diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py
index d0020251b0..eeace5a4a4 100644
--- a/test/bin/pyutils/build_bootc_images.py
+++ b/test/bin/pyutils/build_bootc_images.py
@@ -29,12 +29,41 @@
NEXT_REPO = common.get_env_var('NEXT_REPO')
HOME_DIR = common.get_env_var("HOME")
PULL_SECRET = common.get_env_var('PULL_SECRET', f"{HOME_DIR}/.pull-secret.json")
+BIB_IMAGE = "quay.io/centos-bootc/bootc-image-builder:latest"
+FORCE_REBUILD = False
+
+
+def cleanup_atexit(dry_run):
+ common.print_msg("Running atexit cleanup")
+ # Terminating any running subprocesses
+ for pid in common.find_subprocesses():
+ common.print_msg(f"Terminating {pid} PID")
+ common.terminate_process(pid)
+
+ # Terminate running bootc image builder containers
+ podman_args = [
+ "sudo", "podman", "ps",
+ "--filter", f"ancestor={BIB_IMAGE}",
+ "--format", "{{.ID}}"
+ ]
+ cids = common.run_command_in_shell(podman_args, dry_run)
+ if cids:
+ # Make sure the ids are normalized in a single line
+ cids = re.sub(r'\s+', ' ', cids)
+ common.print_msg(f"Terminating '{cids}' container(s)")
+ common.run_command_in_shell(["sudo", "podman", "stop", cids], dry_run)
def should_skip(file):
- if os.path.exists(file):
- return True
- return False
+ if not os.path.exists(file):
+ return False
+ # Forcing the rebuild if needed
+ if FORCE_REBUILD:
+ common.print_msg(f"Forcing rebuild of '{file}'")
+ return False
+
+ common.print_msg(f"The '{file}' already exists, skipping")
+ return True
def find_latest_rpm(repo_path, version=""):
@@ -160,7 +189,7 @@ def process_containerfile(groupdir, containerfile, dry_run):
# Check if the target artifact exists
if should_skip(cf_targetimg):
- common.record_junit(groupdir, cf_path, "containerfile", "SKIPPED")
+ common.record_junit(cf_path, "process-container", "SKIPPED")
return
# Create the output directories
@@ -178,6 +207,7 @@ def process_containerfile(groupdir, containerfile, dry_run):
os.path.join(IMAGEDIR, "rpm-repos")
]
common.run_command_in_shell(build_args, dry_run, logfile, logfile)
+ common.record_junit(cf_path, "build-container", "OK")
# Run the container export command
if os.path.exists(cf_outdir):
@@ -188,7 +218,9 @@ def process_containerfile(groupdir, containerfile, dry_run):
"-o", cf_outdir, cf_outname
]
common.run_command_in_shell(save_args, dry_run, logfile, logfile)
+ common.record_junit(cf_path, "save-container", "OK")
except Exception:
+ common.record_junit(cf_path, "process-container", "FAILED")
# Propagate the exception to the caller
raise
finally:
@@ -205,7 +237,7 @@ def process_image_bootc(groupdir, bootcfile, dry_run):
# Check if the target artifact exists
if should_skip(bf_targetiso):
- common.record_junit(groupdir, bf_path, "image-bootc", "SKIPPED")
+ common.record_junit(bf_path, "process-bootc-image", "SKIPPED")
return
# Create the output directories
@@ -223,6 +255,7 @@ def process_image_bootc(groupdir, bootcfile, dry_run):
"--authfile", PULL_SECRET, bf_imgref
]
common.run_command_in_shell(pull_args, dry_run, logfile, logfile)
+ common.record_junit(bf_path, "pull-bootc-image", "OK")
# The podman command with security elevation and
# mount of output / container storage
@@ -236,13 +269,15 @@ def process_image_bootc(groupdir, bootcfile, dry_run):
]
# Add the bootc image builder command line using local images
build_args += [
- "quay.io/centos-bootc/bootc-image-builder:latest",
+ BIB_IMAGE,
"--type", "anaconda-iso",
"--local",
bf_imgref
]
common.run_command_in_shell(build_args, dry_run, logfile, logfile)
+ common.record_junit(bf_path, "build-bootc-image", "OK")
except Exception:
+ common.record_junit(bf_path, "process-bootc-image", "FAILED")
# Propagate the exception to the caller
raise
finally:
@@ -257,20 +292,28 @@ def process_image_bootc(groupdir, bootcfile, dry_run):
os.rename(f"{bf_outdir}/bootiso/install.iso", bf_targetiso)
-def process_group(groupdir, dry_run=False):
+def process_group(groupdir, build_type, dry_run=False):
futures = []
- # Parallel processing loop
- with concurrent.futures.ProcessPoolExecutor() as executor:
- # Scan group directory contents sorted by length and then alphabetically
- for file in sorted(os.listdir(groupdir), key=lambda i: (len(i), i)):
- if file.endswith(".containerfile"):
- futures += [executor.submit(process_containerfile, groupdir, file, dry_run)]
- elif file.endswith(".image-bootc"):
- futures += [executor.submit(process_image_bootc, groupdir, file, dry_run)]
- else:
- common.print_msg(f"Skipping unknown file {file}")
-
try:
+ # Open the junit file
+ common.start_junit(groupdir)
+ # Parallel processing loop
+ with concurrent.futures.ProcessPoolExecutor() as executor:
+ # Scan group directory contents sorted by length and then alphabetically
+ for file in sorted(os.listdir(groupdir), key=lambda i: (len(i), i)):
+ if file.endswith(".containerfile"):
+ if build_type and build_type != "containerfile":
+ common.print_msg(f"Skipping '{file}' due to '{build_type}' filter")
+ continue
+ futures.append(executor.submit(process_containerfile, groupdir, file, dry_run))
+ elif file.endswith(".image-bootc"):
+ if build_type and build_type != "image-bootc":
+ common.print_msg(f"Skipping '{file}' due to '{build_type}' filter")
+ continue
+ futures.append(executor.submit(process_image_bootc, groupdir, file, dry_run))
+ else:
+ common.print_msg(f"Skipping unknown file {file}")
+
# Wait for the parallel tasks to complete
for f in concurrent.futures.as_completed(futures):
common.print_msg(f"Task {f} completed")
@@ -284,54 +327,76 @@ def process_group(groupdir, dry_run=False):
common.print_msg(f"Task {f} cancelled")
# Propagate the exception to the caller
raise
+ finally:
+ # Close junit file
+ common.close_junit()
def main():
# Parse command line arguments
- parser = argparse.ArgumentParser(description="Process container files with Podman.")
- parser.add_argument("-d", "--dry-run", action="store_true", help="Dry run: skip executing Podman commands.")
+ parser = argparse.ArgumentParser(description="Build image layers using Bootc Image Builder and Podman.")
+ parser.add_argument("-d", "--dry-run", action="store_true", help="Dry run: skip executing build commands.")
+ parser.add_argument("-f", "--force-rebuild", action="store_true", help="Force rebuilding images that already exist.")
+ parser.add_argument("-E", "--no-extract-images", action="store_true", help="Skip container image extraction.")
+ parser.add_argument("-b", "--build-type", choices=["image-bootc", "containerfile"], help="Only build images of the specified type.")
dirgroup = parser.add_mutually_exclusive_group(required=True)
dirgroup.add_argument("-l", "--layer-dir", type=str, help="Path to the layer directory to process.")
dirgroup.add_argument("-g", "--group-dir", type=str, help="Path to the group directory to process.")
args = parser.parse_args()
- # Convert input directories to absolute paths
- if args.group_dir:
- args.group_dir = os.path.abspath(args.group_dir)
- if args.layer_dir:
- args.layer_dir = os.path.abspath(args.layer_dir)
-
+ success_message = False
try:
+ # Convert input directories to absolute paths
+ if args.group_dir:
+ args.group_dir = os.path.abspath(args.group_dir)
+ dir2process = args.group_dir
+ if args.layer_dir:
+ args.layer_dir = os.path.abspath(args.layer_dir)
+ dir2process = args.layer_dir
+ # Make sure the input directory exists
+ if not os.path.isdir(dir2process):
+ raise Exception(f"The input directory '{dir2process}' does not exist")
# Make sure the local RPM repository exists
if not os.path.isdir(LOCAL_REPO):
raise Exception("Run create_local_repo.sh before building images")
+ # Initialize force rebuild option
+ global FORCE_REBUILD
+ if args.force_rebuild:
+ FORCE_REBUILD = True
# Determine versions of RPM packages
set_rpm_version_info_vars()
- # Prepare container lists for mirroring registries
+ # Prepare container image lists for mirroring registries
common.delete_file(CONTAINER_LIST)
- extract_container_images(SOURCE_VERSION, LOCAL_REPO, CONTAINER_LIST, args.dry_run)
- # The following images are specific to layers that use fake rpms built from source
- extract_container_images(f"4.{FAKE_NEXT_MINOR_VERSION}.*", NEXT_REPO, CONTAINER_LIST, args.dry_run)
- extract_container_images(PREVIOUS_RELEASE_VERSION, PREVIOUS_RELEASE_REPO, CONTAINER_LIST, args.dry_run)
- extract_container_images(YMINUS2_RELEASE_VERSION, YMINUS2_RELEASE_REPO, CONTAINER_LIST, args.dry_run)
+ if args.no_extract_images:
+ common.print_msg("Skipping container image extraction")
+ else:
+ extract_container_images(SOURCE_VERSION, LOCAL_REPO, CONTAINER_LIST, args.dry_run)
+ # The following images are specific to layers that use fake rpms built from source
+ extract_container_images(f"4.{FAKE_NEXT_MINOR_VERSION}.*", NEXT_REPO, CONTAINER_LIST, args.dry_run)
+ extract_container_images(PREVIOUS_RELEASE_VERSION, PREVIOUS_RELEASE_REPO, CONTAINER_LIST, args.dry_run)
+ extract_container_images(YMINUS2_RELEASE_VERSION, YMINUS2_RELEASE_REPO, CONTAINER_LIST, args.dry_run)
# Process individual group directory
if args.group_dir:
- process_group(args.group_dir, args.dry_run)
+ process_group(args.group_dir, args.build_type, args.dry_run)
else:
# Process layer directory contents sorted by length and then alphabetically
for item in sorted(os.listdir(args.layer_dir), key=lambda i: (len(i), i)):
item_path = os.path.join(args.layer_dir, item)
# Check if this item is a directory
if os.path.isdir(item_path):
- process_group(item_path, args.dry_run)
- # Success message
- common.print_msg("Build complete")
+ process_group(item_path, args.build_type, args.dry_run)
+ # Toggle the success flag
+ success_message = True
except Exception as e:
common.print_msg(f"An error occurred: {e}")
traceback.print_exc()
sys.exit(1)
+ finally:
+ cleanup_atexit(args.dry_run)
+ # Exit status message
+ common.print_msg("Build " + ("OK" if success_message else "FAILED"))
if __name__ == "__main__":
diff --git a/test/bin/pyutils/common.py b/test/bin/pyutils/common.py
index 5a2c2c419b..ac44ed79e1 100644
--- a/test/bin/pyutils/common.py
+++ b/test/bin/pyutils/common.py
@@ -2,18 +2,73 @@
import os
import pathlib
+import psutil
import sys
import subprocess
import time
+import threading
from typing import List
PUSHD_DIR_STACK = []
-
-
-def record_junit(groupdir, containerfile, filetype, status):
- # Implement your recording logic here
- pass
+JUNIT_LOGFILE = None
+JUNIT_LOCK = threading.Lock()
+
+
+def start_junit(groupdir):
+ """Create a new junit file with the group name and timestampt header"""
+ # Initialize the junit log file path
+ global JUNIT_LOGFILE
+ group = basename(groupdir)
+ JUNIT_LOGFILE = os.path.join(get_env_var('IMAGEDIR'), "build-logs", group, "junit.xml")
+
+ print_msg(f"Creating '{JUNIT_LOGFILE}'")
+ # Create the output directory
+ create_dir(os.path.dirname(JUNIT_LOGFILE))
+ # Create a new junit file with a header
+ delete_file(JUNIT_LOGFILE)
+ timestamp = get_timestamp("%Y-%m-%dT%H:%M:%S")
+ append_file(JUNIT_LOGFILE, f'''
+''')
+
+
+def close_junit():
+ """Close the junit file"""
+ global JUNIT_LOGFILE
+ if not JUNIT_LOGFILE:
+ raise Exception("Attempt to close junit without starting it first")
+ # Close the unit
+ append_file(JUNIT_LOGFILE, '')
+ # Reset the junit log directory
+ JUNIT_LOGFILE = None
+
+
+def record_junit(object, step, status):
+ """Add a message for the specified object and step with OK, SKIP or FAIL status.
+ Recording messages is synchronized and it can be called from different threads.
+ """
+ try:
+ # BEGIN CRITICAL SECTION
+ JUNIT_LOCK.acquire()
+
+ append_file(JUNIT_LOGFILE, f'')
+ # Add a message according to the status
+ if status == "OK":
+ pass
+ elif status.startswith("SKIP"):
+ append_file(JUNIT_LOGFILE, f'')
+ elif status.startswith("FAIL"):
+ append_file(JUNIT_LOGFILE, f'')
+ else:
+ raise Exception(f"Invalid junit status '{status}'")
+ # Close the test case block
+ append_file(JUNIT_LOGFILE, '')
+ except Exception:
+ # Propagate the exception to the caller
+ raise
+ finally:
+ # END CRITICAL SECTION
+ JUNIT_LOCK.release()
def get_timestamp(format: str = "%H:%M:%S"):
@@ -105,6 +160,12 @@ def read_file(file_path: str):
return content
+def append_file(file_path: str, content: str):
+ """Append the specified content to a file"""
+ with open(file_path, 'a') as file:
+ file.write(content)
+
+
def delete_file(file_path: str):
"""Attempt file deletion ignoring errors when a file does not exist"""
try:
@@ -116,3 +177,42 @@ def delete_file(file_path: str):
def basename(path: str):
"""Return a base name of the path"""
return pathlib.Path(path).name
+
+
+def find_subprocesses(ppid=None):
+ """Find and return a list of all the sub-processes of a parent PID"""
+ # Get current process if not specified
+ if not ppid:
+ ppid = psutil.Process().pid
+ # Get all child process objects recursively
+ children = psutil.Process(ppid).children(recursive=True)
+ # Collect the child process IDs
+ pids = []
+ for child in children:
+ pids.append(child.pid)
+ return pids
+
+
+def terminate_process(pid, wait=True):
+ """Terminate a process, waiting for 10s until it exits"""
+ try:
+ proc = psutil.Process(pid)
+ # Check if the process runs elevated
+ if proc.uids().effective == 0:
+ run_command(["sudo", "kill", "-TERM", f"{pid}"], False)
+ else:
+ proc.terminate()
+ if not wait:
+ return
+
+ # Wait for process to terminate
+ try:
+ proc.wait(timeout=10)
+ except psutil.TimeoutExpired:
+ print_msg(f"The {pid} PID did not exit after 10s")
+ except psutil.NoSuchProcess:
+ # Ignore non-existent processes
+ pass
+ except Exception:
+ # Propagate the exception to the caller
+ raise