From b21d4291eb9cd80e7c37099b5d3aa1dec8df4422 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sat, 27 Apr 2024 06:20:30 +0000 Subject: [PATCH 1/9] Implement atexit cleanup procedure --- test/bin/pyutils/build_bootc_images.py | 40 ++++++++++++++++++++++---- test/bin/pyutils/common.py | 32 +++++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index d0020251b0..754397060c 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -29,10 +29,31 @@ NEXT_REPO = common.get_env_var('NEXT_REPO') HOME_DIR = common.get_env_var("HOME") PULL_SECRET = common.get_env_var('PULL_SECRET', f"{HOME_DIR}/.pull-secret.json") +BIB_IMAGE = "quay.io/centos-bootc/bootc-image-builder:latest" + + +def cleanup_atexit(dry_run): + common.print_msg(f"Running atexit cleanup") + # Terminating any running subprocesses + for pid in common.find_subprocesses(): + common.print_msg(f"Terminating {pid} PID") + common.terminate_process(pid) + + # Terminate running bootc image builder containers + podman_args = [ + "sudo", "podman", "ps", + "--filter", f"ancestor={BIB_IMAGE}", + "--format", "{{.ID}}" + ] + cids = common.run_command_in_shell(podman_args, dry_run) + if cids: + common.print_msg(f"Terminating '{cids}' container(s)") + common.run_command_in_shell(["sudo", "podman", "stop", cids], dry_run) def should_skip(file): if os.path.exists(file): + common.print_msg(f"{file} already exists, skipping") return True return False @@ -236,7 +257,7 @@ def process_image_bootc(groupdir, bootcfile, dry_run): ] # Add the bootc image builder command line using local images build_args += [ - "quay.io/centos-bootc/bootc-image-builder:latest", + BIB_IMAGE, "--type", "anaconda-iso", "--local", bf_imgref @@ -295,13 +316,18 @@ def main(): dirgroup.add_argument("-g", "--group-dir", type=str, help="Path to the group directory to process.") args = parser.parse_args() - # Convert input directories to absolute paths - if args.group_dir: - args.group_dir = os.path.abspath(args.group_dir) - if args.layer_dir: - args.layer_dir = os.path.abspath(args.layer_dir) try: + # Convert input directories to absolute paths + if args.group_dir: + args.group_dir = os.path.abspath(args.group_dir) + dir2process = args.group_dir + if args.layer_dir: + args.layer_dir = os.path.abspath(args.layer_dir) + dir2process = args.layer_dir + # Make sure the input directory exists + if not os.path.isdir(dir2process): + raise Exception(f"The input directory '{dir2process}' does not exist") # Make sure the local RPM repository exists if not os.path.isdir(LOCAL_REPO): raise Exception("Run create_local_repo.sh before building images") @@ -332,6 +358,8 @@ def main(): common.print_msg(f"An error occurred: {e}") traceback.print_exc() sys.exit(1) + finally: + cleanup_atexit(args.dry_run) if __name__ == "__main__": diff --git a/test/bin/pyutils/common.py b/test/bin/pyutils/common.py index 5a2c2c419b..57a0eec1d3 100644 --- a/test/bin/pyutils/common.py +++ b/test/bin/pyutils/common.py @@ -2,6 +2,7 @@ import os import pathlib +import psutil import sys import subprocess import time @@ -116,3 +117,34 @@ def delete_file(file_path: str): def basename(path: str): """Return a base name of the path""" return pathlib.Path(path).name + + +def find_subprocesses(ppid=None): + """Find and return a list of all the sub-processes of a parent PID""" + # Get current process if not specified + if not ppid: + ppid = psutil.Process().pid + # Get all child process objects recursively + children = psutil.Process(ppid).children(recursive=True) + # Collect the child process IDs + pids = [] + for child in children: + pids += [child.pid] + return pids + + +def terminate_process(pid, wait=True): + """Terminate a process, waiting until it exited""" + try: + proc = psutil.Process(pid) + # Check if the process runs elevated + if proc.uids().effective == 0: + run_command(["sudo", "kill", "-TERM", pid], False) + else: + proc.terminate() + # Wait for process to terminate + if wait: + proc.wait() + except psutil.NoSuchProcess: + # Ignore non-existent processes + None From 857a20b7db444243eb7c52ac2367745fa4924a96 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sat, 27 Apr 2024 07:28:31 +0000 Subject: [PATCH 2/9] Add an option to build images of the specified type --- test/bin/pyutils/build_bootc_images.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index 754397060c..db3f8d1de9 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -278,15 +278,21 @@ def process_image_bootc(groupdir, bootcfile, dry_run): os.rename(f"{bf_outdir}/bootiso/install.iso", bf_targetiso) -def process_group(groupdir, dry_run=False): +def process_group(groupdir, build_type, dry_run=False): futures = [] # Parallel processing loop with concurrent.futures.ProcessPoolExecutor() as executor: # Scan group directory contents sorted by length and then alphabetically for file in sorted(os.listdir(groupdir), key=lambda i: (len(i), i)): if file.endswith(".containerfile"): + if build_type and build_type != "containerfile": + common.print_msg(f"Skipping '{file}' due to '{build_type}' filter") + continue futures += [executor.submit(process_containerfile, groupdir, file, dry_run)] elif file.endswith(".image-bootc"): + if build_type and build_type != "image-bootc": + common.print_msg(f"Skipping '{file}' due to '{build_type}' filter") + continue futures += [executor.submit(process_image_bootc, groupdir, file, dry_run)] else: common.print_msg(f"Skipping unknown file {file}") @@ -309,8 +315,9 @@ def process_group(groupdir, dry_run=False): def main(): # Parse command line arguments - parser = argparse.ArgumentParser(description="Process container files with Podman.") - parser.add_argument("-d", "--dry-run", action="store_true", help="Dry run: skip executing Podman commands.") + parser = argparse.ArgumentParser(description="Build image layers using Bootc Image Builder and Podman.") + parser.add_argument("-d", "--dry-run", action="store_true", help="Dry run: skip executing build commands.") + parser.add_argument("-b", "--build-type", choices=["image-bootc", "containerfile"], help="Only build images of the specified type.") dirgroup = parser.add_mutually_exclusive_group(required=True) dirgroup.add_argument("-l", "--layer-dir", type=str, help="Path to the layer directory to process.") dirgroup.add_argument("-g", "--group-dir", type=str, help="Path to the group directory to process.") @@ -344,14 +351,14 @@ def main(): # Process individual group directory if args.group_dir: - process_group(args.group_dir, args.dry_run) + process_group(args.group_dir, args.build_type, args.dry_run) else: # Process layer directory contents sorted by length and then alphabetically for item in sorted(os.listdir(args.layer_dir), key=lambda i: (len(i), i)): item_path = os.path.join(args.layer_dir, item) # Check if this item is a directory if os.path.isdir(item_path): - process_group(item_path, args.dry_run) + process_group(item_path, args.build_type, args.dry_run) # Success message common.print_msg("Build complete") except Exception as e: From ab904f1a426d8d2c85c96fc2fdaf6bf706496cff Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sat, 27 Apr 2024 07:47:02 +0000 Subject: [PATCH 3/9] Add force-rebuild option --- test/bin/pyutils/build_bootc_images.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index db3f8d1de9..6fecc5106d 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -30,10 +30,11 @@ HOME_DIR = common.get_env_var("HOME") PULL_SECRET = common.get_env_var('PULL_SECRET', f"{HOME_DIR}/.pull-secret.json") BIB_IMAGE = "quay.io/centos-bootc/bootc-image-builder:latest" +FORCE_REBUILD = False def cleanup_atexit(dry_run): - common.print_msg(f"Running atexit cleanup") + common.print_msg("Running atexit cleanup") # Terminating any running subprocesses for pid in common.find_subprocesses(): common.print_msg(f"Terminating {pid} PID") @@ -52,10 +53,15 @@ def cleanup_atexit(dry_run): def should_skip(file): - if os.path.exists(file): - common.print_msg(f"{file} already exists, skipping") - return True - return False + if not os.path.exists(file): + return False + # Forcing the rebuild if needed + if FORCE_REBUILD is True: + common.print_msg(f"Forcing rebuild of '{file}'") + return False + + common.print_msg(f"The '{file}' already exists, skipping") + return True def find_latest_rpm(repo_path, version=""): @@ -317,6 +323,7 @@ def main(): # Parse command line arguments parser = argparse.ArgumentParser(description="Build image layers using Bootc Image Builder and Podman.") parser.add_argument("-d", "--dry-run", action="store_true", help="Dry run: skip executing build commands.") + parser.add_argument("-f", "--force-rebuild", action="store_true", help="Force rebuilding images that already exist.") parser.add_argument("-b", "--build-type", choices=["image-bootc", "containerfile"], help="Only build images of the specified type.") dirgroup = parser.add_mutually_exclusive_group(required=True) dirgroup.add_argument("-l", "--layer-dir", type=str, help="Path to the layer directory to process.") @@ -338,6 +345,10 @@ def main(): # Make sure the local RPM repository exists if not os.path.isdir(LOCAL_REPO): raise Exception("Run create_local_repo.sh before building images") + # Initialize force rebuild option + global FORCE_REBUILD + if args.force_rebuild and args.force_rebuild is True: + FORCE_REBUILD = True # Determine versions of RPM packages set_rpm_version_info_vars() From 6f6b1adee8629e88b7a096165e95ec79f6314ef3 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sat, 27 Apr 2024 07:57:01 +0000 Subject: [PATCH 4/9] Fix the exit message to appear in the end --- test/bin/pyutils/build_bootc_images.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index 6fecc5106d..91f163d22f 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -330,7 +330,7 @@ def main(): dirgroup.add_argument("-g", "--group-dir", type=str, help="Path to the group directory to process.") args = parser.parse_args() - + success_message = False try: # Convert input directories to absolute paths if args.group_dir: @@ -370,14 +370,16 @@ def main(): # Check if this item is a directory if os.path.isdir(item_path): process_group(item_path, args.build_type, args.dry_run) - # Success message - common.print_msg("Build complete") + # Toggle the success flag + success_message = True except Exception as e: common.print_msg(f"An error occurred: {e}") traceback.print_exc() sys.exit(1) finally: cleanup_atexit(args.dry_run) + # Exit status message + common.print_msg("Build " + ("OK" if success_message else "FAILED")) if __name__ == "__main__": From 99591a45bc60b4d5e3db24db900596e6f1ad1ab9 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sat, 27 Apr 2024 08:12:16 +0000 Subject: [PATCH 5/9] Add no container image extraction options --- test/bin/pyutils/build_bootc_images.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index 91f163d22f..66a7ce4cd8 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -324,6 +324,7 @@ def main(): parser = argparse.ArgumentParser(description="Build image layers using Bootc Image Builder and Podman.") parser.add_argument("-d", "--dry-run", action="store_true", help="Dry run: skip executing build commands.") parser.add_argument("-f", "--force-rebuild", action="store_true", help="Force rebuilding images that already exist.") + parser.add_argument("-E", "--no-extract-images", action="store_true", help="Skip container image extraction.") parser.add_argument("-b", "--build-type", choices=["image-bootc", "containerfile"], help="Only build images of the specified type.") dirgroup = parser.add_mutually_exclusive_group(required=True) dirgroup.add_argument("-l", "--layer-dir", type=str, help="Path to the layer directory to process.") @@ -352,13 +353,16 @@ def main(): # Determine versions of RPM packages set_rpm_version_info_vars() - # Prepare container lists for mirroring registries - common.delete_file(CONTAINER_LIST) - extract_container_images(SOURCE_VERSION, LOCAL_REPO, CONTAINER_LIST, args.dry_run) - # The following images are specific to layers that use fake rpms built from source - extract_container_images(f"4.{FAKE_NEXT_MINOR_VERSION}.*", NEXT_REPO, CONTAINER_LIST, args.dry_run) - extract_container_images(PREVIOUS_RELEASE_VERSION, PREVIOUS_RELEASE_REPO, CONTAINER_LIST, args.dry_run) - extract_container_images(YMINUS2_RELEASE_VERSION, YMINUS2_RELEASE_REPO, CONTAINER_LIST, args.dry_run) + # Prepare container image lists for mirroring registries + if args.no_extract_images: + common.print_msg("Skipping container image extraction") + else: + common.delete_file(CONTAINER_LIST) + extract_container_images(SOURCE_VERSION, LOCAL_REPO, CONTAINER_LIST, args.dry_run) + # The following images are specific to layers that use fake rpms built from source + extract_container_images(f"4.{FAKE_NEXT_MINOR_VERSION}.*", NEXT_REPO, CONTAINER_LIST, args.dry_run) + extract_container_images(PREVIOUS_RELEASE_VERSION, PREVIOUS_RELEASE_REPO, CONTAINER_LIST, args.dry_run) + extract_container_images(YMINUS2_RELEASE_VERSION, YMINUS2_RELEASE_REPO, CONTAINER_LIST, args.dry_run) # Process individual group directory if args.group_dir: From f82f9a0a8f76fbf44789e4533ff524db9a33843c Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sat, 27 Apr 2024 08:21:58 +0000 Subject: [PATCH 6/9] Optimize force rebuild flag checks --- test/bin/pyutils/build_bootc_images.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index 66a7ce4cd8..25706a48b8 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -56,7 +56,7 @@ def should_skip(file): if not os.path.exists(file): return False # Forcing the rebuild if needed - if FORCE_REBUILD is True: + if FORCE_REBUILD: common.print_msg(f"Forcing rebuild of '{file}'") return False @@ -348,7 +348,7 @@ def main(): raise Exception("Run create_local_repo.sh before building images") # Initialize force rebuild option global FORCE_REBUILD - if args.force_rebuild and args.force_rebuild is True: + if args.force_rebuild: FORCE_REBUILD = True # Determine versions of RPM packages From 25e1c627445f174cdec7ea09442aff29c51c9343 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Mon, 29 Apr 2024 06:22:51 +0000 Subject: [PATCH 7/9] Add junit handling --- test/bin/pyutils/build_bootc_images.py | 53 +++++++++------- test/bin/pyutils/common.py | 83 ++++++++++++++++++++++---- 2 files changed, 106 insertions(+), 30 deletions(-) diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index 25706a48b8..8181b5a0a2 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -48,6 +48,8 @@ def cleanup_atexit(dry_run): ] cids = common.run_command_in_shell(podman_args, dry_run) if cids: + # Make sure the ids are normalized in a single line + cids = re.sub(r'\s+', ' ', cids) common.print_msg(f"Terminating '{cids}' container(s)") common.run_command_in_shell(["sudo", "podman", "stop", cids], dry_run) @@ -187,7 +189,7 @@ def process_containerfile(groupdir, containerfile, dry_run): # Check if the target artifact exists if should_skip(cf_targetimg): - common.record_junit(groupdir, cf_path, "containerfile", "SKIPPED") + common.record_junit(cf_path, "process-container", "SKIPPED") return # Create the output directories @@ -205,6 +207,7 @@ def process_containerfile(groupdir, containerfile, dry_run): os.path.join(IMAGEDIR, "rpm-repos") ] common.run_command_in_shell(build_args, dry_run, logfile, logfile) + common.record_junit(cf_path, "build-container", "OK") # Run the container export command if os.path.exists(cf_outdir): @@ -215,7 +218,9 @@ def process_containerfile(groupdir, containerfile, dry_run): "-o", cf_outdir, cf_outname ] common.run_command_in_shell(save_args, dry_run, logfile, logfile) + common.record_junit(cf_path, "save-container", "OK") except Exception: + common.record_junit(cf_path, "process-container", "FAILED") # Propagate the exception to the caller raise finally: @@ -232,7 +237,7 @@ def process_image_bootc(groupdir, bootcfile, dry_run): # Check if the target artifact exists if should_skip(bf_targetiso): - common.record_junit(groupdir, bf_path, "image-bootc", "SKIPPED") + common.record_junit(bf_path, "process-bootc-image", "SKIPPED") return # Create the output directories @@ -250,6 +255,7 @@ def process_image_bootc(groupdir, bootcfile, dry_run): "--authfile", PULL_SECRET, bf_imgref ] common.run_command_in_shell(pull_args, dry_run, logfile, logfile) + common.record_junit(bf_path, "pull-bootc-image", "OK") # The podman command with security elevation and # mount of output / container storage @@ -269,7 +275,9 @@ def process_image_bootc(groupdir, bootcfile, dry_run): bf_imgref ] common.run_command_in_shell(build_args, dry_run, logfile, logfile) + common.record_junit(bf_path, "build-bootc-image", "OK") except Exception: + common.record_junit(bf_path, "process-bootc-image", "FAILED") # Propagate the exception to the caller raise finally: @@ -286,24 +294,26 @@ def process_image_bootc(groupdir, bootcfile, dry_run): def process_group(groupdir, build_type, dry_run=False): futures = [] - # Parallel processing loop - with concurrent.futures.ProcessPoolExecutor() as executor: - # Scan group directory contents sorted by length and then alphabetically - for file in sorted(os.listdir(groupdir), key=lambda i: (len(i), i)): - if file.endswith(".containerfile"): - if build_type and build_type != "containerfile": - common.print_msg(f"Skipping '{file}' due to '{build_type}' filter") - continue - futures += [executor.submit(process_containerfile, groupdir, file, dry_run)] - elif file.endswith(".image-bootc"): - if build_type and build_type != "image-bootc": - common.print_msg(f"Skipping '{file}' due to '{build_type}' filter") - continue - futures += [executor.submit(process_image_bootc, groupdir, file, dry_run)] - else: - common.print_msg(f"Skipping unknown file {file}") - try: + # Open the junit file + common.start_junit(groupdir) + # Parallel processing loop + with concurrent.futures.ProcessPoolExecutor() as executor: + # Scan group directory contents sorted by length and then alphabetically + for file in sorted(os.listdir(groupdir), key=lambda i: (len(i), i)): + if file.endswith(".containerfile"): + if build_type and build_type != "containerfile": + common.print_msg(f"Skipping '{file}' due to '{build_type}' filter") + continue + futures += [executor.submit(process_containerfile, groupdir, file, dry_run)] + elif file.endswith(".image-bootc"): + if build_type and build_type != "image-bootc": + common.print_msg(f"Skipping '{file}' due to '{build_type}' filter") + continue + futures += [executor.submit(process_image_bootc, groupdir, file, dry_run)] + else: + common.print_msg(f"Skipping unknown file {file}") + # Wait for the parallel tasks to complete for f in concurrent.futures.as_completed(futures): common.print_msg(f"Task {f} completed") @@ -317,6 +327,9 @@ def process_group(groupdir, build_type, dry_run=False): common.print_msg(f"Task {f} cancelled") # Propagate the exception to the caller raise + finally: + # Close junit file + common.close_junit() def main(): @@ -354,10 +367,10 @@ def main(): # Determine versions of RPM packages set_rpm_version_info_vars() # Prepare container image lists for mirroring registries + common.delete_file(CONTAINER_LIST) if args.no_extract_images: common.print_msg("Skipping container image extraction") else: - common.delete_file(CONTAINER_LIST) extract_container_images(SOURCE_VERSION, LOCAL_REPO, CONTAINER_LIST, args.dry_run) # The following images are specific to layers that use fake rpms built from source extract_container_images(f"4.{FAKE_NEXT_MINOR_VERSION}.*", NEXT_REPO, CONTAINER_LIST, args.dry_run) diff --git a/test/bin/pyutils/common.py b/test/bin/pyutils/common.py index 57a0eec1d3..a631978414 100644 --- a/test/bin/pyutils/common.py +++ b/test/bin/pyutils/common.py @@ -6,15 +6,69 @@ import sys import subprocess import time +import threading from typing import List PUSHD_DIR_STACK = [] - - -def record_junit(groupdir, containerfile, filetype, status): - # Implement your recording logic here - pass +JUNIT_LOGFILE = None +JUNIT_LOCK = threading.Lock() + + +def start_junit(groupdir): + """Create a new junit file with the group name and timestampt header""" + # Initialize the junit log file path + global JUNIT_LOGFILE + group = basename(groupdir) + JUNIT_LOGFILE = os.path.join(get_env_var('IMAGEDIR'), "build-logs", group, "junit.xml") + + print_msg(f"Creating '{JUNIT_LOGFILE}'") + # Create the output directory + create_dir(os.path.dirname(JUNIT_LOGFILE)) + # Create a new junit file with a header + delete_file(JUNIT_LOGFILE) + timestamp = get_timestamp("%Y-%m-%dT%H:%M:%S") + append_file(JUNIT_LOGFILE, f''' +''') + + +def close_junit(): + """Close the junit file""" + global JUNIT_LOGFILE + if not JUNIT_LOGFILE: + raise Exception("Attempt to close junit without starting it first") + # Close the unit + append_file(JUNIT_LOGFILE, '') + # Reset the junit log directory + JUNIT_LOGFILE = None + + +def record_junit(object, step, status): + """Add a message for the specified object and step with OK, SKIP or FAIL status. + Recording messages is synchronized and it can be called from different threads. + """ + try: + # BEGIN CRITICAL SECTION + JUNIT_LOCK.acquire() + + append_file(JUNIT_LOGFILE, f'') + # Add a message according to the status + if status == "OK": + pass + elif status.startswith("SKIP"): + append_file(JUNIT_LOGFILE, f'') + elif status.startswith("FAIL"): + append_file(JUNIT_LOGFILE, f'') + else: + raise Exception(f"Invalid junit status '{status}'") + # Close the test case block + append_file(JUNIT_LOGFILE, '') + except Exception: + # Propagate the exception to the caller + raise + finally: + # END CRITICAL SECTION + JUNIT_LOCK.release() def get_timestamp(format: str = "%H:%M:%S"): @@ -106,6 +160,12 @@ def read_file(file_path: str): return content +def append_file(file_path: str, content: str): + """Append the specified content to a file""" + with open(file_path, 'a') as file: + file.write(content) + + def delete_file(file_path: str): """Attempt file deletion ignoring errors when a file does not exist""" try: @@ -134,17 +194,20 @@ def find_subprocesses(ppid=None): def terminate_process(pid, wait=True): - """Terminate a process, waiting until it exited""" + """Terminate a process, waiting for 10s until it exits""" try: proc = psutil.Process(pid) # Check if the process runs elevated if proc.uids().effective == 0: - run_command(["sudo", "kill", "-TERM", pid], False) + run_command(["sudo", "kill", "-TERM", f"{pid}"], False) else: proc.terminate() # Wait for process to terminate if wait: - proc.wait() - except psutil.NoSuchProcess: + try: + proc.wait(timeout=10) + except psutil.TimeoutExpired: + print_msg(f"The {pid} PID did not exit after 10s") + except Exception: # Ignore non-existent processes - None + pass From 4a0350f0047a920b7f4dadb902097df882fde32a Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Mon, 29 Apr 2024 15:55:29 +0000 Subject: [PATCH 8/9] List handling optimizations --- test/bin/pyutils/build_bootc_images.py | 4 ++-- test/bin/pyutils/common.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index 8181b5a0a2..eeace5a4a4 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -305,12 +305,12 @@ def process_group(groupdir, build_type, dry_run=False): if build_type and build_type != "containerfile": common.print_msg(f"Skipping '{file}' due to '{build_type}' filter") continue - futures += [executor.submit(process_containerfile, groupdir, file, dry_run)] + futures.append(executor.submit(process_containerfile, groupdir, file, dry_run)) elif file.endswith(".image-bootc"): if build_type and build_type != "image-bootc": common.print_msg(f"Skipping '{file}' due to '{build_type}' filter") continue - futures += [executor.submit(process_image_bootc, groupdir, file, dry_run)] + futures.append(executor.submit(process_image_bootc, groupdir, file, dry_run)) else: common.print_msg(f"Skipping unknown file {file}") diff --git a/test/bin/pyutils/common.py b/test/bin/pyutils/common.py index a631978414..7f87a564e9 100644 --- a/test/bin/pyutils/common.py +++ b/test/bin/pyutils/common.py @@ -189,7 +189,7 @@ def find_subprocesses(ppid=None): # Collect the child process IDs pids = [] for child in children: - pids += [child.pid] + pids.append(child.pid) return pids From 730529a5f5686d97bf2fefd7aace662f82b14da8 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Tue, 30 Apr 2024 08:58:38 +0000 Subject: [PATCH 9/9] Fix terminate process function --- test/bin/pyutils/common.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/bin/pyutils/common.py b/test/bin/pyutils/common.py index 7f87a564e9..ac44ed79e1 100644 --- a/test/bin/pyutils/common.py +++ b/test/bin/pyutils/common.py @@ -202,12 +202,17 @@ def terminate_process(pid, wait=True): run_command(["sudo", "kill", "-TERM", f"{pid}"], False) else: proc.terminate() + if not wait: + return + # Wait for process to terminate - if wait: - try: - proc.wait(timeout=10) - except psutil.TimeoutExpired: - print_msg(f"The {pid} PID did not exit after 10s") - except Exception: + try: + proc.wait(timeout=10) + except psutil.TimeoutExpired: + print_msg(f"The {pid} PID did not exit after 10s") + except psutil.NoSuchProcess: # Ignore non-existent processes pass + except Exception: + # Propagate the exception to the caller + raise