diff --git a/scripts/auto-rebase/rebase.py b/scripts/auto-rebase/rebase.py index c9839e5247..1951b8bb4d 100755 --- a/scripts/auto-rebase/rebase.py +++ b/scripts/auto-rebase/rebase.py @@ -16,14 +16,12 @@ from pathlib import Path from timeit import default_timer as timer -from git import PushInfo, Repo # GitPython -from github import Github, GithubException, GithubIntegration # pygithub - -APP_ID_ENV = "APP_ID" # GitHub App's ID -KEY_ENV = "KEY" # Path to GitHub App's key -PAT_ENV = "TOKEN" # Personal Access Token -ORG_ENV = "ORG" -REPO_ENV = "REPO" +from github import GithubException # pygithub + +sys.path.append(str(Path(__file__).resolve().parent / '../pyutils')) +import gitutils # noqa: E402 +import ghutils # noqa: E402 + AMD64_RELEASE_ENV = "AMD64_RELEASE" ARM64_RELEASE_ENV = "ARM64_RELEASE" RHOAI_RELEASE_ENV = "RHOAI_RELEASE" @@ -37,9 +35,6 @@ BOT_REMOTE_NAME = "bot-creds" REMOTE_ORIGIN = "origin" -# List of reviewers to always request review from -REVIEWERS = [] - # If True, then just log action such as branch push and PR or comment creation REMOTE_DRY_RUN = False @@ -144,28 +139,13 @@ def run_rebase_cert_manager_sh(release): return RebaseScriptResult(success=result.returncode == 0, output=result.stdout) -def commit_str(commit): - """Returns the first 8 characters of the commit's SHA hash and the commit summary.""" - return f"{commit.hexsha[:8]} - {commit.summary}" - - -def get_installation_access_token(app_id, key_path, org, repo): - """Get a installation access token for a GitHub App installation.""" - integration = GithubIntegration(app_id, Path(key_path).read_text(encoding='utf-8')) - app_installation = integration.get_installation(org, repo) - if app_installation is None: - sys.exit(f"Failed to get app_installation for {org}/{repo}. " + - f"Response: {app_installation.raw_data}") - return integration.get_access_token(app_installation.id).token - - def make_sure_rebase_script_created_new_commits_or_exit(git_repo, base_branch): """Exit the script if the 'rebase.sh' script did not create any new commits.""" if git_repo.active_branch.commit == git_repo.branches[base_branch].commit: logging.info(f"There's no new commit on branch {git_repo.active_branch} compared to '{base_branch}' " "meaning that the rebase.sh script didn't create any commits and " "MicroShift is already rebased on top of given release.\n" - f"Last commit: {commit_str(git_repo.active_branch.commit)}") + f"Last commit: {gitutils.commit_str(git_repo.active_branch.commit)}") sys.exit(0) @@ -199,106 +179,6 @@ def rebase_script_made_changes_considered_functional(git_repo, base_branch): return False -def get_remote_with_token(git_repo, token, org, repo): - """ - Returns the Git remote for the given repository using - the provided installation (or personal) access token. - """ - remote_url = f"https://x-access-token:{token}@github.com/{org}/{repo}" - try: - remote = git_repo.remote(BOT_REMOTE_NAME) - remote.set_url(remote_url) - except ValueError: - git_repo.create_remote(BOT_REMOTE_NAME, remote_url) - - return git_repo.remote(BOT_REMOTE_NAME) - - -def try_get_rebase_branch_ref_from_remote(remote, branch_name): - """ - Get the reference for the given branch on the specified Git remote, - otherwise return None if the branch does not exist. - """ - remote.fetch() - matching_remote_refs = [ref for ref in remote.refs if BOT_REMOTE_NAME + "/" + branch_name == ref.name] - - if len(matching_remote_refs) == 0: - logging.info(f"Branch '{branch_name}' does not exist on remote") - return None - - if len(matching_remote_refs) > 1: - matching_branches = ", ".join([r.name for r in matching_remote_refs]) - logging.warning(f"Found more than one branch matching '{branch_name}' " + - f"on remote: {matching_branches}. Taking first one") - _extra_msgs.append(f"Found more than one branch matching '{branch_name}' " + - f"on remote: {matching_branches}.") - return matching_remote_refs[0] - - if len(matching_remote_refs) == 1: - logging.info(f"Branch '{branch_name}' already exists on remote") - return matching_remote_refs[0] - - return None - - -def is_local_branch_based_on_newer_base_branch_commit(git_repo, base_branch_name, remote_branch_name, local_branch_name): - """ - Compares local and remote rebase branches by looking at their start on base branch. - Returns True if local branch starts on newer commit and needs to be pushed to remote, - otherwise False. - """ - remote_merge_base = git_repo.merge_base(base_branch_name, remote_branch_name) - local_merge_base = git_repo.merge_base(base_branch_name, local_branch_name) - - if remote_merge_base[0] == local_merge_base[0]: - logging.info("Remote branch is up to date. " + - f"Branch-off commit: {commit_str(remote_merge_base[0])}") - return False - - logging.info( - f"Remote branch is older - it needs updating. " - f"Remote branch is on top of {base_branch_name}'s commit: '{commit_str(remote_merge_base[0])}'. " - f"Local branch is on top of {base_branch_name}'s commit '{commit_str(local_merge_base[0])}'" - ) - return True - - -def try_get_pr(gh_repo, org, base_branch, branch_name): - """ - Try to get a pull request for a branch on a GitHub repository. - Returns - - The pull request if it exists and is open, otherwise None. - - If more than one pull request is found, then the first one will be used. - """ - prs = gh_repo.get_pulls(base=base_branch, head=f"{org}:{branch_name}", state="all") - - if prs.totalCount == 0: - logging.info(f"PR for branch {branch_name} does not exist yet on {gh_repo.full_name}") - return None - - pull_req = None - if prs.totalCount > 1: - pull_req = prs[0] - logging.warning( - f"Found more than one PR for branch {branch_name} on {gh_repo.full_name} -" + - f"this is unexpected, continuing with first one of: {[(x.state, x.html_url) for x in prs]}" - ) - - if prs.totalCount == 1: - pull_req = prs[0] - logging.info(f"Found PR #{pull_req.number} for branch {branch_name} on {gh_repo.full_name}: {pull_req.html_url}") - - if pull_req.state == 'closed': - logging.warning(f"PR #{pull_req.number} is not open - new PR will be created") - if pull_req.is_merged(): - logging.warning(f"PR #{pull_req.number} for '{branch_name}' branch is already merged but rebase.sh produced results") - _extra_msgs.append(f"PR #{pull_req.number} for '{branch_name}' was already merged but rebase.sh produced results") - else: - _extra_msgs.append(f"PR #{pull_req.number} for '{branch_name}' exists already but was closed") - return None - return pull_req - - def generate_pr_description(amd_tag, arm_tag, prow_job_url, rebase_script_succeded): # pylint: disable=unused-argument """ Returns a string that represents the body of a pull request (PR) description. @@ -337,79 +217,6 @@ def generate_pr_description(amd_tag, arm_tag, prow_job_url, rebase_script_succed else "# rebase.sh failed - check committed rebase_sh.log\n\n" + base) -def create_pr(gh_repo, base_branch, branch_name, title, desc): - """ - Creates a pull request (and requests reviews) for a given branch on a GitHub repository. - If the `REMOTE_DRY_RUN` variable is True, it logs the PR creation request without actually creating it. - """ - if REMOTE_DRY_RUN: - logging.info(f"[DRY RUN] Create PR: branch='{branch_name}', title='{title}', desc='{desc}'") - logging.info(f"[DRY RUN] Requesting review from {REVIEWERS}") - return None - - pull_req = gh_repo.create_pull( - title=title, body=desc, base=base_branch, head=branch_name, maintainer_can_modify=True) - logging.info(f"Created pull request: {pull_req.html_url}") - try: - pull_req.create_review_request(reviewers=REVIEWERS) - logging.info(f"Requested review from {REVIEWERS}") - except GithubException as err: - logging.info(f"Failed to request review from {REVIEWERS} because: {err}") - return pull_req - - -def update_pr(pull_req, title, desc): - """Updates the title and description of a pull request on a GitHub repository.""" - if REMOTE_DRY_RUN: - logging.info(f"[DRY RUN] Update PR #{pull_req.number}: {title}\n{desc}") - return - - pull_req.edit(title=title, body=desc) - pull_req.update() # arm64 release or prow job url might've changed - logging.info(f"Updated PR #{pull_req.number}: {title}\n{desc}") - - -def post_comment(pull_req, comment=""): - """ - Posts a comment on a GitHub pull request with - the contents of the global `_extra_msgs` list. - """ - if len(_extra_msgs) != 0: - if comment != "": - comment += "\n\n" - comment += "Extra messages:\n - " + "\n - ".join(_extra_msgs) - - if comment.strip() != "": - logging.info(f"Comment to post: {comment}") - if REMOTE_DRY_RUN: - logging.info("[DRY RUN] Posted a comment") - return - issue = pull_req.as_issue() - issue.create_comment(comment) - else: - logging.info("No content for comment") - - -def push_branch_or_die(remote, branch_name): - """ - Attempts to push a branch to a remote Git repository, - and terminates the program if the push fails. - """ - if REMOTE_DRY_RUN: - logging.info(f"[DRY RUN] git push --force {branch_name}") - return - - # TODO add retries - push_result = remote.push(branch_name, force=True) - - if len(push_result) != 1: - sys.exit(f"Unexpected amount ({len(push_result)}) of items in push_result: {push_result}") - if push_result[0].flags & PushInfo.ERROR: - sys.exit(f"Pushing branch failed: {push_result[0].summary}") - if push_result[0].flags & PushInfo.FORCED_UPDATE: - logging.info(f"Branch '{branch_name}' existed and was updated (force push)") - - def get_release_tag(release): """ Given a release string in the format "abc:xyz", returns the "xyz" portion of the string. @@ -462,26 +269,6 @@ def get_expected_branch_name(amd, arm): return f"rebase-{match_amd['version_stream']}_amd64-{match_amd['date']}_arm64-{match_arm['date']}" -def get_base_branch_name(gh_repo, base_branch): - """ - Given a branch name, determine if the rebase branch should be the same - or it should target main, depending on the current stage in the dev cycle. - """ - issues = gh_repo.get_issues(state="open") - for issue in issues: - title = issue.title - if title.startswith("Future Release Branches Frozen For Merging"): - try: - branches_part = title.split('|', 1)[1].strip() - branch_tokens = branches_part.split() - if f"branch:{base_branch}" in branch_tokens: - return "main" - except Exception as e: - logging.warning(f"Failed to parse freeze issue title: {title} ({e})") - continue - return base_branch - - def cleanup_branches(gh_repo): """ Deletes branches with names in the format "rebase-4*" that are @@ -512,29 +299,11 @@ def cleanup_branches(gh_repo): _extra_msgs.append("Deleted following branches: " + ", ".join(deleted_branches)) -def get_token(org, repo): - """ - Returns a token to be used with GitHub API. - It's either Personal Access Token if TOKEN env is set, - or Installation Access Token which is intended to be used with GitHub Apps. - """ - personal_access_token = try_get_env(PAT_ENV, die=False) - if personal_access_token != "": - logging.info("Using Personal Access Token to access GitHub API") - return personal_access_token - - app_id = try_get_env(APP_ID_ENV) - key_path = try_get_env(KEY_ENV) - return get_installation_access_token(app_id, key_path, org, repo) - - def main(): """ The main function of the script. Reads environment variables, retrieves the necessary information from GitHub, performs a rebase, creates a pull request, and cleans up old branches. """ - org = try_get_env(ORG_ENV) - repo = try_get_env(REPO_ENV) release_amd = try_get_env(AMD64_RELEASE_ENV) release_arm = try_get_env(ARM64_RELEASE_ENV) rhoai_release = try_get_env(RHOAI_RELEASE_ENV) @@ -547,11 +316,9 @@ def main(): if REMOTE_DRY_RUN: logging.info("Dry run mode") - token = get_token(org, repo) - gh_repo = Github(token).get_repo(f"{org}/{repo}") - git_repo = Repo('.') + g = gitutils.GitUtils(dry_run=REMOTE_DRY_RUN) base_branch = ( - git_repo.active_branch.name + g.git_repo.active_branch.name if base_branch_override == "" else base_branch_override ) @@ -565,8 +332,8 @@ def main(): if rebases_succeeded: # TODO How can we inform team that rebase job ran successfully just there was nothing new? - make_sure_rebase_script_created_new_commits_or_exit(git_repo, base_branch) - if rebase_script_made_changes_considered_functional(git_repo, base_branch): + make_sure_rebase_script_created_new_commits_or_exit(g.git_repo, base_branch) + if rebase_script_made_changes_considered_functional(g.git_repo, base_branch): logging.info("Detected functional changes made by rebase script - proceeding with creating PR") else: logging.info("Rebase did not produce any change considered to be functional - quitting") @@ -580,47 +347,48 @@ def main(): "rebase_ai_model_serving.sh:\n" + f"{ai_rebase_result.output}") writer.write(output) - if git_repo.active_branch.name == base_branch: + if g.git_repo.active_branch.name == base_branch: # rebase.sh didn't reach the step that would create a branch # so script needs to create it - branch = git_repo.create_head(get_expected_branch_name(release_amd, release_arm)) - branch.checkout() - git_repo.git.add(A=True) - git_repo.index.commit("rebase.sh failure artifacts") + g.checkout_branch(get_expected_branch_name(release_amd, release_arm)) + g.add_files_to_staging_area(all=True) + g.commit("rebase.sh failure artifacts") - rebase_branch_name = git_repo.active_branch.name - adjusted_base_branch = get_base_branch_name(gh_repo, base_branch) + gh = ghutils.GithubUtils(dry_run=REMOTE_DRY_RUN) + + rebase_branch_name = g.git_repo.active_branch.name + adjusted_base_branch = "main" if gh.is_branch_under_active_development(rebase_branch_name) else rebase_branch_name logging.info(f"Adjusted base branch: {adjusted_base_branch}") - git_remote = get_remote_with_token(git_repo, token, org, repo) - remote_branch = try_get_rebase_branch_ref_from_remote(git_remote, rebase_branch_name) # {BOT_REMOTE_NAME}/{rebase_branch_name} + + g.setup_remote_with_token(gh.token, gh.org, gh.repo) + remote_branch = g.get_remote_branch(rebase_branch_name) # {BOT_REMOTE_NAME}/{rebase_branch_name} rbranch_does_not_exists = remote_branch is None rbranch_exists_and_needs_update = ( remote_branch is not None and - is_local_branch_based_on_newer_base_branch_commit(git_repo, adjusted_base_branch, remote_branch.name, rebase_branch_name) + g.is_local_branch_based_on_newer_base_branch_commit(adjusted_base_branch, remote_branch.name, rebase_branch_name) ) if rbranch_does_not_exists or rbranch_exists_and_needs_update: - push_branch_or_die(git_remote, rebase_branch_name) + g.push(rebase_branch_name) prow_job_url = try_create_prow_job_url() pr_title = create_pr_title(rebase_branch_name, rebase_result.success) desc = generate_pr_description(get_release_tag(release_amd), get_release_tag(release_arm), prow_job_url, rebase_result.success) comment = "" - pull_req = try_get_pr(gh_repo, org, adjusted_base_branch, rebase_branch_name) + pull_req = gh.get_existing_pr_for_a_branch(adjusted_base_branch, rebase_branch_name) if pull_req is None: - pull_req = create_pr(gh_repo, adjusted_base_branch, rebase_branch_name, pr_title, desc) + pull_req = gh.create_pr(adjusted_base_branch, rebase_branch_name, pr_title, desc) else: - update_pr(pull_req, pr_title, desc) + gh.update_pr(pull_req, pr_title, desc) comment = f"Rebase job updated the branch\n{desc}" if adjusted_base_branch == "main": - cleanup_branches(gh_repo) - post_comment(pull_req, comment) + cleanup_branches(gh.gh_repo) - post_comment(pull_req, '/verified by automatic-rebase') + gh.post_comment(pull_req, comment, _extra_msgs) - git_remote.remove(git_repo, BOT_REMOTE_NAME) + g.remove_remote_with_token() sys.exit(0 if rebases_succeeded else 1) diff --git a/scripts/pyutils/create-venv.sh b/scripts/pyutils/create-venv.sh new file mode 100755 index 0000000000..ed1b19f04d --- /dev/null +++ b/scripts/pyutils/create-venv.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# shellcheck disable=all +set -o nounset +set -o errexit +set -o pipefail + +SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +ROOT_DIR=$(realpath "${SCRIPT_DIR}/../..") +DEFAULT_DEST_DIR="${ROOT_DIR}/_output/pyutils" +DEST_DIR="${DEST_DIR:-${DEFAULT_DEST_DIR}}" + +if [ ! -d "${DEST_DIR}" ]; then + echo "Setting up virtualenv in ${DEST_DIR}" + python3 -m venv "${DEST_DIR}" + "${DEST_DIR}/bin/python3" -m pip install --upgrade pip + "${DEST_DIR}/bin/python3" -m pip install -r "${SCRIPT_DIR}/requirements.txt" +fi diff --git a/scripts/pyutils/ghutils.py b/scripts/pyutils/ghutils.py new file mode 100644 index 0000000000..6c9f8f67f8 --- /dev/null +++ b/scripts/pyutils/ghutils.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 + +import os +import logging +import sys +from pathlib import Path +from github import GithubIntegration, Github + +APP_ID_ENV = "APP_ID" # GitHub App's ID +KEY_ENV = "KEY" # Path to GitHub App's key +PAT_ENV = "GH_TOKEN" # Personal Access Token + +ORG_ENV = "ORG" +_DEFAULT_ORG = "openshift" +REPO_ENV = "REPO" +_DEFAULT_REPO = "microshift" + + +class GithubUtils: + def __init__(self, dry_run=False): + self.dry_run = dry_run + self.org, self.repo = self._get_org_repo_from_env() + self.token = self._get_gh_token_from_env() + self.gh_repo = Github(self.token).get_repo(f"{self.org}/{self.repo}") + + def _get_org_repo_from_env(self) -> tuple[str, str]: + if self.dry_run: + logging.info(f"[DRY RUN] Using default org and repo: {_DEFAULT_ORG}/{_DEFAULT_REPO}") + return _DEFAULT_ORG, _DEFAULT_REPO + return try_get_env(ORG_ENV, default=_DEFAULT_ORG), try_get_env(REPO_ENV, default=_DEFAULT_REPO) + + def _get_gh_token_from_env(self) -> str: + """ + Returns a token to be used with GitHub API. + It's either Personal Access Token if TOKEN env is set, + or Installation Access Token which is intended to be used with GitHub Apps. + """ + personal_access_token = try_get_env(PAT_ENV) + if personal_access_token != "": + logging.info("Using Personal Access Token to access GitHub API") + return personal_access_token + + app_id = try_get_env(APP_ID_ENV, die=True) + key_path = try_get_env(KEY_ENV, die=True) + integration = GithubIntegration(app_id, Path(key_path).read_text(encoding='utf-8')) + app_installation = integration.get_repo_installation(self.org, self.repo) + if app_installation is None: + sys.exit(f"Failed to get app_installation for {self.org}/{self.repo}. " + + f"Response: {app_installation.raw_data}") + return integration.get_access_token(app_installation.id).token + + def is_branch_under_active_development(self, branch): + """ + Checks title of the issue #1239 in the openshift/microshift repository to check if + given branch is frozen and thus under active development is happening on main branch. + + It returns True if given branch is the first on the list of frozen branches. + In such case the target (base) branch of newly created PR should be switch to main. + """ + if self.dry_run: + logging.info(f"[DRY RUN] Assuming branch {branch} is under active development") + return True + + issue = self.gh_repo.get_issue(number=1239) + title = issue.title + try: + branches_part = title.split('|', 1)[1].strip() + frozen_branches = [x.replace('branch:', '') for x in branches_part.split()] + if len(frozen_branches) == 0: + raise Exception(f"Unexpected amount of branch in the Issue 1239 title: {title}") + # Assuming the first branch name is the release under development right now. + # No job creating PRs should run against the next release branch. + return branch == frozen_branches[0] + except Exception as e: + raise RuntimeError(f"Failed to parse freeze issue title: {title} ({e})") + + def create_pr(self, base_branch, branch_name, title, desc): + """Create a Pull Request""" + if self.dry_run: + logging.info(f"[DRY RUN] Create PR: {base_branch=} <- {branch_name=}: {title=} {desc=}") + return None + + if (self.org == _DEFAULT_ORG and self.repo == _DEFAULT_REPO and + self.is_branch_under_active_development(branch_name)): + base_branch = "main" + + pull_req = self.gh_repo.create_pull( + title=title, body=desc, base=base_branch, head=branch_name, maintainer_can_modify=True) + logging.info(f"Created pull request: {pull_req.html_url}") + return pull_req + + def update_pr(self, pull_req, title, desc): + """Updates the title and description of a pull request on a GitHub repository.""" + if self.dry_run: + logging.info(f"[DRY RUN] Update PR #{pull_req.number}: {title}\n{desc}") + return + + pull_req.edit(title=title, body=desc) + pull_req.update() # arm64 release or prow job url might've changed + logging.info(f"Updated PR #{pull_req.number}: {title}\n{desc}") + + def post_comment(self, pull_req, comment="", extra_msgs=[]): + """Post a comment on a Pull Request with the contents of the extra_msgs list.""" + if self.dry_run: + logging.info(f"[DRY RUN] Post comment: {comment}") + return + + if len(extra_msgs) != 0: + if comment != "": + comment += "\n\n" + comment += "Extra messages:\n - " + "\n - ".join(extra_msgs) + + if comment.strip() != "": + logging.info(f"Comment to post: {comment}") + if self.dry_run: + logging.info("[DRY RUN] Posted a comment") + return + issue = pull_req.as_issue() + issue.create_comment(comment) + else: + logging.info("No content for comment") + + def get_existing_pr_for_a_branch(self, base_branch, branch_name): + """ + Try to get a pull request for a branch on a GitHub repository. + Returns + - The pull request if it exists and is open, otherwise None. + - If more than one pull request is found, then the first one will be used. + """ + prs = self.gh_repo.get_pulls(base=base_branch, head=f"{self.org}:{branch_name}", state="all") + + if prs.totalCount == 0: + logging.info(f"PR for branch {branch_name} does not exist yet on {self.gh_repo.full_name}") + return None + + pull_req = None + if prs.totalCount > 1: + pull_req = prs[0] + logging.warning( + f"Found more than one PR for branch {branch_name} on {self.gh_repo.full_name} -" + + f"this is unexpected, continuing with first one of: {[(x.state, x.html_url) for x in prs]}" + ) + + if prs.totalCount == 1: + pull_req = prs[0] + logging.info(f"Found PR #{pull_req.number} for branch {branch_name} on {self.gh_repo.full_name}: {pull_req.html_url}") + + if pull_req.state == 'closed': + logging.warning(f"PR #{pull_req.number} is not open - new PR will be created") + if pull_req.is_merged(): + logging.warning(f"PR #{pull_req.number} for '{branch_name}' branch is already merged but rebase.sh produced results") + else: + logging.warning(f"PR #{pull_req.number} for '{branch_name}' exists already but was closed") + return None + return pull_req + + +def try_get_env(var_name, default=None, die=False) -> str: + val = os.getenv(var_name) + if val is None or val == "": + if default is not None: + logging.info(f"'{var_name}' env var is unset, using '{default}'") + return default + if die: + raise Exception(f"Could not get environment variable '{var_name}'") + else: + logging.info(f"Could not get environment variable '{var_name}' - ignoring") + return "" + return val diff --git a/scripts/pyutils/gitutils.py b/scripts/pyutils/gitutils.py new file mode 100644 index 0000000000..eaa1705b3d --- /dev/null +++ b/scripts/pyutils/gitutils.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +import logging + +from git import PushInfo, Repo # GitPython + +BOT_REMOTE_NAME = "bot-creds" +REMOTE_ORIGIN = "origin" + + +class GitUtils(): + def __init__(self, dry_run=False): + self.dry_run = dry_run + self.git_repo = Repo(".") + self.remote = None + + def file_changed(self, file_path) -> bool: + changedFiles = [item.a_path for item in self.git_repo.index.diff(None)] + return file_path in changedFiles + + def add_files_to_staging_area(self, file_paths=[], all=False): + if self.dry_run: + if all: + logging.info("[DRY RUN] git add -A") + else: + logging.info(f"[DRY RUN] git add {file_paths}") + return + + if all: + self.git_repo.index.add(A=True) + else: + self.git_repo.index.add(file_paths) + + def commit(self, message): + if self.dry_run: + logging.info(f"[DRY RUN] git commit -m {message}") + return + self.git_repo.index.commit(message) + + def checkout_branch(self, branch_name): + if self.dry_run: + logging.info(f"[DRY RUN] git checkout -b {branch_name}") + return + new_branch = self.git_repo.create_head(branch_name) + new_branch.checkout() + + def setup_remote_with_token(self, token, org, repo): + """ + Sets up the Git remote for the given repository using + the provided installation or personal access token. + """ + if self.dry_run: + logging.info(f"[DRY RUN] git remote add {BOT_REMOTE_NAME} https://x-access-token:TOKEN@github.com/{org}/{repo}") + return + + remote_url = f"https://x-access-token:{token}@github.com/{org}/{repo}" + try: + remote = self.git_repo.remote(BOT_REMOTE_NAME) + remote.set_url(remote_url) + except ValueError: + self.git_repo.create_remote(BOT_REMOTE_NAME, remote_url) + + self.remote = self.git_repo.remote(BOT_REMOTE_NAME) + return self.remote + + def remove_remote_with_token(self): + if self.dry_run: + logging.info(f"[DRY RUN] git remote remove {BOT_REMOTE_NAME}") + return + self.remote.remove(self.git_repo, BOT_REMOTE_NAME) + + def push(self, branch_name): + if self.dry_run: + logging.info(f"[DRY RUN] git push --force {branch_name}") + return + + push_result = self.remote.push(branch_name, force=True) + + if len(push_result) != 1: + raise Exception(f"Unexpected amount ({len(push_result)}) of items in push_result: {push_result}") + if push_result[0].flags & PushInfo.ERROR: + raise Exception(f"Pushing branch failed: {push_result[0].summary}") + if push_result[0].flags & PushInfo.FORCED_UPDATE: + logging.info(f"Branch '{branch_name}' existed and was updated (force push)") + + def get_remote_branch(self, branch_name): + """ + Get the reference for the given branch on the specified Git remote, + otherwise return None if the branch does not exist. + """ + if self.dry_run: + return None + + self.remote.fetch() + matching_remote_refs = [ref for ref in self.remote.refs if BOT_REMOTE_NAME + "/" + branch_name == ref.name] + + if len(matching_remote_refs) == 0: + logging.info(f"Branch '{branch_name}' does not exist on remote") + return None + + if len(matching_remote_refs) > 1: + matching_branches = ", ".join([r.name for r in matching_remote_refs]) + logging.warning(f"Found more than one branch matching '{branch_name}' " + + f"on remote: {matching_branches}. Taking first one") + return matching_remote_refs[0] + + if len(matching_remote_refs) == 1: + logging.info(f"Branch '{branch_name}' already exists on remote") + return matching_remote_refs[0] + + return None + + def is_local_branch_based_on_newer_base_branch_commit(self, base_branch_name, remote_branch_name, local_branch_name): + """ + Compares local and remote rebase branches by looking at their start on base branch. + Returns True if local branch starts on newer commit and should to be pushed to remote, + otherwise False. + """ + if self.dry_run: + return True + + remote_merge_base = self.git_repo.merge_base(base_branch_name, remote_branch_name) + local_merge_base = self.git_repo.merge_base(base_branch_name, local_branch_name) + + if remote_merge_base[0] == local_merge_base[0]: + logging.info("Remote branch is up to date. " + + f"Branch-off commit: {commit_str(remote_merge_base[0])}") + return False + + logging.info( + f"Remote branch is older - it needs updating. " + f"Remote branch is on top of {base_branch_name}'s commit: '{commit_str(remote_merge_base[0])}'. " + f"Local branch is on top of {base_branch_name}'s commit '{commit_str(local_merge_base[0])}'" + ) + return True + + +def commit_str(commit): + """Returns the first 8 characters of the commit's SHA hash and the commit summary.""" + return f"{commit.hexsha[:8]} - {commit.summary}" diff --git a/scripts/pyutils/requirements.txt b/scripts/pyutils/requirements.txt new file mode 100644 index 0000000000..9291dfa8a6 --- /dev/null +++ b/scripts/pyutils/requirements.txt @@ -0,0 +1,2 @@ +pygithub==2.8.1 +gitpython==3.1.45 diff --git a/test/assets/common_versions.sh.template b/test/assets/common_versions.sh.template index 86eb52e609..c2a0dc7a70 100644 --- a/test/assets/common_versions.sh.template +++ b/test/assets/common_versions.sh.template @@ -46,7 +46,7 @@ get_vrel_from_rpm() {{ local -r rpm_dir="$1" # exit if rpm_dir directory does not exist - if [ ! -d "${rpm_dir}" ]; then + if [ ! -d "${{rpm_dir}}" ]; then echo "" return fi @@ -147,7 +147,7 @@ BREW_Y1_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{PREVIOUS BREW_Y2_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{YMINUS2_MINOR_VERSION}}-zstream/{ARCH}/")" BREW_RC_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{MINOR_VERSION}}-rc/{ARCH}/")" BREW_EC_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{MINOR_VERSION}}-ec/{ARCH}/")" -BREW_NIGHTLY_RELEASE_VERSION="$(get_vrel_from_rpm "${BREW_RPM_SOURCE}/4.${MINOR_VERSION}-nightly/${UNAME_M}/")" +BREW_NIGHTLY_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{MINOR_VERSION}}-nightly/{ARCH}/")" export BREW_Y0_RELEASE_VERSION export BREW_Y1_RELEASE_VERSION export BREW_Y2_RELEASE_VERSION diff --git a/test/bin/ci_phase_iso_build.sh b/test/bin/ci_phase_iso_build.sh index 0d52ceaefc..7d1b1541e1 100755 --- a/test/bin/ci_phase_iso_build.sh +++ b/test/bin/ci_phase_iso_build.sh @@ -144,6 +144,9 @@ cd "${ROOTDIR}/test/" # shellcheck source=test/bin/common.sh source "${SCRIPTDIR}/common.sh" +# shellcheck source=test/bin/common_versions_verify.sh +source "${SCRIPTDIR}/common_versions_verify.sh" + if ${COMPOSER_CLI_BUILDS} ; then # Determine and create the ideal number of workers $(dry_run) bash -x ./bin/manage_composer_config.sh create-workers diff --git a/test/bin/common_versions_verify.sh b/test/bin/common_versions_verify.sh new file mode 100644 index 0000000000..5fb443b273 --- /dev/null +++ b/test/bin/common_versions_verify.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Following script serves as a sort of presubmit to make sure that following files stay in sync: +# - test/bin/common_versions.sh +# - test/bin/pyutils/generate_common_versions.py +# - test/assets/common_versions.sh.template + +# Check if there was a change to common_versions.sh on current branch compared to base branch. +# --exit-code: 0 means no changes, 1 means changes hence the inverted condition with '!' +# Because the top commit is Merge Commit of PR branch into base branch (e.g. main), we need to check against the earlier commit (i.e. ^1). +if ! git diff --exit-code "${SCENARIO_BUILD_BRANCH}^1...HEAD" "${ROOTDIR}/test/bin/common_versions.sh"; then + # If the file was changed, regenerate it and compare - diff means that most likely the file was updated manually. + "${ROOTDIR}/test/bin/pyutils/generate_common_versions.py" --update-file + if ! git diff --exit-code "${ROOTDIR}/test/bin/common_versions.sh"; then + echo "ERROR: Discovered that common_versions.sh was updated on the branch under test, but the regenerated version is different" + git diff + exit 1 + fi +fi diff --git a/test/bin/pyutils/generate_common_versions.py b/test/bin/pyutils/generate_common_versions.py index e2f20a4805..c230f231ff 100644 --- a/test/bin/pyutils/generate_common_versions.py +++ b/test/bin/pyutils/generate_common_versions.py @@ -11,6 +11,10 @@ import logging import pathlib +sys.path.append(str(pathlib.Path(__file__).resolve().parent / '../../../scripts/pyutils')) +import gitutils # noqa: E402 +import ghutils # noqa: E402 + ARCH = os.uname().machine # The version of Sonobuoy package used in CNCF tests. @@ -28,7 +32,7 @@ logging.basicConfig( level=logging.INFO, - format='%(levelname)s: %(message)s', + format='%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', stream=sys.stderr ) @@ -96,7 +100,11 @@ def repo_provides_pkg(repo, pkg): try: logging.info(f"Running command: {' '.join(args)}") - subprocess.run(args, stdout=sys.stderr, check=True) + result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, check=True) + output = result.stdout.strip() + logging.info(f"Command's output:\n{output}") + if "Usable URL not found" in output: + return False return True except subprocess.CalledProcessError: return False @@ -203,83 +211,128 @@ def get_release_version_string(repo, var_name): return None -parser = argparse.ArgumentParser(description="Generate common_versions.sh variables.") -parser.add_argument("minor", type=int, help="The minor version number.") - -args = parser.parse_args() - -minor_version = args.minor -previous_minor_version = minor_version - 1 -yminus2_minor_version = minor_version - 2 - -# The current release repository comes from the 'rhocp' stream for release -# branches, or the OpenShift mirror if only a RC or EC is available. It can -# be empty, if no candidate for the current minor has been built yet. -logging.info("Getting CURRENT_RELEASE_REPO") -current_release_repo = get_microshift_repo(minor_version) -current_release_version = get_release_version_string(current_release_repo, "CURRENT_RELEASE_REPO") - -# The previous release repository value should either point to the OpenShift -# mirror URL or the 'rhocp' repository name. -logging.info("Getting PREVIOUS_RELEASE_REPO") -previous_release_repo = get_microshift_repo(previous_minor_version) -previous_release_version = get_release_version_string(previous_release_repo, "PREVIOUS_RELEASE_REPO") - -# The y-2 release repository value should either point to the OpenShift -# mirror URL or the 'rhocp' repository name. It should always come from -# the 'rhocp' stream. -logging.info("Getting YMINUS2_RELEASE_REPO") -yminus2_release_repo = get_microshift_repo(yminus2_minor_version) -yminus2_release_version = get_release_version_string(yminus2_release_repo, "YMINUS2_RELEASE_REPO") - -# The 'rhocp_minor_y' variable should be the minor version number, if the -# current release is available through the 'rhocp' stream, otherwise empty. -rhocp_minor_y = minor_version if repo_provides_pkg(f"rhocp-4.{minor_version}-for-rhel-9-{ARCH}-rpms", "cri-o") else '""' - -# The beta repository, containing dependencies, should point to the -# OpenShift mirror URL. If the mirror for current minor is not -# available yet, it should point to an older release. -logging.info("Getting RHOCP_MINOR_Y_BETA") -rhocp_minor_y_beta = get_dependencies_repo_url(minor_version, 3) - -# The 'rhocp_minor_y' variable should be the previous minor version number, if -# the previous release is available through the 'rhocp' stream, otherwise empty. -rhocp_minor_y1 = previous_minor_version if repo_provides_pkg(f"rhocp-4.{previous_minor_version}-for-rhel-9-{ARCH}-rpms", "cri-o") else '""' - -# The beta repository, containing dependencies, should point to the -# OpenShift mirror URL. The mirror for previous release should always -# be available. -logging.info("Getting RHOCP_MINOR_Y1_BETA") -rhocp_minor_y1_beta = get_dependencies_repo_url(previous_minor_version) - -# The 'rhocp_minor_y2' should always be the y-2 minor version number. -rhocp_minor_y2 = yminus2_minor_version - -template_path = pathlib.Path(__file__).resolve().parent / '../../assets/common_versions.sh.template' - -with open(template_path, 'r') as f: - template_string = f.read() - -output = template_string.format( - minor_version=minor_version, - current_release_repo=current_release_repo, - current_release_version=current_release_version, - previous_release_repo=previous_release_repo, - previous_release_version=previous_release_version, - yminus2_release_repo=yminus2_release_repo, - yminus2_release_version=yminus2_release_version, - rhocp_minor_y=rhocp_minor_y, - rhocp_minor_y_beta=rhocp_minor_y_beta, - rhocp_minor_y1=rhocp_minor_y1, - rhocp_minor_y1_beta=rhocp_minor_y1_beta, - rhocp_minor_y2=rhocp_minor_y2, - CNCF_SONOBUOY_VERSION=CNCF_SONOBUOY_VERSION, - CNCF_SYSTEMD_LOGS_VERSION=CNCF_SYSTEMD_LOGS_VERSION, - GITOPS_VERSION=GITOPS_VERSION, - LATEST_RELEASE_TYPE=LATEST_RELEASE_TYPE, - ARCH=ARCH -) - -output_noarch = output.replace(ARCH, '${UNAME_M}') - -print(output_noarch) +def generate_common_versions(minor_version): + previous_minor_version = minor_version - 1 + yminus2_minor_version = minor_version - 2 + + # The current release repository comes from the 'rhocp' stream for release + # branches, or the OpenShift mirror if only a RC or EC is available. It can + # be empty, if no candidate for the current minor has been built yet. + logging.info("Getting CURRENT_RELEASE_REPO") + current_release_repo = get_microshift_repo(minor_version) + current_release_version = get_release_version_string(current_release_repo, "CURRENT_RELEASE_REPO") + + # The previous release repository value should either point to the OpenShift + # mirror URL or the 'rhocp' repository name. + logging.info("Getting PREVIOUS_RELEASE_REPO") + previous_release_repo = get_microshift_repo(previous_minor_version) + previous_release_version = get_release_version_string(previous_release_repo, "PREVIOUS_RELEASE_REPO") + + # The y-2 release repository value should either point to the OpenShift + # mirror URL or the 'rhocp' repository name. It should always come from + # the 'rhocp' stream. + logging.info("Getting YMINUS2_RELEASE_REPO") + yminus2_release_repo = get_microshift_repo(yminus2_minor_version) + yminus2_release_version = get_release_version_string(yminus2_release_repo, "YMINUS2_RELEASE_REPO") + + # The 'rhocp_minor_y' variable should be the minor version number, if the + # current release is available through the 'rhocp' stream, otherwise empty. + rhocp_minor_y = minor_version if repo_provides_pkg(f"rhocp-4.{minor_version}-for-rhel-9-{ARCH}-rpms", "cri-o") else '""' + + # The beta repository, containing dependencies, should point to the + # OpenShift mirror URL. If the mirror for current minor is not + # available yet, it should point to an older release. + logging.info("Getting RHOCP_MINOR_Y_BETA") + rhocp_minor_y_beta = get_dependencies_repo_url(minor_version, 3) + + # The 'rhocp_minor_y' variable should be the previous minor version number, if + # the previous release is available through the 'rhocp' stream, otherwise empty. + rhocp_minor_y1 = previous_minor_version if repo_provides_pkg(f"rhocp-4.{previous_minor_version}-for-rhel-9-{ARCH}-rpms", "cri-o") else '""' + + # The beta repository, containing dependencies, should point to the + # OpenShift mirror URL. The mirror for previous release should always + # be available. + logging.info("Getting RHOCP_MINOR_Y1_BETA") + rhocp_minor_y1_beta = get_dependencies_repo_url(previous_minor_version) + + # The 'rhocp_minor_y2' should always be the y-2 minor version number. + rhocp_minor_y2 = yminus2_minor_version + + template_path = pathlib.Path(__file__).resolve().parent / '../../assets/common_versions.sh.template' + + with open(template_path, 'r') as f: + template_string = f.read() + + output = template_string.format( + minor_version=minor_version, + current_release_repo=current_release_repo, + current_release_version=current_release_version, + previous_release_repo=previous_release_repo, + previous_release_version=previous_release_version, + yminus2_release_repo=yminus2_release_repo, + yminus2_release_version=yminus2_release_version, + rhocp_minor_y=rhocp_minor_y, + rhocp_minor_y_beta=rhocp_minor_y_beta, + rhocp_minor_y1=rhocp_minor_y1, + rhocp_minor_y1_beta=rhocp_minor_y1_beta, + rhocp_minor_y2=rhocp_minor_y2, + CNCF_SONOBUOY_VERSION=CNCF_SONOBUOY_VERSION, + CNCF_SYSTEMD_LOGS_VERSION=CNCF_SYSTEMD_LOGS_VERSION, + GITOPS_VERSION=GITOPS_VERSION, + LATEST_RELEASE_TYPE=LATEST_RELEASE_TYPE, + ARCH=ARCH + ) + + output_noarch = output.replace(ARCH, '${UNAME_M}') + + return output_noarch + + +def main(): + parser = argparse.ArgumentParser(description="Generate common_versions.sh variables.") + parser.add_argument("minor", type=int, help="The minor version number.") + parser.add_argument("--update-file", default=False, action="store_true", help="Update test/bin/common_versions.sh file.") + parser.add_argument("--create-pr", default=False, action="store_true", + help=("Commit the changes to a new branch, push it to the openshift/microshift, and create a pull request." + + "Implies --update-file. Expects following env vars to be set: ORG, REPO, GH_TOKEN or APP_ID and KEY")) + parser.add_argument("--dry-run", default=False, action="store_true", help="Dry run") + args = parser.parse_args() + + output = generate_common_versions(args.minor) + + if args.update_file or args.create_pr: + logging.info("Updating test/bin/common_versions.sh file") + dest_file = pathlib.Path(__file__).resolve().parent / '../common_versions.sh' + with open(dest_file, 'w') as f: + f.write(output) + else: + print(output) + + if args.create_pr: + g = gitutils.GitUtils(dry_run=args.dry_run) + if not g.file_changed("test/bin/common_versions.sh"): + logging.info("No changes to test/bin/common_versions.sh") + exit(0) + + base_branch = g.git_repo.active_branch.name + if not base_branch.startswith("release-4"): + logging.error(f"Script is expected to be executed on branch starting with 'release-4', but it's {base_branch}") + exit(1) + + gh = ghutils.GithubUtils(dry_run=args.dry_run) + g.setup_remote_with_token(gh.token, gh.org, gh.repo) + new_branch_name = f"{base_branch}-common-versions-update" + g.checkout_branch(new_branch_name) + g.add_files_to_staging_area(["test/bin/common_versions.sh"]) + g.commit("Update common_versions.sh") + g.push(new_branch_name) + + pull_req = gh.get_existing_pr_for_a_branch(base_branch, new_branch_name) + if pull_req is None: + # Assuming the script always runs against `release-4.y` branch for the value in brackets. + pr_title = f"[{base_branch}] NO-ISSUE: Update common_versions.sh" + gh.create_pr(base_branch, new_branch_name, pr_title, "") + + +if __name__ == "__main__": + main()