USHIFT-6209: Automate update of common_version.sh

scripts/pyutils/create-venv.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=all
+set -o nounset
+set -o errexit
+set -o pipefail
+
+SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
+ROOT_DIR=$(realpath "${SCRIPT_DIR}/../..")
+DEFAULT_DEST_DIR="${ROOT_DIR}/_output/pyutils"
+DEST_DIR="${DEST_DIR:-${DEFAULT_DEST_DIR}}"
+
+if [ ! -d "${DEST_DIR}" ]; then
+    echo "Setting up virtualenv in ${DEST_DIR}"
+    python3 -m venv "${DEST_DIR}"
+    "${DEST_DIR}/bin/python3" -m pip install --upgrade pip
+    "${DEST_DIR}/bin/python3" -m pip install -r "${SCRIPT_DIR}/requirements.txt"
+fi

scripts/pyutils/ghutils.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+
+import os
+import logging
+import sys
+from pathlib import Path
+from github import GithubIntegration, Github
+
+APP_ID_ENV = "APP_ID"   # GitHub App's ID
+KEY_ENV = "KEY"         # Path to GitHub App's key
+PAT_ENV = "GH_TOKEN"    # Personal Access Token
+
+ORG_ENV = "ORG"
+_DEFAULT_ORG = "openshift"
+REPO_ENV = "REPO"
+_DEFAULT_REPO = "microshift"
+
+
+class GithubUtils:
+    def __init__(self, dry_run=False):
+        self.dry_run = dry_run
+        self.org, self.repo = self._get_org_repo_from_env()
+        self.token = self._get_gh_token_from_env()
+        self.gh_repo = Github(self.token).get_repo(f"{self.org}/{self.repo}")
+
+    def _get_org_repo_from_env(self) -> tuple[str, str]:
+        if self.dry_run:
+            logging.info(f"[DRY RUN] Using default org and repo: {_DEFAULT_ORG}/{_DEFAULT_REPO}")
+            return _DEFAULT_ORG, _DEFAULT_REPO
+        return try_get_env(ORG_ENV, default=_DEFAULT_ORG), try_get_env(REPO_ENV, default=_DEFAULT_REPO)
+
+    def _get_gh_token_from_env(self) -> str:
+        """
+        Returns a token to be used with GitHub API.
+        It's either Personal Access Token if TOKEN env is set,
+        or Installation Access Token which is intended to be used with GitHub Apps.
+        """
+        personal_access_token = try_get_env(PAT_ENV)
+        if personal_access_token != "":
+            logging.info("Using Personal Access Token to access GitHub API")
+            return personal_access_token
+
+        app_id = try_get_env(APP_ID_ENV, die=True)
+        key_path = try_get_env(KEY_ENV, die=True)
+        integration = GithubIntegration(app_id, Path(key_path).read_text(encoding='utf-8'))
+        app_installation = integration.get_repo_installation(self.org, self.repo)
+        if app_installation is None:
+            sys.exit(f"Failed to get app_installation for {self.org}/{self.repo}. " +
+                     f"Response: {app_installation.raw_data}")
+        return integration.get_access_token(app_installation.id).token
+
+    def is_branch_under_active_development(self, branch):
+        """
+        Checks title of the issue #1239 in the openshift/microshift repository to check if
+        given branch is frozen and thus under active development is happening on main branch.
+
+        It returns True if given branch is the first on the list of frozen branches.
+        In such case the target (base) branch of newly created PR should be switch to main.
+        """
+        if self.dry_run:
+            logging.info(f"[DRY RUN] Assuming branch {branch} is under active development")
+            return True
+
+        issue = self.gh_repo.get_issue(number=1239)
+        title = issue.title
+        try:
+            branches_part = title.split('|', 1)[1].strip()
+            frozen_branches = [x.replace('branch:', '') for x in branches_part.split()]
+            if len(frozen_branches) == 0:
+                raise Exception(f"Unexpected amount of branch in the Issue 1239 title: {title}")
+            # Assuming the first branch name is the release under development right now.
+            # No job creating PRs should run against the next release branch.
+            return branch == frozen_branches[0]
+        except Exception as e:
+            raise RuntimeError(f"Failed to parse freeze issue title: {title} ({e})")
+
+    def create_pr(self, base_branch, branch_name, title, desc):
+        """Create a Pull Request"""
+        if self.dry_run:
+            logging.info(f"[DRY RUN] Create PR: {base_branch=} <- {branch_name=}: {title=} {desc=}")
+            return None
+
+        if (self.org == _DEFAULT_ORG and self.repo == _DEFAULT_REPO and
+                self.is_branch_under_active_development(branch_name)):
+            base_branch = "main"
+
+        pull_req = self.gh_repo.create_pull(
+            title=title, body=desc, base=base_branch, head=branch_name, maintainer_can_modify=True)
+        logging.info(f"Created pull request: {pull_req.html_url}")
+        return pull_req
+
+    def update_pr(self, pull_req, title, desc):
+        """Updates the title and description of a pull request on a GitHub repository."""
+        if self.dry_run:
+            logging.info(f"[DRY RUN] Update PR #{pull_req.number}: {title}\n{desc}")
+            return
+
+        pull_req.edit(title=title, body=desc)
+        pull_req.update()  # arm64 release or prow job url might've changed
+        logging.info(f"Updated PR #{pull_req.number}: {title}\n{desc}")
+
+    def post_comment(self, pull_req, comment="", extra_msgs=[]):
+        """Post a comment on a Pull Request with the contents of the extra_msgs list."""
+        if self.dry_run:
+            logging.info(f"[DRY RUN] Post comment: {comment}")
+            return
+
+        if len(extra_msgs) != 0:
+            if comment != "":
+                comment += "\n\n"
+            comment += "Extra messages:\n - " + "\n - ".join(extra_msgs)
+
+        if comment.strip() != "":
+            logging.info(f"Comment to post: {comment}")
+            if self.dry_run:
+                logging.info("[DRY RUN] Posted a comment")
+                return
+            issue = pull_req.as_issue()
+            issue.create_comment(comment)
+        else:
+            logging.info("No content for comment")
+
+    def get_existing_pr_for_a_branch(self, base_branch, branch_name):
+        """
+        Try to get a pull request for a branch on a GitHub repository.
+        Returns
+        - The pull request if it exists and is open, otherwise None.
+        - If more than one pull request is found, then the first one will be used.
+        """
+        prs = self.gh_repo.get_pulls(base=base_branch, head=f"{self.org}:{branch_name}", state="all")
+
+        if prs.totalCount == 0:
+            logging.info(f"PR for branch {branch_name} does not exist yet on {self.gh_repo.full_name}")
+            return None
+
+        pull_req = None
+        if prs.totalCount > 1:
+            pull_req = prs[0]
+            logging.warning(
+                f"Found more than one PR for branch {branch_name} on {self.gh_repo.full_name} -" +
+                f"this is unexpected, continuing with first one of: {[(x.state, x.html_url) for x in prs]}"
+            )
+
+        if prs.totalCount == 1:
+            pull_req = prs[0]
+            logging.info(f"Found PR #{pull_req.number} for branch {branch_name} on {self.gh_repo.full_name}: {pull_req.html_url}")
+
+        if pull_req.state == 'closed':
+            logging.warning(f"PR #{pull_req.number} is not open - new PR will be created")
+            if pull_req.is_merged():
+                logging.warning(f"PR #{pull_req.number} for '{branch_name}' branch is already merged but rebase.sh produced results")
+            else:
+                logging.warning(f"PR #{pull_req.number} for '{branch_name}' exists already but was closed")
+            return None
+        return pull_req
+
+
+def try_get_env(var_name, default=None, die=False) -> str:
+    val = os.getenv(var_name)
+    if val is None or val == "":
+        if default is not None:
+            logging.info(f"'{var_name}' env var is unset, using '{default}'")
+            return default
+        if die:
+            raise Exception(f"Could not get environment variable '{var_name}'")
+        else:
+            logging.info(f"Could not get environment variable '{var_name}' - ignoring")
+            return ""
+    return val

scripts/pyutils/gitutils.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+
+import logging
+
+from git import PushInfo, Repo  # GitPython
+
+BOT_REMOTE_NAME = "bot-creds"
+REMOTE_ORIGIN = "origin"
+
+
+class GitUtils():
+    def __init__(self, dry_run=False):
+        self.dry_run = dry_run
+        self.git_repo = Repo(".")
+        self.remote = None
+
+    def file_changed(self, file_path) -> bool:
+        changedFiles = [item.a_path for item in self.git_repo.index.diff(None)]
+        return file_path in changedFiles
+
+    def add_files_to_staging_area(self, file_paths=[], all=False):
+        if self.dry_run:
+            if all:
+                logging.info("[DRY RUN] git add -A")
+            else:
+                logging.info(f"[DRY RUN] git add {file_paths}")
+            return
+
+        if all:
+            self.git_repo.index.add(A=True)
+        else:
+            self.git_repo.index.add(file_paths)
+
+    def commit(self, message):
+        if self.dry_run:
+            logging.info(f"[DRY RUN] git commit -m {message}")
+            return
+        self.git_repo.index.commit(message)
+
+    def checkout_branch(self, branch_name):
+        if self.dry_run:
+            logging.info(f"[DRY RUN] git checkout -b {branch_name}")
+            return
+        new_branch = self.git_repo.create_head(branch_name)
+        new_branch.checkout()
+
+    def setup_remote_with_token(self, token, org, repo):
+        """
+        Sets up the Git remote for the given repository using
+        the provided installation or personal access token.
+        """
+        if self.dry_run:
+            logging.info(f"[DRY RUN] git remote add {BOT_REMOTE_NAME} https://x-access-token:TOKEN@github.com/{org}/{repo}")
+            return
+
+        remote_url = f"https://x-access-token:{token}@github.com/{org}/{repo}"
+        try:
+            remote = self.git_repo.remote(BOT_REMOTE_NAME)
+            remote.set_url(remote_url)
+        except ValueError:
+            self.git_repo.create_remote(BOT_REMOTE_NAME, remote_url)
+
+        self.remote = self.git_repo.remote(BOT_REMOTE_NAME)
+        return self.remote
+
+    def remove_remote_with_token(self):
+        if self.dry_run:
+            logging.info(f"[DRY RUN] git remote remove {BOT_REMOTE_NAME}")
+            return
+        self.remote.remove(self.git_repo, BOT_REMOTE_NAME)
+
+    def push(self, branch_name):
+        if self.dry_run:
+            logging.info(f"[DRY RUN] git push --force {branch_name}")
+            return
+
+        push_result = self.remote.push(branch_name, force=True)
+
+        if len(push_result) != 1:
+            raise Exception(f"Unexpected amount ({len(push_result)}) of items in push_result: {push_result}")
+        if push_result[0].flags & PushInfo.ERROR:
+            raise Exception(f"Pushing branch failed: {push_result[0].summary}")
+        if push_result[0].flags & PushInfo.FORCED_UPDATE:
+            logging.info(f"Branch '{branch_name}' existed and was updated (force push)")
+
+    def get_remote_branch(self, branch_name):
+        """
+        Get the reference for the given branch on the specified Git remote,
+        otherwise return None if the branch does not exist.
+        """
+        if self.dry_run:
+            return None
+
+        self.remote.fetch()
+        matching_remote_refs = [ref for ref in self.remote.refs if BOT_REMOTE_NAME + "/" + branch_name == ref.name]
+
+        if len(matching_remote_refs) == 0:
+            logging.info(f"Branch '{branch_name}' does not exist on remote")
+            return None
+
+        if len(matching_remote_refs) > 1:
+            matching_branches = ", ".join([r.name for r in matching_remote_refs])
+            logging.warning(f"Found more than one branch matching '{branch_name}' " +
+                            f"on remote: {matching_branches}. Taking first one")
+            return matching_remote_refs[0]
+
+        if len(matching_remote_refs) == 1:
+            logging.info(f"Branch '{branch_name}' already exists on remote")
+            return matching_remote_refs[0]
+
+        return None
+
+    def is_local_branch_based_on_newer_base_branch_commit(self, base_branch_name, remote_branch_name, local_branch_name):
+        """
+        Compares local and remote rebase branches by looking at their start on base branch.
+        Returns True if local branch starts on newer commit and should to be pushed to remote,
+        otherwise False.
+        """
+        if self.dry_run:
+            return True
+
+        remote_merge_base = self.git_repo.merge_base(base_branch_name, remote_branch_name)
+        local_merge_base = self.git_repo.merge_base(base_branch_name, local_branch_name)
+
+        if remote_merge_base[0] == local_merge_base[0]:
+            logging.info("Remote branch is up to date. " +
+                         f"Branch-off commit: {commit_str(remote_merge_base[0])}")
+            return False
+
+        logging.info(
+            f"Remote branch is older - it needs updating. "
+            f"Remote branch is on top of {base_branch_name}'s commit: '{commit_str(remote_merge_base[0])}'. "
+            f"Local branch is on top of {base_branch_name}'s commit '{commit_str(local_merge_base[0])}'"
+        )
+        return True
+
+
+def commit_str(commit):
+    """Returns the first 8 characters of the commit's SHA hash and the commit summary."""
+    return f"{commit.hexsha[:8]} - {commit.summary}"

scripts/pyutils/requirements.txt

@@ -0,0 +1,2 @@
+pygithub==2.8.1
+gitpython==3.1.45

test/assets/common_versions.sh.template

@@ -46,7 +46,7 @@ get_vrel_from_rpm() {{
     local -r rpm_dir="$1"
 
     # exit if rpm_dir directory does not exist
-    if [ ! -d "${rpm_dir}" ]; then
+    if [ ! -d "${{rpm_dir}}" ]; then
         echo ""
         return
     fi
@@ -147,7 +147,7 @@ BREW_Y1_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{PREVIOUS
 BREW_Y2_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{YMINUS2_MINOR_VERSION}}-zstream/{ARCH}/")"
 BREW_RC_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{MINOR_VERSION}}-rc/{ARCH}/")"
 BREW_EC_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{MINOR_VERSION}}-ec/{ARCH}/")"
-BREW_NIGHTLY_RELEASE_VERSION="$(get_vrel_from_rpm "${BREW_RPM_SOURCE}/4.${MINOR_VERSION}-nightly/${UNAME_M}/")"
+BREW_NIGHTLY_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{MINOR_VERSION}}-nightly/{ARCH}/")"
 export BREW_Y0_RELEASE_VERSION
 export BREW_Y1_RELEASE_VERSION
 export BREW_Y2_RELEASE_VERSION

test/bin/ci_phase_iso_build.sh

@@ -144,6 +144,9 @@ cd "${ROOTDIR}/test/"
 # shellcheck source=test/bin/common.sh
 source "${SCRIPTDIR}/common.sh"
 
+# shellcheck source=test/bin/common_versions_verify.sh
+source "${SCRIPTDIR}/common_versions_verify.sh"
+
 if ${COMPOSER_CLI_BUILDS} ; then
     # Determine and create the ideal number of workers
     $(dry_run) bash -x ./bin/manage_composer_config.sh create-workers

test/bin/common_versions_verify.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Following script serves as a sort of presubmit to make sure that following files stay in sync:
+# - test/bin/common_versions.sh
+# - test/bin/pyutils/generate_common_versions.py
+# - test/assets/common_versions.sh.template
+
+# Check if there was a change to common_versions.sh on current branch compared to base branch.
+# --exit-code: 0 means no changes, 1 means changes hence the inverted condition with '!'
+# Because the top commit is Merge Commit of PR branch into base branch (e.g. main), we need to check against the earlier commit (i.e. ^1).
+if ! git diff --exit-code "${SCENARIO_BUILD_BRANCH}^1...HEAD" "${ROOTDIR}/test/bin/common_versions.sh"; then
+    # If the file was changed, regenerate it and compare - diff means that most likely the file was updated manually.
+    "${ROOTDIR}/test/bin/pyutils/generate_common_versions.py" --update-file
+    if ! git diff --exit-code "${ROOTDIR}/test/bin/common_versions.sh"; then
+        echo "ERROR: Discovered that common_versions.sh was updated on the branch under test, but the regenerated version is different"
+        git diff
+        exit 1
+    fi
+fi