USHIFT-6205: Add MicroShift brew nightly RPMs into periodics scenarios

test/assets/common_versions.sh.template

@@ -45,6 +45,12 @@ get_vrel_from_rhsm() {{
 get_vrel_from_rpm() {{
     local -r rpm_dir="$1"
 
+    # exit if rpm_dir directory does not exist
+    if [ ! -d "${rpm_dir}" ]; then
+        echo ""
+        return
+    fi
+
     local -r rpm_release_info_file=$(find "${{rpm_dir}}" -name "microshift-release-info-*.rpm" | sort | tail -n1)
     if [ -z "${{rpm_release_info_file}}" ]; then
         echo ""
@@ -141,11 +147,13 @@ BREW_Y1_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{PREVIOUS
 BREW_Y2_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{YMINUS2_MINOR_VERSION}}-zstream/{ARCH}/")"
 BREW_RC_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{MINOR_VERSION}}-rc/{ARCH}/")"
 BREW_EC_RELEASE_VERSION="$(get_vrel_from_rpm "${{BREW_RPM_SOURCE}}/4.${{MINOR_VERSION}}-ec/{ARCH}/")"
+BREW_NIGHTLY_RELEASE_VERSION="$(get_vrel_from_rpm "${BREW_RPM_SOURCE}/4.${MINOR_VERSION}-nightly/${UNAME_M}/")"
 export BREW_Y0_RELEASE_VERSION
 export BREW_Y1_RELEASE_VERSION
 export BREW_Y2_RELEASE_VERSION
 export BREW_RC_RELEASE_VERSION
 export BREW_EC_RELEASE_VERSION
+export BREW_NIGHTLY_RELEASE_VERSION
 
 # Set the release type to ec, rc or zstream
 LATEST_RELEASE_TYPE="{LATEST_RELEASE_TYPE}"

test/bin/build_images.sh

@@ -721,6 +721,7 @@ if ${EXTRACT_CONTAINER_IMAGES}; then
     [ -n "${BREW_Y2_RELEASE_VERSION}" ] && extract_container_images "${BREW_Y2_RELEASE_VERSION}" "${BREW_REPO}" "${CONTAINER_LIST}"
     [ -n "${BREW_RC_RELEASE_VERSION}" ] && extract_container_images "${BREW_RC_RELEASE_VERSION}" "${BREW_REPO}" "${CONTAINER_LIST}"
     [ -n "${BREW_EC_RELEASE_VERSION}" ] && extract_container_images "${BREW_EC_RELEASE_VERSION}" "${BREW_REPO}" "${CONTAINER_LIST}"
+    [ -n "${BREW_NIGHTLY_RELEASE_VERSION}" ] && extract_container_images "${BREW_NIGHTLY_RELEASE_VERSION}" "${BREW_REPO}" "${CONTAINER_LIST}"
 fi
 
 trap 'osbuild_logs' EXIT

test/bin/build_rpms.sh

@@ -121,12 +121,12 @@ download_brew_rpms() {
         rm -rf "${BREW_RPM_SOURCE}"
         # Run the download procedure
         bash -x "${SCRIPTDIR}/../../scripts/fetch_tools.sh" brew
-        bash -x "${SCRIPTDIR}/manage_brew_rpms.sh" download "4.${MINOR_VERSION}" "${BREW_RPM_SOURCE}"
         bash -x "${SCRIPTDIR}/manage_brew_rpms.sh" download "4.${MINOR_VERSION}" "${BREW_RPM_SOURCE}" "zstream" || true
         bash -x "${SCRIPTDIR}/manage_brew_rpms.sh" download "4.${PREVIOUS_MINOR_VERSION}" "${BREW_RPM_SOURCE}" "zstream" || true
         bash -x "${SCRIPTDIR}/manage_brew_rpms.sh" download "4.${YMINUS2_MINOR_VERSION}" "${BREW_RPM_SOURCE}" "zstream" || true
         bash -x "${SCRIPTDIR}/manage_brew_rpms.sh" download "4.${MINOR_VERSION}" "${BREW_RPM_SOURCE}" "rc" || true
         bash -x "${SCRIPTDIR}/manage_brew_rpms.sh" download "4.${MINOR_VERSION}" "${BREW_RPM_SOURCE}" "ec" || true
+        bash -x "${SCRIPTDIR}/manage_brew_rpms.sh" download "4.${MINOR_VERSION}" "${BREW_RPM_SOURCE}" "nightly" || true
     else
         echo "WARNING: The Brew Hub site is not accessible, skipping the download"
     fi

test/bin/common_versions.sh

@@ -147,11 +147,13 @@ BREW_Y1_RELEASE_VERSION="$(get_vrel_from_rpm "${BREW_RPM_SOURCE}/4.${PREVIOUS_MI
 BREW_Y2_RELEASE_VERSION="$(get_vrel_from_rpm "${BREW_RPM_SOURCE}/4.${YMINUS2_MINOR_VERSION}-zstream/${UNAME_M}/")"
 BREW_RC_RELEASE_VERSION="$(get_vrel_from_rpm "${BREW_RPM_SOURCE}/4.${MINOR_VERSION}-rc/${UNAME_M}/")"
 BREW_EC_RELEASE_VERSION="$(get_vrel_from_rpm "${BREW_RPM_SOURCE}/4.${MINOR_VERSION}-ec/${UNAME_M}/")"
+BREW_NIGHTLY_RELEASE_VERSION="$(get_vrel_from_rpm "${BREW_RPM_SOURCE}/4.${MINOR_VERSION}-nightly/${UNAME_M}/")"
 export BREW_Y0_RELEASE_VERSION
 export BREW_Y1_RELEASE_VERSION
 export BREW_Y2_RELEASE_VERSION
 export BREW_RC_RELEASE_VERSION
 export BREW_EC_RELEASE_VERSION
+export BREW_NIGHTLY_RELEASE_VERSION
 
 # Set the release type to ec, rc or zstream
 LATEST_RELEASE_TYPE="ec"
@@ -165,4 +167,3 @@ OPENSHIFT_TESTS_PRIVATE_REPO_BRANCH="release-4.${MINOR_VERSION}"
 OPENSHIFT_TESTS_PRIVATE_REPO_COMMIT="61613d96c91db7b2907c24dd257075d3f2201991"
 export OPENSHIFT_TESTS_PRIVATE_REPO_BRANCH
 export OPENSHIFT_TESTS_PRIVATE_REPO_COMMIT
-

test/bin/manage_brew_rpms.sh

@@ -32,7 +32,7 @@ action_access() {
 action_download() {
     local -r ver=$1
     local -r dir=$2
-    local -r ver_type=${3:-ec}
+    local -r ver_type=${3:-nightly}
 
     if [ -z "${ver}" ] || [ -z "${dir}" ] ; then
         echo "ERROR: At least two parameters (version and path) are required"
@@ -48,14 +48,17 @@ action_download() {
     # Attempt downloading the specified build version
     local package
     case ${ver_type} in
+        nightly)
+            package=$(brew list-builds --quiet --package=microshift --state=COMPLETE | grep "^microshift-${ver}" | grep "nightly" | uniq | tail -n1) || true
+            ;;
         zstream)
             package=$(brew list-builds --quiet --package=microshift --state=COMPLETE | grep "^microshift-${ver}" | grep -v "~" | uniq | tail -n1) || true
             ;;
         rc|ec)
             package=$(brew list-builds --quiet --package=microshift --state=COMPLETE | grep "^microshift-${ver}.0~${ver_type}." | tail -n1) || true
             ;;
         *)
-            echo "ERROR: Invalid version_type '${ver_type}'. Valid values are: rc, ec and zstream"
+            echo "ERROR: Invalid version_type '${ver_type}'. Valid values are: nightly, rc, ec and zstream"
             exit 1
             ;;
     esac

test/bin/pyutils/build_bootc_images.py

@@ -113,6 +113,7 @@ def set_rpm_version_info_vars():
     global BREW_Y2_RELEASE_VERSION
     global BREW_RC_RELEASE_VERSION
     global BREW_EC_RELEASE_VERSION
+    global BREW_NIGHTLY_RELEASE_VERSION
 
     FAKE_NEXT_MINOR_VERSION = common.get_env_var('FAKE_NEXT_MINOR_VERSION')
     PREVIOUS_RELEASE_REPO = common.get_env_var('PREVIOUS_RELEASE_REPO')
@@ -124,6 +125,7 @@ def set_rpm_version_info_vars():
     BREW_Y2_RELEASE_VERSION = common.get_env_var('BREW_Y2_RELEASE_VERSION')
     BREW_RC_RELEASE_VERSION = common.get_env_var('BREW_RC_RELEASE_VERSION')
     BREW_EC_RELEASE_VERSION = common.get_env_var('BREW_EC_RELEASE_VERSION')
+    BREW_NIGHTLY_RELEASE_VERSION = common.get_env_var('BREW_NIGHTLY_RELEASE_VERSION')
 
     # The source versions are deduced from the locally built RPMs
     global SOURCE_VERSION
@@ -135,15 +137,6 @@ def set_rpm_version_info_vars():
     SOURCE_VERSION = common.run_command_in_shell(f"rpm -q --queryformat '%{{version}}-%{{release}}' {release_info_rpm}")
     SOURCE_VERSION_BASE = common.run_command_in_shell(f"rpm -q --queryformat '%{{version}}-%{{release}}' {release_info_rpm_base}")
 
-    # The brew versions are deduced from the locally downloaded RPMs.
-    # If RPMs are missing, the version is empty and builds are skipped.
-    global BREW_VERSION
-    try:
-        release_info_rpm_brew = find_latest_rpm(BREW_REPO)
-        BREW_VERSION = common.run_command_in_shell(f"rpm -q --queryformat '%{{version}}-%{{release}}' {release_info_rpm_brew}")
-    except Exception:
-        BREW_VERSION = ""
-
     global SSL_CLIENT_KEY_FILE
     global SSL_CLIENT_CERT_FILE
     # Find the first file matching "*-key.pem" in the entitlements directory
@@ -157,8 +150,7 @@ def set_rpm_version_info_vars():
     # Update selected environment variables based on the global variables.
     # These are used for templating container files and images.
     rpmver_globals_vars = [
-        'SOURCE_VERSION', 'SOURCE_VERSION_BASE', 'BREW_VERSION',
-        'SSL_CLIENT_KEY_FILE', 'SSL_CLIENT_CERT_FILE'
+        'SOURCE_VERSION', 'SOURCE_VERSION_BASE', 'SSL_CLIENT_KEY_FILE', 'SSL_CLIENT_CERT_FILE'
     ]
     for var in rpmver_globals_vars:
         value = globals().get(var)
@@ -640,6 +632,8 @@ def main():
                 extract_container_images(BREW_RC_RELEASE_VERSION, BREW_REPO, CONTAINER_LIST, args.dry_run)
             if BREW_EC_RELEASE_VERSION:
                 extract_container_images(BREW_EC_RELEASE_VERSION, BREW_REPO, CONTAINER_LIST, args.dry_run)
+            if BREW_NIGHTLY_RELEASE_VERSION:
+                extract_container_images(BREW_NIGHTLY_RELEASE_VERSION, BREW_REPO, CONTAINER_LIST, args.dry_run)
         # Sort the images list, only leaving unique entries
         common.sort_uniq_file(CONTAINER_LIST)
 

test/image-blueprints-bootc/layer1-base/group2/rhel96-bootc-brew.containerfile → test/image-blueprints-bootc/layer3-periodic/group1/rhel96-bootc-brew-nightly-with-optional.containerfile

@@ -1,4 +1,4 @@
-# {{- if env.Getenv "BREW_VERSION" "" -}}
+# {{- if env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" "" -}}
 # Note: This comment makes templating add a new line before the code
 FROM localhost/rhel96-test-agent:latest
 
@@ -18,7 +18,15 @@ COPY ./bootc-images/$USHIFT_RPM_REPO_NAME.repo ./bootc-images/microshift-fast-da
 RUN dnf repoinfo --enabled && \
     dnf install -y firewalld systemd-resolved \
         {{ range (env.Getenv "MICROSHIFT_MANDATORY_RPMS" | strings.Split " ") -}}
-        "{{ . }}-{{ env.Getenv "BREW_VERSION" }}" \
+        "{{ . }}-{{ env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" }}" \
+        {{ end -}}
+        {{ range (env.Getenv "MICROSHIFT_OPTIONAL_RPMS" | strings.Split " ") -}}
+        "{{ . }}-{{ env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" }}" \
+        {{ end -}}
+        {{ if and (env.Getenv "UNAME_M" "") (eq "x86_64" .Env.UNAME_M) -}}
+        {{ range (env.Getenv "MICROSHIFT_X86_64_RPMS" | strings.Split " ") -}}
+        "{{ . }}-{{ env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" }}" \
+        {{ end -}}
         {{ end -}}
         && \
     systemctl enable microshift microshift-test-agent && \
@@ -35,6 +43,11 @@ RUN firewall-offline-cmd --zone=public --add-port=22/tcp && \
     firewall-offline-cmd --zone=public --add-port=443/tcp && \
     firewall-offline-cmd --zone=public --add-port=5353/udp && \
     firewall-offline-cmd --zone=public --add-port=6443/tcp && \
+    firewall-offline-cmd --zone=public --add-port=8889/tcp && \
     firewall-offline-cmd --zone=public --add-port=30000-32767/tcp && \
     firewall-offline-cmd --zone=public --add-port=30000-32767/udp
+
+# Prepare system for testing Generic Device Plugin
+COPY --chmod=755 ./bootc-images/build-serialsim.sh /tmp/build-serialsim.sh
+RUN /tmp/build-serialsim.sh && rm -f /tmp/build-serialsim.sh
 # {{- end -}}

test/image-blueprints/layer3-periodic/group1/rhel96-brew-nightly-with-optionals.toml

@@ -0,0 +1,71 @@
+{{- if and (env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" "") (env.Getenv "BREW_Y1_RELEASE_VERSION" "") -}}
+{{- /*
+
+  We wrap this template in a test so that the body of the output is
+  empty when there is no "current" version release. The output file
+  must end up completely empty, so we need to remove whitespace from
+  around the first and last template instructions.
+
+*/ -}}
+
+name = "rhel-9.6-microshift-brew-optionals-4.{{ .Env.MINOR_VERSION}}-nightly"
+description = "A RHEL 9.6 image with already built and released RPMs like EC, RC, or Z-stream release: {{ .Env.BREW_NIGHTLY_RELEASE_VERSION }}"
+version = "0.0.1"
+modules = []
+groups = []
+distro = "rhel-96"
+
+# Parent specification directive recognized by test/bin/build_images.sh to be
+# used with the '--parent' argument of 'osbuild-composer'
+# parent = "rhel-9.6-microshift-brew-optionals-4.{{ .Env.PREVIOUS_MINOR_VERSION }}-zstream"
+
+{{ range (env.Getenv "MICROSHIFT_MANDATORY_RPMS" | strings.Split " ") }}
+[[packages]]
+name = "{{ . }}"
+version = "{{ env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" }}"
+{{ end }}
+
+{{ range (env.Getenv "MICROSHIFT_OPTIONAL_RPMS" | strings.Split " ") }}
+[[packages]]
+name = "{{ . }}"
+version = "{{ env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" }}"
+{{ end }}
+
+{{- if and (env.Getenv "UNAME_M" "") (eq "x86_64" .Env.UNAME_M) }}
+{{ range (env.Getenv "MICROSHIFT_X86_64_RPMS" | strings.Split " ") }}
+[[packages]]
+name = "{{ . }}"
+version = "{{ env.Getenv "BREW_NIGHTLY_RELEASE_VERSION" }}"
+{{ end }}
+{{- end }}
+
+[[packages]]
+name = "microshift-test-agent"
+version = "*"
+
+[[packages]]
+name = "systemd-resolved"
+version = "*"
+
+[customizations.services]
+enabled = ["microshift", "microshift-test-agent"]
+
+[customizations.firewall]
+ports = [
+    "22:tcp",
+    "80:tcp",
+    "443:tcp",
+    "5353:udp",
+    "6443:tcp",
+    "8889:tcp",
+    "30000-32767:tcp",
+    "30000-32767:udp",
+]
+
+[customizations.firewall.services]
+enabled = ["mdns", "ssh", "http", "https"]
+
+[[customizations.firewall.zones]]
+name = "trusted"
+sources = ["10.42.0.0/16", "169.254.169.1", "fd01::/48"]
+{{- end -}}

test/scenarios-bootc/periodics/el96-crel@brew-standard1.sh → test/scenarios-bootc/periodics/el96-lrel@nightly-brew-standard1.sh

@@ -2,7 +2,7 @@
 
 # Sourced from scenario.sh and uses functions defined there.
 
-start_image=rhel96-bootc-brew
+start_image=rhel96-bootc-brew-nightly-with-optional
 
 scenario_create_vms() {
     if ! does_image_exist "${start_image}"; then

test/scenarios-bootc/periodics/el96-crel@brew-standard2.sh → test/scenarios-bootc/periodics/el96-lrel@nightly-brew-standard2.sh

@@ -2,7 +2,7 @@
 
 # Sourced from scenario.sh and uses functions defined there.
 
-start_image=rhel96-bootc-brew
+start_image=rhel96-bootc-brew-nightly-with-optional
 
 scenario_create_vms() {
     if ! does_image_exist "${start_image}"; then

test/scenarios/periodics/el96-lrel@nightly-brew-standard1.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Sourced from scenario.sh and uses functions defined there.
+
+start_image="rhel-9.6-microshift-brew-optionals-4.${MINOR_VERSION}-nightly"
+
+scenario_create_vms() {
+    if ! does_commit_exist "${start_image}"; then
+        echo "Image '${start_image}' not found - skipping test"
+        return 0
+    fi
+
+    prepare_kickstart host1 kickstart.ks.template "${start_image}"
+    launch_vm
+}
+
+scenario_remove_vms() {
+    if ! does_commit_exist "${start_image}"; then
+        echo "Image '${start_image}' not found - skipping test"
+        return 0
+    fi
+
+    remove_vm host1
+}
+
+scenario_run_tests() {
+    if ! does_commit_exist "${start_image}"; then
+        echo "Image '${start_image}' not found - skipping test"
+        return 0
+    fi
+
+    run_tests host1 \
+        --variable "EXPECTED_OS_VERSION:9.6" \
+        suites/standard1/ suites/selinux/validate-selinux-policy.robot
+}

test/scenarios/periodics/el96-lrel@nightly-brew-standard2.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Sourced from scenario.sh and uses functions defined there.
+
+start_image="rhel-9.6-microshift-brew-optionals-4.${MINOR_VERSION}-nightly"
+
+scenario_create_vms() {
+    if ! does_commit_exist "${start_image}"; then
+        echo "Image '${start_image}' not found - skipping test"
+        return 0
+    fi
+
+    prepare_kickstart host1 kickstart.ks.template "${start_image}"
+    launch_vm
+}
+
+scenario_remove_vms() {
+    if ! does_commit_exist "${start_image}"; then
+        echo "Image '${start_image}' not found - skipping test"
+        return 0
+    fi
+
+    remove_vm host1
+}
+
+scenario_run_tests() {
+    if ! does_commit_exist "${start_image}"; then
+        echo "Image '${start_image}' not found - skipping test"
+        return 0
+    fi
+
+    run_tests host1 suites/standard2/
+}