diff --git a/.gitignore b/.gitignore index fb05ccd53a..a4ddfa8fb1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ _output/* _output sshfile ansible/*.txt +**/.venv/ diff --git a/e2e-robot/Makefile b/e2e-robot/Makefile new file mode 100644 index 0000000000..243233bd5f --- /dev/null +++ b/e2e-robot/Makefile @@ -0,0 +1,38 @@ + +.PHONY: setup dry-run with-filter with-tag all + +setup: + python3 -m venv .venv && \ + .venv/bin/python3 -m pip install -r requirements.txt + +dry-run: + .venv/bin/robot \ + --dryrun \ + --outputdir ../_output/e2e-$$(date +'%Y%m%d-%H%M%S') \ + ./tests/microshift.robot + +with-filter: + .venv/bin/robot \ + -v USHIFT_IP:$${USHIFT_IP} \ + -v USHIFT_USER:microshift \ + --test *router* \ + --outputdir ../_output/e2e-$$(date +'%Y%m%d-%H%M%S') \ + -x xunit.xml \ + ./tests/microshift.robot + +with-tag: + .venv/bin/robot \ + -v USHIFT_IP:$${USHIFT_IP} \ + -v USHIFT_USER:microshift \ + --include smoke \ + --outputdir ../_output/e2e-$$(date +'%Y%m%d-%H%M%S') \ + -x xunit.xml \ + ./tests/microshift.robot + +all: + .venv/bin/robot \ + -v USHIFT_IP:$${USHIFT_IP} \ + -v USHIFT_USER:microshift \ + --outputdir ../_output/e2e-$$(date +'%Y%m%d-%H%M%S') \ + -x xunit.xml \ + ./tests/microshift.robot \ No newline at end of file diff --git a/e2e-robot/requirements.txt b/e2e-robot/requirements.txt new file mode 100644 index 0000000000..27103b1c2d --- /dev/null +++ b/e2e-robot/requirements.txt @@ -0,0 +1,27 @@ +bcrypt==4.0.1 +certifi==2022.12.7 +cffi==1.15.1 +charset-normalizer==3.1.0 +click==8.1.3 +colorama==0.4.6 +cryptography==40.0.1 +idna==3.4 +Jinja2==3.1.2 +markdown-it-py==2.2.0 +MarkupSafe==2.1.2 +mdurl==0.1.2 +paramiko==3.1.0 +pathspec==0.11.1 +pycparser==2.21 +Pygments==2.14.0 +PyNaCl==1.5.0 +requests==2.28.2 +rich==13.3.3 +rich-click==1.4 +robotframework==6.0.2 +robotframework-requests==0.9.4 +robotframework-sshlibrary==3.8.0 +robotframework-tidy==4.0.1 +scp==0.14.5 +tomli==2.0.1 +urllib3==1.26.15 diff --git a/e2e-robot/tests/microshift.robot b/e2e-robot/tests/microshift.robot new file mode 100644 index 0000000000..ad79aa0184 --- /dev/null +++ b/e2e-robot/tests/microshift.robot @@ -0,0 +1,167 @@ +*** Settings *** +Documentation MicroShift e2e test suite + +Library SSHLibrary +Library String +Library OperatingSystem +Library Process +Library RequestsLibrary + +Suite Setup Get Kubeconfig +Suite Teardown Remove Kubeconfig + + +*** Variables *** +${USHIFT_IP} ${EMPTY} +${USHIFT_USER} ${EMPTY} + + +*** Test Cases *** +Router Smoke Test + [Documentation] Verify that Router correctly exposes HTTP service + [Tags] smoke + [Setup] Run Keywords + ... Create Hello MicroShift Pod AND + ... Expose Hello MicroShift Pod Via Router AND + ... Open Port 80 tcp + + Wait Until Keyword Succeeds 3x 3s Access Hello Microshift via Router + + [Teardown] Run Keywords + ... Delete Hello MicroShift Pod Route And Service AND + ... Close Port 80 tcp + +Load Balancer Smoke Test + [Documentation] Verify that Load Balancer correctly exposes HTTP service + [Tags] smoke + [Setup] Run Keywords + ... Create Hello MicroShift Pod AND + ... Expose Hello MicroShift Pod Via LB AND + ... Open Port 5678 tcp + + Wait Until Keyword Succeeds 3x 3s Access Hello Microshift via LB + + [Teardown] Run Keywords + ... Delete Hello MicroShift Pod Route And Service AND + ... Close Port 5678 tcp + +Reboot Test + [Documentation] Verify that MicroShift starts successfully after reboot + [Setup] Run Keywords Create Pod With PVC + + Open Connection ${USHIFT_IP} + Login ${USHIFT_USER} allow_agent=True + Execute Command reboot now sudo=True + Close Connection + + Sleep 15s + + Open Connection ${USHIFT_IP} + Set Client Configuration timeout=600s + Wait Until Keyword Succeeds 10x 10s Login ${USHIFT_USER} allow_agent=True + + Wait Until Keyword Succeeds + ... 10x + ... 10s + ... Execute Command + ... [ $(systemctl show -p SubState --value microshift) = "running" ] + ... timeout=10s return_stdout=True return_stderr=True + + ${stdout} ${rc}= Execute Command + ... /etc/greenboot/check/required.d/40_microshift_running_check.sh | tee /tmp/asd.log + ... sudo=True + ... timeout=600s + ... return_stdout=True + ... return_rc=True + Log ${stdout} + Should Be Equal As Integers ${rc} 0 + Close Connection + + Run With Kubeconfig oc wait --for=condition=Ready --timeout=120s pod/test-pod + + [Teardown] Delete Pod With PVC + +Failed Test + Fail Let's see how it looks in Prow + + +*** Keywords *** +Get Kubeconfig + [Documentation] X + Open Connection ${USHIFT_IP} + Login ${USHIFT_USER} allow_agent=True + ${konfig}= Execute Command + ... cat /var/lib/microshift/resources/kubeadmin/${USHIFT_IP}/kubeconfig + ... sudo=True + Should Not Be Empty ${konfig} + ${rand}= Generate Random String + ${path}= Join Path /tmp ${rand} + Create File ${path} ${konfig} + Close Connection + Set Suite Variable \${KUBECONFIG} ${path} + +Remove Kubeconfig + Remove File ${KUBECONFIG} + +Access Hello Microshift via Router + ${result}= Run Process + ... curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${USHIFT_IP}" + ... shell=True timeout=15s + Check HTTP Response ${result} + +Access Hello Microshift via LB + ${result}= Run Process curl -i ${USHIFT_IP}:5678 shell=True timeout=15s + Check HTTP Response ${result} + +Check HTTP Response + [Arguments] ${result} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Match Regexp ${result.stdout} HTTP.*200 + Should Match ${result.stdout} *Hello MicroShift* + +Create Pod With PVC + Run With Kubeconfig oc create -f ../e2e/tests/assets/pod-with-pvc.yaml + Run With Kubeconfig oc wait --for=condition=Ready --timeout=120s pod/test-pod + +Delete Pod With PVC + Run With Kubeconfig oc delete -f ../e2e/tests/assets/pod-with-pvc.yaml True + +Create Hello MicroShift Pod + Run With Kubeconfig oc create -f ../e2e/tests/assets/hello-microshift.yaml + Run With Kubeconfig oc wait pods -l app\=hello-microshift --for condition\=Ready --timeout\=60s + +Expose Hello MicroShift Pod Via Router + Run With Kubeconfig oc expose pod hello-microshift + Run With Kubeconfig oc expose svc hello-microshift --hostname hello-microshift.cluster.local + +Expose Hello MicroShift Pod Via LB + Run With Kubeconfig oc create service loadbalancer hello-microshift --tcp=5678:8080 + +Delete Hello MicroShift Pod Route And Service + Run With Kubeconfig oc delete route hello-microshift True + Run With Kubeconfig oc delete service hello-microshift True + Run With Kubeconfig oc delete -f ../e2e/tests/assets/hello-microshift.yaml True + +Run With Kubeconfig + [Arguments] ${cmd} ${allow_fail}=False + ${result}= Run Process ${cmd} env:KUBECONFIG=${KUBECONFIG} stderr=STDOUT shell=True + Log ${result.stdout} + IF ${allow_fail} == False + Should Be Equal As Integers ${result.rc} 0 + END + +Open Port + [Arguments] ${number} ${protocol} + ${res}= Run Process bash -c + ... if declare -F firewall::open_port; then firewall::open_port ${number} ${protocol}; fi stderr=STDOUT + Log ${res.stdout} + Should Be Equal As Integers ${res.rc} 0 + +Close Port + [Arguments] ${number} ${protocol} + ${res}= Run Process bash -c + ... if declare -F firewall::close_port; then firewall::close_port ${number} ${protocol}; fi stderr=STDOUT + Log ${res.stdout} + Should Be Equal As Integers ${res.rc} 0 diff --git a/e2e/main.sh b/e2e/main.sh new file mode 100755 index 0000000000..49aff1c746 --- /dev/null +++ b/e2e/main.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +OUTPUT_DIR="${ARTIFACT_DIR:-_output}/microshift-e2e-$(date +'%Y%m%d-%H%M%S')/" + +usage() { + echo "Usage: $(basename "${0}") {list|run} [filter]" + echo "" + echo " list Lists tests" + echo " run Runs tests" + echo " filter Simple string to match against test files, e.g. 'reboot', 'boot', 'smoke'" + echo "" + echo " Script expects two environmental variables:" + echo " - USHIFT_IP" + echo " - USHIFT_USER" + echo "" + echo " Script assumes following:" + echo " - Passwordless SSH to \$USHIFT_USER@\$USHIFT_IP is configured" + echo " - Both hosts already exchanged their public keys:" + echo " - Test runner has MicroShift's sshd key in ~/.ssh/known_keys" + echo " - Remote \$USHIFT_USER has test runner's key in ~/.ssh/authorized_keys" + echo " - Passwordless sudo for \$USHIFT_USER" + echo "" + echo " Script aims to be target platform agnostic. It means that for some environments (e.g. GCP)" + echo " it might be required to export firewall::open_port and firewall::close_port functions" + echo " so the tests can open custom ports" + + [ -n "$1" ] && echo -e "\nERROR: $1" + exit 1 +} + +log() { + echo -e "$(date +'%H:%M:%S.%N') $*" +} + +var_should_not_be_empty() { + local var_name=${1} + if [ -z "${!var_name+x}" ]; then + echo >&2 "Environmental variable '${var_name}' is unset" + return 1 + elif [ -z "${!var_name}" ]; then + echo >&2 "Environmental variable '${var_name}' is empty" + return 1 + fi +} + +function_should_be_exported() { + local fname=${1} + if ! declare -F "${fname}"; then + log "WARNING: Function '${fname}' is unexported. It is expected that function is provided for interacting with cloud provider" + return 1 + fi +} + +check_passwordless_ssh() { + ssh -o BatchMode=yes "${USHIFT_USER}@${USHIFT_IP}" "true" || { + echo "Failed to access ${USHIFT_IP}:" + echo " - Test runner should have MicroShift's sshd key in ~/.ssh/known_keys" + echo " - Remote \$USHIFT_USER should have test runner's key in ~/.ssh/authorized_keys" + exit 1 + } +} + +check_passwordless_sudo() { + ssh -o BatchMode=yes "${USHIFT_USER}@${USHIFT_IP}" "sudo --non-interactive true" || { + echo "Failed to run sudo command as ${USHIFT_USER} without password" + exit 1 + } +} + +prechecks() { + var_should_not_be_empty USHIFT_IP && var_should_not_be_empty USHIFT_USER || exit 1 + check_passwordless_ssh + check_passwordless_sudo + + # Just warning for now + # Following functions needed only for runs in CI + function_should_be_exported firewall::open_port || true + function_should_be_exported firewall::close_port || true +} + +microshift_get_konfig() { + tmpfile=$(mktemp /tmp/microshift-e2e-konfig.XXXXXX) + ssh "${USHIFT_USER}@${USHIFT_IP}" 'sudo cat /var/lib/microshift/resources/kubeadmin/'"${USHIFT_IP}"'/kubeconfig' >"${tmpfile}" + echo "${tmpfile}" +} + +microshift_check_readiness() { + local test_output="${1}" + log "Waiting for MicroShift to become ready" + ssh "${USHIFT_USER}@${USHIFT_IP}" \ + "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh" &>"${test_output}/0002-readiness-check.log" +} + +microshift_setup() { + local test_output="${1}" + log "Setting up and starting MicroShift" + ssh "${USHIFT_USER}@${USHIFT_IP}" 'cat << EOF | sudo tee /etc/microshift/config.yaml +--- +apiServer: + subjectAltNames: + - '"${USHIFT_IP}"' +EOF' &>"${test_output}/0001-setup.log" + ssh "${USHIFT_USER}@${USHIFT_IP}" "sudo systemctl enable --now microshift" &>>"${test_output}/0001-setup.log" +} + +microshift_debug_info() { + local test_output="${1}" + log "Gathering debug info to ${test_output}/0020-cluster-debug-info.log" + scp "${SCRIPT_DIR}/../validate-microshift/cluster-debug-info.sh" "${USHIFT_USER}@${USHIFT_IP}:/tmp/cluster-debug-info.sh" + ssh "${USHIFT_USER}@${USHIFT_IP}" "sudo /tmp/cluster-debug-info.sh" &>"${test_output}/0020-cluster-debug-info.log" +} + +microshift_cleanup() { + local test_output="${1}" + log "Cleaning MicroShift" + ssh "${USHIFT_USER}@${USHIFT_IP}" "echo 1 | sudo microshift-cleanup-data --all" &>"${test_output}/0000-cleanup.log" +} + +microshift_health_summary() { + log "Summary of MicroShift health" + + # Because test might be "destructive" (i.e. tear down and set up again MicroShift) + # so these commands are executed via ssh. + # Alternative is to copy kubeconfig second time in the same test. + ssh "${USHIFT_USER}@${USHIFT_IP}" \ + "mkdir -p ~/.kube/ && sudo cat /var/lib/microshift/resources/kubeadmin/kubeconfig > ~/.kube/config ; \ + oc get pods -A ; \ + oc get nodes -o wide ; \ + oc get events -A --sort-by=.metadata.creationTimestamp | head -n 20" +} + +run_test() { + local test=$1 + echo -e "\n\n==================================================" + log "${test} - PREPARING" + + local test_output="${OUTPUT_DIR}/${test}/" + mkdir -p "${test_output}" + + prep_start=$(date +%s) + microshift_cleanup "${test_output}" + microshift_setup "${test_output}" + microshift_check_readiness "${test_output}" + konfig=$(microshift_get_konfig) + trap 'rm -f "${konfig}"' RETURN + prep_dur=$(($(date +%s) - prep_start)) + log "Cleanup, setup, and readiness took $((prep_dur / 60))m $((prep_dur % 60))s." + + log "${test} - RUNNING" + test_start=$(date +%s) + set +e + KUBECONFIG="${konfig}" "${SCRIPT_DIR}/tests/${test}" &>"${test_output}/0010-test.log" + res=$? + set -e + test_dur=$(($(date +%s) - test_start)) + + log "${test} took $((test_dur / 60))m $((test_dur % 60))s." + if [ ${res} -eq 0 ]; then + log "${test} - SUCCESS" + return 0 + fi + + log "${test} - FAILURE" + microshift_health_summary + microshift_debug_info "${test_output}" || true + return 1 +} + +list() { + local -r filter="*${1:-}*.sh" + find "${SCRIPT_DIR}/tests" -maxdepth 1 -iname "${filter}" -printf "%f\n" | sort +} + +run() { + local -r to_run=$(list "${1}") + + prechecks + log "Following tests will run:\n${to_run}" + [ ! -d "${OUTPUT_DIR}" ] && mkdir -p "${OUTPUT_DIR}" + + all_successful=true + for t in ${to_run}; do + run_test "${t}" || all_successful=false + done + "${all_successful}" +} + +[ $# -eq 0 ] && { + usage "Expected arguments" +} + +cmd="$1" +shift +"${cmd}" "${1:-}" diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh new file mode 100755 index 0000000000..14268a5305 --- /dev/null +++ b/e2e/tests/0030-router-smoke-test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +RETRIES=3 +BACKOFF=3s + +# shellcheck disable=SC2317 # Don't warn about unreachable commands in this function +cleanup() { + oc delete route hello-microshift || true + oc delete service hello-microshift || true + oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true + if declare -F firewall::close_port; then firewall::close_port 80 tcp || true; fi +} +trap 'cleanup' EXIT + +declare -F firewall::open_port && firewall::open_port 80 tcp + +oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" +oc expose pod hello-microshift +oc expose svc hello-microshift --hostname hello-microshift.cluster.local +oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s + +for _ in $(seq "${RETRIES}"); do + set +e + response=$(curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${USHIFT_IP}" 2>&1) + result=$? + set -e + + [ ${result} -eq 0 ] && + echo "${response}" | grep -q -E "HTTP.*200" && + echo "${response}" | grep -q "Hello MicroShift" && + exit 0 + + sleep "${BACKOFF}" +done +exit 1 diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh new file mode 100755 index 0000000000..07f6a4d51a --- /dev/null +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +RETRIES=3 +BACKOFF=3s + +# shellcheck disable=SC2317 # Don't warn about unreachable commands in this function +cleanup() { + oc delete service hello-microshift || true + oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true + if declare -F firewall::close_port; then firewall::close_port 5678 tcp || true; fi +} +trap 'cleanup' EXIT + +declare -F firewall::open_port && firewall::open_port 5678 tcp +oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" +oc create service loadbalancer hello-microshift --tcp=5678:8080 +oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s + +for _ in $(seq "${RETRIES}"); do + set +e + response=$(curl -i "${USHIFT_IP}":5678 2>&1) + result=$? + set -e + + [ ${result} -eq 0 ] && + echo "${response}" | grep -q -E "HTTP.*200" && + echo "${response}" | grep -q "Hello MicroShift" && + exit 0 + + sleep "${BACKOFF}" +done +exit 1 diff --git a/e2e/tests/0050-greenboot.sh b/e2e/tests/0050-greenboot.sh new file mode 100755 index 0000000000..967702b603 --- /dev/null +++ b/e2e/tests/0050-greenboot.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +scp "${SCRIPT_PATH}/../../docs/config/busybox_running_check.sh" "${SCRIPT_PATH}/assets/greenboot-test.sh" "${USHIFT_USER}@${USHIFT_IP}":/tmp/ +ssh -q "${USHIFT_USER}@${USHIFT_IP}" "chmod +x /tmp/greenboot-test.sh /tmp/busybox_running_check.sh && sudo /tmp/greenboot-test.sh" diff --git a/e2e/tests/0100-reboot.sh b/e2e/tests/0100-reboot.sh new file mode 100755 index 0000000000..fd13494075 --- /dev/null +++ b/e2e/tests/0100-reboot.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -euo pipefail +PS4='+ $(date "+%T.%N")\011 ' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +RETRIES=5 +BACKOFF=20 + +wait_until() { + local cmd=$* + for _ in $(seq "${RETRIES}"); do + ${cmd} && return 0 + sleep "${BACKOFF}" + done + return 1 +} + +cleanup() { + oc delete -f "${SCRIPT_PATH}/assets/pod-with-pvc.yaml" +} +trap 'cleanup' EXIT + +oc create -f "${SCRIPT_PATH}/assets/pod-with-pvc.yaml" +oc wait --for=condition=Ready --timeout=120s pod/test-pod + +set +e +ssh -v "${USHIFT_USER}@${USHIFT_IP}" "sudo reboot now" +res=$? +set -e + +# Allow for `ssh` command errors (255 exit code) like "connection closed by remote host" +# Fail on other errors (coming from the command executed remotely itself) +if [ "${res}" -ne 0 ] && [ "${res}" -ne 255 ]; then + exit 1 +fi + +wait_until ssh "${USHIFT_USER}@${USHIFT_IP}" "true" +# Just check if KAS is up and serving +wait_until oc get node +ssh "${USHIFT_USER}@${USHIFT_IP}" "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh" +oc wait --for=condition=Ready --timeout=120s pod/test-pod diff --git a/e2e/tests/assets/greenboot-test.sh b/e2e/tests/assets/greenboot-test.sh new file mode 100644 index 0000000000..3935eabfbd --- /dev/null +++ b/e2e/tests/assets/greenboot-test.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' +set -x + +function check_greenboot_exit_status() { + local expectedRC=$1 + local cleanup=$2 + + if [ "${cleanup}" -ne 0 ]; then + echo 1 | microshift-cleanup-data --all + systemctl enable --now microshift || true + fi + + for check_script in $(find /etc/greenboot/check/ -name \*.sh | sort); do + echo Running "${check_script}"... + local currentRC=1 + if ${check_script}; then + currentRC=0 + fi + + if [ ${currentRC} != "${expectedRC}" ]; then + exit 1 + fi + done +} + +# +# Initial check must succeed (set timeout of 180s to speed up the process) +# +tee /etc/greenboot/greenboot.conf &>/dev/null </dev/null </dev/null </dev/null <