From 82c7752011c95095bb7200b94beb0f9e683f6891 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 23 Mar 2023 16:17:42 +0100 Subject: [PATCH 01/39] microshift specific tests --- e2e/helpers.sh | 27 ++++++++++++ e2e/main.sh | 53 +++++++++++++++++++++++ e2e/tests/0030-router-smoke-test.sh | 42 ++++++++++++++++++ e2e/tests/0031-loadbalancer-smoke-test.sh | 40 +++++++++++++++++ e2e/tests/0050-greenboot.sh | 1 + e2e/tests/0100-reboot.sh | 1 + e2e/tests/assets/hello-microshift.yaml | 25 +++++++++++ 7 files changed, 189 insertions(+) create mode 100644 e2e/helpers.sh create mode 100755 e2e/main.sh create mode 100755 e2e/tests/0030-router-smoke-test.sh create mode 100755 e2e/tests/0031-loadbalancer-smoke-test.sh create mode 100755 e2e/tests/0050-greenboot.sh create mode 100755 e2e/tests/0100-reboot.sh create mode 100644 e2e/tests/assets/hello-microshift.yaml diff --git a/e2e/helpers.sh b/e2e/helpers.sh new file mode 100644 index 0000000000..ff223624f2 --- /dev/null +++ b/e2e/helpers.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +gscp() { + local src="${1}" + local dst="${2}" + + gcloud compute \ + --project "${GOOGLE_PROJECT_ID}" \ + --quiet \ + scp \ + --zone "${GOOGLE_COMPUTE_ZONE}" \ + --recurse \ + "${src}" "rhel8user@${INSTANCE_PREFIX}:${dst}" +} + +gssh() { + local cmd="${1}" + + # TODO: Consider flag --quiet + gcloud compute \ + --project "${GOOGLE_PROJECT_ID}" \ + ssh \ + --zone "${GOOGLE_COMPUTE_ZONE}" \ + "rhel8user@${INSTANCE_PREFIX}" \ + --command \ + "${cmd}" +} diff --git a/e2e/main.sh b/e2e/main.sh new file mode 100755 index 0000000000..36b14076c8 --- /dev/null +++ b/e2e/main.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +set -o errtrace +set -o nounset +set -o pipefail + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +var_should_not_be_empty() { + local var=${1} + if [ -z "${!var}" ]; then + echo >&2 "Environmental variable '${var}' is empty" + exit 1 + fi +} + +verify_gcloud() { + var_should_not_be_empty INSTANCE_PREFIX + var_should_not_be_empty GOOGLE_PROJECT_ID + var_should_not_be_empty GOOGLE_COMPUTE_REGION + var_should_not_be_empty GOOGLE_COMPUTE_ZONE +} + +prepare_kubeconfig() { + export IP_ADDRESS="$(gcloud compute instances describe ${INSTANCE_PREFIX} --format='get(networkInterfaces[0].accessConfigs[0].natIP)')" + gssh "sudo cat /var/lib/microshift/resources/kubeadmin/${IP_ADDRESS}/kubeconfig" >/tmp/kubeconfig + export KUBECONFIG=/tmp/kubeconfig +} + +main() { + verify_gcloud + prepare_kubeconfig + + local gather_debug=false + for test in ${SCRIPT_PATH}/tests/*; do + echo "RUNNING $(basename ${test})" + ${test} + res=$? + if [ $res -eq 0 ]; then + echo "SUCCESS" + else + echo "FAILURE" + gather_debug=true + fi + done + + if ${gather_debug}; then + echo "Run cluster-debug-info.sh" + exit 1 + fi +} + +main diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh new file mode 100755 index 0000000000..f28bf25006 --- /dev/null +++ b/e2e/tests/0030-router-smoke-test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +source "${SCRIPT_PATH}/../helpers.sh" + +cleanup() { + oc delete route hello-microshift + oc delete service hello-microshift + oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" +} +trap cleanup EXIT + +gcloud compute firewall-rules update "${INSTANCE_PREFIX}" --allow tcp:22,icmp,tcp:80 +oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" +oc expose pod hello-microshift +oc expose svc hello-microshift --hostname hello-microshift.cluster.local +oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s + +set +x +retries=3 +backoff=3s +for try in $(seq 1 "${retries}"); do + echo "Attempt: ${try}" + echo "Running: curl -vk http://hello-microshift.cluster.local --resolve \"hello-microshift.cluster.local:80:${IP}\"" + RESPONSE=$(curl -vk http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${IP}" 2>&1) + RESULT=$? + echo "Exit code: ${RESULT}" + echo -e "Response: \n${RESPONSE}\n\n" + if [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200 OK"; then + echo "Request fulfilled conditions to be successful (exit code = 0, response contains 'HTTP.*200 OK')" + exit 0 + fi + echo -e "Waiting ${backoff} before next retry\n\n" + sleep "${backoff}" +done +set -x + +exit 1 diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh new file mode 100755 index 0000000000..78a0010720 --- /dev/null +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +source "${SCRIPT_PATH}/../helpers.sh" + +cleanup() { + oc delete service hello-microshift + oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" +} +trap cleanup EXIT + +gcloud compute firewall-rules update "${INSTANCE_PREFIX}" --allow tcp:22,icmp,tcp:5678 +oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" +oc create service loadbalancer hello-microshift --tcp=5678:8080 +oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s + +set +x +retries=3 +backoff=3s +for try in $(seq 1 "${retries}"); do + echo "Attempt: ${try}" + echo "Running: curl -vk ${IP}:5678" + RESPONSE=$(curl -I ${IP}:5678 2>&1) + RESULT=$? + echo "Exit code: ${RESULT}" + echo -e "Response: \n${RESPONSE}\n\n" + if [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200"; then + echo "Request fulfilled conditions to be successful (exit code = 0, response contains 'HTTP.*200')" + exit 0 + fi + echo -e "Waiting ${backoff} before next retry\n\n" + sleep "${backoff}" +done +set -x + +exit 1 diff --git a/e2e/tests/0050-greenboot.sh b/e2e/tests/0050-greenboot.sh new file mode 100755 index 0000000000..f1f641af19 --- /dev/null +++ b/e2e/tests/0050-greenboot.sh @@ -0,0 +1 @@ +#!/usr/bin/env bash diff --git a/e2e/tests/0100-reboot.sh b/e2e/tests/0100-reboot.sh new file mode 100755 index 0000000000..f1f641af19 --- /dev/null +++ b/e2e/tests/0100-reboot.sh @@ -0,0 +1 @@ +#!/usr/bin/env bash diff --git a/e2e/tests/assets/hello-microshift.yaml b/e2e/tests/assets/hello-microshift.yaml new file mode 100644 index 0000000000..47a864d33e --- /dev/null +++ b/e2e/tests/assets/hello-microshift.yaml @@ -0,0 +1,25 @@ +kind: Pod +apiVersion: v1 +metadata: + name: hello-microshift + labels: + app: hello-microshift +spec: + containers: + - name: hello-microshift + image: busybox:1.35 + command: ["/bin/sh"] + args: ["-c", "while true; do echo -ne \"HTTP/1.0 200 OK\r\nContent-Length: 16\r\n\r\nHello MicroShift\" | nc -l -p 8080 ; done"] + ports: + - containerPort: 8080 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + seccompProfile: + type: RuntimeDefault From 99693da1d08ca3ef71ede24bdac472189997ab22 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 30 Mar 2023 11:41:53 +0200 Subject: [PATCH 02/39] fast hello-microshift pod deletion --- e2e/tests/assets/hello-microshift.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e/tests/assets/hello-microshift.yaml b/e2e/tests/assets/hello-microshift.yaml index 47a864d33e..11b27c5b90 100644 --- a/e2e/tests/assets/hello-microshift.yaml +++ b/e2e/tests/assets/hello-microshift.yaml @@ -5,6 +5,7 @@ metadata: labels: app: hello-microshift spec: + terminationGracePeriodSeconds: 0 containers: - name: hello-microshift image: busybox:1.35 From 5f931d707992cb1995080eedd5a5b30bd3319a44 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 30 Mar 2023 11:42:23 +0200 Subject: [PATCH 03/39] remove references to gcloud, expect some exported functions --- e2e/main.sh | 72 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index 36b14076c8..2ce3278fef 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -7,33 +7,71 @@ set -o pipefail SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" var_should_not_be_empty() { - local var=${1} - if [ -z "${!var}" ]; then - echo >&2 "Environmental variable '${var}' is empty" - exit 1 + local var_name=${1} + if [ -z "${!var_name+x}" ]; then + echo >&2 "Environmental variable '${var_name}' is unset" + return 1 + elif [ -z "${!var_name}" ]; then + echo >&2 "Environmental variable '${var_name}' is empty" + return 1 fi } -verify_gcloud() { - var_should_not_be_empty INSTANCE_PREFIX - var_should_not_be_empty GOOGLE_PROJECT_ID - var_should_not_be_empty GOOGLE_COMPUTE_REGION - var_should_not_be_empty GOOGLE_COMPUTE_ZONE +function_should_be_exported() { + local fname=${1} + if ! declare -F "$fname"; then + echo >&2 "Function '$fname' is unexported. It is expected that function is provided for interacting with cloud provider" + return 1 + fi } -prepare_kubeconfig() { - export IP_ADDRESS="$(gcloud compute instances describe ${INSTANCE_PREFIX} --format='get(networkInterfaces[0].accessConfigs[0].natIP)')" - gssh "sudo cat /var/lib/microshift/resources/kubeadmin/${IP_ADDRESS}/kubeconfig" >/tmp/kubeconfig - export KUBECONFIG=/tmp/kubeconfig +file_should_exist() { + local file_var="$1" + + var_should_not_be_empty "$file_var" || return 1 + local file="${!file_var}" + + if [[ ! -e "$file" ]]; then + echo >&2 "File '$file' does not exist" + return 1 + fi + + if [[ ! -f "$file" ]]; then + echo >&2 "'$file' is not a file" + return 1 + fi + + if [[ ! -r "$file" ]]; then + echo >&2 "'$file' is missing 'read' permissions" + return 1 + fi + } +prechecks() { + # TODO: It'd be nice to run all prechecks each time, but exit at the end if any is unfullfiled + file_should_exist KUBECONFIG || exit 1 + var_should_not_be_empty USHIFT_IP || exit 1 + var_should_not_be_empty USHIFT_USER || exit 1 + # TODO Check passwordless SSH + # TODO Check passwordless sudo + + # Just warning for now + function_should_be_exported firewall::open_port # || exit 1 + function_should_be_exported firewall::close_port # || exit 1 +} + +# TODO: check_if_cluster_is_ready() {} - kuttl? + main() { - verify_gcloud - prepare_kubeconfig + prechecks + + export IP="$USHIFT_IP" + export USER="$USHIFT_USER" local gather_debug=false - for test in ${SCRIPT_PATH}/tests/*; do - echo "RUNNING $(basename ${test})" + for test in "${SCRIPT_PATH}"/tests/*.sh; do + echo "RUNNING $(basename "${test}")" ${test} res=$? if [ $res -eq 0 ]; then From 3919930b209d9d690e59d5afbee2cebdd2435b76 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 30 Mar 2023 11:43:43 +0200 Subject: [PATCH 04/39] update router & lb smoke tests --- e2e/tests/0030-router-smoke-test.sh | 23 +++++++--------- e2e/tests/0031-loadbalancer-smoke-test.sh | 33 ++++++++++------------- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh index f28bf25006..1573d1f040 100755 --- a/e2e/tests/0030-router-smoke-test.sh +++ b/e2e/tests/0030-router-smoke-test.sh @@ -5,38 +5,33 @@ IFS=$'\n\t' set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" -source "${SCRIPT_PATH}/../helpers.sh" +# shellcheck disable=SC2317 # Don't warn about unreachable commands in this function cleanup() { oc delete route hello-microshift oc delete service hello-microshift oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" + firewall::close_port tcp:80 } trap cleanup EXIT -gcloud compute firewall-rules update "${INSTANCE_PREFIX}" --allow tcp:22,icmp,tcp:80 +firewall::open_port tcp:80 oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" oc expose pod hello-microshift oc expose svc hello-microshift --hostname hello-microshift.cluster.local oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s -set +x retries=3 backoff=3s -for try in $(seq 1 "${retries}"); do - echo "Attempt: ${try}" - echo "Running: curl -vk http://hello-microshift.cluster.local --resolve \"hello-microshift.cluster.local:80:${IP}\"" - RESPONSE=$(curl -vk http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${IP}" 2>&1) +for _ in $(seq 1 "${retries}"); do + RESPONSE=$(curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${IP}" 2>&1) RESULT=$? - echo "Exit code: ${RESULT}" - echo -e "Response: \n${RESPONSE}\n\n" - if [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200 OK"; then - echo "Request fulfilled conditions to be successful (exit code = 0, response contains 'HTTP.*200 OK')" + [ $RESULT -eq 0 ] && + echo "${RESPONSE}" | grep -q -E "HTTP.*200" && + echo "${RESPONSE}" | grep -q "Hello MicroShift" && exit 0 - fi - echo -e "Waiting ${backoff} before next retry\n\n" + sleep "${backoff}" done -set -x exit 1 diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh index 78a0010720..f340e15a91 100755 --- a/e2e/tests/0031-loadbalancer-smoke-test.sh +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -5,36 +5,31 @@ IFS=$'\n\t' set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" -source "${SCRIPT_PATH}/../helpers.sh" +# shellcheck disable=SC2317 # Don't warn about unreachable commands in this function cleanup() { - oc delete service hello-microshift - oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" + oc delete service hello-microshift + oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" + firewall::close_port tcp:5678 } trap cleanup EXIT -gcloud compute firewall-rules update "${INSTANCE_PREFIX}" --allow tcp:22,icmp,tcp:5678 +firewall::open_port tcp:5678 oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" oc create service loadbalancer hello-microshift --tcp=5678:8080 oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s -set +x retries=3 backoff=3s -for try in $(seq 1 "${retries}"); do - echo "Attempt: ${try}" - echo "Running: curl -vk ${IP}:5678" - RESPONSE=$(curl -I ${IP}:5678 2>&1) - RESULT=$? - echo "Exit code: ${RESULT}" - echo -e "Response: \n${RESPONSE}\n\n" - if [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200"; then - echo "Request fulfilled conditions to be successful (exit code = 0, response contains 'HTTP.*200')" - exit 0 - fi - echo -e "Waiting ${backoff} before next retry\n\n" - sleep "${backoff}" +for _ in $(seq 1 "${retries}"); do + RESPONSE=$(curl -i "${IP}":5678 2>&1) + RESULT=$? + [ $RESULT -eq 0 ] && + echo "${RESPONSE}" | grep -q -E "HTTP.*200" && + echo "${RESPONSE}" | grep -q "Hello MicroShift" && + exit 0 + + sleep "${backoff}" done -set -x exit 1 From 30d1f0f6eb5af82db9ce0716915a353c330ed26e Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 30 Mar 2023 15:33:58 +0200 Subject: [PATCH 05/39] don't use VARs which might override local env --- e2e/main.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index 2ce3278fef..17510c3a36 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -66,9 +66,6 @@ prechecks() { main() { prechecks - export IP="$USHIFT_IP" - export USER="$USHIFT_USER" - local gather_debug=false for test in "${SCRIPT_PATH}"/tests/*.sh; do echo "RUNNING $(basename "${test}")" From 9065200fa6921ab5f847b3694d8073bd9cbe57a5 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 30 Mar 2023 15:34:17 +0200 Subject: [PATCH 06/39] greenboot --- e2e/tests/0050-greenboot.sh | 131 ++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/e2e/tests/0050-greenboot.sh b/e2e/tests/0050-greenboot.sh index f1f641af19..c037995dc0 100755 --- a/e2e/tests/0050-greenboot.sh +++ b/e2e/tests/0050-greenboot.sh @@ -1 +1,132 @@ #!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +scp "$SCRIPT_PATH/../../docs/config/busybox_running_check.sh" "$USHIFT_USER@$USHIFT_IP":~/busybox_running_check.sh + +ssh -q "$USHIFT_USER@$USHIFT_IP" <<'ENDSSH' +cat << 'ENDCAT' > ~/greenboot-test.sh +set -euo pipefail +IFS=$'\n\t' +set -x + +function check_greenboot_exit_status() { + local expectedRC=$1 + local cleanup=$2 + + if [ ${cleanup} -ne 0 ] ; then + echo 1 | microshift-cleanup-data --all + systemctl enable --now microshift || true + fi + + for check_script in $(find /etc/greenboot/check/ -name \*.sh | sort) ; do + echo Running ${check_script}... + local currentRC=1 + if ${check_script} ; then + currentRC=0 + fi + + if [ ${currentRC} != ${expectedRC} ] ; then + exit 1 + fi + done +} + +# +# Initial check must succeed (set timeout of 180s to speed up the process) +# +tee /etc/greenboot/greenboot.conf &>/dev/null </dev/null </dev/null </dev/null < Date: Thu, 30 Mar 2023 16:49:31 +0200 Subject: [PATCH 07/39] move busybox check script, other fixes --- e2e/tests/0030-router-smoke-test.sh | 10 +++++----- e2e/tests/0031-loadbalancer-smoke-test.sh | 8 ++++---- e2e/tests/0050-greenboot.sh | 2 +- .../tests/assets}/busybox_running_check.sh | 0 4 files changed, 10 insertions(+), 10 deletions(-) rename {docs/config => e2e/tests/assets}/busybox_running_check.sh (100%) diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh index 1573d1f040..424ddedcd4 100755 --- a/e2e/tests/0030-router-smoke-test.sh +++ b/e2e/tests/0030-router-smoke-test.sh @@ -8,14 +8,14 @@ SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" # shellcheck disable=SC2317 # Don't warn about unreachable commands in this function cleanup() { - oc delete route hello-microshift - oc delete service hello-microshift - oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" - firewall::close_port tcp:80 + oc delete route hello-microshift || true + oc delete service hello-microshift || true + oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true + firewall::close_port 80 tcp || true } trap cleanup EXIT -firewall::open_port tcp:80 +firewall::open_port 80 tcp oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" oc expose pod hello-microshift oc expose svc hello-microshift --hostname hello-microshift.cluster.local diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh index f340e15a91..56855c7a45 100755 --- a/e2e/tests/0031-loadbalancer-smoke-test.sh +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -8,13 +8,13 @@ SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" # shellcheck disable=SC2317 # Don't warn about unreachable commands in this function cleanup() { - oc delete service hello-microshift - oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" - firewall::close_port tcp:5678 + oc delete service hello-microshift || true + oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true + firewall::close_port 5678 tcp || true } trap cleanup EXIT -firewall::open_port tcp:5678 +firewall::open_port 5678 tcp oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" oc create service loadbalancer hello-microshift --tcp=5678:8080 oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s diff --git a/e2e/tests/0050-greenboot.sh b/e2e/tests/0050-greenboot.sh index c037995dc0..e134f54beb 100755 --- a/e2e/tests/0050-greenboot.sh +++ b/e2e/tests/0050-greenboot.sh @@ -6,7 +6,7 @@ set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" -scp "$SCRIPT_PATH/../../docs/config/busybox_running_check.sh" "$USHIFT_USER@$USHIFT_IP":~/busybox_running_check.sh +scp "$SCRIPT_PATH/assets/busybox_running_check.sh" "$USHIFT_USER@$USHIFT_IP":~/busybox_running_check.sh ssh -q "$USHIFT_USER@$USHIFT_IP" <<'ENDSSH' cat << 'ENDCAT' > ~/greenboot-test.sh diff --git a/docs/config/busybox_running_check.sh b/e2e/tests/assets/busybox_running_check.sh similarity index 100% rename from docs/config/busybox_running_check.sh rename to e2e/tests/assets/busybox_running_check.sh From d67d0d3765a6bc51c1fe08ce76fc6afa54ce9306 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Fri, 31 Mar 2023 07:38:59 +0200 Subject: [PATCH 08/39] fix IP var --- e2e/tests/0030-router-smoke-test.sh | 2 +- e2e/tests/0031-loadbalancer-smoke-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh index 424ddedcd4..26dc377438 100755 --- a/e2e/tests/0030-router-smoke-test.sh +++ b/e2e/tests/0030-router-smoke-test.sh @@ -24,7 +24,7 @@ oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s retries=3 backoff=3s for _ in $(seq 1 "${retries}"); do - RESPONSE=$(curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${IP}" 2>&1) + RESPONSE=$(curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${USHIFT_IP}" 2>&1) RESULT=$? [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200" && diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh index 56855c7a45..0e954a2d99 100755 --- a/e2e/tests/0031-loadbalancer-smoke-test.sh +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -22,7 +22,7 @@ oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s retries=3 backoff=3s for _ in $(seq 1 "${retries}"); do - RESPONSE=$(curl -i "${IP}":5678 2>&1) + RESPONSE=$(curl -i "${USHIFT_IP}":5678 2>&1) RESULT=$? [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200" && From e550e74c710d6e2828e0ac11070d88f5dd2c7c46 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Sat, 1 Apr 2023 17:25:41 +0200 Subject: [PATCH 09/39] PS4 with time, firewall:: rework --- e2e/tests/0030-router-smoke-test.sh | 20 ++++++++++++-------- e2e/tests/0031-loadbalancer-smoke-test.sh | 20 +++++++++++--------- e2e/tests/0050-greenboot.sh | 8 +++++--- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh index 26dc377438..de96e1f9ba 100755 --- a/e2e/tests/0030-router-smoke-test.sh +++ b/e2e/tests/0030-router-smoke-test.sh @@ -2,6 +2,7 @@ set -euo pipefail IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" @@ -11,11 +12,14 @@ cleanup() { oc delete route hello-microshift || true oc delete service hello-microshift || true oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true - firewall::close_port 80 tcp || true + if declare -F firewall::close_port; then firewall::close_port 80 tcp || true; fi } -trap cleanup EXIT +RESULT=1 +# explicit exit after cleanup, to not have cleanup override RESULT +trap 'cleanup; exit $RESULT' EXIT + +declare -F firewall::open_port && firewall::open_port 80 tcp -firewall::open_port 80 tcp oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" oc expose pod hello-microshift oc expose svc hello-microshift --hostname hello-microshift.cluster.local @@ -26,12 +30,12 @@ backoff=3s for _ in $(seq 1 "${retries}"); do RESPONSE=$(curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${USHIFT_IP}" 2>&1) RESULT=$? - [ $RESULT -eq 0 ] && + if [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200" && - echo "${RESPONSE}" | grep -q "Hello MicroShift" && - exit 0 + echo "${RESPONSE}" | grep -q "Hello MicroShift"; then + RESULT=0 + break + fi sleep "${backoff}" done - -exit 1 diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh index 0e954a2d99..7e25ddc773 100755 --- a/e2e/tests/0031-loadbalancer-smoke-test.sh +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -2,6 +2,7 @@ set -euo pipefail IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" @@ -10,11 +11,13 @@ SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" cleanup() { oc delete service hello-microshift || true oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true - firewall::close_port 5678 tcp || true + if declare -F firewall::close_port; then firewall::close_port 5678 tcp || true; fi } -trap cleanup EXIT +RESULT=1 +# explicit exit after cleanup, to not have cleanup override RESULT +trap 'cleanup; exit $RESULT' EXIT -firewall::open_port 5678 tcp +declare -F firewall::open_port && firewall::open_port 80 tcp oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" oc create service loadbalancer hello-microshift --tcp=5678:8080 oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s @@ -24,12 +27,11 @@ backoff=3s for _ in $(seq 1 "${retries}"); do RESPONSE=$(curl -i "${USHIFT_IP}":5678 2>&1) RESULT=$? - [ $RESULT -eq 0 ] && + if [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200" && - echo "${RESPONSE}" | grep -q "Hello MicroShift" && - exit 0 - + echo "${RESPONSE}" | grep -q "Hello MicroShift"; then + RESULT=0 + break + fi sleep "${backoff}" done - -exit 1 diff --git a/e2e/tests/0050-greenboot.sh b/e2e/tests/0050-greenboot.sh index e134f54beb..00f237ccfc 100755 --- a/e2e/tests/0050-greenboot.sh +++ b/e2e/tests/0050-greenboot.sh @@ -2,16 +2,18 @@ set -euo pipefail IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" -scp "$SCRIPT_PATH/assets/busybox_running_check.sh" "$USHIFT_USER@$USHIFT_IP":~/busybox_running_check.sh +scp "$SCRIPT_PATH/assets/busybox_running_check.sh" "$USHIFT_USER@$USHIFT_IP":/tmp/busybox_running_check.sh ssh -q "$USHIFT_USER@$USHIFT_IP" <<'ENDSSH' cat << 'ENDCAT' > ~/greenboot-test.sh set -euo pipefail IFS=$'\n\t' +PS4='+ $(date "+%T.%N")\011 ' set -x function check_greenboot_exit_status() { @@ -92,7 +94,7 @@ EOF SCRIPT_FILE=/etc/greenboot/check/required.d/50_busybox_running_check.sh # The file comes from payload.tar -ln -s ~/busybox_running_check.sh ${SCRIPT_FILE} +ln -s /tmp/busybox_running_check.sh ${SCRIPT_FILE} check_greenboot_exit_status 0 1 rm -f ${SCRIPT_FILE} @@ -126,7 +128,7 @@ rm -f ${YAML_FILE} check_greenboot_exit_status 0 1 ENDCAT -chmod +x ~/greenboot-test.sh ~/busybox_running_check.sh +chmod +x ~/greenboot-test.sh /tmp/busybox_running_check.sh ENDSSH ssh -q "$USHIFT_USER@$USHIFT_IP" "sudo ~/greenboot-test.sh" From 75a816f823736390eb67c1f19333af498227ad07 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Sat, 1 Apr 2023 17:25:57 +0200 Subject: [PATCH 10/39] add cluster-debug-info to e2e --- e2e/cluster-debug-info.sh | 73 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 e2e/cluster-debug-info.sh diff --git a/e2e/cluster-debug-info.sh b/e2e/cluster-debug-info.sh new file mode 100755 index 0000000000..d28eb96add --- /dev/null +++ b/e2e/cluster-debug-info.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +declare -a commands_to_run=() +function to_run() { + cmd="$@" + commands_to_run+=("${cmd}") +} + +to_run oc get cm -n kube-public microshift-version -o=jsonpath='{.data}' +to_run microshift version +to_run microshift version -o yaml +to_run microshift show-config -m effective +to_run oc version +to_run sudo crictl version +to_run uname -a +to_run cat /etc/*-release + +RESOURCES=(nodes pods configmaps deployments daemonsets statefulsets services routes replicasets persistentvolumeclaims persistentvolumes storageclasses endpoints endpointslices csidrivers csinodes) +for resource in ${RESOURCES[*]}; do + to_run oc get "${resource}" -A +done + +to_run oc describe nodes +to_run oc get events -A --sort-by=.metadata.creationTimestamp + +for resource in ${RESOURCES[*]}; do + to_run oc get "${resource}" -A -o yaml +done + +TO_DESCRIBE=(deployments daemonsets statefulsets replicasets) +for ns in $(kubectl get namespace -o jsonpath='{.items..metadata.name}'); do + to_run oc get namespace $ns -o yaml + + for resource_type in ${TO_DESCRIBE[*]}; do + for resource in $(kubectl get $resource_type -n $ns -o name); do + to_run oc describe -n $ns $resource + done + done + + for pod in $(kubectl get pods -n $ns -o name); do + to_run oc describe -n $ns $pod + to_run oc get -n $ns $pod -o yaml + for container in $(kubectl get -n $ns $pod -o jsonpath='{.spec.containers[*].name}'); do + to_run oc logs -n $ns $pod $container + to_run oc logs --previous=true -n $ns $pod $container + done + done +done + +to_run nmcli +to_run ip a +to_run ip route +to_run sudo crictl images --digests +to_run sudo crictl ps +to_run sudo crictl pods +to_run ls -lah /etc/cni/net.d/ +to_run find /etc/cni/net.d/ -type f -exec echo {} \; -exec sudo cat {} \; -exec echo \; +to_run dnf list --installed +to_run dnf history +to_run sudo systemctl -a -l +to_run sudo journalctl -xu microshift +to_run sudo journalctl -xu microshift-etcd.scope + +echo -e "\n=== DEBUG INFORMATION ===\n" +echo "Following commands will be executed:" +for cmd in "${commands_to_run[@]}"; do + echo " - ${cmd}" +done + +for cmd in "${commands_to_run[@]}"; do + echo -e "\n\n > $ ${cmd}" + ${cmd} 2>&1 || true +done From 4ce8a6eb4d6fa553dcf71aa005bcadceca273af3 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Sat, 1 Apr 2023 19:26:13 +0200 Subject: [PATCH 11/39] python test runner --- e2e/helpers.sh | 27 ------- e2e/main.py | 191 +++++++++++++++++++++++++++++++++++++++++++++++++ e2e/main.sh | 88 ----------------------- 3 files changed, 191 insertions(+), 115 deletions(-) delete mode 100644 e2e/helpers.sh create mode 100755 e2e/main.py delete mode 100755 e2e/main.sh diff --git a/e2e/helpers.sh b/e2e/helpers.sh deleted file mode 100644 index ff223624f2..0000000000 --- a/e2e/helpers.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -gscp() { - local src="${1}" - local dst="${2}" - - gcloud compute \ - --project "${GOOGLE_PROJECT_ID}" \ - --quiet \ - scp \ - --zone "${GOOGLE_COMPUTE_ZONE}" \ - --recurse \ - "${src}" "rhel8user@${INSTANCE_PREFIX}:${dst}" -} - -gssh() { - local cmd="${1}" - - # TODO: Consider flag --quiet - gcloud compute \ - --project "${GOOGLE_PROJECT_ID}" \ - ssh \ - --zone "${GOOGLE_COMPUTE_ZONE}" \ - "rhel8user@${INSTANCE_PREFIX}" \ - --command \ - "${cmd}" -} diff --git a/e2e/main.py b/e2e/main.py new file mode 100755 index 0000000000..1844a4e19e --- /dev/null +++ b/e2e/main.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 + +import os +import sys +import time +import logging +import tempfile +import argparse +import subprocess +from pathlib import Path +import xml.etree.ElementTree as ET + +testsuite = ET.Element("testsuite") + +logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s') + +_SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__)) +_TESTS_DIR = Path(f"{_SCRIPT_DIR}/tests") + + +def run_cmd(args, extra_envs=None, strict=False): + env = os.environ.copy() + if extra_envs: + env = {**env, **extra_envs} + + result = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + universal_newlines=True) + if strict and result.returncode != 0: + raise RuntimeError(f"Command {' '.join(args)} failed: {result.stdout}") + return result.returncode == 0, result.stdout + + +def ssh(host, user, cmd, strict=False): + c = ["ssh", f"{user}@{host}", cmd] + return run_cmd(c, strict=strict) + + +def cleanup_microshift(host, user): + logging.debug("Cleaning MicroShift") + _, output = ssh(host, user, "echo 1 | sudo -E microshift-cleanup-data --all", strict=True) + return output + + +def setup_microshift(host, user): + logging.debug("Setting up MicroShift") + config = f"""cat << EOF | sudo tee /etc/microshift/config.yaml +--- +apiServer: + subjectAltNames: + - {host} +EOF""" + _, output = ssh(host, user, config, strict=True) + logging.debug("Starting MicroShift systemd service") + _, output2 = ssh(host, user, "sudo systemctl enable --now microshift", strict=True) + logging.debug("MicroShift systemd service started") + return output + "\n" + output2 + + +def wait_until_microshift_is_fully_functional(host, user): + logging.debug("Waiting until MicroShift is ready") + cmd = ("sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh | " + + "while IFS= read -r line; do printf '%s %s\\n' \"$(date +'%H:%M:%S.%N')\" \"$line\"; done") + _, output = ssh(host, user, cmd, strict=True) + return output + + +def get_kubeconfig(host, user): + logging.debug("Fetching kubeconfig") + cmd = f"sudo cat /var/lib/microshift/resources/kubeadmin/{host}/kubeconfig" + success, output = ssh(host, user, cmd) + if not success: + raise RuntimeError("Failed to obtain KUBECONFIG from the remote") + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as f: + f.write(output) + return f.name + + +def get_cluster_debug_info(host, user): + copy = ["scp", f"{_SCRIPT_DIR}/cluster-debug-info.sh", f"{user}@{host}:/tmp/cluster-debug-info.sh"] + run = "sudo KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig /tmp/cluster-debug-info.sh" + + run_cmd(copy, strict=True) + _, out = ssh(host, user, run, strict=True) + return out + + +def wrapper42(xml_parent, func, *args): + elem = ET.SubElement(xml_parent, func.__name__) + start = time.time() + elem.text = func(*args) + elem.set("elapsed", f"{time.time() - start:.2f}") + + +def run_test(test, host, user): + logging.info(f"Running test case {test.name}") + testcase = ET.Element("testcase", attrib={"name": test.name}) + success = False + konfig = "" + try: + wrapper42(testcase, cleanup_microshift, host, user) + wrapper42(testcase, setup_microshift, host, user) + wrapper42(testcase, wait_until_microshift_is_fully_functional, host, user) + + konfig = get_kubeconfig(host, user) + xml_out = ET.SubElement(testcase, "output") + start = time.time() + logging.info(f"Starting {str(test)}") + success, xml_out.text = run_cmd([str(test)], {"KUBECONFIG": konfig, "USHIFT_IP": host, "USHIFT_USER": user}) + elapsed = time.time() - start + testcase.set("time", f"{elapsed:.2f}") + if not success: + ET.SubElement(testcase, "failure", attrib={"msg": ""}) + logging.info(f"Test case {test.name} - {success=} {elapsed=:.2f}s") + + except Exception as e: + logging.error(f"Exception happened: '{e}'") + ET.SubElement(testcase, "failure", attrib={"msg": f"{e}"}) + wrapper42(testcase, get_cluster_debug_info, host, user) + finally: + if konfig: + os.unlink(konfig) + return testcase, success + + +def run_tests(tests, host, user): + testsuite = ET.Element("testsuite", attrib={"tests": f"{len(tests)}"}) + logging.info(f"Running test suite") + testsuite_start = time.time() + failures = 0 + for t in tests: + test_start = time.time() + testcase, success = run_test(t, host, user) + testsuite.append(testcase) + test_elapsed = time.time() - test_start + if not success: + failures = failures + 1 + logging.info(f"Test case and other activities took {test_elapsed=:.2f}s") + + testsuite_elapsed = time.time() - testsuite_start + testsuite.set("time", f"{testsuite_elapsed:.2f}") + testsuite.set("failures", f"{failures}") + + outdir = os.path.join(os.getenv("ARTIFACT_DIR", "_output"), "junit") + Path(outdir).mkdir(parents=True, exist_ok=True) + tree = ET.ElementTree(testsuite) + tree.write(os.path.join(outdir, "result.xml")) + logging.info(f"Test suite finished in {testsuite_elapsed:.2f}s: {len(tests)} tests, {failures} failures. \ + Details stored in {outdir}/result.xml") + return failures == 0 + + +def list_tests(tests): + [print(t.name) for t in tests] + + +def get_test_files(filter): + filter = f"*{filter}*" if filter else "*.*" + return list(_TESTS_DIR.glob(filter)) + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--filter', type=str) + + subparsers = parser.add_subparsers(dest='command') + + run = subparsers.add_parser("run", help="Run MicroShift end to end test suite") + run.add_argument('--host', required=True, type=str, help="IP address of remote MicroShift instance") + run.add_argument('--user', required=True, type=str, help="User of remote MicroShift instance. Must be configured for passwordless SSH") + + subparsers.add_parser("list", help="List tests from MicroShift end to end test suite") + + return parser.parse_args() + + +def main(): + args = get_args() + tests = get_test_files(args.filter) + + if args.command == "list": + list_tests(tests) + elif args.command == "run": + all_passed = run_tests(tests, args.host, args.user) + sys.exit(0 if all_passed else 1) + + +if __name__ == "__main__": + main() diff --git a/e2e/main.sh b/e2e/main.sh deleted file mode 100755 index 17510c3a36..0000000000 --- a/e2e/main.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -set -o errtrace -set -o nounset -set -o pipefail - -SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" - -var_should_not_be_empty() { - local var_name=${1} - if [ -z "${!var_name+x}" ]; then - echo >&2 "Environmental variable '${var_name}' is unset" - return 1 - elif [ -z "${!var_name}" ]; then - echo >&2 "Environmental variable '${var_name}' is empty" - return 1 - fi -} - -function_should_be_exported() { - local fname=${1} - if ! declare -F "$fname"; then - echo >&2 "Function '$fname' is unexported. It is expected that function is provided for interacting with cloud provider" - return 1 - fi -} - -file_should_exist() { - local file_var="$1" - - var_should_not_be_empty "$file_var" || return 1 - local file="${!file_var}" - - if [[ ! -e "$file" ]]; then - echo >&2 "File '$file' does not exist" - return 1 - fi - - if [[ ! -f "$file" ]]; then - echo >&2 "'$file' is not a file" - return 1 - fi - - if [[ ! -r "$file" ]]; then - echo >&2 "'$file' is missing 'read' permissions" - return 1 - fi - -} - -prechecks() { - # TODO: It'd be nice to run all prechecks each time, but exit at the end if any is unfullfiled - file_should_exist KUBECONFIG || exit 1 - var_should_not_be_empty USHIFT_IP || exit 1 - var_should_not_be_empty USHIFT_USER || exit 1 - # TODO Check passwordless SSH - # TODO Check passwordless sudo - - # Just warning for now - function_should_be_exported firewall::open_port # || exit 1 - function_should_be_exported firewall::close_port # || exit 1 -} - -# TODO: check_if_cluster_is_ready() {} - kuttl? - -main() { - prechecks - - local gather_debug=false - for test in "${SCRIPT_PATH}"/tests/*.sh; do - echo "RUNNING $(basename "${test}")" - ${test} - res=$? - if [ $res -eq 0 ]; then - echo "SUCCESS" - else - echo "FAILURE" - gather_debug=true - fi - done - - if ${gather_debug}; then - echo "Run cluster-debug-info.sh" - exit 1 - fi -} - -main From 5b738f66813c13ab3008b9d608addf62d2290d6d Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Sat, 1 Apr 2023 20:29:48 +0200 Subject: [PATCH 12/39] change printing to satisfy older python? --- e2e/main.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/e2e/main.py b/e2e/main.py index 1844a4e19e..e024b69c52 100755 --- a/e2e/main.py +++ b/e2e/main.py @@ -10,8 +10,6 @@ from pathlib import Path import xml.etree.ElementTree as ET -testsuite = ET.Element("testsuite") - logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s') _SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__)) @@ -113,7 +111,7 @@ def run_test(test, host, user): testcase.set("time", f"{elapsed:.2f}") if not success: ET.SubElement(testcase, "failure", attrib={"msg": ""}) - logging.info(f"Test case {test.name} - {success=} {elapsed=:.2f}s") + logging.info(f"Test case {test.name} - success={success} elapsed={elapsed:.2f}s") except Exception as e: logging.error(f"Exception happened: '{e}'") @@ -127,7 +125,7 @@ def run_test(test, host, user): def run_tests(tests, host, user): testsuite = ET.Element("testsuite", attrib={"tests": f"{len(tests)}"}) - logging.info(f"Running test suite") + logging.info("Running test suite") testsuite_start = time.time() failures = 0 for t in tests: @@ -137,7 +135,7 @@ def run_tests(tests, host, user): test_elapsed = time.time() - test_start if not success: failures = failures + 1 - logging.info(f"Test case and other activities took {test_elapsed=:.2f}s") + logging.info(f"Test case and other activities took {test_elapsed:.2f}s") testsuite_elapsed = time.time() - testsuite_start testsuite.set("time", f"{testsuite_elapsed:.2f}") From 2258e910782f8a8f1677de3af5cba5e9faa06ef0 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Sat, 1 Apr 2023 22:34:45 +0200 Subject: [PATCH 13/39] loadbalancer - fix port --- e2e/main.py | 2 +- e2e/tests/0031-loadbalancer-smoke-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/main.py b/e2e/main.py index e024b69c52..0d4279df3d 100755 --- a/e2e/main.py +++ b/e2e/main.py @@ -110,7 +110,7 @@ def run_test(test, host, user): elapsed = time.time() - start testcase.set("time", f"{elapsed:.2f}") if not success: - ET.SubElement(testcase, "failure", attrib={"msg": ""}) + ET.SubElement(testcase, "failure", attrib={"msg": xml_out.text}) logging.info(f"Test case {test.name} - success={success} elapsed={elapsed:.2f}s") except Exception as e: diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh index 7e25ddc773..f0a2fcf717 100755 --- a/e2e/tests/0031-loadbalancer-smoke-test.sh +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -17,7 +17,7 @@ RESULT=1 # explicit exit after cleanup, to not have cleanup override RESULT trap 'cleanup; exit $RESULT' EXIT -declare -F firewall::open_port && firewall::open_port 80 tcp +declare -F firewall::open_port && firewall::open_port 5678 tcp oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" oc create service loadbalancer hello-microshift --tcp=5678:8080 oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s From 9b92950e18e1056d65707ac8bbce7bb6ffa4269b Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Mon, 3 Apr 2023 14:46:46 +0200 Subject: [PATCH 14/39] bring back bash test runner --- e2e/main.py | 189 ---------------------------------------------------- e2e/main.sh | 134 +++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 189 deletions(-) delete mode 100755 e2e/main.py create mode 100755 e2e/main.sh diff --git a/e2e/main.py b/e2e/main.py deleted file mode 100755 index 0d4279df3d..0000000000 --- a/e2e/main.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import time -import logging -import tempfile -import argparse -import subprocess -from pathlib import Path -import xml.etree.ElementTree as ET - -logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s') - -_SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__)) -_TESTS_DIR = Path(f"{_SCRIPT_DIR}/tests") - - -def run_cmd(args, extra_envs=None, strict=False): - env = os.environ.copy() - if extra_envs: - env = {**env, **extra_envs} - - result = subprocess.run(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=env, - universal_newlines=True) - if strict and result.returncode != 0: - raise RuntimeError(f"Command {' '.join(args)} failed: {result.stdout}") - return result.returncode == 0, result.stdout - - -def ssh(host, user, cmd, strict=False): - c = ["ssh", f"{user}@{host}", cmd] - return run_cmd(c, strict=strict) - - -def cleanup_microshift(host, user): - logging.debug("Cleaning MicroShift") - _, output = ssh(host, user, "echo 1 | sudo -E microshift-cleanup-data --all", strict=True) - return output - - -def setup_microshift(host, user): - logging.debug("Setting up MicroShift") - config = f"""cat << EOF | sudo tee /etc/microshift/config.yaml ---- -apiServer: - subjectAltNames: - - {host} -EOF""" - _, output = ssh(host, user, config, strict=True) - logging.debug("Starting MicroShift systemd service") - _, output2 = ssh(host, user, "sudo systemctl enable --now microshift", strict=True) - logging.debug("MicroShift systemd service started") - return output + "\n" + output2 - - -def wait_until_microshift_is_fully_functional(host, user): - logging.debug("Waiting until MicroShift is ready") - cmd = ("sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh | " + - "while IFS= read -r line; do printf '%s %s\\n' \"$(date +'%H:%M:%S.%N')\" \"$line\"; done") - _, output = ssh(host, user, cmd, strict=True) - return output - - -def get_kubeconfig(host, user): - logging.debug("Fetching kubeconfig") - cmd = f"sudo cat /var/lib/microshift/resources/kubeadmin/{host}/kubeconfig" - success, output = ssh(host, user, cmd) - if not success: - raise RuntimeError("Failed to obtain KUBECONFIG from the remote") - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as f: - f.write(output) - return f.name - - -def get_cluster_debug_info(host, user): - copy = ["scp", f"{_SCRIPT_DIR}/cluster-debug-info.sh", f"{user}@{host}:/tmp/cluster-debug-info.sh"] - run = "sudo KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig /tmp/cluster-debug-info.sh" - - run_cmd(copy, strict=True) - _, out = ssh(host, user, run, strict=True) - return out - - -def wrapper42(xml_parent, func, *args): - elem = ET.SubElement(xml_parent, func.__name__) - start = time.time() - elem.text = func(*args) - elem.set("elapsed", f"{time.time() - start:.2f}") - - -def run_test(test, host, user): - logging.info(f"Running test case {test.name}") - testcase = ET.Element("testcase", attrib={"name": test.name}) - success = False - konfig = "" - try: - wrapper42(testcase, cleanup_microshift, host, user) - wrapper42(testcase, setup_microshift, host, user) - wrapper42(testcase, wait_until_microshift_is_fully_functional, host, user) - - konfig = get_kubeconfig(host, user) - xml_out = ET.SubElement(testcase, "output") - start = time.time() - logging.info(f"Starting {str(test)}") - success, xml_out.text = run_cmd([str(test)], {"KUBECONFIG": konfig, "USHIFT_IP": host, "USHIFT_USER": user}) - elapsed = time.time() - start - testcase.set("time", f"{elapsed:.2f}") - if not success: - ET.SubElement(testcase, "failure", attrib={"msg": xml_out.text}) - logging.info(f"Test case {test.name} - success={success} elapsed={elapsed:.2f}s") - - except Exception as e: - logging.error(f"Exception happened: '{e}'") - ET.SubElement(testcase, "failure", attrib={"msg": f"{e}"}) - wrapper42(testcase, get_cluster_debug_info, host, user) - finally: - if konfig: - os.unlink(konfig) - return testcase, success - - -def run_tests(tests, host, user): - testsuite = ET.Element("testsuite", attrib={"tests": f"{len(tests)}"}) - logging.info("Running test suite") - testsuite_start = time.time() - failures = 0 - for t in tests: - test_start = time.time() - testcase, success = run_test(t, host, user) - testsuite.append(testcase) - test_elapsed = time.time() - test_start - if not success: - failures = failures + 1 - logging.info(f"Test case and other activities took {test_elapsed:.2f}s") - - testsuite_elapsed = time.time() - testsuite_start - testsuite.set("time", f"{testsuite_elapsed:.2f}") - testsuite.set("failures", f"{failures}") - - outdir = os.path.join(os.getenv("ARTIFACT_DIR", "_output"), "junit") - Path(outdir).mkdir(parents=True, exist_ok=True) - tree = ET.ElementTree(testsuite) - tree.write(os.path.join(outdir, "result.xml")) - logging.info(f"Test suite finished in {testsuite_elapsed:.2f}s: {len(tests)} tests, {failures} failures. \ - Details stored in {outdir}/result.xml") - return failures == 0 - - -def list_tests(tests): - [print(t.name) for t in tests] - - -def get_test_files(filter): - filter = f"*{filter}*" if filter else "*.*" - return list(_TESTS_DIR.glob(filter)) - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--filter', type=str) - - subparsers = parser.add_subparsers(dest='command') - - run = subparsers.add_parser("run", help="Run MicroShift end to end test suite") - run.add_argument('--host', required=True, type=str, help="IP address of remote MicroShift instance") - run.add_argument('--user', required=True, type=str, help="User of remote MicroShift instance. Must be configured for passwordless SSH") - - subparsers.add_parser("list", help="List tests from MicroShift end to end test suite") - - return parser.parse_args() - - -def main(): - args = get_args() - tests = get_test_files(args.filter) - - if args.command == "list": - list_tests(tests) - elif args.command == "run": - all_passed = run_tests(tests, args.host, args.user) - sys.exit(0 if all_passed else 1) - - -if __name__ == "__main__": - main() diff --git a/e2e/main.sh b/e2e/main.sh new file mode 100755 index 0000000000..57e67d904b --- /dev/null +++ b/e2e/main.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +set -o errtrace +set -o nounset +set -o pipefail + +SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +log() { + echo -e "$(date +'%H:%M:%S.%N') $*" +} + +var_should_not_be_empty() { + local var_name=${1} + if [ -z "${!var_name+x}" ]; then + echo >&2 "Environmental variable '${var_name}' is unset" + return 1 + elif [ -z "${!var_name}" ]; then + echo >&2 "Environmental variable '${var_name}' is empty" + return 1 + fi +} + +function_should_be_exported() { + local fname=${1} + if ! declare -F "$fname"; then + echo >&2 "Function '$fname' is unexported. It is expected that function is provided for interacting with cloud provider" + return 1 + fi +} + +prechecks() { + var_should_not_be_empty USHIFT_IP || exit 1 + var_should_not_be_empty USHIFT_USER || exit 1 + # TODO Check passwordless SSH + # TODO Check passwordless sudo + + # Just warning for now + # Following functions needed only for runs in CI + function_should_be_exported firewall::open_port + function_should_be_exported firewall::close_port +} + +microshift_get_konfig() { + tmpfile=$(mktemp /tmp/microshift-e2e-konfig.XXXXXX) + ssh "$USHIFT_USER@$USHIFT_IP" 'sudo cat /var/lib/microshift/resources/kubeadmin/'"$USHIFT_IP"'/kubeconfig' >"$tmpfile" + echo "$tmpfile" +} + +microshift_check_readiness() { + log "Waiting for MicroShift to become ready" + ssh "$USHIFT_USER@$USHIFT_IP" \ + "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh | \ + while IFS= read -r line; do printf '%s %s\\n' \"\$(date +'%H:%M:%S.%N')\" \"\$line\"; done" +} + +microshift_setup() { + log "Setting up and starting MicroShift" + ssh "$USHIFT_USER@$USHIFT_IP" 'cat << EOF | sudo tee /etc/microshift/config.yaml +--- +apiServer: + subjectAltNames: + - '"$USHIFT_IP"' +EOF' + ssh "$USHIFT_USER@$USHIFT_IP" "sudo systemctl enable --now microshift" +} + +microshift_debug_info() { + log "Gathering debug info" + scp "$SCRIPT_DIR/cluster-debug-info.sh" "$USHIFT_USER@$USHIFT_IP:/tmp/cluster-debug-info.sh" + ssh "$USHIFT_USER@$USHIFT_IP" "sudo KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig /tmp/cluster-debug-info.sh" +} + +microshift_cleanup() { + log "Cleaning MicroShift" + ssh "$USHIFT_USER@$USHIFT_IP" "echo 1 | sudo microshift-cleanup-data --all" +} + +run_test() { + local test=$1 + echo -e "\n\n==================================================" + log "${test} - RUNNING" + + microshift_cleanup + microshift_setup + microshift_check_readiness + konfig=$(microshift_get_konfig) + + start=$(date +%s) + set +e + KUBECONFIG="$konfig" "${SCRIPT_DIR}/tests/${test}" + res=$? + set -e + rm -f "${konfig}" + + end="$(date +%s)" + duration_total_seconds=$((end - start)) + log "${test} - took ${duration_total_seconds}s" + if [ $res -eq 0 ]; then + log "${test} - SUCCESS" + return 0 + fi + + log "${test} - FAILURE" + microshift_debug_info || true # TODO: > artifacts/file + return 1 +} + +list() { + find "${SCRIPT_DIR}/tests" -maxdepth 1 -iname "*.sh" -printf "%f\n" +} + +run() { + local -r filter="*${1:-}*.sh" + + local -r to_run=$(find "${SCRIPT_DIR}/tests" -maxdepth 1 -iname "$filter" -printf "%f\n") + log "Following tests will run:\n$to_run" + + prechecks + all_successful=true + for t in $to_run; do + run_test "${t}" || all_successful=false + done + "${all_successful}" +} + +[ $# -eq 0 ] && { + echo "usage" + exit 1 +} + +cmd="$1" +shift +"${cmd}" "${1:-}" From 1b995a6491735027f46bd9affe9e5d171e106933 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Mon, 3 Apr 2023 18:02:30 +0200 Subject: [PATCH 15/39] move "inner" greenboot test to separate file --- e2e/tests/0050-greenboot.sh | 127 +---------------------------- e2e/tests/assets/greenboot-test.sh | 117 ++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 125 deletions(-) create mode 100644 e2e/tests/assets/greenboot-test.sh diff --git a/e2e/tests/0050-greenboot.sh b/e2e/tests/0050-greenboot.sh index 00f237ccfc..80c093c273 100755 --- a/e2e/tests/0050-greenboot.sh +++ b/e2e/tests/0050-greenboot.sh @@ -7,128 +7,5 @@ set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" -scp "$SCRIPT_PATH/assets/busybox_running_check.sh" "$USHIFT_USER@$USHIFT_IP":/tmp/busybox_running_check.sh - -ssh -q "$USHIFT_USER@$USHIFT_IP" <<'ENDSSH' -cat << 'ENDCAT' > ~/greenboot-test.sh -set -euo pipefail -IFS=$'\n\t' -PS4='+ $(date "+%T.%N")\011 ' -set -x - -function check_greenboot_exit_status() { - local expectedRC=$1 - local cleanup=$2 - - if [ ${cleanup} -ne 0 ] ; then - echo 1 | microshift-cleanup-data --all - systemctl enable --now microshift || true - fi - - for check_script in $(find /etc/greenboot/check/ -name \*.sh | sort) ; do - echo Running ${check_script}... - local currentRC=1 - if ${check_script} ; then - currentRC=0 - fi - - if [ ${currentRC} != ${expectedRC} ] ; then - exit 1 - fi - done -} - -# -# Initial check must succeed (set timeout of 180s to speed up the process) -# -tee /etc/greenboot/greenboot.conf &>/dev/null </dev/null </dev/null </dev/null </dev/null </dev/null </dev/null </dev/null < Date: Tue, 4 Apr 2023 17:17:43 +0200 Subject: [PATCH 16/39] fix inner greenboot test path --- e2e/tests/0050-greenboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/tests/0050-greenboot.sh b/e2e/tests/0050-greenboot.sh index 80c093c273..9d6c610e1f 100755 --- a/e2e/tests/0050-greenboot.sh +++ b/e2e/tests/0050-greenboot.sh @@ -8,4 +8,4 @@ set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" scp "$SCRIPT_PATH/assets/busybox_running_check.sh" "$SCRIPT_PATH/assets/greenboot-test.sh" "$USHIFT_USER@$USHIFT_IP":/tmp/ -ssh -q "$USHIFT_USER@$USHIFT_IP" "chmod +x ~/greenboot-test.sh /tmp/busybox_running_check.sh && sudo ~/greenboot-test.sh" +ssh -q "$USHIFT_USER@$USHIFT_IP" "chmod +x /tmp/greenboot-test.sh /tmp/busybox_running_check.sh && sudo /tmp/greenboot-test.sh" From c94e5f10ef559055b0ab2088a978bc104b3c5edf Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Tue, 4 Apr 2023 17:18:45 +0200 Subject: [PATCH 17/39] don't duplicate files, move files back to orig loc --- .../config}/busybox_running_check.sh | 0 e2e/cluster-debug-info.sh | 73 ------------------- e2e/main.sh | 2 +- e2e/tests/0050-greenboot.sh | 2 +- 4 files changed, 2 insertions(+), 75 deletions(-) rename {e2e/tests/assets => docs/config}/busybox_running_check.sh (100%) delete mode 100755 e2e/cluster-debug-info.sh diff --git a/e2e/tests/assets/busybox_running_check.sh b/docs/config/busybox_running_check.sh similarity index 100% rename from e2e/tests/assets/busybox_running_check.sh rename to docs/config/busybox_running_check.sh diff --git a/e2e/cluster-debug-info.sh b/e2e/cluster-debug-info.sh deleted file mode 100755 index d28eb96add..0000000000 --- a/e2e/cluster-debug-info.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash - -declare -a commands_to_run=() -function to_run() { - cmd="$@" - commands_to_run+=("${cmd}") -} - -to_run oc get cm -n kube-public microshift-version -o=jsonpath='{.data}' -to_run microshift version -to_run microshift version -o yaml -to_run microshift show-config -m effective -to_run oc version -to_run sudo crictl version -to_run uname -a -to_run cat /etc/*-release - -RESOURCES=(nodes pods configmaps deployments daemonsets statefulsets services routes replicasets persistentvolumeclaims persistentvolumes storageclasses endpoints endpointslices csidrivers csinodes) -for resource in ${RESOURCES[*]}; do - to_run oc get "${resource}" -A -done - -to_run oc describe nodes -to_run oc get events -A --sort-by=.metadata.creationTimestamp - -for resource in ${RESOURCES[*]}; do - to_run oc get "${resource}" -A -o yaml -done - -TO_DESCRIBE=(deployments daemonsets statefulsets replicasets) -for ns in $(kubectl get namespace -o jsonpath='{.items..metadata.name}'); do - to_run oc get namespace $ns -o yaml - - for resource_type in ${TO_DESCRIBE[*]}; do - for resource in $(kubectl get $resource_type -n $ns -o name); do - to_run oc describe -n $ns $resource - done - done - - for pod in $(kubectl get pods -n $ns -o name); do - to_run oc describe -n $ns $pod - to_run oc get -n $ns $pod -o yaml - for container in $(kubectl get -n $ns $pod -o jsonpath='{.spec.containers[*].name}'); do - to_run oc logs -n $ns $pod $container - to_run oc logs --previous=true -n $ns $pod $container - done - done -done - -to_run nmcli -to_run ip a -to_run ip route -to_run sudo crictl images --digests -to_run sudo crictl ps -to_run sudo crictl pods -to_run ls -lah /etc/cni/net.d/ -to_run find /etc/cni/net.d/ -type f -exec echo {} \; -exec sudo cat {} \; -exec echo \; -to_run dnf list --installed -to_run dnf history -to_run sudo systemctl -a -l -to_run sudo journalctl -xu microshift -to_run sudo journalctl -xu microshift-etcd.scope - -echo -e "\n=== DEBUG INFORMATION ===\n" -echo "Following commands will be executed:" -for cmd in "${commands_to_run[@]}"; do - echo " - ${cmd}" -done - -for cmd in "${commands_to_run[@]}"; do - echo -e "\n\n > $ ${cmd}" - ${cmd} 2>&1 || true -done diff --git a/e2e/main.sh b/e2e/main.sh index 57e67d904b..8166e3eb4f 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -67,7 +67,7 @@ EOF' microshift_debug_info() { log "Gathering debug info" - scp "$SCRIPT_DIR/cluster-debug-info.sh" "$USHIFT_USER@$USHIFT_IP:/tmp/cluster-debug-info.sh" + scp "$SCRIPT_DIR/../validate-microshift/cluster-debug-info.sh" "$USHIFT_USER@$USHIFT_IP:/tmp/cluster-debug-info.sh" ssh "$USHIFT_USER@$USHIFT_IP" "sudo KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig /tmp/cluster-debug-info.sh" } diff --git a/e2e/tests/0050-greenboot.sh b/e2e/tests/0050-greenboot.sh index 9d6c610e1f..75ab9d10c6 100755 --- a/e2e/tests/0050-greenboot.sh +++ b/e2e/tests/0050-greenboot.sh @@ -7,5 +7,5 @@ set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" -scp "$SCRIPT_PATH/assets/busybox_running_check.sh" "$SCRIPT_PATH/assets/greenboot-test.sh" "$USHIFT_USER@$USHIFT_IP":/tmp/ +scp "$SCRIPT_PATH/../../docs/config/busybox_running_check.sh" "$SCRIPT_PATH/assets/greenboot-test.sh" "$USHIFT_USER@$USHIFT_IP":/tmp/ ssh -q "$USHIFT_USER@$USHIFT_IP" "chmod +x /tmp/greenboot-test.sh /tmp/busybox_running_check.sh && sudo /tmp/greenboot-test.sh" From 820db7d16c9a9d0b3aeec19c418e98e6d3111d14 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Tue, 4 Apr 2023 17:20:12 +0200 Subject: [PATCH 18/39] cluster-debug-info.sh: use default kubeconfig if unset --- e2e/main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/main.sh b/e2e/main.sh index 8166e3eb4f..3381025dea 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -68,7 +68,7 @@ EOF' microshift_debug_info() { log "Gathering debug info" scp "$SCRIPT_DIR/../validate-microshift/cluster-debug-info.sh" "$USHIFT_USER@$USHIFT_IP:/tmp/cluster-debug-info.sh" - ssh "$USHIFT_USER@$USHIFT_IP" "sudo KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig /tmp/cluster-debug-info.sh" + ssh "$USHIFT_USER@$USHIFT_IP" "sudo /tmp/cluster-debug-info.sh" } microshift_cleanup() { From 2c2f9327e9af34964685dcc7c30b17db6ecf1812 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Tue, 4 Apr 2023 17:22:59 +0200 Subject: [PATCH 19/39] use trap to remove kubeconfig --- e2e/main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/main.sh b/e2e/main.sh index 3381025dea..e1fe1b3bac 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -85,13 +85,13 @@ run_test() { microshift_setup microshift_check_readiness konfig=$(microshift_get_konfig) + trap 'rm -f "${konfig}"' RETURN start=$(date +%s) set +e KUBECONFIG="$konfig" "${SCRIPT_DIR}/tests/${test}" res=$? set -e - rm -f "${konfig}" end="$(date +%s)" duration_total_seconds=$((end - start)) From c5f866673997a3dc345ded712fa7a1905d833439 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Tue, 4 Apr 2023 18:17:00 +0200 Subject: [PATCH 20/39] forward output to files --- e2e/main.sh | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index e1fe1b3bac..14bf865d88 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -5,6 +5,8 @@ set -o nounset set -o pipefail SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +OUTPUT_DIR="${ARTIFACT_DIR:-_output}/microshift-e2e-$(date +'%Y%m%d-%H%M%S')/" +[ ! -d "${OUTPUT_DIR}" ] && mkdir -p "${OUTPUT_DIR}" log() { echo -e "$(date +'%H:%M:%S.%N') $*" @@ -48,32 +50,44 @@ microshift_get_konfig() { } microshift_check_readiness() { + local test_output="${1}" log "Waiting for MicroShift to become ready" ssh "$USHIFT_USER@$USHIFT_IP" \ "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh | \ - while IFS= read -r line; do printf '%s %s\\n' \"\$(date +'%H:%M:%S.%N')\" \"\$line\"; done" + while IFS= read -r line; do printf '%s %s\\n' \"\$(date +'%H:%M:%S.%N')\" \"\$line\"; done" &>"${test_output}/0002-readiness-check.log" } microshift_setup() { + local test_output="${1}" log "Setting up and starting MicroShift" ssh "$USHIFT_USER@$USHIFT_IP" 'cat << EOF | sudo tee /etc/microshift/config.yaml --- apiServer: subjectAltNames: - '"$USHIFT_IP"' -EOF' - ssh "$USHIFT_USER@$USHIFT_IP" "sudo systemctl enable --now microshift" +EOF' &>"${test_output}/0001-setup.log" + ssh "$USHIFT_USER@$USHIFT_IP" "sudo systemctl enable --now microshift" &>>"${test_output}/0001-setup.log" } microshift_debug_info() { - log "Gathering debug info" + local test_output="${1}" + log "Gathering debug info to ${test_output}/0020-cluster-debug-info.log" scp "$SCRIPT_DIR/../validate-microshift/cluster-debug-info.sh" "$USHIFT_USER@$USHIFT_IP:/tmp/cluster-debug-info.sh" - ssh "$USHIFT_USER@$USHIFT_IP" "sudo /tmp/cluster-debug-info.sh" + ssh "$USHIFT_USER@$USHIFT_IP" "sudo /tmp/cluster-debug-info.sh" &>"${test_output}/0020-cluster-debug-info.log" } microshift_cleanup() { + local test_output="${1}" log "Cleaning MicroShift" - ssh "$USHIFT_USER@$USHIFT_IP" "echo 1 | sudo microshift-cleanup-data --all" + ssh "$USHIFT_USER@$USHIFT_IP" "echo 1 | sudo microshift-cleanup-data --all" &>"${test_output}/0000-cleanup.log" +} + +microshift_health_summary() { + local konfig=${1} + log "Summary of MicroShift health" + oc --kubeconfig "${konfig}" get pods -A + oc --kubeconfig "${konfig}" get nodes -o wide + oc --kubeconfig "${konfig}" get events -A --sort-by=.metadata.creationTimestamp | head -n 20 } run_test() { @@ -81,20 +95,22 @@ run_test() { echo -e "\n\n==================================================" log "${test} - RUNNING" - microshift_cleanup - microshift_setup - microshift_check_readiness + local test_output="${OUTPUT_DIR}/${test}/" + mkdir -p "${test_output}" + + microshift_cleanup "${test_output}" + microshift_setup "${test_output}" + microshift_check_readiness "${test_output}" konfig=$(microshift_get_konfig) trap 'rm -f "${konfig}"' RETURN start=$(date +%s) set +e - KUBECONFIG="$konfig" "${SCRIPT_DIR}/tests/${test}" + KUBECONFIG="$konfig" "${SCRIPT_DIR}/tests/${test}" &>"${test_output}/0010-test.log" res=$? set -e - end="$(date +%s)" - duration_total_seconds=$((end - start)) + duration_total_seconds=$(($(date +%s) - start)) log "${test} - took ${duration_total_seconds}s" if [ $res -eq 0 ]; then log "${test} - SUCCESS" @@ -102,7 +118,8 @@ run_test() { fi log "${test} - FAILURE" - microshift_debug_info || true # TODO: > artifacts/file + microshift_health_summary "${konfig}" + microshift_debug_info "${test_output}" || true return 1 } From 5b060daa1ef3c7d7eac2d5b4b44ae073de028a3a Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Wed, 5 Apr 2023 09:35:54 +0200 Subject: [PATCH 21/39] smoke tests - don't fail, capture exit code --- e2e/tests/0030-router-smoke-test.sh | 2 ++ e2e/tests/0031-loadbalancer-smoke-test.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh index de96e1f9ba..8776878349 100755 --- a/e2e/tests/0030-router-smoke-test.sh +++ b/e2e/tests/0030-router-smoke-test.sh @@ -28,8 +28,10 @@ oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s retries=3 backoff=3s for _ in $(seq 1 "${retries}"); do + set +x RESPONSE=$(curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${USHIFT_IP}" 2>&1) RESULT=$? + set -x if [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200" && echo "${RESPONSE}" | grep -q "Hello MicroShift"; then diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh index f0a2fcf717..37c0ba879e 100755 --- a/e2e/tests/0031-loadbalancer-smoke-test.sh +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -25,8 +25,10 @@ oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s retries=3 backoff=3s for _ in $(seq 1 "${retries}"); do + set +x RESPONSE=$(curl -i "${USHIFT_IP}":5678 2>&1) RESULT=$? + set -x if [ $RESULT -eq 0 ] && echo "${RESPONSE}" | grep -q -E "HTTP.*200" && echo "${RESPONSE}" | grep -q "Hello MicroShift"; then From 64d3a3d01ec6c3fd216336be0ca13dcbf84363db Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Wed, 5 Apr 2023 09:36:06 +0200 Subject: [PATCH 22/39] health summary via ssh --- e2e/main.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index 14bf865d88..34f0acf754 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -83,11 +83,16 @@ microshift_cleanup() { } microshift_health_summary() { - local konfig=${1} log "Summary of MicroShift health" - oc --kubeconfig "${konfig}" get pods -A - oc --kubeconfig "${konfig}" get nodes -o wide - oc --kubeconfig "${konfig}" get events -A --sort-by=.metadata.creationTimestamp | head -n 20 + + # Because test might be "destructive" (i.e. tear down and set up again MicroShift) + # so these commands are executed via ssh. + # Alternative is to copy kubeconfig second time in the same time. + ssh "$USHIFT_USER@$USHIFT_IP" \ + "mkdir -p ~/.kube/ && sudo cat /var/lib/microshift/resources/kubeadmin/kubeconfig > ~/.kube/config ; \ + oc get pods -A ; \ + oc get nodes -o wide ; \ + oc get events -A --sort-by=.metadata.creationTimestamp | head -n 20" } run_test() { @@ -118,7 +123,7 @@ run_test() { fi log "${test} - FAILURE" - microshift_health_summary "${konfig}" + microshift_health_summary microshift_debug_info "${test_output}" || true return 1 } From 44bd5992eae0cdd811a7e137296869b33768297f Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:39:30 +0200 Subject: [PATCH 23/39] reuse list() --- e2e/main.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index 34f0acf754..591ed49f3a 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -129,13 +129,12 @@ run_test() { } list() { - find "${SCRIPT_DIR}/tests" -maxdepth 1 -iname "*.sh" -printf "%f\n" + local -r filter="*${1:-}*.sh" + find "${SCRIPT_DIR}/tests" -maxdepth 1 -iname "$filter" -printf "%f\n" } run() { - local -r filter="*${1:-}*.sh" - - local -r to_run=$(find "${SCRIPT_DIR}/tests" -maxdepth 1 -iname "$filter" -printf "%f\n") + local -r to_run=$(list "${1}") log "Following tests will run:\n$to_run" prechecks From 83bf1fb845a57fc3bb2a3c16283a17b4b28d0faf Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:39:48 +0200 Subject: [PATCH 24/39] e2e-reboot --- e2e/tests/0100-reboot.sh | 36 +++++++++++++++++++++++++ e2e/tests/assets/pod-with-pvc.yaml | 42 ++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 e2e/tests/assets/pod-with-pvc.yaml diff --git a/e2e/tests/0100-reboot.sh b/e2e/tests/0100-reboot.sh index f1f641af19..73b84bb1ca 100755 --- a/e2e/tests/0100-reboot.sh +++ b/e2e/tests/0100-reboot.sh @@ -1 +1,37 @@ #!/usr/bin/env bash + +set -euo pipefail +PS4='+ $(date "+%T.%N")\011 ' +set -x + +SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +_RETRIES=5 +_BACKOFF=20 + +wait_until() { + local cmd=$* + for _ in $(seq "${_RETRIES}"); do + ${cmd} && return 0 + sleep "${_BACKOFF}" + done + return 1 +} + +cleanup() { + oc delete -f "${SCRIPT_PATH}/assets/pod-with-pvc.yaml" +} +trap 'cleanup' EXIT + +# TODO: Remove the labels again once https://issues.redhat.com/browse/OCPBUGS-1969 has been fixed upstream +oc label namespaces default "pod-security.kubernetes.io/"{enforce,audit,warn}"-version=v1.24" +oc label namespaces default "pod-security.kubernetes.io/"{enforce,audit,warn}"=privileged" +oc create -f "${SCRIPT_PATH}/assets/pod-with-pvc.yaml" +oc wait --for=condition=Ready --timeout=120s pod/test-pod + +ssh "$USHIFT_USER@$USHIFT_IP" "sudo reboot now" +wait_until ssh "$USHIFT_USER@$USHIFT_IP" "true" +# Just check if KAS is up and serving +wait_until oc get node +ssh "$USHIFT_USER@$USHIFT_IP" "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh" +oc wait --for=condition=Ready --timeout=120s pod/test-pod diff --git a/e2e/tests/assets/pod-with-pvc.yaml b/e2e/tests/assets/pod-with-pvc.yaml new file mode 100644 index 0000000000..1750da5c4c --- /dev/null +++ b/e2e/tests/assets/pod-with-pvc.yaml @@ -0,0 +1,42 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + namespace: default + name: test-claim +spec: + accessModes: + - ReadWriteOnce + storageClassName: topolvm-provisioner + resources: + requests: + storage: 1Gi +--- +kind: Pod +apiVersion: v1 +metadata: + name: test-pod + namespace: default +spec: + securityContext: + runAsNonRoot: false # TODO pick different image + privileged: false + capabilities: + drop: + - 'ALL' + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + containers: + - name: test-container + image: quay.io/openshifttest/hello-openshift:1.2.0 + command: + - sh + - -c + - sleep 1d + volumeMounts: + - mountPath: /vol + name: test-vol + volumes: + - name: test-vol + persistentVolumeClaim: + claimName: test-claim From f8be940e4022d789b950d561763032582bdd71bf Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 12:27:52 +0200 Subject: [PATCH 25/39] more bash sanity to router and lb tests --- e2e/tests/0030-router-smoke-test.sh | 35 +++++++++++------------ e2e/tests/0031-loadbalancer-smoke-test.sh | 34 +++++++++++----------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh index 8776878349..afbd3b76c4 100755 --- a/e2e/tests/0030-router-smoke-test.sh +++ b/e2e/tests/0030-router-smoke-test.sh @@ -7,6 +7,9 @@ set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +RETRIES=3 +BACKOFF=3s + # shellcheck disable=SC2317 # Don't warn about unreachable commands in this function cleanup() { oc delete route hello-microshift || true @@ -14,9 +17,7 @@ cleanup() { oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true if declare -F firewall::close_port; then firewall::close_port 80 tcp || true; fi } -RESULT=1 -# explicit exit after cleanup, to not have cleanup override RESULT -trap 'cleanup; exit $RESULT' EXIT +trap 'cleanup' EXIT declare -F firewall::open_port && firewall::open_port 80 tcp @@ -25,19 +26,17 @@ oc expose pod hello-microshift oc expose svc hello-microshift --hostname hello-microshift.cluster.local oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s -retries=3 -backoff=3s -for _ in $(seq 1 "${retries}"); do - set +x - RESPONSE=$(curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${USHIFT_IP}" 2>&1) - RESULT=$? - set -x - if [ $RESULT -eq 0 ] && - echo "${RESPONSE}" | grep -q -E "HTTP.*200" && - echo "${RESPONSE}" | grep -q "Hello MicroShift"; then - RESULT=0 - break - fi - - sleep "${backoff}" +for _ in $(seq "${RETRIES}"); do + set +e + response=$(curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${USHIFT_IP}" 2>&1) + result=$? + set -e + + [ $result -eq 0 ] && + echo "${response}" | grep -q -E "HTTP.*200" && + echo "${response}" | grep -q "Hello MicroShift" && + exit 0 + + sleep "${BACKOFF}" done +exit 1 diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh index 37c0ba879e..f170b1e91d 100755 --- a/e2e/tests/0031-loadbalancer-smoke-test.sh +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -7,33 +7,33 @@ set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +RETRIES=3 +BACKOFF=3s + # shellcheck disable=SC2317 # Don't warn about unreachable commands in this function cleanup() { oc delete service hello-microshift || true oc delete -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" || true if declare -F firewall::close_port; then firewall::close_port 5678 tcp || true; fi } -RESULT=1 -# explicit exit after cleanup, to not have cleanup override RESULT -trap 'cleanup; exit $RESULT' EXIT +trap 'cleanup' EXIT declare -F firewall::open_port && firewall::open_port 5678 tcp oc create -f "${SCRIPT_PATH}/assets/hello-microshift.yaml" oc create service loadbalancer hello-microshift --tcp=5678:8080 oc wait pods -l app=hello-microshift --for condition=Ready --timeout=60s -retries=3 -backoff=3s -for _ in $(seq 1 "${retries}"); do - set +x - RESPONSE=$(curl -i "${USHIFT_IP}":5678 2>&1) - RESULT=$? - set -x - if [ $RESULT -eq 0 ] && - echo "${RESPONSE}" | grep -q -E "HTTP.*200" && - echo "${RESPONSE}" | grep -q "Hello MicroShift"; then - RESULT=0 - break - fi - sleep "${backoff}" +for _ in $(seq "${RETRIES}"); do + set +e + response=$(curl -i "${USHIFT_IP}":5678 2>&1) + result=$? + set -e + + [ $result -eq 0 ] && + echo "${response}" | grep -q -E "HTTP.*200" && + echo "${response}" | grep -q "Hello MicroShift" && + exit 0 + + sleep "${BACKOFF}" done +exit 1 From d0bc6feb0ad14d568264d8ef88a51a03d6d25034 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 12:28:18 +0200 Subject: [PATCH 26/39] log duration of test itself and preparations --- e2e/main.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index 591ed49f3a..37fa7d3a32 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -103,20 +103,23 @@ run_test() { local test_output="${OUTPUT_DIR}/${test}/" mkdir -p "${test_output}" + prep_start=$(date +%s) microshift_cleanup "${test_output}" microshift_setup "${test_output}" microshift_check_readiness "${test_output}" konfig=$(microshift_get_konfig) trap 'rm -f "${konfig}"' RETURN + prep_dur=$(($(date +%s) - prep_start)) - start=$(date +%s) + test_start=$(date +%s) set +e KUBECONFIG="$konfig" "${SCRIPT_DIR}/tests/${test}" &>"${test_output}/0010-test.log" res=$? set -e + test_dur=$(($(date +%s) - test_start)) - duration_total_seconds=$(($(date +%s) - start)) - log "${test} - took ${duration_total_seconds}s" + log "${test} took $((test_dur / 60))m $((test_dur % 60))s." \ + "Setup and readiness took $((prep_dur / 60))m $((prep_dur % 60))s." if [ $res -eq 0 ]; then log "${test} - SUCCESS" return 0 From e531113e84c41146dc08e1f03c9ad1bcc2c55f62 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 12:28:55 +0200 Subject: [PATCH 27/39] consistent top level var naming --- e2e/tests/0100-reboot.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/tests/0100-reboot.sh b/e2e/tests/0100-reboot.sh index 73b84bb1ca..4230e08540 100755 --- a/e2e/tests/0100-reboot.sh +++ b/e2e/tests/0100-reboot.sh @@ -6,14 +6,14 @@ set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" -_RETRIES=5 -_BACKOFF=20 +RETRIES=5 +BACKOFF=20 wait_until() { local cmd=$* - for _ in $(seq "${_RETRIES}"); do + for _ in $(seq "${RETRIES}"); do ${cmd} && return 0 - sleep "${_BACKOFF}" + sleep "${BACKOFF}" done return 1 } From 3af9abda33a3ec3c32f1ebf59fe7221ecf3b6288 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:35:47 +0200 Subject: [PATCH 28/39] reboot: allow for ssh exit 0 or 255 --- e2e/tests/0100-reboot.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/e2e/tests/0100-reboot.sh b/e2e/tests/0100-reboot.sh index 4230e08540..46941c7dc8 100755 --- a/e2e/tests/0100-reboot.sh +++ b/e2e/tests/0100-reboot.sh @@ -29,7 +29,17 @@ oc label namespaces default "pod-security.kubernetes.io/"{enforce,audit,warn}"=p oc create -f "${SCRIPT_PATH}/assets/pod-with-pvc.yaml" oc wait --for=condition=Ready --timeout=120s pod/test-pod -ssh "$USHIFT_USER@$USHIFT_IP" "sudo reboot now" +set +e +ssh -v "$USHIFT_USER@$USHIFT_IP" "sudo reboot now" +res=$? +set -e + +# Allow for `ssh` command errors (255 exit code) like "connection closed by remote host" +# Fail on other errors (coming from the command executed remotely itself) +if [ "${res}" -ne 0 ] && [ "${res}" -ne 255 ]; then + exit 1 +fi + wait_until ssh "$USHIFT_USER@$USHIFT_IP" "true" # Just check if KAS is up and serving wait_until oc get node From a72e23c2f739cc1f1631c39ff8d1b0d3b1f7ac59 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:36:16 +0200 Subject: [PATCH 29/39] reboot: don't workaround pod security --- e2e/tests/0100-reboot.sh | 3 --- e2e/tests/assets/pod-with-pvc.yaml | 13 +++++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/e2e/tests/0100-reboot.sh b/e2e/tests/0100-reboot.sh index 46941c7dc8..2ca585614e 100755 --- a/e2e/tests/0100-reboot.sh +++ b/e2e/tests/0100-reboot.sh @@ -23,9 +23,6 @@ cleanup() { } trap 'cleanup' EXIT -# TODO: Remove the labels again once https://issues.redhat.com/browse/OCPBUGS-1969 has been fixed upstream -oc label namespaces default "pod-security.kubernetes.io/"{enforce,audit,warn}"-version=v1.24" -oc label namespaces default "pod-security.kubernetes.io/"{enforce,audit,warn}"=privileged" oc create -f "${SCRIPT_PATH}/assets/pod-with-pvc.yaml" oc wait --for=condition=Ready --timeout=120s pod/test-pod diff --git a/e2e/tests/assets/pod-with-pvc.yaml b/e2e/tests/assets/pod-with-pvc.yaml index 1750da5c4c..b0ad97ea01 100644 --- a/e2e/tests/assets/pod-with-pvc.yaml +++ b/e2e/tests/assets/pod-with-pvc.yaml @@ -18,17 +18,18 @@ metadata: namespace: default spec: securityContext: - runAsNonRoot: false # TODO pick different image + runAsNonRoot: true privileged: false - capabilities: - drop: - - 'ALL' - allowPrivilegeEscalation: false seccompProfile: type: RuntimeDefault containers: - name: test-container - image: quay.io/openshifttest/hello-openshift:1.2.0 + image: nginxinc/nginx-unprivileged:latest + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL command: - sh - -c From 4a7cc588dbebb53aef16578a4b2c0450154de7c4 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:36:43 +0200 Subject: [PATCH 30/39] test-runner: change logs --- e2e/main.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index 37fa7d3a32..74ca5586da 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -98,7 +98,7 @@ microshift_health_summary() { run_test() { local test=$1 echo -e "\n\n==================================================" - log "${test} - RUNNING" + log "${test} - PREPARING" local test_output="${OUTPUT_DIR}/${test}/" mkdir -p "${test_output}" @@ -110,7 +110,9 @@ run_test() { konfig=$(microshift_get_konfig) trap 'rm -f "${konfig}"' RETURN prep_dur=$(($(date +%s) - prep_start)) + log "Cleanup, setup, and readiness took $((prep_dur / 60))m $((prep_dur % 60))s." + log "${test} - RUNNING" test_start=$(date +%s) set +e KUBECONFIG="$konfig" "${SCRIPT_DIR}/tests/${test}" &>"${test_output}/0010-test.log" @@ -118,8 +120,7 @@ run_test() { set -e test_dur=$(($(date +%s) - test_start)) - log "${test} took $((test_dur / 60))m $((test_dur % 60))s." \ - "Setup and readiness took $((prep_dur / 60))m $((prep_dur % 60))s." + log "${test} took $((test_dur / 60))m $((test_dur % 60))s." if [ $res -eq 0 ]; then log "${test} - SUCCESS" return 0 From 1be578f38400c8ed5c1acb12b5cbb3fcb9cb5937 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:41:05 +0200 Subject: [PATCH 31/39] create output dir only on run --- e2e/main.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/main.sh b/e2e/main.sh index 74ca5586da..90bb893929 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -6,7 +6,6 @@ set -o pipefail SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" OUTPUT_DIR="${ARTIFACT_DIR:-_output}/microshift-e2e-$(date +'%Y%m%d-%H%M%S')/" -[ ! -d "${OUTPUT_DIR}" ] && mkdir -p "${OUTPUT_DIR}" log() { echo -e "$(date +'%H:%M:%S.%N') $*" @@ -142,6 +141,8 @@ run() { log "Following tests will run:\n$to_run" prechecks + [ ! -d "${OUTPUT_DIR}" ] && mkdir -p "${OUTPUT_DIR}" + all_successful=true for t in $to_run; do run_test "${t}" || all_successful=false From 47dce1a724187a933a19cff108372b6214739fd3 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:16:54 +0200 Subject: [PATCH 32/39] test runner: use different 'set's --- e2e/main.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index 90bb893929..7d4ef4f56d 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash -set -o errtrace -set -o nounset -set -o pipefail +set -euo pipefail SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" OUTPUT_DIR="${ARTIFACT_DIR:-_output}/microshift-e2e-$(date +'%Y%m%d-%H%M%S')/" From afd53de0a8f89ff528ccf4eda7564c6dac5a7518 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:17:23 +0200 Subject: [PATCH 33/39] just warn about firewall::*_port funcs --- e2e/main.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index 7d4ef4f56d..adb460238e 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -36,8 +36,8 @@ prechecks() { # Just warning for now # Following functions needed only for runs in CI - function_should_be_exported firewall::open_port - function_should_be_exported firewall::close_port + function_should_be_exported firewall::open_port || true + function_should_be_exported firewall::close_port || true } microshift_get_konfig() { From 7f2baf55eb616e46cb50222406b81ee3e8b22d2d Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:17:53 +0200 Subject: [PATCH 34/39] usage --- e2e/main.sh | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index adb460238e..7b289f333a 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -5,6 +5,32 @@ set -euo pipefail SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" OUTPUT_DIR="${ARTIFACT_DIR:-_output}/microshift-e2e-$(date +'%Y%m%d-%H%M%S')/" +usage() { + echo "Usage: $(basename "${0}") {list|run} [filter]" + echo "" + echo " list Lists tests" + echo " run Runs tests" + echo " filter Simple string to match against test files, e.g. 'reboot', 'boot', 'smoke'" + echo "" + echo " Script expects two environmental variables:" + echo " - USHIFT_IP" + echo " - USHIFT_USER" + echo "" + echo " Script assumes following:" + echo " - Passwordless SSH to \$USHIFT_USER@\$USHIFT_IP is configured" + echo " - Both hosts already exchanged their public keys:" + echo " - Test runner has MicroShift's sshd key in ~/.ssh/known_keys" + echo " - Remote \$USHIFT_USER has test runner's key in ~/.ssh/authorized_keys" + echo " - Passwordless sudo for \$USHIFT_USER" + echo "" + echo " Script aims to be target platform agnostic. It means that for some environments (e.g. GCP)" + echo " it might be required to export firewall::open_port and firewall::close_port functions" + echo " so the tests can open custom ports" + + [ -n "$1" ] && echo -e "\nERROR: $1" + exit 1 +} + log() { echo -e "$(date +'%H:%M:%S.%N') $*" } @@ -149,8 +175,7 @@ run() { } [ $# -eq 0 ] && { - echo "usage" - exit 1 + usage "Expected arguments" } cmd="$1" From 1c60b7b2275dd16bd60b73c89cb55490f939de2d Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:18:49 +0200 Subject: [PATCH 35/39] add ssh and sudo checks --- e2e/main.sh | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index 7b289f333a..e9ef1cacdf 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -49,16 +49,31 @@ var_should_not_be_empty() { function_should_be_exported() { local fname=${1} if ! declare -F "$fname"; then - echo >&2 "Function '$fname' is unexported. It is expected that function is provided for interacting with cloud provider" + log "WARNING: Function '$fname' is unexported. It is expected that function is provided for interacting with cloud provider" return 1 fi } +check_passwordless_ssh() { + ssh -o BatchMode=yes "$USHIFT_USER@$USHIFT_IP" "true" || { + echo "Failed to access ${USHIFT_IP}:" + echo " - Test runner should have MicroShift's sshd key in ~/.ssh/known_keys" + echo " - Remote \$USHIFT_USER should have test runner's key in ~/.ssh/authorized_keys" + exit 1 + } +} + +check_passwordless_sudo() { + ssh -o BatchMode=yes "$USHIFT_USER@$USHIFT_IP" "sudo --non-interactive true" || { + echo "Failed to run sudo command as ${USHIFT_USER} without password" + exit 1 + } +} + prechecks() { - var_should_not_be_empty USHIFT_IP || exit 1 - var_should_not_be_empty USHIFT_USER || exit 1 - # TODO Check passwordless SSH - # TODO Check passwordless sudo + var_should_not_be_empty USHIFT_IP && var_should_not_be_empty USHIFT_USER || exit 1 + check_passwordless_ssh + check_passwordless_sudo # Just warning for now # Following functions needed only for runs in CI From 0366f168199239b2e1a03cdae3d9fcfb9fc58b87 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:32:38 +0200 Subject: [PATCH 36/39] linter fixes --- e2e/main.sh | 38 +++++++++++------------ e2e/tests/0030-router-smoke-test.sh | 2 +- e2e/tests/0031-loadbalancer-smoke-test.sh | 2 +- e2e/tests/0050-greenboot.sh | 4 +-- e2e/tests/0100-reboot.sh | 6 ++-- e2e/tests/assets/greenboot-test.sh | 6 ++-- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index e9ef1cacdf..ba2ca6dc71 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -48,14 +48,14 @@ var_should_not_be_empty() { function_should_be_exported() { local fname=${1} - if ! declare -F "$fname"; then - log "WARNING: Function '$fname' is unexported. It is expected that function is provided for interacting with cloud provider" + if ! declare -F "${fname}"; then + log "WARNING: Function '${fname}' is unexported. It is expected that function is provided for interacting with cloud provider" return 1 fi } check_passwordless_ssh() { - ssh -o BatchMode=yes "$USHIFT_USER@$USHIFT_IP" "true" || { + ssh -o BatchMode=yes "${USHIFT_USER}@${USHIFT_IP}" "true" || { echo "Failed to access ${USHIFT_IP}:" echo " - Test runner should have MicroShift's sshd key in ~/.ssh/known_keys" echo " - Remote \$USHIFT_USER should have test runner's key in ~/.ssh/authorized_keys" @@ -64,7 +64,7 @@ check_passwordless_ssh() { } check_passwordless_sudo() { - ssh -o BatchMode=yes "$USHIFT_USER@$USHIFT_IP" "sudo --non-interactive true" || { + ssh -o BatchMode=yes "${USHIFT_USER}@${USHIFT_IP}" "sudo --non-interactive true" || { echo "Failed to run sudo command as ${USHIFT_USER} without password" exit 1 } @@ -83,14 +83,14 @@ prechecks() { microshift_get_konfig() { tmpfile=$(mktemp /tmp/microshift-e2e-konfig.XXXXXX) - ssh "$USHIFT_USER@$USHIFT_IP" 'sudo cat /var/lib/microshift/resources/kubeadmin/'"$USHIFT_IP"'/kubeconfig' >"$tmpfile" - echo "$tmpfile" + ssh "${USHIFT_USER}@${USHIFT_IP}" 'sudo cat /var/lib/microshift/resources/kubeadmin/'"${USHIFT_IP}"'/kubeconfig' >"${tmpfile}" + echo "${tmpfile}" } microshift_check_readiness() { local test_output="${1}" log "Waiting for MicroShift to become ready" - ssh "$USHIFT_USER@$USHIFT_IP" \ + ssh "${USHIFT_USER}@${USHIFT_IP}" \ "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh | \ while IFS= read -r line; do printf '%s %s\\n' \"\$(date +'%H:%M:%S.%N')\" \"\$line\"; done" &>"${test_output}/0002-readiness-check.log" } @@ -98,26 +98,26 @@ microshift_check_readiness() { microshift_setup() { local test_output="${1}" log "Setting up and starting MicroShift" - ssh "$USHIFT_USER@$USHIFT_IP" 'cat << EOF | sudo tee /etc/microshift/config.yaml + ssh "${USHIFT_USER}@${USHIFT_IP}" 'cat << EOF | sudo tee /etc/microshift/config.yaml --- apiServer: subjectAltNames: - - '"$USHIFT_IP"' + - '"${USHIFT_IP}"' EOF' &>"${test_output}/0001-setup.log" - ssh "$USHIFT_USER@$USHIFT_IP" "sudo systemctl enable --now microshift" &>>"${test_output}/0001-setup.log" + ssh "${USHIFT_USER}@${USHIFT_IP}" "sudo systemctl enable --now microshift" &>>"${test_output}/0001-setup.log" } microshift_debug_info() { local test_output="${1}" log "Gathering debug info to ${test_output}/0020-cluster-debug-info.log" - scp "$SCRIPT_DIR/../validate-microshift/cluster-debug-info.sh" "$USHIFT_USER@$USHIFT_IP:/tmp/cluster-debug-info.sh" - ssh "$USHIFT_USER@$USHIFT_IP" "sudo /tmp/cluster-debug-info.sh" &>"${test_output}/0020-cluster-debug-info.log" + scp "${SCRIPT_DIR}/../validate-microshift/cluster-debug-info.sh" "${USHIFT_USER}@${USHIFT_IP}:/tmp/cluster-debug-info.sh" + ssh "${USHIFT_USER}@${USHIFT_IP}" "sudo /tmp/cluster-debug-info.sh" &>"${test_output}/0020-cluster-debug-info.log" } microshift_cleanup() { local test_output="${1}" log "Cleaning MicroShift" - ssh "$USHIFT_USER@$USHIFT_IP" "echo 1 | sudo microshift-cleanup-data --all" &>"${test_output}/0000-cleanup.log" + ssh "${USHIFT_USER}@${USHIFT_IP}" "echo 1 | sudo microshift-cleanup-data --all" &>"${test_output}/0000-cleanup.log" } microshift_health_summary() { @@ -126,7 +126,7 @@ microshift_health_summary() { # Because test might be "destructive" (i.e. tear down and set up again MicroShift) # so these commands are executed via ssh. # Alternative is to copy kubeconfig second time in the same time. - ssh "$USHIFT_USER@$USHIFT_IP" \ + ssh "${USHIFT_USER}@${USHIFT_IP}" \ "mkdir -p ~/.kube/ && sudo cat /var/lib/microshift/resources/kubeadmin/kubeconfig > ~/.kube/config ; \ oc get pods -A ; \ oc get nodes -o wide ; \ @@ -153,13 +153,13 @@ run_test() { log "${test} - RUNNING" test_start=$(date +%s) set +e - KUBECONFIG="$konfig" "${SCRIPT_DIR}/tests/${test}" &>"${test_output}/0010-test.log" + KUBECONFIG="${konfig}" "${SCRIPT_DIR}/tests/${test}" &>"${test_output}/0010-test.log" res=$? set -e test_dur=$(($(date +%s) - test_start)) log "${test} took $((test_dur / 60))m $((test_dur % 60))s." - if [ $res -eq 0 ]; then + if [ ${res} -eq 0 ]; then log "${test} - SUCCESS" return 0 fi @@ -172,18 +172,18 @@ run_test() { list() { local -r filter="*${1:-}*.sh" - find "${SCRIPT_DIR}/tests" -maxdepth 1 -iname "$filter" -printf "%f\n" + find "${SCRIPT_DIR}/tests" -maxdepth 1 -iname "${filter}" -printf "%f\n" | sort } run() { local -r to_run=$(list "${1}") - log "Following tests will run:\n$to_run" prechecks + log "Following tests will run:\n${to_run}" [ ! -d "${OUTPUT_DIR}" ] && mkdir -p "${OUTPUT_DIR}" all_successful=true - for t in $to_run; do + for t in ${to_run}; do run_test "${t}" || all_successful=false done "${all_successful}" diff --git a/e2e/tests/0030-router-smoke-test.sh b/e2e/tests/0030-router-smoke-test.sh index afbd3b76c4..14268a5305 100755 --- a/e2e/tests/0030-router-smoke-test.sh +++ b/e2e/tests/0030-router-smoke-test.sh @@ -32,7 +32,7 @@ for _ in $(seq "${RETRIES}"); do result=$? set -e - [ $result -eq 0 ] && + [ ${result} -eq 0 ] && echo "${response}" | grep -q -E "HTTP.*200" && echo "${response}" | grep -q "Hello MicroShift" && exit 0 diff --git a/e2e/tests/0031-loadbalancer-smoke-test.sh b/e2e/tests/0031-loadbalancer-smoke-test.sh index f170b1e91d..07f6a4d51a 100755 --- a/e2e/tests/0031-loadbalancer-smoke-test.sh +++ b/e2e/tests/0031-loadbalancer-smoke-test.sh @@ -29,7 +29,7 @@ for _ in $(seq "${RETRIES}"); do result=$? set -e - [ $result -eq 0 ] && + [ ${result} -eq 0 ] && echo "${response}" | grep -q -E "HTTP.*200" && echo "${response}" | grep -q "Hello MicroShift" && exit 0 diff --git a/e2e/tests/0050-greenboot.sh b/e2e/tests/0050-greenboot.sh index 75ab9d10c6..967702b603 100755 --- a/e2e/tests/0050-greenboot.sh +++ b/e2e/tests/0050-greenboot.sh @@ -7,5 +7,5 @@ set -x SCRIPT_PATH="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" -scp "$SCRIPT_PATH/../../docs/config/busybox_running_check.sh" "$SCRIPT_PATH/assets/greenboot-test.sh" "$USHIFT_USER@$USHIFT_IP":/tmp/ -ssh -q "$USHIFT_USER@$USHIFT_IP" "chmod +x /tmp/greenboot-test.sh /tmp/busybox_running_check.sh && sudo /tmp/greenboot-test.sh" +scp "${SCRIPT_PATH}/../../docs/config/busybox_running_check.sh" "${SCRIPT_PATH}/assets/greenboot-test.sh" "${USHIFT_USER}@${USHIFT_IP}":/tmp/ +ssh -q "${USHIFT_USER}@${USHIFT_IP}" "chmod +x /tmp/greenboot-test.sh /tmp/busybox_running_check.sh && sudo /tmp/greenboot-test.sh" diff --git a/e2e/tests/0100-reboot.sh b/e2e/tests/0100-reboot.sh index 2ca585614e..fd13494075 100755 --- a/e2e/tests/0100-reboot.sh +++ b/e2e/tests/0100-reboot.sh @@ -27,7 +27,7 @@ oc create -f "${SCRIPT_PATH}/assets/pod-with-pvc.yaml" oc wait --for=condition=Ready --timeout=120s pod/test-pod set +e -ssh -v "$USHIFT_USER@$USHIFT_IP" "sudo reboot now" +ssh -v "${USHIFT_USER}@${USHIFT_IP}" "sudo reboot now" res=$? set -e @@ -37,8 +37,8 @@ if [ "${res}" -ne 0 ] && [ "${res}" -ne 255 ]; then exit 1 fi -wait_until ssh "$USHIFT_USER@$USHIFT_IP" "true" +wait_until ssh "${USHIFT_USER}@${USHIFT_IP}" "true" # Just check if KAS is up and serving wait_until oc get node -ssh "$USHIFT_USER@$USHIFT_IP" "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh" +ssh "${USHIFT_USER}@${USHIFT_IP}" "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh" oc wait --for=condition=Ready --timeout=120s pod/test-pod diff --git a/e2e/tests/assets/greenboot-test.sh b/e2e/tests/assets/greenboot-test.sh index c6465bedfc..3935eabfbd 100644 --- a/e2e/tests/assets/greenboot-test.sh +++ b/e2e/tests/assets/greenboot-test.sh @@ -9,19 +9,19 @@ function check_greenboot_exit_status() { local expectedRC=$1 local cleanup=$2 - if [ ${cleanup} -ne 0 ]; then + if [ "${cleanup}" -ne 0 ]; then echo 1 | microshift-cleanup-data --all systemctl enable --now microshift || true fi for check_script in $(find /etc/greenboot/check/ -name \*.sh | sort); do - echo Running ${check_script}... + echo Running "${check_script}"... local currentRC=1 if ${check_script}; then currentRC=0 fi - if [ ${currentRC} != ${expectedRC} ]; then + if [ ${currentRC} != "${expectedRC}" ]; then exit 1 fi done From 9e0dedc480c58be1238aa0dba8dd77ac07b2a404 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:41:20 +0200 Subject: [PATCH 37/39] remove timestamps when running greenboot check --- e2e/main.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e/main.sh b/e2e/main.sh index ba2ca6dc71..77761508bf 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -91,8 +91,7 @@ microshift_check_readiness() { local test_output="${1}" log "Waiting for MicroShift to become ready" ssh "${USHIFT_USER}@${USHIFT_IP}" \ - "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh | \ - while IFS= read -r line; do printf '%s %s\\n' \"\$(date +'%H:%M:%S.%N')\" \"\$line\"; done" &>"${test_output}/0002-readiness-check.log" + "sudo /etc/greenboot/check/required.d/40_microshift_running_check.sh" &>"${test_output}/0002-readiness-check.log" } microshift_setup() { From a67a2beb23a2bb53388d64d2695f5a71636bca40 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Fri, 7 Apr 2023 08:31:51 +0200 Subject: [PATCH 38/39] 'typo' :) --- e2e/main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/main.sh b/e2e/main.sh index 77761508bf..49aff1c746 100755 --- a/e2e/main.sh +++ b/e2e/main.sh @@ -124,7 +124,7 @@ microshift_health_summary() { # Because test might be "destructive" (i.e. tear down and set up again MicroShift) # so these commands are executed via ssh. - # Alternative is to copy kubeconfig second time in the same time. + # Alternative is to copy kubeconfig second time in the same test. ssh "${USHIFT_USER}@${USHIFT_IP}" \ "mkdir -p ~/.kube/ && sudo cat /var/lib/microshift/resources/kubeadmin/kubeconfig > ~/.kube/config ; \ oc get pods -A ; \ From 59035828f6e9781d30469330764e9328d499b10c Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Fri, 7 Apr 2023 14:35:10 +0200 Subject: [PATCH 39/39] e2e in robot framework --- .gitignore | 1 + e2e-robot/Makefile | 38 +++++++ e2e-robot/requirements.txt | 27 +++++ e2e-robot/tests/microshift.robot | 167 +++++++++++++++++++++++++++++++ 4 files changed, 233 insertions(+) create mode 100644 e2e-robot/Makefile create mode 100644 e2e-robot/requirements.txt create mode 100644 e2e-robot/tests/microshift.robot diff --git a/.gitignore b/.gitignore index fb05ccd53a..a4ddfa8fb1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ _output/* _output sshfile ansible/*.txt +**/.venv/ diff --git a/e2e-robot/Makefile b/e2e-robot/Makefile new file mode 100644 index 0000000000..243233bd5f --- /dev/null +++ b/e2e-robot/Makefile @@ -0,0 +1,38 @@ + +.PHONY: setup dry-run with-filter with-tag all + +setup: + python3 -m venv .venv && \ + .venv/bin/python3 -m pip install -r requirements.txt + +dry-run: + .venv/bin/robot \ + --dryrun \ + --outputdir ../_output/e2e-$$(date +'%Y%m%d-%H%M%S') \ + ./tests/microshift.robot + +with-filter: + .venv/bin/robot \ + -v USHIFT_IP:$${USHIFT_IP} \ + -v USHIFT_USER:microshift \ + --test *router* \ + --outputdir ../_output/e2e-$$(date +'%Y%m%d-%H%M%S') \ + -x xunit.xml \ + ./tests/microshift.robot + +with-tag: + .venv/bin/robot \ + -v USHIFT_IP:$${USHIFT_IP} \ + -v USHIFT_USER:microshift \ + --include smoke \ + --outputdir ../_output/e2e-$$(date +'%Y%m%d-%H%M%S') \ + -x xunit.xml \ + ./tests/microshift.robot + +all: + .venv/bin/robot \ + -v USHIFT_IP:$${USHIFT_IP} \ + -v USHIFT_USER:microshift \ + --outputdir ../_output/e2e-$$(date +'%Y%m%d-%H%M%S') \ + -x xunit.xml \ + ./tests/microshift.robot \ No newline at end of file diff --git a/e2e-robot/requirements.txt b/e2e-robot/requirements.txt new file mode 100644 index 0000000000..27103b1c2d --- /dev/null +++ b/e2e-robot/requirements.txt @@ -0,0 +1,27 @@ +bcrypt==4.0.1 +certifi==2022.12.7 +cffi==1.15.1 +charset-normalizer==3.1.0 +click==8.1.3 +colorama==0.4.6 +cryptography==40.0.1 +idna==3.4 +Jinja2==3.1.2 +markdown-it-py==2.2.0 +MarkupSafe==2.1.2 +mdurl==0.1.2 +paramiko==3.1.0 +pathspec==0.11.1 +pycparser==2.21 +Pygments==2.14.0 +PyNaCl==1.5.0 +requests==2.28.2 +rich==13.3.3 +rich-click==1.4 +robotframework==6.0.2 +robotframework-requests==0.9.4 +robotframework-sshlibrary==3.8.0 +robotframework-tidy==4.0.1 +scp==0.14.5 +tomli==2.0.1 +urllib3==1.26.15 diff --git a/e2e-robot/tests/microshift.robot b/e2e-robot/tests/microshift.robot new file mode 100644 index 0000000000..ad79aa0184 --- /dev/null +++ b/e2e-robot/tests/microshift.robot @@ -0,0 +1,167 @@ +*** Settings *** +Documentation MicroShift e2e test suite + +Library SSHLibrary +Library String +Library OperatingSystem +Library Process +Library RequestsLibrary + +Suite Setup Get Kubeconfig +Suite Teardown Remove Kubeconfig + + +*** Variables *** +${USHIFT_IP} ${EMPTY} +${USHIFT_USER} ${EMPTY} + + +*** Test Cases *** +Router Smoke Test + [Documentation] Verify that Router correctly exposes HTTP service + [Tags] smoke + [Setup] Run Keywords + ... Create Hello MicroShift Pod AND + ... Expose Hello MicroShift Pod Via Router AND + ... Open Port 80 tcp + + Wait Until Keyword Succeeds 3x 3s Access Hello Microshift via Router + + [Teardown] Run Keywords + ... Delete Hello MicroShift Pod Route And Service AND + ... Close Port 80 tcp + +Load Balancer Smoke Test + [Documentation] Verify that Load Balancer correctly exposes HTTP service + [Tags] smoke + [Setup] Run Keywords + ... Create Hello MicroShift Pod AND + ... Expose Hello MicroShift Pod Via LB AND + ... Open Port 5678 tcp + + Wait Until Keyword Succeeds 3x 3s Access Hello Microshift via LB + + [Teardown] Run Keywords + ... Delete Hello MicroShift Pod Route And Service AND + ... Close Port 5678 tcp + +Reboot Test + [Documentation] Verify that MicroShift starts successfully after reboot + [Setup] Run Keywords Create Pod With PVC + + Open Connection ${USHIFT_IP} + Login ${USHIFT_USER} allow_agent=True + Execute Command reboot now sudo=True + Close Connection + + Sleep 15s + + Open Connection ${USHIFT_IP} + Set Client Configuration timeout=600s + Wait Until Keyword Succeeds 10x 10s Login ${USHIFT_USER} allow_agent=True + + Wait Until Keyword Succeeds + ... 10x + ... 10s + ... Execute Command + ... [ $(systemctl show -p SubState --value microshift) = "running" ] + ... timeout=10s return_stdout=True return_stderr=True + + ${stdout} ${rc}= Execute Command + ... /etc/greenboot/check/required.d/40_microshift_running_check.sh | tee /tmp/asd.log + ... sudo=True + ... timeout=600s + ... return_stdout=True + ... return_rc=True + Log ${stdout} + Should Be Equal As Integers ${rc} 0 + Close Connection + + Run With Kubeconfig oc wait --for=condition=Ready --timeout=120s pod/test-pod + + [Teardown] Delete Pod With PVC + +Failed Test + Fail Let's see how it looks in Prow + + +*** Keywords *** +Get Kubeconfig + [Documentation] X + Open Connection ${USHIFT_IP} + Login ${USHIFT_USER} allow_agent=True + ${konfig}= Execute Command + ... cat /var/lib/microshift/resources/kubeadmin/${USHIFT_IP}/kubeconfig + ... sudo=True + Should Not Be Empty ${konfig} + ${rand}= Generate Random String + ${path}= Join Path /tmp ${rand} + Create File ${path} ${konfig} + Close Connection + Set Suite Variable \${KUBECONFIG} ${path} + +Remove Kubeconfig + Remove File ${KUBECONFIG} + +Access Hello Microshift via Router + ${result}= Run Process + ... curl -i http://hello-microshift.cluster.local --resolve "hello-microshift.cluster.local:80:${USHIFT_IP}" + ... shell=True timeout=15s + Check HTTP Response ${result} + +Access Hello Microshift via LB + ${result}= Run Process curl -i ${USHIFT_IP}:5678 shell=True timeout=15s + Check HTTP Response ${result} + +Check HTTP Response + [Arguments] ${result} + Log ${result.stdout} + Log ${result.stderr} + Should Be Equal As Integers ${result.rc} 0 + Should Match Regexp ${result.stdout} HTTP.*200 + Should Match ${result.stdout} *Hello MicroShift* + +Create Pod With PVC + Run With Kubeconfig oc create -f ../e2e/tests/assets/pod-with-pvc.yaml + Run With Kubeconfig oc wait --for=condition=Ready --timeout=120s pod/test-pod + +Delete Pod With PVC + Run With Kubeconfig oc delete -f ../e2e/tests/assets/pod-with-pvc.yaml True + +Create Hello MicroShift Pod + Run With Kubeconfig oc create -f ../e2e/tests/assets/hello-microshift.yaml + Run With Kubeconfig oc wait pods -l app\=hello-microshift --for condition\=Ready --timeout\=60s + +Expose Hello MicroShift Pod Via Router + Run With Kubeconfig oc expose pod hello-microshift + Run With Kubeconfig oc expose svc hello-microshift --hostname hello-microshift.cluster.local + +Expose Hello MicroShift Pod Via LB + Run With Kubeconfig oc create service loadbalancer hello-microshift --tcp=5678:8080 + +Delete Hello MicroShift Pod Route And Service + Run With Kubeconfig oc delete route hello-microshift True + Run With Kubeconfig oc delete service hello-microshift True + Run With Kubeconfig oc delete -f ../e2e/tests/assets/hello-microshift.yaml True + +Run With Kubeconfig + [Arguments] ${cmd} ${allow_fail}=False + ${result}= Run Process ${cmd} env:KUBECONFIG=${KUBECONFIG} stderr=STDOUT shell=True + Log ${result.stdout} + IF ${allow_fail} == False + Should Be Equal As Integers ${result.rc} 0 + END + +Open Port + [Arguments] ${number} ${protocol} + ${res}= Run Process bash -c + ... if declare -F firewall::open_port; then firewall::open_port ${number} ${protocol}; fi stderr=STDOUT + Log ${res.stdout} + Should Be Equal As Integers ${res.rc} 0 + +Close Port + [Arguments] ${number} ${protocol} + ${res}= Run Process bash -c + ... if declare -F firewall::close_port; then firewall::close_port ${number} ${protocol}; fi stderr=STDOUT + Log ${res.stdout} + Should Be Equal As Integers ${res.rc} 0