From d5fb87689687eadde87e28ab09f5e7ff7a7c56b9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 2 Feb 2024 17:50:06 +0100 Subject: [PATCH 1/6] Do actual testing --- bot/test.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/bot/test.sh b/bot/test.sh index 9d978cdcd0..5c55bb1768 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -10,4 +10,38 @@ # license: GPLv2 # +# Create tmp file for output of test step +test_outerr=$(mktemp test.outerr.XXXX) + +# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version +# of the ReFrame module available in the current environment +(module load ReFrame/4.3.3 || echo "FAILED to load the ReFrame module") 2>&1 | tee -a ${test_outerr} + +# Check ReFrame came with the hpctestlib and we can import it +(python3 -c 'import hpctestlib.sciapps.gromacs' || echo "FAILED to load hpctestlib") 2>&1 | tee -a ${test_outerr} + +# Clone the EESSI test suite +git clone https://github.com/EESSI/test-suite EESSI-test-suite +export TESTSUITEPREFIX=$PWD/EESSI-test-suite +export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH + +# Check that we can import from the testsuite +(python3 -c 'import eessi.testsuite' || echo "FAILED to import from eessi.testsuite in Python") 2>&1 | tee -a ${test_outerr} + +# Configure ReFrame +export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py +export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests +export RFM_CHECK_SEARCH_RECURSIVE=1 +export RFM_PREFIX=$PWD/reframe_runs + +# Check we can run reframe +(reframe --version || echo "FAILED to run ReFrame") 2>&1 | tee -a ${test_outerr} + +# List the tests we want to run +export REFRAME_ARGS='--tag CI --tag 1_nodes' +(reframe "${REFRAME_ARGS}" --list || echo "FAILED to list ReFrame tests") 2>&1 | tee -a ${test_outerr} + +# Run all tests +reframe "${REFRAME_ARGS}" --run 2>&1 | tee -a ${test_outerr} + exit 0 From c6a09abbe304df04d791fa24769245ceed9e8fa2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 2 Feb 2024 18:05:48 +0100 Subject: [PATCH 2/6] Added a check for the test --- bot/check-test.sh | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 76e0df7f40..eb10678101 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -13,8 +13,33 @@ job_dir=${PWD} job_out="slurm-${SLURM_JOB_ID}.out" job_test_result_file="_bot_job${SLURM_JOB_ID}.test" +# ReFrame prints e.g. +#[----------] start processing checks +#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default +#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default +#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default +#[ FAIL ] (1/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default +#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_d597cff4' +#[ OK ] (2/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default +#P: perf: 8.441 ns/day (r:0, l:None, u:None) +#[ FAIL ] (3/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default +#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_f4194106' +#[----------] all spawned checks have finished +#[ FAILED ] Ran 3/3 test case(s) from 2 check(s) (2 failure(s), 0 skipped, 0 aborted) + +# We will grep for the last and final line, since this reflects the overall result +error_pattern="\[\s+FAILED\s+\]" +grep_out=$(grep "${error_pattern}" ${job_dir}/${job_out}) +[[ $? -eq 0 ]] && ERROR=1 || ERROR=0 + +# Write the result to the job_test_result_file echo "[TEST]" > ${job_test_result_file} -echo "comment_description = (no tests yet)" >> ${job_test_result_file} -echo "status = SUCCESS" >> ${job_test_result_file} +if [[ ${ERROR} -eq 1 ]]; then + echo "comment_description = Test suite failed: ${grep_out}" >> ${job_test_result_file} + echo "status = :cry: FAILURE" >> ${job_test_result_file} +else + echo "comment_description = (no tests yet)" >> ${job_test_result_file} + echo "status = SUCCESS" >> ${job_test_result_file} +fi exit 0 From 730820e7cdac1e65a85a1e5764869d9093c873b4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 2 Feb 2024 18:11:05 +0100 Subject: [PATCH 3/6] I don't really need to build anything, but need a change to one of the easystack files to be able to trigger the bot building pipeline and see if the testing works --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 82190071ab..4c73b5887a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -15,3 +15,4 @@ easyconfigs: options: from-pr: 19573 - scikit-learn-1.3.1-gfbf-2023a.eb + - patchelf-0.18.0-GCCcore-12.3.0.eb From c05bc8e72560e9c0792ff19fc87e2fa0c6f48e89 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 2 Feb 2024 18:23:53 +0100 Subject: [PATCH 4/6] Make sure it notices failures in the test.sh script too --- bot/check-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index eb10678101..57d31aba59 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -28,7 +28,8 @@ job_test_result_file="_bot_job${SLURM_JOB_ID}.test" #[ FAILED ] Ran 3/3 test case(s) from 2 check(s) (2 failure(s), 0 skipped, 0 aborted) # We will grep for the last and final line, since this reflects the overall result -error_pattern="\[\s+FAILED\s+\]" +# Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails +error_pattern="FAILED" grep_out=$(grep "${error_pattern}" ${job_dir}/${job_out}) [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 From 41333a059206d8a644399ae17dea93596c94966d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 2 Feb 2024 18:38:11 +0100 Subject: [PATCH 5/6] Maybe it doesn't like my attempt at emojis --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 57d31aba59..71b66748a9 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -37,7 +37,7 @@ grep_out=$(grep "${error_pattern}" ${job_dir}/${job_out}) echo "[TEST]" > ${job_test_result_file} if [[ ${ERROR} -eq 1 ]]; then echo "comment_description = Test suite failed: ${grep_out}" >> ${job_test_result_file} - echo "status = :cry: FAILURE" >> ${job_test_result_file} + echo "status = FAILURE" >> ${job_test_result_file} else echo "comment_description = (no tests yet)" >> ${job_test_result_file} echo "status = SUCCESS" >> ${job_test_result_file} From 89f82a03df24ab7bea7d794e36f801f20214cf68 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 2 Feb 2024 18:42:58 +0100 Subject: [PATCH 6/6] Make the reported pattern even simpler... --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 71b66748a9..5641b91dd1 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -36,7 +36,7 @@ grep_out=$(grep "${error_pattern}" ${job_dir}/${job_out}) # Write the result to the job_test_result_file echo "[TEST]" > ${job_test_result_file} if [[ ${ERROR} -eq 1 ]]; then - echo "comment_description = Test suite failed: ${grep_out}" >> ${job_test_result_file} + echo "comment_description = Test suite failed" >> ${job_test_result_file} echo "status = FAILURE" >> ${job_test_result_file} else echo "comment_description = (no tests yet)" >> ${job_test_result_file}