diff --git a/bot/check-test.sh b/bot/check-test.sh
index 76e0df7f40..5641b91dd1 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -13,8 +13,34 @@ job_dir=${PWD}
 job_out="slurm-${SLURM_JOB_ID}.out"
 job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
 
+# ReFrame prints e.g.
+#[----------] start processing checks
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
+#[     FAIL ] (1/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
+#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_d597cff4'
+#[       OK ] (2/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
+#P: perf: 8.441 ns/day (r:0, l:None, u:None)
+#[     FAIL ] (3/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
+#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_f4194106'
+#[----------] all spawned checks have finished
+#[  FAILED  ] Ran 3/3 test case(s) from 2 check(s) (2 failure(s), 0 skipped, 0 aborted)
+
+# We will grep for the last and final line, since this reflects the overall result
+# Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
+error_pattern="FAILED"
+grep_out=$(grep "${error_pattern}" ${job_dir}/${job_out})
+[[ $? -eq 0 ]] && ERROR=1 || ERROR=0
+
+# Write the result to the job_test_result_file
 echo "[TEST]" > ${job_test_result_file}
-echo "comment_description = <em>(no tests yet)</em>" >> ${job_test_result_file}
-echo "status = SUCCESS" >> ${job_test_result_file}
+if [[ ${ERROR} -eq 1 ]]; then
+    echo "comment_description = Test suite failed" >> ${job_test_result_file}
+    echo "status = FAILURE" >> ${job_test_result_file}
+else
+    echo "comment_description = <em>(no tests yet)</em>" >> ${job_test_result_file}
+    echo "status = SUCCESS" >> ${job_test_result_file}
+fi
 
 exit 0
diff --git a/bot/test.sh b/bot/test.sh
index 9d978cdcd0..5c55bb1768 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -10,4 +10,38 @@
 # license: GPLv2
 #
 
+# Create tmp file for output of test step
+test_outerr=$(mktemp test.outerr.XXXX)
+
+# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version
+# of the ReFrame module available in the current environment
+(module load ReFrame/4.3.3 || echo "FAILED to load the ReFrame module") 2>&1 | tee -a ${test_outerr}
+
+# Check ReFrame came with the hpctestlib and we can import it
+(python3 -c 'import hpctestlib.sciapps.gromacs' || echo "FAILED to load hpctestlib") 2>&1 | tee -a ${test_outerr}
+
+# Clone the EESSI test suite
+git clone https://github.com/EESSI/test-suite EESSI-test-suite
+export TESTSUITEPREFIX=$PWD/EESSI-test-suite
+export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
+
+# Check that we can import from the testsuite
+(python3 -c 'import eessi.testsuite' || echo "FAILED to import from eessi.testsuite in Python") 2>&1 | tee -a ${test_outerr}
+
+# Configure ReFrame
+export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py
+export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
+export RFM_CHECK_SEARCH_RECURSIVE=1
+export RFM_PREFIX=$PWD/reframe_runs
+
+# Check we can run reframe
+(reframe --version || echo "FAILED to run ReFrame") 2>&1 | tee -a ${test_outerr}
+
+# List the tests we want to run
+export REFRAME_ARGS='--tag CI --tag 1_nodes'
+(reframe "${REFRAME_ARGS}" --list || echo "FAILED to list ReFrame tests") 2>&1 | tee -a ${test_outerr}
+
+# Run all tests
+reframe "${REFRAME_ARGS}" --run 2>&1 | tee -a ${test_outerr}
+
 exit 0
diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 82190071ab..4c73b5887a 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -15,3 +15,4 @@ easyconfigs:
       options:
         from-pr: 19573
   - scikit-learn-1.3.1-gfbf-2023a.eb
+  - patchelf-0.18.0-GCCcore-12.3.0.eb