From 3109e9d1bcda2d3c7b596da036def76a911bc297 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman <rjeffman@redhat.com> Date: Tue, 7 Nov 2023 13:53:09 -0300 Subject: [PATCH] utils/run-tests.sh: Replicate Azure's test grouping When running ansible-freeipa's Azure pipelines for nightly and weekly tests, due to the amount of tests to execute, tests are grouped and executed in parallel jobs. Due to a still unkonwn issue, depending on the order the tests are executed, some random failures may occur and debugging them is hard due to current implementation of the tests. This patch adds support for replicating the tests of a specific Azure test group once the seed used to create groups and the group number are provided, allowing the test failures to be replicated on the developer's workstation where it can be more easily debugged. A new option is added to 'utils/run-tests.sh', '-A SEED.G' that is used to define the seed and group to replicate the tests. The seed is a date, with the format "YYYYMMDD", so, for example '-A 20230611.2' would execute the same tests, in the same order as the second group of tests for date 2023-06-11. To aid in usability 'YYYY-MM-DD' may also be used. When using '-A' neither '-s' (test suites) or specific tests (positional arguments) can be used. Also, to help fixing tests, an option to stop the tests on the first test failure ('-x') was added to the script. --- utils/run-tests.sh | 60 +++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/utils/run-tests.sh b/utils/run-tests.sh index fe1eb720f4..e814aed014 100755 --- a/utils/run-tests.sh +++ b/utils/run-tests.sh @@ -32,7 +32,7 @@ interrupt_exception() { usage() { local prog="${0##*/}" cat <<EOF -usage: ${prog} [-h] [-l] [-e] [-g] [-s TESTS_SUITE] [-i IMAGE] [TEST...] +usage: ${prog} [-h] [-l] [-e] [-K] [-c CONTAINER] [-s TESTS_SUITE] [-x] [-A SEED.GRP] [-i IMAGE] [-m MEMORY] [-v...] [TEST...] ${prog} runs playbook(s) TEST using an ansible-freeipa testing image. EOF @@ -51,10 +51,13 @@ optional arguments: -K keep container, even if tests succeed -l list available images -e force recreation of the virtual environment - -i select image to run the tests (default: fedora-latest) + -i IMAGE select image to run the tests (default: fedora-latest) -m container memory, in GiB (default: 3) -s TEST_SUITE run all playbooks for test suite, which is a directory under ${WHITE}tests${RST} + -A SEED.GROUP Replicate Azure's test group and seed (seed is YYYYMMDD) + -v Increase Ansible verbosity (can be used multiple times) + -x Stop on first error. EOF )" } @@ -179,15 +182,29 @@ IMAGE_TAG="fedora-latest" scenario="" MEMORY=3 hostname="ipaserver.test.local" - +SEED="" +GROUP=0 +SPLITS=0 ANSIBLE_COLLECTIONS=${ANSIBLE_COLLECTIONS:-"containers.podman"} +EXTRA_OPTIONS="" + # Process command options -while getopts ":hc:ei:Klms:v" option +while getopts ":hA:c:ei:Klms:vx" option do case "$option" in h) help && exit 0 ;; + A) + [ ${#ENABLED_MODULES[@]} -eq 0 ] || die -u "Can't use '-A' with '-s'" + SEED="$(cut -d. -f1 <<< "${OPTARG}" | tr -d "-")" + GROUP="$(cut -d. -f2 <<< "${OPTARG}")" + if [ -z "${SEED}" ] || [ -z "${GROUP}" ] + then + die -u "Seed for '-A' must have the format YYYYMMDD.N" + fi + SPLITS=3 + ;; c) scenario="${OPTARG}" ;; e) FORCE_ENV="Y" ;; i) IMAGE_TAG="${OPTARG}" ;; @@ -195,6 +212,7 @@ do l) list_images && exit 0 || exit 1;; m) MEMORY="${OPTARG}" ;; s) + [ ${SPLITS} -ne 0 ] && die -u "Can't use '-A' with '-s'" if [ -d "${TOPDIR}/tests/${OPTARG}" ] then ENABLED_MODULES+=("${OPTARG}") @@ -203,6 +221,7 @@ do fi ;; v) verbose=${verbose:--}${option} ;; + x) EXTRA_OPTIONS="$EXTRA_OPTIONS --exitfirst" ;; *) die -u "Invalid option: ${OPTARG}" ;; esac done @@ -212,13 +231,14 @@ do # shellcheck disable=SC2207 if stat "$test" >/dev/null 2>&1 then + [ ${SPLITS} -ne 0 ] && die -u "Can't define tests and use '-A'" ENABLED_TESTS+=($(basename "${test}" .yml)) else log error "Test not found: ${test}" fi done -[ ${#ENABLED_MODULES[@]} -eq 0 ] && [ ${#ENABLED_TESTS[@]} -eq 0 ] && die -u "No test defined." +[ ${SPLITS} -eq 0 ] && [ ${#ENABLED_MODULES[@]} -eq 0 ] && [ ${#ENABLED_TESTS[@]} -eq 0 ] && die -u "No test defined." # Prepare virtual environment VENV=$(in_python_virtualenv && echo Y || echo N) @@ -373,23 +393,31 @@ EOF # run tests RESULT=0 -# shellcheck disable=SC2086 + export RUN_TESTS_IN_DOCKER=${engine} export IPA_SERVER_HOST="${scenario}" -joined="$(printf "%s," "${ENABLED_MODULES[@]}")" -# shelcheck disable=SC2178 -IPA_ENABLED_MODULES="${joined%,}" -joined="$(printf "%s," "${ENABLED_TESTS[@]}")" -# shelcheck disable=SC2178 -IPA_ENABLED_TESTS="${joined%,}" -export IPA_ENABLED_MODULES IPA_ENABLED_TESTS -[ -n "${IPA_ENABLED_MODULES}" ] && log info "Test suites: ${IPA_ENABLED_MODULES}" -[ -n "${IPA_ENABLED_TESTS}" ] && log info "Individual tests: ${IPA_ENABLED_TESTS}" +if [ ${SPLITS} -ne 0 ] +then + EXTRA_OPTIONS="${EXTRA_OPTIONS} --splits=${SPLITS} --group=${GROUP} --randomly-seed=${SEED}" + log info "Running tests for group ${GROUP} of ${SPLITS} with seed ${SEED}" +else + # shellcheck disable=SC2086 + joined="$(printf "%s," "${ENABLED_MODULES[@]}")" + # shelcheck disable=SC2178 + IPA_ENABLED_MODULES="${joined%,}" + joined="$(printf "%s," "${ENABLED_TESTS[@]}")" + # shelcheck disable=SC2178 + IPA_ENABLED_TESTS="${joined%,}" + export IPA_ENABLED_MODULES IPA_ENABLED_TESTS + [ -n "${IPA_ENABLED_MODULES}" ] && log info "Test suites: ${IPA_ENABLED_MODULES}" + [ -n "${IPA_ENABLED_TESTS}" ] && log info "Individual tests: ${IPA_ENABLED_TESTS}" +fi IPA_VERBOSITY="${verbose}" [ -n "${IPA_VERBOSITY}" ] && export IPA_VERBOSITY -if ! pytest -m "playbook" --verbose --color=yes +# shellcheck disable=SC2086 +if ! pytest -m "playbook" --verbose --color=yes ${EXTRA_OPTIONS} then RESULT=2 log error "Container not stopped for verification: ${scenario}"