From a8ce23526164b6e97798880c18f5634687f71e88 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Sun, 27 Oct 2024 20:23:50 -0300 Subject: [PATCH 1/8] utils/set_test_modules: Allow to ignore Git differences To force setting the IPA_ENABLE_* variables to run all tests, source the script using '-I' or set the environment variable SKIP_GIT_TESTS to 'True'. This will allow the correct selection of Azure pipelines tests to be based on a single environment variable, what will reduce the number of test running templates to a singe file. --- utils/set_test_modules | 51 ++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/utils/set_test_modules b/utils/set_test_modules index daa47dcf55..f9f1d91e4a 100644 --- a/utils/set_test_modules +++ b/utils/set_test_modules @@ -1,8 +1,7 @@ #!/bin/bash -eu # This file shoud be source'd (. set_test_modules) rather than executed. - # -# Set "BASE_BRANCH" to a different branch to compare. +# Set SKIP_GIT_TEST="True" or use -a to prevent git modification comparison. # RED="\033[31;1m" @@ -18,27 +17,45 @@ TOPDIR="$(dirname "${BASH_SOURCE[0]}")/.." pushd "${TOPDIR}" >/dev/null 2>&1 || die "Failed to change directory." +SKIP_GIT_TEST=${SKIP_GIT_TEST:-"False"} + +while getopts ":a" opt +do + case "${opt}" in + a) SKIP_GIT_TEST="True" ;; + *) ;; # ignore other options + esac +done + files_list=$(mktemp) -remote="$(basename $(mktemp -u remote_XXXXXX))" -git remote add ${remote} https://github.com/freeipa/ansible-freeipa -git fetch --prune --no-tags --quiet ${remote} -git diff "${remote}/master" --name-only > "${files_list}" -git remote remove ${remote} +enabled_modules="None" +enabled_tests="None" -# Get all modules that should have tests executed -enabled_modules="$(${python} utils/get_test_modules.py $(cat "${files_list}"))" -[ -z "${enabled_modules}" ] && enabled_modules="None" +if [ "${SKIP_GIT_TEST}" != "True" ] +then + remote="$(basename "$(mktemp -u remote_XXXXXX)")" + git remote add "${remote}" https://github.com/freeipa/ansible-freeipa + git fetch --prune --no-tags --quiet "${remote}" + git diff "${remote}/master" --name-only > "${files_list}" + git remote remove "${remote}" -# Get individual tests that should be executed -mapfile -t tests < <(sed -n "s#.*/\(test_[^/]*\).yml#\1#p" "${files_list}" | tr -d " ") -[ ${#tests[@]} -gt 0 ] && enabled_tests=$(IFS=, ; echo "${tests[*]}") -[ -z "${enabled_tests}" ] && enabled_tests="None" + # shellcheck disable=SC2046 + enabled_modules="$(${python} utils/get_test_modules.py $(cat "${files_list}"))" + [ -z "${enabled_modules}" ] && enabled_modules="None" -[ -n "${enabled_tests}" ] && IPA_ENABLED_TESTS="${enabled_tests},${IPA_ENABLED_TESTS}" -[ -n "${enabled_modules}" ] && IPA_ENABLED_MODULES="${enabled_modules},${IPA_ENABLED_MODULES}" + # Get individual tests that should be executed + mapfile -t tests < <(sed -n 's#.*/\(test_[^/]*\).yml#\1#p' "${files_list}" | tr -d " ") + [ ${#tests[@]} -gt 0 ] && enabled_tests=$(IFS=, ; echo "${tests[*]}") + [ -z "${enabled_tests}" ] && enabled_tests="None" -rm -f "${files_list}" + [ -n "${enabled_tests}" ] && IPA_ENABLED_TESTS="${enabled_tests},${IPA_ENABLED_TESTS}" + [ -n "${enabled_modules}" ] && IPA_ENABLED_MODULES="${enabled_modules},${IPA_ENABLED_MODULES}" + + rm -f "${files_list}" +fi + +# Get all modules that should have tests executed export IPA_ENABLED_MODULES export IPA_ENABLED_TESTS From c979843b1a12ca9689bf2038c13bb93637b2f873 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Fri, 25 Oct 2024 14:18:57 -0300 Subject: [PATCH 2/8] upstream CI: Allow podman options when creating containers When using containers to test ansible-freeipa there's a need to deal with 'podman' the development environment and the Azure environment. In the Azure environment, with Ubuntu hosts, using 'cap-add' does not allow FreeIPA to be installed on the containers, and they need to be executed with privileged mode. On the other hand, on development environments, such as recent Fedora hosts, there's no need to run the container with extra privileges. This patch modifies the utility function 'container_create' to allow the usage of key-value argumes such as "cpus=4" and "privileged", that will be used in the container creation. The currently available options are "privileged", "cpus", "memory" and "hostname". By default "cpus=2" and "hostname=ipaserver.test.local". Also, too make the image build script more self-contained, if the required Ansible collections are not installed, they will be temporarily installed so that the image can be built. --- infra/image/build.sh | 29 ++++++++++++++++++++++++----- infra/image/shcontainer | 27 +++++++++++++++++++-------- infra/image/start.sh | 16 ++++++++++++---- 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/infra/image/build.sh b/infra/image/build.sh index 6da8179191..3220ecce31 100755 --- a/infra/image/build.sh +++ b/infra/image/build.sh @@ -15,7 +15,7 @@ valid_distro() { usage() { local prog="${0##*/}" cat << EOF -usage: ${prog} [-h] [-s] distro +usage: ${prog} [-h] [-p] [-c HOSTNAME] [-s] distro ${prog} build a container image to test ansible-freeipa. EOF } @@ -29,8 +29,9 @@ positional arguments: optional arguments: - -s Deploy IPA server - + -c HOSTNAME Container hostname + -p Give extended privileges to the container + -s Deploy IPA server EOF } @@ -41,11 +42,14 @@ hostname="ipaserver.test.local" memory="3g" quayname="quay.io/ansible-freeipa/upstream-tests" deploy_server="N" +privileged="" -while getopts ":hs" option +while getopts ":hc:ps" option do case "${option}" in h) help && exit 0 ;; + c) hostname="${OPTARG}" ;; + p) privileged="privileged" ;; s) deploy_server="Y" ;; *) die -u "Invalid option: ${option}" ;; esac @@ -82,13 +86,28 @@ container_remove_image_if_exists "${tag}" container_remove_image_if_exists "${server_tag}" container_build "${tag}" "${BASEDIR}/dockerfile/${distro}" "${BASEDIR}" -container_create "${name}" "${tag}" "${hostname}" "${memory}" +container_create "${name}" "${tag}" \ + "hostname=${hostname}" \ + "memory=${memory}" \ + "cpus=${cpus}" \ + "${privileged}" container_commit "${name}" "${quayname}:${tag}" if [ "${deploy_server}" == "Y" ] then deployed=false + # Set path to ansible-freeipa roles + [ -z "${ANSIBLE_ROLES_PATH:-""}" ] && export ANSIBLE_ROLES_PATH="${TOPDIR}/roles" + + # Install collection containers.podman if not available + if [ -z "$(ansible-galaxy collection list containers.podman)" ] + then + tmpdir="$(mktemp -d)" + export ANSIBLE_COLLECTIONS_PATH="${tmpdir}" + ansible-galaxy collection install -p "${tmpdir}" containers.podman + fi + [ "${container_state}" != "running" ] && container_start "${name}" container_wait_for_journald "${name}" diff --git a/infra/image/shcontainer b/infra/image/shcontainer index 6f4e8a8527..0bc3326acc 100644 --- a/infra/image/shcontainer +++ b/infra/image/shcontainer @@ -9,24 +9,35 @@ TOPDIR="$(readlink -f "${SCRIPTDIR}/../..")" container_create() { local name=${1} local image=${2} - local hostname=${3} - local memory=${4:-"3g"} - local cpus=${5:-"2"} + shift 2 + declare -a extra_opts=() + for opt in "$@" + do + [ -z "${opt}" ] && continue + case "${opt}" in + hostname=*) extra_opts+=("--${opt}") ;; + cpus=*) extra_opts+=("--${opt}") ;; + memory=*) extra_opts+=("--${opt}") ;; + privileged) extra_opts+=("--${opt}") ;; + *) log error "container_create: Invalid option: ${opt}" ;; + esac + done - [ -n "${hostname}" ] || die "No hostname given" + # ensure default values are set + [[ " ${extra_opts[*]} " =~ " --cpus=" ]] || extra_opts+=("--cpus=2") + [[ " ${extra_opts[*]} " =~ " --hostname=" ]] \ + || extra_opts+=("--hostname=ipaserver.test.local") log info "= Creating ${name} =" podman create \ --security-opt label=disable \ - --name "${name}" \ - --hostname "${hostname}" \ --network bridge:interface_name=eth0 \ --systemd true \ - --cpus "${cpus}" \ - --memory "${memory}" \ + --name "${name}" \ --memory-swap -1 \ --no-hosts \ --replace \ + "${extra_opts[@]}" \ "${image}" echo } diff --git a/infra/image/start.sh b/infra/image/start.sh index c5a7a34286..0f17189d36 100755 --- a/infra/image/start.sh +++ b/infra/image/start.sh @@ -11,7 +11,7 @@ TOPDIR="$(readlink -f "${BASEDIR}/../..")" usage() { local prog="${0##*/}" cat << EOF -usage: ${prog} [-h] [-l] image +usage: ${prog} [-h] [-l] [-n HOSTNAME ] image ${prog} start a prebuilt ansible-freeipa test container image. EOF } @@ -24,7 +24,14 @@ positional arguments: optional arguments: - -l Try to use local image first, if not found download. + -h Show this message + -l Try to use local image first, if not found download. + -n HOSTNAME Set container hostname + +NOTE: + - The hostname must be the same as the hostname of the container + when FreeIPA was deployed. Use only if you built the image and + defined its hostname. EOF } @@ -45,11 +52,12 @@ name="ansible-freeipa-tests" hostname="ipaserver.test.local" try_local_first="N" -while getopts ":hl" option +while getopts ":hln:" option do case "${option}" in h) help && exit 0 ;; l) try_local_first="Y" ;; + n) hostname="${OPTARG}" ;; *) die -u "Invalid option: ${option}" ;; esac done @@ -79,7 +87,7 @@ fi [ -z "${local_image}" ] && die "Image '${image}' is not valid" -container_create "${name}" "${local_image}" "${hostname}" +container_create "${name}" "${local_image}" "hostname=${hostname}" container_start "${name}" container_wait_for_journald "${name}" container_wait_up "${name}" From 5b33cb5e801635ad7e2aaafebcbbd031749ecfa8 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Tue, 15 Oct 2024 18:08:21 -0300 Subject: [PATCH 3/8] Remove molecule dependencies For some time now, we had some issues with molecule when building test images for ansible-freeipa, and replaced the image creation with custom build scripts that use commom container tools (like Dockerfiles and the build command). As there's no more tasks that require the use of molecule, this patch removes the last bits used by it, and fixes documentation and lint scripts and configuration. --- .ansible-lint | 1 - molecule/c8s-build/Dockerfile | 30 ------------ molecule/c8s-build/molecule.yml | 19 -------- molecule/c8s/molecule.yml | 19 -------- molecule/c9s-build/Dockerfile | 29 ----------- molecule/c9s-build/molecule.yml | 19 -------- molecule/c9s/molecule.yml | 19 -------- molecule/centos-7-build/molecule.yml | 19 -------- molecule/centos-7/molecule.yml | 19 -------- molecule/default | 1 - molecule/fedora-latest-build/Dockerfile | 30 ------------ molecule/fedora-latest-build/molecule.yml | 19 -------- molecule/fedora-latest/molecule.yml | 19 -------- molecule/fedora-rawhide-build/Dockerfile | 30 ------------ molecule/fedora-rawhide-build/molecule.yml | 19 -------- molecule/fedora-rawhide/molecule.yml | 19 -------- molecule/resources/playbooks/library | 1 - molecule/resources/playbooks/module_utils | 1 - .../resources/playbooks/prepare-build.yml | 28 ----------- .../resources/playbooks/prepare-common.yml | 33 ------------- molecule/resources/playbooks/prepare.yml | 48 ------------------- molecule/resources/playbooks/roles | 1 - tests/README.md | 42 ++-------------- utils/lint_check.sh | 1 - 24 files changed, 4 insertions(+), 462 deletions(-) delete mode 100644 molecule/c8s-build/Dockerfile delete mode 100644 molecule/c8s-build/molecule.yml delete mode 100644 molecule/c8s/molecule.yml delete mode 100644 molecule/c9s-build/Dockerfile delete mode 100644 molecule/c9s-build/molecule.yml delete mode 100644 molecule/c9s/molecule.yml delete mode 100644 molecule/centos-7-build/molecule.yml delete mode 100644 molecule/centos-7/molecule.yml delete mode 120000 molecule/default delete mode 100644 molecule/fedora-latest-build/Dockerfile delete mode 100644 molecule/fedora-latest-build/molecule.yml delete mode 100644 molecule/fedora-latest/molecule.yml delete mode 100644 molecule/fedora-rawhide-build/Dockerfile delete mode 100644 molecule/fedora-rawhide-build/molecule.yml delete mode 100644 molecule/fedora-rawhide/molecule.yml delete mode 120000 molecule/resources/playbooks/library delete mode 120000 molecule/resources/playbooks/module_utils delete mode 100644 molecule/resources/playbooks/prepare-build.yml delete mode 100644 molecule/resources/playbooks/prepare-common.yml delete mode 100644 molecule/resources/playbooks/prepare.yml delete mode 120000 molecule/resources/playbooks/roles diff --git a/.ansible-lint b/.ansible-lint index 67136a0371..28fc3ac24f 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -7,7 +7,6 @@ exclude_paths: - .tox/ - .venv/ - .yamllint - - molecule/ - tests/azure/ - meta/runtime.yml - requirements-docker.yml diff --git a/molecule/c8s-build/Dockerfile b/molecule/c8s-build/Dockerfile deleted file mode 100644 index 7bdc176471..0000000000 --- a/molecule/c8s-build/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM quay.io/centos/centos:stream8 -ENV container=docker - -RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ -dnf makecache; \ -dnf --assumeyes install \ - /usr/bin/python3 \ - /usr/bin/python3-config \ - /usr/bin/dnf-3 \ - sudo \ - bash \ - systemd \ - procps-ng \ - iproute && \ -dnf clean all; \ -(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ -rm -f /lib/systemd/system/multi-user.target.wants/*;\ -rm -f /etc/systemd/system/*.wants/*;\ -rm -f /lib/systemd/system/local-fs.target.wants/*; \ -rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ -rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ -rm -f /lib/systemd/system/basic.target.wants/*;\ -rm -f /lib/systemd/system/anaconda.target.wants/*; \ -rm -rf /var/cache/dnf/; - -STOPSIGNAL RTMIN+3 - -VOLUME ["/sys/fs/cgroup"] - -CMD ["/usr/sbin/init"] diff --git a/molecule/c8s-build/molecule.yml b/molecule/c8s-build/molecule.yml deleted file mode 100644 index 8855215190..0000000000 --- a/molecule/c8s-build/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: c8s-build - image: "quay.io/centos/centos:stream8" - dockerfile: Dockerfile - hostname: ipaserver.test.local - dns_servers: - - 8.8.8.8 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare-build.yml -prerun: false diff --git a/molecule/c8s/molecule.yml b/molecule/c8s/molecule.yml deleted file mode 100644 index 79f35c547a..0000000000 --- a/molecule/c8s/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: c8s - image: quay.io/ansible-freeipa/upstream-tests:c8s - pre_build_image: true - hostname: ipaserver.test.local - dns_servers: - - 127.0.0.1 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare.yml -prerun: false diff --git a/molecule/c9s-build/Dockerfile b/molecule/c9s-build/Dockerfile deleted file mode 100644 index 2bb39a10f9..0000000000 --- a/molecule/c9s-build/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM quay.io/centos/centos:stream9 -ENV container=docker - -RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ -dnf makecache; \ -dnf --assumeyes install \ - /usr/bin/python3 \ - /usr/bin/dnf-3 \ - sudo \ - bash \ - systemd \ - procps-ng \ - iproute && \ -dnf clean all; \ -(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ -rm -f /lib/systemd/system/multi-user.target.wants/*;\ -rm -f /etc/systemd/system/*.wants/*;\ -rm -f /lib/systemd/system/local-fs.target.wants/*; \ -rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ -rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ -rm -f /lib/systemd/system/basic.target.wants/*;\ -rm -f /lib/systemd/system/anaconda.target.wants/*; \ -rm -rf /var/cache/dnf/; - -STOPSIGNAL RTMIN+3 - -VOLUME ["/sys/fs/cgroup"] - -CMD ["/usr/sbin/init"] diff --git a/molecule/c9s-build/molecule.yml b/molecule/c9s-build/molecule.yml deleted file mode 100644 index dfa6548045..0000000000 --- a/molecule/c9s-build/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: c9s-build - image: "quay.io/centos/centos:stream9" - dockerfile: Dockerfile - hostname: ipaserver.test.local - dns_servers: - - 8.8.8.8 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare-build.yml -prerun: false diff --git a/molecule/c9s/molecule.yml b/molecule/c9s/molecule.yml deleted file mode 100644 index 7aba522529..0000000000 --- a/molecule/c9s/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: c9s - image: quay.io/ansible-freeipa/upstream-tests:c9s - pre_build_image: true - hostname: ipaserver.test.local - dns_servers: - - 127.0.0.1 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare.yml -prerun: false diff --git a/molecule/centos-7-build/molecule.yml b/molecule/centos-7-build/molecule.yml deleted file mode 100644 index e88e1208ad..0000000000 --- a/molecule/centos-7-build/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: centos-7-build - image: centos/systemd - pre_build_image: true - hostname: ipaserver.test.local - dns_servers: - - 8.8.8.8 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare-build.yml -prerun: false diff --git a/molecule/centos-7/molecule.yml b/molecule/centos-7/molecule.yml deleted file mode 100644 index 5cb077e233..0000000000 --- a/molecule/centos-7/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: centos-7 - image: quay.io/ansible-freeipa/upstream-tests:centos-7 - pre_build_image: true - hostname: ipaserver.test.local - dns_servers: - - 127.0.0.1 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare.yml -prerun: false diff --git a/molecule/default b/molecule/default deleted file mode 120000 index 768be926fb..0000000000 --- a/molecule/default +++ /dev/null @@ -1 +0,0 @@ -fedora-latest \ No newline at end of file diff --git a/molecule/fedora-latest-build/Dockerfile b/molecule/fedora-latest-build/Dockerfile deleted file mode 100644 index f3c2ef6281..0000000000 --- a/molecule/fedora-latest-build/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM fedora:latest -ENV container=docker - -RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ -dnf makecache; \ -dnf --assumeyes install \ - /usr/bin/python3 \ - /usr/bin/python3-config \ - /usr/bin/dnf-3 \ - sudo \ - bash \ - systemd \ - procps-ng \ - iproute && \ -dnf clean all; \ -(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ -rm -f /lib/systemd/system/multi-user.target.wants/*;\ -rm -f /etc/systemd/system/*.wants/*;\ -rm -f /lib/systemd/system/local-fs.target.wants/*; \ -rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ -rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ -rm -f /lib/systemd/system/basic.target.wants/*;\ -rm -f /lib/systemd/system/anaconda.target.wants/*; \ -rm -rf /var/cache/dnf/; - -STOPSIGNAL RTMIN+3 - -VOLUME ["/sys/fs/cgroup"] - -CMD ["/usr/sbin/init"] diff --git a/molecule/fedora-latest-build/molecule.yml b/molecule/fedora-latest-build/molecule.yml deleted file mode 100644 index 10635d3a81..0000000000 --- a/molecule/fedora-latest-build/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: fedora-latest-build - image: "fedora:latest" - dockerfile: Dockerfile - hostname: ipaserver.test.local - dns_servers: - - 8.8.8.8 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare-build.yml -prerun: false diff --git a/molecule/fedora-latest/molecule.yml b/molecule/fedora-latest/molecule.yml deleted file mode 100644 index 94c4fc4a0c..0000000000 --- a/molecule/fedora-latest/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: fedora-latest - image: quay.io/ansible-freeipa/upstream-tests:fedora-latest - pre_build_image: true - hostname: ipaserver.test.local - dns_servers: - - 127.0.0.1 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare.yml -prerun: false diff --git a/molecule/fedora-rawhide-build/Dockerfile b/molecule/fedora-rawhide-build/Dockerfile deleted file mode 100644 index f1a1c124d3..0000000000 --- a/molecule/fedora-rawhide-build/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM fedora:rawhide -ENV container=docker - -RUN rm -fv /var/cache/dnf/metadata_lock.pid; \ -dnf makecache; \ -dnf --assumeyes install \ - /usr/bin/python3 \ - /usr/bin/python3-config \ - /usr/bin/dnf-3 \ - sudo \ - bash \ - systemd \ - procps-ng \ - iproute && \ -dnf clean all; \ -(cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ -rm -f /lib/systemd/system/multi-user.target.wants/*;\ -rm -f /etc/systemd/system/*.wants/*;\ -rm -f /lib/systemd/system/local-fs.target.wants/*; \ -rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ -rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ -rm -f /lib/systemd/system/basic.target.wants/*;\ -rm -f /lib/systemd/system/anaconda.target.wants/*; \ -rm -rf /var/cache/dnf/; - -STOPSIGNAL RTMIN+3 - -VOLUME ["/sys/fs/cgroup"] - -CMD ["/usr/sbin/init"] diff --git a/molecule/fedora-rawhide-build/molecule.yml b/molecule/fedora-rawhide-build/molecule.yml deleted file mode 100644 index c8afa32964..0000000000 --- a/molecule/fedora-rawhide-build/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: fedora-rawhide-build - image: "fedora:rawhide" - dockerfile: Dockerfile - hostname: ipaserver.test.local - dns_servers: - - 8.8.8.8 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare-build.yml -prerun: false diff --git a/molecule/fedora-rawhide/molecule.yml b/molecule/fedora-rawhide/molecule.yml deleted file mode 100644 index 6ff53e9b4e..0000000000 --- a/molecule/fedora-rawhide/molecule.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -driver: - name: docker -platforms: - - name: fedora-rawhide - image: quay.io/ansible-freeipa/upstream-tests:fedora-rawhide - pre_build_image: true - hostname: ipaserver.test.local - dns_servers: - - 127.0.0.1 - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - command: /usr/sbin/init - privileged: true -provisioner: - name: ansible - playbooks: - prepare: ../resources/playbooks/prepare.yml -prerun: false diff --git a/molecule/resources/playbooks/library b/molecule/resources/playbooks/library deleted file mode 120000 index 0140dc6d3e..0000000000 --- a/molecule/resources/playbooks/library +++ /dev/null @@ -1 +0,0 @@ -../../../plugins/modules/ \ No newline at end of file diff --git a/molecule/resources/playbooks/module_utils b/molecule/resources/playbooks/module_utils deleted file mode 120000 index 2b1d4840b3..0000000000 --- a/molecule/resources/playbooks/module_utils +++ /dev/null @@ -1 +0,0 @@ -../../../plugins/module_utils/ \ No newline at end of file diff --git a/molecule/resources/playbooks/prepare-build.yml b/molecule/resources/playbooks/prepare-build.yml deleted file mode 100644 index 5e8e5a92e1..0000000000 --- a/molecule/resources/playbooks/prepare-build.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: Converge - hosts: all - tasks: - - include_tasks: prepare-common.yml - - - name: Ensure sudo package is installed - package: - name: sudo - - - name: Ensure nss package is updated - package: - name: nss - state: latest # noqa 403 - - - include_role: - name: ipaserver - vars: - ipaserver_setup_dns: yes - ipaserver_setup_kra: yes - ipaserver_auto_forwarders: yes - ipaserver_no_dnssec_validation: yes - ipaserver_auto_reverse: yes - ipaadmin_password: SomeADMINpassword - ipadm_password: SomeDMpassword - ipaserver_domain: test.local - ipaserver_realm: TEST.LOCAL - ipaclient_no_ntp: yes diff --git a/molecule/resources/playbooks/prepare-common.yml b/molecule/resources/playbooks/prepare-common.yml deleted file mode 100644 index 1e1a60227d..0000000000 --- a/molecule/resources/playbooks/prepare-common.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -# IPA depends on IPv6 and without it dirsrv service won't start. -- name: Ensure IPv6 is ENABLED - ansible.posix.sysctl: - name: "{{ item.name }}" - value: "{{ item.value }}" - sysctl_set: yes - state: present - reload: yes - with_items: - - name: net.ipv6.conf.all.disable_ipv6 - value: 0 - - name: net.ipv6.conf.lo.disable_ipv6 - value: 0 - - name: net.ipv6.conf.eth0.disable_ipv6 - value: 1 - -# Set fs.protected_regular to 0 -# This is needed in some IPA versions in order to get KRA enabled. -# See https://pagure.io/freeipa/issue/7906 for more information. -- name: stat protected_regular - ansible.builtin.stat: - path: /proc/sys/fs/protected_regular - register: result - -- name: Ensure fs.protected_regular is disabled - ansible.posix.sysctl: - name: fs.protected_regular - value: 0 - sysctl_set: yes - state: present - reload: yes - when: result.stat.exists diff --git a/molecule/resources/playbooks/prepare.yml b/molecule/resources/playbooks/prepare.yml deleted file mode 100644 index deaa092b33..0000000000 --- a/molecule/resources/playbooks/prepare.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: Converge - hosts: all - tasks: - - include_tasks: prepare-common.yml - - # In some distros DS won't start up after reboot - # This is due to a problem in 389-ds. See tickets: - # * https://pagure.io/389-ds-base/issue/47429 - # * https://pagure.io/389-ds-base/issue/51039 - # - # To avoid this problem we create the directories before starting IPA. - - name: Ensure lock dirs for DS exists - ansible.builtin.file: - state: directory - owner: dirsrv - group: dirsrv - path: "{{ item }}" - mode: 0770 - loop: - - /var/lock/dirsrv/ - - /var/lock/dirsrv/slapd-TEST-LOCAL/ - - - name: Ensure IPA server is up an running - ansible.builtin.service: - name: ipa - state: started - - - name: Wait for krb5dkc to be running - ansible.builtin.service_facts: - no_log: True - register: result - until: "'krb5kdc.service' in result.ansible_facts.services and \ - result.ansible_facts.services['krb5kdc.service'].state == 'running'" - retries: 30 - delay: 5 - - - name: Check if TGT is available for admin. - ansible.builtin.shell: - cmd: echo SomeADMINpassword | kinit -c ansible_freeipa_cache admin - register: result - until: not result.failed - retries: 30 - delay: 5 - - - name: Cleanup TGT. - ansible.builtin.shell: - cmd: kdestroy -c ansible_freeipa_cache -A diff --git a/molecule/resources/playbooks/roles b/molecule/resources/playbooks/roles deleted file mode 120000 index e2b799b9d7..0000000000 --- a/molecule/resources/playbooks/roles +++ /dev/null @@ -1 +0,0 @@ -../../../roles/ \ No newline at end of file diff --git a/tests/README.md b/tests/README.md index 1e9d4e4a19..298078b868 100644 --- a/tests/README.md +++ b/tests/README.md @@ -102,48 +102,16 @@ The pytests are tests that will execute small playbooks and then will verify the To select only these tests on a test execution use the option `-m "not playbook"`. -## Running tests in a docker container +## Running tests in a container -It's also possible to run the tests in a container. - -### Creating a container to run the tests - -Before setting up a container you will need to install molecule framework: - -``` -pip install molecule-plugins[docker] -``` - -Now you can start a test container using the following command: -``` -molecule create -s c8s -``` - -Note: Currently the containers available for running the tests are: - * fedora-latest - * centos-7 - * c8s - * c9s - -### Running the tests inside the container +It's also possible to run the tests in a container. Use the script `infra/image/start.sh` to start a container. To run the tests you will use pytest (works the same as for VMs). ``` -RUN_TESTS_IN_DOCKER=1 IPA_SERVER_HOST=c8s pytest +RUN_TESTS_IN_DOCKER=podman IPA_SERVER_HOST=ansbile-freeipa-tests pytest -m "playbook" ``` -### Cleaning up after tests - -After running the tests you should probably destroy the test container using: - -``` -molecule destroy -s c8s -``` - -See [Running the tests](#running-the-tests) section for more information on available options. - - ## Running local tests with upstream CI images To run tests locally using the same images used by upstream CI use `utils/run-tests.sh`. @@ -172,8 +140,6 @@ By default the tests are executed against the latest version of the Fedora image utils/run-tests.sh -i c9s tests/host/test_host.yml ``` - ## Upcoming/desired improvements: -* A script to pre-config the complete test environment using virsh. -* A test matrix to run tests against different distros in parallel (probably using tox). +* A test matrix to run tests against different distros in parallel. diff --git a/utils/lint_check.sh b/utils/lint_check.sh index 25d4f8878c..76c8c2ace1 100755 --- a/utils/lint_check.sh +++ b/utils/lint_check.sh @@ -35,7 +35,6 @@ echo -e "${INFO}Running 'yamllint'...${RST}" yaml_dirs=( "tests" "playbooks" - "molecule" "roles" ) yamllint -f colored "${yaml_dirs[@]}" From 77c34aeca2a68d140f56b9fd3c33954ea483f8e9 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Tue, 15 Oct 2024 10:28:49 -0300 Subject: [PATCH 4/8] upstream ci: Move Azure scripts to infra directory Move Azure scripts to infra directory, as only roles and modules test playbooks should exist in the tests directory. --- {tests => infra}/azure/azure-pipelines.yml | 0 {tests => infra}/azure/build-containers.yml | 0 {tests => infra}/azure/nightly.yml | 0 {tests => infra}/azure/pr-pipeline.yml | 0 {tests => infra}/azure/templates/build_container.yml | 0 {tests => infra}/azure/templates/fast_tests.yml | 0 {tests => infra}/azure/templates/galaxy_pytest_script.yml | 0 {tests => infra}/azure/templates/galaxy_script.yml | 0 {tests => infra}/azure/templates/galaxy_tests.yml | 0 {tests => infra}/azure/templates/group_tests.yml | 0 {tests => infra}/azure/templates/playbook_fast.yml | 0 {tests => infra}/azure/templates/playbook_tests.yml | 0 {tests => infra}/azure/templates/pytest_tests.yml | 0 {tests => infra}/azure/templates/variables.yaml | 0 {tests => infra}/azure/templates/variables_c8s.yaml | 0 {tests => infra}/azure/templates/variables_c9s.yaml | 0 {tests => infra}/azure/templates/variables_centos-7.yaml | 0 {tests => infra}/azure/templates/variables_fedora-latest.yaml | 0 {tests => infra}/azure/templates/variables_fedora-rawhide.yaml | 0 19 files changed, 0 insertions(+), 0 deletions(-) rename {tests => infra}/azure/azure-pipelines.yml (100%) rename {tests => infra}/azure/build-containers.yml (100%) rename {tests => infra}/azure/nightly.yml (100%) rename {tests => infra}/azure/pr-pipeline.yml (100%) rename {tests => infra}/azure/templates/build_container.yml (100%) rename {tests => infra}/azure/templates/fast_tests.yml (100%) rename {tests => infra}/azure/templates/galaxy_pytest_script.yml (100%) rename {tests => infra}/azure/templates/galaxy_script.yml (100%) rename {tests => infra}/azure/templates/galaxy_tests.yml (100%) rename {tests => infra}/azure/templates/group_tests.yml (100%) rename {tests => infra}/azure/templates/playbook_fast.yml (100%) rename {tests => infra}/azure/templates/playbook_tests.yml (100%) rename {tests => infra}/azure/templates/pytest_tests.yml (100%) rename {tests => infra}/azure/templates/variables.yaml (100%) rename {tests => infra}/azure/templates/variables_c8s.yaml (100%) rename {tests => infra}/azure/templates/variables_c9s.yaml (100%) rename {tests => infra}/azure/templates/variables_centos-7.yaml (100%) rename {tests => infra}/azure/templates/variables_fedora-latest.yaml (100%) rename {tests => infra}/azure/templates/variables_fedora-rawhide.yaml (100%) diff --git a/tests/azure/azure-pipelines.yml b/infra/azure/azure-pipelines.yml similarity index 100% rename from tests/azure/azure-pipelines.yml rename to infra/azure/azure-pipelines.yml diff --git a/tests/azure/build-containers.yml b/infra/azure/build-containers.yml similarity index 100% rename from tests/azure/build-containers.yml rename to infra/azure/build-containers.yml diff --git a/tests/azure/nightly.yml b/infra/azure/nightly.yml similarity index 100% rename from tests/azure/nightly.yml rename to infra/azure/nightly.yml diff --git a/tests/azure/pr-pipeline.yml b/infra/azure/pr-pipeline.yml similarity index 100% rename from tests/azure/pr-pipeline.yml rename to infra/azure/pr-pipeline.yml diff --git a/tests/azure/templates/build_container.yml b/infra/azure/templates/build_container.yml similarity index 100% rename from tests/azure/templates/build_container.yml rename to infra/azure/templates/build_container.yml diff --git a/tests/azure/templates/fast_tests.yml b/infra/azure/templates/fast_tests.yml similarity index 100% rename from tests/azure/templates/fast_tests.yml rename to infra/azure/templates/fast_tests.yml diff --git a/tests/azure/templates/galaxy_pytest_script.yml b/infra/azure/templates/galaxy_pytest_script.yml similarity index 100% rename from tests/azure/templates/galaxy_pytest_script.yml rename to infra/azure/templates/galaxy_pytest_script.yml diff --git a/tests/azure/templates/galaxy_script.yml b/infra/azure/templates/galaxy_script.yml similarity index 100% rename from tests/azure/templates/galaxy_script.yml rename to infra/azure/templates/galaxy_script.yml diff --git a/tests/azure/templates/galaxy_tests.yml b/infra/azure/templates/galaxy_tests.yml similarity index 100% rename from tests/azure/templates/galaxy_tests.yml rename to infra/azure/templates/galaxy_tests.yml diff --git a/tests/azure/templates/group_tests.yml b/infra/azure/templates/group_tests.yml similarity index 100% rename from tests/azure/templates/group_tests.yml rename to infra/azure/templates/group_tests.yml diff --git a/tests/azure/templates/playbook_fast.yml b/infra/azure/templates/playbook_fast.yml similarity index 100% rename from tests/azure/templates/playbook_fast.yml rename to infra/azure/templates/playbook_fast.yml diff --git a/tests/azure/templates/playbook_tests.yml b/infra/azure/templates/playbook_tests.yml similarity index 100% rename from tests/azure/templates/playbook_tests.yml rename to infra/azure/templates/playbook_tests.yml diff --git a/tests/azure/templates/pytest_tests.yml b/infra/azure/templates/pytest_tests.yml similarity index 100% rename from tests/azure/templates/pytest_tests.yml rename to infra/azure/templates/pytest_tests.yml diff --git a/tests/azure/templates/variables.yaml b/infra/azure/templates/variables.yaml similarity index 100% rename from tests/azure/templates/variables.yaml rename to infra/azure/templates/variables.yaml diff --git a/tests/azure/templates/variables_c8s.yaml b/infra/azure/templates/variables_c8s.yaml similarity index 100% rename from tests/azure/templates/variables_c8s.yaml rename to infra/azure/templates/variables_c8s.yaml diff --git a/tests/azure/templates/variables_c9s.yaml b/infra/azure/templates/variables_c9s.yaml similarity index 100% rename from tests/azure/templates/variables_c9s.yaml rename to infra/azure/templates/variables_c9s.yaml diff --git a/tests/azure/templates/variables_centos-7.yaml b/infra/azure/templates/variables_centos-7.yaml similarity index 100% rename from tests/azure/templates/variables_centos-7.yaml rename to infra/azure/templates/variables_centos-7.yaml diff --git a/tests/azure/templates/variables_fedora-latest.yaml b/infra/azure/templates/variables_fedora-latest.yaml similarity index 100% rename from tests/azure/templates/variables_fedora-latest.yaml rename to infra/azure/templates/variables_fedora-latest.yaml diff --git a/tests/azure/templates/variables_fedora-rawhide.yaml b/infra/azure/templates/variables_fedora-rawhide.yaml similarity index 100% rename from tests/azure/templates/variables_fedora-rawhide.yaml rename to infra/azure/templates/variables_fedora-rawhide.yaml From 367d30a30c50797c492599987ddfd408c18e89d0 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Tue, 15 Oct 2024 11:16:57 -0300 Subject: [PATCH 5/8] upstream CI: Simplify pipelines enviroment creation Several optimizations have been done to the pipelines, to make them closer to what can be reproduced, with the existing scripts, in a development environment: - Use start.sh and build.sh scripts to build and start containers - Use variables to configure different stages instead of using separate files - Use a commom 'prepare_environment' to create the environment for every pipeline - Use a single file defining testing steps (run_tests.yml) - Remove Centos 7 pipelines - Reduce the number of pipelines in the test matrix due to the amount of time that tests were using - Use Azure "loop" (each) to create test groups The above changes make the pipelines easier to understand and modify. --- infra/azure/azure-pipelines.yml | 32 +++-- infra/azure/build-containers.yml | 10 -- infra/azure/nightly.yml | 119 +++++++----------- infra/azure/pr-pipeline.yml | 31 ++--- infra/azure/templates/build_container.yml | 11 +- infra/azure/templates/fast_tests.yml | 31 ----- .../azure/templates/galaxy_pytest_script.yml | 65 ---------- infra/azure/templates/galaxy_script.yml | 86 ------------- infra/azure/templates/galaxy_tests.yml | 46 ------- infra/azure/templates/group_tests.yml | 52 +++----- infra/azure/templates/playbook_fast.yml | 84 ------------- infra/azure/templates/playbook_tests.yml | 88 ------------- .../azure/templates/prepare_environment.yaml | 30 +++++ infra/azure/templates/pytest_tests.yml | 75 ----------- infra/azure/templates/run_tests.yml | 98 +++++++++++++++ infra/image/build.sh | 3 +- 16 files changed, 218 insertions(+), 643 deletions(-) delete mode 100644 infra/azure/templates/fast_tests.yml delete mode 100644 infra/azure/templates/galaxy_pytest_script.yml delete mode 100644 infra/azure/templates/galaxy_script.yml delete mode 100644 infra/azure/templates/galaxy_tests.yml delete mode 100644 infra/azure/templates/playbook_fast.yml delete mode 100644 infra/azure/templates/playbook_tests.yml create mode 100644 infra/azure/templates/prepare_environment.yaml delete mode 100644 infra/azure/templates/pytest_tests.yml create mode 100644 infra/azure/templates/run_tests.yml diff --git a/infra/azure/azure-pipelines.yml b/infra/azure/azure-pipelines.yml index ab28bb1108..2043940a68 100644 --- a/infra/azure/azure-pipelines.yml +++ b/infra/azure/azure-pipelines.yml @@ -15,8 +15,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core" + skip_git_test: true # Fedora @@ -26,8 +27,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core <2.14" + skip_git_test: true # Fedora @@ -37,19 +39,22 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core >=2.14,<2.15" + skip_git_test: true # Galaxy on Fedora - stage: Galaxy_Fedora_Latest dependsOn: [] jobs: - - template: templates/galaxy_tests.yml + - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core >=2.14,<2.15" + skip_git_test: true + test_galaxy: true # CentOS 9 Stream @@ -59,8 +64,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c9s + distro: c9s ansible_version: "-core >=2.14,<2.15" + skip_git_test: true # CentOS 8 Stream @@ -70,16 +76,6 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c8s - ansible_version: "-core >=2.14,<2.15" - -# CentOS 7 - -- stage: CentOS_7 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - scenario: centos-7 + distro: c8s ansible_version: "-core >=2.14,<2.15" + skip_git_test: true diff --git a/infra/azure/build-containers.yml b/infra/azure/build-containers.yml index 75c3853909..d4b6b25abd 100644 --- a/infra/azure/build-containers.yml +++ b/infra/azure/build-containers.yml @@ -15,16 +15,6 @@ pool: stages: -# Currently, it's not possible to use CentOS container -# -# - stage: CentOS_7 -# dependsOn: [] -# jobs: -# - template: templates/build_container.yml -# parameters: -# job_name_suffix: Centos7 -# distro: centos-7 - - stage: CentOS_8_Stream dependsOn: [] jobs: diff --git a/infra/azure/nightly.yml b/infra/azure/nightly.yml index 89914f8e4b..cbbc988972 100644 --- a/infra/azure/nightly.yml +++ b/infra/azure/nightly.yml @@ -22,8 +22,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core >=2.13,<2.14" + skip_git_test: false - stage: FedoraLatest_Ansible_Core_2_14 dependsOn: [] @@ -31,8 +32,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core >=2.14,<2.15" + skip_git_test: false - stage: FedoraLatest_Ansible_Core_2_15 dependsOn: [] @@ -40,8 +42,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core >=2.15,<2.16" + skip_git_test: true - stage: FedoraLatest_Ansible_latest dependsOn: [] @@ -49,46 +52,44 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "" + skip_git_test: true # Galaxy on Fedora - stage: Galaxy_FedoraLatest_Ansible_Core_2_13 dependsOn: [] jobs: - - template: templates/galaxy_tests.yml + - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core >=2.13,<2.14" + skip_git_test: false + test_galaxy: true - stage: Galaxy_FedoraLatest_Ansible_Core_2_14 dependsOn: [] jobs: - - template: templates/galaxy_tests.yml + - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core >=2.14,<2.15" + skip_git_test: false + test_galaxy: true - stage: Galaxy_FedoraLatest_Ansible_Core_2_15 dependsOn: [] jobs: - - template: templates/galaxy_tests.yml + - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-latest + distro: fedora-latest ansible_version: "-core >=2.15,<2.16" - -- stage: Galaxy_FedoraLatest_Ansible_latest - dependsOn: [] - jobs: - - template: templates/galaxy_tests.yml - parameters: - build_number: $(Build.BuildNumber) - scenario: fedora-latest - ansible_version: "" + skip_git_test: true + test_galaxy: true # Fedora Rawhide @@ -98,8 +99,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-rawhide + distro: fedora-rawhide ansible_version: "-core >=2.13,<2.14" + skip_git_test: false - stage: FedoraRawhide_Ansible_Core_2_14 dependsOn: [] @@ -107,8 +109,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-rawhide + distro: fedora-rawhide ansible_version: "-core >=2.14,<2.15" + skip_git_test: false - stage: FedoraRawhide_Ansible_Core_2_15 dependsOn: [] @@ -116,8 +119,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-rawhide + distro: fedora-rawhide ansible_version: "-core >=2.15,<2.16" + skip_git_test: true - stage: FedoraRawhide_Ansible_latest dependsOn: [] @@ -125,8 +129,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: fedora-rawhide + distro: fedora-rawhide ansible_version: "" + skip_git_test: true # CentoOS 9 Stream @@ -136,8 +141,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c9s + distro: c9s ansible_version: "-core >=2.13,<2.14" + skip_git_test: false - stage: c9s_Ansible_Core_2_14 dependsOn: [] @@ -145,8 +151,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c9s + distro: c9s ansible_version: "-core >=2.14,<2.15" + skip_git_test: false - stage: c9s_Ansible_Core_2_15 dependsOn: [] @@ -154,8 +161,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c9s + distro: c9s ansible_version: "-core >=2.15,<2.16" + skip_git_test: true - stage: c9s_Ansible_latest dependsOn: [] @@ -163,8 +171,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c9s + distro: c9s ansible_version: "" + skip_git_test: true # CentOS 8 Stream @@ -174,8 +183,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c8s + distro: c8s ansible_version: "-core >=2.13,<2.14" + skip_git_test: true - stage: c8s_Ansible_Core_2_14 dependsOn: [] @@ -183,8 +193,9 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c8s + distro: c8s ansible_version: "-core >=2.14,<2.15" + skip_git_test: false - stage: c8s_Ansible_Core_2_15 dependsOn: [] @@ -192,52 +203,6 @@ stages: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - scenario: c8s + distro: c8s ansible_version: "-core >=2.15,<2.16" - -- stage: c8s_Ansible_latest - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - scenario: c8s - ansible_version: "" - -# CentOS 7 - -- stage: CentOS7_Ansible_Core_2_13 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - scenario: centos-7 - ansible_version: "-core >=2.13,<2.14" - -- stage: CentOS7_Ansible_Core_2_14 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - scenario: centos-7 - ansible_version: "-core >=2.14,<2.15" - -- stage: CentOS7_Ansible_Core_2_15 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - scenario: centos-7 - ansible_version: "-core >=2.15,<2.16" - -- stage: CentOS7_Ansible_latest - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - scenario: centos-7 - ansible_version: "" + skip_git_test: true diff --git a/infra/azure/pr-pipeline.yml b/infra/azure/pr-pipeline.yml index 40d78a9b10..bc6e0830b8 100644 --- a/infra/azure/pr-pipeline.yml +++ b/infra/azure/pr-pipeline.yml @@ -12,67 +12,58 @@ stages: - stage: Fedora_Latest dependsOn: [] jobs: - - template: templates/fast_tests.yml + - template: templates/run_tests.yml parameters: build_number: $(Build.BuildNumber) distro: fedora-latest ansible_version: "-core >=2.15,<2.16" + skip_git_test: false # Galaxy on Fedora - stage: Galaxy_Fedora_Latest dependsOn: [] jobs: - - template: templates/fast_tests.yml + - template: templates/run_tests.yml parameters: build_number: $(Build.BuildNumber) distro: fedora-latest ansible_version: "-core >=2.15,<2.16" + skip_git_test: false + test_galaxy: true # CentOS 9 Stream - stage: CentOS_9_Stream dependsOn: [] jobs: - - template: templates/fast_tests.yml + - template: templates/run_tests.yml parameters: build_number: $(Build.BuildNumber) distro: c9s ansible_version: "-core >=2.15,<2.16" + skip_git_test: false # CentOS 8 Stream - stage: CentOS_8_Stream dependsOn: [] jobs: - - template: templates/fast_tests.yml + - template: templates/run_tests.yml parameters: build_number: $(Build.BuildNumber) distro: c8s ansible_version: "-core >=2.15,<2.16" - target_python: "/usr/libexec/platform-python" - -# CentOS 7 cannot be used with current systemd -# -# CentOS 7 -# -# - stage: CentOS_7 -# dependsOn: [] -# jobs: -# - template: templates/fast_tests.yml -# parameters: -# build_number: $(Build.BuildNumber) -# distro: centos-7 -# ansible_version: "-core >=2.15,<2.16" -# target_python: "/usr/bin/python2" + skip_git_test: false # Rawhide - stage: Fedora_Rawhide dependsOn: [] jobs: - - template: templates/fast_tests.yml + - template: templates/run_tests.yml parameters: build_number: $(Build.BuildNumber) distro: fedora-rawhide ansible_version: "-core >=2.15,<2.16" + skip_git_test: false diff --git a/infra/azure/templates/build_container.yml b/infra/azure/templates/build_container.yml index f93881ce2f..49d2f73108 100644 --- a/infra/azure/templates/build_container.yml +++ b/infra/azure/templates/build_container.yml @@ -22,18 +22,15 @@ jobs: retryCountOnTaskFailure: 5 displayName: Install tools - - script: | - rm -rf ~/.ansible - mkdir -p ~/.ansible - ln -snf $(readlink -f roles) ~/.ansible/roles - ln -snf $(readlink -f plugins) ~/.ansible/plugins - displayName: Setup ansible-freeipa using Git repository - - script: ansible-galaxy collection install containers.podman displayName: Install Ansible Galaxy collections - script: infra/image/build.sh -s ${{ parameters.distro }} displayName: Build ${{ parameters.distro }} base image + env: + ANSIBLE_ROLES_PATH: "${PWD}/roles" + ANSIBLE_LIBRARY: "${PWD}/plugins/modules" + ANSIBLE_MODULE_UTILS: "${PWD}/plugins/module_utils" - script: podman login -u="$QUAY_ROBOT_USERNAME" -p="$QUAY_ROBOT_TOKEN" quay.io displayName: Registry login diff --git a/infra/azure/templates/fast_tests.yml b/infra/azure/templates/fast_tests.yml deleted file mode 100644 index ac26ce8f0b..0000000000 --- a/infra/azure/templates/fast_tests.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -parameters: - - name: distro - type: string - default: fedora-latest - - name: build_number - type: string - - name: ansible_version - type: string - default: "" - - name: target_python - type: string - default: "/usr/bin/python3" - -jobs: -- template: playbook_fast.yml - parameters: - group_number: 1 - number_of_groups: 1 - build_number: ${{ parameters.build_number }} - distro: ${{ parameters.distro }} - ansible_version: ${{ parameters.ansible_version }} - python_version: '< 3.12' - target_python: ${{ parameters.target_python }} - -# - template: pytest_tests.yml -# parameters: -# build_number: ${{ parameters.build_number }} -# distro: ${{ parameters.distro }} -# ansible_version: ${{ parameters.ansible_version }} -# python_version: '< 3.12' diff --git a/infra/azure/templates/galaxy_pytest_script.yml b/infra/azure/templates/galaxy_pytest_script.yml deleted file mode 100644 index 3b16416961..0000000000 --- a/infra/azure/templates/galaxy_pytest_script.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -parameters: - - name: build_number - type: string - - name: scenario - type: string - default: fedora-latest - - name: ansible_version - type: string - default: "" - - name: python_version - type: string - default: 3.x - -jobs: -- job: Test_PyTests - displayName: Run pytests on ${{ parameters.scenario }} - timeoutInMinutes: 240 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '${{ parameters.python_version }}' - - - script: | - pip install \ - "molecule-plugins[docker]" \ - "requests<2.29" \ - "ansible${{ parameters.ansible_version }}" - retryCountOnTaskFailure: 5 - displayName: Install molecule and Ansible - - - script: ansible-galaxy collection install community.docker ansible.posix - retryCountOnTaskFailure: 5 - displayName: Install Ansible collections - - - script: pip install -r requirements-tests.txt - retryCountOnTaskFailure: 5 - displayName: Install dependencies - - - script: | - utils/build-galaxy-release.sh -i - molecule create -s ${{ parameters.scenario }} - retryCountOnTaskFailure: 5 - displayName: Setup test container - env: - ANSIBLE_LIBRARY: ./molecule - - - script: | - cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa - pytest \ - -m "not playbook" \ - --verbose \ - --color=yes \ - --junit-xml=TEST-results-pytests.xml - displayName: Run tests - env: - IPA_SERVER_HOST: ${{ parameters.scenario }} - RUN_TESTS_IN_DOCKER: true - IPA_VERBOSITY: "-vvv" - - - task: PublishTestResults@2 - inputs: - mergeTestResults: true - testRunTitle: PlaybookTests-Build${{ parameters.build_number }} - condition: succeededOrFailed() diff --git a/infra/azure/templates/galaxy_script.yml b/infra/azure/templates/galaxy_script.yml deleted file mode 100644 index a00dd0a55f..0000000000 --- a/infra/azure/templates/galaxy_script.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- -parameters: - - name: group_number - type: number - default: 1 - - name: number_of_groups - type: number - default: 1 - - name: scenario - type: string - default: fedora-latest - - name: ansible_version - type: string - default: "" - - name: python_version - type: string - default: 3.x - - name: build_number - type: string - - -jobs: -- job: Test_Group${{ parameters.group_number }} - displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }}) - timeoutInMinutes: 240 - variables: - - template: variables.yaml - - template: variables_${{ parameters.scenario }}.yaml - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '${{ parameters.python_version }}' - - - script: | - pip install \ - "molecule-plugins[docker]" \ - "requests<2.29" \ - "ansible${{ parameters.ansible_version }}" - retryCountOnTaskFailure: 5 - displayName: Install molecule and Ansible - - - script: ansible-galaxy collection install community.docker ansible.posix - retryCountOnTaskFailure: 5 - displayName: Install Ansible collections - - - script: pip install -r requirements-tests.txt - retryCountOnTaskFailure: 5 - displayName: Install dependencies - - - script: | - utils/build-galaxy-release.sh -i - molecule create -s ${{ parameters.scenario }} - retryCountOnTaskFailure: 5 - displayName: Setup test container - env: - ANSIBLE_LIBRARY: ./molecule - - - script: | - python utils/check_test_configuration.py ${{ parameters.scenario }} - displayName: Check scenario test configuration - - - script: | - cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa - pytest \ - -m "playbook" \ - --verbose \ - --color=yes \ - --splits=${{ parameters.number_of_groups }} \ - --group=${{ parameters.group_number }} \ - --randomly-seed=$(date "+%Y%m%d") \ - --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml - displayName: Run playbook tests - env: - IPA_SERVER_HOST: ${{ parameters.scenario }} - RUN_TESTS_IN_DOCKER: true - IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} - IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} - IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }} - IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }} - IPA_VERBOSITY: "-vvv" - - - task: PublishTestResults@2 - inputs: - mergeTestResults: true - testRunTitle: PlaybookTests-Build${{ parameters.build_number }} - condition: succeededOrFailed() diff --git a/infra/azure/templates/galaxy_tests.yml b/infra/azure/templates/galaxy_tests.yml deleted file mode 100644 index 465097de2e..0000000000 --- a/infra/azure/templates/galaxy_tests.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -parameters: - - name: scenario - type: string - default: fedora-latest - - name: build_number - type: string - - name: ansible_version - type: string - default: "" - -jobs: -- template: galaxy_script.yml - parameters: - group_number: 1 - number_of_groups: 3 - build_number: ${{ parameters.build_number }} - scenario: ${{ parameters.scenario }} - ansible_version: ${{ parameters.ansible_version }} - python_version: '< 3.12' - -- template: galaxy_script.yml - parameters: - group_number: 2 - number_of_groups: 3 - build_number: ${{ parameters.build_number }} - scenario: ${{ parameters.scenario }} - ansible_version: ${{ parameters.ansible_version }} - python_version: '< 3.12' - -- template: galaxy_script.yml - parameters: - group_number: 3 - number_of_groups: 3 - build_number: ${{ parameters.build_number }} - scenario: ${{ parameters.scenario }} - ansible_version: ${{ parameters.ansible_version }} - python_version: '< 3.12' - -# Temporarily disable due to issues with ansible docker plugin. -#- template: galaxy_pytest_script.yml -# parameters: -# build_number: ${{ parameters.build_number }} -# scenario: ${{ parameters.scenario }} -# ansible_version: ${{ parameters.ansible_version }} -# python_version: '< 3.12' diff --git a/infra/azure/templates/group_tests.yml b/infra/azure/templates/group_tests.yml index 38f183dd97..da18c2e3fb 100644 --- a/infra/azure/templates/group_tests.yml +++ b/infra/azure/templates/group_tests.yml @@ -1,6 +1,6 @@ --- parameters: - - name: scenario + - name: distro type: string default: fedora-latest - name: build_number @@ -8,39 +8,23 @@ parameters: - name: ansible_version type: string default: "" + - name: skip_git_test + type: boolean + default: false + - name: test_galaxy + type: boolean + default: false jobs: -- template: playbook_tests.yml - parameters: - group_number: 1 - number_of_groups: 3 - build_number: ${{ parameters.build_number }} - scenario: ${{ parameters.scenario }} - ansible_version: ${{ parameters.ansible_version }} - python_version: '< 3.12' -- template: playbook_tests.yml - parameters: - group_number: 2 - number_of_groups: 3 - build_number: ${{ parameters.build_number }} - scenario: ${{ parameters.scenario }} - ansible_version: ${{ parameters.ansible_version }} - python_version: '< 3.12' - -- template: playbook_tests.yml - parameters: - group_number: 3 - number_of_groups: 3 - build_number: ${{ parameters.build_number }} - scenario: ${{ parameters.scenario }} - ansible_version: ${{ parameters.ansible_version }} - python_version: '< 3.12' - -# Temporarily disabled due to ansible docker plugin issue. -#- template: pytest_tests.yml -# parameters: -# build_number: ${{ parameters.build_number }} -# scenario: ${{ parameters.scenario }} -# ansible_version: ${{ parameters.ansible_version }} -# python_version: '< 3.12' +- ${{ each group in split('1,2,3', ',') }}: + - template: run_tests.yml + parameters: + group_number: ${{ group }} + number_of_groups: 3 + build_number: ${{ parameters.build_number }} + distro: ${{ parameters.distro }} + ansible_version: ${{ parameters.ansible_version }} + python_version: '< 3.12' + skip_git_test: ${{ parameters.skip_git_test }} + test_galaxy: ${{ parameters.test_galaxy }} diff --git a/infra/azure/templates/playbook_fast.yml b/infra/azure/templates/playbook_fast.yml deleted file mode 100644 index a00a7875dc..0000000000 --- a/infra/azure/templates/playbook_fast.yml +++ /dev/null @@ -1,84 +0,0 @@ ---- -parameters: - - name: group_number - type: number - default: 1 - - name: number_of_groups - type: number - default: 1 - - name: distro - type: string - default: fedora-latest - - name: ansible_version - type: string - default: "" - - name: python_version - type: string - default: 3.x - - name: build_number - type: string - - name: target_python - type: string - default: "/usr/bin/python3" - -jobs: -- job: Test_Group${{ parameters.group_number }} - displayName: Run playbook tests ${{ parameters.distro }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }}) - timeoutInMinutes: 360 - variables: - - template: variables.yaml - - template: variables_${{ parameters.distro }}.yaml - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '${{ parameters.python_version }}' - - - script: | - pip install "ansible${{ parameters.ansible_version }}" - retryCountOnTaskFailure: 5 - displayName: Install Ansible - - - script: ansible-galaxy collection install containers.podman - retryCountOnTaskFailure: 5 - displayName: Install Ansible collections - - - script: pip install -r requirements-tests.txt - retryCountOnTaskFailure: 5 - displayName: Install dependencies - - - script: | - . utils/set_test_modules - python3 utils/check_test_configuration.py ${{ parameters.distro }} - displayName: Check test configuration - - - script: | - utils/setup_test_container.sh \ - -e podman \ - -a \ - -m 4 \ - -n "ipaserver.test.local" \ - -p ${{ parameters.target_python }} \ - -i ${{ parameters.distro }}-server \ - ${{ parameters.distro }}-test - displayName: Setup target container - - - script: | - . utils/set_test_modules - pytest -m "playbook" --verbose --color=yes --suppress-no-test-exit-code --junit-xml=TEST-results-pr-check.xml - displayName: Run playbook tests - env: - ANSIBLE_ROLES_PATH: "${PWD}/roles" - ANSIBLE_LIBRARY: "${PWD}/plugins" - ANSIBLE_MODULE_UTILS: "${PWD}/plugins/module_utils" - IPA_SERVER_HOST: ${{ parameters.distro }}-test - RUN_TESTS_IN_DOCKER: podman - IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} - IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} - IPA_VERBOSITY: "-vvv" - IPA_PYTHON_PATH: ${{ parameters.target_python }} - - - task: PublishTestResults@2 - inputs: - mergeTestResults: true - testRunTitle: PlaybookTests-Build${{ parameters.build_number }} - condition: succeededOrFailed() diff --git a/infra/azure/templates/playbook_tests.yml b/infra/azure/templates/playbook_tests.yml deleted file mode 100644 index c65c955fb1..0000000000 --- a/infra/azure/templates/playbook_tests.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -parameters: - - name: group_number - type: number - default: 1 - - name: number_of_groups - type: number - default: 1 - - name: scenario - type: string - default: fedora-latest - - name: ansible_version - type: string - default: "" - - name: python_version - type: string - default: 3.x - - name: build_number - type: string - -jobs: -- job: Test_Group${{ parameters.group_number }} - displayName: Run playbook tests ${{ parameters.scenario }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }}) - timeoutInMinutes: 240 - variables: - - template: variables.yaml - - template: variables_${{ parameters.scenario }}.yaml - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '${{ parameters.python_version }}' - - - script: | - pip install \ - "molecule-plugins[docker]" \ - "requests<2.29" \ - "ansible${{ parameters.ansible_version }}" - retryCountOnTaskFailure: 5 - displayName: Install molecule and Ansible - - - script: ansible-galaxy collection install community.docker ansible.posix - retryCountOnTaskFailure: 5 - displayName: Install Ansible collections - - - script: pip install -r requirements-tests.txt - retryCountOnTaskFailure: 5 - displayName: Install dependencies - - - script: | - rm -rf ~/ansible - mkdir -p ~/.ansible/roles ~/.ansible/library ~/.ansible/module_utils - cp -a roles/* ~/.ansible/roles - cp -a plugins/modules/* ~/.ansible/library - cp -a plugins/module_utils/* ~/.ansible/module_utils - molecule create -s ${{ parameters.scenario }} - retryCountOnTaskFailure: 5 - displayName: Setup test container - env: - ANSIBLE_LIBRARY: ./molecule - - - script: | - python utils/check_test_configuration.py ${{ parameters.scenario }} - displayName: Check scenario test configuration - - - script: | - pytest \ - -m "playbook" \ - --verbose \ - --color=yes \ - --splits=${{ parameters.number_of_groups }} \ - --group=${{ parameters.group_number }} \ - --randomly-seed=$(date "+%Y%m%d") \ - --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml - displayName: Run playbook tests - env: - IPA_SERVER_HOST: ${{ parameters.scenario }} - RUN_TESTS_IN_DOCKER: true - IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} - IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} - IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }} - IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }} - IPA_VERBOSITY: "-vvv" - - - task: PublishTestResults@2 - inputs: - mergeTestResults: true - testRunTitle: PlaybookTests-Build${{ parameters.build_number }} - condition: succeededOrFailed() diff --git a/infra/azure/templates/prepare_environment.yaml b/infra/azure/templates/prepare_environment.yaml new file mode 100644 index 0000000000..077d40528e --- /dev/null +++ b/infra/azure/templates/prepare_environment.yaml @@ -0,0 +1,30 @@ +--- +parameters: + - name: distro + type: string + default: fedora-latest + - name: ansible_version + type: string + default: "" + - name: python_version + type: string + default: 3.x + - name: build_number + type: string + +steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.python_version }}' + + - script: | + pip install "ansible${{ parameters.ansible_version }}" -r requirements-tests.txt + retryCountOnTaskFailure: 5 + displayName: Install test dependencies + + - script: ansible-galaxy collection install -r requirements-podman.yml + retryCountOnTaskFailure: 5 + displayName: Install Ansible collections + + - script: infra/image/start.sh ${{ parameters.distro }}-server + displayName: Setup target container for ${{ parameters.distro }} diff --git a/infra/azure/templates/pytest_tests.yml b/infra/azure/templates/pytest_tests.yml deleted file mode 100644 index a7b12110e5..0000000000 --- a/infra/azure/templates/pytest_tests.yml +++ /dev/null @@ -1,75 +0,0 @@ ---- -parameters: - - name: build_number - type: string - - name: scenario - type: string - default: fedora-latest - - name: ansible_version - type: string - default: "" - - name: python_version - type: string - default: 3.x - -jobs: -- job: Test_PyTests - displayName: Run pytests on ${{ parameters.scenario }} - timeoutInMinutes: 240 - variables: - - template: variables.yaml - - template: variables_${{ parameters.scenario }}.yaml - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '${{ parameters.python_version }}' - - - script: | - pip install \ - "molecule-plugins[docker]" \ - "requests<2.29" \ - "ansible${{ parameters.ansible_version }}" - retryCountOnTaskFailure: 5 - displayName: Install molecule and Ansible - - - script: ansible-galaxy collection install community.docker ansible.posix - retryCountOnTaskFailure: 5 - displayName: Install Ansible collections - - - script: pip install -r requirements-tests.txt - retryCountOnTaskFailure: 5 - displayName: Install dependencies - - - script: | - rm -rf ~/.ansible - mkdir -p ~/.ansible/roles ~/.ansible/library ~/.ansible/module_utils - cp -a roles/* ~/.ansible/roles - cp -a plugins/modules/* ~/.ansible/library - cp -a plugins/module_utils/* ~/.ansible/module_utils - molecule create -s ${{ parameters.scenario }} - retryCountOnTaskFailure: 5 - displayName: Setup test container - env: - ANSIBLE_LIBRARY: ./molecule - - - script: | - pytest \ - -m "not playbook" \ - --verbose \ - --color=yes \ - --junit-xml=TEST-results-pytests.xml - displayName: Run tests - env: - IPA_SERVER_HOST: ${{ parameters.scenario }} - RUN_TESTS_IN_DOCKER: true - IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} - IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} - IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }} - IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }} - IPA_VERBOSITY: "-vvv" - - - task: PublishTestResults@2 - inputs: - mergeTestResults: true - testRunTitle: PlaybookTests-Build${{ parameters.build_number }} - condition: succeededOrFailed() diff --git a/infra/azure/templates/run_tests.yml b/infra/azure/templates/run_tests.yml new file mode 100644 index 0000000000..1cc00030bf --- /dev/null +++ b/infra/azure/templates/run_tests.yml @@ -0,0 +1,98 @@ +--- +parameters: + - name: group_number + type: number + default: 1 + - name: number_of_groups + type: number + default: 1 + - name: distro + type: string + default: fedora-latest + - name: ansible_version + type: string + default: "" + - name: python_version + type: string + default: 3.x + - name: build_number + type: string + - name: skip_git_test + type: boolean + default: true + - name: test_type + type: string + default: "playbook" + - name: test_galaxy + type: boolean + default: false + +jobs: +- job: Test_Group${{ parameters.group_number }} + displayName: Run playbook tests ${{ parameters.distro }} (${{ parameters.group_number }}/${{ parameters.number_of_groups }}) + timeoutInMinutes: 360 + variables: + - template: variables.yaml + - template: variables_${{ parameters.distro }}.yaml + steps: + - template: prepare_environment.yaml + parameters: + build_number: ${{ parameters.build_number }} + distro: ${{ parameters.distro }} + ansible_version: ${{ parameters.ansible_version }} + python_version: ${{ parameters.python_version }} + + - bash: echo "##vso[task.setvariable variable=TOPDIR]${PWD}" + displayName: Set repo rootdir + + - script: | + . "${TOPDIR}/utils/set_test_modules" + python3 utils/check_test_configuration.py ${{ parameters.distro }} + displayName: Check test configuration + env: + SKIP_GIT_TEST: ${{ parameters.skip_git_test }} + + - script: | + git fetch --unshallow + utils/build-galaxy-release.sh -i + retryCountOnTaskFailure: 5 + displayName: Build Galaxy release + condition: ${{ parameters.test_galaxy }} + + - script: | + echo "PWD: ${PWD}" + echo "TOPDIR: ${TOPDIR}" + echo "ROLES: ${ANSIBLE_ROLES_PATH}" + echo "LIBRARY: ${ANSIBLE_LIBRARY}" + echo "MODULE_UTILS: ${ANSIBLE_MODULE_UTILS}" + . "${TOPDIR}/utils/set_test_modules" + [ "${{ parameters.test_galaxy }}" == "True" ] && cd ~/.ansible/collections/ansible_collections/freeipa/ansible_freeipa + pytest \ + -m "${{ parameters.test_type }}" \ + --verbose \ + --color=yes \ + --splits=${{ parameters.number_of_groups }} \ + --group=${{ parameters.group_number }} \ + --randomly-seed=$(date "+%Y%m%d") \ + --suppress-no-test-exit-code \ + --junit-xml=TEST-results-pr-check.xml + displayName: Run playbook tests + env: + SKIP_GIT_TEST: ${{ parameters.skip_git_test }} + ${{ if not(parameters.test_galaxy) }}: + ANSIBLE_ROLES_PATH: "${PWD}/roles" + ANSIBLE_LIBRARY: "${PWD}/plugins" + ANSIBLE_MODULE_UTILS: "${PWD}/plugins/module_utils" + IPA_SERVER_HOST: ansible-freeipa-tests + RUN_TESTS_IN_DOCKER: podman + IPA_DISABLED_MODULES: ${{ variables.ipa_disabled_modules }} + IPA_DISABLED_TESTS: ${{ variables.ipa_disabled_tests }} + IPA_ENABLED_MODULES: ${{ variables.ipa_enabled_modules }} + IPA_ENABLED_TESTS: ${{ variables.ipa_enabled_tests }} + IPA_VERBOSITY: "-vvv" + + - task: PublishTestResults@2 + inputs: + mergeTestResults: true + testRunTitle: PlaybookTests-Build${{ parameters.build_number }} + condition: succeededOrFailed() diff --git a/infra/image/build.sh b/infra/image/build.sh index 3220ecce31..8a2491c881 100755 --- a/infra/image/build.sh +++ b/infra/image/build.sh @@ -37,8 +37,7 @@ EOF name="ansible-freeipa-image-builder" hostname="ipaserver.test.local" -# Number of cpus is not available in usptream CI (Ubuntu 22.04). -# cpus="2" +cpus="2" memory="3g" quayname="quay.io/ansible-freeipa/upstream-tests" deploy_server="N" From e1786c9ddcd9e773362047a4bd2d0b6398740cfb Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Wed, 23 Oct 2024 10:39:47 -0300 Subject: [PATCH 6/8] upstream CI: Enable creation of CentOS 10 Stream images --- infra/azure/build-containers.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/infra/azure/build-containers.yml b/infra/azure/build-containers.yml index d4b6b25abd..4b2381024e 100644 --- a/infra/azure/build-containers.yml +++ b/infra/azure/build-containers.yml @@ -33,6 +33,14 @@ stages: job_name_suffix: C9S distro: c9s +- stage: CentOS_10_Stream + dependsOn: [] + jobs: + - template: templates/build_container.yml + parameters: + job_name_suffix: C10S + distro: c10s + - stage: Fedora_Latest dependsOn: [] jobs: From a707d1887dd9d629e17b7c232bc3dc72ac550a9b Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Fri, 25 Oct 2024 14:34:26 -0300 Subject: [PATCH 7/8] upstream CI: Use Ubuntu 24.04 to build test images --- infra/azure/build-containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/azure/build-containers.yml b/infra/azure/build-containers.yml index 4b2381024e..a98b2e1100 100644 --- a/infra/azure/build-containers.yml +++ b/infra/azure/build-containers.yml @@ -11,7 +11,7 @@ schedules: trigger: none pool: - vmImage: 'ubuntu-22.04' + vmImage: 'ubuntu-24.04' stages: From 0f530df092f93744c93cd976aa1d85aec998091e Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Sat, 26 Oct 2024 12:20:22 -0300 Subject: [PATCH 8/8] upstream CI: Update Ansible version ansible-core versions 2.15 and 2.16 and used for all pipelines, but version 2.17 is not used for CentOS 8 Stream, as platform python on this version is 3.6 which is not supported in this ansible-core version. --- infra/azure/azure-pipelines.yml | 10 +-- infra/azure/nightly.yml | 126 ++++++-------------------------- 2 files changed, 27 insertions(+), 109 deletions(-) diff --git a/infra/azure/azure-pipelines.yml b/infra/azure/azure-pipelines.yml index 2043940a68..7456b62eb4 100644 --- a/infra/azure/azure-pipelines.yml +++ b/infra/azure/azure-pipelines.yml @@ -28,7 +28,7 @@ stages: parameters: build_number: $(Build.BuildNumber) distro: fedora-latest - ansible_version: "-core <2.14" + ansible_version: "-core <2.16" skip_git_test: true # Fedora @@ -40,8 +40,8 @@ stages: parameters: build_number: $(Build.BuildNumber) distro: fedora-latest - ansible_version: "-core >=2.14,<2.15" skip_git_test: true + ansible_version: "-core >=2.16,<2.17" # Galaxy on Fedora @@ -52,7 +52,7 @@ stages: parameters: build_number: $(Build.BuildNumber) distro: fedora-latest - ansible_version: "-core >=2.14,<2.15" + ansible_version: "-core >=2.16,<2.17" skip_git_test: true test_galaxy: true @@ -65,7 +65,7 @@ stages: parameters: build_number: $(Build.BuildNumber) distro: c9s - ansible_version: "-core >=2.14,<2.15" + ansible_version: "-core >=2.16,<2.17" skip_git_test: true # CentOS 8 Stream @@ -77,5 +77,5 @@ stages: parameters: build_number: $(Build.BuildNumber) distro: c8s - ansible_version: "-core >=2.14,<2.15" + ansible_version: "-core <2.17" skip_git_test: true diff --git a/infra/azure/nightly.yml b/infra/azure/nightly.yml index cbbc988972..671571a04a 100644 --- a/infra/azure/nightly.yml +++ b/infra/azure/nightly.yml @@ -16,27 +16,7 @@ stages: # Fedora -- stage: FedoraLatest_Ansible_Core_2_13 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-latest - ansible_version: "-core >=2.13,<2.14" - skip_git_test: false - -- stage: FedoraLatest_Ansible_Core_2_14 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-latest - ansible_version: "-core >=2.14,<2.15" - skip_git_test: false - -- stage: FedoraLatest_Ansible_Core_2_15 +- stage: fedora_latest_Ansible_Core_2_15 dependsOn: [] jobs: - template: templates/group_tests.yml @@ -46,7 +26,7 @@ stages: ansible_version: "-core >=2.15,<2.16" skip_git_test: true -- stage: FedoraLatest_Ansible_latest +- stage: fedora_latest_Ansible_Core_2_16 dependsOn: [] jobs: - template: templates/group_tests.yml @@ -56,105 +36,43 @@ stages: ansible_version: "" skip_git_test: true -# Galaxy on Fedora - -- stage: Galaxy_FedoraLatest_Ansible_Core_2_13 +- stage: fedora_latest_Ansible_Core_2_17 dependsOn: [] jobs: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) distro: fedora-latest - ansible_version: "-core >=2.13,<2.14" - skip_git_test: false - test_galaxy: true + ansible_version: "-core >=2.17" + skip_git_test: true -- stage: Galaxy_FedoraLatest_Ansible_Core_2_14 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-latest - ansible_version: "-core >=2.14,<2.15" - skip_git_test: false - test_galaxy: true +# Galaxy on Fedora -- stage: Galaxy_FedoraLatest_Ansible_Core_2_15 +- stage: Galaxy_fedora_latest_Ansible_Core_2_17 dependsOn: [] jobs: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) distro: fedora-latest - ansible_version: "-core >=2.15,<2.16" + ansible_version: "-core >=2.17,<2.18" skip_git_test: true test_galaxy: true # Fedora Rawhide -- stage: FedoraRawhide_Ansible_Core_2_13 +- stage: fedora_rawhide_Ansible_Core_2_17 dependsOn: [] jobs: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) distro: fedora-rawhide - ansible_version: "-core >=2.13,<2.14" - skip_git_test: false - -- stage: FedoraRawhide_Ansible_Core_2_14 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-rawhide - ansible_version: "-core >=2.14,<2.15" - skip_git_test: false - -- stage: FedoraRawhide_Ansible_Core_2_15 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-rawhide - ansible_version: "-core >=2.15,<2.16" - skip_git_test: true - -- stage: FedoraRawhide_Ansible_latest - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: fedora-rawhide - ansible_version: "" + ansible_version: "-core >=2.17,<2.18" skip_git_test: true # CentoOS 9 Stream -- stage: c9s_Ansible_Core_2_13 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c9s - ansible_version: "-core >=2.13,<2.14" - skip_git_test: false - -- stage: c9s_Ansible_Core_2_14 - dependsOn: [] - jobs: - - template: templates/group_tests.yml - parameters: - build_number: $(Build.BuildNumber) - distro: c9s - ansible_version: "-core >=2.14,<2.15" - skip_git_test: false - - stage: c9s_Ansible_Core_2_15 dependsOn: [] jobs: @@ -165,44 +83,44 @@ stages: ansible_version: "-core >=2.15,<2.16" skip_git_test: true -- stage: c9s_Ansible_latest +- stage: c9s_Ansible_Core_2_16 dependsOn: [] jobs: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) distro: c9s - ansible_version: "" + ansible_version: "-core >=2.16,<2.17" skip_git_test: true -# CentOS 8 Stream - -- stage: c8s_Ansible_Core_2_13 +- stage: c9s_Ansible_Core_2_17 dependsOn: [] jobs: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) - distro: c8s - ansible_version: "-core >=2.13,<2.14" + distro: c9s + ansible_version: "-core >=2.17,<2.18" skip_git_test: true -- stage: c8s_Ansible_Core_2_14 +# CentOS 8 Stream only works up to ansible-core 2.16.z + +- stage: c8s_Ansible_Core_2_15 dependsOn: [] jobs: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) distro: c8s - ansible_version: "-core >=2.14,<2.15" - skip_git_test: false + ansible_version: "-core >=2.15,<2.16" + skip_git_test: true -- stage: c8s_Ansible_Core_2_15 +- stage: c8s_Ansible_Core_2_16 dependsOn: [] jobs: - template: templates/group_tests.yml parameters: build_number: $(Build.BuildNumber) distro: c8s - ansible_version: "-core >=2.15,<2.16" + ansible_version: "-core >=2.16,<2.17" skip_git_test: true