From c71a2b33dd4f9897046232d876b5929955558f0a Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Fri, 29 Sep 2023 12:51:59 -0300 Subject: [PATCH 1/3] upstream CI: Fix test selection for CheckPR pipeline. Due to an error on processing Ansible key 'import_tasks' the script that creates a list of modules to test is broken making some modules to be not tested. By fixing the handling of 'import_tasks' and module import, the list is correct again and the list of modules to be tested now include the ones which depend on the modified module. --- utils/get_test_modules.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/utils/get_test_modules.py b/utils/get_test_modules.py index 2f83176e34..3e9306b0dd 100644 --- a/utils/get_test_modules.py +++ b/utils/get_test_modules.py @@ -23,8 +23,6 @@ def get_tasks(task_block): for tasks in task_block: for task in tasks: original_task = task - if "." in task: - task = task.split(".")[-1] if task == "block": _result.update(get_tasks(tasks["block"])) elif task in ["include_tasks", "import_tasks" @@ -127,8 +125,16 @@ def parse_playbooks(test_module): "builtins.__import__", side_effect=import_mock ): # pylint: disable=no-value-for-parameter - loader = SourceFileLoader(playbook, source) - loader.exec_module(types.ModuleType(loader.name)) + try: + loader = SourceFileLoader(playbook, source) + loader.exec_module( + types.ModuleType(loader.name) + ) + except Exception: # pylint: disable=broad-except + # If import fails, we'll assume there's no + # plugin to be loaded. This is of little risk + # it is rare that a plugin includes another. + pass # pylint: disable=no-member candidates = [ f.split(".")[1:] From 319a0d3d869c6829136ec9bf99a028bfeebf38c1 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Thu, 5 Oct 2023 14:36:19 -0300 Subject: [PATCH 2/3] upstream ci: Use a single random seed for spliting tests Dependind on how long it took for the jobs to start, a different seed would be used to group tests and tests could either repeat or not be selected at all. By using a seed based on the day the test run reduces the chance of using different random seeds, and still allow for the tests to be executed in a different order. The execution in different order is important to identify tests that work or fail only if executed after other tests. --- tests/azure/templates/galaxy_script.yml | 1 + tests/azure/templates/playbook_fast.yml | 1 + tests/azure/templates/playbook_tests.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/azure/templates/galaxy_script.yml b/tests/azure/templates/galaxy_script.yml index cd95c65815..a00dd0a55f 100644 --- a/tests/azure/templates/galaxy_script.yml +++ b/tests/azure/templates/galaxy_script.yml @@ -67,6 +67,7 @@ jobs: --color=yes \ --splits=${{ parameters.number_of_groups }} \ --group=${{ parameters.group_number }} \ + --randomly-seed=$(date "+%Y%m%d") \ --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml displayName: Run playbook tests env: diff --git a/tests/azure/templates/playbook_fast.yml b/tests/azure/templates/playbook_fast.yml index ce187a3ccb..2629b3e5b7 100644 --- a/tests/azure/templates/playbook_fast.yml +++ b/tests/azure/templates/playbook_fast.yml @@ -72,6 +72,7 @@ jobs: --suppress-no-test-exit-code \ --splits=${{ parameters.number_of_groups }} \ --group=${{ parameters.group_number }} \ + --randomly-seed=$(date "+%Y%m%d") \ --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml then [ $? -eq 5 ] && true || false diff --git a/tests/azure/templates/playbook_tests.yml b/tests/azure/templates/playbook_tests.yml index ce3200026b..c65c955fb1 100644 --- a/tests/azure/templates/playbook_tests.yml +++ b/tests/azure/templates/playbook_tests.yml @@ -69,6 +69,7 @@ jobs: --color=yes \ --splits=${{ parameters.number_of_groups }} \ --group=${{ parameters.group_number }} \ + --randomly-seed=$(date "+%Y%m%d") \ --junit-xml=TEST-results-group-${{ parameters.group_number }}.xml displayName: Run playbook tests env: From fe2d17e4dffbdaf558d6e5eb6e564945372b86d1 Mon Sep 17 00:00:00 2001 From: Rafael Guterres Jeffman Date: Thu, 5 Oct 2023 14:43:47 -0300 Subject: [PATCH 3/3] upstream ci: Run PR tests using a single job. The usual scenario for PR checks is to execute only a few tests, and searching for the results in several jobs makes it harder to find issues. By using a single job run the tests would take some more time to complete, although not much, as only a small subset is executed, and test verification would be easier and less error prone. --- tests/azure/templates/fast_tests.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/tests/azure/templates/fast_tests.yml b/tests/azure/templates/fast_tests.yml index 9c6c80f3ac..fdb1ea0d13 100644 --- a/tests/azure/templates/fast_tests.yml +++ b/tests/azure/templates/fast_tests.yml @@ -13,30 +13,12 @@ jobs: - template: playbook_fast.yml parameters: group_number: 1 - number_of_groups: 3 + number_of_groups: 1 build_number: ${{ parameters.build_number }} scenario: ${{ parameters.scenario }} ansible_version: ${{ parameters.ansible_version }} python_version: '< 3.12' -- template: playbook_fast.yml - parameters: - group_number: 2 - number_of_groups: 3 - build_number: ${{ parameters.build_number }} - scenario: ${{ parameters.scenario }} - ansible_version: ${{ parameters.ansible_version }} - python_version: '< 3.12' - -- template: playbook_fast.yml - parameters: - group_number: 3 - number_of_groups: 3 - build_number: ${{ parameters.build_number }} - scenario: ${{ parameters.scenario }} - ansible_version: ${{ parameters.ansible_version }}z - python_version: '< 3.12' - # - template: pytest_tests.yml # parameters: # build_number: ${{ parameters.build_number }}