From 1b5a997a94feb679bfb61892c8f777bdc2b2db7d Mon Sep 17 00:00:00 2001 From: Maciek Grzybowski Date: Thu, 23 Sep 2021 12:41:26 +0200 Subject: [PATCH 1/2] RUMM-1625 Add more logs around environment status (simulator, runtimes and devices) when tool fails. --- .../generate_bitrise_yml.py | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/tools/nightly-unit-tests/generate_bitrise_yml.py b/tools/nightly-unit-tests/generate_bitrise_yml.py index b88bd2c5a2..a33cbebd20 100644 --- a/tools/nightly-unit-tests/generate_bitrise_yml.py +++ b/tools/nightly-unit-tests/generate_bitrise_yml.py @@ -23,6 +23,37 @@ from src.semver import Version +def get_environment() -> (Simulators, Runtimes, Devices): + """ + Uses `xcversion simulators` and `xcrun simctl` CLIs to load available + simulators, runtimes and devices. + """ + simulators = Simulators( + xcversion_simulators_output=shell_output('xcversion simulators') + ) + runtimes = Runtimes( + xcrun_simctl_list_runtimes_json_output=shell_output('xcrun simctl list runtimes --json') + ) + devices = Devices( + runtimes=runtimes, + xcrun_simctl_list_devices_json_output=shell_output('xcrun simctl list devices --json') + ) + return simulators, runtimes, devices + + +def dump_environment(simulators: Simulators, runtimes: Runtimes, devices: Devices): + """ + Prints log listing all available simulators, runtimes and devices. + """ + print('\n⚙️ All simulators:') + print_simulators(simulators=simulators.all) + print('\n⚙️ All runtimes:') + print_runtimes(runtimes=runtimes.all) + print('\n⚙️ All devices:') + print_devices(devices=devices.all) + + + def generate_bitrise_yml(test_plan: TestPlan, dry_run: bool): """ Generates `bitrise.yml` file for given Test Plan. @@ -55,22 +86,8 @@ def generate_bitrise_yml(test_plan: TestPlan, dry_run: bool): print(f' → installed {simulator.os_name} {simulator.os_version} Simulator, in: {minutes_elapsed}') # After installing new simulators, load list of available devices: - simulators = Simulators( - xcversion_simulators_output=shell_output('xcversion simulators') - ) - runtimes = Runtimes( - xcrun_simctl_list_runtimes_json_output=shell_output('xcrun simctl list runtimes --json') - ) - devices = Devices( - runtimes=runtimes, - xcrun_simctl_list_devices_json_output=shell_output('xcrun simctl list devices --json') - ) - print('\n⚙️ App simulators:') - print_simulators(simulators=simulators.all) - print('\n⚙️ All runtimes:') - print_runtimes(runtimes=runtimes.all) - print('\n⚙️ All devices:') - print_devices(devices=devices.all) + simulators, runtimes, devices = get_environment() + dump_environment(simulators=simulators, runtimes=runtimes, devices=devices) # Generate `bitrise.yml` print('\n⚙️ Creating `bitrise.yml`:') @@ -189,6 +206,9 @@ def create_test_plan(os_name: str, os_versions: [Version]) -> TestPlan: print('-' * 60) traceback.print_exc(file=sys.stdout) print('-' * 60) + print('Environment dump:') + simulators, runtimes, devices = get_environment() + dump_environment(simulators=simulators, runtimes=runtimes, devices=devices) sys.exit(1) sys.exit(0) From a7b65fd4f66d965d31bb3c9f4f399822ef2b559d Mon Sep 17 00:00:00 2001 From: Maciek Grzybowski Date: Thu, 23 Sep 2021 13:03:27 +0200 Subject: [PATCH 2/2] RUMM-1625 Do not plan test steps below minimal supported iOS version This is to prevent CI failures, when too old simulators can be installed on the host (CI machine). --- .../generate_bitrise_yml.py | 1 - tools/nightly-unit-tests/src/semver.py | 3 +++ tools/nightly-unit-tests/src/test_plan.py | 10 +++++++ tools/nightly-unit-tests/tests/test_semver.py | 26 +++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tools/nightly-unit-tests/tests/test_semver.py diff --git a/tools/nightly-unit-tests/generate_bitrise_yml.py b/tools/nightly-unit-tests/generate_bitrise_yml.py index a33cbebd20..a5615fa18c 100644 --- a/tools/nightly-unit-tests/generate_bitrise_yml.py +++ b/tools/nightly-unit-tests/generate_bitrise_yml.py @@ -53,7 +53,6 @@ def dump_environment(simulators: Simulators, runtimes: Runtimes, devices: Device print_devices(devices=devices.all) - def generate_bitrise_yml(test_plan: TestPlan, dry_run: bool): """ Generates `bitrise.yml` file for given Test Plan. diff --git a/tools/nightly-unit-tests/src/semver.py b/tools/nightly-unit-tests/src/semver.py index 71f803e0fa..e4a9e89b57 100644 --- a/tools/nightly-unit-tests/src/semver.py +++ b/tools/nightly-unit-tests/src/semver.py @@ -44,3 +44,6 @@ def is_newer_than(self, other_version: 'Version'): return True return False + + def is_newer_than_or_equal(self, other_version: 'Version'): + return self.is_newer_than(other_version=other_version) or self == other_version diff --git a/tools/nightly-unit-tests/src/test_plan.py b/tools/nightly-unit-tests/src/test_plan.py index 921ddfa3c1..7248201614 100644 --- a/tools/nightly-unit-tests/src/test_plan.py +++ b/tools/nightly-unit-tests/src/test_plan.py @@ -6,6 +6,7 @@ import random from src.simulators_parser import Simulator +from src.semver import Version # An estimated duration of installing simulator on Bitrise. # It includes ~3-5min margin over measured duration. @@ -19,6 +20,9 @@ # It includes ~10min margin for set-up and tear-down jobs. BITRISE_TIMEOUT_IN_MINUTES = 80 +# The minimal supported iOS version for running unit tests. +MIN_SUPPORTED_IOS_VERSION = Version.parse('11.0.0') + class TestPlanStep: def __init__(self, simulator: Simulator): @@ -62,6 +66,8 @@ def create_randomized_plan(simulators: [Simulator]): :return: a `TestPlan` object """ possible_steps = list(map(lambda s: TestPlanStep(simulator=s), simulators)) + possible_steps = list(filter(lambda step: is_using_supported_ios_version(step), possible_steps)) + planned_steps: [TestPlanStep] = [] random.shuffle(possible_steps) @@ -90,3 +96,7 @@ def total_duration_in_minutes(steps: [TestPlanStep]): for step in steps: total += step.estimated_duration_in_minutes return total + + +def is_using_supported_ios_version(step: TestPlanStep) -> bool: + return step.simulator.os_version.is_newer_than_or_equal(MIN_SUPPORTED_IOS_VERSION) diff --git a/tools/nightly-unit-tests/tests/test_semver.py b/tools/nightly-unit-tests/tests/test_semver.py new file mode 100644 index 0000000000..0689303308 --- /dev/null +++ b/tools/nightly-unit-tests/tests/test_semver.py @@ -0,0 +1,26 @@ +# ----------------------------------------------------------- +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-2020 Datadog, Inc. +# ----------------------------------------------------------- + +import unittest +from src.semver import Version + + +class VersionTestCase(unittest.TestCase): + def test_parsing(self): + self.assertEqual(Version.parse('10.0.3'), Version(major=10, minor=0, patch=3)) + self.assertEqual(Version.parse('11.4'), Version(major=11, minor=4, patch=0)) + self.assertEqual(Version.parse('12'), Version(major=12, minor=0, patch=0)) + + def test_comparing(self): + self.assertTrue(Version.parse('14.0.0').is_newer_than(Version.parse('13.1.2'))) + self.assertTrue(Version.parse('14.1.1').is_newer_than(Version.parse('14.1.0'))) + self.assertTrue(Version.parse('14.2.3').is_newer_than(Version.parse('14.2.2'))) + self.assertFalse(Version.parse('14.0.3').is_newer_than(Version.parse('15.0.2'))) + self.assertFalse(Version.parse('14.0.3').is_newer_than(Version.parse('14.1.0'))) + self.assertFalse(Version.parse('14.0.3').is_newer_than(Version.parse('14.0.4'))) + self.assertFalse(Version.parse('14.0.3').is_newer_than(Version.parse('14.0.3'))) + self.assertTrue(Version.parse('14.0.3').is_newer_than_or_equal(Version.parse('14.0.3'))) + self.assertFalse(Version.parse('14.0.2').is_newer_than_or_equal(Version.parse('14.0.3')))