Merge pull request #606 from DataDog/ncreated/RUMM-1625-do-not-run-ni…

…ghtly-tests-below-supported-OS-versions

tools/nightly-unit-tests/generate_bitrise_yml.py

@@ -23,6 +23,36 @@
 from src.semver import Version
 
 
+def get_environment() -> (Simulators, Runtimes, Devices):
+    """
+    Uses `xcversion simulators` and `xcrun simctl` CLIs to load available
+    simulators, runtimes and devices.
+    """
+    simulators = Simulators(
+        xcversion_simulators_output=shell_output('xcversion simulators')
+    )
+    runtimes = Runtimes(
+        xcrun_simctl_list_runtimes_json_output=shell_output('xcrun simctl list runtimes --json')
+    )
+    devices = Devices(
+        runtimes=runtimes,
+        xcrun_simctl_list_devices_json_output=shell_output('xcrun simctl list devices --json')
+    )
+    return simulators, runtimes, devices
+
+
+def dump_environment(simulators: Simulators, runtimes: Runtimes, devices: Devices):
+    """
+    Prints log listing all available simulators, runtimes and devices.
+    """
+    print('\n⚙️ All simulators:')
+    print_simulators(simulators=simulators.all)
+    print('\n⚙️ All runtimes:')
+    print_runtimes(runtimes=runtimes.all)
+    print('\n⚙️ All devices:')
+    print_devices(devices=devices.all)
+
+
 def generate_bitrise_yml(test_plan: TestPlan, dry_run: bool):
     """
     Generates `bitrise.yml` file for given Test Plan.
@@ -55,22 +85,8 @@ def generate_bitrise_yml(test_plan: TestPlan, dry_run: bool):
             print(f' → installed {simulator.os_name} {simulator.os_version} Simulator, in: {minutes_elapsed}')
 
     # After installing new simulators, load list of available devices:
-    simulators = Simulators(
-        xcversion_simulators_output=shell_output('xcversion simulators')
-    )
-    runtimes = Runtimes(
-        xcrun_simctl_list_runtimes_json_output=shell_output('xcrun simctl list runtimes --json')
-    )
-    devices = Devices(
-        runtimes=runtimes,
-        xcrun_simctl_list_devices_json_output=shell_output('xcrun simctl list devices --json')
-    )
-    print('\n⚙️ App simulators:')
-    print_simulators(simulators=simulators.all)
-    print('\n⚙️ All runtimes:')
-    print_runtimes(runtimes=runtimes.all)
-    print('\n⚙️ All devices:')
-    print_devices(devices=devices.all)
+    simulators, runtimes, devices = get_environment()
+    dump_environment(simulators=simulators, runtimes=runtimes, devices=devices)
 
     # Generate `bitrise.yml`
     print('\n⚙️ Creating `bitrise.yml`:')
@@ -189,6 +205,9 @@ def create_test_plan(os_name: str, os_versions: [Version]) -> TestPlan:
         print('-' * 60)
         traceback.print_exc(file=sys.stdout)
         print('-' * 60)
+        print('Environment dump:')
+        simulators, runtimes, devices = get_environment()
+        dump_environment(simulators=simulators, runtimes=runtimes, devices=devices)
         sys.exit(1)
 
     sys.exit(0)

tools/nightly-unit-tests/src/semver.py

@@ -44,3 +44,6 @@ def is_newer_than(self, other_version: 'Version'):
                     return True
 
         return False
+
+    def is_newer_than_or_equal(self, other_version: 'Version'):
+        return self.is_newer_than(other_version=other_version) or self == other_version

tools/nightly-unit-tests/src/test_plan.py

@@ -6,6 +6,7 @@
 
 import random
 from src.simulators_parser import Simulator
+from src.semver import Version
 
 # An estimated duration of installing simulator on Bitrise.
 # It includes ~3-5min margin over measured duration.
@@ -19,6 +20,9 @@
 # It includes ~10min margin for set-up and tear-down jobs.
 BITRISE_TIMEOUT_IN_MINUTES = 80
 
+# The minimal supported iOS version for running unit tests.
+MIN_SUPPORTED_IOS_VERSION = Version.parse('11.0.0')
+
 
 class TestPlanStep:
     def __init__(self, simulator: Simulator):
@@ -62,6 +66,8 @@ def create_randomized_plan(simulators: [Simulator]):
         :return: a `TestPlan` object
         """
         possible_steps = list(map(lambda s: TestPlanStep(simulator=s), simulators))
+        possible_steps = list(filter(lambda step: is_using_supported_ios_version(step), possible_steps))
+
         planned_steps: [TestPlanStep] = []
 
         random.shuffle(possible_steps)
@@ -90,3 +96,7 @@ def total_duration_in_minutes(steps: [TestPlanStep]):
     for step in steps:
         total += step.estimated_duration_in_minutes
     return total
+
+
+def is_using_supported_ios_version(step: TestPlanStep) -> bool:
+    return step.simulator.os_version.is_newer_than_or_equal(MIN_SUPPORTED_IOS_VERSION)

tools/nightly-unit-tests/tests/test_semver.py

@@ -0,0 +1,26 @@
+# -----------------------------------------------------------
+# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019-2020 Datadog, Inc.
+# -----------------------------------------------------------
+
+import unittest
+from src.semver import Version
+
+
+class VersionTestCase(unittest.TestCase):
+    def test_parsing(self):
+        self.assertEqual(Version.parse('10.0.3'), Version(major=10, minor=0, patch=3))
+        self.assertEqual(Version.parse('11.4'), Version(major=11, minor=4, patch=0))
+        self.assertEqual(Version.parse('12'), Version(major=12, minor=0, patch=0))
+
+    def test_comparing(self):
+        self.assertTrue(Version.parse('14.0.0').is_newer_than(Version.parse('13.1.2')))
+        self.assertTrue(Version.parse('14.1.1').is_newer_than(Version.parse('14.1.0')))
+        self.assertTrue(Version.parse('14.2.3').is_newer_than(Version.parse('14.2.2')))
+        self.assertFalse(Version.parse('14.0.3').is_newer_than(Version.parse('15.0.2')))
+        self.assertFalse(Version.parse('14.0.3').is_newer_than(Version.parse('14.1.0')))
+        self.assertFalse(Version.parse('14.0.3').is_newer_than(Version.parse('14.0.4')))
+        self.assertFalse(Version.parse('14.0.3').is_newer_than(Version.parse('14.0.3')))
+        self.assertTrue(Version.parse('14.0.3').is_newer_than_or_equal(Version.parse('14.0.3')))
+        self.assertFalse(Version.parse('14.0.2').is_newer_than_or_equal(Version.parse('14.0.3')))