add comparisons keyword for 'assert_contains' and 'assert_not_contain…

…s' property which is used to specify list of assertions. the 'mode' key is added to allow status check to be logical OR/AND. update buildspec file. update json schema definition
buildtesters · Oct 30, 2023 · c5f095e · c5f095e
1 parent 56392e9
commit c5f095e
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 46 deletions.
diff --git a/buildtest/buildsystem/checks.py b/buildtest/buildsystem/checks.py
@@ -774,7 +774,7 @@ def contains_check(builder):
     metric_names = list(builder.metadata["metrics"].keys())
 
     # iterate over each metric in buildspec and determine reference check for each metric
-    for metric in builder.status["contains"]:
+    for metric in builder.status["contains"]["comparisons"]:
         name = metric["name"]
         ref_value = metric["ref"]
 
@@ -810,9 +810,11 @@ def contains_check(builder):
         console.print(
             f"[blue]{builder}[/]: testing metric: [red]{name}[/red] if [yellow]{conv_value}[/yellow] in [yellow]{ref_value}[/yellow] - Check: {bool_check}"
         )
-
-    # perform a logical AND on the list and return the boolean result
-    bool_check = all(assert_check)
+    # perform logical OR if mode is set to 'or' or 'OR' otherwise do logical AND
+    if builder.status["contains"].get("mode") in ["or", "OR"]:
+        bool_check = any(assert_check)
+    else:
+        bool_check = all(assert_check)
 
     console.print(f"[blue]{builder}[/]: Contains Check: {bool_check}")
     return bool_check
@@ -836,7 +838,7 @@ def notcontains_check(builder):
     metric_names = list(builder.metadata["metrics"].keys())
 
     # iterate over each metric in buildspec and determine reference check for each metric
-    for metric in builder.status["not_contains"]:
+    for metric in builder.status["not_contains"]["comparisons"]:
         name = metric["name"]
         ref_value = metric["ref"]
 
@@ -873,8 +875,10 @@ def notcontains_check(builder):
             f"[blue]{builder}[/]: testing metric: [red]{name}[/red] if [yellow]{conv_value}[/yellow] not in [yellow]{ref_value}[/yellow] - Check: {bool_check}"
         )
 
-    # perform a logical AND on the list and return the boolean result
-    bool_check = all(assert_check)
+    if builder.status["not_contains"].get("mode") in ["or", "OR"]:
+        bool_check = any(assert_check)
+    else:
+        bool_check = all(assert_check)
 
     console.print(f"[blue]{builder}[/]: Not Contains Check: {bool_check}")
     return bool_check

diff --git a/buildtest/schemas/definitions.schema.json b/buildtest/schemas/definitions.schema.json
@@ -353,48 +353,12 @@
           }
         },
         "contains": {
-          "description": "Check if metric value is in a list of reference values ",
-          "items": {
-            "type": "object",
-            "additionalProperties": false,
-            "required": [
-              "name",
-              "ref"
-            ],
-            "properties": {
-              "name": {
-                "type": "string",
-                "description": "Name of metric to use for comparison"
-              },
-              "ref": {
-                "type": "array",
-                "minItems": 1,
-                "description": "Specify a list of reference value"
-              }
-            }
-          }
+          "description": "Check if metric value is in a list of reference values",
+          "$ref": "#/definitions/status_check_contains_not_contains"
         },
         "not_contains": {
           "description": "Check if metric value not in a list of reference values ",
-          "items": {
-            "type": "object",
-            "additionalProperties": false,
-            "required": [
-              "name",
-              "ref"
-            ],
-            "properties": {
-              "name": {
-                "type": "string",
-                "description": "Name of metric to use for comparison"
-              },
-              "ref": {
-                "type": "array",
-                "minItems": 1,
-                "description": "Specify a list of reference value"
-              }
-            }
-          }
+          "$ref": "#/definitions/status_check_contains_not_contains"
         },
         "is_symlink": {
           "description": "Check for list of files or directory paths that are symbolic links",
@@ -544,6 +508,38 @@
           }
         }
     },
+    "status_check_contains_not_contains": {
+      "type": "object",
+      "required": ["comparisons"],
+      "additionalProperties": false,
+      "properties": {
+        "mode": {
+          "$ref": "#/definitions/mode"
+        },
+        "comparisons": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "name",
+              "ref"
+            ],
+            "properties": {
+              "name": {
+                "type": "string",
+                "description": "Name of metric to use for comparison"
+              },
+              "ref": {
+                "type": "array",
+                "minItems": 1,
+                "description": "Specify a list of reference value"
+              }
+            }
+          }
+        }
+      }
+    },
     "mode": {
       "description": "Determine how the status check is resolved, for instance it can be logical AND or OR",
       "type": "string",

diff --git a/tutorials/perf_checks/contains.yml b/tutorials/perf_checks/contains.yml
@@ -16,9 +16,11 @@ buildspecs:
           item: 1
     status:
       contains:
+        comparisons:
         - name: x
           ref: [1, 2, 4, 8]
       not_contains:
+        comparisons:
         - name: x
           ref: [2, 4]
   assert_contains_fail:
@@ -38,5 +40,6 @@ buildspecs:
           item: 1
     status:
       contains:
+        comparisons:
         - name: x
           ref: ['1', 2, 4, 8]