From 79263d5306797a8eabd59b86c6902d0bb12f88db Mon Sep 17 00:00:00 2001 From: Mark Zhang <46978338+MarkintoshZ@users.noreply.github.com> Date: Tue, 13 Feb 2024 19:37:23 -0600 Subject: [PATCH] Add New Testcase Generators (#311) * Added generator decorator and tests for it - generator decorator supports k8s schema name, field name, field type, paths, and priority configuration * Rename PathIndex to PathSegment Signed-off-by: Tyler Gu * Code style changes Signed-off-by: Tyler Gu * Create test generator directory Signed-off-by: Tyler Gu * Migrate test generator from known schemas Signed-off-by: Tyler Gu * Add init for test_generators to make it a module Signed-off-by: Tyler Gu * Rename tests to be consistent with previous names Signed-off-by: Tyler Gu * Add attributes to TestCase class Signed-off-by: Tyler Gu * Optimize and improve generator decorator - Rename `field` to `property` in decorator - Move test generator matching code to the TestGenerator data class - Support multiple constraints for test generator matching - Tests for multiple constraints * Primitive test generators - Copied value generator code to the function decorator pattern - Moved gen() method declaration in test generator to BaseSchema - Moved gen() method implementation in [Type]Schemas * Fix import error * Custom validate_call decorator for test generators - Include unittest * Fix primitive test generators * Fix unittest * Rename generator to test_generator Signed-off-by: Tyler Gu * Skip the test_generator function for pytest Signed-off-by: Tyler Gu * Prioritize Integer test gens over Number test gens * Migrate test generator priority to int enum * Export all test generators * Add cli schema_match that annotates CRD yaml file * Add manual for test generator decorator * Integrate new test gen into input model Signed-off-by: Tyler Gu * Fix custom mapping Signed-off-by: Tyler Gu * Fix applying property attributes Signed-off-by: Tyler Gu * Fix value with schema Signed-off-by: Tyler Gu * Fix value with anyof schema Signed-off-by: Tyler Gu * Add tests for test execution Signed-off-by: Tyler Gu --------- Signed-off-by: Tyler Gu Co-authored-by: Tyler Gu --- acto/__init__.py | 1 + acto/checker/impl/consistency.py | 12 +- acto/checker/impl/tests/test_state.py | 3 +- acto/cli/__init__.py | 0 acto/cli/schema_match.py | 10 +- acto/common.py | 10 +- acto/engine.py | 65 +- acto/input/input.py | 659 ++++----------- acto/input/k8s_schemas.py | 367 +++++++-- acto/input/kubernetes_property.py | 92 +++ acto/input/property_attribute.py | 31 + acto/input/test_generators/__init__.py | 9 + acto/input/test_generators/cron_job.py | 46 ++ acto/input/test_generators/deployment.py | 27 + acto/input/test_generators/generator.py | 281 +++++++ acto/input/test_generators/pod.py | 541 ++++++++++++ acto/input/test_generators/primitive.py | 767 ++++++++++++++++++ acto/input/test_generators/resource.py | 78 ++ acto/input/test_generators/service.py | 44 + acto/input/test_generators/stateful_set.py | 84 ++ acto/input/test_generators/storage.py | 41 + acto/input/testcase.py | 93 ++- acto/input/value_with_schema.py | 272 ++++--- acto/lib/operator_config.py | 110 ++- acto/reproduce.py | 33 +- acto/schema/base.py | 2 + acto/schema/object.py | 2 +- data/cass-operator/config.json | 9 +- data/cass-operator/custom_mapping.py | 12 + data/cockroach-operator/config.json | 8 +- data/cockroach-operator/custom_mapping.py | 4 + data/knative-operator-eventing/config.json | 3 - .../custom_mapping.py | 8 + data/knative-operator-serving/config.json | 5 +- .../custom_mapping.py | 9 + data/mongodb-community-operator/config.json | 9 +- .../config.json | 8 +- .../custom_mapping.py | 18 + .../config.json | 8 +- .../custom_mapping.py | 19 + data/rabbitmq-operator/config.json | 6 +- data/rabbitmq-operator/custom_mapping.py | 20 + data/redis-operator/config.json | 5 +- .../config.json | 5 +- data/tidb-operator/config.json | 5 +- data/zookeeper-operator/config.json | 5 +- docs/test_generator.md | 138 ++++ pyproject.toml | 3 + requirements-dev.txt | 18 +- requirements.txt | 2 +- test/integration_tests/test_cassop_bugs.py | 1 - test/integration_tests/test_crdb_bugs.py | 1 - test/integration_tests/test_known_schemas.py | 442 +++++++--- test/integration_tests/test_rbop_bugs.py | 12 +- .../test_test_generator_decorator.py | 190 +++++ 55 files changed, 3652 insertions(+), 1001 deletions(-) create mode 100644 acto/cli/__init__.py create mode 100644 acto/input/kubernetes_property.py create mode 100644 acto/input/property_attribute.py create mode 100644 acto/input/test_generators/__init__.py create mode 100644 acto/input/test_generators/cron_job.py create mode 100644 acto/input/test_generators/deployment.py create mode 100644 acto/input/test_generators/generator.py create mode 100644 acto/input/test_generators/pod.py create mode 100644 acto/input/test_generators/primitive.py create mode 100644 acto/input/test_generators/resource.py create mode 100644 acto/input/test_generators/service.py create mode 100644 acto/input/test_generators/stateful_set.py create mode 100644 acto/input/test_generators/storage.py create mode 100644 data/cass-operator/custom_mapping.py create mode 100644 data/cockroach-operator/custom_mapping.py create mode 100644 data/knative-operator-eventing/custom_mapping.py create mode 100644 data/knative-operator-serving/custom_mapping.py create mode 100644 data/percona-server-mongodb-operator/custom_mapping.py create mode 100644 data/percona-xtradb-cluster-operator/custom_mapping.py create mode 100644 data/rabbitmq-operator/custom_mapping.py create mode 100644 docs/test_generator.md create mode 100644 test/integration_tests/test_test_generator_decorator.py diff --git a/acto/__init__.py b/acto/__init__.py index e69de29bb2..258ab44fc6 100644 --- a/acto/__init__.py +++ b/acto/__init__.py @@ -0,0 +1 @@ +DEFAULT_KUBERNETES_VERSION = "v1.27.0" diff --git a/acto/checker/impl/consistency.py b/acto/checker/impl/consistency.py index a55f5edfc2..3e3335063e 100644 --- a/acto/checker/impl/consistency.py +++ b/acto/checker/impl/consistency.py @@ -1,4 +1,5 @@ """State checker""" + import copy import json import re @@ -20,6 +21,7 @@ ) from acto.input import InputModel from acto.input.get_matched_schemas import find_matched_schema +from acto.input.property_attribute import PropertyAttribute from acto.k8s_util.k8sutil import canonicalize_quantity from acto.result import ConsistencyOracleResult, InvalidInputResult from acto.schema import ArraySchema, BaseSchema, ObjectSchema, extract_schema @@ -372,7 +374,8 @@ def check( ) if ( diff_type == "iterable_item_removed" - or corresponding_schema.patch + or corresponding_schema.attributes & PropertyAttribute.Patch + == PropertyAttribute.Patch or input_diff.path[-1] == "ACTOKEY" ): pass @@ -395,7 +398,12 @@ def check( ) should_compare = ( - should_compare or corresponding_schema.mapped + should_compare + or ( + corresponding_schema.attributes + & PropertyAttribute.Mapped + == PropertyAttribute.Mapped + ) ) and must_produce_delta # Policy: pass if any of the matched deltas is equivalent diff --git a/acto/checker/impl/tests/test_state.py b/acto/checker/impl/tests/test_state.py index 8ae07c56a8..fc7bcc2f39 100644 --- a/acto/checker/impl/tests/test_state.py +++ b/acto/checker/impl/tests/test_state.py @@ -44,7 +44,6 @@ def input_model_and_context_mapping() -> ( input_model = DeterministicInputModel( crd=context["crd"]["body"], seed_input=defaultdict(lambda: defaultdict(dict)), - used_fields=[], example_dir=None, num_workers=1, num_cases=1, @@ -57,6 +56,7 @@ def input_model_and_context_mapping() -> ( def checker_func(s: Snapshot, prev_s: Snapshot) -> Optional[OracleResult]: + """Run the consistency checker and return the result.""" api_version = s.input_cr["apiVersion"] checker = ConsistencyChecker( trial_dir="", @@ -107,6 +107,7 @@ def checker_func(s: Snapshot, prev_s: Snapshot) -> Optional[OracleResult]: ), ) def test_consistency_checker(test_case_id, result_dict): + """Test the consistency checker.""" snapshot = load_snapshot("state", test_case_id) snapshot_prev = load_snapshot("state", test_case_id, load_prev=True) oracle_result = checker_func(snapshot, snapshot_prev) diff --git a/acto/cli/__init__.py b/acto/cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acto/cli/schema_match.py b/acto/cli/schema_match.py index b9f83687af..49fed6bebe 100644 --- a/acto/cli/schema_match.py +++ b/acto/cli/schema_match.py @@ -5,6 +5,7 @@ from ruamel.yaml import YAML from acto.input.k8s_schemas import K8sSchemaMatcher +from acto.schema.object import ObjectSchema from acto.schema.schema import extract_schema @@ -22,7 +23,7 @@ def main(): parser.add_argument( "--k8s-version", required=False, - default="1.29", + default="v1.29.0", help="Kubernetes version to match the schema with", ) parser.add_argument( @@ -43,7 +44,7 @@ def main(): # match the schema with Kubernetes resource schemas schema_matcher = K8sSchemaMatcher.from_version(args.k8s_version) - matches = schema_matcher.find_matched_schemas(root) + matches = schema_matcher.find_all_matched_schemas(root) # output the breakdown of the matched schema information df = pd.DataFrame( @@ -53,6 +54,7 @@ def main(): "schema_path": "/".join(schema.path), } for schema, k8s_schema in matches + if k8s_schema.k8s_schema_name is not None ] ) @@ -61,6 +63,8 @@ def main(): # annotate the yaml file with the matched schema information for schema, k8s_schema in matches: + if k8s_schema.k8s_schema_name is None: + continue comment = k8s_schema.k8s_schema_name curr = schema_yaml for segment in schema.path[:-1]: @@ -68,7 +72,7 @@ def main(): curr = curr["items"] else: curr = curr["properties"][segment] - if schema.path[-1] != "ITEM": + if schema.path[-1] != "ITEM" and isinstance(schema, ObjectSchema): curr["properties"].yaml_add_eol_comment(comment, schema.path[-1]) else: curr.yaml_add_eol_comment(comment, "items") diff --git a/acto/common.py b/acto/common.py index bf21551bcd..bc0741d115 100644 --- a/acto/common.py +++ b/acto/common.py @@ -5,7 +5,7 @@ import random import re import string -from typing import Any, Tuple, TypeAlias, Union +from typing import Any, Sequence, Tuple, TypeAlias, Union import deepdiff.model as deepdiff_model import kubernetes @@ -14,15 +14,15 @@ from acto.utils.thread_logger import get_thread_logger -PathIndex: TypeAlias = Union[str, int] +PathSegment: TypeAlias = Union[str, int] class PropertyPath(pydantic.BaseModel): """Path of a field in a dict""" - path: list[PathIndex] + path: list[PathSegment] - def __init__(self, path: list[PathIndex]) -> None: + def __init__(self, path: Sequence[PathSegment]) -> None: """Override constructor to allow positional argument""" super().__init__(path=path) @@ -44,7 +44,7 @@ def __getitem__(self, item: int): def __len__(self): return len(self.path) - def __contains__(self, item: PathIndex): + def __contains__(self, item: PathSegment): return item in self.path diff --git a/acto/engine.py b/acto/engine.py index c43985c28d..759dfc9afe 100644 --- a/acto/engine.py +++ b/acto/engine.py @@ -16,19 +16,15 @@ import jsonpatch import yaml -from acto.acto_config import ACTO_CONFIG from acto.checker.checker_set import CheckerSet from acto.common import kubernetes_client, print_event from acto.constant import CONST from acto.deploy import Deploy from acto.input import InputModel -from acto.input.input import DeterministicInputModel, OverSpecifiedField -from acto.input.known_schemas.base import K8sField -from acto.input.known_schemas.known_schema import find_all_matched_schemas_type +from acto.input.input import DeterministicInputModel from acto.input.testcase import TestCase from acto.input.testplan import TestGroup from acto.input.value_with_schema import ValueWithSchema, attach_schema_to_value -from acto.input.valuegenerator import ArrayGenerator from acto.kubectl_client import KubectlClient from acto.kubernetes_engine import base, kind from acto.lib.operator_config import OperatorConfig @@ -825,72 +821,17 @@ def __init__( if preload_images_ is not None: self.context["preload_images"].update(preload_images_) - # Apply custom fields - if operator_config.analysis is not None: - used_fields = self.context["analysis_result"]["used_fields"] - else: - used_fields = None self.input_model: DeterministicInputModel = input_model( crd=self.context["crd"]["body"], seed_input=self.seed, - used_fields=used_fields, example_dir=operator_config.example_dir, num_workers=num_workers, num_cases=num_cases, mount=mount, + kubernetes_version=operator_config.kubernetes_version, + custom_module_path=operator_config.custom_module, ) - applied_custom_k8s_fields = False - - if operator_config.k8s_fields is not None: - module = importlib.import_module(operator_config.k8s_fields) - if hasattr(module, "BLACKBOX") and ACTO_CONFIG.mode == "blackbox": - applied_custom_k8s_fields = True - for k8s_field in module.BLACKBOX: - self.input_model.apply_k8s_schema(k8s_field) - elif hasattr(module, "WHITEBOX") and ACTO_CONFIG.mode == "whitebox": - applied_custom_k8s_fields = True - for k8s_field in module.WHITEBOX: - self.input_model.apply_k8s_schema(k8s_field) - if not applied_custom_k8s_fields: - # default to use the known_schema module to automatically find the mapping - # from CRD to K8s schema - logger.info( - "Using known_schema to find the mapping from CRD to K8s schema" - ) - tuples = find_all_matched_schemas_type(self.input_model.root_schema) - for match_tuple in tuples: - logger.debug( - "Found matched schema: %s -> %s", - match_tuple[0].path, - match_tuple[1], - ) - k8s_schema = K8sField(match_tuple[0].path, match_tuple[1]) - self.input_model.apply_k8s_schema(k8s_schema) - - if operator_config.custom_fields is not None: - if ACTO_CONFIG.mode == "blackbox": - pruned_list = [] - module = importlib.import_module(operator_config.custom_fields) - for custom_field in module.custom_fields: - pruned_list.append(custom_field.path) - self.input_model.apply_custom_field(custom_field) - else: - pruned_list = [] - module = importlib.import_module(operator_config.custom_fields) - for custom_field in module.custom_fields: - pruned_list.append(custom_field.path) - self.input_model.apply_custom_field(custom_field) - else: - pruned_list = [] - tuples = find_all_matched_schemas_type(self.input_model.root_schema) - for match_tuple in tuples: - custom_field = OverSpecifiedField( - match_tuple[0].path, - array=isinstance(match_tuple[1], ArrayGenerator), - ) - self.input_model.apply_custom_field(custom_field) - self.sequence_base = 20 if delta_from else 0 if operator_config.custom_oracle is not None: diff --git a/acto/input/input.py b/acto/input/input.py index f53daa1973..3f2b406ae5 100644 --- a/acto/input/input.py +++ b/acto/input/input.py @@ -1,6 +1,6 @@ import abc import glob -import inspect +import importlib import json import logging import operator @@ -9,89 +9,50 @@ from functools import reduce from typing import List, Optional, Tuple +import pydantic import yaml -from acto.common import is_subfield, random_string -from acto.input import known_schemas +from acto import DEFAULT_KUBERNETES_VERSION +from acto.common import is_subfield +from acto.input import k8s_schemas, property_attribute from acto.input.get_matched_schemas import find_matched_schema -from acto.input.valuegenerator import extract_schema_with_value_generator -from acto.schema import BaseSchema, IntegerSchema -from acto.utils import get_thread_logger, is_prefix +from acto.input.test_generators.generator import get_testcases +from acto.schema import BaseSchema +from acto.schema.schema import extract_schema +from acto.utils import get_thread_logger -from .known_schemas import K8sField from .testcase import TestCase from .testplan import DeterministicTestPlan, TestGroup, TestPlan from .value_with_schema import attach_schema_to_value -def covered_by_k8s(k8s_fields: list[list[str]], path: list[str]) -> bool: - if path[-1] == "additional_properties": - return True +class CustomKubernetesMapping(pydantic.BaseModel): + """Class for specifying custom mapping""" - for k8s_path in k8s_fields: - if is_prefix(k8s_path, path): - return True - return False + schema_path: list[str] + kubernetes_schema_name: str -class CustomField: - def __init__(self, path, used_fields: Optional[list]) -> None: - self.path = path - self.used_fields = used_fields +class InputMetadata(pydantic.BaseModel): + """Metadata for the result of input model""" + total_number_of_schemas: int = 0 + number_of_matched_kubernetes_schemas: int = 0 + total_number_of_test_cases: int = 0 + number_of_run_test_cases: int = 0 + number_of_primitive_test_cases: int = 0 + number_of_semantic_test_cases: int = 0 + number_of_misoperations: int = 0 + number_of_pruned_test_cases: int = 0 -class CopiedOverField(CustomField): - """For pruning the fields that are simply copied over to other resources - All the subfields of this field (excluding this field) will be pruned - """ - - def __init__( - self, path, used_fields: Optional[list] = None, array: bool = False - ) -> None: - super().__init__(path, used_fields) - - -class OverSpecifiedField(CustomField): - """For pruning the fields that are simply copied over to other resources - - All the subfields of this field (excluding this field) will be pruned - """ - - def __init__( - self, path, used_fields: Optional[list] = None, array: bool = False - ) -> None: - super().__init__(path, used_fields) - - -class ProblematicField(CustomField): - """For pruning the field that can not be simply generated using Acto"s current generation mechanism. - - All the subfields of this field (including this field itself) will be pruned - """ - - def __init__(self, path, array: bool = False, string: bool = False) -> None: - super().__init__(path, []) - - -class PatchField(CustomField): - """For pruning the field that can not be simply generated using Acto"s current generation mechanism. - - All the subfields of this field (including this field itself) will be pruned - """ - - def __init__(self, path) -> None: - super().__init__(path, None) - - -class MappedField(CustomField): - """For annotating the field to be checked against the system state""" - - def __init__(self, path) -> None: - super().__init__(path, None) +# The number of test cases to form a group +CHUNK_SIZE = 10 class InputModel(abc.ABC): + """An abstract class for input model""" + NORMAL = "NORMAL" OVERSPECIFIED = "OVERSPECIFIED" COPIED_OVER = "COPIED_OVER" @@ -174,17 +135,21 @@ def __init__( self, crd: dict, seed_input: Optional[dict], - used_fields: list, example_dir: Optional[str], num_workers: int, num_cases: int, mount: Optional[list] = None, + kubernetes_version: str = DEFAULT_KUBERNETES_VERSION, + custom_module_path: Optional[str] = None, ) -> None: + # Mount allows to only test a subtree of the CRD if mount is not None: self.mount = mount else: self.mount = ["spec"] # We model the cr.spec as the input - self.root_schema = extract_schema_with_value_generator( + + # Load the CRD + self.root_schema = extract_schema( [], crd["spec"]["versions"][-1]["schema"]["openAPIV3Schema"] ) @@ -199,13 +164,13 @@ def __init__( docs = yaml.load_all(example_file, Loader=yaml.FullLoader) for doc in docs: example_docs.append(doc) - for example_doc in example_docs: self.root_schema.load_examples(example_doc) - self.used_fields = used_fields self.num_workers = num_workers self.num_cases = num_cases # number of test cases to run at a time + + # Initialize the seed input if seed_input is not None: initial_value = seed_input initial_value["metadata"]["name"] = "test-cluster" @@ -214,37 +179,81 @@ def __init__( ) else: self.seed_input = None + self.k8s_paths = find_matched_schema(self.root_schema) self.thread_vars = threading.local() - self.metadata = { - "normal_schemas": 0, - "pruned_by_overspecified": 0, - "pruned_by_copied": 0, - "num_normal_testcases": 0, - "num_overspecified_testcases": 0, - "num_copiedover_testcases": 0, - } # to fill in the generate_test_plan function - + # Initialize the metadata, to be filled in the generate_test_plan + self.metadata = InputMetadata() self.normal_test_plan_partitioned: list[ list[list[tuple[str, TestCase]]] ] = [] - self.overspecified_test_plan_partitioned: list[ - list[list[tuple[str, TestCase]]] - ] = [] - self.copiedover_test_plan_partitioned: list[ - list[list[tuple[str, TestCase]]] - ] = [] - self.semantic_test_plan_partitioned: list[ - list[tuple[str, list[TestCase]]] - ] = [] - self.additional_semantic_test_plan_partitioned: list[ - list[list[tuple[str, TestCase]]] - ] = [] - self.discarded_tests: dict[str, list] = {} + override_matches: Optional[list[tuple[BaseSchema, str]]] = None + if custom_module_path is not None: + custom_module = importlib.import_module(custom_module_path) + + # We need to do very careful sanitization here because we are + # loading user-provided module + if hasattr(custom_module, "KUBERNETES_TYPE_MAPPING"): + custum_kubernetes_type_mapping = ( + custom_module.KUBERNETES_TYPE_MAPPING + ) + if isinstance(custum_kubernetes_type_mapping, list): + override_matches = [] + for custom_mapping in custum_kubernetes_type_mapping: + if isinstance(custom_mapping, CustomKubernetesMapping): + try: + schema = self.get_schema_by_path( + custom_mapping.schema_path + ) + except KeyError as exc: + raise RuntimeError( + "Schema path of the custom mapping is invalid: " + f"{custom_mapping.schema_path}" + ) from exc + + override_matches.append( + (schema, custom_mapping.kubernetes_schema_name) + ) + else: + raise TypeError( + "Expected CustomKubernetesMapping in KUBERNETES_TYPE_MAPPING, " + f"but got {type(custom_mapping)}" + ) + + # Do the matching from CRD to Kubernetes schemas + mounted_schema = self.get_schema_by_path(self.mount) + self.metadata.total_number_of_schemas = len( + mounted_schema.get_all_schemas()[0] + ) + + # Match the Kubernetes schemas to subproperties of the root schema + kubernetes_schema_matcher = k8s_schemas.K8sSchemaMatcher.from_version( + kubernetes_version, override_matches + ) + top_matched_schemas = ( + kubernetes_schema_matcher.find_top_level_matched_schemas( + mounted_schema + ) + ) + for base_schema, k8s_schema_name in top_matched_schemas: + logging.info( + "Matched schema %s to k8s schema %s", + base_schema.get_path(), + k8s_schema_name, + ) + self.full_matched_schemas = ( + kubernetes_schema_matcher.expand_top_level_matched_schemas( + top_matched_schemas + ) + ) + + # Apply custom property attributes based on the property_attribute module + self.apply_custom_field() + def set_worker_id(self, worker_id: int): """Claim this thread"s id, so that we can split the test plan among threads""" @@ -256,9 +265,6 @@ def set_worker_id(self, worker_id: int): self.thread_vars.id = worker_id # so that we can run the test case itself right after the setup self.thread_vars.normal_test_plan = DeterministicTestPlan() - self.thread_vars.overspecified_test_plan = DeterministicTestPlan() - self.thread_vars.copiedover_test_plan = DeterministicTestPlan() - self.thread_vars.additional_semantic_test_plan = DeterministicTestPlan() self.thread_vars.semantic_test_plan = TestPlan( self.root_schema.to_tree() ) @@ -268,27 +274,6 @@ def set_worker_id(self, worker_id: int): TestGroup(group) ) - for group in self.overspecified_test_plan_partitioned[worker_id]: - self.thread_vars.overspecified_test_plan.add_testcase_group( - TestGroup(group) - ) - - for group in self.copiedover_test_plan_partitioned[worker_id]: - self.thread_vars.copiedover_test_plan.add_testcase_group( - TestGroup(group) - ) - - for group_ in self.additional_semantic_test_plan_partitioned[worker_id]: - self.thread_vars.additional_semantic_test_plan.add_testcase_group( - TestGroup(group_) - ) - - for key, value in self.semantic_test_plan_partitioned[worker_id]: - path = json.loads(key) - self.thread_vars.semantic_test_plan.add_testcases_by_path( - value, path - ) - def generate_test_plan( self, delta_from: Optional[str] = None, @@ -297,299 +282,76 @@ def generate_test_plan( """Generate test plan based on CRD""" logger = get_thread_logger(with_prefix=False) - existing_testcases = {} - if delta_from is not None: - with open(delta_from, "r", encoding="utf-8") as delta_from_file: - existing_testcases = json.load(delta_from_file)[ - "normal_testcases" - ] - - # Calculate the unused fields using used_fields from static analysis - # tree: TreeNode = self.root_schema.to_tree() - # for field in self.used_fields: - # field = field[1:] - # node = tree.get_node_by_path(field) - # if node is None: - # logger.warning(f"Field {field} not found in CRD") - # continue - - # node.set_used() - - # def func(overspecified_fields: list, unused_fields: list, node: TreeNode) -> bool: - # if len(node.children) == 0: - # return False - - # if not node.used: - # return False - - # used_child = [] - # for child in node.children.values(): - # if child.used: - # used_child.append(child) - # else: - # unused_fields.append(child.path) - - # if len(used_child) == 0: - # overspecified_fields.append(node.path) - # return False - # elif len(used_child) == len(node.children): - # return True - # else: - # return True - - # overspecified_fields = [] - # unused_fields = [] - # tree.traverse_func(partial(func, overspecified_fields, unused_fields)) - # for field in overspecified_fields: - # logger.info("Overspecified field: %s", field) - # for field in unused_fields: - # logger.info("Unused field: %s", field) - - ######################################## - # Get all K8s schemas - ######################################## - k8s_int_tests = [] - k8s_str_tests = [] - for name, obj in inspect.getmembers(known_schemas): - if inspect.isclass(obj): - if issubclass(obj, known_schemas.K8sIntegerSchema): - for _, class_member in inspect.getmembers(obj): - if isinstance(class_member, known_schemas.K8sTestCase): - k8s_int_tests.append(class_member) - elif issubclass(obj, known_schemas.K8sStringSchema): - for _, class_member in inspect.getmembers(obj): - if isinstance(class_member, known_schemas.K8sTestCase): - k8s_str_tests.append(class_member) - - logger.info("Got %d K8s integer tests", len(k8s_int_tests)) - logger.info("Got %d K8s string tests", len(k8s_str_tests)) - ######################################## # Generate test plan ######################################## - planned_normal_testcases = {} normal_testcases = {} - semantic_testcases = {} - additional_semantic_testcases = {} - overspecified_testcases = {} - copiedover_testcases = {} - num_normal_testcases = 0 - num_overspecified_testcases = 0 - num_copiedover_testcases = 0 - num_semantic_testcases = 0 - num_additional_semantic_testcases = 0 - - num_total_semantic_tests = 0 - num_total_invalid_tests = 0 - mounted_schema = self.get_schema_by_path(self.mount) - ( - normal_schemas, - semantic_schemas, - ) = mounted_schema.get_normal_semantic_schemas() - logger.info("Got %d normal schemas", len(normal_schemas)) - logger.info("Got %d semantic schemas", len(semantic_schemas)) - - ( - normal_schemas, - pruned_by_overspecified, - pruned_by_copied, - ) = mounted_schema.get_all_schemas() - for schema in normal_schemas: - # Skip if the schema is not in the focus fields - if focus_fields is not None: - logger.info("focusing on %s", focus_fields) - focused = False - for focus_field in focus_fields: - logger.info( - "Comparing %s with %s", schema.path, focus_field - ) - if is_subfield(schema.path, focus_field): - focused = True - break - if not focused: - continue + test_cases = get_testcases(self.root_schema, self.full_matched_schemas) - path = ( - json.dumps(schema.path) - .replace('"ITEM"', "0") - .replace("additional_properties", "ACTOKEY") - ) - testcases, semantic_testcases_ = schema.test_cases() - planned_normal_testcases[path] = testcases - if len(semantic_testcases_) > 0: - semantic_testcases[path] = semantic_testcases_ - num_semantic_testcases += len(semantic_testcases_) - if path in existing_testcases: - continue - normal_testcases[path] = testcases - num_normal_testcases += len(testcases) - - if isinstance(schema, known_schemas.K8sSchema): - for testcase in testcases: - if isinstance(testcase, known_schemas.K8sInvalidTestCase): - num_total_invalid_tests += 1 - num_total_semantic_tests += 1 - - for semantic_testcase in semantic_testcases_: - if isinstance( - semantic_testcase, known_schemas.K8sInvalidTestCase - ): - num_total_invalid_tests += 1 - num_total_semantic_tests += 1 - - if not isinstance( - schema, known_schemas.K8sSchema - ) and not covered_by_k8s(self.k8s_paths, list(schema.path)): - if isinstance(schema, IntegerSchema): - additional_semantic_testcases[path] = list(k8s_int_tests) - num_additional_semantic_testcases += len(k8s_int_tests) - # elif isinstance(schema, StringSchema): - # additional_semantic_testcases[path] = list(k8s_str_tests) - # num_additional_semantic_testcases += len(k8s_str_tests) - - for schema in pruned_by_overspecified: - # Skip if the schema is not in the focus fields + num_test_cases = 0 + num_run_test_cases = 0 + num_primitive_test_cases = 0 + num_semantic_test_cases = 0 + num_misoperations = 0 + num_pruned_test_cases = 0 + for path, test_case_list in test_cases: + # First, check if the path is in the focus fields if focus_fields is not None: - logger.info(f"focusing on {focus_fields}") focused = False for focus_field in focus_fields: - logger.info(f"Comparing {schema.path} with {focus_field}") - if is_subfield(schema.path, focus_field): + if is_subfield(path, focus_field): focused = True break if not focused: continue - testcases, semantic_testcases_ = schema.test_cases() - path = ( - json.dumps(schema.path) + path_str = ( + json.dumps(path) .replace('"ITEM"', "0") - .replace("additional_properties", random_string(5)) + .replace("additional_properties", "ACTOKEY") ) - if len(semantic_testcases_) > 0: - semantic_testcases[path] = semantic_testcases_ - num_semantic_testcases += len(semantic_testcases_) - overspecified_testcases[path] = testcases - num_overspecified_testcases += len(testcases) - if isinstance(schema, known_schemas.K8sSchema): - for testcase in testcases: - if isinstance(testcase, known_schemas.K8sInvalidTestCase): - num_total_invalid_tests += 1 - num_total_semantic_tests += 1 - - for semantic_testcase in semantic_testcases_: - if isinstance( - semantic_testcase, known_schemas.K8sInvalidTestCase - ): - num_total_invalid_tests += 1 - num_total_semantic_tests += 1 - - for schema in pruned_by_copied: - # Skip if the schema is not in the focus fields - if focus_fields is not None: - logger.info(f"focusing on {focus_fields}") - focused = False - for focus_field in focus_fields: - logger.info(f"Comparing {schema.path} with {focus_field}") - if is_subfield(schema.path, focus_field): - focused = True - break - if not focused: + # Filter by test case attributes + filtered_test_case_list = [] + for test_case in test_case_list: + num_test_cases += 1 + if test_case.primitive: + num_primitive_test_cases += 1 + if test_case.kubernetes_schema and test_case.primitive: + # This is a primitive test case for a k8s schema + # Primitive test cases are pruned for k8s schemas + num_pruned_test_cases += 1 continue - - testcases, semantic_testcases_ = schema.test_cases() - path = ( - json.dumps(schema.path) - .replace('"ITEM"', "0") - .replace("additional_properties", random_string(5)) - ) - if len(semantic_testcases_) > 0: - semantic_testcases[path] = semantic_testcases_ - num_semantic_testcases += len(semantic_testcases_) - copiedover_testcases[path] = testcases - num_copiedover_testcases += len(testcases) - - if isinstance(schema, known_schemas.K8sSchema): - for testcase in testcases: - if isinstance(testcase, known_schemas.K8sInvalidTestCase): - num_total_invalid_tests += 1 - num_total_semantic_tests += 1 - - for semantic_testcase in semantic_testcases_: - if isinstance( - semantic_testcase, known_schemas.K8sInvalidTestCase - ): - num_total_invalid_tests += 1 - num_total_semantic_tests += 1 - - logger.info( - "Parsed [%d] fields from normal schema", len(normal_schemas) - ) - logger.info( - "Parsed [%d] fields from over-specified schema", - len(pruned_by_overspecified), - ) + if test_case.semantic: + num_semantic_test_cases += 1 + if test_case.invalid: + num_misoperations += 1 + filtered_test_case_list.append(test_case) + num_run_test_cases += 1 + + normal_testcases[path_str] = filtered_test_case_list + + self.metadata.total_number_of_test_cases = num_test_cases + self.metadata.number_of_run_test_cases = num_run_test_cases + self.metadata.number_of_primitive_test_cases = num_pruned_test_cases + self.metadata.number_of_semantic_test_cases = num_semantic_test_cases + self.metadata.number_of_misoperations = num_misoperations + self.metadata.number_of_pruned_test_cases = num_pruned_test_cases + + logger.info("Generated %d test cases in total", num_test_cases) + logger.info("Generated %d test cases to run", num_run_test_cases) logger.info( - "Parsed [%d] fields from copied-over schema", len(pruned_by_copied) - ) - - logger.info( - "Generated [%d] test cases for normal schemas", num_normal_testcases - ) - logger.info( - "Generated [%d] test cases for overspecified schemas", - num_overspecified_testcases, - ) - logger.info( - "Generated [%d] test cases for copiedover schemas", - num_copiedover_testcases, - ) - logger.info( - "Generated [%d] test cases for semantic schemas", - num_semantic_testcases, - ) - logger.info( - "Generated [%d] test cases for additional semantic schemas", - num_additional_semantic_testcases, - ) - - logger.info("Generated [%d] semantic tests", num_total_semantic_tests) - logger.info("Generated [%d] invalid tests", num_total_invalid_tests) - - self.metadata["pruned_by_overspecified"] = len(pruned_by_overspecified) - self.metadata["pruned_by_copied"] = len(pruned_by_copied) - self.metadata["semantic_schemas"] = len(semantic_testcases) - self.metadata["num_normal_testcases"] = num_normal_testcases - self.metadata[ - "num_overspecified_testcases" - ] = num_overspecified_testcases - self.metadata["num_copiedover_testcases"] = num_copiedover_testcases - self.metadata["num_semantic_testcases"] = num_semantic_testcases - self.metadata[ - "num_additional_semantic_testcases" - ] = num_additional_semantic_testcases - - logger.info( - f"Generated {num_normal_testcases + num_semantic_testcases} normal testcases" + "Generated %d primitive test cases", num_primitive_test_cases ) + logger.info("Generated %d semantic test cases", num_semantic_test_cases) + logger.info("Generated %d misoperations", num_misoperations) + logger.info("Generated %d pruned test cases", num_pruned_test_cases) normal_test_plan_items = list(normal_testcases.items()) - overspecified_test_plan_items = list(overspecified_testcases.items()) - copiedover_test_plan_items = list(copiedover_testcases.items()) - semantic_test_plan_items = list(semantic_testcases.items()) # randomize to reduce skewness among workers random.shuffle(normal_test_plan_items) - random.shuffle(overspecified_test_plan_items) - random.shuffle(copiedover_test_plan_items) - random.shuffle(semantic_test_plan_items) - - # run semantic testcases anyway - normal_test_plan_items.extend(semantic_test_plan_items) - - CHUNK_SIZE = 10 def split_into_subgroups( test_plan_items, @@ -605,57 +367,23 @@ def split_into_subgroups( return subgroups normal_subgroups = split_into_subgroups(normal_test_plan_items) - overspecified_subgroups = split_into_subgroups( - overspecified_test_plan_items - ) - copiedover_subgroups = split_into_subgroups(copiedover_test_plan_items) # Initialize the three test plans, and assign test cases to them # according to the number of workers for i in range(self.num_workers): self.normal_test_plan_partitioned.append([]) - self.overspecified_test_plan_partitioned.append([]) - self.copiedover_test_plan_partitioned.append([]) - self.semantic_test_plan_partitioned.append([]) - self.additional_semantic_test_plan_partitioned.append([]) for i in range(0, len(normal_subgroups)): self.normal_test_plan_partitioned[i % self.num_workers].append( normal_subgroups[i] ) - for i in range(0, len(overspecified_subgroups)): - self.overspecified_test_plan_partitioned[ - i % self.num_workers - ].append(overspecified_subgroups[i]) - - for i in range(0, len(copiedover_subgroups)): - self.copiedover_test_plan_partitioned[i % self.num_workers].append( - copiedover_subgroups[i] - ) - - for i in range(0, len(semantic_test_plan_items)): - self.semantic_test_plan_partitioned[i % self.num_workers].append( - semantic_test_plan_items[i] - ) - # appending empty lists to avoid no test cases distributed to certain # work nodes assert self.num_workers == len(self.normal_test_plan_partitioned) - assert self.num_workers == len(self.overspecified_test_plan_partitioned) - assert self.num_workers == len(self.copiedover_test_plan_partitioned) return { - "delta_from": delta_from, - "existing_testcases": existing_testcases, "normal_testcases": normal_testcases, - "overspecified_testcases": overspecified_testcases, - "copiedover_testcases": copiedover_testcases, - "semantic_testcases": semantic_testcases, - "planned_normal_testcases": planned_normal_testcases, - "normal_subgroups": normal_subgroups, - "overspecified_subgroups": overspecified_subgroups, - "copiedover_subgroups": copiedover_subgroups, } def next_test( @@ -746,82 +474,27 @@ def discard_test_case(self): del self.thread_vars.test_plan[self.thread_vars.curr_field] self.thread_vars.curr_field = None - def apply_custom_field(self, custom_field: CustomField): + def apply_custom_field(self): """Applies custom field to the input model Relies on the __setitem__ and __getitem__ methods of schema class """ - path = custom_field.path - if len(path) == 0: - self.root_schema = custom_field.custom_schema( - self.root_schema, custom_field.used_fields - ) - - # fetch the parent schema - curr = self.root_schema - for idx in path: - curr = curr[idx] - if isinstance(custom_field, PatchField): - curr.patch = True - s1, s2, s3 = curr.get_all_schemas() - for s in s1: - s.patch = True - for s in s2: - s.patch = True - for s in s3: - s.patch = True - - if isinstance(custom_field, MappedField): - curr.mapped = True - s1, s2, s3 = curr.get_all_schemas() - for s in s1: - s.mapped = True - for s in s2: - s.mapped = True - for s in s3: - s.mapped = True + for ( + property_path, + attribute, + ) in property_attribute.PROPERTY_ATTRIBUTES.items(): + schema = reduce( + operator.getitem, property_path.path, self.root_schema + ) - if isinstance(custom_field, CopiedOverField): - curr.copied_over = True - elif isinstance(custom_field, OverSpecifiedField): - curr.over_specified = True - s1, s2, s3 = curr.get_all_schemas() + s1, s2, s3 = schema.get_all_schemas() for s in s1: - s.over_specified = True + s.attributes |= attribute for s in s2: - s.over_specified = True + s.attributes |= attribute for s in s3: - s.over_specified = True - elif isinstance(custom_field, ProblematicField): - curr.problematic = True - elif isinstance(custom_field, PatchField) or isinstance( - custom_field, MappedField - ): - pass # do nothing, already handled above - else: - raise Exception("Unknown custom field type") - - def apply_k8s_schema(self, k8s_field: K8sField): - path = k8s_field.path - if len(path) == 0: - self.root_schema = k8s_field.custom_schema(self.root_schema) - - # fetch the parent schema - curr = self.root_schema - for idx in path[:-1]: - curr = curr[idx] - - # construct new schema - custom_schema = k8s_field.custom_schema(curr[path[-1]]) - - # replace old schema with the new one - curr[path[-1]] = custom_schema - - def apply_candidates(self, candidates: dict, path: list): - """Apply candidates file onto schema""" - # TODO - candidates_list = self.candidates_dict_to_list(candidates, path) + s.attributes |= attribute def apply_default_value(self, default_value_result: dict): """Takes default value result from static analysis and apply to schema @@ -836,23 +509,15 @@ def apply_default_value(self, default_value_result: dict): for k, v in decoded_value.items(): decoded_value[k] = json.loads(v) logging.info( - "Setting default value for %s to %s" - % (path + [k], decoded_value[k]) + "Setting default value for %s to %s", + path + [k], + decoded_value[k], ) self.get_schema_by_path(path + [k]).set_default( decoded_value[k] ) else: logging.info( - "Setting default value for %s to %s" % (path, decoded_value) + "Setting default value for %s to %s", path, decoded_value ) self.get_schema_by_path(path).set_default(value) - - def candidates_dict_to_list(self, candidates: dict, path: list) -> list: - if "candidates" in candidates: - return [(path, candidates["candidates"])] - else: - ret = [] - for key, value in candidates.items(): - ret.extend(self.candidates_dict_to_list(value, path + [key])) - return ret diff --git a/acto/input/k8s_schemas.py b/acto/input/k8s_schemas.py index 93b65bd498..b7adb81931 100644 --- a/acto/input/k8s_schemas.py +++ b/acto/input/k8s_schemas.py @@ -4,13 +4,15 @@ to Kubernetes schemas. It is used for generating Kubernetes CRD schemas from acto schemas. """ + # pylint: disable=redefined-outer-name +import json import sys from abc import ABC, abstractmethod from collections import defaultdict from difflib import SequenceMatcher -from typing import Callable +from typing import Callable, Optional import requests @@ -30,6 +32,14 @@ class KubernetesSchema(ABC): """Base class for Kubernetes schema matching""" + def __init__( + self, + schema_spec: dict, + schema_name: Optional[str] = None, + ) -> None: + self.k8s_schema_name = schema_name + self.schema_spec = schema_spec + @abstractmethod def match(self, schema: BaseSchema) -> bool: """Determines if the schema matches the Kubernetes schema""" @@ -38,13 +48,34 @@ def match(self, schema: BaseSchema) -> bool: def dump_schema(self) -> dict: """Dumps the Kubernetes schema into a dictionary (for debugging)""" + @abstractmethod + def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: + """Resolves k8s schema properties into k8s schema objects""" + + +class KubernetesUninitializedSchema(KubernetesSchema): + """Class for uninitialized Kubernetes schema matching""" + + def __init__(self) -> None: + super().__init__({}) + + def match(self, schema) -> bool: + raise RuntimeError("Uninitialized schema") + + def dump_schema(self) -> dict: + raise RuntimeError("Uninitialized schema") + + def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: + raise RuntimeError("Uninitialized schema") + class KubernetesObjectSchema(KubernetesSchema): """Class for Kubernetes object schema matching""" - def __init__(self, schema_name, schema_spec) -> None: - self.k8s_schema_name: str = schema_name - self.schema_spec = schema_spec + def __init__( + self, schema_spec: dict, schema_name: Optional[str] = None + ) -> None: + super().__init__(schema_spec, schema_name) self.properties: dict[str, KubernetesSchema] = {} def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: @@ -55,22 +86,26 @@ def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: "properties" ].items(): self.properties[property_name] = resolve(property_spec) + self.properties[property_name].update(resolve) def match(self, schema) -> bool: if ( not isinstance(schema, ObjectSchema) - or len(self.properties) != len(schema.properties) or len(self.properties) == 0 + or len(self.properties) != len(schema.properties) ): return False + num_mismatch = 0 for property_name, property_schema in self.properties.items(): if property_name not in schema.properties: # not a match if property is not in schema - return False + num_mismatch += 1 elif not property_schema.match(schema.properties[property_name]): # not a match if schema does not match property schema - return False - return True + num_mismatch += 1 + + # TODO: How to do approximate matching? + return bool(num_mismatch == 0) def dump_schema(self) -> dict: properties = {} @@ -102,6 +137,9 @@ def match(self, schema) -> bool: def dump_schema(self) -> dict: return {"type": "string"} + def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: + pass + class KubernetesBooleanSchema(KubernetesSchema): """Class for Kubernetes boolean schema matching""" @@ -112,6 +150,9 @@ def match(self, schema) -> bool: def dump_schema(self) -> dict: return {"type": "boolean"} + def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: + pass + class KubernetesIntegerSchema(KubernetesSchema): """Class for Kubernetes integer schema matching""" @@ -122,6 +163,9 @@ def match(self, schema) -> bool: def dump_schema(self) -> dict: return {"type": "integer"} + def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: + pass + class KubernetesFloatSchema(KubernetesSchema): """Class for Kubernetes float schema matching""" @@ -132,13 +176,18 @@ def match(self, schema) -> bool: def dump_schema(self) -> dict: return {"type": "number"} + def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: + pass + class KubernetesMapSchema(KubernetesSchema): """Class for Kubernetes map schema matching""" - def __init__(self, value_cls: KubernetesSchema) -> None: - super().__init__() - self.value: KubernetesSchema = value_cls + def __init__( + self, schema_spec: dict, schema_name: Optional[str] = None + ) -> None: + super().__init__(schema_spec, schema_name) + self.value: KubernetesSchema = KubernetesUninitializedSchema() def match(self, schema) -> bool: # Dict schema requires additional_properties to be set @@ -160,13 +209,19 @@ def dump_schema(self) -> dict: "additionalProperties": self.value.dump_schema(), } + def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: + self.value = resolve(self.schema_spec["additionalProperties"]) + self.value.update(resolve) + class KubernetesArraySchema(KubernetesSchema): """Class for Kubernetes array schema matching""" - def __init__(self, item_cls: KubernetesSchema) -> None: - super().__init__() - self.item: KubernetesSchema = item_cls + def __init__( + self, schema_spec: dict, schema_name: Optional[str] = None + ) -> None: + super().__init__(schema_spec, schema_name) + self.item: KubernetesSchema = KubernetesUninitializedSchema() def match(self, schema) -> bool: # List schema requires items to be set @@ -185,6 +240,10 @@ def match(self, schema) -> bool: def dump_schema(self) -> dict: return {"type": "array", "items": self.item.dump_schema()} + def update(self, resolve: Callable[[dict], KubernetesSchema]) -> None: + self.item = resolve(self.schema_spec["items"]) + self.item.update(resolve) + class KubernetesDatetimeSchema(KubernetesSchema): """Class for Kubernetes datetime schema matching""" @@ -195,6 +254,9 @@ def match(self, schema) -> bool: def dump_schema(self) -> dict: return {"type": "string", "format": "date-time"} + def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: + pass + class KubernetesOpaqueSchema(KubernetesSchema): """Class for Kubernetes opaque schema matching""" @@ -205,6 +267,9 @@ def match(self, schema) -> bool: def dump_schema(self) -> dict: return {"type": "object"} + def update(self, resolve: Callable[[dict], "KubernetesSchema"]) -> None: + pass + class ObjectMetaSchema(KubernetesObjectSchema): """Class for Kubernetes ObjectMeta schema matching""" @@ -213,7 +278,15 @@ def match(self, schema) -> bool: if isinstance(schema, OpaqueSchema): return True if isinstance(schema, ObjectSchema): - return super().match(schema) + # ObjectMeta is a special case + # Its schema is generated specially by controller-gen + if "name" in schema.properties and "namespace" in schema.properties: + for key, sub_schema in schema.properties.items(): + if key not in self.properties: + return False + if not self.properties[key].match(sub_schema): + return False + return True return False @@ -221,14 +294,14 @@ def fetch_k8s_schema_spec(version: str) -> dict: """Fetches the Kubernetes schema spec from the Kubernetes repo Args: - version (str): the Kubernetes version e.g. "1.29" + version (str): the Kubernetes version e.g. "v1.29.0" Returns: dict: the Kubernetes schema spec """ # pylint: disable=line-too-long resp = requests.get( - f"https://raw.githubusercontent.com/kubernetes/kubernetes/release-{version}/api/openapi-spec/swagger.json", + f"https://raw.githubusercontent.com/kubernetes/kubernetes/{version}/api/openapi-spec/swagger.json", timeout=5, ) return resp.json() @@ -237,7 +310,11 @@ def fetch_k8s_schema_spec(version: str) -> dict: class K8sSchemaMatcher: """Find Kubernetes schemas that match the given schema""" - def __init__(self, schema_definitions: dict) -> None: + def __init__( + self, + schema_definitions: dict, + custom_mappings: Optional[list[tuple[BaseSchema, str]]] = None, + ) -> None: """Initializes the Kubernetes schema matcher with the kubernetes schema definitions @@ -250,20 +327,37 @@ def __init__(self, schema_definitions: dict) -> None: schema_definitions ) ) + self._custom_mapping_dict = ( + { + json.dumps(base_schema.path): kubernetes_schema + for base_schema, kubernetes_schema in custom_mappings + } + if custom_mappings is not None + else {} + ) # mapping from the encoded schema path to the Kubernetes schema name + + @property + def k8s_models(self) -> dict[str, KubernetesSchema]: + """Returns the Kubernetes models""" + return self._k8s_models @classmethod - def from_version(cls, version: str): + def from_version( + cls, + version: str, + custom_mappings: Optional[list[tuple[BaseSchema, str]]] = None, + ): """Factory method that creates a Kubernetes schema matcher from a Kubernetes version Args: - version (str): the Kubernetes version e.g. "1.29" + version (str): the Kubernetes version e.g. "v1.29.0" Returns: K8sSchemaMatcher: the Kubernetes schema matcher """ schema_definitions = fetch_k8s_schema_spec(version)["definitions"] - return cls(schema_definitions) + return cls(schema_definitions, custom_mappings) def _generate_schema_name_to_property_name_mapping( self, schema_definitions: dict @@ -286,10 +380,41 @@ def _generate_schema_name_to_property_name_mapping( return dict(schema_name_to_property_name) def _generate_k8s_models( - self, schema_definitions: dict - ) -> dict[str, KubernetesObjectSchema]: + self, schema_definitions: dict[str, dict] + ) -> dict[str, KubernetesSchema]: """Generates a dictionary of Kubernetes models for schema matching""" - k8s_models: dict[str, KubernetesObjectSchema] = {} + k8s_models: dict[str, KubernetesSchema] = {} + + def resolve_named_kubernetes_schema( + schema_name, schema_spec: dict + ) -> KubernetesSchema: + if schema_name.endswith("ObjectMeta"): + return ObjectMetaSchema(schema_spec, schema_name) + if schema_spec["type"] == "string": + return KubernetesStringSchema(schema_spec, schema_name) + if schema_spec["type"] == "boolean": + return KubernetesBooleanSchema(schema_spec, schema_name) + if schema_spec["type"] == "integer": + return KubernetesIntegerSchema(schema_spec, schema_name) + if schema_spec["type"] == "number": + return KubernetesFloatSchema(schema_spec, schema_name) + if schema_spec["type"] == "object": + if ( + "additionalProperties" in schema_spec + and "properties" in schema_spec + ): + raise NotImplementedError( + "Object with both additional properties and properties" + ) + if "additionalProperties" in schema_spec: + return KubernetesMapSchema(schema_spec, schema_name) + if "properties" in schema_spec: + return KubernetesObjectSchema(schema_spec, schema_name) + return KubernetesOpaqueSchema(schema_spec, schema_name) + if schema_spec["type"] == "array": + return KubernetesArraySchema(schema_spec, schema_name) + else: + raise KeyError(f"Cannot resolve type {schema_spec}") def resolve(schema_spec: dict) -> KubernetesSchema: """Resolves schema type from k8s schema spec""" @@ -301,54 +426,62 @@ def resolve(schema_spec: dict) -> KubernetesSchema: except KeyError as exc: raise KeyError(f"Cannot resolve type {type_str}") from exc elif schema_spec["type"] == "string": - return KubernetesStringSchema() + return KubernetesStringSchema(schema_spec) elif schema_spec["type"] == "boolean": - return KubernetesBooleanSchema() + return KubernetesBooleanSchema(schema_spec) elif schema_spec["type"] == "integer": - return KubernetesIntegerSchema() + return KubernetesIntegerSchema(schema_spec) elif schema_spec["type"] == "number": - return KubernetesFloatSchema() + return KubernetesFloatSchema(schema_spec) elif schema_spec["type"] == "object": if "additionalProperties" in schema_spec: - return KubernetesMapSchema( - resolve(schema_spec["additionalProperties"]) - ) - return KubernetesOpaqueSchema() + return KubernetesMapSchema(schema_spec) + if "properties" in schema_spec: + return KubernetesObjectSchema(schema_spec) + return KubernetesOpaqueSchema(schema_spec) elif schema_spec["type"] == "array": - return KubernetesArraySchema(resolve(schema_spec["items"])) + return KubernetesArraySchema(schema_spec) else: raise KeyError(f"Cannot resolve type {schema_spec}") + # First initialize all k8s models for schema_name, schema_spec in schema_definitions.items(): if schema_name.startswith("io.k8s.apiextensions-apiserver"): continue - schema: KubernetesObjectSchema - if schema_name.endswith("ObjectMeta"): - schema = ObjectMetaSchema(schema_name, schema_spec) - else: - schema = KubernetesObjectSchema(schema_name, schema_spec) - - k8s_models[schema_name] = schema + resolved_schema = resolve_named_kubernetes_schema( + schema_name, schema_spec + ) + k8s_models[schema_name] = resolved_schema - for schema in k8s_models.values(): - schema.update(resolve) + # Second pass to resolve the references + for k8s_schema in k8s_models.values(): + k8s_schema.update(resolve) return k8s_models def _rank_matched_k8s_schemas( self, schema: BaseSchema, - matched_schemas: [tuple[BaseSchema, KubernetesSchema]], + matched_schemas: list[tuple[BaseSchema, KubernetesSchema]], ) -> int: """returns the index of the best matched schemas using heuristic""" # 1. Give priority to the schemas that have been used with the same # property name in k8s schema specs + if len(schema.path) < 2: + raise RuntimeError( + f"Schema path too short {schema.path} for " + f"{[schema.k8s_schema_name for _, schema in matched_schemas]}" + ) schema_name = ( schema.path[-2] if schema.path[-1] == "ITEM" else schema.path[-1] ) name_matched = [] for i, (_, k8s_schema) in enumerate(matched_schemas): + if k8s_schema.k8s_schema_name is None: + raise RuntimeError( + f"Kubernetes schema name not found for {k8s_schema}" + ) observed_schema_names = self._schema_name_to_property_name.get( k8s_schema.k8s_schema_name, set() ) @@ -372,6 +505,10 @@ def _rank_matched_k8s_schemas( for i, (_, matched_schema) in enumerate(matched_schemas): if name_matched and i not in name_matched: continue + if matched_schema.k8s_schema_name is None: + raise RuntimeError( + f"Kubernetes schema name not found for {matched_schema}" + ) seq_matcher.set_seq2(matched_schema.k8s_schema_name.split(".")[-1]) ratio = seq_matcher.ratio() if ratio > max_ratio: @@ -379,12 +516,24 @@ def _rank_matched_k8s_schemas( max_ratio_schema_idx = i return max_ratio_schema_idx - def find_matched_schemas( + def find_all_matched_schemas( self, schema: BaseSchema ) -> list[tuple[BaseSchema, KubernetesSchema]]: - """Finds all Kubernetes schemas that match the given schema""" + """Finds all Kubernetes schemas that match the given schema + including matches of anonymous Kubernetes schemas""" + top_level_matches = self.find_top_level_matched_schemas(schema) + return self.expand_top_level_matched_schemas(top_level_matches) + + def find_named_matched_schemas( + self, schema: BaseSchema + ) -> list[tuple[BaseSchema, KubernetesSchema]]: + """Finds all named Kubernetes schemas that match the given schema, + without anonymous Kubernetes schemas""" matched_schemas: list[tuple[BaseSchema, KubernetesSchema]] = [] for kubernetes_schema in self._k8s_models.values(): + if not isinstance(kubernetes_schema, KubernetesObjectSchema): + # Avoid Opaque schemas for Kubernetes named schemas + continue if kubernetes_schema.match(schema): matched_schemas.append((schema, kubernetes_schema)) if matched_schemas: @@ -392,14 +541,134 @@ def find_matched_schemas( matched_schemas = [matched_schemas[idx]] if isinstance(schema, ObjectSchema): for sub_schema in schema.properties.values(): - matched_schemas.extend(self.find_matched_schemas(sub_schema)) + matched_schemas.extend( + self.find_named_matched_schemas(sub_schema) + ) + elif isinstance(schema, ArraySchema): + matched_schemas.extend( + self.find_named_matched_schemas(schema.get_item_schema()) + ) + + return matched_schemas + + def find_top_level_matched_schemas( + self, schema: BaseSchema + ) -> list[tuple[BaseSchema, KubernetesSchema]]: + """Finds all Kubernetes schemas that match the given schema + at the top level, without returning the matched sub-schemas. + The returned matches are guaranteed to be named Kubernetes schemas""" + matched_schemas: list[tuple[BaseSchema, KubernetesSchema]] = [] + + # First check the custom mapping + if (schema_key := json.dumps(schema.path)) in self._custom_mapping_dict: + kubernetes_schema_name = self._custom_mapping_dict[schema_key] + return [(schema, self._k8s_models[kubernetes_schema_name])] + + # Look through the Kubernetes schemas + for kubernetes_schema in self._k8s_models.values(): + if not isinstance(kubernetes_schema, KubernetesObjectSchema): + # Avoid Opaque schemas for Kubernetes named schemas + continue + if kubernetes_schema.match(schema): + matched_schemas.append((schema, kubernetes_schema)) + if matched_schemas: + idx = self._rank_matched_k8s_schemas(schema, matched_schemas) + matched_schemas = [matched_schemas[idx]] + elif isinstance(schema, ObjectSchema): + for sub_schema in schema.properties.values(): + matched_schemas.extend( + self.find_top_level_matched_schemas(sub_schema) + ) elif isinstance(schema, ArraySchema): matched_schemas.extend( - self.find_matched_schemas(schema.get_item_schema()) + self.find_top_level_matched_schemas(schema.get_item_schema()) ) return matched_schemas + def expand_top_level_matched_schemas( + self, top_level_matches: list[tuple[BaseSchema, KubernetesSchema]] + ) -> list[tuple[BaseSchema, KubernetesSchema]]: + """Expands the top level matches to include all sub-schemas""" + matched_schemas: list[tuple[BaseSchema, KubernetesSchema]] = [] + # BFS to find all matches + to_explore: list[tuple[BaseSchema, KubernetesSchema]] = list( + top_level_matches + ) + while to_explore: + crd_schema, k8s_schema = to_explore.pop() + + # Handle custom mapping here, override the matched k8s_schema + if ( + schema_key := json.dumps(crd_schema.path) + ) in self._custom_mapping_dict: + kubernetes_schema_name = self._custom_mapping_dict[schema_key] + k8s_schema = self._k8s_models[kubernetes_schema_name] + matched_schemas.append((crd_schema, k8s_schema)) + + if isinstance(crd_schema, ObjectSchema): + if isinstance(k8s_schema, KubernetesObjectSchema): + for key, sub_schema in crd_schema.properties.items(): + if key in k8s_schema.properties: + to_explore.append( + (sub_schema, k8s_schema.properties[key]) + ) + elif ( + isinstance(k8s_schema, KubernetesMapSchema) + and crd_schema.additional_properties is not None + ): + to_explore.append( + (crd_schema.additional_properties, k8s_schema.value) + ) + else: + raise RuntimeError( + "CRD schema type does not match k8s schema type" + f"({crd_schema}, {k8s_schema})" + ) + elif isinstance(crd_schema, ArraySchema): + if isinstance(k8s_schema, KubernetesArraySchema): + to_explore.append( + (crd_schema.get_item_schema(), k8s_schema.item) + ) + else: + raise RuntimeError( + "CRD schema type does not match k8s schema type" + f"({crd_schema}, {k8s_schema})" + ) + + return matched_schemas + + def override_schema_matches( + self, + existing_matches: list[tuple[BaseSchema, KubernetesSchema]], + override: list[tuple[BaseSchema, KubernetesSchema]], + ) -> list[tuple[BaseSchema, KubernetesSchema]]: + """Override the existing matches with a list of custom matches + + Args: + existing_matches: a full list of matches, containing subproperties + override: custom top-level schema matches + + Return: + overriden full list of matches + """ + matched_schema_map: dict[str, tuple[BaseSchema, KubernetesSchema]] = {} + + for schema, kubernetes_schema in existing_matches: + matched_schema_map[json.dumps(schema.path)] = ( + schema, + kubernetes_schema, + ) + + expanded_overrides = self.expand_top_level_matched_schemas(override) + for schema, kubernetes_schema in expanded_overrides: + matched_schema_map[json.dumps(schema.path)] = ( + schema, + kubernetes_schema, + ) + + return list(matched_schema_map.values()) + def dump_k8s_schemas(self) -> dict: """Dumps all Kubernetes schemas into a dictionary (for debugging)""" return { @@ -427,8 +696,8 @@ def dump_k8s_schemas(self) -> dict: ["root"], crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"] ) - schema_matcher = K8sSchemaMatcher.from_version("1.29") - matched = schema_matcher.find_matched_schemas(spec_schema) + schema_matcher = K8sSchemaMatcher.from_version("1.29.0") + matched = schema_matcher.find_all_matched_schemas(spec_schema) for schema, k8s_schema in matched: # pylint: disable-next=invalid-name diff --git a/acto/input/kubernetes_property.py b/acto/input/kubernetes_property.py new file mode 100644 index 0000000000..0d72730906 --- /dev/null +++ b/acto/input/kubernetes_property.py @@ -0,0 +1,92 @@ +import enum + +import pydantic + +from acto.common import PropertyPath + + +class KubernetesProperty(pydantic.BaseModel): + """Class specifying how a property is corresponded to a Kubernetes property""" + + kubernetes_schema: str + path: list + + def __hash__(self) -> int: + return hash((self.kubernetes_schema, self.path)) + + def __eq__(self, __value: object) -> bool: + if not isinstance(__value, KubernetesProperty): + return False + return ( + self.kubernetes_schema == __value.kubernetes_schema + and self.path == __value.path + ) + + +class SemanticTag(enum.Enum): + """Enum of Semantic tags, used to associate with test generators""" + + # pylint: disable=invalid-name + Replicas = enum.auto() + ConcurrencyPolicy = enum.auto() + Schedule = enum.auto() + ImagePullPolicy = enum.auto() + Name = enum.auto() + PreemptionPolicy = enum.auto() + RestartPolicy = enum.auto() + + +KUBERNETES_TO_SEMANTIC = { + KubernetesProperty( + kubernetes_schema="io.k8s.api.apps.v1.StatefulSetSpec", + path=["replicas"], + ): SemanticTag.Replicas, + KubernetesProperty( + kubernetes_schema="io.k8s.api.apps.v1.DeploymentSpec", + path=["replicas"], + ): SemanticTag.Replicas, + KubernetesProperty( + kubernetes_schema="io.k8s.api.apps.v1.DeploymentSpec", + path=["replicas"], + ): SemanticTag.Replicas, + KubernetesProperty( + kubernetes_schema="io.k8s.api.batch.v1.CronJobSpec", + path=["concurrencyPolicy"], + ): SemanticTag.ConcurrencyPolicy, + KubernetesProperty( + kubernetes_schema="io.k8s.api.batch.v1.CronJobSpec", + path=["schedule"], + ): SemanticTag.Schedule, + KubernetesProperty( + kubernetes_schema="io.k8s.api.core.v1.Container", + path=["imagePullPolicy"], + ): SemanticTag.ImagePullPolicy, + KubernetesProperty( + kubernetes_schema="io.k8s.api.core.v1.Container", + path=["name"], + ): SemanticTag.Name, + KubernetesProperty( + kubernetes_schema="io.k8s.api.core.v1.PodSpec", + path=["preemptionPolicy"], + ): SemanticTag.PreemptionPolicy, + KubernetesProperty( + kubernetes_schema="io.k8s.api.core.v1.PodSpec", + path=["restartPolicy"], + ): SemanticTag.RestartPolicy, +} + +PROPERTY_TO_SEMANTIC: dict[PropertyPath, SemanticTag] = {} + + +def tag_kubernetes_property_semantic( + kubernetes_property: KubernetesProperty, semantic_tag: SemanticTag +): + """Tag a Kubernetes property with semantic, this is for the anonymous Kubernetes schemas""" + KUBERNETES_TO_SEMANTIC[kubernetes_property] = semantic_tag + + +def tag_property_semantic( + property_path: PropertyPath, semantic_tag: SemanticTag +): + """Tag a property with a semantic tag""" + PROPERTY_TO_SEMANTIC[property_path] = semantic_tag diff --git a/acto/input/property_attribute.py b/acto/input/property_attribute.py new file mode 100644 index 0000000000..72c6ccda66 --- /dev/null +++ b/acto/input/property_attribute.py @@ -0,0 +1,31 @@ +from collections import defaultdict +from enum import Flag, auto + +from acto.common import PropertyPath + + +class PropertyAttribute(Flag): + """PropertyAttribute is a flag that represents the attribute of a property""" + + # pylint: disable=invalid-name + # Patch is a flag that represents the property behaves as a patch + Patch = auto() + + # Mapped is a flag that represents the property is mapped to Kubernetes + # core resource and should be checked by consistency checker + Mapped = auto() + + # Prune is a flag that represents the property should not generate tests + Prune = auto() + + +PROPERTY_ATTRIBUTES: dict[PropertyPath, PropertyAttribute] = defaultdict( + lambda: PropertyAttribute(0) +) + + +def tag_property_attribute( + property_path: list[str], attribute: PropertyAttribute +): + """Tag a property with attribute""" + PROPERTY_ATTRIBUTES[PropertyPath(property_path)] |= attribute diff --git a/acto/input/test_generators/__init__.py b/acto/input/test_generators/__init__.py new file mode 100644 index 0000000000..a3f9f1d630 --- /dev/null +++ b/acto/input/test_generators/__init__.py @@ -0,0 +1,9 @@ +from .cron_job import * +from .deployment import * +from .generator import TEST_GENERATORS, get_testcases, test_generator +from .pod import * +from .primitive import * +from .resource import * +from .service import * +from .stateful_set import * +from .storage import * diff --git a/acto/input/test_generators/cron_job.py b/acto/input/test_generators/cron_job.py new file mode 100644 index 0000000000..da7552becf --- /dev/null +++ b/acto/input/test_generators/cron_job.py @@ -0,0 +1,46 @@ +# pylint: disable=unused-argument +from acto.input.test_generators.generator import Priority, test_generator +from acto.input.testcase import TestCase +from acto.schema.string import StringSchema + + +@test_generator(property_name="concurrentPolicy", priority=Priority.SEMANTIC) +def concurrent_policy_tests(schema: StringSchema) -> list[TestCase]: + """Generate test cases for concurrentPolicy field""" + invalid_test = TestCase( + "k8s-invalid_concurrency_policy_change", + lambda x: True, + lambda x: "InvalidConcurrencyPolicy", + lambda x: "Forbid", + invalid=True, + semantic=True, + ) + change_test = TestCase( + "k8s-concurrency_policy_change", + lambda x: True, + lambda x: "Forbid" if x == "Replace" else "Replace", + lambda x: "Forbid", + semantic=True, + ) + return [invalid_test, change_test] + + +@test_generator(property_name="schedule", priority=Priority.SEMANTIC) +def schedule_tests(schema: StringSchema) -> list[TestCase]: + """Generate test cases for schedule field""" + invalid_test = TestCase( + "k8s-invalid_cronjob_schedule_change", + lambda x: True, + lambda x: "InvalidSchedule", + lambda x: "0 * * * *", + invalid=True, + semantic=True, + ) + change_test = TestCase( + "k8s-cronjob_schedule_change", + lambda x: True, + lambda x: "0 * * * *" if x == "1 * * * *" else "1 * * * *", + lambda x: "0 * * * *", + semantic=True, + ) + return [invalid_test, change_test] diff --git a/acto/input/test_generators/deployment.py b/acto/input/test_generators/deployment.py new file mode 100644 index 0000000000..fa7093ca1d --- /dev/null +++ b/acto/input/test_generators/deployment.py @@ -0,0 +1,27 @@ +# pylint: disable=unused-argument +from acto.input.test_generators.generator import Priority, test_generator +from acto.input.testcase import TestCase +from acto.schema.object import ObjectSchema + + +@test_generator( + k8s_schema_name="apps.v1.DeploymentStrategy", priority=Priority.SEMANTIC +) +def deployment_strategy_tests(schema: ObjectSchema) -> list[TestCase]: + """Generate test cases for deploymentStrategy field""" + invalid_test = TestCase( + "k8s-invalid_deployment_strategy", + lambda x: True, + lambda x: {"type": "INVALID_DEPLOYMENT_STRATEGY"}, + lambda x: None, + invalid=True, + semantic=True, + ) + change_test = TestCase( + "k8s-deployment_strategy_change", + lambda x: x != {"type": "RollingUpdate"}, + lambda x: {"type": "RollingUpdate"}, + lambda x: {"type": "Recreate"}, + semantic=True, + ) + return [invalid_test, change_test] diff --git a/acto/input/test_generators/generator.py b/acto/input/test_generators/generator.py new file mode 100644 index 0000000000..9b3f885567 --- /dev/null +++ b/acto/input/test_generators/generator.py @@ -0,0 +1,281 @@ +"""This module provides a decorator for generating test cases for a schema and +a function to get all test cases for a schema.""" + +import inspect +from dataclasses import dataclass +from enum import IntEnum +from functools import wraps +from typing import Callable, Literal, Optional + +import pytest + +from acto.input.k8s_schemas import KubernetesSchema +from acto.input.property_attribute import PropertyAttribute +from acto.input.testcase import TestCase +from acto.schema import ( + AnyOfSchema, + ArraySchema, + BaseSchema, + BooleanSchema, + IntegerSchema, + NumberSchema, + ObjectSchema, + OneOfSchema, + OpaqueSchema, + StringSchema, +) + + +class Priority(IntEnum): + """Priority enum for test generators""" + + PRIMITIVE = 0 + SEMANTIC = 1 + CUSTOM = 2 + + +@dataclass +class TestGenerator: + """A test generator object""" + + k8s_schema_name: Optional[str] + property_name: Optional[str] + property_type: Optional[ + Literal[ + "AnyOf", + "Array", + "Boolean", + "Integer", + "Number", + "Object", + "OneOf", + "Opaque", + "String", + ] + ] + paths: Optional[list[str]] + priority: Priority + func: Callable[[BaseSchema], list[TestCase]] + + def match( + self, + schema: BaseSchema, + matched_schema: Optional[KubernetesSchema], + ) -> bool: + """Check if the test generator matches the schema""" + return all( + [ + self._match_path(schema), + self._match_property_name(schema), + self._match_property_type(schema), + self._match_k8s_schema_name(matched_schema), + ] + ) + + def _match_path(self, schema: BaseSchema) -> bool: + if not self.paths: + return True + path_str = "/".join(schema.path) + return any(path_str.endswith(path) for path in self.paths) + + def _match_property_name(self, schema: BaseSchema) -> bool: + return ( + self.property_name is None + or len(schema.path) > 0 + and self.property_name == schema.path[-1] + ) + + def _match_property_type(self, schema: BaseSchema) -> bool: + if self.property_type is None: + return True + matching_types = { + "AnyOf": AnyOfSchema, + "Array": ArraySchema, + "Boolean": BooleanSchema, + "Integer": IntegerSchema, + "Number": NumberSchema, + "Object": ObjectSchema, + "OneOf": OneOfSchema, + "Opaque": OpaqueSchema, + "String": StringSchema, + } + if schema_type_obj := matching_types.get(self.property_type): + if isinstance(schema, schema_type_obj): + return True + else: + raise ValueError(f"Unknown schema type: {self.property_type}") + return False + + def _match_k8s_schema_name( + self, matched_schema: Optional[KubernetesSchema] + ) -> bool: + return ( + self.k8s_schema_name is None + or matched_schema is not None + and matched_schema.k8s_schema_name is not None + and matched_schema.k8s_schema_name.endswith(self.k8s_schema_name) + ) + + +# singleton +# global variable for registered test generators +TEST_GENERATORS: list[TestGenerator] = [] + + +def validate_call(func: Callable) -> Callable: + """Validates the `schema` argument type for call to a test generator + function""" + + error_msg = ( + "Argument `schema` of function {} got type {} but expected type {}" + ) + + @wraps(func) + def wrapped_func(*args, **kwargs): + # TODO: handle parameter name other than `schema` and improve error msg + if schema_arg := inspect.signature(func).parameters.get("schema"): + if not isinstance(args[0], schema_arg.annotation): + raise TypeError( + error_msg.format(func, schema_arg.annotation, type(args[0])) + ) + return func(*args, **kwargs) + + return wrapped_func + + +@pytest.mark.skip(reason="not a test") +def test_generator( + k8s_schema_name: Optional[str] = None, + property_name: Optional[str] = None, + property_type: Optional[ + Literal[ + "AnyOf", + "Array", + "Boolean", + "Integer", + "Number", + "Object", + "OneOf", + "Opaque", + "String", + ] + ] = None, + paths: Optional[list[str]] = None, + priority: Priority = Priority.CUSTOM, +) -> Callable[..., Callable[[BaseSchema], list[TestCase]]]: + """Annotates a function as a test generator + + Args: + k8s_schema_name (str, optional): Kubernetes schema name. Defaults to None. + field_name (str, optional): field/property name. Defaults to None. + field_type (str, optional): field/property type. Defaults to None. + paths (list[str], optional): Path suffixes. Defaults to None. + priority (int, optional): Priority. Defaults to 0.""" + assert ( + k8s_schema_name is not None + or property_name is not None + or property_type is not None + or paths is not None + ), "One of k8s_schema_name, schema_name, schema_type, paths must be specified" + + def wrapped_func(func: Callable[[BaseSchema], list[TestCase]]): + func = validate_call(func) + gen_obj = TestGenerator( + k8s_schema_name, + property_name, + property_type, + paths, + priority, + func, + ) + TEST_GENERATORS.append(gen_obj) + return func + + return wrapped_func + + +# TODO: .spec.replicas for StatefulSchema + + +def get_testcases( + schema: BaseSchema, + full_matched_schemas: list[tuple[BaseSchema, KubernetesSchema]], +) -> list[tuple[list[str], list[TestCase]]]: + """Get all test cases for a schema from registered test generators + + Args: + schema (BaseSchema): The schema to generate test cases for + matched_schemas (list[tuple[BaseSchema, KubernetesSchema]]): A list of + matched schemas, including the sub-schemas, but does not include + anonymous schemas + full_matched_schemas (list[tuple[BaseSchema, KubernetesSchema]]): The + complete list of matched schemas, including anonymous schemas + + Returns: + list[tuple[list[str], list[TestCase]]]: A list of tuples, each containing + the path and the test cases for the schema + """ + full_matched_schemas_set: set[str] = { + "/".join(s.path) for s, _ in full_matched_schemas + } + matched_named_schema_dict: dict[str, KubernetesSchema] = { + "/".join(s.path): m + for s, m in full_matched_schemas + if m.k8s_schema_name is not None + } + + def get_testcases_helper( + schema: BaseSchema, + ) -> list[tuple[list[str], list[TestCase]]]: + test_cases: list[tuple[list[str], list[TestCase]]] = [] + generator_candidates: list[TestGenerator] = [] + + # check paths + path_str = "/".join(schema.path) + matched_schema = matched_named_schema_dict.get(path_str) + + for test_generator_ in TEST_GENERATORS: + if test_generator_.match(schema, matched_schema): + generator_candidates.append(test_generator_) + + # sort by priority + generator_candidates.sort(key=lambda x: x.priority, reverse=True) + if len(generator_candidates) > 0: + test_cases.append( + (schema.path, generator_candidates[0].func(schema)), + ) + + # check sub schemas + if isinstance(schema, ArraySchema): + test_cases.extend(get_testcases_helper(schema.get_item_schema())) + elif isinstance(schema, ObjectSchema): + for sub_schema in schema.properties.values(): + test_cases.extend(get_testcases_helper(sub_schema)) + if schema.additional_properties: + test_cases.extend( + get_testcases_helper(schema.additional_properties) + ) + + if path_str in full_matched_schemas_set: + # This schema is a semantic match + # Set kubernetes_schema to True for all test cases + # This is used by the input model to filter out test cases + for _, test_case_list in test_cases: + for test_case in test_case_list: + test_case.kubernetes_schema = True + + if schema.copied_over or schema.over_specified or schema.problematic: + # Prune the test cases + for _, test_case_list in test_cases: + for test_case in test_case_list: + test_case.kubernetes_schema = True + + if schema.attributes & PropertyAttribute.Prune: + # Prune the test cases + for _, test_case_list in test_cases: + for test_case in test_case_list: + test_case.kubernetes_schema = True + + return test_cases + + return get_testcases_helper(schema) diff --git a/acto/input/test_generators/pod.py b/acto/input/test_generators/pod.py new file mode 100644 index 0000000000..5c7ba7a3a9 --- /dev/null +++ b/acto/input/test_generators/pod.py @@ -0,0 +1,541 @@ +# pylint: disable=unused-argument +import enum + +from acto.input.test_generators.generator import Priority, test_generator +from acto.input.testcase import TestCase +from acto.schema.array import ArraySchema +from acto.schema.object import ObjectSchema +from acto.schema.string import StringSchema + +UnAchievableNodeAffinity = { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.io/hostname", + "operator": "In", + "values": [ + "NULL", + ], + } + ] + } + ] + } +} + +AllOnOneNodePodAffinity = { + "requiredDuringSchedulingIgnoredDuringExecution": [ + { + "labelSelector": { + "matchExpressions": [ + { + "key": "app.kubernetes.io/name", + "operator": "In", + "values": ["test-cluster"], + } + ] + }, + "topologyKey": "kubernetes.io/hostname", + } + ] +} + +PlainNodeAffinity = { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "kubernetes.io/hostname", + "operator": "In", + "values": [ + "kind-worker", + "kind-worker2", + "kind-worker3", + "kind-control-plane", + ], + } + ] + } + ] + } +} + +PlainPodAffinity = { + "requiredDuringSchedulingIgnoredDuringExecution": [ + { + "labelSelector": { + "matchExpressions": [ + { + "key": "app.kubernetes.io/name", + "operator": "In", + "values": ["test-cluster"], + } + ] + }, + "topologyKey": "kubernetes.io/os", + } + ] +} + +UnAchievablePodAffinity = { + "requiredDuringSchedulingIgnoredDuringExecution": [ + { + "labelSelector": { + "matchExpressions": [ + { + "key": "app.kubernetes.io/name", + "operator": "In", + "values": ["test-cluster"], + } + ] + }, + "topologyKey": "NULL", + } + ] +} + +AllOnDifferentNodesAntiAffinity = { + "requiredDuringSchedulingIgnoredDuringExecution": [ + { + "labelSelector": { + "matchExpressions": [ + { + "key": "app.kubernetes.io/name", + "operator": "In", + "values": ["test-cluster"], + } + ] + }, + "topologyKey": "kubernetes.io/hostname", + } + ] +} + +AllOnOneNodePodAffinity = { + "requiredDuringSchedulingIgnoredDuringExecution": [ + { + "labelSelector": { + "matchExpressions": [ + { + "key": "app.kubernetes.io/name", + "operator": "In", + "values": ["test-cluster"], + } + ] + }, + "topologyKey": "kubernetes.io/hostname", + } + ] +} + + +class AffinityValues(enum.Enum): + """Some predefined values for affinity""" + + ALL_ON_ONE_NODE = { + "podAffinity": AllOnOneNodePodAffinity, + } + PLAIN = { + "nodeAffinity": PlainNodeAffinity, + } + UNACHIEVABLE = { + "nodeAffinity": UnAchievableNodeAffinity, + } + ALL_ON_DIFFERENT_NODES = { + "podAntiAffinity": AllOnDifferentNodesAntiAffinity, + } + + +@test_generator(k8s_schema_name="core.v1.Affinity", priority=Priority.SEMANTIC) +def affinity_tests(schema: ObjectSchema) -> list[TestCase]: + """Test generator for CoreV1 Affinity""" + all_on_one_node_test = TestCase( + name="k8s-all_on_one_node", + precondition=lambda x: x != AffinityValues.ALL_ON_ONE_NODE, + mutator=lambda x: AffinityValues.ALL_ON_ONE_NODE, + setup=lambda x: None, + semantic=True, + ) + all_on_different_nodes_test = TestCase( + name="k8s-all_on_different_nodes", + precondition=lambda x: x != AffinityValues.ALL_ON_DIFFERENT_NODES, + mutator=lambda x: AffinityValues.ALL_ON_DIFFERENT_NODES, + setup=lambda x: None, + semantic=True, + ) + invalid_test = TestCase( + name="k8s-invalid_affinity", + precondition=lambda x: x != AffinityValues.ALL_ON_DIFFERENT_NODES, + mutator=lambda x: AffinityValues.ALL_ON_DIFFERENT_NODES, + setup=lambda x: None, + invalid=True, + semantic=True, + ) + null_test = TestCase( + name="k8s-null_affinity", + precondition=lambda x: x is not None, + mutator=lambda x: None, + setup=lambda x: AffinityValues.ALL_ON_DIFFERENT_NODES, + semantic=True, + ) + return [ + all_on_one_node_test, + all_on_different_nodes_test, + invalid_test, + null_test, + ] + + +class PodSecurityContextValues(enum.Enum): + """Some predefined values for PodSecurityContext""" + + DEFAULT = { + "runAsGroup": 1000, + "runAsUser": 1000, + "supplementalGroups": [1000], + } + ROOT = { + "runAsGroup": 0, + "runAsUser": 0, + "supplementalGroups": [0], + } + BAD = { + "runAsUser": 500, + "runAsGroup": 500, + "fsGroup": 500, + "supplementalGroups": [500], + } + + +@test_generator( + k8s_schema_name="core.v1.PodSecurityContext", priority=Priority.SEMANTIC +) +def pod_security_context_tests(schema: ObjectSchema) -> list[TestCase]: + """Test generator for PodSecurityContext""" + bad_security_context_test = TestCase( + name="k8s-bad_security_context", + precondition=lambda x: x != PodSecurityContextValues.BAD, + mutator=lambda x: PodSecurityContextValues.BAD, + setup=lambda x: None, + invalid=True, + semantic=True, + ) + root_security_context_test = TestCase( + name="k8s-root_security_context", + precondition=lambda x: x != PodSecurityContextValues.ROOT, + mutator=lambda x: PodSecurityContextValues.ROOT, + setup=lambda x: None, + semantic=True, + ) + normal_security_context_test = TestCase( + name="k8s-normal_security_context", + precondition=lambda x: x != PodSecurityContextValues.DEFAULT, + mutator=lambda x: PodSecurityContextValues.DEFAULT, + setup=lambda x: None, + semantic=True, + ) + return [ + bad_security_context_test, + root_security_context_test, + normal_security_context_test, + ] + + +class TolerationValues(enum.Enum): + """Some predefined values for Toleration""" + + PLAIN = { + "key": "test-key", + "operator": "Equal", + "value": "test-value", + "effect": "NoExecute", + "tolerationSeconds": 3600, + } + CONTROL_PLANE_TOLERATION = { + "key": "node-role.kubernetes.io/control-plane", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 3600, + } + INVALID = { + "key": "test-key", + "operator": "Equal", + "value": "test-value", + "effect": "INVALID_EFFECT", + "tolerationSeconds": 0, + } + + +@test_generator( + k8s_schema_name="core.v1.Toleration", priority=Priority.SEMANTIC +) +def toleration_tests(schema: ObjectSchema) -> list[TestCase]: + """Test generator for Toleration""" + plain_toleration_test = TestCase( + name="k8s-plain_toleration", + precondition=lambda x: x != TolerationValues.PLAIN, + mutator=lambda x: TolerationValues.PLAIN, + setup=lambda x: None, + semantic=True, + ) + control_plane_toleration_test = TestCase( + name="k8s-control_plane_toleration", + precondition=lambda x: x != TolerationValues.CONTROL_PLANE_TOLERATION, + mutator=lambda x: TolerationValues.CONTROL_PLANE_TOLERATION, + setup=lambda x: None, + semantic=True, + ) + invalid_toleration_test = TestCase( + name="k8s-invalid_toleration", + precondition=lambda x: x != TolerationValues.INVALID, + mutator=lambda x: TolerationValues.INVALID, + setup=lambda x: None, + invalid=True, + semantic=True, + ) + return [ + plain_toleration_test, + control_plane_toleration_test, + invalid_toleration_test, + ] + + +@test_generator( + k8s_schema_name="core.v1.Tolerations", priority=Priority.SEMANTIC +) +def tolerations_tests(schema: ArraySchema) -> list[TestCase]: + """Test generator for Tolerations""" + tolerations_pop_test = TestCase( + name="k8s-tolerations_pop", + precondition=lambda x: x and len(x) > 0, + mutator=lambda x: x[:-1], + setup=lambda x: [TolerationValues.PLAIN], + semantic=True, + ) + return [tolerations_pop_test] + + +class ImagePullPolicyValues(enum.Enum): + """Some predefined values for ImagePullPolicy""" + + ALWAYS = "Always" + NEVER = "Never" + IF_NOT_PRESENT = "IfNotPresent" + + +@test_generator(property_name="imagePullPolicy", priority=Priority.SEMANTIC) +def image_pull_policy_tests(schema: StringSchema) -> list[TestCase]: + """Test generator for imagePullPolicy""" + change_test = TestCase( + name="k8s-change_image_pull_policy", + precondition=lambda x: x != ImagePullPolicyValues.ALWAYS, + mutator=lambda x: ImagePullPolicyValues.ALWAYS, + setup=lambda x: ImagePullPolicyValues.NEVER, + semantic=True, + ) + invalid_test = TestCase( + name="k8s-invalid_image_pull_policy", + precondition=lambda x: True, + mutator=lambda x: "INVALID_IMAGE_PULL_POLICY", + setup=lambda x: ImagePullPolicyValues.NEVER, + invalid=True, + semantic=True, + ) + return [change_test, invalid_test] + + +@test_generator( + k8s_schema_name="core.v1.GRPCAction", priority=Priority.SEMANTIC +) +def grpc_action_tests(schema: ObjectSchema) -> list[TestCase]: + """Test generator for grpc action""" + invalid_test = TestCase( + name="k8s-invalid_grpc_action", + precondition=lambda x: True, + mutator=lambda x: {"port": 1234, "service": "invalid-service"}, + setup=lambda x: None, + semantic=True, + invalid=True, + ) + return [invalid_test] + + +@test_generator(k8s_schema_name="core.v1.Probe", priority=Priority.SEMANTIC) +def liveness_probe_tests(schema: ObjectSchema) -> list[TestCase]: + """Test generator for liveness probe""" + invalid_test = TestCase( + name="k8s-http_probe", + precondition=lambda x: True, + mutator=lambda x: {"httpGet": {"path": "/invalid-path"}}, + setup=lambda x: None, + invalid=True, + semantic=True, + ) + invalid_tcp_test = TestCase( + name="k8s-tcp_probe", + precondition=lambda x: True, + mutator=lambda x: {"tcpSocket": {"port": 1234}}, + setup=lambda x: None, + invalid=True, + semantic=True, + ) + invalid_exec_test = TestCase( + name="k8s-exec_probe", + precondition=lambda x: True, + mutator=lambda x: {"exec": {"command": ["invalid-command"]}}, + setup=lambda x: None, + invalid=True, + semantic=True, + ) + return [invalid_test, invalid_tcp_test, invalid_exec_test] + + +@test_generator(k8s_schema_name="core.v1.Container", priority=Priority.SEMANTIC) +def container_tests(schema: ObjectSchema) -> list[TestCase]: + """Test generator for container""" + invalid_test = TestCase( + name="k8s-container_invalid_name", + precondition=lambda x: True, + mutator=lambda x: {"name": "INVALID_NAME", "image": "nginx"}, + setup=lambda x: None, + invalid=True, + semantic=True, + ) + return [invalid_test] + + +@test_generator(property_name="name", priority=Priority.SEMANTIC) +def invalid_name_tests(schema: StringSchema) -> list[TestCase]: + """Test generator for invalid name""" + # TODO: inherit basic tests + invalid_test = TestCase( + name="invalid-name", + precondition=lambda x: True, + mutator=lambda x: "INVALID_NAME", + setup=lambda x: None, + invalid=True, + semantic=True, + ) + return [invalid_test] + + +class PreemptionPolicyValues(enum.Enum): + """Some predefined values for PreemptionPolicy""" + + NEVER = "Never" + PREMEPTION_LOW_PRIORITY = "PreemptLowerPriority" + + +@test_generator(property_name="preemptionPolicy", priority=Priority.SEMANTIC) +def preemption_policy_tests(schema: StringSchema) -> list[TestCase]: + """Test generator for preemption policy""" + policy_change_test = TestCase( + name="k8s-change_preemption_policy", + precondition=lambda x: x != PreemptionPolicyValues.NEVER, + mutator=lambda x: PreemptionPolicyValues.NEVER, + setup=lambda x: PreemptionPolicyValues.PREMEPTION_LOW_PRIORITY, + semantic=True, + ) + return [policy_change_test] + + +@test_generator(property_name="restartPolicy", priority=Priority.SEMANTIC) +def restart_policy_tests(schema: StringSchema) -> list[TestCase]: + """Test generator for restart policy""" + invalid_test = TestCase( + name="k8s-invalid_restart_policy", + precondition=lambda x: True, + mutator=lambda x: "INVALID_RESTART_POLICY", + setup=lambda x: None, + invalid=True, + semantic=True, + ) + change_test = TestCase( + name="k8s-restart_policy_change", + precondition=lambda x: x != "Always", + mutator=lambda x: "Always", + setup=lambda x: "Never", + semantic=True, + ) + return [invalid_test, change_test] + + +@test_generator(property_name="priorityClassName", priority=Priority.SEMANTIC) +def priority_class_name_tests(schema: StringSchema) -> list[TestCase]: + """Test generator for priority class name""" + invalid_test = TestCase( + name="k8s-invalid_priority_class_name", + precondition=lambda x: True, + mutator=lambda x: "INVALID_PRIORITY_CLASS_NAME", + setup=lambda x: None, + invalid=True, + semantic=True, + ) + change_test = TestCase( + name="k8s-priority_class_name_change", + precondition=lambda x: x != "system-cluster-critical", + mutator=lambda x: "system-cluster-critical", + setup=lambda x: "system-node-critical", + semantic=True, + ) + return [invalid_test, change_test] + + +@test_generator(property_name="serviceAccountName", priority=Priority.SEMANTIC) +def service_account_name_tests(schema: StringSchema) -> list[TestCase]: + """Test generator for service account name""" + invalid_test = TestCase( + name="invalid-service-account-name", + precondition=lambda x: True, + mutator=lambda x: "INVALID_SERVICE_ACCOUNT_NAME", + setup=lambda x: None, + invalid=True, + semantic=True, + ) + change_test = TestCase( + name="k8s-service_account_name_change", + precondition=lambda x: x != "default", + mutator=lambda x: "default", + setup=lambda x: "system:serviceaccount:default:default", + semantic=True, + ) + return [invalid_test, change_test] + + +@test_generator(property_name="whenUnsatisfiable", priority=Priority.SEMANTIC) +def when_unsatisfiable_tests(schema: StringSchema) -> list[TestCase]: + """Test generator for when unsatisfiable""" + invalid_test = TestCase( + name="k8s-invalid_value", + precondition=lambda x: True, + mutator=lambda x: "INVALID_WHEN_UNSATISFIABLE", + setup=lambda x: None, + invalid=True, + semantic=True, + ) + change_test = TestCase( + name="k8s-when_unsatisfiable_change", + precondition=lambda x: x != "ScheduleAnyway", + mutator=lambda x: "ScheduleAnyway", + setup=lambda x: None, + semantic=True, + ) + return [invalid_test, change_test] + + +@test_generator( + k8s_schema_name="core.v1.TopologySpreadConstraint", + priority=Priority.SEMANTIC, +) +def topology_spread_constraint_tests(schema: ObjectSchema) -> list[TestCase]: + """Test generator for topology spread constraint""" + invalid_test = TestCase( + name="k8s-invalid_topology_spread_constraint", + precondition=lambda x: True, + mutator=lambda x: {"topologyKey": "INVALID_TOPOLOGY_KEY"}, + setup=lambda x: None, + ) + return [invalid_test] diff --git a/acto/input/test_generators/primitive.py b/acto/input/test_generators/primitive.py new file mode 100644 index 0000000000..739e7a2d03 --- /dev/null +++ b/acto/input/test_generators/primitive.py @@ -0,0 +1,767 @@ +"""Testcase generators for primitive types""" + +# pylint: disable=unused-argument, invalid-name, unused-variable + +import random + +import exrex + +from acto.input.test_generators.generator import Priority, test_generator +from acto.input.testcase import EnumTestCase, SchemaPrecondition, TestCase +from acto.schema import ( + AnyOfSchema, + ArraySchema, + BaseSchema, + BooleanSchema, + IntegerSchema, + ObjectSchema, + OpaqueSchema, + StringSchema, +) +from acto.utils.thread_logger import get_thread_logger + + +def resolve_testcases(schema: BaseSchema) -> list[TestCase]: + """Get testcases for a schema""" + if isinstance(schema, AnyOfSchema): + return any_of_tests(schema) + elif isinstance(schema, ArraySchema): + return array_tests(schema) + elif isinstance(schema, BooleanSchema): + return boolean_tests(schema) + elif isinstance(schema, IntegerSchema): + return integer_tests(schema) + # elif isinstance(schema, NumberSchema): + # return number_tests(schema) + elif isinstance(schema, ObjectSchema): + return object_tests(schema) + elif isinstance(schema, OpaqueSchema): + return [] + elif isinstance(schema, StringSchema): + return string_tests(schema) + else: + raise NotImplementedError + + +@test_generator(property_type="AnyOf", priority=Priority.PRIMITIVE) +def any_of_tests(schema: AnyOfSchema): + """Generate testcases for AnyOf type""" + + ret: list[TestCase] = [] + if schema.enum is not None: + for case in schema.enum: + ret.append(EnumTestCase(case, primitive=True)) + else: + for sub_schema in schema.possibilities: + testcases = resolve_testcases(sub_schema) + for testcase in testcases: + testcase.add_precondition( + SchemaPrecondition(sub_schema).precondition + ) + ret.extend(testcases) + return ret + + +@test_generator(property_type="Array", priority=Priority.PRIMITIVE) +def array_tests(schema: ArraySchema): + """Representation of an array node + + It handles + - minItems + - maxItems + - items + - uniqueItems + """ + default_min_items = 0 + default_max_items = 5 + + DELETION_TEST = "array-deletion" + PUSH_TEST = "array-push" + POP_TEST = "array-pop" + EMPTY_TEST = "array-empty" + + def push_precondition(prev): + if prev is None: + return False + if len(prev) >= schema.max_items: + return False + return True + + def push_mutator(prev): + new_item = schema.item_schema.gen() + return prev + [new_item] + + def push_setup(prev): + logger = get_thread_logger(with_prefix=True) + if len(schema.examples) > 0: + for example in schema.examples: + if len(example) > 1: + logger.info( + "Using example for setting up field [%s]: [%s]", + schema.path, + schema.examples[0], + ) + return example + if prev is None: + return schema.gen() + return schema.gen(size=schema.min_items) + + def pop_precondition(prev): + if prev is None: + return False + if len(prev) <= schema.min_items: + return False + if len(prev) == 0: + return False + return True + + def pop_mutator(prev): + prev.pop() + return prev + + def pop_setup(prev): + logger = get_thread_logger(with_prefix=True) + + if len(schema.examples) > 0: + for example in schema.examples: + if len(example) > 1: + logger.info( + "Using example for setting up field [%s]: [%s]", + schema.path, + schema.examples[0], + ) + return example + if prev is None: + return schema.gen(size=schema.min_items + 1) + return schema.gen(size=schema.max_items) + + def empty_precondition(prev): + return prev != [] + + def empty_mutator(prev): + return [] + + def empty_setup(prev): + return prev + + def delete(prev): + return schema.empty_value() + + def delete_precondition(prev): + return ( + prev is not None + and prev != schema.default + and prev != schema.empty_value() + ) + + def delete_setup(prev): + logger = get_thread_logger(with_prefix=True) + if len(schema.examples) > 0: + logger.info( + "Using example for setting up field [%s]: [%s]", + schema.path, + schema.examples[0], + ) + example_without_default = [ + x for x in schema.enum if x != schema.default + ] + if len(example_without_default) > 0: + return random.choice(example_without_default) + else: + return schema.gen(exclude_value=schema.default) + else: + return schema.gen(exclude_value=schema.default) + + ret = [ + TestCase( + DELETION_TEST, + delete_precondition, + delete, + delete_setup, + primitive=True, + ) + ] + if schema.enum is not None: + for case in schema.enum: + ret.append(EnumTestCase(case, primitive=True)) + else: + ret.append( + TestCase( + PUSH_TEST, + push_precondition, + push_mutator, + push_setup, + primitive=True, + ) + ) + ret.append( + TestCase( + POP_TEST, + pop_precondition, + pop_mutator, + pop_setup, + primitive=True, + ) + ) + ret.append( + TestCase( + EMPTY_TEST, + empty_precondition, + empty_mutator, + empty_setup, + primitive=True, + ) + ) + return ret + + +@test_generator(property_type="Boolean", priority=Priority.PRIMITIVE) +def boolean_tests(schema: BooleanSchema): + """Generate testcases for Boolean type""" + DELETION_TEST = "boolean-deletion" + TOGGLE_OFF_TEST = "boolean-toggle-off" + TOGGLE_ON_TEST = "boolean-toggle-on" + + def toggle_on_precondition(prev): + if prev is None and schema.default is False: + return True + elif prev is False: + return True + else: + return False + + def toggle_on(prev): + return True + + def toggle_on_setup(prev): + return False + + def toggle_off_precondition(prev): + if prev is None and schema.default is True: + return True + elif prev is True: + return True + else: + return False + + def toggle_off(prev): + return False + + def toggle_off_setup(prev): + return True + + def delete(prev): + return schema.empty_value() + + def delete_precondition(prev): + return ( + prev is not None + and prev != schema.default + and prev != schema.empty_value() + ) + + def delete_setup(prev): + logger = get_thread_logger(with_prefix=True) + if len(schema.examples) > 0: + logger.info( + "Using example for setting up field [%s]: [%s]", + schema.path, + schema.examples[0], + ) + example_without_default = [ + x for x in schema.enum if x != schema.default + ] + if len(example_without_default) > 0: + return random.choice(example_without_default) + else: + return schema.gen(exclude_value=schema.default) + else: + return schema.gen(exclude_value=schema.default) + + ret = [ + TestCase( + DELETION_TEST, + delete_precondition, + delete, + delete_setup, + primitive=True, + ) + ] + if schema.enum is not None: + for case in schema.enum: + ret.append(EnumTestCase(case, primitive=True)) + else: + ret.append( + TestCase( + TOGGLE_OFF_TEST, + toggle_off_precondition, + toggle_off, + toggle_off_setup, + primitive=True, + ) + ) + ret.append( + TestCase( + TOGGLE_ON_TEST, + toggle_on_precondition, + toggle_on, + toggle_on_setup, + primitive=True, + ) + ) + return ret + + +# Commented out because there is no need to support number type yet +# @test_generator(property_type="Number", priority=Priority.PRIMITIVE) +# def number_tests(schema: NumberSchema): +# """Generate testcases for Number type + +# It handles +# - minimum +# - maximum +# - exclusiveMinimum +# - exclusiveMaximum +# - multipleOf +# """ + +# default_minimum = 0 +# default_maximum = 5 + +# DELETION_TEST = "number-deletion" +# INCREASE_TEST = "number-increase" +# DECREASE_TEST = "number-decrease" +# EMPTY_TEST = "number-empty" +# CHANGE_TEST = "number-change" + +# def increase_precondition(prev): +# if prev is None: +# return False +# if schema.multiple_of is None: +# return prev < schema.maximum +# else: +# return prev < schema.maximum - schema.multiple_of + +# def increase(prev): +# if schema.multiple_of is not None: +# return prev + schema.multiple_of +# else: +# return random.uniform(prev, schema.maximum) + +# def increase_setup(prev): +# return schema.minimum + +# def decrease_precondition(prev): +# if prev is None: +# return False +# if schema.multiple_of is None: +# return prev > schema.minimum +# else: +# return prev > schema.minimum + schema.multiple_of + +# def decrease(prev): +# if schema.multiple_of is not None: +# return prev - schema.multiple_of +# else: +# return random.uniform(schema.minimum, prev) + +# def decrease_setup(prev): +# return schema.maximum + +# def empty_precondition(prev): +# return prev != 0 + +# def empty_mutator(prev): +# return 0 + +# def empty_setup(prev): +# return 1 + +# def change_precondition(prev): +# return prev is not None + +# def change(prev): +# """Test case to change the value to another one""" +# logger = get_thread_logger(with_prefix=True) +# if schema.enum is not None: +# logger.fatal( +# "Number field with enum should not call change to mutate" +# ) +# if schema.multiple_of is not None: +# new_number = random.randrange( +# schema.minimum, schema.maximum + 1, schema.multiple_of +# ) +# else: +# new_number = random.uniform(schema.minimum, schema.maximum) +# if prev == new_number: +# logger.error( +# "Failed to change, generated the same number with previous one" +# ) +# return new_number + +# def change_setup(prev): +# return 2 + +# def delete(prev): +# return schema.empty_value() + +# def delete_precondition(prev): +# return ( +# prev is not None +# and prev != schema.default +# and prev != schema.empty_value() +# ) + +# def delete_setup(prev): +# logger = get_thread_logger(with_prefix=True) +# if len(schema.examples) > 0: +# logger.info( +# "Using example for setting up field [%s]: [%s]", +# schema.path, +# schema.examples[0], +# ) +# example_without_default = [ +# x for x in schema.enum if x != schema.default +# ] +# if len(example_without_default) > 0: +# return random.choice(example_without_default) +# else: +# return schema.gen(exclude_value=schema.default) +# else: +# return schema.gen(exclude_value=schema.default) + +# testcases = [ +# TestCase(DELETION_TEST, delete_precondition, delete, delete_setup) +# ] +# if schema.enum is not None: +# for case in schema.enum: +# testcases.append(EnumTestCase(case)) +# else: +# testcases.append( +# TestCase(CHANGE_TEST, change_precondition, change, change_setup) +# ) +# return testcases + + +@test_generator(property_type="Integer", priority=Priority.PRIMITIVE) +def integer_tests(schema: IntegerSchema): + """Generate testcases for Integer type + + It handles + - minimum + - maximum + - exclusiveMinimum + - exclusiveMaximum + - multipleOf + """ + default_minimum = 0 + default_maximum = 5 + + DELETION_TEST = "integer-deletion" + INCREASE_TEST = "integer-increase" + DECREASE_TEST = "integer-decrease" + EMPTY_TEST = "integer-empty" + CHANGE_TEST = "integer-change" + + def increase_precondition(prev): + if prev is None: + return False + if schema.multiple_of is None: + return prev < schema.maximum + else: + return prev < schema.maximum - schema.multiple_of + + def increase(prev): + if schema.multiple_of is not None: + return random.randrange( + prev, schema.maximum + 1, schema.multiple_of + ) + else: + return min(schema.maximum, prev * 2) + + def increase_setup(prev): + return schema.minimum + + def decrease_precondition(prev): + if prev is None: + return False + if schema.multiple_of is None: + return prev > schema.minimum + else: + return prev > schema.minimum + schema.multiple_of + + def decrease(prev): + if schema.multiple_of is not None: + return random.randrange(schema.minimum, prev, schema.multiple_of) + else: + return max(schema.minimum, int(prev / 2)) + + def decrease_setup(prev): + return schema.maximum + + def change_precondition(prev): + return prev is not None + + def change(prev): + return prev + 2 + + def change_setup(prev): + return 2 + + def delete(prev): + return schema.empty_value() + + def delete_precondition(prev): + return ( + prev is not None + and prev != schema.default + and prev != schema.empty_value() + ) + + def delete_setup(prev): + logger = get_thread_logger(with_prefix=True) + if len(schema.examples) > 0: + logger.info( + "Using example for setting up field [%s]: [%s]", + schema.path, + schema.examples[0], + ) + example_without_default = [ + x for x in schema.enum if x != schema.default + ] + if len(example_without_default) > 0: + return random.choice(example_without_default) + else: + return schema.gen(exclude_value=schema.default) + else: + return schema.gen(exclude_value=schema.default) + + testcases = [ + TestCase( + DELETION_TEST, + delete_precondition, + delete, + delete_setup, + primitive=True, + ) + ] + if schema.enum is not None: + for case in schema.enum: + testcases.append(EnumTestCase(case)) + else: + testcases.append( + TestCase( + CHANGE_TEST, + change_precondition, + change, + change_setup, + primitive=True, + ) + ) + return testcases + + +@test_generator(property_type="Object", priority=Priority.PRIMITIVE) +def object_tests(schema: ObjectSchema): + """Generate testcases for Object type + + It handles + - properties + - additionalProperties + - required + - minProperties + - maxProperties + TODO: + - dependencies + - patternProperties + - regexp + """ + + DELETION_TEST = "object-deletion" + EMPTY_TEST = "object-empty" + + def empty_precondition(prev): + return prev != {} + + def empty_mutator(prev): + return {} + + def empty_setup(prev): + return prev + + def delete(prev): + return schema.empty_value() + + def delete_precondition(prev): + return ( + prev is not None + and prev != schema.default + and prev != schema.empty_value() + ) + + def delete_setup(prev): + logger = get_thread_logger(with_prefix=True) + if len(schema.examples) > 0: + logger.info( + "Using example for setting up field [%s]: [%s]", + schema.path, + schema.examples[0], + ) + example_without_default = [ + x for x in schema.enum if x != schema.default + ] + if len(example_without_default) > 0: + return random.choice(example_without_default) + else: + return schema.gen(exclude_value=schema.default) + else: + return schema.gen(exclude_value=schema.default) + + ret = [ + TestCase( + DELETION_TEST, + delete_precondition, + delete, + delete_setup, + primitive=True, + ) + ] + if schema.enum is not None: + for case in schema.enum: + ret.append(EnumTestCase(case)) + else: + ret.append( + TestCase( + EMPTY_TEST, + empty_precondition, + empty_mutator, + empty_setup, + primitive=True, + ) + ) + return ret + + +@test_generator(property_type="Opaque", priority=Priority.PRIMITIVE) +def opaque_gen(schema: OpaqueSchema): + """Opaque schema to handle the fields that do not have a schema""" + return [] + + +@test_generator(property_type="String", priority=Priority.PRIMITIVE) +def string_tests(schema: StringSchema): + """Representation of a generic value generator for string schema + + It handles + - minLength + - maxLength + - pattern + """ + default_max_length = 10 + DELETION_TEST = "string-deletion" + CHANGE_TEST = "string-change" + EMPTY_TEST = "string-empty" + + def change_precondition(prev): + return prev is not None + + def change(prev): + """Test case to change the value to another one""" + logger = get_thread_logger(with_prefix=True) + if schema.enum is not None: + logger.fatal( + "String field with enum should not call change to mutate" + ) + if schema.pattern is not None: + new_string = exrex.getone(schema.pattern, schema.max_length) + else: + new_string = "ACTOKEY" + if prev == new_string: + logger.error( + "Failed to change, generated the same string with previous one" + ) + return new_string + + def change_setup(prev): + logger = get_thread_logger(with_prefix=True) + if len(schema.examples) > 0: + logger.info( + "Using example for setting up field [%s]: [%s]", + schema.path, + schema.examples[0], + ) + return schema.examples[0] + else: + return schema.gen() + + def empty_precondition(prev): + return prev != "" + + def empty_mutator(prev): + return "" + + def empty_setup(prev): + return prev + + def delete(prev): + return schema.empty_value() + + def delete_precondition(prev): + return ( + prev is not None + and prev != schema.default + and prev != schema.empty_value() + ) + + def delete_setup(prev): + logger = get_thread_logger(with_prefix=True) + if len(schema.examples) > 0: + logger.info( + "Using example for setting up field [%s]: [%s]", + schema.path, + schema.examples[0], + ) + example_without_default = [ + x for x in schema.enum if x != schema.default + ] + if len(example_without_default) > 0: + return random.choice(example_without_default) + else: + return schema.gen(exclude_value=schema.default) + else: + return schema.gen(exclude_value=schema.default) + + ret = [ + TestCase( + DELETION_TEST, + delete_precondition, + delete, + delete_setup, + primitive=True, + ) + ] + if schema.enum is not None: + for case in schema.enum: + ret.append(EnumTestCase(case, primitive=True)) + else: + change_testcase = TestCase( + CHANGE_TEST, + change_precondition, + change, + change_setup, + primitive=True, + ) + ret.append(change_testcase) + ret.append( + TestCase( + EMPTY_TEST, + empty_precondition, + empty_mutator, + empty_setup, + primitive=True, + ) + ) + return ret + + +# TODO: Default test generator for Kubernetes schemas diff --git a/acto/input/test_generators/resource.py b/acto/input/test_generators/resource.py new file mode 100644 index 0000000000..77f026d8c4 --- /dev/null +++ b/acto/input/test_generators/resource.py @@ -0,0 +1,78 @@ +# pylint: disable=unused-argument +from acto.input.test_generators.generator import Priority, test_generator +from acto.input.testcase import TestCase +from acto.k8s_util.k8sutil import ( + canonicalize_quantity, + double_quantity, + half_quantity, +) +from acto.schema.base import BaseSchema +from acto.schema.object import ObjectSchema + + +@test_generator( + k8s_schema_name="apimachinery.pkg.api.resource.Quantity", + priority=Priority.SEMANTIC, +) +def quantity_tests(schema: BaseSchema) -> list[TestCase]: + """Generate test cases for quantity field""" + increase_test = TestCase( + "k8s-quantity_increase", + lambda x: x is not None and canonicalize_quantity(x) != "INVALID", + double_quantity, + lambda x: "2000m", + ) + decrease_test = TestCase( + "k8s-quantity_decrease", + lambda x: x is not None and canonicalize_quantity(x) != "INVALID", + half_quantity, + lambda x: "1000m", + ) + return [increase_test, decrease_test] + + +@test_generator( + k8s_schema_name="core.v1.ResourceRequirements", priority=Priority.SEMANTIC +) +def resource_requirements_tests(schema: ObjectSchema) -> list[TestCase]: + """Generate test cases for resourceRequirements field""" + invalid_test = TestCase( + "invalid-resourceRequirements", + lambda x: True, + lambda x: {"limits": {"hugepages-2Mi": "1000m"}}, + lambda x: None, + invalid=True, + semantic=True, + ) + change_test = TestCase( + "resourceRequirements-change", + lambda x: x != {"limits": {"cpu": "1000m"}}, + lambda x: {"limits": {"cpu": "1000m"}}, + lambda x: {"limits": {"cpu": "2000m"}}, + semantic=True, + ) + return [invalid_test, change_test] + + +@test_generator( + k8s_schema_name="core.v1.VolumeResourceRequirements", + priority=Priority.SEMANTIC, +) +def volume_resource_requirements_tests(schema: ObjectSchema) -> list[TestCase]: + """Generate test cases for volumeResourceRequirements field""" + invalid_test = TestCase( + "k8s-invalid-volumeResourceRequirements", + lambda x: True, + lambda x: {"request": {"INVALID": "1000m"}}, + lambda x: None, + invalid=True, + semantic=True, + ) + change_test = TestCase( + "k8s-volumeResourceRequirements-change", + lambda x: x != {"request": {"storage": "1000Mi"}}, + lambda x: {"request": {"storage": "1000Mi"}}, + lambda x: {"request": {"storage": "2000Mi"}}, + semantic=True, + ) + return [invalid_test, change_test] diff --git a/acto/input/test_generators/service.py b/acto/input/test_generators/service.py new file mode 100644 index 0000000000..cc0c8ccbc5 --- /dev/null +++ b/acto/input/test_generators/service.py @@ -0,0 +1,44 @@ +# pylint: disable=unused-argument +from acto.input.test_generators.generator import Priority, test_generator +from acto.input.testcase import TestCase +from acto.schema.object import ObjectSchema + + +def service_type_tests(schema: ObjectSchema) -> list[TestCase]: + """Generate test cases for serviceType field""" + invalid_test = TestCase( + "invalid-serviceType", + lambda x: True, + lambda x: "InvalidServiceType", + lambda x: "ClusterIP", + ) + change_test = TestCase( + "serviceType-change", + lambda x: x != "NodePort", + lambda x: "NodePort", + lambda x: "ClusterIP", + ) + return [invalid_test, change_test] + + +@test_generator( + k8s_schema_name="networking.v1.IngressTLS", priority=Priority.SEMANTIC +) +def ingress_tls_tests(schema: ObjectSchema) -> list[TestCase]: + """Generate test cases for ingressTLS field""" + invalid_test = TestCase( + "k8s-non_existent_secret", + lambda x: True, + lambda x: {"hosts": ["test.com"], "secretName": "non-existent"}, + lambda x: None, + invalid=True, + semantic=True, + ) + change_test = TestCase( + "k8s-ingressTLS-change", + lambda x: x != {"hosts": ["example.com"]}, + lambda x: {"hosts": ["example.com"]}, + lambda x: {"hosts": ["example.org"]}, + semantic=True, + ) + return [invalid_test, change_test] diff --git a/acto/input/test_generators/stateful_set.py b/acto/input/test_generators/stateful_set.py new file mode 100644 index 0000000000..03aaf3da1c --- /dev/null +++ b/acto/input/test_generators/stateful_set.py @@ -0,0 +1,84 @@ +# pylint: disable=unused-argument +from acto.input.test_generators.generator import Priority, test_generator +from acto.input.testcase import Store, TestCase +from acto.schema.integer import IntegerSchema +from acto.schema.object import ObjectSchema + + +def scale_down_up_precondition(prev, store: Store) -> bool: + """Precondition for scaleDownUp test case""" + if store.data is None: + store.data = True + return False + else: + return True + + +def scale_up_down_precondition(prev, store: Store) -> bool: + """Precondition for scaleUpDown test case""" + if store.data is None: + store.data = True + return False + else: + return True + + +@test_generator(property_name="replicas", priority=Priority.SEMANTIC) +def replicas_tests(schema: IntegerSchema) -> list[TestCase]: + """Generate test cases for replicas field""" + invalid_test = TestCase( + "k8s-invalid_replicas", + lambda x: True, + lambda x: 0, + lambda x: 1, + invalid=True, + semantic=True, + ) + scale_down_up_test = TestCase( + "k8s-scaleDownUp", + scale_down_up_precondition, + lambda x: 4 if x is None else x + 2, + lambda x: 1 if x is None else x - 2, + Store(), + semantic=True, + ) + scale_up_down_test = TestCase( + "k8s-scaleUpDown", + scale_up_down_precondition, + lambda x: 1 if x is None else x - 2, + lambda x: 5 if x is None else x + 2, + Store(), + semantic=True, + ) + overload_test = TestCase( + "k8s-overload", + lambda x: True, + lambda x: 1000, + lambda x: 1, + invalid=True, + semantic=True, + ) + return [invalid_test, scale_down_up_test, scale_up_down_test, overload_test] + + +@test_generator( + property_name="statefulSetUpdateStrategy", priority=Priority.SEMANTIC +) +def stateful_set_update_strategy_tests(schema: ObjectSchema) -> list[TestCase]: + """Generate test cases for StatefulSetUpdateStrategy field""" + invalid_test = TestCase( + "k8s-invalid_update_strategy", + lambda x: True, + lambda x: {"type": "INVALID_STATEFUL_SET_UPDATE_STRATEGY"}, + lambda x: None, + invalid=True, + semantic=True, + ) + change_test = TestCase( + "k8s-update_strategy_change", + lambda x: x != {"type": "RollingUpdate"}, + lambda x: {"type": "RollingUpdate"}, + lambda x: {"type": "OnDelete"}, + semantic=True, + ) + return [invalid_test, change_test] diff --git a/acto/input/test_generators/storage.py b/acto/input/test_generators/storage.py new file mode 100644 index 0000000000..2543ee47d9 --- /dev/null +++ b/acto/input/test_generators/storage.py @@ -0,0 +1,41 @@ +# pylint: disable=unused-argument +from acto.input.test_generators.generator import Priority, test_generator +from acto.input.testcase import TestCase +from acto.schema.array import ArraySchema +from acto.schema.string import StringSchema + + +@test_generator(property_name="accessModes", priority=Priority.SEMANTIC) +def access_mode_tests(schema: ArraySchema) -> list[TestCase]: + """Generate test cases for accessModes field""" + invalid_test = TestCase( + "k8s-invalid_access_mode", + lambda x: True, + lambda x: ["InvalidAccessMode"], + lambda x: ["ReadWriteOnce"], + invalid=True, + semantic=True, + ) + change_test = TestCase( + "k8s-change_access_mode", + lambda x: True, + lambda x: ["ReadWriteOnce"] + if x == ["ReadWriteMany"] + else ["ReadWriteMany"], + lambda x: ["ReadWriteOnce"], + semantic=True, + ) + return [invalid_test, change_test] + + +@test_generator(property_name="apiVersion", priority=Priority.SEMANTIC) +def api_version_tests(schema: StringSchema) -> list[TestCase]: + """Generate test cases for apiVersion field""" + change_test = TestCase( + "k8s-change_api_version", + lambda x: True, + lambda x: "v1" if x == "v2" else "v2", + lambda x: "v1", + semantic=True, + ) + return [change_test] diff --git a/acto/input/testcase.py b/acto/input/testcase.py index 9fb0818d11..ff8626d777 100644 --- a/acto/input/testcase.py +++ b/acto/input/testcase.py @@ -1,24 +1,37 @@ -from typing import Callable, Any, Union +import dataclasses +from typing import Any, Callable, Optional, Union from acto.utils import get_thread_logger +@dataclasses.dataclass class Store: + """A store to store data for test cases""" def __init__(self) -> None: - self.data = None + self.data: Any = None class TestCase: + """Class represent a test case""" + # disable pytest to collect this class __test__ = False - def __init__(self, - name: str, - precondition: Union[Callable[[Any], bool], Callable[[Any, Store], bool]], - mutator: Callable[[Any], Any], - setup: Callable[[Any], Any], - store: Store = None) -> None: + def __init__( + self, + name: str, + precondition: Union[ + Callable[[Any], bool], Callable[[Any, Store], bool] + ], + mutator: Callable[[Any], Any], + setup: Callable[[Any], Any], + store: Optional[Store] = None, + primitive: bool = False, + semantic: bool = False, + invalid: bool = False, + kubernetes_schema: bool = False, + ) -> None: """Class represent a test case Args: @@ -36,7 +49,14 @@ def __init__(self, self.setup = setup self.store = store + # test attributes + self.primitive = primitive + self.semantic = semantic + self.invalid = invalid + self.kubernetes_schema = kubernetes_schema + def test_precondition(self, prev) -> bool: + """Test whether the previous value satisfies the precondition""" logger = get_thread_logger(with_prefix=True) ret = True @@ -45,23 +65,28 @@ def test_precondition(self, prev) -> bool: ret = ret and precondition(prev, self.store) else: ret = ret and precondition(prev) - logger.debug('Precondition [%s] Result [%s]' % (precondition.__name__, ret)) + logger.debug( + "Precondition [%s] Result [%s]", precondition.__name__, ret + ) return ret def run_setup(self, prev): + """Run setup to set up the precondition""" return self.setup(prev) - def add_precondition(self, precondition: callable): + def add_precondition(self, precondition: Callable): + """Add a precondition to the test case""" self.preconditions.append(precondition) def __str__(self) -> str: return self.name def to_dict(self) -> dict: + """serialize to dict""" ret = { - 'precondition': list(map(lambda x: x.__name__, self.preconditions)), - 'mutator': self.mutator.__name__ + "precondition": list(map(lambda x: x.__name__, self.preconditions)), + "mutator": self.mutator.__name__, } return ret @@ -69,51 +94,67 @@ def to_dict(self) -> dict: class K8sTestCase(TestCase): """Class represent a test case for k8s, purely for test case name purposes""" - def __init__(self, - precondition: Callable, - mutator: Callable, - setup: Callable, - store: Store = None) -> None: - name = f'k8s-{mutator.__name__}' + def __init__( + self, + precondition: Callable, + mutator: Callable, + setup: Callable, + store: Optional[Store] = None, + ) -> None: + name = f"k8s-{mutator.__name__}" super().__init__(name, precondition, mutator, setup, store) class K8sInvalidTestCase(K8sTestCase): """Class represent a test case for k8s, purely for test case name purposes""" - def __init__(self, - precondition: Callable, - mutator: Callable, - setup: Callable, - store: Store = None) -> None: + def __init__( + self, + precondition: Callable, + mutator: Callable, + setup: Callable, + store: Store = None, + ) -> None: super().__init__(precondition, mutator, setup, store) class EnumTestCase(TestCase): + """Class represent a test case for enum fields""" - def __init__(self, case) -> None: + def __init__( + self, + case, + primitive: bool = False, + ) -> None: self.case = case - super().__init__(case, self.enum_precondition, self.enum_mutator, self.setup) + super().__init__( + case, self.enum_precondition, self.enum_mutator, self.enum_setup + ) @staticmethod def enum_precondition(_): + """Always true""" return True def enum_mutator(self, _): + """Always return the case""" return self.case @staticmethod - def setup(_): + def enum_setup(_): """Never going to be called""" # FIXME: assert happens for rabbitmq's service.type field assert () class SchemaPrecondition: + """Precondition for schema validation""" + # XXX: Probably could use partial def __init__(self, schema) -> None: self.schema = schema def precondition(self, prev): + """Precondition for schema validation""" return self.schema.validate(prev) diff --git a/acto/input/value_with_schema.py b/acto/input/value_with_schema.py index 4423507638..c47819100f 100644 --- a/acto/input/value_with_schema.py +++ b/acto/input/value_with_schema.py @@ -1,64 +1,88 @@ +import enum import random import string from abc import abstractmethod - -import yaml - -from acto.schema import (AnyOfSchema, ArraySchema, BooleanSchema, - IntegerSchema, NumberSchema, ObjectSchema, - OpaqueSchema, StringSchema) +from typing import Optional + +from acto.schema import ( + AnyOfSchema, + ArraySchema, + BooleanSchema, + IntegerSchema, + NumberSchema, + ObjectSchema, + OpaqueSchema, + StringSchema, +) from acto.utils import get_thread_logger -class ValueWithSchema(): +class ValueWithSchema: + """A concrete value with a schema attached""" def __init__(self) -> None: pass @abstractmethod def raw_value(self) -> object: + """Return the raw value of the object""" return None @abstractmethod - def mutate(self): + def mutate(self, p_delete=0.05, p_replace=0.1): + """Mutate a small portion of the value""" return @abstractmethod - def update(self): + def update(self, value): + """Update the value with a new value""" return @abstractmethod def get_value_by_path(self, path: list): + """Fetch the value specified by path""" return @abstractmethod def create_path(self, path: list): + """Ensures the path exists""" return @abstractmethod def set_value_by_path(self, value, path): + """Set the value specified by path""" + return + + @abstractmethod + def value(self): + """Return the value""" return class ValueWithObjectSchema(ValueWithSchema): + """Value with ObjectSchema attached""" def __init__(self, value, schema) -> None: self.schema = schema - if value == None: + if value is None: self.store = None elif isinstance(value, dict): self.store = {} for k, v in value.items(): - self.store[k] = attach_schema_to_value(v, self.schema.get_property_schema(k)) + self.store[k] = attach_schema_to_value( + v, self.schema.get_property_schema(k) + ) else: - raise TypeError('Value [%s] has type [%s] Path [%s]' % - (value, type(value), self.schema.get_path())) + raise TypeError( + f"Value [{value}] has type [{type(value)}] Path [{self.schema.get_path()}]" + ) def value(self): + """Return the value""" return self.store def __str__(self) -> str: - if self.store == None: + if self.store is None: ret = None else: ret = {} @@ -67,7 +91,7 @@ def __str__(self) -> str: return str(ret) def raw_value(self) -> dict: - if self.store == None: + if self.store is None: return None else: ret = {} @@ -76,16 +100,16 @@ def raw_value(self) -> dict: return ret def mutate(self, p_delete=0.05, p_replace=0.1): - '''Mutate a small portion of the value - + """Mutate a small portion of the value + - Replace with null - Replace with a new value - mutate a child TODO: generate a property that didn't exist before - ''' + """ logger = get_thread_logger(with_prefix=True) - if self.store == None: + if self.store is None: value = self.schema.gen() self.update(value) else: @@ -99,33 +123,44 @@ def mutate(self, p_delete=0.05, p_replace=0.1): properties = self.schema.get_properties() if len(properties) == 0: # XXX: Handle additional properties better - if self.schema.get_additional_properties() == None: - logger.warning('Object schema is opaque %s', self.schema.get_path()) + if self.schema.get_additional_properties() is None: + logger.warning( + "Object schema is opaque %s", self.schema.get_path() + ) return else: letters = string.ascii_lowercase - key = ''.join(random.choice(letters) for i in range(5)) - self.__setitem__(key, self.schema.get_additional_properties().gen()) + key = "".join(random.choice(letters) for i in range(5)) + self[ + key + ] = self.schema.get_additional_properties().gen() else: - child_key = random.choice(list(self.schema.get_properties())) + child_key = random.choice( + list(self.schema.get_properties()) + ) if child_key not in self.store: - self.__setitem__(child_key, - self.schema.get_property_schema(child_key).gen()) + self[child_key] = ( + self.schema.get_property_schema(child_key).gen(), + ) self.store[child_key].mutate() def update(self, value): - if value == None: + if isinstance(value, enum.Enum): + value = value.value + if value is None: self.store = None elif isinstance(value, dict): self.store = {} for k, v in value.items(): - self.store[k] = attach_schema_to_value(v, self.schema.get_property_schema(k)) + self.store[k] = attach_schema_to_value( + v, self.schema.get_property_schema(k) + ) else: - raise TypeError('Value [%s] Path [%s]' % (value, self.schema.get_path())) + raise TypeError(f"Value [{value}] Path [{self.schema.get_path()}]") def get_value_by_path(self, path: list): - '''Fetch the value specified by path''' - if self.store == None: + """Fetch the value specified by path""" + if self.store is None: return None if len(path) == 0: return self.raw_value() @@ -137,15 +172,15 @@ def get_value_by_path(self, path: list): return self.store[key].get_value_by_path(path) def create_path(self, path: list): - '''Ensures the path exists''' + """Ensures the path exists""" if len(path) == 0: return key = path.pop(0) - if self.store == None: + if self.store is None: self.update(self.schema.gen(minimum=True)) - self.__setitem__(key, None) + self[key] = None elif key not in self.store: - self.__setitem__(key, None) + self[key] = None self.store[key].create_path(path) def set_value_by_path(self, value, path): @@ -159,32 +194,37 @@ def __getitem__(self, key): return self.store[key] def __setitem__(self, key, value): - self.store[key] = attach_schema_to_value(value, self.schema.get_property_schema(key)) + self.store[key] = attach_schema_to_value( + value, self.schema.get_property_schema(key) + ) - def __contains__(self, item: string): + def __contains__(self, item: str): # in operator return item in self.store class ValueWithArraySchema(ValueWithSchema): + """Value with ArraySchema attached""" def __init__(self, value, schema) -> None: self.schema = schema - if value == None: + if value is None: self.store = None elif isinstance(value, list): self.store = [] for i in value: - self.store.append(attach_schema_to_value(i, self.schema.get_item_schema())) + self.store.append( + attach_schema_to_value(i, self.schema.get_item_schema()) + ) else: - raise TypeError('Value [%s] Path [%s]' % (value, self.schema.get_path())) + raise TypeError(f"Value [{value}] Path [{self.schema.get_path()}]") def value(self): return self.store def __str__(self) -> str: - if self.store == None: - return 'None' + if self.store is None: + return "None" else: ret = [] for i in self.store: @@ -192,7 +232,7 @@ def __str__(self) -> str: return str(ret) def raw_value(self) -> list: - if self.store == None: + if self.store is None: return None else: ret = [] @@ -201,14 +241,14 @@ def raw_value(self) -> list: return ret def mutate(self, p_delete=0.05, p_replace=0.1): - '''Mutate a small portion of the value - + """Mutate a small portion of the value + - Replace with null - Delete an item - Append an item - mutate an item - ''' - if self.store == None: + """ + if self.store is None: value = self.schema.gen() self.update(value) elif len(self.store) == 0: @@ -230,21 +270,28 @@ def mutate(self, p_delete=0.05, p_replace=0.1): self.store[index].mutate() def update(self, value): - if value == None: + if isinstance(value, enum.Enum): + value = value.value + if value is None: self.store = None elif isinstance(value, list): self.store = [] for i in value: - self.store.append(attach_schema_to_value(i, self.schema.get_item_schema())) + self.store.append( + attach_schema_to_value(i, self.schema.get_item_schema()) + ) else: - raise TypeError('Value [%s] Path [%s]' % (value, self.schema.get_path())) + raise TypeError(f"Value [{value}] Path [{self.schema.get_path()}]") def append(self, value): - self.store.append(attach_schema_to_value(value, self.schema.get_item_schema())) + """Append a value to the array""" + self.store.append( + attach_schema_to_value(value, self.schema.get_item_schema()) + ) def get_value_by_path(self, path: list): - '''Fetch the value specified by path''' - if self.store == None: + """Fetch the value specified by path""" + if self.store is None: return None if len(path) == 0: return self.raw_value() @@ -256,17 +303,17 @@ def get_value_by_path(self, path: list): return self.store[key].get_value_by_path(path) def create_path(self, path: list): - '''Ensures the path exists''' + """Ensures the path exists""" if len(path) == 0: return key = path.pop(0) - if self.store == None: + if self.store is None: self.store = [] - for i in range(0, key): + for _ in range(0, key): self.append(None) self.append(None) elif key >= len(self.store): - for i in range(len(self.store), key): + for _ in range(len(self.store), key): self.append(None) self.append(None) self.store[key].create_path(path) @@ -282,7 +329,9 @@ def __getitem__(self, key): return self.store[key] def __setitem__(self, key, value): - self.store[key] = attach_schema_to_value(value, self.schema.get_item_schema()) + self.store[key] = attach_schema_to_value( + value, self.schema.get_item_schema() + ) def __contains__(self, item: int): # in operator @@ -290,25 +339,25 @@ def __contains__(self, item: int): class ValueWithAnyOfSchema(ValueWithSchema): - '''Value with AnyOfSchema attached - + """Value with AnyOfSchema attached + store here is an instance of ValueWithSchema - ''' + """ def __init__(self, value, schema) -> None: self.schema = schema - if value == None: + if value is None: self.store = None for possible_schema in self.schema.get_possibilities(): if self.__validate(value, possible_schema): self.store = attach_schema_to_value(value, possible_schema) return - raise TypeError('Value [%s] Path [%s]' % (value, self.schema.get_path())) + raise TypeError(f"Value [{value}] Path [{self.schema.get_path()}]") def __validate(self, value, schema) -> bool: # XXX: Fragile! Use a complete validation utility from library - if value == None: + if value is None: return True elif isinstance(value, dict) and isinstance(schema, ObjectSchema): return True @@ -320,33 +369,35 @@ def __validate(self, value, schema) -> bool: return True elif isinstance(value, int) and isinstance(schema, IntegerSchema): return True - elif isinstance(value, (float, int)) and isinstance(schema, NumberSchema): + elif isinstance(value, (float, int)) and isinstance( + schema, NumberSchema + ): return True else: return False def __str__(self) -> str: - if self.schema == None: - ret = 'None' + if self.schema is None: + ret = "None" else: ret = str(self.store) return ret - def raw_value(self) -> dict: - '''serialization''' - if self.store == None: + def raw_value(self) -> Optional[dict]: + """serialization""" + if self.store is None: return None else: return self.store.raw_value() def mutate(self, p_delete=0.05, p_replace=0.1): - '''Mutate a small portion of the value - + """Mutate a small portion of the value + - Replace with null - Replace with a new value - Mutate depend on the current schema - ''' - if self.store == None: + """ + if self.store is None: value = self.schema.gen() self.update(value) else: @@ -359,27 +410,26 @@ def mutate(self, p_delete=0.05, p_replace=0.1): self.store.mutate() def update(self, value): - if value == None: + if value is None: self.store = None else: for possible_schema in self.schema.get_possibilities(): if self.__validate(value, possible_schema): self.store = attach_schema_to_value(value, possible_schema) return - raise TypeError('Value [%s] Path [%s]' % (value, self.schema.get_path())) + raise TypeError(f"Value [{value}] Path [{self.schema.get_path()}]") def get_value_by_path(self, path: list): - '''Fetch the value specified by path''' - if self.store == None: + """Fetch the value specified by path""" + if self.store is None: return None else: return self.store.get_value_by_path(path) def create_path(self, path: list): - '''Ensures the path exists''' + """Ensures the path exists""" if len(path) == 0: return - key = path.pop(0) # XXX: Complicated, no use case yet, let's implement later raise NotImplementedError @@ -390,9 +440,12 @@ def set_value_by_path(self, value, path): else: self.store.set_value_by_path(value, path) + def value(self): + return self.store + class ValueWithBasicSchema(ValueWithSchema): - '''Value with schema attached for Number/Integer, Bool, String''' + """Value with schema attached for Number/Integer, Bool, String""" def __init__(self, value, schema) -> None: self.schema = schema @@ -405,20 +458,19 @@ def value(self): return self.store def __str__(self) -> str: - if self.store == None: - ret = 'None' + if self.store is None: + ret = "None" else: ret = str(self.store) return ret - def raw_value(self) -> dict: - '''serialization''' + def raw_value(self) -> Optional[dict]: + """serialization""" return self.store def mutate(self, p_delete=0.05, p_replace=0.1): - '''Generate a new value or set to null - ''' - if self.store == None: + """Generate a new value or set to null""" + if self.store is None: self.store = self.schema.gen() else: dice = random.random() @@ -428,6 +480,8 @@ def mutate(self, p_delete=0.05, p_replace=0.1): self.update(self.schema.gen()) def update(self, value): + if isinstance(value, enum.Enum): + value = value.value if value is None: self.store = None else: @@ -435,24 +489,24 @@ def update(self, value): def get_value_by_path(self, path: list): if len(path) > 0: - raise Exception('Reached basic value, but path is not exhausted') + raise RuntimeError("Reached basic value, but path is not exhausted") return self.store def create_path(self, path: list): if len(path) == 0: return else: - raise Exception('Reached basic value, but path is not exhausted') + raise RuntimeError("Reached basic value, but path is not exhausted") def set_value_by_path(self, value, path): if len(path) == 0: self.update(value) else: - raise Exception('Reached basic value, but path is not exhausted') + raise RuntimeError("Reached basic value, but path is not exhausted") class ValueWithOpaqueSchema(ValueWithSchema): - '''Value with an opaque schema''' + """Value with an opaque schema""" def __init__(self, value, schema) -> None: self.schema = schema @@ -461,14 +515,29 @@ def __init__(self, value, schema) -> None: def raw_value(self) -> object: return self.store - def mutate(self): + def mutate(self, p_delete=0.05, p_replace=0.1): return def update(self, value): + if isinstance(value, enum.Enum): + value = value.value + self.store = value + + def get_value_by_path(self, path: list): + return self.store + + def create_path(self, path: list): + return + + def set_value_by_path(self, value, path): self.store = value + def value(self): + return self.store + def attach_schema_to_value(value, schema): + """Attach schema to value and return a ValueWithSchema object""" if isinstance(schema, ObjectSchema): return ValueWithObjectSchema(value, schema) elif isinstance(schema, ArraySchema): @@ -479,20 +548,3 @@ def attach_schema_to_value(value, schema): return ValueWithOpaqueSchema(value, schema) else: return ValueWithBasicSchema(value, schema) - - -if __name__ == '__main__': - with open('data/rabbitmq-operator/operator.yaml', 'r') as operator_yaml: - parsed_operator_documents = yaml.load_all(operator_yaml, Loader=yaml.FullLoader) - for document in parsed_operator_documents: - if document['kind'] == 'CustomResourceDefinition': - spec_schema = ObjectSchema(document['spec']['versions'][0]['schema'] - ['openAPIV3Schema']['properties']['spec']) - - with open('data/rabbitmq-operator/cr.yaml', 'r') as cr_yaml: - cr = yaml.load(cr_yaml, Loader=yaml.FullLoader) - value = attach_schema_to_value(cr['spec'], spec_schema) - print(type(spec_schema)) - print(str(value)) - value.mutate() - print(value.raw_value()) \ No newline at end of file diff --git a/acto/lib/operator_config.py b/acto/lib/operator_config.py index 2ac0511426..445f6d633c 100644 --- a/acto/lib/operator_config.py +++ b/acto/lib/operator_config.py @@ -7,35 +7,42 @@ class ApplyStep(BaseModel, extra="forbid"): """Configuration for each step of kubectl apply""" - file: str = Field( - description="Path to the file for kubectl apply") + + file: str = Field(description="Path to the file for kubectl apply") operator: bool = Field( description="If the file contains the operator deployment", - default=False) + default=False, + ) operator_container_name: Optional[str] = Field( description="The container name of the operator in the operator pod", - default=None) + default=None, + ) namespace: Optional[str] = Field( - description="Namespace for applying the file. If not specified, " + - "use the namespace in the file or Acto namespace. " + - "If set to null, use the namespace in the file", - default=DELEGATED_NAMESPACE) + description="Namespace for applying the file. If not specified, " + + "use the namespace in the file or Acto namespace. " + + "If set to null, use the namespace in the file", + default=DELEGATED_NAMESPACE, + ) class WaitStep(BaseModel, extra="forbid"): """Configuration for each step of waiting for the operator""" + duration: int = Field( - description="Wait for the specified seconds", - default=10) + description="Wait for the specified seconds", default=10 + ) class DeployStep(BaseModel, extra="forbid"): + """A step of deploying a resource""" + apply: ApplyStep = Field( - description="Configuration for each step of kubectl apply", - default=None) + description="Configuration for each step of kubectl apply", default=None + ) wait: WaitStep = Field( description="Configuration for each step of waiting for the operator", - default=None) + default=None, + ) # TODO: Add support for helm and kustomize # helm: str = Field( @@ -46,77 +53,96 @@ class DeployStep(BaseModel, extra="forbid"): class DeployConfig(BaseModel, extra="forbid"): """Configuration for deploying the operator""" + steps: List[DeployStep] = Field( - description="Steps to deploy the operator", - min_length=1) + description="Steps to deploy the operator", min_length=1 + ) class AnalysisConfig(BaseModel, extra="forbid"): "Configuration for static analysis" github_link: str = Field( - description="HTTPS URL for cloning the operator repo") + description="HTTPS URL for cloning the operator repo" + ) commit: str = Field( - description="Commit hash to specify the version to conduct static analysis") + description="Commit hash to specify the version to conduct static analysis" + ) type: str = Field(description="Type name of the CR") package: str = Field( - description="Package name in which the type of the CR is defined") + description="Package name in which the type of the CR is defined" + ) entrypoint: Optional[str] = Field( - description="The relative path of the main package for the operator, " + - "required if the main is not in the root directory") + description="The relative path of the main package for the operator, " + + "required if the main is not in the root directory" + ) class KubernetesEngineConfig(BaseModel, extra="forbid"): + """Configuration for Kubernetes""" + feature_gates: Dict[str, bool] = Field( - description="Path to the feature gates file", default=None) + description="Path to the feature gates file", default=None + ) class OperatorConfig(BaseModel, extra="forbid"): """Configuration for porting operators to Acto""" + deploy: DeployConfig analysis: Optional[AnalysisConfig] = Field( - default=None, - description="Configuration for static analysis") + default=None, description="Configuration for static analysis" + ) seed_custom_resource: str = Field(description="Path to the seed CR file") num_nodes: int = Field( - description="Number of workers in the Kubernetes cluster", default=4) + description="Number of workers in the Kubernetes cluster", default=4 + ) wait_time: int = Field( description="Timeout duration (seconds) for the resettable timer for system convergence", - default=60) + default=60, + ) collect_coverage: bool = False custom_oracle: Optional[str] = Field( - default=None, description="Path to the custom oracle file") + default=None, description="Path to the custom oracle file" + ) diff_ignore_fields: Optional[List[str]] = Field(default_factory=list) kubernetes_version: str = Field( - default="v1.22.9", description="Kubernetes version") + default="v1.22.9", description="Kubernetes version" + ) kubernetes_engine: KubernetesEngineConfig = Field( default=KubernetesEngineConfig(), - description="Configuration for the Kubernetes engine") + description="Configuration for the Kubernetes engine", + ) monkey_patch: Optional[str] = Field( - default=None, description="Path to the monkey patch file") - custom_fields: Optional[str] = Field( - default=None, description="Path to the custom fields file") - crd_name: Optional[str] = Field( - default=None, description="Name of the CRD") - blackbox_custom_fields: Optional[str] = Field( - default=None, description="Path to the blackbox custom fields file") + default=None, description="Path to the monkey patch file" + ) + custom_module: Optional[str] = Field( + default=None, + description="Path to the custom module, in the Python module path format", + ) + crd_name: Optional[str] = Field(default=None, description="Name of the CRD") k8s_fields: Optional[str] = Field( - default=None, description="Path to the k8s fields file") + default=None, description="Path to the k8s fields file" + ) example_dir: Optional[str] = Field( - default=None, description="Path to the example dir") + default=None, description="Path to the example dir" + ) context: Optional[str] = Field( - default=None, description="Path to the context file") + default=None, description="Path to the context file" + ) focus_fields: Optional[List[List[str]]] = Field( - default=None, description="List of focus fields") + default=None, description="List of focus fields" + ) if __name__ == "__main__": import json import jsonref + print( json.dumps( - jsonref.replace_refs( - OperatorConfig.model_json_schema()), - indent=4)) + jsonref.replace_refs(OperatorConfig.model_json_schema()), indent=4 + ) + ) diff --git a/acto/reproduce.py b/acto/reproduce.py index bdcdc0b088..43557c49df 100644 --- a/acto/reproduce.py +++ b/acto/reproduce.py @@ -11,6 +11,7 @@ import jsonpatch import yaml +from acto import DEFAULT_KUBERNETES_VERSION from acto.engine import Acto from acto.input import TestCase from acto.input.input import DeterministicInputModel @@ -66,16 +67,18 @@ def __init__(self, path, schema) -> None: class ReproInputModel(DeterministicInputModel): """Input model for reproducing""" + # pylint: disable=super-init-not-called, unused-argument def __init__( self, crd: dict, seed_input: dict, - used_fields: list, example_dir: str, num_workers: int, num_cases: int, reproduce_dir: str, mount: Optional[list] = None, + kubernetes_version: str = DEFAULT_KUBERNETES_VERSION, + custom_module_path: Optional[str] = None, ) -> None: logger = get_thread_logger(with_prefix=True) # WARNING: Not sure the initialization is correct @@ -105,10 +108,10 @@ def __init__( self.metadata = {} def initialize(self, initial_value: dict): - pass + """Override""" - def set_worker_id(self, id: int): - pass + def set_worker_id(self, worker_id: int): + """Override""" def set_mode(self, mode: str): pass @@ -141,18 +144,21 @@ def next_test(self) -> list: ] # return the first test case def apply_k8s_schema(self, k8s_field): - pass + """Override""" -def repro_precondition(v): +def repro_precondition(_): + """Precondition for reproducing""" return True -def repro_mutator(cr, v): +def repro_mutator(cr, _): + """Mutator for reproducing""" return cr -def repro_setup(v): +def repro_setup(_): + """Setup for reproducing""" return None @@ -163,6 +169,7 @@ def reproduce( acto_namespace: int, **kwargs, ) -> List[OracleResults]: + """Reproduce the trial folder""" os.makedirs(workdir_path, exist_ok=True) # Setting up log infra logging.basicConfig( @@ -211,12 +218,15 @@ def reproduce_postdiff( acto_namespace: int, **kwargs, ) -> bool: + """Reproduce the trial folder with post-diff test""" + _ = kwargs with open(operator_config, "r", encoding="utf-8") as config_file: config = OperatorConfig(**json.load(config_file)) post_diff_test_dir = os.path.join(workdir_path, "post_diff_test") logs = glob(workdir_path + "/*/operator-*.log") for log in logs: - open(log, "w", encoding="utf-8").close() + with open(log, "w", encoding="utf-8") as _: + pass p = PostDiffTest( testrun_dir=workdir_path, config=config, @@ -262,12 +272,11 @@ def reproduce_postdiff( parser.add_argument("--context", dest="context", help="Cached context data") args = parser.parse_args() - workdir_path = "testrun-%s" % datetime.now().strftime("%Y-%m-%d-%H-%M") + workdir_path_ = f"testrun-{datetime.now().strftime('%Y-%m-%d-%H-%M')}" - is_reproduce = True start_time = datetime.now() reproduce( - workdir_path=workdir_path, + workdir_path=workdir_path_, reproduce_dir=args.reproduce_dir, operator_config=args.config, acto_namespace=args.acto_namespace, diff --git a/acto/schema/base.py b/acto/schema/base.py index 0e12a9d273..af172eb5af 100644 --- a/acto/schema/base.py +++ b/acto/schema/base.py @@ -4,6 +4,7 @@ import jsonschema import jsonschema.exceptions +from acto.input.property_attribute import PropertyAttribute from acto.utils.thread_logger import get_thread_logger @@ -150,6 +151,7 @@ def __init__(self, path: list, schema: dict) -> None: self.enum = None if "enum" not in schema else schema["enum"] self.examples: list[Any] = [] + self.attributes = PropertyAttribute(value=0) self.copied_over = False self.over_specified = False self.problematic = False diff --git a/acto/schema/object.py b/acto/schema/object.py index 7137ceb9ef..f7b70fef7c 100644 --- a/acto/schema/object.py +++ b/acto/schema/object.py @@ -28,7 +28,7 @@ def __init__(self, path: list, schema: dict) -> None: from .schema import extract_schema super().__init__(path, schema) - self.properties = {} + self.properties: dict[str, BaseSchema] = {} self.additional_properties = None self.required = [] logger = get_thread_logger(with_prefix=True) diff --git a/data/cass-operator/config.json b/data/cass-operator/config.json index acbe01bab3..3fbc2bf50f 100644 --- a/data/cass-operator/config.json +++ b/data/cass-operator/config.json @@ -21,11 +21,10 @@ ] }, "crd_name": "cassandradatacenters.cassandra.datastax.com", - "custom_fields": "data.cass-operator.prune", - "blackbox_custom_fields": "data.cass-operator.prune_blackbox", - "k8s_fields": "data.cass-operator.k8s_mapping", + "custom_module": "data.cass-operator.custom_mapping", + "kubernetes_version": "v1.23.0", "seed_custom_resource": "data/cass-operator/cr.yaml", - "example_dir": "data/cass-operator/examples", + "example_dir": "data/cass-operator/examples", "analysis": { "github_link": "https://github.com/k8ssandra/cass-operator.git", "commit": "241e71cdd32bd9f8a7e5c00d5427cdcaf9f55497", @@ -49,4 +48,4 @@ "\\['stateful_set'\\]\\['cluster1\\-test\\-cluster\\-default\\-sts'\\]\\['spec'\\]\\['persistent_volume_claim_retention_policy'\\]", "\\['cassandra\\.datastax\\.com/resource\\-hash'\\]" ] -} \ No newline at end of file +} diff --git a/data/cass-operator/custom_mapping.py b/data/cass-operator/custom_mapping.py new file mode 100644 index 0000000000..30015fda33 --- /dev/null +++ b/data/cass-operator/custom_mapping.py @@ -0,0 +1,12 @@ +from acto.input.property_attribute import ( + PropertyAttribute, + tag_property_attribute, +) +from acto.input.test_generators.generator import test_generator +from acto.input.test_generators.stateful_set import replicas_tests + +tag_property_attribute( + ["spec", "managementApiAuth", "manual"], PropertyAttribute.Prune +) + +test_generator(property_name="size")(replicas_tests) diff --git a/data/cockroach-operator/config.json b/data/cockroach-operator/config.json index a96b33c952..868960aaba 100644 --- a/data/cockroach-operator/config.json +++ b/data/cockroach-operator/config.json @@ -9,12 +9,10 @@ } ] }, - "crd_name": null, - "custom_fields": "data.cockroach-operator.prune", - "blackbox_custom_fields": "data.cockroach-operator.prune_blackbox", - "k8s_fields": "data.cockroach-operator.k8s_mapping", + "crd_name": null, + "custom_module": "data.cockroach-operator.custom_mapping", "seed_custom_resource": "data/cockroach-operator/cr.yaml", - "example_dir": "data/cockroach-operator/examples", + "example_dir": "data/cockroach-operator/examples", "analysis": { "github_link": "https://github.com/cockroachdb/cockroach-operator.git", "commit": "d84779cf14612576703e2cea4c9bff153570e8b3", diff --git a/data/cockroach-operator/custom_mapping.py b/data/cockroach-operator/custom_mapping.py new file mode 100644 index 0000000000..70b75a5dd3 --- /dev/null +++ b/data/cockroach-operator/custom_mapping.py @@ -0,0 +1,4 @@ +from acto.input.test_generators.generator import test_generator +from acto.input.test_generators.stateful_set import replicas_tests + +test_generator(property_name="nodes")(replicas_tests) diff --git a/data/knative-operator-eventing/config.json b/data/knative-operator-eventing/config.json index 264b5e61cd..ba8e33ad14 100644 --- a/data/knative-operator-eventing/config.json +++ b/data/knative-operator-eventing/config.json @@ -10,9 +10,6 @@ ] }, "crd_name": "knativeeventings.operator.knative.dev", - "custom_fields": "data.knative-operator-eventing.prune", - "blackbox_custom_fields": "data.knative-operator-eventing.prune_blackbox", - "k8s_fields": "data.knative-operator-eventing.k8s_mapping", "example_dir": "data/knative-operator-eventing/examples", "seed_custom_resource": "data/knative-operator-eventing/cr.yaml", "analysis": { diff --git a/data/knative-operator-eventing/custom_mapping.py b/data/knative-operator-eventing/custom_mapping.py new file mode 100644 index 0000000000..ba686f12d7 --- /dev/null +++ b/data/knative-operator-eventing/custom_mapping.py @@ -0,0 +1,8 @@ +from acto.input.property_attribute import ( + PropertyAttribute, + tag_property_attribute, +) + +tag_property_attribute( + ["spec", "registry", "override"], PropertyAttribute.Prune +) diff --git a/data/knative-operator-serving/config.json b/data/knative-operator-serving/config.json index 2ecaa2e060..bc85d4e60d 100644 --- a/data/knative-operator-serving/config.json +++ b/data/knative-operator-serving/config.json @@ -10,9 +10,6 @@ ] }, "crd_name": "knativeservings.operator.knative.dev", - "custom_fields": "data.knative-operator-serving.prune", - "blackbox_custom_fields": "data.knative-operator-serving.prune_blackbox", - "k8s_fields": "data.knative-operator-serving.k8s_mapping", "example_dir": "data/knative-operator-serving/examples", "seed_custom_resource": "data/knative-operator-serving/cr.yaml", "analysis": { @@ -21,5 +18,5 @@ "entrypoint": "cmd/operator", "type": "KnativeServing", "package": "knative.dev/operator/pkg/apis/operator/v1beta1" - } + } } diff --git a/data/knative-operator-serving/custom_mapping.py b/data/knative-operator-serving/custom_mapping.py new file mode 100644 index 0000000000..5075a7e489 --- /dev/null +++ b/data/knative-operator-serving/custom_mapping.py @@ -0,0 +1,9 @@ +from acto.input.property_attribute import ( + PropertyAttribute, + tag_property_attribute, +) + +tag_property_attribute(["spec", "ingress", "istio"], PropertyAttribute.Prune) +tag_property_attribute( + ["spec", "registry", "override"], PropertyAttribute.Prune +) diff --git a/data/mongodb-community-operator/config.json b/data/mongodb-community-operator/config.json index 8025987fe9..e9f7dfe668 100644 --- a/data/mongodb-community-operator/config.json +++ b/data/mongodb-community-operator/config.json @@ -9,12 +9,9 @@ } ] }, - "crd_name": null, - "custom_fields": "data.mongodb-community-operator.prune", - "blackbox_custom_fields": "data.mongodb-community-operator.prune_blackbox", - "k8s_fields": "data.mongodb-community-operator.k8s_mapping", + "crd_name": null, "seed_custom_resource": "data/mongodb-community-operator/cr.yaml", - "example_dir": "data/mongodb-community-operator/config/samples", + "example_dir": "data/mongodb-community-operator/config/samples", "analysis": { "github_link": "https://github.com/mongodb/mongodb-kubernetes-operator.git", "commit": "5c78a242de9a6f71093def52a23bc829fa312ae1", @@ -25,4 +22,4 @@ "diff_ignore_fields": [ "\\['metadata'\\]\\['annotations'\\]\\['agent\\.mongodb\\.com\\\/version'\\]" ] -} \ No newline at end of file +} diff --git a/data/percona-server-mongodb-operator/config.json b/data/percona-server-mongodb-operator/config.json index 66dddf1a19..e77d0d555e 100644 --- a/data/percona-server-mongodb-operator/config.json +++ b/data/percona-server-mongodb-operator/config.json @@ -11,11 +11,9 @@ ] }, "crd_name": "perconaservermongodbs.psmdb.percona.com", - "custom_fields": "data.percona-server-mongodb-operator.prune", - "blackbox_custom_fields": "data.percona-server-mongodb-operator.prune_blackbox", - "k8s_fields": "data.percona-server-mongodb-operator.k8s_mapping", + "custom_module": "data.percona-server-mongodb-operator.custom_mapping", "seed_custom_resource": "data/percona-server-mongodb-operator/cr.yaml", - "example_dir": "data/percona-server-mongodb-operator/examples", + "example_dir": "data/percona-server-mongodb-operator/examples", "analysis": { "github_link": "https://github.com/percona/percona-server-mongodb-operator.git", "commit": "54950f7e56cde893c4b36a061c6335598b84873d", @@ -32,4 +30,4 @@ "\\['metadata'\\]\\['annotations'\\]\\['percona\\.com\\\/ssl\\-internal\\-hash'\\]", "\\['metadata'\\]\\['annotations'\\]\\['percona\\.com\\\/last\\-config\\-hash'\\]" ] -} \ No newline at end of file +} diff --git a/data/percona-server-mongodb-operator/custom_mapping.py b/data/percona-server-mongodb-operator/custom_mapping.py new file mode 100644 index 0000000000..e1576d01ac --- /dev/null +++ b/data/percona-server-mongodb-operator/custom_mapping.py @@ -0,0 +1,18 @@ +from acto.input.property_attribute import ( + PropertyAttribute, + tag_property_attribute, +) + +tag_property_attribute(["spec", "pmm"], PropertyAttribute.Prune) +tag_property_attribute(["spec", "crVersion"], PropertyAttribute.Prune) +tag_property_attribute(["spec", "mongod", "security"], PropertyAttribute.Prune) +tag_property_attribute( + ["spec", "mongod", "setParameter"], PropertyAttribute.Prune +) +tag_property_attribute( + ["spec", "mongod", "replication"], PropertyAttribute.Prune +) +tag_property_attribute( + ["spec", "mongod", "operationProfiling"], PropertyAttribute.Prune +) +tag_property_attribute(["spec", "mongod", "storage"], PropertyAttribute.Mapped) diff --git a/data/percona-xtradb-cluster-operator/config.json b/data/percona-xtradb-cluster-operator/config.json index dad35b56cd..44b8162400 100644 --- a/data/percona-xtradb-cluster-operator/config.json +++ b/data/percona-xtradb-cluster-operator/config.json @@ -10,11 +10,9 @@ ] }, "crd_name": "perconaxtradbclusters.pxc.percona.com", - "custom_fields": "data.percona-xtradb-cluster-operator.prune", - "blackbox_custom_fields": "data.percona-xtradb-cluster-operator.prune_blackbox", - "k8s_fields": "data.percona-xtradb-cluster-operator.k8s_mapping", + "custom_module": "data.percona-xtradb-cluster-operator.custom_mapping", "seed_custom_resource": "data/percona-xtradb-cluster-operator/cr.yaml", - "example_dir": "data/percona-xtradb-cluster-operator/examples", + "example_dir": "data/percona-xtradb-cluster-operator/examples", "analysis": { "github_link": "https://github.com/percona/percona-xtradb-cluster-operator", "commit": "e797d016cfbf847ff0a45ce1b1a1d10ad70a2fd3", @@ -27,4 +25,4 @@ "\\['spec'\\]\\['containers'\\]\\[.*\\]\\['env'\\]\\[.*\\]\\['value'\\]", "\\['config_map'\\]\\['auto\\-test\\-cluster\\-pxc'\\]" ] -} \ No newline at end of file +} diff --git a/data/percona-xtradb-cluster-operator/custom_mapping.py b/data/percona-xtradb-cluster-operator/custom_mapping.py new file mode 100644 index 0000000000..9e3969f362 --- /dev/null +++ b/data/percona-xtradb-cluster-operator/custom_mapping.py @@ -0,0 +1,19 @@ +from acto.input.property_attribute import ( + PropertyAttribute, + tag_property_attribute, +) +from acto.input.test_generators.generator import test_generator +from acto.input.test_generators.service import service_type_tests +from acto.input.test_generators.stateful_set import replicas_tests + +tag_property_attribute(["spec", "pmm"], PropertyAttribute.Prune) +tag_property_attribute(["spec", "backup"], PropertyAttribute.Prune) +tag_property_attribute(["spec", "proxysql"], PropertyAttribute.Prune) +tag_property_attribute(["spec", "updateStrategy"], PropertyAttribute.Prune) +tag_property_attribute(["spec", "upgradeOptions"], PropertyAttribute.Prune) + +test_generator(paths=["haproxy/size"])(replicas_tests) +test_generator(paths=["proxysql/size"])(replicas_tests) +test_generator(paths=["pxc/size"])(replicas_tests) + +test_generator(property_name="replicasServiceType")(service_type_tests) diff --git a/data/rabbitmq-operator/config.json b/data/rabbitmq-operator/config.json index ea92e0dcab..a930e60a07 100644 --- a/data/rabbitmq-operator/config.json +++ b/data/rabbitmq-operator/config.json @@ -10,9 +10,7 @@ ] }, "crd_name": null, - "custom_fields": "data.rabbitmq-operator.prune", - "blackbox_custom_fields": "data.rabbitmq-operator.prune_blackbox", - "k8s_fields": "data.rabbitmq-operator.k8s_mapping", + "custom_module": "data.rabbitmq-operator.custom_mapping", "seed_custom_resource": "data/rabbitmq-operator/cr.yaml", "example_dir": "data/rabbitmq-operator/examples", "analysis": { @@ -29,4 +27,4 @@ "\\['secret'\\]\\['test\\-cluster\\-server\\-token\\-.*'\\]", "\\['service'\\]\\['test\\-cluster'\\]\\['spec'\\]\\['ports'\\]" ] -} \ No newline at end of file +} diff --git a/data/rabbitmq-operator/custom_mapping.py b/data/rabbitmq-operator/custom_mapping.py new file mode 100644 index 0000000000..e0d8d16f24 --- /dev/null +++ b/data/rabbitmq-operator/custom_mapping.py @@ -0,0 +1,20 @@ +from acto.input.input import CustomKubernetesMapping +from acto.input.property_attribute import ( + PropertyAttribute, + tag_property_attribute, +) + +KUBERNETES_TYPE_MAPPING: list[CustomKubernetesMapping] = [ + CustomKubernetesMapping( + schema_path=["spec", "override", "statefulSet", "spec"], + kubernetes_schema_name="io.k8s.api.apps.v1.StatefulSetSpec", + ), + CustomKubernetesMapping( + schema_path=["spec", "override", "service", "spec"], + kubernetes_schema_name="io.k8s.api.core.v1.ServiceSpec", + ), +] + +tag_property_attribute( + ["spec", "override", "statefulSet", "spec"], PropertyAttribute.Patch +) diff --git a/data/redis-operator/config.json b/data/redis-operator/config.json index 6158eb7549..50aedf7e33 100644 --- a/data/redis-operator/config.json +++ b/data/redis-operator/config.json @@ -16,9 +16,6 @@ ] }, "crd_name": null, - "custom_fields": "data.redis-operator.prune", - "blackbox_custom_fields": "data.redis-operator.prune_blackbox", - "k8s_fields": "data.redis-operator.k8s_mapping", "seed_custom_resource": "data/redis-operator/cr.yaml", "example_dir": "data/redis-operator/examples", "analysis": { @@ -34,4 +31,4 @@ "\\['service_account'\\]\\['default'\\]\\['secrets'\\]\\[.*\\]\\['name'\\]", "\\['service_account'\\]\\['redisoperator'\\]\\['secrets'\\]\\[.*\\]\\['name'\\]" ] -} \ No newline at end of file +} diff --git a/data/redis-ot-container-kit-operator/config.json b/data/redis-ot-container-kit-operator/config.json index 9600d76561..25147c554d 100644 --- a/data/redis-ot-container-kit-operator/config.json +++ b/data/redis-ot-container-kit-operator/config.json @@ -10,9 +10,6 @@ ] }, "crd_name": "redisclusters.redis.redis.opstreelabs.in", - "custom_fields": "data.redis-ot-container-kit-operator.prune", - "blackbox_custom_fields": "data.redis-ot-container-kit-operator.prune_blackbox", - "k8s_fields": "data.redis-ot-container-kit-operator.k8s_mapping", "seed_custom_resource": "data/redis-ot-container-kit-operator/cr_cluster.yaml", "example_dir": "data/redis-ot-container-kit-operator/examples", "analysis": { @@ -22,4 +19,4 @@ "type": "RedisCluster", "package": "redis-operator/api/v1beta1" } -} \ No newline at end of file +} diff --git a/data/tidb-operator/config.json b/data/tidb-operator/config.json index d4b3c8b2ac..4d3f33ffb5 100644 --- a/data/tidb-operator/config.json +++ b/data/tidb-operator/config.json @@ -15,10 +15,7 @@ } ] }, - "crd_name": "tidbclusters.pingcap.com", - "custom_fields": "data.tidb-operator.prune", - "blackbox_custom_fields": "data.tidb-operator.prune_blackbox", - "k8s_fields": "data.tidb-operator.k8s_mapping", + "crd_name": "tidbclusters.pingcap.com", "seed_custom_resource": "data/tidb-operator/cr.yaml", "example_dir": "data/tidb-operator/examples", "analysis": { diff --git a/data/zookeeper-operator/config.json b/data/zookeeper-operator/config.json index 43e838e33b..eba1041e52 100644 --- a/data/zookeeper-operator/config.json +++ b/data/zookeeper-operator/config.json @@ -10,10 +10,7 @@ ] }, "crd_name": "zookeeperclusters.zookeeper.pravega.io", - "custom_fields": "data.zookeeper-operator.prune", "custom_oracle": "data.zookeeper-operator.oracle", - "blackbox_custom_fields": "data.zookeeper-operator.prune_blackbox", - "k8s_fields": "data.zookeeper-operator.k8s_mapping", "seed_custom_resource": "data/zookeeper-operator/cr.yaml", "example_dir": "data/zookeeper-operator/examples", "analysis": { @@ -28,4 +25,4 @@ "\\['config_map'\\]\\['zookeeper\\-operator\\-lock'\\]\\['metadata'\\]\\['owner_references'\\]\\[.*\\]\\['name'\\]", "\\['service'\\]\\['test\\-cluster\\-admin\\-server'\\]\\['spec'\\]\\['ports'\\]\\[.*\\]\\['node_port'\\]" ] -} \ No newline at end of file +} diff --git a/docs/test_generator.md b/docs/test_generator.md new file mode 100644 index 0000000000..07d484b079 --- /dev/null +++ b/docs/test_generator.md @@ -0,0 +1,138 @@ +# Test Generator + +## Introduction + +Acto creates test cases for each type of CRD schemas using a set of builtin test generator functions. These functions provide: + +- Primitive test cases for primitive property types such as string, number, and boolean. +- Semantic test cases for matched kubernetes schemas. + such as `v1.Pod` and `v1.StatefulSet`. + +Additional custom test generators can be specified to + +- [Add semantic test cases for operator specific schemas.](#custom-semantic-test-generator) +- [Override or extend the default test generation testcases.](#override-or-extend-default-test-generator) +- [Apply built-in test generators to more schemas.](#apply-built-in-test-generators-to-more-schemas) + +## Test Generator Functions + +Test generators are functions that takes a schema as argument and returns a list of test cases. + +```python +@test_generator(...) +def test_generator_function(schema: BaseSchema) -> List[TestCase]: + ... +``` + +To specify which schema in the CRD should the test generator function handle, annotate the function with the `@test_generator` decorator and pass schema contraints as arguments. A schema would be handled by a test generator function if all constraints are satified and the function has the highest priority out of all matched functions. The properties of the schema that can be constrained are: + +- `k8s_schema_name` - `str | None` + + The name suffix of the kubernetes schema if the schema matches to one. + +- `property_name` - `str | None` + + The name of the property within the CRD schema. + +- `property_type` - `str | None` + + The type of the property within the CRD schema. The type can be one of the following: `AnyOf`, `Array`, `Boolean`, `Integer`, `Number`, `Object`, `OneOf`, `Opaque`, `String` + +- `paths` - `List[str] | None` + + The list of path suffixes of the schema within the CRD schema. For example, the path `spec.cluster.replicas` would be specified with `['spec.cluster.replicas']` or `['cluster.resplicas']`. + +- `priority` - `Priority` + + The priority of the test generator. The priority is used to determine which test generator to apply to a schema when multiple test generators are matched to the same schema. The priority can be one of the following (shown in increasing order of priority): + + - `Priority.PRIMITIVE` + - `Priority.SEMANTIC` + - `Priority.CUSTOM` (default value) + +## Examples + +### Custom Semantic Test Generator + +For operator specific schemas, you can implement custom test generators to add semantic test cases. For example, the following is a test generator for the `updateStrategy` property in the Percona XtraDB Cluster Operator CRD. This schema is of type `string` and could be configured with `SmartUpdate`, `RollingUpdate`, or `OnDelete`. + +```python +from acto.input.test_generators import test_generator +from acto.input.testcase import TestCase +from acto.schema import StringSchema + +@test_generator(property_name='updateStrategy', property_type="String") +def update_strategy_tests(schema: StringSchema) -> List[TestCase]: + return [ + TestCase( + name='update-strategy-smart-update', + preconditions=lambda x: x != 'SmartUpdate', + mutators=lambda x: 'SmartUpdate', + setup=lambda x: None, + ), + TestCase( + name='update-strategy-rolling-update', + preconditions=lambda x: x != 'RollingUpdate', + mutators=lambda x: 'RollingUpdate', + setup=lambda x: None, + ), + TestCase( + name='update-strategy-on-delete', + preconditions=lambda x: x != 'OnDelete', + mutators=lambda x: 'OnDelete', + setup=lambda x: None, + ), + TestCase( + name='update-strategy-invalid', + preconditions=lambda x: x in ['SmartUpdate', 'RollingUpdate', 'OnDelete'], + mutators=lambda x: 'InvalidUpdateStrategy', + setup=lambda x: None, + invalid=True, + ), + ] +``` + +The generator function will attach to any schemas with property name `updateStrategy` and is of type `String`. Since the property name `updateStrategy` is unique in the Percona XtraDB Cluster Operator CRD, the generator function will only be applied to that schema. In the case where the property name is not unique, `paths` can be used to specify the schema path directly. + +Notice that an additional test case `InvalidUpdateStrategy` is added to test the behavior when the property is set to an invalid value. This is a common pattern for test generators to include a test case that tests the behavior when the property is set to an invalid value. This allows Acto to also test for misoperations (e.g. from typos). + +### Override or Extend Default Test Generator + +Sometimes, you want to add addtional test cases to the default test generator. For example, the default test generator for `v1.StatefulSet.spec.replicas` only generates test cases for integer values but many CRDs allow string values as well. The following test generator inherites the test cases from the default test generator using function composition and adds additional test cases for string values. + +```python +from acto.input.test_generators import test_generator, replicas_tests +from acto.input.testcase import TestCase +from acto.schema import StringSchema + +@test_generator(property_name='updateStrategy') +def custom_replicas_tests(schema) -> List[TestCase]: + return [ + *replicas_tests(schema), # inherit test cases from the builtin replicas test generator + TestCase( + name='replicas-string-1', + preconditions=lambda x: x is not None and isinstance(x, int), + mutators=lambda x: '1', + setup=lambda x: None, + ), + TestCase( + name='replicas-string-invalid', + preconditions=lambda x: x != "-1", + mutators=lambda x: '-1', + setup=lambda x: None, + invalid=True, + ), + ] +``` + +### Apply Built-in Test Generators to More Schemas + +You can also apply built-in test generators to a wider range of schemas. For example, the `replicas_tests` test generator only applies to properties of name `replicas`. The following test generator applies the test to `minReplicas` schema as well. + +```python +from acto.input.test_generators import test_generator, replicas_tests + +@test_generator(property_name='minReplicas') +def extended_replicas_test(schema) -> List[TestCase]: + return replicas_tests(schema) +``` diff --git a/pyproject.toml b/pyproject.toml index c42312ecde..5842c593fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,8 @@ markers = [ "all_bug_reproduction: mark a test to reproduce all bugs in the suite", "kubernetes_engine: mark a test for cluster set up", ] +python_functions = "test_*" +python_classes = "" [tool.pylint."messages control"] disable = [ @@ -87,6 +89,7 @@ disable = [ "dangerous-default-value", "duplicate-code", "missing-module-docstring", + "fixme", ] [tool.mypy] diff --git a/requirements-dev.txt b/requirements-dev.txt index 98e76ec246..03f12df4eb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --extra=dev --output-file=requirements-dev.txt @@ -14,8 +14,6 @@ ansible-core==2.16.2 # via acto (pyproject.toml) astroid==3.0.2 # via pylint -async-timeout==4.0.3 - # via aiohttp attrs==23.1.0 # via # aiohttp @@ -54,8 +52,6 @@ distlib==0.3.8 # via virtualenv docker==6.1.3 # via acto (pyproject.toml) -exceptiongroup==1.2.0 - # via pytest exrex==0.11.0 # via acto (pyproject.toml) filelock==3.13.1 @@ -194,22 +190,10 @@ six==1.16.0 # python-dateutil tabulate==0.9.0 # via acto (pyproject.toml) -tomli==2.0.1 - # via - # black - # build - # coverage - # mypy - # pip-tools - # pylint - # pyproject-hooks - # pytest tomlkit==0.12.3 # via pylint typing-extensions==4.9.0 # via - # astroid - # black # mypy # pydantic # pydantic-core diff --git a/requirements.txt b/requirements.txt index 443aff7680..f2c1b51408 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --output-file=requirements.txt diff --git a/test/integration_tests/test_cassop_bugs.py b/test/integration_tests/test_cassop_bugs.py index 945caeb749..28fc525a0a 100644 --- a/test/integration_tests/test_cassop_bugs.py +++ b/test/integration_tests/test_cassop_bugs.py @@ -50,7 +50,6 @@ def __init__(self, methodName: str = "runTest") -> None: self.input_model: InputModel = DeterministicInputModel( crd=self.context["crd"]["body"], seed_input=self.seed, - used_fields=self.context["analysis_result"]["used_fields"], example_dir=self.config.example_dir, num_workers=1, num_cases=1, diff --git a/test/integration_tests/test_crdb_bugs.py b/test/integration_tests/test_crdb_bugs.py index fb6a72145c..a1ff260058 100644 --- a/test/integration_tests/test_crdb_bugs.py +++ b/test/integration_tests/test_crdb_bugs.py @@ -48,7 +48,6 @@ def __init__(self, methodName: str = "runTest") -> None: self.input_model: InputModel = DeterministicInputModel( crd=self.context["crd"]["body"], seed_input=self.seed, - used_fields=self.context["analysis_result"]["used_fields"], example_dir=self.config.example_dir, num_workers=1, num_cases=1, diff --git a/test/integration_tests/test_known_schemas.py b/test/integration_tests/test_known_schemas.py index 8231c67b9f..9b81b89ff0 100644 --- a/test/integration_tests/test_known_schemas.py +++ b/test/integration_tests/test_known_schemas.py @@ -1,4 +1,4 @@ -# pylint: disable=missing-docstring, line-too-long, redefine-outer-name +# pylint: disable=missing-docstring, line-too-long import os import pathlib @@ -6,11 +6,11 @@ import yaml -from acto.input.get_matched_schemas import field_matched from acto.input.k8s_schemas import K8sSchemaMatcher, KubernetesSchema -from acto.input.known_schemas import * from acto.input.valuegenerator import extract_schema_with_value_generator from acto.schema import extract_schema +from acto.schema.base import BaseSchema +from acto.schema.object import ObjectSchema test_dir = pathlib.Path(__file__).parent.resolve() test_data_dir = os.path.join(test_dir, "test_data") @@ -21,16 +21,21 @@ class TestSchema(unittest.TestCase): @classmethod def setUpClass(cls): - cls.schema_matcher = K8sSchemaMatcher.from_version("1.29") + cls.schema_matcher = K8sSchemaMatcher.from_version("v1.29.0") + cls.schema_matcher_1_20 = K8sSchemaMatcher.from_version("v1.20.0") + cls.schema_matcher_1_21 = K8sSchemaMatcher.from_version("v1.21.0") + cls.schema_matcher_1_23 = K8sSchemaMatcher.from_version("v1.23.0") def assert_exists( self, suffix: str, schema_name: str, - matches: [tuple[BaseSchema, KubernetesSchema]], + matches: list[tuple[BaseSchema, KubernetesSchema]], ): applied = 0 for schema, k8s_schema in matches: + if k8s_schema.k8s_schema_name is None: + continue schema_path = "/".join(schema.path) if schema_path.endswith(suffix): self.assertTrue( @@ -51,7 +56,7 @@ def test_rabbitmq_crd(self): spec_schema = extract_schema( [], crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"] ) - matches = self.schema_matcher.find_matched_schemas(spec_schema) + matches = self.schema_matcher.find_all_matched_schemas(spec_schema) self.assert_exists("labelSelector", "v1.LabelSelector", matches) self.assert_exists("exec", "v1.ExecAction", matches) self.assert_exists("httpGet/httpHeaders/ITEM", "v1.HTTPHeader", matches) @@ -59,37 +64,61 @@ def test_rabbitmq_crd(self): self.assert_exists("seLinuxOptions", "v1.SELinuxOptions", matches) self.assert_exists("seccompProfile", "v1.SeccompProfile", matches) self.assert_exists("volumeMounts/ITEM", "v1.VolumeMount", matches) - self.assert_exists("configMapKeyRef", "v1.ConfigMapKeySelector", matches) + self.assert_exists( + "configMapKeyRef", "v1.ConfigMapKeySelector", matches + ) self.assert_exists("secretKeyRef", "v1.SecretKeySelector", matches) self.assert_exists("envFrom/ITEM", "v1.EnvFromSource", matches) - self.assert_exists("Containers/ITEM/ports/ITEM", "v1.ContainerPort", matches) + self.assert_exists( + "Containers/ITEM/ports/ITEM", "v1.ContainerPort", matches + ) self.assert_exists("capabilities", "v1.Capabilities", matches) self.assert_exists("volumeDevices/ITEM", "v1.VolumeDevice", matches) self.assert_exists("tolerations/ITEM", "v1.Toleration", matches) - self.assert_exists("spec/dataSource", "v1.TypedLocalObjectReference", matches) + self.assert_exists( + "spec/dataSource", "v1.TypedLocalObjectReference", matches + ) self.assert_exists("affinity/nodeAffinity", "v1.NodeAffinity", matches) - self.assert_exists("imagePullSecrets/ITEM", "v1.LocalObjectReference", matches) + self.assert_exists( + "imagePullSecrets/ITEM", "v1.LocalObjectReference", matches + ) self.assert_exists("volumes/ITEM/nfs", "v1.NFSVolumeSource", matches) - self.assert_exists("volumes/ITEM/hostPath", "v1.HostPathVolumeSource", matches) + self.assert_exists( + "volumes/ITEM/hostPath", "v1.HostPathVolumeSource", matches + ) self.assert_exists("securityContext/sysctls/ITEM", "v1.Sysctl", matches) def test_cassop_crd(self): with open( os.path.join(test_data_dir, "cassop_crd.yaml"), "r", - encoding="utf-8" + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) spec_schema = extract_schema( [], crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"] ) - matches = self.schema_matcher.find_matched_schemas(spec_schema) + matches = self.schema_matcher.find_all_matched_schemas(spec_schema) self.assert_exists("affinity/nodeAffinity", "v1.NodeAffinity", matches) - self.assert_exists("podAffinityTerm/labelSelector", "v1.LabelSelector", matches) - self.assert_exists("podAffinityTerm/namespaceSelector", "v1.LabelSelector", matches) - self.assert_exists("requiredDuringSchedulingIgnoredDuringExecution/ITEM/labelSelector", "v1.LabelSelector", matches) - self.assert_exists("requiredDuringSchedulingIgnoredDuringExecution/ITEM/namespaceSelector", "v1.LabelSelector", matches) - self.assert_exists("configMapKeyRef", "v1.ConfigMapKeySelector", matches) + self.assert_exists( + "podAffinityTerm/labelSelector", "v1.LabelSelector", matches + ) + self.assert_exists( + "podAffinityTerm/namespaceSelector", "v1.LabelSelector", matches + ) + self.assert_exists( + "requiredDuringSchedulingIgnoredDuringExecution/ITEM/labelSelector", + "v1.LabelSelector", + matches, + ) + self.assert_exists( + "requiredDuringSchedulingIgnoredDuringExecution/ITEM/namespaceSelector", + "v1.LabelSelector", + matches, + ) + self.assert_exists( + "configMapKeyRef", "v1.ConfigMapKeySelector", matches + ) self.assert_exists("fieldRef", "v1.ObjectFieldSelector", matches) self.assert_exists("secretKeyRef", "v1.SecretKeySelector", matches) self.assert_exists("envFrom/ITEM", "v1.EnvFromSource", matches) @@ -99,50 +128,140 @@ def test_cassop_crd(self): self.assert_exists("ports/ITEM", "v1.ContainerPort", matches) self.assert_exists("readinessProbe/exec", "v1.ExecAction", matches) self.assert_exists("readinessProbe/grpc", "v1.GRPCAction", matches) - self.assert_exists("containers/ITEM/securityContext", "v1.SecurityContext", matches) + self.assert_exists( + "containers/ITEM/securityContext", "v1.SecurityContext", matches + ) self.assert_exists("volumeDevices/ITEM", "v1.VolumeDevice", matches) self.assert_exists("volumeMounts/ITEM", "v1.VolumeMount", matches) - self.assert_exists("podTemplateSpec/spec/dnsConfig", "v1.PodDNSConfig", matches) + self.assert_exists( + "podTemplateSpec/spec/dnsConfig", "v1.PodDNSConfig", matches + ) self.assert_exists("podTemplateSpec/spec/os", "v1.PodOS", matches) - self.assert_exists("spec/readinessGates/ITEM", "v1.PodReadinessGate", matches) - self.assert_exists("podTemplateSpec/spec/securityContext", "v1.PodSecurityContext", matches) + self.assert_exists( + "spec/readinessGates/ITEM", "v1.PodReadinessGate", matches + ) + self.assert_exists( + "podTemplateSpec/spec/securityContext", + "v1.PodSecurityContext", + matches, + ) self.assert_exists("spec/tolerations/ITEM", "v1.Toleration", matches) - self.assert_exists("topologySpreadConstraints/ITEM/labelSelector", "v1.LabelSelector", matches) - self.assert_exists("volumes/ITEM/awsElasticBlockStore", "v1.AWSElasticBlockStoreVolumeSource", matches) - self.assert_exists("volumes/ITEM/azureDisk", "v1.AzureDiskVolumeSource", matches) - self.assert_exists("volumes/ITEM/azureFile", "v1.AzureFileVolumeSource", matches) - self.assert_exists("volumes/ITEM/cephfs", "v1.CephFSVolumeSource", matches) - self.assert_exists("volumes/ITEM/cinder", "v1.CinderVolumeSource", matches) - self.assert_exists("volumes/ITEM/configMap", "v1.ConfigMapVolumeSource", matches) + self.assert_exists( + "topologySpreadConstraints/ITEM/labelSelector", + "v1.LabelSelector", + matches, + ) + self.assert_exists( + "volumes/ITEM/awsElasticBlockStore", + "v1.AWSElasticBlockStoreVolumeSource", + matches, + ) + self.assert_exists( + "volumes/ITEM/azureDisk", "v1.AzureDiskVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/azureFile", "v1.AzureFileVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/cephfs", "v1.CephFSVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/cinder", "v1.CinderVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/configMap", "v1.ConfigMapVolumeSource", matches + ) self.assert_exists("volumes/ITEM/csi", "v1.CSIVolumeSource", matches) - self.assert_exists("items/ITEM/fieldRef", "v1.ObjectFieldSelector", matches) - self.assert_exists("volumeClaimTemplate/spec/dataSource", "v1.TypedLocalObjectReference", matches) + self.assert_exists( + "items/ITEM/fieldRef", "v1.ObjectFieldSelector", matches + ) + self.assert_exists( + "volumeClaimTemplate/spec/dataSource", + "v1.TypedLocalObjectReference", + matches, + ) # self.assert_exists("volumeClaimTemplate/spec/dataSourceRef", "v1.TypedObjectReference", matches) - self.assert_exists("volumeClaimTemplate/spec/selector", "v1.LabelSelector", matches) + self.assert_exists( + "volumeClaimTemplate/spec/selector", "v1.LabelSelector", matches + ) self.assert_exists("volumes/ITEM/fc", "v1.FCVolumeSource", matches) - self.assert_exists("volumes/ITEM/flexVolume", "v1.FlexVolumeSource", matches) - self.assert_exists("volumes/ITEM/flocker", "v1.FlockerVolumeSource", matches) - self.assert_exists("volumes/ITEM/gcePersistentDisk", "v1.GCEPersistentDiskVolumeSource", matches) - self.assert_exists("volumes/ITEM/gitRepo", "v1.GitRepoVolumeSource", matches) - self.assert_exists("volumes/ITEM/glusterfs", "v1.GlusterfsVolumeSource", matches) - self.assert_exists("volumes/ITEM/hostPath", "v1.HostPathVolumeSource", matches) - self.assert_exists("volumes/ITEM/iscsi", "v1.ISCSIVolumeSource", matches) + self.assert_exists( + "volumes/ITEM/flexVolume", "v1.FlexVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/flocker", "v1.FlockerVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/gcePersistentDisk", + "v1.GCEPersistentDiskVolumeSource", + matches, + ) + self.assert_exists( + "volumes/ITEM/gitRepo", "v1.GitRepoVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/glusterfs", "v1.GlusterfsVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/hostPath", "v1.HostPathVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/iscsi", "v1.ISCSIVolumeSource", matches + ) self.assert_exists("volumes/ITEM/nfs", "v1.NFSVolumeSource", matches) - self.assert_exists("volumes/ITEM/persistentVolumeClaim", "v1.PersistentVolumeClaimVolumeSource", matches) - self.assert_exists("volumes/ITEM/photonPersistentDisk", "v1.PhotonPersistentDiskVolumeSource", matches) - self.assert_exists("volumes/ITEM/portworxVolume", "v1.PortworxVolumeSource", matches) - self.assert_exists("sources/ITEM/configMap", "v1.ConfigMapProjection", matches) - self.assert_exists("volumes/ITEM/quobyte", "v1.QuobyteVolumeSource", matches) + self.assert_exists( + "volumes/ITEM/persistentVolumeClaim", + "v1.PersistentVolumeClaimVolumeSource", + matches, + ) + self.assert_exists( + "volumes/ITEM/photonPersistentDisk", + "v1.PhotonPersistentDiskVolumeSource", + matches, + ) + self.assert_exists( + "volumes/ITEM/portworxVolume", "v1.PortworxVolumeSource", matches + ) + self.assert_exists( + "sources/ITEM/configMap", "v1.ConfigMapProjection", matches + ) + self.assert_exists( + "volumes/ITEM/quobyte", "v1.QuobyteVolumeSource", matches + ) self.assert_exists("volumes/ITEM/rbd", "v1.RBDVolumeSource", matches) - self.assert_exists("volumes/ITEM/scaleIO", "v1.ScaleIOVolumeSource", matches) - self.assert_exists("volumes/ITEM/secret", "v1.SecretVolumeSource", matches) - self.assert_exists("volumes/ITEM/storageos", "v1.StorageOSVolumeSource", matches) - self.assert_exists("volumes/ITEM/vsphereVolume", "v1.VsphereVirtualDiskVolumeSource", matches) - self.assert_exists("ITEM/pvcSpec/dataSource", "v1.TypedLocalObjectReference", matches) + self.assert_exists( + "volumes/ITEM/scaleIO", "v1.ScaleIOVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/secret", "v1.SecretVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/storageos", "v1.StorageOSVolumeSource", matches + ) + self.assert_exists( + "volumes/ITEM/vsphereVolume", + "v1.VsphereVirtualDiskVolumeSource", + matches, + ) + self.assert_exists( + "ITEM/pvcSpec/dataSource", "v1.TypedLocalObjectReference", matches + ) # self.assert_exists("ITEM/pvcSpec/dataSourceRef", "v1.TypedObjectReference", matches) - self.assert_exists("storageConfig/cassandraDataVolumeClaimSpec/dataSource", "v1.TypedLocalObjectReference", matches) - self.assert_exists("storageConfig/cassandraDataVolumeClaimSpec/dataSourceRef", "v1alpha2.ResourceClaimParametersReference", matches) - self.assert_exists("storageConfig/cassandraDataVolumeClaimSpec/selector", "v1.LabelSelector", matches) + self.assert_exists( + "storageConfig/cassandraDataVolumeClaimSpec/dataSource", + "v1.TypedLocalObjectReference", + matches, + ) + self.assert_exists( + "storageConfig/cassandraDataVolumeClaimSpec/dataSourceRef", + "v1alpha2.ResourceClaimParametersReference", + matches, + ) + self.assert_exists( + "storageConfig/cassandraDataVolumeClaimSpec/selector", + "v1.LabelSelector", + matches, + ) self.assert_exists("spec/tolerations/ITEM", "v1.Toleration", matches) def test_strimzi_kafka_crd(self): @@ -156,26 +275,47 @@ def test_strimzi_kafka_crd(self): spec_schema = extract_schema( [], crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"] ) - matches = self.schema_matcher.find_matched_schemas(spec_schema) - self.assert_exists("namespaceSelector/matchExpressions/ITEM", "v1.LabelSelectorRequirement", matches) - self.assert_exists("labelSelector/matchExpressions/ITEM", "v1.LabelSelectorRequirement", matches) - self.assert_exists("container/securityContext", "v1.SecurityContext", matches) + matches = self.schema_matcher.find_all_matched_schemas(spec_schema) + self.assert_exists( + "namespaceSelector/matchExpressions/ITEM", + "v1.LabelSelectorRequirement", + matches, + ) + self.assert_exists( + "labelSelector/matchExpressions/ITEM", + "v1.LabelSelectorRequirement", + matches, + ) + self.assert_exists( + "container/securityContext", "v1.SecurityContext", matches + ) self.assert_exists("resources/claims/ITEM", "v1.ResourceClaim", matches) - self.assert_exists("configMapKeyRef", "v1.ConfigMapKeySelector", matches) - self.assert_exists("imagePullSecrets/ITEM", "v1.LocalObjectReference", matches) - self.assert_exists("pod/securityContext", "v1.PodSecurityContext", matches) + self.assert_exists( + "configMapKeyRef", "v1.ConfigMapKeySelector", matches + ) + self.assert_exists( + "imagePullSecrets/ITEM", "v1.LocalObjectReference", matches + ) + self.assert_exists( + "pod/securityContext", "v1.PodSecurityContext", matches + ) self.assert_exists("affinity/nodeAffinity", "v1.NodeAffinity", matches) self.assert_exists("tolerations/ITEM", "v1.Toleration", matches) self.assert_exists("hostAliases/ITEM", "v1.HostAlias", matches) - self.assert_exists("networkPolicyPeers/ITEM/ipBlock", "v1.IPBlock", matches) + self.assert_exists( + "networkPolicyPeers/ITEM/ipBlock", "v1.IPBlock", matches + ) # The following are schemas that are descendants of already matched schemas self.assert_exists("capabilities", "v1.Capabilities", matches) self.assert_exists("seLinuxOptions", "v1.SELinuxOptions", matches) self.assert_exists("seccompProfile", "v1.SeccompProfile", matches) + @unittest.skip("Need approaximate matching for the RabbitMQ's StatefulSet") def test_statefulset_match(self): with open( - os.path.join(test_data_dir, "rabbitmq_crd.yaml"), "r" + os.path.join(test_data_dir, "rabbitmq_crd.yaml"), + "r", + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) spec_schema = ObjectSchema( @@ -184,28 +324,49 @@ def test_statefulset_match(self): "properties" ]["spec"]["properties"]["override"]["properties"][ "statefulSet" + ][ + "properties" + ][ + "spec" ], ) - self.assertTrue(StatefulSetSchema.Match(spec_schema)) + self.assertTrue( + self.schema_matcher_1_20.k8s_models[ + "io.k8s.api.apps.v1.StatefulSetSpec" + ].match(spec_schema) + ) + # @unittest.skip("Need approaximate matching for the schema path") def test_service_match(self): with open( - os.path.join(test_data_dir, "rabbitmq_crd.yaml"), "r" + os.path.join(test_data_dir, "rabbitmq_crd.yaml"), + "r", + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) spec_schema = ObjectSchema( ["root"], crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"][ "properties" - ]["spec"]["properties"]["override"]["properties"]["service"], + ]["spec"]["properties"]["override"]["properties"]["service"][ + "properties" + ][ + "spec" + ], ) - self.assertTrue(ServiceSchema.Match(spec_schema)) + self.assertTrue( + self.schema_matcher_1_21.k8s_models[ + "io.k8s.api.core.v1.ServiceSpec" + ].match(spec_schema) + ) def test_affinity_match(self): with open( - os.path.join(test_data_dir, "rabbitmq_crd.yaml"), "r" + os.path.join(test_data_dir, "rabbitmq_crd.yaml"), + "r", + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) spec_schema = ObjectSchema( @@ -215,25 +376,17 @@ def test_affinity_match(self): ]["spec"]["properties"]["affinity"], ) - self.assertTrue(AffinitySchema.Match(spec_schema)) - - def test_tolerations_match(self): - with open( - os.path.join(test_data_dir, "rabbitmq_crd.yaml"), "r" - ) as operator_yaml: - crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) - spec_schema = ArraySchema( - ["root"], - crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"][ - "properties" - ]["spec"]["properties"]["tolerations"], - ) - - self.assertTrue(TolerationsSchema.Match(spec_schema)) + self.assertTrue( + self.schema_matcher_1_23.k8s_models[ + "io.k8s.api.core.v1.Affinity" + ].match(spec_schema) + ) - def test_tolerations_not_match(self): + def test_toleration_match(self): with open( - os.path.join(test_data_dir, "rabbitmq_crd.yaml"), "r" + os.path.join(test_data_dir, "rabbitmq_crd.yaml"), + "r", + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) spec_schema = ObjectSchema( @@ -243,11 +396,17 @@ def test_tolerations_not_match(self): ]["spec"]["properties"]["tolerations"]["items"], ) - self.assertFalse(TolerationsSchema.Match(spec_schema)) + self.assertTrue( + self.schema_matcher.k8s_models[ + "io.k8s.api.core.v1.Toleration" + ].match(spec_schema) + ) def test_resources_match(self): with open( - os.path.join(test_data_dir, "rabbitmq_crd.yaml"), "r" + os.path.join(test_data_dir, "rabbitmq_crd.yaml"), + "r", + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) spec_schema = ObjectSchema( @@ -257,14 +416,28 @@ def test_resources_match(self): ]["spec"]["properties"]["resources"], ) - self.assertTrue(ResourceRequirementsSchema.Match(spec_schema)) + print( + self.schema_matcher_1_20.k8s_models[ + "io.k8s.api.core.v1.ResourceRequirements" + ].dump_schema() + ) + + print(spec_schema) + + self.assertTrue( + self.schema_matcher_1_20.k8s_models[ + "io.k8s.api.core.v1.ResourceRequirements" + ].match(spec_schema) + ) def test_container_match(self): with open( - os.path.join(test_data_dir, "rabbitmq_crd.yaml"), "r" + os.path.join(test_data_dir, "rabbitmq_crd.yaml"), + "r", + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) - spec_schema = ObjectSchema( + container_schema = ObjectSchema( ["root"], crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"][ "properties" @@ -290,14 +463,56 @@ def test_container_match(self): "items" ], ) + probe_schema = ObjectSchema( + ["root"], + crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"][ + "properties" + ]["spec"]["properties"]["override"]["properties"][ + "statefulSet" + ][ + "properties" + ][ + "spec" + ][ + "properties" + ][ + "template" + ][ + "properties" + ][ + "spec" + ][ + "properties" + ][ + "containers" + ][ + "items" + ][ + "properties" + ][ + "livenessProbe" + ], + ) + + self.assertTrue( + self.schema_matcher_1_21.k8s_models[ + "io.k8s.api.core.v1.Probe" + ].match(probe_schema) + ) - self.assertTrue(ContainerSchema.Match(spec_schema)) + self.assertTrue( + self.schema_matcher_1_21.k8s_models[ + "io.k8s.api.core.v1.Container" + ].match(container_schema) + ) with open( os.path.join( - test_data_dir, "psmdb.percona.com_perconaservermongodbs.yaml" + test_data_dir, + "psmdb.percona.com_perconaservermongodbs.yaml", ), "r", + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) spec_schema = ObjectSchema( @@ -311,11 +526,15 @@ def test_container_match(self): ], ) - self.assertTrue(ContainerSchema.Match(spec_schema)) + self.assertTrue( + self.schema_matcher_1_23.k8s_models[ + "io.k8s.api.core.v1.Container" + ].match(spec_schema) + ) - def test_resources_match(self): + def test_ingress_tls_match(self): with open( - os.path.join(test_data_dir, "crdb_crd.yaml"), "r" + os.path.join(test_data_dir, "crdb_crd.yaml"), "r", encoding="utf-8" ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) tls_schema = ObjectSchema( @@ -331,13 +550,17 @@ def test_resources_match(self): ], ) - self.assertTrue(IngressTLSSchema.Match(tls_schema)) - - self.assertTrue(field_matched(tls_schema, IngressTLSSchema)) + self.assertTrue( + self.schema_matcher.k8s_models[ + "io.k8s.api.networking.v1.IngressTLS" + ].match(tls_schema) + ) def test_pod_spec_match(self): with open( - os.path.join(test_data_dir, "cassop_crd.yaml"), "r" + os.path.join(test_data_dir, "cassop_crd.yaml"), + "r", + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) @@ -363,17 +586,11 @@ def test_pod_spec_match(self): # for tuple in tuples: # print(f"Found matches schema: {tuple[0].path} -> {tuple[1]}") # k8s_schema = K8sField(tuple[0].path, tuple[1]) - print(LivenessProbeSchema.Match(spec_schema)) - - def test_find_matches(self): - with open( - os.path.join(test_data_dir, "rabbitmq_crd.yaml"), "r" - ) as operator_yaml: - crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) - spec_schema = extract_schema( - [], crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"] + self.assertTrue( + self.schema_matcher_1_23.k8s_models[ + "io.k8s.api.core.v1.Probe" + ].match(spec_schema) ) - print(find_all_matched_schemas(spec_schema)) def test_pvc_match(self): with open( @@ -381,6 +598,7 @@ def test_pvc_match(self): test_data_dir, "databases.spotahome.com_redisfailovers.yaml" ), "r", + encoding="utf-8", ) as operator_yaml: crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) spec_schema = ObjectSchema( @@ -391,10 +609,18 @@ def test_pvc_match(self): "properties" ][ "persistentVolumeClaim" + ][ + "properties" + ][ + "spec" ], ) - self.assertTrue(PersistentVolumeClaimSchema.Match(spec_schema)) + self.assertTrue( + self.schema_matcher_1_23.k8s_models[ + "io.k8s.api.core.v1.PersistentVolumeClaimSpec" + ].match(spec_schema) + ) if __name__ == "__main__": diff --git a/test/integration_tests/test_rbop_bugs.py b/test/integration_tests/test_rbop_bugs.py index f64df07605..7152e6a019 100644 --- a/test/integration_tests/test_rbop_bugs.py +++ b/test/integration_tests/test_rbop_bugs.py @@ -1,5 +1,5 @@ """Integration tests for the rabbitmq-operator bugs.""" -import importlib + import json import os import pathlib @@ -49,20 +49,12 @@ def __init__(self, methodName: str = "runTest") -> None: self.input_model: InputModel = DeterministicInputModel( crd=self.context["crd"]["body"], seed_input=self.seed, - used_fields=self.context["analysis_result"]["used_fields"], example_dir=self.config.example_dir, num_workers=1, num_cases=1, + custom_module_path=self.config.custom_module, ) - # The oracle depends on the custom fields - if self.config.custom_fields: - module = importlib.import_module(self.config.custom_fields) - for custom_field in module.custom_fields: - self.input_model.apply_custom_field(custom_field) - else: - raise AttributeError("No custom fields specified in config") - def test_rbop_928(self): """Test rabbitmq-operator rabbitmq-operator-928.""" # https://github.com/rabbitmq/cluster-operator/issues/928 diff --git a/test/integration_tests/test_test_generator_decorator.py b/test/integration_tests/test_test_generator_decorator.py new file mode 100644 index 0000000000..bb23c639fe --- /dev/null +++ b/test/integration_tests/test_test_generator_decorator.py @@ -0,0 +1,190 @@ +# pylint: disable=missing-docstring, line-too-long + +import os +import pathlib +import unittest + +import yaml + +from acto.input.k8s_schemas import K8sSchemaMatcher +from acto.input.test_generators.generator import ( + TEST_GENERATORS, + get_testcases, + test_generator, +) +from acto.input.testcase import TestCase +from acto.schema import extract_schema +from acto.schema.integer import IntegerSchema + +test_dir = pathlib.Path(__file__).parent.resolve() +test_data_dir = os.path.join(test_dir, "test_data") + + +def gen(_): + return [TestCase("test", lambda x: True, lambda x: None, lambda x: None)] + + +class TestTestGeneratorDecorator(unittest.TestCase): + """This class tests the schema matching code for various CRDs.""" + + @classmethod + def setUpClass(cls): + with open( + os.path.join(test_data_dir, "rabbitmq_crd.yaml"), + "r", + encoding="utf-8", + ) as operator_yaml: + rabbitmq_crd = yaml.load(operator_yaml, Loader=yaml.FullLoader) + schema_matcher = K8sSchemaMatcher.from_version("v1.29.0") + cls.spec_schema = extract_schema( + [], rabbitmq_crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"] + ) + cls.matches = schema_matcher.find_named_matched_schemas(cls.spec_schema) + + def test_path_suffix(self): + TEST_GENERATORS.clear() + test_generator(paths=["serviceAccountToken/expirationSeconds"])(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 1) + + TEST_GENERATORS.clear() + test_generator( + paths=[ + "serviceAccountToken/expirationSeconds", + "volumes/ITEM/quobyte/user", + ] + )(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 2) + + def test_k8s_schema_name(self): + TEST_GENERATORS.clear() + test_generator(k8s_schema_name="v1.NodeAffinity")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 2) + + TEST_GENERATORS.clear() + test_generator(k8s_schema_name="HTTPHeader")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 15) + + def test_field_name(self): + TEST_GENERATORS.clear() + test_generator(property_name="ports")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 4) + + TEST_GENERATORS.clear() + test_generator(property_name="image")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 5) + + def test_field_type(self): + TEST_GENERATORS.clear() + test_generator(property_type="AnyOf")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 51) + + TEST_GENERATORS.clear() + test_generator(property_type="Array")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 173) + + TEST_GENERATORS.clear() + test_generator(property_type="Boolean")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 73) + + TEST_GENERATORS.clear() + test_generator(property_type="Integer")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 106) + + TEST_GENERATORS.clear() + test_generator(property_type="Number")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 106) + + TEST_GENERATORS.clear() + test_generator(property_type="Object")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 368) + + # TEST_GENERATORS.clear() + # generator(field_type="OneOf")(gen) + # testcases = get_testcases(self.spec_schema, self.matches) + # self.assertEqual(len(testcases), 0) + + # TEST_GENERATORS.clear() + # generator(field_type="Opaque")(gen) + # testcases = get_testcases(self.spec_schema, self.matches) + # self.assertEqual(len(testcases), 0) + + TEST_GENERATORS.clear() + test_generator(property_type="String")(gen) + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 585) + + def test_priority(self): + TEST_GENERATORS.clear() + + @test_generator(property_type="Integer", priority=0) + def gen0(_): + return [ + TestCase( + "integer-test", + lambda x: True, + lambda x: None, + lambda x: None, + ) + ] + + @test_generator(property_name="replicas", priority=1) + def gen1(_): + return [ + TestCase( + "replicas-test", + lambda x: True, + lambda x: None, + lambda x: None, + ) + ] + + testcases = get_testcases(self.spec_schema, self.matches) + for path, tests in testcases: + if path[-1] == "replicas": + self.assertEqual(tests[0].name, "replicas-test") + else: + self.assertEqual(tests[0].name, "integer-test") + + def test_multiple_constraints(self): + TEST_GENERATORS.clear() + + test_generator( + property_type="Integer", + property_name="type", + )(gen) + + test_generator( + property_name="type", + paths=["seLinuxOptions/type"], + )(gen) + + test_generator( + property_type="String", + paths=["hostPath/type"], + )(gen) + + testcases = get_testcases(self.spec_schema, self.matches) + self.assertEqual(len(testcases), 0 + 4 + 1) + + def test_func_call_validation(self): + TEST_GENERATORS.clear() + + # pylint: disable=unused-argument + @test_generator(property_type="String") + def gen0(schema: IntegerSchema): + return [] + + with self.assertRaises(TypeError): + get_testcases(self.spec_schema, self.matches)