Use dataclass to define build configs

Signed-off-by: Nikolay Yurin <[email protected]>

.github/scripts/matrix.py

@@ -1,11 +1,25 @@
 #!/usr/bin/env python3
 
-from json import dumps
-from enum import Enum
 import os
+import dataclasses
+import json
+
+from enum import Enum
+from typing import Any, Dict, List, Optional, Final, Set, Union
+
+MANAGED_OWNER: Final[str] = "kernel-patches"
+MANAGED_REPOS: Final[Set[str]] = {
+    f"{MANAGED_OWNER}/bpf",
+    f"{MANAGED_OWNER}/vmtest",
+}
+# We need to run on ubuntu 20.04 because our rootfs is based on debian buster and we
+# otherwise get library versioning issue such as
+# `./test_verifier: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by ./test_verifier)`
+DEFAULT_RUNNER: Final[str] = "ubuntu-20.04"
+DEFAULT_LLVM_VERSION: Final[int] = 17
 
 
-class Arch(Enum):
+class Arch(str, Enum):
     """
     CPU architecture supported by CI.
     """
@@ -15,18 +29,99 @@ class Arch(Enum):
     X86_64 = "x86_64"
 
 
+class Compiler(str, Enum):
+    GCC = "gcc"
+    LLVM = "llvm"
+
+
+@dataclasses.dataclass
+class Toolchain:
+    compiler: Compiler
+    # This is relevant ONLY for LLVM and should not be required for GCC
+    version: int
+
+    @property
+    def short_name(self) -> str:
+        return str(self.compiler.value)
+
+    @property
+    def full_name(self) -> str:
+        if self.compiler == Compiler.GCC:
+            return self.short_name
+
+        return f"{self.short_name}-{self.version}"
+
+    def to_dict(self) -> Dict[str, Union[str, int]]:
+        return {
+            "name": self.short_name,
+            "fullname": self.full_name,
+            "version": self.version,
+        }
+
+
+@dataclasses.dataclass
+class BuildConfig:
+    arch: Arch
+    toolchain: Toolchain
+    kernel: str = "LATEST"
+    run_veristat: bool = False
+    parallel_tests: bool = False
+    build_release: bool = False
+
+    @property
+    def runs_on(self) -> List[str]:
+        if is_managed_repo():
+            return ["self-hosted", self.arch.value]
+        return [DEFAULT_RUNNER]
+
+    @property
+    def tests(self) -> Dict[str, Any]:
+        tests_list = [
+            "test_progs",
+            "test_progs_parallel",
+            "test_progs_no_alu32",
+            "test_progs_no_alu32_parallel",
+            "test_maps",
+            "test_verifier",
+        ]
+
+        if not self.parallel_tests:
+            tests_list = [test for test in tests_list if not test.endswith("parallel")]
+
+        return {"include": [generate_test_config(test) for test in tests_list]}
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "arch": self.arch.value,
+            "toolchain": self.toolchain.to_dict(),
+            "kernel": self.kernel,
+            "run_veristat": self.run_veristat,
+            "parallel_tests": self.parallel_tests,
+            "build_release": self.build_release,
+            "runs_on": self.runs_on,
+            "tests": self.tests,
+        }
+
+
+def is_managed_repo() -> bool:
+    return (
+        os.environ["GITHUB_REPOSITORY_OWNER"] == MANAGED_OWNER
+        and os.environ["GITHUB_REPOSITORY"] in MANAGED_REPOS
+    )
+
+
 def set_output(name, value):
     """Write an output variable to the GitHub output file."""
     with open(os.getenv("GITHUB_OUTPUT"), "a", encoding="utf-8") as file:
         file.write(f"{name}={value}\n")
 
 
-def generate_test_config(test):
+def generate_test_config(test: str) -> Dict[str, Union[str, int]]:
     """Create the configuration for the provided test."""
-    experimental = test.endswith("_parallel")
+    is_parallel = test.endswith("_parallel")
     config = {
         "test": test,
-        "continue_on_error": experimental,
+        "continue_on_error": is_parallel,
         # While in experimental mode, parallel jobs may get stuck
         # anywhere, including in user space where the kernel won't detect
         # a problem and panic. We add a second layer of (smaller) timeouts
@@ -36,112 +131,50 @@ def generate_test_config(test):
         # non-experimental jobs, 360 is the default which will be
         # superseded by the overall workflow timeout (but we need to
         # specify something).
-        "timeout_minutes": 30 if experimental else 360,
+        "timeout_minutes": 30 if is_parallel else 360,
     }
     return config
 
 
-def get_tests(config):
-    tests = [
-        "test_progs",
-        "test_progs_parallel",
-        "test_progs_no_alu32",
-        "test_progs_no_alu32_parallel",
-        "test_maps",
-        "test_verifier",
+if __name__ == "__main__":
+    matrix = [
+        BuildConfig(
+            arch=Arch.X86_64,
+            toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION),
+            run_veristat=True,
+            parallel_tests=True,
+        ),
+        BuildConfig(
+            arch=Arch.X86_64,
+            toolchain=Toolchain(compiler=Compiler.LLVM, version=DEFAULT_LLVM_VERSION),
+            build_release=True,
+        ),
+        BuildConfig(
+            arch=Arch.X86_64,
+            toolchain=Toolchain(compiler=Compiler.LLVM, version=18),
+            build_release=True,
+        ),
+        BuildConfig(
+            arch=Arch.AARCH64,
+            toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION),
+            build_release=True,
+        ),
+        # BuildConfig(
+        #     arch=Arch.AARCH64,
+        #     toolchain=Toolchain(
+        #         compiler=Compiler.LLVM,
+        #         version=DEFAULT_LLVM_VERSION
+        #     ),
+        # ),
+        BuildConfig(
+            arch=Arch.S390X,
+            toolchain=Toolchain(compiler=Compiler.GCC, version=DEFAULT_LLVM_VERSION),
+        ),
     ]
-    if config.get("parallel_tests", False):
-        return tests
-    return [test for test in tests if not test.endswith("parallel")]
-
-
-matrix = [
-    {
-        "kernel": "LATEST",
-        "runs_on": [],
-        "arch": Arch.X86_64.value,
-        "toolchain": "gcc",
-        "llvm-version": "17",
-        "run_veristat": True,
-        "parallel_tests": True,
-    },
-    {
-        "kernel": "LATEST",
-        "runs_on": [],
-        "arch": Arch.X86_64.value,
-        "toolchain": "llvm",
-        "llvm-version": "17",
-        "build_release": True,
-    },
-    {
-        "kernel": "LATEST",
-        "runs_on": [],
-        "arch": Arch.X86_64.value,
-        "toolchain": "llvm",
-        "llvm-version": "18",
-        "build_release": True,
-    },
-    {
-        "kernel": "LATEST",
-        "runs_on": [],
-        "arch": Arch.AARCH64.value,
-        "toolchain": "gcc",
-        "llvm-version": "17",
-    },
-    # {
-    #     "kernel": "LATEST",
-    #     "runs_on": [],
-    #     "arch": Arch.AARCH64.value,
-    #     "toolchain": "llvm",
-    #     "llvm-version": "16",
-    # },
-    {
-        "kernel": "LATEST",
-        "runs_on": [],
-        "arch": Arch.S390X.value,
-        "toolchain": "gcc",
-        "llvm-version": "17",
-    },
-]
-self_hosted_repos = [
-    "kernel-patches/bpf",
-    "kernel-patches/vmtest",
-]
-
-for idx in range(len(matrix) - 1, -1, -1):
-    if matrix[idx]["toolchain"] == "gcc":
-        matrix[idx]["toolchain_full"] = "gcc"
-    else:
-        matrix[idx]["toolchain_full"] = "llvm-" + matrix[idx]["llvm-version"]
-
-    # Set run_veristat to false by default.
-    matrix[idx]["run_veristat"] = matrix[idx].get("run_veristat", False)
-    # Set build_release to false by default.
-    matrix[idx]["build_release"] = matrix[idx].get("build_release", False)
-    # Feel in the tests
-    matrix[idx]["tests"] = {
-        "include": [generate_test_config(test) for test in get_tests(matrix[idx])]
-    }
+    # Outside of those repositories we only run on x86_64
+    if not is_managed_repo():
+        matrix = [config for config in matrix if config.arch == Arch.X86_64]
 
-# Only a few repository within "kernel-patches" use self-hosted runners.
-if (
-    os.environ["GITHUB_REPOSITORY_OWNER"] != "kernel-patches"
-    or os.environ["GITHUB_REPOSITORY"] not in self_hosted_repos
-):
-    # Outside of those repositories, we only run on x86_64 GH hosted runners (ubuntu-20.04)
-    # We need to run on ubuntu 20.04 because our rootfs is based on debian buster and we
-    # otherwise get library versioning issue such as
-    # `./test_verifier: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found (required by ./test_verifier)`
-    for idx in range(len(matrix) - 1, -1, -1):
-        if matrix[idx]["arch"] != Arch.X86_64.value:
-            del matrix[idx]
-        else:
-            matrix[idx]["runs_on"] = ["ubuntu-20.04"]
-else:
-    # Otherwise, run on (self-hosted, arch) runners
-    for idx in range(len(matrix) - 1, -1, -1):
-        matrix[idx]["runs_on"].extend(["self-hosted", matrix[idx]["arch"]])
-
-build_matrix = {"include": matrix}
-
-set_output("build_matrix", dumps(build_matrix))
+    json_matrix = json.dumps({"include": [config.to_dict() for config in matrix]})
+    print(json_matrix)
+    set_output("build_matrix", json_matrix)

.github/workflows/test.yml

@@ -30,7 +30,7 @@ jobs:
   build-and-test:
     # Setting name to arch-compiler here to avoid lengthy autogenerated names due to matrix
     # e.g build-and-test x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
-    name: "${{ matrix.arch }}-${{ matrix.toolchain_full }}"
+    name: "${{ matrix.arch }}-${{ matrix.toolchain.fullname }}"
     uses: ./.github/workflows/kernel-build-test.yml
     needs: [set-matrix]
     permissions:
@@ -41,10 +41,10 @@ jobs:
       matrix: ${{ fromJSON(needs.set-matrix.outputs.build-matrix) }}
     with:
       arch: ${{ matrix.arch }}
-      toolchain_full: ${{ matrix.toolchain_full }}
-      toolchain: ${{ matrix.toolchain }}
+      toolchain_full: ${{ matrix.toolchain.fullname }}
+      toolchain: ${{ matrix.toolchain.name }}
       runs_on: ${{ toJSON(matrix.runs_on) }}
-      llvm-version: ${{ matrix.llvm-version }}
+      llvm-version: ${{ matrix.toolchain.version }}
       kernel: ${{ matrix.kernel }}
       tests: ${{ toJSON(matrix.tests) }}
       run_veristat: ${{ matrix.run_veristat }}