Ensure uniqueness of pipeline_id and datasource_id

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

vulnerabilities/pipelines/v2_importers/collect_fix_commits.py

@@ -3,184 +3,221 @@
 
 class CollectLinuxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_linux_fix_commits"
+    datasource_id = "linux_fix_commits"
     repo_url = "https://github.com/torvalds/linux"
 
 
 class CollectBusyBoxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_busybox_fix_commits"
+    datasource_id = "busybox_fix_commits"
     repo_url = "https://github.com/mirror/busybox"
 
 
 class CollectNginxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_nginx_fix_commits"
+    datasource_id = "nginx_fix_commits"
     repo_url = "https://github.com/nginx/nginx"
 
 
 class CollectApacheTomcatFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_apache_tomcat_fix_commits"
+    datasource_id = "apache_tomcat_fix_commits"
     repo_url = "https://github.com/apache/tomcat"
 
 
 class CollectMysqlServerFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_mysql_server_fix_commits"
+    datasource_id = "mysql_server_fix_commits"
     repo_url = "https://github.com/mysql/mysql-server"
 
 
 class CollectPostgresqlFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_postgresql_fix_commits"
+    datasource_id = "postgresql_fix_commits"
     repo_url = "https://github.com/postgres/postgres"
 
 
 class CollectMongodbFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_mongodb_fix_commits"
+    datasource_id = "mongodb_fix_commits"
     repo_url = "https://github.com/mongodb/mongo"
 
 
 class CollectRedisFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_redis_fix_commits"
+    datasource_id = "redis_fix_commits"
     repo_url = "https://github.com/redis/redis"
 
 
 class CollectSqliteFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_sqlite_fix_commits"
+    datasource_id = "sqlite_fix_commits"
     repo_url = "https://github.com/sqlite/sqlite"
 
 
 class CollectPhpFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_php_fix_commits"
+    datasource_id = "php_fix_commits"
     repo_url = "https://github.com/php/php-src"
 
 
 class CollectPythonCpythonFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_python_cpython_fix_commits"
+    datasource_id = "python_cpython_fix_commits"
     repo_url = "https://github.com/python/cpython"
 
 
 class CollectRubyFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_ruby_fix_commits"
+    datasource_id = "ruby_fix_commits"
     repo_url = "https://github.com/ruby/ruby"
 
 
 class CollectGoFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_go_fix_commits"
+    datasource_id = "go_fix_commits"
     repo_url = "https://github.com/golang/go"
 
 
 class CollectNodeJsFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_node_js_fix_commits"
+    datasource_id = "node_js_fix_commits"
     repo_url = "https://github.com/nodejs/node"
 
 
 class CollectRustFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_rust_fix_commits"
+    datasource_id = "rust_fix_commits"
     repo_url = "https://github.com/rust-lang/rust"
 
 
 class CollectOpenjdkFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_openjdk_fix_commits"
+    datasource_id = "openjdk_fix_commits"
     repo_url = "https://github.com/openjdk/jdk"
 
 
 class CollectSwiftFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_swift_fix_commits"
+    datasource_id = "swift_fix_commits"
     repo_url = "https://github.com/swiftlang/swift"
 
 
 class CollectDjangoFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_django_fix_commits"
+    datasource_id = "django_fix_commits"
     repo_url = "https://github.com/django/django"
 
 
 class CollectRailsFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_rails_fix_commits"
+    datasource_id = "rails_fix_commits"
     repo_url = "https://github.com/rails/rails"
 
 
 class CollectLaravelFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_laravel_fix_commits"
+    datasource_id = "laravel_fix_commits"
     repo_url = "https://github.com/laravel/framework"
 
 
 class CollectSpringFrameworkFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_spring_framework_fix_commits"
+    datasource_id = "spring_framework_fix_commits"
     repo_url = "https://github.com/spring-projects/spring-framework"
 
 
 class CollectReactFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_react_fix_commits"
+    datasource_id = "react_fix_commits"
     repo_url = "https://github.com/facebook/react"
 
 
 class CollectAngularFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_angular_fix_commits"
+    datasource_id = "angular_fix_commits"
     repo_url = "https://github.com/angular/angular"
 
 
 class CollectWordpressFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_wordpress_fix_commits"
+    datasource_id = "wordpress_fix_commits"
     repo_url = "https://github.com/WordPress/WordPress"
 
 
 class CollectDockerMobyFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_docker_moby_fix_commits"
+    datasource_id = "docker_moby_fix_commits"
     repo_url = "https://github.com/moby/moby"
 
 
 class CollectKubernetesFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_kubernetes_fix_commits"
+    datasource_id = "kubernetes_fix_commits"
     repo_url = "https://github.com/kubernetes/kubernetes"
 
 
 class CollectQemuFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_qemu_fix_commits"
+    datasource_id = "qemu_fix_commits"
     repo_url = "https://gitlab.com/qemu-project/qemu"
 
 
 class CollectXenProjectFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_xen_project_fix_commits"
+    datasource_id = "xen_project_fix_commits"
     repo_url = "https://github.com/xen-project/xen"
 
 
 class CollectVirtualboxFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_virtualbox_fix_commits"
+    datasource_id = "virtualbox_fix_commits"
     repo_url = "https://github.com/mirror/vbox"
 
 
 class CollectContainerdFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_containerd_fix_commits"
+    datasource_id = "containerd_fix_commits"
     repo_url = "https://github.com/containerd/containerd"
 
 
 class CollectAnsibleFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_ansible_fix_commits"
+    datasource_id = "ansible_fix_commits"
     repo_url = "https://github.com/ansible/ansible"
 
 
 class CollectTerraformFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_terraform_fix_commits"
+    datasource_id = "terraform_fix_commits"
     repo_url = "https://github.com/hashicorp/terraform"
 
 
 class CollectWiresharkFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_wireshark_fix_commits"
+    datasource_id = "wireshark_fix_commits"
     repo_url = "https://gitlab.com/wireshark/wireshark"
 
 
 class CollectTcpdumpFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_tcpdump_fix_commits"
+    datasource_id = "tcpdump_fix_commits"
     repo_url = "https://github.com/the-tcpdump-group/tcpdump"
 
 
 class CollectGitFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_git_fix_commits"
+    datasource_id = "git_fix_commits"
     repo_url = "https://github.com/git/git"
 
 
 class CollectJenkinsFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_jenkins_fix_commits"
+    datasource_id = "jenkins_fix_commits"
     repo_url = "https://github.com/jenkinsci/jenkins"
 
 
 class CollectGitlabFixCommitsPipeline(CollectVCSFixCommitPipeline):
     pipeline_id = "collect_gitlab_fix_commits"
+    datasource_id = "gitlab_fix_commits"
     repo_url = "https://gitlab.com/gitlab-org/gitlab-foss"

vulnerabilities/pipelines/v2_importers/glibc_importer.py

@@ -30,6 +30,7 @@ class GlibcImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     """
 
     pipeline_id = "glibc_importer_v2"
+    datasource_id = "glibc"
     spdx_license_expression = "LGPL-2.1-only"
     license_url = "https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=LICENSES"
     repo_url = "git+https://sourceware.org/git/glibc.git"

vulnerabilities/pipelines/v2_importers/linux_kernel_importer.py

@@ -30,6 +30,7 @@ class LinuxKernelPipeline(VulnerableCodeBaseImporterPipelineV2):
     """
 
     pipeline_id = "linux_kernel_cves_fix_commits"
+    datasource_id = "linux_kernel"
     spdx_license_expression = "Apache-2.0"
     license_url = "https://github.com/nluedtke/linux_kernel_cves/blob/master/LICENSE"
     run_once = True

vulnerabilities/pipes/vcs_collector_utils.py

@@ -34,7 +34,7 @@ class CollectVCSFixCommitPipeline(VulnerableCodeBaseImporterPipelineV2):
 
     pipeline_id = "collect_vcs_fix_commits"
 
-    datasource_id = pipeline_id.replace("collect_", "")
+    datasource_id = "vcs_fix_commits"
 
     repo_url: str
     patterns: list[str] = [

vulnerabilities/tests/pipelines/test_pipeline_id.py

@@ -7,11 +7,13 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import collections
 import importlib
 import inspect
 import unittest
 from pathlib import Path
 
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
 from vulnerabilities.pipelines import VulnerableCodePipeline
 
 PIPELINE_DIR = Path(__file__).parent.parent.parent / "pipelines"
@@ -20,6 +22,7 @@
 class PipelineTests(unittest.TestCase):
     def setUp(self):
         self.pipeline_dict = self.collect_pipeline_ids()
+        self.v2_pipeline_dict = self.collect_v2_pipeline_ids()
 
     def collect_pipeline_ids(self):
         """Return pipeline_ids from all the VulnerableCodePipeline."""
@@ -40,6 +43,32 @@ def collect_pipeline_ids(self):
 
         return pipeline_dict
 
+    def import_all_pipeline_modules(self):
+        """Import all modules under vulnerabilities.pipelines recursively."""
+        for pipeline in PIPELINE_DIR.rglob("*.py"):
+            if pipeline.name == "__init__.py":
+                continue
+
+            module_name = ".".join(pipeline.relative_to(PIPELINE_DIR).with_suffix("").parts)
+            importlib.import_module(f"vulnerabilities.pipelines.{module_name}")
+
+    def collect_v2_pipeline_ids(self):
+        """Return pipeline_id and datasource_id from all VulnerableCodeBaseImporterPipelineV2 subclasses."""
+        self.import_all_pipeline_modules()
+        importlib.import_module("vulnerabilities.pipes.vcs_collector_utils")
+
+        pipeline_dict = {}
+        for obj in self._all_subclasses(VulnerableCodeBaseImporterPipelineV2):
+            pipeline_dict[obj] = (obj.pipeline_id, obj.datasource_id)
+
+        return pipeline_dict
+
+    def _all_subclasses(self, cls):
+        subclasses = set(cls.__subclasses__())
+        for subclass in list(subclasses):
+            subclasses.update(self._all_subclasses(subclass))
+        return subclasses
+
     def test_no_empty_pipeline_ids(self):
         empty_pipeline_ids = [cls for cls, pid in self.pipeline_dict.items() if pid == ""]
 
@@ -64,3 +93,73 @@ def test_unique_pipeline_ids(self):
         pipeline_ids = self.pipeline_dict.values()
         unique_ids = set(pipeline_ids)
         assert len(pipeline_ids) == len(unique_ids), "`pipeline_id` should be unique."
+
+    def test_v2_pipeline_ids_are_defined(self):
+        missing_pipeline_ids = [
+            cls.__name__
+            for cls, (pipeline_id, _) in self.v2_pipeline_dict.items()
+            if pipeline_id is None or pipeline_id == ""
+        ]
+
+        if missing_pipeline_ids:
+            error_messages = [
+                f"{pipeline} has missing pipeline_id." for pipeline in missing_pipeline_ids
+            ]
+            error_message = (
+                "`pipeline_id` for V2 pipelines must be defined and non-empty:\n"
+                + "\n".join(error_messages)
+            )
+            assert False, error_message
+
+    def test_v2_datasource_ids_are_defined(self):
+        missing_datasource_ids = [
+            cls.__name__
+            for cls, (_, datasource_id) in self.v2_pipeline_dict.items()
+            if datasource_id is None or datasource_id == ""
+        ]
+
+        if missing_datasource_ids:
+            error_messages = [
+                f"{pipeline} has missing datasource_id." for pipeline in missing_datasource_ids
+            ]
+            error_message = (
+                "`datasource_id` for V2 pipelines must be defined and non-empty:\n"
+                + "\n".join(error_messages)
+            )
+            assert False, error_message
+
+    def test_unique_v2_pipeline_ids(self):
+        pipeline_to_classes = collections.defaultdict(list)
+        for cls, (pipeline_id, _) in self.v2_pipeline_dict.items():
+            pipeline_to_classes[pipeline_id].append(cls.__name__)
+
+        duplicates = {
+            pipeline_id: classes
+            for pipeline_id, classes in pipeline_to_classes.items()
+            if pipeline_id and len(classes) > 1
+        }
+
+        assert not duplicates, (
+            "Duplicate V2 pipeline_id values found: "
+            + ", ".join(
+                f"{pipeline_id}: {classes}" for pipeline_id, classes in duplicates.items()
+            )
+        )
+
+    def test_unique_v2_datasource_ids(self):
+        datasource_to_classes = collections.defaultdict(list)
+        for cls, (_, datasource_id) in self.v2_pipeline_dict.items():
+            datasource_to_classes[datasource_id].append(cls.__name__)
+
+        duplicates = {
+            datasource_id: classes
+            for datasource_id, classes in datasource_to_classes.items()
+            if datasource_id and len(classes) > 1
+        }
+
+        assert not duplicates, (
+            "Duplicate V2 datasource_id values found: "
+            + ", ".join(
+                f"{datasource_id}: {classes}" for datasource_id, classes in duplicates.items()
+            )
+        )