mark unfurl should be a step in unfurling version range pipeline

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

vulnerabilities/importers/__init__.py

@@ -120,39 +120,8 @@
         ubuntu_osv_importer_v2.UbuntuOSVImporterPipeline,
         alpine_linux_importer_v2.AlpineLinuxImporterPipeline,
         linux_kernel_importer_v2.LinuxKernelPipeline,
-        github_importer.GitHubAPIImporterPipeline,
-        gitlab_importer.GitLabImporterPipeline,
-        github_osv.GithubOSVImporter,
-        pypa_importer.PyPaImporterPipeline,
-        npm_importer.NpmImporterPipeline,
-        nginx_importer.NginxImporterPipeline,
-        pysec_importer.PyPIImporterPipeline,
-        fireeye_importer_v2.FireeyeImporterPipeline,
-        apache_tomcat.ApacheTomcatImporter,
-        postgresql.PostgreSQLImporter,
-        debian.DebianImporter,
-        curl.CurlImporter,
-        epss.EPSSImporter,
-        vulnrichment.VulnrichImporter,
-        alpine_linux_importer.AlpineLinuxImporterPipeline,
-        ruby.RubyImporter,
-        apache_kafka.ApacheKafkaImporter,
         openssl_importer_v2.OpenSSLImporterPipeline,
-        redhat.RedhatImporter,
-        archlinux.ArchlinuxImporter,
-        debian_oval.DebianOvalImporter,
-        retiredotnet.RetireDotnetImporter,
-        apache_httpd.ApacheHTTPDImporter,
-        mozilla.MozillaImporter,
-        gentoo.GentooImporter,
-        istio.IstioImporter,
-        project_kb_msr2019.ProjectKBMSRImporter,
-        suse_scores.SUSESeverityScoreImporter,
-        elixir_security.ElixirSecurityImporter,
-        xen.XenImporter,
-        ubuntu_usn.UbuntuUSNImporter,
-        fireeye.FireyeImporter,
-        oss_fuzz.OSSFuzzImporter,
+        fireeye_importer_v2.FireeyeImporterPipeline,
         collect_fix_commits_v2.CollectLinuxFixCommitsPipeline,
         collect_fix_commits_v2.CollectBusyBoxFixCommitsPipeline,
         collect_fix_commits_v2.CollectNginxFixCommitsPipeline,
@@ -190,6 +159,37 @@
         collect_fix_commits_v2.CollectGitFixCommitsPipeline,
         collect_fix_commits_v2.CollectJenkinsFixCommitsPipeline,
         collect_fix_commits_v2.CollectGitlabFixCommitsPipeline,
+        # github_importer.GitHubAPIImporterPipeline,
+        # gitlab_importer.GitLabImporterPipeline,
+        # github_osv.GithubOSVImporter,
+        # pypa_importer.PyPaImporterPipeline,
+        # npm_importer.NpmImporterPipeline,
+        # nginx_importer.NginxImporterPipeline,
+        # pysec_importer.PyPIImporterPipeline,
+        # apache_tomcat.ApacheTomcatImporter,
+        # postgresql.PostgreSQLImporter,
+        # debian.DebianImporter,
+        # curl.CurlImporter,
+        # epss.EPSSImporter,
+        # vulnrichment.VulnrichImporter,
+        # alpine_linux_importer.AlpineLinuxImporterPipeline,
+        # apache_kafka.ApacheKafkaImporter,
+        # ruby.RubyImporter,
+        # redhat.RedhatImporter,
+        # archlinux.ArchlinuxImporter,
+        # debian_oval.DebianOvalImporter,
+        # retiredotnet.RetireDotnetImporter,
+        # apache_httpd.ApacheHTTPDImporter,
+        # mozilla.MozillaImporter,
+        # gentoo.GentooImporter,
+        # istio.IstioImporter,
+        # project_kb_msr2019.ProjectKBMSRImporter,
+        # suse_scores.SUSESeverityScoreImporter,
+        # elixir_security.ElixirSecurityImporter,
+        # xen.XenImporter,
+        # ubuntu_usn.UbuntuUSNImporter,
+        # fireeye.FireyeImporter,
+        # oss_fuzz.OSSFuzzImporter,
     ]
 )
 

vulnerabilities/improvers/__init__.py

@@ -27,39 +27,13 @@
     enhance_with_metasploit as enhance_with_metasploit_v2,
 )
 from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2
-from vulnerabilities.pipelines.v2_improvers import (
-    mark_all_impacts_unfurled as mark_all_impacts_unfurled_v2,
-)
 from vulnerabilities.pipelines.v2_improvers import reference_collect_commits
 from vulnerabilities.pipelines.v2_improvers import relate_severities
 from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2
 from vulnerabilities.utils import create_registry
 
 IMPROVERS_REGISTRY = create_registry(
     [
-        valid_versions.GitHubBasicImprover,
-        valid_versions.GitLabBasicImprover,
-        valid_versions.NginxBasicImprover,
-        valid_versions.ApacheHTTPDImprover,
-        valid_versions.DebianBasicImprover,
-        valid_versions.NpmImprover,
-        valid_versions.ElixirImprover,
-        valid_versions.ApacheTomcatImprover,
-        valid_versions.ApacheKafkaImprover,
-        valid_versions.IstioImprover,
-        valid_versions.DebianOvalImprover,
-        valid_versions.OSSFuzzImprover,
-        valid_versions.RubyImprover,
-        valid_versions.GithubOSVImprover,
-        vulnerability_status.VulnerabilityStatusImprover,
-        valid_versions.CurlImprover,
-        flag_ghost_packages.FlagGhostPackagePipeline,
-        enhance_with_kev.VulnerabilityKevPipeline,
-        enhance_with_metasploit.MetasploitImproverPipeline,
-        enhance_with_exploitdb.ExploitDBImproverPipeline,
-        compute_package_risk.ComputePackageRiskPipeline,
-        compute_package_version_rank.ComputeVersionRankPipeline,
-        populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
         exploitdb_v2.ExploitDBImproverPipeline,
         enhance_with_kev_v2.VulnerabilityKevPipeline,
         flag_ghost_packages_v2.FlagGhostPackagePipeline,
@@ -72,6 +46,28 @@
         compute_advisory_todo_v2.ComputeToDo,
         reference_collect_commits.CollectReferencesFixCommitsPipeline,
         enhance_with_github_poc.GithubPocsImproverPipeline,
-        mark_all_impacts_unfurled_v2.MarkAllImpactsUnfurledPipeline,
+        # valid_versions.GitHubBasicImprover,
+        # valid_versions.GitLabBasicImprover,
+        # valid_versions.NginxBasicImprover,
+        # valid_versions.ApacheHTTPDImprover,
+        # valid_versions.DebianBasicImprover,
+        # valid_versions.NpmImprover,
+        # valid_versions.ElixirImprover,
+        # valid_versions.ApacheTomcatImprover,
+        # valid_versions.ApacheKafkaImprover,
+        # valid_versions.IstioImprover,
+        # valid_versions.DebianOvalImprover,
+        # valid_versions.OSSFuzzImprover,
+        # valid_versions.RubyImprover,
+        # valid_versions.GithubOSVImprover,
+        # vulnerability_status.VulnerabilityStatusImprover,
+        # valid_versions.CurlImprover,
+        # flag_ghost_packages.FlagGhostPackagePipeline,
+        # enhance_with_kev.VulnerabilityKevPipeline,
+        # enhance_with_metasploit.MetasploitImproverPipeline,
+        # enhance_with_exploitdb.ExploitDBImproverPipeline,
+        # compute_package_risk.ComputePackageRiskPipeline,
+        # compute_package_version_rank.ComputeVersionRankPipeline,
+        # populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
     ]
 )

vulnerabilities/pipelines/v2_improvers/mark_all_impacts_unfurled.py

@@ -13,20 +13,27 @@
 
 from aboutcode.pipeline import LoopProgress
 from django.db import transaction
+from django.db.models import Exists
 from django.db.models import F
+from django.db.models import OuterRef
 from django.db.models import Q
 from django.utils import timezone
 from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS
 from packageurl import PackageURL
 from univers.version_range import RANGE_CLASS_BY_SCHEMES
 from univers.version_range import VersionRange
 
+from vulnerabilities.models import AdvisoryV2
 from vulnerabilities.models import ImpactedPackage
 from vulnerabilities.models import ImpactedPackageAffecting
+from vulnerabilities.models import ImpactedPackageFixedBy
 from vulnerabilities.models import PackageV2
 from vulnerabilities.models import PipelineSchedule
 from vulnerabilities.pipelines import VulnerableCodePipeline
 from vulnerabilities.pipes.fetchcode_utils import get_versions
+from vulnerabilities.pipes.group_advisories import group_advisory_for_package
+from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk
+from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS
 from vulnerabilities.utils import update_purl_version
 
 
@@ -39,15 +46,15 @@ class UnfurlVersionRangePipeline(VulnerableCodePipeline):
 
     pipeline_id = "unfurl_version_range_v2"
 
-    run_interval = 2
+    run_interval = 1
     run_priority = PipelineSchedule.ExecutionPriority.HIGH
 
     # Days elapsed before version range is re-unfurled
     reunfurl_after_days = 2
 
     @classmethod
     def steps(cls):
-        return (cls.unfurl_version_range,)
+        return (cls.unfurl_version_range, cls.mark_all_impacts_unfurled)
 
     def unfurl_version_range(self):
         processed_impacted_packages_count = 0
@@ -115,6 +122,15 @@ def unfurl_version_range(self):
         self.log(f"Successfully processed {processed_impacted_packages_count:,d} ImpactedPackage.")
         self.log(f"{processed_affected_packages_count:,d} new Impact-Package relation created.")
 
+    def mark_all_impacts_unfurled(self):
+        while True:
+            advisories = list(latest_advisories_with_all_impacts_unfurled()[:100])
+
+            if not advisories:
+                break
+
+            complete_advisories_import(AdvisoryV2.objects.filter(id__in=[a.id for a in advisories]))
+
 
 def get_affected_purls(versions, impact, logger):
     affecting_version_range = VersionRange.from_string(impact.affecting_vers)
@@ -200,3 +216,56 @@ def impacted_package_qs(cutoff_day=2):
         .order_by("advisory__id", F("last_range_unfurl_at").asc(nulls_first=True))
         .only("pk", "affecting_vers", "advisory", "base_purl")
     )
+
+
+@transaction.atomic
+def complete_advisories_import(advisories):
+
+    advisory_ids = list(advisories.values_list("id", flat=True))
+
+    if not advisory_ids:
+        return
+
+    AdvisoryV2.objects.filter(id__in=advisory_ids).update(_all_impacts_unfurled=True)
+
+    affecting_package_ids = set(
+        ImpactedPackageAffecting.objects.filter(
+            impacted_package__advisory_id__in=advisory_ids
+        ).values_list(
+            "package_id",
+            flat=True,
+        )
+    )
+
+    fixed_by_package_ids = set(
+        ImpactedPackageFixedBy.objects.filter(
+            impacted_package__advisory_id__in=advisory_ids
+        ).values_list(
+            "package_id",
+            flat=True,
+        )
+    )
+
+    compute_package_risk_score_bulk(PackageV2.objects.filter(id__in=affecting_package_ids))
+
+    group_package_ids = affecting_package_ids | fixed_by_package_ids
+
+    for package in PackageV2.objects.filter(
+        id__in=group_package_ids, type__in=TYPES_WITH_MULTIPLE_IMPORTERS
+    ).iterator(chunk_size=2000):
+        group_advisory_for_package(package)
+
+
+def latest_advisories_with_all_impacts_unfurled():
+    remaining_unfurled_impacts = ImpactedPackage.objects.filter(
+        advisory_id=OuterRef("pk"),
+        last_range_unfurl_at__isnull=True,
+        advisory__is_latest=True,
+    )
+
+    return (
+        AdvisoryV2.objects.filter(_all_impacts_unfurled=False, is_latest=True)
+        .annotate(has_remaining_unfurled=Exists(remaining_unfurled_impacts))
+        .filter(has_remaining_unfurled=False)
+        .order_by("id")
+    )

vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py

@@ -14,10 +14,8 @@
 
 from vulnerabilities.models import AdvisoryV2
 from vulnerabilities.models import ImpactedPackage
-from vulnerabilities.pipelines.v2_improvers.mark_all_impacts_unfurled import (
-    MarkAllImpactsUnfurledPipeline,
-)
-from vulnerabilities.pipelines.v2_improvers.mark_all_impacts_unfurled import (
+from vulnerabilities.pipelines.v2_improvers.unfurl_version_range import UnfurlVersionRangePipeline
+from vulnerabilities.pipelines.v2_improvers.unfurl_version_range import (
     latest_advisories_with_all_impacts_unfurled,
 )
 
@@ -125,7 +123,7 @@ def test_pipeline_marks_matching_advisories():
         last_range_unfurl_at=timezone.now(),
     )
 
-    pipeline = MarkAllImpactsUnfurledPipeline()
+    pipeline = UnfurlVersionRangePipeline()
     pipeline.mark_all_impacts_unfurled()
 
     advisory.refresh_from_db()
@@ -149,7 +147,7 @@ def test_pipeline_does_not_mark_partial_advisory():
         last_range_unfurl_at=None,
     )
 
-    pipeline = MarkAllImpactsUnfurledPipeline()
+    pipeline = UnfurlVersionRangePipeline()
     pipeline.mark_all_impacts_unfurled()
 
     advisory.refresh_from_db()