Migrate debian importer to importer-improver model

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

requirements.txt

@@ -107,7 +107,7 @@ toml==0.10.2
 tomli==2.0.1
 traitlets==5.1.1
 typing_extensions==4.1.1
-univers==30.4.0
+univers==30.5.1
 urllib3==1.26.9
 wcwidth==0.2.5
 websocket-client==0.59.0

vulnerabilities/helpers.py

@@ -26,6 +26,7 @@
 import logging
 import os
 import re
+from collections import defaultdict
 from functools import total_ordering
 from typing import List
 from typing import Optional
@@ -280,3 +281,18 @@ def dedupe(original: List) -> List:
         if i not in new_list:
             new_list.append(i)
     return new_list
+
+
+def get_affected_packages_by_patched_package(
+    affected_packages: List[AffectedPackage],
+):
+    """
+    Return a mapping of list of vulnerable purls keyed by
+    purl which fix those vulnerable package.
+    """
+    affected_packages_by_patched_package = defaultdict(list)
+    for package in affected_packages:
+        affected_packages_by_patched_package[package.patched_package].append(
+            package.vulnerable_package
+        )
+    return affected_packages_by_patched_package

vulnerabilities/importer.py

@@ -150,7 +150,9 @@ def get_fixed_purl(self):
         return fixed_purl
 
     @classmethod
-    def merge(cls, affected_packages: Iterable):
+    def merge(
+        cls, affected_packages: Iterable
+    ) -> Tuple[PackageURL, List[VersionRange], List[Version]]:
         """
         Return a tuple with all attributes of AffectedPackage as a set
         for all values in the given iterable of AffectedPackage

vulnerabilities/importers/__init__.py

@@ -20,6 +20,7 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 from vulnerabilities.importers import alpine_linux
+from vulnerabilities.importers import debian
 from vulnerabilities.importers import github
 from vulnerabilities.importers import nginx
 from vulnerabilities.importers import nvd
@@ -35,6 +36,7 @@
     openssl.OpensslImporter,
     redhat.RedhatImporter,
     pysec.PyPIImporter,
+    debian.DebianImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/debian.py

@@ -21,115 +21,204 @@
 #  VulnerableCode is a free software tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-import dataclasses
+import logging
 from typing import Any
+from typing import Iterable
 from typing import List
 from typing import Mapping
-from typing import Set
 
 import requests
-from dateutil import parser as dateparser
+from django.db.models.query import QuerySet
 from packageurl import PackageURL
+from univers.version_range import DebianVersionRange
+from univers.versions import DebianVersion
 
+from vulnerabilities.helpers import AffectedPackage as LegacyAffectedPackage
+from vulnerabilities.helpers import get_affected_packages_by_patched_package
+from vulnerabilities.helpers import get_item
 from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
+from vulnerabilities.importer import UnMergeablePackageError
+from vulnerabilities.improver import MAX_CONFIDENCE
+from vulnerabilities.improver import Improver
+from vulnerabilities.improver import Inference
+from vulnerabilities.models import Advisory
 
+logger = logging.getLogger(__name__)
 
-class DebianImporter(Importer):
-    def __enter__(self):
-        if self.response_is_new():
-            self._api_response = self._fetch()
-
-        else:
-            self._api_response = {}
-
-    def updated_advisories(self) -> Set[AdvisoryData]:
-        advisories = []
-
-        for pkg_name, records in self._api_response.items():
-            advisories.extend(self._parse(pkg_name, records))
-
-        return self.batch_advisories(advisories)
-
-    def _fetch(self) -> Mapping[str, Any]:
-        return requests.get(self.config.debian_tracker_url).json()
-
-    def _parse(self, pkg_name: str, records: Mapping[str, Any]) -> List[AdvisoryData]:
-        advisories = []
-        ignored_versions = {"3.8.20-4."}
 
+class DebianImporter(Importer):
+    # From: Tushar Goel <tgoel@nexb.com>
+    # Date: Thu, May 12, 2022 at 11:42 PM +00:00
+    # Subject: Usage of Debian Security Data in VulnerableCode
+    # To: <team@security.debian.org>
+
+    # Hey,
+
+    # We would like to integrate the debian security data in vulnerablecode
+    # [1][2] which is a FOSS db of FOSS vulnerability data. We were not able
+    # to know under which license the debian security data comes. We would
+    # be grateful to have your acknowledgement over usage of the debian
+    # security data in vulnerablecode and have some kind of licensing
+    # declaration from your side.
+
+    # [1] - https://github.com/nexB/vulnerablecode
+    # [2] - https://github.com/nexB/vulnerablecode/pull/723
+
+    spdx_license_expression = "MIT"
+    license_url = "https://www.debian.org/license"
+
+    api_url = "https://security-tracker.debian.org/tracker/data/json"
+
+    def get_response(self):
+        response = requests.get(self.api_url)
+        if response.status_code == 200:
+            return response.json()
+        raise Exception(
+            f"Failed to fetch data from {self.api_url!r} with status code: {response.status_code!r}"
+        )
+
+    def advisory_data(self) -> Iterable[AdvisoryData]:
+        response = self.get_response()
+        for pkg_name, records in response.items():
+            yield from self.parse(pkg_name, records)
+
+    def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryData]:
         for cve_id, record in records.items():
-            impacted_purls, resolved_purls = [], []
+            affected_versions = []
+            fixed_versions = []
             if not cve_id.startswith("CVE"):
+                logger.error(f"Invalid CVE ID: {cve_id} in {record} in package {pkg_name}")
                 continue
 
             # vulnerabilities starting with something else may not be public yet
             # see for instance https://web.archive.org/web/20201215213725/https://security-tracker.debian.org/tracker/TEMP-0000000-A2EB44
             # TODO: this would need to be revisited though to ensure we are not missing out on anything
+            # https://github.com/nexB/vulnerablecode/issues/730
 
-            for release_name, release_record in record["releases"].items():
-                if not release_record.get("repositories", {}).get(release_name):
-                    continue
-
-                version = release_record["repositories"][release_name]
+            releases = record["releases"].items()
+            for release_name, release_record in releases:
+                version = get_item(release_record, "repositories", release_name)
 
-                if version in ignored_versions:
+                if not version:
+                    logger.error(
+                        f"Version not found for {release_name} in {record} in package {pkg_name}"
+                    )
                     continue
 
                 purl = PackageURL(
                     name=pkg_name,
                     type="deb",
                     namespace="debian",
-                    version=version,
                     qualifiers={"distro": release_name},
                 )
 
                 if release_record.get("status", "") == "resolved":
-                    resolved_purls.append(purl)
+                    fixed_versions.append(version)
                 else:
-                    impacted_purls.append(purl)
-
-                if (
-                    "fixed_version" in release_record
-                    and release_record["fixed_version"] not in ignored_versions
-                ):
-                    resolved_purls.append(
-                        PackageURL(
-                            name=pkg_name,
-                            type="deb",
-                            namespace="debian",
-                            version=release_record["fixed_version"],
-                            qualifiers={"distro": release_name},
-                        )
-                    )
+                    affected_versions.append(version)
+
+                if "fixed_version" in release_record:
+                    fixed_versions.append(release_record["fixed_version"])
 
             references = []
             debianbug = record.get("debianbug")
             if debianbug:
                 bug_url = f"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug={debianbug}"
-                references.append(Reference(url=bug_url, reference_id=debianbug))
-            advisories.append(
-                AdvisoryData(
-                    vulnerability_id=cve_id,
-                    affected_packages=nearest_patched_package(impacted_purls, resolved_purls),
-                    summary=record.get("description", ""),
-                    references=references,
+                references.append(Reference(url=bug_url, reference_id=str(debianbug)))
+            # Use list(dict.fromkeys) to remove duplicates
+            affected_versions = list(dict.fromkeys(affected_versions))
+            fixed_versions = list(dict.fromkeys(fixed_versions))
+            if affected_versions:
+                affected_version_range = DebianVersionRange.from_versions(affected_versions)
+            else:
+                affected_version_range = None
+            affected_packages = []
+            for fixed_version in fixed_versions:
+                affected_packages.append(
+                    AffectedPackage(
+                        package=purl,
+                        affected_version_range=affected_version_range,
+                        fixed_version=DebianVersion(fixed_version),
+                    )
                 )
+            yield AdvisoryData(
+                aliases=[cve_id],
+                summary=record.get("description", ""),
+                affected_packages=affected_packages,
+                references=references,
             )
 
-        return advisories
 
-    def response_is_new(self):
+class DebianBasicImprover(Improver):
+    @property
+    def interesting_advisories(self) -> QuerySet:
+        return Advisory.objects.filter(created_by=DebianImporter.qualified_name)
+
+    def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
         """
-        Return True if a request response is for new data likely changed or
-        updated since we last checked.
+        Yield Inferences for the given advisory data
         """
-        head = requests.head(self.config.debian_tracker_url)
-        date_str = head.headers.get("last-modified")
-        last_modified_date = dateparser.parse(date_str)
-        if self.config.last_run_date:
-            return self.config.last_run_date < last_modified_date
+        if not advisory_data.affected_packages:
+            return
+        try:
+            purl, affected_version_ranges, fixed_versions = AffectedPackage.merge(
+                advisory_data.affected_packages
+            )
+        except UnMergeablePackageError:
+            logger.error(f"Cannot merge with different purls {advisory_data.affected_packages!r}")
+            return
+
+        pkg_type = purl.type
+        pkg_namespace = purl.namespace
+        pkg_name = purl.name
+        pkg_qualifiers = purl.qualifiers
+        fixed_purls = [
+            PackageURL(
+                type=pkg_type,
+                namespace=pkg_namespace,
+                name=pkg_name,
+                version=str(version),
+                qualifiers=pkg_qualifiers,
+            )
+            for version in fixed_versions
+        ]
+        if not affected_version_ranges:
+            for fixed_purl in fixed_purls:
+                yield Inference.from_advisory_data(
+                    advisory_data,  # We are getting all valid versions to get this inference
+                    confidence=MAX_CONFIDENCE,
+                    affected_purls=[],
+                    fixed_purl=fixed_purl,
+                )
+        else:
+            aff_versions = set()
+            for affected_version_range in affected_version_ranges:
+                for constraint in affected_version_range.constraints:
+                    aff_versions.add(constraint.version.string)
+            affected_purls = [
+                PackageURL(
+                    type=pkg_type,
+                    namespace=pkg_namespace,
+                    name=pkg_name,
+                    version=version,
+                    qualifiers=pkg_qualifiers,
+                )
+                for version in aff_versions
+            ]
+            affected_packages: List[LegacyAffectedPackage] = nearest_patched_package(
+                vulnerable_packages=affected_purls, resolved_packages=fixed_purls
+            )
 
-        return True
+            for (fixed_package, affected_packages,) in get_affected_packages_by_patched_package(
+                affected_packages=affected_packages
+            ).items():
+                yield Inference.from_advisory_data(
+                    advisory_data,
+                    confidence=MAX_CONFIDENCE,  # We are getting all valid versions to get this inference
+                    affected_purls=affected_packages,
+                    fixed_purl=fixed_package,
+                )

vulnerabilities/importers/github.py

@@ -37,6 +37,7 @@
 from vulnerabilities import helpers
 from vulnerabilities import severity_systems
 from vulnerabilities.helpers import AffectedPackage as LegacyAffectedPackage
+from vulnerabilities.helpers import get_affected_packages_by_patched_package
 from vulnerabilities.helpers import get_item
 from vulnerabilities.helpers import nearest_patched_package
 from vulnerabilities.importer import AdvisoryData
@@ -465,18 +466,10 @@ def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
                 vulnerable_packages=affected_purls, resolved_packages=unaffected_purls
             )
 
-            unique_patched_packages_with_affected_packages = {}
-            for package in affected_packages:
-                if package.patched_package not in unique_patched_packages_with_affected_packages:
-                    unique_patched_packages_with_affected_packages[package.patched_package] = []
-                unique_patched_packages_with_affected_packages[package.patched_package].append(
-                    package.vulnerable_package
-                )
-
             for (
                 fixed_package,
                 affected_packages,
-            ) in unique_patched_packages_with_affected_packages.items():
+            ) in get_affected_packages_by_patched_package(affected_packages).items():
                 yield Inference.from_advisory_data(
                     advisory_data,
                     confidence=100,  # We are getting all valid versions to get this inference

vulnerabilities/improvers/__init__.py

@@ -28,6 +28,7 @@
     default.DefaultImprover,
     importers.nginx.NginxBasicImprover,
     importers.github.GitHubBasicImprover,
+    importers.debian.DebianBasicImprover,
 ]
 
 IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY}

vulnerabilities/templates/vulnerabilities.html

@@ -36,7 +36,7 @@ <h1 class="title">
           </tr>
           {% for vulnerability in  vulnerabilities %}
               <tr>
-                  <td><a href="{% url 'vulnerability_view' vulnerability.pk %}">{{vulnerability.vulcoid}}</a></td>
+                  <td><a href="{% url 'vulnerability_view' vulnerability.pk %}">{{vulnerability.vulnerability_id}}</a></td>
                   <td>{{vulnerability.vulnerable_package_count}}</td>
                   <td>{{vulnerability.patched_package_count}}</td>
               </tr>

vulnerabilities/tests/conftest.py

@@ -46,7 +46,6 @@ def no_rmtree(monkeypatch):
     "test_archlinux.py",
     "test_data_source.py",
     "test_debian_oval.py",
-    "test_debian.py",
     "test_elixir_security.py",
     "test_gentoo.py",
     "test_importer_yielder.py",