Simplify patch inference, fix arch importer bug and remove useless code

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

Author: sbs2001

vulnerabilities/helpers.py

@@ -119,71 +119,40 @@ def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5):
 
 
 def nearest_patched_package(vulnerable_packages, resolved_packages):
-    if not vulnerable_packages:
-        return []
-
-    def create_package_by_version_obj_mapping(packages, overwrite=False):
-        # overwrite=True, returns version->PackageURL mapping.
-        # overwrite=False, returns version->[PackageURL] mapping.
-        if not packages:
-            return {}
-
-        package_by_version_obj_mapping = {}
-        version_class = version_class_by_package_type[packages[0].type]
-        for package in packages:
-            version_object = version_class(package.version)
-            if not overwrite:
-                if version_object in package_by_version_obj_mapping:
-                    package_by_version_obj_mapping[version_object].append(package)
-                else:
-                    package_by_version_obj_mapping[version_object] = [package]
-            else:
-                package_by_version_obj_mapping[version_object] = package
-
-        return package_by_version_obj_mapping
-
-    vulnerable_packages_by_version_obj = create_package_by_version_obj_mapping(vulnerable_packages)
-    resolved_package_by_version_obj = create_package_by_version_obj_mapping(
-        resolved_packages, overwrite=True
+    # This class is used to  get around bisect module's lack of supplying custom
+    # compartor. Get rid of this once we use python 3.10 which supports this.
+    # See https://github.com/python/cpython/pull/20556
+    class PackageURLWithVersionComparator:
+        def __init__(self, package):
+            self.package = package
+            self.version_object = version_class_by_package_type[package.type](package.version)
+
+        def __eq__(self, other):
+            return self.version_object == other.version_object
+
+        def __lt__(self, other):
+            return self.version_object < other.version_object
+
+    vulnerable_packages = sorted(
+        [PackageURLWithVersionComparator(package) for package in vulnerable_packages]
     )
-
-    vulnerable_versions = list(vulnerable_packages_by_version_obj.keys())
-    resolved_versions = list(resolved_package_by_version_obj.keys())
-
-    patched_version_by_vulnerable_versions = nearest_patched_versions(
-        vulnerable_versions, resolved_versions
+    resolved_packages = sorted(
+        [PackageURLWithVersionComparator(package) for package in resolved_packages]
     )
 
-    affected_packages_with_patched_package = []
-    for vulnerable_version, patched_version in patched_version_by_vulnerable_versions.items():
-        for vulnerable_package in vulnerable_packages_by_version_obj[vulnerable_version]:
-            affected_packages_with_patched_package.append(
-                AffectedPackageWithPatchedPackage(
-                    vulnerable_package=vulnerable_package,
-                    patched_package=resolved_package_by_version_obj.get(patched_version),
-                )
-            )
-
-    return affected_packages_with_patched_package
+    resolved_package_count = len(resolved_packages)
+    affected_package_with_patched_package_objects = []
 
+    for vulnerable_package in vulnerable_packages:
+        patched_package_index = bisect.bisect_right(resolved_packages, vulnerable_package)
+        patched_package = None
+        if patched_package_index < resolved_package_count:
+            patched_package = resolved_packages[patched_package_index].package
 
-def nearest_patched_versions(vulnerable_versions, resolved_versions):
-    """
-    Returns a mapping of vulnerable_version -> nearest_safe_version
-    """
+        affected_package_with_patched_package_objects.append(
+            AffectedPackageWithPatchedPackage(
+                vulnerable_package=vulnerable_package.package, patched_package=patched_package
+            )
+        )
 
-    vulnerable_versions = sorted(vulnerable_versions)
-    resolved_versions = sorted(resolved_versions)
-    resolved_version_count = len(resolved_versions)
-    nearest_patch_for_version = {}
-    for vulnerable_version in vulnerable_versions:
-        nearest_patch_for_version[vulnerable_version] = None
-        if not resolved_versions:
-            continue
-
-        patched_version_index = bisect.bisect_right(resolved_versions, vulnerable_version)
-        if patched_version_index >= resolved_version_count:
-            continue
-        nearest_patch_for_version[vulnerable_version] = resolved_versions[patched_version_index]
-
-    return nearest_patch_for_version
+    return affected_package_with_patched_package_objects

vulnerabilities/import_runner.py

@@ -158,59 +158,10 @@ def process_advisories(data_source: DataSource) -> None:
                     f"Failed to process advisory: {advisory!r}:\n" + traceback.format_exc()
                 )
 
-    # find_conflicting_relations handles in-memory conflicts
-    # conflicts = find_conflicting_relations(bulk_create_vuln_pkg_refs)
-
     models.PackageRelatedVulnerability.objects.bulk_create(
-        [i.to_model_object() for i in bulk_create_vuln_pkg_refs if i not in conflicts]
+        [i.to_model_object() for i in bulk_create_vuln_pkg_refs]
     )
 
-    # handle_conflicts([i.to_model_object() for i in conflicts])
-
-
-# def find_conflicting_relations(
-#     relations: Set[Set[PackageRelatedVulnerabilityInserter]],
-# ) -> Set[PackageRelatedVulnerabilityInserter]:
-
-#     # Chop off `is_vulnerable` flag from PackageRelatedVulnerabilityInserter and create a list of
-#     # tuples of format (rel.package, rel.vulnerability)
-
-#     relation_tuples = [(rel.package, rel.vulnerability) for rel in relations]
-#     relation_counter = Counter(relation_tuples).most_common()
-
-#     # If a (rel.package, rel.vulnerability) occurs twice then that means the
-#     # PackageRelatedVulnerabilityInserter objects
-#     # (rel.package, rel.vulnerability, is_vulnerable=True) and
-#     # (rel.package, rel.vulnerability, is_vulnerable=False) both existed which is conflicting data.
-#     # We detect and return these conflicts.
-
-#     conflicts = set()
-#     for rel, count in relation_counter:
-#         if count < 2:
-#             # All the subsequent entries from here on would have count == 1 which is of no interest
-#             # since conflicts exist in pairs with `is_vulnerable=True` and `is_vulnerable=False`.
-#             break
-
-#         # `rel` is of format (pkg, vuln)
-#         conflicts.add(
-#             PackageRelatedVulnerabilityInserter(
-#                 vulnerability=rel[1], package=rel[0], is_vulnerable=True
-#             )
-#         )
-
-#         conflicts.add(
-#             PackageRelatedVulnerabilityInserter(
-#                 vulnerability=rel[1], package=rel[0], is_vulnerable=False
-#             )
-#         )
-
-#     return conflicts
-
-
-# def handle_conflicts(conflicts):
-#     conflicts = serializers.serialize("json", [i for i in conflicts])
-#     models.ImportProblem.objects.create(conflicting_model=conflicts)
-
 
 def _get_or_create_vulnerability(
     advisory: Advisory,

vulnerabilities/importers/archlinux.py

@@ -88,9 +88,9 @@ def _parse(self, record) -> List[Advisory]:
         advisories = []
 
         for cve_id in record["issues"]:
-            impacted_purls, resolved_purls = [], []
             affected_packages_with_patched_package = []
             for name in record["packages"]:
+                impacted_purls, resolved_purls = [], []
                 impacted_purls.append(
                     PackageURL(
                         name=name,

vulnerabilities/migrations/0003_populate_patched_package.py

@@ -1,13 +1,33 @@
 # Generated by Django 3.0.13 on 2021-04-04 06:40
 
+import bisect
 from sys import stdout
-from multiprocessing import Pool
 
 from django.db import migrations
 from univers.versions import version_class_by_package_type
-from vulnerabilities.helpers import nearest_patched_versions
 
 
+def nearest_patched_versions(vulnerable_versions, resolved_versions):
+    """
+    Returns a mapping of vulnerable_version -> nearest_safe_version
+    """
+
+    vulnerable_versions = sorted(vulnerable_versions)
+    resolved_versions = sorted(resolved_versions)
+    resolved_version_count = len(resolved_versions)
+    nearest_patch_for_version = {}
+    for vulnerable_version in vulnerable_versions:
+        nearest_patch_for_version[vulnerable_version] = None
+        if not resolved_versions:
+            continue
+
+        patched_version_index = bisect.bisect_right(resolved_versions, vulnerable_version)
+        if patched_version_index >= resolved_version_count:
+            continue
+        nearest_patch_for_version[vulnerable_version] = resolved_versions[patched_version_index]
+
+    return nearest_patch_for_version
+
 def _get_tuple_key(packagerelatedvulnerability):
     return (
         packagerelatedvulnerability.vulnerability,

vulnerabilities/tests/test_archlinux.py

@@ -68,7 +68,6 @@ def test_import(self):
             models.PackageRelatedVulnerability.objects.filter(patched_package__isnull=False).count()
             == 8
         )
-        print(models.Package.objects.all())
         assert models.Package.objects.count() == 10
 
         self.assert_for_package(