Merge pull request #259 from sbs2001/custom_vuln_ids

Handle vulnerabilities which don't have any vulnerability ids

Author: sbs2001

requirements.txt

@@ -55,3 +55,4 @@ zipp==0.6.0
 requests==2.23.0
 toml==0.10.2
 PyYAML==5.3.1
+freezegun==1.1.0

vulnerabilities/api.py

@@ -63,7 +63,6 @@ class Meta:
 
 
 class HyperLinkedVulnerabilitySerializer(serializers.HyperlinkedModelSerializer):
-    vulnerability_id = serializers.CharField(source="cve_id")
 
     class Meta:
         model = Vulnerability
@@ -177,7 +176,6 @@ def bulk_search(self, request):
 
 
 class VulnerabilityFilterSet(filters.FilterSet):
-    vulnerability_id = filters.CharFilter(field_name="cve_id")
 
     class Meta:
         model = Vulnerability
@@ -208,13 +206,13 @@ def bulk_search(self, request):
                 },
             )
 
-        for cve_id in request.data["vulnerabilities"]:
-            filter_list.append(cve_id)
+        for vulnerability_id in request.data["vulnerabilities"]:
+            filter_list.append(vulnerability_id)
             # This handles the case when the said cve doesnt exist in db
-            response[cve_id] = {}
-        res = Vulnerability.objects.filter(cve_id__in=filter_list)
+            response[vulnerability_id] = {}
+        res = Vulnerability.objects.filter(vulnerability_id__in=filter_list)
         for vuln in res:
-            response[vuln.cve_id] = MinimalVulnerabilitySerializer(
+            response[vuln.vulnerability_id] = MinimalVulnerabilitySerializer(
                 vuln, context={"request": request}
             ).data
         return Response(response)

vulnerabilities/data_source.py

@@ -81,14 +81,14 @@ class Advisory:
     impacted_package_urls: Iterable[PackageURL]
     resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list)
     vuln_references: List[Reference] = dataclasses.field(default_factory=list)
-    cve_id: Optional[str] = None
+    vulnerability_id: Optional[str] = None
 
     def __hash__(self):
         s = "{}{}{}{}".format(
             self.summary,
-            "".join(sorted([str(p) for p in self.impacted_package_urls])),
-            "".join(sorted([str(p) for p in self.resolved_package_urls])),
-            self.cve_id,
+            ''.join(sorted([str(p) for p in self.impacted_package_urls])),
+            ''.join(sorted([str(p) for p in self.resolved_package_urls])),
+            self.vulnerability_id,
         )
         return hash(s)
 
@@ -539,8 +539,6 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
                     summary=description,
                     impacted_package_urls=affected_purls,
                     resolved_package_urls=safe_purls,
-                    cve_id=vuln_id,
-                    vuln_references=references,
-                )
-            )
+                    vulnerability_id=vuln_id,
+                    vuln_references=references))
         return all_adv

vulnerabilities/fixtures/debian.json

@@ -3,7 +3,7 @@
   "model": "vulnerabilities.vulnerability",
   "pk": 1,
   "fields": {
-    "cve_id": "CVE-2014-8242",
+    "vulnerability_id": "CVE-2014-8242",
     "summary": ""
 
   }
@@ -12,7 +12,7 @@
   "model": "vulnerabilities.vulnerability",
   "pk": 2,
   "fields": {
-    "cve_id": "CVE-2009-1382",
+    "vulnerability_id": "CVE-2009-1382",
     "summary": ""
 
   }
@@ -21,7 +21,7 @@
   "model": "vulnerabilities.vulnerability",
   "pk": 3,
   "fields": {
-    "cve_id": "CVE-2009-2459",
+    "vulnerability_id": "CVE-2009-2459",
     "summary": ""
 
   }

vulnerabilities/fixtures/openssl.json

@@ -153,12 +153,23 @@ def process_advisories(data_source: DataSource) -> None:
                         existing_ref = get_vuln_pkg_refs(vuln, pkg)
                         if not existing_ref:
                             bulk_create_vuln_pkg_refs.add(pkg_vuln_ref)
+                            # A vulnerability-package relationship does not exist already
+                            # if either the vulnerability or the package is just created.
 
                         else:
-                            # This handles conflicts between existing data and obtained data
-                            if existing_ref[0].is_vulnerable != pkg_vuln_ref.is_vulnerable:
-                                handle_conflicts([existing_ref[0], pkg_vuln_ref.to_model_object()])
-                                existing_ref.delete()
+                            # insert only if it there is no existing vulnerability-package relationship.  # nopep8
+                            existing_ref = get_vuln_pkg_refs(vuln, pkg)
+                            if not existing_ref:
+                                bulk_create_vuln_pkg_refs.add(pkg_vuln_ref)
+
+                            else:
+                                # This handles conflicts between existing data and obtained data
+                                if existing_ref[0].is_vulnerable != pkg_vuln_ref.is_vulnerable:
+                                    handle_conflicts(
+                                        [existing_ref[0], pkg_vuln_ref.to_model_object()]
+                                    )
+                                    existing_ref.delete()
+
             except Exception:
                 # TODO: store error but continue
                 logger.error(
@@ -223,26 +234,13 @@ def _get_or_create_vulnerability(
     advisory: Advisory,
 ) -> Tuple[models.Vulnerability, bool]:
 
-    if advisory.cve_id:
-        query_kwargs = {"cve_id": advisory.cve_id}
-    elif advisory.summary:
-        query_kwargs = {"summary": advisory.summary}
-    else:
-        return models.Vulnerability.objects.create(), True
-
-    try:
-        vuln, created = models.Vulnerability.objects.get_or_create(**query_kwargs)
-        # Eventually we only want to keep summary from NVD and ignore other descriptions.
-        if advisory.summary and vuln.summary != advisory.summary:
-            vuln.summary = advisory.summary
-            vuln.save()
-        return vuln, created
-
-    except Exception:
-        logger.error(
-            f"Failed to _get_or_create_vulnerability: {query_kwargs!r}:\n" + traceback.format_exc()
-        )
-        raise
+    vuln, created = models.Vulnerability.objects.get_or_create(vulnerability_id=advisory.vulnerability_id)  # nopep8
+    # Eventually we only want to keep summary from NVD and ignore other descriptions.
+    if advisory.summary and vuln.summary != advisory.summary:
+        vuln.summary = advisory.summary
+        vuln.save()
+
+    return vuln, created
 
 
 def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]:

vulnerabilities/import_runner.py

@@ -58,16 +58,16 @@
             'debian_tracker_url': 'https://security-tracker.debian.org/tracker/data/json'
         },
     },
-#     {
-#         'name': 'safetydb',
-#         'license': 'cc-by-nc-4.0',
-#         'last_run': None,
-#         'data_source': 'SafetyDbDataSource',
-#         'data_source_cfg': {
-#             'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json',  # nopep8
-#             'etags': {}
-#         },
-#     },
+    {
+        'name': 'safetydb',
+        'license': 'cc-by-nc-4.0',
+        'last_run': None,
+        'data_source': 'SafetyDbDataSource',
+        'data_source_cfg': {
+            'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json',  # nopep8
+            'etags': {}
+        },
+    },
     {
         'name': 'npm',
         'license': 'mit',

vulnerabilities/importer_yielder.py

@@ -47,3 +47,4 @@
 from vulnerabilities.importers.suse_scores import SUSESeverityScoreDataSource
 from vulnerabilities.importers.ubuntu import UbuntuDataSource
 from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource
+from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource

vulnerabilities/importers/__init__.py

@@ -187,7 +187,7 @@ def _load_advisories(
                         impacted_package_urls=[],
                         resolved_package_urls=resolved_purls,
                         vuln_references=references,
-                        cve_id=vuln_ids[0].upper() if vuln_ids[0] != "CVE-????-?????" else None,
+                        vulnerability_id=vuln_ids[0] if vuln_ids[0] != "CVE-????-?????" else None,
                     )
                 )
 

vulnerabilities/importers/alpine_linux.py

@@ -82,7 +82,7 @@ def to_advisory(self, advisory_page):
 
             advisories.append(
                 Advisory(
-                    cve_id=cve_id,
+                    vulnerability_id=cve_id,
                     summary=cve_description_paragraph.text,
                     impacted_package_urls=affected_packages,
                     resolved_package_urls=fixed_packages,