Sync with main branch and fix latest tests.

* Added incremental time id in import_runner.py
  to prevent vulnerability id conflicts

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

Author: sbs2001

vulnerabilities/import_runner.py

@@ -139,13 +139,19 @@ def process_advisories(data_source: DataSource, create_vulcodes) -> None:
     # Treat updated_advisories and added_advisories as same. Eventually
     # we want to  refactor all data sources to  provide advisories via a
     # single method.
+    vulcoid = datetime.datetime.now()
     advisory_batches = chain(data_source.updated_advisories(), data_source.added_advisories())
     for batch in advisory_batches:
         for advisory in batch:
             try:
+
                 if not advisory.identifier and not create_vulcodes:
                     continue
 
+                if not advisory.identifier:
+                    advisory.identifier = "VULCOID-" + vulcoid.strftime("%Y-%m-%d-%H:%M:%S")
+                    vulcoid += datetime.timedelta(seconds=1)
+
                 vuln, vuln_created = _get_or_create_vulnerability(advisory)
                 for vuln_ref in advisory.vuln_references:
                     ref = VulnerabilityReferenceInserter(
@@ -154,9 +160,11 @@ def process_advisories(data_source: DataSource, create_vulcodes) -> None:
                         reference_id=vuln_ref.reference_id,
                     )
 
-                    if vuln_created or not vuln_ref_exists(vuln, vuln_ref.url, vuln_ref.reference_id):
-                        # A vulnerability reference can't exist if the vulnerability is just created so
-                        # insert it
+                    if vuln_created or not vuln_ref_exists(
+                        vuln, vuln_ref.url, vuln_ref.reference_id
+                    ):
+                        # A vulnerability reference can't exist if the vulnerability is
+                        # just created, so insert it
                         bulk_create_vuln_refs.add(ref)
 
                 for purl in chain(advisory.impacted_package_urls, advisory.resolved_package_urls):
@@ -176,25 +184,27 @@ def process_advisories(data_source: DataSource, create_vulcodes) -> None:
                         existing_ref = get_vuln_pkg_refs(vuln, pkg)
                         if not existing_ref:
                             bulk_create_vuln_pkg_refs.add(pkg_vuln_ref)
-                            # A vulnerability-package relationship does not exist already if either the
-                            # vulnerability or the package is just created.
+                            # A vulnerability-package relationship does not exist already
+                            # if either the vulnerability or the package is just created.
 
                         else:
-                            # insert only if it there is no existing vulnerability-package relationship.
+                            # insert only if it there is no existing vulnerability-package relationship.  # nopep8
                             existing_ref = get_vuln_pkg_refs(vuln, pkg)
                             if not existing_ref:
                                 bulk_create_vuln_pkg_refs.add(pkg_vuln_ref)
 
                             else:
                                 # This handles conflicts between existing data and obtained data
                                 if existing_ref[0].is_vulnerable != pkg_vuln_ref.is_vulnerable:
-                                    handle_conflicts([existing_ref[0], pkg_vuln_ref.to_model_object()])
+                                    handle_conflicts(
+                                        [existing_ref[0], pkg_vuln_ref.to_model_object()]
+                                    )
                                     existing_ref.delete()
+
             except Exception:
                 # TODO: store error but continue
                 logger.error(
-                    f"Failed to process advisory: {advisory!r}:\n"
-                    + traceback.format_exc()
+                    f"Failed to process advisory: {advisory!r}:\n" + traceback.format_exc()
                 )
 
     models.VulnerabilityReference.objects.bulk_create(
@@ -259,17 +269,20 @@ def _get_or_create_vulnerability(
     advisory: Advisory,
 ) -> Tuple[models.Vulnerability, bool]:
 
-    vuln, created = models.Vulnerability.objects.get_or_create(identifier=advisory.identifier)
+    try:
+        vuln, created = models.Vulnerability.objects.get_or_create(identifier=advisory.identifier)
 
-    # Eventually we only want to keep summary from NVD and ignore other descriptions.
-    if advisory.summary and vuln.summary != advisory.summary:
-        vuln.summary = advisory.summary
-        vuln.save()
+        # Eventually we only want to keep summary from NVD and ignore other descriptions.
+        if advisory.summary and vuln.summary != advisory.summary:
+            vuln.summary = advisory.summary
+            vuln.save()
+
+        return vuln, created
 
     except Exception:
         logger.error(
-            f"Failed to _get_or_create_vulnerability: {query_kwargs!r}:\n"
-            + traceback.format_exc())
+            f"Failed to _get_or_create_vulnerability: {query_kwargs!r}:\n" + traceback.format_exc()
+        )
         raise
 
 

vulnerabilities/importer_yielder.py

@@ -61,16 +61,16 @@
             'debian_tracker_url': 'https://security-tracker.debian.org/tracker/data/json'
         },
     },
-#     {
-#         'name': 'safetydb',
-#         'license': 'cc-by-nc-4.0',
-#         'last_run': None,
-#         'data_source': 'SafetyDbDataSource',
-#         'data_source_cfg': {
-#             'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json',  # nopep8
-#             'etags': {}
-#         },
-#     },
+    # {
+    #     'name': 'safetydb',
+    #     'license': 'cc-by-nc-4.0',
+    #     'last_run': None,
+    #     'data_source': 'SafetyDbDataSource',
+    #     'data_source_cfg': {
+    #         'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json',  # nopep8
+    #         'etags': {}
+    #     },
+    # },
     {
         'name': 'npm',
         'license': 'mit',

vulnerabilities/importers/apache_tomcat.py

@@ -116,7 +116,7 @@ def to_advisories(self, apache_tomcat_advisory_html):
                         summary="",
                         impacted_package_urls=affected_packages,
                         resolved_package_urls=fixed_package,
-                        cve_id=cve_id,
+                        identifier=cve_id,
                         vuln_references=references,
                     )
                 )

vulnerabilities/importers/kaybee.py

@@ -66,7 +66,7 @@ def yaml_file_to_advisory(yaml_path):
             references.append(Reference(url=f"{commit['repository']}/{commit['id']}"))
 
     return Advisory(
-        cve_id=vuln_id,
+        identifier=vuln_id,
         summary=summary,
         impacted_package_urls=impacted_packages,
         resolved_package_urls=resolved_packages,

vulnerabilities/importers/nginx.py

@@ -109,7 +109,7 @@ def to_advisories(self, data):
 
             advisories.append(
                 Advisory(
-                    cve_id=cve_id,
+                    identifier=cve_id,
                     summary=summary,
                     impacted_package_urls=vulnerable_packages,
                     resolved_package_urls=fixed_packages,

vulnerabilities/importers/postgresql.py

@@ -105,7 +105,7 @@ def to_advisories(data):
 
         advisories.append(
             Advisory(
-                cve_id=cve_id,
+                identifier=cve_id,
                 summary=summary,
                 vuln_references=references,
                 impacted_package_urls=affected_packages,

vulnerabilities/importers/safety_db.py

@@ -25,16 +25,17 @@
 
 import asyncio
 import dataclasses
-import json
+import re
+import logging
 from typing import Any
 from typing import Iterable
 from typing import Mapping
 from typing import Set
 from typing import Tuple
 
+import requests
 from dephell_specifier import RangeSpecifier
 from packageurl import PackageURL
-import requests
 from schema import Or
 from schema import Regex
 from schema import Schema
@@ -44,22 +45,21 @@
 from vulnerabilities.data_source import DataSourceConfiguration
 from vulnerabilities.data_source import Reference
 from vulnerabilities.package_managers import PypiVersionAPI
-from vulnerabilities.helpers import create_etag
+
+logger = logging.getLogger(__name__)
 
 
 def validate_schema(advisory_dict):
 
-    scheme = {
-        str: [
-            {
-                "advisory": str,
-                "cve": Or(None, Regex(r"CVE-\d+-\d+")),
-                "id": Regex(r"^pyup.io-\d"),
-                "specs": list,
-                "v": str,
-            }
-        ]
-    }
+    scheme = [
+        {
+            "advisory": str,
+            "cve": Or(None, str),
+            "id": Regex(r"^pyup.io-\d"),
+            "specs": list,
+            "v": str,
+        }
+    ]
 
     Schema(scheme).validate(advisory_dict)
 
@@ -77,7 +77,6 @@ class SafetyDbDataSource(DataSource):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._api_response = self._fetch()
-        # validate_schema(self._api_response)
 
     def __enter__(self):
         self._versions = PypiVersionAPI()
@@ -91,9 +90,8 @@ def set_api(self, packages):
         asyncio.run(self._versions.load_api(packages))
 
     def _fetch(self) -> Mapping[str, Any]:
-        if create_etag(data_src=self, url=self.config.url, etag_key="ETag"):
+        if self.create_etag(self.config.url):
             return requests.get(self.config.url).json()
-
         return []
 
     def collect_packages(self):
@@ -103,22 +101,33 @@ def updated_advisories(self) -> Set[Advisory]:
         advisories = []
 
         for package_name in self._api_response:
+            if package_name == "$meta":
+                # This is the first entry in the data feed. It contains metadata of the feed.
+                # Skip it.
+                continue
+
+            try:
+                validate_schema(self._api_response[package_name])
+
+            except Exception as e:
+                logger.error(e)
+                continue
+
             all_package_versions = self.versions.get(package_name)
-            if len(all_package_versions) == 0:
+            if not len(all_package_versions):
                 # PyPi does not have data about this package, we skip these
                 continue
 
             for advisory in self._api_response[package_name]:
-
                 impacted_purls, resolved_purls = categorize_versions(
                     package_name, all_package_versions, advisory["specs"]
                 )
 
-                cve_ids = advisory.get("cve") or [""]
-
-                # meaning if cve_ids is not [''] but either ['CVE-123'] or ['CVE-123, CVE-124']
-                if len(cve_ids[0]):
-                    cve_ids = [s.strip() for s in cve_ids.split(",")]
+                if advisory["cve"]:
+                    # Check on advisory["cve"] instead of using `get` because it can have null value
+                    cve_ids = re.findall(r"CVE-\d+-\d+", advisory["cve"])
+                else:
+                    cve_ids = [None]
 
                 reference = [Reference(reference_id=advisory["id"])]
 
@@ -135,17 +144,29 @@ def updated_advisories(self) -> Set[Advisory]:
 
         return self.batch_advisories(advisories)
 
+    def create_etag(self, url):
+        etag = requests.head(url).headers.get("ETag")
+        if not etag:
+            # Kind of inaccurate to return True since etag is
+            # not created
+            return True
+        elif url in self.config.etags:
+            if self.config.etags[url] == etag:
+                return False
+        self.config.etags[url] = etag
+        return True
+
 
 def categorize_versions(
     package_name: str,
     all_versions: Set[str],
-    version_ranges: Iterable[str],
+    version_specs: Iterable[str],
 ) -> Tuple[Set[PackageURL], Set[PackageURL]]:
     """
     :return: impacted, resolved purls
     """
     impacted_versions, impacted_purls = set(), set()
-    ranges = [RangeSpecifier(s) for s in version_ranges]
+    ranges = [RangeSpecifier(s) for s in version_specs]
 
     for version in all_versions:
         if any([version in r for r in ranges]):

vulnerabilities/management/commands/push.py

@@ -44,7 +44,7 @@ def cd(newdir):
 def get_vulcodes():
 
     vulcodes = models.Vulnerability.objects.filter(
-        identifier__startswith="VULCODE"
+        identifier__startswith="VULCOID"
     ).select_related()
     for vuln in vulcodes:
         yield {

vulnerabilities/models.py

@@ -45,9 +45,7 @@ class Vulnerability(models.Model):
 
     def save(self, *args, **kwargs):
         if not self.identifier:
-            # Replace `str(datetime.now())` with our custom identifier TBD.
-            self.identifier = "VULCODE-" + str(datetime.now())
-
+            self.identifier = "VULCOID-" + datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
         super().save(*args, **kwargs)
 
     @property

vulnerabilities/tests/test_apache_tomcat.py

@@ -89,7 +89,7 @@ def test_to_advisories(self):
                             reference_id="",
                         ),
                     ],
-                    cve_id="CVE-2016-0763",
+                    identifier="CVE-2016-0763",
                 ),
                 Advisory(
                     summary="",
@@ -127,7 +127,7 @@ def test_to_advisories(self):
                             reference_id="",
                         ),
                     ],
-                    cve_id="CVE-2015-5351",
+                    identifier="CVE-2015-5351",
                 ),
                 Advisory(
                     summary="",
@@ -169,7 +169,7 @@ def test_to_advisories(self):
                             reference_id="",
                         ),
                     ],
-                    cve_id="CVE-2016-0706",
+                    identifier="CVE-2016-0706",
                 ),
                 Advisory(
                     summary="",
@@ -207,16 +207,16 @@ def test_to_advisories(self):
                             reference_id="",
                         ),
                     ],
-                    cve_id="CVE-2016-0714",
+                    identifier="CVE-2016-0714",
                 ),
             ],
-            key=lambda x: x.cve_id,
+            key=lambda x: x.identifier,
         )
 
         with open(TEST_DATA) as f:
             found_advisories = self.data_src.to_advisories(f)
 
-        found_advisories.sort(key=lambda x: x.cve_id)
+        found_advisories.sort(key=lambda x: x.identifier)
 
         for i in range(len(found_advisories)):
             found_advisories[i].vuln_references.sort(key=lambda x: x.url)