Fix github_osv expected values

Add a ruby version test

Fix merge conflict

Add support for nuget , hex , gem , cargo

Add support for maven , go and packagist ecosystem.

Import data from github advisory-database using osv format and add support for all osv ecosystems

Signed-off-by: ziadhany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -16,6 +16,7 @@
 from vulnerabilities.importers import elixir_security
 from vulnerabilities.importers import gentoo
 from vulnerabilities.importers import github
+from vulnerabilities.importers import github_osv
 from vulnerabilities.importers import gitlab
 from vulnerabilities.importers import istio
 from vulnerabilities.importers import mozilla
@@ -59,6 +60,7 @@
     elixir_security.ElixirSecurityImporter,
     apache_tomcat.ApacheTomcatImporter,
     xen.XenImporter,
+    github_osv.GithubOSVImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/github_osv.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+import logging
+from io import BytesIO
+from pathlib import Path
+from typing import Iterable
+from zipfile import ZipFile
+
+import requests
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import Importer
+from vulnerabilities.importers.osv import parse_advisory_data
+
+logger = logging.getLogger(__name__)
+
+
+class GithubOSVImporter(Importer):
+    license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
+    spdx_license_expression = "CC-BY-4.0"
+    url = "https://codeload.github.com/github/advisory-database/zip/refs/heads/main"
+
+    def advisory_data(self) -> Iterable[AdvisoryData]:
+        response = requests.get(self.url).content
+        with ZipFile(BytesIO(response)) as zip_file:
+            for file_name in filter(
+                lambda filename_list: ("github-reviewed" in Path(filename_list).parts)
+                and Path(filename_list).suffix == ".json",
+                zip_file.namelist(),
+            ):
+                with zip_file.open(file_name) as f:
+                    raw_data = json.load(f)
+                    try:
+                        yield parse_advisory_data(
+                            raw_data,
+                            supported_ecosystems=[
+                                "pypi",
+                                "npm",
+                                "maven",
+                                "golang",
+                                "composer",
+                                "hex",
+                                "gem",
+                                "nuget",
+                                "cargo",
+                            ],
+                        )
+                    except Exception as e:
+                        logger.error(f"Invalid file name: {file_name} - {e}")

vulnerabilities/importers/osv.py

@@ -16,7 +16,6 @@
 from packageurl import PackageURL
 from univers.version_range import RANGE_CLASS_BY_SCHEMES
 from univers.versions import InvalidVersion
-from univers.versions import PypiVersion
 from univers.versions import SemverVersion
 from univers.versions import Version
 
@@ -30,8 +29,20 @@
 
 logger = logging.getLogger(__name__)
 
+PURL_TYPE_BY_OSV_SCHEME = {
+    "npm": "npm",
+    "pypi": "pypi",
+    "maven": "maven",
+    "nuget": "nuget",
+    "packagist": "composer",
+    "rubygems": "gem",
+    "go": "golang",
+    "hex": "hex",
+    "cargo": "cargo",
+}
 
-def parse_advisory_data(raw_data: dict, supported_ecosystem) -> Optional[AdvisoryData]:
+
+def parse_advisory_data(raw_data: dict, supported_ecosystems: List) -> Optional[AdvisoryData]:
     """
     Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and
     a ``supported_ecosystem`` string.
@@ -53,18 +64,23 @@ def parse_advisory_data(raw_data: dict, supported_ecosystem) -> Optional[Advisor
 
     for affected_pkg in raw_data.get("affected") or []:
         purl = get_affected_purl(affected_pkg=affected_pkg, raw_id=raw_id)
-        if purl.type != supported_ecosystem:
+        if purl.type in PURL_TYPE_BY_OSV_SCHEME.keys():
+            purl = purl._replace(type=PURL_TYPE_BY_OSV_SCHEME[purl.type])
+
+        if purl.type not in supported_ecosystems:
             logger.error(f"Unsupported package type: {purl!r} in OSV: {raw_id!r}")
             continue
 
         affected_version_range = get_affected_version_range(
             affected_pkg=affected_pkg,
             raw_id=raw_id,
-            supported_ecosystem=supported_ecosystem,
+            supported_ecosystem=purl.type,
         )
 
         for fixed_range in affected_pkg.get("ranges") or []:
-            fixed_version = get_fixed_versions(fixed_range=fixed_range, raw_id=raw_id)
+            fixed_version = get_fixed_versions(
+                fixed_range=fixed_range, raw_id=raw_id, supported_ecosystem=purl.type
+            )
 
             for version in fixed_version:
                 affected_packages.append(
@@ -74,7 +90,6 @@ def parse_advisory_data(raw_data: dict, supported_ecosystem) -> Optional[Advisor
                         fixed_version=version,
                     )
                 )
-
     return AdvisoryData(
         aliases=aliases,
         summary=summary,
@@ -113,14 +128,19 @@ def get_severities(raw_data) -> Iterable[VulnerabilitySeverity]:
     """
     Yield VulnerabilitySeverity extracted from a mapping of OSV ``raw_data``
     """
-    for severity in raw_data.get("severity") or []:
-        if severity.get("type") == "CVSS_V3":
-            vector = severity["score"]
-            system = SCORING_SYSTEMS["cvssv3.1"]
-            score = system.compute(vector)
-            yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)
-        else:
-            logger.error(f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}")
+    try:
+        for severity in raw_data.get("severity") or []:
+            if severity.get("type") == "CVSS_V3":
+                vector = severity["score"]
+                system = SCORING_SYSTEMS["cvssv3.1"]
+                score = system.compute(vector)
+                yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)
+            else:
+                logger.error(
+                    f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}"
+                )
+    except Exception as e:
+        logger.error(f"Invalid severity {e}")
 
     ecosystem_specific = raw_data.get("ecosystem_specific") or {}
     severity = ecosystem_specific.get("severity")
@@ -199,18 +219,19 @@ def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):
             )
 
 
-def get_fixed_versions(fixed_range, raw_id) -> List[Version]:
+def get_fixed_versions(fixed_range, raw_id, supported_ecosystem) -> List[Version]:
     """
     Return a list of unique fixed univers Versions given a ``fixed_range``
     univers VersionRange and a ``raw_id``.
 
     For example::
 
-    >>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj")
+    >>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj", supported_ecosystem="pypi",)
     []
     >>> get_fixed_versions(
     ...   fixed_range={"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}]},
-    ...   raw_id="GHSA-j3f7-7rmc-6wqj"
+    ...   raw_id="GHSA-j3f7-7rmc-6wqj",
+    ...   supported_ecosystem="pypi",
     ... )
     [PypiVersion(string='1.7.0')]
     """
@@ -221,21 +242,32 @@ def get_fixed_versions(fixed_range, raw_id) -> List[Version]:
 
     fixed_range_type = fixed_range["type"]
 
-    for version in extract_fixed_versions(fixed_range):
+    version_class = None
+    try:
+        version_class = RANGE_CLASS_BY_SCHEMES[supported_ecosystem].version_class
+    except KeyError:
+        logger.error(
+            f"Unknown version range for ecosystem {supported_ecosystem} for OSV id: {raw_id!r}"
+        )
 
-        # FIXME: ECOSYSTEM does not imply PyPI!!!!
+    for version in extract_fixed_versions(fixed_range):
         if fixed_range_type == "ECOSYSTEM":
             try:
-                fixed_versions.append(PypiVersion(version))
+                if not version_class:
+                    raise InvalidVersion(
+                        f"Unsupported version for ecosystem: {supported_ecosystem}"
+                    )
+                fixed_versions.append(version_class(version))
             except InvalidVersion:
-                logger.error(f"Invalid PypiVersion: {version!r} for OSV id: {raw_id!r}")
+                logger.error(
+                    f"Invalid version class: {version_class} - {version!r} for OSV id: {raw_id!r}"
+                )
 
         elif fixed_range_type == "SEMVER":
             try:
                 fixed_versions.append(SemverVersion(version))
             except InvalidVersion:
                 logger.error(f"Invalid SemverVersion: {version!r} for OSV id: {raw_id!r}")
-
         else:
             logger.error(f"Unsupported fixed version type: {version!r} for OSV id: {raw_id!r}")
 

vulnerabilities/importers/pypa.py

@@ -27,7 +27,7 @@ class PyPaImporter(Importer):
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
         for raw_data in fork_and_get_files(self.url):
-            yield parse_advisory_data(raw_data=raw_data, supported_ecosystem="pypi")
+            yield parse_advisory_data(raw_data=raw_data, supported_ecosystems=["pypi"])
 
 
 class ForkError(Exception):

vulnerabilities/importers/pysec.py

@@ -38,4 +38,4 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                     continue
                 with zip_file.open(file_name) as f:
                     vul_info = json.load(f)
-                    yield parse_advisory_data(raw_data=vul_info, supported_ecosystem="pypi")
+                    yield parse_advisory_data(raw_data=vul_info, supported_ecosystems=["pypi"])

vulnerabilities/tests/test_data/github_osv/github_osv_expected_1.json

@@ -0,0 +1,121 @@
+{
+  "aliases": [
+    "CVE-2015-8315",
+    "GHSA-3fx5-fwvr-xrjg"
+  ],
+  "summary": "Regular Expression Denial of Service in ms\nVersions of `ms` prior to 0.7.1 are affected by a regular expression denial of service vulnerability when extremely long version strings are parsed.\n\n## Proof of Concept\n```javascript\nvar ms = require('ms');\nvar genstr = function (len, chr) {\n   var result = \"\";\n   for (i=0; i<=len; i++) {\n       result = result + chr;\n   }\n\n   return result;\n}\n\nms(genstr(process.argv[2], \"5\") + \" minutea\");\n\n```\n\n### Results\nShowing increase in execution time based on the input string.\n```\n$ time node ms.js 10000\n\nreal\t0m0.758s\nuser\t0m0.724s\nsys\t0m0.031s\n\n$ time node ms.js 20000\n\nreal\t0m2.580s\nuser\t0m2.494s\nsys\t0m0.047s\n\n$ time node ms.js 30000\n\nreal\t0m5.747s\nuser\t0m5.483s\nsys\t0m0.080s\n\n$ time node ms.js 80000\n\nreal\t0m41.022s\nuser\t0m38.894s\nsys\t0m0.529s\n```",
+  "affected_packages": [
+    {
+      "package": {
+        "type": "npm",
+        "namespace": null,
+        "name": "ms",
+        "version": null,
+        "qualifiers": null,
+        "subpath": null
+      },
+      "affected_version_range": null,
+      "fixed_version": "0.7.1"
+    }
+  ],
+  "references": [
+    {
+      "reference_id": "",
+      "url": "https://nvd.nist.gov/vuln/detail/CVE-2015-8315",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "https://github.com/unshiftio/millisecond/",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "https://support.f5.com/csp/article/K46337613?utm_source=f5support&amp;utm_medium=RSS",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "https://www.npmjs.com/advisories/46",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "http://www.openwall.com/lists/oss-security/2016/04/20/11",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "http://www.securityfocus.com/bid/96389",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    }
+  ],
+  "date_published": "2017-10-24T18:33:36+00:00",
+  "weaknesses": []
+}