From 17b0d7438a1850ddcf89506894e62b72be899476 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 18 Mar 2021 14:46:11 +0530 Subject: [PATCH 01/17] Remove dephell_specifier and use universal-versions in safetydb Signed-off-by: Shivam Sandbhor --- requirements.txt | 2 +- vulnerabilities/importers/safety_db.py | 29 +++++++++++++++----------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/requirements.txt b/requirements.txt index 37b289ae2..d1ae8229f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ cached-property==1.5.1 cffi==1.14.0 contextlib2==0.5.5 decorator==4.4.2 -dephell-specifier==0.2.1 +universal-versions @ git+https://github.com/sbs2001/universal-versions.git dj-database-url==0.4.2 Django==3.0.13 django-filter==2.2.0 diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py index 2cacd9f80..dd69e79f0 100755 --- a/vulnerabilities/importers/safety_db.py +++ b/vulnerabilities/importers/safety_db.py @@ -34,7 +34,8 @@ from typing import Tuple import requests -from dephell_specifier import RangeSpecifier +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import PYPIVersion from packageurl import PackageURL from schema import Or from schema import Regex @@ -101,9 +102,9 @@ def updated_advisories(self) -> Set[Advisory]: advisories = [] for package_name in self._api_response: - if package_name == "$meta": + if package_name == "$meta" or package_name == "cumin": # This is the first entry in the data feed. It contains metadata of the feed. - # Skip it. + # Skip it. The 'cumin' entry is wrong continue try: @@ -119,15 +120,15 @@ def updated_advisories(self) -> Set[Advisory]: continue for advisory in self._api_response[package_name]: - impacted_purls, resolved_purls = categorize_versions( - package_name, all_package_versions, advisory["specs"] - ) - if advisory["cve"]: # Check on advisory["cve"] instead of using `get` because it can have null value cve_ids = re.findall(r"CVE-\d+-\d+", advisory["cve"]) else: - cve_ids = [None] + continue + + impacted_purls, resolved_purls = categorize_versions( + package_name, all_package_versions, advisory["specs"] + ) reference = [Reference(reference_id=advisory["id"])] @@ -166,12 +167,17 @@ def categorize_versions( :return: impacted, resolved purls """ impacted_versions, impacted_purls = set(), set() - ranges = [RangeSpecifier(s) for s in version_specs] + vurl_spec = f"pypi:{ ','.join(version_specs) }" + vurl_spec = VersionSpecifier.from_version_spec_string(vurl_spec) for version in all_versions: - if any([version in r for r in ranges]): - impacted_versions.add(version) + try: + version_object = PYPIVersion(version) + except: + continue + if version_object in vurl_spec: + impacted_versions.add(version) impacted_purls.add( PackageURL( name=package_name, @@ -183,5 +189,4 @@ def categorize_versions( resolved_purls = set() for version in all_versions - impacted_versions: resolved_purls.add(PackageURL(name=package_name, type="pypi", version=version)) - return impacted_purls, resolved_purls From d316abccc3f21a61df33ffd86012485e5e49f188 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Wed, 24 Mar 2021 15:27:46 +0530 Subject: [PATCH 02/17] Refactor npm importer to use universal_versions Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/npm.py | 71 ++++++++++++++++++++++++++++--- vulnerabilities/tests/test_npm.py | 22 +++++----- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index 055f3c945..6d43a10ad 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -32,7 +32,8 @@ from urllib.request import urlopen from dateutil.parser import parse -from dephell_specifier import RangeSpecifier +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import SemverVersion from packageurl import PackageURL from vulnerabilities.data_source import Advisory @@ -86,8 +87,15 @@ def process_file(self, file) -> List[Advisory]: advisories = [] package_name = record["module_name"].strip() all_versions = self.versions.get(package_name) - aff_range = record.get("vulnerable_versions", "") - fixed_range = record.get("patched_versions", "") + aff_range = record.get("vulnerable_versions") + if not aff_range: + aff_range = "" + fixed_range = record.get("patched_versions") + if not fixed_range: + fixed_range = "" + + if aff_range == "*" or fixed_range == "*": + return [] impacted_versions, resolved_versions = categorize_versions( all_versions, aff_range, fixed_range @@ -120,6 +128,36 @@ def _versions_to_purls(package_name, versions): return {PackageURL.from_string(s) for s in purls} +def normalize_ranges(version_range_string): + """ + Splits version range strings with "||" operator into separate ranges + Example: + >>> z = normalize_ranges(">=6.1.3 < 7.0.0 || >=7.0.3") + >>> assert z == [">=6.1.3,<7.0.0", ">=7.0.3"] + """ + + version_ranges = version_range_string.split("||") + version_ranges = list(map(str.strip, version_ranges)) + for id, version_range in enumerate(version_ranges): + + # TODO: This cryptic, simplify this if possible + version_ranges[id] = ",".join(version_range.split()) + version_ranges[id] = version_ranges[id].replace(">=,", ">=") + version_ranges[id] = version_ranges[id].replace("<=,", "<=") + version_ranges[id] = version_ranges[id].replace("<=,", "<=") + version_ranges[id] = version_ranges[id].replace("<,", "<") + version_ranges[id] = version_ranges[id].replace(">,", ">") + + # "x" is interpretted as wild card character here. These are not part of semver + # spec. We replace the "x" with aribitarily large number to simulate the effect. + if ".x." in version_ranges[id]: + version_ranges[id] = version_ranges[id].replace(".x", ".10000.0") + if ".x" in version_ranges[id]: + version_ranges[id] = version_ranges[id].replace(".x", ".10000") + + return version_ranges + + def categorize_versions( all_versions: Set[str], affected_version_range: str, @@ -135,14 +173,33 @@ def categorize_versions( # NPM registry has no data regarding this package, we skip these return set(), set() - aff_spec = RangeSpecifier(affected_version_range) - fix_spec = RangeSpecifier(fixed_version_range) - aff_ver, fix_ver = set(), set() + aff_spec = [] + fix_spec = [] + + if affected_version_range: + aff_specs = normalize_ranges(affected_version_range) + aff_spec = [ + VersionSpecifier.from_scheme_version_spec_string("semver", spec) + for spec in aff_specs + if len(spec) >= 3 + ] + if fixed_version_range: + fix_specs = normalize_ranges(fixed_version_range) + fix_spec = [ + VersionSpecifier.from_scheme_version_spec_string("semver", spec) + for spec in fix_specs + if len(spec) >= 3 + ] + aff_ver, fix_ver = set(), set() # Unaffected version is that version which is in the fixed_version_range # or which is absent in the affected_version_range for ver in all_versions: - if ver in fix_spec or ver not in aff_spec: + ver_obj = SemverVersion(ver) + + if not any([ver_obj in spec for spec in aff_spec]) or any( + [ver_obj in spec for spec in fix_spec] + ): fix_ver.add(ver) else: aff_ver.add(ver) diff --git a/vulnerabilities/tests/test_npm.py b/vulnerabilities/tests/test_npm.py index f73bfe76a..19e856a39 100644 --- a/vulnerabilities/tests/test_npm.py +++ b/vulnerabilities/tests/test_npm.py @@ -40,9 +40,9 @@ MOCK_VERSION_API = NpmVersionAPI( cache={ - "jquery": {"3.4", "3.8"}, - "kerberos": {"0.5.8", "1.2"}, - "@hapi/subtext": {"3.7", "4.1.1", "6.1.3", "7.0.0", "7.0.5"}, + "jquery": {"3.4.0", "3.8.0"}, + "kerberos": {"0.5.8", "1.2.0"}, + "@hapi/subtext": {"3.7.0", "4.1.1", "6.1.3", "7.0.0", "7.0.5"}, } ) @@ -96,10 +96,10 @@ def test_import(self, _): assert models.Package.objects.count() == expected_package_count self.assert_for_package( - "jquery", {"3.4"}, {"3.8"}, "1518", vulnerability_id="CVE-2020-11022" + "jquery", {"3.4.0"}, {"3.8.0"}, "1518", vulnerability_id="CVE-2020-11022" ) # nopep8 - self.assert_for_package("kerberos", {"0.5.8"}, {"1.2"}, "1514") - self.assert_for_package("subtext", {"4.1.1", "7.0.0"}, {"3.7", "6.1.3", "7.0.5"}, "1476") + self.assert_for_package("kerberos", {"0.5.8"}, {"1.2.0"}, "1514") + self.assert_for_package("subtext", {"4.1.1", "7.0.0"}, {"3.7.0", "6.1.3", "7.0.5"}, "1476") def assert_for_package( self, @@ -130,7 +130,7 @@ def assert_for_package( def test_categorize_versions_simple_ranges(): - all_versions = {"3.4", "3.8"} + all_versions = {"3.4.0", "3.8.0"} impacted_ranges = "<3.5.0" resolved_ranges = ">=3.5.0" @@ -138,12 +138,12 @@ def test_categorize_versions_simple_ranges(): all_versions, impacted_ranges, resolved_ranges ) - assert impacted_versions == {"3.4"} - assert resolved_versions == {"3.8"} + assert impacted_versions == {"3.4.0"} + assert resolved_versions == {"3.8.0"} def test_categorize_versions_complex_ranges(): - all_versions = {"3.7", "4.1.1", "6.1.3", "7.0.0", "7.0.5"} + all_versions = {"3.7.0", "4.1.1", "6.1.3", "7.0.0", "7.0.5"} impacted_ranges = ">=4.1.0 <6.1.3 || >= 7.0.0 <7.0.3" resolved_ranges = ">=6.1.3 <7.0.0 || >=7.0.3" @@ -152,4 +152,4 @@ def test_categorize_versions_complex_ranges(): ) assert impacted_versions == {"4.1.1", "7.0.0"} - assert resolved_versions == {"3.7", "6.1.3", "7.0.5"} + assert resolved_versions == {"3.7.0", "6.1.3", "7.0.5"} From 82aee93f8adb0dc9b53019d38d24e1ab2d581822 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 25 Mar 2021 10:27:27 +0530 Subject: [PATCH 03/17] Refactor elixir importer to use universal_versions Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/elixir_security.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index f10f436b1..71062a962 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -24,7 +24,8 @@ from typing import List, Set import yaml -from dephell_specifier import RangeSpecifier +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import SemverVersion from packageurl import PackageURL from vulnerabilities.data_source import GitDataSource @@ -86,9 +87,13 @@ def get_versions_for_pkg_from_range_list(self, version_range_list, pkg_name): all_version_list = self.pkg_manager_api.get(pkg_name) if not version_range_list: return [], all_version_list - version_ranges = {RangeSpecifier(r) for r in version_range_list} + version_ranges = { + VersionSpecifier.from_scheme_version_spec_string("semver", r) + for r in version_range_list + } for version in all_version_list: - if any([version in v for v in version_ranges]): + version_obj = SemverVersion(version) + if any([version_obj in v for v in version_ranges]): safe_pkg_versions.append(version) vuln_pkg_versions = set(all_version_list) - set(safe_pkg_versions) From 7dc7a1ed660a372374ccded9efe41741fa9d7583 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 25 Mar 2021 11:19:27 +0530 Subject: [PATCH 04/17] Refactor Istio to use universal_versions Istio importer now only considers public releases Signed-off-by: Shivam Sandbhor --- vulnerabilities/helpers.py | 8 ++ vulnerabilities/importers/istio.py | 16 ++- vulnerabilities/package_managers.py | 2 +- vulnerabilities/tests/test_istio.py | 149 ++++++++++++++++++++-------- 4 files changed, 131 insertions(+), 44 deletions(-) diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index 05975b96e..330f0b9fb 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -98,3 +98,11 @@ def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5): session.mount("https://", adapter) session.mount("http://", adapter) return session + + +def contains_alpha(string): + """ + Return True if the input 'string' contains any alphabet + """ + + return any([c.isalpha() for c in string]) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 5e29e739b..9b76176af 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -23,13 +23,15 @@ import asyncio import re from typing import List, Set - import yaml -from dephell_specifier import RangeSpecifier +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import SemverVersion from packageurl import PackageURL + from vulnerabilities.data_source import Advisory, GitDataSource, Reference from vulnerabilities.package_managers import GitHubTagsAPI +from vulnerabilities.helpers import contains_alpha is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match @@ -47,6 +49,8 @@ def __enter__(self): def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) + # Only keep public releases + self.version_api.cache = list(filter(contains_alpha, self.version_api.cache)) def updated_advisories(self) -> Set[Advisory]: files = self._added_files.union(self._updated_files) @@ -64,9 +68,13 @@ def get_pkg_versions_from_ranges(self, version_range_list): all_version = self.version_api.get("istio/istio") safe_pkg_versions = [] vuln_pkg_versions = [] - version_ranges = [RangeSpecifier(r) for r in version_range_list] + version_ranges = [ + VersionSpecifier.from_scheme_version_spec_string("semver", r) + for r in version_range_list + ] for version in all_version: - if any([version in v for v in version_ranges]): + version_obj = SemverVersion(version) + if any([version_obj in v for v in version_ranges]): vuln_pkg_versions.append(version) safe_pkg_versions = set(all_version) - set(vuln_pkg_versions) diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index 501949201..7422bf3e7 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -34,7 +34,7 @@ class VersionAPI: def __init__(self, cache: Mapping[str, Set[str]] = None): - self.cache = cache or {} + self.cache = cache or set() def get(self, package_name: str) -> Set[str]: return self.cache.get(package_name, set()) diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index b907afe0f..8269ed092 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -44,15 +44,16 @@ def setUpClass(cls): cls.data_src.version_api = GitHubTagsAPI( { "istio/istio": [ - "1.1.0-rc.0", - "1.1.0-rc.1", - "1.1.0-rc.2", - "1.1.0-rc.3", - "1.1.0-rc.4", - "1.1.0-rc.5", - "1.1.0-rc.6", - "1.1.0-snapshot.2", - "1.1.0-snapshot.3", + "1.0.0", + "1.1.0", + "1.1.1", + "1.1.17", + "1.2.1", + "1.2.7", + "1.3.0", + "1.3.1", + "1.3.2", + "1.9.1", ] } ) @@ -78,105 +79,175 @@ def test_process_file(self): path = os.path.join(BASE_DIR, "test_data/istio/test_file.md") expected_data = [ Advisory( - summary=("Incorrect access control."), + summary="Incorrect access control.", + vulnerability_id="CVE-2019-12243", impacted_package_urls={ PackageURL( type="golang", + namespace=None, name="istio", - version="1.1.0-snapshot.2", + version="1.1.1", + qualifiers={}, + subpath=None, ), PackageURL( - type="golang", + type="github", + namespace=None, name="istio", - version="1.1.0-snapshot.3", + version="1.3.1", + qualifiers={}, + subpath=None, ), PackageURL( type="github", + namespace=None, + name="istio", + version="1.3.0", + qualifiers={}, + subpath=None, + ), + PackageURL( + type="golang", + namespace=None, name="istio", - version="1.1.0-snapshot.2", + version="1.3.1", + qualifiers={}, + subpath=None, ), PackageURL( type="github", + namespace=None, name="istio", - version="1.1.0-snapshot.3", + version="1.1.0", + qualifiers={}, + subpath=None, ), - }, - resolved_package_urls={ PackageURL( type="golang", + namespace=None, name="istio", - version="1.1.0-rc.2", + version="1.2.1", + qualifiers={}, + subpath=None, ), PackageURL( - type="golang", + type="github", + namespace=None, name="istio", - version="1.1.0-rc.4", + version="1.1.1", + qualifiers={}, + subpath=None, ), PackageURL( - type="golang", + type="github", + namespace=None, name="istio", - version="1.1.0-rc.3", + version="1.2.1", + qualifiers={}, + subpath=None, ), PackageURL( type="golang", + namespace=None, name="istio", - version="1.1.0-rc.0", + version="1.1.0", + qualifiers={}, + subpath=None, ), PackageURL( type="golang", + namespace=None, name="istio", - version="1.1.0-rc.5", + version="1.3.0", + qualifiers={}, + subpath=None, ), + }, + resolved_package_urls={ PackageURL( - type="golang", + type="github", + namespace=None, name="istio", - version="1.1.0-rc.1", + version="1.1.17", + qualifiers={}, + subpath=None, ), PackageURL( type="golang", + namespace=None, name="istio", - version="1.1.0-rc.6", + version="1.0.0", + qualifiers={}, + subpath=None, ), PackageURL( type="github", + namespace=None, name="istio", - version="1.1.0-rc.2", + version="1.0.0", + qualifiers={}, + subpath=None, ), PackageURL( - type="github", + type="golang", + namespace=None, name="istio", - version="1.1.0-rc.4", + version="1.9.1", + qualifiers={}, + subpath=None, ), PackageURL( type="github", + namespace=None, name="istio", - version="1.1.0-rc.3", + version="1.3.2", + qualifiers={}, + subpath=None, ), PackageURL( - type="github", + type="golang", + namespace=None, name="istio", - version="1.1.0-rc.0", + version="1.3.2", + qualifiers={}, + subpath=None, ), PackageURL( - type="github", + type="golang", + namespace=None, name="istio", - version="1.1.0-rc.5", + version="1.2.7", + qualifiers={}, + subpath=None, + ), + PackageURL( + type="golang", + namespace=None, + name="istio", + version="1.1.17", + qualifiers={}, + subpath=None, ), PackageURL( type="github", + namespace=None, name="istio", - version="1.1.0-rc.1", + version="1.2.7", + qualifiers={}, + subpath=None, ), PackageURL( type="github", + namespace=None, name="istio", - version="1.1.0-rc.6", + version="1.9.1", + qualifiers={}, + subpath=None, ), }, - vulnerability_id="CVE-2019-12243", + references=[], ) ] found_data = self.data_src.process_file(path) - assert expected_data == found_data From bf90402a0b676d1b33bc04cc920e225d01d231bc Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 25 Mar 2021 12:23:57 +0530 Subject: [PATCH 05/17] Refactor rust importer to use universal_versions Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/rust.py | 38 ++++++++++++++++++++--------- vulnerabilities/package_managers.py | 2 +- vulnerabilities/tests/test_rust.py | 21 ++++++++++------ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index 13bda2407..d4b3a2dba 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -21,17 +21,19 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import asyncio -from itertools import chain import re +from itertools import chain from typing import Optional, Mapping from typing import Set from typing import Tuple from urllib.error import HTTPError from urllib.request import urlopen -from dephell_specifier import RangeSpecifier -from packageurl import PackageURL import toml +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import SemverVersion +from packageurl import PackageURL + from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource @@ -100,15 +102,24 @@ def _load_advisory(self, path: str) -> Optional[Advisory]: all_versions = self.crates_api.get(crate_name) + # FIXME: Avoid wildcard version ranges for now. + # See https://github.com/RustSec/advisory-db/discussions/831 affected_ranges = { - RangeSpecifier(r) + VersionSpecifier.from_scheme_version_spec_string("semver", r) for r in chain.from_iterable(record.get("affected", {}).get("functions", {}).values()) + if r != "*" } unaffected_ranges = { - RangeSpecifier(r) for r in record.get("versions", {}).get("unaffected", []) + VersionSpecifier.from_scheme_version_spec_string("semver", r) + for r in record.get("versions", {}).get("unaffected", []) + if r != "*" + } + resolved_ranges = { + VersionSpecifier.from_scheme_version_spec_string("semver", r) + for r in record.get("versions", {}).get("patched", []) + if r != "*" } - resolved_ranges = {RangeSpecifier(r) for r in record.get("versions", {}).get("patched", [])} unaffected, affected = categorize_versions( all_versions, unaffected_ranges, affected_ranges, resolved_ranges @@ -142,26 +153,29 @@ def _load_advisory(self, path: str) -> Optional[Advisory]: def categorize_versions( all_versions: Set[str], - unaffected_versions: Set[RangeSpecifier], - affected_versions: Set[RangeSpecifier], - resolved_versions: Set[RangeSpecifier], + unaffected_versions: Set[VersionSpecifier], + affected_versions: Set[VersionSpecifier], + resolved_versions: Set[VersionSpecifier], ) -> Tuple[Set[str], Set[str]]: """ Categorize all versions of a crate according to the given version ranges. :return: unaffected versions, affected versions """ + unaffected, affected = set(), set() if not any(unaffected_versions.union(affected_versions).union(resolved_versions)): return unaffected, affected + # TODO: This is probably wrong for version in all_versions: - if affected_versions and all([version in av for av in affected_versions]): + version_obj = SemverVersion(version) + if affected_versions and all([version_obj in av for av in affected_versions]): affected.add(version) - elif unaffected_versions and all([version in av for av in unaffected_versions]): + elif unaffected_versions and all([version_obj in av for av in unaffected_versions]): unaffected.add(version) - elif resolved_versions and all([version in av for av in resolved_versions]): + elif resolved_versions and all([version_obj in av for av in resolved_versions]): unaffected.add(version) # If some versions were not classified above, one or more of the given ranges might be empty, so diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index 7422bf3e7..501949201 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -34,7 +34,7 @@ class VersionAPI: def __init__(self, cache: Mapping[str, Set[str]] = None): - self.cache = cache or set() + self.cache = cache or {} def get(self, package_name: str) -> Set[str]: return self.cache.get(package_name, set()) diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py index 4a64847b8..6840e3e44 100644 --- a/vulnerabilities/tests/test_rust.py +++ b/vulnerabilities/tests/test_rust.py @@ -26,7 +26,8 @@ import zipfile from unittest.mock import patch -from dephell_specifier import RangeSpecifier + +from universal_versions.version_specifier import VersionSpecifier from django.test import TestCase @@ -51,9 +52,12 @@ def test_categorize_versions(): flatbuffers_versions = MOCKED_CRATES_API_VERSIONS["flatbuffers"] - unaffected_ranges = {RangeSpecifier("< 0.4.0")} - affected_ranges = {RangeSpecifier(">= 0.4.0"), RangeSpecifier("<= 0.6.0")} - resolved_ranges = {RangeSpecifier(">= 0.6.1")} + unaffected_ranges = {VersionSpecifier.from_scheme_version_spec_string("semver", "< 0.4.0")} + affected_ranges = { + VersionSpecifier.from_scheme_version_spec_string("semver", ">= 0.4.0"), + VersionSpecifier.from_scheme_version_spec_string("semver", "<= 0.6.0"), + } + resolved_ranges = {VersionSpecifier.from_scheme_version_spec_string("semver", ">= 0.6.1")} unaffected_versions, affected_versions = categorize_versions( set(flatbuffers_versions), @@ -73,9 +77,9 @@ def test_categorize_versions(): def test_categorize_versions_without_affected_ranges(): all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"} - unaffected_ranges = {RangeSpecifier("< 1.2")} + unaffected_ranges = {VersionSpecifier.from_scheme_version_spec_string("semver", "< 1.2")} affected_ranges = set() - resolved_ranges = {RangeSpecifier(">= 3.0")} + resolved_ranges = {VersionSpecifier.from_scheme_version_spec_string("semver", ">= 3.0")} unaffected_versions, affected_versions = categorize_versions( all_versions, @@ -99,7 +103,10 @@ def test_categorize_versions_with_only_affected_ranges(): all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"} unaffected_ranges = set() - affected_ranges = {RangeSpecifier("> 1.2"), RangeSpecifier("<= 2.1")} + affected_ranges = { + VersionSpecifier.from_scheme_version_spec_string("semver", "> 1.2"), + VersionSpecifier.from_scheme_version_spec_string("semver", "<= 2.1"), + } resolved_ranges = set() unaffected_versions, affected_versions = categorize_versions( From a401218265c07aa528f32967bd9a79463454ac40 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 25 Mar 2021 14:04:52 +0530 Subject: [PATCH 06/17] Refactor nginx importer to use universal_versions Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/nginx.py | 34 +++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/importers/nginx.py b/vulnerabilities/importers/nginx.py index 19955af07..195191cea 100644 --- a/vulnerabilities/importers/nginx.py +++ b/vulnerabilities/importers/nginx.py @@ -27,7 +27,8 @@ import requests from packageurl import PackageURL from bs4 import BeautifulSoup -from dephell_specifier import RangeSpecifier +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import SemverVersion from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource @@ -58,10 +59,10 @@ def set_api(self): def updated_advisories(self): advisories = [] - if create_etag(data_src=self, url=self.url, etag_key="ETag"): - self.set_api() - data = requests.get(self.url).content - advisories.extend(self.to_advisories(data)) + # if create_etag(data_src=self, url=self.url, etag_key="ETag"): + self.set_api() + data = requests.get(self.url).content + advisories.extend(self.to_advisories(data)) return self.batch_advisories(advisories) def to_advisories(self, data): @@ -126,13 +127,15 @@ def extract_fixed_pkgs(self, vuln_info): raw_ranges = version_info.split(",") version_ranges = [] for rng in raw_ranges: - # Eg. "1.7.3+" gets converted to RangeSpecifier("^1.7.3") + # Eg. "1.7.3+" gets converted to VersionSpecifier.from_scheme_version_spec_string("semver","^1.7.3") # The advisory in this case uses `+` in the sense that any version # with greater or equal `minor` version satisfies the range. # "1.7.4" satisifes "1.7.3+", but "1.8.4" does not. "1.7.3+" has same # semantics as that of "^1.7.3" - version_ranges.append(RangeSpecifier("^" + rng[:-1])) + version_ranges.append( + VersionSpecifier.from_scheme_version_spec_string("semver", "^" + rng[:-1]) + ) valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges) @@ -148,16 +151,26 @@ def extract_vuln_pkgs(self, vuln_info): version_ranges = [] windows_only = False for version_info in version_infos.split(", "): + if version_info == "all": + # This is misleading since eventually some version get fixed. + continue + if "-" not in version_info: # These are discrete versions - version_ranges.append(RangeSpecifier(version_info[0])) + version_ranges.append( + VersionSpecifier.from_scheme_version_spec_string("semver", version_info[0]) + ) continue windows_only = "nginx/Windows" in version_info version_info = version_info.replace("nginx/Windows", "") lower_bound, upper_bound = version_info.split("-") - version_ranges.append(RangeSpecifier(f">={lower_bound},<={upper_bound}")) + version_ranges.append( + VersionSpecifier.from_scheme_version_spec_string( + "semver", f">={lower_bound},<={upper_bound}" + ) + ) valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges) qualifiers = {} @@ -173,7 +186,8 @@ def extract_vuln_pkgs(self, vuln_info): def find_valid_versions(versions, version_ranges): valid_versions = set() for version in versions: - if any([version in ver_range for ver_range in version_ranges]): + version_obj = SemverVersion(version) + if any([version_obj in ver_range for ver_range in version_ranges]): valid_versions.add(version) return valid_versions From d57a5d6ce8f604660d4dca3f6953c85852f2a0e3 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 25 Mar 2021 14:28:29 +0530 Subject: [PATCH 07/17] Refactor apache tomcat importer to use universal versions Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/apache_tomcat.py | 10 +- vulnerabilities/tests/test_apache_tomcat.py | 117 +++++++++++--------- 2 files changed, 71 insertions(+), 56 deletions(-) diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index 3ec166855..eac84abaf 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -28,6 +28,8 @@ import requests from bs4 import BeautifulSoup from dephell_specifier import RangeSpecifier +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import MavenVersion from packageurl import PackageURL from vulnerabilities.data_source import Advisory @@ -101,7 +103,7 @@ def to_advisories(self, apache_tomcat_advisory_html): type="maven", namespace="apache", name="tomcat", version=version ) for version in self.version_api.get("org.apache.tomcat:tomcat") - if version in version_range + if MavenVersion(version) in version_range ] ) @@ -110,6 +112,8 @@ def to_advisories(self, apache_tomcat_advisory_html): type="maven", namespace="apache", name="tomcat", version=fixed_version ) ] + if fixed_package in affected_packages: + raise ValueError("Bing bong") advisories.append( Advisory( @@ -147,4 +151,6 @@ def parse_version_ranges(string): else: lower_bound = upper_bound = version_range - yield RangeSpecifier(">=" + lower_bound + "<=" + upper_bound) + yield VersionSpecifier.from_scheme_version_spec_string( + "maven", f">={lower_bound},<={upper_bound}" + ) diff --git a/vulnerabilities/tests/test_apache_tomcat.py b/vulnerabilities/tests/test_apache_tomcat.py index 8d11f45ff..fd395dee0 100644 --- a/vulnerabilities/tests/test_apache_tomcat.py +++ b/vulnerabilities/tests/test_apache_tomcat.py @@ -50,7 +50,16 @@ def test_to_advisories(self): expected_advisories = [ Advisory( summary="", - impacted_package_urls=[ + vulnerability_id="CVE-2015-5351", + impacted_package_urls={ + PackageURL( + type="maven", + namespace="apache", + name="tomcat", + version="8.0.0.M1", + qualifiers={}, + subpath=None, + ), PackageURL( type="maven", namespace="apache", @@ -67,8 +76,8 @@ def test_to_advisories(self): qualifiers={}, subpath=None, ), - ], - resolved_package_urls=[ + }, + resolved_package_urls={ PackageURL( type="maven", namespace="apache", @@ -77,32 +86,39 @@ def test_to_advisories(self): qualifiers={}, subpath=None, ) - ], + }, references=[ Reference( - url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0763", reference_id="", + url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-5351", + severities=[], + ), + Reference( + reference_id="", + url="https://svn.apache.org/viewvc?view=rev&rev=1720652", + severities=[], ), Reference( - url="https://svn.apache.org/viewvc?view=rev&rev=1725926", reference_id="", + url="https://svn.apache.org/viewvc?view=rev&rev=1720655", + severities=[], ), ], - vulnerability_id="CVE-2016-0763", ), Advisory( summary="", - impacted_package_urls=[ + vulnerability_id="CVE-2016-0706", + impacted_package_urls={ PackageURL( type="maven", namespace="apache", name="tomcat", - version="8.0.0.M1", + version="9.0.0.M1", qualifiers={}, subpath=None, ) - ], - resolved_package_urls=[ + }, + resolved_package_urls={ PackageURL( type="maven", namespace="apache", @@ -111,44 +127,25 @@ def test_to_advisories(self): qualifiers={}, subpath=None, ) - ], + }, references=[ Reference( - url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-5351", - reference_id="", - ), - Reference( - url="https://svn.apache.org/viewvc?view=rev&rev=1720652", reference_id="", + url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0706", + severities=[], ), Reference( - url="https://svn.apache.org/viewvc?view=rev&rev=1720655", reference_id="", + url="https://svn.apache.org/viewvc?view=rev&rev=1722799", + severities=[], ), ], - vulnerability_id="CVE-2015-5351", ), Advisory( summary="", - impacted_package_urls=[ - PackageURL( - type="maven", - namespace="apache", - name="tomcat", - version="9.0.0.M1", - qualifiers={}, - subpath=None, - ), - PackageURL( - type="maven", - namespace="apache", - name="tomcat", - version="9.0.0.M2", - qualifiers={}, - subpath=None, - ), - ], - resolved_package_urls=[ + vulnerability_id="CVE-2016-0714", + impacted_package_urls=set(), + resolved_package_urls={ PackageURL( type="maven", namespace="apache", @@ -157,22 +154,37 @@ def test_to_advisories(self): qualifiers={}, subpath=None, ) - ], + }, references=[ Reference( - url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0706", reference_id="", + url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0714", + severities=[], + ), + Reference( + reference_id="", + url="https://svn.apache.org/viewvc?view=rev&rev=1725263", + severities=[], ), Reference( - url="https://svn.apache.org/viewvc?view=rev&rev=1722799", reference_id="", + url="https://svn.apache.org/viewvc?view=rev&rev=1725914", + severities=[], ), ], - vulnerability_id="CVE-2016-0706", ), Advisory( summary="", - impacted_package_urls=[ + vulnerability_id="CVE-2016-0763", + impacted_package_urls={ + PackageURL( + type="maven", + namespace="apache", + name="tomcat", + version="9.0.0.M1", + qualifiers={}, + subpath=None, + ), PackageURL( type="maven", namespace="apache", @@ -180,9 +192,9 @@ def test_to_advisories(self): version="9.0.0.M2", qualifiers={}, subpath=None, - ) - ], - resolved_package_urls=[ + ), + }, + resolved_package_urls={ PackageURL( type="maven", namespace="apache", @@ -191,22 +203,19 @@ def test_to_advisories(self): qualifiers={}, subpath=None, ) - ], + }, references=[ Reference( - url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0714", - reference_id="", - ), - Reference( - url="https://svn.apache.org/viewvc?view=rev&rev=1725263", reference_id="", + url="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-0763", + severities=[], ), Reference( - url="https://svn.apache.org/viewvc?view=rev&rev=1725914", reference_id="", + url="https://svn.apache.org/viewvc?view=rev&rev=1725926", + severities=[], ), ], - vulnerability_id="CVE-2016-0714", ), ] From 323a4b9f5df1637b0c174530ae9b82b733a2b18f Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 25 Mar 2021 15:48:03 +0530 Subject: [PATCH 08/17] Refactor apache kakfa importer to use universal versions Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/apache_kafka.py | 31 +++++++++++++++++----- vulnerabilities/importers/apache_tomcat.py | 1 - vulnerabilities/tests/test_apache_kafka.py | 20 +++++++++----- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/vulnerabilities/importers/apache_kafka.py b/vulnerabilities/importers/apache_kafka.py index 3eb0f5771..1994b46bf 100644 --- a/vulnerabilities/importers/apache_kafka.py +++ b/vulnerabilities/importers/apache_kafka.py @@ -24,8 +24,9 @@ import requests from bs4 import BeautifulSoup -from dephell_specifier import RangeSpecifier from packageurl import PackageURL +from universal_versions.versions import MavenVersion +from universal_versions.version_specifier import VersionSpecifier from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource @@ -71,13 +72,23 @@ def to_advisory(self, advisory_page): fixed_packages = [ PackageURL(type="apache", name="kafka", version=version) for version in self.version_api.get("apache/kafka") - if any([version in version_range for version_range in fixed_version_ranges]) + if any( + [ + MavenVersion(version) in version_range + for version_range in fixed_version_ranges + ] + ) ] affected_packages = [ PackageURL(type="apache", name="kafka", version=version) for version in self.version_api.get("apache/kafka") - if any([version in version_range for version_range in affected_version_ranges]) + if any( + [ + MavenVersion(version) in version_range + for version_range in affected_version_ranges + ] + ) ] advisories.append( @@ -107,14 +118,22 @@ def to_version_ranges(version_range_text): lower_bound, upper_bound = range_expression.split("to") lower_bound = f">={lower_bound}" upper_bound = f"<={upper_bound}" - version_ranges.append(RangeSpecifier(f"{lower_bound},{upper_bound}")) + version_ranges.append( + VersionSpecifier.from_scheme_version_spec_string( + "maven", f"{lower_bound},{upper_bound}" + ) + ) elif "and later" in range_expression: # eg range_expression == "2.1.1 and later" range_expression = range_expression.replace("and later", "") - version_ranges.append(RangeSpecifier(f">={range_expression}")) + version_ranges.append( + VersionSpecifier.from_scheme_version_spec_string("maven", f">={range_expression}") + ) else: # eg range_expression == "3.0.0" - version_ranges.append(RangeSpecifier(range_expression)) + version_ranges.append( + VersionSpecifier.from_scheme_version_spec_string("maven", range_expression) + ) return version_ranges diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index eac84abaf..472cd85a3 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -27,7 +27,6 @@ import requests from bs4 import BeautifulSoup -from dephell_specifier import RangeSpecifier from universal_versions.version_specifier import VersionSpecifier from universal_versions.versions import MavenVersion from packageurl import PackageURL diff --git a/vulnerabilities/tests/test_apache_kafka.py b/vulnerabilities/tests/test_apache_kafka.py index 2c39ee700..69a6e5ee6 100644 --- a/vulnerabilities/tests/test_apache_kafka.py +++ b/vulnerabilities/tests/test_apache_kafka.py @@ -23,8 +23,8 @@ import os from unittest import TestCase -from dephell_specifier import RangeSpecifier from packageurl import PackageURL +from universal_versions.version_specifier import VersionSpecifier from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import Reference @@ -39,19 +39,25 @@ class TestApacheKafkaDataSource(TestCase): def test_to_version_ranges(self): # Check single version - assert [RangeSpecifier("==3.2.2")] == to_version_ranges("3.2.2") + assert [ + VersionSpecifier.from_scheme_version_spec_string("maven", "=3.2.2") + ] == to_version_ranges("3.2.2") # Check range with lower and upper bounds - assert [RangeSpecifier(">=3.2.2, <=3.2.3")] == to_version_ranges("3.2.2 to 3.2.3") + assert [ + VersionSpecifier.from_scheme_version_spec_string("maven", ">=3.2.2, <=3.2.3") + ] == to_version_ranges("3.2.2 to 3.2.3") # Check range with "and later" - assert [RangeSpecifier(">=3.2.2")] == to_version_ranges("3.2.2 and later") + assert [ + VersionSpecifier.from_scheme_version_spec_string("maven", ">=3.2.2") + ] == to_version_ranges("3.2.2 and later") # Check combination of above cases assert [ - RangeSpecifier(">=3.2.2"), - RangeSpecifier(">=3.2.2, <=3.2.3"), - RangeSpecifier("==3.2.2"), + VersionSpecifier.from_scheme_version_spec_string("maven", ">=3.2.2"), + VersionSpecifier.from_scheme_version_spec_string("maven", ">=3.2.2, <=3.2.3"), + VersionSpecifier.from_scheme_version_spec_string("maven", "==3.2.2"), ] == to_version_ranges("3.2.2 and later, 3.2.2 to 3.2.3, 3.2.2") def test_to_advisory(self): From 2cbc23e958e5288b3a2e11264c6c1581bf555030 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 25 Mar 2021 16:07:17 +0530 Subject: [PATCH 09/17] Refactor ruby importer to use universal versions Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/ruby.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index a22006923..35e8cf281 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -24,9 +24,9 @@ from typing import Set from typing import List -from dephell_specifier import RangeSpecifier -from dephell_specifier.range_specifier import InvalidSpecifier from packageurl import PackageURL +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import SemverVersion from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource @@ -81,7 +81,6 @@ def collect_packages(self): def process_file(self, path) -> List[Advisory]: record = load_yaml(path) package_name = record.get("gem") - if not package_name: return @@ -137,17 +136,14 @@ def process_file(self, path) -> List[Advisory]: def categorize_versions(all_versions, unaffected_version_ranges): for id, elem in enumerate(unaffected_version_ranges): - try: - unaffected_version_ranges[id] = RangeSpecifier(elem.replace(" ", "")) - except InvalidSpecifier: - continue + unaffected_version_ranges[id] = VersionSpecifier.from_scheme_version_spec_string("semver",elem) safe_versions = set() for i in all_versions: - for ver_rng in unaffected_version_ranges: - - if i in ver_rng: + vobj = SemverVersion(i) + for ver_rng in unaffected_version_ranges: + if vobj in ver_rng: safe_versions.add(i) return (safe_versions, all_versions - safe_versions) From 6f643c90bb7641a6b9768688379e0cb4aa7a5902 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Tue, 30 Mar 2021 11:04:44 +0530 Subject: [PATCH 10/17] Refactor github importer and fix doctests in tomcat Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/apache_tomcat.py | 23 +++++++++++++--------- vulnerabilities/importers/github.py | 18 ++++++++++++----- vulnerabilities/importers/npm.py | 2 +- vulnerabilities/importers/ruby.py | 4 +++- vulnerabilities/tests/test_github.py | 5 +++-- 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index 472cd85a3..6652aa737 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -129,16 +129,21 @@ def to_advisories(self, apache_tomcat_advisory_html): def parse_version_ranges(string): """ - This method yields Rangespecifier objects obtained by + This method yields VersionSpecifier objects obtained by parsing `string`. - >> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9")) - [RangeSpecifier(<=9.0.0.M9,>=9.0.0.M1)] - - >> list(parse_version_ranges("Affects: 9.0.0.M1")) - [RangeSpecifier(>=9.0.0.M1<=9.0.0.M1)] - - >> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9, 1.2.3 to 3.4.5")) - [RangeSpecifier(<=9.0.0.M9,>=9.0.0.M1), RangeSpecifier(<=3.4.5,>=1.2.3)] + >>> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9")) == [ + ... VersionSpecifier.from_scheme_version_spec_string('maven','<=9.0.0.M9,>=9.0.0.M1') + ... ] + True + >>> list(parse_version_ranges("Affects: 9.0.0.M1")) == [ + ... VersionSpecifier.from_scheme_version_spec_string('maven','>=9.0.0.M1,<=9.0.0.M1') + ... ] + True + >>> list(parse_version_ranges("Affects: 9.0.0.M1 to 9.0.0.M9, 1.2.3 to 3.4.5")) == [ + ... VersionSpecifier.from_scheme_version_spec_string('maven','<=9.0.0.M9,>=9.0.0.M1'), + ... VersionSpecifier.from_scheme_version_spec_string('maven','<=3.4.5,>=1.2.3') + ... ] + True """ version_rng_txt = string.split("Affects:")[-1].strip() version_ranges = version_rng_txt.split(",") diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index 258e83f14..9fe6e3db3 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -32,6 +32,8 @@ import requests from dephell_specifier import RangeSpecifier +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import version_class_by_package_type from packageurl import PackageURL from vulnerabilities.data_source import Advisory @@ -200,7 +202,7 @@ def process_response(self) -> List[Advisory]: ns, pkg_name = self.process_name(ecosystem, name) aff_range = adv["node"]["vulnerableVersionRange"] aff_vers, unaff_vers = self.categorize_versions( - aff_range, self.version_api.get(name) + self.version_api.package_type, aff_range, self.version_api.get(name) ) affected_purls = { PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type) @@ -252,8 +254,14 @@ def process_response(self) -> List[Advisory]: @staticmethod def categorize_versions( - version_range: str, all_versions: Set[str] - ) -> Tuple[Set[str], Set[str]]: # nopep8 - version_range = RangeSpecifier(version_range) - affected_versions = {version for version in all_versions if version in version_range} + package_type: str, version_range: str, all_versions: Set[str] + ) -> Tuple[Set[str], Set[str]]: + version_class = version_class_by_package_type[package_type] + version_scheme = version_class.scheme + version_range = VersionSpecifier.from_scheme_version_spec_string( + version_scheme, version_range + ) + affected_versions = { + version for version in all_versions if version_class(version) in version_range + } return (affected_versions, all_versions - affected_versions) diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index 6d43a10ad..b70f2b560 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -140,7 +140,7 @@ def normalize_ranges(version_range_string): version_ranges = list(map(str.strip, version_ranges)) for id, version_range in enumerate(version_ranges): - # TODO: This cryptic, simplify this if possible + # TODO: This is cryptic, simplify this if possible version_ranges[id] = ",".join(version_range.split()) version_ranges[id] = version_ranges[id].replace(">=,", ">=") version_ranges[id] = version_ranges[id].replace("<=,", "<=") diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index 35e8cf281..069425df4 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -136,7 +136,9 @@ def process_file(self, path) -> List[Advisory]: def categorize_versions(all_versions, unaffected_version_ranges): for id, elem in enumerate(unaffected_version_ranges): - unaffected_version_ranges[id] = VersionSpecifier.from_scheme_version_spec_string("semver",elem) + unaffected_version_ranges[id] = VersionSpecifier.from_scheme_version_spec_string( + "semver", elem + ) safe_versions = set() for i in all_versions: diff --git a/vulnerabilities/tests/test_github.py b/vulnerabilities/tests/test_github.py index 6753efd56..24de42d20 100644 --- a/vulnerabilities/tests/test_github.py +++ b/vulnerabilities/tests/test_github.py @@ -55,7 +55,6 @@ def setUpClass(cls): "endpoint": "https://api.example.com/graphql", "ecosystems": ["MAVEN"], } - # os.environ = {'GH_TOKEN':'abc'} with patch.dict(os.environ, {"GH_TOKEN": "abc"}): cls.data_src = GitHubAPIDataSource(1, config=data_source_cfg) @@ -66,7 +65,9 @@ def test_categorize_versions(self): eg_version_range = ">= 3.3.0, < 3.3.5" eg_versions = {"3.3.6", "3.3.0", "3.3.4", "3.2.0"} - aff_vers, safe_vers = self.data_src.categorize_versions(eg_version_range, eg_versions) + aff_vers, safe_vers = self.data_src.categorize_versions( + "pypi", eg_version_range, eg_versions + ) exp_safe_vers = {"3.3.6", "3.2.0"} exp_aff_vers = {"3.3.0", "3.3.4"} From 5e6b29ef164cec6e2a1d7fa5c4a0b98ada123c90 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Tue, 30 Mar 2021 17:06:04 +0530 Subject: [PATCH 11/17] Refactor oval data sources to use universal versions Signed-off-by: Shivam Sandbhor --- vulnerabilities/data_source.py | 17 ++++++++++++++--- vulnerabilities/importers/github.py | 3 +-- vulnerabilities/oval_parser.py | 10 ++++------ vulnerabilities/tests/test_debian_oval.py | 19 +++++++++---------- vulnerabilities/tests/test_suse.py | 18 ++++++++---------- vulnerabilities/tests/test_ubuntu.py | 15 +++++++-------- 6 files changed, 43 insertions(+), 39 deletions(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index c911518ec..e899440d4 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -41,6 +41,8 @@ import pygit2 from packageurl import PackageURL +from universal_versions.version_specifier import VersionSpecifier +from universal_versions.versions import version_class_by_package_type from vulnerabilities.oval_parser import OvalParser from vulnerabilities.severity_systems import ScoringSystem @@ -514,7 +516,7 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis "type" key. Example value of pkg_metadata: - {"type":"deb","qualifiers":{"distro":"buster"} } + {"type":"deb","qualifiers":{"distro":"buster"} } """ all_adv = [] @@ -536,7 +538,14 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis for package_name in test_data["package_list"]: if package_name and len(package_name) >= 50: continue - aff_ver_range = test_data["version_ranges"] or set() + + affected_version_range = test_data["version_ranges"] or set() + version_class = version_class_by_package_type[pkg_metadata["type"]] + version_scheme = version_class.scheme + + affected_version_range = VersionSpecifier.from_scheme_version_spec_string( + version_scheme, affected_version_range + ) all_versions = self.pkg_manager_api.get(package_name) # FIXME: what is this 50 DB limit? that's too small for versions @@ -546,7 +555,9 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis all_versions = set(filter(lambda x: len(x) < 50, all_versions)) if not all_versions: continue - affected_versions = set(filter(lambda x: x in aff_ver_range, all_versions)) + affected_versions = set( + filter(lambda x: version_class(x) in affected_version_range, all_versions) + ) safe_versions = all_versions - affected_versions for version in affected_versions: diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index 9fe6e3db3..9d27c5c48 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -31,10 +31,9 @@ from typing import Optional import requests -from dephell_specifier import RangeSpecifier +from packageurl import PackageURL from universal_versions.version_specifier import VersionSpecifier from universal_versions.versions import version_class_by_package_type -from packageurl import PackageURL from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource diff --git a/vulnerabilities/oval_parser.py b/vulnerabilities/oval_parser.py index bcf80cba5..059df3d9b 100755 --- a/vulnerabilities/oval_parser.py +++ b/vulnerabilities/oval_parser.py @@ -28,8 +28,6 @@ from typing import Set import xml.etree.ElementTree as ET -from dephell_specifier import RangeSpecifier - from vulnerabilities.lib_oval import OvalDefinition from vulnerabilities.lib_oval import OvalDocument from vulnerabilities.lib_oval import OvalObject @@ -68,7 +66,7 @@ def get_data(self) -> List[Dict]: continue test_data = {"package_list": []} test_data["package_list"].extend(self.get_pkgs_from_obj(test_obj)) - version_ranges = self.get_version_ranges_from_state(test_state) + version_ranges = self.get_version_range_from_state(test_state) test_data["version_ranges"] = version_ranges definition_data["test_data"].append(test_data) @@ -131,9 +129,9 @@ def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]: return pkg_list - def get_version_ranges_from_state(self, state: OvalState) -> Optional[RangeSpecifier]: + def get_version_range_from_state(self, state: OvalState) -> Optional[str]: """ - Return a version range(s)? from a state + Return a version range from a state """ for var in state.element: operation = var.get("operation") @@ -160,7 +158,7 @@ def get_version_ranges_from_state(self, state: OvalState) -> Optional[RangeSpeci if version_range in x_version_ranges: version_range = x_version_ranges[version_range] - return RangeSpecifier(version_range) + return version_range @staticmethod def get_urls_from_definition(definition: OvalDefinition) -> Set[str]: diff --git a/vulnerabilities/tests/test_debian_oval.py b/vulnerabilities/tests/test_debian_oval.py index 127228aae..aa89ad22f 100644 --- a/vulnerabilities/tests/test_debian_oval.py +++ b/vulnerabilities/tests/test_debian_oval.py @@ -6,7 +6,6 @@ from collections import OrderedDict import asyncio -from dephell_specifier import RangeSpecifier from packageurl import PackageURL from vulnerabilities.oval_parser import OvalParser @@ -73,35 +72,35 @@ def test_get_data_from_xml_doc(self, mock_write): ), Advisory( summary="security update", + vulnerability_id="CVE-2001-1593", impacted_package_urls={ PackageURL( type="deb", namespace=None, name="a2ps", - version="0.11.1+dfsg-5+deb7u1", - qualifiers=OrderedDict([("distro", "wheezy")]), + version="1.11.1+dfsg-5+deb7u1", + qualifiers={"distro": "wheezy"}, subpath=None, - ) - }, - resolved_package_urls={ + ), PackageURL( type="deb", namespace=None, name="a2ps", version="2.3.9", - qualifiers=OrderedDict([("distro", "wheezy")]), + qualifiers={"distro": "wheezy"}, subpath=None, ), PackageURL( type="deb", namespace=None, name="a2ps", - version="1.11.1+dfsg-5+deb7u1", - qualifiers=OrderedDict([("distro", "wheezy")]), + version="0.11.1+dfsg-5+deb7u1", + qualifiers={"distro": "wheezy"}, subpath=None, ), }, - vulnerability_id="CVE-2001-1593", + resolved_package_urls=set(), + references=[], ), ] diff --git a/vulnerabilities/tests/test_suse.py b/vulnerabilities/tests/test_suse.py index 5eda669eb..a4695d826 100644 --- a/vulnerabilities/tests/test_suse.py +++ b/vulnerabilities/tests/test_suse.py @@ -2,8 +2,6 @@ import unittest import xml.etree.ElementTree as ET -from dephell_specifier import RangeSpecifier - from vulnerabilities.oval_parser import OvalParser @@ -95,7 +93,7 @@ def test_get_pkgs_from_obj(self): # it's object due to filters to avoid such tests in the first place assert pkg_set2 == {"cacti"} - def test_get_version_ranges_from_state(self): + def test_get_version_range_from_state(self): assert len(self.parsed_oval.oval_document.getStates()) == 4 @@ -103,11 +101,11 @@ def test_get_version_ranges_from_state(self): state_2 = self.parsed_oval.oval_document.getStates()[1] exp_range_1 = None - exp_range_2 = RangeSpecifier("<1.2.11-lp151.3.6") + exp_range_2 = "<1.2.11-lp151.3.6" # In a full run we wont get exp_range1 because we won't obtain # it's state due to filters to avoid such tests in the first place - assert self.parsed_oval.get_version_ranges_from_state(state_1) == exp_range_1 - assert self.parsed_oval.get_version_ranges_from_state(state_2) == exp_range_2 + assert self.parsed_oval.get_version_range_from_state(state_1) == exp_range_1 + assert self.parsed_oval.get_version_range_from_state(state_2) == exp_range_2 def test_get_urls_from_definition(self): @@ -135,11 +133,11 @@ def test_get_data(self): "test_data": [ { "package_list": ["cacti"], - "version_ranges": RangeSpecifier("<1.2.11-lp151.3.6"), + "version_ranges": "<1.2.11-lp151.3.6", }, { "package_list": ["cacti-spine"], - "version_ranges": RangeSpecifier("<1.2.11-lp151.3.6"), + "version_ranges": "<1.2.11-lp151.3.6", }, ], "description": '\n Cacti 0.8.7e and earlier allows remote authenticated administrators to gain privileges by modifying the "Data Input Method" for the "Linux - Get Memory Usage" setting to contain arbitrary commands.\n ', @@ -155,11 +153,11 @@ def test_get_data(self): "test_data": [ { "package_list": ["apache2-mod_perl"], - "version_ranges": RangeSpecifier("<2.0.11-lp151.3.3"), + "version_ranges": "<2.0.11-lp151.3.3", }, { "package_list": ["apache2-mod_perl-devel"], - "version_ranges": RangeSpecifier("<2.0.11-lp151.3.3"), + "version_ranges": "<2.0.11-lp151.3.3", }, ], "description": "\n mod_perl 2.0 through 2.0.10 allows attackers to execute arbitrary Perl code by placing it in a user-owned .htaccess file, because (contrary to the documentation) there is no configuration option that permits Perl code for the administrator's control of HTTP request processing without also permitting unprivileged users to run Perl code in the context of the user account that runs Apache HTTP Server processes.\n ", diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 3b2eff174..4d1ee9bda 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -6,7 +6,6 @@ from collections import OrderedDict import asyncio -from dephell_specifier import RangeSpecifier from packageurl import PackageURL from vulnerabilities.oval_parser import OvalParser @@ -90,18 +89,18 @@ def test_get_pkgs_from_obj(self): assert pkg_set1 == {"potrace"} assert pkg_set2 == {"tor"} - def test_get_version_ranges_from_state(self): + def test_get_version_range_from_state(self): assert len(self.parsed_oval.oval_document.getStates()) == 2 state_1 = self.parsed_oval.oval_document.getStates()[0] state_2 = self.parsed_oval.oval_document.getStates()[1] - exp_range_1 = RangeSpecifier("<1.14-2") - exp_range_2 = RangeSpecifier("<0.2.8.9-1ubuntu1") + exp_range_1 = "<1.14-2" + exp_range_2 = "<0.2.8.9-1ubuntu1" - assert self.parsed_oval.get_version_ranges_from_state(state_1) == exp_range_1 - assert self.parsed_oval.get_version_ranges_from_state(state_2) == exp_range_2 + assert self.parsed_oval.get_version_range_from_state(state_1) == exp_range_1 + assert self.parsed_oval.get_version_range_from_state(state_2) == exp_range_2 def test_get_urls_from_definition(self): @@ -131,7 +130,7 @@ def test_get_data(self): "test_data": [ { "package_list": ["potrace"], - "version_ranges": RangeSpecifier("<1.14-2"), + "version_ranges": "<1.14-2", } ], "description": "Heap-based buffer overflow in the bm_readbody_bmp function in bitmap_io.c in potrace before 1.13 allows remote attackers to have unspecified impact via a crafted BMP image, a different vulnerability than CVE-2016-8698, CVE-2016-8699, CVE-2016-8700, CVE-2016-8701, and CVE-2016-8702.", @@ -146,7 +145,7 @@ def test_get_data(self): "test_data": [ { "package_list": ["tor"], - "version_ranges": RangeSpecifier("<0.2.8.9-1ubuntu1"), + "version_ranges": "<0.2.8.9-1ubuntu1", } ], "description": "Tor before 0.2.8.9 and 0.2.9.x before 0.2.9.4-alpha had internal functions that were entitled to expect that buf_t data had NUL termination, but the implementation of or/buffers.c did not ensure that NUL termination was present, which allows remote attackers to cause a denial of service (client, hidden service, relay, or authority crash) via crafted data.", From 0ac1ab3ebbdecd7aa181f1565978e3f79aed39cc Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Wed, 31 Mar 2021 08:07:01 +0530 Subject: [PATCH 12/17] Refactor tests Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/elixir_security.py | 4 +-- vulnerabilities/importers/rust.py | 38 ++++++++++++-------- vulnerabilities/importers/safety_db.py | 9 +++-- vulnerabilities/tests/test_rust.py | 24 ++++++------- vulnerabilities/tests/test_safety_db.py | 15 +++----- 5 files changed, 47 insertions(+), 43 deletions(-) diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 71062a962..34827ac7e 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -87,10 +87,10 @@ def get_versions_for_pkg_from_range_list(self, version_range_list, pkg_name): all_version_list = self.pkg_manager_api.get(pkg_name) if not version_range_list: return [], all_version_list - version_ranges = { + version_ranges = [ VersionSpecifier.from_scheme_version_spec_string("semver", r) for r in version_range_list - } + ] for version in all_version_list: version_obj = SemverVersion(version) if any([version_obj in v for v in version_ranges]): diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index d4b3a2dba..cf6fcd26d 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -23,7 +23,9 @@ import asyncio import re from itertools import chain -from typing import Optional, Mapping +from typing import Optional +from typing import Mapping +from typing import List from typing import Set from typing import Tuple from urllib.error import HTTPError @@ -104,22 +106,22 @@ def _load_advisory(self, path: str) -> Optional[Advisory]: # FIXME: Avoid wildcard version ranges for now. # See https://github.com/RustSec/advisory-db/discussions/831 - affected_ranges = { + affected_ranges = [ VersionSpecifier.from_scheme_version_spec_string("semver", r) for r in chain.from_iterable(record.get("affected", {}).get("functions", {}).values()) if r != "*" - } + ] - unaffected_ranges = { + unaffected_ranges = [ VersionSpecifier.from_scheme_version_spec_string("semver", r) for r in record.get("versions", {}).get("unaffected", []) if r != "*" - } - resolved_ranges = { + ] + resolved_ranges = [ VersionSpecifier.from_scheme_version_spec_string("semver", r) for r in record.get("versions", {}).get("patched", []) if r != "*" - } + ] unaffected, affected = categorize_versions( all_versions, unaffected_ranges, affected_ranges, resolved_ranges @@ -153,9 +155,9 @@ def _load_advisory(self, path: str) -> Optional[Advisory]: def categorize_versions( all_versions: Set[str], - unaffected_versions: Set[VersionSpecifier], - affected_versions: Set[VersionSpecifier], - resolved_versions: Set[VersionSpecifier], + unaffected_version_ranges: List[VersionSpecifier], + affected_version_ranges: List[VersionSpecifier], + resolved_version_ranges: List[VersionSpecifier], ) -> Tuple[Set[str], Set[str]]: """ Categorize all versions of a crate according to the given version ranges. @@ -165,24 +167,30 @@ def categorize_versions( unaffected, affected = set(), set() - if not any(unaffected_versions.union(affected_versions).union(resolved_versions)): + if ( + not unaffected_version_ranges + and not affected_version_ranges + and not resolved_version_ranges + ): return unaffected, affected # TODO: This is probably wrong for version in all_versions: version_obj = SemverVersion(version) - if affected_versions and all([version_obj in av for av in affected_versions]): + if affected_version_ranges and all([version_obj in av for av in affected_version_ranges]): affected.add(version) - elif unaffected_versions and all([version_obj in av for av in unaffected_versions]): + elif unaffected_version_ranges and all( + [version_obj in av for av in unaffected_version_ranges] + ): unaffected.add(version) - elif resolved_versions and all([version_obj in av for av in resolved_versions]): + elif resolved_version_ranges and all([version_obj in av for av in resolved_version_ranges]): unaffected.add(version) # If some versions were not classified above, one or more of the given ranges might be empty, so # the remaining versions default to either affected or unaffected. uncategorized_versions = all_versions - unaffected.union(affected) if uncategorized_versions: - if not affected_versions: + if not affected_version_ranges: affected.update(uncategorized_versions) else: unaffected.update(uncategorized_versions) diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py index dd69e79f0..875a0c928 100755 --- a/vulnerabilities/importers/safety_db.py +++ b/vulnerabilities/importers/safety_db.py @@ -167,8 +167,11 @@ def categorize_versions( :return: impacted, resolved purls """ impacted_versions, impacted_purls = set(), set() - vurl_spec = f"pypi:{ ','.join(version_specs) }" - vurl_spec = VersionSpecifier.from_version_spec_string(vurl_spec) + # vurl_spec = f"pypi:{ ','.join(version_specs) }" + # vurl_spec = VersionSpecifier.from_version_spec_string(vurl_spec) + vurl_specs = [] + for version_spec in version_specs: + vurl_specs.append(VersionSpecifier.from_scheme_version_spec_string("pypi", version_spec)) for version in all_versions: try: @@ -176,7 +179,7 @@ def categorize_versions( except: continue - if version_object in vurl_spec: + if any([version_object in vurl_spec for vurl_spec in vurl_specs]): impacted_versions.add(version) impacted_purls.add( PackageURL( diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py index 6840e3e44..7d0c789af 100644 --- a/vulnerabilities/tests/test_rust.py +++ b/vulnerabilities/tests/test_rust.py @@ -52,12 +52,12 @@ def test_categorize_versions(): flatbuffers_versions = MOCKED_CRATES_API_VERSIONS["flatbuffers"] - unaffected_ranges = {VersionSpecifier.from_scheme_version_spec_string("semver", "< 0.4.0")} - affected_ranges = { + unaffected_ranges = [VersionSpecifier.from_scheme_version_spec_string("semver", "< 0.4.0")] + affected_ranges = [ VersionSpecifier.from_scheme_version_spec_string("semver", ">= 0.4.0"), VersionSpecifier.from_scheme_version_spec_string("semver", "<= 0.6.0"), - } - resolved_ranges = {VersionSpecifier.from_scheme_version_spec_string("semver", ">= 0.6.1")} + ] + resolved_ranges = [VersionSpecifier.from_scheme_version_spec_string("semver", ">= 0.6.1")] unaffected_versions, affected_versions = categorize_versions( set(flatbuffers_versions), @@ -77,9 +77,9 @@ def test_categorize_versions(): def test_categorize_versions_without_affected_ranges(): all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"} - unaffected_ranges = {VersionSpecifier.from_scheme_version_spec_string("semver", "< 1.2")} - affected_ranges = set() - resolved_ranges = {VersionSpecifier.from_scheme_version_spec_string("semver", ">= 3.0")} + unaffected_ranges = [VersionSpecifier.from_scheme_version_spec_string("semver", "< 1.2")] + affected_ranges = [] + resolved_ranges = [VersionSpecifier.from_scheme_version_spec_string("semver", ">= 3.0")] unaffected_versions, affected_versions = categorize_versions( all_versions, @@ -102,12 +102,12 @@ def test_categorize_versions_without_affected_ranges(): def test_categorize_versions_with_only_affected_ranges(): all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"} - unaffected_ranges = set() - affected_ranges = { + unaffected_ranges = [] + affected_ranges = [ VersionSpecifier.from_scheme_version_spec_string("semver", "> 1.2"), VersionSpecifier.from_scheme_version_spec_string("semver", "<= 2.1"), - } - resolved_ranges = set() + ] + resolved_ranges = [] unaffected_versions, affected_versions = categorize_versions( all_versions, @@ -130,7 +130,7 @@ def test_categorize_versions_with_only_affected_ranges(): def test_categorize_versions_without_any_ranges(): all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"} - unaffected, affected = categorize_versions(all_versions, set(), set(), set()) + unaffected, affected = categorize_versions(all_versions, [], [], []) assert len(unaffected) == 0 assert len(affected) == 0 diff --git a/vulnerabilities/tests/test_safety_db.py b/vulnerabilities/tests/test_safety_db.py index 1d6e52bbd..2d3621141 100644 --- a/vulnerabilities/tests/test_safety_db.py +++ b/vulnerabilities/tests/test_safety_db.py @@ -74,10 +74,10 @@ def test_import(self): with patch("vulnerabilities.importers.SafetyDbDataSource.set_api"): runner.run() - assert models.Vulnerability.objects.count() == 9 - assert models.VulnerabilityReference.objects.count() == 9 - assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=False).count() == 18 - assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=True).count() == 18 + assert models.Vulnerability.objects.count() == 8 + assert models.VulnerabilityReference.objects.count() == 8 + assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=False).count() == 15 + assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=True).count() == 17 expected_package_count = sum([len(v) for v in MOCK_VERSION_API.cache.values()]) assert models.Package.objects.count() == expected_package_count @@ -106,13 +106,6 @@ def test_import(self): cve_ids={"CVE-2016-6186"}, ) - self.assert_by_vulnerability( - "pyup.io-38115", - "zulip", - {"2.0"}, - {"2.1.1", "2.1.2", "2.1.3"}, - ) - self.assert_by_vulnerability( "pyup.io-38114", "zulip", From bda5f0d9ac26e7b207eaaff0c7d37eb338e588ef Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Wed, 31 Mar 2021 08:21:16 +0530 Subject: [PATCH 13/17] Update packaging dep Signed-off-by: Shivam Sandbhor --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d1ae8229f..edae28b86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ jedi==0.17.0 lxml==4.6.3 more-itertools==8.0.2 packageurl-python==0.9.3 -packaging==19.2 +packaging==20.9 parso==0.7.0 pexpect==4.8.0 pickleshare==0.7.5 From 049f05caf22d2fa4ea5f30dd8471b6aa0837fd49 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 2 Apr 2021 10:09:39 +0530 Subject: [PATCH 14/17] Remove debug statements, improve doc, simplify code Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/apache_tomcat.py | 2 -- vulnerabilities/importers/istio.py | 3 --- vulnerabilities/importers/npm.py | 4 +++- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index 6652aa737..b72d5a8b0 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -111,8 +111,6 @@ def to_advisories(self, apache_tomcat_advisory_html): type="maven", namespace="apache", name="tomcat", version=fixed_version ) ] - if fixed_package in affected_packages: - raise ValueError("Bing bong") advisories.append( Advisory( diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 9b76176af..7c20cd16a 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -31,7 +31,6 @@ from vulnerabilities.data_source import Advisory, GitDataSource, Reference from vulnerabilities.package_managers import GitHubTagsAPI -from vulnerabilities.helpers import contains_alpha is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match @@ -49,8 +48,6 @@ def __enter__(self): def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) - # Only keep public releases - self.version_api.cache = list(filter(contains_alpha, self.version_api.cache)) def updated_advisories(self) -> Set[Advisory]: files = self._added_files.union(self._updated_files) diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index b70f2b560..f1b524dc0 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -130,7 +130,9 @@ def _versions_to_purls(package_name, versions): def normalize_ranges(version_range_string): """ - Splits version range strings with "||" operator into separate ranges + - Splits version range strings with "||" operator into separate ranges. + - Removes spaces between range operator and range operands + - Normalizes 'x' ranges Example: >>> z = normalize_ranges(">=6.1.3 < 7.0.0 || >=7.0.3") >>> assert z == [">=6.1.3,<7.0.0", ">=7.0.3"] From 653f949e61a184ef9dc6e9df09e8a071f05ab515 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 2 Apr 2021 13:37:38 +0530 Subject: [PATCH 15/17] Refactor universal versions to univers Signed-off-by: Shivam Sandbhor --- requirements.txt | 2 +- vulnerabilities/data_source.py | 4 ++-- vulnerabilities/importers/apache_kafka.py | 4 ++-- vulnerabilities/importers/apache_tomcat.py | 4 ++-- vulnerabilities/importers/elixir_security.py | 4 ++-- vulnerabilities/importers/github.py | 4 ++-- vulnerabilities/importers/istio.py | 4 ++-- vulnerabilities/importers/nginx.py | 4 ++-- vulnerabilities/importers/npm.py | 4 ++-- vulnerabilities/importers/ruby.py | 4 ++-- vulnerabilities/importers/rust.py | 4 ++-- vulnerabilities/importers/safety_db.py | 4 ++-- vulnerabilities/tests/test_apache_kafka.py | 2 +- vulnerabilities/tests/test_rust.py | 2 +- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/requirements.txt b/requirements.txt index edae28b86..707e6db68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ cached-property==1.5.1 cffi==1.14.0 contextlib2==0.5.5 decorator==4.4.2 -universal-versions @ git+https://github.com/sbs2001/universal-versions.git +univers @ git+https://github.com/sbs2001/univers.git dj-database-url==0.4.2 Django==3.0.13 django-filter==2.2.0 diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index e899440d4..3010eea9e 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -41,8 +41,8 @@ import pygit2 from packageurl import PackageURL -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import version_class_by_package_type +from univers.version_specifier import VersionSpecifier +from univers.versions import version_class_by_package_type from vulnerabilities.oval_parser import OvalParser from vulnerabilities.severity_systems import ScoringSystem diff --git a/vulnerabilities/importers/apache_kafka.py b/vulnerabilities/importers/apache_kafka.py index 1994b46bf..1b763cb7a 100644 --- a/vulnerabilities/importers/apache_kafka.py +++ b/vulnerabilities/importers/apache_kafka.py @@ -25,8 +25,8 @@ import requests from bs4 import BeautifulSoup from packageurl import PackageURL -from universal_versions.versions import MavenVersion -from universal_versions.version_specifier import VersionSpecifier +from univers.versions import MavenVersion +from univers.version_specifier import VersionSpecifier from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index b72d5a8b0..99fda05d6 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -27,8 +27,8 @@ import requests from bs4 import BeautifulSoup -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import MavenVersion +from univers.version_specifier import VersionSpecifier +from univers.versions import MavenVersion from packageurl import PackageURL from vulnerabilities.data_source import Advisory diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 34827ac7e..c0c937718 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -24,8 +24,8 @@ from typing import List, Set import yaml -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import SemverVersion +from univers.version_specifier import VersionSpecifier +from univers.versions import SemverVersion from packageurl import PackageURL from vulnerabilities.data_source import GitDataSource diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index 9d27c5c48..cea315efc 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -32,8 +32,8 @@ import requests from packageurl import PackageURL -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import version_class_by_package_type +from univers.version_specifier import VersionSpecifier +from univers.versions import version_class_by_package_type from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 7c20cd16a..02b34de4b 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -25,8 +25,8 @@ from typing import List, Set import yaml -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import SemverVersion +from univers.version_specifier import VersionSpecifier +from univers.versions import SemverVersion from packageurl import PackageURL from vulnerabilities.data_source import Advisory, GitDataSource, Reference diff --git a/vulnerabilities/importers/nginx.py b/vulnerabilities/importers/nginx.py index 195191cea..11e49b738 100644 --- a/vulnerabilities/importers/nginx.py +++ b/vulnerabilities/importers/nginx.py @@ -27,8 +27,8 @@ import requests from packageurl import PackageURL from bs4 import BeautifulSoup -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import SemverVersion +from univers.version_specifier import VersionSpecifier +from univers.versions import SemverVersion from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index f1b524dc0..2c5988cfc 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -32,8 +32,8 @@ from urllib.request import urlopen from dateutil.parser import parse -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import SemverVersion +from univers.version_specifier import VersionSpecifier +from univers.versions import SemverVersion from packageurl import PackageURL from vulnerabilities.data_source import Advisory diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index 069425df4..f22df5aa5 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -25,8 +25,8 @@ from typing import List from packageurl import PackageURL -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import SemverVersion +from univers.version_specifier import VersionSpecifier +from univers.versions import SemverVersion from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index cf6fcd26d..1c3ecd98f 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -32,8 +32,8 @@ from urllib.request import urlopen import toml -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import SemverVersion +from univers.version_specifier import VersionSpecifier +from univers.versions import SemverVersion from packageurl import PackageURL diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py index 875a0c928..228aec425 100755 --- a/vulnerabilities/importers/safety_db.py +++ b/vulnerabilities/importers/safety_db.py @@ -34,8 +34,8 @@ from typing import Tuple import requests -from universal_versions.version_specifier import VersionSpecifier -from universal_versions.versions import PYPIVersion +from univers.version_specifier import VersionSpecifier +from univers.versions import PYPIVersion from packageurl import PackageURL from schema import Or from schema import Regex diff --git a/vulnerabilities/tests/test_apache_kafka.py b/vulnerabilities/tests/test_apache_kafka.py index 69a6e5ee6..2de04390d 100644 --- a/vulnerabilities/tests/test_apache_kafka.py +++ b/vulnerabilities/tests/test_apache_kafka.py @@ -24,7 +24,7 @@ from unittest import TestCase from packageurl import PackageURL -from universal_versions.version_specifier import VersionSpecifier +from univers.version_specifier import VersionSpecifier from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import Reference diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py index 7d0c789af..96be46720 100644 --- a/vulnerabilities/tests/test_rust.py +++ b/vulnerabilities/tests/test_rust.py @@ -27,7 +27,7 @@ from unittest.mock import patch -from universal_versions.version_specifier import VersionSpecifier +from univers.version_specifier import VersionSpecifier from django.test import TestCase From 7c4bbf4fd1609b4c6f40a8c4da28fd1761ebdc28 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 2 Apr 2021 13:41:02 +0530 Subject: [PATCH 16/17] Update attrs dep Signed-off-by: Shivam Sandbhor --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 707e6db68..db773fc0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aiohttp==3.7.4 asgiref==3.2.7 -attrs==19.3.0 +attrs==20.3.0 backcall==0.1.0 beautifulsoup4==4.7.1 cached-property==1.5.1 From 0ca5e1689db5b4bf6c8f03bdefe7b24cee1feeb5 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 8 Apr 2021 09:57:19 +0530 Subject: [PATCH 17/17] Install univers from pypi Signed-off-by: Shivam Sandbhor --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index db773fc0a..cbd4bd2c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ cached-property==1.5.1 cffi==1.14.0 contextlib2==0.5.5 decorator==4.4.2 -univers @ git+https://github.com/sbs2001/univers.git +univers==21.4.8 dj-database-url==0.4.2 Django==3.0.13 django-filter==2.2.0