diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index 5cd72146b..02c63688a 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -30,6 +30,7 @@ from dateutil import parser as dateparser from django.db.models.query import QuerySet from packageurl import PackageURL +from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import build_range_from_github_advisory_constraint from vulnerabilities import severity_systems @@ -48,6 +49,7 @@ from vulnerabilities.package_managers import VersionAPI from vulnerabilities.package_managers import get_api_package_name from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage +from vulnerabilities.utils import dedupe from vulnerabilities.utils import get_affected_packages_by_patched_package from vulnerabilities.utils import get_item from vulnerabilities.utils import nearest_patched_package @@ -153,6 +155,9 @@ severity publishedAt } + firstPatchedVersion{ + identifier + } package { name } @@ -236,60 +241,64 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: return for vulnerability in vulnerabilities: + aliases = [] affected_packages = [] - aliases = set() github_advisory = get_item(vulnerability, "node") if not github_advisory: logger.error(f"No node found in {vulnerability!r}") continue - name = get_item(github_advisory, "package", "name") - if not name: - logger.error(f"No name found in {github_advisory!r}") - continue - - purl = get_purl(pkg_type=package_type, github_name=name) - if not purl: - continue - - vulnerable_range = get_item(github_advisory, "vulnerableVersionRange") - if not vulnerable_range: - logger.error(f"No affected range found in {github_advisory!r}") - continue - - affected_range = None - try: - affected_range = build_range_from_github_advisory_constraint( - package_type, vulnerable_range - ) - except InvalidVersionRange: - logger.error(f"Could not parse affected range {vulnerable_range!r}") - continue - - if affected_range != NotImplementedError: - affected_packages.append( - AffectedPackage( - package=purl, - affected_version_range=affected_range, - ) - ) - advisory = get_item(github_advisory, "advisory") if not advisory: logger.error(f"No advisory found in {github_advisory!r}") continue + summary = get_item(advisory, "summary") or "" + references = get_item(advisory, "references") or [] if references: urls = (ref["url"] for ref in references) references = [Reference.from_url(u) for u in urls] - summary = get_item(advisory, "summary") + date_published = get_item(advisory, "publishedAt") + if date_published: + date_published = dateparser.parse(date_published) + + name = get_item(github_advisory, "package", "name") + if name: + purl = get_purl(pkg_type=package_type, github_name=name) + if purl: + affected_range = get_item(github_advisory, "vulnerableVersionRange") + fixed_version = get_item(github_advisory, "firstPatchedVersion", "identifier") + if affected_range: + try: + affected_range = build_range_from_github_advisory_constraint( + package_type, affected_range + ) + except InvalidVersionRange as e: + logger.error(f"Could not parse affected range {affected_range!r} {e!r}") + affected_range = None + if fixed_version: + try: + fixed_version = RANGE_CLASS_BY_SCHEMES[package_type].version_class( + fixed_version + ) + except Exception as e: + logger.error(f"Invalid fixed version {fixed_version!r} {e!r}") + fixed_version = None + if affected_range or fixed_version: + affected_packages.append( + AffectedPackage( + package=purl, + affected_version_range=affected_range, + fixed_version=fixed_version, + ) + ) identifiers = get_item(advisory, "identifiers") or [] for identifier in identifiers: value = identifier["value"] identifier_type = identifier["type"] - aliases.add(value) + aliases.append(value) # attach the GHSA with severity score if identifier_type == "GHSA": # Each Node has only one GHSA, hence exit after attaching @@ -310,12 +319,8 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: else: logger.error(f"Unknown identifier type {identifier_type!r} and value {value!r}") - date_published = get_item(advisory, "publishedAt") - if date_published: - date_published = dateparser.parse(date_published) - yield AdvisoryData( - aliases=sorted(list(aliases)), + aliases=sorted(dedupe(aliases)), summary=summary, references=references, affected_packages=affected_packages, diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index 3f77b9ea0..caf5e3a5c 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -395,9 +395,6 @@ class ComposerVersionAPI(VersionAPI): package_type = "composer" def fetch(self, pkg: str) -> Iterable[PackageVersion]: - if "/" not in pkg: - raise Exception(f"Composer package: {pkg!r} does not have a vendor/name structure.") - response = get_response(url=f"https://repo.packagist.org/p/{pkg}.json") if response: yield from self.extract_versions(response, pkg) diff --git a/vulnerabilities/tests/test_data/github_api/composer-expected.json b/vulnerabilities/tests/test_data/github_api/composer-expected.json index 4da9122cb..bb33f3d76 100644 --- a/vulnerabilities/tests/test_data/github_api/composer-expected.json +++ b/vulnerabilities/tests/test_data/github_api/composer-expected.json @@ -163,7 +163,7 @@ "subpath": null }, "affected_version_range": "vers:composer/<22.1.0", - "fixed_version": null + "fixed_version": "22.1.0" } ], "references": [ diff --git a/vulnerabilities/tests/test_data/github_api/composer.json b/vulnerabilities/tests/test_data/github_api/composer.json index 72e6bc701..2079c2d0f 100644 --- a/vulnerabilities/tests/test_data/github_api/composer.json +++ b/vulnerabilities/tests/test_data/github_api/composer.json @@ -150,6 +150,9 @@ "package": { "name": "librenms/librenms" }, + "firstPatchedVersion": { + "identifier" :"22.1.0" + }, "vulnerableVersionRange": "< 22.1.0" } } diff --git a/vulnerabilities/tests/test_data/github_api/gem-expected.json b/vulnerabilities/tests/test_data/github_api/gem-expected.json index 07d98054b..d5aee434b 100644 --- a/vulnerabilities/tests/test_data/github_api/gem-expected.json +++ b/vulnerabilities/tests/test_data/github_api/gem-expected.json @@ -16,7 +16,7 @@ "subpath": null }, "affected_version_range": "vers:gem/<=1.3.1", - "fixed_version": null + "fixed_version": "1.3.2" } ], "references": [ diff --git a/vulnerabilities/tests/test_data/github_api/gem.json b/vulnerabilities/tests/test_data/github_api/gem.json index db91c7eda..4df89cd29 100644 --- a/vulnerabilities/tests/test_data/github_api/gem.json +++ b/vulnerabilities/tests/test_data/github_api/gem.json @@ -57,6 +57,9 @@ "package": { "name": "webrick" }, + "firstPatchedVersion": { + "identifier" :"1.3.2" + }, "vulnerableVersionRange": "<= 1.3.1" } }, diff --git a/vulnerabilities/tests/test_data/github_api/golang-expected.json b/vulnerabilities/tests/test_data/github_api/golang-expected.json index a06494b32..f4bbfd396 100644 --- a/vulnerabilities/tests/test_data/github_api/golang-expected.json +++ b/vulnerabilities/tests/test_data/github_api/golang-expected.json @@ -16,7 +16,7 @@ "subpath": null }, "affected_version_range": "vers:golang/<1.3.3", - "fixed_version": null + "fixed_version": "1.3.3" } ], "references": [ diff --git a/vulnerabilities/tests/test_data/github_api/golang.json b/vulnerabilities/tests/test_data/github_api/golang.json index 03d9005d1..21e53268e 100644 --- a/vulnerabilities/tests/test_data/github_api/golang.json +++ b/vulnerabilities/tests/test_data/github_api/golang.json @@ -45,6 +45,9 @@ "package": { "name": "github.com/moby/moby" }, + "firstPatchedVersion": { + "identifier" :"1.3.3" + }, "vulnerableVersionRange": "< 1.3.3" } }, diff --git a/vulnerabilities/tests/test_data/github_api/maven-expected.json b/vulnerabilities/tests/test_data/github_api/maven-expected.json index 2ad169b4f..ca45ae433 100644 --- a/vulnerabilities/tests/test_data/github_api/maven-expected.json +++ b/vulnerabilities/tests/test_data/github_api/maven-expected.json @@ -152,7 +152,7 @@ "subpath": null }, "affected_version_range": "vers:maven/>=9.0.0|<9.0.31", - "fixed_version": null + "fixed_version": "9.0.1" } ], "references": [ diff --git a/vulnerabilities/tests/test_data/github_api/maven.json b/vulnerabilities/tests/test_data/github_api/maven.json index 147962df4..13e1621a4 100644 --- a/vulnerabilities/tests/test_data/github_api/maven.json +++ b/vulnerabilities/tests/test_data/github_api/maven.json @@ -139,6 +139,9 @@ "package": { "name": "org.apache.tomcat.embed:tomcat-embed-core" }, + "firstPatchedVersion": { + "identifier" :"9.0.1" + }, "vulnerableVersionRange": ">= 9.0.0, < 9.0.31" } } diff --git a/vulnerabilities/tests/test_data/github_api/nuget-expected.json b/vulnerabilities/tests/test_data/github_api/nuget-expected.json index d88a02235..fa3df350a 100644 --- a/vulnerabilities/tests/test_data/github_api/nuget-expected.json +++ b/vulnerabilities/tests/test_data/github_api/nuget-expected.json @@ -16,7 +16,7 @@ "subpath": null }, "affected_version_range": "vers:nuget/<=4.5.1-alpha001", - "fixed_version": null + "fixed_version": "4.5.1" } ], "references": [ diff --git a/vulnerabilities/tests/test_data/github_api/nuget.json b/vulnerabilities/tests/test_data/github_api/nuget.json index 4d5d60354..09ba73c54 100644 --- a/vulnerabilities/tests/test_data/github_api/nuget.json +++ b/vulnerabilities/tests/test_data/github_api/nuget.json @@ -33,6 +33,9 @@ "package": { "name": "RazorEngine" }, + "firstPatchedVersion": { + "identifier" :"4.5.1" + }, "vulnerableVersionRange": "<= 4.5.1-alpha001" } }, diff --git a/vulnerabilities/tests/test_data/github_api/pypi-expected.json b/vulnerabilities/tests/test_data/github_api/pypi-expected.json index 5f37bf59a..6469325eb 100644 --- a/vulnerabilities/tests/test_data/github_api/pypi-expected.json +++ b/vulnerabilities/tests/test_data/github_api/pypi-expected.json @@ -15,7 +15,7 @@ "subpath": null }, "affected_version_range": "vers:pypi/<9.0.0", - "fixed_version": null + "fixed_version": "9.0.0" } ], "references": [ diff --git a/vulnerabilities/tests/test_data/github_api/pypi.json b/vulnerabilities/tests/test_data/github_api/pypi.json index 2f63948e2..eb4e0b4e0 100644 --- a/vulnerabilities/tests/test_data/github_api/pypi.json +++ b/vulnerabilities/tests/test_data/github_api/pypi.json @@ -29,6 +29,9 @@ "package": { "name": "Pillow" }, + "firstPatchedVersion": { + "identifier" :"9.0.0" + }, "vulnerableVersionRange": "< 9.0.0" } }, diff --git a/vulnerabilities/tests/test_utils.py b/vulnerabilities/tests/test_utils.py index 7335973c3..066fa044e 100644 --- a/vulnerabilities/tests/test_utils.py +++ b/vulnerabilities/tests/test_utils.py @@ -23,6 +23,7 @@ from packageurl import PackageURL from vulnerabilities.utils import AffectedPackage +from vulnerabilities.utils import get_item from vulnerabilities.utils import nearest_patched_package from vulnerabilities.utils import split_markdown_front_matter @@ -94,3 +95,17 @@ def test_split_markdown_front_matter(): results = split_markdown_front_matter(text) assert results == expected + + +def test_get_item(): + d1 = {"a": {"b": {"c": None}}} + assert get_item(d1, "a", "b", "c", "d") == None + d2 = {"a": {"b": {"c": {"d": None}}}} + assert get_item(d2, "a", "b", "c", "e") == None + d3 = ["a", "b", "c", "d"] + assert get_item(d3, "a", "b") == None + d4 = {"a": {"b": {"c": {"d": []}}}} + assert get_item(d4, "a", "b", "c", "d", "e") == None + d5 = {"a": {"b": {"c": "d"}}} + assert get_item(d5, "a", "b", "c", "d") == None + assert get_item(d5, "a", "b", "c") == "d" diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index dd1b985c2..0e7334ec8 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -217,12 +217,15 @@ def get_item(dictionary: dict, *attributes): 'd' >>> assert(get_item({'a': {'b': {'c': 'd'}}}, 'a', 'b', 'e')) == None """ - if not dictionary: - return for attribute in attributes: + if not dictionary: + return + if not isinstance(dictionary, dict): + logger.error("dictionary must be of type `dict`") + return if attribute not in dictionary: logger.error(f"Missing attribute {attribute} in {dictionary}") - return None + return dictionary = dictionary[attribute] return dictionary