diff --git a/requirements.txt b/requirements.txt index b92834fcc..24d84536f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -106,7 +106,7 @@ toml==0.10.2 tomli==2.0.1 traitlets==5.1.1 typing_extensions==4.1.1 -univers==30.7.0 +univers==30.9.0 urllib3==1.26.9 wcwidth==0.2.5 websocket-client==0.59.0 diff --git a/setup.cfg b/setup.cfg index b1bf0b721..692e444f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,7 +65,7 @@ install_requires = #essentials packageurl-python>=0.9.4 - univers>=30.3.1 + univers>=30.9.0 license-expression>=21.6.14 # file and data formats diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 34082ae6c..70a42646b 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -8,6 +8,7 @@ # from vulnerabilities.importers import alpine_linux +from vulnerabilities.importers import archlinux from vulnerabilities.importers import debian from vulnerabilities.importers import github from vulnerabilities.importers import gitlab @@ -29,6 +30,7 @@ debian.DebianImporter, gitlab.GitLabAPIImporter, pypa.PyPaImporter, + archlinux.ArchlinuxImporter, ] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 91e367897..b1fb861ac 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -31,6 +31,7 @@ from vulnerabilities.references import WireSharkReference from vulnerabilities.references import XsaReference from vulnerabilities.references import ZbxReference +from vulnerabilities.utils import fetch_response from vulnerabilities.utils import is_cve LOGGER = logging.getLogger(__name__) @@ -58,16 +59,6 @@ def advisory_data(self) -> Iterable[AdvisoryData]: yield from process_record(record) -def fetch_response(url): - """ - Fetch and return `response` from the `url` - """ - response = requests.get(url) - if response.status_code == 200: - return response - raise Exception(f"Failed to fetch data from {url!r} with status code: {response.status_code!r}") - - def fetch_advisory_directory_links(page_response_content: str) -> List[str]: """ Return a list of advisory directory links present in `page_response_content` html string diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index f72e96c7a..a059ab9e3 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -6,66 +6,64 @@ # See https://github.com/nexB/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import dataclasses -import json + from typing import Iterable from typing import List from typing import Mapping -from typing import Set from urllib.request import urlopen from packageurl import PackageURL +from univers.version_range import ArchLinuxVersionRange +from univers.versions import ArchLinuxVersion from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.utils import nearest_patched_package +from vulnerabilities.utils import fetch_response class ArchlinuxImporter(Importer): - def __enter__(self): - self._api_response = self._fetch() - - def updated_advisories(self) -> Set[AdvisoryData]: - advisories = [] + url = "https://security.archlinux.org/json" + spdx_license_expression = "MIT" + license_url = "https://github.com/archlinux/arch-security-tracker/blob/master/LICENSE" - for record in self._api_response: - advisories.extend(self._parse(record)) + def fetch(self) -> Iterable[Mapping]: + response = fetch_response(self.url) + return response.json() - return self.batch_advisories(advisories) + def advisory_data(self) -> Iterable[AdvisoryData]: + for record in self.fetch(): + yield from self.parse_advisory(record) - def _fetch(self) -> Iterable[Mapping]: - with urlopen(self.config.archlinux_tracker_url) as response: - return json.load(response) - - def _parse(self, record) -> List[AdvisoryData]: + def parse_advisory(self, record) -> List[AdvisoryData]: advisories = [] - - for cve_id in record["issues"]: + aliases = record.get("issues") or [] + for alias in aliases: affected_packages = [] for name in record["packages"]: - impacted_purls, resolved_purls = [], [] - impacted_purls.append( - PackageURL( + summary = record.get("type") or "" + if summary == "unknown": + summary = "" + affected = record.get("affected") or "" + affected_version_range = ( + ArchLinuxVersionRange.from_versions([affected]) if affected else None + ) + fixed = record.get("fixed") or "" + fixed_version = ArchLinuxVersion(fixed) if fixed else None + affected_packages = [] + affected_package = AffectedPackage( + package=PackageURL( name=name, - type="pacman", + type="alpm", namespace="archlinux", - version=record["affected"], - ) + ), + affected_version_range=affected_version_range, + fixed_version=fixed_version, ) - - if record["fixed"]: - resolved_purls.append( - PackageURL( - name=name, - type="pacman", - namespace="archlinux", - version=record["fixed"], - ) - ) - affected_packages.extend(nearest_patched_package(impacted_purls, resolved_purls)) + affected_packages.append(affected_package) references = [] references.append( @@ -89,9 +87,9 @@ def _parse(self, record) -> List[AdvisoryData]: ) advisories.append( - Advisory( - vulnerability_id=cve_id, - summary="", + AdvisoryData( + aliases=[alias], + summary=summary, affected_packages=affected_packages, references=references, ) diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index bbce075aa..15962e14a 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -29,7 +29,6 @@ def no_rmtree(monkeypatch): "test_apache_kafka.py", "test_apache_tomcat.py", "test_api.py", - "test_archlinux.py", "test_debian_oval.py", "test_elixir_security.py", "test_gentoo.py", diff --git a/vulnerabilities/tests/test_archlinux.py b/vulnerabilities/tests/test_archlinux.py index 51e671116..3b1908e73 100644 --- a/vulnerabilities/tests/test_archlinux.py +++ b/vulnerabilities/tests/test_archlinux.py @@ -16,87 +16,38 @@ from vulnerabilities import models from vulnerabilities.import_runner import ImportRunner +from vulnerabilities.importers import archlinux +from vulnerabilities.tests import util_tests BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/") - - -class ArchlinuxImportTest(TestCase): - @classmethod - def setUpClass(cls) -> None: - fixture_path = os.path.join(TEST_DATA, "archlinux.json") - with open(fixture_path) as f: - cls.mock_response = json.load(f) - - cls.importer = models.Importer.objects.create( - name="archlinux_unittests", - license="", - last_run=None, - data_source="ArchlinuxImporter", - data_source_cfg={ - "archlinux_tracker_url": "https://security.example.com/json", - }, - ) - - @classmethod - def tearDownClass(cls) -> None: - pass - - def test_import(self): - runner = ImportRunner(self.importer, 5) - - with patch( - "vulnerabilities.importers.ArchlinuxImporter._fetch", return_value=self.mock_response - ): - runner.run() - assert models.Vulnerability.objects.count() == 6 - assert models.VulnerabilityReference.objects.count() == 10 - assert models.PackageRelatedVulnerability.objects.all().count() == 12 - assert ( - models.PackageRelatedVulnerability.objects.filter(patched_package__isnull=False).count() - == 8 - ) - assert models.Package.objects.count() == 10 - - self.assert_for_package( - "squid", - "4.10-2", - cve_ids={"CVE-2020-11945", "CVE-2019-12521", "CVE-2019-12519"}, - ) - self.assert_for_package("openconnect", "1:8.05-1", cve_ids={"CVE-2020-12823"}) - self.assert_for_package( - "wireshark-common", - "2.6.0-1", - cve_ids={"CVE-2018-11362", "CVE-2018-11361"}, - ) - self.assert_for_package( - "wireshark-gtk", - "2.6.0-1", - cve_ids={"CVE-2018-11362", "CVE-2018-11361"}, - ) - self.assert_for_package( - "wireshark-cli", - "2.6.0-1", - cve_ids={"CVE-2018-11362", "CVE-2018-11361"}, - ) - self.assert_for_package( - "wireshark-qt", - "2.6.0-1", - cve_ids={"CVE-2018-11362", "CVE-2018-11361"}, - ) - self.assert_for_package("wireshark-common", "2.6.1-1") - self.assert_for_package("wireshark-gtk", "2.6.1-1") - self.assert_for_package("wireshark-cli", "2.6.1-1") - self.assert_for_package("wireshark-qt", "2.6.1-1") - - def assert_for_package(self, name, version, cve_ids=None): - qs = models.Package.objects.filter( - name=name, - version=version, - type="pacman", - namespace="archlinux", - ) - assert qs - - if cve_ids: - assert cve_ids == {v.vulnerability_id for v in qs[0].vulnerabilities.all()} +TEST_DATA = os.path.join(BASE_DIR, "test_data/archlinux") + + +def test_parse_advisory_single(): + record = { + "name": "AVG-2781", + "packages": ["python-pyjwt"], + "status": "Unknown", + "severity": "Unknown", + "type": "unknown", + "affected": "2.3.0-1", + "fixed": "2.4.0-1", + "ticket": None, + "issues": ["CVE-2022-29217"], + "advisories": [], + } + + advisory_data = archlinux.ArchlinuxImporter().parse_advisory(record) + result = [data.to_dict() for data in advisory_data] + expected_file = os.path.join(TEST_DATA, f"parse-advisory-archlinux-expected.json") + util_tests.check_results_against_json(result, expected_file) + + +@patch("vulnerabilities.importers.archlinux.ArchlinuxImporter.fetch") +def test_archlinux_importer(mock_response): + with open(os.path.join(TEST_DATA, "archlinux-multi.json")) as f: + mock_response.return_value = json.load(f) + + expected_file = os.path.join(TEST_DATA, f"archlinux-multi-expected.json") + result = [data.to_dict() for data in list(archlinux.ArchlinuxImporter().advisory_data())] + util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_data/archlinux/archlinux-multi-expected.json b/vulnerabilities/tests/test_data/archlinux/archlinux-multi-expected.json new file mode 100644 index 000000000..6ead1f6d7 --- /dev/null +++ b/vulnerabilities/tests/test_data/archlinux/archlinux-multi-expected.json @@ -0,0 +1,172 @@ +[ + { + "aliases": [ + "CVE-2022-29217" + ], + "summary": "", + "affected_packages": [ + { + "package": { + "type": "alpm", + "namespace": "archlinux", + "name": "python-pyjwt", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:alpm/2.3.0-1", + "fixed_version": "2.4.0-1" + } + ], + "references": [ + { + "reference_id": "AVG-2781", + "url": "https://security.archlinux.org/AVG-2781", + "severities": [ + { + "system": "archlinux", + "value": "Unknown" + } + ] + } + ], + "date_published": null + }, + { + "aliases": [ + "CVE-2022-26710" + ], + "summary": "", + "affected_packages": [ + { + "package": { + "type": "alpm", + "namespace": "archlinux", + "name": "wpewebkit", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:alpm/2.36.3-1", + "fixed_version": "2.36.4-1" + } + ], + "references": [ + { + "reference_id": "AVG-2780", + "url": "https://security.archlinux.org/AVG-2780", + "severities": [ + { + "system": "archlinux", + "value": "Unknown" + } + ] + } + ], + "date_published": null + }, + { + "aliases": [ + "CVE-2022-22677" + ], + "summary": "", + "affected_packages": [ + { + "package": { + "type": "alpm", + "namespace": "archlinux", + "name": "wpewebkit", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:alpm/2.36.3-1", + "fixed_version": "2.36.4-1" + } + ], + "references": [ + { + "reference_id": "AVG-2780", + "url": "https://security.archlinux.org/AVG-2780", + "severities": [ + { + "system": "archlinux", + "value": "Unknown" + } + ] + } + ], + "date_published": null + }, + { + "aliases": [ + "CVE-2022-22662" + ], + "summary": "", + "affected_packages": [ + { + "package": { + "type": "alpm", + "namespace": "archlinux", + "name": "wpewebkit", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:alpm/2.36.3-1", + "fixed_version": "2.36.4-1" + } + ], + "references": [ + { + "reference_id": "AVG-2780", + "url": "https://security.archlinux.org/AVG-2780", + "severities": [ + { + "system": "archlinux", + "value": "Unknown" + } + ] + } + ], + "date_published": null + }, + { + "aliases": [ + "CVE-2016-3189" + ], + "summary": "denial of service", + "affected_packages": [ + { + "package": { + "type": "alpm", + "namespace": "archlinux", + "name": "bzip2", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:alpm/1.0.6-5", + "fixed_version": "1.0.6-6" + } + ], + "references": [ + { + "reference_id": "AVG-4", + "url": "https://security.archlinux.org/AVG-4", + "severities": [ + { + "system": "archlinux", + "value": "Low" + } + ] + }, + { + "reference_id": "ASA-201702-19", + "url": "https://security.archlinux.org/ASA-201702-19", + "severities": [] + } + ], + "date_published": null + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/archlinux/archlinux-multi.json b/vulnerabilities/tests/test_data/archlinux/archlinux-multi.json new file mode 100644 index 000000000..14bfb17cf --- /dev/null +++ b/vulnerabilities/tests/test_data/archlinux/archlinux-multi.json @@ -0,0 +1,54 @@ +[ + { + "name": "AVG-2781", + "packages": [ + "python-pyjwt" + ], + "status": "Unknown", + "severity": "Unknown", + "type": "unknown", + "affected": "2.3.0-1", + "fixed": "2.4.0-1", + "ticket": null, + "issues": [ + "CVE-2022-29217" + ], + "advisories": [] + }, + { + "name": "AVG-2780", + "packages": [ + "wpewebkit" + ], + "status": "Unknown", + "severity": "Unknown", + "type": "unknown", + "affected": "2.36.3-1", + "fixed": "2.36.4-1", + "ticket": null, + "issues": [ + "CVE-2022-26710", + "CVE-2022-22677", + "CVE-2022-22662" + ], + "advisories": [] + }, + { + "name": "AVG-4", + "packages": [ + "bzip2" + ], + "status": "Fixed", + "severity": "Low", + "type": "denial of service", + "affected": "1.0.6-5", + "fixed": "1.0.6-6", + "ticket": null, + "issues": [ + "CVE-2016-3189" + ], + "advisories": [ + "ASA-201702-19" + ] + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/archlinux/parse-advisory-archlinux-expected.json b/vulnerabilities/tests/test_data/archlinux/parse-advisory-archlinux-expected.json new file mode 100644 index 000000000..12d7ad947 --- /dev/null +++ b/vulnerabilities/tests/test_data/archlinux/parse-advisory-archlinux-expected.json @@ -0,0 +1,35 @@ +[ + { + "aliases": [ + "CVE-2022-29217" + ], + "summary": "", + "affected_packages": [ + { + "package": { + "type": "alpm", + "namespace": "archlinux", + "name": "python-pyjwt", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:alpm/2.3.0-1", + "fixed_version": "2.4.0-1" + } + ], + "references": [ + { + "reference_id": "AVG-2781", + "url": "https://security.archlinux.org/AVG-2781", + "severities": [ + { + "system": "archlinux", + "value": "Unknown" + } + ] + } + ], + "date_published": null + } +] \ No newline at end of file diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 737b26171..c027b6bdc 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -410,3 +410,13 @@ def base32_custom(btes): + _base32_table[c & 0x3FF] # bits 21 - 30 # bits 31 - 40 ) return bytes(encoded) + + +def fetch_response(url): + """ + Fetch and return `response` from the `url` + """ + response = requests.get(url) + if response.status_code == 200: + return response + raise Exception(f"Failed to fetch data from {url!r} with status code: {response.status_code!r}")