From 9f3496f193e443ea1ae0b10c336ab855c4ceeb09 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Wed, 15 Jul 2020 14:30:53 +0530 Subject: [PATCH 1/9] Add data class for vulnerability reference Signed-off-by: Shivam Sandbhor --- vulnerabilities/data_source.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 0dbfd7677..e73f5cb1c 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -43,6 +43,16 @@ from vulnerabilities.oval_parser import OvalParser +@dataclasses.dataclass +class VulnerabilityReferenceUnit: + + url : str = '' + reference_id : str = '' + + def __post_init__(self) : + if not any([self.url, self.reference_id]): + raise TypeError + @dataclasses.dataclass class Advisory: """ @@ -57,8 +67,7 @@ class Advisory: summary: str impacted_package_urls: Iterable[PackageURL] resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list) - reference_urls: Sequence[str] = dataclasses.field(default_factory=list) - reference_ids: Sequence[str] = dataclasses.field(default_factory=list) + vuln_references = Sequence[VulnerabilityReferenceUnit] = dataclasses.field(default_factory=list) cve_id: Optional[str] = None def __hash__(self): From f5c100d29f1756d6e5fd98461afa8de728d49afd Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 16 Jul 2020 19:26:27 +0530 Subject: [PATCH 2/9] Refactor OvalDataSource to work with VulnerabilityReferenceUnit dataclass Signed-off-by: Shivam Sandbhor --- vulnerabilities/data_source.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index e73f5cb1c..c131e0c09 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -67,15 +67,14 @@ class Advisory: summary: str impacted_package_urls: Iterable[PackageURL] resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list) - vuln_references = Sequence[VulnerabilityReferenceUnit] = dataclasses.field(default_factory=list) + vuln_references : List[VulnerabilityReferenceUnit] = dataclasses.field(default_factory=list) cve_id: Optional[str] = None def __hash__(self): - s = '{}{}{}{}{}'.format( + s = '{}{}{}{}'.format( self.summary, ''.join(sorted([str(p) for p in self.impacted_package_urls])), ''.join(sorted([str(p) for p in self.resolved_package_urls])), - ''.join(sorted(self.reference_urls)), self.cve_id, ) return hash(s) @@ -481,8 +480,8 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis description = definition_data['description'] affected_purls = set() safe_purls = set() - urls = definition_data['reference_urls'] - + references = [VulnerabilityReferenceUnit(url=url) for url in definition_data['reference_urls']] + for test_data in definition_data['test_data']: for package in test_data['package_list']: pkg_name = package @@ -520,5 +519,5 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis impacted_package_urls=affected_purls, resolved_package_urls=safe_purls, cve_id=vuln_id, - reference_urls=urls)) + vuln_references=references)) return all_adv From 6eaf82cf5487f4c8cf46911714eb98eb6c022cc0 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 16 Jul 2020 19:26:56 +0530 Subject: [PATCH 3/9] Refactor importer_runner.py to work with VulnerabilityReferenceUnit dataclass Signed-off-by: Shivam Sandbhor --- vulnerabilities/import_runner.py | 78 ++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index c5331d4e0..13ee53709 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# Copyright (c) nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/vulnerablecode/ # The VulnerableCode software is licensed under the Apache License version 2.0. # Data generated with VulnerableCode require an acknowledgment. @@ -18,7 +18,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. No content created from # VulnerableCode should be considered or used as legal advice. Consult an Attorney # for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. import dataclasses @@ -165,37 +165,56 @@ def _process_added_advisories(data_source: DataSource) -> None: def _create_vulnerability_and_references(advisory: Advisory): vuln, vuln_created = _get_or_create_vulnerability(advisory) - vuln_references = set() + new_vuln_references = set() if vuln_created: # This means vulnerability didn't previously exist in the DB, so add # the references to bulk create queue without any hesitation - for id_ in advisory.reference_ids: - vuln_references.add(VulnerabilityReferenceInserter(vulnerability=vuln, reference_id=id_)) # nopep8 - - for url in advisory.reference_urls: - vuln_references.add(VulnerabilityReferenceInserter(vulnerability=vuln, url=url)) + for ref in advisory.vuln_references: + new_vuln_references.add(VulnerabilityReferenceInserter(vulnerability=vuln, **dataclasses.asdict(ref))) else: - vuln_refs_qs = models.VulnerabilityReference.objects.filter(vulnerability=vuln) - vuln_ids = {ref.reference_id for ref in vuln_refs_qs} - vuln_urls = {ref.url for ref in vuln_refs_qs} + # build indexes of existing references to either update or create new ones + # and avoid creating duplicates + + existing_vuln_references = models.VulnerabilityReference.objects.filter(vulnerability=vuln) + refs_by_id_url = {(ref.id, ref.url): ref for ref in existing_vuln_references if ref.id and ref.url} + refs_by_id = {ref.id: ref for ref in existing_vuln_references if ref.id} + refs_by_url = {ref.url: ref for ref in existing_vuln_references if ref.url} + + for adv_ref in advisory.vuln_references: + if adv_ref.reference_id and adv_ref.url: + existing_ref = refs_by_id_url.get((adv_ref.reference_id, adv_ref.url)) + + elif adv_ref.reference_id: + existing_ref = refs_by_id.get(adv_ref.reference_id) + + elif adv_ref.url: + existing_ref = refs_by_url.get(adv_ref.url) + + if existing_ref: + # update ref as needed + if not adv_ref.reference_id or not adv_ref.url: + continue - for id_ in advisory.reference_ids: - # Add the item preventing duplicates pass to through. - if id_ not in vuln_ids: - vuln_ids.add(id_) - vuln_references.add(VulnerabilityReferenceInserter( - vulnerability=vuln, reference_id=id_)) + if adv_ref.reference_id and not existing_ref.reference_id: + existing_ref.reference_id = adv_ref.reference_id - for url in advisory.reference_urls: - # Add the item preventing duplicates pass to through. - if url not in vuln_urls: - vuln_urls.add(url) - vuln_references.add(VulnerabilityReferenceInserter( - vulnerability=vuln, url=url)) + if adv_ref.url and not existing_ref.url: + existing_ref.url = adv_ref.url - return vuln, vuln_created, vuln_references + existing_ref.save() + + else: + new_vuln_references.add( + VulnerabilityReferenceInserter( + vulnerability=vuln, + reference_id=adv_ref.reference_id, + url=adv_ref.url, + ) + ) + + return vuln, vuln_created, new_vuln_references def _create_pkg_vuln_refs(vuln: models.Vulnerability, vuln_created: bool, purls: Sequence[PackageURL], is_vulnerable: bool): # nopep8 @@ -371,13 +390,12 @@ def _insert_vulnerabilities_and_references(batch: Set[Advisory]) -> Set[models.V vulnerabilities.add(vuln) - for id_ in advisory.reference_ids: - models.VulnerabilityReference.objects.get_or_create( - vulnerability=vuln, reference_id=id_) - - for url in advisory.reference_urls: + # TODO: Refactor this method to be atleast as efficient as + # one used by _process_updated_advisories + + for ref in advisory.vuln_references: models.VulnerabilityReference.objects.get_or_create( - vulnerability=vuln, url=url) + vulnerability=vuln, reference_id=ref.reference_id, url=ref.url) return vulnerabilities From eb4b1c4ea2a4553737cadff888d180ef538fe5b9 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 16 Jul 2020 19:28:14 +0530 Subject: [PATCH 4/9] Refactor all importers to work with VulnerabilityReferenceUnit dataclass Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/alpine_linux.py | 25 +++++++++++- vulnerabilities/importers/archlinux.py | 19 +++++++-- vulnerabilities/importers/debian.py | 10 +++-- vulnerabilities/importers/gentoo.py | 5 ++- vulnerabilities/importers/npm.py | 50 +++++++++++++---------- vulnerabilities/importers/openssl.py | 7 ++-- vulnerabilities/importers/redhat.py | 23 +++++++++-- vulnerabilities/importers/retiredotnet.py | 9 +++- vulnerabilities/importers/ruby.py | 9 +++- vulnerabilities/importers/rust.py | 18 ++++++-- vulnerabilities/importers/safety_db.py | 7 +++- vulnerabilities/importers/ubuntu_usn.py | 12 +++--- 12 files changed, 143 insertions(+), 51 deletions(-) diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 36b95359b..4e4225afe 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -34,6 +34,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource +from vulnerabilities.data_source import VulnerabilityReferenceUnit def validate_schema(advisory_dict): @@ -136,12 +137,34 @@ def _load_advisories( for vuln_ids in fixed_vulns: vuln_ids = vuln_ids.split() + references = [] + for reference_id in vuln_ids[1:]: + + if reference_id.startswith('XSA'): + xsa_id = reference_id.split('-')[-1] + references.append(VulnerabilityReferenceUnit( + reference_id=reference_id, + url='https://xenbits.xen.org/xsa/advisory-{}.html'.format(xsa_id) + )) + + elif reference_id.startswith('ZBX'): + references.append(VulnerabilityReferenceUnit( + reference_id=reference_id, + url='https://support.zabbix.com/browse/{}'.format(reference_id) + )) + + elif reference_id.startswith('wnpa-sec'): + references.append(VulnerabilityReferenceUnit( + reference_id=reference_id, + url='https://www.wireshark.org/security/{}.html'.format(reference_id) + )) + advisories.append(Advisory( summary='', impacted_package_urls=[], resolved_package_urls=resolved_purls, - reference_ids=vuln_ids[1:], + vuln_references=references, cve_id=vuln_ids[0] if vuln_ids[0] != 'CVE-????-?????' else None, )) diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index 37c80b658..acd26a2a9 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -31,7 +31,7 @@ from packageurl import PackageURL from schema import Regex, Schema, Or -from vulnerabilities.data_source import DataSource, DataSourceConfiguration, Advisory +from vulnerabilities.data_source import DataSource, DataSourceConfiguration, Advisory,VulnerabilityReferenceUnit def validate_schema(advisory_dict): @@ -101,14 +101,25 @@ def _parse(self, record) -> List[Advisory]: version=record['fixed'], )) - reference_urls = [f'https://security.archlinux.org/{a}' for a in record['advisories']] - + vuln_references = [] + vuln_references.append(VulnerabilityReferenceUnit( + reference_id=record['name'], + url='https://security.archlinux.org/{}'.format(record['name'])) + ) + + for ref in record['advisories']: + vuln_references.append(VulnerabilityReferenceUnit( + reference_id=ref, + url='https://security.archlinux.org/{}'.format(record['name']))) + print(vuln_references) + + advisories.append(Advisory( cve_id=cve_id, summary='', impacted_package_urls=impacted_purls, resolved_package_urls=resolved_purls, - reference_urls=reference_urls, + vuln_references=vuln_references )) return advisories diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index b9cd3d5ec..ea49ed212 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -37,6 +37,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration +from vulnerabilities.data_source import VulnerabilityReferenceUnit def validate_schema(advisory_dict): @@ -131,18 +132,21 @@ def _parse(self, pkg_name: str, records: Mapping[str, Any]) -> List[Advisory]: qualifiers={'distro': release_name}, )) - reference_urls = [] + references = [] debianbug = record.get('debianbug') if debianbug: bug_url = f'https://bugs.debian.org/cgi-bin/bugreport.cgi?bug={debianbug}' - reference_urls.append(bug_url) + references.append(VulnerabilityReferenceUnit( + url=bug_url, + reference_id=debianbug + )) advisories.append(Advisory( cve_id=cve_id, summary=record.get('description', ''), impacted_package_urls=impacted_purls, resolved_package_urls=resolved_purls, - reference_urls=reference_urls, + vuln_references=references, )) return advisories diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index 95255d34e..3aaa6a85b 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -28,6 +28,7 @@ from vulnerabilities.data_source import GitDataSource from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit class GentooDataSource(GitDataSource): @@ -51,6 +52,8 @@ def process_file(self, file): xml_data = {} xml_root = ET.parse(file).getroot() glsa = "GLSA-" + xml_root.attrib["id"] + vuln_reference = [VulnerabilityReferenceUnit(reference_id=glsa ,url="https://security.gentoo.org/glsa/{}".format(xml_root.attrib["id"]))] + for child in xml_root: if child.tag == "references": xml_data["cves"] = self.cves_from_reference(child) @@ -73,7 +76,7 @@ def process_file(self, file): summary=xml_data["description"], impacted_package_urls=xml_data["affected_purls"], resolved_package_urls=xml_data["unaffected_purls"], - reference_ids=[glsa], + vuln_references=vuln_reference ) advisory_list.append(advisory) return advisory_list diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index 2862ba64b..289bc92e6 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -37,6 +37,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource +from vulnerabilities.data_source import VulnerabilityReferenceUnit NPM_URL = 'https://registry.npmjs.org{}' @@ -67,29 +68,34 @@ def process_file(self, file) -> List[Advisory]: with open(file) as f: record = json.load(f) advisories = [] - package_name = record['module_name'] - all_versions = self.versions.get(package_name) - aff_range = record.get('vulnerable_versions', '') - fixed_range = record.get('patched_versions', '') - - impacted_versions, resolved_versions = categorize_versions( - all_versions, - aff_range, - fixed_range - ) - - impacted_purls = _versions_to_purls(package_name, impacted_versions) - resolved_purls = _versions_to_purls(package_name, resolved_versions) - - for cve_id in record.get('cves') or ['']: - advisories.append(Advisory( - summary=record.get('overview', ''), - cve_id=cve_id, - impacted_package_urls=impacted_purls, - resolved_package_urls=resolved_purls, - reference_urls=[NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}')], - )) + for record in records: + package_name = record['module_name'] + all_versions = self.versions.get(package_name) + aff_range = record.get('vulnerable_versions', '') + fixed_range = record.get('patched_versions', '') + + impacted_versions, resolved_versions = categorize_versions( + all_versions, + aff_range, + fixed_range + ) + + impacted_purls = _versions_to_purls(package_name, impacted_versions) + resolved_purls = _versions_to_purls(package_name, resolved_versions) + vuln_reference = [VulnerabilityReferenceUnit( + url=NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}'), + reference_id=record['id'] + )] + + for cve_id in record.get('cves') or ['']: + advisories.append(Advisory( + summary=record.get('overview', ''), + cve_id=cve_id, + impacted_package_urls=impacted_purls, + resolved_package_urls=resolved_purls, + vuln_references=vuln_reference, + )) return advisories diff --git a/vulnerabilities/importers/openssl.py b/vulnerabilities/importers/openssl.py index 778b747fc..b6fc6c220 100644 --- a/vulnerabilities/importers/openssl.py +++ b/vulnerabilities/importers/openssl.py @@ -28,6 +28,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource +from vulnerabilities.data_source import VulnerabilityReferenceUnit import xml.etree.ElementTree as ET @@ -69,8 +70,8 @@ def to_advisories(xml_response: str) -> Set[Advisory]: if info: commit_hash = info[0].attrib['hash'] - ref_urls.append("https://github.com/openssl/openssl/commit/" - + commit_hash) + ref_urls.append(VulnerabilityReferenceUnit(url="https://github.com/openssl/openssl/commit/" + + commit_hash)) if info.tag == 'description': # Description summary = re.sub(r'\s+', ' ', info.text).strip() @@ -88,7 +89,7 @@ def to_advisories(xml_response: str) -> Set[Advisory]: summary=summary, impacted_package_urls=vuln_purls, resolved_package_urls=safe_purls, - reference_urls=ref_urls) + vuln_references=ref_urls) advisories.append(advisory) return advisories diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index 98910582f..54e37493b 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -27,7 +27,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration - +from vulnerabilities.data_source import VulnerabilityReferenceUnit class RedhatDataSource(DataSource): CONFIG_CLASS = DataSourceConfiguration @@ -71,12 +71,29 @@ def to_advisory(advisory_data): if rpm_to_purl(rpm): affected_purls.append(rpm_to_purl(rpm)) + references = [] + if advisory_data.get('bugzilla'): + bugzilla = advisory_data.get('bugzilla') + references.append(VulnerabilityReferenceUnit( + url='https://bugzilla.redhat.com/show_bug.cgi?id={}'.format(bugzilla), + reference_id=bugzilla + )) + + for rhsa in advisory_data['advisories']: + references.append(VulnerabilityReferenceUnit( + url='https://access.redhat.com/errata/{}'.format(rhsa), + reference_id=rhsa, + )) + + references.append(VulnerabilityReferenceUnit( + url=advisory_data['resource_url'] + )) + return Advisory( summary=advisory_data['bugzilla_description'], cve_id=advisory_data['CVE'], - reference_ids=advisory_data['advisories'], impacted_package_urls=affected_purls, - reference_urls=[advisory_data['resource_url']], + vuln_references=references, ) diff --git a/vulnerabilities/importers/retiredotnet.py b/vulnerabilities/importers/retiredotnet.py index a01e1a0a5..9976718d3 100644 --- a/vulnerabilities/importers/retiredotnet.py +++ b/vulnerabilities/importers/retiredotnet.py @@ -30,6 +30,7 @@ from vulnerabilities.data_source import GitDataSource from vulnerabilities.data_source import GitDataSourceConfiguration from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit class RetireDotnetDataSource(GitDataSource): @@ -90,9 +91,13 @@ def process_file(self, path) -> List[Advisory]: version=pkg['fix'], type='nuget')) + vuln_reference = [VulnerabilityReferenceUnit( + url=json_doc['link'], + )] + return Advisory( summary=json_doc['description'], impacted_package_urls=affected_purls, resolved_package_urls=fixed_purls, - reference_urls=[json_doc['link']], - cve_id=vuln_id) + cve_id=vuln_id, + vuln_references=vuln_reference) diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index 86d01b9bc..ee4ca2b2a 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -32,6 +32,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource +from vulnerabilities.data_source import VulnerabilityReferenceUnit class RubyDataSource(GitDataSource): @@ -104,11 +105,17 @@ def process_file(self, path) -> List[Advisory]: version=version, ) for version in safe_versions} + references = [] + if record.get('url'): + references.append( + VulnerabilityReferenceUnit(url=record.get('url')) + ) + return Advisory( summary=record.get('description', ''), impacted_package_urls=impacted_purls, resolved_package_urls=resolved_purls, - reference_urls=[record.get('url', '')], + vuln_references=references, cve_id=cve_id ) diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index dc5ac2e42..40dedcee3 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -34,6 +34,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource +from vulnerabilities.data_source import VulnerabilityReferenceUnit class RustDataSource(GitDataSource): @@ -79,8 +80,13 @@ def _load_advisory(self, path: str) -> Optional[Advisory]: record = toml.load(f) advisory = record.get('advisory', {}) + references = [] crate_name = advisory['package'] - reference_url = advisory.get('url', '') + if advisory.get('url'): + references.append(VulnerabilityReferenceUnit( + url=advisory['url'] + )) + all_versions = self.crates_api.get(crate_name) affected_ranges = {RangeSpecifier(r) for r @@ -104,14 +110,20 @@ def _load_advisory(self, path: str) -> Optional[Advisory]: if alias.startswith('CVE-'): cve_id = alias break + + + references.append(VulnerabilityReferenceUnit( + reference_id=advisory['id'], + url='https://rustsec.org/advisories/{}.html'.format(advisory['id']) + ) + ) return Advisory( summary=advisory.get('description', ''), impacted_package_urls=impacted_purls, resolved_package_urls=resolved_purls, - reference_urls=[reference_url] if reference_url else [], - reference_ids=[advisory['id']], cve_id=cve_id, + vuln_references=references ) diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py index 736af4364..b4c7f7b31 100755 --- a/vulnerabilities/importers/safety_db.py +++ b/vulnerabilities/importers/safety_db.py @@ -42,6 +42,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration +from vulnerabilities.data_source import VulnerabilityReferenceUnit def validate_schema(advisory_dict): @@ -107,11 +108,15 @@ def updated_advisories(self) -> Set[Advisory]: if len(cve_ids[0]): cve_ids = [s.strip() for s in cve_ids.split(',')] + reference = [VulnerabilityReferenceUnit( + reference_id=advisory['id'] + )] + for cve_id in cve_ids: advisories.append(Advisory( cve_id=cve_id, summary=advisory['advisory'], - reference_ids=[advisory['id']], + vuln_references=reference, impacted_package_urls=impacted_purls, resolved_package_urls=resolved_purls, )) diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py index bc52a285e..6a2dadc5a 100644 --- a/vulnerabilities/importers/ubuntu_usn.py +++ b/vulnerabilities/importers/ubuntu_usn.py @@ -29,7 +29,7 @@ from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import Advisory - +from vulnerabilities.data_source import VulnerabilityReferenceUnit @dataclasses.dataclass class USNDBConfiguration: @@ -63,7 +63,7 @@ def create_etag(self, url): def to_advisories(usn_db): advisories = [] for usn in usn_db: - usnid_data = get_usn_references(usn_db[usn]['id']) + reference = get_usn_references(usn_db[usn]['id']) for release in usn_db[usn]['releases']: pkg_dict = usn_db[usn]['releases'][release] safe_purls = get_purls(pkg_dict) @@ -80,16 +80,14 @@ def to_advisories(usn_db): impacted_package_urls=[], resolved_package_urls=safe_purls, summary='', - reference_urls=usnid_data['reference_url'], - reference_ids=[usnid_data['reference_id']])) + vuln_references=[reference])) return advisories def get_usn_references(usn_id): - return {'reference_id': 'USN-' + usn_id, - 'reference_url': ['https://usn.ubuntu.com/{}/'.format(usn_id)] - } + return VulnerabilityReferenceUnit(reference_id='USN-' + usn_id, + url='https://usn.ubuntu.com/{}/'.format(usn_id)) def fetch(url): From d0e4b95d017d882008b0ea0219324585b26e8b8d Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 16 Jul 2020 19:28:35 +0530 Subject: [PATCH 5/9] Refactor all importer tests to work with VulnerabilityReferenceUnit dataclass Signed-off-by: Shivam Sandbhor --- vulnerabilities/tests/test_archlinux.py | 3 +-- vulnerabilities/tests/test_debian_oval.py | 4 --- vulnerabilities/tests/test_gentoo.py | 4 +-- vulnerabilities/tests/test_import_runner.py | 3 ++- vulnerabilities/tests/test_openssl.py | 19 ++++++------- vulnerabilities/tests/test_redhat_importer.py | 23 +++++----------- vulnerabilities/tests/test_retiredotnet.py | 4 +-- vulnerabilities/tests/test_ruby.py | 7 +++-- vulnerabilities/tests/test_safety_db.py | 1 + vulnerabilities/tests/test_suse_backports.py | 10 ------- vulnerabilities/tests/test_ubuntu.py | 27 ++++++++++--------- vulnerabilities/tests/test_ubuntu_usn.py | 22 ++++++++------- 12 files changed, 55 insertions(+), 72 deletions(-) diff --git a/vulnerabilities/tests/test_archlinux.py b/vulnerabilities/tests/test_archlinux.py index 7a271dd39..83148ab63 100644 --- a/vulnerabilities/tests/test_archlinux.py +++ b/vulnerabilities/tests/test_archlinux.py @@ -63,9 +63,8 @@ def test_import(self): return_value=self.mock_response ): runner.run() - assert models.Vulnerability.objects.count() == 6 - assert models.VulnerabilityReference.objects.count() == 4 + assert models.VulnerabilityReference.objects.count() == 10 assert models.PackageRelatedVulnerability.objects.filter( is_vulnerable=True).count() == 12 assert models.PackageRelatedVulnerability.objects.filter( diff --git a/vulnerabilities/tests/test_debian_oval.py b/vulnerabilities/tests/test_debian_oval.py index 51c5408ae..291da20e9 100644 --- a/vulnerabilities/tests/test_debian_oval.py +++ b/vulnerabilities/tests/test_debian_oval.py @@ -70,8 +70,6 @@ def test_get_data_from_xml_doc(self, mock_write): version='2.3.9', qualifiers=OrderedDict([('distro', 'wheezy')]), subpath=None)}, - reference_urls=set(), - reference_ids=[], cve_id='CVE-2002-2443' ), Advisory( @@ -98,8 +96,6 @@ def test_get_data_from_xml_doc(self, mock_write): version='0:1.11.1+dfsg-5+deb7u1', qualifiers=OrderedDict([('distro', 'wheezy')]), subpath=None)}, - reference_urls=set(), - reference_ids=[], cve_id='CVE-2001-1593') } diff --git a/vulnerabilities/tests/test_gentoo.py b/vulnerabilities/tests/test_gentoo.py index 8385f1750..12fec233b 100644 --- a/vulnerabilities/tests/test_gentoo.py +++ b/vulnerabilities/tests/test_gentoo.py @@ -30,6 +30,7 @@ from vulnerabilities.importers.gentoo import GentooDataSource from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/gentoo/glsa-201709-09.xml") @@ -107,8 +108,7 @@ def test_process_file(self): version='1.9.7', qualifiers=OrderedDict(), subpath=None)}, - reference_urls=[], - reference_ids=['GLSA-201709-09'], + vuln_references=[VulnerabilityReferenceUnit(url='https://security.gentoo.org/glsa/201709-09',reference_id='GLSA-201709-09')], cve_id='CVE-2017-9800')] found_data = self.data_src.process_file(TEST_DATA) diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index 2ab0dc46c..e8a7a54e7 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -28,6 +28,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import PackageURL +from vulnerabilities.data_source import VulnerabilityReferenceUnit from vulnerabilities.import_runner import ImportRunner from vulnerabilities.import_runner import _insert_vulnerabilities_and_references @@ -70,7 +71,7 @@ def save(self): Advisory( cve_id='MOCK-CVE-2020-1337', summary='vulnerability description here', - reference_urls=['https://example.com/with/more/info/MOCK-CVE-2020-1337'], + vuln_references=[VulnerabilityReferenceUnit(url='https://example.com/with/more/info/MOCK-CVE-2020-1337')], impacted_package_urls=[PackageURL(name='mock-webserver', type='pypi', version='1.2.33')], resolved_package_urls=[PackageURL(name='mock-webserver', type='pypi', version='1.2.34')], ) diff --git a/vulnerabilities/tests/test_openssl.py b/vulnerabilities/tests/test_openssl.py index 178794281..9e44da0f8 100644 --- a/vulnerabilities/tests/test_openssl.py +++ b/vulnerabilities/tests/test_openssl.py @@ -28,6 +28,7 @@ from vulnerabilities.importers.openssl import OpenSSLDataSource from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data/', 'openssl_xml_data.xml') @@ -86,10 +87,9 @@ def test_to_advisory(self): version='1.1.1g', qualifiers=OrderedDict(), subpath=None)}, - reference_urls=[ - 'https://github.com/openssl/openssl/commit/' + - 'eb563247aef3e83dda7679c43f9649270462e5b1'], - reference_ids=[], + vuln_references=[VulnerabilityReferenceUnit( + url='https://github.com/openssl/openssl/commit/' + + 'eb563247aef3e83dda7679c43f9649270462e5b1')], cve_id='CVE-2020-1967'), Advisory( summary='There is an overflow bug in the x64_64 Montgomery squaring procedure ' @@ -189,11 +189,12 @@ def test_to_advisory(self): qualifiers=OrderedDict(), subpath=None), PackageURL(type='generic', namespace=None, name='openssl', version='1.0.2u', qualifiers=OrderedDict(), subpath=None)}, - reference_urls=['https://github.com/openssl/openssl/commit/' + - '419102400a2811582a7a3d4a4e317d72e5ce0a8f', - 'https://github.com/openssl/openssl/commit/' + - 'f1c5eea8a817075d31e43f5876993c6710238c98'], - reference_ids=[], + vuln_references=[VulnerabilityReferenceUnit( + url='https://github.com/openssl/openssl/commit/' + + '419102400a2811582a7a3d4a4e317d72e5ce0a8f'), + VulnerabilityReferenceUnit( + url='https://github.com/openssl/openssl/commit/' + + 'f1c5eea8a817075d31e43f5876993c6710238c98')], cve_id='CVE-2019-1551') ] found_data = OpenSSLDataSource.to_advisories(data) diff --git a/vulnerabilities/tests/test_redhat_importer.py b/vulnerabilities/tests/test_redhat_importer.py index 22a8252fc..d9af07991 100644 --- a/vulnerabilities/tests/test_redhat_importer.py +++ b/vulnerabilities/tests/test_redhat_importer.py @@ -29,6 +29,7 @@ import vulnerabilities.importers.redhat as redhat from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data/', 'redhat.json') @@ -74,11 +75,7 @@ def test_to_advisory(self): qualifiers=OrderedDict(), subpath=None)], resolved_package_urls=[], - reference_urls=[ - 'https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-9401.json'], - reference_ids=[ - 'RHSA-2017:1931', - 'RHSA-2017:0725'], + vuln_references=sorted([VulnerabilityReferenceUnit(url='https://bugzilla.redhat.com/show_bug.cgi?id=1396383', reference_id='1396383'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:1931', reference_id='RHSA-2017:1931'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:0725', reference_id='RHSA-2017:0725'), VulnerabilityReferenceUnit(url='https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-9401.json', reference_id='')], key=lambda x: x.url), cve_id='CVE-2016-9401'), Advisory( summary=('CVE-2016-10200 kernel: l2tp: Race condition ' @@ -106,26 +103,20 @@ def test_to_advisory(self): qualifiers=OrderedDict(), subpath=None)], resolved_package_urls=[], - reference_urls=[ - 'https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-10200.json'], - reference_ids=[ - 'RHSA-2017:1842', - 'RHSA-2017:2437', - 'RHSA-2017:2077', - 'RHSA-2017:2444'], + vuln_references=sorted([VulnerabilityReferenceUnit(url='https://bugzilla.redhat.com/show_bug.cgi?id=1430347', reference_id='1430347'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:1842', reference_id='RHSA-2017:1842'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:2437', reference_id='RHSA-2017:2437'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:2077', reference_id='RHSA-2017:2077'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:2444', reference_id='RHSA-2017:2444'), VulnerabilityReferenceUnit(url='https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-10200.json', reference_id='')],key=lambda x:x.url), cve_id='CVE-2016-10200'), Advisory( summary=('CVE-2017-12168 Kernel: kvm: ARM64: ' 'assert failure when accessing PMCCNTR register'), impacted_package_urls=[], resolved_package_urls=[], - reference_urls=[ - 'https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2017-12168.json'], - reference_ids=[], + vuln_references=sorted([VulnerabilityReferenceUnit(url='https://bugzilla.redhat.com/show_bug.cgi?id=1492984', reference_id='1492984'), VulnerabilityReferenceUnit(url='https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2017-12168.json', reference_id='')], key=lambda x:x.url), cve_id='CVE-2017-12168'), } found_data = set() for adv in data: - found_data.add(redhat.to_advisory(adv)) + adv = redhat.to_advisory(adv) + adv.vuln_references = sorted(adv.vuln_references, key=lambda x : x.url) + found_data.add(adv) assert expected_data == found_data diff --git a/vulnerabilities/tests/test_retiredotnet.py b/vulnerabilities/tests/test_retiredotnet.py index 9ea38e297..bce9f617a 100644 --- a/vulnerabilities/tests/test_retiredotnet.py +++ b/vulnerabilities/tests/test_retiredotnet.py @@ -28,6 +28,7 @@ from vulnerabilities.importers.retiredotnet import RetireDotnetDataSource from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -114,8 +115,7 @@ def test_process_file(self): version='1.1.5', qualifiers=OrderedDict(), subpath=None)}, - reference_urls=['https://github.com/aspnet/Announcements/issues/359'], - reference_ids=[], + vuln_references=[VulnerabilityReferenceUnit(url='https://github.com/aspnet/Announcements/issues/359', reference_id='')], cve_id='CVE-2019-0982') found_data = self.data_src.process_file(path) diff --git a/vulnerabilities/tests/test_ruby.py b/vulnerabilities/tests/test_ruby.py index 8e3e98044..830c3b482 100644 --- a/vulnerabilities/tests/test_ruby.py +++ b/vulnerabilities/tests/test_ruby.py @@ -31,6 +31,7 @@ from vulnerabilities.importers.ruby import RubyDataSource from vulnerabilities.data_source import GitDataSourceConfiguration from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -82,8 +83,7 @@ def test_process_file(self, mock_write): version='2.0.3', qualifiers=OrderedDict(), subpath=None)}, - reference_urls=['https://github.com/sinatra/sinatra/pull/1379'], - reference_ids=[], + vuln_references=[VulnerabilityReferenceUnit(url='https://github.com/sinatra/sinatra/pull/1379')], cve_id='CVE-2018-7212'), Advisory( summary=('Sinatra before 2.0.2 has XSS via the 400 Bad Request ' @@ -111,8 +111,7 @@ def test_process_file(self, mock_write): version='2.0.3', qualifiers=OrderedDict(), subpath=None)}, - reference_urls=['https://github.com/sinatra/sinatra/issues/1428'], - reference_ids=[], + vuln_references=[VulnerabilityReferenceUnit(url='https://github.com/sinatra/sinatra/issues/1428')], cve_id='CVE-2018-11627'), None} diff --git a/vulnerabilities/tests/test_safety_db.py b/vulnerabilities/tests/test_safety_db.py index 110afca63..e019e7199 100644 --- a/vulnerabilities/tests/test_safety_db.py +++ b/vulnerabilities/tests/test_safety_db.py @@ -30,6 +30,7 @@ from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importers.safety_db import VersionAPI from vulnerabilities.importers.safety_db import categorize_versions +from vulnerabilities.data_source import VulnerabilityReferenceUnit BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data/') diff --git a/vulnerabilities/tests/test_suse_backports.py b/vulnerabilities/tests/test_suse_backports.py index 4a4e58d98..fa36c9534 100644 --- a/vulnerabilities/tests/test_suse_backports.py +++ b/vulnerabilities/tests/test_suse_backports.py @@ -65,8 +65,6 @@ def test_process_file(self): version='3.0.10-1.1.1', qualifiers=OrderedDict(), subpath=None)], - reference_urls=[], - reference_ids=[], cve_id='CVE-2009-1313'), Advisory( summary='', @@ -79,8 +77,6 @@ def test_process_file(self): version='3.5-1.1.5', qualifiers=OrderedDict(), subpath=None)], - reference_urls=[], - reference_ids=[], cve_id='CVE-2009-1313'), Advisory( summary='', @@ -93,8 +89,6 @@ def test_process_file(self): version='3.0.10-1.1.1', qualifiers=OrderedDict(), subpath=None)], - reference_urls=[], - reference_ids=[], cve_id='CVE-2009-1313'), Advisory( summary='', @@ -107,8 +101,6 @@ def test_process_file(self): version='0.7.0.r4359-15.9.2', qualifiers=OrderedDict(), subpath=None)], - reference_urls=[], - reference_ids=[], cve_id='CVE-2009-0365'), Advisory( summary='', @@ -121,8 +113,6 @@ def test_process_file(self): version='0.7.0.r4359-15.9.2', qualifiers=OrderedDict(), subpath=None)], - reference_urls=[], - reference_ids=[], cve_id='CVE-2009-0578'), ] diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 14aa69bc9..55b5582b9 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -12,6 +12,7 @@ from vulnerabilities.oval_parser import OvalParser from vulnerabilities.importers.ubuntu import UbuntuDataSource from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/") @@ -224,14 +225,13 @@ def test_get_data_from_xml_doc(self, mock_write): version='2.14-2', qualifiers=OrderedDict(), subpath=None)}, - reference_urls={ - 'http://www.openwall.com/lists/oss-security/2016/10/18/11', - 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860', - 'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html', - 'https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce', - 'https://blog.torproject.org/blog/tor-0289-released-important-fixes', - 'https://trac.torproject.org/projects/tor/ticket/20384'}, - reference_ids=[], + vuln_references=sorted([ + VulnerabilityReferenceUnit(url='http://www.openwall.com/lists/oss-security/2016/10/18/11'), + VulnerabilityReferenceUnit(url='https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860'), + VulnerabilityReferenceUnit(url='http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html'), + VulnerabilityReferenceUnit(url='https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce'), + VulnerabilityReferenceUnit(url='https://blog.torproject.org/blog/tor-0289-released-important-fixes'), + VulnerabilityReferenceUnit(url='https://trac.torproject.org/projects/tor/ticket/20384')],key=lambda x : x.url), cve_id='CVE-2016-8860'), Advisory( summary=('Heap-based buffer overflow in the bm_readbody_bmp function' @@ -262,11 +262,10 @@ def test_get_data_from_xml_doc(self, mock_write): version='2.14-2', qualifiers=OrderedDict(), subpath=None)}, - reference_urls={ - 'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html', - 'https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/', - 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703'}, - reference_ids=[], + vuln_references=sorted([ + VulnerabilityReferenceUnit(url='http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html'), + VulnerabilityReferenceUnit(url='https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/'), + VulnerabilityReferenceUnit(url='https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703')],key=lambda x: x.url), cve_id='CVE-2016-8703')} xml_doc = ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml")) @@ -274,6 +273,8 @@ def test_get_data_from_xml_doc(self, mock_write): with patch('vulnerabilities.importers.ubuntu.UbuntuDataSource.batch_advisories', new=return_adv): data = {i for i in self.ubuntu_data_src.get_data_from_xml_doc(xml_doc,{"type":"deb"})} + for adv in data : + adv.vuln_references = sorted(adv.vuln_references, key=lambda x : x.url) assert expected_data == data def test_create_etag(self): diff --git a/vulnerabilities/tests/test_ubuntu_usn.py b/vulnerabilities/tests/test_ubuntu_usn.py index 9a520a408..dc40fbf09 100644 --- a/vulnerabilities/tests/test_ubuntu_usn.py +++ b/vulnerabilities/tests/test_ubuntu_usn.py @@ -31,6 +31,7 @@ from packageurl import PackageURL from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit import vulnerabilities.importers.ubuntu_usn as ubuntu_usn BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -49,10 +50,10 @@ def setUpClass(cls): def test_get_usn_references(self): eg_usn = '435-1' - expected_references = { - 'reference_id': 'USN-435-1', - 'reference_url': ['https://usn.ubuntu.com/435-1/'], - } + expected_references = VulnerabilityReferenceUnit( + reference_id='USN-435-1', + url='https://usn.ubuntu.com/435-1/' + ) found_references = ubuntu_usn.get_usn_references(eg_usn) assert found_references == expected_references @@ -153,8 +154,9 @@ def test_to_advisories(self): subpath=None, ), }, - reference_urls=['https://usn.ubuntu.com/763-1/'], - reference_ids=['USN-763-1'], + vuln_references=[VulnerabilityReferenceUnit( + url='https://usn.ubuntu.com/763-1/', + reference_id='USN-763-1')], cve_id='CVE-2009-0698', ), Advisory( @@ -178,10 +180,12 @@ def test_to_advisories(self): subpath=None, ), }, - reference_urls=['https://usn.ubuntu.com/763-1/'], - reference_ids=['USN-763-1'], + vuln_references = [VulnerabilityReferenceUnit( + url='https://usn.ubuntu.com/763-1/', + reference_id='USN-763-1') + ], cve_id='CVE-2009-1274', - ), + ), } found_advisories = set(self.data_src.to_advisories(self.db)) From 747b9efc0ba1abfe3bd4678a7baf9c4c6fd27663 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 16 Jul 2020 19:43:11 +0530 Subject: [PATCH 6/9] Do the style changes to follow coding standards Signed-off-by: Shivam Sandbhor --- vulnerabilities/data_source.py | 14 +- vulnerabilities/import_runner.py | 22 ++- vulnerabilities/importers/alpine_linux.py | 105 ++++++------ vulnerabilities/importers/archlinux.py | 102 ++++++----- vulnerabilities/importers/gentoo.py | 16 +- vulnerabilities/importers/npm.py | 48 +++--- vulnerabilities/importers/openssl.py | 57 +++--- vulnerabilities/importers/redhat.py | 46 ++--- vulnerabilities/importers/rust.py | 79 +++++---- vulnerabilities/importers/safety_db.py | 68 ++++---- vulnerabilities/importers/ubuntu_usn.py | 53 +++--- vulnerabilities/tests/test_gentoo.py | 12 +- vulnerabilities/tests/test_import_runner.py | 19 +- vulnerabilities/tests/test_redhat_importer.py | 162 +++++++++++++----- vulnerabilities/tests/test_retiredotnet.py | 93 ++++++---- vulnerabilities/tests/test_ruby.py | 116 ++++++++----- vulnerabilities/tests/test_ubuntu_usn.py | 138 +++++++-------- 17 files changed, 645 insertions(+), 505 deletions(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index c131e0c09..3a4599b08 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -46,13 +46,14 @@ @dataclasses.dataclass class VulnerabilityReferenceUnit: - url : str = '' - reference_id : str = '' + url: str = '' + reference_id: str = '' - def __post_init__(self) : + def __post_init__(self): if not any([self.url, self.reference_id]): raise TypeError + @dataclasses.dataclass class Advisory: """ @@ -67,7 +68,7 @@ class Advisory: summary: str impacted_package_urls: Iterable[PackageURL] resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list) - vuln_references : List[VulnerabilityReferenceUnit] = dataclasses.field(default_factory=list) + vuln_references: List[VulnerabilityReferenceUnit] = dataclasses.field(default_factory=list) cve_id: Optional[str] = None def __hash__(self): @@ -480,8 +481,9 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis description = definition_data['description'] affected_purls = set() safe_purls = set() - references = [VulnerabilityReferenceUnit(url=url) for url in definition_data['reference_urls']] - + references = [VulnerabilityReferenceUnit(url=url) + for url in definition_data['reference_urls']] + for test_data in definition_data['test_data']: for package in test_data['package_list']: pkg_name = package diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 13ee53709..c58cdeee4 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -51,7 +51,7 @@ class VulnerabilityReferenceInserter: vulnerability: models.Vulnerability reference_id: Optional[str] = '' - url: Optional[str] = '' + url: Optional[str] = '' def __post_init__(self): if not any([self.reference_id, self.url]): @@ -171,14 +171,18 @@ def _create_vulnerability_and_references(advisory: Advisory): # This means vulnerability didn't previously exist in the DB, so add # the references to bulk create queue without any hesitation for ref in advisory.vuln_references: - new_vuln_references.add(VulnerabilityReferenceInserter(vulnerability=vuln, **dataclasses.asdict(ref))) + new_vuln_references.add( + VulnerabilityReferenceInserter( + vulnerability=vuln, + **dataclasses.asdict(ref))) else: # build indexes of existing references to either update or create new ones # and avoid creating duplicates existing_vuln_references = models.VulnerabilityReference.objects.filter(vulnerability=vuln) - refs_by_id_url = {(ref.id, ref.url): ref for ref in existing_vuln_references if ref.id and ref.url} + refs_by_id_url = { + (ref.id, ref.url): ref for ref in existing_vuln_references if ref.id and ref.url} refs_by_id = {ref.id: ref for ref in existing_vuln_references if ref.id} refs_by_url = {ref.url: ref for ref in existing_vuln_references if ref.url} @@ -191,7 +195,7 @@ def _create_vulnerability_and_references(advisory: Advisory): elif adv_ref.url: existing_ref = refs_by_url.get(adv_ref.url) - + if existing_ref: # update ref as needed if not adv_ref.reference_id or not adv_ref.url: @@ -208,8 +212,8 @@ def _create_vulnerability_and_references(advisory: Advisory): else: new_vuln_references.add( VulnerabilityReferenceInserter( - vulnerability=vuln, - reference_id=adv_ref.reference_id, + vulnerability=vuln, + reference_id=adv_ref.reference_id, url=adv_ref.url, ) ) @@ -222,7 +226,7 @@ def _create_pkg_vuln_refs(vuln: models.Vulnerability, vuln_created: bool, purls: for purl in purls: pkg, pkg_created = _get_or_create_package(purl) vuln_pkg_ref = PackageRelatedVulnerabilityInserter( - package=pkg, vulnerability=vuln, is_vulnerable=is_vulnerable) + package=pkg, vulnerability=vuln, is_vulnerable=is_vulnerable) if pkg_created or vuln_created: new_refs.add(vuln_pkg_ref) @@ -390,9 +394,9 @@ def _insert_vulnerabilities_and_references(batch: Set[Advisory]) -> Set[models.V vulnerabilities.add(vuln) - # TODO: Refactor this method to be atleast as efficient as + # TODO: Refactor this method to be atleast as efficient as # one used by _process_updated_advisories - + for ref in advisory.vuln_references: models.VulnerabilityReference.objects.get_or_create( vulnerability=vuln, reference_id=ref.reference_id, url=ref.url) diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 4e4225afe..845f3bedf 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -34,7 +34,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import VulnerabilityReferenceUnit def validate_schema(advisory_dict): @@ -54,14 +54,12 @@ def validate_schema(advisory_dict): Regex(r"XSA-\d{3}"), Regex(r"ZBX-\d{4}"), Regex(r"wnpa-sec-\d{4}-\d{2}"), - - ) ], "", # FIXME: Remove the None when below issue gets fixed # https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb/-/issues/1 - None + None, ), }, } @@ -73,13 +71,13 @@ def validate_schema(advisory_dict): class AlpineDataSource(GitDataSource): - def __enter__(self): super(AlpineDataSource, self).__enter__() - if not getattr(self, '_added_files', None): + if not getattr(self, "_added_files", None): self._added_files, self._updated_files = self.file_changes( - recursive=True, file_ext='yaml') + recursive=True, file_ext="yaml" + ) def updated_advisories(self) -> Set[Advisory]: files = self._updated_files.union(self._added_files) @@ -95,42 +93,36 @@ def _process_file(self, path) -> List[Advisory]: with open(path) as f: record = yaml.safe_load(f) - if record['packages'] is None: + if record["packages"] is None: return advisories validate_schema(record) - for p in record['packages']: - advisories.extend(self._load_advisories( - p['pkg'], - record['distroversion'], - record['reponame'], - record['archs'], - )) + for p in record["packages"]: + advisories.extend( + self._load_advisories( + p["pkg"], record["distroversion"], record["reponame"], record["archs"], + ) + ) return advisories def _load_advisories( - self, - pkg_infos: Mapping[str, Any], - distroversion: str, - reponame: str, - archs: Iterable[str], + self, pkg_infos: Mapping[str, Any], distroversion: str, reponame: str, archs: Iterable[str], ) -> List[Advisory]: advisories = [] - for version, fixed_vulns in pkg_infos['secfixes'].items(): + for version, fixed_vulns in pkg_infos["secfixes"].items(): if fixed_vulns is None: continue resolved_purls = { PackageURL( - name=pkg_infos['name'], - type='alpine', + name=pkg_infos["name"], + type="alpine", version=version, - qualifiers={ - 'arch': arch, 'distroversion': distroversion, 'reponame': reponame}, + qualifiers={"arch": arch, "distroversion": distroversion, "reponame": reponame}, ) for arch in archs } @@ -140,32 +132,41 @@ def _load_advisories( references = [] for reference_id in vuln_ids[1:]: - if reference_id.startswith('XSA'): - xsa_id = reference_id.split('-')[-1] - references.append(VulnerabilityReferenceUnit( - reference_id=reference_id, - url='https://xenbits.xen.org/xsa/advisory-{}.html'.format(xsa_id) - )) - - elif reference_id.startswith('ZBX'): - references.append(VulnerabilityReferenceUnit( - reference_id=reference_id, - url='https://support.zabbix.com/browse/{}'.format(reference_id) - )) - - elif reference_id.startswith('wnpa-sec'): - references.append(VulnerabilityReferenceUnit( - reference_id=reference_id, - url='https://www.wireshark.org/security/{}.html'.format(reference_id) - )) - - - advisories.append(Advisory( - summary='', - impacted_package_urls=[], - resolved_package_urls=resolved_purls, - vuln_references=references, - cve_id=vuln_ids[0] if vuln_ids[0] != 'CVE-????-?????' else None, - )) + if reference_id.startswith("XSA"): + xsa_id = reference_id.split("-")[-1] + references.append( + VulnerabilityReferenceUnit( + reference_id=reference_id, + url="https://xenbits.xen.org/xsa/advisory-{}.html".format(xsa_id), + ) + ) + + elif reference_id.startswith("ZBX"): + references.append( + VulnerabilityReferenceUnit( + reference_id=reference_id, + url="https://support.zabbix.com/browse/{}".format(reference_id), + ) + ) + + elif reference_id.startswith("wnpa-sec"): + references.append( + VulnerabilityReferenceUnit( + reference_id=reference_id, + url="https://www.wireshark.org/security/{}.html".format( + reference_id + ), + ) + ) + + advisories.append( + Advisory( + summary="", + impacted_package_urls=[], + resolved_package_urls=resolved_purls, + vuln_references=references, + cve_id=vuln_ids[0] if vuln_ids[0] != "CVE-????-?????" else None, + ) + ) return advisories diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index acd26a2a9..c19c6df62 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -31,23 +31,26 @@ from packageurl import PackageURL from schema import Regex, Schema, Or -from vulnerabilities.data_source import DataSource, DataSourceConfiguration, Advisory,VulnerabilityReferenceUnit +from vulnerabilities.data_source import ( + DataSource, + DataSourceConfiguration, + Advisory, + VulnerabilityReferenceUnit, +) def validate_schema(advisory_dict): scheme = { - - 'advisories': list, - 'affected': str, - 'fixed': Or(None, str), - 'issues': [Regex(r'CVE-\d+-\d+')], - 'name': str, - 'packages': [str], - 'status': str, - 'ticket': object, - 'type': str, - 'severity': str, - + "advisories": list, + "affected": str, + "fixed": Or(None, str), + "issues": [Regex(r"CVE-\d+-\d+")], + "name": str, + "packages": [str], + "status": str, + "ticket": object, + "type": str, + "severity": str, } Schema(scheme).validate(advisory_dict) @@ -83,43 +86,50 @@ def _fetch(self) -> Iterable[Mapping]: def _parse(self, record) -> List[Advisory]: advisories = [] - for cve_id in record['issues']: + for cve_id in record["issues"]: impacted_purls, resolved_purls = set(), set() - for name in record['packages']: - impacted_purls.add(PackageURL( - name=name, - type='pacman', - namespace='archlinux', - version=record['affected'], - )) - - if record['fixed']: - resolved_purls.add(PackageURL( - name=name, - type='pacman', - namespace='archlinux', - version=record['fixed'], - )) + for name in record["packages"]: + impacted_purls.add( + PackageURL( + name=name, type="pacman", namespace="archlinux", version=record["affected"], + ) + ) + + if record["fixed"]: + resolved_purls.add( + PackageURL( + name=name, + type="pacman", + namespace="archlinux", + version=record["fixed"], + ) + ) vuln_references = [] - vuln_references.append(VulnerabilityReferenceUnit( - reference_id=record['name'], - url='https://security.archlinux.org/{}'.format(record['name'])) + vuln_references.append( + VulnerabilityReferenceUnit( + reference_id=record["name"], + url="https://security.archlinux.org/{}".format(record["name"]), + ) + ) + + for ref in record["advisories"]: + vuln_references.append( + VulnerabilityReferenceUnit( + reference_id=ref, + url="https://security.archlinux.org/{}".format(record["name"]), + ) ) - - for ref in record['advisories']: - vuln_references.append(VulnerabilityReferenceUnit( - reference_id=ref, - url='https://security.archlinux.org/{}'.format(record['name']))) print(vuln_references) - - - advisories.append(Advisory( - cve_id=cve_id, - summary='', - impacted_package_urls=impacted_purls, - resolved_package_urls=resolved_purls, - vuln_references=vuln_references - )) + + advisories.append( + Advisory( + cve_id=cve_id, + summary="", + impacted_package_urls=impacted_purls, + resolved_package_urls=resolved_purls, + vuln_references=vuln_references, + ) + ) return advisories diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index 3aaa6a85b..1bc680804 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -52,7 +52,12 @@ def process_file(self, file): xml_data = {} xml_root = ET.parse(file).getroot() glsa = "GLSA-" + xml_root.attrib["id"] - vuln_reference = [VulnerabilityReferenceUnit(reference_id=glsa ,url="https://security.gentoo.org/glsa/{}".format(xml_root.attrib["id"]))] + vuln_reference = [ + VulnerabilityReferenceUnit( + reference_id=glsa, + url="https://security.gentoo.org/glsa/{}".format(xml_root.attrib["id"]), + ) + ] for child in xml_root: if child.tag == "references": @@ -76,7 +81,7 @@ def process_file(self, file): summary=xml_data["description"], impacted_package_urls=xml_data["affected_purls"], resolved_package_urls=xml_data["unaffected_purls"], - vuln_references=vuln_reference + vuln_references=vuln_reference, ) advisory_list.append(advisory) return advisory_list @@ -99,11 +104,8 @@ def affected_and_safe_purls(affected_elem): for pkg in affected_elem: for info in pkg: - pkg_ns, pkg_name, = pkg.attrib["name"].split('/') - purl = PackageURL( - type="ebuild", name=pkg_name, version=info.text, - namespace=pkg_ns - ) + pkg_ns, pkg_name, = pkg.attrib["name"].split("/") + purl = PackageURL(type="ebuild", name=pkg_name, version=info.text, namespace=pkg_ns) if info.attrib.get("range"): if len(info.attrib.get("range")) > 2: diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index 289bc92e6..c020c7980 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -70,44 +70,44 @@ def process_file(self, file) -> List[Advisory]: advisories = [] for record in records: - package_name = record['module_name'] + package_name = record["module_name"] all_versions = self.versions.get(package_name) - aff_range = record.get('vulnerable_versions', '') - fixed_range = record.get('patched_versions', '') + aff_range = record.get("vulnerable_versions", "") + fixed_range = record.get("patched_versions", "") impacted_versions, resolved_versions = categorize_versions( - all_versions, - aff_range, - fixed_range + all_versions, aff_range, fixed_range ) impacted_purls = _versions_to_purls(package_name, impacted_versions) resolved_purls = _versions_to_purls(package_name, resolved_versions) - vuln_reference = [VulnerabilityReferenceUnit( + vuln_reference = [ + VulnerabilityReferenceUnit( url=NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}'), - reference_id=record['id'] - )] - - for cve_id in record.get('cves') or ['']: - advisories.append(Advisory( - summary=record.get('overview', ''), - cve_id=cve_id, - impacted_package_urls=impacted_purls, - resolved_package_urls=resolved_purls, - vuln_references=vuln_reference, - )) + reference_id=record["id"], + ) + ] + + for cve_id in record.get("cves") or [""]: + advisories.append( + Advisory( + summary=record.get("overview", ""), + cve_id=cve_id, + impacted_package_urls=impacted_purls, + resolved_package_urls=resolved_purls, + vuln_references=vuln_reference, + ) + ) return advisories def _versions_to_purls(package_name, versions): - purls = {f'pkg:npm/{quote(package_name)}@{v}' for v in versions} + purls = {f"pkg:npm/{quote(package_name)}@{v}" for v in versions} return {PackageURL.from_string(s) for s in purls} def categorize_versions( - all_versions: Set[str], - aff_version_range: str, - fixed_version_range: str, + all_versions: Set[str], aff_version_range: str, fixed_version_range: str, ) -> Tuple[Set[str], Set[str]]: """ Seperate list of affected versions and unaffected versions from all versions @@ -147,9 +147,9 @@ def get(self, package_name: str) -> Set[str]: if package_name not in self.cache: releases = set() try: - with urlopen(f'https://registry.npmjs.org/{package_name}') as response: + with urlopen(f"https://registry.npmjs.org/{package_name}") as response: data = json.load(response) - releases = {v for v in data.get('versions', {})} + releases = {v for v in data.get("versions", {})} except HTTPError as e: if e.code == 404: # NPM registry has no data regarding this package, we skip these diff --git a/vulnerabilities/importers/openssl.py b/vulnerabilities/importers/openssl.py index b6fc6c220..a551027c5 100644 --- a/vulnerabilities/importers/openssl.py +++ b/vulnerabilities/importers/openssl.py @@ -50,46 +50,51 @@ def to_advisories(xml_response: str) -> Set[Advisory]: root = ET.fromstring(xml_response) for element in root: if element.tag == "issue": - cve_id = '' - summary = '' + cve_id = "" + summary = "" safe_pkg_versions = [] vuln_pkg_versions = [] ref_urls = [] for info in element: - if info.tag == 'cve': - cve_id = 'CVE-' + info.attrib.get('name') + if info.tag == "cve": + cve_id = "CVE-" + info.attrib.get("name") - if info.tag == 'affects': + if info.tag == "affects": # Vulnerable package versions - vuln_pkg_versions.append(info.attrib.get('version')) + vuln_pkg_versions.append(info.attrib.get("version")) - if info.tag == 'fixed': + if info.tag == "fixed": # Fixed package versions - safe_pkg_versions.append(info.attrib.get('version')) + safe_pkg_versions.append(info.attrib.get("version")) if info: - commit_hash = info[0].attrib['hash'] - ref_urls.append(VulnerabilityReferenceUnit(url="https://github.com/openssl/openssl/commit/" - + commit_hash)) - if info.tag == 'description': + commit_hash = info[0].attrib["hash"] + ref_urls.append( + VulnerabilityReferenceUnit( + url="https://github.com/openssl/openssl/commit/" + commit_hash + ) + ) + if info.tag == "description": # Description - summary = re.sub(r'\s+', ' ', info.text).strip() + summary = re.sub(r"\s+", " ", info.text).strip() - safe_purls = {PackageURL(name=pkg_name, - type=pkg_type, - version=version) - for version in safe_pkg_versions} - vuln_purls = {PackageURL(name=pkg_name, - type=pkg_type, - version=version) - for version in vuln_pkg_versions} + safe_purls = { + PackageURL(name=pkg_name, type=pkg_type, version=version) + for version in safe_pkg_versions + } + vuln_purls = { + PackageURL(name=pkg_name, type=pkg_type, version=version) + for version in vuln_pkg_versions + } - advisory = Advisory(cve_id=cve_id, - summary=summary, - impacted_package_urls=vuln_purls, - resolved_package_urls=safe_purls, - vuln_references=ref_urls) + advisory = Advisory( + cve_id=cve_id, + summary=summary, + impacted_package_urls=vuln_purls, + resolved_package_urls=safe_purls, + vuln_references=ref_urls, + ) advisories.append(advisory) return advisories diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index 54e37493b..72066ad64 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -29,6 +29,7 @@ from vulnerabilities.data_source import DataSourceConfiguration from vulnerabilities.data_source import VulnerabilityReferenceUnit + class RedhatDataSource(DataSource): CONFIG_CLASS = DataSourceConfiguration @@ -66,32 +67,33 @@ def fetch(): def to_advisory(advisory_data): affected_purls = [] - if advisory_data.get('affected_packages'): - for rpm in advisory_data['affected_packages']: + if advisory_data.get("affected_packages"): + for rpm in advisory_data["affected_packages"]: if rpm_to_purl(rpm): affected_purls.append(rpm_to_purl(rpm)) references = [] - if advisory_data.get('bugzilla'): - bugzilla = advisory_data.get('bugzilla') - references.append(VulnerabilityReferenceUnit( - url='https://bugzilla.redhat.com/show_bug.cgi?id={}'.format(bugzilla), - reference_id=bugzilla - )) - - for rhsa in advisory_data['advisories']: - references.append(VulnerabilityReferenceUnit( - url='https://access.redhat.com/errata/{}'.format(rhsa), - reference_id=rhsa, - )) - - references.append(VulnerabilityReferenceUnit( - url=advisory_data['resource_url'] - )) + if advisory_data.get("bugzilla"): + bugzilla = advisory_data.get("bugzilla") + references.append( + VulnerabilityReferenceUnit( + url="https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla), + reference_id=bugzilla, + ) + ) + + for rhsa in advisory_data["advisories"]: + references.append( + VulnerabilityReferenceUnit( + url="https://access.redhat.com/errata/{}".format(rhsa), reference_id=rhsa, + ) + ) + + references.append(VulnerabilityReferenceUnit(url=advisory_data["resource_url"])) return Advisory( - summary=advisory_data['bugzilla_description'], - cve_id=advisory_data['CVE'], + summary=advisory_data["bugzilla_description"], + cve_id=advisory_data["CVE"], impacted_package_urls=affected_purls, vuln_references=references, ) @@ -101,11 +103,11 @@ def rpm_to_purl(rpm_string): # Red Hat uses `-:0` instead of just `-` to separate # package name and version - components = rpm_string.split('-0:') + components = rpm_string.split("-0:") if len(components) != 2: return name, version = components if version[0].isdigit(): - return PackageURL(name=name, type='rpm', version=version, namespace="redhat") + return PackageURL(name=name, type="rpm", version=version, namespace="redhat") diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index 40dedcee3..6a88a6419 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -38,21 +38,20 @@ class RustDataSource(GitDataSource): - def __enter__(self): super(RustDataSource, self).__enter__() - if not getattr(self, '_added_files', None): + if not getattr(self, "_added_files", None): self._added_files, self._updated_files = self.file_changes( - subdir='crates', # TODO Consider importing the advisories for cargo, etc as well. + subdir="crates", # TODO Consider importing the advisories for cargo, etc as well. recursive=True, - file_ext='toml', + file_ext="toml", ) @property def crates_api(self): - if not hasattr(self, '_crates_api'): - setattr(self, '_crates_api', VersionAPI()) + if not hasattr(self, "_crates_api"): + setattr(self, "_crates_api", VersionAPI()) return self._crates_api def added_advisories(self) -> Set[Advisory]: @@ -62,10 +61,10 @@ def updated_advisories(self) -> Set[Advisory]: return self._load_advisories(self._updated_files) def _load_advisories(self, files) -> Set[Advisory]: - files = [f for f in files if not f.endswith('-0000.toml')] # skip temporary files + files = [f for f in files if not f.endswith("-0000.toml")] # skip temporary files while files: - batch, files = files[:self.batch_size], files[self.batch_size:] + batch, files = files[: self.batch_size], files[self.batch_size:] advisories = set() @@ -78,60 +77,60 @@ def _load_advisories(self, files) -> Set[Advisory]: def _load_advisory(self, path: str) -> Optional[Advisory]: with open(path) as f: record = toml.load(f) - advisory = record.get('advisory', {}) + advisory = record.get("advisory", {}) references = [] - crate_name = advisory['package'] - if advisory.get('url'): - references.append(VulnerabilityReferenceUnit( - url=advisory['url'] - )) + crate_name = advisory["package"] + if advisory.get("url"): + references.append(VulnerabilityReferenceUnit(url=advisory["url"])) all_versions = self.crates_api.get(crate_name) - affected_ranges = {RangeSpecifier(r) for r - in chain.from_iterable( - record.get('affected', {}).get('functions', {}).values())} + affected_ranges = { + RangeSpecifier(r) + for r in chain.from_iterable(record.get("affected", {}).get("functions", {}).values()) + } - unaffected_ranges = {RangeSpecifier(r) for r - in record.get('versions', {}).get('unaffected', [])} - resolved_ranges = {RangeSpecifier(r) for r - in record.get('versions', {}).get('patched', [])} + unaffected_ranges = { + RangeSpecifier(r) for r in record.get("versions", {}).get("unaffected", []) + } + resolved_ranges = {RangeSpecifier(r) for r in record.get("versions", {}).get("patched", [])} unaffected, affected = categorize_versions( - all_versions, unaffected_ranges, affected_ranges, resolved_ranges) + all_versions, unaffected_ranges, affected_ranges, resolved_ranges + ) - impacted_purls = {PackageURL(type='cargo', name=crate_name, version=v) for v in affected} - resolved_purls = {PackageURL(type='cargo', name=crate_name, version=v) for v in unaffected} + impacted_purls = {PackageURL(type="cargo", name=crate_name, version=v) for v in affected} + resolved_purls = {PackageURL(type="cargo", name=crate_name, version=v) for v in unaffected} cve_id = None - if 'aliases' in advisory: - for alias in advisory['aliases']: - if alias.startswith('CVE-'): + if "aliases" in advisory: + for alias in advisory["aliases"]: + if alias.startswith("CVE-"): cve_id = alias break - - references.append(VulnerabilityReferenceUnit( - reference_id=advisory['id'], - url='https://rustsec.org/advisories/{}.html'.format(advisory['id']) + references.append( + VulnerabilityReferenceUnit( + reference_id=advisory["id"], + url="https://rustsec.org/advisories/{}.html".format(advisory["id"]), ) ) return Advisory( - summary=advisory.get('description', ''), + summary=advisory.get("description", ""), impacted_package_urls=impacted_purls, resolved_package_urls=resolved_purls, cve_id=cve_id, - vuln_references=references + vuln_references=references, ) def categorize_versions( - all_versions: Set[str], - unaffected_versions: Set[RangeSpecifier], - affected_versions: Set[RangeSpecifier], - resolved_versions: Set[RangeSpecifier], + all_versions: Set[str], + unaffected_versions: Set[RangeSpecifier], + affected_versions: Set[RangeSpecifier], + resolved_versions: Set[RangeSpecifier], ) -> Tuple[Set[str], Set[str]]: """ Categorize all versions of a crate according to the given version ranges. @@ -174,10 +173,10 @@ def get(self, package_name: str) -> Set[str]: releases = set() try: - with urlopen(f'https://crates.io/api/v1/crates/{package_name}') as response: + with urlopen(f"https://crates.io/api/v1/crates/{package_name}") as response: response = json.load(response) - for version_info in response['versions']: - releases.add(version_info['num']) + for version_info in response["versions"]: + releases.add(version_info["num"]) except HTTPError as e: if e.code == 404: pass diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py index b4c7f7b31..8366dd042 100755 --- a/vulnerabilities/importers/safety_db.py +++ b/vulnerabilities/importers/safety_db.py @@ -48,16 +48,15 @@ def validate_schema(advisory_dict): scheme = { - str: - [ - { - "advisory": str, - "cve": Or(None, Regex(r"CVE-\d+-\d+")), - "id": Regex(r"^pyup.io-\d"), - "specs": list, - "v": str - } - ] + str: [ + { + "advisory": str, + "cve": Or(None, Regex(r"CVE-\d+-\d+")), + "id": Regex(r"^pyup.io-\d"), + "specs": list, + "v": str, + } + ] } Schema(scheme).validate(advisory_dict) @@ -100,34 +99,33 @@ def updated_advisories(self) -> Set[Advisory]: for advisory in self._api_response[package_name]: impacted_purls, resolved_purls = categorize_versions( - package_name, all_package_versions, advisory['specs']) + package_name, all_package_versions, advisory["specs"] + ) - cve_ids = advisory.get('cve') or [''] + cve_ids = advisory.get("cve") or [""] # meaning if cve_ids is not [''] but either ['CVE-123'] or ['CVE-123, CVE-124'] if len(cve_ids[0]): - cve_ids = [s.strip() for s in cve_ids.split(',')] + cve_ids = [s.strip() for s in cve_ids.split(",")] + + reference = [VulnerabilityReferenceUnit(reference_id=advisory["id"])] - reference = [VulnerabilityReferenceUnit( - reference_id=advisory['id'] - )] - for cve_id in cve_ids: - advisories.append(Advisory( - cve_id=cve_id, - summary=advisory['advisory'], - vuln_references=reference, - impacted_package_urls=impacted_purls, - resolved_package_urls=resolved_purls, - )) + advisories.append( + Advisory( + cve_id=cve_id, + summary=advisory["advisory"], + vuln_references=reference, + impacted_package_urls=impacted_purls, + resolved_package_urls=resolved_purls, + ) + ) return self.batch_advisories(advisories) def categorize_versions( - package_name: str, - all_versions: Set[str], - version_specs: Iterable[str], + package_name: str, all_versions: Set[str], version_specs: Iterable[str], ) -> Tuple[Set[PackageURL], Set[PackageURL]]: """ :return: impacted, resolved purls @@ -139,19 +137,11 @@ def categorize_versions( if any([version in r for r in ranges]): impacted_versions.add(version) - impacted_purls.add(PackageURL( - name=package_name, - type='pypi', - version=version, - )) + impacted_purls.add(PackageURL(name=package_name, type="pypi", version=version,)) resolved_purls = set() for version in all_versions - impacted_versions: - resolved_purls.add(PackageURL( - name=package_name, - type='pypi', - version=version - )) + resolved_purls.add(PackageURL(name=package_name, type="pypi", version=version)) return impacted_purls, resolved_purls @@ -166,9 +156,9 @@ def get(self, package_name: str) -> Set[str]: if package_name not in self.cache: releases = set() try: - with urlopen(f'https://pypi.org/pypi/{package_name}/json') as response: + with urlopen(f"https://pypi.org/pypi/{package_name}/json") as response: json_file = json.load(response) - releases = set(json_file['releases']) + releases = set(json_file["releases"]) except HTTPError as e: if e.code == 404: # PyPi does not have data about this package diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py index 6a2dadc5a..42405ca4d 100644 --- a/vulnerabilities/importers/ubuntu_usn.py +++ b/vulnerabilities/importers/ubuntu_usn.py @@ -31,6 +31,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import VulnerabilityReferenceUnit + @dataclasses.dataclass class USNDBConfiguration: etags: list @@ -48,7 +49,7 @@ def updated_advisories(self): return self.batch_advisories(advisories) def create_etag(self, url): - etag = requests.head(url).headers.get('etag') + etag = requests.head(url).headers.get("etag") if not etag: return True @@ -63,31 +64,35 @@ def create_etag(self, url): def to_advisories(usn_db): advisories = [] for usn in usn_db: - reference = get_usn_references(usn_db[usn]['id']) - for release in usn_db[usn]['releases']: - pkg_dict = usn_db[usn]['releases'][release] + reference = get_usn_references(usn_db[usn]["id"]) + for release in usn_db[usn]["releases"]: + pkg_dict = usn_db[usn]["releases"][release] safe_purls = get_purls(pkg_dict) - for cve in usn_db[usn].get('cves', ['']): + for cve in usn_db[usn].get("cves", [""]): # The db sometimes contains entries like # {'cves': ['python-pgsql vulnerabilities', 'CVE-2006-2313', 'CVE-2006-2314']} # This `if` filters entries like 'python-pgsql vulnerabilities' - if not cve.startswith('CVE-'): + if not cve.startswith("CVE-"): continue - advisories.append(Advisory( - cve_id=cve, - impacted_package_urls=[], - resolved_package_urls=safe_purls, - summary='', - vuln_references=[reference])) + advisories.append( + Advisory( + cve_id=cve, + impacted_package_urls=[], + resolved_package_urls=safe_purls, + summary="", + vuln_references=[reference], + ) + ) return advisories def get_usn_references(usn_id): - return VulnerabilityReferenceUnit(reference_id='USN-' + usn_id, - url='https://usn.ubuntu.com/{}/'.format(usn_id)) + return VulnerabilityReferenceUnit( + reference_id="USN-" + usn_id, url="https://usn.ubuntu.com/{}/".format(usn_id) + ) def fetch(url): @@ -99,29 +104,21 @@ def fetch(url): def get_purls(pkg_dict): purls = set() - for pkg_name in pkg_dict.get('sources', []): - version = pkg_dict['sources'][pkg_name]['version'] + for pkg_name in pkg_dict.get("sources", []): + version = pkg_dict["sources"][pkg_name]["version"] # The db sometimes contains entries like {'postgresql': {'version': ''}} # This `if` ignores such entries if not version: continue - purls.add(PackageURL(name=pkg_name, - version=version, - type='deb', - namespace='ubuntu', - )) + purls.add(PackageURL(name=pkg_name, version=version, type="deb", namespace="ubuntu",)) - for pkg_name in pkg_dict['binaries']: - version = pkg_dict['binaries'][pkg_name]['version'] + for pkg_name in pkg_dict["binaries"]: + version = pkg_dict["binaries"][pkg_name]["version"] # The db sometimes contains entries like {'postgresql': {'version': ''}} # This `if` ignores such entries if not version: continue - purls.add(PackageURL(name=pkg_name, - version=version, - type='deb', - namespace='ubuntu', - )) + purls.add(PackageURL(name=pkg_name, version=version, type="deb", namespace="ubuntu",)) return purls diff --git a/vulnerabilities/tests/test_gentoo.py b/vulnerabilities/tests/test_gentoo.py index 12fec233b..e3d508c2b 100644 --- a/vulnerabilities/tests/test_gentoo.py +++ b/vulnerabilities/tests/test_gentoo.py @@ -89,9 +89,10 @@ def test_process_file(self): exp_data = [ Advisory( - summary=('A command injection vulnerability in ' - 'Subversion may allow remote\n ' - 'attackers to execute arbitrary code.\n '), + summary=( + 'A command injection vulnerability in ' + 'Subversion may allow remote\n ' + 'attackers to execute arbitrary code.\n '), impacted_package_urls={ PackageURL( type='ebuild', @@ -108,7 +109,10 @@ def test_process_file(self): version='1.9.7', qualifiers=OrderedDict(), subpath=None)}, - vuln_references=[VulnerabilityReferenceUnit(url='https://security.gentoo.org/glsa/201709-09',reference_id='GLSA-201709-09')], + vuln_references=[ + VulnerabilityReferenceUnit( + url='https://security.gentoo.org/glsa/201709-09', + reference_id='GLSA-201709-09')], cve_id='CVE-2017-9800')] found_data = self.data_src.process_file(TEST_DATA) diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index e8a7a54e7..bdc4d5e5e 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -71,11 +71,20 @@ def save(self): Advisory( cve_id='MOCK-CVE-2020-1337', summary='vulnerability description here', - vuln_references=[VulnerabilityReferenceUnit(url='https://example.com/with/more/info/MOCK-CVE-2020-1337')], - impacted_package_urls=[PackageURL(name='mock-webserver', type='pypi', version='1.2.33')], - resolved_package_urls=[PackageURL(name='mock-webserver', type='pypi', version='1.2.34')], - ) -] + vuln_references=[ + VulnerabilityReferenceUnit( + url='https://example.com/with/more/info/MOCK-CVE-2020-1337')], + impacted_package_urls=[ + PackageURL( + name='mock-webserver', + type='pypi', + version='1.2.33')], + resolved_package_urls=[ + PackageURL( + name='mock-webserver', + type='pypi', + version='1.2.34')], + )] def make_import_runner(added_advs=None, updated_advs=None): diff --git a/vulnerabilities/tests/test_redhat_importer.py b/vulnerabilities/tests/test_redhat_importer.py index d9af07991..3b1e32b8c 100644 --- a/vulnerabilities/tests/test_redhat_importer.py +++ b/vulnerabilities/tests/test_redhat_importer.py @@ -32,7 +32,7 @@ from vulnerabilities.data_source import VulnerabilityReferenceUnit BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, 'test_data/', 'redhat.json') +TEST_DATA = os.path.join(BASE_DIR, "test_data/", "redhat.json") def load_test_data(): @@ -41,82 +41,156 @@ def load_test_data(): class TestRedhat(unittest.TestCase): - def test_rpm_to_purl(self): assert redhat.rpm_to_purl("foobar") is None assert redhat.rpm_to_purl("foo-bar-devel-0:sys76") is None assert redhat.rpm_to_purl("kernel-0:2.6.32-754.el6") == PackageURL( - type='rpm', - namespace='redhat', - name='kernel', - version='2.6.32-754.el6', + type="rpm", + namespace="redhat", + name="kernel", + version="2.6.32-754.el6", qualifiers=OrderedDict(), - subpath=None) + subpath=None, + ) def test_to_advisory(self): data = load_test_data() expected_data = { Advisory( - summary='CVE-2016-9401 bash: popd controlled free', + summary="CVE-2016-9401 bash: popd controlled free", impacted_package_urls=[ PackageURL( - type='rpm', - namespace='redhat', - name='bash', - version='4.2.46-28.el7', + type="rpm", + namespace="redhat", + name="bash", + version="4.2.46-28.el7", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='rpm', - namespace='redhat', - name='bash', - version='4.1.2-48.el6', + type="rpm", + namespace="redhat", + name="bash", + version="4.1.2-48.el6", qualifiers=OrderedDict(), - subpath=None)], + subpath=None, + ), + ], resolved_package_urls=[], - vuln_references=sorted([VulnerabilityReferenceUnit(url='https://bugzilla.redhat.com/show_bug.cgi?id=1396383', reference_id='1396383'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:1931', reference_id='RHSA-2017:1931'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:0725', reference_id='RHSA-2017:0725'), VulnerabilityReferenceUnit(url='https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-9401.json', reference_id='')], key=lambda x: x.url), - cve_id='CVE-2016-9401'), + vuln_references=sorted( + [ + VulnerabilityReferenceUnit( + url="https://bugzilla.redhat.com/show_bug.cgi?id=1396383", + reference_id="1396383", + ), + VulnerabilityReferenceUnit( + url="https://access.redhat.com/errata/RHSA-2017:1931", + reference_id="RHSA-2017:1931", + ), + VulnerabilityReferenceUnit( + url="https://access.redhat.com/errata/RHSA-2017:0725", + reference_id="RHSA-2017:0725", + ), + VulnerabilityReferenceUnit( + url="https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-9401.json", # nopep8 + reference_id="", + ), + ], + key=lambda x: x.url, + ), + cve_id="CVE-2016-9401", + ), Advisory( - summary=('CVE-2016-10200 kernel: l2tp: Race condition ' - 'in the L2TPv3 IP encapsulation feature'), + summary=( + "CVE-2016-10200 kernel: l2tp: Race condition " + "in the L2TPv3 IP encapsulation feature" + ), impacted_package_urls=[ PackageURL( - type='rpm', - namespace='redhat', - name='kernel-rt', - version='3.10.0-693.rt56.617.el7', + type="rpm", + namespace="redhat", + name="kernel-rt", + version="3.10.0-693.rt56.617.el7", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='rpm', - namespace='redhat', - name='kernel', - version='3.10.0-693.el7', + type="rpm", + namespace="redhat", + name="kernel", + version="3.10.0-693.el7", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='rpm', - namespace='redhat', - name='kernel', - version='3.10.0-514.28.1.el7', + type="rpm", + namespace="redhat", + name="kernel", + version="3.10.0-514.28.1.el7", qualifiers=OrderedDict(), - subpath=None)], + subpath=None, + ), + ], resolved_package_urls=[], - vuln_references=sorted([VulnerabilityReferenceUnit(url='https://bugzilla.redhat.com/show_bug.cgi?id=1430347', reference_id='1430347'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:1842', reference_id='RHSA-2017:1842'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:2437', reference_id='RHSA-2017:2437'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:2077', reference_id='RHSA-2017:2077'), VulnerabilityReferenceUnit(url='https://access.redhat.com/errata/RHSA-2017:2444', reference_id='RHSA-2017:2444'), VulnerabilityReferenceUnit(url='https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-10200.json', reference_id='')],key=lambda x:x.url), - cve_id='CVE-2016-10200'), + vuln_references=sorted( + [ + VulnerabilityReferenceUnit( + url="https://bugzilla.redhat.com/show_bug.cgi?id=1430347", + reference_id="1430347", + ), + VulnerabilityReferenceUnit( + url="https://access.redhat.com/errata/RHSA-2017:1842", + reference_id="RHSA-2017:1842", + ), + VulnerabilityReferenceUnit( + url="https://access.redhat.com/errata/RHSA-2017:2437", + reference_id="RHSA-2017:2437", + ), + VulnerabilityReferenceUnit( + url="https://access.redhat.com/errata/RHSA-2017:2077", + reference_id="RHSA-2017:2077", + ), + VulnerabilityReferenceUnit( + url="https://access.redhat.com/errata/RHSA-2017:2444", + reference_id="RHSA-2017:2444", + ), + VulnerabilityReferenceUnit( + url="https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-10200.json", # nopep8 + reference_id="", + ), + ], + key=lambda x: x.url, + ), + cve_id="CVE-2016-10200", + ), Advisory( - summary=('CVE-2017-12168 Kernel: kvm: ARM64: ' - 'assert failure when accessing PMCCNTR register'), + summary=( + "CVE-2017-12168 Kernel: kvm: ARM64: " + "assert failure when accessing PMCCNTR register" + ), impacted_package_urls=[], resolved_package_urls=[], - vuln_references=sorted([VulnerabilityReferenceUnit(url='https://bugzilla.redhat.com/show_bug.cgi?id=1492984', reference_id='1492984'), VulnerabilityReferenceUnit(url='https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2017-12168.json', reference_id='')], key=lambda x:x.url), - cve_id='CVE-2017-12168'), + vuln_references=sorted( + [ + VulnerabilityReferenceUnit( + url="https://bugzilla.redhat.com/show_bug.cgi?id=1492984", + reference_id="1492984", + ), + VulnerabilityReferenceUnit( + url="https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2017-12168.json", # nopep8 + reference_id="", + ), + ], + key=lambda x: x.url, + ), + cve_id="CVE-2017-12168", + ), } found_data = set() for adv in data: adv = redhat.to_advisory(adv) - adv.vuln_references = sorted(adv.vuln_references, key=lambda x : x.url) + adv.vuln_references = sorted(adv.vuln_references, key=lambda x: x.url) found_data.add(adv) assert expected_data == found_data diff --git a/vulnerabilities/tests/test_retiredotnet.py b/vulnerabilities/tests/test_retiredotnet.py index bce9f617a..dd22c8e7c 100644 --- a/vulnerabilities/tests/test_retiredotnet.py +++ b/vulnerabilities/tests/test_retiredotnet.py @@ -34,11 +34,11 @@ class TestRetireDotnetDataSource(TestCase): - @classmethod def setUpClass(cls): data_source_cfg = { - 'repository_url': 'https://test.net', } + "repository_url": "https://test.net", + } cls.data_src = RetireDotnetDataSource(1, config=data_source_cfg) def test_vuln_id_from_desc(self): @@ -55,68 +55,85 @@ def test_process_file(self): path = os.path.join(BASE_DIR, "test_data/retiredotnet/test_file.json") expected_data = Advisory( - summary=('Microsoft Security Advisory CVE-2019-0982: ' - 'ASP.NET Core Denial of Service Vulnerability'), + summary=( + "Microsoft Security Advisory CVE-2019-0982: " + "ASP.NET Core Denial of Service Vulnerability" + ), impacted_package_urls={ PackageURL( - type='nuget', + type="nuget", namespace=None, - name='Microsoft.AspNetCore.SignalR.Protocols.MessagePack', - version='1.0.4', + name="Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + version="1.0.4", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='nuget', + type="nuget", namespace=None, - name='Microsoft.AspNetCore.SignalR.Protocols.MessagePack', - version='1.0.0', + name="Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + version="1.0.0", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='nuget', + type="nuget", namespace=None, - name='Microsoft.AspNetCore.SignalR.Protocols.MessagePack', - version='1.0.2', + name="Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + version="1.0.2", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='nuget', + type="nuget", namespace=None, - name='Microsoft.AspNetCore.SignalR.Protocols.MessagePack', - version='1.0.3', + name="Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + version="1.0.3", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='nuget', + type="nuget", namespace=None, - name='Microsoft.AspNetCore.SignalR.Protocols.MessagePack', - version='1.1.0', + name="Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + version="1.1.0", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='nuget', + type="nuget", namespace=None, - name='Microsoft.AspNetCore.SignalR.Protocols.MessagePack', - version='1.0.1', + name="Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + version="1.0.1", qualifiers=OrderedDict(), - subpath=None)}, + subpath=None, + ), + }, resolved_package_urls={ PackageURL( - type='nuget', + type="nuget", namespace=None, - name='Microsoft.AspNetCore.SignalR.Protocols.MessagePack', - version='1.0.11', + name="Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + version="1.0.11", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='nuget', + type="nuget", namespace=None, - name='Microsoft.AspNetCore.SignalR.Protocols.MessagePack', - version='1.1.5', + name="Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + version="1.1.5", qualifiers=OrderedDict(), - subpath=None)}, - vuln_references=[VulnerabilityReferenceUnit(url='https://github.com/aspnet/Announcements/issues/359', reference_id='')], - cve_id='CVE-2019-0982') + subpath=None, + ), + }, + vuln_references=[ + VulnerabilityReferenceUnit( + url="https://github.com/aspnet/Announcements/issues/359", reference_id="" + ) + ], + cve_id="CVE-2019-0982", + ) found_data = self.data_src.process_file(path) diff --git a/vulnerabilities/tests/test_ruby.py b/vulnerabilities/tests/test_ruby.py index 830c3b482..8aa4bb48b 100644 --- a/vulnerabilities/tests/test_ruby.py +++ b/vulnerabilities/tests/test_ruby.py @@ -35,85 +35,108 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, 'test_data', 'ruby') +TEST_DATA = os.path.join(BASE_DIR, "test_data", "ruby") MOCK_ADDED_FILES = [] -for filepath in pathlib.Path(TEST_DATA).glob('**/*.yml'): +for filepath in pathlib.Path(TEST_DATA).glob("**/*.yml"): MOCK_ADDED_FILES.append(filepath.absolute()) class RubyDataSourceTest(TestCase): - @classmethod def setUpClass(cls): data_source_cfg = { - 'repository_url': 'https://github.com/rubysec/ruby-advisory-db.git', } + "repository_url": "https://github.com/rubysec/ruby-advisory-db.git", + } cls.data_src = RubyDataSource(1, config=data_source_cfg) - @patch('vulnerabilities.importers.ruby.rubyAPI.get_all_version_of_package', - return_value={'1.0.0', '1.8.0', '2.0.3'}) + @patch( + "vulnerabilities.importers.ruby.rubyAPI.get_all_version_of_package", + return_value={"1.0.0", "1.8.0", "2.0.3"}, + ) def test_process_file(self, mock_write): expected_advisories = { Advisory( - summary=('An issue was discovered in' - ' rack-protection/lib/rack/protection/path_traversal.rb\n' - 'in Sinatra 2.x before 2.0.1 on Windows.' - ' Path traversal is possible via backslash\ncharacters.\n'), + summary=( + "An issue was discovered in" + " rack-protection/lib/rack/protection/path_traversal.rb\n" + "in Sinatra 2.x before 2.0.1 on Windows." + " Path traversal is possible via backslash\ncharacters.\n" + ), impacted_package_urls={ PackageURL( - type='gem', + type="gem", namespace=None, - name='sinatra', - version='1.8.0', + name="sinatra", + version="1.8.0", qualifiers=OrderedDict(), - subpath=None)}, + subpath=None, + ) + }, resolved_package_urls={ PackageURL( - type='gem', + type="gem", namespace=None, - name='sinatra', - version='1.0.0', + name="sinatra", + version="1.0.0", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='gem', + type="gem", namespace=None, - name='sinatra', - version='2.0.3', + name="sinatra", + version="2.0.3", qualifiers=OrderedDict(), - subpath=None)}, - vuln_references=[VulnerabilityReferenceUnit(url='https://github.com/sinatra/sinatra/pull/1379')], - cve_id='CVE-2018-7212'), + subpath=None, + ), + }, + vuln_references=[ + VulnerabilityReferenceUnit(url="https://github.com/sinatra/sinatra/pull/1379") + ], + cve_id="CVE-2018-7212", + ), Advisory( - summary=('Sinatra before 2.0.2 has XSS via the 400 Bad Request ' - 'page that occurs upon a params parser exception.\n'), + summary=( + "Sinatra before 2.0.2 has XSS via the 400 Bad Request " + "page that occurs upon a params parser exception.\n" + ), impacted_package_urls={ PackageURL( - type='gem', + type="gem", namespace=None, - name='sinatra', - version='1.0.0', + name="sinatra", + version="1.0.0", qualifiers=OrderedDict(), - subpath=None), + subpath=None, + ), PackageURL( - type='gem', + type="gem", namespace=None, - name='sinatra', - version='1.8.0', + name="sinatra", + version="1.8.0", qualifiers=OrderedDict(), - subpath=None)}, + subpath=None, + ), + }, resolved_package_urls={ PackageURL( - type='gem', + type="gem", namespace=None, - name='sinatra', - version='2.0.3', + name="sinatra", + version="2.0.3", qualifiers=OrderedDict(), - subpath=None)}, - vuln_references=[VulnerabilityReferenceUnit(url='https://github.com/sinatra/sinatra/issues/1428')], - cve_id='CVE-2018-11627'), - None} + subpath=None, + ) + }, + vuln_references=[ + VulnerabilityReferenceUnit(url="https://github.com/sinatra/sinatra/issues/1428") + ], + cve_id="CVE-2018-11627", + ), + None, + } found_advisories = set() @@ -123,13 +146,12 @@ def test_process_file(self, mock_write): def test_categorize_versions(self): - all_versions = {'1.0.0', '1.2.0', '9.0.2', '0.2.3'} - safe_ver_ranges = ['==1.0.0', '>1.2.0'] + all_versions = {"1.0.0", "1.2.0", "9.0.2", "0.2.3"} + safe_ver_ranges = ["==1.0.0", ">1.2.0"] - exp_safe_vers = {'1.0.0', '9.0.2'} - exp_aff_vers = {'1.2.0', '0.2.3'} + exp_safe_vers = {"1.0.0", "9.0.2"} + exp_aff_vers = {"1.2.0", "0.2.3"} - safe_vers, aff_vers = self.data_src.categorize_versions( - all_versions, safe_ver_ranges) + safe_vers, aff_vers = self.data_src.categorize_versions(all_versions, safe_ver_ranges) assert exp_aff_vers == aff_vers assert exp_safe_vers == safe_vers diff --git a/vulnerabilities/tests/test_ubuntu_usn.py b/vulnerabilities/tests/test_ubuntu_usn.py index dc40fbf09..5e3adf0b4 100644 --- a/vulnerabilities/tests/test_ubuntu_usn.py +++ b/vulnerabilities/tests/test_ubuntu_usn.py @@ -35,24 +35,23 @@ import vulnerabilities.importers.ubuntu_usn as ubuntu_usn BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, 'test_data/', 'ubuntu_usn_db', 'database-all.json.bz2') +TEST_DATA = os.path.join(BASE_DIR, "test_data/", "ubuntu_usn_db", "database-all.json.bz2") class TestUbuntuUSNDataSource(TestCase): @classmethod def setUpClass(cls): - data_src_cfg = {'etags': {}, 'db_url': 'http://exampledb.com'} + data_src_cfg = {"etags": {}, "db_url": "http://exampledb.com"} cls.data_src = ubuntu_usn.UbuntuUSNDataSource(batch_size=1, config=data_src_cfg) - with open(TEST_DATA, 'rb') as f: + with open(TEST_DATA, "rb") as f: cls.raw_data = f.read() cls.db = json.loads(bz2.decompress(cls.raw_data)) def test_get_usn_references(self): - eg_usn = '435-1' + eg_usn = "435-1" expected_references = VulnerabilityReferenceUnit( - reference_id='USN-435-1', - url='https://usn.ubuntu.com/435-1/' + reference_id="USN-435-1", url="https://usn.ubuntu.com/435-1/" ) found_references = ubuntu_usn.get_usn_references(eg_usn) @@ -62,65 +61,65 @@ def test_fetch(self): mock_response = MagicMock() mock_response.content = self.raw_data - with patch('vulnerabilities.importers.ubuntu_usn.requests.get', return_value=mock_response): - assert ubuntu_usn.fetch('www.db.com') == self.db + with patch("vulnerabilities.importers.ubuntu_usn.requests.get", return_value=mock_response): + assert ubuntu_usn.fetch("www.db.com") == self.db def test_get_purls(self): - eg_pkg_dict_1 = self.db['763-1']['releases']['hardy'] - eg_pkg_dict_2 = self.db['763-1']['releases']['dapper'] - eg_pkg_dict_3 = self.db['763-1']['releases']['intrepid'] + eg_pkg_dict_1 = self.db["763-1"]["releases"]["hardy"] + eg_pkg_dict_2 = self.db["763-1"]["releases"]["dapper"] + eg_pkg_dict_3 = self.db["763-1"]["releases"]["intrepid"] exp_pkgs_1 = { PackageURL( - type='deb', - namespace='ubuntu', - name='xine-lib', - version='1.1.11.1-1ubuntu3.4', + type="deb", + namespace="ubuntu", + name="xine-lib", + version="1.1.11.1-1ubuntu3.4", qualifiers=OrderedDict(), subpath=None, ), PackageURL( - type='deb', - namespace='ubuntu', - name='libxine1', - version='1.1.11.1-1ubuntu3.4', + type="deb", + namespace="ubuntu", + name="libxine1", + version="1.1.11.1-1ubuntu3.4", qualifiers=OrderedDict(), subpath=None, ), } exp_pkgs_2 = { PackageURL( - type='deb', - namespace='ubuntu', - name='libxine-main1', - version='1.1.1+ubuntu2-7.12', + type="deb", + namespace="ubuntu", + name="libxine-main1", + version="1.1.1+ubuntu2-7.12", qualifiers=OrderedDict(), subpath=None, ), PackageURL( - type='deb', - namespace='ubuntu', - name='xine-lib', - version='1.1.1+ubuntu2-7.12', + type="deb", + namespace="ubuntu", + name="xine-lib", + version="1.1.1+ubuntu2-7.12", qualifiers=OrderedDict(), subpath=None, ), } exp_pkgs_3 = { PackageURL( - type='deb', - namespace='ubuntu', - name='xine-lib', - version='1.1.15-0ubuntu3.3', + type="deb", + namespace="ubuntu", + name="xine-lib", + version="1.1.15-0ubuntu3.3", qualifiers=OrderedDict(), subpath=None, ), PackageURL( - type='deb', - namespace='ubuntu', - name='libxine1', - version='1.1.15-0ubuntu3.3', + type="deb", + namespace="ubuntu", + name="libxine1", + version="1.1.15-0ubuntu3.3", qualifiers=OrderedDict(), subpath=None, ), @@ -134,58 +133,61 @@ def test_to_advisories(self): expected_advisories = { Advisory( - summary='', + summary="", impacted_package_urls=[], resolved_package_urls={ PackageURL( - type='deb', - namespace='ubuntu', - name='xine-lib', - version='1.1.15-0ubuntu3.3', + type="deb", + namespace="ubuntu", + name="xine-lib", + version="1.1.15-0ubuntu3.3", qualifiers=OrderedDict(), subpath=None, ), PackageURL( - type='deb', - namespace='ubuntu', - name='libxine1', - version='1.1.15-0ubuntu3.3', + type="deb", + namespace="ubuntu", + name="libxine1", + version="1.1.15-0ubuntu3.3", qualifiers=OrderedDict(), subpath=None, ), }, - vuln_references=[VulnerabilityReferenceUnit( - url='https://usn.ubuntu.com/763-1/', - reference_id='USN-763-1')], - cve_id='CVE-2009-0698', + vuln_references=[ + VulnerabilityReferenceUnit( + url="https://usn.ubuntu.com/763-1/", reference_id="USN-763-1" + ) + ], + cve_id="CVE-2009-0698", ), Advisory( - summary='', + summary="", impacted_package_urls=[], resolved_package_urls={ PackageURL( - type='deb', - namespace='ubuntu', - name='xine-lib', - version='1.1.15-0ubuntu3.3', + type="deb", + namespace="ubuntu", + name="xine-lib", + version="1.1.15-0ubuntu3.3", qualifiers=OrderedDict(), subpath=None, ), PackageURL( - type='deb', - namespace='ubuntu', - name='libxine1', - version='1.1.15-0ubuntu3.3', + type="deb", + namespace="ubuntu", + name="libxine1", + version="1.1.15-0ubuntu3.3", qualifiers=OrderedDict(), subpath=None, ), }, - vuln_references = [VulnerabilityReferenceUnit( - url='https://usn.ubuntu.com/763-1/', - reference_id='USN-763-1') - ], - cve_id='CVE-2009-1274', - ), + vuln_references=[ + VulnerabilityReferenceUnit( + url="https://usn.ubuntu.com/763-1/", reference_id="USN-763-1" + ) + ], + cve_id="CVE-2009-1274", + ), } found_advisories = set(self.data_src.to_advisories(self.db)) @@ -195,9 +197,9 @@ def test_create_etag(self): assert self.data_src.config.etags == {} mock_response = MagicMock() - mock_response.headers = {'etag': '2131151243&2191'} + mock_response.headers = {"etag": "2131151243&2191"} - with patch('vulnerabilities.importers.ubuntu.requests.head', return_value=mock_response): - assert self.data_src.create_etag('https://example.org') - assert self.data_src.config.etags == {'https://example.org': '2131151243&2191'} - assert not self.data_src.create_etag('https://example.org') + with patch("vulnerabilities.importers.ubuntu.requests.head", return_value=mock_response): + assert self.data_src.create_etag("https://example.org") + assert self.data_src.config.etags == {"https://example.org": "2131151243&2191"} + assert not self.data_src.create_etag("https://example.org") From c53176670eab18268a9938f6dff359366287d1cd Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 23 Jul 2020 11:55:30 +0530 Subject: [PATCH 7/9] Refactor github api importer Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/github.py | 25 +++++++++++++++------- vulnerabilities/tests/test_github.py | 31 +++++++++++++++++----------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index 92e0063b9..2ca5c1943 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -38,6 +38,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration +from vulnerabilities.data_source import VulnerabilityReferenceUnit # set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET'} @@ -167,24 +168,33 @@ def process_response(self) -> List[Advisory]: ) affected_purls = { - PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type) + PackageURL(name=pkg_name, namespace=ns, + version=version, type=pkg_type) for version in aff_vers } unaffected_purls = { - PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type) + PackageURL(name=pkg_name, namespace=ns, + version=version, type=pkg_type) for version in unaff_vers } cve_ids = set() - ref_ids = set() + vuln_references = [] vuln_desc = adv["node"]["advisory"]["summary"] for vuln in adv["node"]["advisory"]["identifiers"]: if vuln["type"] == "CVE": cve_ids.add(vuln["value"]) - else: - ref_ids.add(vuln["value"]) + + elif vuln["type"] == "GHSA": + ghsa = vuln['value'] + vuln_references.append(VulnerabilityReferenceUnit( + reference_id=ghsa, + url="https://github.com/advisories/{}".format( + ghsa) + )) + for cve_id in cve_ids: adv_list.append( Advisory( @@ -192,7 +202,7 @@ def process_response(self) -> List[Advisory]: summary=vuln_desc, impacted_package_urls=affected_purls, resolved_package_urls=unaffected_purls, - reference_ids=ref_ids, + vuln_references=vuln_references, ) ) return adv_list @@ -200,7 +210,8 @@ def process_response(self) -> List[Advisory]: @staticmethod def categorize_versions(version_range: str, all_versions: Set[str]) -> Tuple[Set[str], Set[str]]: # nopep8 version_range = RangeSpecifier(version_range) - affected_versions = {version for version in all_versions if version in version_range} + affected_versions = { + version for version in all_versions if version in version_range} return (affected_versions, all_versions - affected_versions) diff --git a/vulnerabilities/tests/test_github.py b/vulnerabilities/tests/test_github.py index eb65c359d..bbbbb15d7 100644 --- a/vulnerabilities/tests/test_github.py +++ b/vulnerabilities/tests/test_github.py @@ -32,13 +32,14 @@ from requests.models import Response from packageurl import PackageURL +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilityReferenceUnit from vulnerabilities.importers.github import GitHubAPIDataSource from vulnerabilities.importers.github import MavenVersionAPI from vulnerabilities.importers.github import ComposerVersionAPI from vulnerabilities.importers.github import NugetVersionAPI from vulnerabilities.importers.github import GitHubTokenError from vulnerabilities.importers.github import query -from vulnerabilities.data_source import Advisory BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data') @@ -186,8 +187,9 @@ def test_process_response(self): subpath=None, ), }, - reference_urls=[], - reference_ids={'GHSA-qcxh-w3j9-58qr'}, + vuln_references=[VulnerabilityReferenceUnit( + url='https://github.com/advisories/GHSA-qcxh-w3j9-58qr', + reference_id='GHSA-qcxh-w3j9-58qr')], cve_id='CVE-2019-0199', ), Advisory( @@ -212,8 +214,9 @@ def test_process_response(self): subpath=None, ) }, - reference_urls=[], - reference_ids={'GHSA-qcxh-w3j9-58qr'}, + vuln_references=[VulnerabilityReferenceUnit( + url='https://github.com/advisories/GHSA-qcxh-w3j9-58qr', + reference_id='GHSA-qcxh-w3j9-58qr')], cve_id='CVE-2019-0199', ), Advisory( @@ -237,8 +240,9 @@ def test_process_response(self): subpath=None, ), }, - reference_urls=[], - reference_ids={'GHSA-c9hw-wf7x-jp9j'}, + vuln_references=[VulnerabilityReferenceUnit( + url='https://github.com/advisories/GHSA-c9hw-wf7x-jp9j', + reference_id='GHSA-c9hw-wf7x-jp9j')], cve_id='CVE-2020-1938', ), Advisory( @@ -262,8 +266,9 @@ def test_process_response(self): subpath=None, ), }, - reference_urls=[], - reference_ids={'GHSA-c9hw-wf7x-jp9j'}, + vuln_references=[VulnerabilityReferenceUnit( + url='https://github.com/advisories/GHSA-c9hw-wf7x-jp9j', + reference_id='GHSA-c9hw-wf7x-jp9j')], cve_id='CVE-2020-1938', ), Advisory( @@ -288,8 +293,9 @@ def test_process_response(self): subpath=None, ) }, - reference_urls=[], - reference_ids={'GHSA-c9hw-wf7x-jp9j'}, + vuln_references=[VulnerabilityReferenceUnit( + url='https://github.com/advisories/GHSA-c9hw-wf7x-jp9j', + reference_id='GHSA-c9hw-wf7x-jp9j')], cve_id='CVE-2020-1938', ), ] @@ -433,7 +439,8 @@ def test_artifact_url(self): def test_extract_versions(self): expected_versions = {'1.2.2', '1.2.3', '1.3.0'} - assert expected_versions == self.version_api.extract_versions(self.response) + assert expected_versions == self.version_api.extract_versions( + self.response) def test_load_to_api(self): From a195a5dd952527b5f42d3688f8cf92e216c30f41 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 23 Jul 2020 14:52:07 +0530 Subject: [PATCH 8/9] Fix wrong ref urls in archlinux importer Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/archlinux.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index c19c6df62..53d8c1482 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# Copyright (c) nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/vulnerablecode/ # The VulnerableCode software is licensed under the Apache License version 2.0. # Data generated with VulnerableCode require an acknowledgment. @@ -18,7 +18,7 @@ # OR CONDITIONS OF ANY KIND, either express or implied. No content created from # VulnerableCode should be considered or used as legal advice. Consult an Attorney # for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. import dataclasses import json @@ -31,12 +31,10 @@ from packageurl import PackageURL from schema import Regex, Schema, Or -from vulnerabilities.data_source import ( - DataSource, - DataSourceConfiguration, - Advisory, - VulnerabilityReferenceUnit, -) +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import DataSource +from vulnerabilities.data_source import DataSourceConfiguration +from vulnerabilities.data_source import VulnerabilityReferenceUnit def validate_schema(advisory_dict): @@ -117,10 +115,9 @@ def _parse(self, record) -> List[Advisory]: vuln_references.append( VulnerabilityReferenceUnit( reference_id=ref, - url="https://security.archlinux.org/{}".format(record["name"]), + url="https://security.archlinux.org/{}".format(ref), ) ) - print(vuln_references) advisories.append( Advisory( From 52c7e89200bdd397351d4a3cc8449300269a55a9 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sun, 26 Jul 2020 10:22:23 +0530 Subject: [PATCH 9/9] Change VulnerabilityReferenceUnit to Reference Signed-off-by: Shivam Sandbhor --- vulnerabilities/data_source.py | 6 +- vulnerabilities/importers/alpine_linux.py | 8 +-- vulnerabilities/importers/archlinux.py | 6 +- vulnerabilities/importers/debian.py | 4 +- vulnerabilities/importers/gentoo.py | 4 +- vulnerabilities/importers/github.py | 4 +- vulnerabilities/importers/npm.py | 56 +++++++++---------- vulnerabilities/importers/openssl.py | 4 +- vulnerabilities/importers/redhat.py | 8 +-- vulnerabilities/importers/retiredotnet.py | 4 +- vulnerabilities/importers/ruby.py | 4 +- vulnerabilities/importers/rust.py | 6 +- vulnerabilities/importers/safety_db.py | 4 +- vulnerabilities/importers/ubuntu_usn.py | 4 +- vulnerabilities/tests/test_gentoo.py | 4 +- vulnerabilities/tests/test_github.py | 12 ++-- vulnerabilities/tests/test_import_runner.py | 4 +- vulnerabilities/tests/test_openssl.py | 8 +-- vulnerabilities/tests/test_redhat_importer.py | 26 ++++----- vulnerabilities/tests/test_retiredotnet.py | 4 +- vulnerabilities/tests/test_ruby.py | 6 +- vulnerabilities/tests/test_safety_db.py | 2 +- vulnerabilities/tests/test_ubuntu.py | 20 +++---- vulnerabilities/tests/test_ubuntu_usn.py | 8 +-- 24 files changed, 107 insertions(+), 109 deletions(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 3a4599b08..091b72394 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -44,7 +44,7 @@ @dataclasses.dataclass -class VulnerabilityReferenceUnit: +class Reference: url: str = '' reference_id: str = '' @@ -68,7 +68,7 @@ class Advisory: summary: str impacted_package_urls: Iterable[PackageURL] resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list) - vuln_references: List[VulnerabilityReferenceUnit] = dataclasses.field(default_factory=list) + vuln_references: List[Reference] = dataclasses.field(default_factory=list) cve_id: Optional[str] = None def __hash__(self): @@ -481,7 +481,7 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis description = definition_data['description'] affected_purls = set() safe_purls = set() - references = [VulnerabilityReferenceUnit(url=url) + references = [Reference(url=url) for url in definition_data['reference_urls']] for test_data in definition_data['test_data']: diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 845f3bedf..78085db26 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -34,7 +34,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference def validate_schema(advisory_dict): @@ -135,7 +135,7 @@ def _load_advisories( if reference_id.startswith("XSA"): xsa_id = reference_id.split("-")[-1] references.append( - VulnerabilityReferenceUnit( + Reference( reference_id=reference_id, url="https://xenbits.xen.org/xsa/advisory-{}.html".format(xsa_id), ) @@ -143,7 +143,7 @@ def _load_advisories( elif reference_id.startswith("ZBX"): references.append( - VulnerabilityReferenceUnit( + Reference( reference_id=reference_id, url="https://support.zabbix.com/browse/{}".format(reference_id), ) @@ -151,7 +151,7 @@ def _load_advisories( elif reference_id.startswith("wnpa-sec"): references.append( - VulnerabilityReferenceUnit( + Reference( reference_id=reference_id, url="https://www.wireshark.org/security/{}.html".format( reference_id diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index 53d8c1482..95e674735 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -34,7 +34,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference def validate_schema(advisory_dict): @@ -105,7 +105,7 @@ def _parse(self, record) -> List[Advisory]: vuln_references = [] vuln_references.append( - VulnerabilityReferenceUnit( + Reference( reference_id=record["name"], url="https://security.archlinux.org/{}".format(record["name"]), ) @@ -113,7 +113,7 @@ def _parse(self, record) -> List[Advisory]: for ref in record["advisories"]: vuln_references.append( - VulnerabilityReferenceUnit( + Reference( reference_id=ref, url="https://security.archlinux.org/{}".format(ref), ) diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index ea49ed212..0715d1c8b 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -37,7 +37,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference def validate_schema(advisory_dict): @@ -136,7 +136,7 @@ def _parse(self, pkg_name: str, records: Mapping[str, Any]) -> List[Advisory]: debianbug = record.get('debianbug') if debianbug: bug_url = f'https://bugs.debian.org/cgi-bin/bugreport.cgi?bug={debianbug}' - references.append(VulnerabilityReferenceUnit( + references.append(Reference( url=bug_url, reference_id=debianbug )) diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index 1bc680804..57b273b9c 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -28,7 +28,7 @@ from vulnerabilities.data_source import GitDataSource from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference class GentooDataSource(GitDataSource): @@ -53,7 +53,7 @@ def process_file(self, file): xml_root = ET.parse(file).getroot() glsa = "GLSA-" + xml_root.attrib["id"] vuln_reference = [ - VulnerabilityReferenceUnit( + Reference( reference_id=glsa, url="https://security.gentoo.org/glsa/{}".format(xml_root.attrib["id"]), ) diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index 2ca5c1943..66fca5d0d 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -38,7 +38,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference # set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET'} @@ -189,7 +189,7 @@ def process_response(self) -> List[Advisory]: elif vuln["type"] == "GHSA": ghsa = vuln['value'] - vuln_references.append(VulnerabilityReferenceUnit( + vuln_references.append(Reference( reference_id=ghsa, url="https://github.com/advisories/{}".format( ghsa) diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index c020c7980..86a1e063b 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -37,7 +37,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference NPM_URL = 'https://registry.npmjs.org{}' @@ -68,36 +68,34 @@ def process_file(self, file) -> List[Advisory]: with open(file) as f: record = json.load(f) advisories = [] - - for record in records: - package_name = record["module_name"] - all_versions = self.versions.get(package_name) - aff_range = record.get("vulnerable_versions", "") - fixed_range = record.get("patched_versions", "") - - impacted_versions, resolved_versions = categorize_versions( - all_versions, aff_range, fixed_range + package_name = record["module_name"] + all_versions = self.versions.get(package_name) + aff_range = record.get("vulnerable_versions", "") + fixed_range = record.get("patched_versions", "") + + impacted_versions, resolved_versions = categorize_versions( + all_versions, aff_range, fixed_range + ) + + impacted_purls = _versions_to_purls(package_name, impacted_versions) + resolved_purls = _versions_to_purls(package_name, resolved_versions) + vuln_reference = [ + Reference( + url=NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}'), + reference_id=record["id"], ) - - impacted_purls = _versions_to_purls(package_name, impacted_versions) - resolved_purls = _versions_to_purls(package_name, resolved_versions) - vuln_reference = [ - VulnerabilityReferenceUnit( - url=NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}'), - reference_id=record["id"], - ) - ] - - for cve_id in record.get("cves") or [""]: - advisories.append( - Advisory( - summary=record.get("overview", ""), - cve_id=cve_id, - impacted_package_urls=impacted_purls, - resolved_package_urls=resolved_purls, - vuln_references=vuln_reference, - ) + ] + + for cve_id in record.get("cves") or [""]: + advisories.append( + Advisory( + summary=record.get("overview", ""), + cve_id=cve_id, + impacted_package_urls=impacted_purls, + resolved_package_urls=resolved_purls, + vuln_references=vuln_reference, ) + ) return advisories diff --git a/vulnerabilities/importers/openssl.py b/vulnerabilities/importers/openssl.py index a551027c5..ffc01ec15 100644 --- a/vulnerabilities/importers/openssl.py +++ b/vulnerabilities/importers/openssl.py @@ -28,7 +28,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference import xml.etree.ElementTree as ET @@ -71,7 +71,7 @@ def to_advisories(xml_response: str) -> Set[Advisory]: if info: commit_hash = info[0].attrib["hash"] ref_urls.append( - VulnerabilityReferenceUnit( + Reference( url="https://github.com/openssl/openssl/commit/" + commit_hash ) ) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index 72066ad64..d6e69b922 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -27,7 +27,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference class RedhatDataSource(DataSource): @@ -76,7 +76,7 @@ def to_advisory(advisory_data): if advisory_data.get("bugzilla"): bugzilla = advisory_data.get("bugzilla") references.append( - VulnerabilityReferenceUnit( + Reference( url="https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla), reference_id=bugzilla, ) @@ -84,12 +84,12 @@ def to_advisory(advisory_data): for rhsa in advisory_data["advisories"]: references.append( - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/errata/{}".format(rhsa), reference_id=rhsa, ) ) - references.append(VulnerabilityReferenceUnit(url=advisory_data["resource_url"])) + references.append(Reference(url=advisory_data["resource_url"])) return Advisory( summary=advisory_data["bugzilla_description"], diff --git a/vulnerabilities/importers/retiredotnet.py b/vulnerabilities/importers/retiredotnet.py index 9976718d3..76322e8c2 100644 --- a/vulnerabilities/importers/retiredotnet.py +++ b/vulnerabilities/importers/retiredotnet.py @@ -30,7 +30,7 @@ from vulnerabilities.data_source import GitDataSource from vulnerabilities.data_source import GitDataSourceConfiguration from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference class RetireDotnetDataSource(GitDataSource): @@ -91,7 +91,7 @@ def process_file(self, path) -> List[Advisory]: version=pkg['fix'], type='nuget')) - vuln_reference = [VulnerabilityReferenceUnit( + vuln_reference = [Reference( url=json_doc['link'], )] diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index ee4ca2b2a..3b36b2379 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -32,7 +32,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference class RubyDataSource(GitDataSource): @@ -108,7 +108,7 @@ def process_file(self, path) -> List[Advisory]: references = [] if record.get('url'): references.append( - VulnerabilityReferenceUnit(url=record.get('url')) + Reference(url=record.get('url')) ) return Advisory( diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index 6a88a6419..886efce4c 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -34,7 +34,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference class RustDataSource(GitDataSource): @@ -82,7 +82,7 @@ def _load_advisory(self, path: str) -> Optional[Advisory]: references = [] crate_name = advisory["package"] if advisory.get("url"): - references.append(VulnerabilityReferenceUnit(url=advisory["url"])) + references.append(Reference(url=advisory["url"])) all_versions = self.crates_api.get(crate_name) @@ -111,7 +111,7 @@ def _load_advisory(self, path: str) -> Optional[Advisory]: break references.append( - VulnerabilityReferenceUnit( + Reference( reference_id=advisory["id"], url="https://rustsec.org/advisories/{}.html".format(advisory["id"]), ) diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py index 8366dd042..72dd8e48a 100755 --- a/vulnerabilities/importers/safety_db.py +++ b/vulnerabilities/importers/safety_db.py @@ -42,7 +42,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference def validate_schema(advisory_dict): @@ -108,7 +108,7 @@ def updated_advisories(self) -> Set[Advisory]: if len(cve_ids[0]): cve_ids = [s.strip() for s in cve_ids.split(",")] - reference = [VulnerabilityReferenceUnit(reference_id=advisory["id"])] + reference = [Reference(reference_id=advisory["id"])] for cve_id in cve_ids: advisories.append( diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py index 42405ca4d..fe39850ad 100644 --- a/vulnerabilities/importers/ubuntu_usn.py +++ b/vulnerabilities/importers/ubuntu_usn.py @@ -29,7 +29,7 @@ from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference @dataclasses.dataclass @@ -90,7 +90,7 @@ def to_advisories(usn_db): def get_usn_references(usn_id): - return VulnerabilityReferenceUnit( + return Reference( reference_id="USN-" + usn_id, url="https://usn.ubuntu.com/{}/".format(usn_id) ) diff --git a/vulnerabilities/tests/test_gentoo.py b/vulnerabilities/tests/test_gentoo.py index e3d508c2b..6659e4a77 100644 --- a/vulnerabilities/tests/test_gentoo.py +++ b/vulnerabilities/tests/test_gentoo.py @@ -30,7 +30,7 @@ from vulnerabilities.importers.gentoo import GentooDataSource from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/gentoo/glsa-201709-09.xml") @@ -110,7 +110,7 @@ def test_process_file(self): qualifiers=OrderedDict(), subpath=None)}, vuln_references=[ - VulnerabilityReferenceUnit( + Reference( url='https://security.gentoo.org/glsa/201709-09', reference_id='GLSA-201709-09')], cve_id='CVE-2017-9800')] diff --git a/vulnerabilities/tests/test_github.py b/vulnerabilities/tests/test_github.py index bbbbb15d7..f6e0c27c2 100644 --- a/vulnerabilities/tests/test_github.py +++ b/vulnerabilities/tests/test_github.py @@ -33,7 +33,7 @@ from packageurl import PackageURL from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference from vulnerabilities.importers.github import GitHubAPIDataSource from vulnerabilities.importers.github import MavenVersionAPI from vulnerabilities.importers.github import ComposerVersionAPI @@ -187,7 +187,7 @@ def test_process_response(self): subpath=None, ), }, - vuln_references=[VulnerabilityReferenceUnit( + vuln_references=[Reference( url='https://github.com/advisories/GHSA-qcxh-w3j9-58qr', reference_id='GHSA-qcxh-w3j9-58qr')], cve_id='CVE-2019-0199', @@ -214,7 +214,7 @@ def test_process_response(self): subpath=None, ) }, - vuln_references=[VulnerabilityReferenceUnit( + vuln_references=[Reference( url='https://github.com/advisories/GHSA-qcxh-w3j9-58qr', reference_id='GHSA-qcxh-w3j9-58qr')], cve_id='CVE-2019-0199', @@ -240,7 +240,7 @@ def test_process_response(self): subpath=None, ), }, - vuln_references=[VulnerabilityReferenceUnit( + vuln_references=[Reference( url='https://github.com/advisories/GHSA-c9hw-wf7x-jp9j', reference_id='GHSA-c9hw-wf7x-jp9j')], cve_id='CVE-2020-1938', @@ -266,7 +266,7 @@ def test_process_response(self): subpath=None, ), }, - vuln_references=[VulnerabilityReferenceUnit( + vuln_references=[Reference( url='https://github.com/advisories/GHSA-c9hw-wf7x-jp9j', reference_id='GHSA-c9hw-wf7x-jp9j')], cve_id='CVE-2020-1938', @@ -293,7 +293,7 @@ def test_process_response(self): subpath=None, ) }, - vuln_references=[VulnerabilityReferenceUnit( + vuln_references=[Reference( url='https://github.com/advisories/GHSA-c9hw-wf7x-jp9j', reference_id='GHSA-c9hw-wf7x-jp9j')], cve_id='CVE-2020-1938', diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index bdc4d5e5e..5af1975b1 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -28,7 +28,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import PackageURL -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference from vulnerabilities.import_runner import ImportRunner from vulnerabilities.import_runner import _insert_vulnerabilities_and_references @@ -72,7 +72,7 @@ def save(self): cve_id='MOCK-CVE-2020-1337', summary='vulnerability description here', vuln_references=[ - VulnerabilityReferenceUnit( + Reference( url='https://example.com/with/more/info/MOCK-CVE-2020-1337')], impacted_package_urls=[ PackageURL( diff --git a/vulnerabilities/tests/test_openssl.py b/vulnerabilities/tests/test_openssl.py index 9e44da0f8..23ebb813e 100644 --- a/vulnerabilities/tests/test_openssl.py +++ b/vulnerabilities/tests/test_openssl.py @@ -28,7 +28,7 @@ from vulnerabilities.importers.openssl import OpenSSLDataSource from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data/', 'openssl_xml_data.xml') @@ -87,7 +87,7 @@ def test_to_advisory(self): version='1.1.1g', qualifiers=OrderedDict(), subpath=None)}, - vuln_references=[VulnerabilityReferenceUnit( + vuln_references=[Reference( url='https://github.com/openssl/openssl/commit/' + 'eb563247aef3e83dda7679c43f9649270462e5b1')], cve_id='CVE-2020-1967'), @@ -189,10 +189,10 @@ def test_to_advisory(self): qualifiers=OrderedDict(), subpath=None), PackageURL(type='generic', namespace=None, name='openssl', version='1.0.2u', qualifiers=OrderedDict(), subpath=None)}, - vuln_references=[VulnerabilityReferenceUnit( + vuln_references=[Reference( url='https://github.com/openssl/openssl/commit/' + '419102400a2811582a7a3d4a4e317d72e5ce0a8f'), - VulnerabilityReferenceUnit( + Reference( url='https://github.com/openssl/openssl/commit/' + 'f1c5eea8a817075d31e43f5876993c6710238c98')], cve_id='CVE-2019-1551') diff --git a/vulnerabilities/tests/test_redhat_importer.py b/vulnerabilities/tests/test_redhat_importer.py index 3b1e32b8c..8447c508a 100644 --- a/vulnerabilities/tests/test_redhat_importer.py +++ b/vulnerabilities/tests/test_redhat_importer.py @@ -29,7 +29,7 @@ import vulnerabilities.importers.redhat as redhat from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/", "redhat.json") @@ -80,19 +80,19 @@ def test_to_advisory(self): resolved_package_urls=[], vuln_references=sorted( [ - VulnerabilityReferenceUnit( + Reference( url="https://bugzilla.redhat.com/show_bug.cgi?id=1396383", reference_id="1396383", ), - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/errata/RHSA-2017:1931", reference_id="RHSA-2017:1931", ), - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/errata/RHSA-2017:0725", reference_id="RHSA-2017:0725", ), - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-9401.json", # nopep8 reference_id="", ), @@ -135,27 +135,27 @@ def test_to_advisory(self): resolved_package_urls=[], vuln_references=sorted( [ - VulnerabilityReferenceUnit( + Reference( url="https://bugzilla.redhat.com/show_bug.cgi?id=1430347", reference_id="1430347", ), - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/errata/RHSA-2017:1842", reference_id="RHSA-2017:1842", ), - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/errata/RHSA-2017:2437", reference_id="RHSA-2017:2437", ), - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/errata/RHSA-2017:2077", reference_id="RHSA-2017:2077", ), - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/errata/RHSA-2017:2444", reference_id="RHSA-2017:2444", ), - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2016-10200.json", # nopep8 reference_id="", ), @@ -173,11 +173,11 @@ def test_to_advisory(self): resolved_package_urls=[], vuln_references=sorted( [ - VulnerabilityReferenceUnit( + Reference( url="https://bugzilla.redhat.com/show_bug.cgi?id=1492984", reference_id="1492984", ), - VulnerabilityReferenceUnit( + Reference( url="https://access.redhat.com/hydra/rest/securitydata/cve/CVE-2017-12168.json", # nopep8 reference_id="", ), diff --git a/vulnerabilities/tests/test_retiredotnet.py b/vulnerabilities/tests/test_retiredotnet.py index dd22c8e7c..fa29d7899 100644 --- a/vulnerabilities/tests/test_retiredotnet.py +++ b/vulnerabilities/tests/test_retiredotnet.py @@ -28,7 +28,7 @@ from vulnerabilities.importers.retiredotnet import RetireDotnetDataSource from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -128,7 +128,7 @@ def test_process_file(self): ), }, vuln_references=[ - VulnerabilityReferenceUnit( + Reference( url="https://github.com/aspnet/Announcements/issues/359", reference_id="" ) ], diff --git a/vulnerabilities/tests/test_ruby.py b/vulnerabilities/tests/test_ruby.py index 8aa4bb48b..0e5ec5969 100644 --- a/vulnerabilities/tests/test_ruby.py +++ b/vulnerabilities/tests/test_ruby.py @@ -31,7 +31,7 @@ from vulnerabilities.importers.ruby import RubyDataSource from vulnerabilities.data_source import GitDataSourceConfiguration from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -93,7 +93,7 @@ def test_process_file(self, mock_write): ), }, vuln_references=[ - VulnerabilityReferenceUnit(url="https://github.com/sinatra/sinatra/pull/1379") + Reference(url="https://github.com/sinatra/sinatra/pull/1379") ], cve_id="CVE-2018-7212", ), @@ -131,7 +131,7 @@ def test_process_file(self, mock_write): ) }, vuln_references=[ - VulnerabilityReferenceUnit(url="https://github.com/sinatra/sinatra/issues/1428") + Reference(url="https://github.com/sinatra/sinatra/issues/1428") ], cve_id="CVE-2018-11627", ), diff --git a/vulnerabilities/tests/test_safety_db.py b/vulnerabilities/tests/test_safety_db.py index e019e7199..e0220846f 100644 --- a/vulnerabilities/tests/test_safety_db.py +++ b/vulnerabilities/tests/test_safety_db.py @@ -30,7 +30,7 @@ from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importers.safety_db import VersionAPI from vulnerabilities.importers.safety_db import categorize_versions -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data/') diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 55b5582b9..b12ff29c9 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -12,7 +12,7 @@ from vulnerabilities.oval_parser import OvalParser from vulnerabilities.importers.ubuntu import UbuntuDataSource from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/") @@ -226,12 +226,12 @@ def test_get_data_from_xml_doc(self, mock_write): qualifiers=OrderedDict(), subpath=None)}, vuln_references=sorted([ - VulnerabilityReferenceUnit(url='http://www.openwall.com/lists/oss-security/2016/10/18/11'), - VulnerabilityReferenceUnit(url='https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860'), - VulnerabilityReferenceUnit(url='http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html'), - VulnerabilityReferenceUnit(url='https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce'), - VulnerabilityReferenceUnit(url='https://blog.torproject.org/blog/tor-0289-released-important-fixes'), - VulnerabilityReferenceUnit(url='https://trac.torproject.org/projects/tor/ticket/20384')],key=lambda x : x.url), + Reference(url='http://www.openwall.com/lists/oss-security/2016/10/18/11'), + Reference(url='https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860'), + Reference(url='http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html'), + Reference(url='https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce'), + Reference(url='https://blog.torproject.org/blog/tor-0289-released-important-fixes'), + Reference(url='https://trac.torproject.org/projects/tor/ticket/20384')],key=lambda x : x.url), cve_id='CVE-2016-8860'), Advisory( summary=('Heap-based buffer overflow in the bm_readbody_bmp function' @@ -263,9 +263,9 @@ def test_get_data_from_xml_doc(self, mock_write): qualifiers=OrderedDict(), subpath=None)}, vuln_references=sorted([ - VulnerabilityReferenceUnit(url='http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html'), - VulnerabilityReferenceUnit(url='https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/'), - VulnerabilityReferenceUnit(url='https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703')],key=lambda x: x.url), + Reference(url='http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html'), + Reference(url='https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/'), + Reference(url='https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703')],key=lambda x: x.url), cve_id='CVE-2016-8703')} xml_doc = ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml")) diff --git a/vulnerabilities/tests/test_ubuntu_usn.py b/vulnerabilities/tests/test_ubuntu_usn.py index 5e3adf0b4..7e155a654 100644 --- a/vulnerabilities/tests/test_ubuntu_usn.py +++ b/vulnerabilities/tests/test_ubuntu_usn.py @@ -31,7 +31,7 @@ from packageurl import PackageURL from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import VulnerabilityReferenceUnit +from vulnerabilities.data_source import Reference import vulnerabilities.importers.ubuntu_usn as ubuntu_usn BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -50,7 +50,7 @@ def setUpClass(cls): def test_get_usn_references(self): eg_usn = "435-1" - expected_references = VulnerabilityReferenceUnit( + expected_references = Reference( reference_id="USN-435-1", url="https://usn.ubuntu.com/435-1/" ) @@ -154,7 +154,7 @@ def test_to_advisories(self): ), }, vuln_references=[ - VulnerabilityReferenceUnit( + Reference( url="https://usn.ubuntu.com/763-1/", reference_id="USN-763-1" ) ], @@ -182,7 +182,7 @@ def test_to_advisories(self): ), }, vuln_references=[ - VulnerabilityReferenceUnit( + Reference( url="https://usn.ubuntu.com/763-1/", reference_id="USN-763-1" ) ],