From cd4a9566ec57e380cf949b74897ba176da3da12b Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sat, 6 Jun 2020 09:59:21 +0530 Subject: [PATCH 1/9] Make the config of updateable Signed-off-by: Shivam Sandbhor --- vulnerabilities/data_source.py | 7 ++++--- vulnerabilities/import_runner.py | 2 ++ vulnerabilities/importers/rust.py | 2 +- vulnerabilities/tests/test_import_runner.py | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 05928a460..ca68d1b83 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -75,7 +75,7 @@ class InvalidConfigurationError(Exception): @dataclasses.dataclass class DataSourceConfiguration: - batch_size: int + pass class DataSource(ContextManager): @@ -105,8 +105,9 @@ def __init__( :param config: Optional dictionary with subclass-specific configuration """ config = config or {} + self.batch_size = batch_size try: - self.config = self.__class__.CONFIG_CLASS(batch_size, **config) + self.config = self.__class__.CONFIG_CLASS(**config) # These really should be declared in DataSourceConfiguration above but that would # prevent DataSource subclasses from declaring mandatory parameters (i.e. positional # arguments) @@ -183,7 +184,7 @@ def batch_advisories(self, advisories: List[Advisory]) -> Set[Advisory]: advisories = advisories[:] # copy the list as we are mutating it in the loop below while advisories: - b, advisories = advisories[:self.config.batch_size], advisories[self.config.batch_size:] + b, advisories = advisories[:self.batch_size], advisories[self.batch_size:] yield set(b) diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index c8176d1b5..9b049fd3d 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -21,6 +21,7 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. +import dataclasses import datetime import logging from typing import Dict @@ -77,6 +78,7 @@ def run(self, cutoff_date: datetime.datetime = None) -> None: _process_updated_advisories(data_source) self.importer.last_run = datetime.datetime.now(tz=datetime.timezone.utc) + self.importer.data_source_cfg = dataclasses.asdict(data_source.config) self.importer.save() logger.debug(f'Successfully finished import for {self.importer.name}.') diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index 6c085a899..dc5ac2e42 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -64,7 +64,7 @@ def _load_advisories(self, files) -> Set[Advisory]: files = [f for f in files if not f.endswith('-0000.toml')] # skip temporary files while files: - batch, files = files[:self.config.batch_size], files[self.config.batch_size:] + batch, files = files[:self.batch_size], files[self.batch_size:] advisories = set() diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index 44b1dd897..ce5572c8b 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -46,7 +46,7 @@ def updated_advisories(self): def _yield_advisories(self, advisories): while advisories: - b, advisories = advisories[:self.config.batch_size], advisories[self.config.batch_size:] + b, advisories = advisories[:self.batch_size], advisories[self.batch_size:] yield b From 3f5904abc0927493d612762c551cdf82f4978a12 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 29 May 2020 19:01:30 +0530 Subject: [PATCH 2/9] Remove tests for old ubuntu importer Signed-off-by: Shivam Sandbhor --- vulnerabilities/tests/test_data_dump.py | 48 --------------------- vulnerabilities/tests/test_importers.py | 57 ------------------------- 2 files changed, 105 deletions(-) delete mode 100644 vulnerabilities/tests/test_data_dump.py delete mode 100644 vulnerabilities/tests/test_importers.py diff --git a/vulnerabilities/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py deleted file mode 100644 index 1772bf8b6..000000000 --- a/vulnerabilities/tests/test_data_dump.py +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnerableCode software is licensed under the Apache License version 2.0. -# Data generated with VulnerableCode require an acknowledgment. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnerableCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import os - -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, 'test_data/') - - -def test_ubuntu_data_dump(setUbuntuData): - """ - Check basic data import - """ - assert Vulnerability.objects.filter(cve_id='CVE-2002-2439') - pkgs = Package.objects.filter(name='gcc-4.6') - assert pkgs - - pkg = pkgs[0] - assert 'deb' == pkg.type - assert 'ubuntu' == pkg.namespace - - -CVE_IDS = ('CVE-2018-11362', 'CVE-2018-11361', 'CVE-2018-11360', - 'CVE-2018-11359', 'CVE-2018-11358', 'CVE-2018-11357', - 'CVE-2018-11356', 'CVE-2018-11355', 'CVE-2018-11354') diff --git a/vulnerabilities/tests/test_importers.py b/vulnerabilities/tests/test_importers.py deleted file mode 100644 index 417bab236..000000000 --- a/vulnerabilities/tests/test_importers.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnerableCode software is licensed under the Apache License version 2.0. -# Data generated with VulnerableCode require an acknowledgment. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnerableCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -from os.path import dirname -from os.path import join - -from vulnerabilities.importers import ubuntu - - -def test_ubuntu_extract_cves(): - ubuntu_testfile = join(dirname(__file__), 'test_data', 'ubuntu_main.html') - - with open(ubuntu_testfile) as f: - test_input = f.read() - - cves = ubuntu.extract_cves(test_input) - - expected = { - 'cve_id': 'CVE-2002-2439', - 'package_name': 'gcc-4.6', - 'vulnerability_status': 'low' - } - assert expected == cves[0] - - expected = { - 'cve_id': 'CVE-2013-0157', - 'package_name': 'util-linux', - 'vulnerability_status': 'low', - } - assert expected == cves[50] - - expected = { - 'cve_id': 'CVE-2017-9986', - 'package_name': 'linux-lts-xenial', - 'vulnerability_status': 'medium', - } - assert expected == cves[-1] From 42a9945b3b052588bd453f102a905e7730801108 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 29 May 2020 19:02:42 +0530 Subject: [PATCH 3/9] Avoid collecting 'binaries' of OVAL files and add tests for UbuntuDataSource(yet to be implemented) Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/oval_parser.py | 28 ++++- vulnerabilities/tests/test_ubuntu.py | 127 +++++++++++++++++++++-- 2 files changed, 143 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/importers/oval_parser.py b/vulnerabilities/importers/oval_parser.py index 5752dbc5e..21f724f1b 100755 --- a/vulnerabilities/importers/oval_parser.py +++ b/vulnerabilities/importers/oval_parser.py @@ -1,3 +1,26 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import re from typing import Optional from typing import List from typing import Dict @@ -103,8 +126,9 @@ def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]: if var.get('var_ref'): var_elem = self.oval_document.getElementByID( var.get('var_ref')) - for vals in var_elem.element: - pkg_list.append(vals.text) + comment = var_elem.element.get('comment') + pkg_name = re.match("'.+'", comment).group().replace("'","") + pkg_list.append(pkg_name) else: pkg_list.append(var.text) diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index b3636ddc2..31ae12356 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -1,12 +1,16 @@ import os import unittest +from unittest.mock import patch import xml.etree.ElementTree as ET +from collections import OrderedDict +import asyncio from dephell_specifier import RangeSpecifier - +from packageurl import PackageURL from vulnerabilities.importers.oval_parser import OvalParser - +from vulnerabilities.importers.ubuntu import UbuntuDataSource +from vulnerabilities.data_source import Advisory BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/") @@ -84,8 +88,8 @@ def test_get_pkgs_from_obj(self): pkg_set1 = set(self.parsed_oval.get_pkgs_from_obj(obj_t2)) pkg_set2 = set(self.parsed_oval.get_pkgs_from_obj(obj_t1)) - assert pkg_set1 == {"potrace", "libpotrace0"} - assert pkg_set2 == {"tor", "tor-geoipdb"} + assert pkg_set1 == {"potrace"} + assert pkg_set2 == {"tor"} def test_get_versionsrngs_from_state(self): @@ -99,7 +103,7 @@ def test_get_versionsrngs_from_state(self): assert self.parsed_oval.get_versionsrngs_from_state(state_1) == exp_range_1 assert self.parsed_oval.get_versionsrngs_from_state(state_2) == exp_range_2 - + def test_get_urls_from_definition(self): def1_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html', @@ -107,7 +111,7 @@ def test_get_urls_from_definition(self): 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703' } - assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1) + assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1) def2_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html', 'https://trac.torproject.org/projects/tor/ticket/20384', @@ -117,7 +121,7 @@ def test_get_urls_from_definition(self): 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860', } - assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2) + assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2) def test_get_data(self): @@ -125,7 +129,7 @@ def test_get_data(self): { "test_data": [ { - "package_list": ["libpotrace0", "potrace"], + "package_list": ["potrace"], "version_ranges": RangeSpecifier("<1.14-2"), } ], @@ -141,7 +145,7 @@ def test_get_data(self): { "test_data": [ { - "package_list": ["tor", "tor-geoipdb"], + "package_list": ["tor"], "version_ranges": RangeSpecifier("<0.2.8.9-1ubuntu1"), } ], @@ -160,4 +164,107 @@ def test_get_data(self): ] assert expected_data == self.parsed_oval.get_data() - + +#This is horrible, there must be a better way +async def mock(a,b): + pass + +class TestUbuntuDataSource(unittest.TestCase): + + @classmethod + def setUpClass(cls): + pass + + @patch( + 'vulnerabilities.importers.ubuntu.VersionAPI.get', + return_value={ + '0.3.0', + '0.2.0', + '2.14-2'}) + @patch('vulnerabilities.importers.ubuntu.VersionAPI.load_api',new=mock) + def test_get_data_from_xml_doc(self, mock_write): + + data_source_cfg = { + 'releases': 'eg-ubuntu'} + ubuntu_data_src = UbuntuDataSource( + batch_size=1, config=data_source_cfg) + expected_data = { + Advisory( + summary=('Tor before 0.2.8.9 and 0.2.9.x before 0.2.9.4-alpha had ' + 'internal functions that were entitled to expect that buf_t data had ' + 'NUL termination, but the implementation of or/buffers.c did not ' + 'ensure that NUL termination was present, which allows remote ' + 'attackers to cause a denial of service (client, hidden ' + 'service, relay, or authority crash) via crafted data.'), + impacted_package_urls={ + PackageURL( + type='deb', + namespace=None, + name='tor', + version='0.2.0', + qualifiers=OrderedDict(), + subpath=None)}, + resolved_package_urls={ + PackageURL( + type='deb', + namespace=None, + name='tor', + version='0.3.0', + qualifiers=OrderedDict(), + subpath=None), + PackageURL( + type='deb', + namespace=None, + name='tor', + version='2.14-2', + qualifiers=OrderedDict(), + subpath=None)}, + reference_urls={ + 'http://www.openwall.com/lists/oss-security/2016/10/18/11', + 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860', + 'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html', + 'https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce', + 'https://blog.torproject.org/blog/tor-0289-released-important-fixes', + 'https://trac.torproject.org/projects/tor/ticket/20384'}, + reference_ids=[], + cve_id='CVE-2016-8860'), + Advisory( + summary=('Heap-based buffer overflow in the bm_readbody_bmp function' + ' in bitmap_io.c in potrace before 1.13 allows remote attackers to ' + 'have unspecified impact via a crafted BMP image, a different ' + 'vulnerability than CVE-2016-8698, CVE-2016-8699, ' + 'CVE-2016-8700, CVE-2016-8701, and CVE-2016-8702.'), + impacted_package_urls={ + PackageURL( + type='deb', + namespace=None, + name='potrace', + version='0.3.0', + qualifiers=OrderedDict(), + subpath=None), + PackageURL( + type='deb', + namespace=None, + name='potrace', + version='0.2.0', + qualifiers=OrderedDict(), + subpath=None)}, + resolved_package_urls={ + PackageURL( + type='deb', + namespace=None, + name='potrace', + version='2.14-2', + qualifiers=OrderedDict(), + subpath=None)}, + reference_urls={ + 'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html', + 'https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/', + 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703'}, + reference_ids=[], + cve_id='CVE-2016-8703')} + + xml_doc = ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml")) + data = set(ubuntu_data_src.get_data_from_xml_doc(xml_doc)) + + assert expected_data == data From ea9974b406036e428dd8d57217ff727f08ae61ae Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 29 May 2020 19:04:55 +0530 Subject: [PATCH 4/9] Add ubuntu data source which uses OVAL files Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/__init__.py | 1 + vulnerabilities/importers/ubuntu.py | 153 +++++++++++++++--- .../migrations/0009_ubuntu_importer.py | 53 ++++++ 3 files changed, 183 insertions(+), 24 deletions(-) create mode 100644 vulnerabilities/migrations/0009_ubuntu_importer.py diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 7f50ca158..a1bb94f85 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -27,3 +27,4 @@ from vulnerabilities.importers.rust import RustDataSource from vulnerabilities.importers.safety_db import SafetyDbDataSource from vulnerabilities.importers.ruby import RubyDataSource +from vulnerabilities.importers.ubuntu import UbuntuDataSource diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index 20389b186..079637a66 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -1,4 +1,3 @@ -# # Copyright (c) 2017 nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/vulnerablecode/ # The VulnerableCode software is licensed under the Apache License version 2.0. @@ -21,36 +20,142 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -from urllib.request import urlopen -import bs4 +import asyncio +import bz2 +import dataclasses +from typing import Iterable +from typing import List +from typing import Mapping +from typing import Set +import xml.etree.ElementTree as ET + + +from aiohttp import ClientSession +from aiohttp.client_exceptions import ClientResponseError +import requests +from packageurl import PackageURL + + +from vulnerabilities.data_source import DataSource, DataSourceConfiguration, Advisory +from vulnerabilities.importers import oval_parser + + + +@dataclasses.dataclass +class UbuntuConfiguration(DataSourceConfiguration): + releases: list + +class UbuntuDataSource(DataSource): + + CONFIG_CLASS = UbuntuConfiguration + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + #we could avoid setting translations, and have it + #set by default in the OvalParser, but we don't yet know + #whether all OVAL providers use the same format + self.translations = {'less than':'<'} + self._versions = VersionAPI() + + def _fetch(self) : + base_url = 'https://people.canonical.com/~ubuntu-security/oval/' + file_name = 'com.ubuntu.{}.cve.oval.xml.bz2' + releases = self.config.releases + for release in releases: + resp = requests.get(base_url + file_name.format(release)) + extracted = bz2.decompress(resp.content) + yield ET.ElementTree(ET.fromstring(extracted.decode('utf-8'))) + + def added_advisories(self) -> List[Advisory] : + advisories = [] + for oval_file in self._fetch(): + advisories.extend(self.get_data_from_xml_doc(oval_file)) + return advisories + + @staticmethod + def _collect_pkgs(parsed_oval_data) -> Set : + all_pkgs = set() + for definition_data in parsed_oval_data: + for test_data in definition_data['test_data']: + for package in test_data['package_list']: + all_pkgs.add(package) + + return all_pkgs + + + def get_data_from_xml_doc(self, xml_doc) -> List[Advisory] : + all_adv = [] + oval_doc = oval_parser.OvalParser(self.translations, xml_doc) + raw_data = oval_doc.get_data() + all_pkgs = self._collect_pkgs(raw_data) + + asyncio.run(self._versions.load_api(all_pkgs)) + + for definition_data in raw_data: #definition_data -> Advisory + vuln_id = definition_data['vuln_id'] + description = definition_data['description'] + affected_purls = set() + safe_purls = set() + urls = definition_data['reference_urls'] + for test_data in definition_data['test_data'] : + for package in test_data['package_list']: + pkg_name = package + aff_ver_range = test_data['version_ranges'] + all_versions = self._versions.get(package) + #This filter is to filter out long versions. + #50 is limit because that's what db permits atm + all_versions = set(filter(lambda x : len(x)<50,all_versions)) + if not all_versions: + continue + affected_versions = set(filter(lambda x: x in aff_ver_range,all_versions)) + safe_versions = all_versions - affected_versions + for version in affected_versions: + #should we add a qualifier like 'distro:ubuntu'? + pkg_url = PackageURL(name=pkg_name,type='deb',version=version) + affected_purls.add(pkg_url) -UBUNTU_ROOT_URL = 'https://people.canonical.com/~ubuntu-security/cve/main.html' + for version in safe_versions: + #should we add a qualifier like 'distro:ubuntu'? + pkg_url = PackageURL(name=pkg_name,type='deb',version=version) + safe_purls.add(pkg_url) + all_adv.append(Advisory(summary=description,impacted_package_urls=affected_purls, + resolved_package_urls=safe_purls,cve_id=vuln_id,reference_urls=urls)) + return all_adv -def extract_cves(html): - soup = bs4.BeautifulSoup(html, 'lxml') - # Exclude the header row which has no class attribute - rows = soup.find_all('tr', attrs={'class': True}) - cves = [] - for row in rows: - columns = row.text.split() - cves.append({ - 'cve_id': columns[0], - 'package_name': columns[1], - 'vulnerability_status': row.get('class')[0], - }) +class VersionAPI: + def __init__(self, cache: Mapping[str, Set[str]] = None): + self.cache = cache or {} - return cves + def get(self, package_name: str) -> Set[str]: + return self.cache[package_name] + async def load_api(self, pkg_set): + async with ClientSession() as session: + await asyncio.gather(*[self.set_api(pkg, session) for pkg in pkg_set if pkg not in self.cache]) -def scrape_cves(): - """ - Runs the full scraping process of Ubuntu CVEs. - """ - html = urlopen(UBUNTU_ROOT_URL).read() - cves = extract_cves(html) - return cves + async def set_api(self, pkg, session): + url = ('https://api.launchpad.net/1.0/ubuntu/+archive/' + 'primary?ws.op=getPublishedSources&' + 'source_name={}&exact_match=true'.format(pkg)) + try: + all_versions = set() + while(True): + response = await session.request(method='GET', url=url) + response.raise_for_status() + resp_json = await response.json() + if resp_json['entries'] == [] : + self.cache[pkg] = {} + break + for release in resp_json['entries']: + all_versions.add(release['source_package_version']) + if resp_json.get('next_collection_link') : + url = resp_json['next_collection_link'] + else: + break + self.cache[pkg] = all_versions + except ClientResponseError: + self.cache[pkg] = {} \ No newline at end of file diff --git a/vulnerabilities/migrations/0009_ubuntu_importer.py b/vulnerabilities/migrations/0009_ubuntu_importer.py new file mode 100644 index 000000000..0c2ef3a5a --- /dev/null +++ b/vulnerabilities/migrations/0009_ubuntu_importer.py @@ -0,0 +1,53 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +from django.db import migrations + + +def add_ubuntu_importer(apps, _): + Importer = apps.get_model('vulnerabilities', 'Importer') + + Importer.objects.create( + name='ubuntu', + license='', + last_run=None, + data_source='UbuntuDataSource', + data_source_cfg={'releases':['bionic','trusty','focal','eoan','xenial']}, + ) + + +def remove_ubuntu_importer(apps, _): + Importer = apps.get_model('vulnerabilities', 'Importer') + qs = Importer.objects.filter(name='ubuntu') + if qs: + qs[0].delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0008_ruby_importer'), + ] + + operations = [ + migrations.RunPython(add_ubuntu_importer, remove_ubuntu_importer), + ] \ No newline at end of file From a35db614da46762e927da290b5dd8dc495e96ad4 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 29 May 2020 20:11:33 +0530 Subject: [PATCH 5/9] Fix style Signed-off-by: Shivam Sandbhor --- requirements.txt | 1 + vulnerabilities/importers/oval_parser.py | 4 +- vulnerabilities/importers/ubuntu.py | 83 ++++++++++++++---------- 3 files changed, 51 insertions(+), 37 deletions(-) diff --git a/requirements.txt b/requirements.txt index 31dd46e25..874fb4100 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +aiohttp==3.6.2 asgiref==3.2.7 attrs==19.3.0 backcall==0.1.0 diff --git a/vulnerabilities/importers/oval_parser.py b/vulnerabilities/importers/oval_parser.py index 21f724f1b..9e2ebf6cd 100755 --- a/vulnerabilities/importers/oval_parser.py +++ b/vulnerabilities/importers/oval_parser.py @@ -126,8 +126,8 @@ def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]: if var.get('var_ref'): var_elem = self.oval_document.getElementByID( var.get('var_ref')) - comment = var_elem.element.get('comment') - pkg_name = re.match("'.+'", comment).group().replace("'","") + comment = var_elem.element.get('comment') + pkg_name = re.match("'.+'", comment).group().replace("'", "") pkg_list.append(pkg_name) else: pkg_list.append(var.text) diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index 079637a66..644e2399c 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -41,49 +41,49 @@ from vulnerabilities.importers import oval_parser - @dataclasses.dataclass class UbuntuConfiguration(DataSourceConfiguration): releases: list + class UbuntuDataSource(DataSource): CONFIG_CLASS = UbuntuConfiguration + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - #we could avoid setting translations, and have it - #set by default in the OvalParser, but we don't yet know - #whether all OVAL providers use the same format - self.translations = {'less than':'<'} + # we could avoid setting translations, and have it + # set by default in the OvalParser, but we don't yet know + # whether all OVAL providers use the same format + self.translations = {'less than': '<'} self._versions = VersionAPI() - def _fetch(self) : + def _fetch(self): base_url = 'https://people.canonical.com/~ubuntu-security/oval/' file_name = 'com.ubuntu.{}.cve.oval.xml.bz2' - releases = self.config.releases + releases = self.config.releases for release in releases: resp = requests.get(base_url + file_name.format(release)) extracted = bz2.decompress(resp.content) yield ET.ElementTree(ET.fromstring(extracted.decode('utf-8'))) - def added_advisories(self) -> List[Advisory] : + def added_advisories(self) -> List[Advisory]: advisories = [] for oval_file in self._fetch(): advisories.extend(self.get_data_from_xml_doc(oval_file)) - return advisories + return advisories - @staticmethod - def _collect_pkgs(parsed_oval_data) -> Set : + @staticmethod + def _collect_pkgs(parsed_oval_data) -> Set: all_pkgs = set() - for definition_data in parsed_oval_data: + for definition_data in parsed_oval_data: for test_data in definition_data['test_data']: for package in test_data['package_list']: all_pkgs.add(package) return all_pkgs - - def get_data_from_xml_doc(self, xml_doc) -> List[Advisory] : + def get_data_from_xml_doc(self, xml_doc) -> List[Advisory]: all_adv = [] oval_doc = oval_parser.OvalParser(self.translations, xml_doc) raw_data = oval_doc.get_data() @@ -91,41 +91,53 @@ def get_data_from_xml_doc(self, xml_doc) -> List[Advisory] : asyncio.run(self._versions.load_api(all_pkgs)) - for definition_data in raw_data: #definition_data -> Advisory + for definition_data in raw_data: # definition_data -> Advisory vuln_id = definition_data['vuln_id'] description = definition_data['description'] affected_purls = set() safe_purls = set() urls = definition_data['reference_urls'] - for test_data in definition_data['test_data'] : + for test_data in definition_data['test_data']: for package in test_data['package_list']: pkg_name = package aff_ver_range = test_data['version_ranges'] all_versions = self._versions.get(package) - #This filter is to filter out long versions. - #50 is limit because that's what db permits atm - all_versions = set(filter(lambda x : len(x)<50,all_versions)) + # This filter is to filter out long versions. + # 50 is limit because that's what db permits atm + all_versions = set( + filter( + lambda x: len(x) < 50, + all_versions)) if not all_versions: continue - affected_versions = set(filter(lambda x: x in aff_ver_range,all_versions)) + affected_versions = set( + filter( + lambda x: x in aff_ver_range, + all_versions)) safe_versions = all_versions - affected_versions for version in affected_versions: - #should we add a qualifier like 'distro:ubuntu'? - pkg_url = PackageURL(name=pkg_name,type='deb',version=version) + # should we add a qualifier like 'distro:ubuntu'? + pkg_url = PackageURL( + name=pkg_name, type='deb', version=version) affected_purls.add(pkg_url) for version in safe_versions: - #should we add a qualifier like 'distro:ubuntu'? - pkg_url = PackageURL(name=pkg_name,type='deb',version=version) + # should we add a qualifier like 'distro:ubuntu'? + pkg_url = PackageURL( + name=pkg_name, type='deb', version=version) safe_purls.add(pkg_url) - all_adv.append(Advisory(summary=description,impacted_package_urls=affected_purls, - resolved_package_urls=safe_purls,cve_id=vuln_id,reference_urls=urls)) + all_adv.append( + Advisory( + summary=description, + impacted_package_urls=affected_purls, + resolved_package_urls=safe_purls, + cve_id=vuln_id, + reference_urls=urls)) return all_adv - class VersionAPI: def __init__(self, cache: Mapping[str, Set[str]] = None): self.cache = cache or {} @@ -135,27 +147,28 @@ def get(self, package_name: str) -> Set[str]: async def load_api(self, pkg_set): async with ClientSession() as session: - await asyncio.gather(*[self.set_api(pkg, session) for pkg in pkg_set if pkg not in self.cache]) + await asyncio.gather(*[self.set_api(pkg, session) + for pkg in pkg_set if pkg not in self.cache]) - async def set_api(self, pkg, session): + async def set_api(self, pkg, session): url = ('https://api.launchpad.net/1.0/ubuntu/+archive/' - 'primary?ws.op=getPublishedSources&' - 'source_name={}&exact_match=true'.format(pkg)) + 'primary?ws.op=getPublishedSources&' + 'source_name={}&exact_match=true'.format(pkg)) try: all_versions = set() while(True): response = await session.request(method='GET', url=url) response.raise_for_status() resp_json = await response.json() - if resp_json['entries'] == [] : + if resp_json['entries'] == []: self.cache[pkg] = {} break for release in resp_json['entries']: all_versions.add(release['source_package_version']) - if resp_json.get('next_collection_link') : - url = resp_json['next_collection_link'] + if resp_json.get('next_collection_link'): + url = resp_json['next_collection_link'] else: break self.cache[pkg] = all_versions except ClientResponseError: - self.cache[pkg] = {} \ No newline at end of file + self.cache[pkg] = {} From 34de081cb5b93eb6e6a8a2f59cb09522a4feafb6 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 5 Jun 2020 15:57:46 +0530 Subject: [PATCH 6/9] Add OvalDataSource, refactor UbuntuDataSource to inherit from OvalDataSource Signed-off-by: Shivam Sandbhor --- vulnerabilities/data_source.py | 118 +++++++++++++++++- vulnerabilities/importers/__init__.py | 1 + vulnerabilities/importers/ubuntu.py | 93 +++----------- vulnerabilities/{importers => }/lib_oval.py | 0 .../{importers => }/oval_parser.py | 2 +- vulnerabilities/tests/test_suse.py | 2 +- vulnerabilities/tests/test_ubuntu.py | 4 +- 7 files changed, 132 insertions(+), 88 deletions(-) rename vulnerabilities/{importers => }/lib_oval.py (100%) rename vulnerabilities/{importers => }/oval_parser.py (99%) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index ca68d1b83..171169021 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -35,10 +35,13 @@ from typing import Sequence from typing import Set from typing import Tuple +import xml.etree.ElementTree as ET import pygit2 from packageurl import PackageURL +from vulnerabilities.oval_parser import OvalParser + @dataclasses.dataclass class Advisory: @@ -257,7 +260,8 @@ def file_changes( return {str(p) for p in path.glob(glob) if p.is_file()}, set() - return self._collect_file_changes(subdir=subdir, recursive=recursive, file_ext=file_ext) + return self._collect_file_changes( + subdir=subdir, recursive=recursive, file_ext=file_ext) def _collect_file_changes( self, @@ -269,7 +273,8 @@ def _collect_file_changes( previous_commit = None added_files, updated_files = set(), set() - for commit in self._repo.walk(self._repo.head.target, pygit2.GIT_SORT_TIME): + for commit in self._repo.walk( + self._repo.head.target, pygit2.GIT_SORT_TIME): commit_time = commit.commit_time + commit.commit_time_offset # convert to UTC if commit_time < self.cutoff_timestamp: @@ -280,13 +285,16 @@ def _collect_file_changes( continue for d in commit.tree.diff_to_tree(previous_commit.tree).deltas: - if not _include_file(d.new_file.path, subdir, recursive, file_ext) or d.is_binary: + if not _include_file( + d.new_file.path, subdir, recursive, file_ext) or d.is_binary: continue - abspath = os.path.join(self.config.working_directory, d.new_file.path) + abspath = os.path.join( + self.config.working_directory, d.new_file.path) # TODO # Just filtering on the two status values for "added" and "modified" is too - # simplistic. This does not cover file renames, copies & deletions. + # simplistic. This does not cover file renames, copies & + # deletions. if d.status == pygit2.GIT_DELTA_ADDED: added_files.add(abspath) elif d.status == pygit2.GIT_DELTA_MODIFIED: @@ -381,3 +389,103 @@ def _include_file( match = match and path.endswith(f'.{file_ext}') return match + + +class OvalDataSource(DataSource): + + @staticmethod + def create_purl(pkg_name: str, pkg_version: str, pkg_data: Mapping): + """ + Note: pkg_data must include 'type' of package + """ + return PackageURL(name=pkg_name, version=pkg_version, **pkg_data) + + @staticmethod + def _collect_pkgs(parsed_oval_data: Mapping) -> Set: + """ + Helper method, used for loading the API. It expects data from + OvalParser.get_data() . + """ + all_pkgs = set() + for definition_data in parsed_oval_data: + for test_data in definition_data['test_data']: + for package in test_data['package_list']: + all_pkgs.add(package) + + return all_pkgs + + def _fetch() -> Tuple[Mapping, Iterable[ET.ElementTree]]: + """ + This method contains logic to fetch OVAL files and yield them into + a tuple of file's metadata and it's ET.ElementTree. + Subclasses must implement this method. + """ + raise NotImplementedError + + def added_advisories(self) -> List[Advisory]: + advisories = [] + for metadata, oval_file in self._fetch(): + advisories.extend(self.get_data_from_xml_doc(oval_file, metadata)) + return advisories + + def set_api(self, all_pkgs: Iterable[str]): + """ + This method loads the self.pkg_manager_api with the specified packages. It fetches + and caches the data about these packages exposes them through + self.pkg_manager_api.get() + """ + raise NotImplementedError + + def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> List[Advisory]: + """ + The orchestration method of the OvalDataSource. Breaks an OVAL xml + ElementTree into a list of Advisory. + """ + all_adv = [] + oval_doc = OvalParser(self.translations, xml_doc) + raw_data = oval_doc.get_data() + all_pkgs = self._collect_pkgs(raw_data) + self.set_api(all_pkgs) + for definition_data in raw_data: # definition_data -> Advisory + vuln_id = definition_data['vuln_id'] + description = definition_data['description'] + affected_purls = set() + safe_purls = set() + urls = definition_data['reference_urls'] + for test_data in definition_data['test_data']: + for package in test_data['package_list']: + pkg_name = package + aff_ver_range = test_data['version_ranges'] + all_versions = self.pkg_manager_api.get(package) + # This filter is to filter out long versions. + # 50 is limit because that's what db permits atm + all_versions = set( + filter( + lambda x: len(x) < 50, + all_versions)) + if not all_versions: + continue + affected_versions = set( + filter( + lambda x: x in aff_ver_range, + all_versions)) + safe_versions = all_versions - affected_versions + + for version in affected_versions: + pkg_url = self.create_purl( + pkg_name=pkg_name, pkg_version=version, pkg_data=pkg_metadata) + affected_purls.add(pkg_url) + + for version in safe_versions: + pkg_url = self.create_purl( + pkg_name=pkg_name, pkg_version=version, pkg_data=pkg_metadata) + safe_purls.add(pkg_url) + + all_adv.append( + Advisory( + summary=description, + impacted_package_urls=affected_purls, + resolved_package_urls=safe_purls, + cve_id=vuln_id, + reference_urls=urls)) + return all_adv diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index a1bb94f85..a6a6cce08 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -20,6 +20,7 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. + from vulnerabilities.importers.alpine_linux import AlpineDataSource from vulnerabilities.importers.archlinux import ArchlinuxDataSource from vulnerabilities.importers.debian import DebianDataSource diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index 644e2399c..093da5c01 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -31,14 +31,13 @@ import xml.etree.ElementTree as ET -from aiohttp import ClientSession +from aiohttp import ClientSession, ClientTimeout from aiohttp.client_exceptions import ClientResponseError import requests from packageurl import PackageURL -from vulnerabilities.data_source import DataSource, DataSourceConfiguration, Advisory -from vulnerabilities.importers import oval_parser +from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration, Advisory @dataclasses.dataclass @@ -46,7 +45,7 @@ class UbuntuConfiguration(DataSourceConfiguration): releases: list -class UbuntuDataSource(DataSource): +class UbuntuDataSource(OvalDataSource): CONFIG_CLASS = UbuntuConfiguration @@ -56,86 +55,21 @@ def __init__(self, *args, **kwargs): # set by default in the OvalParser, but we don't yet know # whether all OVAL providers use the same format self.translations = {'less than': '<'} - self._versions = VersionAPI() + self.pkg_manager_api = VersionAPI() def _fetch(self): base_url = 'https://people.canonical.com/~ubuntu-security/oval/' file_name = 'com.ubuntu.{}.cve.oval.xml.bz2' releases = self.config.releases for release in releases: + print("getting ", release) resp = requests.get(base_url + file_name.format(release)) extracted = bz2.decompress(resp.content) - yield ET.ElementTree(ET.fromstring(extracted.decode('utf-8'))) - - def added_advisories(self) -> List[Advisory]: - advisories = [] - for oval_file in self._fetch(): - advisories.extend(self.get_data_from_xml_doc(oval_file)) - return advisories - - @staticmethod - def _collect_pkgs(parsed_oval_data) -> Set: - all_pkgs = set() - for definition_data in parsed_oval_data: - for test_data in definition_data['test_data']: - for package in test_data['package_list']: - all_pkgs.add(package) - - return all_pkgs - - def get_data_from_xml_doc(self, xml_doc) -> List[Advisory]: - all_adv = [] - oval_doc = oval_parser.OvalParser(self.translations, xml_doc) - raw_data = oval_doc.get_data() - all_pkgs = self._collect_pkgs(raw_data) - - asyncio.run(self._versions.load_api(all_pkgs)) - - for definition_data in raw_data: # definition_data -> Advisory - vuln_id = definition_data['vuln_id'] - description = definition_data['description'] - affected_purls = set() - safe_purls = set() - urls = definition_data['reference_urls'] - for test_data in definition_data['test_data']: - for package in test_data['package_list']: - pkg_name = package - aff_ver_range = test_data['version_ranges'] - all_versions = self._versions.get(package) - # This filter is to filter out long versions. - # 50 is limit because that's what db permits atm - all_versions = set( - filter( - lambda x: len(x) < 50, - all_versions)) - if not all_versions: - continue - affected_versions = set( - filter( - lambda x: x in aff_ver_range, - all_versions)) - safe_versions = all_versions - affected_versions - - for version in affected_versions: - # should we add a qualifier like 'distro:ubuntu'? - pkg_url = PackageURL( - name=pkg_name, type='deb', version=version) - affected_purls.add(pkg_url) - - for version in safe_versions: - # should we add a qualifier like 'distro:ubuntu'? - pkg_url = PackageURL( - name=pkg_name, type='deb', version=version) - safe_purls.add(pkg_url) - - all_adv.append( - Advisory( - summary=description, - impacted_package_urls=affected_purls, - resolved_package_urls=safe_purls, - cve_id=vuln_id, - reference_urls=urls)) - return all_adv + print("done ") + yield ({'type': 'deb'}, ET.ElementTree(ET.fromstring(extracted.decode('utf-8')))) + + def set_api(self, packages): + asyncio.run(self.pkg_manager_api.load_api(packages)) class VersionAPI: @@ -146,7 +80,9 @@ def get(self, package_name: str) -> Set[str]: return self.cache[package_name] async def load_api(self, pkg_set): - async with ClientSession() as session: + # This is debatable + timeout = ClientTimeout(total=None) + async with ClientSession(raise_for_status=True, timeout=timeout) as session: await asyncio.gather(*[self.set_api(pkg, session) for pkg in pkg_set if pkg not in self.cache]) @@ -156,9 +92,8 @@ async def set_api(self, pkg, session): 'source_name={}&exact_match=true'.format(pkg)) try: all_versions = set() - while(True): + while True: response = await session.request(method='GET', url=url) - response.raise_for_status() resp_json = await response.json() if resp_json['entries'] == []: self.cache[pkg] = {} diff --git a/vulnerabilities/importers/lib_oval.py b/vulnerabilities/lib_oval.py similarity index 100% rename from vulnerabilities/importers/lib_oval.py rename to vulnerabilities/lib_oval.py diff --git a/vulnerabilities/importers/oval_parser.py b/vulnerabilities/oval_parser.py similarity index 99% rename from vulnerabilities/importers/oval_parser.py rename to vulnerabilities/oval_parser.py index 9e2ebf6cd..f951391f4 100755 --- a/vulnerabilities/importers/oval_parser.py +++ b/vulnerabilities/oval_parser.py @@ -30,7 +30,7 @@ from dephell_specifier import RangeSpecifier -from vulnerabilities.importers.lib_oval import ( +from vulnerabilities.lib_oval import ( OvalDefinition, OvalDocument, OvalTest, OvalObject, OvalState) diff --git a/vulnerabilities/tests/test_suse.py b/vulnerabilities/tests/test_suse.py index 45d73eda7..1d3a7045d 100644 --- a/vulnerabilities/tests/test_suse.py +++ b/vulnerabilities/tests/test_suse.py @@ -5,7 +5,7 @@ from dephell_specifier import RangeSpecifier -from vulnerabilities.importers.oval_parser import OvalParser +from vulnerabilities.oval_parser import OvalParser BASE_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 31ae12356..6ac022104 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -8,7 +8,7 @@ from dephell_specifier import RangeSpecifier from packageurl import PackageURL -from vulnerabilities.importers.oval_parser import OvalParser +from vulnerabilities.oval_parser import OvalParser from vulnerabilities.importers.ubuntu import UbuntuDataSource from vulnerabilities.data_source import Advisory @@ -265,6 +265,6 @@ def test_get_data_from_xml_doc(self, mock_write): cve_id='CVE-2016-8703')} xml_doc = ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml")) - data = set(ubuntu_data_src.get_data_from_xml_doc(xml_doc)) + data = set(ubuntu_data_src.get_data_from_xml_doc(xml_doc,{"type":"deb"})) assert expected_data == data From 0a857ed2813f2bf0dad9c1d7900cecff884141e9 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sat, 6 Jun 2020 12:37:02 +0530 Subject: [PATCH 7/9] Store etags and utilise them, cleanup the ubunutu VersionAPI Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/ubuntu.py | 34 ++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index 093da5c01..c8796fe66 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -43,6 +43,7 @@ @dataclasses.dataclass class UbuntuConfiguration(DataSourceConfiguration): releases: list + etags: dict class UbuntuDataSource(OvalDataSource): @@ -62,15 +63,32 @@ def _fetch(self): file_name = 'com.ubuntu.{}.cve.oval.xml.bz2' releases = self.config.releases for release in releases: - print("getting ", release) - resp = requests.get(base_url + file_name.format(release)) + file_url = base_url + file_name.format(release) + if not self.create_etag(file_url): + continue + resp = requests.get(file_url) extracted = bz2.decompress(resp.content) - print("done ") - yield ({'type': 'deb'}, ET.ElementTree(ET.fromstring(extracted.decode('utf-8')))) + yield ( + {'type': 'deb', 'namespace': 'ubuntu'}, + ET.ElementTree(ET.fromstring(extracted.decode('utf-8'))) + ) def set_api(self, packages): asyncio.run(self.pkg_manager_api.load_api(packages)) + def create_etag(self, url): + + etag = requests.head(url).headers.get('ETag') + if not etag: + # Kind of inaccurate to return True since etag is + # not created + return True + elif url in self.config.etags: + if self.config.etags[url] == etag: + return False + self.config.etags[url] = etag + return True + class VersionAPI: def __init__(self, cache: Mapping[str, Set[str]] = None): @@ -80,13 +98,13 @@ def get(self, package_name: str) -> Set[str]: return self.cache[package_name] async def load_api(self, pkg_set): - # This is debatable - timeout = ClientTimeout(total=None) - async with ClientSession(raise_for_status=True, timeout=timeout) as session: + async with ClientSession(raise_for_status=True) as session: await asyncio.gather(*[self.set_api(pkg, session) for pkg in pkg_set if pkg not in self.cache]) async def set_api(self, pkg, session): + if pkg in self.cache: + return url = ('https://api.launchpad.net/1.0/ubuntu/+archive/' 'primary?ws.op=getPublishedSources&' 'source_name={}&exact_match=true'.format(pkg)) @@ -105,5 +123,5 @@ async def set_api(self, pkg, session): else: break self.cache[pkg] = all_versions - except ClientResponseError: + except (ClientResponseError, asyncio.exceptions.TimeoutError): self.cache[pkg] = {} From 82ce33e5fa373522b108b398426de8c435206970 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sat, 6 Jun 2020 13:53:51 +0530 Subject: [PATCH 8/9] Cleanup OvalDataSource and UbuntuDataSource, add etags to ubuntu migration script Signed-off-by: Shivam Sandbhor --- vulnerabilities/data_source.py | 2 +- vulnerabilities/importers/ubuntu.py | 16 ++++++++++------ .../migrations/0009_ubuntu_importer.py | 3 ++- vulnerabilities/tests/test_ubuntu.py | 11 ++++++++--- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 171169021..cc46d162b 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -488,4 +488,4 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis resolved_package_urls=safe_purls, cve_id=vuln_id, reference_urls=urls)) - return all_adv + return self.batch_advisories(all_adv) diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index c8796fe66..78eaf7ee8 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -31,13 +31,12 @@ import xml.etree.ElementTree as ET -from aiohttp import ClientSession, ClientTimeout +from aiohttp import ClientSession from aiohttp.client_exceptions import ClientResponseError import requests -from packageurl import PackageURL -from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration, Advisory +from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration @dataclasses.dataclass @@ -69,9 +68,14 @@ def _fetch(self): resp = requests.get(file_url) extracted = bz2.decompress(resp.content) yield ( - {'type': 'deb', 'namespace': 'ubuntu'}, - ET.ElementTree(ET.fromstring(extracted.decode('utf-8'))) - ) + {'type': 'deb', 'namespace': 'ubuntu'}, + ET.ElementTree(ET.fromstring(extracted.decode('utf-8'))) + ) + # In case every file is latest, _fetch won't yield anything(due to checking for new etags), + # this would return None to added_advisories + # which will cause error, hence this + # function return an empty list + return [] def set_api(self, packages): asyncio.run(self.pkg_manager_api.load_api(packages)) diff --git a/vulnerabilities/migrations/0009_ubuntu_importer.py b/vulnerabilities/migrations/0009_ubuntu_importer.py index 0c2ef3a5a..638df9025 100644 --- a/vulnerabilities/migrations/0009_ubuntu_importer.py +++ b/vulnerabilities/migrations/0009_ubuntu_importer.py @@ -31,7 +31,8 @@ def add_ubuntu_importer(apps, _): license='', last_run=None, data_source='UbuntuDataSource', - data_source_cfg={'releases':['bionic','trusty','focal','eoan','xenial']}, + data_source_cfg={'releases':['bionic','trusty','focal','eoan','xenial'], + 'etags':{}}, ) diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 6ac022104..93f52d100 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -169,6 +169,9 @@ def test_get_data(self): async def mock(a,b): pass +def return_adv(_,a): + return a + class TestUbuntuDataSource(unittest.TestCase): @classmethod @@ -185,7 +188,7 @@ def setUpClass(cls): def test_get_data_from_xml_doc(self, mock_write): data_source_cfg = { - 'releases': 'eg-ubuntu'} + 'releases': 'eg-ubuntu',"etags":{}} ubuntu_data_src = UbuntuDataSource( batch_size=1, config=data_source_cfg) expected_data = { @@ -265,6 +268,8 @@ def test_get_data_from_xml_doc(self, mock_write): cve_id='CVE-2016-8703')} xml_doc = ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml")) - data = set(ubuntu_data_src.get_data_from_xml_doc(xml_doc,{"type":"deb"})) - + # Dirty quick patch to deal with batch_advisories + with patch('vulnerabilities.importers.ubuntu.UbuntuDataSource.batch_advisories', + new=return_adv): + data = {i for i in ubuntu_data_src.get_data_from_xml_doc(xml_doc,{"type":"deb"})} assert expected_data == data From 4e19a120fea2d25a93d343dd572c4498d9d236b1 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sat, 6 Jun 2020 18:08:03 +0530 Subject: [PATCH 9/9] Add tests for etags and change added_advisories to updated_advisories in OvalDataSource Signed-off-by: Shivam Sandbhor --- vulnerabilities/data_source.py | 48 ++++++++++++++----- vulnerabilities/tests/test_data_source.py | 58 ++++++++++++++++++++++- vulnerabilities/tests/test_ubuntu.py | 31 ++++++++---- 3 files changed, 115 insertions(+), 22 deletions(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index cc46d162b..8270dc65c 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -392,10 +392,15 @@ def _include_file( class OvalDataSource(DataSource): - + """ + All data sources which collect data from OVAL files must inherit from this + `OvalDataSource` class. Subclasses must implement the methods `_fetch` and `set_api`. + """ @staticmethod - def create_purl(pkg_name: str, pkg_version: str, pkg_data: Mapping): + def create_purl(pkg_name: str, pkg_version: str, pkg_data: Mapping) -> PackageURL: """ + Helper method for creating different purls for subclasses without them reimplementing + get_data_from_xml_doc method Note: pkg_data must include 'type' of package """ return PackageURL(name=pkg_name, version=pkg_version, **pkg_data) @@ -404,7 +409,7 @@ def create_purl(pkg_name: str, pkg_version: str, pkg_data: Mapping): def _collect_pkgs(parsed_oval_data: Mapping) -> Set: """ Helper method, used for loading the API. It expects data from - OvalParser.get_data() . + OvalParser.get_data(). """ all_pkgs = set() for definition_data in parsed_oval_data: @@ -419,27 +424,42 @@ def _fetch() -> Tuple[Mapping, Iterable[ET.ElementTree]]: This method contains logic to fetch OVAL files and yield them into a tuple of file's metadata and it's ET.ElementTree. Subclasses must implement this method. + + Note: Mapping MUST INCLUDE "type" key. Example values of Mapping + {"type":"deb","qualifiers":{"distro":"buster"} } + """ raise NotImplementedError - def added_advisories(self) -> List[Advisory]: + def updated_advisories(self) -> List[Advisory]: + """ + Note: metadata MUST INCLUDE "type" key, implement _fetch accordingly. + """ advisories = [] for metadata, oval_file in self._fetch(): advisories.extend(self.get_data_from_xml_doc(oval_file, metadata)) - return advisories + return self.batch_advisories(advisories) def set_api(self, all_pkgs: Iterable[str]): """ This method loads the self.pkg_manager_api with the specified packages. It fetches - and caches the data about these packages exposes them through - self.pkg_manager_api.get() + and caches all the versions of these packages and exposes them through + self.pkg_manager_api.get(). Example + + >>> self.set_api(['electron']) + Assume 'electron' has only versions 1.0.0 and 1.2.0 + >>> assert self.pkg_manager_api.get('electron') == {'1.0.0','1.2.0'} + """ raise NotImplementedError def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> List[Advisory]: """ - The orchestration method of the OvalDataSource. Breaks an OVAL xml - ElementTree into a list of Advisory. + The orchestration method of the OvalDataSource. This method breaks an OVAL xml + ElementTree into a list of `Advisory`. + + Note: pkg_metadata MUST INCLUDE "type" key. Example value of pkg_metadata, + {"type":"deb","qualifiers":{"distro":"buster"} } """ all_adv = [] oval_doc = OvalParser(self.translations, xml_doc) @@ -447,18 +467,22 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis all_pkgs = self._collect_pkgs(raw_data) self.set_api(all_pkgs) for definition_data in raw_data: # definition_data -> Advisory + + # These fields are definition level, i.e common for all + # elements connected/linked to an OvalDefinition vuln_id = definition_data['vuln_id'] description = definition_data['description'] affected_purls = set() safe_purls = set() urls = definition_data['reference_urls'] + for test_data in definition_data['test_data']: for package in test_data['package_list']: pkg_name = package aff_ver_range = test_data['version_ranges'] all_versions = self.pkg_manager_api.get(package) - # This filter is to filter out long versions. - # 50 is limit because that's what db permits atm + # This filter is for filtering out long versions. + # 50 is limit because that's what db permits atm. all_versions = set( filter( lambda x: len(x) < 50, @@ -488,4 +512,4 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis resolved_package_urls=safe_purls, cve_id=vuln_id, reference_urls=urls)) - return self.batch_advisories(all_adv) + return all_adv diff --git a/vulnerabilities/tests/test_data_source.py b/vulnerabilities/tests/test_data_source.py index b26c4eaf4..315c8e5c1 100644 --- a/vulnerabilities/tests/test_data_source.py +++ b/vulnerabilities/tests/test_data_source.py @@ -27,16 +27,29 @@ from unittest import TestCase from unittest.mock import MagicMock from unittest.mock import patch +import xml.etree.ElementTree as ET import pygit2 import pytest +from packageurl import PackageURL -from vulnerabilities.data_source import GitDataSource, _include_file +from vulnerabilities.data_source import GitDataSource, _include_file, OvalDataSource from vulnerabilities.data_source import InvalidConfigurationError +from vulnerabilities.oval_parser import OvalParser BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data/') +def load_oval_data(): + etrees_of_oval = {} + for f in os.listdir(TEST_DATA): + if f.endswith('oval_data.xml'): + path = os.path.join(TEST_DATA, f) + provider = f.split("_")[0] + etrees_of_oval[provider] = ET.parse(path) + return etrees_of_oval + + def mk_ds(**kwargs): # just for convenience, since this is a manadory parameter we always pass a value @@ -331,3 +344,46 @@ def test_file_changes_include_fixed_advisories(self): assert len(added_files) == 0 assert len(updated_files) == 1 assert os.path.join(self.repodir, 'crates/hyper/RUSTSEC-2020-0008.toml') in updated_files + +class TestOvalDataSource(TestCase): + + @classmethod + def setUpClass(cls): + cls.oval_data_src = OvalDataSource(1) + + def test_create_purl(self): + purl1 = PackageURL(name="ffmpeg",type="test",version="1.2.0") + + assert purl1 == self.oval_data_src.create_purl(pkg_name="ffmpeg", + pkg_version="1.2.0", pkg_data={"type":"test"}) + + purl2 = PackageURL(name="notepad",type="example",version="7.9.6",namespace="ns", + qualifiers={"distro":"sample"},subpath="root") + assert purl2 == self.oval_data_src.create_purl(pkg_name="notepad", + pkg_version="7.9.6",pkg_data={ + "namespace":"ns","qualifiers":{"distro":"sample"}, + "subpath":"root","type":"example" + } + ) + + def test__collect_pkgs(self): + + xmls = load_oval_data() + + expected_suse_pkgs = {'cacti-spine', 'apache2-mod_perl', 'cacti', 'apache2-mod_perl-devel'} + expected_ubuntu_pkgs = {'potrace', 'tor'} + + translations = {"less than": "<"} + + found_suse_pkgs = self.oval_data_src._collect_pkgs( + OvalParser(translations,xmls['suse']).get_data()) + + found_ubuntu_pkgs = self.oval_data_src._collect_pkgs( + OvalParser(translations,xmls['ubuntu']).get_data()) + + assert found_suse_pkgs == expected_suse_pkgs + assert found_ubuntu_pkgs == expected_ubuntu_pkgs + + + + diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 93f52d100..14aa69bc9 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -1,6 +1,7 @@ import os import unittest from unittest.mock import patch +from unittest.mock import MagicMock import xml.etree.ElementTree as ET from collections import OrderedDict import asyncio @@ -15,6 +16,9 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/") +class MockResponse: + + headers = {"ETag":"0x1234"} class TestUbuntuOvalParser(unittest.TestCase): @classmethod @@ -165,7 +169,7 @@ def test_get_data(self): assert expected_data == self.parsed_oval.get_data() -#This is horrible, there must be a better way +#This is horrible, there might be a better way async def mock(a,b): pass @@ -176,7 +180,10 @@ class TestUbuntuDataSource(unittest.TestCase): @classmethod def setUpClass(cls): - pass + data_source_cfg = { + 'releases': 'eg-ubuntu',"etags":{}} + cls.ubuntu_data_src = UbuntuDataSource( + batch_size=1, config=data_source_cfg) @patch( 'vulnerabilities.importers.ubuntu.VersionAPI.get', @@ -186,11 +193,6 @@ def setUpClass(cls): '2.14-2'}) @patch('vulnerabilities.importers.ubuntu.VersionAPI.load_api',new=mock) def test_get_data_from_xml_doc(self, mock_write): - - data_source_cfg = { - 'releases': 'eg-ubuntu',"etags":{}} - ubuntu_data_src = UbuntuDataSource( - batch_size=1, config=data_source_cfg) expected_data = { Advisory( summary=('Tor before 0.2.8.9 and 0.2.9.x before 0.2.9.4-alpha had ' @@ -268,8 +270,19 @@ def test_get_data_from_xml_doc(self, mock_write): cve_id='CVE-2016-8703')} xml_doc = ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml")) - # Dirty quick patch to deal with batch_advisories + # Dirty quick patch to mock batch_advisories with patch('vulnerabilities.importers.ubuntu.UbuntuDataSource.batch_advisories', new=return_adv): - data = {i for i in ubuntu_data_src.get_data_from_xml_doc(xml_doc,{"type":"deb"})} + data = {i for i in self.ubuntu_data_src.get_data_from_xml_doc(xml_doc,{"type":"deb"})} assert expected_data == data + + def test_create_etag(self): + + assert self.ubuntu_data_src.config.etags == {} + with patch('vulnerabilities.importers.ubuntu.requests.head', return_value=MockResponse()): + assert True == self.ubuntu_data_src.create_etag("https://example.org") + assert self.ubuntu_data_src.config.etags == {"https://example.org":"0x1234"} + assert False == self.ubuntu_data_src.create_etag("https://example.org") + + +