diff --git a/requirements.txt b/requirements.txt index 874fb4100..31dd46e25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -aiohttp==3.6.2 asgiref==3.2.7 attrs==19.3.0 backcall==0.1.0 diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 8270dc65c..ca68d1b83 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -35,13 +35,10 @@ from typing import Sequence from typing import Set from typing import Tuple -import xml.etree.ElementTree as ET import pygit2 from packageurl import PackageURL -from vulnerabilities.oval_parser import OvalParser - @dataclasses.dataclass class Advisory: @@ -260,8 +257,7 @@ def file_changes( return {str(p) for p in path.glob(glob) if p.is_file()}, set() - return self._collect_file_changes( - subdir=subdir, recursive=recursive, file_ext=file_ext) + return self._collect_file_changes(subdir=subdir, recursive=recursive, file_ext=file_ext) def _collect_file_changes( self, @@ -273,8 +269,7 @@ def _collect_file_changes( previous_commit = None added_files, updated_files = set(), set() - for commit in self._repo.walk( - self._repo.head.target, pygit2.GIT_SORT_TIME): + for commit in self._repo.walk(self._repo.head.target, pygit2.GIT_SORT_TIME): commit_time = commit.commit_time + commit.commit_time_offset # convert to UTC if commit_time < self.cutoff_timestamp: @@ -285,16 +280,13 @@ def _collect_file_changes( continue for d in commit.tree.diff_to_tree(previous_commit.tree).deltas: - if not _include_file( - d.new_file.path, subdir, recursive, file_ext) or d.is_binary: + if not _include_file(d.new_file.path, subdir, recursive, file_ext) or d.is_binary: continue - abspath = os.path.join( - self.config.working_directory, d.new_file.path) + abspath = os.path.join(self.config.working_directory, d.new_file.path) # TODO # Just filtering on the two status values for "added" and "modified" is too - # simplistic. This does not cover file renames, copies & - # deletions. + # simplistic. This does not cover file renames, copies & deletions. if d.status == pygit2.GIT_DELTA_ADDED: added_files.add(abspath) elif d.status == pygit2.GIT_DELTA_MODIFIED: @@ -389,127 +381,3 @@ def _include_file( match = match and path.endswith(f'.{file_ext}') return match - - -class OvalDataSource(DataSource): - """ - All data sources which collect data from OVAL files must inherit from this - `OvalDataSource` class. Subclasses must implement the methods `_fetch` and `set_api`. - """ - @staticmethod - def create_purl(pkg_name: str, pkg_version: str, pkg_data: Mapping) -> PackageURL: - """ - Helper method for creating different purls for subclasses without them reimplementing - get_data_from_xml_doc method - Note: pkg_data must include 'type' of package - """ - return PackageURL(name=pkg_name, version=pkg_version, **pkg_data) - - @staticmethod - def _collect_pkgs(parsed_oval_data: Mapping) -> Set: - """ - Helper method, used for loading the API. It expects data from - OvalParser.get_data(). - """ - all_pkgs = set() - for definition_data in parsed_oval_data: - for test_data in definition_data['test_data']: - for package in test_data['package_list']: - all_pkgs.add(package) - - return all_pkgs - - def _fetch() -> Tuple[Mapping, Iterable[ET.ElementTree]]: - """ - This method contains logic to fetch OVAL files and yield them into - a tuple of file's metadata and it's ET.ElementTree. - Subclasses must implement this method. - - Note: Mapping MUST INCLUDE "type" key. Example values of Mapping - {"type":"deb","qualifiers":{"distro":"buster"} } - - """ - raise NotImplementedError - - def updated_advisories(self) -> List[Advisory]: - """ - Note: metadata MUST INCLUDE "type" key, implement _fetch accordingly. - """ - advisories = [] - for metadata, oval_file in self._fetch(): - advisories.extend(self.get_data_from_xml_doc(oval_file, metadata)) - return self.batch_advisories(advisories) - - def set_api(self, all_pkgs: Iterable[str]): - """ - This method loads the self.pkg_manager_api with the specified packages. It fetches - and caches all the versions of these packages and exposes them through - self.pkg_manager_api.get(). Example - - >>> self.set_api(['electron']) - Assume 'electron' has only versions 1.0.0 and 1.2.0 - >>> assert self.pkg_manager_api.get('electron') == {'1.0.0','1.2.0'} - - """ - raise NotImplementedError - - def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> List[Advisory]: - """ - The orchestration method of the OvalDataSource. This method breaks an OVAL xml - ElementTree into a list of `Advisory`. - - Note: pkg_metadata MUST INCLUDE "type" key. Example value of pkg_metadata, - {"type":"deb","qualifiers":{"distro":"buster"} } - """ - all_adv = [] - oval_doc = OvalParser(self.translations, xml_doc) - raw_data = oval_doc.get_data() - all_pkgs = self._collect_pkgs(raw_data) - self.set_api(all_pkgs) - for definition_data in raw_data: # definition_data -> Advisory - - # These fields are definition level, i.e common for all - # elements connected/linked to an OvalDefinition - vuln_id = definition_data['vuln_id'] - description = definition_data['description'] - affected_purls = set() - safe_purls = set() - urls = definition_data['reference_urls'] - - for test_data in definition_data['test_data']: - for package in test_data['package_list']: - pkg_name = package - aff_ver_range = test_data['version_ranges'] - all_versions = self.pkg_manager_api.get(package) - # This filter is for filtering out long versions. - # 50 is limit because that's what db permits atm. - all_versions = set( - filter( - lambda x: len(x) < 50, - all_versions)) - if not all_versions: - continue - affected_versions = set( - filter( - lambda x: x in aff_ver_range, - all_versions)) - safe_versions = all_versions - affected_versions - - for version in affected_versions: - pkg_url = self.create_purl( - pkg_name=pkg_name, pkg_version=version, pkg_data=pkg_metadata) - affected_purls.add(pkg_url) - - for version in safe_versions: - pkg_url = self.create_purl( - pkg_name=pkg_name, pkg_version=version, pkg_data=pkg_metadata) - safe_purls.add(pkg_url) - - all_adv.append( - Advisory( - summary=description, - impacted_package_urls=affected_purls, - resolved_package_urls=safe_purls, - cve_id=vuln_id, - reference_urls=urls)) - return all_adv diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index a6a6cce08..7f50ca158 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -20,7 +20,6 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. - from vulnerabilities.importers.alpine_linux import AlpineDataSource from vulnerabilities.importers.archlinux import ArchlinuxDataSource from vulnerabilities.importers.debian import DebianDataSource @@ -28,4 +27,3 @@ from vulnerabilities.importers.rust import RustDataSource from vulnerabilities.importers.safety_db import SafetyDbDataSource from vulnerabilities.importers.ruby import RubyDataSource -from vulnerabilities.importers.ubuntu import UbuntuDataSource diff --git a/vulnerabilities/lib_oval.py b/vulnerabilities/importers/lib_oval.py similarity index 100% rename from vulnerabilities/lib_oval.py rename to vulnerabilities/importers/lib_oval.py diff --git a/vulnerabilities/oval_parser.py b/vulnerabilities/importers/oval_parser.py similarity index 76% rename from vulnerabilities/oval_parser.py rename to vulnerabilities/importers/oval_parser.py index f951391f4..5752dbc5e 100755 --- a/vulnerabilities/oval_parser.py +++ b/vulnerabilities/importers/oval_parser.py @@ -1,26 +1,3 @@ -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnerableCode software is licensed under the Apache License version 2.0. -# Data generated with VulnerableCode require an acknowledgment. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnerableCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import re from typing import Optional from typing import List from typing import Dict @@ -30,7 +7,7 @@ from dephell_specifier import RangeSpecifier -from vulnerabilities.lib_oval import ( +from vulnerabilities.importers.lib_oval import ( OvalDefinition, OvalDocument, OvalTest, OvalObject, OvalState) @@ -126,9 +103,8 @@ def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]: if var.get('var_ref'): var_elem = self.oval_document.getElementByID( var.get('var_ref')) - comment = var_elem.element.get('comment') - pkg_name = re.match("'.+'", comment).group().replace("'", "") - pkg_list.append(pkg_name) + for vals in var_elem.element: + pkg_list.append(vals.text) else: pkg_list.append(var.text) diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index 78eaf7ee8..20389b186 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -1,3 +1,4 @@ +# # Copyright (c) 2017 nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/vulnerablecode/ # The VulnerableCode software is licensed under the Apache License version 2.0. @@ -20,112 +21,36 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. +from urllib.request import urlopen -import asyncio -import bz2 -import dataclasses -from typing import Iterable -from typing import List -from typing import Mapping -from typing import Set -import xml.etree.ElementTree as ET - - -from aiohttp import ClientSession -from aiohttp.client_exceptions import ClientResponseError -import requests - - -from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration - - -@dataclasses.dataclass -class UbuntuConfiguration(DataSourceConfiguration): - releases: list - etags: dict - - -class UbuntuDataSource(OvalDataSource): - - CONFIG_CLASS = UbuntuConfiguration - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # we could avoid setting translations, and have it - # set by default in the OvalParser, but we don't yet know - # whether all OVAL providers use the same format - self.translations = {'less than': '<'} - self.pkg_manager_api = VersionAPI() +import bs4 - def _fetch(self): - base_url = 'https://people.canonical.com/~ubuntu-security/oval/' - file_name = 'com.ubuntu.{}.cve.oval.xml.bz2' - releases = self.config.releases - for release in releases: - file_url = base_url + file_name.format(release) - if not self.create_etag(file_url): - continue - resp = requests.get(file_url) - extracted = bz2.decompress(resp.content) - yield ( - {'type': 'deb', 'namespace': 'ubuntu'}, - ET.ElementTree(ET.fromstring(extracted.decode('utf-8'))) - ) - # In case every file is latest, _fetch won't yield anything(due to checking for new etags), - # this would return None to added_advisories - # which will cause error, hence this - # function return an empty list - return [] - def set_api(self, packages): - asyncio.run(self.pkg_manager_api.load_api(packages)) +UBUNTU_ROOT_URL = 'https://people.canonical.com/~ubuntu-security/cve/main.html' - def create_etag(self, url): - etag = requests.head(url).headers.get('ETag') - if not etag: - # Kind of inaccurate to return True since etag is - # not created - return True - elif url in self.config.etags: - if self.config.etags[url] == etag: - return False - self.config.etags[url] = etag - return True +def extract_cves(html): + soup = bs4.BeautifulSoup(html, 'lxml') + # Exclude the header row which has no class attribute + rows = soup.find_all('tr', attrs={'class': True}) -class VersionAPI: - def __init__(self, cache: Mapping[str, Set[str]] = None): - self.cache = cache or {} + cves = [] + for row in rows: + columns = row.text.split() + cves.append({ + 'cve_id': columns[0], + 'package_name': columns[1], + 'vulnerability_status': row.get('class')[0], + }) - def get(self, package_name: str) -> Set[str]: - return self.cache[package_name] + return cves - async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: - await asyncio.gather(*[self.set_api(pkg, session) - for pkg in pkg_set if pkg not in self.cache]) - async def set_api(self, pkg, session): - if pkg in self.cache: - return - url = ('https://api.launchpad.net/1.0/ubuntu/+archive/' - 'primary?ws.op=getPublishedSources&' - 'source_name={}&exact_match=true'.format(pkg)) - try: - all_versions = set() - while True: - response = await session.request(method='GET', url=url) - resp_json = await response.json() - if resp_json['entries'] == []: - self.cache[pkg] = {} - break - for release in resp_json['entries']: - all_versions.add(release['source_package_version']) - if resp_json.get('next_collection_link'): - url = resp_json['next_collection_link'] - else: - break - self.cache[pkg] = all_versions - except (ClientResponseError, asyncio.exceptions.TimeoutError): - self.cache[pkg] = {} +def scrape_cves(): + """ + Runs the full scraping process of Ubuntu CVEs. + """ + html = urlopen(UBUNTU_ROOT_URL).read() + cves = extract_cves(html) + return cves diff --git a/vulnerabilities/migrations/0009_ubuntu_importer.py b/vulnerabilities/tests/test_data_dump.py similarity index 64% rename from vulnerabilities/migrations/0009_ubuntu_importer.py rename to vulnerabilities/tests/test_data_dump.py index 638df9025..1772bf8b6 100644 --- a/vulnerabilities/migrations/0009_ubuntu_importer.py +++ b/vulnerabilities/tests/test_data_dump.py @@ -1,3 +1,4 @@ +# # Copyright (c) 2017 nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/vulnerablecode/ # The VulnerableCode software is licensed under the Apache License version 2.0. @@ -20,35 +21,28 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -from django.db import migrations - - -def add_ubuntu_importer(apps, _): - Importer = apps.get_model('vulnerabilities', 'Importer') +import os - Importer.objects.create( - name='ubuntu', - license='', - last_run=None, - data_source='UbuntuDataSource', - data_source_cfg={'releases':['bionic','trusty','focal','eoan','xenial'], - 'etags':{}}, - ) +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, 'test_data/') -def remove_ubuntu_importer(apps, _): - Importer = apps.get_model('vulnerabilities', 'Importer') - qs = Importer.objects.filter(name='ubuntu') - if qs: - qs[0].delete() +def test_ubuntu_data_dump(setUbuntuData): + """ + Check basic data import + """ + assert Vulnerability.objects.filter(cve_id='CVE-2002-2439') + pkgs = Package.objects.filter(name='gcc-4.6') + assert pkgs -class Migration(migrations.Migration): + pkg = pkgs[0] + assert 'deb' == pkg.type + assert 'ubuntu' == pkg.namespace - dependencies = [ - ('vulnerabilities', '0008_ruby_importer'), - ] - operations = [ - migrations.RunPython(add_ubuntu_importer, remove_ubuntu_importer), - ] \ No newline at end of file +CVE_IDS = ('CVE-2018-11362', 'CVE-2018-11361', 'CVE-2018-11360', + 'CVE-2018-11359', 'CVE-2018-11358', 'CVE-2018-11357', + 'CVE-2018-11356', 'CVE-2018-11355', 'CVE-2018-11354') diff --git a/vulnerabilities/tests/test_data_source.py b/vulnerabilities/tests/test_data_source.py index 315c8e5c1..b26c4eaf4 100644 --- a/vulnerabilities/tests/test_data_source.py +++ b/vulnerabilities/tests/test_data_source.py @@ -27,29 +27,16 @@ from unittest import TestCase from unittest.mock import MagicMock from unittest.mock import patch -import xml.etree.ElementTree as ET import pygit2 import pytest -from packageurl import PackageURL -from vulnerabilities.data_source import GitDataSource, _include_file, OvalDataSource +from vulnerabilities.data_source import GitDataSource, _include_file from vulnerabilities.data_source import InvalidConfigurationError -from vulnerabilities.oval_parser import OvalParser BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, 'test_data/') -def load_oval_data(): - etrees_of_oval = {} - for f in os.listdir(TEST_DATA): - if f.endswith('oval_data.xml'): - path = os.path.join(TEST_DATA, f) - provider = f.split("_")[0] - etrees_of_oval[provider] = ET.parse(path) - return etrees_of_oval - - def mk_ds(**kwargs): # just for convenience, since this is a manadory parameter we always pass a value @@ -344,46 +331,3 @@ def test_file_changes_include_fixed_advisories(self): assert len(added_files) == 0 assert len(updated_files) == 1 assert os.path.join(self.repodir, 'crates/hyper/RUSTSEC-2020-0008.toml') in updated_files - -class TestOvalDataSource(TestCase): - - @classmethod - def setUpClass(cls): - cls.oval_data_src = OvalDataSource(1) - - def test_create_purl(self): - purl1 = PackageURL(name="ffmpeg",type="test",version="1.2.0") - - assert purl1 == self.oval_data_src.create_purl(pkg_name="ffmpeg", - pkg_version="1.2.0", pkg_data={"type":"test"}) - - purl2 = PackageURL(name="notepad",type="example",version="7.9.6",namespace="ns", - qualifiers={"distro":"sample"},subpath="root") - assert purl2 == self.oval_data_src.create_purl(pkg_name="notepad", - pkg_version="7.9.6",pkg_data={ - "namespace":"ns","qualifiers":{"distro":"sample"}, - "subpath":"root","type":"example" - } - ) - - def test__collect_pkgs(self): - - xmls = load_oval_data() - - expected_suse_pkgs = {'cacti-spine', 'apache2-mod_perl', 'cacti', 'apache2-mod_perl-devel'} - expected_ubuntu_pkgs = {'potrace', 'tor'} - - translations = {"less than": "<"} - - found_suse_pkgs = self.oval_data_src._collect_pkgs( - OvalParser(translations,xmls['suse']).get_data()) - - found_ubuntu_pkgs = self.oval_data_src._collect_pkgs( - OvalParser(translations,xmls['ubuntu']).get_data()) - - assert found_suse_pkgs == expected_suse_pkgs - assert found_ubuntu_pkgs == expected_ubuntu_pkgs - - - - diff --git a/vulnerabilities/tests/test_importers.py b/vulnerabilities/tests/test_importers.py new file mode 100644 index 000000000..417bab236 --- /dev/null +++ b/vulnerabilities/tests/test_importers.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +from os.path import dirname +from os.path import join + +from vulnerabilities.importers import ubuntu + + +def test_ubuntu_extract_cves(): + ubuntu_testfile = join(dirname(__file__), 'test_data', 'ubuntu_main.html') + + with open(ubuntu_testfile) as f: + test_input = f.read() + + cves = ubuntu.extract_cves(test_input) + + expected = { + 'cve_id': 'CVE-2002-2439', + 'package_name': 'gcc-4.6', + 'vulnerability_status': 'low' + } + assert expected == cves[0] + + expected = { + 'cve_id': 'CVE-2013-0157', + 'package_name': 'util-linux', + 'vulnerability_status': 'low', + } + assert expected == cves[50] + + expected = { + 'cve_id': 'CVE-2017-9986', + 'package_name': 'linux-lts-xenial', + 'vulnerability_status': 'medium', + } + assert expected == cves[-1] diff --git a/vulnerabilities/tests/test_suse.py b/vulnerabilities/tests/test_suse.py index 1d3a7045d..45d73eda7 100644 --- a/vulnerabilities/tests/test_suse.py +++ b/vulnerabilities/tests/test_suse.py @@ -5,7 +5,7 @@ from dephell_specifier import RangeSpecifier -from vulnerabilities.oval_parser import OvalParser +from vulnerabilities.importers.oval_parser import OvalParser BASE_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 14aa69bc9..b3636ddc2 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -1,24 +1,16 @@ import os import unittest -from unittest.mock import patch -from unittest.mock import MagicMock import xml.etree.ElementTree as ET -from collections import OrderedDict -import asyncio from dephell_specifier import RangeSpecifier -from packageurl import PackageURL -from vulnerabilities.oval_parser import OvalParser -from vulnerabilities.importers.ubuntu import UbuntuDataSource -from vulnerabilities.data_source import Advisory + +from vulnerabilities.importers.oval_parser import OvalParser + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/") -class MockResponse: - - headers = {"ETag":"0x1234"} class TestUbuntuOvalParser(unittest.TestCase): @classmethod @@ -92,8 +84,8 @@ def test_get_pkgs_from_obj(self): pkg_set1 = set(self.parsed_oval.get_pkgs_from_obj(obj_t2)) pkg_set2 = set(self.parsed_oval.get_pkgs_from_obj(obj_t1)) - assert pkg_set1 == {"potrace"} - assert pkg_set2 == {"tor"} + assert pkg_set1 == {"potrace", "libpotrace0"} + assert pkg_set2 == {"tor", "tor-geoipdb"} def test_get_versionsrngs_from_state(self): @@ -107,7 +99,7 @@ def test_get_versionsrngs_from_state(self): assert self.parsed_oval.get_versionsrngs_from_state(state_1) == exp_range_1 assert self.parsed_oval.get_versionsrngs_from_state(state_2) == exp_range_2 - + def test_get_urls_from_definition(self): def1_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html', @@ -115,7 +107,7 @@ def test_get_urls_from_definition(self): 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703' } - assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1) + assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1) def2_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html', 'https://trac.torproject.org/projects/tor/ticket/20384', @@ -125,7 +117,7 @@ def test_get_urls_from_definition(self): 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860', } - assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2) + assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2) def test_get_data(self): @@ -133,7 +125,7 @@ def test_get_data(self): { "test_data": [ { - "package_list": ["potrace"], + "package_list": ["libpotrace0", "potrace"], "version_ranges": RangeSpecifier("<1.14-2"), } ], @@ -149,7 +141,7 @@ def test_get_data(self): { "test_data": [ { - "package_list": ["tor"], + "package_list": ["tor", "tor-geoipdb"], "version_ranges": RangeSpecifier("<0.2.8.9-1ubuntu1"), } ], @@ -168,121 +160,4 @@ def test_get_data(self): ] assert expected_data == self.parsed_oval.get_data() - -#This is horrible, there might be a better way -async def mock(a,b): - pass - -def return_adv(_,a): - return a - -class TestUbuntuDataSource(unittest.TestCase): - - @classmethod - def setUpClass(cls): - data_source_cfg = { - 'releases': 'eg-ubuntu',"etags":{}} - cls.ubuntu_data_src = UbuntuDataSource( - batch_size=1, config=data_source_cfg) - - @patch( - 'vulnerabilities.importers.ubuntu.VersionAPI.get', - return_value={ - '0.3.0', - '0.2.0', - '2.14-2'}) - @patch('vulnerabilities.importers.ubuntu.VersionAPI.load_api',new=mock) - def test_get_data_from_xml_doc(self, mock_write): - expected_data = { - Advisory( - summary=('Tor before 0.2.8.9 and 0.2.9.x before 0.2.9.4-alpha had ' - 'internal functions that were entitled to expect that buf_t data had ' - 'NUL termination, but the implementation of or/buffers.c did not ' - 'ensure that NUL termination was present, which allows remote ' - 'attackers to cause a denial of service (client, hidden ' - 'service, relay, or authority crash) via crafted data.'), - impacted_package_urls={ - PackageURL( - type='deb', - namespace=None, - name='tor', - version='0.2.0', - qualifiers=OrderedDict(), - subpath=None)}, - resolved_package_urls={ - PackageURL( - type='deb', - namespace=None, - name='tor', - version='0.3.0', - qualifiers=OrderedDict(), - subpath=None), - PackageURL( - type='deb', - namespace=None, - name='tor', - version='2.14-2', - qualifiers=OrderedDict(), - subpath=None)}, - reference_urls={ - 'http://www.openwall.com/lists/oss-security/2016/10/18/11', - 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860', - 'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html', - 'https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce', - 'https://blog.torproject.org/blog/tor-0289-released-important-fixes', - 'https://trac.torproject.org/projects/tor/ticket/20384'}, - reference_ids=[], - cve_id='CVE-2016-8860'), - Advisory( - summary=('Heap-based buffer overflow in the bm_readbody_bmp function' - ' in bitmap_io.c in potrace before 1.13 allows remote attackers to ' - 'have unspecified impact via a crafted BMP image, a different ' - 'vulnerability than CVE-2016-8698, CVE-2016-8699, ' - 'CVE-2016-8700, CVE-2016-8701, and CVE-2016-8702.'), - impacted_package_urls={ - PackageURL( - type='deb', - namespace=None, - name='potrace', - version='0.3.0', - qualifiers=OrderedDict(), - subpath=None), - PackageURL( - type='deb', - namespace=None, - name='potrace', - version='0.2.0', - qualifiers=OrderedDict(), - subpath=None)}, - resolved_package_urls={ - PackageURL( - type='deb', - namespace=None, - name='potrace', - version='2.14-2', - qualifiers=OrderedDict(), - subpath=None)}, - reference_urls={ - 'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html', - 'https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/', - 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703'}, - reference_ids=[], - cve_id='CVE-2016-8703')} - - xml_doc = ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml")) - # Dirty quick patch to mock batch_advisories - with patch('vulnerabilities.importers.ubuntu.UbuntuDataSource.batch_advisories', - new=return_adv): - data = {i for i in self.ubuntu_data_src.get_data_from_xml_doc(xml_doc,{"type":"deb"})} - assert expected_data == data - - def test_create_etag(self): - - assert self.ubuntu_data_src.config.etags == {} - with patch('vulnerabilities.importers.ubuntu.requests.head', return_value=MockResponse()): - assert True == self.ubuntu_data_src.create_etag("https://example.org") - assert self.ubuntu_data_src.config.etags == {"https://example.org":"0x1234"} - assert False == self.ubuntu_data_src.create_etag("https://example.org") - - - +