diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 8270dc65c..f67d0be01 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -479,6 +479,8 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis for test_data in definition_data['test_data']: for package in test_data['package_list']: pkg_name = package + if package and len(pkg_name) >= 50: + continue aff_ver_range = test_data['version_ranges'] all_versions = self.pkg_manager_api.get(package) # This filter is for filtering out long versions. diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index e54aadd0b..4590dee4b 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -160,7 +160,7 @@ def _get_or_create_vulnerability(advisory: Advisory) -> Tuple[models.Vulnerabili def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]: - version = packageurl.normalize_version(p.version, encode=True) + version = p.version query_kwargs = { 'name': packageurl.normalize_name(p.name, p.type, encode=True), diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 6bf486f20..ad76c402c 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -31,3 +31,4 @@ from vulnerabilities.importers.ubuntu import UbuntuDataSource from vulnerabilities.importers.retiredotnet import RetireDotnetDataSource from vulnerabilities.importers.suse_backports import SUSEBackportsDataSource +from vulnerabilities.importers.debian_oval import DebianOvalDataSource diff --git a/vulnerabilities/importers/debian_oval.py b/vulnerabilities/importers/debian_oval.py new file mode 100644 index 000000000..934743b04 --- /dev/null +++ b/vulnerabilities/importers/debian_oval.py @@ -0,0 +1,126 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + + +import asyncio +import dataclasses +from typing import Iterable +from typing import List +from typing import Mapping +from typing import Set +import xml.etree.ElementTree as ET + + +from aiohttp import ClientSession +from aiohttp.client_exceptions import ClientResponseError, ServerDisconnectedError +import requests + + +from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration + + +@dataclasses.dataclass +class DebianOvalConfiguration(DataSourceConfiguration): + releases: list + etags: dict + + +class DebianOvalDataSource(OvalDataSource): + + CONFIG_CLASS = DebianOvalConfiguration + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # we could avoid setting translations, and have it + # set by default in the OvalParser, but we don't yet know + # whether all OVAL providers use the same format + self.translations = {'less than': '<'} + self.pkg_manager_api = VersionAPI() + + def _fetch(self): + base_url = 'https://www.debian.org/security/oval/' + file_name = 'oval-definitions-{}.xml' + releases = self.config.releases + for release in releases: + file_url = base_url + file_name.format(release) + if not self.create_etag(file_url): + continue + resp = requests.get(file_url).content + yield ( + {'type': 'deb', 'namespace': 'debian', + 'qualifiers': {'distro': release} + }, + ET.ElementTree(ET.fromstring(resp.decode('utf-8'))) + ) + return [] + + def set_api(self, packages): + asyncio.run(self.pkg_manager_api.load_api(packages)) + + def create_etag(self, url): + etag = requests.head(url).headers.get('ETag') + if not etag: + return True + elif url in self.config.etags: + if self.config.etags[url] == etag: + return False + self.config.etags[url] = etag + return True + + +class VersionAPI: + def __init__(self, cache: Mapping[str, Set[str]] = None): + self.cache = cache or {} + + def get(self, package_name: str) -> Set[str]: + return self.cache[package_name] + + async def load_api(self, pkg_set): + # Need to set the headers, because the Debian API upgrades + # the connection to HTTP 2.0 + async with ClientSession( + raise_for_status=True, + headers={'Connection': 'keep-alive'} + ) as session: + await asyncio.gather(*[self.set_api(pkg, session) + for pkg in pkg_set if pkg not in self.cache]) + + async def set_api(self, pkg, session, retry_count=5): + if pkg in self.cache: + return + url = ('https://sources.debian.org/api/src/{}'.format(pkg)) + try: + all_versions = set() + response = await session.request(method='GET', url=url) + resp_json = await response.json() + + if resp_json.get('error') or not resp_json.get('versions'): + self.cache[pkg] = {} + return + for release in resp_json['versions']: + all_versions.add(release['version']) + + self.cache[pkg] = all_versions + # TODO : Handle ServerDisconnectedError by using some sort of + # retry mechanism + except (ClientResponseError, asyncio.exceptions.TimeoutError, ServerDisconnectedError): + self.cache[pkg] = {} diff --git a/vulnerabilities/migrations/0012_debian_oval_importer.py b/vulnerabilities/migrations/0012_debian_oval_importer.py new file mode 100644 index 000000000..66137548e --- /dev/null +++ b/vulnerabilities/migrations/0012_debian_oval_importer.py @@ -0,0 +1,54 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +from django.db import migrations + + +def add_debian_oval_importer(apps, _): + Importer = apps.get_model('vulnerabilities', 'Importer') + + Importer.objects.create( + name='debian_oval', + license='', + last_run=None, + data_source='DebianOvalDataSource', + data_source_cfg={'releases':['wheezy','stretch','jessie','buster'], + 'etags':{}}, + ) + + +def remove_debian_oval_importer(apps, _): + Importer = apps.get_model('vulnerabilities', 'Importer') + qs = Importer.objects.filter(name='debian_oval') + if qs: + qs[0].delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0011_susebackports_importer'), + ] + + operations = [ + migrations.RunPython(add_debian_oval_importer, remove_debian_oval_importer), + ] \ No newline at end of file diff --git a/vulnerabilities/oval_parser.py b/vulnerabilities/oval_parser.py index f951391f4..69d129b7c 100755 --- a/vulnerabilities/oval_parser.py +++ b/vulnerabilities/oval_parser.py @@ -57,6 +57,10 @@ def get_data(self) -> List[Dict]: definition_data = {'test_data': []} definition_data['description'] = definition.getMetadata( ).getDescription() # this could use some data cleaning + + if not definition_data['description']: + definition_data['description'] = '' + definition_data['vuln_id'] = self.get_vuln_id_from_definition( definition) definition_data['reference_urls'] = self.get_urls_from_definition( @@ -166,6 +170,9 @@ def get_urls_from_definition(definition: OvalDefinition) -> Set[str]: @staticmethod def get_vuln_id_from_definition(definition): + # SUSE and Ubuntu OVAL files will get cves via this loop for child in definition.element.iter(): if child.get('ref_id'): return child.get('ref_id') + # Debian OVAL files will get cves via this + return definition.getMetadata().getTitle() diff --git a/vulnerabilities/tests/test_data/debian_oval_data.xml b/vulnerabilities/tests/test_data/debian_oval_data.xml new file mode 100644 index 000000000..6aabf53bb --- /dev/null +++ b/vulnerabilities/tests/test_data/debian_oval_data.xml @@ -0,0 +1,128 @@ + + + + Debian + 5.11.2 + 2020-06-08T23:31:02.188-04:00 + + + + + CVE-2001-1593 + + Debian GNU/Linux 7 + a2ps + + security update + + 2014-03-31 + + DSA-2892 + Several vulnerabilities have been found in a2ps, an <q>Anything to + PostScript</q> converter and pretty-printer. The Common Vulnerabilities and + Exposures project identifies the following problems: + The spy_user function which is called when a2ps is invoked with the + --debug flag insecurely used temporary files. + Brian M. Carlson reported that a2ps's fixps script does not invoke + gs with the -dSAFER option. Consequently executing fixps on a + malicious PostScript file could result in files being deleted or + arbitrary commands being executed with the privileges of the user + running fixps. + + + + + + + + + + + + + + + + CVE-2002-2443 + + Debian GNU/Linux 7 + krb5 + + denial of service + + 2013-05-29 + + DSA-2701 + It was discovered that the kpasswd service running on UDP port 464 + could respond to response packets, creating a packet loop and a denial + of service condition. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /etc + debian_version + (\d+)\.\d + 1 + + + + + a2ps + + + krb5 + + + + + + + + 7 + + + 0:1:4.14-1.1+deb7u1 + + + 0:1.10.1+dfsg-5+deb7u1 + + + + + \ No newline at end of file diff --git a/vulnerabilities/tests/test_debian_oval.py b/vulnerabilities/tests/test_debian_oval.py new file mode 100644 index 000000000..51c5408ae --- /dev/null +++ b/vulnerabilities/tests/test_debian_oval.py @@ -0,0 +1,118 @@ +import os +import unittest +from unittest.mock import patch +from unittest.mock import MagicMock +import xml.etree.ElementTree as ET +from collections import OrderedDict +import asyncio + +from dephell_specifier import RangeSpecifier +from packageurl import PackageURL + +from vulnerabilities.oval_parser import OvalParser +from vulnerabilities.importers.debian_oval import DebianOvalDataSource +from vulnerabilities.data_source import Advisory + + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/") + + +async def mock(a, b): + pass + + +def return_adv(_, a): + return a + + +class TestDebianOvalDataSource(unittest.TestCase): + + @classmethod + def setUpClass(cls): + data_source_cfg = { + 'releases': 'eg-debian_oval', "etags": {}} + cls.debian_oval_data_src = DebianOvalDataSource( + batch_size=1, config=data_source_cfg) + + @patch( + 'vulnerabilities.importers.debian_oval.VersionAPI.get', + return_value={ + '0:1.11.1+dfsg-5+deb7u1', + '0:0.11.1+dfsg-5+deb7u1', + '2.3.9'}) + @patch('vulnerabilities.importers.debian_oval.VersionAPI.load_api', new=mock) + def test_get_data_from_xml_doc(self, mock_write): + expected_data = { + Advisory( + summary='denial of service', + impacted_package_urls={ + PackageURL( + type='deb', + namespace=None, + name='krb5', + version='0:0.11.1+dfsg-5+deb7u1', + qualifiers=OrderedDict([('distro', 'wheezy')]), + subpath=None + )}, + resolved_package_urls={ + PackageURL( + type='deb', + namespace=None, + name='krb5', + version='0:1.11.1+dfsg-5+deb7u1', + qualifiers=OrderedDict([('distro', 'wheezy')]), + subpath=None), + PackageURL( + type='deb', + namespace=None, + name='krb5', + version='2.3.9', + qualifiers=OrderedDict([('distro', 'wheezy')]), + subpath=None)}, + reference_urls=set(), + reference_ids=[], + cve_id='CVE-2002-2443' + ), + Advisory( + summary='security update', + impacted_package_urls={ + PackageURL( + type='deb', + namespace=None, + name='a2ps', + version='0:0.11.1+dfsg-5+deb7u1', + qualifiers=OrderedDict([('distro', 'wheezy')]), + subpath=None + )}, + resolved_package_urls={ + PackageURL(type='deb', + namespace=None, + name='a2ps', + version='2.3.9', + qualifiers=OrderedDict([('distro', 'wheezy')]), + subpath=None), + PackageURL(type='deb', + namespace=None, + name='a2ps', + version='0:1.11.1+dfsg-5+deb7u1', + qualifiers=OrderedDict([('distro', 'wheezy')]), + subpath=None)}, + reference_urls=set(), + reference_ids=[], + cve_id='CVE-2001-1593') + + } + + xml_doc = ET.parse(os.path.join(TEST_DATA, "debian_oval_data.xml")) + # Dirty quick patch to mock batch_advisories + with patch('vulnerabilities.importers.debian_oval.DebianOvalDataSource.batch_advisories', + new=return_adv): + data = {i for i in self.debian_oval_data_src.get_data_from_xml_doc( + xml_doc, + { + "type": "deb", + "qualifiers": {"distro": "wheezy"} + }) + } + assert expected_data == data