Add Debian OVAL importer

vulnerabilities/data_source.py

@@ -479,6 +479,8 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
             for test_data in definition_data['test_data']:
                 for package in test_data['package_list']:
                     pkg_name = package
+                    if package and len(pkg_name) >= 50:
+                        continue
                     aff_ver_range = test_data['version_ranges']
                     all_versions = self.pkg_manager_api.get(package)
                     # This filter is for filtering out long versions.

vulnerabilities/import_runner.py

@@ -160,7 +160,7 @@ def _get_or_create_vulnerability(advisory: Advisory) -> Tuple[models.Vulnerabili
 
 
 def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]:
-    version = packageurl.normalize_version(p.version, encode=True)
+    version = p.version
 
     query_kwargs = {
         'name': packageurl.normalize_name(p.name, p.type, encode=True),

vulnerabilities/importers/__init__.py

@@ -31,3 +31,4 @@
 from vulnerabilities.importers.ubuntu import UbuntuDataSource
 from vulnerabilities.importers.retiredotnet import RetireDotnetDataSource
 from vulnerabilities.importers.suse_backports import SUSEBackportsDataSource
+from vulnerabilities.importers.debian_oval import DebianOvalDataSource

vulnerabilities/importers/debian_oval.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+
+import asyncio
+import dataclasses
+from typing import Iterable
+from typing import List
+from typing import Mapping
+from typing import Set
+import xml.etree.ElementTree as ET
+
+
+from aiohttp import ClientSession
+from aiohttp.client_exceptions import ClientResponseError, ServerDisconnectedError
+import requests
+
+
+from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration
+
+
+@dataclasses.dataclass
+class DebianOvalConfiguration(DataSourceConfiguration):
+    releases: list
+    etags: dict
+
+
+class DebianOvalDataSource(OvalDataSource):
+
+    CONFIG_CLASS = DebianOvalConfiguration
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # we could avoid setting translations, and have it
+        # set by default in the OvalParser, but we don't yet know
+        # whether all OVAL providers use the same format
+        self.translations = {'less than': '<'}
+        self.pkg_manager_api = VersionAPI()
+
+    def _fetch(self):
+        base_url = 'https://www.debian.org/security/oval/'
+        file_name = 'oval-definitions-{}.xml'
+        releases = self.config.releases
+        for release in releases:
+            file_url = base_url + file_name.format(release)
+            if not self.create_etag(file_url):
+                continue
+            resp = requests.get(file_url).content
+            yield (
+                {'type': 'deb', 'namespace': 'debian',
+                    'qualifiers': {'distro': release}
+                 },
+                ET.ElementTree(ET.fromstring(resp.decode('utf-8')))
+            )
+        return []
+
+    def set_api(self, packages):
+        asyncio.run(self.pkg_manager_api.load_api(packages))
+
+    def create_etag(self, url):
+        etag = requests.head(url).headers.get('ETag')
+        if not etag:
+            return True
+        elif url in self.config.etags:
+            if self.config.etags[url] == etag:
+                return False
+        self.config.etags[url] = etag
+        return True
+
+
+class VersionAPI:
+    def __init__(self, cache: Mapping[str, Set[str]] = None):
+        self.cache = cache or {}
+
+    def get(self, package_name: str) -> Set[str]:
+        return self.cache[package_name]
+
+    async def load_api(self, pkg_set):
+        # Need to set the headers, because the Debian API upgrades
+        # the connection to HTTP 2.0
+        async with ClientSession(
+            raise_for_status=True,
+            headers={'Connection': 'keep-alive'}
+        ) as session:
+            await asyncio.gather(*[self.set_api(pkg, session)
+                                   for pkg in pkg_set if pkg not in self.cache])
+
+    async def set_api(self, pkg, session, retry_count=5):
+        if pkg in self.cache:
+            return
+        url = ('https://sources.debian.org/api/src/{}'.format(pkg))
+        try:
+            all_versions = set()
+            response = await session.request(method='GET', url=url)
+            resp_json = await response.json()
+
+            if resp_json.get('error') or not resp_json.get('versions'):
+                self.cache[pkg] = {}
+                return
+            for release in resp_json['versions']:
+                all_versions.add(release['version'])
+
+            self.cache[pkg] = all_versions
+        # TODO : Handle ServerDisconnectedError by using some sort of
+        # retry mechanism
+        except (ClientResponseError, asyncio.exceptions.TimeoutError, ServerDisconnectedError):
+            self.cache[pkg] = {}

vulnerabilities/migrations/0012_debian_oval_importer.py

@@ -0,0 +1,54 @@
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from django.db import migrations
+
+
+def add_debian_oval_importer(apps, _):
+    Importer = apps.get_model('vulnerabilities', 'Importer')
+
+    Importer.objects.create(
+        name='debian_oval',
+        license='',
+        last_run=None,
+        data_source='DebianOvalDataSource',
+        data_source_cfg={'releases':['wheezy','stretch','jessie','buster'],
+        'etags':{}},
+    )
+
+
+def remove_debian_oval_importer(apps, _):
+    Importer = apps.get_model('vulnerabilities', 'Importer')
+    qs = Importer.objects.filter(name='debian_oval')
+    if qs:
+        qs[0].delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0011_susebackports_importer'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_debian_oval_importer, remove_debian_oval_importer),
+    ]

vulnerabilities/oval_parser.py

@@ -57,6 +57,10 @@ def get_data(self) -> List[Dict]:
             definition_data = {'test_data': []}
             definition_data['description'] = definition.getMetadata(
             ).getDescription()  # this could use some data cleaning
+
+            if not definition_data['description']:
+                definition_data['description'] = ''
+
             definition_data['vuln_id'] = self.get_vuln_id_from_definition(
                 definition)
             definition_data['reference_urls'] = self.get_urls_from_definition(
@@ -166,6 +170,9 @@ def get_urls_from_definition(definition: OvalDefinition) -> Set[str]:
 
     @staticmethod
     def get_vuln_id_from_definition(definition):
+        # SUSE and Ubuntu OVAL files will get cves via this loop
         for child in definition.element.iter():
             if child.get('ref_id'):
                 return child.get('ref_id')
+        # Debian OVAL files will get cves via this
+        return definition.getMetadata().getTitle()

vulnerabilities/tests/test_data/debian_oval_data.xml

@@ -0,0 +1,128 @@
+<?xml version='1.0' encoding='utf-8'?>
+<oval_definitions xmlns:ind-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" xmlns:linux-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux" xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5" xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:unix-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#unix" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5" xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent independent-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#linux linux-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#unix unix-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-common-5 oval-common-schema.xsd">
+    <generator>
+        <oval:product_name>Debian</oval:product_name>
+        <oval:schema_version>5.11.2</oval:schema_version>
+        <oval:timestamp>2020-06-08T23:31:02.188-04:00</oval:timestamp>
+    </generator>
+    <definitions>
+        <definition class="vulnerability" id="oval:org.debian:def:20011593" version="1">
+        <metadata>
+            <title>CVE-2001-1593</title>
+            <affected family="unix">
+            <platform>Debian GNU/Linux 7</platform>
+            <product>a2ps</product>
+            </affected>
+            <description>security update</description>
+            <debian>
+            <date>2014-03-31</date>
+            <moreinfo>
+    DSA-2892
+    Several vulnerabilities have been found in a2ps, an &lt;q&gt;Anything to
+    PostScript&lt;/q&gt; converter and pretty-printer. The Common Vulnerabilities and
+    Exposures project identifies the following problems:
+    The spy_user function which is called when a2ps is invoked with the
+        --debug flag insecurely used temporary files.
+    Brian M. Carlson reported that a2ps's fixps script does not invoke
+        gs with the -dSAFER option. Consequently executing fixps on a
+        malicious PostScript file could result in files being deleted or
+        arbitrary commands being executed with the privileges of the user
+        running fixps.
+    </moreinfo>
+            </debian>
+        </metadata>
+        <criteria comment="Release section" operator="AND">
+            <criterion comment="Debian 7 is installed" test_ref="oval:org.debian.oval:tst:1"/>
+            <criteria comment="Architecture section" operator="OR">
+            <criteria comment="Architecture independent section" operator="AND">
+                <criterion comment="all architecture" test_ref="oval:org.debian.oval:tst:2"/>
+                <criterion comment="a2ps DPKG is earlier than 1:4.14-1.1+deb7u1" test_ref="oval:org.debian.oval:tst:3"/>
+            </criteria>
+            </criteria>
+        </criteria>
+        </definition>
+        <definition class="vulnerability" id="oval:org.debian:def:20022443" version="1">
+        <metadata>
+            <title>CVE-2002-2443</title>
+            <affected family="unix">
+            <platform>Debian GNU/Linux 7</platform>
+            <product>krb5</product>
+            </affected>
+            <description>denial of service</description>
+            <debian>
+            <date>2013-05-29</date>
+            <moreinfo>
+    DSA-2701
+    It was discovered that the kpasswd service running on UDP port 464
+    could respond to response packets, creating a packet loop and a denial
+    of service condition.
+    </moreinfo>
+            </debian>
+        </metadata>
+        <criteria comment="Release section" operator="AND">
+            <criterion comment="Debian 7 is installed" test_ref="oval:org.debian.oval:tst:1"/>
+            <criteria comment="Architecture section" operator="OR">
+            <criteria comment="Architecture independent section" operator="AND">
+                <criterion comment="all architecture" test_ref="oval:org.debian.oval:tst:2"/>
+                <criterion comment="krb5 DPKG is earlier than 1.10.1+dfsg-5+deb7u1" test_ref="oval:org.debian.oval:tst:4"/>
+            </criteria>
+            </criteria>
+        </criteria>
+        </definition>
+    </definitions>
+
+    <tests>
+        <textfilecontent54_test check="all" check_existence="at_least_one_exists" comment="Debian GNU/Linux 7 is installed" id="oval:org.debian.oval:tst:1" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent">
+        <object object_ref="oval:org.debian.oval:obj:1"/>
+        <state state_ref="oval:org.debian.oval:ste:1"/>
+        </textfilecontent54_test>
+
+        <uname_test check="all" check_existence="at_least_one_exists" comment="Installed architecture is all" id="oval:org.debian.oval:tst:2" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#unix">
+            <object object_ref="oval:org.debian.oval:obj:2"/>
+            </uname_test>
+            <dpkginfo_test check="all" check_existence="at_least_one_exists" comment="a2ps is earlier than 1:4.14-1.1+deb7u1" id="oval:org.debian.oval:tst:3" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+            <object object_ref="oval:org.debian.oval:obj:3"/>
+            <state state_ref="oval:org.debian.oval:ste:2"/>
+            </dpkginfo_test>
+
+            <dpkginfo_test check="all" check_existence="at_least_one_exists" comment="krb5 is earlier than 1.10.1+dfsg-5+deb7u1" id="oval:org.debian.oval:tst:4" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+      <object object_ref="oval:org.debian.oval:obj:4"/>
+      <state state_ref="oval:org.debian.oval:ste:3"/>
+    </dpkginfo_test>
+
+    </tests>
+
+    <objects>
+
+    <textfilecontent54_object id="oval:org.debian.oval:obj:1" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent">
+      <path>/etc</path>
+      <filename>debian_version</filename>
+      <pattern operation="pattern match">(\d+)\.\d</pattern>
+      <instance datatype="int">1</instance>
+    </textfilecontent54_object>
+
+    <uname_object id="oval:org.debian.oval:obj:2" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#unix"/>
+        <dpkginfo_object id="oval:org.debian.oval:obj:3" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+        <name>a2ps</name>
+    </dpkginfo_object>
+       <dpkginfo_object id="oval:org.debian.oval:obj:4" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+      <name>krb5</name>
+    </dpkginfo_object>
+
+
+    </objects>
+
+    <states>
+        <textfilecontent54_state id="oval:org.debian.oval:ste:1" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent">
+        <subexpression operation="equals">7</subexpression>
+        </textfilecontent54_state>
+        <dpkginfo_state id="oval:org.debian.oval:ste:2" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+        <evr datatype="debian_evr_string" operation="less than">0:1:4.14-1.1+deb7u1</evr>
+        </dpkginfo_state>
+          <dpkginfo_state id="oval:org.debian.oval:ste:3" version="1" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux">
+      <evr datatype="debian_evr_string" operation="less than">0:1.10.1+dfsg-5+deb7u1</evr>
+    </dpkginfo_state>
+    </states>
+
+
+</oval_definitions>