Avoid collecting 'binaries' of OVAL files and add tests for UbuntuDataSource(yet to be implemented)

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

Author: sbs2001

vulnerabilities/importers/oval_parser.py

@@ -1,3 +1,26 @@
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import re
 from typing import Optional
 from typing import List
 from typing import Dict
@@ -103,8 +126,9 @@ def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]:
             if var.get('var_ref'):
                 var_elem = self.oval_document.getElementByID(
                     var.get('var_ref'))
-                for vals in var_elem.element:
-                    pkg_list.append(vals.text)
+                comment  = var_elem.element.get('comment')
+                pkg_name = re.match("'.+'", comment).group().replace("'","")
+                pkg_list.append(pkg_name)
             else:
                 pkg_list.append(var.text)
 

vulnerabilities/tests/test_ubuntu.py

@@ -1,12 +1,16 @@
 import os
 import unittest
+from unittest.mock import patch
 import xml.etree.ElementTree as ET
+from collections import OrderedDict
+import asyncio
 
 from dephell_specifier import RangeSpecifier
-
+from packageurl import PackageURL
 
 from vulnerabilities.importers.oval_parser import OvalParser
-
+from vulnerabilities.importers.ubuntu import UbuntuDataSource
+from vulnerabilities.data_source import Advisory
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 TEST_DATA = os.path.join(BASE_DIR, "test_data/")
@@ -84,8 +88,8 @@ def test_get_pkgs_from_obj(self):
         pkg_set1 = set(self.parsed_oval.get_pkgs_from_obj(obj_t2))
         pkg_set2 = set(self.parsed_oval.get_pkgs_from_obj(obj_t1))
 
-        assert pkg_set1 == {"potrace", "libpotrace0"}
-        assert pkg_set2 == {"tor", "tor-geoipdb"}
+        assert pkg_set1 == {"potrace"}
+        assert pkg_set2 == {"tor"}
 
     def test_get_versionsrngs_from_state(self):
 
@@ -99,15 +103,15 @@ def test_get_versionsrngs_from_state(self):
 
         assert self.parsed_oval.get_versionsrngs_from_state(state_1) == exp_range_1
         assert self.parsed_oval.get_versionsrngs_from_state(state_2) == exp_range_2
-    
+
     def test_get_urls_from_definition(self):
 
         def1_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html',
                      'https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/',
                      'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703'
                     }
 
-        assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1)            
+        assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1)
 
         def2_urls = {'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html',
                      'https://trac.torproject.org/projects/tor/ticket/20384',
@@ -117,15 +121,15 @@ def test_get_urls_from_definition(self):
                      'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860',
                     }
 
-        assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2)       
+        assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2)
 
     def test_get_data(self):
 
         expected_data = [
             {
                 "test_data": [
                     {
-                        "package_list": ["libpotrace0", "potrace"],
+                        "package_list": ["potrace"],
                         "version_ranges": RangeSpecifier("<1.14-2"),
                     }
                 ],
@@ -141,7 +145,7 @@ def test_get_data(self):
             {
                 "test_data": [
                     {
-                        "package_list": ["tor", "tor-geoipdb"],
+                        "package_list": ["tor"],
                         "version_ranges": RangeSpecifier("<0.2.8.9-1ubuntu1"),
                     }
                 ],
@@ -160,4 +164,107 @@ def test_get_data(self):
         ]
 
         assert expected_data == self.parsed_oval.get_data()
-    
+
+#This is horrible, there must be a better way
+async def mock(a,b):
+    pass
+
+class TestUbuntuDataSource(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        pass
+
+    @patch(
+        'vulnerabilities.importers.ubuntu.VersionAPI.get',
+        return_value={
+            '0.3.0',
+            '0.2.0',
+            '2.14-2'})
+    @patch('vulnerabilities.importers.ubuntu.VersionAPI.load_api',new=mock)
+    def test_get_data_from_xml_doc(self, mock_write):
+
+        data_source_cfg = {
+            'releases': 'eg-ubuntu'}
+        ubuntu_data_src = UbuntuDataSource(
+            batch_size=1, config=data_source_cfg)
+        expected_data = {
+            Advisory(
+                summary=('Tor before 0.2.8.9 and 0.2.9.x before 0.2.9.4-alpha had '
+                'internal functions that were entitled to expect that buf_t data had '
+                'NUL termination, but the implementation of or/buffers.c did not '
+                'ensure that NUL termination was present, which allows remote '
+                'attackers to cause a denial of service (client, hidden '
+                'service, relay, or authority crash) via crafted data.'),
+                impacted_package_urls={
+                    PackageURL(
+                        type='deb',
+                        namespace=None,
+                        name='tor',
+                        version='0.2.0',
+                        qualifiers=OrderedDict(),
+                        subpath=None)},
+                resolved_package_urls={
+                    PackageURL(
+                        type='deb',
+                        namespace=None,
+                        name='tor',
+                        version='0.3.0',
+                        qualifiers=OrderedDict(),
+                        subpath=None),
+                    PackageURL(
+                        type='deb',
+                        namespace=None,
+                        name='tor',
+                        version='2.14-2',
+                        qualifiers=OrderedDict(),
+                        subpath=None)},
+                reference_urls={
+                    'http://www.openwall.com/lists/oss-security/2016/10/18/11',
+                    'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8860',
+                    'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8860.html',
+                    'https://github.com/torproject/tor/commit/3cea86eb2fbb65949673eb4ba8ebb695c87a57ce',
+                    'https://blog.torproject.org/blog/tor-0289-released-important-fixes',
+                    'https://trac.torproject.org/projects/tor/ticket/20384'},
+                reference_ids=[],
+                cve_id='CVE-2016-8860'),
+            Advisory(
+                summary=('Heap-based buffer overflow in the bm_readbody_bmp function'
+                ' in bitmap_io.c in potrace before 1.13 allows remote attackers to '
+                'have unspecified impact via a crafted BMP image, a different '
+                'vulnerability than CVE-2016-8698, CVE-2016-8699, '
+                'CVE-2016-8700, CVE-2016-8701, and CVE-2016-8702.'),
+                impacted_package_urls={
+                    PackageURL(
+                        type='deb',
+                        namespace=None,
+                        name='potrace',
+                        version='0.3.0',
+                        qualifiers=OrderedDict(),
+                        subpath=None),
+                    PackageURL(
+                        type='deb',
+                        namespace=None,
+                        name='potrace',
+                        version='0.2.0',
+                        qualifiers=OrderedDict(),
+                        subpath=None)},
+                resolved_package_urls={
+                    PackageURL(
+                        type='deb',
+                        namespace=None,
+                        name='potrace',
+                        version='2.14-2',
+                        qualifiers=OrderedDict(),
+                        subpath=None)},
+                reference_urls={
+                    'http://people.canonical.com/~ubuntu-security/cve/2016/CVE-2016-8703.html',
+                    'https://blogs.gentoo.org/ago/2016/08/08/potrace-multiplesix-heap-based-buffer-overflow-in-bm_readbody_bmp-bitmap_io-c/',
+                    'https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2016-8703'},
+                reference_ids=[],
+                cve_id='CVE-2016-8703')}
+
+        xml_doc = ET.parse(os.path.join(TEST_DATA, "ubuntu_oval_data.xml"))
+        data = set(ubuntu_data_src.get_data_from_xml_doc(xml_doc))
+
+        assert expected_data == data