Harden Ubuntu OVAL importer

This importer no longer worked with failures to process some version
ranges as RangeSpecifier(). These change imprivae error handling and
crude logging using print statements for now.
Also apply minor code formatting.

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

vulnerabilities/data_source.py

@@ -446,29 +446,48 @@ def _collect_pkgs(parsed_oval_data: Mapping) -> Set:
 
     def _fetch(self) -> Tuple[Mapping, Iterable[ET.ElementTree]]:
         """
-        This method  contains logic to fetch OVAL files and yield them into
-        a tuple of file's metadata and it's ET.ElementTree.
+        Return a two-tuple of ({mapping of Package URL data}, it's ET.ElementTree)
         Subclasses must implement this method.
 
-        Note: Mapping MUST INCLUDE "type" key. Example values of Mapping
-              {"type":"deb","qualifiers":{"distro":"buster"} }
+        Note:  Package URL data MUST INCLUDE a Package URL "type" key so
+        implement _fetch() accordingly.
+        For example::
 
+              {"type":"deb","qualifiers":{"distro":"buster"} }
         """
+        # TODO: enforce that we receive the proper data here
         raise NotImplementedError
 
     def updated_advisories(self) -> List[Advisory]:
-        """
-        Note: metadata MUST INCLUDE "type" key, implement _fetch accordingly.
-        """
-        for metadata, oval_file in self._fetch():
+        for purl_data, oval_etree in self._fetch():
+            if 'type' not in purl_data:
+                ets = (oval_etree and ET.tostring(oval_etree)) or 'NO DATA'
+                msg = (
+                    "Failed to get updated_advisories for Ubuntu: purl_data is "
+                    f"missing a package type {purl_data!r}\n"
+                    f"with OVAL XML:\n"
+                    f"{ets}\n"
+                    f"... continuing..."
+                )
+                print(msg)
+                logger.error(msg)
+                continue
+
             try:
-                oval_data = self.get_data_from_xml_doc(oval_file, metadata)
+                oval_data = self.get_data_from_xml_doc(oval_etree, purl_data)
                 yield oval_data
             except Exception:
-                logger.error(
-                    f"Failed to get updated_advisories: {oval_file!r} "
-                    "with {metadata!r}:\n" + traceback.format_exc()
+                ets = (oval_etree and ET.tostring(oval_etree)) or 'NO DATA'
+                tb = traceback.format_exc()
+                msg = (
+                    f"Failed to get updated_advisories for Ubuntu:"
+                    f"with {purl_data!r}\n"
+                    f"and with OVAL XML:\n"
+                    f"{ets}\n{tb}\n"
+                    f"... continuing..."
                 )
+                print(msg)
+                logger.error(msg)
                 continue
 
     def set_api(self, all_pkgs: Iterable[str]):
@@ -486,34 +505,66 @@ def set_api(self, all_pkgs: Iterable[str]):
 
     def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> List[Advisory]:
         """
-        The orchestration method of the OvalDataSource. This method breaks an OVAL xml
-        ElementTree into a list of `Advisory`.
+        The orchestration method of the OvalDataSource. This method breaks an
+        OVAL xml ElementTree into a list of `Advisory`.
 
-        Note: pkg_metadata MUST INCLUDE "type" key. Example value of pkg_metadata,
+        Note: pkg_metadata is a mapping of Package URL data that MUST INCLUDE
+        "type" key.
+
+        Example value of pkg_metadata:
               {"type":"deb","qualifiers":{"distro":"buster"} }
         """
+        if 'type' not in pkg_metadata:
+            ets = xml_doc and ET.tostring(xml_doc) or 'NO DATA'
+            msg = (
+                "Failed to get_data_from_xml_doc: pkg_metadata is "
+                f"missing a package type {pkg_metadata!r}\n"
+                f"with OVAL XML:\n"
+                f"{ets}"
+            )
+            print(msg)
+            logger.error(msg)
+            raise Exception(msg)
+
         all_adv = []
         oval_doc = OvalParser(self.translations, xml_doc)
-        raw_data = oval_doc.get_data()
+        try:
+            raw_data = oval_doc.get_data()
+        except Exception:
+            ets = xml_doc and ET.tostring(xml_doc) or 'NO DATA'
+            tb = traceback.format_exc()
+            msg = (
+                f"Failed to get_data_from_xml_doc:"
+                f"with {pkg_metadata!r}\n"
+                f"and with OVAL XML:\n"
+                f"{ets}\n{tb}"
+            )
+            print(msg)
+            logger.error(msg)
+            raise Exception(msg)
+
         all_pkgs = self._collect_pkgs(raw_data)
         self.set_api(all_pkgs)
-        for definition_data in raw_data:  # definition_data -> Advisory
 
-            # These fields are definition level, i.e common for all
-            # elements connected/linked to an OvalDefinition
-            vuln_id = definition_data["vuln_id"]
-            description = definition_data["description"]
+        # convert definition_data to Advisory objects
+        for definition_data in raw_data:
+            # These fields are definition level, i.e common for all elements
+            # connected/linked to an OvalDefinition
+            vuln_id = definition_data['vuln_id']
+            description = definition_data['description']
             affected_purls = set()
             safe_purls = set()
-            references = [Reference(url=url) for url in definition_data["reference_urls"]]
-
-            for test_data in definition_data["test_data"]:
-                for package in test_data["package_list"]:
+            references = [Reference(url=url) for url in definition_data['reference_urls']]
+            for test_data in definition_data['test_data']:
+                for package in test_data['package_list']:
                     pkg_name = package
                     if package and len(pkg_name) >= 50:
                         continue
-                    aff_ver_range = test_data["version_ranges"]
+                    aff_ver_range = test_data['version_ranges'] or set()
                     all_versions = self.pkg_manager_api.get(package)
+
+                    # FIXME: what is this 50 DB limit? that's too small for versions
+                    # FIXME: we should not drop data this way
                     # This filter is for filtering out long versions.
                     # 50 is limit because that's what db permits atm.
                     all_versions = set(filter(lambda x: len(x) < 50, all_versions))
@@ -524,13 +575,17 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
 
                     for version in affected_versions:
                         pkg_url = self.create_purl(
-                            pkg_name=pkg_name, pkg_version=version, pkg_data=pkg_metadata
+                            pkg_name=pkg_name,
+                            pkg_version=version,
+                            pkg_data=pkg_metadata,
                         )
                         affected_purls.add(pkg_url)
 
                     for version in safe_versions:
                         pkg_url = self.create_purl(
-                            pkg_name=pkg_name, pkg_version=version, pkg_data=pkg_metadata
+                            pkg_name=pkg_name,
+                            pkg_version=version,
+                            pkg_data=pkg_metadata,
                         )
                         safe_purls.add(pkg_url)
 

vulnerabilities/importers/ubuntu.py

@@ -59,21 +59,25 @@ def __init__(self, *args, **kwargs):
 
     def _fetch(self):
         releases = self.config.releases
-        for release in releases:
+        for i, release in enumerate(releases, 1):
             file_url = f"https://people.canonical.com/~ubuntu-security/oval/com.ubuntu.{release}.cve.oval.xml.bz2"  # nopep8
-            if not create_etag(data_src=self, url=file_url, etag_key="ETag"):
+#             if not create_etag(data_src=self, url=file_url, etag_key="ETag"):
+#                 print(f"Ubuntu Oval Etag not changed, not re-fetching: {file_url}")
+#                 continue
+
+            print(f"Fetching Ubuntu Oval: {file_url}")
+            response = requests.get(file_url)
+            if response.status_code != requests.codes.ok:
+                print(f"Failed to fetch Ubuntu Oval: HTTP {response.status_code} : {file_url}")
                 continue
-            resp = requests.get(file_url)
-            extracted = bz2.decompress(resp.content)
+
+            extracted = bz2.decompress(response.content)
             yield (
                 {"type": "deb", "namespace": "ubuntu"},
                 ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))),
             )
-        # In case every file is latest, _fetch won't yield anything(due to checking for new etags),
-        # this would return None to added_advisories
-        # which will cause error, hence this
-        # function return an empty list
-        return []
+
+        print(f"Fetched {i} Ubuntu Oval releases https://people.canonical.com/~ubuntu-security/oval/")  # nopep8
 
     def set_api(self, packages):
         asyncio.run(self.pkg_manager_api.load_api(packages))

vulnerabilities/lib_oval.py

@@ -27,71 +27,71 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-"""Library to simplify working with the OVAL XML structure
+# Library to simplify working with the OVAL XML structure
 
 
-Authors: Gunnar Engelbach <Gunnar.Engelbach@ThreatGuard.com>
+# Authors: Gunnar Engelbach <Gunnar.Engelbach@ThreatGuard.com>
 
 
 
-Available classes:
-    - OvalDocument:    operations at the OVAL document level, such as reading in an existing OVAL document from
-file, creating a new one, finding or adding OVAL elements
-    - OvalElement:    the base class for OVAL elements.  Implements a few common methods inherited by the
-subclasses for definition, test, state, object, and variable
-    - OvalDefinition:    a type of OVAL element with certain attributes available.  Additional classes used by the OvalDefinition class:
-        - OvalMetadata:    the metadata associated with a definition, which includes the definition title and description.  Metadata also contains:
-            - OvalAffected:    The family and platforms affected by this definition
-            - OvalRepositoryInformation:    Additional information added by the OVAL repository
-    - OvalTest:    for working with OVAL test elements
-    - OvalObject:    for working with OVAL object elements
-    - OvalState:    for working with OVAL state elements
-    - OvalVariable:    for working with OVAL variable elements
+# Available classes:
+#     - OvalDocument:    operations at the OVAL document level, such as reading in an existing OVAL document from
+# file, creating a new one, finding or adding OVAL elements
+#     - OvalElement:    the base class for OVAL elements.  Implements a few common methods inherited by the
+# subclasses for definition, test, state, object, and variable
+#     - OvalDefinition:    a type of OVAL element with certain attributes available.  Additional classes used by the OvalDefinition class:
+#         - OvalMetadata:    the metadata associated with a definition, which includes the definition title and description.  Metadata also contains:
+#             - OvalAffected:    The family and platforms affected by this definition
+#             - OvalRepositoryInformation:    Additional information added by the OVAL repository
+#     - OvalTest:    for working with OVAL test elements
+#     - OvalObject:    for working with OVAL object elements
+#     - OvalState:    for working with OVAL state elements
+#     - OvalVariable:    for working with OVAL variable elements
 
 
 
-Available exceptions:
-    - None at this time
+# Available exceptions:
+#     - None at this time
 
 
-:Usage:
+# :Usage:
 
-1. Create an OvalDocument:
+# 1. Create an OvalDocument:
 
-    >>> tree = ElementTree()
-    >>> tree.parse("OvalTest.xml")
-    >>> document = OvalDocument(tree)
+#     >>> tree = ElementTree()
+#     >>> tree.parse("OvalTest.xml")
+#     >>> document = OvalDocument(tree)
 
-2. Find an oval element within the loaded document:
+# 2. Find an oval element within the loaded document:
 
-    >>> element = document.getElementByID("oval:org.mitre.oval:def:22382")
-    >>> if element is not None:
-    >>>    ....
+#     >>> element = document.getElementByID("oval:org.mitre.oval:def:22382")
+#     >>> if element is not None:
+#     >>>    ....
 
-3. Read an XML file with a single OVAL Definition (error checking omitted for brevity):
+# 3. Read an XML file with a single OVAL Definition (error checking omitted for brevity):
 
-    >>> tree = ElementTree()    
-    >>> tree.parse('test-definition.xml')
-    >>> root = tree.getroot()    
-    >>> definition = lib_oval.OvalDefinition(root)
+#     >>> tree = ElementTree()    
+#     >>> tree.parse('test-definition.xml')
+#     >>> root = tree.getroot()    
+#     >>> definition = lib_oval.OvalDefinition(root)
 
-4. Change information in the definition from #3 and write the changes
+# 4. Change information in the definition from #3 and write the changes
 
-    >>> meta = definition.getMetadata()
-    >>> repo = meta.getOvalRepositoryInformation()
-    >>> repo.setMinimumSchemaVersion("5.9")
-    >>> tree.write("outfilename.xml", UTF-8", True)
+#     >>> meta = definition.getMetadata()
+#     >>> repo = meta.getOvalRepositoryInformation()
+#     >>> repo.setMinimumSchemaVersion("5.9")
+#     >>> tree.write("outfilename.xml", UTF-8", True)
 
 
 
 
 
-TODO:
-    - Add exceptions that give more detail about why a value of None is sometimes returned
-    - Expand use of find() to allow for the possibility that the XML document is not using namespaces
-    - Lots of pydoc to be added
-    - Redo getter/setter for OvalRepository status elements.
-"""
+# TODO:
+#     - Add exceptions that give more detail about why a value of None is sometimes returned
+#     - Expand use of find() to allow for the possibility that the XML document is not using namespaces
+#     - Lots of pydoc to be added
+#     - Redo getter/setter for OvalRepository status elements.
+
 
 import os, xml.etree
 from xml.etree import ElementTree