Make RedHat CVE import more robust

This importer failed at times toward the end of the fetch step when
reaching (and trying repeatidly) to re-fetch the last page.
The fix consists in fetching larger batches at once (10K instead of 1k)
which measn fewer API calls and less risk to be throttled and properly
handling exceptions and HTTP response codes and breaking rather than
retrying. This also adds some minimal logging.

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

vulnerabilities/importers/redhat.py

@@ -38,40 +38,57 @@ class RedhatDataSource(DataSource):
     CONFIG_CLASS = DataSourceConfiguration
 
     def __enter__(self):
-        self.redhat_response = fetch()
 
-    def updated_advisories(self):
-        processed_advisories = []
-        for advisory_data in self.redhat_response:
-            processed_advisories.append(to_advisory(advisory_data))
+        self.redhat_cves = fetch()
 
+    def updated_advisories(self):
+        processed_advisories = list(map(to_advisory, self.redhat_cves))
         return self.batch_advisories(processed_advisories)
 
 
 def fetch():
-
-    response = []
+    """
+    Return a list of CVE data mappings fetched from the RedHat API.
+    See:
+        https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0/html/red_hat_security_data_api/index
+    """
+    cves = []
     page_no = 1
-    url = "https://access.redhat.com/hydra/rest/securitydata/cve.json?page={}"
+    url_template = "https://access.redhat.com/hydra/rest/securitydata/cve.json?per_page=10000&page={}"
 
+    cve_data = None
     while True:
-        resp_json = requests.get(url.format(page_no)).json()
-        page_no += 1
-        if not resp_json:
+        current_url = url_template.format(page_no)
+        try:
+            print(f'Fetching: {current_url}')
+            response = requests.get(current_url)
+            if response.status_code != requests.codes.ok:
+                # TODO: log me
+                print(f'Failed to fetch results from {current_url}')
+                break
+            cve_data = response.json()
+        except Exception as e:
+            # TODO: log me
+            msg = f'Failed to fetch results from {current_url}:\n{e}'
+            print(msg)
             break
 
-        for advisory in resp_json:
-            response.append(advisory)
+        if not cve_data:
+            break
+        cves.extend(cve_data)
+        page_no += 1
 
-    return response
+    print(f'Fetched {len(cves)} CVEs from: {current_url}')
+    return cves
 
 
 def to_advisory(advisory_data):
     affected_purls = []
     if advisory_data.get("affected_packages"):
         for rpm in advisory_data["affected_packages"]:
-            if rpm_to_purl(rpm):
-                affected_purls.append(rpm_to_purl(rpm))
+            purl = rpm_to_purl(rpm)
+            if purl:
+                affected_purls.append(purl)
 
     references = []
     bugzilla = advisory_data.get("bugzilla")
@@ -137,14 +154,16 @@ def to_advisory(advisory_data):
     references.append(Reference(severities=redhat_scores, url=advisory_data["resource_url"]))
 
     return Advisory(
+        cve_id=advisory_data["CVE"],
         summary=advisory_data["bugzilla_description"],
-        vulnerability_id=advisory_data["CVE"],
         impacted_package_urls=affected_purls,
         vuln_references=references,
     )
 
 
 def rpm_to_purl(rpm_string):
+    # FIXME: there is code in scancode to handle RPM conversion AND this should
+    # be all be part of the pcakageurl library
 
     # Red Hat uses `-:0` instead of just `-` to separate
     # package name and version
@@ -155,4 +174,4 @@ def rpm_to_purl(rpm_string):
     name, version = components
 
     if version[0].isdigit():
-        return PackageURL(name=name, type="rpm", version=version, namespace="redhat")
+        return PackageURL(namespace="redhat", name=name, type="rpm", version=version)