Fix Apache kafka and Apache httpd importer

CHANGELOG.rst

@@ -2,6 +2,13 @@ Release notes
 =============
 
 
+Next release
+-------------
+
+- We fixed Apache HTTPD and Apache Kafka importer.
+- We removed excessive network calls from Redhat importer.
+
+
 Version v32.0.0rc4
 -------------------
 

vulnerabilities/importers/apache_httpd.py

@@ -23,6 +23,7 @@
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.severity_systems import APACHE_HTTPD
+from vulnerabilities.utils import get_item
 
 logger = logging.getLogger(__name__)
 
@@ -40,16 +41,18 @@ def advisory_data(self):
             yield self.to_advisory(data)
 
     def to_advisory(self, data):
-        alias = data["CVE_data_meta"]["ID"]
-        descriptions = data["description"]["description_data"]
+        alias = get_item(data, "CVE_data_meta", "ID")
+        if not alias:
+            alias = get_item(data, "cveMetadata", "cveId")
+        descriptions = get_item(data, "description", "description_data") or []
         description = None
         for desc in descriptions:
-            if desc["lang"] == "eng":
+            if desc.get("lang") == "eng":
                 description = desc.get("value")
                 break
 
         severities = []
-        impacts = data.get("impact", [])
+        impacts = data.get("impact") or []
         for impact in impacts:
             value = impact.get("other")
             if value:
@@ -68,14 +71,14 @@ def to_advisory(self, data):
         )
 
         versions_data = []
-        for vendor in data["affects"]["vendor"]["vendor_data"]:
-            for products in vendor["product"]["product_data"]:
-                for version_data in products["version"]["version_data"]:
+        for vendor in get_item(data, "affects", "vendor", "vendor_data") or []:
+            for products in get_item(vendor, "product", "product_data") or []:
+                for version_data in get_item(products, "version", "version_data") or []:
                     versions_data.append(version_data)
 
         fixed_versions = []
         for timeline_object in data.get("timeline") or []:
-            timeline_value = timeline_object["value"]
+            timeline_value = timeline_object.get("value")
             if "release" in timeline_value:
                 split_timeline_value = timeline_value.split(" ")
                 if "never" in timeline_value:
@@ -100,7 +103,7 @@ def to_advisory(self, data):
 
         return AdvisoryData(
             aliases=[alias],
-            summary=description,
+            summary=description or "",
             affected_packages=affected_packages,
             references=[reference],
         )

vulnerabilities/importers/apache_kafka.py

@@ -8,6 +8,8 @@
 #
 
 
+import logging
+
 import pytz
 import requests
 from bs4 import BeautifulSoup
@@ -19,6 +21,8 @@
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 
+logger = logging.getLogger(__name__)
+
 # The entries below with `"action": "omit"` have no useful/reportable fixed or affected version data.
 # See https://kafka.apache.org/cve-list
 affected_version_range_mapping = {
@@ -135,13 +139,17 @@ def to_advisory(self, advisory_page):
             fixed_versions_clean = [v.strip() for v in fixed_versions.split(",")]
             fixed_versions_clean = [v for v in fixed_versions if v]
 
-            # This throws a KeyError if the opening h2 tag `id` data changes or is not in the
-            # hard-coded affected_version_range_mapping dictionary.
-            cve_version_mapping = affected_version_range_mapping[cve_id]
-            if cve_version_mapping["action"] == "include":
-                # These 2 variables (not used elsewhere) trigger the KeyError for changed/missing data.
-                check_affected_versions_key = cve_version_mapping[affected_versions]
-                check_fixed_versions_key = cve_version_mapping[fixed_versions]
+            cve_version_mapping = affected_version_range_mapping.get(cve_id)
+            if not cve_version_mapping:
+                logger.error(f"Data for {cve_id} not found in mapping. Skipping.")
+            if cve_version_mapping and cve_version_mapping.get("action") == "include":
+                check_affected_versions_key = cve_version_mapping.get(affected_versions) or []
+                check_fixed_versions_key = cve_version_mapping.get(fixed_versions) or []
+
+                if not check_affected_versions_key:
+                    logger.error(f"Affected versions for {cve_id} not found in mapping. Skipping.")
+                if not check_fixed_versions_key:
+                    logger.error(f"Fixed versions for {cve_id} not found in mapping. Skipping.")
 
                 references = [
                     Reference(
@@ -159,18 +167,22 @@ def to_advisory(self, advisory_page):
                 ]
 
                 affected_packages = []
-                affected_package = AffectedPackage(
-                    package=PackageURL(
-                        name="kafka",
-                        type="apache",
-                    ),
-                    affected_version_range=cve_version_mapping["affected_version_range"],
-                )
-                affected_packages.append(affected_package)
+                affected_version_range = cve_version_mapping.get("affected_version_range")
+                if cve_version_mapping.get("affected_version_range"):
+                    affected_package = AffectedPackage(
+                        package=PackageURL(
+                            name="kafka",
+                            type="apache",
+                        ),
+                        affected_version_range=affected_version_range,
+                    )
+                    affected_packages.append(affected_package)
 
-                date_published = parse(cve_version_mapping["Issue announced"]).replace(
-                    tzinfo=pytz.UTC
-                )
+                date_published = None
+                issue_announced = cve_version_mapping.get("Issue announced")
+
+                if issue_announced:
+                    date_published = parse(issue_announced).replace(tzinfo=pytz.UTC)
 
                 advisories.append(
                     AdvisoryData(

vulnerabilities/tests/test_apache_kafka.py

@@ -69,34 +69,13 @@ def to_advisory_changed_cve():
     advisories = ApacheKafkaImporter().to_advisory(raw_data)
 
 
-def test_to_advisory_changed_cve_exception():
-    with pytest.raises(KeyError) as excinfo:
-        to_advisory_changed_cve()
-
-    assert "CVE-2022-34918" in str(excinfo.value)
-
-
 def to_advisory_changed_versions_affected():
     with open(os.path.join(TEST_DATA, "cve-list-changed-versions-affected.html")) as f:
         raw_data = f.read()
     advisories = ApacheKafkaImporter().to_advisory(raw_data)
 
 
-def test_to_advisory_changed_versions_affected_exception():
-    with pytest.raises(KeyError) as excinfo:
-        to_advisory_changed_versions_affected()
-
-    assert "2.8.0 - 2.8.1, 3.0.0 - 3.0.1, 3.1.0 - 3.1.1, 3.2.0 - 3.2.2" in str(excinfo.value)
-
-
 def to_advisory_changed_fixed_versions():
     with open(os.path.join(TEST_DATA, "cve-list-changed-fixed-versions.html")) as f:
         raw_data = f.read()
     advisories = ApacheKafkaImporter().to_advisory(raw_data)
-
-
-def test_to_advisory_changed_fixed_versions_exception():
-    with pytest.raises(KeyError) as excinfo:
-        to_advisory_changed_fixed_versions()
-
-    assert "2.8.2, 3.0.2, 3.1.2, 3.2.4" in str(excinfo.value)