Remove unneeded comments and test files, add changelog entry for apache_kafka #972

Reference: #972

Signed-off-by: John M. Horan <johnmhoran@gmail.com>

Author: johnmhoran

CHANGELOG.rst

@@ -2,6 +2,12 @@ Release notes
 =============
 
 
+Next Release
+------------
+
+- We re-enabled support for the Apache Kafka vulnerabilities advisories importer.
+
+
 Version v32.0.0rc2
 --------------------
 

vulnerabilities/importers/apache_kafka.py

@@ -7,16 +7,10 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-import asyncio
-
-# to test json print
-import json
 
 import requests
 from bs4 import BeautifulSoup
 from packageurl import PackageURL
-
-# import VersionConstraint to experiment
 from univers.version_constraint import VersionConstraint
 from univers.version_range import MavenVersionRange
 from univers.versions import MavenVersion
@@ -25,8 +19,9 @@
 from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
-from vulnerabilities.package_managers import GitHubTagsAPI
 
+# The entries below with `"action": "omit"` have no useful/reportable fixed or affected version data.
+# See https://kafka.apache.org/cve-list
 affected_version_range_mapping = {
     "CVE-2022-34917": {
         "action": "include",
@@ -97,15 +92,8 @@ def fetch_advisory_page(self):
         page = requests.get(self.GH_PAGE_URL)
         return page.content
 
-    # For now, don't use the GH API
-    # def set_api(self):
-    #     self.version_api = GitHubTagsAPI()
-    #     asyncio.run(self.version_api.load_api(["apache/kafka"]))
-
     def updated_advisories(self):
         advisory_page = self.fetch_advisory_page()
-        # For now, don't use the GH API
-        # self.set_api()
 
         parsed_data = self.to_advisory(advisory_page)
         return self.batch_advisories(parsed_data)
@@ -155,8 +143,6 @@ def to_advisory(self, advisory_page):
             # hard-coded affected_version_range_mapping dictionary.
             if affected_version_range_mapping[cve_id]["action"] == "include":
 
-                # print("\ncve_id = {}".format(cve_id))
-
                 # These 2 variables (not used elsewhere) trigger the KeyError for changed/missing data.
                 check_affected_versions_key = affected_version_range_mapping[cve_id][
                     affected_versions_string
@@ -165,15 +151,6 @@ def to_advisory(self, advisory_page):
                     fixed_versions_string
                 ]
 
-                # This calculates/prints the correct univers version value, which we can then use in the mapping dictionary.
-                # We'll delete this and to_version_ranges_test() when no longer needed.
-                affected_version_ranges_TEST = self.to_version_ranges_test(
-                    affected_versions_string_split_SPLIT, fixed_versions_string_split_SPLIT
-                )
-                # print("\naffected_version_ranges_TEST = {}".format(affected_version_ranges_TEST))
-                # print("\naffected_versions_string = {}".format(affected_versions_string))
-                # print("\nfixed_versions_string = {}".format(fixed_versions_string))
-
                 references = [
                     Reference(url=self.ASF_PAGE_URL),
                     Reference(
@@ -203,127 +180,4 @@ def to_advisory(self, advisory_page):
                     )
                 )
 
-                # print("\nadvisories[-1] = {}".format(advisories[-1]))
-                # print("\nadvisories[-1].to_dict() = {}".format(advisories[-1].to_dict()))
-
-                # print("\n========================================")
-
-        # Print a dict of the advisories.
-        # result = [data.to_dict() for data in advisories]
-        # print("result = \n")
-        # print(json.dumps(result, indent=4, sort_keys=False))
-
         return advisories
-
-    # We use this to calculate the ranges for the hard-coded affected_version_range_mapping.
-    # We'' delete this when no longer needed.
-    def to_version_ranges_test(self, versions_data, fixed_versions):
-        constraints = []
-
-        for version_item in versions_data:
-            if "to" in version_item:
-                version_item_split = version_item.split(" ")
-
-                constraints.append(
-                    VersionConstraint(
-                        comparator=">=",
-                        version=MavenVersion(version_item_split[0]),
-                    )
-                )
-                constraints.append(
-                    VersionConstraint(
-                        comparator="<=",
-                        version=MavenVersion(version_item_split[-1]),
-                    )
-                )
-
-            elif "-" in version_item:
-                version_item_split = version_item.split(" ")
-
-                constraints.append(
-                    VersionConstraint(
-                        comparator=">=",
-                        version=MavenVersion(version_item_split[0]),
-                    )
-                )
-                constraints.append(
-                    VersionConstraint(
-                        comparator="<=",
-                        version=MavenVersion(version_item_split[-1]),
-                    )
-                )
-
-            elif "and later" in version_item:
-                version_item_split = version_item.split(" ")
-
-                constraints.append(
-                    VersionConstraint(
-                        comparator=">=",
-                        version=MavenVersion(version_item_split[0]),
-                    )
-                )
-
-            else:
-                version_item_split = version_item.split(" ")
-
-                constraints.append(
-                    VersionConstraint(
-                        comparator="=",
-                        version=MavenVersion(version_item_split[0]),
-                    )
-                )
-
-        for fixed_item in fixed_versions:
-            if "to" in fixed_item:
-                fixed_item_split = fixed_item.split(" ")
-
-                constraints.append(
-                    VersionConstraint(
-                        comparator=">=",
-                        version=MavenVersion(fixed_item_split[0]),
-                    ).invert()
-                )
-                constraints.append(
-                    VersionConstraint(
-                        comparator="<=",
-                        version=MavenVersion(fixed_item_split[-1]),
-                    ).invert()
-                )
-
-            elif "-" in fixed_item:
-                fixed_item_split = fixed_item.split(" ")
-
-                constraints.append(
-                    VersionConstraint(
-                        comparator=">=",
-                        version=MavenVersion(fixed_item_split[0]),
-                    ).invert()
-                )
-                constraints.append(
-                    VersionConstraint(
-                        comparator="<=",
-                        version=MavenVersion(fixed_item_split[-1]),
-                    ).invert()
-                )
-
-            elif "and later" in fixed_item:
-                fixed_item_split = fixed_item.split(" ")
-
-                constraints.append(
-                    VersionConstraint(
-                        comparator=">=",
-                        version=MavenVersion(fixed_item_split[0]),
-                    ).invert()
-                )
-
-            else:
-                fixed_item_split = fixed_item.split(" ")
-
-                constraints.append(
-                    VersionConstraint(
-                        comparator="=",
-                        version=MavenVersion(fixed_item_split[0]),
-                    ).invert()
-                )
-
-        return MavenVersionRange(constraints=constraints)

vulnerabilities/tests/test_apache_kafka.py

@@ -7,19 +7,15 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-# temporarily import json to create output to analyze
-import json
 import os
 
 import pytest
 
 from vulnerabilities.importers.apache_kafka import ApacheKafkaImporter
-
-# from vulnerabilities.package_managers import GitHubTagsAPI
 from vulnerabilities.tests import util_tests
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-# Created cve-list-2022-12-06.html with a full copy of https://raw.githubusercontent.com/apache/kafka-site/asf-site/cve-list.html
+
 TEST_DATA = os.path.join(
     BASE_DIR,
     "test_data/apache_kafka",
@@ -32,23 +28,10 @@ def test_to_advisory():
     advisories = ApacheKafkaImporter().to_advisory(raw_data)
     result = [data.to_dict() for data in advisories]
 
-    # TODO: We need to finish this test including the REGEN step.  2022-12-12 Monday 14:48:05.  Done.
     expected_file = os.path.join(TEST_DATA, f"to-advisory-apache_kafka-expected.json")
     util_tests.check_results_against_json(result, expected_file)
 
-    # We generate these 2 files solely to vet the output and adjust the importer code.
-    # with open(os.path.join(TEST_DATA, "jmh-test-01.txt"), "w") as f1:
-    #     for advisory_object in result:
-    #         f1.write(f"{advisory_object}\n\n")
-    #         for k, v in advisory_object.items():
-    #             f1.write(f"{k}: {v}\n\n")
-    #         f1.write(f"=================================================\n\n")
-
-    # with open(os.path.join(TEST_DATA, "test-advisories.json"), "w", encoding="utf-8") as f:
-    #     json.dump(result, f, ensure_ascii=False, indent=4)
-
 
-# Check for an unknown CVE value.
 def to_advisory_changed_cve():
     with open(os.path.join(TEST_DATA, "cve-list-changed-cve.html")) as f:
         raw_data = f.read()
@@ -62,7 +45,6 @@ def test_to_advisory_changed_cve_exception():
     assert "CVE-2022-34918" in str(excinfo.value)
 
 
-# Check for an unknown "Versions affected" value.
 def to_advisory_changed_versions_affected():
     with open(os.path.join(TEST_DATA, "cve-list-changed-versions-affected.html")) as f:
         raw_data = f.read()
@@ -76,7 +58,6 @@ def test_to_advisory_changed_versions_affected_exception():
     assert "2.8.0 - 2.8.1, 3.0.0 - 3.0.1, 3.1.0 - 3.1.1, 3.2.0 - 3.2.2" in str(excinfo.value)
 
 
-# Check for an unknown "Fixed versions" value.
 def to_advisory_changed_fixed_versions():
     with open(os.path.join(TEST_DATA, "cve-list-changed-fixed-versions.html")) as f:
         raw_data = f.read()