Collect kafka cves

vulnerabilities/importer_yielder.py

@@ -241,6 +241,13 @@
             "etags": {}
         },
     },
+    {
+        'name': 'apache_kafka',
+        'license': '',
+        'last_run': None,
+        'data_source': 'ApacheKafkaDataSource',
+        'data_source_cfg': {},
+    },
 
 ]
 

vulnerabilities/importers/__init__.py

@@ -45,3 +45,4 @@
 from vulnerabilities.importers.ubuntu import UbuntuDataSource
 from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource
 from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource
+from vulnerabilities.importers.apache_kafka import ApacheKafkaDataSource

vulnerabilities/importers/apache_kafka.py

@@ -0,0 +1,120 @@
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import asyncio
+
+import requests
+from bs4 import BeautifulSoup
+from dephell_specifier import RangeSpecifier
+from packageurl import PackageURL
+
+from vulnerabilities.data_source import Advisory
+from vulnerabilities.data_source import DataSource
+from vulnerabilities.data_source import Reference
+from vulnerabilities.package_managers import GitHubTagsAPI
+
+GH_PAGE_URL = "https://raw.githubusercontent.com/apache/kafka-site/asf-site/cve-list.html"
+ASF_PAGE_URL = "https://kafka.apache.org/cve-list"
+
+
+class ApacheKafkaDataSource(DataSource):
+    @staticmethod
+    def fetch_advisory_page():
+        page = requests.get(GH_PAGE_URL)
+        return page.content
+
+    def set_api(self):
+        self.version_api = GitHubTagsAPI()
+        asyncio.run(self.version_api.load_api(["apache/kafka"]))
+
+    def updated_advisories(self):
+        advisory_page = self.fetch_advisory_page()
+        self.set_api()
+        parsed_data = self.to_advisory(advisory_page)
+        return self.batch_advisories(parsed_data)
+
+    def to_advisory(self, advisory_page):
+        advisories = []
+        advisory_page = BeautifulSoup(advisory_page, features="lxml")
+        cve_section_beginnings = advisory_page.find_all("h2")
+        for cve_section_beginning in cve_section_beginnings:
+            cve_id = cve_section_beginning.text.split("\n")[0]
+            cve_description_paragraph = cve_section_beginning.find_next_sibling("p")
+            cve_data_table = cve_section_beginning.find_next_sibling("table")
+            cve_data_table_rows = cve_data_table.find_all("tr")
+            affected_versions_row = cve_data_table_rows[0]
+            fixed_versions_row = cve_data_table_rows[1]
+            affected_version_ranges = to_version_ranges(
+                affected_versions_row.find_all("td")[1].text
+            )
+            fixed_version_ranges = to_version_ranges(fixed_versions_row.find_all("td")[1].text)
+
+            fixed_packages = [
+                PackageURL(type="apache", name="kafka", version=version)
+                for version in self.version_api.get("apache/kafka")
+                if any([version in version_range for version_range in fixed_version_ranges])
+            ]
+
+            affected_packages = [
+                PackageURL(type="apache", name="kafka", version=version)
+                for version in self.version_api.get("apache/kafka")
+                if any([version in version_range for version_range in affected_version_ranges])
+            ]
+
+            advisories.append(
+                Advisory(
+                    cve_id=cve_id,
+                    summary=cve_description_paragraph.text,
+                    impacted_package_urls=affected_packages,
+                    resolved_package_urls=fixed_packages,
+                    vuln_references=[
+                        Reference(url=ASF_PAGE_URL),
+                        Reference(
+                            url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve_id}",
+                            reference_id=cve_id,
+                        ),
+                    ],
+                )
+            )
+        return advisories
+
+
+def to_version_ranges(version_range_text):
+    version_ranges = []
+    range_expressions = version_range_text.split(",")
+    for range_expression in range_expressions:
+        if "to" in range_expression:
+            # eg range_expression == "3.2.0 to 3.2.1"
+            lower_bound, upper_bound = range_expression.split("to")
+            lower_bound = f">={lower_bound}"
+            upper_bound = f"<={upper_bound}"
+            version_ranges.append(RangeSpecifier(f"{lower_bound},{upper_bound}"))
+
+        elif "and later" in range_expression:
+            # eg range_expression == "2.1.1 and later"
+            range_expression = range_expression.replace("and later", "")
+            version_ranges.append(RangeSpecifier(f">={range_expression}"))
+
+        else:
+            # eg  range_expression == "3.0.0"
+            version_ranges.append(RangeSpecifier(range_expression))
+    return version_ranges

vulnerabilities/tests/test_apache_kafka.py

@@ -0,0 +1,102 @@
+# Copyright (c)  nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import os
+from unittest import TestCase
+
+from dephell_specifier import RangeSpecifier
+from packageurl import PackageURL
+
+from vulnerabilities.data_source import Advisory
+from vulnerabilities.data_source import Reference
+from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.importers.apache_kafka import ApacheKafkaDataSource
+from vulnerabilities.importers.apache_kafka import to_version_ranges
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_kafka", "cve-list.html")
+
+
+class TestApacheKafkaDataSource(TestCase):
+    def test_to_version_ranges(self):
+        # Check single version
+        assert [RangeSpecifier("==3.2.2")] == to_version_ranges("3.2.2")
+
+        # Check range with lower and upper bounds
+        assert [RangeSpecifier(">=3.2.2, <=3.2.3")] == to_version_ranges("3.2.2 to 3.2.3")
+
+        # Check range with "and later"
+        assert [RangeSpecifier(">=3.2.2")] == to_version_ranges("3.2.2 and later")
+
+        # Check combination of above cases
+        assert [
+            RangeSpecifier(">=3.2.2"),
+            RangeSpecifier(">=3.2.2, <=3.2.3"),
+            RangeSpecifier("==3.2.2"),
+        ] == to_version_ranges("3.2.2 and later, 3.2.2 to 3.2.3, 3.2.2")
+
+    def test_to_advisory(self):
+        data_source = ApacheKafkaDataSource(batch_size=1)
+        data_source.version_api = GitHubTagsAPI(
+            cache={"apache/kafka": ["2.1.2", "0.10.2.2"]}
+        )
+        expected_data = [
+            Advisory(
+                summary="In Apache Kafka versions between 0.11.0.0 and 2.1.0, it is possible to "
+                "manually\n    craft a Produce request which bypasses transaction/idempotent ACL "
+                "validation.\n    Only authenticated clients with Write permission on the "
+                "respective topics are\n    able to exploit this vulnerability. Users should "
+                "upgrade to 2.1.1 or later\n    where this vulnerability has been fixed.",
+                impacted_package_urls=[
+                    PackageURL(
+                        type="apache",
+                        namespace=None,
+                        name="kafka",
+                        version="0.10.2.2",
+                        qualifiers={},
+                        subpath=None,
+                    )
+                ],
+                resolved_package_urls=[
+                    PackageURL(
+                        type="apache",
+                        namespace=None,
+                        name="kafka",
+                        version="2.1.2",
+                        qualifiers={},
+                        subpath=None,
+                    )
+                ],
+                vuln_references=[
+                    Reference(url="https://kafka.apache.org/cve-list", reference_id=""),
+                    Reference(
+                        url="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-17196",
+                        reference_id="CVE-2018-17196",
+                    ),
+                ],
+                cve_id="CVE-2018-17196",
+            )
+        ]
+        with open(TEST_DATA) as f:
+            found_data = data_source.to_advisory(f)
+
+        assert found_data == expected_data

vulnerabilities/tests/test_data/apache_kafka/cve-list.html

@@ -0,0 +1,40 @@
+<!--#include virtual="includes/_header.htm" -->
+<body class="page-cve-list ">
+    <!--#include virtual="includes/_top.htm" -->
+    <div class="content">
+        <!--#include virtual="includes/_nav.htm" -->
+        <div class="right">
+
+    <h1>Apache Kafka Security Vulnerabilities</h1>
+
+    This page lists all security vulnerabilities fixed in released versions of Apache Kafka
+
+    <h2><a href="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-17196">CVE-2018-17196</a>
+    Authenticated clients with Write permission may bypass transaction/idempotent ACL validation</h2>
+    <p>In Apache Kafka versions between 0.11.0.0 and 2.1.0, it is possible to manually
+    craft a Produce request which bypasses transaction/idempotent ACL validation.
+    Only authenticated clients with Write permission on the respective topics are
+    able to exploit this vulnerability. Users should upgrade to 2.1.1 or later
+    where this vulnerability has been fixed.</p>
+
+    <table class="data-table">
+    <tbody>
+      <tr>
+        <td>Versions affected</td>
+        <td>0.11.0.0 to 2.1.0, 0.10.2.2</td>
+      </tr>
+      <tr>
+        <td>Fixed versions</td>
+        <td>2.1.1 and later</td>
+      </tr>
+      <tr>
+        <td>Impact</td>
+        <td>This issue could result in privilege escalation.</td>
+      </tr>
+      <tr>
+        <td>Issue announced</td>
+        <td>10 July 2019</td>
+      </tr>
+    </tbody>
+    </table>
+    <!--#include virtual="includes/_footer.htm" -->