Openjdk importer branch

vulnerabilities/importers/__init__.py

@@ -40,6 +40,7 @@
 from vulnerabilities.pipelines import nginx_importer
 from vulnerabilities.pipelines import npm_importer
 from vulnerabilities.pipelines import nvd_importer
+from vulnerabilities.pipelines import openjdk_importer
 from vulnerabilities.pipelines import pypa_importer
 from vulnerabilities.pipelines import pysec_importer
 
@@ -78,6 +79,7 @@
     github_importer.GitHubAPIImporterPipeline,
     nvd_importer.NVDImporterPipeline,
     pysec_importer.PyPIImporterPipeline,
+    openjdk_importer.OpenJDKImporterPipeline,
 ]
 
 IMPORTERS_REGISTRY = {

vulnerabilities/pipelines/openjdk_importer.py

@@ -0,0 +1,178 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import urllib.parse as urlparse
+from typing import Iterable
+import logging
+from traceback import format_exc as traceback_format_exc
+
+import requests
+from bs4 import BeautifulSoup
+from packageurl import PackageURL
+from univers.version_range import GenericVersionRange
+from univers.versions import GenericVersion
+
+from vulnerabilities import severity_systems
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import Importer
+from vulnerabilities.importer import Reference
+from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
+from vulnerabilities.utils import get_advisory_url
+
+
+class OpenJDKImporterPipeline(VulnerableCodeBaseImporterPipeline):
+    """Collect advisories from OpenJDK."""
+
+    root_url = "https://openjdk.org/groups/vulnerability/advisories/"
+    license_url = "https://openjdk.org/legal/"
+    spdx_license_expression = "CC-BY-4.0"
+    importer_name = "OpenJDK Importer"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.collect_and_store_advisories,
+            cls.import_new_advisories,
+        )
+
+    def advisories_count(self) -> int:
+        advisory_count = 0
+        try:
+            response = requests.get(self.root_url)
+            response.raise_for_status()
+
+            advisory_data = response.text
+
+
+        except requests.HTTPError as http_err:
+            self.log(
+                f"HTTP error occurred: {http_err} \n {traceback_format_exc()}",
+                level=logging.ERROR,
+            )
+            return advisory_count
+
+        advisory_count = advisory_data.count("<li>")
+        return advisory_count
+
+    def advisory_data(self):
+
+        extracted_urls = []
+        data_by_urls = {}
+
+        # scraping the advisory urls
+        data = requests.get(self.root_url).text
+        soup = BeautifulSoup(data, features="lxml")
+        li_tags = soup.find_all("li")
+        li_contents = [li.get_text(strip=True) for li in li_tags]
+
+        for link in li_contents:
+            link = link.split()[0]  # for getting only the year links
+            link = "-".join(link.split("/"))
+            link = self.root_url + link
+            extracted_urls.append(link)
+            data_by_urls[link] = requests.get(link).text
+
+        for url, data in data_by_urls.items():
+            yield from to_advisories(data, date_published=link)
+
+
+def to_advisories(data, date_published) -> Iterable[AdvisoryData]:
+    advisories = []
+    soup_spec = BeautifulSoup(data, features="lxml")
+    table_tags = soup_spec.find_all("table")
+    # print(soup_spec.find_all('th'))
+
+    # tables containing the vulnurability info of OpenJDK and OpenJFX
+
+    # For OpenJDK vulnerability data
+    try:
+        OpenJDK_table = table_tags[0]
+        extract_table_advisories(advisories, OpenJDK_table)
+    except IndexError:
+        pass
+
+    # For OpenJFX vulnerability data
+    try:
+        OpenJFX_table = table_tags[1]
+        extract_table_advisories(advisories, OpenJFX_table)
+    except IndexError:
+        pass
+
+    return advisories
+
+
+def extract_table_advisories(advisories, OpenJDK_table):
+    OpenJDK_tr_tags = OpenJDK_table.select("tr")
+    # print(OpenJDK_tr_tags)
+    th_tags = OpenJDK_tr_tags[1].find_all("th")
+
+    # to get the possible affected versions
+    possible_affected_versions = []
+    for i in range(3, len(th_tags)):
+        possible_affected_versions.append(th_tags[i].text)
+
+    scoring_system = OpenJDK_tr_tags[1].find("a").text
+    link = OpenJDK_tr_tags[1].find("a").attrs["href"]
+    for i in range(2, len(OpenJDK_tr_tags)):
+        td_tags = OpenJDK_tr_tags[i].select("td")
+        affected_version_list = []
+
+        try:
+            cve_id, component, severity_score, *affected_ver = td_tags
+            # print(severity_score.text)
+            cve_url = cve_id.find("a").attrs["href"]
+            cve_id = cve_id.text
+            component = "".join(component.text.split("\n"))
+            [cvss_score, cvss_vector] = severity_score.text.split("\n")
+
+            references = []
+            severities = []
+            for index, version in enumerate(possible_affected_versions):
+                if affected_ver[index].text:
+                    affected_version_list.append(version)
+                    severities.append(
+                        VulnerabilitySeverity(
+                            system=scoring_system,
+                            value=cvss_score,
+                            scoring_elements=cvss_vector,
+                        )
+                    )
+
+            references.append(Reference(url=link, severities=severities))
+
+            affected_packages = []
+            if affected_version_list:
+                affected_packages.append(
+                    AffectedPackage(
+                        package=PackageURL(
+                            name="openjdk",
+                            type="generic",
+                        ),
+                        affected_version_range=GenericVersionRange.from_versions(
+                            affected_version_list
+                        )
+                        if affected_version_list
+                        else None,
+                    )
+                )
+
+            advisories.append(
+                AdvisoryData(
+                    aliases=[cve_id],
+                    summary=component,
+                    references=references,
+                    affected_packages=affected_packages,
+                    url=cve_url,
+                )
+            )
+
+        # if vulnerability table is empty
+        except LookupError:
+            pass

vulnerabilities/templates/vulnerability_details.html

@@ -409,10 +409,20 @@
                         {% endfor %}
                 </div>
 
+<<<<<<< HEAD
 
             <div class="tab-div content" data-content="exploits">
                 {% for exploit in vulnerability.exploits.all %}
                     <table class="table is-bordered is-striped is-fullwidth gray-header-border">
+=======
+            <div class="tab-div content" data-content="known-exploited-vulnerabilities">
+                <div class="has-text-weight-bold tab-nested-div ml-1 mb-1 mt-1">
+                    Known Exploited Vulnerabilities
+                </div>
+                {% if vulnerability.kev %}
+                <table class="table vcio-table width-100-pct mt-2">
+                        <tbody>
+>>>>>>> e5ca080b (Add matched affected/fixed-by Packages with correct sorting #1287)
                             <tr>
                                 <th class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left"
                                     data-tooltip="The source of the exploit information, such as CISA KEV, Exploit-DB, Metasploit, Packet Storm, or others."> Data source </th>

vulnerabilities/templatetags/url_filters.py

@@ -5,7 +5,6 @@
 
 register = template.Library()
 
-
-@register.filter(name="url_quote")
+@register.filter(name='url_quote')
 def url_quote_filter(value):
     return quote(str(value))

vulnerabilities/tests/test_view.py

@@ -207,7 +207,7 @@ def test_sorted_queryset(self):
 
         pkg_package_urls = [obj.package_url for obj in sorted_pkgs_qs_all]
         sorted_purls = os.path.join(TEST_DIR, "sorted_purls.txt")
-        with open(sorted_purls, "r") as f:
+        with open(sorted_purls, 'r') as f:
             expected_content = f.read().splitlines()
             assert pkg_package_urls == expected_content
 
@@ -236,40 +236,16 @@ def test_get_purl_version_class(self):
 
 
 class TestCustomFilters:
-    @pytest.mark.parametrize(
-        "input_value, expected_output",
-        [
-            (
-                "pkg:rpm/redhat/katello-client-bootstrap@1.1.0-2?arch=el6sat",
-                "pkg%3Arpm/redhat/katello-client-bootstrap%401.1.0-2%3Farch%3Del6sat",
-            ),
-            (
-                "pkg:alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main",
-                "pkg%3Aalpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain",
-            ),
-            ("pkg:nginx/nginx@0.9.0?os=windows", "pkg%3Anginx/nginx%400.9.0%3Fos%3Dwindows"),
-            (
-                "pkg:deb/ubuntu/nginx@0.6.34-2ubuntu1~intrepid1",
-                "pkg%3Adeb/ubuntu/nginx%400.6.34-2ubuntu1~intrepid1",
-            ),
-            (
-                "pkg:rpm/redhat/openssl@1:1.0.2k-16.el7_6?arch=1",
-                "pkg%3Arpm/redhat/openssl%401%3A1.0.2k-16.el7_6%3Farch%3D1",
-            ),
-            (
-                "pkg:golang/google.golang.org/genproto#googleapis/api/annotations",
-                "pkg%3Agolang/google.golang.org/genproto%23googleapis/api/annotations",
-            ),
-            (
-                "pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib",
-                "pkg%3Acocoapods/GoogleUtilities%407.5.2%23NSData%2Bzlib",
-            ),
-            (
-                "pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2",
-                "pkg%3Aconda/absl-py%400.4.1%3Fbuild%3Dpy36h06a4308_0%26channel%3Dmain%26subdir%3Dlinux-64%26type%3Dtar.bz2",
-            ),
-        ],
-    )
+    @pytest.mark.parametrize("input_value, expected_output", [
+        ("pkg:rpm/redhat/katello-client-bootstrap@1.1.0-2?arch=el6sat", "pkg%3Arpm/redhat/katello-client-bootstrap%401.1.0-2%3Farch%3Del6sat"),
+        ("pkg:alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main", "pkg%3Aalpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain"),
+        ("pkg:nginx/nginx@0.9.0?os=windows", "pkg%3Anginx/nginx%400.9.0%3Fos%3Dwindows"),
+        ("pkg:deb/ubuntu/nginx@0.6.34-2ubuntu1~intrepid1", "pkg%3Adeb/ubuntu/nginx%400.6.34-2ubuntu1~intrepid1"),
+        ("pkg:rpm/redhat/openssl@1:1.0.2k-16.el7_6?arch=1", "pkg%3Arpm/redhat/openssl%401%3A1.0.2k-16.el7_6%3Farch%3D1"),
+        ("pkg:golang/google.golang.org/genproto#googleapis/api/annotations", "pkg%3Agolang/google.golang.org/genproto%23googleapis/api/annotations"),
+        ("pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib", "pkg%3Acocoapods/GoogleUtilities%407.5.2%23NSData%2Bzlib"),
+        ("pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2", "pkg%3Aconda/absl-py%400.4.1%3Fbuild%3Dpy36h06a4308_0%26channel%3Dmain%26subdir%3Dlinux-64%26type%3Dtar.bz2"),
+    ])
     def test_url_quote_filter(self, input_value, expected_output):
         filtered = url_quote_filter(input_value)
         assert filtered == expected_output

vulnerabilities/views.py

@@ -52,7 +52,6 @@ def purl_sort_key(purl: models.Package):
         purl_sort_version = purl_version_class(purl.version)
     return (purl.type, purl.namespace, purl.name, purl_sort_version, purl.qualifiers, purl.subpath)
 
-
 def get_purl_version_class(purl: models.Package):
     RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange
     purl_version_class = None
@@ -173,12 +172,7 @@ def get_context_data(self, **kwargs):
                 try:
                     vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements)
                     severity_vectors.append(vector_values)
-                except (
-                    CVSS2MalformedError,
-                    CVSS3MalformedError,
-                    CVSS4MalformedError,
-                    NotImplementedError,
-                ):
+                except (CVSS2MalformedError, CVSS3MalformedError, NotImplementedError):
                     logging.error(f"CVSSMalformedError for {s.scoring_elements}")
 
             if s.value:
@@ -201,10 +195,7 @@ def get_context_data(self, **kwargs):
                     and (fixed_by_package.name == sorted_affected_package.name)
                     and (fixed_by_package.qualifiers == sorted_affected_package.qualifiers)
                     and (fixed_by_package.subpath == sorted_affected_package.subpath)
-                    and (
-                        fixed_by_version_class(fixed_by_package.version)
-                        > sorted_affected_version_class(sorted_affected_package.version)
-                    )
+                    and (fixed_by_version_class(fixed_by_package.version) > sorted_affected_version_class(sorted_affected_package.version))
                 ):
                     matched_fixed_by_packages.append(fixed_by_package.purl)
             affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages