diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 3394dd989..5fc6fe968 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -40,6 +40,7 @@ from vulnerabilities.pipelines import nginx_importer from vulnerabilities.pipelines import npm_importer from vulnerabilities.pipelines import nvd_importer +from vulnerabilities.pipelines import openjdk_importer from vulnerabilities.pipelines import pypa_importer from vulnerabilities.pipelines import pysec_importer @@ -78,6 +79,7 @@ github_importer.GitHubAPIImporterPipeline, nvd_importer.NVDImporterPipeline, pysec_importer.PyPIImporterPipeline, + openjdk_importer.OpenJDKImporterPipeline, ] IMPORTERS_REGISTRY = { diff --git a/vulnerabilities/pipelines/openjdk_importer.py b/vulnerabilities/pipelines/openjdk_importer.py new file mode 100644 index 000000000..263d52763 --- /dev/null +++ b/vulnerabilities/pipelines/openjdk_importer.py @@ -0,0 +1,178 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import urllib.parse as urlparse +from typing import Iterable +import logging +from traceback import format_exc as traceback_format_exc + +import requests +from bs4 import BeautifulSoup +from packageurl import PackageURL +from univers.version_range import GenericVersionRange +from univers.versions import GenericVersion + +from vulnerabilities import severity_systems +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Importer +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.utils import get_advisory_url + + +class OpenJDKImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories from OpenJDK.""" + + root_url = "https://openjdk.org/groups/vulnerability/advisories/" + license_url = "https://openjdk.org/legal/" + spdx_license_expression = "CC-BY-4.0" + importer_name = "OpenJDK Importer" + + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + def advisories_count(self) -> int: + advisory_count = 0 + try: + response = requests.get(self.root_url) + response.raise_for_status() + + advisory_data = response.text + + + except requests.HTTPError as http_err: + self.log( + f"HTTP error occurred: {http_err} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + return advisory_count + + advisory_count = advisory_data.count("
  • ") + return advisory_count + + def advisory_data(self): + + extracted_urls = [] + data_by_urls = {} + + # scraping the advisory urls + data = requests.get(self.root_url).text + soup = BeautifulSoup(data, features="lxml") + li_tags = soup.find_all("li") + li_contents = [li.get_text(strip=True) for li in li_tags] + + for link in li_contents: + link = link.split()[0] # for getting only the year links + link = "-".join(link.split("/")) + link = self.root_url + link + extracted_urls.append(link) + data_by_urls[link] = requests.get(link).text + + for url, data in data_by_urls.items(): + yield from to_advisories(data, date_published=link) + + +def to_advisories(data, date_published) -> Iterable[AdvisoryData]: + advisories = [] + soup_spec = BeautifulSoup(data, features="lxml") + table_tags = soup_spec.find_all("table") + # print(soup_spec.find_all('th')) + + # tables containing the vulnurability info of OpenJDK and OpenJFX + + # For OpenJDK vulnerability data + try: + OpenJDK_table = table_tags[0] + extract_table_advisories(advisories, OpenJDK_table) + except IndexError: + pass + + # For OpenJFX vulnerability data + try: + OpenJFX_table = table_tags[1] + extract_table_advisories(advisories, OpenJFX_table) + except IndexError: + pass + + return advisories + + +def extract_table_advisories(advisories, OpenJDK_table): + OpenJDK_tr_tags = OpenJDK_table.select("tr") + # print(OpenJDK_tr_tags) + th_tags = OpenJDK_tr_tags[1].find_all("th") + + # to get the possible affected versions + possible_affected_versions = [] + for i in range(3, len(th_tags)): + possible_affected_versions.append(th_tags[i].text) + + scoring_system = OpenJDK_tr_tags[1].find("a").text + link = OpenJDK_tr_tags[1].find("a").attrs["href"] + for i in range(2, len(OpenJDK_tr_tags)): + td_tags = OpenJDK_tr_tags[i].select("td") + affected_version_list = [] + + try: + cve_id, component, severity_score, *affected_ver = td_tags + # print(severity_score.text) + cve_url = cve_id.find("a").attrs["href"] + cve_id = cve_id.text + component = "".join(component.text.split("\n")) + [cvss_score, cvss_vector] = severity_score.text.split("\n") + + references = [] + severities = [] + for index, version in enumerate(possible_affected_versions): + if affected_ver[index].text: + affected_version_list.append(version) + severities.append( + VulnerabilitySeverity( + system=scoring_system, + value=cvss_score, + scoring_elements=cvss_vector, + ) + ) + + references.append(Reference(url=link, severities=severities)) + + affected_packages = [] + if affected_version_list: + affected_packages.append( + AffectedPackage( + package=PackageURL( + name="openjdk", + type="generic", + ), + affected_version_range=GenericVersionRange.from_versions( + affected_version_list + ) + if affected_version_list + else None, + ) + ) + + advisories.append( + AdvisoryData( + aliases=[cve_id], + summary=component, + references=references, + affected_packages=affected_packages, + url=cve_url, + ) + ) + + # if vulnerability table is empty + except LookupError: + pass diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index d1f2fb6de..9a4bdef8c 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -409,10 +409,20 @@ {% endfor %} +<<<<<<< HEAD
    {% for exploit in vulnerability.exploits.all %} +======= +
    +
    + Known Exploited Vulnerabilities +
    + {% if vulnerability.kev %} +
    + +>>>>>>> e5ca080b (Add matched affected/fixed-by Packages with correct sorting #1287) diff --git a/vulnerabilities/templatetags/url_filters.py b/vulnerabilities/templatetags/url_filters.py index a6dda1dd8..9a3f1a2f9 100644 --- a/vulnerabilities/templatetags/url_filters.py +++ b/vulnerabilities/templatetags/url_filters.py @@ -5,7 +5,6 @@ register = template.Library() - -@register.filter(name="url_quote") +@register.filter(name='url_quote') def url_quote_filter(value): return quote(str(value)) diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index 692305f8d..3f75e87de 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -207,7 +207,7 @@ def test_sorted_queryset(self): pkg_package_urls = [obj.package_url for obj in sorted_pkgs_qs_all] sorted_purls = os.path.join(TEST_DIR, "sorted_purls.txt") - with open(sorted_purls, "r") as f: + with open(sorted_purls, 'r') as f: expected_content = f.read().splitlines() assert pkg_package_urls == expected_content @@ -236,40 +236,16 @@ def test_get_purl_version_class(self): class TestCustomFilters: - @pytest.mark.parametrize( - "input_value, expected_output", - [ - ( - "pkg:rpm/redhat/katello-client-bootstrap@1.1.0-2?arch=el6sat", - "pkg%3Arpm/redhat/katello-client-bootstrap%401.1.0-2%3Farch%3Del6sat", - ), - ( - "pkg:alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main", - "pkg%3Aalpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain", - ), - ("pkg:nginx/nginx@0.9.0?os=windows", "pkg%3Anginx/nginx%400.9.0%3Fos%3Dwindows"), - ( - "pkg:deb/ubuntu/nginx@0.6.34-2ubuntu1~intrepid1", - "pkg%3Adeb/ubuntu/nginx%400.6.34-2ubuntu1~intrepid1", - ), - ( - "pkg:rpm/redhat/openssl@1:1.0.2k-16.el7_6?arch=1", - "pkg%3Arpm/redhat/openssl%401%3A1.0.2k-16.el7_6%3Farch%3D1", - ), - ( - "pkg:golang/google.golang.org/genproto#googleapis/api/annotations", - "pkg%3Agolang/google.golang.org/genproto%23googleapis/api/annotations", - ), - ( - "pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib", - "pkg%3Acocoapods/GoogleUtilities%407.5.2%23NSData%2Bzlib", - ), - ( - "pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2", - "pkg%3Aconda/absl-py%400.4.1%3Fbuild%3Dpy36h06a4308_0%26channel%3Dmain%26subdir%3Dlinux-64%26type%3Dtar.bz2", - ), - ], - ) + @pytest.mark.parametrize("input_value, expected_output", [ + ("pkg:rpm/redhat/katello-client-bootstrap@1.1.0-2?arch=el6sat", "pkg%3Arpm/redhat/katello-client-bootstrap%401.1.0-2%3Farch%3Del6sat"), + ("pkg:alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main", "pkg%3Aalpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain"), + ("pkg:nginx/nginx@0.9.0?os=windows", "pkg%3Anginx/nginx%400.9.0%3Fos%3Dwindows"), + ("pkg:deb/ubuntu/nginx@0.6.34-2ubuntu1~intrepid1", "pkg%3Adeb/ubuntu/nginx%400.6.34-2ubuntu1~intrepid1"), + ("pkg:rpm/redhat/openssl@1:1.0.2k-16.el7_6?arch=1", "pkg%3Arpm/redhat/openssl%401%3A1.0.2k-16.el7_6%3Farch%3D1"), + ("pkg:golang/google.golang.org/genproto#googleapis/api/annotations", "pkg%3Agolang/google.golang.org/genproto%23googleapis/api/annotations"), + ("pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib", "pkg%3Acocoapods/GoogleUtilities%407.5.2%23NSData%2Bzlib"), + ("pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2", "pkg%3Aconda/absl-py%400.4.1%3Fbuild%3Dpy36h06a4308_0%26channel%3Dmain%26subdir%3Dlinux-64%26type%3Dtar.bz2"), + ]) def test_url_quote_filter(self, input_value, expected_output): filtered = url_quote_filter(input_value) assert filtered == expected_output diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 51cdcd049..ba16325bd 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -52,7 +52,6 @@ def purl_sort_key(purl: models.Package): purl_sort_version = purl_version_class(purl.version) return (purl.type, purl.namespace, purl.name, purl_sort_version, purl.qualifiers, purl.subpath) - def get_purl_version_class(purl: models.Package): RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange purl_version_class = None @@ -173,12 +172,7 @@ def get_context_data(self, **kwargs): try: vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) severity_vectors.append(vector_values) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ): + except (CVSS2MalformedError, CVSS3MalformedError, NotImplementedError): logging.error(f"CVSSMalformedError for {s.scoring_elements}") if s.value: @@ -201,10 +195,7 @@ def get_context_data(self, **kwargs): and (fixed_by_package.name == sorted_affected_package.name) and (fixed_by_package.qualifiers == sorted_affected_package.qualifiers) and (fixed_by_package.subpath == sorted_affected_package.subpath) - and ( - fixed_by_version_class(fixed_by_package.version) - > sorted_affected_version_class(sorted_affected_package.version) - ) + and (fixed_by_version_class(fixed_by_package.version) > sorted_affected_version_class(sorted_affected_package.version)) ): matched_fixed_by_packages.append(fixed_by_package.purl) affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages
    Data source