diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 9b11c7920..95c1f3345 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -18,6 +18,7 @@ from vulnerabilities.pipelines import enhance_with_kev from vulnerabilities.pipelines import enhance_with_metasploit from vulnerabilities.pipelines import flag_ghost_packages +from vulnerabilities.pipelines.liferay_advisories import LiferayAdvisoryPipeline IMPROVERS_REGISTRY = [ valid_versions.GitHubBasicImprover, @@ -45,6 +46,7 @@ compute_package_version_rank.ComputeVersionRankPipeline, collect_commits.CollectFixCommitsPipeline, add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline, + LiferayAdvisoryPipeline, ] IMPROVERS_REGISTRY = { diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py index f2e9009e8..7e1a90fda 100644 --- a/vulnerabilities/improvers/default.py +++ b/vulnerabilities/improvers/default.py @@ -44,9 +44,9 @@ def interesting_advisories(self) -> QuerySet: return ( Advisory.objects.filter(Q(created_by=self.importer.qualified_name)) .order_by("-date_collected") - .paginated() + .iterator() ) - return Advisory.objects.all().order_by("-date_collected").paginated() + return Advisory.objects.all().order_by("-date_collected").iterator() def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: if not advisory_data: diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py index 916f36f59..434034bcc 100644 --- a/vulnerabilities/improvers/valid_versions.py +++ b/vulnerabilities/improvers/valid_versions.py @@ -64,8 +64,8 @@ class ValidVersionImprover(Improver): @property def interesting_advisories(self) -> QuerySet: if issubclass(self.importer, VulnerableCodeBaseImporterPipeline): - return Advisory.objects.filter(Q(created_by=self.importer.pipeline_id)).paginated() - return Advisory.objects.filter(Q(created_by=self.importer.qualified_name)).paginated() + return Advisory.objects.filter(Q(created_by=self.importer.pipeline_id)).iterator() + return Advisory.objects.filter(Q(created_by=self.importer.qualified_name)).iterator() def get_package_versions( self, package_url: PackageURL, until: Optional[datetime] = None @@ -222,7 +222,7 @@ class NginxBasicImprover(Improver): @property def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=NginxImporterPipeline.pipeline_id).paginated() + return Advisory.objects.filter(created_by=NginxImporterPipeline.pipeline_id).iterator() def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: all_versions = list(self.fetch_nginx_version_from_git_tags()) diff --git a/vulnerabilities/improvers/vulnerability_status.py b/vulnerabilities/improvers/vulnerability_status.py index 214e6dc35..cd717da1d 100644 --- a/vulnerabilities/improvers/vulnerability_status.py +++ b/vulnerabilities/improvers/vulnerability_status.py @@ -40,7 +40,7 @@ def interesting_advisories(self) -> QuerySet: return ( Advisory.objects.filter(Q(created_by=NVDImporterPipeline.pipeline_id)) .distinct("aliases") - .paginated() + .iterator() ) def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: diff --git a/vulnerabilities/management/commands/export.py b/vulnerabilities/management/commands/export.py index 08685e33d..0da38bbdc 100644 --- a/vulnerabilities/management/commands/export.py +++ b/vulnerabilities/management/commands/export.py @@ -159,7 +159,7 @@ def packages_by_type_ns_name(): "fixing_vulnerabilities__weaknesses", "fixing_vulnerabilities__severities", ) - .paginated() + .iterator() ) for tp_ns_name, packages in groupby(qs, key=by_purl_type_ns_name): diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index d74db9f35..1b3613933 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -170,7 +170,7 @@ def import_new_advisories(self): imported_advisory_count = 0 progress = LoopProgress(total_iterations=new_advisories_count, logger=self.log) - for advisory in progress.iter(new_advisories.paginated()): + for advisory in progress.iter(new_advisories.iterator()): self.import_advisory(advisory=advisory) if advisory.date_imported: imported_advisory_count += 1 diff --git a/vulnerabilities/pipelines/add_cvss31_to_CVEs.py b/vulnerabilities/pipelines/add_cvss31_to_CVEs.py index acda42b52..9de6f9aae 100644 --- a/vulnerabilities/pipelines/add_cvss31_to_CVEs.py +++ b/vulnerabilities/pipelines/add_cvss31_to_CVEs.py @@ -47,7 +47,7 @@ def process_cve_advisory_mapping(self): batch_size = 1000 results = [] - for severity in progress.iter(nvd_severities.paginated(per_page=batch_size)): + for severity in progress.iter(nvd_severities.iterator()): print(severity.url) cve_pattern = re.compile(r"(CVE-\d{4}-\d{4,7})").search cve_match = cve_pattern(severity.url) diff --git a/vulnerabilities/pipelines/collect_commits.py b/vulnerabilities/pipelines/collect_commits.py index 92145c051..403c3e6fa 100644 --- a/vulnerabilities/pipelines/collect_commits.py +++ b/vulnerabilities/pipelines/collect_commits.py @@ -51,9 +51,8 @@ def collect_and_store_fix_commits(self): total_iterations=affected_by_package_related_vulnerabilities.count(), logger=self.log ) - for apv in progress.iter( - affected_by_package_related_vulnerabilities.paginated(per_page=500) - ): + # Remove the per_page parameter + for apv in progress.iter(affected_by_package_related_vulnerabilities.iterator()): vulnerability = apv.vulnerability for reference in vulnerability.references.all(): if not "/commit/" in reference.url: diff --git a/vulnerabilities/pipelines/flag_ghost_packages.py b/vulnerabilities/pipelines/flag_ghost_packages.py index 7daee4115..33e504676 100644 --- a/vulnerabilities/pipelines/flag_ghost_packages.py +++ b/vulnerabilities/pipelines/flag_ghost_packages.py @@ -49,7 +49,7 @@ def detect_and_flag_ghost_packages(logger=None): ) grouped_packages = groupby( - interesting_packages_qs.paginated(), + interesting_packages_qs.iterator(), key=lambda pkg: (pkg.type, pkg.namespace, pkg.name), ) diff --git a/vulnerabilities/pipelines/liferay_advisories.py b/vulnerabilities/pipelines/liferay_advisories.py new file mode 100644 index 000000000..e4fb3ad73 --- /dev/null +++ b/vulnerabilities/pipelines/liferay_advisories.py @@ -0,0 +1,187 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import re +from typing import Iterable + +import requests +from bs4 import BeautifulSoup +from packageurl import PackageURL +from requests.exceptions import HTTPError +from requests.exceptions import RequestException +from requests.exceptions import Timeout + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Reference +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline + + +class LiferayAdvisoryPipeline(VulnerableCodeBaseImporterPipeline): + """Pipeline to import Liferay security advisories.""" + + pipeline_id = "liferay_advisories" + description = "Import Liferay security advisories" + license_url = "https://liferay.dev/portal/security/known-vulnerabilities" + spdx_license_expression = "CC-BY-4.0" + importer_name = "Liferay Security Advisories" + + @classmethod + def steps(cls): + return ( + cls.fetch_advisories, + cls.parse_advisories, + cls.collect_and_store_advisories, # Changed from collect_advisories + cls.import_new_advisories, + ) + + def fetch_advisories(self): + """Fetch HTML content from Liferay's security page.""" + try: + response = requests.get( + self.license_url, + headers={"User-Agent": "Mozilla/5.0"}, + timeout=30, + ) + response.raise_for_status() + self.html_content = response.text + self.log(f"Response size: {len(self.html_content)} bytes") + except (Timeout, HTTPError, RequestException) as e: + self.log(f"Request error: {e}") + raise + except Exception as e: + self.log(f"Unexpected error: {e}") + raise + + def parse_advisories(self): + """Parse HTML to extract advisories.""" + self.parsed_advisories = [] + soup = BeautifulSoup(self.html_content, "html.parser") + + # Look for CVE IDs anywhere in the page + cve_patterns = soup.find_all(text=re.compile(r"CVE-\d{4}-\d+", re.IGNORECASE)) + self.log(f"Found {len(cve_patterns)} potential CVE mentions") + + if not cve_patterns: + self.log("No CVE IDs found in the page") + return + + # Process each CVE mention + processed_cves = set() + for cve_text in cve_patterns: + # Extract the CVE ID + match = re.search(r"(CVE-\d{4}-\d+)", cve_text, re.IGNORECASE) + if not match: + continue + + cve_id = match.group(1).upper() + if cve_id in processed_cves: + continue + + processed_cves.add(cve_id) + + # Find the parent container + parent = cve_text.parent + container = None + for _ in range(5): + if not parent: + break + if parent.name in ["article", "section", "div"] and len(parent.get_text()) > 100: + container = parent + break + parent = parent.parent + + if not container: + continue + + # Extract information + try: + # Description - get all text from paragraphs + paragraphs = container.find_all("p") + description = " ".join(p.get_text(strip=True) for p in paragraphs) + + # Affected versions + affected_versions = [] + version_section = container.find( + text=re.compile(r"affected|versions", re.IGNORECASE) + ) + + if version_section: + version_list = None + parent = version_section.parent + for _ in range(3): + if not parent: + break + lists = parent.find_all(["ul", "ol"]) + if lists: + version_list = lists[0] + break + parent = parent.parent + + if version_list: + affected_versions = [ + li.get_text(strip=True) for li in version_list.find_all("li") + ] + + # References - all links in the container + references = [] + for a in container.find_all("a", href=True): + if a["href"].startswith("http"): + references.append(a["href"]) + + self.parsed_advisories.append( + { + "cve_id": cve_id, + "summary": description[:500] if description else "", + "affected_versions": affected_versions, + "references": references, + } + ) + + self.log(f"Successfully parsed advisory for {cve_id}") + + except Exception as e: + self.log(f"Error parsing advisory for {cve_id}: {e}") + + def parse_version_to_purl(self, version_text): + """Convert version text to a PackageURL.""" + version_text = version_text.strip() + if "DXP" in version_text.upper(): + match = re.search(r"\d+(?:\.\d+)*", version_text) + if not match: + raise ValueError(f"No version found in {version_text}") + return PackageURL(type="liferay", name="dxp", version=match.group()) + else: + match = re.search(r"\d+(?:\.\d+)*", version_text) + if not match: + raise ValueError(f"No version found in {version_text}") + return PackageURL(type="liferay", name="portal", version=match.group()) + + def collect_advisories(self) -> Iterable[AdvisoryData]: + """Generate advisory data from parsed content.""" + for advisory in self.parsed_advisories: + affected_packages = [] + for version in advisory.get("affected_versions", []): + try: + purl = self.parse_version_to_purl(version) + affected_packages.append(AffectedPackage(package=purl)) + except ValueError as e: + self.log(f"Skipping invalid version {version}: {e}") + + yield AdvisoryData( + aliases=[advisory["cve_id"]], + summary=advisory["summary"], + references=[Reference(url=url) for url in advisory.get("references", [])], + affected_packages=affected_packages, + url=self.license_url, + ) + + def advisories_count(self): + if hasattr(self, "parsed_advisories"): + return len(self.parsed_advisories) + return 0