|
| 1 | +# |
| 2 | +# Copyright (c) nexB Inc. and others. All rights reserved. |
| 3 | +# VulnerableCode is a trademark of nexB Inc. |
| 4 | +# SPDX-License-Identifier: Apache-2.0 |
| 5 | +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. |
| 6 | +# See https://github.com/aboutcode-org/vulnerablecode for support or download. |
| 7 | +# See https://aboutcode.org for more information about nexB OSS projects. |
| 8 | +# |
| 9 | + |
| 10 | +import json |
| 11 | +import shutil |
| 12 | +from pathlib import Path |
| 13 | + |
| 14 | +import dateparser |
| 15 | +from django.core.exceptions import ValidationError |
| 16 | +from fetchcode.vcs import fetch_via_vcs |
| 17 | + |
| 18 | +from vulnerabilities.importer import AdvisoryData |
| 19 | +from vulnerabilities.importer import ReferenceV2 |
| 20 | +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 |
| 21 | + |
| 22 | + |
| 23 | +class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2): |
| 24 | + """ |
| 25 | + Pipeline to collect fix commits from Aosp Dataset: |
| 26 | + """ |
| 27 | + |
| 28 | + pipeline_id = "aosp_dataset_fix_commits" |
| 29 | + spdx_license_expression = "Apache-2.0" |
| 30 | + license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE" |
| 31 | + importer_name = "aosp_dataset" |
| 32 | + qualified_name = "aosp_dataset_fix_commits" |
| 33 | + |
| 34 | + @classmethod |
| 35 | + def steps(cls): |
| 36 | + return ( |
| 37 | + cls.clone, |
| 38 | + cls.collect_and_store_advisories, |
| 39 | + cls.clean_downloads, |
| 40 | + ) |
| 41 | + |
| 42 | + def clone(self): |
| 43 | + self.repo_url = "git+https://github.com/quarkslab/aosp_dataset" |
| 44 | + self.log(f"Cloning `{self.repo_url}`") |
| 45 | + self.vcs_response = fetch_via_vcs(self.repo_url) |
| 46 | + |
| 47 | + def advisories_count(self): |
| 48 | + root = Path(self.vcs_response.dest_dir) |
| 49 | + return sum(1 for _ in root.rglob("*.json")) |
| 50 | + |
| 51 | + def collect_advisories(self): |
| 52 | + self.log(f"Processing aosp_dataset fix commits.") |
| 53 | + base_path = Path(self.vcs_response.dest_dir) / "cves" |
| 54 | + for file_path in base_path.rglob("*.json"): |
| 55 | + if not file_path.name.startswith("CVE-"): |
| 56 | + continue |
| 57 | + |
| 58 | + with open(file_path) as f: |
| 59 | + vulnerability_data = json.load(f) |
| 60 | + |
| 61 | + vulnerability_id = vulnerability_data.get("cveId", []) |
| 62 | + if ( |
| 63 | + not vulnerability_id or "," in vulnerability_id |
| 64 | + ): # escape invalid multiple CVE-2017-13077, CVE-2017-13078 |
| 65 | + continue |
| 66 | + |
| 67 | + summary = vulnerability_data.get("vulnerabilityType") |
| 68 | + date_reported = vulnerability_data.get("dateReported") |
| 69 | + date_published = dateparser.parse(date_reported) if date_reported else None |
| 70 | + |
| 71 | + references = [] |
| 72 | + for commit_data in vulnerability_data.get("fixes", []): |
| 73 | + vcs_url = commit_data.get("patchUrl") |
| 74 | + |
| 75 | + if not vcs_url: |
| 76 | + continue |
| 77 | + |
| 78 | + ref = ReferenceV2(reference_type="commit", url=vcs_url) |
| 79 | + references.append(ref) |
| 80 | + |
| 81 | + yield AdvisoryData( |
| 82 | + advisory_id=vulnerability_id, |
| 83 | + summary=summary, |
| 84 | + references_v2=references, |
| 85 | + date_published=date_published, |
| 86 | + url=f"https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/{file_path.name}", |
| 87 | + ) |
| 88 | + |
| 89 | + def clean_downloads(self): |
| 90 | + """Cleanup any temporary repository data.""" |
| 91 | + self.log("Cleaning up local repository resources.") |
| 92 | + if hasattr(self, "repo") and self.repo.working_dir: |
| 93 | + shutil.rmtree(path=self.repo.working_dir) |
| 94 | + |
| 95 | + def on_failure(self): |
| 96 | + """Ensure cleanup is always performed on failure.""" |
| 97 | + self.clean_downloads() |
0 commit comments