From 05983b009b6681b861f2278f1ee168b4cf483462 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sun, 7 Feb 2021 14:56:04 +0530 Subject: [PATCH 1/3] Change alpine data source Use https://secdb.alpinelinux.org/ instead of https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb Signed-off-by: Shivam Sandbhor --- vulnerabilities/importer_yielder.py | 5 +-- vulnerabilities/importers/alpine_linux.py | 54 ++++++++++++++++------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 9400979b3..02f88e3e7 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -38,10 +38,7 @@ 'license': '', 'last_run': None, 'data_source': 'AlpineDataSource', - 'data_source_cfg': { - 'branch': None, - 'repository_url': 'https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb', - }, + 'data_source_cfg': {}, }, { 'name': 'archlinux', diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 29a555682..a3855303f 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -26,15 +26,19 @@ from typing import Mapping from typing import Set +import requests +import yaml +from bs4 import BeautifulSoup from packageurl import PackageURL from schema import Or from schema import Regex from schema import Schema from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import GitDataSource +from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import Reference -from vulnerabilities.helpers import load_yaml + +BASE_URL = "https://secdb.alpinelinux.org/" def validate_schema(advisory_dict): @@ -54,6 +58,7 @@ def validate_schema(advisory_dict): Regex(r"XSA-\d{3}"), Regex(r"ZBX-\d{4}"), Regex(r"wnpa-sec-\d{4}-\d{2}"), + Regex(r"GHSA-.{4}-.{4}-.{4}"), ) ], "", @@ -70,31 +75,50 @@ def validate_schema(advisory_dict): Schema(scheme).validate(advisory_dict) -class AlpineDataSource(GitDataSource): - def __enter__(self): - super(AlpineDataSource, self).__enter__() +class AlpineDataSource(DataSource): - if not getattr(self, "_added_files", None): - self._added_files, self._updated_files = self.file_changes( - recursive=True, file_ext="yaml" - ) + @staticmethod + def fetch_advisory_links(): + index_page = BeautifulSoup(requests.get(BASE_URL).content, features="lxml") + + alpine_versions = [ + link.text for link in index_page.find_all("a") + if link.text.startswith("v") + ] + + advisory_directory_links = [ + f"{BASE_URL}{version}" for version in alpine_versions + ] + + advisory_links = [] + for advisory_directory_link in advisory_directory_links: + advisory_directory_page = requests.get(advisory_directory_link).content + advisory_directory_page = BeautifulSoup(advisory_directory_page, features="lxml") + advisory_links.extend([ + f"{advisory_directory_link}{anchore_tag.text}" + for anchore_tag in advisory_directory_page.find_all("a") + if anchore_tag.text.endswith("yaml") + ]) + + return advisory_links def updated_advisories(self) -> Set[Advisory]: - files = self._updated_files.union(self._added_files) advisories = [] - for f in files: - advisories.extend(self._process_file(f)) + advisory_links = self.fetch_advisory_links() + for link in advisory_links: + advisories.extend(self._process_link(link)) return self.batch_advisories(advisories) - def _process_file(self, path) -> List[Advisory]: + def _process_link(self, link) -> List[Advisory]: advisories = [] + yaml_response = requests.get(link).content + record = yaml.safe_load(yaml_response) - record = load_yaml(path) if record["packages"] is None: return advisories - validate_schema(record) + validate_schema(record) for p in record["packages"]: advisories.extend( self._load_advisories( From 787ffe36231ae4351ea09753f158845eff015c5c Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sun, 7 Feb 2021 14:57:26 +0530 Subject: [PATCH 2/3] Update tests for alpine importer Signed-off-by: Shivam Sandbhor --- vulnerabilities/tests/test_alpine.py | 220 ++++++++++-------- .../test_data/alpine/v3.11/community.yaml | 15 -- 2 files changed, 124 insertions(+), 111 deletions(-) delete mode 100644 vulnerabilities/tests/test_data/alpine/v3.11/community.yaml diff --git a/vulnerabilities/tests/test_alpine.py b/vulnerabilities/tests/test_alpine.py index e7aa4b61d..ed7399e38 100644 --- a/vulnerabilities/tests/test_alpine.py +++ b/vulnerabilities/tests/test_alpine.py @@ -18,109 +18,137 @@ # OR CONDITIONS OF ANY KIND, either express or implied. No content created from # VulnerableCode should be considered or used as legal advice. Consult an Attorney # for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import os -from unittest.mock import patch -from django.test import TestCase +import os +import yaml +from unittest import TestCase +from unittest.mock import patch, MagicMock -from vulnerabilities import models -from vulnerabilities.import_runner import ImportRunner +from packageurl import PackageURL -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, 'test_data/') +from vulnerabilities.data_source import Advisory, Reference +from vulnerabilities.importers.alpine_linux import AlpineDataSource -MOCK_ADDED_FILES = {os.path.join(TEST_DATA, 'alpine', p) for p in { - 'v3.11/main.yaml', -}} -MOCK_UPDATED_FILES = {os.path.join(TEST_DATA, 'alpine', p) for p in { - 'v3.11/community.yaml', -}} +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data", "alpine", "v3.11") -@patch('vulnerabilities.importers.AlpineDataSource.file_changes', - return_value=(MOCK_ADDED_FILES, MOCK_UPDATED_FILES)) -@patch('vulnerabilities.importers.AlpineDataSource._ensure_repository') class AlpineImportTest(TestCase): - @classmethod - def setUpClass(cls) -> None: - cls.importer = models.Importer.objects.create( - name='alpine_unittests', - license='', - last_run=None, - data_source='AlpineDataSource', - data_source_cfg={ - 'repository_url': 'https://example.com/unit-tests/alpine-secdb', - 'working_directory': os.path.join(TEST_DATA, 'alpine'), - 'create_working_directory': False, - 'remove_working_directory': False, - }, - ) - - @classmethod - def tearDownClass(cls) -> None: - pass - - def test_import(self, *_): - runner = ImportRunner(self.importer, 5) - - runner.run() - - assert models.Vulnerability.objects.count() == 7 - assert models.VulnerabilityReference.objects.count() == 1 - assert models.PackageRelatedVulnerability.objects.filter( - is_vulnerable=False).count() == 8 - assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=True).count() == 0 - - assert models.Package.objects.count() == 5 - - self.assert_for_package('cacti', '1.2.8-r0', cve_ids={'CVE-2019-17358'}, arch='armv7') - self.assert_for_package('cacti', '1.2.8-r0', cve_ids={'CVE-2019-17358'}, arch='x86_64') - self.assert_for_package('xen', '4.12.1-r0', vuln_ref='XSA-295', arch='x86_64') - - self.assert_for_package( - 'ansible', - '2.9.3-r0', - cve_ids={'CVE-2019-14904', 'CVE-2019-14905'}, - arch='x86_64' - ) - self.assert_for_package( - 'ansible', - '2.8.6-r0', - cve_ids={'CVE-2019-14846', 'CVE-2019-14856', 'CVE-2019-14858'}, - arch='x86_64' - ) - - def assert_for_package(self, name, version, cve_ids=None, vuln_ref=None, arch=None): - qs = models.Package.objects.filter(name=name, version=version) - assert qs - - if arch: - pkg = qs.get(qualifiers__arch=arch) - else: - pkg = qs[0] - - qs = models.PackageRelatedVulnerability.objects.filter(package=pkg) - assert qs - - if cve_ids is None and vuln_ref is None: - return - - vulns = {rp.vulnerability for rp in qs} - - if cve_ids: - assert cve_ids == {v.cve_id for v in vulns} - - if vuln_ref: - vuln_refs = set() - - for vuln in vulns: - vuln_refs.update( - {v.reference_id for v in - models.VulnerabilityReference.objects.filter(vulnerability=vuln)} - ) - - assert vuln_ref in vuln_refs + def setUpClass(cls): + cls.data_source = AlpineDataSource(batch_size=1) + + def test__process_link(self): + expected_advisories = [ + Advisory( + summary="", + impacted_package_urls=[], + resolved_package_urls={ + PackageURL( + type="alpine", + namespace=None, + name="ansible", + version="2.9.3-r0", + qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, + subpath=None, + ) + }, + vuln_references=[], + cve_id="CVE-2019-14904", + ), + Advisory( + summary="", + impacted_package_urls=[], + resolved_package_urls={ + PackageURL( + type="alpine", + namespace=None, + name="ansible", + version="2.9.3-r0", + qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, + subpath=None, + ) + }, + vuln_references=[], + cve_id="CVE-2019-14905", + ), + Advisory( + summary="", + impacted_package_urls=[], + resolved_package_urls={ + PackageURL( + type="alpine", + namespace=None, + name="ansible", + version="2.8.6-r0", + qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, + subpath=None, + ) + }, + vuln_references=[], + cve_id="CVE-2019-14846", + ), + Advisory( + summary="", + impacted_package_urls=[], + resolved_package_urls={ + PackageURL( + type="alpine", + namespace=None, + name="ansible", + version="2.8.6-r0", + qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, + subpath=None, + ) + }, + vuln_references=[], + cve_id="CVE-2019-14856", + ), + Advisory( + summary="", + impacted_package_urls=[], + resolved_package_urls={ + PackageURL( + type="alpine", + namespace=None, + name="ansible", + version="2.8.6-r0", + qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, + subpath=None, + ) + }, + vuln_references=[], + cve_id="CVE-2019-14858", + ), + Advisory( + summary="", + impacted_package_urls=[], + resolved_package_urls={ + PackageURL( + type="alpine", + namespace=None, + name="xen", + version="4.12.1-r0", + qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, + subpath=None, + ) + }, + vuln_references=[ + Reference( + url="https://xenbits.xen.org/xsa/advisory-295.html", reference_id="XSA-295" + ) + ], + cve_id=None, + ), + ] + mock_requests = MagicMock() + mock_content = MagicMock() + with open(os.path.join(TEST_DATA, "main.yaml")) as f: + mock_requests.get = lambda x: mock_content + mock_content.content = f + with patch("vulnerabilities.importers.alpine_linux.requests", new=mock_requests): + found_advisories = self.data_source._process_link("does not matter") + assert expected_advisories == found_advisories diff --git a/vulnerabilities/tests/test_data/alpine/v3.11/community.yaml b/vulnerabilities/tests/test_data/alpine/v3.11/community.yaml deleted file mode 100644 index b1a4f2df8..000000000 --- a/vulnerabilities/tests/test_data/alpine/v3.11/community.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -distroversion: v3.11 -reponame: community -archs: - - x86_64 - - armv7 -urlprefix: http://dl-cdn.alpinelinux.org/alpine -apkurl: "{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-{{pkg.ver}}.apk" -packages: - - pkg: - name: cacti - secfixes: - 1.2.8-r0: - - CVE-2019-17358 - From 32249ba2c59dde3eb08f24fb08c1a2aa6a4918fc Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sun, 7 Feb 2021 14:57:54 +0530 Subject: [PATCH 3/3] Update alpine data source info in SOURCES.rst Signed-off-by: Shivam Sandbhor --- SOURCES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SOURCES.rst b/SOURCES.rst index 4bba04be2..ab024f532 100644 --- a/SOURCES.rst +++ b/SOURCES.rst @@ -3,7 +3,7 @@ +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ |rust | https://github.com/RustSec/advisory-db |rust crates | +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ -|alpine | https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb |alpine packages | +|alpine | https://secdb.alpinelinux.org/ |alpine packages | +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ |archlinux | https://security.archlinux.org/json |arch packages | +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+