Skip to content

Change Alpine data source to use new source #339

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sbs2001 merged 3 commits into from
Feb 7, 2021
Merged

Change Alpine data source to use new source #339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
05983b0
Change alpine data source
sbs2001 Feb 7, 2021
787ffe3
Update tests for alpine importer
sbs2001 Feb 7, 2021
32249ba
Update alpine data source info in SOURCES.rst
sbs2001 Feb 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion SOURCES.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
+----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
|rust | https://github.com/RustSec/advisory-db |rust crates |
+----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
|alpine | https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb |alpine packages |
|alpine | https://secdb.alpinelinux.org/ |alpine packages |
+----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
|archlinux | https://security.archlinux.org/json |arch packages |
+----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
Expand Down
5 changes: 1 addition & 4 deletions vulnerabilities/importer_yielder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,7 @@
'license': '',
'last_run': None,
'data_source': 'AlpineDataSource',
'data_source_cfg': {
'branch': None,
'repository_url': 'https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb',
},
'data_source_cfg': {},
},
{
'name': 'archlinux',
Expand Down
54 changes: 39 additions & 15 deletions vulnerabilities/importers/alpine_linux.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,19 @@
from typing import Mapping
from typing import Set

import requests
import yaml
from bs4 import BeautifulSoup
from packageurl import PackageURL
from schema import Or
from schema import Regex
from schema import Schema

from vulnerabilities.data_source import Advisory
from vulnerabilities.data_source import GitDataSource
from vulnerabilities.data_source import DataSource
from vulnerabilities.data_source import Reference
from vulnerabilities.helpers import load_yaml

BASE_URL = "https://secdb.alpinelinux.org/"


def validate_schema(advisory_dict):
Expand All @@ -54,6 +58,7 @@ def validate_schema(advisory_dict):
Regex(r"XSA-\d{3}"),
Regex(r"ZBX-\d{4}"),
Regex(r"wnpa-sec-\d{4}-\d{2}"),
Regex(r"GHSA-.{4}-.{4}-.{4}"),
)
],
"",
Expand All @@ -70,31 +75,50 @@ def validate_schema(advisory_dict):
Schema(scheme).validate(advisory_dict)


class AlpineDataSource(GitDataSource):
def __enter__(self):
super(AlpineDataSource, self).__enter__()
class AlpineDataSource(DataSource):

if not getattr(self, "_added_files", None):
self._added_files, self._updated_files = self.file_changes(
recursive=True, file_ext="yaml"
)
@staticmethod
def fetch_advisory_links():
index_page = BeautifulSoup(requests.get(BASE_URL).content, features="lxml")

alpine_versions = [
link.text for link in index_page.find_all("a")
if link.text.startswith("v")
]

advisory_directory_links = [
f"{BASE_URL}{version}" for version in alpine_versions
]

advisory_links = []
for advisory_directory_link in advisory_directory_links:
advisory_directory_page = requests.get(advisory_directory_link).content
advisory_directory_page = BeautifulSoup(advisory_directory_page, features="lxml")
advisory_links.extend([
f"{advisory_directory_link}{anchore_tag.text}"
for anchore_tag in advisory_directory_page.find_all("a")
if anchore_tag.text.endswith("yaml")
])

return advisory_links

def updated_advisories(self) -> Set[Advisory]:
files = self._updated_files.union(self._added_files)
advisories = []
for f in files:
advisories.extend(self._process_file(f))
advisory_links = self.fetch_advisory_links()
for link in advisory_links:
advisories.extend(self._process_link(link))

return self.batch_advisories(advisories)

def _process_file(self, path) -> List[Advisory]:
def _process_link(self, link) -> List[Advisory]:
advisories = []
yaml_response = requests.get(link).content
record = yaml.safe_load(yaml_response)

record = load_yaml(path)
if record["packages"] is None:
return advisories
validate_schema(record)

validate_schema(record)
for p in record["packages"]:
advisories.extend(
self._load_advisories(
Expand Down
220 changes: 124 additions & 96 deletions vulnerabilities/tests/test_alpine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,109 +18,137 @@
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
# for any legal advice.
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
# VulnerableCode is a free software tool from nexB Inc. and others.
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
import os
from unittest.mock import patch

from django.test import TestCase
import os
import yaml
from unittest import TestCase
from unittest.mock import patch, MagicMock

from vulnerabilities import models
from vulnerabilities.import_runner import ImportRunner
from packageurl import PackageURL

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_DATA = os.path.join(BASE_DIR, 'test_data/')
from vulnerabilities.data_source import Advisory, Reference
from vulnerabilities.importers.alpine_linux import AlpineDataSource

MOCK_ADDED_FILES = {os.path.join(TEST_DATA, 'alpine', p) for p in {
'v3.11/main.yaml',
}}

MOCK_UPDATED_FILES = {os.path.join(TEST_DATA, 'alpine', p) for p in {
'v3.11/community.yaml',
}}
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_DATA = os.path.join(BASE_DIR, "test_data", "alpine", "v3.11")


@patch('vulnerabilities.importers.AlpineDataSource.file_changes',
return_value=(MOCK_ADDED_FILES, MOCK_UPDATED_FILES))
@patch('vulnerabilities.importers.AlpineDataSource._ensure_repository')
class AlpineImportTest(TestCase):

@classmethod
def setUpClass(cls) -> None:
cls.importer = models.Importer.objects.create(
name='alpine_unittests',
license='',
last_run=None,
data_source='AlpineDataSource',
data_source_cfg={
'repository_url': 'https://example.com/unit-tests/alpine-secdb',
'working_directory': os.path.join(TEST_DATA, 'alpine'),
'create_working_directory': False,
'remove_working_directory': False,
},
)

@classmethod
def tearDownClass(cls) -> None:
pass

def test_import(self, *_):
runner = ImportRunner(self.importer, 5)

runner.run()

assert models.Vulnerability.objects.count() == 7
assert models.VulnerabilityReference.objects.count() == 1
assert models.PackageRelatedVulnerability.objects.filter(
is_vulnerable=False).count() == 8
assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=True).count() == 0

assert models.Package.objects.count() == 5

self.assert_for_package('cacti', '1.2.8-r0', cve_ids={'CVE-2019-17358'}, arch='armv7')
self.assert_for_package('cacti', '1.2.8-r0', cve_ids={'CVE-2019-17358'}, arch='x86_64')
self.assert_for_package('xen', '4.12.1-r0', vuln_ref='XSA-295', arch='x86_64')

self.assert_for_package(
'ansible',
'2.9.3-r0',
cve_ids={'CVE-2019-14904', 'CVE-2019-14905'},
arch='x86_64'
)
self.assert_for_package(
'ansible',
'2.8.6-r0',
cve_ids={'CVE-2019-14846', 'CVE-2019-14856', 'CVE-2019-14858'},
arch='x86_64'
)

def assert_for_package(self, name, version, cve_ids=None, vuln_ref=None, arch=None):
qs = models.Package.objects.filter(name=name, version=version)
assert qs

if arch:
pkg = qs.get(qualifiers__arch=arch)
else:
pkg = qs[0]

qs = models.PackageRelatedVulnerability.objects.filter(package=pkg)
assert qs

if cve_ids is None and vuln_ref is None:
return

vulns = {rp.vulnerability for rp in qs}

if cve_ids:
assert cve_ids == {v.cve_id for v in vulns}

if vuln_ref:
vuln_refs = set()

for vuln in vulns:
vuln_refs.update(
{v.reference_id for v in
models.VulnerabilityReference.objects.filter(vulnerability=vuln)}
)

assert vuln_ref in vuln_refs
def setUpClass(cls):
cls.data_source = AlpineDataSource(batch_size=1)

def test__process_link(self):
expected_advisories = [
Advisory(
summary="",
impacted_package_urls=[],
resolved_package_urls={
PackageURL(
type="alpine",
namespace=None,
name="ansible",
version="2.9.3-r0",
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
subpath=None,
)
},
vuln_references=[],
cve_id="CVE-2019-14904",
),
Advisory(
summary="",
impacted_package_urls=[],
resolved_package_urls={
PackageURL(
type="alpine",
namespace=None,
name="ansible",
version="2.9.3-r0",
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
subpath=None,
)
},
vuln_references=[],
cve_id="CVE-2019-14905",
),
Advisory(
summary="",
impacted_package_urls=[],
resolved_package_urls={
PackageURL(
type="alpine",
namespace=None,
name="ansible",
version="2.8.6-r0",
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
subpath=None,
)
},
vuln_references=[],
cve_id="CVE-2019-14846",
),
Advisory(
summary="",
impacted_package_urls=[],
resolved_package_urls={
PackageURL(
type="alpine",
namespace=None,
name="ansible",
version="2.8.6-r0",
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
subpath=None,
)
},
vuln_references=[],
cve_id="CVE-2019-14856",
),
Advisory(
summary="",
impacted_package_urls=[],
resolved_package_urls={
PackageURL(
type="alpine",
namespace=None,
name="ansible",
version="2.8.6-r0",
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
subpath=None,
)
},
vuln_references=[],
cve_id="CVE-2019-14858",
),
Advisory(
summary="",
impacted_package_urls=[],
resolved_package_urls={
PackageURL(
type="alpine",
namespace=None,
name="xen",
version="4.12.1-r0",
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
subpath=None,
)
},
vuln_references=[
Reference(
url="https://xenbits.xen.org/xsa/advisory-295.html", reference_id="XSA-295"
)
],
cve_id=None,
),
]
mock_requests = MagicMock()
mock_content = MagicMock()
with open(os.path.join(TEST_DATA, "main.yaml")) as f:
mock_requests.get = lambda x: mock_content
mock_content.content = f
with patch("vulnerabilities.importers.alpine_linux.requests", new=mock_requests):
found_advisories = self.data_source._process_link("does not matter")
assert expected_advisories == found_advisories
15 changes: 0 additions & 15 deletions vulnerabilities/tests/test_data/alpine/v3.11/community.yaml
View file
Open in desktop

This file was deleted.