Skip to content

Commit 837bb8f

Browse files
sbs2001sbs2001
authored
Merge pull request #339 from sbs2001/333-alpine-change-source
Change Alpine data source to use new source
2 parents 6609bcf + 32249ba commit 837bb8f

5 files changed

Lines changed: 165 additions & 131 deletions

File tree

‎SOURCES.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
+----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
44
|rust | https://github.com/RustSec/advisory-db |rust crates |
55
+----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
6-
|alpine | https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb |alpine packages |
6+
|alpine | https://secdb.alpinelinux.org/ |alpine packages |
77
+----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+
88
|archlinux | https://security.archlinux.org/json |arch packages |
99
+----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+

‎vulnerabilities/importer_yielder.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,7 @@
3838
'license': '',
3939
'last_run': None,
4040
'data_source': 'AlpineDataSource',
41-
'data_source_cfg': {
42-
'branch': None,
43-
'repository_url': 'https://gitlab.alpinelinux.org/alpine/infra/alpine-secdb',
44-
},
41+
'data_source_cfg': {},
4542
},
4643
{
4744
'name': 'archlinux',

‎vulnerabilities/importers/alpine_linux.py‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,19 @@
2626
from typing import Mapping
2727
from typing import Set
2828

29+
import requests
30+
import yaml
31+
from bs4 import BeautifulSoup
2932
from packageurl import PackageURL
3033
from schema import Or
3134
from schema import Regex
3235
from schema import Schema
3336

3437
from vulnerabilities.data_source import Advisory
35-
from vulnerabilities.data_source import GitDataSource
38+
from vulnerabilities.data_source import DataSource
3639
from vulnerabilities.data_source import Reference
37-
from vulnerabilities.helpers import load_yaml
40+
41+
BASE_URL = "https://secdb.alpinelinux.org/"
3842

3943

4044
def validate_schema(advisory_dict):
@@ -54,6 +58,7 @@ def validate_schema(advisory_dict):
5458
Regex(r"XSA-\d{3}"),
5559
Regex(r"ZBX-\d{4}"),
5660
Regex(r"wnpa-sec-\d{4}-\d{2}"),
61+
Regex(r"GHSA-.{4}-.{4}-.{4}"),
5762
)
5863
],
5964
"",
@@ -70,31 +75,50 @@ def validate_schema(advisory_dict):
7075
Schema(scheme).validate(advisory_dict)
7176

7277

73-
class AlpineDataSource(GitDataSource):
74-
def __enter__(self):
75-
super(AlpineDataSource, self).__enter__()
78+
class AlpineDataSource(DataSource):
7679

77-
if not getattr(self, "_added_files", None):
78-
self._added_files, self._updated_files = self.file_changes(
79-
recursive=True, file_ext="yaml"
80-
)
80+
@staticmethod
81+
def fetch_advisory_links():
82+
index_page = BeautifulSoup(requests.get(BASE_URL).content, features="lxml")
83+
84+
alpine_versions = [
85+
link.text for link in index_page.find_all("a")
86+
if link.text.startswith("v")
87+
]
88+
89+
advisory_directory_links = [
90+
f"{BASE_URL}{version}" for version in alpine_versions
91+
]
92+
93+
advisory_links = []
94+
for advisory_directory_link in advisory_directory_links:
95+
advisory_directory_page = requests.get(advisory_directory_link).content
96+
advisory_directory_page = BeautifulSoup(advisory_directory_page, features="lxml")
97+
advisory_links.extend([
98+
f"{advisory_directory_link}{anchore_tag.text}"
99+
for anchore_tag in advisory_directory_page.find_all("a")
100+
if anchore_tag.text.endswith("yaml")
101+
])
102+
103+
return advisory_links
81104

82105
def updated_advisories(self) -> Set[Advisory]:
83-
files = self._updated_files.union(self._added_files)
84106
advisories = []
85-
for f in files:
86-
advisories.extend(self._process_file(f))
107+
advisory_links = self.fetch_advisory_links()
108+
for link in advisory_links:
109+
advisories.extend(self._process_link(link))
87110

88111
return self.batch_advisories(advisories)
89112

90-
def _process_file(self, path) -> List[Advisory]:
113+
def _process_link(self, link) -> List[Advisory]:
91114
advisories = []
115+
yaml_response = requests.get(link).content
116+
record = yaml.safe_load(yaml_response)
92117

93-
record = load_yaml(path)
94118
if record["packages"] is None:
95119
return advisories
96-
validate_schema(record)
97120

121+
validate_schema(record)
98122
for p in record["packages"]:
99123
advisories.extend(
100124
self._load_advisories(

‎vulnerabilities/tests/test_alpine.py‎

Lines changed: 124 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -18,109 +18,137 @@
1818
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
1919
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
2020
# for any legal advice.
21-
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
21+
# VulnerableCode is a free software tool from nexB Inc. and others.
2222
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
23-
import os
24-
from unittest.mock import patch
2523

26-
from django.test import TestCase
24+
import os
25+
import yaml
26+
from unittest import TestCase
27+
from unittest.mock import patch, MagicMock
2728

28-
from vulnerabilities import models
29-
from vulnerabilities.import_runner import ImportRunner
29+
from packageurl import PackageURL
3030

31-
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
32-
TEST_DATA = os.path.join(BASE_DIR, 'test_data/')
31+
from vulnerabilities.data_source import Advisory, Reference
32+
from vulnerabilities.importers.alpine_linux import AlpineDataSource
3333

34-
MOCK_ADDED_FILES = {os.path.join(TEST_DATA, 'alpine', p) for p in {
35-
'v3.11/main.yaml',
36-
}}
3734

38-
MOCK_UPDATED_FILES = {os.path.join(TEST_DATA, 'alpine', p) for p in {
39-
'v3.11/community.yaml',
40-
}}
35+
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
36+
TEST_DATA = os.path.join(BASE_DIR, "test_data", "alpine", "v3.11")
4137

4238

43-
@patch('vulnerabilities.importers.AlpineDataSource.file_changes',
44-
return_value=(MOCK_ADDED_FILES, MOCK_UPDATED_FILES))
45-
@patch('vulnerabilities.importers.AlpineDataSource._ensure_repository')
4639
class AlpineImportTest(TestCase):
47-
4840
@classmethod
49-
def setUpClass(cls) -> None:
50-
cls.importer = models.Importer.objects.create(
51-
name='alpine_unittests',
52-
license='',
53-
last_run=None,
54-
data_source='AlpineDataSource',
55-
data_source_cfg={
56-
'repository_url': 'https://example.com/unit-tests/alpine-secdb',
57-
'working_directory': os.path.join(TEST_DATA, 'alpine'),
58-
'create_working_directory': False,
59-
'remove_working_directory': False,
60-
},
61-
)
62-
63-
@classmethod
64-
def tearDownClass(cls) -> None:
65-
pass
66-
67-
def test_import(self, *_):
68-
runner = ImportRunner(self.importer, 5)
69-
70-
runner.run()
71-
72-
assert models.Vulnerability.objects.count() == 7
73-
assert models.VulnerabilityReference.objects.count() == 1
74-
assert models.PackageRelatedVulnerability.objects.filter(
75-
is_vulnerable=False).count() == 8
76-
assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=True).count() == 0
77-
78-
assert models.Package.objects.count() == 5
79-
80-
self.assert_for_package('cacti', '1.2.8-r0', cve_ids={'CVE-2019-17358'}, arch='armv7')
81-
self.assert_for_package('cacti', '1.2.8-r0', cve_ids={'CVE-2019-17358'}, arch='x86_64')
82-
self.assert_for_package('xen', '4.12.1-r0', vuln_ref='XSA-295', arch='x86_64')
83-
84-
self.assert_for_package(
85-
'ansible',
86-
'2.9.3-r0',
87-
cve_ids={'CVE-2019-14904', 'CVE-2019-14905'},
88-
arch='x86_64'
89-
)
90-
self.assert_for_package(
91-
'ansible',
92-
'2.8.6-r0',
93-
cve_ids={'CVE-2019-14846', 'CVE-2019-14856', 'CVE-2019-14858'},
94-
arch='x86_64'
95-
)
96-
97-
def assert_for_package(self, name, version, cve_ids=None, vuln_ref=None, arch=None):
98-
qs = models.Package.objects.filter(name=name, version=version)
99-
assert qs
100-
101-
if arch:
102-
pkg = qs.get(qualifiers__arch=arch)
103-
else:
104-
pkg = qs[0]
105-
106-
qs = models.PackageRelatedVulnerability.objects.filter(package=pkg)
107-
assert qs
108-
109-
if cve_ids is None and vuln_ref is None:
110-
return
111-
112-
vulns = {rp.vulnerability for rp in qs}
113-
114-
if cve_ids:
115-
assert cve_ids == {v.cve_id for v in vulns}
116-
117-
if vuln_ref:
118-
vuln_refs = set()
119-
120-
for vuln in vulns:
121-
vuln_refs.update(
122-
{v.reference_id for v in
123-
models.VulnerabilityReference.objects.filter(vulnerability=vuln)}
124-
)
125-
126-
assert vuln_ref in vuln_refs
41+
def setUpClass(cls):
42+
cls.data_source = AlpineDataSource(batch_size=1)
43+
44+
def test__process_link(self):
45+
expected_advisories = [
46+
Advisory(
47+
summary="",
48+
impacted_package_urls=[],
49+
resolved_package_urls={
50+
PackageURL(
51+
type="alpine",
52+
namespace=None,
53+
name="ansible",
54+
version="2.9.3-r0",
55+
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
56+
subpath=None,
57+
)
58+
},
59+
vuln_references=[],
60+
cve_id="CVE-2019-14904",
61+
),
62+
Advisory(
63+
summary="",
64+
impacted_package_urls=[],
65+
resolved_package_urls={
66+
PackageURL(
67+
type="alpine",
68+
namespace=None,
69+
name="ansible",
70+
version="2.9.3-r0",
71+
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
72+
subpath=None,
73+
)
74+
},
75+
vuln_references=[],
76+
cve_id="CVE-2019-14905",
77+
),
78+
Advisory(
79+
summary="",
80+
impacted_package_urls=[],
81+
resolved_package_urls={
82+
PackageURL(
83+
type="alpine",
84+
namespace=None,
85+
name="ansible",
86+
version="2.8.6-r0",
87+
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
88+
subpath=None,
89+
)
90+
},
91+
vuln_references=[],
92+
cve_id="CVE-2019-14846",
93+
),
94+
Advisory(
95+
summary="",
96+
impacted_package_urls=[],
97+
resolved_package_urls={
98+
PackageURL(
99+
type="alpine",
100+
namespace=None,
101+
name="ansible",
102+
version="2.8.6-r0",
103+
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
104+
subpath=None,
105+
)
106+
},
107+
vuln_references=[],
108+
cve_id="CVE-2019-14856",
109+
),
110+
Advisory(
111+
summary="",
112+
impacted_package_urls=[],
113+
resolved_package_urls={
114+
PackageURL(
115+
type="alpine",
116+
namespace=None,
117+
name="ansible",
118+
version="2.8.6-r0",
119+
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
120+
subpath=None,
121+
)
122+
},
123+
vuln_references=[],
124+
cve_id="CVE-2019-14858",
125+
),
126+
Advisory(
127+
summary="",
128+
impacted_package_urls=[],
129+
resolved_package_urls={
130+
PackageURL(
131+
type="alpine",
132+
namespace=None,
133+
name="xen",
134+
version="4.12.1-r0",
135+
qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"},
136+
subpath=None,
137+
)
138+
},
139+
vuln_references=[
140+
Reference(
141+
url="https://xenbits.xen.org/xsa/advisory-295.html", reference_id="XSA-295"
142+
)
143+
],
144+
cve_id=None,
145+
),
146+
]
147+
mock_requests = MagicMock()
148+
mock_content = MagicMock()
149+
with open(os.path.join(TEST_DATA, "main.yaml")) as f:
150+
mock_requests.get = lambda x: mock_content
151+
mock_content.content = f
152+
with patch("vulnerabilities.importers.alpine_linux.requests", new=mock_requests):
153+
found_advisories = self.data_source._process_link("does not matter")
154+
assert expected_advisories == found_advisories

‎vulnerabilities/tests/test_data/alpine/v3.11/community.yaml‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

0 commit comments

Comments
 (0)