Skip to content

Commit 6b4ddf1

Browse files
lohani2280lohani2280
committed
Collect vulnerabilities from arch linux #20
Signed-off-by: Ayush Lohani <lohani.ayush01@gmail.com>
1 parent 55a633d commit 6b4ddf1

5 files changed

Lines changed: 350 additions & 0 deletions

File tree

‎vulnerabilities/data_dump.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,29 @@ def ubuntu_dump(html):
6868
vulnerability=vulnerability,
6969
package=package
7070
)
71+
72+
73+
def archlinux_dump(extract_data):
74+
"""
75+
Save data scraped from archlinux' security tracker.
76+
Args:
77+
extract_data(generator): data collected from archlinux' security tracker
78+
"""
79+
for raw_data in extract_data:
80+
for data in raw_data:
81+
vulnerability = Vulnerability.objects.create(
82+
summary=data['description'],
83+
)
84+
VulnerabilityReference.objects.create(
85+
vulnerability=vulnerability,
86+
reference_id=data['vulnerability_id'],
87+
source='archlinux',
88+
)
89+
package = Package.objects.create(
90+
name=data['package_name'],
91+
version=data['version'],
92+
)
93+
ImpactedPackage.objects.create(
94+
vulnerability=vulnerability,
95+
package=package
96+
)

‎vulnerabilities/scraper/archlinux.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#
2+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
4+
# The VulnerableCode software is licensed under the Apache License version 2.0.
5+
# Data generated with VulnerableCode require an acknowledgment.
6+
#
7+
# You may not use this software except in compliance with the License.
8+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
9+
# Unless required by applicable law or agreed to in writing, software distributed
10+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
# specific language governing permissions and limitations under the License.
13+
#
14+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
15+
# derivative work, you must accompany this data with the following acknowledgment:
16+
#
17+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
18+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
19+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
20+
# for any legal advice.
21+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
22+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
23+
24+
import json
25+
from urllib.request import urlopen
26+
27+
28+
ARCHLINUX_TRACKER_URL = 'https://security.archlinux.org/json'
29+
30+
31+
def extract_vulnerabilities(arch_data):
32+
"""
33+
Return a sequence of mappings for each existing combination of
34+
package and vulnerability from a mapping of arch linux vulnerabilities
35+
data.
36+
"""
37+
for item in arch_data:
38+
package_vulnerabilities = []
39+
vulnerabilities = item['issues']
40+
packages_name = item['packages']
41+
42+
if not vulnerabilities or not packages_name:
43+
continue
44+
45+
for package in packages_name:
46+
affected_version = item['affected']
47+
fixed_version = item['fixed']
48+
for vulnerability in vulnerabilities:
49+
package_vulnerabilities.append({
50+
'package_name': package,
51+
'vulnerability_id': vulnerability,
52+
'description': item['type'],
53+
'status': item['status'],
54+
'severity': item['severity'],
55+
'version': affected_version
56+
})
57+
58+
for vulnerability in vulnerabilities:
59+
if not fixed_version:
60+
fixed_version = ''
61+
package_vulnerabilities.append({
62+
'package_name': package,
63+
'vulnerability_id': vulnerability,
64+
'description': item['type'],
65+
'status': item['status'],
66+
'severity': item['severity'],
67+
'version': fixed_version
68+
})
69+
yield package_vulnerabilities
70+
71+
72+
def scrape_vulnerabilities():
73+
"""
74+
Scrape arch linux' security tracker.
75+
"""
76+
json_content = urlopen(ARCHLINUX_TRACKER_URL).read()
77+
return extract_vulnerabilities(json.loads(json_content))

‎vulnerabilities/tests/test_data/archlinux.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
[
2+
{
3+
"name": "AVG-837",
4+
"packages": [
5+
"libarchive"
6+
],
7+
"status": "Vulnerable",
8+
"severity": "High",
9+
"type": "multiple issues",
10+
"affected": "3.3.3-1",
11+
"fixed": "3.3.3-2",
12+
"ticket": null,
13+
"issues": [
14+
"CVE-2019-1000020",
15+
"CVE-2019-1000019",
16+
"CVE-2018-1000880",
17+
"CVE-2018-1000879",
18+
"CVE-2018-1000878",
19+
"CVE-2018-1000877"
20+
],
21+
"advisories": []
22+
},
23+
{
24+
"name": "AVG-57",
25+
"packages": [
26+
"python2-django",
27+
"python-django"
28+
],
29+
"status": "Fixed",
30+
"severity": "High",
31+
"type": "multiple issues",
32+
"affected": "1.10.2-1",
33+
"fixed": "1.10.3-1",
34+
"ticket": null,
35+
"issues": [
36+
"CVE-2016-9014",
37+
"CVE-2016-9013"
38+
],
39+
"advisories": [
40+
"ASA-201611-15",
41+
"ASA-201611-14"
42+
]
43+
}
44+
]

‎vulnerabilities/tests/test_data_dump.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,10 @@
3131
from vulnerabilities.models import Package
3232
from vulnerabilities.data_dump import debian_dump
3333
from vulnerabilities.data_dump import ubuntu_dump
34+
from vulnerabilities.data_dump import archlinux_dump
3435
from vulnerabilities.scraper import debian
3536
from vulnerabilities.scraper import ubuntu
37+
from vulnerabilities.scraper import archlinux
3638

3739

3840
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -85,3 +87,24 @@ def test_ubuntu_data_dump(self):
8587
reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0]
8688
self.assertEqual(reference.reference_id, 'CVE-2002-2439')
8789
self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6')
90+
91+
def test_archlinux_data_dump(self):
92+
"""
93+
Scrape data from Archlinux' main tracker, save it
94+
in the database and verify entries.
95+
"""
96+
with open(os.path.join(TEST_DATA, 'archlinux.json')) as f:
97+
test_data = json.loads(f.read())
98+
99+
extract_data = archlinux.extract_vulnerabilities(test_data)
100+
archlinux_dump(extract_data)
101+
102+
self.assertEqual(20, Vulnerability.objects.count())
103+
self.assertEqual(20, VulnerabilityReference.objects.count())
104+
self.assertEqual(20, Package.objects.count())
105+
106+
self.assertTrue(Vulnerability.objects.filter(
107+
summary='multiple issues'))
108+
109+
self.assertEqual(Package.objects.filter(name='libarchive')[0].name, 'libarchive')
110+
self.assertEqual(Package.objects.filter(name='python-django')[0].name, 'python-django')

‎vulnerabilities/tests/test_scrapers.py‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
from vulnerabilities.scraper import debian
2929
from vulnerabilities.scraper import ubuntu
30+
from vulnerabilities.scraper import archlinux
3031

3132

3233
def test_ubuntu_extract_cves():
@@ -93,3 +94,182 @@ def test_debian_extract_vulnerabilities():
9394
]
9495

9596
assert expected == debian.extract_vulnerabilities(test_data)
97+
98+
99+
def test_archlinux_extract_vulnerabilities():
100+
archlinux_test_file = join(dirname(__file__), 'test_data', 'archlinux.json')
101+
102+
with open(archlinux_test_file) as f:
103+
test_data = json.loads(f.read())
104+
105+
G = archlinux.extract_vulnerabilities(test_data)
106+
107+
expected = [
108+
{
109+
'package_name': 'libarchive',
110+
'vulnerability_id': 'CVE-2019-1000020',
111+
'description': 'multiple issues',
112+
'status': 'Vulnerable',
113+
'severity': 'High',
114+
'version': '3.3.3-1'
115+
},
116+
{
117+
'package_name': 'libarchive',
118+
'vulnerability_id': 'CVE-2019-1000019',
119+
'description': 'multiple issues',
120+
'status': 'Vulnerable',
121+
'severity': 'High',
122+
'version': '3.3.3-1'
123+
},
124+
{
125+
'package_name': 'libarchive',
126+
'vulnerability_id': 'CVE-2018-1000880',
127+
'description': 'multiple issues',
128+
'status': 'Vulnerable',
129+
'severity': 'High',
130+
'version': '3.3.3-1'
131+
},
132+
{
133+
'package_name': 'libarchive',
134+
'vulnerability_id': 'CVE-2018-1000879',
135+
'description': 'multiple issues',
136+
'status': 'Vulnerable',
137+
'severity': 'High',
138+
'version': '3.3.3-1'
139+
},
140+
{
141+
'package_name': 'libarchive',
142+
'vulnerability_id': 'CVE-2018-1000878',
143+
'description': 'multiple issues',
144+
'status': 'Vulnerable',
145+
'severity': 'High',
146+
'version': '3.3.3-1'
147+
},
148+
{
149+
'package_name': 'libarchive',
150+
'vulnerability_id': 'CVE-2018-1000877',
151+
'description': 'multiple issues',
152+
'status': 'Vulnerable',
153+
'severity': 'High',
154+
'version': '3.3.3-1'
155+
},
156+
{
157+
'package_name': 'libarchive',
158+
'vulnerability_id': 'CVE-2019-1000020',
159+
'description': 'multiple issues',
160+
'status': 'Vulnerable',
161+
'severity': 'High',
162+
'version': '3.3.3-2'
163+
},
164+
{
165+
'package_name': 'libarchive',
166+
'vulnerability_id': 'CVE-2019-1000019',
167+
'description': 'multiple issues',
168+
'status': 'Vulnerable',
169+
'severity': 'High',
170+
'version': '3.3.3-2'
171+
},
172+
{
173+
'package_name': 'libarchive',
174+
'vulnerability_id': 'CVE-2018-1000880',
175+
'description': 'multiple issues',
176+
'status': 'Vulnerable',
177+
'severity': 'High',
178+
'version': '3.3.3-2'
179+
},
180+
{
181+
'package_name': 'libarchive',
182+
'vulnerability_id': 'CVE-2018-1000879',
183+
'description': 'multiple issues',
184+
'status': 'Vulnerable',
185+
'severity': 'High',
186+
'version': '3.3.3-2'
187+
},
188+
{
189+
'package_name': 'libarchive',
190+
'vulnerability_id': 'CVE-2018-1000878',
191+
'description': 'multiple issues',
192+
'status': 'Vulnerable',
193+
'severity': 'High',
194+
'version': '3.3.3-2'
195+
},
196+
{
197+
'package_name': 'libarchive',
198+
'vulnerability_id': 'CVE-2018-1000877',
199+
'description': 'multiple issues',
200+
'status': 'Vulnerable',
201+
'severity': 'High',
202+
'version': '3.3.3-2'
203+
}
204+
]
205+
206+
assert expected == next(G)
207+
208+
expected = [
209+
{
210+
'package_name': 'python2-django',
211+
'vulnerability_id': 'CVE-2016-9014',
212+
'description': 'multiple issues',
213+
'status': 'Fixed',
214+
'severity': 'High',
215+
'version': '1.10.2-1'
216+
},
217+
{
218+
'package_name': 'python2-django',
219+
'vulnerability_id': 'CVE-2016-9013',
220+
'description': 'multiple issues',
221+
'status': 'Fixed',
222+
'severity': 'High',
223+
'version': '1.10.2-1'
224+
},
225+
{
226+
'package_name': 'python2-django',
227+
'vulnerability_id': 'CVE-2016-9014',
228+
'description': 'multiple issues',
229+
'status': 'Fixed',
230+
'severity': 'High',
231+
'version': '1.10.3-1'
232+
},
233+
{
234+
'package_name': 'python2-django',
235+
'vulnerability_id': 'CVE-2016-9013',
236+
'description': 'multiple issues',
237+
'status': 'Fixed',
238+
'severity': 'High',
239+
'version': '1.10.3-1'
240+
},
241+
{
242+
'package_name': 'python-django',
243+
'vulnerability_id': 'CVE-2016-9014',
244+
'description': 'multiple issues',
245+
'status': 'Fixed',
246+
'severity': 'High',
247+
'version': '1.10.2-1'
248+
},
249+
{
250+
'package_name': 'python-django',
251+
'vulnerability_id': 'CVE-2016-9013',
252+
'description': 'multiple issues',
253+
'status': 'Fixed',
254+
'severity': 'High',
255+
'version': '1.10.2-1'
256+
},
257+
{
258+
'package_name': 'python-django',
259+
'vulnerability_id': 'CVE-2016-9014',
260+
'description': 'multiple issues',
261+
'status': 'Fixed',
262+
'severity': 'High',
263+
'version': '1.10.3-1'
264+
},
265+
{
266+
'package_name': 'python-django',
267+
'vulnerability_id': 'CVE-2016-9013',
268+
'description': 'multiple issues',
269+
'status': 'Fixed',
270+
'severity': 'High',
271+
'version': '1.10.3-1'
272+
}
273+
]
274+
275+
assert expected == next(G)

0 commit comments

Comments
 (0)