Skip to content

Commit 5d6c681

Browse files
lohani2280lohani2280
committed
Collect vulnerabilities from arch linux #20
Signed-off-by: Ayush Lohani <lohani.ayush01@gmail.com>
1 parent 55a633d commit 5d6c681

5 files changed

Lines changed: 228 additions & 0 deletions

File tree

‎vulnerabilities/data_dump.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,28 @@ def ubuntu_dump(html):
6868
vulnerability=vulnerability,
6969
package=package
7070
)
71+
72+
73+
def archlinux_dump(extract_data):
74+
"""
75+
Save data scraped from archlinux' security tracker.
76+
Args:
77+
extract_data(list): data collected from archlinux' security tracker
78+
"""
79+
for data in extract_data:
80+
vulnerability = Vulnerability.objects.create(
81+
summary=data['description'],
82+
)
83+
VulnerabilityReference.objects.create(
84+
vulnerability=vulnerability,
85+
reference_id=data['vulnerability_id'],
86+
source="archlinux",
87+
)
88+
package = Package.objects.create(
89+
name=data['package_name'],
90+
version=data['version'],
91+
)
92+
ImpactedPackage.objects.create(
93+
vulnerability=vulnerability,
94+
package=package
95+
)

‎vulnerabilities/scraper/archlinux.py‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#
2+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
4+
# The VulnerableCode software is licensed under the Apache License version 2.0.
5+
# Data generated with VulnerableCode require an acknowledgment.
6+
#
7+
# You may not use this software except in compliance with the License.
8+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
9+
# Unless required by applicable law or agreed to in writing, software distributed
10+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
# specific language governing permissions and limitations under the License.
13+
#
14+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
15+
# derivative work, you must accompany this data with the following acknowledgment:
16+
#
17+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
18+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
19+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
20+
# for any legal advice.
21+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
22+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
23+
24+
import json
25+
from urllib.request import urlopen
26+
27+
28+
ARCHLINUX_TRACKER_URL = 'https://security.archlinux.org/json'
29+
30+
31+
def extract_vulnerabilities(arch_data):
32+
"""
33+
Return a sequence of mappings for each existing combination of
34+
package and vulnerability from a mapping of arch linux vulnerabilities
35+
data.
36+
"""
37+
package_vulnerabilities = []
38+
39+
for item in arch_data:
40+
vulnerabilities = item["issues"]
41+
package_name = item["packages"][0]
42+
affected_version = item["affected"]
43+
fixed_version = item["fixed"]
44+
if not vulnerabilities or not package_name:
45+
continue
46+
47+
for vulnerability in vulnerabilities:
48+
if not affected_version:
49+
break
50+
package_vulnerabilities.append({
51+
'package_name': package_name,
52+
'vulnerability_id': vulnerability,
53+
'description': item["type"],
54+
'status': item["status"],
55+
'severity': item["severity"],
56+
'version': affected_version
57+
})
58+
59+
for vulnerability in vulnerabilities:
60+
if not fixed_version:
61+
break
62+
package_vulnerabilities.append({
63+
'package_name': package_name,
64+
'vulnerability_id': vulnerability,
65+
'description': item["type"],
66+
'status': item["status"],
67+
'severity': item["severity"],
68+
'version': fixed_version
69+
})
70+
return package_vulnerabilities
71+
72+
73+
def scrape_vulnerabilities():
74+
"""
75+
Scrape arch linux' security tracker.
76+
"""
77+
json_content = urlopen(ARCHLINUX_TRACKER_URL).read()
78+
return extract_vulnerabilities(json.loads(json_content))

‎vulnerabilities/tests/test_data/archlinux.json‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
[
2+
{
3+
"name": "AVG-837",
4+
"packages": [
5+
"libarchive"
6+
],
7+
"status": "Vulnerable",
8+
"severity": "High",
9+
"type": "multiple issues",
10+
"affected": "3.3.3-1",
11+
"fixed": "3.3.3-2",
12+
"ticket": null,
13+
"issues": [
14+
"CVE-2019-1000020",
15+
"CVE-2019-1000019",
16+
"CVE-2018-1000880",
17+
"CVE-2018-1000879",
18+
"CVE-2018-1000878",
19+
"CVE-2018-1000877"
20+
],
21+
"advisories": []
22+
},
23+
{
24+
"name": "AVG-886",
25+
"packages": [
26+
"libtiff"
27+
],
28+
"status": "Vulnerable",
29+
"severity": "Medium",
30+
"type": "multiple issues",
31+
"affected": "4.0.10-1",
32+
"fixed": "4.0.10-2",
33+
"ticket": null,
34+
"issues": [
35+
"CVE-2019-7663",
36+
"CVE-2019-6128"
37+
],
38+
"advisories": []
39+
},
40+
{
41+
"name": "AVG-340",
42+
"packages": [
43+
"ipsec-tools"
44+
],
45+
"status": "Vulnerable",
46+
"severity": "Medium",
47+
"type": "denial of service",
48+
"affected": "0.8.2-8",
49+
"fixed": "0.8.3-1",
50+
"ticket": null,
51+
"issues": [
52+
"CVE-2016-10396"
53+
],
54+
"advisories": []
55+
}
56+
]

‎vulnerabilities/tests/test_data_dump.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
from vulnerabilities.data_dump import ubuntu_dump
3434
from vulnerabilities.scraper import debian
3535
from vulnerabilities.scraper import ubuntu
36+
from vulnerabilities.scraper import archlinux
3637

3738

3839
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -85,3 +86,31 @@ def test_ubuntu_data_dump(self):
8586
reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0]
8687
self.assertEqual(reference.reference_id, 'CVE-2002-2439')
8788
self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6')
89+
90+
def test_archlinux_data_dump(self):
91+
"""
92+
Scrape data from Archlinux' main tracker, save it
93+
in the database and verify entries.
94+
"""
95+
with open(os.path.join(TEST_DATA, 'archlinux.json')) as f:
96+
test_data = json.loads(f.read())
97+
98+
extract_data = archlinux.extract_vulnerabilities(test_data)
99+
debian_dump(extract_data)
100+
101+
self.assertEqual(3, Vulnerability.objects.count())
102+
self.assertEqual(3, VulnerabilityReference.objects.count())
103+
self.assertEqual(3, Package.objects.count())
104+
105+
self.assertTrue(Vulnerability.objects.get(
106+
summary='denial of service'))
107+
108+
self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-837'))
109+
110+
self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-886'))
111+
112+
self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-340'))
113+
114+
self.assertEqual(Package.objects.filter(name='libarchive')[0].name, 'libarchive')
115+
self.assertTrue(Package.objects.get(name='libtiff'))
116+
self.assertEqual(Package.objects.filter(version='0.8.3-1')[0].version, '0.8.3-1')

‎vulnerabilities/tests/test_scrapers.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
from vulnerabilities.scraper import debian
2929
from vulnerabilities.scraper import ubuntu
30+
from vulnerabilities.scraper import archlinux
3031

3132

3233
def test_ubuntu_extract_cves():
@@ -93,3 +94,42 @@ def test_debian_extract_vulnerabilities():
9394
]
9495

9596
assert expected == debian.extract_vulnerabilities(test_data)
97+
98+
99+
def test_archlinux_extract_vulnerabilities():
100+
archlinux_test_file = join(dirname(__file__), 'test_data', 'archlinux.json')
101+
102+
with open(archlinux_test_file) as f:
103+
test_data = json.loads(f.read())
104+
105+
expected = [
106+
{
107+
'package_name': 'libarchive',
108+
'vulnerability_id': 'AVG-837',
109+
'description': 'multiple issues',
110+
'status': 'Vulnerable',
111+
'severity': 'High',
112+
'affected_version': '3.3.3-1',
113+
'fixed_version': '3.3.3-2'
114+
},
115+
{
116+
'package_name': 'libtiff',
117+
'vulnerability_id': 'AVG-886',
118+
'description': 'multiple issues',
119+
'status': 'Vulnerable',
120+
'severity': 'Medium',
121+
'affected_version': '4.0.10-1',
122+
'fixed_version': '4.0.10-2'
123+
},
124+
{
125+
'package_name': 'ipsec-tools',
126+
'vulnerability_id': 'AVG-340',
127+
'description': 'denial of service',
128+
'status': 'Vulnerable',
129+
'severity': 'Medium',
130+
'affected_version': '0.8.2-8',
131+
'fixed_version': '0.8.3-1'
132+
},
133+
]
134+
135+
assert expected == archlinux.extract_vulnerabilities(test_data)

0 commit comments

Comments
 (0)