Skip to content

Commit 7343bde

Browse files
pombredannepombredanne
authored
Merge pull request #33 from lohani2280/test_branch_1
Collect vulnerabilities from archlinux Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
2 parents 5895b36 + 43a35ce commit 7343bde

7 files changed

Lines changed: 187 additions & 6 deletions

File tree

‎README.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ python manage.py shell
6262
```
6363

6464
```
65-
from vulnerabilities.scraper import debian, ubuntu
66-
from vulnerabilities.data_dump import debian_dump, ubuntu_dump
65+
from vulnerabilities.scraper import archlinux, debian, ubuntu
66+
from vulnerabilities.data_dump import archlinux_dump, debian_dump, ubuntu_dump
6767
6868
# May be needed on macOS
6969
# import ssl; ssl._create_default_https_context = ssl._create_unverified_context
@@ -73,6 +73,9 @@ ubuntu_dump(ubuntu_cves)
7373
7474
debian_vulnerabilities = debian.scrape_vulnerabilities()
7575
debian_dump(debian_vulnerabilities)
76+
77+
archlinux_vulnerabilities = archlinux.scrape_vulnerabilities()
78+
archlinux_dump(archlinux_vulnerabilities)
7679
```
7780

7881
API

‎vulnerabilities/data_dump.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323

2424
from vulnerabilities.models import ImpactedPackage
2525
from vulnerabilities.models import Package
26+
from vulnerabilities.models import PackageReference
27+
from vulnerabilities.models import ResolvedPackage
2628
from vulnerabilities.models import Vulnerability
2729
from vulnerabilities.models import VulnerabilityReference
2830

@@ -68,3 +70,59 @@ def ubuntu_dump(html):
6870
vulnerability=vulnerability,
6971
package=package
7072
)
73+
74+
75+
def archlinux_dump(extract_data):
76+
"""
77+
Save data scraped from archlinux' security tracker.
78+
"""
79+
for item in extract_data:
80+
cves = item['issues']
81+
group = item['name']
82+
83+
advisories = set(item['advisories'])
84+
vulnerabilities = cves + list(advisories)
85+
vulnerabilities.append(group)
86+
packages_name = item['packages']
87+
88+
affected_version = item['affected']
89+
fixed_version = item['fixed']
90+
if not fixed_version:
91+
fixed_version = 'None'
92+
93+
vulnerability = Vulnerability.objects.create(
94+
summary=item['type'],
95+
)
96+
97+
for vulnerability_id in vulnerabilities:
98+
VulnerabilityReference.objects.create(
99+
vulnerability=vulnerability,
100+
reference_id=vulnerability_id,
101+
url='https://security.archlinux.org/{}'.format(vulnerability_id)
102+
)
103+
104+
for package_name in packages_name:
105+
package_affected = Package.objects.create(
106+
name=package_name,
107+
version=affected_version
108+
)
109+
ImpactedPackage.objects.create(
110+
vulnerability=vulnerability,
111+
package=package_affected
112+
)
113+
PackageReference.objects.create(
114+
package=package_affected,
115+
repository='https://security.archlinux.org/package/{}'.format(package_name)
116+
)
117+
package_fixed = Package.objects.create(
118+
name=package_name,
119+
version=fixed_version
120+
)
121+
ResolvedPackage.objects.create(
122+
vulnerability=vulnerability,
123+
package=package_fixed
124+
)
125+
PackageReference.objects.create(
126+
package=package_fixed,
127+
repository='https://security.archlinux.org/package/{}'.format(package_name)
128+
)

‎vulnerabilities/migrations/0005_auto_20190411_1644.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Generated by Django 2.2 on 2019-04-11 16:44
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('vulnerabilities', '0004_auto_20190407_1838'),
10+
]
11+
12+
operations = [
13+
migrations.AlterField(
14+
model_name='packagereference',
15+
name='repository',
16+
field=models.CharField(blank=True, help_text='Repository URL eg:http://central.maven.org', max_length=100),
17+
),
18+
]

‎vulnerabilities/models.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ class PackageReference(models.Model):
9999
"""
100100
package = models.ForeignKey(Package, on_delete=models.CASCADE)
101101
repository = models.CharField(
102-
max_length=50,
102+
max_length=100,
103103
help_text='Repository URL eg:http://central.maven.org',
104104
blank=True,
105105
)

‎vulnerabilities/scraper/archlinux.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#
2+
# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/vulnerablecode/
4+
# The VulnerableCode software is licensed under the Apache License version 2.0.
5+
# Data generated with VulnerableCode require an acknowledgment.
6+
#
7+
# You may not use this software except in compliance with the License.
8+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
9+
# Unless required by applicable law or agreed to in writing, software distributed
10+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
# specific language governing permissions and limitations under the License.
13+
#
14+
# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
15+
# derivative work, you must accompany this data with the following acknowledgment:
16+
#
17+
# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
18+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
19+
# VulnerableCode should be considered or used as legal advice. Consult an Attorney
20+
# for any legal advice.
21+
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
22+
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
23+
24+
import json
25+
from urllib.request import urlopen
26+
27+
ARCHLINUX_TRACKER_URL = 'https://security.archlinux.org/json'
28+
29+
30+
def scrape_vulnerabilities():
31+
"""
32+
Fetch and return data scraped from archlinux security tracker.
33+
"""
34+
json_content = urlopen(ARCHLINUX_TRACKER_URL).read()
35+
arch_data = json.loads(json_content)
36+
return arch_data

‎vulnerabilities/tests/test_data/archlinux.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
[
2+
{
3+
"name": "AVG-708",
4+
"packages": [
5+
"wireshark-common",
6+
"wireshark-gtk",
7+
"wireshark-cli",
8+
"wireshark-qt"
9+
],
10+
"status": "Fixed",
11+
"severity": "Critical",
12+
"type": "multiple issues",
13+
"affected": "2.6.0-1",
14+
"fixed": "2.6.1-1",
15+
"ticket": null,
16+
"issues": [
17+
"CVE-2018-11362",
18+
"CVE-2018-11361",
19+
"CVE-2018-11360",
20+
"CVE-2018-11359",
21+
"CVE-2018-11358",
22+
"CVE-2018-11357",
23+
"CVE-2018-11356",
24+
"CVE-2018-11355",
25+
"CVE-2018-11354"
26+
],
27+
"advisories": [
28+
"ASA-201805-25",
29+
"ASA-201805-24",
30+
"ASA-201805-22",
31+
"ASA-201805-23"
32+
]
33+
}
34+
]

‎vulnerabilities/tests/test_data_dump.py‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,16 @@
2626

2727
from django.test import TestCase
2828

29-
from vulnerabilities.models import Vulnerability
30-
from vulnerabilities.models import VulnerabilityReference
31-
from vulnerabilities.models import Package
29+
from vulnerabilities.data_dump import archlinux_dump
3230
from vulnerabilities.data_dump import debian_dump
3331
from vulnerabilities.data_dump import ubuntu_dump
32+
from vulnerabilities.models import ImpactedPackage
33+
from vulnerabilities.models import Package
34+
from vulnerabilities.models import PackageReference
35+
from vulnerabilities.models import ResolvedPackage
36+
from vulnerabilities.models import Vulnerability
37+
from vulnerabilities.models import VulnerabilityReference
38+
from vulnerabilities.scraper import archlinux
3439
from vulnerabilities.scraper import debian
3540
from vulnerabilities.scraper import ubuntu
3641

@@ -85,3 +90,30 @@ def test_ubuntu_data_dump(self):
8590
reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0]
8691
self.assertEqual(reference.reference_id, 'CVE-2002-2439')
8792
self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6')
93+
94+
def test_archlinux_data_dump(self):
95+
"""
96+
Scrape data from Archlinux' main tracker, save it
97+
in the database and verify entries.
98+
"""
99+
with open(os.path.join(TEST_DATA, 'archlinux.json')) as f:
100+
test_data = json.loads(f.read())
101+
102+
archlinux_dump(test_data)
103+
104+
self.assertEqual(1, Vulnerability.objects.count())
105+
self.assertEqual(14, VulnerabilityReference.objects.count())
106+
self.assertEqual(8, Package.objects.count())
107+
self.assertEqual(8, PackageReference.objects.count())
108+
self.assertEqual(4, ImpactedPackage.objects.count())
109+
self.assertEqual(4, ResolvedPackage.objects.count())
110+
111+
self.assertTrue(Vulnerability.objects.get(summary='multiple issues'))
112+
113+
self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2018-11360'))
114+
115+
self.assertTrue(VulnerabilityReference.objects.get(reference_id='ASA-201805-24'))
116+
117+
self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-708'))
118+
119+
self.assertEqual(Package.objects.filter(name='wireshark-cli')[0].name, 'wireshark-cli')

0 commit comments

Comments
 (0)