Skip to content

Commit 9312d26

Browse files
haikoscholhaikoschol
committed
Add unique CVE ID field to Vulnerability
This change adds the field cve_id to the Vulnerability model and based on that, improves the data import for Arch Linux. The improvements made have been discussed in issue #20: - For each CVE in a given AVG, exactly one Vulnerability is stored - For each CVE, one VulnerabilityReference to its page on security.archlinux.org is stored - Each ASA mentioned in an AVG is stored as a VulnerabilityReference Since there is no production deployment of vulnerablecode yet, I took the opportunity of changing the models to remove all migrations and create a new one that creates the whole schema. Since the cve_id field on Vulnerability has a unique constraint set, I needed to make some changes to the import code that belong to issue #28. I kept them minimal however so #28 is still open and needs to be addressed later. closes #20 Signed-off-by: Haiko Schol <hs@haikoschol.com>
1 parent 58e3ff3 commit 9312d26

16 files changed

Lines changed: 236 additions & 347 deletions

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ python -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests
5959

6060
For Django based tests
6161
```
62-
DJANGO_DEV=1 python manage.py test vulnerabilities/tests
62+
DJANGO_DEV=1 python manage.py test vulnerabilities.tests
6363
```
6464

6565
## Data import

‎vulnerabilities/data_dump.py‎

Lines changed: 79 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -34,15 +34,11 @@ def debian_dump(extract_data, base_release='jessie'):
3434
Save data scraped from Debian' security tracker.
3535
"""
3636
for data in extract_data:
37-
vulnerability = Vulnerability.objects.create(
38-
summary=data.get('description', ''),
39-
)
40-
VulnerabilityReference.objects.create(
41-
vulnerability=vulnerability,
42-
reference_id=data.get('vulnerability_id', ''),
37+
vulnerability, _ = Vulnerability.objects.get_or_create(
38+
cve_id=data['cve_id'],
4339
)
4440

45-
pkg_name = data.get('package_name', '')
41+
pkg_name = data['package_name']
4642
package = Package.objects.create(
4743
name=pkg_name,
4844
type='deb',
@@ -83,15 +79,11 @@ def ubuntu_dump(html):
8379
Dump data scraped from Ubuntu's security tracker.
8480
"""
8581
for data in html:
86-
vulnerability = Vulnerability.objects.create(
87-
summary='',
88-
)
89-
VulnerabilityReference.objects.create(
90-
vulnerability=vulnerability,
91-
reference_id=data.get('cve_id'),
82+
vulnerability, _ = Vulnerability.objects.get_or_create(
83+
cve_id=data['cve_id'],
9284
)
9385
package = Package.objects.create(
94-
name=data.get('package_name'),
86+
name=data['package_name'],
9587
type='deb',
9688
namespace='ubuntu'
9789
)
@@ -105,90 +97,96 @@ def archlinux_dump(extract_data):
10597
"""
10698
Save data scraped from archlinux' security tracker.
10799
"""
108-
for item in extract_data:
109-
cves = item['issues']
110-
group = item['name']
111-
112-
advisories = set(item['advisories'])
113-
vulnerabilities = cves + list(advisories)
114-
vulnerabilities.append(group)
115-
packages_name = item['packages']
116-
117-
affected_version = item['affected']
118-
fixed_version = item['fixed']
119-
if not fixed_version:
120-
fixed_version = 'None'
121-
122-
vulnerability = Vulnerability.objects.create(
123-
summary=item['type'],
124-
)
100+
base_url = 'https://security.archlinux.org'
125101

126-
for vulnerability_id in vulnerabilities:
127-
VulnerabilityReference.objects.create(
128-
vulnerability=vulnerability,
129-
reference_id=vulnerability_id,
130-
url=f'https://security.archlinux.org/{vulnerability_id}',
131-
)
102+
for avg in extract_data:
103+
affected_packages = []
104+
fixed_packages = []
132105

133-
for package_name in packages_name:
134-
package_affected = Package.objects.create(
106+
for package_name in avg['packages']:
107+
ap, _ = Package.objects.get_or_create(
135108
name=package_name,
136109
type='pacman',
137110
namespace='archlinux',
138-
version=affected_version
139-
)
140-
ImpactedPackage.objects.create(
141-
vulnerability=vulnerability,
142-
package=package_affected
143-
)
144-
PackageReference.objects.create(
145-
package=package_affected,
146-
repository=f'https://security.archlinux.org/package/{package_name}',
111+
version=avg['affected'],
147112
)
148-
package_fixed = Package.objects.create(
113+
affected_packages.append(ap)
114+
115+
fp, _ = Package.objects.get_or_create(
149116
name=package_name,
150117
type='pacman',
151118
namespace='archlinux',
152-
version=fixed_version
119+
version=avg['fixed'],
153120
)
154-
ResolvedPackage.objects.create(
121+
fixed_packages.append(fp)
122+
123+
for cve_id in avg['issues']:
124+
vulnerability, _ = Vulnerability.objects.get_or_create(
125+
cve_id=cve_id,
126+
)
127+
VulnerabilityReference.objects.create(
155128
vulnerability=vulnerability,
156-
package=package_fixed
129+
url=f'{base_url}/{cve_id}',
157130
)
158-
PackageReference.objects.create(
159-
package=package_fixed,
160-
repository=f'https://security.archlinux.org/package/{package_name}',
131+
avg_name = avg['name']
132+
VulnerabilityReference.objects.create(
133+
vulnerability=vulnerability,
134+
reference_id=avg_name,
135+
url=f'{base_url}/{avg_name}',
161136
)
162137

138+
for asa in avg['advisories']:
139+
VulnerabilityReference.objects.create(
140+
vulnerability=vulnerability,
141+
reference_id=asa,
142+
url=f'{base_url}/{asa}',
143+
)
144+
145+
for ap in affected_packages:
146+
ImpactedPackage.objects.get_or_create(
147+
vulnerability=vulnerability,
148+
package=ap,
149+
)
150+
151+
for fp in fixed_packages:
152+
ResolvedPackage.objects.get_or_create(
153+
vulnerability=vulnerability,
154+
package=fp,
155+
)
156+
163157

164158
def npm_dump(extract_data):
165159
for data in extract_data:
166-
vulnerability = Vulnerability.objects.create(
167-
summary=data.get('summary'),
168-
)
169-
VulnerabilityReference.objects.create(
170-
vulnerability=vulnerability,
171-
reference_id=data.get('vulnerability_id'),
172-
)
160+
package_name = data['package_name']
161+
advisory = data['advisory']
173162

174-
affected_versions = data.get('affected_version', [])
175-
for version in affected_versions:
176-
package_affected = Package.objects.create(
177-
name=data.get('package_name'),
178-
version=version,
179-
)
180-
ImpactedPackage.objects.create(
181-
vulnerability=vulnerability,
182-
package=package_affected
163+
for cve_id in data['cve_ids']:
164+
vulnerability, _ = Vulnerability.objects.get_or_create(
165+
cve_id=cve_id,
183166
)
184167

185-
fixed_versions = data.get('fixed_version', [])
186-
for version in fixed_versions:
187-
package_fixed = Package.objects.create(
188-
name=data.get('package_name'),
189-
version=version
190-
)
191-
ResolvedPackage.objects.create(
192-
vulnerability=vulnerability,
193-
package=package_fixed
194-
)
168+
if advisory:
169+
VulnerabilityReference.objects.create(
170+
vulnerability=vulnerability,
171+
url=advisory,
172+
)
173+
174+
for version in data['affected_versions']:
175+
package_affected = Package.objects.create(
176+
name=package_name,
177+
version=version,
178+
)
179+
ImpactedPackage.objects.create(
180+
vulnerability=vulnerability,
181+
package=package_affected
182+
)
183+
184+
for version in data['fixed_versions']:
185+
package_fixed = Package.objects.create(
186+
name=package_name,
187+
version=version
188+
)
189+
ResolvedPackage.objects.create(
190+
vulnerability=vulnerability,
191+
package=package_fixed
192+
)

‎vulnerabilities/migrations/0001_initial.py‎

Lines changed: 37 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
# -*- coding: utf-8 -*-
2-
# Generated by Django 1.11.4 on 2017-09-20 22:34
3-
from __future__ import unicode_literals
1+
# Generated by Django 2.2.4 on 2019-11-04 18:01
42

53
from django.db import migrations, models
64
import django.db.models.deletion
@@ -24,51 +22,49 @@ class Migration(migrations.Migration):
2422
name='Package',
2523
fields=[
2624
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
27-
('platform', models.CharField(blank=True, help_text='Package platform eg:maven', max_length=50)),
28-
('name', models.CharField(blank=True, help_text='Package name', max_length=50)),
29-
('version', models.CharField(blank=True, help_text='Package version', max_length=50)),
25+
('type', models.CharField(blank=True, help_text='A short code to identify the type of this package. For example: gem for a Rubygem, docker for a container, pypi for a Python Wheel or Egg, maven for a Maven Jar, deb for a Debian package, etc.', max_length=16, null=True)),
26+
('namespace', models.CharField(blank=True, help_text='Package name prefix, such as Maven groupid, Docker image owner, GitHub user or organization, etc.', max_length=255, null=True)),
27+
('name', models.CharField(blank=True, help_text='Name of the package.', max_length=100, null=True)),
28+
('version', models.CharField(blank=True, help_text='Version of the package.', max_length=50, null=True)),
29+
('qualifiers', models.CharField(blank=True, help_text='Extra qualifying data for a package such as the name of an OS, architecture, distro, etc.', max_length=1024, null=True)),
30+
('subpath', models.CharField(blank=True, help_text='Extra subpath within a package, relative to the package root.', max_length=200, null=True)),
3031
],
32+
options={
33+
'abstract': False,
34+
},
3135
),
3236
migrations.CreateModel(
33-
name='PackageReference',
37+
name='Vulnerability',
3438
fields=[
3539
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
36-
('repository', models.CharField(blank=True, help_text='Repository URL eg:http://central.maven.org', max_length=50)),
37-
('platform', models.CharField(blank=True, help_text='Platform eg:maven', max_length=50)),
38-
('name', models.CharField(blank=True, help_text='Package reference name eg:org.apache.commons.io', max_length=50)),
39-
('version', models.CharField(blank=True, help_text='Reference version', max_length=50)),
40-
('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package')),
40+
('cve_id', models.CharField(help_text='CVE ID', max_length=50, null=True, unique=True)),
41+
('summary', models.TextField(blank=True, help_text='Summary of the vulnerability')),
42+
('cvss', models.FloatField(help_text='CVSS Score', max_length=100, null=True)),
4143
],
4244
),
4345
migrations.CreateModel(
4446
name='ResolvedPackage',
4547
fields=[
4648
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
4749
('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package')),
50+
('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability')),
4851
],
4952
),
5053
migrations.CreateModel(
51-
name='Vulnerability',
52-
fields=[
53-
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
54-
('summary', models.CharField(blank=True, help_text='Summary of the vulnerability', max_length=50)),
55-
('cvss', models.FloatField(help_text='CVSS Score', max_length=50, null=True)),
56-
],
57-
),
58-
migrations.CreateModel(
59-
name='VulnerabilityReference',
54+
name='PackageReference',
6055
fields=[
6156
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
62-
('source', models.CharField(blank=True, help_text='Source(s) name eg:NVD', max_length=50)),
63-
('reference_id', models.CharField(blank=True, help_text='Reference ID, eg:CVE-ID', max_length=50)),
64-
('url', models.URLField(blank=True, help_text='URL of Vulnerability data', max_length=1024)),
65-
('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability')),
57+
('repository', models.CharField(blank=True, help_text='Repository URL eg:http://central.maven.org', max_length=100)),
58+
('platform', models.CharField(blank=True, help_text='Platform eg:maven', max_length=50)),
59+
('name', models.CharField(blank=True, help_text='Package reference name eg:org.apache.commons.io', max_length=50)),
60+
('version', models.CharField(blank=True, help_text='Reference version', max_length=50)),
61+
('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package')),
6662
],
6763
),
6864
migrations.AddField(
69-
model_name='resolvedpackage',
70-
name='vulnerability',
71-
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability'),
65+
model_name='package',
66+
name='vulnerabilities',
67+
field=models.ManyToManyField(through='vulnerabilities.ImpactedPackage', to='vulnerabilities.Vulnerability'),
7268
),
7369
migrations.AddField(
7470
model_name='impactedpackage',
@@ -80,12 +76,21 @@ class Migration(migrations.Migration):
8076
name='vulnerability',
8177
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability'),
8278
),
83-
migrations.AlterUniqueTogether(
84-
name='vulnerabilityreference',
85-
unique_together=set([('vulnerability', 'source', 'reference_id', 'url')]),
79+
migrations.CreateModel(
80+
name='VulnerabilityReference',
81+
fields=[
82+
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
83+
('source', models.CharField(blank=True, help_text='Source(s) name eg:NVD', max_length=50)),
84+
('reference_id', models.CharField(blank=True, help_text='Reference ID, eg:DSA-4465-1', max_length=50)),
85+
('url', models.URLField(blank=True, help_text='URL of Vulnerability data', max_length=1024)),
86+
('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability')),
87+
],
88+
options={
89+
'unique_together': {('vulnerability', 'source', 'reference_id', 'url')},
90+
},
8691
),
8792
migrations.AlterUniqueTogether(
8893
name='impactedpackage',
89-
unique_together=set([('vulnerability', 'package')]),
94+
unique_together={('vulnerability', 'package')},
9095
),
9196
]

‎vulnerabilities/migrations/0002_package_vulnerabilities.py‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎vulnerabilities/migrations/0003_auto_20190406_0950.py‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎vulnerabilities/migrations/0004_auto_20190407_1838.py‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

0 commit comments

Comments
 (0)