From bf07d2b6f6ec8b9d254ad1b411cee4edf8b6a034 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 2 Apr 2020 23:10:36 +0530 Subject: [PATCH 1/2] Add hash arrangements in models and importers Signed-off-by: Shivam Sandbhor --- requirements.txt | 3 ++- vulnerabilities/models.py | 13 ++++++++++--- vulnerabilities/scraper/npm.py | 28 +++++++++++++++++++++++++--- vulnerabilities/scraper/ruby.py | 30 +++++++++++++++++++++++++----- vulnerabilities/tests/test_npm.py | 2 +- 5 files changed, 63 insertions(+), 13 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1d079bd0d..7a4e5a484 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,4 +30,5 @@ tqdm==4.41.1 wcwidth==0.1.7 whitenoise==5.0.1 zipp==0.6.0 -pytoml==0.1.21 \ No newline at end of file +pytoml==0.1.21 +xxhash==1.4.3 \ No newline at end of file diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 47704e59f..74956cb63 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -31,8 +31,10 @@ class Vulnerability(models.Model): A software vulnerability with minimal information. Identifiers other than CVE ID are stored as VulnerabilityReference. """ - cve_id = models.CharField(max_length=50, help_text='CVE ID', unique=True, null=True) - summary = models.TextField(help_text='Summary of the vulnerability', blank=True) + cve_id = models.CharField( + max_length=50, help_text='CVE ID', unique=True, null=True) + summary = models.TextField( + help_text='Summary of the vulnerability', blank=True) cvss = models.FloatField(max_length=100, help_text='CVSS Score', null=True) def __str__(self): @@ -68,7 +70,8 @@ class Package(PackageURLMixin): A software package with minimal identifying information. Other identifiers are stored as PackageReference. """ - vulnerabilities = models.ManyToManyField(to='Vulnerability', through='ImpactedPackage') + vulnerabilities = models.ManyToManyField( + to='Vulnerability', through='ImpactedPackage') def __str__(self): return self.name @@ -123,3 +126,7 @@ class PackageReference(models.Model): def __str__(self): return self.platform + + +class AdvisoryHashes(models.Model): + hash = models.BigIntegerField(primary_key=True, unique=True) diff --git a/vulnerabilities/scraper/npm.py b/vulnerabilities/scraper/npm.py index 133132aec..d1602de1d 100644 --- a/vulnerabilities/scraper/npm.py +++ b/vulnerabilities/scraper/npm.py @@ -22,10 +22,13 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import json +import xxhash from dephell_specifier import RangeSpecifier from urllib.request import urlopen from urllib.error import HTTPError +from vulnerabilities.models import AdvisoryHashes + NPM_URL = 'https://registry.npmjs.org{}' PAGE = '/-/npm/v1/security/advisories?page=0' @@ -93,7 +96,7 @@ def extract_data(JSON): continue # NPM registry has no data regarding this package finally we skip these - package_vulnerabilities.append({ + package_vulnerability = { 'package_name': package_name, 'summary': obj.get('overview', ''), 'cve_ids': obj.get('cves', []), @@ -101,7 +104,18 @@ def extract_data(JSON): 'affected_versions': affected_versions, 'severity': obj.get('severity', ''), 'advisory': obj.get('url', ''), - }) + } + + pkg_vuln_hash = xxhash.xxh32(json.dumps( + package_vulnerability, sort_keys=True)).intdigest() + hash_query = AdvisoryHashes.objects.filter(hash=pkg_vuln_hash) + if hash_query: + # In the past we already had this same data, so much work + # for nothing + continue + package_vulnerabilities.append(package_vulnerability) + AdvisoryHashes.objects.create(hash=pkg_vuln_hash) + return package_vulnerabilities @@ -115,7 +129,15 @@ def scrape_vulnerabilities(): try: cururl = NPM_URL.format(nextpage) response = json.load(urlopen(cururl)) - package_vulnerabilities.extend(extract_data(response)) + resp_hash = xxhash.xxh32(json.dumps( + response, sort_keys=True)).intdigest() + hash_query = AdvisoryHashes.objects.filter(hash=resp_hash) + + if not hash_query: + + package_vulnerabilities.extend(extract_data(response)) + AdvisoryHashes.objects.create(hash=resp_hash) + nextpage = response.get('urls', {}).get('next') except HTTPError as error: diff --git a/vulnerabilities/scraper/ruby.py b/vulnerabilities/scraper/ruby.py index 367e8a4b5..dae5b069f 100644 --- a/vulnerabilities/scraper/ruby.py +++ b/vulnerabilities/scraper/ruby.py @@ -1,18 +1,29 @@ -import os +import saneyaml import urllib.request from urllib.error import HTTPError from zipfile import ZipFile from io import BytesIO -import saneyaml from dephell_specifier import RangeSpecifier from urllib.request import urlopen +from xxhash import xxh32 + +from vulnerabilities.models import AdvisoryHashes RUBYSEC_DB_URL = 'https://github.com/rubysec/ruby-advisory-db/archive/master.zip' def rubygem_advisories(url, prefix='ruby-advisory-db-master/gems/'): + hash_of_zip = xxh32() with urlopen(url) as response: - with ZipFile(BytesIO(response.read())) as zf: + with BytesIO(response.read()) as zfbytes: + chunk = zfbytes.read() + hash_of_zip.update(chunk) + hash_of_zip = hash_of_zip.intdigest() + hash_query = AdvisoryHashes.objects.filter(hash=hash_of_zip) + if hash_query: + return [] + AdvisoryHashes.objects.create(hash=hash_of_zip) + zf = ZipFile(zfbytes) for path in zf.namelist(): if path.startswith(prefix) and path.endswith('.yml'): yield saneyaml.load(zf.open(path)) @@ -67,11 +78,20 @@ def import_vulnerabilities(): break affected_versions = all_versions - unaffected_versions - vulnerability_package_dicts.append({ + vuln_pkg_dict = { 'package_name': package_name, 'cve_id': vulnerability_id, 'fixed_versions': unaffected_versions, 'affected_versions': affected_versions, 'advisory': advisory_url - }) + } + pkg_vuln_hash = xxhash.xxh32(json.dumps( + vuln_pkg_dict, sort_keys=True)).intdigest() + hash_query = AdvisoryHashes.objects.filter(hash=pkg_vuln_hash) + + if hash_query: + continue + + AdvisoryHashes.objects.create(hash=pkg_vuln_hash) + vulnerability_package_dicts.append(vuln_pkg_dict) return vulnerability_package_dicts diff --git a/vulnerabilities/tests/test_npm.py b/vulnerabilities/tests/test_npm.py index ddf69aff6..e11d58983 100644 --- a/vulnerabilities/tests/test_npm.py +++ b/vulnerabilities/tests/test_npm.py @@ -41,7 +41,7 @@ def test_get_all_versions(): @pytest.mark.webtest -def test_extract_data(): +def test_extract_data(db): with open(os.path.join(TEST_DATA, 'npm_test.json')) as f: test_data = json.load(f) From 29348eec042c15718d89dd9d78753ab956f4ef9a Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 3 Apr 2020 00:01:56 +0530 Subject: [PATCH 2/2] Add migration scripts Signed-off-by: Shivam Sandbhor --- .../migrations/0002_auto_20200402_1606.py | 23 +++++++++++++++++++ .../migrations/0003_auto_20200402_1611.py | 18 +++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 vulnerabilities/migrations/0002_auto_20200402_1606.py create mode 100644 vulnerabilities/migrations/0003_auto_20200402_1611.py diff --git a/vulnerabilities/migrations/0002_auto_20200402_1606.py b/vulnerabilities/migrations/0002_auto_20200402_1606.py new file mode 100644 index 000000000..8fbaac6b8 --- /dev/null +++ b/vulnerabilities/migrations/0002_auto_20200402_1606.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-04-02 16:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='AdvisoryHashes', + fields=[ + ('hash', models.IntegerField(primary_key=True, serialize=False, unique=True)), + ], + ), + migrations.AlterModelOptions( + name='vulnerability', + options={'verbose_name_plural': 'Vulnerabilities'}, + ), + ] diff --git a/vulnerabilities/migrations/0003_auto_20200402_1611.py b/vulnerabilities/migrations/0003_auto_20200402_1611.py new file mode 100644 index 000000000..82d08a3a3 --- /dev/null +++ b/vulnerabilities/migrations/0003_auto_20200402_1611.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-04-02 16:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0002_auto_20200402_1606'), + ] + + operations = [ + migrations.AlterField( + model_name='advisoryhashes', + name='hash', + field=models.BigIntegerField(primary_key=True, serialize=False, unique=True), + ), + ]