From 0f124c1404724306608dec0b9cb7d40deb72a1a4 Mon Sep 17 00:00:00 2001 From: lohani2280 Date: Thu, 28 Feb 2019 07:22:26 +0530 Subject: [PATCH 1/3] Collect vulnerabilities from arch linux https://github.com/nexB/vulnerablecode/issues/20 Signed-off-by: lohani2280 --- vulnerabilities/data_dump.py | 58 +++++++++++++++++++ vulnerabilities/scraper/archlinux.py | 36 ++++++++++++ .../tests/test_data/archlinux.json | 34 +++++++++++ vulnerabilities/tests/test_data_dump.py | 38 +++++++++++- 4 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 vulnerabilities/scraper/archlinux.py create mode 100644 vulnerabilities/tests/test_data/archlinux.json diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py index 45d94515e..561c91bb2 100644 --- a/vulnerabilities/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -23,6 +23,8 @@ from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import Package +from vulnerabilities.models import PackageReference +from vulnerabilities.models import ResolvedPackage from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference @@ -68,3 +70,59 @@ def ubuntu_dump(html): vulnerability=vulnerability, package=package ) + + +def archlinux_dump(extract_data): + """ + Save data scraped from archlinux' security tracker. + """ + for item in extract_data: + cves = item['issues'] + group = item['name'] + + advisories = set(item['advisories']) + vulnerabilities = cves + list(advisories) + vulnerabilities.append(group) + packages_name = item['packages'] + + affected_version = item['affected'] + fixed_version = item['fixed'] + if not fixed_version: + fixed_version = 'None' + + vulnerability = Vulnerability.objects.create( + summary=item['type'], + ) + + for vulnerability_id in vulnerabilities: + VulnerabilityReference.objects.create( + vulnerability=vulnerability, + reference_id=vulnerability_id, + url='https://security.archlinux.org/{}'.format(vulnerability_id) + ) + + for package_name in packages_name: + package_affected = Package.objects.create( + name=package_name, + version=affected_version + ) + ImpactedPackage.objects.create( + vulnerability=vulnerability, + package=package_affected + ) + PackageReference.objects.create( + package=package_affected, + repository='https://security.archlinux.org/package/{}'.format(package_name) + ) + package_fixed = Package.objects.create( + name=package_name, + version=fixed_version + ) + ResolvedPackage.objects.create( + vulnerability=vulnerability, + package=package_fixed + ) + PackageReference.objects.create( + package=package_fixed, + repository='https://security.archlinux.org/package/{}'.format(package_name) + ) diff --git a/vulnerabilities/scraper/archlinux.py b/vulnerabilities/scraper/archlinux.py new file mode 100644 index 000000000..1b50ae3b7 --- /dev/null +++ b/vulnerabilities/scraper/archlinux.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json +from urllib.request import urlopen + +ARCHLINUX_TRACKER_URL = 'https://security.archlinux.org/json' + + +def scrape_vulnerabilities(): + """ + Fetch and return data scraped from archlinux security tracker. + """ + json_content = urlopen(ARCHLINUX_TRACKER_URL).read() + arch_data = json.loads(json_content) + return arch_data diff --git a/vulnerabilities/tests/test_data/archlinux.json b/vulnerabilities/tests/test_data/archlinux.json new file mode 100644 index 000000000..35df28955 --- /dev/null +++ b/vulnerabilities/tests/test_data/archlinux.json @@ -0,0 +1,34 @@ +[ + { + "name": "AVG-708", + "packages": [ + "wireshark-common", + "wireshark-gtk", + "wireshark-cli", + "wireshark-qt" + ], + "status": "Fixed", + "severity": "Critical", + "type": "multiple issues", + "affected": "2.6.0-1", + "fixed": "2.6.1-1", + "ticket": null, + "issues": [ + "CVE-2018-11362", + "CVE-2018-11361", + "CVE-2018-11360", + "CVE-2018-11359", + "CVE-2018-11358", + "CVE-2018-11357", + "CVE-2018-11356", + "CVE-2018-11355", + "CVE-2018-11354" + ], + "advisories": [ + "ASA-201805-25", + "ASA-201805-24", + "ASA-201805-22", + "ASA-201805-23" + ] + } +] diff --git a/vulnerabilities/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py index d796a50d3..d538ab88f 100644 --- a/vulnerabilities/tests/test_data_dump.py +++ b/vulnerabilities/tests/test_data_dump.py @@ -26,11 +26,16 @@ from django.test import TestCase -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import Package +from vulnerabilities.data_dump import archlinux_dump from vulnerabilities.data_dump import debian_dump from vulnerabilities.data_dump import ubuntu_dump +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import Package +from vulnerabilities.models import PackageReference +from vulnerabilities.models import ResolvedPackage +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.scraper import archlinux from vulnerabilities.scraper import debian from vulnerabilities.scraper import ubuntu @@ -85,3 +90,30 @@ def test_ubuntu_data_dump(self): reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0] self.assertEqual(reference.reference_id, 'CVE-2002-2439') self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6') + + def test_archlinux_data_dump(self): + """ + Scrape data from Archlinux' main tracker, save it + in the database and verify entries. + """ + with open(os.path.join(TEST_DATA, 'archlinux.json')) as f: + test_data = json.loads(f.read()) + + archlinux_dump(test_data) + + self.assertEqual(1, Vulnerability.objects.count()) + self.assertEqual(14, VulnerabilityReference.objects.count()) + self.assertEqual(8, Package.objects.count()) + self.assertEqual(8, PackageReference.objects.count()) + self.assertEqual(4, ImpactedPackage.objects.count()) + self.assertEqual(4, ResolvedPackage.objects.count()) + + self.assertTrue(Vulnerability.objects.get(summary='multiple issues')) + + self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2018-11360')) + + self.assertTrue(VulnerabilityReference.objects.get(reference_id='ASA-201805-24')) + + self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-708')) + + self.assertEqual(Package.objects.filter(name='wireshark-cli')[0].name, 'wireshark-cli') \ No newline at end of file From ca646245f990ee9d686f1b3d637cebfa52fcef4b Mon Sep 17 00:00:00 2001 From: lohani2280 Date: Wed, 10 Apr 2019 03:02:30 +0530 Subject: [PATCH 2/3] Fix tests Signed-off-by: lohani2280 --- .../migrations/0005_auto_20190411_1644.py | 18 ++++++++++++++++++ vulnerabilities/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 vulnerabilities/migrations/0005_auto_20190411_1644.py diff --git a/vulnerabilities/migrations/0005_auto_20190411_1644.py b/vulnerabilities/migrations/0005_auto_20190411_1644.py new file mode 100644 index 000000000..1fb58a022 --- /dev/null +++ b/vulnerabilities/migrations/0005_auto_20190411_1644.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2019-04-11 16:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0004_auto_20190407_1838'), + ] + + operations = [ + migrations.AlterField( + model_name='packagereference', + name='repository', + field=models.CharField(blank=True, help_text='Repository URL eg:http://central.maven.org', max_length=100), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 33395e696..348a474b1 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -99,7 +99,7 @@ class PackageReference(models.Model): """ package = models.ForeignKey(Package, on_delete=models.CASCADE) repository = models.CharField( - max_length=50, + max_length=100, help_text='Repository URL eg:http://central.maven.org', blank=True, ) From 43a35ce70f0e05154a4b6f6a5a4e38969b3fac08 Mon Sep 17 00:00:00 2001 From: lohani2280 Date: Thu, 11 Apr 2019 22:17:35 +0530 Subject: [PATCH 3/3] Update README on how to add archlinux data into the database. Signed-off-by: lohani2280 --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 51a5e6509..733fda04d 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,8 @@ python manage.py shell ``` ``` -from vulnerabilities.scraper import debian, ubuntu -from vulnerabilities.data_dump import debian_dump, ubuntu_dump +from vulnerabilities.scraper import archlinux, debian, ubuntu +from vulnerabilities.data_dump import archlinux_dump, debian_dump, ubuntu_dump # May be needed on macOS # import ssl; ssl._create_default_https_context = ssl._create_unverified_context @@ -73,6 +73,9 @@ ubuntu_dump(ubuntu_cves) debian_vulnerabilities = debian.scrape_vulnerabilities() debian_dump(debian_vulnerabilities) + +archlinux_vulnerabilities = archlinux.scrape_vulnerabilities() +archlinux_dump(archlinux_vulnerabilities) ``` API