diff --git a/requirements.txt b/requirements.txt index 970beca98..306a7a1e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ lxml==4.3.3 django==2.2.4 djangorestframework==3.9.2 django-filter==2.1.0 +packageurl-python==0.8.7 # Tests pytest==3.2.3 diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index b828826b4..fd82bd0f2 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -22,7 +22,11 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. from rest_framework import serializers +from rest_framework import status from rest_framework import viewsets +from rest_framework.response import Response + +from packageurl import PackageURL from vulnerabilities.models import Package from vulnerabilities.models import PackageReference @@ -72,7 +76,7 @@ class Meta: fields = [ 'name', 'version', - 'platform', + 'package_url', 'vulnerabilities', 'references', ] @@ -81,4 +85,27 @@ class Meta: class PackageViewSet(viewsets.ReadOnlyModelViewSet): queryset = Package.objects.all() serializer_class = PackageSerializer - filter_fields = ('name', 'version', 'platform') + filter_fields = ('name', 'version') + + def filter_queryset(self, qs): + purl = self.request.query_params.get('package_url') + if not purl: + return super().filter_queryset(qs) + + try: + purl = PackageURL.from_string(purl) + except ValueError as ve: + raise serializers.ValidationError( + detail={'error': f'"{purl}" is not a valid Package URL: {ve}'}, + ) + + # Remove "qualifiers" here because it is stored as one string in the model. + # For example, a row in the database could have the "qualifiers" column + # stored as "foo=bar&spam=eggs". If an API request contains a PURL with + # "spam=eggs&foo=bar", the DB query would not include that row. + attrs = {k: v for k, v in purl.to_dict().items() if v and k != 'qualifiers'} + + # TODO + # Since we are filtering on all the Package URL fields except "qualifiers", + # we'll eventually need database indices on them. + return self.queryset.filter(**attrs) diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py index 01a1727f7..bdf164010 100644 --- a/vulnerabilities/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -29,7 +29,7 @@ from vulnerabilities.models import VulnerabilityReference -def debian_dump(extract_data): +def debian_dump(extract_data, base_release='jessie'): """ Save data scraped from Debian' security tracker. """ @@ -45,7 +45,10 @@ def debian_dump(extract_data): pkg_name = data.get('package_name', '') package = Package.objects.create( name=pkg_name, + type='deb', + namespace='debian', version=data.get('version', ''), + qualifiers=f'distro={base_release}', ) if data['status'] == 'open': @@ -63,7 +66,10 @@ def debian_dump(extract_data): if fixed_version: package = Package.objects.create( name=pkg_name, + type='deb', + namespace='debian', version=fixed_version, + qualifiers=f'distro={base_release}', ) ResolvedPackage.objects.create( @@ -86,6 +92,8 @@ def ubuntu_dump(html): ) package = Package.objects.create( name=data.get('package_name'), + type='deb', + namespace='ubuntu' ) ImpactedPackage.objects.create( vulnerability=vulnerability, @@ -119,12 +127,14 @@ def archlinux_dump(extract_data): VulnerabilityReference.objects.create( vulnerability=vulnerability, reference_id=vulnerability_id, - url='https://security.archlinux.org/{}'.format(vulnerability_id) + url=f'https://security.archlinux.org/{vulnerability_id}', ) for package_name in packages_name: package_affected = Package.objects.create( name=package_name, + type='pacman', + namespace='archlinux', version=affected_version ) ImpactedPackage.objects.create( @@ -133,10 +143,12 @@ def archlinux_dump(extract_data): ) PackageReference.objects.create( package=package_affected, - repository='https://security.archlinux.org/package/{}'.format(package_name) + repository=f'https://security.archlinux.org/package/{package_name}', ) package_fixed = Package.objects.create( name=package_name, + type='pacman', + namespace='archlinux', version=fixed_version ) ResolvedPackage.objects.create( @@ -145,5 +157,5 @@ def archlinux_dump(extract_data): ) PackageReference.objects.create( package=package_fixed, - repository='https://security.archlinux.org/package/{}'.format(package_name) + repository=f'https://security.archlinux.org/package/{package_name}', ) diff --git a/vulnerabilities/migrations/0006_auto_20190927_1438.py b/vulnerabilities/migrations/0006_auto_20190927_1438.py new file mode 100644 index 000000000..5d5d6b0a3 --- /dev/null +++ b/vulnerabilities/migrations/0006_auto_20190927_1438.py @@ -0,0 +1,47 @@ +# Generated by Django 2.2.4 on 2019-09-27 14:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0005_auto_20190411_1644'), + ] + + operations = [ + migrations.RemoveField( + model_name='package', + name='platform', + ), + migrations.AddField( + model_name='package', + name='namespace', + field=models.CharField(blank=True, help_text='Package name prefix, such as Maven groupid, Docker image owner, GitHub user or organization, etc.', max_length=255, null=True), + ), + migrations.AddField( + model_name='package', + name='qualifiers', + field=models.CharField(blank=True, help_text='Extra qualifying data for a package such as the name of an OS, architecture, distro, etc.', max_length=1024, null=True), + ), + migrations.AddField( + model_name='package', + name='subpath', + field=models.CharField(blank=True, help_text='Extra subpath within a package, relative to the package root.', max_length=200, null=True), + ), + migrations.AddField( + model_name='package', + name='type', + field=models.CharField(blank=True, help_text='A short code to identify the type of this package. For example: gem for a Rubygem, docker for a container, pypi for a Python Wheel or Egg, maven for a Maven Jar, deb for a Debian package, etc.', max_length=16, null=True), + ), + migrations.AlterField( + model_name='package', + name='name', + field=models.CharField(blank=True, help_text='Name of the package.', max_length=100, null=True), + ), + migrations.AlterField( + model_name='package', + name='version', + field=models.CharField(blank=True, help_text='Version of the package.', max_length=50, null=True), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 348a474b1..aaec56f9b 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -23,6 +23,8 @@ from django.db import models +from packageurl.contrib.django_models import PackageURLMixin + class Vulnerability(models.Model): """ @@ -58,14 +60,11 @@ def __str__(self): return self.source -class Package(models.Model): +class Package(PackageURLMixin): """ A software package with minimal identifying information. Other identifiers are stored as PackageReference. """ - platform = models.CharField(max_length=50, help_text='Package platform eg:maven', blank=True) - name = models.CharField(max_length=50, help_text='Package name', blank=True) - version = models.CharField(max_length=50, help_text='Package version', blank=True) vulnerabilities = models.ManyToManyField(to='Vulnerability', through='ImpactedPackage') def __str__(self): diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 4e3a76b9c..b6cc8b7de 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -26,10 +26,10 @@ from django.test import TestCase -from vulnerabilities.models import Package from vulnerabilities.api import PackageSerializer from vulnerabilities.data_dump import debian_dump from vulnerabilities.data_dump import ubuntu_dump +from vulnerabilities.models import Package from vulnerabilities.scraper import debian from vulnerabilities.scraper import ubuntu @@ -38,16 +38,26 @@ TEST_DATA = os.path.join(BASE_DIR, 'test_data/') -class TestResponse(TestCase): - def test_debian_response(self): +class TestDebianResponse(TestCase): + @classmethod + def setUpTestData(cls): with open(os.path.join(TEST_DATA, 'debian.json')) as f: test_data = json.load(f) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) + + Package.objects.create( + name='mimetex', + version='1.50-1.1', + type='deb', + namespace='ubuntu' + ) + + def test_query_by_name(self): response = self.client.get('/api/packages/?name=mimetex', format='json').data - self.assertEqual(4, response['count']) + self.assertEqual(5, response['count']) first_result = response['results'][0] self.assertEqual('mimetex', first_result['name']) @@ -56,6 +66,47 @@ def test_debian_response(self): self.assertIn('1.50-1.1', versions) self.assertIn('1.74-1', versions) + purls = {r['package_url'] for r in response['results']} + self.assertIn('pkg:deb/debian/mimetex@1.50-1.1?distro=jessie', purls) + self.assertIn('pkg:deb/debian/mimetex@1.74-1?distro=jessie', purls) + + def test_query_by_invalid_package_url(self): + url = '/api/packages/?package_url=invalid_purl' + response = self.client.get(url, format='json') + + self.assertEqual(400, response.status_code) + self.assertIn('error', response.data) + error = response.data['error'] + self.assertIn('invalid_purl', error) + + def test_query_by_package_url(self): + url = '/api/packages/?package_url=pkg:deb/debian/mimetex@1.50-1.1?distro=jessie' + response = self.client.get(url, format='json').data + + self.assertEqual(2, response['count']) + + first_result = response['results'][0] + self.assertEqual('mimetex', first_result['name']) + + versions = {r['version'] for r in response['results']} + self.assertIn('1.50-1.1', versions) + self.assertNotIn('1.74-1', versions) + + def test_query_by_package_url_without_namespace(self): + url = '/api/packages/?package_url=pkg:deb/mimetex@1.50-1.1' + response = self.client.get(url, format='json').data + + self.assertEqual(3, response['count']) + + first_result = response['results'][0] + self.assertEqual('mimetex', first_result['name']) + + purls = {r['package_url'] for r in response['results']} + self.assertIn('pkg:deb/debian/mimetex@1.50-1.1?distro=jessie', purls) + self.assertIn('pkg:deb/ubuntu/mimetex@1.50-1.1', purls) + + +class TestUbuntuResponse(TestCase): def test_ubuntu_response(self): with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f: test_data = f.read() @@ -64,32 +115,26 @@ def test_ubuntu_response(self): ubuntu_dump(extract_data) response = self.client.get('/api/packages/?name=automake', format='json') - expected = [{ - "name": "automake", - "version": "", - "platform": "", - "vulnerabilities": [{ - "summary": "", - "cvss": None, - "references": [{ - "reference_id": "CVE-2012-3386", - "source": "", - "url": "", - }] - }], - "references": [], - }] - - self.assertEqual(expected, response.data.get('results')) + result = response.data.get('results')[0] + self.assertEqual('automake', result['name']) + self.assertEqual(None, result['version']) + self.assertEqual(1, len(result['vulnerabilities'])) + + vuln = result['vulnerabilities'][0] + self.assertEqual(1, len(vuln['references'])) + self.assertEqual('CVE-2012-3386', vuln['references'][0]['reference_id']) class TestSerializers(TestCase): - def test_serializers(self): + @classmethod + def setUpTestData(cls): with open(os.path.join(TEST_DATA, 'debian.json')) as f: test_data = json.load(f) + extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) + def test_package_serializer(self): pk = Package.objects.filter(name="mimetex") response = PackageSerializer(pk, many=True).data @@ -101,3 +146,7 @@ def test_serializers(self): versions = {r['version'] for r in response} self.assertIn('1.50-1.1', versions) self.assertIn('1.74-1', versions) + + purls = {r['package_url'] for r in response} + self.assertIn('pkg:deb/debian/mimetex@1.50-1.1?distro=jessie', purls) + self.assertIn('pkg:deb/debian/mimetex@1.74-1?distro=jessie', purls) diff --git a/vulnerabilities/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py index 766ba65bc..b1734ccd2 100644 --- a/vulnerabilities/tests/test_data_dump.py +++ b/vulnerabilities/tests/test_data_dump.py @@ -45,9 +45,8 @@ class TestDebianDataDump(TestCase): - @classmethod - def setUpTestData(self): + def setUpTestData(cls): with open(os.path.join(TEST_DATA, 'debian.json')) as f: test_data = json.load(f) @@ -88,7 +87,12 @@ def test_Package(self): self.assertEqual(5, Package.objects.count()) self.assertTrue(Package.objects.filter(name='mimetex')) - self.assertTrue(Package.objects.get(name='librsync')) + + pkg = Package.objects.get(name='librsync') + self.assertEqual('0.9.7-10', pkg.version) + self.assertEqual('deb', pkg.type) + self.assertEqual('debian', pkg.namespace) + self.assertIn('distro=jessie', pkg.qualifiers) def test_ImpactedPackage(self): """ @@ -117,7 +121,7 @@ def test_ResolvedPackage(self): class TestUbuntuDataDump(TestCase): @classmethod - def setUpTestData(self): + def setUpTestData(cls): with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f: test_data = f.read() @@ -130,13 +134,17 @@ def test_data_dump(self): """ reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0] self.assertEqual(reference.reference_id, 'CVE-2002-2439') - self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6') + pkgs = Package.objects.filter(name='gcc-4.6') + self.assertTrue(pkgs) + pkg = pkgs[0] + self.assertEqual('deb', pkg.type) + self.assertEqual('ubuntu', pkg.namespace) -class TestArchLinuxDataDump(TestCase): +class TestArchLinuxDataDump(TestCase): @classmethod - def setUpTestData(self): + def setUpTestData(cls): with open(os.path.join(TEST_DATA, 'archlinux.json')) as f: test_data = json.load(f) @@ -163,7 +171,12 @@ def test_Package(self): Check that all packages from the test data are stored in the database """ self.assertEqual(8, Package.objects.count()) - self.assertTrue(Package.objects.filter(name='wireshark-cli')) + pkgs = Package.objects.filter(name='wireshark-cli') + self.assertTrue(pkgs) + + for pkg in pkgs: + self.assertEqual('pacman', pkg.type) + self.assertEqual('archlinux', pkg.namespace) def test_PackageReference(self): """ diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 69581c2a7..912186003 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -58,12 +58,10 @@ class TestPackage(TestCase): def test_package(self): Package.objects.create( name="Firefox", - platform="Maven", version="1.5.4" ) self.assertTrue(Package.objects.get(name="Firefox")) - self.assertTrue(Package.objects.get(platform="Maven")) self.assertTrue(Package.objects.get(version="1.5.4"))