Merge pull request #114 from haikoschol/packageurl

Use Package URL for packages

Signed-off-by: Haiko Schol <hs@haikoschol.com>

Author: haikoschol

requirements.txt

@@ -5,6 +5,7 @@ lxml==4.3.3
 django==2.2.4
 djangorestframework==3.9.2
 django-filter==2.1.0
+packageurl-python==0.8.7
 
 # Tests
 pytest==3.2.3

vulnerabilities/api.py

@@ -22,7 +22,11 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 from rest_framework import serializers
+from rest_framework import status
 from rest_framework import viewsets
+from rest_framework.response import Response
+
+from packageurl import PackageURL
 
 from vulnerabilities.models import Package
 from vulnerabilities.models import PackageReference
@@ -72,7 +76,7 @@ class Meta:
         fields = [
             'name',
             'version',
-            'platform',
+            'package_url',
             'vulnerabilities',
             'references',
         ]
@@ -81,4 +85,27 @@ class Meta:
 class PackageViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Package.objects.all()
     serializer_class = PackageSerializer
-    filter_fields = ('name', 'version', 'platform')
+    filter_fields = ('name', 'version')
+
+    def filter_queryset(self, qs):
+        purl = self.request.query_params.get('package_url')
+        if not purl:
+            return super().filter_queryset(qs)
+
+        try:
+            purl = PackageURL.from_string(purl)
+        except ValueError as ve:
+            raise serializers.ValidationError(
+                detail={'error': f'"{purl}" is not a valid Package URL: {ve}'},
+            )
+
+        # Remove "qualifiers" here because it is stored as one string in the model.
+        # For example, a row in the database could have the "qualifiers" column
+        # stored as "foo=bar&spam=eggs". If an API request contains a PURL with
+        # "spam=eggs&foo=bar", the DB query would not include that row.
+        attrs = {k: v for k, v in purl.to_dict().items() if v and k != 'qualifiers'}
+
+        # TODO
+        # Since we are filtering on all the Package URL fields except "qualifiers",
+        # we'll eventually need database indices on them.
+        return self.queryset.filter(**attrs)

vulnerabilities/data_dump.py

@@ -29,7 +29,7 @@
 from vulnerabilities.models import VulnerabilityReference
 
 
-def debian_dump(extract_data):
+def debian_dump(extract_data, base_release='jessie'):
     """
     Save data scraped from Debian' security tracker.
     """
@@ -45,7 +45,10 @@ def debian_dump(extract_data):
         pkg_name = data.get('package_name', '')
         package = Package.objects.create(
             name=pkg_name,
+            type='deb',
+            namespace='debian',
             version=data.get('version', ''),
+            qualifiers=f'distro={base_release}',
         )
 
         if data['status'] == 'open':
@@ -63,7 +66,10 @@ def debian_dump(extract_data):
             if fixed_version:
                 package = Package.objects.create(
                     name=pkg_name,
+                    type='deb',
+                    namespace='debian',
                     version=fixed_version,
+                    qualifiers=f'distro={base_release}',
                 )
 
                 ResolvedPackage.objects.create(
@@ -86,6 +92,8 @@ def ubuntu_dump(html):
         )
         package = Package.objects.create(
             name=data.get('package_name'),
+            type='deb',
+            namespace='ubuntu'
         )
         ImpactedPackage.objects.create(
             vulnerability=vulnerability,
@@ -119,12 +127,14 @@ def archlinux_dump(extract_data):
             VulnerabilityReference.objects.create(
                 vulnerability=vulnerability,
                 reference_id=vulnerability_id,
-                url='https://security.archlinux.org/{}'.format(vulnerability_id)
+                url=f'https://security.archlinux.org/{vulnerability_id}',
             )
 
         for package_name in packages_name:
             package_affected = Package.objects.create(
                 name=package_name,
+                type='pacman',
+                namespace='archlinux',
                 version=affected_version
             )
             ImpactedPackage.objects.create(
@@ -133,10 +143,12 @@ def archlinux_dump(extract_data):
             )
             PackageReference.objects.create(
                 package=package_affected,
-                repository='https://security.archlinux.org/package/{}'.format(package_name)
+                repository=f'https://security.archlinux.org/package/{package_name}',
             )
             package_fixed = Package.objects.create(
                 name=package_name,
+                type='pacman',
+                namespace='archlinux',
                 version=fixed_version
             )
             ResolvedPackage.objects.create(
@@ -145,5 +157,5 @@ def archlinux_dump(extract_data):
             )
             PackageReference.objects.create(
                 package=package_fixed,
-                repository='https://security.archlinux.org/package/{}'.format(package_name)
+                repository=f'https://security.archlinux.org/package/{package_name}',
             )

vulnerabilities/migrations/0006_auto_20190927_1438.py

@@ -0,0 +1,47 @@
+# Generated by Django 2.2.4 on 2019-09-27 14:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0005_auto_20190411_1644'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='package',
+            name='platform',
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='namespace',
+            field=models.CharField(blank=True, help_text='Package name prefix, such as Maven groupid, Docker image owner, GitHub user or organization, etc.', max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='qualifiers',
+            field=models.CharField(blank=True, help_text='Extra qualifying data for a package such as the name of an OS, architecture, distro, etc.', max_length=1024, null=True),
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='subpath',
+            field=models.CharField(blank=True, help_text='Extra subpath within a package, relative to the package root.', max_length=200, null=True),
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='type',
+            field=models.CharField(blank=True, help_text='A short code to identify the type of this package. For example: gem for a Rubygem, docker for a container, pypi for a Python Wheel or Egg, maven for a Maven Jar, deb for a Debian package, etc.', max_length=16, null=True),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='name',
+            field=models.CharField(blank=True, help_text='Name of the package.', max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='version',
+            field=models.CharField(blank=True, help_text='Version of the package.', max_length=50, null=True),
+        ),
+    ]

vulnerabilities/models.py

@@ -23,6 +23,8 @@
 
 from django.db import models
 
+from packageurl.contrib.django_models import PackageURLMixin
+
 
 class Vulnerability(models.Model):
     """
@@ -58,14 +60,11 @@ def __str__(self):
         return self.source
 
 
-class Package(models.Model):
+class Package(PackageURLMixin):
     """
     A software package with minimal identifying information.
     Other identifiers are stored as PackageReference.
     """
-    platform = models.CharField(max_length=50, help_text='Package platform eg:maven', blank=True)
-    name = models.CharField(max_length=50, help_text='Package name', blank=True)
-    version = models.CharField(max_length=50, help_text='Package version', blank=True)
     vulnerabilities = models.ManyToManyField(to='Vulnerability', through='ImpactedPackage')
 
     def __str__(self):

vulnerabilities/tests/test_api.py

@@ -26,10 +26,10 @@
 
 from django.test import TestCase
 
-from vulnerabilities.models import Package
 from vulnerabilities.api import PackageSerializer
 from vulnerabilities.data_dump import debian_dump
 from vulnerabilities.data_dump import ubuntu_dump
+from vulnerabilities.models import Package
 from vulnerabilities.scraper import debian
 from vulnerabilities.scraper import ubuntu
 
@@ -38,16 +38,26 @@
 TEST_DATA = os.path.join(BASE_DIR, 'test_data/')
 
 
-class TestResponse(TestCase):
-    def test_debian_response(self):
+class TestDebianResponse(TestCase):
+    @classmethod
+    def setUpTestData(cls):
         with open(os.path.join(TEST_DATA, 'debian.json')) as f:
             test_data = json.load(f)
 
         extract_data = debian.extract_vulnerabilities(test_data)
         debian_dump(extract_data)
+
+        Package.objects.create(
+            name='mimetex',
+            version='1.50-1.1',
+            type='deb',
+            namespace='ubuntu'
+        )
+
+    def test_query_by_name(self):
         response = self.client.get('/api/packages/?name=mimetex', format='json').data
 
-        self.assertEqual(4, response['count'])
+        self.assertEqual(5, response['count'])
 
         first_result = response['results'][0]
         self.assertEqual('mimetex', first_result['name'])
@@ -56,6 +66,47 @@ def test_debian_response(self):
         self.assertIn('1.50-1.1', versions)
         self.assertIn('1.74-1', versions)
 
+        purls = {r['package_url'] for r in response['results']}
+        self.assertIn('pkg:deb/debian/mimetex@1.50-1.1?distro=jessie', purls)
+        self.assertIn('pkg:deb/debian/mimetex@1.74-1?distro=jessie', purls)
+
+    def test_query_by_invalid_package_url(self):
+        url = '/api/packages/?package_url=invalid_purl'
+        response = self.client.get(url, format='json')
+
+        self.assertEqual(400, response.status_code)
+        self.assertIn('error', response.data)
+        error = response.data['error']
+        self.assertIn('invalid_purl', error)
+
+    def test_query_by_package_url(self):
+        url = '/api/packages/?package_url=pkg:deb/debian/mimetex@1.50-1.1?distro=jessie'
+        response = self.client.get(url, format='json').data
+
+        self.assertEqual(2, response['count'])
+
+        first_result = response['results'][0]
+        self.assertEqual('mimetex', first_result['name'])
+
+        versions = {r['version'] for r in response['results']}
+        self.assertIn('1.50-1.1', versions)
+        self.assertNotIn('1.74-1', versions)
+
+    def test_query_by_package_url_without_namespace(self):
+        url = '/api/packages/?package_url=pkg:deb/mimetex@1.50-1.1'
+        response = self.client.get(url, format='json').data
+
+        self.assertEqual(3, response['count'])
+
+        first_result = response['results'][0]
+        self.assertEqual('mimetex', first_result['name'])
+
+        purls = {r['package_url'] for r in response['results']}
+        self.assertIn('pkg:deb/debian/mimetex@1.50-1.1?distro=jessie', purls)
+        self.assertIn('pkg:deb/ubuntu/mimetex@1.50-1.1', purls)
+
+
+class TestUbuntuResponse(TestCase):
     def test_ubuntu_response(self):
         with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f:
             test_data = f.read()
@@ -64,32 +115,26 @@ def test_ubuntu_response(self):
         ubuntu_dump(extract_data)
         response = self.client.get('/api/packages/?name=automake', format='json')
 
-        expected = [{
-            "name": "automake",
-            "version": "",
-            "platform": "",
-            "vulnerabilities": [{
-                "summary": "",
-                "cvss": None,
-                "references": [{
-                    "reference_id": "CVE-2012-3386",
-                    "source": "",
-                    "url": "",
-                }]
-            }],
-            "references": [],
-        }]
-
-        self.assertEqual(expected, response.data.get('results'))
+        result = response.data.get('results')[0]
+        self.assertEqual('automake', result['name'])
+        self.assertEqual(None, result['version'])
+        self.assertEqual(1, len(result['vulnerabilities']))
+
+        vuln = result['vulnerabilities'][0]
+        self.assertEqual(1, len(vuln['references']))
+        self.assertEqual('CVE-2012-3386', vuln['references'][0]['reference_id'])
 
 
 class TestSerializers(TestCase):
-    def test_serializers(self):
+    @classmethod
+    def setUpTestData(cls):
         with open(os.path.join(TEST_DATA, 'debian.json')) as f:
             test_data = json.load(f)
+
         extract_data = debian.extract_vulnerabilities(test_data)
         debian_dump(extract_data)
 
+    def test_package_serializer(self):
         pk = Package.objects.filter(name="mimetex")
         response = PackageSerializer(pk, many=True).data
 
@@ -101,3 +146,7 @@ def test_serializers(self):
         versions = {r['version'] for r in response}
         self.assertIn('1.50-1.1', versions)
         self.assertIn('1.74-1', versions)
+
+        purls = {r['package_url'] for r in response}
+        self.assertIn('pkg:deb/debian/mimetex@1.50-1.1?distro=jessie', purls)
+        self.assertIn('pkg:deb/debian/mimetex@1.74-1?distro=jessie', purls)