Store Package URL data in the Package model

Signed-off-by: Haiko Schol <hs@haikoschol.com>

Author: haikoschol

requirements.txt

@@ -5,6 +5,7 @@ lxml==4.3.3
 django==2.2.4
 djangorestframework==3.9.2
 django-filter==2.1.0
+packageurl-python==0.8.7
 
 # Tests
 pytest==3.2.3

vulnerabilities/api.py

@@ -72,7 +72,6 @@ class Meta:
         fields = [
             'name',
             'version',
-            'platform',
             'vulnerabilities',
             'references',
         ]
@@ -81,4 +80,4 @@ class Meta:
 class PackageViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Package.objects.all()
     serializer_class = PackageSerializer
-    filter_fields = ('name', 'version', 'platform')
+    filter_fields = ('name', 'version')

vulnerabilities/data_dump.py

@@ -29,7 +29,7 @@
 from vulnerabilities.models import VulnerabilityReference
 
 
-def debian_dump(extract_data):
+def debian_dump(extract_data, base_release='jessie'):
     """
     Save data scraped from Debian' security tracker.
     """
@@ -45,7 +45,10 @@ def debian_dump(extract_data):
         pkg_name = data.get('package_name', '')
         package = Package.objects.create(
             name=pkg_name,
+            type='deb',
+            namespace='debian',
             version=data.get('version', ''),
+            qualifiers=f'distro={base_release}',
         )
 
         if data['status'] == 'open':
@@ -63,7 +66,10 @@ def debian_dump(extract_data):
             if fixed_version:
                 package = Package.objects.create(
                     name=pkg_name,
+                    type='deb',
+                    namespace='debian',
                     version=fixed_version,
+                    qualifiers=f'distro={base_release}',
                 )
 
                 ResolvedPackage.objects.create(
@@ -86,6 +92,8 @@ def ubuntu_dump(html):
         )
         package = Package.objects.create(
             name=data.get('package_name'),
+            type='deb',
+            namespace='ubuntu'
         )
         ImpactedPackage.objects.create(
             vulnerability=vulnerability,
@@ -119,12 +127,14 @@ def archlinux_dump(extract_data):
             VulnerabilityReference.objects.create(
                 vulnerability=vulnerability,
                 reference_id=vulnerability_id,
-                url='https://security.archlinux.org/{}'.format(vulnerability_id)
+                url=f'https://security.archlinux.org/{vulnerability_id}',
             )
 
         for package_name in packages_name:
             package_affected = Package.objects.create(
                 name=package_name,
+                type='pacman',
+                namespace='archlinux',
                 version=affected_version
             )
             ImpactedPackage.objects.create(
@@ -133,10 +143,12 @@ def archlinux_dump(extract_data):
             )
             PackageReference.objects.create(
                 package=package_affected,
-                repository='https://security.archlinux.org/package/{}'.format(package_name)
+                repository=f'https://security.archlinux.org/package/{package_name}',
             )
             package_fixed = Package.objects.create(
                 name=package_name,
+                type='pacman',
+                namespace='archlinux',
                 version=fixed_version
             )
             ResolvedPackage.objects.create(
@@ -145,5 +157,5 @@ def archlinux_dump(extract_data):
             )
             PackageReference.objects.create(
                 package=package_fixed,
-                repository='https://security.archlinux.org/package/{}'.format(package_name)
+                repository=f'https://security.archlinux.org/package/{package_name}',
             )

vulnerabilities/migrations/0006_auto_20190927_1438.py

@@ -0,0 +1,47 @@
+# Generated by Django 2.2.4 on 2019-09-27 14:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0005_auto_20190411_1644'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='package',
+            name='platform',
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='namespace',
+            field=models.CharField(blank=True, help_text='Package name prefix, such as Maven groupid, Docker image owner, GitHub user or organization, etc.', max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='qualifiers',
+            field=models.CharField(blank=True, help_text='Extra qualifying data for a package such as the name of an OS, architecture, distro, etc.', max_length=1024, null=True),
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='subpath',
+            field=models.CharField(blank=True, help_text='Extra subpath within a package, relative to the package root.', max_length=200, null=True),
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='type',
+            field=models.CharField(blank=True, help_text='A short code to identify the type of this package. For example: gem for a Rubygem, docker for a container, pypi for a Python Wheel or Egg, maven for a Maven Jar, deb for a Debian package, etc.', max_length=16, null=True),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='name',
+            field=models.CharField(blank=True, help_text='Name of the package.', max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='version',
+            field=models.CharField(blank=True, help_text='Version of the package.', max_length=50, null=True),
+        ),
+    ]

vulnerabilities/models.py

@@ -23,6 +23,8 @@
 
 from django.db import models
 
+from packageurl.contrib.django_models import PackageURLMixin
+
 
 class Vulnerability(models.Model):
     """
@@ -58,14 +60,11 @@ def __str__(self):
         return self.source
 
 
-class Package(models.Model):
+class Package(PackageURLMixin):
     """
     A software package with minimal identifying information.
     Other identifiers are stored as PackageReference.
     """
-    platform = models.CharField(max_length=50, help_text='Package platform eg:maven', blank=True)
-    name = models.CharField(max_length=50, help_text='Package name', blank=True)
-    version = models.CharField(max_length=50, help_text='Package version', blank=True)
     vulnerabilities = models.ManyToManyField(to='Vulnerability', through='ImpactedPackage')
 
     def __str__(self):

vulnerabilities/tests/test_api.py

@@ -64,23 +64,14 @@ def test_ubuntu_response(self):
         ubuntu_dump(extract_data)
         response = self.client.get('/api/packages/?name=automake', format='json')
 
-        expected = [{
-            "name": "automake",
-            "version": "",
-            "platform": "",
-            "vulnerabilities": [{
-                "summary": "",
-                "cvss": None,
-                "references": [{
-                    "reference_id": "CVE-2012-3386",
-                    "source": "",
-                    "url": "",
-                }]
-            }],
-            "references": [],
-        }]
-
-        self.assertEqual(expected, response.data.get('results'))
+        result = response.data.get('results')[0]
+        self.assertEqual('automake', result['name'])
+        self.assertEqual(None, result['version'])
+        self.assertEqual(1, len(result['vulnerabilities']))
+
+        vuln = result['vulnerabilities'][0]
+        self.assertEqual(1, len(vuln['references']))
+        self.assertEqual('CVE-2012-3386', vuln['references'][0]['reference_id'])
 
 
 class TestSerializers(TestCase):

vulnerabilities/tests/test_data_dump.py

@@ -88,7 +88,12 @@ def test_Package(self):
         self.assertEqual(5, Package.objects.count())
 
         self.assertTrue(Package.objects.filter(name='mimetex'))
-        self.assertTrue(Package.objects.get(name='librsync'))
+
+        pkg = Package.objects.get(name='librsync')
+        self.assertEqual('0.9.7-10', pkg.version)
+        self.assertEqual('deb', pkg.type)
+        self.assertEqual('debian', pkg.namespace)
+        self.assertIn('distro=jessie', pkg.qualifiers)
 
     def test_ImpactedPackage(self):
         """
@@ -130,7 +135,12 @@ def test_data_dump(self):
         """
         reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0]
         self.assertEqual(reference.reference_id, 'CVE-2002-2439')
-        self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6')
+        pkgs = Package.objects.filter(name='gcc-4.6')
+        self.assertTrue(pkgs)
+
+        pkg = pkgs[0]
+        self.assertEqual('deb', pkg.type)
+        self.assertEqual('ubuntu', pkg.namespace)
 
 
 class TestArchLinuxDataDump(TestCase):
@@ -163,7 +173,12 @@ def test_Package(self):
         Check that all packages from the test data are stored in the database
         """
         self.assertEqual(8, Package.objects.count())
-        self.assertTrue(Package.objects.filter(name='wireshark-cli'))
+        pkgs = Package.objects.filter(name='wireshark-cli')
+        self.assertTrue(pkgs)
+
+        for pkg in pkgs:
+            self.assertEqual('pacman', pkg.type)
+            self.assertEqual('archlinux', pkg.namespace)
 
     def test_PackageReference(self):
         """

vulnerabilities/tests/test_models.py

@@ -58,12 +58,10 @@ class TestPackage(TestCase):
     def test_package(self):
         Package.objects.create(
             name="Firefox",
-            platform="Maven",
             version="1.5.4"
         )
 
         self.assertTrue(Package.objects.get(name="Firefox"))
-        self.assertTrue(Package.objects.get(platform="Maven"))
         self.assertTrue(Package.objects.get(version="1.5.4"))