Consider "status" field in Debian data import

Currently only two status values are recognized:
"open" maps to ImpactedPackage
"resolved" maps to ResolvedPackage

Signed-off-by: Haiko Schol <hs@haikoschol.com>

Author: haikoschol

vulnerabilities/data_dump.py

@@ -41,15 +41,36 @@ def debian_dump(extract_data):
             vulnerability=vulnerability,
             reference_id=data.get('vulnerability_id', ''),
         )
+
+        pkg_name = data.get('package_name', '')
         package = Package.objects.create(
-            name=data.get('package_name', ''),
-            version=data.get('fixed_version', ''),
-        )
-        ImpactedPackage.objects.create(
-            vulnerability=vulnerability,
-            package=package
+            name=pkg_name,
+            version=data.get('version', ''),
         )
 
+        if data['status'] == 'open':
+            ImpactedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=package
+            )
+        else:
+            ResolvedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=package
+            )
+
+            fixed_version = data.get('fixed_version')
+            if fixed_version:
+                package = Package.objects.create(
+                    name=pkg_name,
+                    version=fixed_version,
+                )
+
+                ResolvedPackage.objects.create(
+                    vulnerability=vulnerability,
+                    package=package
+                )
+
 
 def ubuntu_dump(html):
     """

vulnerabilities/scraper/debian.py

@@ -48,14 +48,18 @@ def extract_vulnerabilities(debian_data, base_release='jessie'):
             if not release:
                 continue
 
+            status = release.get('status')
+            if status not in ('open', 'resolved'):
+                continue
+
             # the latest version of this package in base_release
             version = release.get('repositories', {}).get(base_release, '')
 
             package_vulnerabilities.append({
                 'package_name': package_name,
                 'vulnerability_id': vulnerability,
                 'description': details.get('description', ''),
-                'status': release.get('status', ''),
+                'status': status,
                 'urgency': release.get('urgency', ''),
                 'version': version,
                 'fixed_version': release.get('fixed_version', '')

vulnerabilities/tests/test_api.py

@@ -41,43 +41,20 @@
 class TestResponse(TestCase):
     def test_debian_response(self):
         with open(os.path.join(TEST_DATA, 'debian.json')) as f:
-            test_data = json.loads(f.read())
+            test_data = json.load(f)
 
         extract_data = debian.extract_vulnerabilities(test_data)
         debian_dump(extract_data)
-        response = self.client.get('/api/packages/?name=mimetex', format='json')
+        response = self.client.get('/api/packages/?name=mimetex', format='json').data
 
-        expected = [{
-            "name": "mimetex",
-            "version": "1.50-1.1",
-            "platform": "",
-            "vulnerabilities": [{
-                "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX",
-                "cvss": None,
-                "references": [{
-                    "reference_id": "CVE-2009-1382",
-                    "source": "",
-                    "url": "",
-                }]
-            }],
-            "references": [],
-        }, {
-            "name": "mimetex",
-            "version": "1.50-1.1",
-            "platform": "",
-            "vulnerabilities": [{
-                "summary": "Multiple unspecified vulnerabilities in mimeTeX",
-                "cvss": None,
-                "references": [{
-                    "reference_id": "CVE-2009-2459",
-                    "source": "",
-                    "url": "",
-                }]
-            }],
-            "references": [],
-        }]
+        self.assertEqual(4, response['count'])
 
-        self.assertEqual(expected, response.data.get('results'))
+        first_result = response['results'][0]
+        self.assertEqual('mimetex', first_result['name'])
+
+        versions = {r['version'] for r in response['results']}
+        self.assertIn('1.50-1.1', versions)
+        self.assertIn('1.74-1', versions)
 
     def test_ubuntu_response(self):
         with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f:
@@ -109,44 +86,18 @@ def test_ubuntu_response(self):
 class TestSerializers(TestCase):
     def test_serializers(self):
         with open(os.path.join(TEST_DATA, 'debian.json')) as f:
-            test_data = json.loads(f.read())
+            test_data = json.load(f)
         extract_data = debian.extract_vulnerabilities(test_data)
         debian_dump(extract_data)
 
         pk = Package.objects.filter(name="mimetex")
         response = PackageSerializer(pk, many=True).data
 
-        expected = [
-            {
-                "name": "mimetex",
-                "version": "1.50-1.1",
-                "platform": "",
-                "vulnerabilities": [{
-                    "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX",
-                    "cvss": None,
-                    "references": [{
-                        "reference_id": "CVE-2009-1382",
-                        "source": "",
-                        "url": "",
-                    }]
-                }],
-                "references": [],
-            },
-            {
-                "name": "mimetex",
-                "version": "1.50-1.1",
-                "platform": "",
-                "vulnerabilities": [{
-                    "summary": "Multiple unspecified vulnerabilities in mimeTeX",
-                    "cvss": None,
-                    "references": [{
-                        "reference_id": "CVE-2009-2459",
-                        "source": "",
-                        "url": "",
-                    }]
-                }],
-                "references": [],
-            }
-        ]
-
-        self.assertEqual(expected, response)
+        self.assertEqual(4, len(response))
+
+        first_result = response[0]
+        self.assertEqual('mimetex', first_result['name'])
+
+        versions = {r['version'] for r in response}
+        self.assertIn('1.50-1.1', versions)
+        self.assertIn('1.74-1', versions)

vulnerabilities/tests/test_data_dump.py

@@ -83,13 +83,37 @@ def test_Package(self):
         """
         Check that all packages from the test data are stored in the database
         """
-        # There are three rows in Package because currently the models allow duplicates
+        # There are five rows in Package because currently the models allow duplicates
         # (see issue #28).
-        self.assertEqual(3, Package.objects.count())
+        self.assertEqual(5, Package.objects.count())
 
         self.assertTrue(Package.objects.filter(name='mimetex'))
         self.assertTrue(Package.objects.get(name='librsync'))
 
+    def test_ImpactedPackage(self):
+        """
+        Check that all impacted packages from the test data are stored in the database
+        """
+        impacted_pkgs = ImpactedPackage.objects.all()
+        impacted_pkg = impacted_pkgs[0]
+
+        self.assertEqual(1, len(impacted_pkgs))
+        self.assertEqual('librsync', impacted_pkg.package.name)
+        self.assertEqual('0.9.7-10', impacted_pkg.package.version)
+
+    def test_ResolvedPackage(self):
+        """
+        Check that all resolved packages from the test data are stored in the database
+        """
+        resolved_pkgs = ResolvedPackage.objects.all()
+        resolved_pkg = resolved_pkgs[0]
+        versions = [rp.package.version for rp in resolved_pkgs]
+
+        self.assertEqual(4, len(resolved_pkgs))
+        self.assertEqual('mimetex', resolved_pkg.package.name)
+        self.assertIn('1.50-1.1', versions)
+        self.assertIn('1.74-1', versions)
+
 
 class TestUbuntuDataDump(TestCase):
     @classmethod