Merge pull request #60 from haikoschol/import-considers-status

Debian import considers status

Author: pombredanne

.travis.yml

@@ -11,7 +11,7 @@ env:
   - SECRET_KEY="i1bn=oly)w*2yl-5yc&f!vvgt)p)fh3_2$r#spa!*sw36f5ov7"
 
 before_script:
-  - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
+  - pycodestyle --exclude=migrations,settings.py,lib --max-line-length=100 .
   - psql -c "CREATE DATABASE vulnerablecode;" -U postgres
   - ./manage.py migrate
 

README.md

@@ -53,7 +53,7 @@ the code Django includes for this purpose: `SECRET_KEY=$(python -c "from django.
 ## Tests
 
 ```
-pycodestyle --exclude=migrations,settings.py,venv,tests --max-line-length=100 .
+pycodestyle --exclude=migrations,settings.py,venv --max-line-length=100 .
 python -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py
 ```
 

vulnerabilities/data_dump.py

@@ -41,15 +41,36 @@ def debian_dump(extract_data):
             vulnerability=vulnerability,
             reference_id=data.get('vulnerability_id', ''),
         )
+
+        pkg_name = data.get('package_name', '')
         package = Package.objects.create(
-            name=data.get('package_name', ''),
-            version=data.get('fixed_version', ''),
-        )
-        ImpactedPackage.objects.create(
-            vulnerability=vulnerability,
-            package=package
+            name=pkg_name,
+            version=data.get('version', ''),
         )
 
+        if data['status'] == 'open':
+            ImpactedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=package
+            )
+        else:
+            ResolvedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=package
+            )
+
+            fixed_version = data.get('fixed_version')
+            if fixed_version:
+                package = Package.objects.create(
+                    name=pkg_name,
+                    version=fixed_version,
+                )
+
+                ResolvedPackage.objects.create(
+                    vulnerability=vulnerability,
+                    package=package
+                )
+
 
 def ubuntu_dump(html):
     """

vulnerabilities/scraper/debian.py

@@ -48,12 +48,20 @@ def extract_vulnerabilities(debian_data, base_release='jessie'):
             if not release:
                 continue
 
+            status = release.get('status')
+            if status not in ('open', 'resolved'):
+                continue
+
+            # the latest version of this package in base_release
+            version = release.get('repositories', {}).get(base_release, '')
+
             package_vulnerabilities.append({
                 'package_name': package_name,
                 'vulnerability_id': vulnerability,
                 'description': details.get('description', ''),
-                'status': release.get('status', ''),
+                'status': status,
                 'urgency': release.get('urgency', ''),
+                'version': version,
                 'fixed_version': release.get('fixed_version', '')
             })
 

vulnerabilities/tests/test_api.py

@@ -41,43 +41,20 @@
 class TestResponse(TestCase):
     def test_debian_response(self):
         with open(os.path.join(TEST_DATA, 'debian.json')) as f:
-            test_data = json.loads(f.read())
+            test_data = json.load(f)
 
         extract_data = debian.extract_vulnerabilities(test_data)
         debian_dump(extract_data)
-        response = self.client.get('/api/packages/?name=mimetex', format='json')
+        response = self.client.get('/api/packages/?name=mimetex', format='json').data
 
-        expected = [{
-            "name": "mimetex",
-            "version": "1.50-1.1",
-            "platform": "",
-            "vulnerabilities": [{
-                "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX",
-                "cvss": None,
-                "references": [{
-                    "reference_id": "CVE-2009-2458",
-                    "source": "",
-                    "url": "",
-                }]
-            }],
-            "references": [],
-        }, {
-            "name": "mimetex",
-            "version": "1.50-1.1",
-            "platform": "",
-            "vulnerabilities": [{
-                "summary": "Multiple unspecified vulnerabilities in mimeTeX.",
-                "cvss": None,
-                "references": [{
-                    "reference_id": "CVE-2009-2459",
-                    "source": "",
-                    "url": "",
-                }]
-            }],
-            "references": [],
-        }]
+        self.assertEqual(4, response['count'])
 
-        self.assertEqual(expected, response.data.get('results'))
+        first_result = response['results'][0]
+        self.assertEqual('mimetex', first_result['name'])
+
+        versions = {r['version'] for r in response['results']}
+        self.assertIn('1.50-1.1', versions)
+        self.assertIn('1.74-1', versions)
 
     def test_ubuntu_response(self):
         with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f:
@@ -109,44 +86,18 @@ def test_ubuntu_response(self):
 class TestSerializers(TestCase):
     def test_serializers(self):
         with open(os.path.join(TEST_DATA, 'debian.json')) as f:
-            test_data = json.loads(f.read())
+            test_data = json.load(f)
         extract_data = debian.extract_vulnerabilities(test_data)
         debian_dump(extract_data)
 
         pk = Package.objects.filter(name="mimetex")
         response = PackageSerializer(pk, many=True).data
 
-        expected = [
-            {
-                "name": "mimetex",
-                "version": "1.50-1.1",
-                "platform": "",
-                "vulnerabilities": [{
-                    "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX",
-                    "cvss": None,
-                    "references": [{
-                        "reference_id": "CVE-2009-2458",
-                        "source": "",
-                        "url": "",
-                    }]
-                }],
-                "references": [],
-            },
-            {
-                "name": "mimetex",
-                "version": "1.50-1.1",
-                "platform": "",
-                "vulnerabilities": [{
-                    "summary": "Multiple unspecified vulnerabilities in mimeTeX.",
-                    "cvss": None,
-                    "references": [{
-                        "reference_id": "CVE-2009-2459",
-                        "source": "",
-                        "url": "",
-                    }]
-                }],
-                "references": [],
-            }
-        ]
-
-        self.assertEqual(expected, response)
+        self.assertEqual(4, len(response))
+
+        first_result = response[0]
+        self.assertEqual('mimetex', first_result['name'])
+
+        versions = {r['version'] for r in response}
+        self.assertIn('1.50-1.1', versions)
+        self.assertIn('1.74-1', versions)