Scrapes data from scrapers and dump in the database

app/app/settings.py

@@ -31,12 +31,14 @@
 # Application definition
 
 INSTALLED_APPS = [
+    'vulncode_app.apps.VulncodeAppConfig',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+
 ]
 
 MIDDLEWARE = [

app/vulncode_app/data_dump.py

@@ -0,0 +1,75 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from vulncode_app.models import Vulnerability
+from vulncode_app.models import VulnerabilityReference
+from vulncode_app.models import ImpactedPackage
+from vulncode_app.models import ResolvedPackage
+from vulncode_app.models import Package
+from vulncode_app.models import PackageReference
+
+from scraper import debian, ubuntu
+
+
+def debian_data():
+    """
+    Scrape debian' security tracker.
+    """
+    json_data = debian.json_data()
+    extract_data = debian.extract_data(json_data)
+
+    return extract_data
+
+
+def debian_dump(extract_data):
+    """
+    Save data scraped from Debian' security tracker.
+    """
+    for data in extract_data:
+        vulnerability = Vulnerability(summary=data.get('description'))
+        vulnerability_reference = VulnerabilityReference(reference_id=data.get('vulnerability_id'))
+        package = Package(name=data.get('package_name'), version=data.get('fixed_version'))
+
+        vulnerability.save()
+        vulnerability_reference.save()
+        package.save()
+
+
+def ubuntu_data():
+    """
+    Scrape Ubuntu' main security tracker.
+    """
+    data = ubuntu.scrape_cves()
+    return data
+
+
+def ubuntu_dump():
+    """
+    Dump data scraped from Ubuntu's security tracker.
+    """
+    for data in extract_data:
+        vulnerability_reference = VulnerabilityReference(reference_id=data.get('cve_id'))
+        package = ImpactedPackage(name=data.get('package_name'))
+
+        vulnerability_reference.save()
+        package.save()

app/vulncode_app/test_data_dump.py

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from django.test import TestCase
+
+from vulncode_app.models import Vulnerability
+from vulncode_app.models import VulnerabilityReference
+from vulncode_app.models import Package
+from vulncode_app.data_dump import debian_dump
+
+import json
+
+from scraper import debian
+
+
+class TestDataDump(TestCase):
+    def test_data_dump(self):
+        """
+        Scrape data from Debian' main tracker, dump it
+        in the database and verify entries.
+        """
+        with open("tests/test_data/debian.json") as f:
+            test_data = json.loads(f.read())
+
+        extract_data = debian.extract_data(test_data)
+        data_dump = debian_dump(extract_data)
+
+        for i in range(3):
+            self.assertEqual(3, len(Vulnerability.objects.all()))
+            self.assertEqual(3, len(VulnerabilityReference.objects.all()))
+            self.assertEqual(3, len(Package.objects.all()))
+            self.assertEqual(extract_data[i].get('description'),
+                             Vulnerability.objects.get(pk=i+1).summary)
+            self.assertEqual(extract_data[i].get('vulnerability_id'),
+                             VulnerabilityReference.objects.get(pk=i+1).reference_id)
+            self.assertEqual(extract_data[i].get('package_name'),
+                             Package.objects.get(pk=i+1).name)
+            self.assertEqual(extract_data[i].get('fixed_version'),
+                             Package.objects.get(pk=i+1).version)

app/vulncode_app/test_models.py

@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from django.test import TestCase
+
+from vulncode_app.models import Vulnerability
+from vulncode_app.models import VulnerabilityReference
+from vulncode_app.models import ImpactedPackage
+from vulncode_app.models import ResolvedPackage
+from vulncode_app.models import Package
+from vulncode_app.models import PackageReference
+
+
+class TestVulnerability(TestCase):
+    def test_vulnerability(self):
+        summary_create = Vulnerability.objects.create(summary="Affected package xyz")
+        summary_get = Vulnerability.objects.get(pk=summary_create.pk)
+
+        self.assertEqual(str(summary_create), "Affected package xyz")
+
+
+class TestVulnerabilityReference(TestCase):
+    def test_vulnerability_reference(self):
+        ref_id_create = VulnerabilityReference.objects.create(reference_id="CVE-2017-8564")
+        ref_id_get = VulnerabilityReference.objects.get(pk=ref_id_create.pk)
+
+        self.assertEqual(str(ref_id_create), "CVE-2017-8564")
+
+
+class TestPackage(TestCase):
+    def test_package(self):
+        package_name_create = Package.objects.create(name="Firefox")
+        package_name_get = Package.objects.get(pk=package_name_create.pk)
+
+        self.assertEqual(str(package_name_create), "Firefox")
+
+
+class TestPackageReference(TestCase):
+    def test_package_reference(self):
+        platform_create = PackageReference.objects.create(platform="Maven")
+        platform_get = PackageReference.objects.get(pk=platform_create.pk)
+
+        self.assertEqual(str(platform_create), "Maven")

data_dump.py

@@ -0,0 +1,62 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from vulncode_app.models import Vulnerability
+from vulncode_app.models import VulnerabilityReference
+from vulncode_app.models import ImpactedPackage
+from vulncode_app.models import ResolvedPackage
+from vulncode_app.models import Package
+from vulncode_app.models import PackageReference
+
+from scraper import debian
+from scraper import ubuntu
+
+
+def debian_dump():
+
+    json_data = debian.json_data()
+    extracted_data = debian.extract_data(json_data)
+
+    for i, v in enumerate(extract_data):
+        vulnerability = Vulnerability(summary=extract_data[i].get('description'))
+        vulnerability_reference = VulnerabilityReference(
+                                              reference_id=extract_data[i].get('vulnerability_id'))
+        package = ImpactedPackage(name=extract_data[i].get('package'),
+                                  version=extract_data[i].get('fixed_version'))
+
+    vulnerability.save()
+    vulnerability_reference.save()
+    package.save()
+
+
+def ubuntu_dump():
+
+    data = ubuntu.scrape_cves()
+
+    for i, v in enumerate(extract_data):
+        vulnerability_reference = VulnerabilityReference(
+                                              reference_id=data[i].get('cve_id'))
+        package = ImpactedPackage(name=data[i].get('package_name'))
+
+    vulnerability_reference.save()
+    package.save()

scraper/debian.py

@@ -24,7 +24,6 @@
 import json
 from urllib.request import urlopen
 
-
 DEBIAN_TRACKER_URL = 'https://security-tracker.debian.org/tracker/data/json'
 
 
@@ -60,6 +59,7 @@ def extract_data(debian_data, base_release='jessie'):
             package_vulns.append({
                 'package_name': package_name,
                 'vulnerability_id': vulnerability,
+                'description': details.get('description'),
                 'status': release.get('status'),
                 'urgency': release.get('urgency'),
                 'fixed_version': release.get('fixed_version')

tests/test_data/debian.json

@@ -3,7 +3,7 @@
 "CVE-2009-2458": {
 	"scope": "remote", 
 	"debianbug": 537254, 
-	"description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX, when downloaded before 20090713, allow remote attackers to execute arbitrary code via a TeX file with long (1) picture, (2) circle, or (3) input tags.", 
+	"description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", 
 	"releases": 
 		{"stretch": 
 			{"status": "resolved", 
@@ -34,7 +34,7 @@
 "CVE-2009-2459": 
 	{"scope": "un-remote", 
 	"debianbug": 537254, 
-	"description": "Multiple unspecified vulnerabilities in mimeTeX, when downloaded before 20090713, have unknown impact and attack vectors related to the (1) \\environ, (2) \\input, and (3) \\counter TeX directives.", 
+	"description": "Multiple unspecified vulnerabilities in mimeTeX.", 
 	"releases": 
 	{"stretch": 
 		{"status": "resolved", 

tests/test_scrapers.py

@@ -71,21 +71,24 @@ def test_debian_extract_data():
             'package_name': 'mimetex',
             'status': 'resolved',
             'urgency': 'medium',
-            'vulnerability_id': 'CVE-2009-2458'
+            'vulnerability_id': 'CVE-2009-2458',
+            'description': 'Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX'
         },
         {
             'fixed_version': '1.50-1.1',
             'package_name': 'mimetex',
             'status': 'not-resolved',
             'urgency': 'medium',
-            'vulnerability_id': 'CVE-2009-2459'
+            'vulnerability_id': 'CVE-2009-2459',
+            'description': 'Multiple unspecified vulnerabilities in mimeTeX.'
         },
         {
-            'fixed_version': None,
             'package_name': 'git-repair',
+            'vulnerability_id': 'TEMP-0807341-84E914',
+            'description': None,
             'status': 'open',
             'urgency': 'unimportant',
-            'vulnerability_id': 'TEMP-0807341-84E914'
+            'fixed_version': None
         }
     ]