Scrapes data from scrapers and dump in the database

.gitignore

@@ -103,3 +103,6 @@ ENV/
 
 # PyCharm
 .idea/
+
+# Database
+*.sqlite3*

.travis.yml

@@ -7,9 +7,12 @@ install:
 
 before_script:
   - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
+  - cd app/
+  - python3 manage.py migrate
 
 script:
   - python3.6 -m pytest -v tests/
+  - python3.6 manage.py test
 
 notifications:
   email: false

README.md

@@ -12,12 +12,13 @@ Clone the source code:
 git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode
 ```
 
-Activate a virtualenv and install dependencies:
+Activate a virtualenv, install dependencies, and run the database migrations:
 
 ```
 python3.6 -m venv .
 source bin/activate
 pip install -r requirements.txt
+app/manage.py migrate
 ```
 
 Tests
@@ -28,18 +29,28 @@ pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
 cd app/
 python3.6 -m pytest -v tests/
 ```
+
 For Django based tests
 ```
 cd app/
-python3 manage.py test
+./manage.py test
 ```
 
-Scrape
-------
+Scrape and save to the database
+-------------------------------
+
+```
+cd app/
+./manage.py shell
+```
 
 ```
 from scraper import debian, ubuntu
+from vulncode_app.data_dump import debian_dump, ubuntu_dump
+
+debian_vulnerabilities = debian.scrape_vulnerabilities()
+ubuntu_cves = ubuntu.scrape_cves()
 
-debian.scrape_cves()
-ubuntu.scrape_cves()
+debian_dump(debian_vulnerabilities)
+ubuntu_dump(ubuntu_cves)
 ```

app/app/settings.py

@@ -31,6 +31,7 @@
 # Application definition
 
 INSTALLED_APPS = [
+    'vulncode_app.apps.VulncodeAppConfig',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',

app/app/urls.py

@@ -13,10 +13,9 @@
     1. Import the include() function: from django.conf.urls import url, include
     2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
 """
+
 from django.conf.urls import url, include
-from django.contrib import admin
 
 urlpatterns = [
     url(r'^vulncode_app/', include('vulncode_app.urls')),
-    url(r'^admin/', admin.site.urls),
 ]

scraper/debian.py → app/scraper/debian.py

@@ -24,25 +24,16 @@
 import json
 from urllib.request import urlopen
 
-
 DEBIAN_TRACKER_URL = 'https://security-tracker.debian.org/tracker/data/json'
 
 
-def json_data(url=DEBIAN_TRACKER_URL):
-    """
-    Return Debian vulnerabilities data fetched from `url`.
-    """
-    debian_data = urlopen(url).read()
-    return json.loads(debian_data)
-
-
-def extract_data(debian_data, base_release='jessie'):
+def extract_vulnerabilities(debian_data, base_release='jessie'):
     """
     Return a sequence of mappings for each existing combination of
     package and vulnerability from a mapping of Debian vulnerabilities
     data.
     """
-    package_vulns = []
+    package_vulnerabilities = []
 
     for package_name, vulnerabilities in debian_data.items():
         if not vulnerabilities or not package_name:
@@ -57,11 +48,21 @@ def extract_data(debian_data, base_release='jessie'):
             if not release:
                 continue
 
-            package_vulns.append({
+            package_vulnerabilities.append({
                 'package_name': package_name,
                 'vulnerability_id': vulnerability,
-                'status': release.get('status'),
-                'urgency': release.get('urgency'),
-                'fixed_version': release.get('fixed_version')
+                'description': details.get('description', ''),
+                'status': release.get('status', ''),
+                'urgency': release.get('urgency', ''),
+                'fixed_version': release.get('fixed_version', '')
             })
-    return package_vulns
+
+    return package_vulnerabilities
+
+
+def scrape_vulnerabilities():
+    """
+    Scrape debian' security tracker.
+    """
+    json_content = urlopen(DEBIAN_TRACKER_URL).read()
+    return extract_vulnerabilities(json.loads(json_content))

tests/test_api_data.py → app/tests/test_api_data.py

@@ -23,7 +23,7 @@
 
 import json
 
-from api_data import extract_fields
+from vulncode_app.api_data import extract_fields
 
 
 test_data = """

tests/test_data/debian.json → app/tests/test_data/debian.json

@@ -3,7 +3,7 @@
 "CVE-2009-2458": {
 	"scope": "remote", 
 	"debianbug": 537254, 
-	"description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX, when downloaded before 20090713, allow remote attackers to execute arbitrary code via a TeX file with long (1) picture, (2) circle, or (3) input tags.", 
+	"description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", 
 	"releases": 
 		{"stretch": 
 			{"status": "resolved", 
@@ -34,7 +34,7 @@
 "CVE-2009-2459": 
 	{"scope": "un-remote", 
 	"debianbug": 537254, 
-	"description": "Multiple unspecified vulnerabilities in mimeTeX, when downloaded before 20090713, have unknown impact and attack vectors related to the (1) \\environ, (2) \\input, and (3) \\counter TeX directives.", 
+	"description": "Multiple unspecified vulnerabilities in mimeTeX.", 
 	"releases": 
 	{"stretch": 
 		{"status": "resolved", 

tests/test_scrapers.py → app/tests/test_scrapers.py

@@ -30,7 +30,7 @@
 
 
 def test_ubuntu_extract_cves():
-    ubuntu_testfile = join(dirname(__file__), 'ubuntu', 'main.html')
+    ubuntu_testfile = join(dirname(__file__), 'test_data', 'ubuntu_main.html')
 
     with open(ubuntu_testfile) as f:
         test_input = f.read()
@@ -59,7 +59,7 @@ def test_ubuntu_extract_cves():
     assert expected == cves[-1]
 
 
-def test_debian_extract_data():
+def test_debian_extract_vulnerabilities():
     debian_test_file = join(dirname(__file__), 'test_data', 'debian.json')
 
     with open(debian_test_file) as f:
@@ -71,22 +71,25 @@ def test_debian_extract_data():
             'package_name': 'mimetex',
             'status': 'resolved',
             'urgency': 'medium',
-            'vulnerability_id': 'CVE-2009-2458'
+            'vulnerability_id': 'CVE-2009-2458',
+            'description': 'Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX'
         },
         {
             'fixed_version': '1.50-1.1',
             'package_name': 'mimetex',
             'status': 'not-resolved',
             'urgency': 'medium',
-            'vulnerability_id': 'CVE-2009-2459'
+            'vulnerability_id': 'CVE-2009-2459',
+            'description': 'Multiple unspecified vulnerabilities in mimeTeX.'
         },
         {
-            'fixed_version': None,
             'package_name': 'git-repair',
+            'vulnerability_id': 'TEMP-0807341-84E914',
+            'description': '',
             'status': 'open',
             'urgency': 'unimportant',
-            'vulnerability_id': 'TEMP-0807341-84E914'
+            'fixed_version': ''
         }
     ]
 
-    assert expected == debian.extract_data(test_data)
+    assert expected == debian.extract_vulnerabilities(test_data)

app/vulncode_app/data_dump.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from vulncode_app.models import Vulnerability
+from vulncode_app.models import VulnerabilityReference
+from vulncode_app.models import Package
+
+
+def debian_dump(extract_data):
+    """
+    Save data scraped from Debian' security tracker.
+    """
+    for data in extract_data:
+        vulnerability = Vulnerability.objects.create(
+            summary=data.get('description', ''),
+        )
+        VulnerabilityReference.objects.create(
+            vulnerability=vulnerability,
+            reference_id=data.get('vulnerability_id', ''),
+        )
+        Package.objects.create(
+            name=data.get('package_name', ''),
+            version=data.get('fixed_version', ''),
+        )
+
+
+def ubuntu_dump(html):
+    """
+    Dump data scraped from Ubuntu's security tracker.
+    """
+    for data in html:
+        vulnerability = Vulnerability.objects.create(
+            summary='',
+        )
+        VulnerabilityReference.objects.create(
+            vulnerability=vulnerability,
+            reference_id=data.get('cve_id'),
+        )
+        Package.objects.create(
+            name=data.get('package_name'),
+        )

app/vulncode_app/migrations/0001_initial.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.4 on 2017-08-08 09:11
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ImpactedPackage',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Package',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('platform', models.CharField(blank=True, help_text='Package platform eg:maven', max_length=50)),
+                ('name', models.CharField(blank=True, help_text='Package name', max_length=50)),
+                ('version', models.CharField(blank=True, help_text='Package version', max_length=50)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='PackageReference',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('repository', models.CharField(blank=True, help_text='Repository URL eg:http://central.maven.org', max_length=50)),
+                ('platform', models.CharField(blank=True, help_text='Platform eg:maven', max_length=50)),
+                ('name', models.CharField(blank=True, help_text='Package reference name eg:org.apache.commons.io', max_length=50)),
+                ('version', models.CharField(blank=True, help_text='Reference version', max_length=50)),
+                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ResolvedPackage',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Vulnerability',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('summary', models.CharField(blank=True, help_text='Summary of the vulnerability', max_length=50)),
+                ('cvss', models.FloatField(help_text='CVSS Score', max_length=50, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='VulnerabilityReference',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('source', models.CharField(blank=True, help_text="Source's name eg:NVD", max_length=50)),
+                ('reference_id', models.CharField(blank=True, help_text='Reference ID, eg:CVE-ID', max_length=50)),
+                ('url', models.URLField(blank=True, help_text='URL of Vulnerability data', max_length=1024)),
+                ('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='resolvedpackage',
+            name='vulnerability',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability'),
+        ),
+        migrations.AddField(
+            model_name='impactedpackage',
+            name='package',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package'),
+        ),
+        migrations.AddField(
+            model_name='impactedpackage',
+            name='vulnerability',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='vulnerabilityreference',
+            unique_together=set([('vulnerability', 'source', 'reference_id')]),
+        ),
+    ]