Merge pull request #25 from nexB/data_dump

Scrapes data from scrapers and dump in the database

Author: tdruez

.gitignore

@@ -103,3 +103,6 @@ ENV/
 
 # PyCharm
 .idea/
+
+# Database
+*.sqlite3*

.travis.yml

@@ -7,9 +7,12 @@ install:
 
 before_script:
   - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
+  - cd app/
+  - python3 manage.py migrate
 
 script:
   - python3.6 -m pytest -v tests/
+  - python3.6 manage.py test
 
 notifications:
   email: false

README.md

@@ -12,12 +12,13 @@ Clone the source code:
 git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode
 ```
 
-Activate a virtualenv and install dependencies:
+Activate a virtualenv, install dependencies, and run the database migrations:
 
 ```
 python3.6 -m venv .
 source bin/activate
 pip install -r requirements.txt
+app/manage.py migrate
 ```
 
 Tests
@@ -28,18 +29,28 @@ pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
 cd app/
 python3.6 -m pytest -v tests/
 ```
+
 For Django based tests
 ```
 cd app/
-python3 manage.py test
+./manage.py test
 ```
 
-Scrape
-------
+Scrape and save to the database
+-------------------------------
+
+```
+cd app/
+./manage.py shell
+```
 
 ```
 from scraper import debian, ubuntu
+from vulncode_app.data_dump import debian_dump, ubuntu_dump
+
+debian_vulnerabilities = debian.scrape_vulnerabilities()
+ubuntu_cves = ubuntu.scrape_cves()
 
-debian.scrape_cves()
-ubuntu.scrape_cves()
+debian_dump(debian_vulnerabilities)
+ubuntu_dump(ubuntu_cves)
 ```

app/app/settings.py

@@ -31,6 +31,7 @@
 # Application definition
 
 INSTALLED_APPS = [
+    'vulncode_app.apps.VulncodeAppConfig',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',

app/app/urls.py

@@ -13,10 +13,9 @@
     1. Import the include() function: from django.conf.urls import url, include
     2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
 """
+
 from django.conf.urls import url, include
-from django.contrib import admin
 
 urlpatterns = [
     url(r'^vulncode_app/', include('vulncode_app.urls')),
-    url(r'^admin/', admin.site.urls),
 ]

scraper/debian.py app/scraper/debian.pyscraper/debian.py renamed to app/scraper/debian.py

@@ -24,25 +24,16 @@
 import json
 from urllib.request import urlopen
 
-
 DEBIAN_TRACKER_URL = 'https://security-tracker.debian.org/tracker/data/json'
 
 
-def json_data(url=DEBIAN_TRACKER_URL):
-    """
-    Return Debian vulnerabilities data fetched from `url`.
-    """
-    debian_data = urlopen(url).read()
-    return json.loads(debian_data)
-
-
-def extract_data(debian_data, base_release='jessie'):
+def extract_vulnerabilities(debian_data, base_release='jessie'):
     """
     Return a sequence of mappings for each existing combination of
     package and vulnerability from a mapping of Debian vulnerabilities
     data.
     """
-    package_vulns = []
+    package_vulnerabilities = []
 
     for package_name, vulnerabilities in debian_data.items():
         if not vulnerabilities or not package_name:
@@ -57,11 +48,21 @@ def extract_data(debian_data, base_release='jessie'):
             if not release:
                 continue
 
-            package_vulns.append({
+            package_vulnerabilities.append({
                 'package_name': package_name,
                 'vulnerability_id': vulnerability,
-                'status': release.get('status'),
-                'urgency': release.get('urgency'),
-                'fixed_version': release.get('fixed_version')
+                'description': details.get('description', ''),
+                'status': release.get('status', ''),
+                'urgency': release.get('urgency', ''),
+                'fixed_version': release.get('fixed_version', '')
             })
-    return package_vulns
+
+    return package_vulnerabilities
+
+
+def scrape_vulnerabilities():
+    """
+    Scrape debian' security tracker.
+    """
+    json_content = urlopen(DEBIAN_TRACKER_URL).read()
+    return extract_vulnerabilities(json.loads(json_content))

scraper/ubuntu.py app/scraper/ubuntu.pyscraper/ubuntu.py renamed to app/scraper/ubuntu.py

@@ -23,7 +23,7 @@
 
 import json
 
-from api_data import extract_fields
+from vulncode_app.api_data import extract_fields
 
 
 test_data = """

tests/test_api_data.py app/tests/test_api_data.pytests/test_api_data.py renamed to app/tests/test_api_data.py

@@ -3,7 +3,7 @@
 "CVE-2009-2458": {
 	"scope": "remote", 
 	"debianbug": 537254, 
-	"description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX, when downloaded before 20090713, allow remote attackers to execute arbitrary code via a TeX file with long (1) picture, (2) circle, or (3) input tags.", 
+	"description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", 
 	"releases": 
 		{"stretch": 
 			{"status": "resolved", 
@@ -34,7 +34,7 @@
 "CVE-2009-2459": 
 	{"scope": "un-remote", 
 	"debianbug": 537254, 
-	"description": "Multiple unspecified vulnerabilities in mimeTeX, when downloaded before 20090713, have unknown impact and attack vectors related to the (1) \\environ, (2) \\input, and (3) \\counter TeX directives.", 
+	"description": "Multiple unspecified vulnerabilities in mimeTeX.", 
 	"releases": 
 	{"stretch": 
 		{"status": "resolved",