Merge pull request #26 from nexB/json_api

JSON API #8

Author: tdruez

.travis.yml

@@ -3,16 +3,14 @@ python: 3.6
 
 install:
   - pip install -r requirements.txt
-  - pip install pycodestyle
 
 before_script:
   - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
-  - cd app/
-  - python3 manage.py migrate
+  - ./manage.py migrate
 
 script:
-  - python3.6 -m pytest -v tests/
-  - python3.6 manage.py test
+  - python3.6 -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py
+  - ./manage.py test vulnerabilities/tests
 
 notifications:
   email: false

README.md

@@ -18,39 +18,52 @@ Activate a virtualenv, install dependencies, and run the database migrations:
 python3.6 -m venv .
 source bin/activate
 pip install -r requirements.txt
-app/manage.py migrate
+./manage.py migrate
 ```
 
 Tests
 -----
 
 ```
 pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
-cd app/
-python3.6 -m pytest -v tests/
+python3.6 -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py 
 ```
 
 For Django based tests
 ```
-cd app/
-./manage.py test
+./manage.py test vulnerabilities/tests
 ```
 
 Scrape and save to the database
 -------------------------------
 
 ```
-cd app/
 ./manage.py shell
 ```
 
 ```
-from scraper import debian, ubuntu
-from vulncode_app.data_dump import debian_dump, ubuntu_dump
+from vulnerabilities.scraper import debian, ubuntu
+from vulnerabilities.data_dump import debian_dump, ubuntu_dump
+
+# May be needed on macOS
+# import ssl; ssl._create_default_https_context = ssl._create_unverified_context
 
-debian_vulnerabilities = debian.scrape_vulnerabilities()
 ubuntu_cves = ubuntu.scrape_cves()
+ubuntu_dump(ubuntu_cves)
 
+debian_vulnerabilities = debian.scrape_vulnerabilities()
 debian_dump(debian_vulnerabilities)
-ubuntu_dump(ubuntu_cves)
+```
+
+API
+----
+Start the webserver
+
+```
+./manage.py runserver
+```
+
+In your browser access:
+```
+http://127.0.0.1:8000/vulnerabilities/api/<package_name>
 ```

app/manage.py manage.pyapp/manage.py renamed to manage.py

@@ -3,7 +3,7 @@
 import sys
 
 if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "vulnerablecode.settings")
     try:
         from django.core.management import execute_from_command_line
     except ImportError:

requirements.txt

@@ -1,6 +1,10 @@
+setuptools==36.5.0
+
 beautifulsoup4==4.6.0
-lxml==3.8.0
-django==1.11.4
+lxml==4.0.0
+django==1.11.5
+djangorestframework==3.6.4
 
 # Tests
-pytest==3.1.3
+pytest==3.2.2
+pycodestyle==2.3.1

app/app/__init__.py vulnerabilities/__init__.pyapp/app/__init__.py renamed to vulnerabilities/__init__.py

@@ -4,5 +4,5 @@
 from django.apps import AppConfig
 
 
-class VulncodeAppConfig(AppConfig):
-    name = 'vulncode_app'
+class VulnerabilitiesAppConfig(AppConfig):
+    name = 'vulnerabilities'

app/vulncode_app/api_data.py vulnerabilities/api_data.pyapp/vulncode_app/api_data.py renamed to vulnerabilities/api_data.py

@@ -21,9 +21,10 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-from vulncode_app.models import Vulnerability
-from vulncode_app.models import VulnerabilityReference
-from vulncode_app.models import Package
+from vulnerabilities.models import ImpactedPackage
+from vulnerabilities.models import Package
+from vulnerabilities.models import Vulnerability
+from vulnerabilities.models import VulnerabilityReference
 
 
 def debian_dump(extract_data):
@@ -38,10 +39,14 @@ def debian_dump(extract_data):
             vulnerability=vulnerability,
             reference_id=data.get('vulnerability_id', ''),
         )
-        Package.objects.create(
+        package = Package.objects.create(
             name=data.get('package_name', ''),
             version=data.get('fixed_version', ''),
         )
+        ImpactedPackage.objects.create(
+            vulnerability=vulnerability,
+            package=package
+        )
 
 
 def ubuntu_dump(html):
@@ -56,6 +61,10 @@ def ubuntu_dump(html):
             vulnerability=vulnerability,
             reference_id=data.get('cve_id'),
         )
-        Package.objects.create(
+        package = Package.objects.create(
             name=data.get('package_name'),
         )
+        ImpactedPackage.objects.create(
+            vulnerability=vulnerability,
+            package=package
+        )

app/vulncode_app/apps.py vulnerabilities/apps.pyapp/vulncode_app/apps.py renamed to vulnerabilities/apps.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.4 on 2017-08-08 09:11
+# Generated by Django 1.11.4 on 2017-09-20 22:34
 from __future__ import unicode_literals
 
 from django.db import migrations, models
@@ -37,14 +37,14 @@ class Migration(migrations.Migration):
                 ('platform', models.CharField(blank=True, help_text='Platform eg:maven', max_length=50)),
                 ('name', models.CharField(blank=True, help_text='Package reference name eg:org.apache.commons.io', max_length=50)),
                 ('version', models.CharField(blank=True, help_text='Reference version', max_length=50)),
-                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package')),
+                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package')),
             ],
         ),
         migrations.CreateModel(
             name='ResolvedPackage',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package')),
+                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package')),
             ],
         ),
         migrations.CreateModel(
@@ -59,29 +59,33 @@ class Migration(migrations.Migration):
             name='VulnerabilityReference',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('source', models.CharField(blank=True, help_text="Source's name eg:NVD", max_length=50)),
+                ('source', models.CharField(blank=True, help_text='Source(s) name eg:NVD', max_length=50)),
                 ('reference_id', models.CharField(blank=True, help_text='Reference ID, eg:CVE-ID', max_length=50)),
                 ('url', models.URLField(blank=True, help_text='URL of Vulnerability data', max_length=1024)),
-                ('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability')),
+                ('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability')),
             ],
         ),
         migrations.AddField(
             model_name='resolvedpackage',
             name='vulnerability',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability'),
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability'),
         ),
         migrations.AddField(
             model_name='impactedpackage',
             name='package',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package'),
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package'),
         ),
         migrations.AddField(
             model_name='impactedpackage',
             name='vulnerability',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability'),
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability'),
         ),
         migrations.AlterUniqueTogether(
             name='vulnerabilityreference',
-            unique_together=set([('vulnerability', 'source', 'reference_id')]),
+            unique_together=set([('vulnerability', 'source', 'reference_id', 'url')]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='impactedpackage',
+            unique_together=set([('vulnerability', 'package')]),
         ),
     ]

app/vulncode_app/data_dump.py vulnerabilities/data_dump.pyapp/vulncode_app/data_dump.py renamed to vulnerabilities/data_dump.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2017-09-22 19:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='package',
+            name='vulnerabilities',
+            field=models.ManyToManyField(through='vulnerabilities.ImpactedPackage', to='vulnerabilities.Vulnerability'),
+        ),
+    ]