JSON API #8

.travis.yml

@@ -3,16 +3,14 @@ python: 3.6
 
 install:
   - pip install -r requirements.txt
-  - pip install pycodestyle
 
 before_script:
   - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
-  - cd app/
-  - python3 manage.py migrate
+  - ./manage.py migrate
 
 script:
-  - python3.6 -m pytest -v tests/
-  - python3.6 manage.py test
+  - python3.6 -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py
+  - ./manage.py test vulnerabilities/tests
 
 notifications:
   email: false

README.md

@@ -18,39 +18,52 @@ Activate a virtualenv, install dependencies, and run the database migrations:
 python3.6 -m venv .
 source bin/activate
 pip install -r requirements.txt
-app/manage.py migrate
+./manage.py migrate
 ```
 
 Tests
 -----
 
 ```
 pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 .
-cd app/
-python3.6 -m pytest -v tests/
+python3.6 -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py 
 ```
 
 For Django based tests
 ```
-cd app/
-./manage.py test
+./manage.py test vulnerabilities/tests
 ```
 
 Scrape and save to the database
 -------------------------------
 
 ```
-cd app/
 ./manage.py shell
 ```
 
 ```
-from scraper import debian, ubuntu
-from vulncode_app.data_dump import debian_dump, ubuntu_dump
+from vulnerabilities.scraper import debian, ubuntu
+from vulnerabilities.data_dump import debian_dump, ubuntu_dump
+
+# May be needed on macOS
+# import ssl; ssl._create_default_https_context = ssl._create_unverified_context
 
-debian_vulnerabilities = debian.scrape_vulnerabilities()
 ubuntu_cves = ubuntu.scrape_cves()
+ubuntu_dump(ubuntu_cves)
 
+debian_vulnerabilities = debian.scrape_vulnerabilities()
 debian_dump(debian_vulnerabilities)
-ubuntu_dump(ubuntu_cves)
+```
+
+API
+----
+Start the webserver
+
+```
+./manage.py runserver
+```
+
+In your browser access:
+```
+http://127.0.0.1:8000/vulnerabilities/api/<package_name>
 ```

app/manage.py → manage.py

@@ -3,7 +3,7 @@
 import sys
 
 if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "vulnerablecode.settings")
     try:
         from django.core.management import execute_from_command_line
     except ImportError:

requirements.txt

@@ -1,6 +1,10 @@
+setuptools==36.5.0
+
 beautifulsoup4==4.6.0
-lxml==3.8.0
-django==1.11.4
+lxml==4.0.0
+django==1.11.5
+djangorestframework==3.6.4
 
 # Tests
-pytest==3.1.3
+pytest==3.2.2
+pycodestyle==2.3.1

app/vulncode_app/apps.py → vulnerabilities/apps.py

@@ -4,5 +4,5 @@
 from django.apps import AppConfig
 
 
-class VulncodeAppConfig(AppConfig):
-    name = 'vulncode_app'
+class VulnerabilitiesAppConfig(AppConfig):
+    name = 'vulnerabilities'

app/vulncode_app/data_dump.py → vulnerabilities/data_dump.py

@@ -21,9 +21,10 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-from vulncode_app.models import Vulnerability
-from vulncode_app.models import VulnerabilityReference
-from vulncode_app.models import Package
+from vulnerabilities.models import ImpactedPackage
+from vulnerabilities.models import Package
+from vulnerabilities.models import Vulnerability
+from vulnerabilities.models import VulnerabilityReference
 
 
 def debian_dump(extract_data):
@@ -38,10 +39,14 @@ def debian_dump(extract_data):
             vulnerability=vulnerability,
             reference_id=data.get('vulnerability_id', ''),
         )
-        Package.objects.create(
+        package = Package.objects.create(
             name=data.get('package_name', ''),
             version=data.get('fixed_version', ''),
         )
+        ImpactedPackage.objects.create(
+            vulnerability=vulnerability,
+            package=package
+        )
 
 
 def ubuntu_dump(html):
@@ -56,6 +61,10 @@ def ubuntu_dump(html):
             vulnerability=vulnerability,
             reference_id=data.get('cve_id'),
         )
-        Package.objects.create(
+        package = Package.objects.create(
             name=data.get('package_name'),
         )
+        ImpactedPackage.objects.create(
+            vulnerability=vulnerability,
+            package=package
+        )

app/vulncode_app/migrations/0001_initial.py → vulnerabilities/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.4 on 2017-08-08 09:11
+# Generated by Django 1.11.4 on 2017-09-20 22:34
 from __future__ import unicode_literals
 
 from django.db import migrations, models
@@ -37,14 +37,14 @@ class Migration(migrations.Migration):
                 ('platform', models.CharField(blank=True, help_text='Platform eg:maven', max_length=50)),
                 ('name', models.CharField(blank=True, help_text='Package reference name eg:org.apache.commons.io', max_length=50)),
                 ('version', models.CharField(blank=True, help_text='Reference version', max_length=50)),
-                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package')),
+                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package')),
             ],
         ),
         migrations.CreateModel(
             name='ResolvedPackage',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package')),
+                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package')),
             ],
         ),
         migrations.CreateModel(
@@ -59,29 +59,33 @@ class Migration(migrations.Migration):
             name='VulnerabilityReference',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('source', models.CharField(blank=True, help_text="Source's name eg:NVD", max_length=50)),
+                ('source', models.CharField(blank=True, help_text='Source(s) name eg:NVD', max_length=50)),
                 ('reference_id', models.CharField(blank=True, help_text='Reference ID, eg:CVE-ID', max_length=50)),
                 ('url', models.URLField(blank=True, help_text='URL of Vulnerability data', max_length=1024)),
-                ('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability')),
+                ('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability')),
             ],
         ),
         migrations.AddField(
             model_name='resolvedpackage',
             name='vulnerability',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability'),
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability'),
         ),
         migrations.AddField(
             model_name='impactedpackage',
             name='package',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package'),
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package'),
         ),
         migrations.AddField(
             model_name='impactedpackage',
             name='vulnerability',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability'),
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability'),
         ),
         migrations.AlterUniqueTogether(
             name='vulnerabilityreference',
-            unique_together=set([('vulnerability', 'source', 'reference_id')]),
+            unique_together=set([('vulnerability', 'source', 'reference_id', 'url')]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='impactedpackage',
+            unique_together=set([('vulnerability', 'package')]),
         ),
     ]

vulnerabilities/migrations/0002_package_vulnerabilities.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.5 on 2017-09-22 19:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='package',
+            name='vulnerabilities',
+            field=models.ManyToManyField(through='vulnerabilities.ImpactedPackage', to='vulnerabilities.Vulnerability'),
+        ),
+    ]

app/vulncode_app/models.py → vulnerabilities/models.py

@@ -48,7 +48,10 @@ class VulnerabilityReference(models.Model):
     url = models.URLField(max_length=1024, help_text='URL of Vulnerability data', blank=True)
 
     class Meta:
-        unique_together = ('vulnerability', 'source', 'reference_id')
+        unique_together = ('vulnerability', 'source', 'reference_id', 'url')
+
+    def __str__(self):
+        return self.source
 
 
 class ImpactedPackage(models.Model):
@@ -58,6 +61,9 @@ class ImpactedPackage(models.Model):
     vulnerability = models.ForeignKey('Vulnerability')
     package = models.ForeignKey('Package')
 
+    class Meta:
+        unique_together = ('vulnerability', 'package')
+
 
 class ResolvedPackage(models.Model):
     """
@@ -76,6 +82,7 @@ class Package(models.Model):
     platform = models.CharField(max_length=50, help_text='Package platform eg:maven', blank=True)
     name = models.CharField(max_length=50, help_text='Package name', blank=True)
     version = models.CharField(max_length=50, help_text='Package version', blank=True)
+    vulnerabilities = models.ManyToManyField(to='Vulnerability', through='ImpactedPackage')
 
     def __str__(self):
         return self.name

vulnerabilities/serializers.py

@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from rest_framework import serializers
+
+from vulnerabilities.models import Package
+from vulnerabilities.models import PackageReference
+from vulnerabilities.models import Vulnerability
+from vulnerabilities.models import VulnerabilityReference
+
+
+class PackageReferenceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = PackageReference
+        fields = ('repository', 'platform', 'name', 'version')
+
+
+class VulnerabilityReferenceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = VulnerabilityReference
+        fields = ('source', 'reference_id', 'url')
+
+
+class VulnerabilitySerializer(serializers.ModelSerializer):
+    references = VulnerabilityReferenceSerializer(source='vulnerabilityreference_set', many=True)
+
+    class Meta:
+        model = Vulnerability
+        fields = ('summary', 'references')
+
+
+class PackageSerializer(serializers.ModelSerializer):
+    vulnerabilities = VulnerabilitySerializer(many=True)
+
+    class Meta:
+        model = Package
+        fields = ('name', 'version', 'platform', 'vulnerabilities')