JSON API #8

.travis.yml

@@ -12,7 +12,7 @@ before_script:
 
 script:
   - python3.6 -m pytest -v tests/
-  - python3.6 manage.py test
+  - python3.6 manage.py test vulncode_app/tests
 
 notifications:
   email: false

README.md

@@ -33,7 +33,7 @@ python3.6 -m pytest -v tests/
 For Django based tests
 ```
 cd app/
-./manage.py test
+./manage.py test vulncode_app/tests
 ```
 
 Scrape and save to the database
@@ -54,3 +54,17 @@ ubuntu_cves = ubuntu.scrape_cves()
 debian_dump(debian_vulnerabilities)
 ubuntu_dump(ubuntu_cves)
 ```
+
+API
+----
+Start the server
+
+```
+cd app/
+./manage.py runserver
+```
+
+In your browser use
+```
+localhost:8000/vulncode_app/api/<package name>
+```

app/vulncode_app/data_dump.py

@@ -21,9 +21,10 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
+from vulncode_app.models import ImpactedPackage
+from vulncode_app.models import Package
 from vulncode_app.models import Vulnerability
 from vulncode_app.models import VulnerabilityReference
-from vulncode_app.models import Package
 
 
 def debian_dump(extract_data):
@@ -38,10 +39,14 @@ def debian_dump(extract_data):
             vulnerability=vulnerability,
             reference_id=data.get('vulnerability_id', ''),
         )
-        Package.objects.create(
+        package = Package.objects.create(
             name=data.get('package_name', ''),
             version=data.get('fixed_version', ''),
         )
+        ImpactedPackage.objects.create(
+            vulnerability=vulnerability,
+            package=package
+        )
 
 
 def ubuntu_dump(html):
@@ -56,6 +61,10 @@ def ubuntu_dump(html):
             vulnerability=vulnerability,
             reference_id=data.get('cve_id'),
         )
-        Package.objects.create(
+        package = Package.objects.create(
             name=data.get('package_name'),
         )
+        ImpactedPackage.objects.create(
+            vulnerability=vulnerability,
+            package=package
+        )

app/vulncode_app/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.4 on 2017-08-08 09:11
+# Generated by Django 1.11.2 on 2017-08-24 08:31
 from __future__ import unicode_literals
 
 from django.db import migrations, models
@@ -59,7 +59,7 @@ class Migration(migrations.Migration):
             name='VulnerabilityReference',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('source', models.CharField(blank=True, help_text="Source's name eg:NVD", max_length=50)),
+                ('source', models.CharField(blank=True, help_text='Source(s) name eg:NVD', max_length=50)),
                 ('reference_id', models.CharField(blank=True, help_text='Reference ID, eg:CVE-ID', max_length=50)),
                 ('url', models.URLField(blank=True, help_text='URL of Vulnerability data', max_length=1024)),
                 ('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability')),
@@ -84,4 +84,8 @@ class Migration(migrations.Migration):
             name='vulnerabilityreference',
             unique_together=set([('vulnerability', 'source', 'reference_id')]),
         ),
+        migrations.AlterUniqueTogether(
+            name='impactedpackage',
+            unique_together=set([('vulnerability', 'package')]),
+        ),
     ]

app/vulncode_app/migrations/0002_auto_20170907_2111.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.4 on 2017-09-07 21:11
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulncode_app', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='vulnerabilityreference',
+            unique_together=set([('vulnerability', 'source', 'reference_id', 'url')]),
+        ),
+    ]

app/vulncode_app/models.py

@@ -48,7 +48,10 @@ class VulnerabilityReference(models.Model):
     url = models.URLField(max_length=1024, help_text='URL of Vulnerability data', blank=True)
 
     class Meta:
-        unique_together = ('vulnerability', 'source', 'reference_id')
+        unique_together = ('vulnerability', 'source', 'reference_id', 'url')
+
+    def __str__(self):
+        return self.source
 
 
 class ImpactedPackage(models.Model):
@@ -58,6 +61,9 @@ class ImpactedPackage(models.Model):
     vulnerability = models.ForeignKey('Vulnerability')
     package = models.ForeignKey('Package')
 
+    class Meta:
+        unique_together = ('vulnerability', 'package')
+
 
 class ResolvedPackage(models.Model):
     """

app/vulncode_app/serializers.py

@@ -0,0 +1,66 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from rest_framework import serializers
+
+from vulncode_app.models import ImpactedPackage
+from vulncode_app.models import Package
+from vulncode_app.models import PackageReference
+from vulncode_app.models import Vulnerability
+from vulncode_app.models import VulnerabilityReference
+
+
+class PackageReferenceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = PackageReference
+        fields = ('repository', 'platform', 'name', 'version')
+
+
+class VulnerabilityReferenceSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = VulnerabilityReference
+        fields = ('source', 'reference_id', 'url')
+
+
+class VulnerabilitySerializer(serializers.ModelSerializer):
+    reference = VulnerabilityReferenceSerializer(source='vulnerabilityreference_set', many=True)
+
+    class Meta:
+        model = Vulnerability
+        fields = ('summary', 'reference')
+
+
+class ImpactedPackageSerializer(serializers.ModelSerializer):
+    vulnerability = VulnerabilitySerializer()
+
+    class Meta:
+        model = ImpactedPackage
+        fields = ('vulnerability',)
+
+
+class PackageSerializer(serializers.ModelSerializer):
+    vulnerabilities = ImpactedPackageSerializer(source='impactedpackage_set', many=True)
+
+    class Meta:
+        model = Package
+        fields = ('name', 'version', 'vulnerabilities')

app/vulncode_app/tests/test_api.py

@@ -0,0 +1,158 @@
+#
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import json
+
+from django.test import TestCase
+
+from vulncode_app.models import Package
+from vulncode_app.serializers import PackageSerializer
+from vulncode_app.data_dump import debian_dump
+from vulncode_app.data_dump import ubuntu_dump
+from scraper import debian
+from scraper import ubuntu
+
+
+class TestResponse(TestCase):
+    def test_debian_response(self):
+        with open('tests/test_data/debian.json') as f:
+            test_data = json.loads(f.read())
+
+        extract_data = debian.extract_vulnerabilities(test_data)
+        debian_dump(extract_data)
+        response = self.client.get('/vulncode_app/api/mimetex', format='json')
+
+        expected = [{
+            "name": "mimetex",
+            "version": "1.50-1.1",
+            "vulnerabilities": [{
+                "vulnerability": {
+                    "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX",
+                    "reference": [{
+                        "reference_id": "CVE-2009-2458",
+                        "source": "",
+                        "url": ""
+                    }]
+                }
+            }]
+        }, {
+            "name": "mimetex",
+            "version": "1.50-1.1",
+            "vulnerabilities": [{
+                "vulnerability": {
+                    "summary": "Multiple unspecified vulnerabilities in mimeTeX.",
+                    "reference": [{
+                        "reference_id": "CVE-2009-2459",
+                        "source": "",
+                        "url": ""
+                    }]
+                }
+            }]
+        }]
+
+        self.assertEqual(expected, response.data)
+
+    def test_ubuntu_response(self):
+        with open('tests/test_data/ubuntu_main.html') as f:
+            test_data = f.read()
+
+        extract_data = ubuntu.extract_cves(test_data)
+        ubuntu_dump(extract_data)
+        response = self.client.get('/vulncode_app/api/automake', format='json')
+
+        expected = [{
+            "name": "automake",
+            "version": "",
+            "vulnerabilities": [{
+                "vulnerability": {
+                    "summary": "",
+                    "reference": [{
+                        "reference_id": "CVE-2012-3386",
+                        "source": "",
+                        "url": ""
+                    }]
+                }
+            }]
+        }]
+
+        self.assertEqual(expected, response.data)
+
+    def test_blank_response(self):
+        response_invalid = self.client.get('/vulncode_app/api/', format='json')
+        response_blank = self.client.get('/vulncode_app/api/abbpcc', format='json')
+
+        self.assertEqual(404, response_invalid.status_code)
+        self.assertEqual([], response_blank.data)
+
+
+class TestSerializers(TestCase):
+    def test_serializers(self):
+        with open('tests/test_data/debian.json') as f:
+            test_data = json.loads(f.read())
+        extract_data = debian.extract_vulnerabilities(test_data)
+        debian_dump(extract_data)
+
+        pk = Package.objects.filter(name="mimetex")
+        response = PackageSerializer(pk, many=True).data
+
+        expected = [
+            {
+                "name": "mimetex",
+                "version": "1.50-1.1",
+                "vulnerabilities": [
+                    {
+                        "vulnerability": {
+                            "summary": "Multiple stack-based buffer overflows in mimetex.cgi in "
+                                       "mimeTeX",
+                            "reference": [
+                                {
+                                    "reference_id": "CVE-2009-2458",
+                                    "source": "",
+                                    "url": ""
+                                }
+                            ]
+                        }
+                    }
+                ]
+            },
+            {
+                "name": "mimetex",
+                "version": "1.50-1.1",
+                "vulnerabilities": [
+                    {
+                        "vulnerability": {
+                            "summary": "Multiple unspecified vulnerabilities in mimeTeX.",
+                            "reference": [
+                                {
+                                    "reference_id": "CVE-2009-2459",
+                                    "source": "",
+                                    "url": ""
+                                }
+                            ]
+                        }
+                    }
+                ]
+            }
+        ]
+
+        self.assertEqual(expected, response)

app/vulncode_app/urls.py

@@ -23,10 +23,20 @@
 
 from django.conf.urls import url
 
-from . import views
+from rest_framework.urlpatterns import format_suffix_patterns
+
+from vulncode_app import views
 
 
 urlpatterns = [
-    url(r'(?P<name>[a-z]+)/(?P<version>[0-9]+)', views.package_version, name='package_version'),
-    url(r'^(?P<name>[a-z]+)', views.package, name='package'),
+    url(r'^cve-search/(?P<name>[a-z]+)/(?P<version>[0-9]+)',
+        views.package_version,
+        name='package_version'),
+    url(r'^cve-search/(?P<name>[a-z]+)',
+        views.package,
+        name='package'),
+    url(r'^api/(?P<package_name>[a-z]+)',
+        views.VulnerabilityData.as_view()),
 ]
+
+urlpatterns = format_suffix_patterns(urlpatterns)