From bfa5bc7b12ddd3da4c50cdcddc1a1b2b3ad7b674 Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 15 Aug 2017 03:56:39 +0530 Subject: [PATCH 01/29] Add serializers based on models #8 * updated views * updated urls.py Signed-off-by: Kartik sibal --- app/vulncode_app/serializers.py | 46 +++++++++++++++++++++++++++++++++ app/vulncode_app/urls.py | 18 ++++++++++--- app/vulncode_app/views.py | 35 +++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 app/vulncode_app/serializers.py diff --git a/app/vulncode_app/serializers.py b/app/vulncode_app/serializers.py new file mode 100644 index 000000000..a433337d8 --- /dev/null +++ b/app/vulncode_app/serializers.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +from rest_framework import serializers + +from vulncode_app.models import Vulnerability +from vulncode_app.models import VulnerabilityReference +from vulncode_app.models import Package + + +class VulnerabilitySerializer(serializers.ModelSerializer): + class Meta: + model = Vulnerability + exclude = ('id', 'cvss') + + +class VulnerabilityReferenceSerializer(serializers.ModelSerializer): + class Meta: + model = VulnerabilityReference + exclude = ('id', 'source', 'url', 'vulnerability') + + +class PackageSerializer(serializers.ModelSerializer): + class Meta: + model = Package + fields = ('name', 'version') diff --git a/app/vulncode_app/urls.py b/app/vulncode_app/urls.py index 6eec081f6..da3afa2d8 100644 --- a/app/vulncode_app/urls.py +++ b/app/vulncode_app/urls.py @@ -22,11 +22,21 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. from django.conf.urls import url - -from . import views +from rest_framework.urlpatterns import format_suffix_patterns +from vulncode_app import views urlpatterns = [ - url(r'(?P[a-z]+)/(?P[0-9]+)', views.package_version, name='package_version'), - url(r'^(?P[a-z]+)', views.package, name='package'), + url(r'^cve-search/(?P[a-z]+)/(?P[0-9]+)', + views.package_version, + name='package_version'), + + url(r'^cve-search/(?P[a-z]+)', + views.package, + name='package'), + + url(r'^data/(?P[a-z]+)', + views.VulnerabilityData.as_view()), ] + +urlpatterns = format_suffix_patterns(urlpatterns) diff --git a/app/vulncode_app/views.py b/app/vulncode_app/views.py index 254b8b311..a95e986a2 100644 --- a/app/vulncode_app/views.py +++ b/app/vulncode_app/views.py @@ -24,10 +24,45 @@ import json from django.http import HttpResponse +from drf_multiple_model.views import MultipleModelAPIView +from rest_framework.views import APIView +from rest_framework.response import Response +from rest_framework import status +from vulncode_app.serializers import VulnerabilitySerializer +from vulncode_app.serializers import PackageSerializer +from vulncode_app.serializers import VulnerabilityReferenceSerializer +from vulncode_app.models import Vulnerability +from vulncode_app.models import Package +from vulncode_app.models import VulnerabilityReference from vulncode_app import api_data +class VulnerabilityData(APIView): + def get(self, request, pkg_name): + vulnerability_data = [] + reference_data = [] + + pk = Package.objects.filter(name=pkg_name).values('id') + + for index in range(len(pk)): + vulnerability_data.append( + VulnerabilitySerializer( + Vulnerability.objects.filter(pk=pk[index]['id']), + many=True).data + ) + reference_data.append( + VulnerabilityReferenceSerializer( + VulnerabilityReference.objects.filter(pk=pk[index]['id']), + many=True).data + ) + + package_data = PackageSerializer(Package.objects.filter(name=pkg_name), many=True) + data = [*vulnerability_data, *package_data.data, *reference_data] + + return Response(data) + + def package(request, name): """ Queries the cve-search api with just From 89f2bb14e359702ea0b6f076d4927449fe2b6f1e Mon Sep 17 00:00:00 2001 From: Kartik Sibal Date: Tue, 15 Aug 2017 04:04:43 +0530 Subject: [PATCH 02/29] Add DRF to requirements.txt Signed-off-by: Kartik sibal --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 889b545db..e18b17772 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ beautifulsoup4==4.6.0 lxml==3.8.0 django==1.11.4 +djangorestframework==3.6 # Tests pytest==3.1.3 From c75ae67ffa676c07d41530bc0205b9ba8c237252 Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 15 Aug 2017 04:06:42 +0530 Subject: [PATCH 03/29] Remove un-used imports #8 Signed-off-by: kartik sibal --- app/vulncode_app/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/vulncode_app/views.py b/app/vulncode_app/views.py index a95e986a2..0d6731e3c 100644 --- a/app/vulncode_app/views.py +++ b/app/vulncode_app/views.py @@ -24,7 +24,6 @@ import json from django.http import HttpResponse -from drf_multiple_model.views import MultipleModelAPIView from rest_framework.views import APIView from rest_framework.response import Response from rest_framework import status From 585327ad356dd8c0e6dbc32fb6b363a91a3769fc Mon Sep 17 00:00:00 2001 From: Kartik Date: Mon, 21 Aug 2017 02:32:13 +0530 Subject: [PATCH 04/29] Change data output format #8 * use natural impactpackage link * add impact package link in data_dump Signed-off-by: Kartik sibal --- app/vulncode_app/data_dump.py | 14 ++++++++++++-- app/vulncode_app/serializers.py | 27 +++++++++++++++++++++++---- app/vulncode_app/views.py | 33 +++++++++------------------------ 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/app/vulncode_app/data_dump.py b/app/vulncode_app/data_dump.py index 71bcd44bc..7eb609ce3 100644 --- a/app/vulncode_app/data_dump.py +++ b/app/vulncode_app/data_dump.py @@ -23,6 +23,7 @@ from vulncode_app.models import Vulnerability from vulncode_app.models import VulnerabilityReference +from vulncode_app.models import ImpactedPackage from vulncode_app.models import Package @@ -38,11 +39,16 @@ def debian_dump(extract_data): vulnerability=vulnerability, reference_id=data.get('vulnerability_id', ''), ) - Package.objects.create( + package = Package.objects.create( name=data.get('package_name', ''), version=data.get('fixed_version', ''), ) + impacted_package = ImpactedPackage.objects.create( + vulnerability=vulnerability, + package=package + ) + def ubuntu_dump(html): """ @@ -56,6 +62,10 @@ def ubuntu_dump(html): vulnerability=vulnerability, reference_id=data.get('cve_id'), ) - Package.objects.create( + package = Package.objects.create( name=data.get('package_name'), ) + ImpactedPackage.objects.create( + vulnerability=vulnerability, + package=package + ) diff --git a/app/vulncode_app/serializers.py b/app/vulncode_app/serializers.py index a433337d8..c919ac7ad 100644 --- a/app/vulncode_app/serializers.py +++ b/app/vulncode_app/serializers.py @@ -26,21 +26,40 @@ from vulncode_app.models import Vulnerability from vulncode_app.models import VulnerabilityReference from vulncode_app.models import Package +from vulncode_app.models import PackageReference +from vulncode_app.models import ImpactedPackage -class VulnerabilitySerializer(serializers.ModelSerializer): +class PackageReferenceSerializer(serializers.ModelSerializer): class Meta: - model = Vulnerability - exclude = ('id', 'cvss') + model = PackageReference + fields = ('name', 'version') class VulnerabilityReferenceSerializer(serializers.ModelSerializer): class Meta: model = VulnerabilityReference - exclude = ('id', 'source', 'url', 'vulnerability') + fields = ('reference_id',) + + +class VulnerabilitySerializer(serializers.ModelSerializer): + reference = VulnerabilityReferenceSerializer(source='vulnerabilityreference_set', many=True) + + class Meta: + model = Vulnerability + fields = ('summary', 'reference') class PackageSerializer(serializers.ModelSerializer): class Meta: model = Package fields = ('name', 'version') + + +class ImpactedPackageSerializer(serializers.ModelSerializer): + package = PackageSerializer() + vulnerability = VulnerabilitySerializer() + + class Meta: + model = ImpactedPackage + fields = ('package', 'vulnerability') diff --git a/app/vulncode_app/views.py b/app/vulncode_app/views.py index 0d6731e3c..c778fc202 100644 --- a/app/vulncode_app/views.py +++ b/app/vulncode_app/views.py @@ -28,38 +28,23 @@ from rest_framework.response import Response from rest_framework import status -from vulncode_app.serializers import VulnerabilitySerializer -from vulncode_app.serializers import PackageSerializer -from vulncode_app.serializers import VulnerabilityReferenceSerializer -from vulncode_app.models import Vulnerability +from vulncode_app.serializers import ImpactedPackageSerializer +from vulncode_app.models import ImpactedPackage from vulncode_app.models import Package -from vulncode_app.models import VulnerabilityReference from vulncode_app import api_data class VulnerabilityData(APIView): def get(self, request, pkg_name): - vulnerability_data = [] - reference_data = [] + vulnerability = [] + response = [] + pk = Package.objects.filter(name=pkg_name) - pk = Package.objects.filter(name=pkg_name).values('id') + for i, v in enumerate(pk): + vulnerability.append(ImpactedPackage.objects.filter(pk=v.id)) + response += ImpactedPackageSerializer(vulnerability[i], many=True).data - for index in range(len(pk)): - vulnerability_data.append( - VulnerabilitySerializer( - Vulnerability.objects.filter(pk=pk[index]['id']), - many=True).data - ) - reference_data.append( - VulnerabilityReferenceSerializer( - VulnerabilityReference.objects.filter(pk=pk[index]['id']), - many=True).data - ) - - package_data = PackageSerializer(Package.objects.filter(name=pkg_name), many=True) - data = [*vulnerability_data, *package_data.data, *reference_data] - - return Response(data) + return Response(response) def package(request, name): From 37d16b2bfd604bc491eb416580db8a615d5e9d17 Mon Sep 17 00:00:00 2001 From: Kartik Date: Mon, 21 Aug 2017 06:33:32 +0530 Subject: [PATCH 05/29] Change API data format to as discussed #8 Signed-off-by: Kartik sibal --- app/vulncode_app/urls.py | 2 +- app/vulncode_app/views.py | 24 ++++++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/app/vulncode_app/urls.py b/app/vulncode_app/urls.py index da3afa2d8..2a7aaf89c 100644 --- a/app/vulncode_app/urls.py +++ b/app/vulncode_app/urls.py @@ -35,7 +35,7 @@ views.package, name='package'), - url(r'^data/(?P[a-z]+)', + url(r'^data/(?P[a-z]+)', views.VulnerabilityData.as_view()), ] diff --git a/app/vulncode_app/views.py b/app/vulncode_app/views.py index c778fc202..00e31e0bc 100644 --- a/app/vulncode_app/views.py +++ b/app/vulncode_app/views.py @@ -35,14 +35,30 @@ class VulnerabilityData(APIView): - def get(self, request, pkg_name): + def get(self, request, package_name): vulnerability = [] - response = [] - pk = Package.objects.filter(name=pkg_name) + extract = [] + summary = [] + reference_id = [] + version = [] + + pk = Package.objects.filter(name=package_name) for i, v in enumerate(pk): vulnerability.append(ImpactedPackage.objects.filter(pk=v.id)) - response += ImpactedPackageSerializer(vulnerability[i], many=True).data + extract += ImpactedPackageSerializer(vulnerability[i], many=True).data + summary.append(extract[i]['vulnerability']['summary']) + reference_id.append(extract[i]['vulnerability']['reference'][0]['reference_id']) + version.append(extract[i]['package']['version']) + + response = { + 'name': package_name, + 'version': set(version), + 'vulnerabilities': { + 'summary': set(summary), + 'reference_id': set(reference_id) + } + } return Response(response) From 96d6bfd3a32cdee7edd4a700a7f5de7c9fe712c2 Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 22 Aug 2017 17:28:36 +0530 Subject: [PATCH 06/29] Add test cases and change data output format #8 * use dict instead of sets in data output Signed-off-by: Kartik sibal --- app/vulncode_app/test_api.py | 83 ++++++++++++++++++++++++++++++++++++ app/vulncode_app/views.py | 19 ++++----- 2 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 app/vulncode_app/test_api.py diff --git a/app/vulncode_app/test_api.py b/app/vulncode_app/test_api.py new file mode 100644 index 000000000..574b527ec --- /dev/null +++ b/app/vulncode_app/test_api.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import json + +from drf_multiple_model.views import MultipleModelAPIView +from django.test import TestCase +from rest_framework.response import Response + +from vulncode_app.data_dump import debian_dump +from vulncode_app.data_dump import ubuntu_dump + +from scraper import debian +from scraper import ubuntu + + +class TestSerializers(TestCase): + def test_debian_response(self): + with open('tests/test_data/debian.json') as f: + test_data = json.loads(f.read()) + + extract_data = debian.extract_vulnerabilities(test_data) + debian_dump(extract_data) + response = self.client.get('/vulncode_app/data/mimetex', format='json') + + expected = { + "name": "mimetex", + "vulnerabilities": [ + { + "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", + "reference_id": "CVE-2009-2458", + "version": "1.50-1.1" + }, + { + "summary": "Multiple unspecified vulnerabilities in mimeTeX.", + "reference_id": "CVE-2009-2459", + "version": "1.50-1.1" + } + ] + } + + self.assertEqual(expected, response.data) + + def test_ubuntu_response(self): + with open('tests/test_data/ubuntu_main.html') as f: + test_data = f.read() + + extract_data = ubuntu.extract_cves(test_data) + ubuntu_dump(extract_data) + response = self.client.get('/vulncode_app/data/automake', format='json') + + expected = { + "name": "automake", + "vulnerabilities": [ + { + "summary": "", + "reference_id": "CVE-2012-3386", + "version": "" + } + ] + } + + self.assertEqual(expected, response.data) diff --git a/app/vulncode_app/views.py b/app/vulncode_app/views.py index 00e31e0bc..c5247dfd6 100644 --- a/app/vulncode_app/views.py +++ b/app/vulncode_app/views.py @@ -38,26 +38,23 @@ class VulnerabilityData(APIView): def get(self, request, package_name): vulnerability = [] extract = [] - summary = [] - reference_id = [] - version = [] + data = [] pk = Package.objects.filter(name=package_name) for i, v in enumerate(pk): vulnerability.append(ImpactedPackage.objects.filter(pk=v.id)) extract += ImpactedPackageSerializer(vulnerability[i], many=True).data - summary.append(extract[i]['vulnerability']['summary']) - reference_id.append(extract[i]['vulnerability']['reference'][0]['reference_id']) - version.append(extract[i]['package']['version']) + + data.append({ + 'summary': extract[i]['vulnerability']['summary'], + 'reference_id': extract[i]['vulnerability']['reference'][0]['reference_id'], + 'version': extract[i]['package']['version'] + }) response = { 'name': package_name, - 'version': set(version), - 'vulnerabilities': { - 'summary': set(summary), - 'reference_id': set(reference_id) - } + 'vulnerabilities': data } return Response(response) From 0705c24d8b5a0f34a937c21e96c3109b9cbe5550 Mon Sep 17 00:00:00 2001 From: Kartik Date: Wed, 23 Aug 2017 07:29:32 +0530 Subject: [PATCH 07/29] Updated serialization logic Signed-off-by: Kartik sibal --- app/vulncode_app/data_dump.py | 9 ++++++--- app/vulncode_app/serializers.py | 17 +++++++++-------- app/vulncode_app/views.py | 23 ++--------------------- 3 files changed, 17 insertions(+), 32 deletions(-) diff --git a/app/vulncode_app/data_dump.py b/app/vulncode_app/data_dump.py index 7eb609ce3..cd8d6485a 100644 --- a/app/vulncode_app/data_dump.py +++ b/app/vulncode_app/data_dump.py @@ -43,13 +43,16 @@ def debian_dump(extract_data): name=data.get('package_name', ''), version=data.get('fixed_version', ''), ) - impacted_package = ImpactedPackage.objects.create( vulnerability=vulnerability, - package=package + package_fk=package ) + package.impacted_package.add( + impacted_package + ) + package.save() - + def ubuntu_dump(html): """ Dump data scraped from Ubuntu's security tracker. diff --git a/app/vulncode_app/serializers.py b/app/vulncode_app/serializers.py index c919ac7ad..9927fd3c9 100644 --- a/app/vulncode_app/serializers.py +++ b/app/vulncode_app/serializers.py @@ -50,16 +50,17 @@ class Meta: fields = ('summary', 'reference') -class PackageSerializer(serializers.ModelSerializer): - class Meta: - model = Package - fields = ('name', 'version') - - class ImpactedPackageSerializer(serializers.ModelSerializer): - package = PackageSerializer() vulnerability = VulnerabilitySerializer() class Meta: model = ImpactedPackage - fields = ('package', 'vulnerability') + fields = ('vulnerability',) + + +class PackageSerializer(serializers.ModelSerializer): + vulnerability = ImpactedPackageSerializer(source='impactedpackage_set', many=True) + + class Meta: + model = Package + fields = ('name', 'version', 'vulnerability') diff --git a/app/vulncode_app/views.py b/app/vulncode_app/views.py index c5247dfd6..c84113168 100644 --- a/app/vulncode_app/views.py +++ b/app/vulncode_app/views.py @@ -28,34 +28,15 @@ from rest_framework.response import Response from rest_framework import status -from vulncode_app.serializers import ImpactedPackageSerializer -from vulncode_app.models import ImpactedPackage +from vulncode_app.serializers import PackageSerializer from vulncode_app.models import Package from vulncode_app import api_data class VulnerabilityData(APIView): def get(self, request, package_name): - vulnerability = [] - extract = [] - data = [] - pk = Package.objects.filter(name=package_name) - - for i, v in enumerate(pk): - vulnerability.append(ImpactedPackage.objects.filter(pk=v.id)) - extract += ImpactedPackageSerializer(vulnerability[i], many=True).data - - data.append({ - 'summary': extract[i]['vulnerability']['summary'], - 'reference_id': extract[i]['vulnerability']['reference'][0]['reference_id'], - 'version': extract[i]['package']['version'] - }) - - response = { - 'name': package_name, - 'vulnerabilities': data - } + response = PackageSerializer(pk, many=True).data return Response(response) From f08d614bb30e53164ef0c45eeda6c136de5eece2 Mon Sep 17 00:00:00 2001 From: Kartik Date: Wed, 23 Aug 2017 07:30:47 +0530 Subject: [PATCH 08/29] Add many to many field on Package #8 Signed-off-by: kartik sibal --- app/vulncode_app/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/vulncode_app/models.py b/app/vulncode_app/models.py index 713423555..0cddb3f17 100644 --- a/app/vulncode_app/models.py +++ b/app/vulncode_app/models.py @@ -50,13 +50,16 @@ class VulnerabilityReference(models.Model): class Meta: unique_together = ('vulnerability', 'source', 'reference_id') + def __str__(self): + return self.source + class ImpactedPackage(models.Model): """ Relates a vulnerability to package(s) impacted by it. """ vulnerability = models.ForeignKey('Vulnerability') - package = models.ForeignKey('Package') + package_fk = models.ForeignKey('Package') class ResolvedPackage(models.Model): @@ -73,6 +76,7 @@ class Package(models.Model): A software package with minimal identifying information. Other identifiers are stored as PackageReference. """ + impacted_package = models.ManyToManyField('ImpactedPackage') platform = models.CharField(max_length=50, help_text='Package platform eg:maven', blank=True) name = models.CharField(max_length=50, help_text='Package name', blank=True) version = models.CharField(max_length=50, help_text='Package version', blank=True) From 692d72d3c7448eef3dd0c464e9436c328e227b99 Mon Sep 17 00:00:00 2001 From: Kartik Date: Wed, 23 Aug 2017 07:31:24 +0530 Subject: [PATCH 09/29] Remove un-used imports #8 Signed-off-by: Kartik sibal --- app/vulncode_app/test_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/vulncode_app/test_api.py b/app/vulncode_app/test_api.py index 574b527ec..5144e2dcf 100644 --- a/app/vulncode_app/test_api.py +++ b/app/vulncode_app/test_api.py @@ -23,7 +23,6 @@ import json -from drf_multiple_model.views import MultipleModelAPIView from django.test import TestCase from rest_framework.response import Response From fcadb1777ccd1266302b6eaf7cbb0b49b6e500e5 Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 24 Aug 2017 12:42:06 +0530 Subject: [PATCH 10/29] Add test cases #8 * sort imports Signed-off-by: Kartik sibal --- app/vulncode_app/data_dump.py | 8 +-- app/vulncode_app/serializers.py | 14 ++-- app/vulncode_app/test_api.py | 116 ++++++++++++++++++++++++++------ app/vulncode_app/urls.py | 1 + 4 files changed, 107 insertions(+), 32 deletions(-) diff --git a/app/vulncode_app/data_dump.py b/app/vulncode_app/data_dump.py index cd8d6485a..05bc882b6 100644 --- a/app/vulncode_app/data_dump.py +++ b/app/vulncode_app/data_dump.py @@ -21,10 +21,10 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -from vulncode_app.models import Vulnerability -from vulncode_app.models import VulnerabilityReference from vulncode_app.models import ImpactedPackage from vulncode_app.models import Package +from vulncode_app.models import VulnerabilityReference +from vulncode_app.models import Vulnerability def debian_dump(extract_data): @@ -52,7 +52,7 @@ def debian_dump(extract_data): ) package.save() - + def ubuntu_dump(html): """ Dump data scraped from Ubuntu's security tracker. @@ -70,5 +70,5 @@ def ubuntu_dump(html): ) ImpactedPackage.objects.create( vulnerability=vulnerability, - package=package + package_fk=package ) diff --git a/app/vulncode_app/serializers.py b/app/vulncode_app/serializers.py index 9927fd3c9..1254815e7 100644 --- a/app/vulncode_app/serializers.py +++ b/app/vulncode_app/serializers.py @@ -23,23 +23,23 @@ from rest_framework import serializers -from vulncode_app.models import Vulnerability -from vulncode_app.models import VulnerabilityReference +from vulncode_app.models import ImpactedPackage from vulncode_app.models import Package from vulncode_app.models import PackageReference -from vulncode_app.models import ImpactedPackage +from vulncode_app.models import VulnerabilityReference +from vulncode_app.models import Vulnerability class PackageReferenceSerializer(serializers.ModelSerializer): class Meta: model = PackageReference - fields = ('name', 'version') + fields = ('name', 'version', 'repository', 'paltform', ) class VulnerabilityReferenceSerializer(serializers.ModelSerializer): class Meta: model = VulnerabilityReference - fields = ('reference_id',) + fields = ('reference_id', 'source', 'url') class VulnerabilitySerializer(serializers.ModelSerializer): @@ -59,8 +59,8 @@ class Meta: class PackageSerializer(serializers.ModelSerializer): - vulnerability = ImpactedPackageSerializer(source='impactedpackage_set', many=True) + vulnerabilities = ImpactedPackageSerializer(source='impactedpackage_set', many=True) class Meta: model = Package - fields = ('name', 'version', 'vulnerability') + fields = ('name', 'version', 'vulnerabilities') diff --git a/app/vulncode_app/test_api.py b/app/vulncode_app/test_api.py index 5144e2dcf..ac1e9778c 100644 --- a/app/vulncode_app/test_api.py +++ b/app/vulncode_app/test_api.py @@ -25,6 +25,10 @@ from django.test import TestCase from rest_framework.response import Response +from collections import OrderedDict + +from vulncode_app.models import Package +from vulncode_app.serializers import PackageSerializer from vulncode_app.data_dump import debian_dump from vulncode_app.data_dump import ubuntu_dump @@ -33,7 +37,7 @@ from scraper import ubuntu -class TestSerializers(TestCase): +class TestResponse(TestCase): def test_debian_response(self): with open('tests/test_data/debian.json') as f: test_data = json.loads(f.read()) @@ -42,21 +46,36 @@ def test_debian_response(self): debian_dump(extract_data) response = self.client.get('/vulncode_app/data/mimetex', format='json') - expected = { - "name": "mimetex", - "vulnerabilities": [ - { - "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", + expected = [ + { + "name": "mimetex", + "version": "1.50-1.1", + "vulnerabilities": [{ + "vulnerability": { + "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", + "reference": [ + { "reference_id": "CVE-2009-2458", - "version": "1.50-1.1" - }, - { - "summary": "Multiple unspecified vulnerabilities in mimeTeX.", + "source": "", + "url": "" + } + ]}}] + }, + { + "name": "mimetex", + "version": "1.50-1.1", + "vulnerabilities": [{ + "vulnerability": { + "summary": "Multiple unspecified vulnerabilities in mimeTeX.", + "reference": [ + { "reference_id": "CVE-2009-2459", - "version": "1.50-1.1" - } - ] + "source": "", + "url": "" + } + ]}}] } + ] self.assertEqual(expected, response.data) @@ -68,15 +87,70 @@ def test_ubuntu_response(self): ubuntu_dump(extract_data) response = self.client.get('/vulncode_app/data/automake', format='json') - expected = { - "name": "automake", - "vulnerabilities": [ + expected = [ + { + "name": "automake", + "version": "", + "vulnerabilities": [ + { + "vulnerability": { + "summary": "", + "reference": [ { - "summary": "", - "reference_id": "CVE-2012-3386", - "version": "" - } - ] + "reference_id": "CVE-2012-3386", + "source": "", + "url": "" + }]}}] } + ] self.assertEqual(expected, response.data) + + +class TestSerializers(TestCase): + def test_serializers(self): + with open('tests/test_data/debian.json') as f: + test_data = json.loads(f.read()) + extract_data = debian.extract_vulnerabilities(test_data) + debian_dump(extract_data) + + pk = Package.objects.filter(name="mimetex") + response = PackageSerializer(pk, many=True).data + + expected = [OrderedDict([ + ('name', 'mimetex'), + ('version', '1.50-1.1'), + ('vulnerabilities', + [OrderedDict([ + ('vulnerability', + OrderedDict([ + ( + 'summary', + 'Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX' + ), + ('reference', + [OrderedDict([ + ('reference_id', 'CVE-2009-2458'), + ('source', ''), + ('url', '') + ])])]))])])]), + + OrderedDict([ + ('name', 'mimetex'), + ('version', '1.50-1.1'), + ('vulnerabilities', + [OrderedDict([ + ('vulnerability', + OrderedDict([ + ( + 'summary', + 'Multiple unspecified vulnerabilities in mimeTeX.' + ), + ('reference', + [OrderedDict([ + ('reference_id', 'CVE-2009-2459'), + ('source', ''), + ('url', '') + ])])]))])])])] + + self.assertEqual(expected, response) diff --git a/app/vulncode_app/urls.py b/app/vulncode_app/urls.py index 2a7aaf89c..2dbeabe38 100644 --- a/app/vulncode_app/urls.py +++ b/app/vulncode_app/urls.py @@ -23,6 +23,7 @@ from django.conf.urls import url from rest_framework.urlpatterns import format_suffix_patterns + from vulncode_app import views From d41215ddafdcb28e4d61adeb5b880024dbf5af87 Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 24 Aug 2017 13:39:58 +0530 Subject: [PATCH 11/29] New migrations #8 Signed-off-by: kartik sibal --- app/vulncode_app/migrations/0001_initial.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/vulncode_app/migrations/0001_initial.py b/app/vulncode_app/migrations/0001_initial.py index 2b500623d..8c8fb2935 100644 --- a/app/vulncode_app/migrations/0001_initial.py +++ b/app/vulncode_app/migrations/0001_initial.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.4 on 2017-08-08 09:11 +# Generated by Django 1.11.2 on 2017-08-24 07:57 from __future__ import unicode_literals from django.db import migrations, models @@ -27,6 +27,7 @@ class Migration(migrations.Migration): ('platform', models.CharField(blank=True, help_text='Package platform eg:maven', max_length=50)), ('name', models.CharField(blank=True, help_text='Package name', max_length=50)), ('version', models.CharField(blank=True, help_text='Package version', max_length=50)), + ('impacted_package', models.ManyToManyField(to='vulncode_app.ImpactedPackage')), ], ), migrations.CreateModel( @@ -59,7 +60,7 @@ class Migration(migrations.Migration): name='VulnerabilityReference', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('source', models.CharField(blank=True, help_text="Source's name eg:NVD", max_length=50)), + ('source', models.CharField(blank=True, help_text='Source(s) name eg:NVD', max_length=50)), ('reference_id', models.CharField(blank=True, help_text='Reference ID, eg:CVE-ID', max_length=50)), ('url', models.URLField(blank=True, help_text='URL of Vulnerability data', max_length=1024)), ('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability')), @@ -72,7 +73,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='impactedpackage', - name='package', + name='package_fk', field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package'), ), migrations.AddField( From 9015a68de79de1ca50250d26eaacceec7524ee14 Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 24 Aug 2017 14:06:25 +0530 Subject: [PATCH 12/29] Update models #8 * remove M2M on package * rename package_fk to package Signed-off-by: Kartik sibal --- app/vulncode_app/migrations/0001_initial.py | 9 ++++++--- app/vulncode_app/models.py | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/app/vulncode_app/migrations/0001_initial.py b/app/vulncode_app/migrations/0001_initial.py index 8c8fb2935..fb4ef5f0f 100644 --- a/app/vulncode_app/migrations/0001_initial.py +++ b/app/vulncode_app/migrations/0001_initial.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.2 on 2017-08-24 07:57 +# Generated by Django 1.11.2 on 2017-08-24 08:31 from __future__ import unicode_literals from django.db import migrations, models @@ -27,7 +27,6 @@ class Migration(migrations.Migration): ('platform', models.CharField(blank=True, help_text='Package platform eg:maven', max_length=50)), ('name', models.CharField(blank=True, help_text='Package name', max_length=50)), ('version', models.CharField(blank=True, help_text='Package version', max_length=50)), - ('impacted_package', models.ManyToManyField(to='vulncode_app.ImpactedPackage')), ], ), migrations.CreateModel( @@ -73,7 +72,7 @@ class Migration(migrations.Migration): ), migrations.AddField( model_name='impactedpackage', - name='package_fk', + name='package', field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package'), ), migrations.AddField( @@ -85,4 +84,8 @@ class Migration(migrations.Migration): name='vulnerabilityreference', unique_together=set([('vulnerability', 'source', 'reference_id')]), ), + migrations.AlterUniqueTogether( + name='impactedpackage', + unique_together=set([('vulnerability', 'package')]), + ), ] diff --git a/app/vulncode_app/models.py b/app/vulncode_app/models.py index 0cddb3f17..358e8877d 100644 --- a/app/vulncode_app/models.py +++ b/app/vulncode_app/models.py @@ -59,7 +59,10 @@ class ImpactedPackage(models.Model): Relates a vulnerability to package(s) impacted by it. """ vulnerability = models.ForeignKey('Vulnerability') - package_fk = models.ForeignKey('Package') + package = models.ForeignKey('Package') + + class Meta: + unique_together = ('vulnerability', 'package') class ResolvedPackage(models.Model): @@ -76,7 +79,6 @@ class Package(models.Model): A software package with minimal identifying information. Other identifiers are stored as PackageReference. """ - impacted_package = models.ManyToManyField('ImpactedPackage') platform = models.CharField(max_length=50, help_text='Package platform eg:maven', blank=True) name = models.CharField(max_length=50, help_text='Package name', blank=True) version = models.CharField(max_length=50, help_text='Package version', blank=True) From bdcb807f69feda32e58255766931b8d7bd44646c Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 24 Aug 2017 14:07:13 +0530 Subject: [PATCH 13/29] Minor formatting changes #8 Signed-off-by: Kartik sibal --- app/vulncode_app/data_dump.py | 10 +++------- app/vulncode_app/serializers.py | 4 ++-- app/vulncode_app/test_api.py | 3 ++- app/vulncode_app/views.py | 6 +++--- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/app/vulncode_app/data_dump.py b/app/vulncode_app/data_dump.py index 05bc882b6..5d8521a9d 100644 --- a/app/vulncode_app/data_dump.py +++ b/app/vulncode_app/data_dump.py @@ -23,8 +23,8 @@ from vulncode_app.models import ImpactedPackage from vulncode_app.models import Package -from vulncode_app.models import VulnerabilityReference from vulncode_app.models import Vulnerability +from vulncode_app.models import VulnerabilityReference def debian_dump(extract_data): @@ -45,12 +45,8 @@ def debian_dump(extract_data): ) impacted_package = ImpactedPackage.objects.create( vulnerability=vulnerability, - package_fk=package - ) - package.impacted_package.add( - impacted_package + package=package ) - package.save() def ubuntu_dump(html): @@ -70,5 +66,5 @@ def ubuntu_dump(html): ) ImpactedPackage.objects.create( vulnerability=vulnerability, - package_fk=package + package=package ) diff --git a/app/vulncode_app/serializers.py b/app/vulncode_app/serializers.py index 1254815e7..f8579a251 100644 --- a/app/vulncode_app/serializers.py +++ b/app/vulncode_app/serializers.py @@ -26,14 +26,14 @@ from vulncode_app.models import ImpactedPackage from vulncode_app.models import Package from vulncode_app.models import PackageReference -from vulncode_app.models import VulnerabilityReference from vulncode_app.models import Vulnerability +from vulncode_app.models import VulnerabilityReference class PackageReferenceSerializer(serializers.ModelSerializer): class Meta: model = PackageReference - fields = ('name', 'version', 'repository', 'paltform', ) + fields = ('name', 'version', 'repository', 'platform', ) class VulnerabilityReferenceSerializer(serializers.ModelSerializer): diff --git a/app/vulncode_app/test_api.py b/app/vulncode_app/test_api.py index ac1e9778c..7e5b48831 100644 --- a/app/vulncode_app/test_api.py +++ b/app/vulncode_app/test_api.py @@ -23,9 +23,10 @@ import json +from collections import OrderedDict + from django.test import TestCase from rest_framework.response import Response -from collections import OrderedDict from vulncode_app.models import Package from vulncode_app.serializers import PackageSerializer diff --git a/app/vulncode_app/views.py b/app/vulncode_app/views.py index c84113168..d1eed9f92 100644 --- a/app/vulncode_app/views.py +++ b/app/vulncode_app/views.py @@ -23,13 +23,14 @@ import json -from django.http import HttpResponse from rest_framework.views import APIView +from django.http import HttpResponse from rest_framework.response import Response from rest_framework import status -from vulncode_app.serializers import PackageSerializer from vulncode_app.models import Package +from vulncode_app.serializers import PackageSerializer + from vulncode_app import api_data @@ -37,7 +38,6 @@ class VulnerabilityData(APIView): def get(self, request, package_name): pk = Package.objects.filter(name=package_name) response = PackageSerializer(pk, many=True).data - return Response(response) From 42c7be2d23d5f020693efab6232998b2dac3cfa6 Mon Sep 17 00:00:00 2001 From: Kartik Date: Fri, 25 Aug 2017 15:34:04 +0530 Subject: [PATCH 14/29] Add test cases #8 * test blank response Signed-off-by: Kartik sibal --- app/vulncode_app/test_api.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/vulncode_app/test_api.py b/app/vulncode_app/test_api.py index 7e5b48831..8503faf07 100644 --- a/app/vulncode_app/test_api.py +++ b/app/vulncode_app/test_api.py @@ -101,12 +101,20 @@ def test_ubuntu_response(self): "reference_id": "CVE-2012-3386", "source": "", "url": "" - }]}}] + } + ]}}] } ] self.assertEqual(expected, response.data) + def test_blank_response(self): + response_invalid = self.client.get('/vulncode_app/data/', format='json') + response_blank = self.client.get('/vulncode_app/data/abbpcc', format='json') + + self.assertEqual(404, response_invalid.status_code) + self.assertEqual([], response_blank.data) + class TestSerializers(TestCase): def test_serializers(self): From d1821c44a676405e57c997551c2d8754ead0ccb6 Mon Sep 17 00:00:00 2001 From: Kartik Date: Fri, 25 Aug 2017 15:44:09 +0530 Subject: [PATCH 15/29] Update unique together #8 * add url in vulnerability reference unique together Signed-off-by: Kartik sibal --- app/vulncode_app/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/vulncode_app/models.py b/app/vulncode_app/models.py index 358e8877d..f0f06cec4 100644 --- a/app/vulncode_app/models.py +++ b/app/vulncode_app/models.py @@ -48,7 +48,7 @@ class VulnerabilityReference(models.Model): url = models.URLField(max_length=1024, help_text='URL of Vulnerability data', blank=True) class Meta: - unique_together = ('vulnerability', 'source', 'reference_id') + unique_together = ('vulnerability', 'source', 'reference_id', 'url') def __str__(self): return self.source From 8158a84e0b000cdc14625261f13a757496e2884e Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 7 Sep 2017 18:22:17 -0300 Subject: [PATCH 16/29] Add missing migration file along some code cleanup #8 Signed-off-by: Thomas Druez --- app/vulncode_app/data_dump.py | 2 +- .../migrations/0002_auto_20170907_2111.py | 19 +++ app/vulncode_app/serializers.py | 2 +- app/vulncode_app/test_api.py | 157 +++++++++--------- app/vulncode_app/urls.py | 3 +- app/vulncode_app/views.py | 9 +- 6 files changed, 101 insertions(+), 91 deletions(-) create mode 100644 app/vulncode_app/migrations/0002_auto_20170907_2111.py diff --git a/app/vulncode_app/data_dump.py b/app/vulncode_app/data_dump.py index 5d8521a9d..d309e428d 100644 --- a/app/vulncode_app/data_dump.py +++ b/app/vulncode_app/data_dump.py @@ -43,7 +43,7 @@ def debian_dump(extract_data): name=data.get('package_name', ''), version=data.get('fixed_version', ''), ) - impacted_package = ImpactedPackage.objects.create( + ImpactedPackage.objects.create( vulnerability=vulnerability, package=package ) diff --git a/app/vulncode_app/migrations/0002_auto_20170907_2111.py b/app/vulncode_app/migrations/0002_auto_20170907_2111.py new file mode 100644 index 000000000..36d648389 --- /dev/null +++ b/app/vulncode_app/migrations/0002_auto_20170907_2111.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.4 on 2017-09-07 21:11 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulncode_app', '0001_initial'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='vulnerabilityreference', + unique_together=set([('vulnerability', 'source', 'reference_id', 'url')]), + ), + ] diff --git a/app/vulncode_app/serializers.py b/app/vulncode_app/serializers.py index f8579a251..6ffedd948 100644 --- a/app/vulncode_app/serializers.py +++ b/app/vulncode_app/serializers.py @@ -33,7 +33,7 @@ class PackageReferenceSerializer(serializers.ModelSerializer): class Meta: model = PackageReference - fields = ('name', 'version', 'repository', 'platform', ) + fields = ('name', 'version', 'repository', 'platform') class VulnerabilityReferenceSerializer(serializers.ModelSerializer): diff --git a/app/vulncode_app/test_api.py b/app/vulncode_app/test_api.py index 8503faf07..e8bea84a8 100644 --- a/app/vulncode_app/test_api.py +++ b/app/vulncode_app/test_api.py @@ -23,17 +23,12 @@ import json -from collections import OrderedDict - from django.test import TestCase -from rest_framework.response import Response from vulncode_app.models import Package from vulncode_app.serializers import PackageSerializer - from vulncode_app.data_dump import debian_dump from vulncode_app.data_dump import ubuntu_dump - from scraper import debian from scraper import ubuntu @@ -47,36 +42,33 @@ def test_debian_response(self): debian_dump(extract_data) response = self.client.get('/vulncode_app/data/mimetex', format='json') - expected = [ - { - "name": "mimetex", - "version": "1.50-1.1", - "vulnerabilities": [{ + expected = [{ + "name": "mimetex", + "version": "1.50-1.1", + "vulnerabilities": [{ "vulnerability": { - "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", - "reference": [ - { - "reference_id": "CVE-2009-2458", - "source": "", - "url": "" - } - ]}}] - }, - { - "name": "mimetex", - "version": "1.50-1.1", - "vulnerabilities": [{ + "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", + "reference": [{ + "reference_id": "CVE-2009-2458", + "source": "", + "url": "" + }] + } + }] + }, { + "name": "mimetex", + "version": "1.50-1.1", + "vulnerabilities": [{ "vulnerability": { - "summary": "Multiple unspecified vulnerabilities in mimeTeX.", - "reference": [ - { - "reference_id": "CVE-2009-2459", - "source": "", - "url": "" - } - ]}}] - } - ] + "summary": "Multiple unspecified vulnerabilities in mimeTeX.", + "reference": [{ + "reference_id": "CVE-2009-2459", + "source": "", + "url": "" + }] + } + }] + }] self.assertEqual(expected, response.data) @@ -88,23 +80,20 @@ def test_ubuntu_response(self): ubuntu_dump(extract_data) response = self.client.get('/vulncode_app/data/automake', format='json') - expected = [ - { - "name": "automake", - "version": "", - "vulnerabilities": [ - { - "vulnerability": { - "summary": "", - "reference": [ - { + expected = [{ + "name": "automake", + "version": "", + "vulnerabilities": [{ + "vulnerability": { + "summary": "", + "reference": [{ "reference_id": "CVE-2012-3386", "source": "", "url": "" - } - ]}}] - } - ] + }] + } + }] + }] self.assertEqual(expected, response.data) @@ -126,40 +115,44 @@ def test_serializers(self): pk = Package.objects.filter(name="mimetex") response = PackageSerializer(pk, many=True).data - expected = [OrderedDict([ - ('name', 'mimetex'), - ('version', '1.50-1.1'), - ('vulnerabilities', - [OrderedDict([ - ('vulnerability', - OrderedDict([ - ( - 'summary', - 'Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX' - ), - ('reference', - [OrderedDict([ - ('reference_id', 'CVE-2009-2458'), - ('source', ''), - ('url', '') - ])])]))])])]), - - OrderedDict([ - ('name', 'mimetex'), - ('version', '1.50-1.1'), - ('vulnerabilities', - [OrderedDict([ - ('vulnerability', - OrderedDict([ - ( - 'summary', - 'Multiple unspecified vulnerabilities in mimeTeX.' - ), - ('reference', - [OrderedDict([ - ('reference_id', 'CVE-2009-2459'), - ('source', ''), - ('url', '') - ])])]))])])])] + expected = [ + { + "name": "mimetex", + "version": "1.50-1.1", + "vulnerabilities": [ + { + "vulnerability": { + "summary": "Multiple stack-based buffer overflows in mimetex.cgi in " + "mimeTeX", + "reference": [ + { + "reference_id": "CVE-2009-2458", + "source": "", + "url": "" + } + ] + } + } + ] + }, + { + "name": "mimetex", + "version": "1.50-1.1", + "vulnerabilities": [ + { + "vulnerability": { + "summary": "Multiple unspecified vulnerabilities in mimeTeX.", + "reference": [ + { + "reference_id": "CVE-2009-2459", + "source": "", + "url": "" + } + ] + } + } + ] + } + ] self.assertEqual(expected, response) diff --git a/app/vulncode_app/urls.py b/app/vulncode_app/urls.py index 2dbeabe38..4bb461af7 100644 --- a/app/vulncode_app/urls.py +++ b/app/vulncode_app/urls.py @@ -22,6 +22,7 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. from django.conf.urls import url + from rest_framework.urlpatterns import format_suffix_patterns from vulncode_app import views @@ -31,11 +32,9 @@ url(r'^cve-search/(?P[a-z]+)/(?P[0-9]+)', views.package_version, name='package_version'), - url(r'^cve-search/(?P[a-z]+)', views.package, name='package'), - url(r'^data/(?P[a-z]+)', views.VulnerabilityData.as_view()), ] diff --git a/app/vulncode_app/views.py b/app/vulncode_app/views.py index d1eed9f92..a286b5678 100644 --- a/app/vulncode_app/views.py +++ b/app/vulncode_app/views.py @@ -23,21 +23,20 @@ import json -from rest_framework.views import APIView from django.http import HttpResponse + +from rest_framework.views import APIView from rest_framework.response import Response -from rest_framework import status from vulncode_app.models import Package from vulncode_app.serializers import PackageSerializer - from vulncode_app import api_data class VulnerabilityData(APIView): def get(self, request, package_name): - pk = Package.objects.filter(name=package_name) - response = PackageSerializer(pk, many=True).data + package = Package.objects.filter(name=package_name) + response = PackageSerializer(package, many=True).data return Response(response) From 0317f8abd6d0a5c199ac529ccdd4432a6e9255b1 Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 12 Sep 2017 10:02:19 +0530 Subject: [PATCH 17/29] Move tests to vulncode_app/tests #8 Signed-off-by: kartik sibal --- app/vulncode_app/{ => tests}/test_api.py | 0 app/vulncode_app/{ => tests}/test_data_dump.py | 0 app/vulncode_app/{ => tests}/test_models.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename app/vulncode_app/{ => tests}/test_api.py (100%) rename app/vulncode_app/{ => tests}/test_data_dump.py (100%) rename app/vulncode_app/{ => tests}/test_models.py (100%) diff --git a/app/vulncode_app/test_api.py b/app/vulncode_app/tests/test_api.py similarity index 100% rename from app/vulncode_app/test_api.py rename to app/vulncode_app/tests/test_api.py diff --git a/app/vulncode_app/test_data_dump.py b/app/vulncode_app/tests/test_data_dump.py similarity index 100% rename from app/vulncode_app/test_data_dump.py rename to app/vulncode_app/tests/test_data_dump.py diff --git a/app/vulncode_app/test_models.py b/app/vulncode_app/tests/test_models.py similarity index 100% rename from app/vulncode_app/test_models.py rename to app/vulncode_app/tests/test_models.py From d533162869348313d156ddb89c1a3be077a5371b Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 12 Sep 2017 10:03:27 +0530 Subject: [PATCH 18/29] Re-order serializers according to models #8 Signed-off-by: kartik sibal --- app/vulncode_app/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/vulncode_app/serializers.py b/app/vulncode_app/serializers.py index 6ffedd948..4eb8ad4da 100644 --- a/app/vulncode_app/serializers.py +++ b/app/vulncode_app/serializers.py @@ -33,13 +33,13 @@ class PackageReferenceSerializer(serializers.ModelSerializer): class Meta: model = PackageReference - fields = ('name', 'version', 'repository', 'platform') + fields = ('repository', 'platform', 'name', 'version') class VulnerabilityReferenceSerializer(serializers.ModelSerializer): class Meta: model = VulnerabilityReference - fields = ('reference_id', 'source', 'url') + fields = ('source', 'reference_id', 'url') class VulnerabilitySerializer(serializers.ModelSerializer): From 5469f990a91a6aac893b83a209fe878527f4e678 Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 12 Sep 2017 10:04:26 +0530 Subject: [PATCH 19/29] change API url to api/ from data/ #8 Signed-off-by: kartik sibal --- app/vulncode_app/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/vulncode_app/urls.py b/app/vulncode_app/urls.py index 4bb461af7..3fc04b6d3 100644 --- a/app/vulncode_app/urls.py +++ b/app/vulncode_app/urls.py @@ -35,7 +35,7 @@ url(r'^cve-search/(?P[a-z]+)', views.package, name='package'), - url(r'^data/(?P[a-z]+)', + url(r'^api/(?P[a-z]+)', views.VulnerabilityData.as_view()), ] From 0bf63542bbfa8369fa95d014b5d14322954c019a Mon Sep 17 00:00:00 2001 From: Kartik Sibal Date: Tue, 12 Sep 2017 10:12:36 +0530 Subject: [PATCH 20/29] Update travis according to new dir structure Signed-off-by: kartik sibal --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e358372d2..dd8866696 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ before_script: script: - python3.6 -m pytest -v tests/ - - python3.6 manage.py test + - python3.6 manage.py test vulncode_app/tests notifications: email: false From 9b277d6e621b57e1e46157ac5d4c3282f75960c4 Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 12 Sep 2017 10:15:16 +0530 Subject: [PATCH 21/29] Update tests according to changed api url #8 Signed-off-by: kartik sibal --- app/vulncode_app/tests/test_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/vulncode_app/tests/test_api.py b/app/vulncode_app/tests/test_api.py index e8bea84a8..471b53bfd 100644 --- a/app/vulncode_app/tests/test_api.py +++ b/app/vulncode_app/tests/test_api.py @@ -40,7 +40,7 @@ def test_debian_response(self): extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) - response = self.client.get('/vulncode_app/data/mimetex', format='json') + response = self.client.get('/vulncode_app/api/mimetex', format='json') expected = [{ "name": "mimetex", @@ -78,7 +78,7 @@ def test_ubuntu_response(self): extract_data = ubuntu.extract_cves(test_data) ubuntu_dump(extract_data) - response = self.client.get('/vulncode_app/data/automake', format='json') + response = self.client.get('/vulncode_app/api/automake', format='json') expected = [{ "name": "automake", @@ -98,8 +98,8 @@ def test_ubuntu_response(self): self.assertEqual(expected, response.data) def test_blank_response(self): - response_invalid = self.client.get('/vulncode_app/data/', format='json') - response_blank = self.client.get('/vulncode_app/data/abbpcc', format='json') + response_invalid = self.client.get('/vulncode_app/api/', format='json') + response_blank = self.client.get('/vulncode_app/api/abbpcc', format='json') self.assertEqual(404, response_invalid.status_code) self.assertEqual([], response_blank.data) From 16ea31ad22b32eeb577690c31774a3da0c8d7274 Mon Sep 17 00:00:00 2001 From: Kartik Sibal Date: Tue, 12 Sep 2017 22:26:56 +0530 Subject: [PATCH 22/29] Update readme for API instructions #8 Signed-off-by: Kartik sibal --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f7deaa486..76c8d9903 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ python3.6 -m pytest -v tests/ For Django based tests ``` cd app/ -./manage.py test +./manage.py test vulncode_app/tests ``` Scrape and save to the database @@ -54,3 +54,17 @@ ubuntu_cves = ubuntu.scrape_cves() debian_dump(debian_vulnerabilities) ubuntu_dump(ubuntu_cves) ``` + +API +---- +Start the server + +``` +cd app/ +./manage.py runserver +``` + +In your browser use +``` +localhost:8000/vulncode_app/api/ +``` From 86c359a3e579d1cb1c643ca4a65df9d6160fb6e5 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 20 Sep 2017 19:51:14 -0300 Subject: [PATCH 23/29] Refactor the codebase structure - Following Django best-practices - Simplify the layout - Use proper naming convention Signed-off-by: Thomas Druez --- .travis.yml | 14 +++------- README.md | 27 +++++++++---------- .../migrations/0002_auto_20170907_2111.py | 19 ------------- app/manage.py => manage.py | 2 +- requirements.txt | 11 +++++--- {app/app => vulnerabilities}/__init__.py | 0 .../api_data.py | 0 {app/vulncode_app => vulnerabilities}/apps.py | 4 +-- .../data_dump.py | 8 +++--- .../migrations/0001_initial.py | 16 +++++------ .../migrations}/__init__.py | 0 .../models.py | 0 {app => vulnerabilities}/scraper/debian.py | 0 {app => vulnerabilities}/scraper/ubuntu.py | 0 .../serializers.py | 10 +++---- .../tests/test_api.py | 20 +++++++------- .../tests/test_api_data.py | 2 +- .../tests/test_data/debian.json | 0 .../tests/test_data/ubuntu_main.html | 0 .../tests/test_data_dump.py | 14 +++++----- .../tests/test_models.py | 8 +++--- .../tests/test_scrapers.py | 4 +-- {app/vulncode_app => vulnerabilities}/urls.py | 2 +- .../vulncode_app => vulnerabilities}/views.py | 6 ++--- .../migrations => vulnerablecode}/__init__.py | 0 {app/app => vulnerablecode}/settings.py | 7 ++--- {app/app => vulnerablecode}/urls.py | 2 +- {app/app => vulnerablecode}/wsgi.py | 2 +- 28 files changed, 77 insertions(+), 101 deletions(-) delete mode 100644 app/vulncode_app/migrations/0002_auto_20170907_2111.py rename app/manage.py => manage.py (90%) rename {app/app => vulnerabilities}/__init__.py (100%) rename {app/vulncode_app => vulnerabilities}/api_data.py (100%) rename {app/vulncode_app => vulnerabilities}/apps.py (58%) rename {app/vulncode_app => vulnerabilities}/data_dump.py (92%) rename {app/vulncode_app => vulnerabilities}/migrations/0001_initial.py (90%) rename {app/vulncode_app => vulnerabilities/migrations}/__init__.py (100%) rename {app/vulncode_app => vulnerabilities}/models.py (100%) rename {app => vulnerabilities}/scraper/debian.py (100%) rename {app => vulnerabilities}/scraper/ubuntu.py (100%) rename {app/vulncode_app => vulnerabilities}/serializers.py (90%) rename {app/vulncode_app => vulnerabilities}/tests/test_api.py (89%) rename {app => vulnerabilities}/tests/test_api_data.py (98%) rename {app => vulnerabilities}/tests/test_data/debian.json (100%) rename {app => vulnerabilities}/tests/test_data/ubuntu_main.html (100%) rename {app/vulncode_app => vulnerabilities}/tests/test_data_dump.py (90%) rename {app/vulncode_app => vulnerabilities}/tests/test_models.py (94%) rename {app => vulnerabilities}/tests/test_scrapers.py (97%) rename {app/vulncode_app => vulnerabilities}/urls.py (98%) rename {app/vulncode_app => vulnerabilities}/views.py (94%) rename {app/vulncode_app/migrations => vulnerablecode}/__init__.py (100%) rename {app/app => vulnerablecode}/settings.py (95%) rename {app/app => vulnerablecode}/urls.py (91%) rename {app/app => vulnerablecode}/wsgi.py (81%) diff --git a/.travis.yml b/.travis.yml index dd8866696..325f3a433 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,22 +3,14 @@ python: 3.6 install: - pip install -r requirements.txt - - pip install pycodestyle before_script: - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . - - cd app/ - - python3 manage.py migrate + - ./manage.py migrate script: - - python3.6 -m pytest -v tests/ - - python3.6 manage.py test vulncode_app/tests + - python3.6 -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py + - ./manage.py test vulnerabilities/tests notifications: email: false - webhooks: - urls: - - https://webhooks.gitter.im/e/b119fa557626081e1f36 - on_success: change # options: [always|never|change] default: always - on_failure: always # options: [always|never|change] default: always - on_start: never # options: [always|never|change] default: always diff --git a/README.md b/README.md index 76c8d9903..3998d21be 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Activate a virtualenv, install dependencies, and run the database migrations: python3.6 -m venv . source bin/activate pip install -r requirements.txt -app/manage.py migrate +./manage.py migrate ``` Tests @@ -26,45 +26,44 @@ Tests ``` pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . -cd app/ -python3.6 -m pytest -v tests/ +python3.6 -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py ``` For Django based tests ``` -cd app/ -./manage.py test vulncode_app/tests +./manage.py test vulnerabilities/tests ``` Scrape and save to the database ------------------------------- ``` -cd app/ ./manage.py shell ``` ``` -from scraper import debian, ubuntu -from vulncode_app.data_dump import debian_dump, ubuntu_dump +from vulnerabilities.scraper import debian, ubuntu +from vulnerabilities.data_dump import debian_dump, ubuntu_dump + +# May be needed on macOS +# import ssl; ssl._create_default_https_context = ssl._create_unverified_context -debian_vulnerabilities = debian.scrape_vulnerabilities() ubuntu_cves = ubuntu.scrape_cves() +ubuntu_dump(ubuntu_cves) +debian_vulnerabilities = debian.scrape_vulnerabilities() debian_dump(debian_vulnerabilities) -ubuntu_dump(ubuntu_cves) ``` API ---- -Start the server +Start the webserver ``` -cd app/ ./manage.py runserver ``` -In your browser use +In your browser access: ``` -localhost:8000/vulncode_app/api/ +http://127.0.0.1:8000/vulnerabilities/api/ ``` diff --git a/app/vulncode_app/migrations/0002_auto_20170907_2111.py b/app/vulncode_app/migrations/0002_auto_20170907_2111.py deleted file mode 100644 index 36d648389..000000000 --- a/app/vulncode_app/migrations/0002_auto_20170907_2111.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.4 on 2017-09-07 21:11 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('vulncode_app', '0001_initial'), - ] - - operations = [ - migrations.AlterUniqueTogether( - name='vulnerabilityreference', - unique_together=set([('vulnerability', 'source', 'reference_id', 'url')]), - ), - ] diff --git a/app/manage.py b/manage.py similarity index 90% rename from app/manage.py rename to manage.py index 7525f8727..e96501b79 100755 --- a/app/manage.py +++ b/manage.py @@ -3,7 +3,7 @@ import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "vulnerablecode.settings") try: from django.core.management import execute_from_command_line except ImportError: diff --git a/requirements.txt b/requirements.txt index e18b17772..048ff222b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,10 @@ +setuptools==36.5.0 + beautifulsoup4==4.6.0 -lxml==3.8.0 -django==1.11.4 -djangorestframework==3.6 +lxml==4.0.0 +django==1.11.5 +djangorestframework==3.6.4 # Tests -pytest==3.1.3 +pytest==3.2.2 +pycodestyle==2.3.1 diff --git a/app/app/__init__.py b/vulnerabilities/__init__.py similarity index 100% rename from app/app/__init__.py rename to vulnerabilities/__init__.py diff --git a/app/vulncode_app/api_data.py b/vulnerabilities/api_data.py similarity index 100% rename from app/vulncode_app/api_data.py rename to vulnerabilities/api_data.py diff --git a/app/vulncode_app/apps.py b/vulnerabilities/apps.py similarity index 58% rename from app/vulncode_app/apps.py rename to vulnerabilities/apps.py index 66e79cbd6..8decd30f6 100644 --- a/app/vulncode_app/apps.py +++ b/vulnerabilities/apps.py @@ -4,5 +4,5 @@ from django.apps import AppConfig -class VulncodeAppConfig(AppConfig): - name = 'vulncode_app' +class VulnerabilitiesAppConfig(AppConfig): + name = 'vulnerabilities' diff --git a/app/vulncode_app/data_dump.py b/vulnerabilities/data_dump.py similarity index 92% rename from app/vulncode_app/data_dump.py rename to vulnerabilities/data_dump.py index d309e428d..45d94515e 100644 --- a/app/vulncode_app/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -21,10 +21,10 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -from vulncode_app.models import ImpactedPackage -from vulncode_app.models import Package -from vulncode_app.models import Vulnerability -from vulncode_app.models import VulnerabilityReference +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference def debian_dump(extract_data): diff --git a/app/vulncode_app/migrations/0001_initial.py b/vulnerabilities/migrations/0001_initial.py similarity index 90% rename from app/vulncode_app/migrations/0001_initial.py rename to vulnerabilities/migrations/0001_initial.py index fb4ef5f0f..43b2635cf 100644 --- a/app/vulncode_app/migrations/0001_initial.py +++ b/vulnerabilities/migrations/0001_initial.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.2 on 2017-08-24 08:31 +# Generated by Django 1.11.4 on 2017-09-20 22:34 from __future__ import unicode_literals from django.db import migrations, models @@ -37,14 +37,14 @@ class Migration(migrations.Migration): ('platform', models.CharField(blank=True, help_text='Platform eg:maven', max_length=50)), ('name', models.CharField(blank=True, help_text='Package reference name eg:org.apache.commons.io', max_length=50)), ('version', models.CharField(blank=True, help_text='Reference version', max_length=50)), - ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package')), + ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package')), ], ), migrations.CreateModel( name='ResolvedPackage', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package')), + ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package')), ], ), migrations.CreateModel( @@ -62,27 +62,27 @@ class Migration(migrations.Migration): ('source', models.CharField(blank=True, help_text='Source(s) name eg:NVD', max_length=50)), ('reference_id', models.CharField(blank=True, help_text='Reference ID, eg:CVE-ID', max_length=50)), ('url', models.URLField(blank=True, help_text='URL of Vulnerability data', max_length=1024)), - ('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability')), + ('vulnerability', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability')), ], ), migrations.AddField( model_name='resolvedpackage', name='vulnerability', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability'), ), migrations.AddField( model_name='impactedpackage', name='package', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Package'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Package'), ), migrations.AddField( model_name='impactedpackage', name='vulnerability', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulncode_app.Vulnerability'), + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='vulnerabilities.Vulnerability'), ), migrations.AlterUniqueTogether( name='vulnerabilityreference', - unique_together=set([('vulnerability', 'source', 'reference_id')]), + unique_together=set([('vulnerability', 'source', 'reference_id', 'url')]), ), migrations.AlterUniqueTogether( name='impactedpackage', diff --git a/app/vulncode_app/__init__.py b/vulnerabilities/migrations/__init__.py similarity index 100% rename from app/vulncode_app/__init__.py rename to vulnerabilities/migrations/__init__.py diff --git a/app/vulncode_app/models.py b/vulnerabilities/models.py similarity index 100% rename from app/vulncode_app/models.py rename to vulnerabilities/models.py diff --git a/app/scraper/debian.py b/vulnerabilities/scraper/debian.py similarity index 100% rename from app/scraper/debian.py rename to vulnerabilities/scraper/debian.py diff --git a/app/scraper/ubuntu.py b/vulnerabilities/scraper/ubuntu.py similarity index 100% rename from app/scraper/ubuntu.py rename to vulnerabilities/scraper/ubuntu.py diff --git a/app/vulncode_app/serializers.py b/vulnerabilities/serializers.py similarity index 90% rename from app/vulncode_app/serializers.py rename to vulnerabilities/serializers.py index 4eb8ad4da..144ce363e 100644 --- a/app/vulncode_app/serializers.py +++ b/vulnerabilities/serializers.py @@ -23,11 +23,11 @@ from rest_framework import serializers -from vulncode_app.models import ImpactedPackage -from vulncode_app.models import Package -from vulncode_app.models import PackageReference -from vulncode_app.models import Vulnerability -from vulncode_app.models import VulnerabilityReference +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import Package +from vulnerabilities.models import PackageReference +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference class PackageReferenceSerializer(serializers.ModelSerializer): diff --git a/app/vulncode_app/tests/test_api.py b/vulnerabilities/tests/test_api.py similarity index 89% rename from app/vulncode_app/tests/test_api.py rename to vulnerabilities/tests/test_api.py index 471b53bfd..c4686f6b4 100644 --- a/app/vulncode_app/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -25,12 +25,12 @@ from django.test import TestCase -from vulncode_app.models import Package -from vulncode_app.serializers import PackageSerializer -from vulncode_app.data_dump import debian_dump -from vulncode_app.data_dump import ubuntu_dump -from scraper import debian -from scraper import ubuntu +from vulnerabilities.models import Package +from vulnerabilities.serializers import PackageSerializer +from vulnerabilities.data_dump import debian_dump +from vulnerabilities.data_dump import ubuntu_dump +from vulnerabilities.scraper import debian +from vulnerabilities.scraper import ubuntu class TestResponse(TestCase): @@ -40,7 +40,7 @@ def test_debian_response(self): extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) - response = self.client.get('/vulncode_app/api/mimetex', format='json') + response = self.client.get('/vulnerabilities/api/mimetex', format='json') expected = [{ "name": "mimetex", @@ -78,7 +78,7 @@ def test_ubuntu_response(self): extract_data = ubuntu.extract_cves(test_data) ubuntu_dump(extract_data) - response = self.client.get('/vulncode_app/api/automake', format='json') + response = self.client.get('/vulnerabilities/api/automake', format='json') expected = [{ "name": "automake", @@ -98,8 +98,8 @@ def test_ubuntu_response(self): self.assertEqual(expected, response.data) def test_blank_response(self): - response_invalid = self.client.get('/vulncode_app/api/', format='json') - response_blank = self.client.get('/vulncode_app/api/abbpcc', format='json') + response_invalid = self.client.get('/vulnerabilities/api/', format='json') + response_blank = self.client.get('/vulnerabilities/api/abbpcc', format='json') self.assertEqual(404, response_invalid.status_code) self.assertEqual([], response_blank.data) diff --git a/app/tests/test_api_data.py b/vulnerabilities/tests/test_api_data.py similarity index 98% rename from app/tests/test_api_data.py rename to vulnerabilities/tests/test_api_data.py index 9f1a4bc54..78d2cee7a 100644 --- a/app/tests/test_api_data.py +++ b/vulnerabilities/tests/test_api_data.py @@ -23,7 +23,7 @@ import json -from vulncode_app.api_data import extract_fields +from vulnerabilities.api_data import extract_fields test_data = """ diff --git a/app/tests/test_data/debian.json b/vulnerabilities/tests/test_data/debian.json similarity index 100% rename from app/tests/test_data/debian.json rename to vulnerabilities/tests/test_data/debian.json diff --git a/app/tests/test_data/ubuntu_main.html b/vulnerabilities/tests/test_data/ubuntu_main.html similarity index 100% rename from app/tests/test_data/ubuntu_main.html rename to vulnerabilities/tests/test_data/ubuntu_main.html diff --git a/app/vulncode_app/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py similarity index 90% rename from app/vulncode_app/tests/test_data_dump.py rename to vulnerabilities/tests/test_data_dump.py index cb359ece4..8e3ab2299 100644 --- a/app/vulncode_app/tests/test_data_dump.py +++ b/vulnerabilities/tests/test_data_dump.py @@ -25,13 +25,13 @@ from django.test import TestCase -from vulncode_app.models import Vulnerability -from vulncode_app.models import VulnerabilityReference -from vulncode_app.models import Package -from vulncode_app.data_dump import debian_dump -from vulncode_app.data_dump import ubuntu_dump -from scraper import debian -from scraper import ubuntu +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import Package +from vulnerabilities.data_dump import debian_dump +from vulnerabilities.data_dump import ubuntu_dump +from vulnerabilities.scraper import debian +from vulnerabilities.scraper import ubuntu class TestDataDump(TestCase): diff --git a/app/vulncode_app/tests/test_models.py b/vulnerabilities/tests/test_models.py similarity index 94% rename from app/vulncode_app/tests/test_models.py rename to vulnerabilities/tests/test_models.py index 2f5fe0cac..69581c2a7 100644 --- a/app/vulncode_app/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -23,10 +23,10 @@ from django.test import TestCase -from vulncode_app.models import Vulnerability -from vulncode_app.models import VulnerabilityReference -from vulncode_app.models import Package -from vulncode_app.models import PackageReference +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import Package +from vulnerabilities.models import PackageReference class TestVulnerability(TestCase): diff --git a/app/tests/test_scrapers.py b/vulnerabilities/tests/test_scrapers.py similarity index 97% rename from app/tests/test_scrapers.py rename to vulnerabilities/tests/test_scrapers.py index 13cf4e96b..49f669909 100644 --- a/app/tests/test_scrapers.py +++ b/vulnerabilities/tests/test_scrapers.py @@ -25,8 +25,8 @@ from os.path import dirname from os.path import join -from scraper import debian -from scraper import ubuntu +from vulnerabilities.scraper import debian +from vulnerabilities.scraper import ubuntu def test_ubuntu_extract_cves(): diff --git a/app/vulncode_app/urls.py b/vulnerabilities/urls.py similarity index 98% rename from app/vulncode_app/urls.py rename to vulnerabilities/urls.py index 3fc04b6d3..4f28487a0 100644 --- a/app/vulncode_app/urls.py +++ b/vulnerabilities/urls.py @@ -25,7 +25,7 @@ from rest_framework.urlpatterns import format_suffix_patterns -from vulncode_app import views +from vulnerabilities import views urlpatterns = [ diff --git a/app/vulncode_app/views.py b/vulnerabilities/views.py similarity index 94% rename from app/vulncode_app/views.py rename to vulnerabilities/views.py index a286b5678..14b561ede 100644 --- a/app/vulncode_app/views.py +++ b/vulnerabilities/views.py @@ -28,9 +28,9 @@ from rest_framework.views import APIView from rest_framework.response import Response -from vulncode_app.models import Package -from vulncode_app.serializers import PackageSerializer -from vulncode_app import api_data +from vulnerabilities.models import Package +from vulnerabilities.serializers import PackageSerializer +from vulnerabilities import api_data class VulnerabilityData(APIView): diff --git a/app/vulncode_app/migrations/__init__.py b/vulnerablecode/__init__.py similarity index 100% rename from app/vulncode_app/migrations/__init__.py rename to vulnerablecode/__init__.py diff --git a/app/app/settings.py b/vulnerablecode/settings.py similarity index 95% rename from app/app/settings.py rename to vulnerablecode/settings.py index 157066daa..d541733c0 100644 --- a/app/app/settings.py +++ b/vulnerablecode/settings.py @@ -31,13 +31,14 @@ # Application definition INSTALLED_APPS = [ - 'vulncode_app.apps.VulncodeAppConfig', 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'vulnerabilities', + 'rest_framework', ] MIDDLEWARE = [ @@ -50,7 +51,7 @@ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -ROOT_URLCONF = 'app.urls' +ROOT_URLCONF = 'vulnerabilities.urls' TEMPLATES = [ { @@ -68,7 +69,7 @@ }, ] -WSGI_APPLICATION = 'app.wsgi.application' +WSGI_APPLICATION = 'vulnerablecode.wsgi.application' # Database diff --git a/app/app/urls.py b/vulnerablecode/urls.py similarity index 91% rename from app/app/urls.py rename to vulnerablecode/urls.py index 37fd405a7..0c4e0501c 100644 --- a/app/app/urls.py +++ b/vulnerablecode/urls.py @@ -17,5 +17,5 @@ from django.conf.urls import url, include urlpatterns = [ - url(r'^vulncode_app/', include('vulncode_app.urls')), + url(r'^vulnerabilities/', include('vulnerabilities.urls')), ] diff --git a/app/app/wsgi.py b/vulnerablecode/wsgi.py similarity index 81% rename from app/app/wsgi.py rename to vulnerablecode/wsgi.py index cefd1ede9..2ac78d339 100644 --- a/app/app/wsgi.py +++ b/vulnerablecode/wsgi.py @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "vunerablecode.settings") application = get_wsgi_application() From b0168779d70001f20bc9221b2dca58b61c201002 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 20 Sep 2017 21:34:50 -0300 Subject: [PATCH 24/29] Fix failing tests following codebase structure changes Signed-off-by: Thomas Druez --- vulnerabilities/tests/test_api.py | 19 ++++++++++++------- vulnerabilities/tests/test_data_dump.py | 9 +++++++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index c4686f6b4..e89967513 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -22,6 +22,7 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import json +import os from django.test import TestCase @@ -33,14 +34,18 @@ from vulnerabilities.scraper import ubuntu +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, 'test_data/') + + class TestResponse(TestCase): def test_debian_response(self): - with open('tests/test_data/debian.json') as f: + with open(os.path.join(TEST_DATA, 'debian.json')) as f: test_data = json.loads(f.read()) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) - response = self.client.get('/vulnerabilities/api/mimetex', format='json') + response = self.client.get('/api/mimetex', format='json') expected = [{ "name": "mimetex", @@ -73,12 +78,12 @@ def test_debian_response(self): self.assertEqual(expected, response.data) def test_ubuntu_response(self): - with open('tests/test_data/ubuntu_main.html') as f: + with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f: test_data = f.read() extract_data = ubuntu.extract_cves(test_data) ubuntu_dump(extract_data) - response = self.client.get('/vulnerabilities/api/automake', format='json') + response = self.client.get('/api/automake', format='json') expected = [{ "name": "automake", @@ -98,8 +103,8 @@ def test_ubuntu_response(self): self.assertEqual(expected, response.data) def test_blank_response(self): - response_invalid = self.client.get('/vulnerabilities/api/', format='json') - response_blank = self.client.get('/vulnerabilities/api/abbpcc', format='json') + response_invalid = self.client.get('/api/', format='json') + response_blank = self.client.get('/api/abbpcc', format='json') self.assertEqual(404, response_invalid.status_code) self.assertEqual([], response_blank.data) @@ -107,7 +112,7 @@ def test_blank_response(self): class TestSerializers(TestCase): def test_serializers(self): - with open('tests/test_data/debian.json') as f: + with open(os.path.join(TEST_DATA, 'debian.json')) as f: test_data = json.loads(f.read()) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) diff --git a/vulnerabilities/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py index 8e3ab2299..d796a50d3 100644 --- a/vulnerabilities/tests/test_data_dump.py +++ b/vulnerabilities/tests/test_data_dump.py @@ -22,6 +22,7 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import json +import os from django.test import TestCase @@ -34,13 +35,17 @@ from vulnerabilities.scraper import ubuntu +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, 'test_data/') + + class TestDataDump(TestCase): def test_debian_data_dump(self): """ Scrape data from Debian' main tracker, save it in the database and verify entries. """ - with open('tests/test_data/debian.json') as f: + with open(os.path.join(TEST_DATA, 'debian.json')) as f: test_data = json.loads(f.read()) extract_data = debian.extract_vulnerabilities(test_data) @@ -71,7 +76,7 @@ def test_ubuntu_data_dump(self): Scrape data from Ubuntu' main tracker, save it in the database and verify entries. """ - with open('tests/test_data/ubuntu_main.html') as f: + with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f: test_data = f.read() data = ubuntu.extract_cves(test_data) From 23fc544e8f5fda1d653454b993b2b26cc49a007c Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 20 Sep 2017 21:41:11 -0300 Subject: [PATCH 25/29] Add back Gitter webhooks removed by mistake Signed-off-by: Thomas Druez --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 325f3a433..c545c0ebe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,3 +14,9 @@ script: notifications: email: false + webhooks: + urls: + - https://webhooks.gitter.im/e/b119fa557626081e1f36 + on_success: change # options: [always|never|change] default: always + on_failure: always # options: [always|never|change] default: always + on_start: never # options: [always|never|change] default: always From a860befc8ddcd83163c02cb4baed54a0ebd587ca Mon Sep 17 00:00:00 2001 From: Kartik Date: Fri, 22 Sep 2017 22:54:56 +0530 Subject: [PATCH 26/29] Update URL regex #8 Signed-off-by: Kartik sibal --- vulnerabilities/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/urls.py b/vulnerabilities/urls.py index 4f28487a0..5f90c3579 100644 --- a/vulnerabilities/urls.py +++ b/vulnerabilities/urls.py @@ -35,7 +35,7 @@ url(r'^cve-search/(?P[a-z]+)', views.package, name='package'), - url(r'^api/(?P[a-z]+)', + url(r'^api/(?P.+)', views.VulnerabilityData.as_view()), ] From c42a7753a518853ac2551cf055e471f6df4667e2 Mon Sep 17 00:00:00 2001 From: Kartik Date: Fri, 22 Sep 2017 22:55:58 +0530 Subject: [PATCH 27/29] Included platform in package serializer #8 Signed-off-by: kartik sibal --- vulnerabilities/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/serializers.py b/vulnerabilities/serializers.py index 144ce363e..1a2942d59 100644 --- a/vulnerabilities/serializers.py +++ b/vulnerabilities/serializers.py @@ -43,11 +43,11 @@ class Meta: class VulnerabilitySerializer(serializers.ModelSerializer): - reference = VulnerabilityReferenceSerializer(source='vulnerabilityreference_set', many=True) + references = VulnerabilityReferenceSerializer(source='vulnerabilityreference_set', many=True) class Meta: model = Vulnerability - fields = ('summary', 'reference') + fields = ('summary', 'references') class ImpactedPackageSerializer(serializers.ModelSerializer): @@ -63,4 +63,4 @@ class PackageSerializer(serializers.ModelSerializer): class Meta: model = Package - fields = ('name', 'version', 'vulnerabilities') + fields = ('name', 'version', 'platform', 'vulnerabilities') From 2d6d5eb8683ccdf360817684b8af8f1b7b2b219e Mon Sep 17 00:00:00 2001 From: Kartik Date: Fri, 22 Sep 2017 22:56:51 +0530 Subject: [PATCH 28/29] Update tests according to changes #8 Signed-off-by: kartik sibal --- vulnerabilities/tests/test_api.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index e89967513..5bd8a09d1 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -50,10 +50,11 @@ def test_debian_response(self): expected = [{ "name": "mimetex", "version": "1.50-1.1", + "platform":"", "vulnerabilities": [{ "vulnerability": { "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", - "reference": [{ + "references": [{ "reference_id": "CVE-2009-2458", "source": "", "url": "" @@ -63,10 +64,11 @@ def test_debian_response(self): }, { "name": "mimetex", "version": "1.50-1.1", + "platform":"", "vulnerabilities": [{ "vulnerability": { "summary": "Multiple unspecified vulnerabilities in mimeTeX.", - "reference": [{ + "references": [{ "reference_id": "CVE-2009-2459", "source": "", "url": "" @@ -88,10 +90,11 @@ def test_ubuntu_response(self): expected = [{ "name": "automake", "version": "", + "platform":"", "vulnerabilities": [{ "vulnerability": { "summary": "", - "reference": [{ + "references": [{ "reference_id": "CVE-2012-3386", "source": "", "url": "" @@ -124,12 +127,13 @@ def test_serializers(self): { "name": "mimetex", "version": "1.50-1.1", + "platform":"", "vulnerabilities": [ { "vulnerability": { "summary": "Multiple stack-based buffer overflows in mimetex.cgi in " "mimeTeX", - "reference": [ + "references": [ { "reference_id": "CVE-2009-2458", "source": "", @@ -143,11 +147,12 @@ def test_serializers(self): { "name": "mimetex", "version": "1.50-1.1", + "platform":"", "vulnerabilities": [ { "vulnerability": { "summary": "Multiple unspecified vulnerabilities in mimeTeX.", - "reference": [ + "references": [ { "reference_id": "CVE-2009-2459", "source": "", From 2c5f78da9ba59fa7cd179e86532518612efc5bc4 Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 22 Sep 2017 16:26:12 -0300 Subject: [PATCH 29/29] Add a `vulnerabilities` m2m field on the Package model Set the new 2m field on the PackageSerializer Signed-off-by: Thomas Druez --- .../0002_package_vulnerabilities.py | 20 ++++ vulnerabilities/models.py | 1 + vulnerabilities/serializers.py | 11 +-- vulnerabilities/tests/test_api.py | 97 ++++++++----------- 4 files changed, 61 insertions(+), 68 deletions(-) create mode 100644 vulnerabilities/migrations/0002_package_vulnerabilities.py diff --git a/vulnerabilities/migrations/0002_package_vulnerabilities.py b/vulnerabilities/migrations/0002_package_vulnerabilities.py new file mode 100644 index 000000000..595b8063e --- /dev/null +++ b/vulnerabilities/migrations/0002_package_vulnerabilities.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.5 on 2017-09-22 19:18 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='package', + name='vulnerabilities', + field=models.ManyToManyField(through='vulnerabilities.ImpactedPackage', to='vulnerabilities.Vulnerability'), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index f0f06cec4..97b0e6dd1 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -82,6 +82,7 @@ class Package(models.Model): platform = models.CharField(max_length=50, help_text='Package platform eg:maven', blank=True) name = models.CharField(max_length=50, help_text='Package name', blank=True) version = models.CharField(max_length=50, help_text='Package version', blank=True) + vulnerabilities = models.ManyToManyField(to='Vulnerability', through='ImpactedPackage') def __str__(self): return self.name diff --git a/vulnerabilities/serializers.py b/vulnerabilities/serializers.py index 1a2942d59..ec9a0540c 100644 --- a/vulnerabilities/serializers.py +++ b/vulnerabilities/serializers.py @@ -23,7 +23,6 @@ from rest_framework import serializers -from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import Package from vulnerabilities.models import PackageReference from vulnerabilities.models import Vulnerability @@ -50,16 +49,8 @@ class Meta: fields = ('summary', 'references') -class ImpactedPackageSerializer(serializers.ModelSerializer): - vulnerability = VulnerabilitySerializer() - - class Meta: - model = ImpactedPackage - fields = ('vulnerability',) - - class PackageSerializer(serializers.ModelSerializer): - vulnerabilities = ImpactedPackageSerializer(source='impactedpackage_set', many=True) + vulnerabilities = VulnerabilitySerializer(many=True) class Meta: model = Package diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 5bd8a09d1..0c35d3c6b 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -50,30 +50,26 @@ def test_debian_response(self): expected = [{ "name": "mimetex", "version": "1.50-1.1", - "platform":"", + "platform": "", "vulnerabilities": [{ - "vulnerability": { - "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", - "references": [{ - "reference_id": "CVE-2009-2458", - "source": "", - "url": "" - }] - } + "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", + "references": [{ + "reference_id": "CVE-2009-2458", + "source": "", + "url": "", + }] }] }, { "name": "mimetex", "version": "1.50-1.1", - "platform":"", + "platform": "", "vulnerabilities": [{ - "vulnerability": { - "summary": "Multiple unspecified vulnerabilities in mimeTeX.", - "references": [{ - "reference_id": "CVE-2009-2459", - "source": "", - "url": "" - }] - } + "summary": "Multiple unspecified vulnerabilities in mimeTeX.", + "references": [{ + "reference_id": "CVE-2009-2459", + "source": "", + "url": "", + }] }] }] @@ -90,16 +86,14 @@ def test_ubuntu_response(self): expected = [{ "name": "automake", "version": "", - "platform":"", + "platform": "", "vulnerabilities": [{ - "vulnerability": { - "summary": "", - "references": [{ - "reference_id": "CVE-2012-3386", - "source": "", - "url": "" - }] - } + "summary": "", + "references": [{ + "reference_id": "CVE-2012-3386", + "source": "", + "url": "", + }] }] }] @@ -127,41 +121,28 @@ def test_serializers(self): { "name": "mimetex", "version": "1.50-1.1", - "platform":"", - "vulnerabilities": [ - { - "vulnerability": { - "summary": "Multiple stack-based buffer overflows in mimetex.cgi in " - "mimeTeX", - "references": [ - { - "reference_id": "CVE-2009-2458", - "source": "", - "url": "" - } - ] - } - } - ] + "platform": "", + "vulnerabilities": [{ + "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", + "references": [{ + "reference_id": "CVE-2009-2458", + "source": "", + "url": "", + }] + }] }, { "name": "mimetex", "version": "1.50-1.1", - "platform":"", - "vulnerabilities": [ - { - "vulnerability": { - "summary": "Multiple unspecified vulnerabilities in mimeTeX.", - "references": [ - { - "reference_id": "CVE-2009-2459", - "source": "", - "url": "" - } - ] - } - } - ] + "platform": "", + "vulnerabilities": [{ + "summary": "Multiple unspecified vulnerabilities in mimeTeX.", + "references": [{ + "reference_id": "CVE-2009-2459", + "source": "", + "url": "", + }] + }] } ]