diff --git a/.travis.yml b/.travis.yml index de9364aef..2ea5c455c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: - SECRET_KEY="i1bn=oly)w*2yl-5yc&f!vvgt)p)fh3_2$r#spa!*sw36f5ov7" before_script: - - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . + - pycodestyle --exclude=migrations,settings.py,lib --max-line-length=100 . - psql -c "CREATE DATABASE vulnerablecode;" -U postgres - ./manage.py migrate diff --git a/README.md b/README.md index 1ac426ca6..f186a3395 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ the code Django includes for this purpose: `SECRET_KEY=$(python -c "from django. ## Tests ``` -pycodestyle --exclude=migrations,settings.py,venv,tests --max-line-length=100 . +pycodestyle --exclude=migrations,settings.py,venv --max-line-length=100 . python -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py ``` diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py index 561c91bb2..01a1727f7 100644 --- a/vulnerabilities/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -41,15 +41,36 @@ def debian_dump(extract_data): vulnerability=vulnerability, reference_id=data.get('vulnerability_id', ''), ) + + pkg_name = data.get('package_name', '') package = Package.objects.create( - name=data.get('package_name', ''), - version=data.get('fixed_version', ''), - ) - ImpactedPackage.objects.create( - vulnerability=vulnerability, - package=package + name=pkg_name, + version=data.get('version', ''), ) + if data['status'] == 'open': + ImpactedPackage.objects.create( + vulnerability=vulnerability, + package=package + ) + else: + ResolvedPackage.objects.create( + vulnerability=vulnerability, + package=package + ) + + fixed_version = data.get('fixed_version') + if fixed_version: + package = Package.objects.create( + name=pkg_name, + version=fixed_version, + ) + + ResolvedPackage.objects.create( + vulnerability=vulnerability, + package=package + ) + def ubuntu_dump(html): """ diff --git a/vulnerabilities/scraper/debian.py b/vulnerabilities/scraper/debian.py index 74a93fe58..fe84ffc4f 100644 --- a/vulnerabilities/scraper/debian.py +++ b/vulnerabilities/scraper/debian.py @@ -48,12 +48,20 @@ def extract_vulnerabilities(debian_data, base_release='jessie'): if not release: continue + status = release.get('status') + if status not in ('open', 'resolved'): + continue + + # the latest version of this package in base_release + version = release.get('repositories', {}).get(base_release, '') + package_vulnerabilities.append({ 'package_name': package_name, 'vulnerability_id': vulnerability, 'description': details.get('description', ''), - 'status': release.get('status', ''), + 'status': status, 'urgency': release.get('urgency', ''), + 'version': version, 'fixed_version': release.get('fixed_version', '') }) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 6b72f6bd0..4e3a76b9c 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -41,43 +41,20 @@ class TestResponse(TestCase): def test_debian_response(self): with open(os.path.join(TEST_DATA, 'debian.json')) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) - response = self.client.get('/api/packages/?name=mimetex', format='json') + response = self.client.get('/api/packages/?name=mimetex', format='json').data - expected = [{ - "name": "mimetex", - "version": "1.50-1.1", - "platform": "", - "vulnerabilities": [{ - "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", - "cvss": None, - "references": [{ - "reference_id": "CVE-2009-2458", - "source": "", - "url": "", - }] - }], - "references": [], - }, { - "name": "mimetex", - "version": "1.50-1.1", - "platform": "", - "vulnerabilities": [{ - "summary": "Multiple unspecified vulnerabilities in mimeTeX.", - "cvss": None, - "references": [{ - "reference_id": "CVE-2009-2459", - "source": "", - "url": "", - }] - }], - "references": [], - }] + self.assertEqual(4, response['count']) - self.assertEqual(expected, response.data.get('results')) + first_result = response['results'][0] + self.assertEqual('mimetex', first_result['name']) + + versions = {r['version'] for r in response['results']} + self.assertIn('1.50-1.1', versions) + self.assertIn('1.74-1', versions) def test_ubuntu_response(self): with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f: @@ -109,44 +86,18 @@ def test_ubuntu_response(self): class TestSerializers(TestCase): def test_serializers(self): with open(os.path.join(TEST_DATA, 'debian.json')) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) pk = Package.objects.filter(name="mimetex") response = PackageSerializer(pk, many=True).data - expected = [ - { - "name": "mimetex", - "version": "1.50-1.1", - "platform": "", - "vulnerabilities": [{ - "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", - "cvss": None, - "references": [{ - "reference_id": "CVE-2009-2458", - "source": "", - "url": "", - }] - }], - "references": [], - }, - { - "name": "mimetex", - "version": "1.50-1.1", - "platform": "", - "vulnerabilities": [{ - "summary": "Multiple unspecified vulnerabilities in mimeTeX.", - "cvss": None, - "references": [{ - "reference_id": "CVE-2009-2459", - "source": "", - "url": "", - }] - }], - "references": [], - } - ] - - self.assertEqual(expected, response) + self.assertEqual(4, len(response)) + + first_result = response[0] + self.assertEqual('mimetex', first_result['name']) + + versions = {r['version'] for r in response} + self.assertIn('1.50-1.1', versions) + self.assertIn('1.74-1', versions) diff --git a/vulnerabilities/tests/test_data/debian.json b/vulnerabilities/tests/test_data/debian.json index 843d43842..379a9a6f5 100644 --- a/vulnerabilities/tests/test_data/debian.json +++ b/vulnerabilities/tests/test_data/debian.json @@ -1,105 +1,150 @@ -{"mimetex": { - -"CVE-2009-2458": { - "scope": "remote", - "debianbug": 537254, - "description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", - "releases": - {"stretch": - {"status": "resolved", - "repositories": {"stretch": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "jessie": - {"status": "resolved", - "repositories": {"jessie": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "buster": - {"status": "resolved", - "repositories": {"buster": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "wheezy": - {"status": "resolved", - "repositories": {"wheezy": "1.73-2"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "sid": - {"status": "resolved", - "repositories": {"sid": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}}}, - -"CVE-2009-2459": - {"scope": "un-remote", - "debianbug": 537254, - "description": "Multiple unspecified vulnerabilities in mimeTeX.", - "releases": - {"stretch": - {"status": "resolved", - "repositories": {"stretch": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "jessie": - {"status": "not-resolved", - "repositories": {"jessie": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "buster": - {"status": "resolved", - "repositories": {"buster": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "wheezy": - {"status": "resolved", - "repositories": {"wheezy": "1.73-2"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "sid": - {"status": "resolved", - "repositories": {"sid": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}}}}, - -"git-repair": { - "TEMP-0807341-84E914": - {"debianbug": 807341, - "releases": - {"jessie": - {"status": "open", - "repositories": {"jessie": "1.20140914"}, - "urgency": "unimportant"}, - "sid": - {"status": "resolved", - "repositories": {"sid": "1.20151215-1"}, - "urgency": "unimportant", - "fixed_version": "1.20151215-1"}}}}, - -"sysvinit": { - "TEMP-0517018-A83CE6": - {"debianbug": 517018, - "releases": - {"stretch": - {"status": "open", - "repositories": {"stretch": "2.88dsf-59.9"}, - "urgency": "unimportant"}, - - "buster": - {"status": "open", - "repositories": {"buster": "2.88dsf-59.9"}, - "urgency": "unimportant"}, - - "wheezy": - {"status": "open", - "repositories": {"wheezy": "2.88dsf-41+deb7u1"}, - "urgency": "unimportant"}, - - "sid": - {"status": "open", - "repositories": {"sid": "2.88dsf-59.9"}, - "urgency": "unimportant"} - } - } - } - } \ No newline at end of file +{ + "librsync": { + "CVE-2014-8242": { + "scope": "remote", + "debianbug": 776246, + "description": "librsync before 1.0.0 uses a truncated MD4 checksum to match blocks", + "releases": { + "stretch": { + "status": "open", + "nodsa_reason": "", + "repositories": { + "stretch": "0.9.7-10" + }, + "nodsa": "Minor issue, too instrusive to backport", + "urgency": "low" + }, + "jessie": { + "status": "open", + "nodsa_reason": "", + "repositories": { + "jessie": "0.9.7-10" + }, + "nodsa": "Minor issue, too instrusive to backport", + "urgency": "low" + }, + "sid": { + "status": "resolved", + "repositories": { + "sid": "2.0.2-1" + }, + "fixed_version": "2.0.2-1", + "urgency": "low" + }, + "bullseye": { + "status": "resolved", + "repositories": { + "bullseye": "2.0.2-1" + }, + "fixed_version": "2.0.2-1", + "urgency": "low" + }, + "buster": { + "status": "open", + "nodsa_reason": "", + "repositories": { + "buster": "0.9.7-10" + }, + "nodsa": "Minor issue, too instrusive to backport", + "urgency": "low" + } + } + } + }, + "mimetex": { + "CVE-2009-1382": { + "scope": "remote", + "debianbug": 537254, + "description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", + "releases": { + "stretch": { + "status": "resolved", + "repositories": { + "stretch": "1.74-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "jessie": { + "status": "resolved", + "repositories": { + "jessie": "1.74-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "sid": { + "status": "resolved", + "repositories": { + "sid": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "bullseye": { + "status": "resolved", + "repositories": { + "bullseye": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "buster": { + "status": "resolved", + "repositories": { + "buster": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + } + } + }, + "CVE-2009-2459": { + "scope": "remote", + "debianbug": 537254, + "description": "Multiple unspecified vulnerabilities in mimeTeX", + "releases": { + "stretch": { + "status": "resolved", + "repositories": { + "stretch": "1.74-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "jessie": { + "status": "resolved", + "repositories": { + "jessie": "1.74-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "sid": { + "status": "resolved", + "repositories": { + "sid": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "bullseye": { + "status": "resolved", + "repositories": { + "bullseye": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "buster": { + "status": "resolved", + "repositories": { + "buster": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + } + } + } + } +} diff --git a/vulnerabilities/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py index d538ab88f..766ba65bc 100644 --- a/vulnerabilities/tests/test_data_dump.py +++ b/vulnerabilities/tests/test_data_dump.py @@ -44,76 +44,149 @@ TEST_DATA = os.path.join(BASE_DIR, 'test_data/') -class TestDataDump(TestCase): - def test_debian_data_dump(self): - """ - Scrape data from Debian' main tracker, save it - in the database and verify entries. - """ +class TestDebianDataDump(TestCase): + + @classmethod + def setUpTestData(self): with open(os.path.join(TEST_DATA, 'debian.json')) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) + def test_Vulnerability(self): + """ + Check that all vulnerabilities from the test data are stored in the database + """ self.assertEqual(3, Vulnerability.objects.count()) - self.assertEqual(3, VulnerabilityReference.objects.count()) - self.assertEqual(3, Package.objects.count()) self.assertTrue(Vulnerability.objects.get( summary='Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX')) self.assertTrue(Vulnerability.objects.get( - summary='Multiple unspecified vulnerabilities in mimeTeX.')) + summary='Multiple unspecified vulnerabilities in mimeTeX')) - self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2009-2458')) + self.assertTrue(Vulnerability.objects.get( + summary='librsync before 1.0.0 uses a truncated MD4 checksum \ +to match blocks')) + def test_VulnerabilityReference(self): + """ + Check that all vulnerability references from the test data are stored in the database + """ + self.assertEqual(3, VulnerabilityReference.objects.count()) + self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2009-1382')) self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2009-2459')) + self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2014-8242')) - self.assertTrue(VulnerabilityReference.objects.get(reference_id='TEMP-0807341-84E914')) + def test_Package(self): + """ + Check that all packages from the test data are stored in the database + """ + # There are five rows in Package because currently the models allow duplicates + # (see issue #28). + self.assertEqual(5, Package.objects.count()) - self.assertEqual(Package.objects.filter(name='mimetex')[0].name, 'mimetex') - self.assertTrue(Package.objects.get(name='git-repair')) - self.assertEqual(Package.objects.filter(version='1.50-1.1')[0].version, '1.50-1.1') + self.assertTrue(Package.objects.filter(name='mimetex')) + self.assertTrue(Package.objects.get(name='librsync')) - def test_ubuntu_data_dump(self): + def test_ImpactedPackage(self): """ - Scrape data from Ubuntu' main tracker, save it - in the database and verify entries. + Check that all impacted packages from the test data are stored in the database """ + impacted_pkgs = ImpactedPackage.objects.all() + impacted_pkg = impacted_pkgs[0] + + self.assertEqual(1, len(impacted_pkgs)) + self.assertEqual('librsync', impacted_pkg.package.name) + self.assertEqual('0.9.7-10', impacted_pkg.package.version) + + def test_ResolvedPackage(self): + """ + Check that all resolved packages from the test data are stored in the database + """ + resolved_pkgs = ResolvedPackage.objects.all() + resolved_pkg = resolved_pkgs[0] + versions = [rp.package.version for rp in resolved_pkgs] + + self.assertEqual(4, len(resolved_pkgs)) + self.assertEqual('mimetex', resolved_pkg.package.name) + self.assertIn('1.50-1.1', versions) + self.assertIn('1.74-1', versions) + + +class TestUbuntuDataDump(TestCase): + @classmethod + def setUpTestData(self): with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f: test_data = f.read() data = ubuntu.extract_cves(test_data) ubuntu_dump(data) + def test_data_dump(self): + """ + Check basic data import + """ reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0] self.assertEqual(reference.reference_id, 'CVE-2002-2439') self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6') - def test_archlinux_data_dump(self): - """ - Scrape data from Archlinux' main tracker, save it - in the database and verify entries. - """ + +class TestArchLinuxDataDump(TestCase): + + @classmethod + def setUpTestData(self): with open(os.path.join(TEST_DATA, 'archlinux.json')) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) archlinux_dump(test_data) + def test_Vulnerability(self): + """ + Check that all vulnerabilities from the test data are stored in the database + """ self.assertEqual(1, Vulnerability.objects.count()) + self.assertTrue(Vulnerability.objects.get(summary='multiple issues')) + + def test_VulnerabilityReference(self): + """ + Check that all vulnerability references from the test data are stored in the database + """ self.assertEqual(14, VulnerabilityReference.objects.count()) + self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2018-11360')) + self.assertTrue(VulnerabilityReference.objects.get(reference_id='ASA-201805-24')) + self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-708')) + + def test_Package(self): + """ + Check that all packages from the test data are stored in the database + """ self.assertEqual(8, Package.objects.count()) - self.assertEqual(8, PackageReference.objects.count()) - self.assertEqual(4, ImpactedPackage.objects.count()) - self.assertEqual(4, ResolvedPackage.objects.count()) + self.assertTrue(Package.objects.filter(name='wireshark-cli')) - self.assertTrue(Vulnerability.objects.get(summary='multiple issues')) + def test_PackageReference(self): + """ + Check that all package references from the test data are stored in the database + """ + self.assertEqual(8, PackageReference.objects.count()) - self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2018-11360')) + def test_ImpactedPackage(self): + """ + Check that all impacted packages from the test data are stored in the database + """ + impacted_pkgs = ImpactedPackage.objects.all() + impacted_pkg = impacted_pkgs[0] - self.assertTrue(VulnerabilityReference.objects.get(reference_id='ASA-201805-24')) + self.assertEqual(4, len(impacted_pkgs)) + self.assertEqual('2.6.0-1', impacted_pkg.package.version) - self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-708')) + def test_ResolvedPackage(self): + """ + Check that all resolved packages from the test data are stored in the database + """ + resolved_pkgs = ResolvedPackage.objects.all() + resolved_pkg = resolved_pkgs[0] - self.assertEqual(Package.objects.filter(name='wireshark-cli')[0].name, 'wireshark-cli') \ No newline at end of file + self.assertEqual(4, len(resolved_pkgs)) + self.assertEqual('2.6.1-1', resolved_pkg.package.version) diff --git a/vulnerabilities/tests/test_import_cli.py b/vulnerabilities/tests/test_import_cli.py index 06976eb6a..d055f7d9c 100644 --- a/vulnerabilities/tests/test_import_cli.py +++ b/vulnerabilities/tests/test_import_cli.py @@ -31,17 +31,17 @@ class ImportCommandTest(TestCase): def test_list_sources(self): buf = StringIO() - + call_command('import', '--list', stdout=buf) - + out = buf.getvalue() self.assertIn('debian', out) self.assertIn('ubuntu', out) self.assertIn('archlinux', out) def test_missing_sources(self): - with self.assertRaises(CommandError) as cm: - call_command('import', stdout=StringIO()) + with self.assertRaises(CommandError) as cm: + call_command('import', stdout=StringIO()) err = str(cm.exception) self.assertIn('Please provide at least one data source', err) diff --git a/vulnerabilities/tests/test_scrapers.py b/vulnerabilities/tests/test_scrapers.py index 49f669909..bb32f57b9 100644 --- a/vulnerabilities/tests/test_scrapers.py +++ b/vulnerabilities/tests/test_scrapers.py @@ -63,32 +63,35 @@ def test_debian_extract_vulnerabilities(): debian_test_file = join(dirname(__file__), 'test_data', 'debian.json') with open(debian_test_file) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) expected = [ { - 'fixed_version': '1.50-1.1', + 'package_name': 'librsync', + 'vulnerability_id': 'CVE-2014-8242', + 'description': 'librsync before 1.0.0 uses a truncated MD4 checksum to match blocks', + 'status': 'open', + 'urgency': 'low', + 'version': '0.9.7-10', + 'fixed_version': '' + }, + { 'package_name': 'mimetex', + 'vulnerability_id': 'CVE-2009-1382', + 'description': 'Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX', 'status': 'resolved', 'urgency': 'medium', - 'vulnerability_id': 'CVE-2009-2458', - 'description': 'Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX' + 'version': '1.74-1', + 'fixed_version': '1.50-1.1' }, { - 'fixed_version': '1.50-1.1', 'package_name': 'mimetex', - 'status': 'not-resolved', - 'urgency': 'medium', 'vulnerability_id': 'CVE-2009-2459', - 'description': 'Multiple unspecified vulnerabilities in mimeTeX.' - }, - { - 'package_name': 'git-repair', - 'vulnerability_id': 'TEMP-0807341-84E914', - 'description': '', - 'status': 'open', - 'urgency': 'unimportant', - 'fixed_version': '' + 'description': 'Multiple unspecified vulnerabilities in mimeTeX', + 'status': 'resolved', + 'urgency': 'medium', + 'version': '1.74-1', + 'fixed_version': '1.50-1.1' } ]