From 97d2491492800168627d29154f071205450f5463 Mon Sep 17 00:00:00 2001 From: Haiko Schol Date: Mon, 23 Sep 2019 16:21:04 +0100 Subject: [PATCH 1/5] Replace test data for Debian security tracker The previous fixture had some strange values like "not-resolved" for "status" and "un-remote" for "scope". It doesn't look like those values are used by the Debian security tracker. To address issue #38 we need records with "status" values "resolved" and "open". Signed-off-by: Haiko Schol --- vulnerabilities/tests/test_api.py | 8 +- vulnerabilities/tests/test_data/debian.json | 255 ++++++++++++-------- vulnerabilities/tests/test_data_dump.py | 17 +- vulnerabilities/tests/test_scrapers.py | 32 +-- 4 files changed, 176 insertions(+), 136 deletions(-) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 6b72f6bd0..eb582c9fc 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -55,7 +55,7 @@ def test_debian_response(self): "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", "cvss": None, "references": [{ - "reference_id": "CVE-2009-2458", + "reference_id": "CVE-2009-1382", "source": "", "url": "", }] @@ -66,7 +66,7 @@ def test_debian_response(self): "version": "1.50-1.1", "platform": "", "vulnerabilities": [{ - "summary": "Multiple unspecified vulnerabilities in mimeTeX.", + "summary": "Multiple unspecified vulnerabilities in mimeTeX", "cvss": None, "references": [{ "reference_id": "CVE-2009-2459", @@ -125,7 +125,7 @@ def test_serializers(self): "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", "cvss": None, "references": [{ - "reference_id": "CVE-2009-2458", + "reference_id": "CVE-2009-1382", "source": "", "url": "", }] @@ -137,7 +137,7 @@ def test_serializers(self): "version": "1.50-1.1", "platform": "", "vulnerabilities": [{ - "summary": "Multiple unspecified vulnerabilities in mimeTeX.", + "summary": "Multiple unspecified vulnerabilities in mimeTeX", "cvss": None, "references": [{ "reference_id": "CVE-2009-2459", diff --git a/vulnerabilities/tests/test_data/debian.json b/vulnerabilities/tests/test_data/debian.json index 843d43842..379a9a6f5 100644 --- a/vulnerabilities/tests/test_data/debian.json +++ b/vulnerabilities/tests/test_data/debian.json @@ -1,105 +1,150 @@ -{"mimetex": { - -"CVE-2009-2458": { - "scope": "remote", - "debianbug": 537254, - "description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", - "releases": - {"stretch": - {"status": "resolved", - "repositories": {"stretch": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "jessie": - {"status": "resolved", - "repositories": {"jessie": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "buster": - {"status": "resolved", - "repositories": {"buster": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "wheezy": - {"status": "resolved", - "repositories": {"wheezy": "1.73-2"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "sid": - {"status": "resolved", - "repositories": {"sid": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}}}, - -"CVE-2009-2459": - {"scope": "un-remote", - "debianbug": 537254, - "description": "Multiple unspecified vulnerabilities in mimeTeX.", - "releases": - {"stretch": - {"status": "resolved", - "repositories": {"stretch": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "jessie": - {"status": "not-resolved", - "repositories": {"jessie": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "buster": - {"status": "resolved", - "repositories": {"buster": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "wheezy": - {"status": "resolved", - "repositories": {"wheezy": "1.73-2"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}, - "sid": - {"status": "resolved", - "repositories": {"sid": "1.74-1"}, - "urgency": "medium", - "fixed_version": "1.50-1.1"}}}}, - -"git-repair": { - "TEMP-0807341-84E914": - {"debianbug": 807341, - "releases": - {"jessie": - {"status": "open", - "repositories": {"jessie": "1.20140914"}, - "urgency": "unimportant"}, - "sid": - {"status": "resolved", - "repositories": {"sid": "1.20151215-1"}, - "urgency": "unimportant", - "fixed_version": "1.20151215-1"}}}}, - -"sysvinit": { - "TEMP-0517018-A83CE6": - {"debianbug": 517018, - "releases": - {"stretch": - {"status": "open", - "repositories": {"stretch": "2.88dsf-59.9"}, - "urgency": "unimportant"}, - - "buster": - {"status": "open", - "repositories": {"buster": "2.88dsf-59.9"}, - "urgency": "unimportant"}, - - "wheezy": - {"status": "open", - "repositories": {"wheezy": "2.88dsf-41+deb7u1"}, - "urgency": "unimportant"}, - - "sid": - {"status": "open", - "repositories": {"sid": "2.88dsf-59.9"}, - "urgency": "unimportant"} - } - } - } - } \ No newline at end of file +{ + "librsync": { + "CVE-2014-8242": { + "scope": "remote", + "debianbug": 776246, + "description": "librsync before 1.0.0 uses a truncated MD4 checksum to match blocks", + "releases": { + "stretch": { + "status": "open", + "nodsa_reason": "", + "repositories": { + "stretch": "0.9.7-10" + }, + "nodsa": "Minor issue, too instrusive to backport", + "urgency": "low" + }, + "jessie": { + "status": "open", + "nodsa_reason": "", + "repositories": { + "jessie": "0.9.7-10" + }, + "nodsa": "Minor issue, too instrusive to backport", + "urgency": "low" + }, + "sid": { + "status": "resolved", + "repositories": { + "sid": "2.0.2-1" + }, + "fixed_version": "2.0.2-1", + "urgency": "low" + }, + "bullseye": { + "status": "resolved", + "repositories": { + "bullseye": "2.0.2-1" + }, + "fixed_version": "2.0.2-1", + "urgency": "low" + }, + "buster": { + "status": "open", + "nodsa_reason": "", + "repositories": { + "buster": "0.9.7-10" + }, + "nodsa": "Minor issue, too instrusive to backport", + "urgency": "low" + } + } + } + }, + "mimetex": { + "CVE-2009-1382": { + "scope": "remote", + "debianbug": 537254, + "description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", + "releases": { + "stretch": { + "status": "resolved", + "repositories": { + "stretch": "1.74-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "jessie": { + "status": "resolved", + "repositories": { + "jessie": "1.74-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "sid": { + "status": "resolved", + "repositories": { + "sid": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "bullseye": { + "status": "resolved", + "repositories": { + "bullseye": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "buster": { + "status": "resolved", + "repositories": { + "buster": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + } + } + }, + "CVE-2009-2459": { + "scope": "remote", + "debianbug": 537254, + "description": "Multiple unspecified vulnerabilities in mimeTeX", + "releases": { + "stretch": { + "status": "resolved", + "repositories": { + "stretch": "1.74-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "jessie": { + "status": "resolved", + "repositories": { + "jessie": "1.74-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "sid": { + "status": "resolved", + "repositories": { + "sid": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "bullseye": { + "status": "resolved", + "repositories": { + "bullseye": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + }, + "buster": { + "status": "resolved", + "repositories": { + "buster": "1.76-1" + }, + "fixed_version": "1.50-1.1", + "urgency": "medium" + } + } + } + } +} diff --git a/vulnerabilities/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py index d538ab88f..ef6f9a7de 100644 --- a/vulnerabilities/tests/test_data_dump.py +++ b/vulnerabilities/tests/test_data_dump.py @@ -51,7 +51,7 @@ def test_debian_data_dump(self): in the database and verify entries. """ with open(os.path.join(TEST_DATA, 'debian.json')) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) @@ -64,17 +64,12 @@ def test_debian_data_dump(self): summary='Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX')) self.assertTrue(Vulnerability.objects.get( - summary='Multiple unspecified vulnerabilities in mimeTeX.')) - - self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2009-2458')) + summary='Multiple unspecified vulnerabilities in mimeTeX')) + self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2009-1382')) self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2009-2459')) - - self.assertTrue(VulnerabilityReference.objects.get(reference_id='TEMP-0807341-84E914')) - - self.assertEqual(Package.objects.filter(name='mimetex')[0].name, 'mimetex') - self.assertTrue(Package.objects.get(name='git-repair')) - self.assertEqual(Package.objects.filter(version='1.50-1.1')[0].version, '1.50-1.1') + self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2014-8242')) + self.assertTrue(Package.objects.get(name='librsync')) def test_ubuntu_data_dump(self): """ @@ -116,4 +111,4 @@ def test_archlinux_data_dump(self): self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-708')) - self.assertEqual(Package.objects.filter(name='wireshark-cli')[0].name, 'wireshark-cli') \ No newline at end of file + self.assertEqual(Package.objects.filter(name='wireshark-cli')[0].name, 'wireshark-cli') diff --git a/vulnerabilities/tests/test_scrapers.py b/vulnerabilities/tests/test_scrapers.py index 49f669909..e2e2761ff 100644 --- a/vulnerabilities/tests/test_scrapers.py +++ b/vulnerabilities/tests/test_scrapers.py @@ -63,32 +63,32 @@ def test_debian_extract_vulnerabilities(): debian_test_file = join(dirname(__file__), 'test_data', 'debian.json') with open(debian_test_file) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) expected = [ { - 'fixed_version': '1.50-1.1', + 'package_name': 'librsync', + 'vulnerability_id': 'CVE-2014-8242', + 'description': 'librsync before 1.0.0 uses a truncated MD4 checksum to match blocks', + 'status': 'open', + 'urgency': 'low', + 'fixed_version': '' + }, + { 'package_name': 'mimetex', + 'vulnerability_id': 'CVE-2009-1382', + 'description': 'Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX', 'status': 'resolved', 'urgency': 'medium', - 'vulnerability_id': 'CVE-2009-2458', - 'description': 'Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX' + 'fixed_version': '1.50-1.1' }, { - 'fixed_version': '1.50-1.1', 'package_name': 'mimetex', - 'status': 'not-resolved', - 'urgency': 'medium', 'vulnerability_id': 'CVE-2009-2459', - 'description': 'Multiple unspecified vulnerabilities in mimeTeX.' - }, - { - 'package_name': 'git-repair', - 'vulnerability_id': 'TEMP-0807341-84E914', - 'description': '', - 'status': 'open', - 'urgency': 'unimportant', - 'fixed_version': '' + 'description': 'Multiple unspecified vulnerabilities in mimeTeX', + 'status': 'resolved', + 'urgency': 'medium', + 'fixed_version': '1.50-1.1' } ] From 0e119b0ec30f85081b0fbb5125848e8633156f80 Mon Sep 17 00:00:00 2001 From: Haiko Schol Date: Mon, 23 Sep 2019 17:30:24 +0100 Subject: [PATCH 2/5] Include "version" attribute in debian.extract_vulnerabilities() Before this change, only "fixed_version" was retained. This is the lowest version of the package that includes a fix for the given vulnerability. However in the meantime, newer versions of the package might have become available in the Debian release passed in as "base_release". Signed-off-by: Haiko Schol --- vulnerabilities/scraper/debian.py | 4 ++++ vulnerabilities/tests/test_scrapers.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/vulnerabilities/scraper/debian.py b/vulnerabilities/scraper/debian.py index 74a93fe58..c76d14cde 100644 --- a/vulnerabilities/scraper/debian.py +++ b/vulnerabilities/scraper/debian.py @@ -48,12 +48,16 @@ def extract_vulnerabilities(debian_data, base_release='jessie'): if not release: continue + # the latest version of this package in base_release + version = release.get('repositories', {}).get(base_release, '') + package_vulnerabilities.append({ 'package_name': package_name, 'vulnerability_id': vulnerability, 'description': details.get('description', ''), 'status': release.get('status', ''), 'urgency': release.get('urgency', ''), + 'version': version, 'fixed_version': release.get('fixed_version', '') }) diff --git a/vulnerabilities/tests/test_scrapers.py b/vulnerabilities/tests/test_scrapers.py index e2e2761ff..bb32f57b9 100644 --- a/vulnerabilities/tests/test_scrapers.py +++ b/vulnerabilities/tests/test_scrapers.py @@ -72,6 +72,7 @@ def test_debian_extract_vulnerabilities(): 'description': 'librsync before 1.0.0 uses a truncated MD4 checksum to match blocks', 'status': 'open', 'urgency': 'low', + 'version': '0.9.7-10', 'fixed_version': '' }, { @@ -80,6 +81,7 @@ def test_debian_extract_vulnerabilities(): 'description': 'Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX', 'status': 'resolved', 'urgency': 'medium', + 'version': '1.74-1', 'fixed_version': '1.50-1.1' }, { @@ -88,6 +90,7 @@ def test_debian_extract_vulnerabilities(): 'description': 'Multiple unspecified vulnerabilities in mimeTeX', 'status': 'resolved', 'urgency': 'medium', + 'version': '1.74-1', 'fixed_version': '1.50-1.1' } ] From ec9abb63b1109c64da4cb50c9b16560c0626da84 Mon Sep 17 00:00:00 2001 From: Haiko Schol Date: Tue, 24 Sep 2019 13:16:02 +0100 Subject: [PATCH 3/5] Split up data dump tests into separate classes and methods Signed-off-by: Haiko Schol --- vulnerabilities/tests/test_data_dump.py | 106 ++++++++++++++++++------ 1 file changed, 80 insertions(+), 26 deletions(-) diff --git a/vulnerabilities/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py index ef6f9a7de..987e164e9 100644 --- a/vulnerabilities/tests/test_data_dump.py +++ b/vulnerabilities/tests/test_data_dump.py @@ -44,21 +44,21 @@ TEST_DATA = os.path.join(BASE_DIR, 'test_data/') -class TestDataDump(TestCase): - def test_debian_data_dump(self): - """ - Scrape data from Debian' main tracker, save it - in the database and verify entries. - """ +class TestDebianDataDump(TestCase): + + @classmethod + def setUpTestData(self): with open(os.path.join(TEST_DATA, 'debian.json')) as f: test_data = json.load(f) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) + def test_Vulnerability(self): + """ + Check that all vulnerabilities from the test data are stored in the database + """ self.assertEqual(3, Vulnerability.objects.count()) - self.assertEqual(3, VulnerabilityReference.objects.count()) - self.assertEqual(3, Package.objects.count()) self.assertTrue(Vulnerability.objects.get( summary='Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX')) @@ -66,49 +66,103 @@ def test_debian_data_dump(self): self.assertTrue(Vulnerability.objects.get( summary='Multiple unspecified vulnerabilities in mimeTeX')) + self.assertTrue(Vulnerability.objects.get( + summary='librsync before 1.0.0 uses a truncated MD4 checksum \ +to match blocks')) + + def test_VulnerabilityReference(self): + """ + Check that all vulnerability references from the test data are stored in the database + """ + self.assertEqual(3, VulnerabilityReference.objects.count()) self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2009-1382')) self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2009-2459')) self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2014-8242')) - self.assertTrue(Package.objects.get(name='librsync')) - def test_ubuntu_data_dump(self): + def test_Package(self): """ - Scrape data from Ubuntu' main tracker, save it - in the database and verify entries. + Check that all packages from the test data are stored in the database """ + # There are three rows in Package because currently the models allow duplicates + # (see issue #28). + self.assertEqual(3, Package.objects.count()) + + self.assertTrue(Package.objects.filter(name='mimetex')) + self.assertTrue(Package.objects.get(name='librsync')) + + +class TestUbuntuDataDump(TestCase): + @classmethod + def setUpTestData(self): with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f: test_data = f.read() data = ubuntu.extract_cves(test_data) ubuntu_dump(data) + def test_data_dump(self): + """ + Check basic data import + """ reference = VulnerabilityReference.objects.filter(reference_id='CVE-2002-2439')[0] self.assertEqual(reference.reference_id, 'CVE-2002-2439') self.assertTrue(Package.objects.filter(name='gcc-4.6')[0].name, 'gcc-4.6') - def test_archlinux_data_dump(self): - """ - Scrape data from Archlinux' main tracker, save it - in the database and verify entries. - """ + +class TestArchLinuxDataDump(TestCase): + + @classmethod + def setUpTestData(self): with open(os.path.join(TEST_DATA, 'archlinux.json')) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) archlinux_dump(test_data) + def test_Vulnerability(self): + """ + Check that all vulnerabilities from the test data are stored in the database + """ self.assertEqual(1, Vulnerability.objects.count()) + self.assertTrue(Vulnerability.objects.get(summary='multiple issues')) + + def test_VulnerabilityReference(self): + """ + Check that all vulnerability references from the test data are stored in the database + """ self.assertEqual(14, VulnerabilityReference.objects.count()) + self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2018-11360')) + self.assertTrue(VulnerabilityReference.objects.get(reference_id='ASA-201805-24')) + self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-708')) + + def test_Package(self): + """ + Check that all packages from the test data are stored in the database + """ self.assertEqual(8, Package.objects.count()) - self.assertEqual(8, PackageReference.objects.count()) - self.assertEqual(4, ImpactedPackage.objects.count()) - self.assertEqual(4, ResolvedPackage.objects.count()) + self.assertTrue(Package.objects.filter(name='wireshark-cli')) - self.assertTrue(Vulnerability.objects.get(summary='multiple issues')) + def test_PackageReference(self): + """ + Check that all package references from the test data are stored in the database + """ + self.assertEqual(8, PackageReference.objects.count()) - self.assertTrue(VulnerabilityReference.objects.get(reference_id='CVE-2018-11360')) + def test_ImpactedPackage(self): + """ + Check that all impacted packages from the test data are stored in the database + """ + impacted_pkgs = ImpactedPackage.objects.all() + impacted_pkg = impacted_pkgs[0] - self.assertTrue(VulnerabilityReference.objects.get(reference_id='ASA-201805-24')) + self.assertEqual(4, len(impacted_pkgs)) + self.assertEqual('2.6.0-1', impacted_pkg.package.version) - self.assertTrue(VulnerabilityReference.objects.get(reference_id='AVG-708')) + def test_ResolvedPackage(self): + """ + Check that all resolved packages from the test data are stored in the database + """ + resolved_pkgs = ResolvedPackage.objects.all() + resolved_pkg = resolved_pkgs[0] - self.assertEqual(Package.objects.filter(name='wireshark-cli')[0].name, 'wireshark-cli') + self.assertEqual(4, len(resolved_pkgs)) + self.assertEqual('2.6.1-1', resolved_pkg.package.version) From de031114eaa1eae4a7b3467a895e4060fa135ebc Mon Sep 17 00:00:00 2001 From: Haiko Schol Date: Tue, 24 Sep 2019 13:19:00 +0100 Subject: [PATCH 4/5] Include tests in pycodestyle check Signed-off-by: Haiko Schol --- .travis.yml | 2 +- README.md | 2 +- vulnerabilities/tests/test_import_cli.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index de9364aef..2ea5c455c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ env: - SECRET_KEY="i1bn=oly)w*2yl-5yc&f!vvgt)p)fh3_2$r#spa!*sw36f5ov7" before_script: - - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . + - pycodestyle --exclude=migrations,settings.py,lib --max-line-length=100 . - psql -c "CREATE DATABASE vulnerablecode;" -U postgres - ./manage.py migrate diff --git a/README.md b/README.md index 1ac426ca6..f186a3395 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ the code Django includes for this purpose: `SECRET_KEY=$(python -c "from django. ## Tests ``` -pycodestyle --exclude=migrations,settings.py,venv,tests --max-line-length=100 . +pycodestyle --exclude=migrations,settings.py,venv --max-line-length=100 . python -m pytest -v vulnerabilities/tests/test_scrapers.py vulnerabilities/tests/test_api_data.py ``` diff --git a/vulnerabilities/tests/test_import_cli.py b/vulnerabilities/tests/test_import_cli.py index 06976eb6a..d055f7d9c 100644 --- a/vulnerabilities/tests/test_import_cli.py +++ b/vulnerabilities/tests/test_import_cli.py @@ -31,17 +31,17 @@ class ImportCommandTest(TestCase): def test_list_sources(self): buf = StringIO() - + call_command('import', '--list', stdout=buf) - + out = buf.getvalue() self.assertIn('debian', out) self.assertIn('ubuntu', out) self.assertIn('archlinux', out) def test_missing_sources(self): - with self.assertRaises(CommandError) as cm: - call_command('import', stdout=StringIO()) + with self.assertRaises(CommandError) as cm: + call_command('import', stdout=StringIO()) err = str(cm.exception) self.assertIn('Please provide at least one data source', err) From 905a899776fc5909b38b6ff174ce6f82cbcc3fc2 Mon Sep 17 00:00:00 2001 From: Haiko Schol Date: Tue, 24 Sep 2019 17:29:52 +0100 Subject: [PATCH 5/5] Consider "status" field in Debian data import Currently only two status values are recognized: "open" maps to ImpactedPackage "resolved" maps to ResolvedPackage Signed-off-by: Haiko Schol --- vulnerabilities/data_dump.py | 33 ++++++++-- vulnerabilities/scraper/debian.py | 6 +- vulnerabilities/tests/test_api.py | 85 ++++++------------------- vulnerabilities/tests/test_data_dump.py | 28 +++++++- 4 files changed, 76 insertions(+), 76 deletions(-) diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py index 561c91bb2..01a1727f7 100644 --- a/vulnerabilities/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -41,15 +41,36 @@ def debian_dump(extract_data): vulnerability=vulnerability, reference_id=data.get('vulnerability_id', ''), ) + + pkg_name = data.get('package_name', '') package = Package.objects.create( - name=data.get('package_name', ''), - version=data.get('fixed_version', ''), - ) - ImpactedPackage.objects.create( - vulnerability=vulnerability, - package=package + name=pkg_name, + version=data.get('version', ''), ) + if data['status'] == 'open': + ImpactedPackage.objects.create( + vulnerability=vulnerability, + package=package + ) + else: + ResolvedPackage.objects.create( + vulnerability=vulnerability, + package=package + ) + + fixed_version = data.get('fixed_version') + if fixed_version: + package = Package.objects.create( + name=pkg_name, + version=fixed_version, + ) + + ResolvedPackage.objects.create( + vulnerability=vulnerability, + package=package + ) + def ubuntu_dump(html): """ diff --git a/vulnerabilities/scraper/debian.py b/vulnerabilities/scraper/debian.py index c76d14cde..fe84ffc4f 100644 --- a/vulnerabilities/scraper/debian.py +++ b/vulnerabilities/scraper/debian.py @@ -48,6 +48,10 @@ def extract_vulnerabilities(debian_data, base_release='jessie'): if not release: continue + status = release.get('status') + if status not in ('open', 'resolved'): + continue + # the latest version of this package in base_release version = release.get('repositories', {}).get(base_release, '') @@ -55,7 +59,7 @@ def extract_vulnerabilities(debian_data, base_release='jessie'): 'package_name': package_name, 'vulnerability_id': vulnerability, 'description': details.get('description', ''), - 'status': release.get('status', ''), + 'status': status, 'urgency': release.get('urgency', ''), 'version': version, 'fixed_version': release.get('fixed_version', '') diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index eb582c9fc..4e3a76b9c 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -41,43 +41,20 @@ class TestResponse(TestCase): def test_debian_response(self): with open(os.path.join(TEST_DATA, 'debian.json')) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) - response = self.client.get('/api/packages/?name=mimetex', format='json') + response = self.client.get('/api/packages/?name=mimetex', format='json').data - expected = [{ - "name": "mimetex", - "version": "1.50-1.1", - "platform": "", - "vulnerabilities": [{ - "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", - "cvss": None, - "references": [{ - "reference_id": "CVE-2009-1382", - "source": "", - "url": "", - }] - }], - "references": [], - }, { - "name": "mimetex", - "version": "1.50-1.1", - "platform": "", - "vulnerabilities": [{ - "summary": "Multiple unspecified vulnerabilities in mimeTeX", - "cvss": None, - "references": [{ - "reference_id": "CVE-2009-2459", - "source": "", - "url": "", - }] - }], - "references": [], - }] + self.assertEqual(4, response['count']) - self.assertEqual(expected, response.data.get('results')) + first_result = response['results'][0] + self.assertEqual('mimetex', first_result['name']) + + versions = {r['version'] for r in response['results']} + self.assertIn('1.50-1.1', versions) + self.assertIn('1.74-1', versions) def test_ubuntu_response(self): with open(os.path.join(TEST_DATA, 'ubuntu_main.html')) as f: @@ -109,44 +86,18 @@ def test_ubuntu_response(self): class TestSerializers(TestCase): def test_serializers(self): with open(os.path.join(TEST_DATA, 'debian.json')) as f: - test_data = json.loads(f.read()) + test_data = json.load(f) extract_data = debian.extract_vulnerabilities(test_data) debian_dump(extract_data) pk = Package.objects.filter(name="mimetex") response = PackageSerializer(pk, many=True).data - expected = [ - { - "name": "mimetex", - "version": "1.50-1.1", - "platform": "", - "vulnerabilities": [{ - "summary": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX", - "cvss": None, - "references": [{ - "reference_id": "CVE-2009-1382", - "source": "", - "url": "", - }] - }], - "references": [], - }, - { - "name": "mimetex", - "version": "1.50-1.1", - "platform": "", - "vulnerabilities": [{ - "summary": "Multiple unspecified vulnerabilities in mimeTeX", - "cvss": None, - "references": [{ - "reference_id": "CVE-2009-2459", - "source": "", - "url": "", - }] - }], - "references": [], - } - ] - - self.assertEqual(expected, response) + self.assertEqual(4, len(response)) + + first_result = response[0] + self.assertEqual('mimetex', first_result['name']) + + versions = {r['version'] for r in response} + self.assertIn('1.50-1.1', versions) + self.assertIn('1.74-1', versions) diff --git a/vulnerabilities/tests/test_data_dump.py b/vulnerabilities/tests/test_data_dump.py index 987e164e9..766ba65bc 100644 --- a/vulnerabilities/tests/test_data_dump.py +++ b/vulnerabilities/tests/test_data_dump.py @@ -83,13 +83,37 @@ def test_Package(self): """ Check that all packages from the test data are stored in the database """ - # There are three rows in Package because currently the models allow duplicates + # There are five rows in Package because currently the models allow duplicates # (see issue #28). - self.assertEqual(3, Package.objects.count()) + self.assertEqual(5, Package.objects.count()) self.assertTrue(Package.objects.filter(name='mimetex')) self.assertTrue(Package.objects.get(name='librsync')) + def test_ImpactedPackage(self): + """ + Check that all impacted packages from the test data are stored in the database + """ + impacted_pkgs = ImpactedPackage.objects.all() + impacted_pkg = impacted_pkgs[0] + + self.assertEqual(1, len(impacted_pkgs)) + self.assertEqual('librsync', impacted_pkg.package.name) + self.assertEqual('0.9.7-10', impacted_pkg.package.version) + + def test_ResolvedPackage(self): + """ + Check that all resolved packages from the test data are stored in the database + """ + resolved_pkgs = ResolvedPackage.objects.all() + resolved_pkg = resolved_pkgs[0] + versions = [rp.package.version for rp in resolved_pkgs] + + self.assertEqual(4, len(resolved_pkgs)) + self.assertEqual('mimetex', resolved_pkg.package.name) + self.assertIn('1.50-1.1', versions) + self.assertIn('1.74-1', versions) + class TestUbuntuDataDump(TestCase): @classmethod