From 05ec4d67021e76a866f6c3aa1b5014455134740d Mon Sep 17 00:00:00 2001 From: Kartik Date: Fri, 7 Jul 2017 09:35:56 +0530 Subject: [PATCH 01/14] Style changes in Debian #6 Signed-off-by: Kartik Sibal --- scraper/debian.py | 78 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 scraper/debian.py diff --git a/scraper/debian.py b/scraper/debian.py new file mode 100644 index 000000000..ac5bbda82 --- /dev/null +++ b/scraper/debian.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import bs4 as bs +import re +from urllib.request import urlopen + + +def debian_data(): + cve_id = [] + package_name = [] + vulnerability_status = [] + links = [] + + # Return vulnerability data from Debian's dataset + parent_url = urlopen("https://security-tracker.debian.org/tracker/") + soup = bs.BeautifulSoup(parent_url, "lxml") + + # Extract links of child datasets + for tag in soup.find_all('a'): + href = tag.get('href') + + if re.findall('^/track+.*', href): + links.append(href) + + for child_links in range(6): + # Extracts package info from all the child datasets + child_url = urlopen("https://security-tracker.debian.org" + + links[child_links + 2]) + soup = bs.BeautifulSoup(child_url, "lxml") + + for tag in soup.find_all('a'): + href = tag.get('href') + + if re.search('/tracker/CVE-(.+)', href): + id = re.findall('(?<=/tracker/).*', href) + cve_id.append(id[0]) + + if re.search('^/tracker/TEMP-+.*', href): + id = re.findall('(?<=/tracker/).*', href) + cve_id.append(id[0]) + + if re.search('/tracker/source-package/(.+)', href): + pkg = re.findall('(?<=/tracker/source-package/).*', href) + package_name.append(pkg[0]) + + # if package name is empty, use the previous package name + if href == "/tracker/source-package/": + package_name.append(pkg) + + for tag in soup.find_all('td'): + + if "medium**" in tag or "medium" in tag or "low" in tag or "low**" in tag or "not yet assigned" in tag: + vulnerability_status.append(tag.text) + elif tag.find_all("span", {"class": "red"}) and tag.text == "high**" or tag.text == "high": + vulnerability_status.append(tag.text) + + return cve_id, package_name, vulnerability_status From e7e89977b556f0d4dbbb41dc7e771186a89497fd Mon Sep 17 00:00:00 2001 From: Kartik Date: Fri, 7 Jul 2017 09:36:50 +0530 Subject: [PATCH 02/14] Style changes iUbuntu #7 Signed-off-by: Kartik Sibal --- scraper/ubuntu.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 scraper/ubuntu.py diff --git a/scraper/ubuntu.py b/scraper/ubuntu.py new file mode 100644 index 000000000..209d01e90 --- /dev/null +++ b/scraper/ubuntu.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import bs4 as bs +import re +from urllib.request import urlopen + + +def ubuntu_data(): + cve_id = [] + package_name = [] + vulnerability_status = [] + + url = urlopen("https://people.canonical.com/~ubuntu-security/cve/main.html") + soup = bs.BeautifulSoup(url, "lxml") + + """ + Scrape vulnerability status. + Ubuntu provides a general vulnerability + status of a package across all it's releases. + """ + for tag in soup.find_all('tr'): + if re.match('<\w+\s\w+="(\w+)">', str(tag)): + status = re.findall('<\w+\s\w+="(\w+)">', str(tag)) + vulnerability_status.append(status[0]) + + for tag in soup.find_all('a'): + href = tag.get('href', None) + + if re.findall('^CVE.+', href): + cve_id.append(href) + + if re.match('\pkg+.*', href): + pkg = re.findall('pkg/(.+)\.html', href) + package_name.append(pkg[0]) + + return cve_id, package_name, vulnerability_status From a3dc2c4c49d69008f90018bf2bf063d5a8552454 Mon Sep 17 00:00:00 2001 From: Kartik Date: Fri, 7 Jul 2017 09:36:50 +0530 Subject: [PATCH 03/14] Style changes in Ubuntu #7 Signed-off-by: Kartik Sibal --- scraper/ubuntu.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 scraper/ubuntu.py diff --git a/scraper/ubuntu.py b/scraper/ubuntu.py new file mode 100644 index 000000000..209d01e90 --- /dev/null +++ b/scraper/ubuntu.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import bs4 as bs +import re +from urllib.request import urlopen + + +def ubuntu_data(): + cve_id = [] + package_name = [] + vulnerability_status = [] + + url = urlopen("https://people.canonical.com/~ubuntu-security/cve/main.html") + soup = bs.BeautifulSoup(url, "lxml") + + """ + Scrape vulnerability status. + Ubuntu provides a general vulnerability + status of a package across all it's releases. + """ + for tag in soup.find_all('tr'): + if re.match('<\w+\s\w+="(\w+)">', str(tag)): + status = re.findall('<\w+\s\w+="(\w+)">', str(tag)) + vulnerability_status.append(status[0]) + + for tag in soup.find_all('a'): + href = tag.get('href', None) + + if re.findall('^CVE.+', href): + cve_id.append(href) + + if re.match('\pkg+.*', href): + pkg = re.findall('pkg/(.+)\.html', href) + package_name.append(pkg[0]) + + return cve_id, package_name, vulnerability_status From 988726b3a17f00e09da82d10f824b7b1329a9702 Mon Sep 17 00:00:00 2001 From: Kartik Date: Sat, 8 Jul 2017 02:23:56 +0530 Subject: [PATCH 04/14] File name change Signed-off-by: Kartik Sibal --- scraper/scraper_debian.py | 77 --------------------------------------- scraper/scraper_ubuntu.py | 57 ----------------------------- 2 files changed, 134 deletions(-) delete mode 100644 scraper/scraper_debian.py delete mode 100644 scraper/scraper_ubuntu.py diff --git a/scraper/scraper_debian.py b/scraper/scraper_debian.py deleted file mode 100644 index 190d176ff..000000000 --- a/scraper/scraper_debian.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnerableCode software is licensed under the Apache License version 2.0. -# Data generated with VulnerableCode require an acknowledgment. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnerableCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import bs4 as bs -import re -from urllib.request import urlopen - - -def debian_data(): - cve_id = [] - package_name = [] - vulnerability_status = [] - links = [] - - #Return vulnerability data from Debian's dataset - parent_url = urlopen("https://security-tracker.debian.org/tracker/") - soup = bs.BeautifulSoup (parent_url, "lxml") - - #Extract links of child datasets - for tag in soup.find_all ('a'): - href = tag.get('href') - - if re.findall('^/track+.*', href): - links.append (href) - - for child_links in range (6): - #Extracts package info from all the child datasets - child_url = urlopen("https://security-tracker.debian.org" + links[child_links + 2]) - soup = bs.BeautifulSoup (child_url, "lxml") - - for tag in soup.find_all ('a'): - href = tag.get('href') - - if re.search('/tracker/CVE-(.+)', href): - id = re.findall ('(?<=/tracker/).*', href) - cve_id.append(id[0]) - - if re.search('^/tracker/TEMP-+.*', href): - id = re.findall ('(?<=/tracker/).*', href) - cve_id.append(id[0]) - - if re.search('/tracker/source-package/(.+)', href): - pkg = re.findall ('(?<=/tracker/source-package/).*', href) - package_name.append(pkg[0]) - - #if package name is empty, use the previous package name - if href == "/tracker/source-package/": - package_name.append(pkg) - - for tag in soup.find_all('td'): - if "medium**" in tag or "medium" in tag or "low" in tag or "low**" in tag or "not yet assigned" in tag: - vulnerability_status.append (tag.text) - - elif tag.find_all("span", {"class":"red"}) and tag.text == "high**" or tag.text == "high": - vulnerability_status.append (tag.text) - - return cve_id, package_name, vulnerability_status diff --git a/scraper/scraper_ubuntu.py b/scraper/scraper_ubuntu.py deleted file mode 100644 index be3edc5a8..000000000 --- a/scraper/scraper_ubuntu.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnerableCode software is licensed under the Apache License version 2.0. -# Data generated with VulnerableCode require an acknowledgment. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnerableCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import bs4 as bs -import re -from urllib.request import urlopen - - -def ubuntu_data(): - cve_id = [] - package_name = [] - vulnerability_status = [] - - url = urlopen("https://people.canonical.com/~ubuntu-security/cve/main.html") - soup = bs.BeautifulSoup (url, "lxml") - - """ - Scrape vulnerability status. - Ubuntu provides a general vulnerability - status of a package across all it's releases. - """ - for tag in soup.find_all('tr'): - if re.match('<\w+\s\w+="(\w+)">', str(tag)): - status = re.findall('<\w+\s\w+="(\w+)">', str(tag)) - vulnerability_status.append(status[0]) - - for tag in soup.find_all('a'): - href = tag.get ('href', None) - - if re.findall ('^CVE.+', href): - cve_id.append(href) - - if re.match('\pkg+.*', href): - pkg = re.findall ('pkg/(.+)\.html', href) - package_name.append(pkg[0]) - - return cve_id, package_name, vulnerability_status From 3d1b9389c75035b6197a767d3b52d66c11f27d7e Mon Sep 17 00:00:00 2001 From: Kartik Date: Mon, 10 Jul 2017 13:28:36 +0530 Subject: [PATCH 05/14] Minor changes #5 Signed-off-by: Kartik Sibal --- app/vulncode_app/models.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/app/vulncode_app/models.py b/app/vulncode_app/models.py index 02642ad72..9635dfe87 100644 --- a/app/vulncode_app/models.py +++ b/app/vulncode_app/models.py @@ -26,38 +26,35 @@ class Vulnerability(models.Model): - vulnerability_id = models.AutoField(primary_key=True) - summary = models.TextField(max_length=50) - cvss = models.FloatField(max_length=50) + summary = models.TextField(max_length=50, help_text="Summary of the vulnerability") + cvss = models.FloatField(max_length=50, help_text="CVSS Score") class VulnerabilityReference(models.Model): - vulnerability_id = models.ForeignKey('Vulnerability') - source = models.CharField(max_length=50) - reference_id = models.CharField(max_length=50) - url = models.URLField(max_length=50) + vulnerability = models.ForeignKey('Vulnerability') + source = models.CharField(max_length=50, help_text="Source's name eg:NVD") + reference_id = models.CharField(max_length=50, help_text="Reference ID, eg:CVE-ID") + url = models.URLField(max_length=1024, help_text="URL of Vulnerability data") class ImpactedPackage(models.Model): - vulnerability_id = models.ForeignKey('Vulnerability') - package_id = models.ForeignKey('Package') + vulnerability_id = models.ForeignKey('Vulnerability') + package_id = models.ForeignKey('Package') class ResolvedPackage(models.Model): - vulnerability_id = models.ForeignKey('Vulnerability') - package_id = models.ForeignKey('Package') + vulnerability_id = models.ForeignKey('Vulnerability') + package_id = models.ForeignKey('Package') class Package(models.Model): - package_id = models.AutoField(primary_key=True) - platform = models.CharField(max_length=50) - name = models.CharField(max_length=50) - version = models.FloatField(max_length=50) + platform = models.CharField(max_length=50, help_text="Package platform eg:maven") + name = models.CharField(max_length=50, help_text="Package name") + version = models.CharField(max_length=50, help_text="Pacakge version") class PackageReference(models.Model): - package_id = models.ForeignKey('Package') - repository = models.CharField(max_length=50) - platform = models.CharField(max_length=50) - name = models.CharField(max_length=50) - version = models.FloatField(max_length=50) + repository = models.CharField(max_length=50, help_text="Repository URL eg:http://central.maven.org") + platform = models.CharField(max_length=50, help_text="Platform eg:maven") + name = models.CharField(max_length=50, help_text="Package reference name eg:org.apache.commons.io") + version = models.CharField(max_length=50, help_text="Reference version") From 2285036eca976c852533d297d9aa151ca1fb732b Mon Sep 17 00:00:00 2001 From: Kartik Date: Tue, 11 Jul 2017 09:59:08 +0530 Subject: [PATCH 06/14] Add test cases for #6 Debian and #7 Ubuntu Signed-off-by: Kartik Sibal --- scraper/debian.py | 9 ++++-- scraper/ubuntu.py | 22 +++++++++------ test_scrapers.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 test_scrapers.py diff --git a/scraper/debian.py b/scraper/debian.py index ac5bbda82..cfdb33efd 100644 --- a/scraper/debian.py +++ b/scraper/debian.py @@ -32,10 +32,13 @@ def debian_data(): vulnerability_status = [] links = [] - # Return vulnerability data from Debian's dataset parent_url = urlopen("https://security-tracker.debian.org/tracker/") - soup = bs.BeautifulSoup(parent_url, "lxml") + data = bs.BeautifulSoup(parent_url, "lxml") + return data + + +def extracted_data_debian(data): # Extract links of child datasets for tag in soup.find_all('a'): href = tag.get('href') @@ -44,7 +47,7 @@ def debian_data(): links.append(href) for child_links in range(6): - # Extracts package info from all the child datasets + # Extract package info from all the child datasets child_url = urlopen("https://security-tracker.debian.org" + links[child_links + 2]) soup = bs.BeautifulSoup(child_url, "lxml") diff --git a/scraper/ubuntu.py b/scraper/ubuntu.py index 209d01e90..425c084d4 100644 --- a/scraper/ubuntu.py +++ b/scraper/ubuntu.py @@ -27,31 +27,35 @@ def ubuntu_data(): - cve_id = [] - package_name = [] - vulnerability_status = [] - url = urlopen("https://people.canonical.com/~ubuntu-security/cve/main.html") - soup = bs.BeautifulSoup(url, "lxml") + data = bs.BeautifulSoup(url, "lxml") + + return data + +def extracted_data_ubuntu(data): """ Scrape vulnerability status. Ubuntu provides a general vulnerability status of a package across all it's releases. """ - for tag in soup.find_all('tr'): + cve_id = [] + package_name = [] + vulnerability_status = [] + + for tag in data.find_all('tr'): if re.match('<\w+\s\w+="(\w+)">', str(tag)): status = re.findall('<\w+\s\w+="(\w+)">', str(tag)) vulnerability_status.append(status[0]) - for tag in soup.find_all('a'): + for tag in data.find_all('a'): href = tag.get('href', None) if re.findall('^CVE.+', href): cve_id.append(href) - if re.match('\pkg+.*', href): + if re.match('pkg+.*', href): pkg = re.findall('pkg/(.+)\.html', href) package_name.append(pkg[0]) - return cve_id, package_name, vulnerability_status + return cve_id, vulnerability_status, package_name diff --git a/test_scrapers.py b/test_scrapers.py new file mode 100644 index 000000000..2185cd642 --- /dev/null +++ b/test_scrapers.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import bs4 as bs +from mock import Mock +import pytest +from urllib.request import urlopen + +# Ubuntu test data +ubuntu_test_data = """ + +CVE-2002-2439 +gcc-4.4 +needs-triage* +needs-triage +DNE +DNE +DNE +DNE +DNE +Mitre +LP +Debian +""" + +# Debian test data +debian_test_data = """ +389-ds-base +CVE-2016-5416 +not yet assigned? +""" + + +def test_ubuntu_data(): + from scraper import ubuntu + test_data = bs.BeautifulSoup(ubuntu_test_data, "lxml") + extracted_data = ubuntu.extracted_data_ubuntu(test_data) + + assert extracted_data == (['CVE-2002-2439'], ['High'], ['gcc-4.4']) + + +def test_debian_data(): + # Fix Me: The test data doesn't accurately depict the + # actual debian website on which, the code under testing + # is written + from scraper import debian + test_data = bs.BeautifulSoup(debian_test_data, "lxml") + extracted_data = debian.extracted_data_debian(test_data) + + assert extracted_data == (["not yet assigned"], ["CVE-2016-5416"], + ["389-ds-base"]) From 980265c906afca0284b8792b0dd82438b992f5f3 Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 11 Jul 2017 10:55:36 -0700 Subject: [PATCH 07/14] Add lxml in the requirements.txt #6 Signed-off-by: Thomas Druez --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2eae01052..6b087fe22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ beautifulsoup4==4.6.0 +lxml==3.8.0 # Tests pytest==3.1.3 From e7af9609ab55771f047519453eaf9b018c2f7bc9 Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 13 Jul 2017 15:55:30 +0530 Subject: [PATCH 08/14] Debian test case and style changes #6 Signed-off-by: Kartik Sibal --- api_data.py | 2 +- scraper/debian.py | 66 ++++++++++++++++++++++++++--------------------- test_scrapers.py | 18 ++++++------- 3 files changed, 47 insertions(+), 39 deletions(-) diff --git a/api_data.py b/api_data.py index 2a8dfe96a..2d193e635 100644 --- a/api_data.py +++ b/api_data.py @@ -52,4 +52,4 @@ def extract_fields(data, fields_names): cve-search' api. Takes as input data, fields requested """ return [{name: item.get(name) for name in fields_names} - for item in data] + for item in data] diff --git a/scraper/debian.py b/scraper/debian.py index cfdb33efd..5f58647f2 100644 --- a/scraper/debian.py +++ b/scraper/debian.py @@ -26,21 +26,19 @@ from urllib.request import urlopen -def debian_data(): - cve_id = [] - package_name = [] - vulnerability_status = [] - links = [] - +def main_data(): parent_url = urlopen("https://security-tracker.debian.org/tracker/") data = bs.BeautifulSoup(parent_url, "lxml") return data -def extracted_data_debian(data): +def child_data(data): + links = [] + child_data = ' ' + # Extract links of child datasets - for tag in soup.find_all('a'): + for tag in data.find_all('a'): href = tag.get('href') if re.findall('^/track+.*', href): @@ -49,33 +47,43 @@ def extracted_data_debian(data): for child_links in range(6): # Extract package info from all the child datasets child_url = urlopen("https://security-tracker.debian.org" - + links[child_links + 2]) - soup = bs.BeautifulSoup(child_url, "lxml") + + links[child_links + 2]).read() + + child_data = child_data + str(child_url) - for tag in soup.find_all('a'): - href = tag.get('href') + return child_data - if re.search('/tracker/CVE-(.+)', href): - id = re.findall('(?<=/tracker/).*', href) - cve_id.append(id[0]) - if re.search('^/tracker/TEMP-+.*', href): - id = re.findall('(?<=/tracker/).*', href) - cve_id.append(id[0]) +def extracted_data_debian(data): + cve_id = [] + package_name = [] + vulnerability_status = [] + + data = bs.BeautifulSoup(data, "lxml") + + for tag in data.find_all('a'): + href = tag.get('href') + + if re.search('/tracker/CVE-(.+)', href): + id = re.findall('(?<=/tracker/).*', href) + cve_id.append(id[0]) - if re.search('/tracker/source-package/(.+)', href): - pkg = re.findall('(?<=/tracker/source-package/).*', href) - package_name.append(pkg[0]) + if re.search('^/tracker/TEMP-+.*', href): + id = re.findall('(?<=/tracker/).*', href) + cve_id.append(id[0]) - # if package name is empty, use the previous package name - if href == "/tracker/source-package/": - package_name.append(pkg) + if re.search('/tracker/source-package/(.+)', href): + pkg = re.findall('(?<=/tracker/source-package/).*', href) + package_name.append(pkg[0]) - for tag in soup.find_all('td'): + # if package name is empty, use the previous package name + if href == "/tracker/source-package/": + package_name.append(pkg) - if "medium**" in tag or "medium" in tag or "low" in tag or "low**" in tag or "not yet assigned" in tag: - vulnerability_status.append(tag.text) - elif tag.find_all("span", {"class": "red"}) and tag.text == "high**" or tag.text == "high": - vulnerability_status.append(tag.text) + for tag in data.find_all('td'): + if "medium**" in tag or "medium" in tag or "low" in tag or "low**" in tag or "not yet assigned" in tag: + vulnerability_status.append(tag.text) + elif tag.find_all("span", {"class": "red"}) and tag.text == "high**" or tag.text == "high": + vulnerability_status.append(tag.text) return cve_id, package_name, vulnerability_status diff --git a/test_scrapers.py b/test_scrapers.py index 2185cd642..6e3cddcbf 100644 --- a/test_scrapers.py +++ b/test_scrapers.py @@ -38,7 +38,7 @@ DNE DNE DNE -Mitre +Mitre LP Debian """ @@ -56,16 +56,16 @@ def test_ubuntu_data(): test_data = bs.BeautifulSoup(ubuntu_test_data, "lxml") extracted_data = ubuntu.extracted_data_ubuntu(test_data) - assert extracted_data == (['CVE-2002-2439'], ['High'], ['gcc-4.4']) + assert extracted_data == (['CVE-2002-2439'], + ['High'], + ['gcc-4.4']) def test_debian_data(): - # Fix Me: The test data doesn't accurately depict the - # actual debian website on which, the code under testing - # is written from scraper import debian - test_data = bs.BeautifulSoup(debian_test_data, "lxml") - extracted_data = debian.extracted_data_debian(test_data) + # test_data = bs.BeautifulSoup(debian_test_data, "lxml") + extracted_data = debian.extracted_data_debian(debian_test_data) - assert extracted_data == (["not yet assigned"], ["CVE-2016-5416"], - ["389-ds-base"]) + assert extracted_data == (['CVE-2016-5416'], + ['389-ds-base'], + ['not yet assigned']) From 9b34f7add89d54894fd0591766babd0085290f16 Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 13 Jul 2017 17:06:42 +0530 Subject: [PATCH 09/14] Minor changes #6 Signed-off-by: Kartik Sibal --- app/app/urls.py | 2 +- test_scrapers.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/app/app/urls.py b/app/app/urls.py index c339d6fda..9a30fd723 100644 --- a/app/app/urls.py +++ b/app/app/urls.py @@ -13,7 +13,7 @@ 1. Import the include() function: from django.conf.urls import url, include 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ -from django.conf.urls import url,include +from django.conf.urls import url, include from django.contrib import admin urlpatterns = [ diff --git a/test_scrapers.py b/test_scrapers.py index 6e3cddcbf..db9c38393 100644 --- a/test_scrapers.py +++ b/test_scrapers.py @@ -22,9 +22,6 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import bs4 as bs -from mock import Mock -import pytest -from urllib.request import urlopen # Ubuntu test data ubuntu_test_data = """ @@ -63,7 +60,6 @@ def test_ubuntu_data(): def test_debian_data(): from scraper import debian - # test_data = bs.BeautifulSoup(debian_test_data, "lxml") extracted_data = debian.extracted_data_debian(debian_test_data) assert extracted_data == (['CVE-2016-5416'], From fee110a4d3e3ec541bb4b00f48f64b101ccfc829 Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 13 Jul 2017 17:23:59 +0530 Subject: [PATCH 10/14] Minor style changes #5 Signed-off-by: Kartik Sibal --- app/vulncode_app/models.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/vulncode_app/models.py b/app/vulncode_app/models.py index 9635dfe87..6f816f357 100644 --- a/app/vulncode_app/models.py +++ b/app/vulncode_app/models.py @@ -54,7 +54,11 @@ class Package(models.Model): class PackageReference(models.Model): - repository = models.CharField(max_length=50, help_text="Repository URL eg:http://central.maven.org") - platform = models.CharField(max_length=50, help_text="Platform eg:maven") - name = models.CharField(max_length=50, help_text="Package reference name eg:org.apache.commons.io") - version = models.CharField(max_length=50, help_text="Reference version") + repository = models.CharField(max_length=50, + help_text="Repository URL eg:http://central.maven.org") + platform = models.CharField(max_length=50, + help_text="Platform eg:maven") + name = models.CharField(max_length=50, + help_text="Package reference name eg:org.apache.commons.io") + version = models.CharField(max_length=50, + help_text="Reference version") From 0b70285e5f160b423cfd8520b860294d40a4814f Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 13 Jul 2017 17:29:39 +0530 Subject: [PATCH 11/14] Minor changes #5 Signed-off-by: Kartik Sibal --- app/vulncode_app/models.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/vulncode_app/models.py b/app/vulncode_app/models.py index 6f816f357..cf7208469 100644 --- a/app/vulncode_app/models.py +++ b/app/vulncode_app/models.py @@ -38,13 +38,13 @@ class VulnerabilityReference(models.Model): class ImpactedPackage(models.Model): - vulnerability_id = models.ForeignKey('Vulnerability') - package_id = models.ForeignKey('Package') + vulnerability = models.ForeignKey('Vulnerability') + package = models.ForeignKey('Package') class ResolvedPackage(models.Model): - vulnerability_id = models.ForeignKey('Vulnerability') - package_id = models.ForeignKey('Package') + vulnerability = models.ForeignKey('Vulnerability') + package = models.ForeignKey('Package') class Package(models.Model): @@ -54,6 +54,7 @@ class Package(models.Model): class PackageReference(models.Model): + package = models.ForeignKey('Package') repository = models.CharField(max_length=50, help_text="Repository URL eg:http://central.maven.org") platform = models.CharField(max_length=50, From 6569bb22b24c08fd8608eeab04b953c392da79cd Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 13 Jul 2017 11:57:03 -0700 Subject: [PATCH 12/14] Move tests files in a tests/ directory #6 - Add tests runner in the travis config - Cleanup Signed-off-by: Thomas Druez --- .travis.yml | 8 ++- README.md | 2 +- test_api_data.py => tests/test_api_data.py | 75 +++++++++++----------- test_scrapers.py => tests/test_scrapers.py | 38 +++++------ 4 files changed, 67 insertions(+), 56 deletions(-) rename test_api_data.py => tests/test_api_data.py (53%) rename test_scrapers.py => tests/test_scrapers.py (69%) diff --git a/.travis.yml b/.travis.yml index 50dfb9237..9c41e3c50 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,18 @@ language: python python: 3.6 -install: pip install pycodestyle +install: + - pip install -r requirements.txt + - pip install pycodestyle before_script: - pycodestyle --exclude=migrations,settings.py --max-line-length=100 . +script: + - python3.6 -m pytest -v tests/ + notifications: + email: false webhooks: urls: - https://webhooks.gitter.im/e/b119fa557626081e1f36 diff --git a/README.md b/README.md index c2d4016ab..8c534ad08 100644 --- a/README.md +++ b/README.md @@ -24,5 +24,5 @@ Tests ----- ``` -python3.6 -m pytest -v +python3.6 -m pytest -v tests/ ``` diff --git a/test_api_data.py b/tests/test_api_data.py similarity index 53% rename from test_api_data.py rename to tests/test_api_data.py index c45036921..5ea514e07 100644 --- a/test_api_data.py +++ b/tests/test_api_data.py @@ -21,49 +21,52 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import api_data as api import json +from api_data import extract_fields + + test_data = """ [{ - "Modified": "2008-11-15T00:00:00", - "Published": "2007-02-19T21:28:00", - "access": { - "authentication": "NONE", - "complexity": "MEDIUM", - "vector": "NETWORK" - }, - "cvss": 4.3, - "cvss-time": "2007-02-20T14:55:00", - "id": "CVE-2007-1004", - "impact": { - "availability": "NONE", - "confidentiality": "NONE", - "integrity": "PARTIAL" - }, - "reason": "Link", - "references": [ - "http://securityreason.com/securityalert/2264", - "http://www.securityfocus.com/archive/1/archive/1/460369/100/0/threaded", - "http://www.securityfocus.com/archive/1/archive/1/460412/100/0/threaded", - "http://www.securityfocus.com/archive/1/archive/1/460617/100/0/threaded", - "http://www.securityfocus.com/bid/22601", - "http://xforce.iss.net/xforce/xfdb/32580" - ], - "summary": "Mozilla Firefox might allow remote", - "vulnerable_configuration": [ - "cpe:2.3:a:mozilla:firefox:2.0:rc3" - ], - "vulnerable_configuration_cpe_2_2": [ - "cpe:/a:mozilla:firefox:2.0:rc3" - ]}] + "Modified": "2008-11-15T00:00:00", + "Published": "2007-02-19T21:28:00", + "access": { + "authentication": "NONE", + "complexity": "MEDIUM", + "vector": "NETWORK" + }, + "cvss": 4.3, + "cvss-time": "2007-02-20T14:55:00", + "id": "CVE-2007-1004", + "impact": { + "availability": "NONE", + "confidentiality": "NONE", + "integrity": "PARTIAL" + }, + "reason": "Link", + "references": [ + "http://securityreason.com/securityalert/2264", + "http://www.securityfocus.com/archive/1/archive/1/460369/100/0/threaded", + "http://www.securityfocus.com/archive/1/archive/1/460412/100/0/threaded", + "http://www.securityfocus.com/archive/1/archive/1/460617/100/0/threaded", + "http://www.securityfocus.com/bid/22601", + "http://xforce.iss.net/xforce/xfdb/32580" + ], + "summary": "Mozilla Firefox might allow remote", + "vulnerable_configuration": [ + "cpe:2.3:a:mozilla:firefox:2.0:rc3" + ], + "vulnerable_configuration_cpe_2_2": [ + "cpe:/a:mozilla:firefox:2.0:rc3" + ] +}] """ def test_extract_fields_data(): fields_names = ['id', 'cvss', 'summary'] data = json.loads(test_data) - extracted_data = api.extract_fields(data=data, fields_names=fields_names) + extracted_data = extract_fields(data=data, fields_names=fields_names) assert extracted_data == [{'cvss': 4.3, 'id': 'CVE-2007-1004', 'summary': 'Mozilla Firefox might allow remote'}] @@ -72,13 +75,13 @@ def test_extract_fields_data(): def test_extract_fields(): fields_names = [] data = json.loads(test_data) - extracted_data = api.extract_fields(data=data, fields_names=fields_names) + extracted_data = extract_fields(data=data, fields_names=fields_names) assert extracted_data == [{}] fields_names = [''] - extracted_data = api.extract_fields(data=data, fields_names=fields_names) + extracted_data = extract_fields(data=data, fields_names=fields_names) assert extracted_data == [{'': None}] fields_names = ['invalid_field'] - extracted_data = api.extract_fields(data=data, fields_names=fields_names) + extracted_data = extract_fields(data=data, fields_names=fields_names) assert extracted_data == [{'invalid_field': None}] diff --git a/test_scrapers.py b/tests/test_scrapers.py similarity index 69% rename from test_scrapers.py rename to tests/test_scrapers.py index db9c38393..91b44545e 100644 --- a/test_scrapers.py +++ b/tests/test_scrapers.py @@ -23,33 +23,36 @@ import bs4 as bs +from scraper import ubuntu +from scraper import debian + + # Ubuntu test data ubuntu_test_data = """ - -CVE-2002-2439 -gcc-4.4 -needs-triage* -needs-triage -DNE -DNE -DNE -DNE -DNE -Mitre -LP -Debian + + CVE-2002-2439 + gcc-4.4 + needs-triage* + needs-triage + DNE + DNE + DNE + DNE + DNE + Mitre + LP + Debian """ # Debian test data debian_test_data = """ -389-ds-base -CVE-2016-5416 -not yet assigned? + 389-ds-base + CVE-2016-5416 + not yet assigned? """ def test_ubuntu_data(): - from scraper import ubuntu test_data = bs.BeautifulSoup(ubuntu_test_data, "lxml") extracted_data = ubuntu.extracted_data_ubuntu(test_data) @@ -59,7 +62,6 @@ def test_ubuntu_data(): def test_debian_data(): - from scraper import debian extracted_data = debian.extracted_data_debian(debian_test_data) assert extracted_data == (['CVE-2016-5416'], From 4abd00bade7c4c444c40ec10d3a3c6917106f55b Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 13 Jul 2017 16:02:52 -0700 Subject: [PATCH 13/14] Refactor and refine the code and structure for scrapers #6 Signed-off-by: Thomas Druez --- .travis.yml | 2 +- README.md | 11 ++++ scraper/debian.py | 69 ++++++++++++----------- scraper/ubuntu.py | 29 +++++----- tests/test_scrapers.py | 124 ++++++++++++++++++++++++++++++----------- 5 files changed, 155 insertions(+), 80 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c41e3c50..606ad44a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ install: - pip install pycodestyle before_script: - - pycodestyle --exclude=migrations,settings.py --max-line-length=100 . + - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . script: - python3.6 -m pytest -v tests/ diff --git a/README.md b/README.md index 8c534ad08..ed43acf1f 100644 --- a/README.md +++ b/README.md @@ -24,5 +24,16 @@ Tests ----- ``` +pycodestyle --exclude=migrations,settings.py,lib --max-line-length=100 . python3.6 -m pytest -v tests/ ``` + +Scrape +------ + +``` +from scraper import debian, ubuntu + +debian.scrape_cves() +ubuntu.scrape_cves() +``` diff --git a/scraper/debian.py b/scraper/debian.py index 5f58647f2..ea78b1929 100644 --- a/scraper/debian.py +++ b/scraper/debian.py @@ -21,47 +21,35 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import bs4 as bs +import logging import re from urllib.request import urlopen +import bs4 -def main_data(): - parent_url = urlopen("https://security-tracker.debian.org/tracker/") - data = bs.BeautifulSoup(parent_url, "lxml") - return data +DEBIAN_ROOT_URL = 'https://security-tracker.debian.org' -def child_data(data): - links = [] - child_data = ' ' - - # Extract links of child datasets - for tag in data.find_all('a'): - href = tag.get('href') - - if re.findall('^/track+.*', href): - links.append(href) - - for child_links in range(6): - # Extract package info from all the child datasets - child_url = urlopen("https://security-tracker.debian.org" - + links[child_links + 2]).read() - - child_data = child_data + str(child_url) - - return child_data +def extract_tracker_paths(html): + """ + Return a list of tracker URL paths extracted from the given `html` input. + """ + soup = bs4.BeautifulSoup(html, 'lxml') + tracker_links = soup.findAll('a', href=re.compile('^/track+.*')) + return [link.get('href') for link in tracker_links] -def extracted_data_debian(data): +def extract_cves_from_tracker(html): + """ + Return all CVEs extracted from the given `html` input. + """ cve_id = [] package_name = [] vulnerability_status = [] + soup = bs4.BeautifulSoup(html, 'lxml') - data = bs.BeautifulSoup(data, "lxml") - - for tag in data.find_all('a'): + for tag in soup.find_all('a'): href = tag.get('href') if re.search('/tracker/CVE-(.+)', href): @@ -77,13 +65,30 @@ def extracted_data_debian(data): package_name.append(pkg[0]) # if package name is empty, use the previous package name - if href == "/tracker/source-package/": + if href == '/tracker/source-package/': package_name.append(pkg) - for tag in data.find_all('td'): - if "medium**" in tag or "medium" in tag or "low" in tag or "low**" in tag or "not yet assigned" in tag: + for tag in soup.find_all('td'): + if 'medium**' in tag or 'medium' in tag or 'low' in tag or 'low**' in tag or 'not yet assigned' in tag: vulnerability_status.append(tag.text) - elif tag.find_all("span", {"class": "red"}) and tag.text == "high**" or tag.text == "high": + elif tag.find_all('span', {'class': 'red'}) and tag.text == 'high**' or tag.text == 'high': vulnerability_status.append(tag.text) return cve_id, package_name, vulnerability_status + + +def scrape_cves(): + """ + Runs the full scraping process of Debian CVEs. + """ + tracker_root_html = urlopen(f'{DEBIAN_ROOT_URL}/tracker/').read() + tracker_paths = extract_tracker_paths(tracker_root_html) + + cves = [] + for tracker_path in tracker_paths: + tracker_url = f'{DEBIAN_ROOT_URL}{tracker_path}/' + logging.info(f'Visiting: {tracker_url}') + html = urlopen(tracker_url).read() + cves.append(extract_cves_from_tracker(html)) + + return cves diff --git a/scraper/ubuntu.py b/scraper/ubuntu.py index 425c084d4..f70f5ac8a 100644 --- a/scraper/ubuntu.py +++ b/scraper/ubuntu.py @@ -21,34 +21,28 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import bs4 as bs import re from urllib.request import urlopen +import bs4 -def ubuntu_data(): - url = urlopen("https://people.canonical.com/~ubuntu-security/cve/main.html") - data = bs.BeautifulSoup(url, "lxml") - return data +UBUNTU_ROOT_URL = 'https://people.canonical.com/~ubuntu-security/cve/main.html' -def extracted_data_ubuntu(data): - """ - Scrape vulnerability status. - Ubuntu provides a general vulnerability - status of a package across all it's releases. - """ +def extract_cves(html): + soup = bs4.BeautifulSoup(html, 'lxml') + cve_id = [] package_name = [] vulnerability_status = [] - for tag in data.find_all('tr'): + for tag in soup.find_all('tr'): if re.match('<\w+\s\w+="(\w+)">', str(tag)): status = re.findall('<\w+\s\w+="(\w+)">', str(tag)) vulnerability_status.append(status[0]) - for tag in data.find_all('a'): + for tag in soup.find_all('a'): href = tag.get('href', None) if re.findall('^CVE.+', href): @@ -59,3 +53,12 @@ def extracted_data_ubuntu(data): package_name.append(pkg[0]) return cve_id, vulnerability_status, package_name + + +def scrape_cves(): + """ + Runs the full scraping process of Ubuntu CVEs. + """ + html = urlopen(UBUNTU_ROOT_URL).read() + cves = extract_cves(html) + return cves diff --git a/tests/test_scrapers.py b/tests/test_scrapers.py index 91b44545e..9fc31eeb0 100644 --- a/tests/test_scrapers.py +++ b/tests/test_scrapers.py @@ -21,49 +21,105 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import bs4 as bs - from scraper import ubuntu from scraper import debian -# Ubuntu test data -ubuntu_test_data = """ +def test_ubuntu_extract_cves(): + test_input = """ - CVE-2002-2439 - gcc-4.4 - needs-triage* - needs-triage - DNE - DNE - DNE - DNE - DNE - Mitre - LP - Debian -""" + CVE-2002-2439 + gcc-4.4 + needs-triage* + needs-triage + DNE + DNE + DNE + DNE + DNE + + Mitre + LP + Debian + + + """ + + expected = ( + ['CVE-2002-2439'], + ['High'], + ['gcc-4.4'], + ) + assert expected == ubuntu.extract_cves(test_input) + -# Debian test data -debian_test_data = """ - 389-ds-base - CVE-2016-5416 - not yet assigned? -""" +def test_debian_extract_tracker_paths(): + test_input = """ + + """ + expected = [ + '/tracker/status/release/unstable', + '/tracker/status/release/testing', + '/tracker/status/release/stable', + '/tracker/status/release/stable-backports', + '/tracker/status/release/oldstable', + '/tracker/status/release/oldstable-backports', + '/tracker/status/release/oldoldstable', + '/tracker/status/release/oldoldstable-backports', + '/tracker/status/dtsa-candidates', + '/tracker/status/todo', + '/tracker/status/undetermined', + '/tracker/status/unimportant', + '/tracker/status/itp', + '/tracker/status/unreported', + '/tracker/data/unknown-packages', + '/tracker/data/fake-names', + '/tracker/data/missing-epochs', + '/tracker/data/latently-vulnerable', + '/tracker/data/funny-versions', + '/tracker/data/releases', + '/tracker/data/json', + ] -def test_ubuntu_data(): - test_data = bs.BeautifulSoup(ubuntu_test_data, "lxml") - extracted_data = ubuntu.extracted_data_ubuntu(test_data) + assert expected == debian.extract_tracker_paths(test_input) - assert extracted_data == (['CVE-2002-2439'], - ['High'], - ['gcc-4.4']) +def test_debian_extract_cves_from_tracker(): + test_input = """ + + 389-ds-base + CVE-2016-5416 + not yet assigned? + + """ -def test_debian_data(): - extracted_data = debian.extracted_data_debian(debian_test_data) + expected = ( + ['CVE-2016-5416'], + ['389-ds-base'], + ['not yet assigned'], + ) - assert extracted_data == (['CVE-2016-5416'], - ['389-ds-base'], - ['not yet assigned']) + assert expected == debian.extract_cves_from_tracker(test_input) From f0e46ceb1cfcc7cb40a841676484f604d6761b37 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 13 Jul 2017 16:12:40 -0700 Subject: [PATCH 14/14] Simplify status condition in extract_cves_from_tracker #6 Signed-off-by: Thomas Druez --- scraper/debian.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraper/debian.py b/scraper/debian.py index ea78b1929..be17f438e 100644 --- a/scraper/debian.py +++ b/scraper/debian.py @@ -69,7 +69,7 @@ def extract_cves_from_tracker(html): package_name.append(pkg) for tag in soup.find_all('td'): - if 'medium**' in tag or 'medium' in tag or 'low' in tag or 'low**' in tag or 'not yet assigned' in tag: + if 'medium' in tag or 'low' in tag or 'not yet assigned' in tag: vulnerability_status.append(tag.text) elif tag.find_all('span', {'class': 'red'}) and tag.text == 'high**' or tag.text == 'high': vulnerability_status.append(tag.text)