diff --git a/.travis.yml b/.travis.yml index 50dfb9237..606ad44a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,18 @@ language: python python: 3.6 -install: pip install pycodestyle +install: + - pip install -r requirements.txt + - pip install pycodestyle before_script: - - pycodestyle --exclude=migrations,settings.py --max-line-length=100 . + - pycodestyle --exclude=migrations,settings.py,lib,tests --max-line-length=100 . + +script: + - python3.6 -m pytest -v tests/ notifications: + email: false webhooks: urls: - https://webhooks.gitter.im/e/b119fa557626081e1f36 diff --git a/README.md b/README.md index c2d4016ab..ed43acf1f 100644 --- a/README.md +++ b/README.md @@ -24,5 +24,16 @@ Tests ----- ``` -python3.6 -m pytest -v +pycodestyle --exclude=migrations,settings.py,lib --max-line-length=100 . +python3.6 -m pytest -v tests/ +``` + +Scrape +------ + +``` +from scraper import debian, ubuntu + +debian.scrape_cves() +ubuntu.scrape_cves() ``` diff --git a/api_data.py b/api_data.py index 2a8dfe96a..2d193e635 100644 --- a/api_data.py +++ b/api_data.py @@ -52,4 +52,4 @@ def extract_fields(data, fields_names): cve-search' api. Takes as input data, fields requested """ return [{name: item.get(name) for name in fields_names} - for item in data] + for item in data] diff --git a/app/app/urls.py b/app/app/urls.py index c339d6fda..9a30fd723 100644 --- a/app/app/urls.py +++ b/app/app/urls.py @@ -13,7 +13,7 @@ 1. Import the include() function: from django.conf.urls import url, include 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ -from django.conf.urls import url,include +from django.conf.urls import url, include from django.contrib import admin urlpatterns = [ diff --git a/app/vulncode_app/models.py b/app/vulncode_app/models.py index 02642ad72..cf7208469 100644 --- a/app/vulncode_app/models.py +++ b/app/vulncode_app/models.py @@ -26,38 +26,40 @@ class Vulnerability(models.Model): - vulnerability_id = models.AutoField(primary_key=True) - summary = models.TextField(max_length=50) - cvss = models.FloatField(max_length=50) + summary = models.TextField(max_length=50, help_text="Summary of the vulnerability") + cvss = models.FloatField(max_length=50, help_text="CVSS Score") class VulnerabilityReference(models.Model): - vulnerability_id = models.ForeignKey('Vulnerability') - source = models.CharField(max_length=50) - reference_id = models.CharField(max_length=50) - url = models.URLField(max_length=50) + vulnerability = models.ForeignKey('Vulnerability') + source = models.CharField(max_length=50, help_text="Source's name eg:NVD") + reference_id = models.CharField(max_length=50, help_text="Reference ID, eg:CVE-ID") + url = models.URLField(max_length=1024, help_text="URL of Vulnerability data") class ImpactedPackage(models.Model): - vulnerability_id = models.ForeignKey('Vulnerability') - package_id = models.ForeignKey('Package') + vulnerability = models.ForeignKey('Vulnerability') + package = models.ForeignKey('Package') class ResolvedPackage(models.Model): - vulnerability_id = models.ForeignKey('Vulnerability') - package_id = models.ForeignKey('Package') + vulnerability = models.ForeignKey('Vulnerability') + package = models.ForeignKey('Package') class Package(models.Model): - package_id = models.AutoField(primary_key=True) - platform = models.CharField(max_length=50) - name = models.CharField(max_length=50) - version = models.FloatField(max_length=50) + platform = models.CharField(max_length=50, help_text="Package platform eg:maven") + name = models.CharField(max_length=50, help_text="Package name") + version = models.CharField(max_length=50, help_text="Pacakge version") class PackageReference(models.Model): - package_id = models.ForeignKey('Package') - repository = models.CharField(max_length=50) - platform = models.CharField(max_length=50) - name = models.CharField(max_length=50) - version = models.FloatField(max_length=50) + package = models.ForeignKey('Package') + repository = models.CharField(max_length=50, + help_text="Repository URL eg:http://central.maven.org") + platform = models.CharField(max_length=50, + help_text="Platform eg:maven") + name = models.CharField(max_length=50, + help_text="Package reference name eg:org.apache.commons.io") + version = models.CharField(max_length=50, + help_text="Reference version") diff --git a/requirements.txt b/requirements.txt index 2eae01052..6b087fe22 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ beautifulsoup4==4.6.0 +lxml==3.8.0 # Tests pytest==3.1.3 diff --git a/scraper/debian.py b/scraper/debian.py new file mode 100644 index 000000000..be17f438e --- /dev/null +++ b/scraper/debian.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import logging +import re +from urllib.request import urlopen + +import bs4 + + +DEBIAN_ROOT_URL = 'https://security-tracker.debian.org' + + +def extract_tracker_paths(html): + """ + Return a list of tracker URL paths extracted from the given `html` input. + """ + soup = bs4.BeautifulSoup(html, 'lxml') + tracker_links = soup.findAll('a', href=re.compile('^/track+.*')) + return [link.get('href') for link in tracker_links] + + +def extract_cves_from_tracker(html): + """ + Return all CVEs extracted from the given `html` input. + """ + cve_id = [] + package_name = [] + vulnerability_status = [] + soup = bs4.BeautifulSoup(html, 'lxml') + + for tag in soup.find_all('a'): + href = tag.get('href') + + if re.search('/tracker/CVE-(.+)', href): + id = re.findall('(?<=/tracker/).*', href) + cve_id.append(id[0]) + + if re.search('^/tracker/TEMP-+.*', href): + id = re.findall('(?<=/tracker/).*', href) + cve_id.append(id[0]) + + if re.search('/tracker/source-package/(.+)', href): + pkg = re.findall('(?<=/tracker/source-package/).*', href) + package_name.append(pkg[0]) + + # if package name is empty, use the previous package name + if href == '/tracker/source-package/': + package_name.append(pkg) + + for tag in soup.find_all('td'): + if 'medium' in tag or 'low' in tag or 'not yet assigned' in tag: + vulnerability_status.append(tag.text) + elif tag.find_all('span', {'class': 'red'}) and tag.text == 'high**' or tag.text == 'high': + vulnerability_status.append(tag.text) + + return cve_id, package_name, vulnerability_status + + +def scrape_cves(): + """ + Runs the full scraping process of Debian CVEs. + """ + tracker_root_html = urlopen(f'{DEBIAN_ROOT_URL}/tracker/').read() + tracker_paths = extract_tracker_paths(tracker_root_html) + + cves = [] + for tracker_path in tracker_paths: + tracker_url = f'{DEBIAN_ROOT_URL}{tracker_path}/' + logging.info(f'Visiting: {tracker_url}') + html = urlopen(tracker_url).read() + cves.append(extract_cves_from_tracker(html)) + + return cves diff --git a/scraper/scraper_debian.py b/scraper/scraper_debian.py deleted file mode 100644 index 190d176ff..000000000 --- a/scraper/scraper_debian.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. -# http://nexb.com and https://github.com/nexB/vulnerablecode/ -# The VulnerableCode software is licensed under the Apache License version 2.0. -# Data generated with VulnerableCode require an acknowledgment. -# -# You may not use this software except in compliance with the License. -# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software distributed -# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -# CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# -# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode -# derivative work, you must accompany this data with the following acknowledgment: -# -# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, either express or implied. No content created from -# VulnerableCode should be considered or used as legal advice. Consult an Attorney -# for any legal advice. -# VulnerableCode is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/vulnerablecode/ for support and download. - -import bs4 as bs -import re -from urllib.request import urlopen - - -def debian_data(): - cve_id = [] - package_name = [] - vulnerability_status = [] - links = [] - - #Return vulnerability data from Debian's dataset - parent_url = urlopen("https://security-tracker.debian.org/tracker/") - soup = bs.BeautifulSoup (parent_url, "lxml") - - #Extract links of child datasets - for tag in soup.find_all ('a'): - href = tag.get('href') - - if re.findall('^/track+.*', href): - links.append (href) - - for child_links in range (6): - #Extracts package info from all the child datasets - child_url = urlopen("https://security-tracker.debian.org" + links[child_links + 2]) - soup = bs.BeautifulSoup (child_url, "lxml") - - for tag in soup.find_all ('a'): - href = tag.get('href') - - if re.search('/tracker/CVE-(.+)', href): - id = re.findall ('(?<=/tracker/).*', href) - cve_id.append(id[0]) - - if re.search('^/tracker/TEMP-+.*', href): - id = re.findall ('(?<=/tracker/).*', href) - cve_id.append(id[0]) - - if re.search('/tracker/source-package/(.+)', href): - pkg = re.findall ('(?<=/tracker/source-package/).*', href) - package_name.append(pkg[0]) - - #if package name is empty, use the previous package name - if href == "/tracker/source-package/": - package_name.append(pkg) - - for tag in soup.find_all('td'): - if "medium**" in tag or "medium" in tag or "low" in tag or "low**" in tag or "not yet assigned" in tag: - vulnerability_status.append (tag.text) - - elif tag.find_all("span", {"class":"red"}) and tag.text == "high**" or tag.text == "high": - vulnerability_status.append (tag.text) - - return cve_id, package_name, vulnerability_status diff --git a/scraper/scraper_ubuntu.py b/scraper/ubuntu.py similarity index 59% rename from scraper/scraper_ubuntu.py rename to scraper/ubuntu.py index be3edc5a8..f70f5ac8a 100644 --- a/scraper/scraper_ubuntu.py +++ b/scraper/ubuntu.py @@ -21,37 +21,44 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import bs4 as bs import re from urllib.request import urlopen +import bs4 -def ubuntu_data(): - cve_id = [] - package_name = [] - vulnerability_status = [] - - url = urlopen("https://people.canonical.com/~ubuntu-security/cve/main.html") - soup = bs.BeautifulSoup (url, "lxml") - - """ - Scrape vulnerability status. - Ubuntu provides a general vulnerability - status of a package across all it's releases. - """ - for tag in soup.find_all('tr'): - if re.match('<\w+\s\w+="(\w+)">', str(tag)): - status = re.findall('<\w+\s\w+="(\w+)">', str(tag)) - vulnerability_status.append(status[0]) - - for tag in soup.find_all('a'): - href = tag.get ('href', None) - - if re.findall ('^CVE.+', href): - cve_id.append(href) - - if re.match('\pkg+.*', href): - pkg = re.findall ('pkg/(.+)\.html', href) - package_name.append(pkg[0]) - - return cve_id, package_name, vulnerability_status + +UBUNTU_ROOT_URL = 'https://people.canonical.com/~ubuntu-security/cve/main.html' + + +def extract_cves(html): + soup = bs4.BeautifulSoup(html, 'lxml') + + cve_id = [] + package_name = [] + vulnerability_status = [] + + for tag in soup.find_all('tr'): + if re.match('<\w+\s\w+="(\w+)">', str(tag)): + status = re.findall('<\w+\s\w+="(\w+)">', str(tag)) + vulnerability_status.append(status[0]) + + for tag in soup.find_all('a'): + href = tag.get('href', None) + + if re.findall('^CVE.+', href): + cve_id.append(href) + + if re.match('pkg+.*', href): + pkg = re.findall('pkg/(.+)\.html', href) + package_name.append(pkg[0]) + + return cve_id, vulnerability_status, package_name + + +def scrape_cves(): + """ + Runs the full scraping process of Ubuntu CVEs. + """ + html = urlopen(UBUNTU_ROOT_URL).read() + cves = extract_cves(html) + return cves diff --git a/test_api_data.py b/tests/test_api_data.py similarity index 53% rename from test_api_data.py rename to tests/test_api_data.py index c45036921..5ea514e07 100644 --- a/test_api_data.py +++ b/tests/test_api_data.py @@ -21,49 +21,52 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import api_data as api import json +from api_data import extract_fields + + test_data = """ [{ - "Modified": "2008-11-15T00:00:00", - "Published": "2007-02-19T21:28:00", - "access": { - "authentication": "NONE", - "complexity": "MEDIUM", - "vector": "NETWORK" - }, - "cvss": 4.3, - "cvss-time": "2007-02-20T14:55:00", - "id": "CVE-2007-1004", - "impact": { - "availability": "NONE", - "confidentiality": "NONE", - "integrity": "PARTIAL" - }, - "reason": "Link", - "references": [ - "http://securityreason.com/securityalert/2264", - "http://www.securityfocus.com/archive/1/archive/1/460369/100/0/threaded", - "http://www.securityfocus.com/archive/1/archive/1/460412/100/0/threaded", - "http://www.securityfocus.com/archive/1/archive/1/460617/100/0/threaded", - "http://www.securityfocus.com/bid/22601", - "http://xforce.iss.net/xforce/xfdb/32580" - ], - "summary": "Mozilla Firefox might allow remote", - "vulnerable_configuration": [ - "cpe:2.3:a:mozilla:firefox:2.0:rc3" - ], - "vulnerable_configuration_cpe_2_2": [ - "cpe:/a:mozilla:firefox:2.0:rc3" - ]}] + "Modified": "2008-11-15T00:00:00", + "Published": "2007-02-19T21:28:00", + "access": { + "authentication": "NONE", + "complexity": "MEDIUM", + "vector": "NETWORK" + }, + "cvss": 4.3, + "cvss-time": "2007-02-20T14:55:00", + "id": "CVE-2007-1004", + "impact": { + "availability": "NONE", + "confidentiality": "NONE", + "integrity": "PARTIAL" + }, + "reason": "Link", + "references": [ + "http://securityreason.com/securityalert/2264", + "http://www.securityfocus.com/archive/1/archive/1/460369/100/0/threaded", + "http://www.securityfocus.com/archive/1/archive/1/460412/100/0/threaded", + "http://www.securityfocus.com/archive/1/archive/1/460617/100/0/threaded", + "http://www.securityfocus.com/bid/22601", + "http://xforce.iss.net/xforce/xfdb/32580" + ], + "summary": "Mozilla Firefox might allow remote", + "vulnerable_configuration": [ + "cpe:2.3:a:mozilla:firefox:2.0:rc3" + ], + "vulnerable_configuration_cpe_2_2": [ + "cpe:/a:mozilla:firefox:2.0:rc3" + ] +}] """ def test_extract_fields_data(): fields_names = ['id', 'cvss', 'summary'] data = json.loads(test_data) - extracted_data = api.extract_fields(data=data, fields_names=fields_names) + extracted_data = extract_fields(data=data, fields_names=fields_names) assert extracted_data == [{'cvss': 4.3, 'id': 'CVE-2007-1004', 'summary': 'Mozilla Firefox might allow remote'}] @@ -72,13 +75,13 @@ def test_extract_fields_data(): def test_extract_fields(): fields_names = [] data = json.loads(test_data) - extracted_data = api.extract_fields(data=data, fields_names=fields_names) + extracted_data = extract_fields(data=data, fields_names=fields_names) assert extracted_data == [{}] fields_names = [''] - extracted_data = api.extract_fields(data=data, fields_names=fields_names) + extracted_data = extract_fields(data=data, fields_names=fields_names) assert extracted_data == [{'': None}] fields_names = ['invalid_field'] - extracted_data = api.extract_fields(data=data, fields_names=fields_names) + extracted_data = extract_fields(data=data, fields_names=fields_names) assert extracted_data == [{'invalid_field': None}] diff --git a/tests/test_scrapers.py b/tests/test_scrapers.py new file mode 100644 index 000000000..9fc31eeb0 --- /dev/null +++ b/tests/test_scrapers.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +from scraper import ubuntu +from scraper import debian + + +def test_ubuntu_extract_cves(): + test_input = """ + + CVE-2002-2439 + gcc-4.4 + needs-triage* + needs-triage + DNE + DNE + DNE + DNE + DNE + + Mitre + LP + Debian + + + """ + + expected = ( + ['CVE-2002-2439'], + ['High'], + ['gcc-4.4'], + ) + assert expected == ubuntu.extract_cves(test_input) + + +def test_debian_extract_tracker_paths(): + test_input = """ + + """ + + expected = [ + '/tracker/status/release/unstable', + '/tracker/status/release/testing', + '/tracker/status/release/stable', + '/tracker/status/release/stable-backports', + '/tracker/status/release/oldstable', + '/tracker/status/release/oldstable-backports', + '/tracker/status/release/oldoldstable', + '/tracker/status/release/oldoldstable-backports', + '/tracker/status/dtsa-candidates', + '/tracker/status/todo', + '/tracker/status/undetermined', + '/tracker/status/unimportant', + '/tracker/status/itp', + '/tracker/status/unreported', + '/tracker/data/unknown-packages', + '/tracker/data/fake-names', + '/tracker/data/missing-epochs', + '/tracker/data/latently-vulnerable', + '/tracker/data/funny-versions', + '/tracker/data/releases', + '/tracker/data/json', + ] + + assert expected == debian.extract_tracker_paths(test_input) + + +def test_debian_extract_cves_from_tracker(): + test_input = """ + + 389-ds-base + CVE-2016-5416 + not yet assigned? + + """ + + expected = ( + ['CVE-2016-5416'], + ['389-ds-base'], + ['not yet assigned'], + ) + + assert expected == debian.extract_cves_from_tracker(test_input)