From 9adbf7d14d64aad61df4bc5e26c2dda83ae2faf5 Mon Sep 17 00:00:00 2001 From: Islam ElHakmi Date: Thu, 12 Mar 2020 16:30:49 +0200 Subject: [PATCH 1/6] feat: add lwn scraper Signed-off-by: Islam ElHakmi --- vulnerabilities/data_dump.py | 29 ++++- vulnerabilities/management/commands/import.py | 5 +- vulnerabilities/scraper/lwn.py | 112 ++++++++++++++++++ 3 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 vulnerabilities/scraper/lwn.py diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py index 831252c37..5909ab305 100644 --- a/vulnerabilities/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -27,7 +27,7 @@ from vulnerabilities.models import ResolvedPackage from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference - +import json def debian_dump(extract_data, base_release='jessie'): """ @@ -262,3 +262,30 @@ def rust_dump(extract_data): vulnerability=vulnerability, package=unaffected_package ) + +def lwn_dump(extract_data): + for package_name in extract_data: + for vuln in extract_data[package_name]: + ap, _ = Package.objects.get_or_create( + name=package_name, + namespace=vuln['distributor'], + ) + + vulnerability = Vulnerability.objects.create( + summary=vuln['summary'], + ) + + VulnerabilityReference.objects.create( + vulnerability=vulnerability, + url=vuln['advisory_link'], + reference_id=vuln['advisory_id'] + ) + for cve in vuln['cve_ids']: + vulnerability, _ = Vulnerability.objects.get_or_create( + cve_id=cve, + ) + VulnerabilityReference.objects.create( + vulnerability=vulnerability, + url=vuln['advisory_link'], + reference_id=vuln['advisory_id'] + ) diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index ebf59b89c..5500179e2 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -24,7 +24,7 @@ from django.core.management.base import BaseCommand, CommandError from vulnerabilities import data_dump as dd -from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby, rust +from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby, rust, lwn IMPORTERS = { 'rust': lambda: dd.rust_dump(rust.import_vulnerabilities()), @@ -32,7 +32,8 @@ 'npm': lambda: dd.npm_dump(npm.scrape_vulnerabilities()), 'debian': lambda: dd.debian_dump(debian.scrape_vulnerabilities()), 'ubuntu': lambda: dd.ubuntu_dump(ubuntu.scrape_cves()), - 'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities()) + 'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities()), + 'lwn': lambda: dd.lwn_dump(lwn.scrape_vulnerabilities()) } diff --git a/vulnerabilities/scraper/lwn.py b/vulnerabilities/scraper/lwn.py new file mode 100644 index 000000000..942f1f385 --- /dev/null +++ b/vulnerabilities/scraper/lwn.py @@ -0,0 +1,112 @@ +# Author: Navonil Das (@NavonilDas) +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +from bs4 import BeautifulSoup as bs +import requests as rq +import re + + +base_url = "https://lwn.net/" + + +def extractPackageData(advisoryLink,dist,advisoryId): + + content = rq.get(advisoryLink).content + soup = bs(content,"html.parser") + text = soup.find('div',{'class':'ArticleText'}).get_text() + phrases = text.split('\n') + cves = [] + references = [] + summary = "" + for i in range(len(phrases)): + words = phrases[i].split() + if phrases[i].startswith('Subject:'): + summary = phrases[i+1].strip() + for word in words: + if word.startswith('CVE-') and word != 'CVE-ID': + cves.append(word) + elif word.startswith('https://') or word.startswith('http://'): + references.append(word) + + cves = list(set(cves)) + + dist = re.sub(r'\W+', '', dist).replace('_','').lower() + + return {'cve_ids':cves,'references':references,'summary':summary,'advisory_id':advisoryId,'distributor':dist,'advisory_link':advisoryLink} + +def getDistributors(): + url = base_url+"Alerts/" + content = rq.get(url).content + soup = bs(content,"html.parser") + dists = [] + distsLinks = [] + tables = soup.find_all('table',{'cellspacing':"4",}) + + for table in tables: + distsLinks += table.find_all('a') + + for a in distsLinks: + dists.append(a['href']) + + return dists + +def scrape_vulnerabilities(): + dists = getDistributors() + packagesVulns = {} + dists = dists[:5] + for dist in dists: + distUrl = base_url+"Alerts/"+dist+"?n=10" + distContent = rq.get(distUrl).content + distSoup = bs(distContent,"html.parser") + articleSoup = distSoup.find('div',{'class':'ArticleText'}) + text = articleSoup.get_text() + total = int(text[text.find("(")+1:text.find(")")].split()[0]) + curr_offset = 0 + while curr_offset < total: + + table = articleSoup.find('table',{'cellpadding':4}) + + data = table.find_all('tr') + data = data[1:] + for row in data: + rowElements = row.find_all('td') + aTag = rowElements[0].find('a') + advisoryLink = base_url[:-1]+aTag['href'] + advisoryId = aTag.get_text() + package_names = rowElements[1].get_text().split(',') + date = rowElements[2].get_text() + for package_name in package_names: + extracted_data = extractPackageData(advisoryLink,dist,advisoryId) + if packagesVulns.get(package_name): + packagesVulns[package_name].append(extracted_data) + else: + packagesVulns[package_name] = [extracted_data] + + + curr_offset += 100 + distUrl = distUrl+"&offset="+str(curr_offset) + distSoup = bs(distContent,"html.parser") + articleSoup = distSoup.find('div',{'class':'ArticleText'}) + break + + return packagesVulns From a85f24b345e9140ce1b02a81cf9fae64bb6426a0 Mon Sep 17 00:00:00 2001 From: Islam ElHakmi Date: Thu, 12 Mar 2020 16:36:32 +0200 Subject: [PATCH 2/6] fix: remove limitations Signed-off-by: Islam ElHakmi --- vulnerabilities/data_dump.py | 1 - vulnerabilities/scraper/lwn.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py index 5909ab305..0808b7f43 100644 --- a/vulnerabilities/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -27,7 +27,6 @@ from vulnerabilities.models import ResolvedPackage from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference -import json def debian_dump(extract_data, base_release='jessie'): """ diff --git a/vulnerabilities/scraper/lwn.py b/vulnerabilities/scraper/lwn.py index 942f1f385..e50e7f875 100644 --- a/vulnerabilities/scraper/lwn.py +++ b/vulnerabilities/scraper/lwn.py @@ -75,7 +75,7 @@ def scrape_vulnerabilities(): packagesVulns = {} dists = dists[:5] for dist in dists: - distUrl = base_url+"Alerts/"+dist+"?n=10" + distUrl = base_url+"Alerts/"+dist+"?n=100" distContent = rq.get(distUrl).content distSoup = bs(distContent,"html.parser") articleSoup = distSoup.find('div',{'class':'ArticleText'}) @@ -107,6 +107,6 @@ def scrape_vulnerabilities(): distUrl = distUrl+"&offset="+str(curr_offset) distSoup = bs(distContent,"html.parser") articleSoup = distSoup.find('div',{'class':'ArticleText'}) - break + return packagesVulns From c831cfe8e1934b349d93d7123f32cb687795ffae Mon Sep 17 00:00:00 2001 From: Islam ElHakmi Date: Thu, 12 Mar 2020 16:55:28 +0200 Subject: [PATCH 3/6] fix: fix CI lint Signed-off-by: Islam ElHakmi --- vulnerabilities/scraper/lwn.py | 154 +++++++++++++++++---------------- 1 file changed, 80 insertions(+), 74 deletions(-) diff --git a/vulnerabilities/scraper/lwn.py b/vulnerabilities/scraper/lwn.py index e50e7f875..d91dd0cb0 100644 --- a/vulnerabilities/scraper/lwn.py +++ b/vulnerabilities/scraper/lwn.py @@ -29,84 +29,90 @@ base_url = "https://lwn.net/" -def extractPackageData(advisoryLink,dist,advisoryId): - - content = rq.get(advisoryLink).content - soup = bs(content,"html.parser") - text = soup.find('div',{'class':'ArticleText'}).get_text() - phrases = text.split('\n') - cves = [] - references = [] - summary = "" - for i in range(len(phrases)): - words = phrases[i].split() - if phrases[i].startswith('Subject:'): - summary = phrases[i+1].strip() - for word in words: - if word.startswith('CVE-') and word != 'CVE-ID': - cves.append(word) - elif word.startswith('https://') or word.startswith('http://'): - references.append(word) - - cves = list(set(cves)) - - dist = re.sub(r'\W+', '', dist).replace('_','').lower() - - return {'cve_ids':cves,'references':references,'summary':summary,'advisory_id':advisoryId,'distributor':dist,'advisory_link':advisoryLink} +def extractPackageData(advisoryLink, dist, advisoryId): + + content = rq.get(advisoryLink).content + soup = bs(content, "html.parser") + text = soup.find('div', {'class': 'ArticleText'}).get_text() + phrases = text.split('\n') + cves = [] + references = [] + summary = "" + for i in range(len(phrases)): + words = phrases[i].split() + if phrases[i].startswith('Subject:'): + summary = phrases[i + 1].strip() + for word in words: + if word.startswith('CVE-') and word != 'CVE-ID': + cves.append(word) + elif word.startswith('https://') or word.startswith('http://'): + references.append(word) + + cves = list(set(cves)) + + dist = re.sub(r'\W+', '', dist).replace('_', '').lower() + + return { + 'cve_ids': cves, + 'references': references, + 'summary': summary, + 'advisory_id': advisoryId, + 'distributor': dist, + 'advisory_link': advisoryLink} + def getDistributors(): - url = base_url+"Alerts/" - content = rq.get(url).content - soup = bs(content,"html.parser") - dists = [] - distsLinks = [] - tables = soup.find_all('table',{'cellspacing':"4",}) + url = base_url + "Alerts/" + content = rq.get(url).content + soup = bs(content, "html.parser") + dists = [] + distsLinks = [] + tables = soup.find_all('table', {'cellspacing': "4", }) + + for table in tables: + distsLinks += table.find_all('a') - for table in tables: - distsLinks += table.find_all('a') + for a in distsLinks: + dists.append(a['href']) - for a in distsLinks: - dists.append(a['href']) + return dists - return dists def scrape_vulnerabilities(): - dists = getDistributors() - packagesVulns = {} - dists = dists[:5] - for dist in dists: - distUrl = base_url+"Alerts/"+dist+"?n=100" - distContent = rq.get(distUrl).content - distSoup = bs(distContent,"html.parser") - articleSoup = distSoup.find('div',{'class':'ArticleText'}) - text = articleSoup.get_text() - total = int(text[text.find("(")+1:text.find(")")].split()[0]) - curr_offset = 0 - while curr_offset < total: - - table = articleSoup.find('table',{'cellpadding':4}) - - data = table.find_all('tr') - data = data[1:] - for row in data: - rowElements = row.find_all('td') - aTag = rowElements[0].find('a') - advisoryLink = base_url[:-1]+aTag['href'] - advisoryId = aTag.get_text() - package_names = rowElements[1].get_text().split(',') - date = rowElements[2].get_text() - for package_name in package_names: - extracted_data = extractPackageData(advisoryLink,dist,advisoryId) - if packagesVulns.get(package_name): - packagesVulns[package_name].append(extracted_data) - else: - packagesVulns[package_name] = [extracted_data] - - - curr_offset += 100 - distUrl = distUrl+"&offset="+str(curr_offset) - distSoup = bs(distContent,"html.parser") - articleSoup = distSoup.find('div',{'class':'ArticleText'}) - - - return packagesVulns + dists = getDistributors() + packagesVulns = {} + for dist in dists: + distUrl = base_url + "Alerts/" + dist + "?n=100" + distContent = rq.get(distUrl).content + distSoup = bs(distContent, "html.parser") + articleSoup = distSoup.find('div', {'class': 'ArticleText'}) + text = articleSoup.get_text() + total = int(text[text.find("(") + 1:text.find(")")].split()[0]) + curr_offset = 0 + while curr_offset < total: + + table = articleSoup.find('table', {'cellpadding': 4}) + + data = table.find_all('tr') + data = data[1:] + for row in data: + rowElements = row.find_all('td') + aTag = rowElements[0].find('a') + advisoryLink = base_url[:-1] + aTag['href'] + advisoryId = aTag.get_text() + package_names = rowElements[1].get_text().split(',') + date = rowElements[2].get_text() + for package_name in package_names: + extracted_data = extractPackageData( + advisoryLink, dist, advisoryId) + if packagesVulns.get(package_name): + packagesVulns[package_name].append(extracted_data) + else: + packagesVulns[package_name] = [extracted_data] + + curr_offset += 100 + distUrl = distUrl + "&offset=" + str(curr_offset) + distSoup = bs(distContent, "html.parser") + articleSoup = distSoup.find('div', {'class': 'ArticleText'}) + + return packagesVulns From 08b08b36d14a37a142e7eee2af61e575341badf3 Mon Sep 17 00:00:00 2001 From: Islam ElHakmi Date: Thu, 12 Mar 2020 16:57:22 +0200 Subject: [PATCH 4/6] fix: fix lint errors Signed-off-by: Islam ElHakmi --- vulnerabilities/data_dump.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py index 0808b7f43..602960f4c 100644 --- a/vulnerabilities/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -28,6 +28,7 @@ from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference + def debian_dump(extract_data, base_release='jessie'): """ Save data scraped from Debian' security tracker. @@ -262,6 +263,7 @@ def rust_dump(extract_data): package=unaffected_package ) + def lwn_dump(extract_data): for package_name in extract_data: for vuln in extract_data[package_name]: From 2a3e5f8664436efcde61b73e5813a94b461e35ae Mon Sep 17 00:00:00 2001 From: Islam ElHakmi Date: Thu, 12 Mar 2020 17:11:00 +0200 Subject: [PATCH 5/6] fix: use urllib instead of requests Signed-off-by: Islam ElHakmi --- vulnerabilities/scraper/lwn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/scraper/lwn.py b/vulnerabilities/scraper/lwn.py index d91dd0cb0..f19fd4064 100644 --- a/vulnerabilities/scraper/lwn.py +++ b/vulnerabilities/scraper/lwn.py @@ -22,7 +22,7 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. from bs4 import BeautifulSoup as bs -import requests as rq +from urllib.request import urlopen import re @@ -31,7 +31,7 @@ def extractPackageData(advisoryLink, dist, advisoryId): - content = rq.get(advisoryLink).content + content = urlopen(advisoryLink).read() soup = bs(content, "html.parser") text = soup.find('div', {'class': 'ArticleText'}).get_text() phrases = text.split('\n') @@ -63,7 +63,7 @@ def extractPackageData(advisoryLink, dist, advisoryId): def getDistributors(): url = base_url + "Alerts/" - content = rq.get(url).content + content = urlopen(url).read() soup = bs(content, "html.parser") dists = [] distsLinks = [] @@ -83,7 +83,7 @@ def scrape_vulnerabilities(): packagesVulns = {} for dist in dists: distUrl = base_url + "Alerts/" + dist + "?n=100" - distContent = rq.get(distUrl).content + distContent = urlopen(distUrl).read() distSoup = bs(distContent, "html.parser") articleSoup = distSoup.find('div', {'class': 'ArticleText'}) text = articleSoup.get_text() From 3df3d73ae7437d119cff9f259e59b4b3f22a4e65 Mon Sep 17 00:00:00 2001 From: Islam ElHakmi Date: Fri, 13 Mar 2020 13:24:05 +0200 Subject: [PATCH 6/6] fix: improve loop code Signed-off-by: Islam ElHakmi --- vulnerabilities/scraper/lwn.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/scraper/lwn.py b/vulnerabilities/scraper/lwn.py index f19fd4064..71db8fdf0 100644 --- a/vulnerabilities/scraper/lwn.py +++ b/vulnerabilities/scraper/lwn.py @@ -1,5 +1,5 @@ -# Author: Navonil Das (@NavonilDas) -# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# Author: Islam Hiko (@EslamHiko) +# Copyright (c) 2020 nexB Inc. and others. All rights reserved. # http://nexb.com and https://github.com/nexB/vulnerablecode/ # The VulnerableCode software is licensed under the Apache License version 2.0. # Data generated with VulnerableCode require an acknowledgment. @@ -88,8 +88,7 @@ def scrape_vulnerabilities(): articleSoup = distSoup.find('div', {'class': 'ArticleText'}) text = articleSoup.get_text() total = int(text[text.find("(") + 1:text.find(")")].split()[0]) - curr_offset = 0 - while curr_offset < total: + for curr_offset in range(0, total, 100): table = articleSoup.find('table', {'cellpadding': 4}) @@ -110,7 +109,6 @@ def scrape_vulnerabilities(): else: packagesVulns[package_name] = [extracted_data] - curr_offset += 100 distUrl = distUrl + "&offset=" + str(curr_offset) distSoup = bs(distContent, "html.parser") articleSoup = distSoup.find('div', {'class': 'ArticleText'})