feat: add lwn scraper

Author: EslamHiko

vulnerabilities/data_dump.py

@@ -27,7 +27,7 @@
 from vulnerabilities.models import ResolvedPackage
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityReference
-
+import json
 
 def debian_dump(extract_data, base_release='jessie'):
     """
@@ -262,3 +262,30 @@ def rust_dump(extract_data):
                 vulnerability=vulnerability,
                 package=unaffected_package
             )
+
+def lwn_dump(extract_data):
+    for package_name in extract_data:
+        for vuln in extract_data[package_name]:
+            ap, _ = Package.objects.get_or_create(
+                name=package_name,
+                namespace=vuln['distributor'],
+            )
+
+            vulnerability = Vulnerability.objects.create(
+                summary=vuln['summary'],
+            )
+
+            VulnerabilityReference.objects.create(
+                vulnerability=vulnerability,
+                url=vuln['advisory_link'],
+                reference_id=vuln['advisory_id']
+            )
+            for cve in vuln['cve_ids']:
+                vulnerability, _ = Vulnerability.objects.get_or_create(
+                    cve_id=cve,
+                )
+                VulnerabilityReference.objects.create(
+                    vulnerability=vulnerability,
+                    url=vuln['advisory_link'],
+                    reference_id=vuln['advisory_id']
+                )

vulnerabilities/management/commands/import.py

@@ -24,15 +24,16 @@
 from django.core.management.base import BaseCommand, CommandError
 
 from vulnerabilities import data_dump as dd
-from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby, rust
+from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby, rust, lwn
 
 IMPORTERS = {
     'rust': lambda: dd.rust_dump(rust.import_vulnerabilities()),
     'ruby': lambda: dd.ruby_dump(ruby.import_vulnerabilities()),
     'npm': lambda: dd.npm_dump(npm.scrape_vulnerabilities()),
     'debian': lambda: dd.debian_dump(debian.scrape_vulnerabilities()),
     'ubuntu': lambda: dd.ubuntu_dump(ubuntu.scrape_cves()),
-    'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities())
+    'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities()),
+    'lwn': lambda: dd.lwn_dump(lwn.scrape_vulnerabilities())
 }
 
 

vulnerabilities/scraper/lwn.py

@@ -0,0 +1,112 @@
+# Author: Navonil Das (@NavonilDas)
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from bs4 import BeautifulSoup as bs
+import requests as rq
+import re
+
+
+base_url = "https://lwn.net/"
+
+
+def extractPackageData(advisoryLink,dist,advisoryId):
+
+	content = rq.get(advisoryLink).content
+	soup = bs(content,"html.parser")
+	text = soup.find('div',{'class':'ArticleText'}).get_text()
+	phrases = text.split('\n')
+	cves = []
+	references = []
+	summary = ""
+	for i in range(len(phrases)):
+		words = phrases[i].split()
+		if phrases[i].startswith('Subject:'):
+			summary = phrases[i+1].strip()
+		for word in words:
+			if word.startswith('CVE-') and word != 'CVE-ID':
+				cves.append(word)
+			elif word.startswith('https://') or word.startswith('http://'):
+				references.append(word)
+
+	cves = list(set(cves))
+
+	dist = re.sub(r'\W+', '', dist).replace('_','').lower()
+
+	return {'cve_ids':cves,'references':references,'summary':summary,'advisory_id':advisoryId,'distributor':dist,'advisory_link':advisoryLink}
+
+def getDistributors():
+	url = base_url+"Alerts/"
+	content = rq.get(url).content
+	soup = bs(content,"html.parser")
+	dists = []
+	distsLinks = []
+	tables = soup.find_all('table',{'cellspacing':"4",})
+
+	for table in tables:
+		distsLinks += table.find_all('a')
+
+	for a in distsLinks:
+		dists.append(a['href'])
+
+	return dists
+
+def scrape_vulnerabilities():
+	dists = getDistributors()
+	packagesVulns = {}
+	dists = dists[:5]
+	for dist in dists:
+		distUrl = base_url+"Alerts/"+dist+"?n=10"
+		distContent = rq.get(distUrl).content
+		distSoup = bs(distContent,"html.parser")
+		articleSoup = distSoup.find('div',{'class':'ArticleText'})
+		text = articleSoup.get_text()
+		total = int(text[text.find("(")+1:text.find(")")].split()[0])
+		curr_offset = 0
+		while curr_offset < total:
+
+			table = articleSoup.find('table',{'cellpadding':4})
+
+			data = table.find_all('tr')
+			data = data[1:]
+			for row in data:
+				rowElements = row.find_all('td')
+				aTag = rowElements[0].find('a')
+				advisoryLink = base_url[:-1]+aTag['href']
+				advisoryId = aTag.get_text()
+				package_names = rowElements[1].get_text().split(',')
+				date = rowElements[2].get_text()
+				for package_name in package_names:
+					extracted_data = extractPackageData(advisoryLink,dist,advisoryId)
+					if packagesVulns.get(package_name):
+						packagesVulns[package_name].append(extracted_data)
+					else:
+						packagesVulns[package_name] = [extracted_data]
+
+
+			curr_offset += 100
+			distUrl = distUrl+"&offset="+str(curr_offset)
+			distSoup = bs(distContent,"html.parser")
+			articleSoup = distSoup.find('div',{'class':'ArticleText'})
+			break
+
+	return packagesVulns