Refactor scraper logic and datastructure #16

scraper/debian.py

@@ -21,74 +21,47 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-import logging
-import re
+import json
 from urllib.request import urlopen
 
-import bs4
 
+DEBIAN_TRACKER_URL = 'https://security-tracker.debian.org/tracker/data/json'
 
-DEBIAN_ROOT_URL = 'https://security-tracker.debian.org'
 
-
-def extract_tracker_paths(html):
-    """
-    Return a list of tracker URL paths extracted from the given `html` input.
+def json_data(url=DEBIAN_TRACKER_URL):
     """
-    soup = bs4.BeautifulSoup(html, 'lxml')
-    tracker_links = soup.findAll('a', href=re.compile('^/track+.*'))
-    return [link.get('href') for link in tracker_links]
-
-
-def extract_cves_from_tracker(html):
+    Return Debian vulnerabilities data fetched from `url`.
     """
-    Return all CVEs extracted from the given `html` input.
-    """
-    cve_id = []
-    package_name = []
-    vulnerability_status = []
-    soup = bs4.BeautifulSoup(html, 'lxml')
-
-    for tag in soup.find_all('a'):
-        href = tag.get('href')
-
-        if re.search('/tracker/CVE-(.+)', href):
-            id = re.findall('(?<=/tracker/).*', href)
-            cve_id.append(id[0])
+    debian_data = urlopen(url).read()
+    return json.loads(debian_data)
 
-        if re.search('^/tracker/TEMP-+.*', href):
-            id = re.findall('(?<=/tracker/).*', href)
-            cve_id.append(id[0])
 
-        if re.search('/tracker/source-package/(.+)', href):
-            pkg = re.findall('(?<=/tracker/source-package/).*', href)
-            package_name.append(pkg[0])
-
-        # if package name is empty, use the previous package name
-        if href == '/tracker/source-package/':
-            package_name.append(pkg)
-
-    for tag in soup.find_all('td'):
-        if 'medium' in tag or 'low' in tag or 'not yet assigned' in tag:
-            vulnerability_status.append(tag.text)
-        elif tag.find_all('span', {'class': 'red'}) and tag.text == 'high**' or tag.text == 'high':
-            vulnerability_status.append(tag.text)
-
-    return cve_id, package_name, vulnerability_status
-
-
-def scrape_cves():
+def extract_data(debian_data, base_release='jessie'):
     """
-    Runs the full scraping process of Debian CVEs.
+    Return a sequence of mappings for each existing combination of
+    package and vulnerability from a mapping of Debian vulnerabilities
+    data.
     """
-    tracker_root_html = urlopen(f'{DEBIAN_ROOT_URL}/tracker/').read()
-    tracker_paths = extract_tracker_paths(tracker_root_html)
-
-    cves = []
-    for tracker_path in tracker_paths:
-        tracker_url = f'{DEBIAN_ROOT_URL}{tracker_path}/'
-        logging.info(f'Visiting: {tracker_url}')
-        html = urlopen(tracker_url).read()
-        cves.append(extract_cves_from_tracker(html))
-
-    return cves
+    package_vulns = []
+
+    for package_name, vulnerabilities in debian_data.items():
+        if not vulnerabilities or not package_name:
+            continue
+
+        for vulnerability, details in vulnerabilities.items():
+            releases = details.get('releases')
+            if not releases:
+                continue
+
+            release = releases.get(base_release)
+            if not release:
+                continue
+
+            package_vulns.append({
+                'package_name': package_name,
+                'vulnerability_id': vulnerability,
+                'status': release.get('status'),
+                'urgency': release.get('urgency'),
+                'fixed_version': release.get('fixed_version')
+            })
+    return package_vulns

scraper/ubuntu.py

@@ -21,7 +21,6 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-import re
 from urllib.request import urlopen
 
 import bs4
@@ -33,26 +32,19 @@
 def extract_cves(html):
     soup = bs4.BeautifulSoup(html, 'lxml')
 
-    cve_id = []
-    package_name = []
-    vulnerability_status = []
+    # Exclude the header row which has no class attribute
+    rows = soup.find_all('tr', attrs={'class': True})
 
-    for tag in soup.find_all('tr'):
-        if re.match('<\w+\s\w+="(\w+)">', str(tag)):
-            status = re.findall('<\w+\s\w+="(\w+)">', str(tag))
-            vulnerability_status.append(status[0])
+    cves = []
+    for row in rows:
+        columns = row.text.split()
+        cves.append({
+            'cve_id': columns[0],
+            'package_name': columns[1],
+            'vulnerability_status': row.get('class')[0],
+        })
 
-    for tag in soup.find_all('a'):
-        href = tag.get('href', None)
-
-        if re.findall('^CVE.+', href):
-            cve_id.append(href)
-
-        if re.match('pkg+.*', href):
-            pkg = re.findall('pkg/(.+)\.html', href)
-            package_name.append(pkg[0])
-
-    return cve_id, vulnerability_status, package_name
+    return cves
 
 
 def scrape_cves():

tests/test_data/debian.json

@@ -0,0 +1,105 @@
+{"mimetex": {
+
+"CVE-2009-2458": {
+	"scope": "remote", 
+	"debianbug": 537254, 
+	"description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX, when downloaded before 20090713, allow remote attackers to execute arbitrary code via a TeX file with long (1) picture, (2) circle, or (3) input tags.", 
+	"releases": 
+		{"stretch": 
+			{"status": "resolved", 
+			 "repositories": {"stretch": "1.74-1"}, 
+			 "urgency": "medium", 
+			 "fixed_version": "1.50-1.1"}, 
+		"jessie": 
+			{"status": "resolved", 
+			"repositories": {"jessie": "1.74-1"}, 
+			"urgency": "medium", 
+			"fixed_version": "1.50-1.1"}, 
+		"buster": 
+			{"status": "resolved", 
+			"repositories": {"buster": "1.74-1"}, 
+			"urgency": "medium", 
+			"fixed_version": "1.50-1.1"}, 
+		"wheezy": 
+			{"status": "resolved", 
+			"repositories": {"wheezy": "1.73-2"}, 
+			"urgency": "medium", 
+			"fixed_version": "1.50-1.1"}, 
+		"sid": 
+			{"status": "resolved", 
+			"repositories": {"sid": "1.74-1"}, 
+			"urgency": "medium", 
+			"fixed_version": "1.50-1.1"}}}, 
+
+"CVE-2009-2459": 
+	{"scope": "un-remote", 
+	"debianbug": 537254, 
+	"description": "Multiple unspecified vulnerabilities in mimeTeX, when downloaded before 20090713, have unknown impact and attack vectors related to the (1) \\environ, (2) \\input, and (3) \\counter TeX directives.", 
+	"releases": 
+	{"stretch": 
+		{"status": "resolved", 
+		"repositories": {"stretch": "1.74-1"}, 
+		"urgency": "medium", 
+		"fixed_version": "1.50-1.1"}, 
+	"jessie": 
+		{"status": "not-resolved", 
+		"repositories": {"jessie": "1.74-1"}, 
+		"urgency": "medium", 
+		"fixed_version": "1.50-1.1"}, 
+	"buster": 
+		{"status": "resolved", 
+		"repositories": {"buster": "1.74-1"}, 
+		"urgency": "medium", 
+		"fixed_version": "1.50-1.1"}, 
+	"wheezy": 
+		{"status": "resolved", 
+		"repositories": {"wheezy": "1.73-2"},
+		"urgency": "medium", 
+		"fixed_version": "1.50-1.1"}, 
+	"sid": 
+		{"status": "resolved", 
+		"repositories": {"sid": "1.74-1"}, 
+		"urgency": "medium", 
+		"fixed_version": "1.50-1.1"}}}}, 
+
+"git-repair": {
+	"TEMP-0807341-84E914": 
+		{"debianbug": 807341, 
+		"releases": 
+			{"jessie": 
+				{"status": "open", 
+				 "repositories": {"jessie": "1.20140914"}, 
+				 "urgency": "unimportant"}, 
+			"sid": 
+				{"status": "resolved", 
+				"repositories": {"sid": "1.20151215-1"}, 
+		 		"urgency": "unimportant", 
+		 		"fixed_version": "1.20151215-1"}}}}, 
+
+"sysvinit": {
+	"TEMP-0517018-A83CE6": 
+		{"debianbug": 517018, 
+		"releases": 
+			{"stretch": 
+				{"status": "open",
+				 "repositories": {"stretch": "2.88dsf-59.9"}, 
+				 "urgency": "unimportant"}, 
+
+			"buster": 	
+				{"status": "open", 
+				"repositories": {"buster": "2.88dsf-59.9"}, 
+				"urgency": "unimportant"}, 
+
+			"wheezy": 
+				{"status": "open", 
+				"repositories": {"wheezy": "2.88dsf-41+deb7u1"}, 
+				"urgency": "unimportant"}, 
+
+			"sid": 		
+				{"status": "open", 
+				"repositories": {"sid": "2.88dsf-59.9"}, 
+				"urgency": "unimportant"}
+				}	
+			}
+		}
+	}