Skip to content

Commit 3d78b73

Browse files
tdrueztdruez
authored
Merge pull request #17 from nexB/scraper_datastructure
Refactor scraper logic and datastructure #16
2 parents ece0cdc + 3e5009e commit 3d78b73

5 files changed

Lines changed: 2059 additions & 168 deletions

File tree

‎scraper/debian.py‎

Lines changed: 33 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -21,74 +21,47 @@
2121
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
2222
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
2323

24-
import logging
25-
import re
24+
import json
2625
from urllib.request import urlopen
2726

28-
import bs4
2927

28+
DEBIAN_TRACKER_URL = 'https://security-tracker.debian.org/tracker/data/json'
3029

31-
DEBIAN_ROOT_URL = 'https://security-tracker.debian.org'
3230

33-
34-
def extract_tracker_paths(html):
35-
"""
36-
Return a list of tracker URL paths extracted from the given `html` input.
31+
def json_data(url=DEBIAN_TRACKER_URL):
3732
"""
38-
soup = bs4.BeautifulSoup(html, 'lxml')
39-
tracker_links = soup.findAll('a', href=re.compile('^/track+.*'))
40-
return [link.get('href') for link in tracker_links]
41-
42-
43-
def extract_cves_from_tracker(html):
33+
Return Debian vulnerabilities data fetched from `url`.
4434
"""
45-
Return all CVEs extracted from the given `html` input.
46-
"""
47-
cve_id = []
48-
package_name = []
49-
vulnerability_status = []
50-
soup = bs4.BeautifulSoup(html, 'lxml')
51-
52-
for tag in soup.find_all('a'):
53-
href = tag.get('href')
54-
55-
if re.search('/tracker/CVE-(.+)', href):
56-
id = re.findall('(?<=/tracker/).*', href)
57-
cve_id.append(id[0])
35+
debian_data = urlopen(url).read()
36+
return json.loads(debian_data)
5837

59-
if re.search('^/tracker/TEMP-+.*', href):
60-
id = re.findall('(?<=/tracker/).*', href)
61-
cve_id.append(id[0])
6238

63-
if re.search('/tracker/source-package/(.+)', href):
64-
pkg = re.findall('(?<=/tracker/source-package/).*', href)
65-
package_name.append(pkg[0])
66-
67-
# if package name is empty, use the previous package name
68-
if href == '/tracker/source-package/':
69-
package_name.append(pkg)
70-
71-
for tag in soup.find_all('td'):
72-
if 'medium' in tag or 'low' in tag or 'not yet assigned' in tag:
73-
vulnerability_status.append(tag.text)
74-
elif tag.find_all('span', {'class': 'red'}) and tag.text == 'high**' or tag.text == 'high':
75-
vulnerability_status.append(tag.text)
76-
77-
return cve_id, package_name, vulnerability_status
78-
79-
80-
def scrape_cves():
39+
def extract_data(debian_data, base_release='jessie'):
8140
"""
82-
Runs the full scraping process of Debian CVEs.
41+
Return a sequence of mappings for each existing combination of
42+
package and vulnerability from a mapping of Debian vulnerabilities
43+
data.
8344
"""
84-
tracker_root_html = urlopen(f'{DEBIAN_ROOT_URL}/tracker/').read()
85-
tracker_paths = extract_tracker_paths(tracker_root_html)
86-
87-
cves = []
88-
for tracker_path in tracker_paths:
89-
tracker_url = f'{DEBIAN_ROOT_URL}{tracker_path}/'
90-
logging.info(f'Visiting: {tracker_url}')
91-
html = urlopen(tracker_url).read()
92-
cves.append(extract_cves_from_tracker(html))
93-
94-
return cves
45+
package_vulns = []
46+
47+
for package_name, vulnerabilities in debian_data.items():
48+
if not vulnerabilities or not package_name:
49+
continue
50+
51+
for vulnerability, details in vulnerabilities.items():
52+
releases = details.get('releases')
53+
if not releases:
54+
continue
55+
56+
release = releases.get(base_release)
57+
if not release:
58+
continue
59+
60+
package_vulns.append({
61+
'package_name': package_name,
62+
'vulnerability_id': vulnerability,
63+
'status': release.get('status'),
64+
'urgency': release.get('urgency'),
65+
'fixed_version': release.get('fixed_version')
66+
})
67+
return package_vulns

‎scraper/ubuntu.py‎

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
2222
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
2323

24-
import re
2524
from urllib.request import urlopen
2625

2726
import bs4
@@ -33,26 +32,19 @@
3332
def extract_cves(html):
3433
soup = bs4.BeautifulSoup(html, 'lxml')
3534

36-
cve_id = []
37-
package_name = []
38-
vulnerability_status = []
35+
# Exclude the header row which has no class attribute
36+
rows = soup.find_all('tr', attrs={'class': True})
3937

40-
for tag in soup.find_all('tr'):
41-
if re.match('<\w+\s\w+="(\w+)">', str(tag)):
42-
status = re.findall('<\w+\s\w+="(\w+)">', str(tag))
43-
vulnerability_status.append(status[0])
38+
cves = []
39+
for row in rows:
40+
columns = row.text.split()
41+
cves.append({
42+
'cve_id': columns[0],
43+
'package_name': columns[1],
44+
'vulnerability_status': row.get('class')[0],
45+
})
4446

45-
for tag in soup.find_all('a'):
46-
href = tag.get('href', None)
47-
48-
if re.findall('^CVE.+', href):
49-
cve_id.append(href)
50-
51-
if re.match('pkg+.*', href):
52-
pkg = re.findall('pkg/(.+)\.html', href)
53-
package_name.append(pkg[0])
54-
55-
return cve_id, vulnerability_status, package_name
47+
return cves
5648

5749

5850
def scrape_cves():

‎tests/test_data/debian.json‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
{"mimetex": {
2+
3+
"CVE-2009-2458": {
4+
"scope": "remote",
5+
"debianbug": 537254,
6+
"description": "Multiple stack-based buffer overflows in mimetex.cgi in mimeTeX, when downloaded before 20090713, allow remote attackers to execute arbitrary code via a TeX file with long (1) picture, (2) circle, or (3) input tags.",
7+
"releases":
8+
{"stretch":
9+
{"status": "resolved",
10+
"repositories": {"stretch": "1.74-1"},
11+
"urgency": "medium",
12+
"fixed_version": "1.50-1.1"},
13+
"jessie":
14+
{"status": "resolved",
15+
"repositories": {"jessie": "1.74-1"},
16+
"urgency": "medium",
17+
"fixed_version": "1.50-1.1"},
18+
"buster":
19+
{"status": "resolved",
20+
"repositories": {"buster": "1.74-1"},
21+
"urgency": "medium",
22+
"fixed_version": "1.50-1.1"},
23+
"wheezy":
24+
{"status": "resolved",
25+
"repositories": {"wheezy": "1.73-2"},
26+
"urgency": "medium",
27+
"fixed_version": "1.50-1.1"},
28+
"sid":
29+
{"status": "resolved",
30+
"repositories": {"sid": "1.74-1"},
31+
"urgency": "medium",
32+
"fixed_version": "1.50-1.1"}}},
33+
34+
"CVE-2009-2459":
35+
{"scope": "un-remote",
36+
"debianbug": 537254,
37+
"description": "Multiple unspecified vulnerabilities in mimeTeX, when downloaded before 20090713, have unknown impact and attack vectors related to the (1) \\environ, (2) \\input, and (3) \\counter TeX directives.",
38+
"releases":
39+
{"stretch":
40+
{"status": "resolved",
41+
"repositories": {"stretch": "1.74-1"},
42+
"urgency": "medium",
43+
"fixed_version": "1.50-1.1"},
44+
"jessie":
45+
{"status": "not-resolved",
46+
"repositories": {"jessie": "1.74-1"},
47+
"urgency": "medium",
48+
"fixed_version": "1.50-1.1"},
49+
"buster":
50+
{"status": "resolved",
51+
"repositories": {"buster": "1.74-1"},
52+
"urgency": "medium",
53+
"fixed_version": "1.50-1.1"},
54+
"wheezy":
55+
{"status": "resolved",
56+
"repositories": {"wheezy": "1.73-2"},
57+
"urgency": "medium",
58+
"fixed_version": "1.50-1.1"},
59+
"sid":
60+
{"status": "resolved",
61+
"repositories": {"sid": "1.74-1"},
62+
"urgency": "medium",
63+
"fixed_version": "1.50-1.1"}}}},
64+
65+
"git-repair": {
66+
"TEMP-0807341-84E914":
67+
{"debianbug": 807341,
68+
"releases":
69+
{"jessie":
70+
{"status": "open",
71+
"repositories": {"jessie": "1.20140914"},
72+
"urgency": "unimportant"},
73+
"sid":
74+
{"status": "resolved",
75+
"repositories": {"sid": "1.20151215-1"},
76+
"urgency": "unimportant",
77+
"fixed_version": "1.20151215-1"}}}},
78+
79+
"sysvinit": {
80+
"TEMP-0517018-A83CE6":
81+
{"debianbug": 517018,
82+
"releases":
83+
{"stretch":
84+
{"status": "open",
85+
"repositories": {"stretch": "2.88dsf-59.9"},
86+
"urgency": "unimportant"},
87+
88+
"buster":
89+
{"status": "open",
90+
"repositories": {"buster": "2.88dsf-59.9"},
91+
"urgency": "unimportant"},
92+
93+
"wheezy":
94+
{"status": "open",
95+
"repositories": {"wheezy": "2.88dsf-41+deb7u1"},
96+
"urgency": "unimportant"},
97+
98+
"sid":
99+
{"status": "open",
100+
"repositories": {"sid": "2.88dsf-59.9"},
101+
"urgency": "unimportant"}
102+
}
103+
}
104+
}
105+
}

0 commit comments

Comments
 (0)