Skip to content

Commit 2ab09b7

Browse files
sbs2001sbs2001
committed
Add ruby gems advisory importer
Signed-off-by: sbs2001 <shivam.sandbhor@gmail.com>
1 parent d389531 commit 2ab09b7

4 files changed

Lines changed: 186 additions & 23 deletions

File tree

‎requirements.txt‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,33 @@
1-
setuptools==36.5.0
2-
1+
attrs==19.3.0
32
beautifulsoup4==4.7.1
4-
lxml==4.3.3
5-
django==2.2.8
6-
djangorestframework==3.9.2
3+
dephell-specifier==0.2.1
4+
dj-database-url==0.4.2
5+
Django==2.2.8
76
django-filter==2.2.0
7+
djangorestframework==3.9.2
8+
gunicorn==19.7.1
9+
importlib-metadata==1.3.0
10+
lxml==4.3.3
11+
more-itertools==8.0.2
812
packageurl-python==0.8.7
9-
semantic-version==2.8.2
10-
11-
# Tests
12-
pytest==5.3.2
13-
pytest-django==3.7.0
13+
packaging==19.2
14+
pluggy==0.13.1
15+
psycopg2==2.8.4
16+
py==1.8.0
1417
pycodestyle==2.5.0
15-
16-
# Deployment
17-
gunicorn==19.7.1
18+
pyparsing==2.4.5
19+
pytest==5.3.2
20+
pytest-dependency==0.4.0
21+
pytest-django==3.7.0
22+
pytest-mock==1.13.0
23+
pytz==2019.3
24+
PyYAML==5.3
25+
saneyaml==0.4
26+
semantic-version==2.8.2
27+
six==1.13.0
28+
soupsieve==1.9.5
29+
sqlparse==0.3.0
30+
tqdm==4.41.1
31+
wcwidth==0.1.7
1832
whitenoise==5.0.1
19-
dj_database_url==0.4.2
20-
psycopg2==2.8.4
33+
zipp==0.6.0

‎vulnerabilities/data_dump.py‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -144,14 +144,14 @@ def archlinux_dump(extract_data):
144144

145145
for ap in affected_packages:
146146
ImpactedPackage.objects.get_or_create(
147-
vulnerability=vulnerability,
148-
package=ap,
147+
vulnerability=vulnerability,
148+
package=ap,
149149
)
150150

151151
for fp in fixed_packages:
152152
ResolvedPackage.objects.get_or_create(
153-
vulnerability=vulnerability,
154-
package=fp,
153+
vulnerability=vulnerability,
154+
package=fp,
155155
)
156156

157157

@@ -190,3 +190,44 @@ def npm_dump(extract_data):
190190
vulnerability=vulnerability,
191191
package=package_fixed
192192
)
193+
194+
195+
def ruby_dump(extract_data):
196+
for package_data in extract_data:
197+
198+
if(package_data['severity']):
199+
vulnerability, _ = Vulnerability.objects.get_or_create(
200+
cve_id=package_data['cve_id'],
201+
summary=package_data['summary'],
202+
cvss=package_data['severity']
203+
)
204+
else:
205+
vulnerability, _ = Vulnerability.objects.get_or_create(
206+
cve_id=package_data['cve_id'],
207+
summary=package_data['summary']
208+
)
209+
210+
VulnerabilityReference.objects.get_or_create(
211+
vulnerability=vulnerability,
212+
url=package_data['advisory']
213+
)
214+
215+
for version in package_data['affected_versions']:
216+
affected_package = Package.objects.create(
217+
name=package_data['package_name'],
218+
version=version
219+
)
220+
ImpactedPackage.objects.create(
221+
vulnerability=vulnerability,
222+
package=affected_package
223+
)
224+
225+
for version in package_data['fixed_versions']:
226+
unaffected_package = Package.objects.create(
227+
name=package_data['package_name'],
228+
version=version
229+
)
230+
ResolvedPackage.objects.create(
231+
vulnerability=vulnerability,
232+
package=unaffected_package
233+
)

‎vulnerabilities/management/commands/import.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,21 +24,25 @@
2424
from django.core.management.base import BaseCommand, CommandError
2525

2626
from vulnerabilities import data_dump as dd
27-
from vulnerabilities.scraper import debian, ubuntu, archlinux, npm
27+
from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby
2828

2929
IMPORTERS = {
30+
'ruby': lambda: dd.ruby_dump(ruby.import_vulnerabilities()),
3031
'npm': lambda: dd.npm_dump(npm.scrape_vulnerabilities()),
3132
'debian': lambda: dd.debian_dump(debian.scrape_vulnerabilities()),
3233
'ubuntu': lambda: dd.ubuntu_dump(ubuntu.scrape_cves()),
33-
'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities()),
34+
'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities())
3435
}
3536

3637

3738
class Command(BaseCommand):
3839
help = 'Import vulnerability data'
3940

4041
def add_arguments(self, parser):
41-
parser.add_argument('--list', action='store_true', help='List available data sources')
42+
parser.add_argument(
43+
'--list',
44+
action='store_true',
45+
help='List available data sources')
4246

4347
parser.add_argument('--all', action='store_true',
4448
help='Import data from all available sources')
@@ -69,7 +73,8 @@ def validate_sources(self, sources):
6973
raise CommandError(f'Unknown data sources: {unknown}')
7074

7175
def list_sources(self):
72-
self.stdout.write('Vulnerability data can be imported from the following sources:')
76+
self.stdout.write(
77+
'Vulnerability data can be imported from the following sources:')
7378
self.stdout.write(', '.join(IMPORTERS.keys()))
7479

7580
def import_data(self, sources):

‎vulnerabilities/scraper/ruby.py‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
import os
2+
import urllib.request
3+
import saneyaml
4+
5+
from shutil import rmtree
6+
from urllib.error import HTTPError
7+
from zipfile import ZipFile
8+
from itertools import chain
9+
from dephell_specifier import RangeSpecifier
10+
11+
RUBYCVE_LINK = 'https://github.com/rubysec/ruby-advisory-db/archive/master.zip'
12+
DOWNLOAD_PATH = os.path.dirname(os.path.realpath(__file__))
13+
14+
15+
def get_rubycve_db():
16+
pathToZip, _ = urllib.request.urlretrieve(
17+
RUBYCVE_LINK, os.path.join(
18+
DOWNLOAD_PATH, 'ruby.zip'))
19+
ZipFile(pathToZip).extractall(DOWNLOAD_PATH)
20+
os.remove(pathToZip)
21+
22+
23+
def path_of_yaml_of_all_packages():
24+
gemPath = os.path.join(DOWNLOAD_PATH, 'ruby-advisory-db-master', 'gems')
25+
rubiesPath = os.path.join(
26+
DOWNLOAD_PATH,
27+
'ruby-advisory-db-master',
28+
'rubies')
29+
for (
30+
packagePath,
31+
_,
32+
yamlNames) in chain(
33+
os.walk(gemPath),
34+
os.walk(rubiesPath)):
35+
for yamlName in yamlNames:
36+
yield os.path.join(packagePath, yamlName)
37+
38+
39+
def get_all_versions_of_package(package_name):
40+
url_to_load = 'https://rubygems.org/api/v1/versions/' + package_name + '.yaml'
41+
try:
42+
page = urllib.request.urlopen(url_to_load)
43+
package_history = saneyaml.load(page)
44+
except HTTPError:
45+
return []
46+
for version in package_history:
47+
yield version['number']
48+
49+
50+
def get_patched_range(spec_list):
51+
if spec_list:
52+
def remove_space(string): return string.replace(' ', '')
53+
spec_list = list(map(remove_space, spec_list))
54+
for spec in spec_list:
55+
if 'rc' in spec:
56+
continue
57+
yield RangeSpecifier(spec)
58+
else:
59+
return None
60+
61+
62+
def import_vulnerabilities():
63+
get_rubycve_db()
64+
ids = set()
65+
vulnerability_to_package_map = []
66+
for vulnerability_path in path_of_yaml_of_all_packages():
67+
with open(vulnerability_path) as yamlFile:
68+
vulnerability = saneyaml.load(yamlFile)
69+
package_name = vulnerability.get(
70+
'engine', vulnerability.get('gem'))
71+
summary = vulnerability.get('description', '')
72+
if 'cve' in vulnerability:
73+
vulnerability_id = 'CVE-{}'.format(vulnerability['cve'])
74+
if vulnerability_id in ids:
75+
continue
76+
ids.add(vulnerability_id)
77+
else:
78+
continue
79+
severity = vulnerability.get(
80+
'cvss_v3', vulnerability.get('cvss_v2'))
81+
advisoryUrl = vulnerability.get('url')
82+
specs = list(
83+
get_patched_range(
84+
vulnerability.get('patched_versions')))
85+
allVersions = set(list(get_all_versions_of_package(package_name)))
86+
unaffected_versions = set()
87+
if specs:
88+
for version in allVersions:
89+
for spec in specs:
90+
if(version in spec):
91+
unaffected_versions.add(version)
92+
break
93+
affected_versions = allVersions - unaffected_versions
94+
vulnerability_to_package_map.append({
95+
'package_name': package_name,
96+
'summary': summary,
97+
'cve_id': vulnerability_id,
98+
'fixed_versions': unaffected_versions,
99+
'affected_versions': affected_versions,
100+
'severity': severity,
101+
'advisory': advisoryUrl
102+
})
103+
rmtree(os.path.join(DOWNLOAD_PATH, 'ruby-advisory-db-master'))
104+
return vulnerability_to_package_map

0 commit comments

Comments
 (0)