diff --git a/requirements.txt b/requirements.txt index 90701d507..e173d2c33 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,3 +51,4 @@ traitlets==4.3.3 wcwidth==0.1.7 whitenoise==5.0.1 zipp==0.6.0 +requests==2.23.0 diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 96dd34d12..c8176d1b5 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -212,7 +212,6 @@ def _bulk_insert_impacted_and_resolved_packages( for advisory in batch: vuln = _advisory_to_vulnerability(advisory, vulnerabilities) - vulnerabilities.remove(vuln) # minor optimization for impacted_purl in advisory.impacted_package_urls: # TODO Figure out when/how it happens that a package is missing from the dict and fix it diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 9734e1f2e..7f50ca158 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -26,3 +26,4 @@ from vulnerabilities.importers.npm import NpmDataSource from vulnerabilities.importers.rust import RustDataSource from vulnerabilities.importers.safety_db import SafetyDbDataSource +from vulnerabilities.importers.ruby import RubyDataSource diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index 03ca1e7f9..86d01b9bc 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -1,76 +1,162 @@ -import urllib.request -from urllib.error import HTTPError -from zipfile import ZipFile -from io import BytesIO -import yaml +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +from json import JSONDecodeError +from typing import Set +from typing import List + from dephell_specifier import RangeSpecifier -from urllib.request import urlopen - -RUBYSEC_DB_URL = 'https://github.com/rubysec/ruby-advisory-db/archive/master.zip' - - -def rubygem_advisories(url, prefix='ruby-advisory-db-master/gems/'): - with urlopen(url) as response: - with ZipFile(BytesIO(response.read())) as zf: - for path in zf.namelist(): - if path.startswith(prefix) and path.endswith('.yml'): - yield yaml.safe_load(zf.open(path)) - - -def get_all_versions_of_package(package_name): - url_to_load = 'https://rubygems.org/api/v1/versions/' + package_name + '.yaml' - try: - page = urllib.request.urlopen(url_to_load) - package_history = yaml.safe_load(page) - except HTTPError: - return [] - for version in package_history: - yield version['number'] - - -def get_patched_range(spec_list): - spec_list = [string.replace(' ', '') for string in spec_list] - for spec in spec_list: - if 'rc' in spec: - continue - yield RangeSpecifier(spec) - - -def import_vulnerabilities(): - vulnerability_package_dicts = [] - for vulnerability in rubygem_advisories(RUBYSEC_DB_URL): - - package_name = vulnerability.get( - 'gem') - - if not package_name: - continue - - if 'cve' in vulnerability: - vulnerability_id = 'CVE-{}'.format(vulnerability['cve']) - else: - continue - - advisory_url = vulnerability.get('url') - patched_version_ranges = list( - get_patched_range( - vulnerability.get('patched_versions', []))) - all_versions = set(get_all_versions_of_package(package_name)) - unaffected_versions = set() - - if patched_version_ranges: - for version in all_versions: - for spec in patched_version_ranges: - if version in spec: - unaffected_versions.add(version) - break - - affected_versions = all_versions - unaffected_versions - vulnerability_package_dicts.append({ - 'package_name': package_name, - 'cve_id': vulnerability_id, - 'fixed_versions': unaffected_versions, - 'affected_versions': affected_versions, - 'advisory': advisory_url - }) - return vulnerability_package_dicts +from dephell_specifier.range_specifier import InvalidSpecifier +from packageurl import PackageURL +import requests +import yaml + +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import GitDataSource + + +class RubyDataSource(GitDataSource): + + def __enter__(self): + super(RubyDataSource, self).__enter__() + + if not getattr(self, '_added_files', None): + self._added_files, self._updated_files = self.file_changes( + recursive=True, file_ext='yml', subdir='./gems') + + def updated_advisories(self) -> Set[Advisory]: + files = self._updated_files + advisories = [] + for f in files: + processed_data = self.process_file(f) + if processed_data: + advisories.append(processed_data) + return self.batch_advisories(advisories) + + def added_advisories(self) -> Set[Advisory]: + files = self._added_files + advisories = [] + for f in files: + processed_data = self.process_file(f) + if processed_data: + advisories.append(processed_data) + return self.batch_advisories(advisories) + + def process_file(self, path) -> List[Advisory]: + with open(path) as f: + record = yaml.safe_load(f) + package_name = record.get( + 'gem') + + if not package_name: + return + + if 'cve' in record: + cve_id = 'CVE-{}'.format(record['cve']) + else: + return + + safe_version_ranges = record.get('patched_versions', []) + # this case happens when the advisory contain only 'patched_versions' field + # and it has value None(i.e it is empty :( ). + if not safe_version_ranges: + safe_version_ranges = [] + safe_version_ranges += record.get('unaffected_versions', []) + safe_version_ranges = [i for i in safe_version_ranges if i] + + if not getattr(self, 'pkg_manager_api', None): + self.pkg_manager_api = rubyAPI() + all_vers = self.pkg_manager_api.get_all_version_of_package( + package_name) + safe_versions, affected_versions = self.categorize_versions( + all_vers, safe_version_ranges) + + impacted_purls = { + PackageURL( + name=package_name, + type='gem', + version=version, + ) for version in affected_versions} + + resolved_purls = { + PackageURL( + name=package_name, + type='gem', + version=version, + ) for version in safe_versions} + + return Advisory( + summary=record.get('description', ''), + impacted_package_urls=impacted_purls, + resolved_package_urls=resolved_purls, + reference_urls=[record.get('url', '')], + cve_id=cve_id + ) + + @staticmethod + def categorize_versions(all_versions, unaffected_version_ranges): + + for id, elem in enumerate(unaffected_version_ranges): + try: + unaffected_version_ranges[id] = RangeSpecifier( + elem.replace(' ', '')) + except InvalidSpecifier: + continue + + safe_versions = set() + for i in all_versions: + for ver_rng in unaffected_version_ranges: + + if i in ver_rng: + + safe_versions.add(i) + + return (safe_versions, all_versions-safe_versions) + + +class rubyAPI: + + base_endpt = 'https://rubygems.org/api/v1/versions/{}.json' + + def __init__(self): + self.client = requests.Session() + self.cache = {} + + def call_api(self, pkg_name) -> List: + end_pt = self.base_endpt.format(pkg_name) + try: + resp = self.client.get(end_pt) + return resp.json() + # this covers 404 alright + except JSONDecodeError: + return [] + + def get_all_version_of_package(self, pkg_name) -> Set[str]: + all_versions = set() + if self.cache.get(pkg_name): + return self.cache.get(pkg_name) + + json_resp = self.call_api(pkg_name) + for release in json_resp: + all_versions.add(release['number']) + self.cache[pkg_name] = all_versions + return all_versions diff --git a/vulnerabilities/migrations/0008_ruby_importer.py b/vulnerabilities/migrations/0008_ruby_importer.py new file mode 100644 index 000000000..b11dde9f9 --- /dev/null +++ b/vulnerabilities/migrations/0008_ruby_importer.py @@ -0,0 +1,55 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +from django.db import migrations + + +def add_ruby_importer(apps, _): + Importer = apps.get_model('vulnerabilities', 'Importer') + + Importer.objects.create( + name='ruby', + license='', + last_run=None, + data_source='RubyDataSource', + data_source_cfg={ + 'repository_url': 'https://github.com/rubysec/ruby-advisory-db.git', + }, + ) + + +def remove_ruby_importer(apps, _): + Importer = apps.get_model('vulnerabilities', 'Importer') + qs = Importer.objects.filter(name='ruby') + if qs: + qs[0].delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('vulnerabilities', '0007_npm_importer'), + ] + + operations = [ + migrations.RunPython(add_ruby_importer, remove_ruby_importer), + ] diff --git a/vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125675.yml b/vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125675.yml new file mode 100644 index 000000000..12e317da0 --- /dev/null +++ b/vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125675.yml @@ -0,0 +1,9 @@ +--- +gem: sidekiq +osvdb: 125675 +url: https://github.com/mperham/sidekiq/pull/2422 +title: Sidekiq Gem for Ruby Multiple Unspecified CSRF +date: 2015-07-06 +description: Sidekiq::Web lacks CSRF protection +patched_versions: + - ">= 3.4.2" diff --git a/vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125676.yml b/vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125676.yml new file mode 100644 index 000000000..18ba94428 --- /dev/null +++ b/vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125676.yml @@ -0,0 +1,14 @@ +--- +gem: sidekiq +osvdb: 125676 +url: https://github.com/mperham/sidekiq/issues/2330 +title: | + Sidekiq Gem for Ruby web/views/queue.erb CurrentMessagesInQueue Element + Reflected XSS +date: 2015-06-04 +description: XSS via queue name in Sidekiq::Web +patched_versions: + - ">= 3.4.0" +related: + osvdb: + - 125677 diff --git a/vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125678.yml b/vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125678.yml new file mode 100644 index 000000000..1566d10a7 --- /dev/null +++ b/vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125678.yml @@ -0,0 +1,9 @@ +--- +gem: sidekiq +osvdb: 125678 +url: https://github.com/mperham/sidekiq/pull/2309 +title: Sidekiq Gem for Ruby web/views/queue.erb msg.display_class Element XSS +date: 2015-04-21 +description: XSS via job arguments display class in Sidekiq::Web +patched_versions: + - ">= 3.4.0" diff --git a/vulnerabilities/tests/test_data/ruby/sinatra/CVE-2018-11627.yml b/vulnerabilities/tests/test_data/ruby/sinatra/CVE-2018-11627.yml new file mode 100644 index 000000000..38f7c07a0 --- /dev/null +++ b/vulnerabilities/tests/test_data/ruby/sinatra/CVE-2018-11627.yml @@ -0,0 +1,15 @@ +--- +gem: sinatra +cve: 2018-11627 +url: https://github.com/sinatra/sinatra/issues/1428 +title: XSS via the 400 Bad Request page +date: 2018-05-31 +description: | + Sinatra before 2.0.2 has XSS via the 400 Bad Request page that occurs upon a params parser exception. + +cvss_v3: 6.1 + +patched_versions: + - ">= 2.0.2" + + diff --git a/vulnerabilities/tests/test_data/ruby/sinatra/CVE-2018-7212.yml b/vulnerabilities/tests/test_data/ruby/sinatra/CVE-2018-7212.yml new file mode 100644 index 000000000..88cd18cbd --- /dev/null +++ b/vulnerabilities/tests/test_data/ruby/sinatra/CVE-2018-7212.yml @@ -0,0 +1,19 @@ +--- +gem: sinatra +cve: 2018-7212 +url: https://github.com/sinatra/sinatra/pull/1379 +date: 2018-01-09 +title: sinatra ruby gem path traversal via backslash characters on Windows +description: | + An issue was discovered in rack-protection/lib/rack/protection/path_traversal.rb + in Sinatra 2.x before 2.0.1 on Windows. Path traversal is possible via backslash + characters. + +cvss_v3: 5.3 +cvss_v2: 5.0 + +patched_versions: + - ">= 2.0.1" + +unaffected_versions: + - "<= 1.0.0" diff --git a/vulnerabilities/tests/test_ruby.py b/vulnerabilities/tests/test_ruby.py new file mode 100644 index 000000000..8e3e98044 --- /dev/null +++ b/vulnerabilities/tests/test_ruby.py @@ -0,0 +1,136 @@ +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import os +import pathlib +from unittest.mock import patch +from unittest import TestCase +from collections import OrderedDict + +from packageurl import PackageURL + +from vulnerabilities.importers.ruby import RubyDataSource +from vulnerabilities.data_source import GitDataSourceConfiguration +from vulnerabilities.data_source import Advisory + + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, 'test_data', 'ruby') + +MOCK_ADDED_FILES = [] + +for filepath in pathlib.Path(TEST_DATA).glob('**/*.yml'): + MOCK_ADDED_FILES.append(filepath.absolute()) + + +class RubyDataSourceTest(TestCase): + + @classmethod + def setUpClass(cls): + data_source_cfg = { + 'repository_url': 'https://github.com/rubysec/ruby-advisory-db.git', } + cls.data_src = RubyDataSource(1, config=data_source_cfg) + + @patch('vulnerabilities.importers.ruby.rubyAPI.get_all_version_of_package', + return_value={'1.0.0', '1.8.0', '2.0.3'}) + def test_process_file(self, mock_write): + expected_advisories = { + Advisory( + summary=('An issue was discovered in' + ' rack-protection/lib/rack/protection/path_traversal.rb\n' + 'in Sinatra 2.x before 2.0.1 on Windows.' + ' Path traversal is possible via backslash\ncharacters.\n'), + impacted_package_urls={ + PackageURL( + type='gem', + namespace=None, + name='sinatra', + version='1.8.0', + qualifiers=OrderedDict(), + subpath=None)}, + resolved_package_urls={ + PackageURL( + type='gem', + namespace=None, + name='sinatra', + version='1.0.0', + qualifiers=OrderedDict(), + subpath=None), + PackageURL( + type='gem', + namespace=None, + name='sinatra', + version='2.0.3', + qualifiers=OrderedDict(), + subpath=None)}, + reference_urls=['https://github.com/sinatra/sinatra/pull/1379'], + reference_ids=[], + cve_id='CVE-2018-7212'), + Advisory( + summary=('Sinatra before 2.0.2 has XSS via the 400 Bad Request ' + 'page that occurs upon a params parser exception.\n'), + impacted_package_urls={ + PackageURL( + type='gem', + namespace=None, + name='sinatra', + version='1.0.0', + qualifiers=OrderedDict(), + subpath=None), + PackageURL( + type='gem', + namespace=None, + name='sinatra', + version='1.8.0', + qualifiers=OrderedDict(), + subpath=None)}, + resolved_package_urls={ + PackageURL( + type='gem', + namespace=None, + name='sinatra', + version='2.0.3', + qualifiers=OrderedDict(), + subpath=None)}, + reference_urls=['https://github.com/sinatra/sinatra/issues/1428'], + reference_ids=[], + cve_id='CVE-2018-11627'), + None} + + found_advisories = set() + + for p in MOCK_ADDED_FILES: + found_advisories.add(self.data_src.process_file(p)) + assert found_advisories == expected_advisories + + def test_categorize_versions(self): + + all_versions = {'1.0.0', '1.2.0', '9.0.2', '0.2.3'} + safe_ver_ranges = ['==1.0.0', '>1.2.0'] + + exp_safe_vers = {'1.0.0', '9.0.2'} + exp_aff_vers = {'1.2.0', '0.2.3'} + + safe_vers, aff_vers = self.data_src.categorize_versions( + all_versions, safe_ver_ranges) + assert exp_aff_vers == aff_vers + assert exp_safe_vers == safe_vers