|
1 | | -import urllib.request |
2 | | -from urllib.error import HTTPError |
3 | | -from zipfile import ZipFile |
4 | | -from io import BytesIO |
5 | | -import yaml |
| 1 | +# Copyright (c) 2017 nexB Inc. and others. All rights reserved. |
| 2 | +# http://nexb.com and https://github.com/nexB/vulnerablecode/ |
| 3 | +# The VulnerableCode software is licensed under the Apache License version 2.0. |
| 4 | +# Data generated with VulnerableCode require an acknowledgment. |
| 5 | +# |
| 6 | +# You may not use this software except in compliance with the License. |
| 7 | +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 |
| 8 | +# Unless required by applicable law or agreed to in writing, software distributed |
| 9 | +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR |
| 10 | +# CONDITIONS OF ANY KIND, either express or implied. See the License for the |
| 11 | +# specific language governing permissions and limitations under the License. |
| 12 | +# |
| 13 | +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode |
| 14 | +# derivative work, you must accompany this data with the following acknowledgment: |
| 15 | +# |
| 16 | +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES |
| 17 | +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from |
| 18 | +# VulnerableCode should be considered or used as legal advice. Consult an Attorney |
| 19 | +# for any legal advice. |
| 20 | +# VulnerableCode is a free software code scanning tool from nexB Inc. and others. |
| 21 | +# Visit https://github.com/nexB/vulnerablecode/ for support and download. |
| 22 | + |
| 23 | +from json import JSONDecodeError |
| 24 | +from typing import Set |
| 25 | +from typing import List |
| 26 | + |
6 | 27 | from dephell_specifier import RangeSpecifier |
7 | | -from urllib.request import urlopen |
8 | | - |
9 | | -RUBYSEC_DB_URL = 'https://github.com/rubysec/ruby-advisory-db/archive/master.zip' |
10 | | - |
11 | | - |
12 | | -def rubygem_advisories(url, prefix='ruby-advisory-db-master/gems/'): |
13 | | - with urlopen(url) as response: |
14 | | - with ZipFile(BytesIO(response.read())) as zf: |
15 | | - for path in zf.namelist(): |
16 | | - if path.startswith(prefix) and path.endswith('.yml'): |
17 | | - yield yaml.safe_load(zf.open(path)) |
18 | | - |
19 | | - |
20 | | -def get_all_versions_of_package(package_name): |
21 | | - url_to_load = 'https://rubygems.org/api/v1/versions/' + package_name + '.yaml' |
22 | | - try: |
23 | | - page = urllib.request.urlopen(url_to_load) |
24 | | - package_history = yaml.safe_load(page) |
25 | | - except HTTPError: |
26 | | - return [] |
27 | | - for version in package_history: |
28 | | - yield version['number'] |
29 | | - |
30 | | - |
31 | | -def get_patched_range(spec_list): |
32 | | - spec_list = [string.replace(' ', '') for string in spec_list] |
33 | | - for spec in spec_list: |
34 | | - if 'rc' in spec: |
35 | | - continue |
36 | | - yield RangeSpecifier(spec) |
37 | | - |
38 | | - |
39 | | -def import_vulnerabilities(): |
40 | | - vulnerability_package_dicts = [] |
41 | | - for vulnerability in rubygem_advisories(RUBYSEC_DB_URL): |
42 | | - |
43 | | - package_name = vulnerability.get( |
44 | | - 'gem') |
45 | | - |
46 | | - if not package_name: |
47 | | - continue |
48 | | - |
49 | | - if 'cve' in vulnerability: |
50 | | - vulnerability_id = 'CVE-{}'.format(vulnerability['cve']) |
51 | | - else: |
52 | | - continue |
53 | | - |
54 | | - advisory_url = vulnerability.get('url') |
55 | | - patched_version_ranges = list( |
56 | | - get_patched_range( |
57 | | - vulnerability.get('patched_versions', []))) |
58 | | - all_versions = set(get_all_versions_of_package(package_name)) |
59 | | - unaffected_versions = set() |
60 | | - |
61 | | - if patched_version_ranges: |
62 | | - for version in all_versions: |
63 | | - for spec in patched_version_ranges: |
64 | | - if version in spec: |
65 | | - unaffected_versions.add(version) |
66 | | - break |
67 | | - |
68 | | - affected_versions = all_versions - unaffected_versions |
69 | | - vulnerability_package_dicts.append({ |
70 | | - 'package_name': package_name, |
71 | | - 'cve_id': vulnerability_id, |
72 | | - 'fixed_versions': unaffected_versions, |
73 | | - 'affected_versions': affected_versions, |
74 | | - 'advisory': advisory_url |
75 | | - }) |
76 | | - return vulnerability_package_dicts |
| 28 | +from dephell_specifier.range_specifier import InvalidSpecifier |
| 29 | +from packageurl import PackageURL |
| 30 | +import requests |
| 31 | +import yaml |
| 32 | + |
| 33 | +from vulnerabilities.data_source import Advisory |
| 34 | +from vulnerabilities.data_source import GitDataSource |
| 35 | + |
| 36 | + |
| 37 | +class RubyDataSource(GitDataSource): |
| 38 | + |
| 39 | + def __enter__(self): |
| 40 | + super(RubyDataSource, self).__enter__() |
| 41 | + |
| 42 | + if not getattr(self, '_added_files', None): |
| 43 | + self._added_files, self._updated_files = self.file_changes( |
| 44 | + recursive=True, file_ext='yml', subdir='./gems') |
| 45 | + |
| 46 | + def updated_advisories(self) -> Set[Advisory]: |
| 47 | + files = self._updated_files |
| 48 | + advisories = [] |
| 49 | + for f in files: |
| 50 | + processed_data = self.process_file(f) |
| 51 | + if processed_data: |
| 52 | + advisories.append(processed_data) |
| 53 | + return self.batch_advisories(advisories) |
| 54 | + |
| 55 | + def added_advisories(self) -> Set[Advisory]: |
| 56 | + files = self._added_files |
| 57 | + advisories = [] |
| 58 | + for f in files: |
| 59 | + processed_data = self.process_file(f) |
| 60 | + if processed_data: |
| 61 | + advisories.append(processed_data) |
| 62 | + return self.batch_advisories(advisories) |
| 63 | + |
| 64 | + def process_file(self, path) -> List[Advisory]: |
| 65 | + with open(path) as f: |
| 66 | + record = yaml.safe_load(f) |
| 67 | + package_name = record.get( |
| 68 | + 'gem') |
| 69 | + |
| 70 | + if not package_name: |
| 71 | + return |
| 72 | + |
| 73 | + if 'cve' in record: |
| 74 | + cve_id = 'CVE-{}'.format(record['cve']) |
| 75 | + else: |
| 76 | + return |
| 77 | + |
| 78 | + safe_version_ranges = record.get('patched_versions', []) |
| 79 | + # this case happens when the advisory contain only 'patched_versions' field |
| 80 | + # and it has value None(i.e it is empty :( ). |
| 81 | + if not safe_version_ranges: |
| 82 | + safe_version_ranges = [] |
| 83 | + safe_version_ranges += record.get('unaffected_versions', []) |
| 84 | + safe_version_ranges = [i for i in safe_version_ranges if i] |
| 85 | + |
| 86 | + if not getattr(self, 'pkg_manager_api', None): |
| 87 | + self.pkg_manager_api = rubyAPI() |
| 88 | + all_vers = self.pkg_manager_api.get_all_version_of_package( |
| 89 | + package_name) |
| 90 | + safe_versions, affected_versions = self.categorize_versions( |
| 91 | + all_vers, safe_version_ranges) |
| 92 | + |
| 93 | + impacted_purls = { |
| 94 | + PackageURL( |
| 95 | + name=package_name, |
| 96 | + type='gem', |
| 97 | + version=version, |
| 98 | + ) for version in affected_versions} |
| 99 | + |
| 100 | + resolved_purls = { |
| 101 | + PackageURL( |
| 102 | + name=package_name, |
| 103 | + type='gem', |
| 104 | + version=version, |
| 105 | + ) for version in safe_versions} |
| 106 | + |
| 107 | + return Advisory( |
| 108 | + summary=record.get('description', ''), |
| 109 | + impacted_package_urls=impacted_purls, |
| 110 | + resolved_package_urls=resolved_purls, |
| 111 | + reference_urls=[record.get('url', '')], |
| 112 | + cve_id=cve_id |
| 113 | + ) |
| 114 | + |
| 115 | + @staticmethod |
| 116 | + def categorize_versions(all_versions, unaffected_version_ranges): |
| 117 | + |
| 118 | + for id, elem in enumerate(unaffected_version_ranges): |
| 119 | + try: |
| 120 | + unaffected_version_ranges[id] = RangeSpecifier( |
| 121 | + elem.replace(' ', '')) |
| 122 | + except InvalidSpecifier: |
| 123 | + continue |
| 124 | + |
| 125 | + safe_versions = set() |
| 126 | + for i in all_versions: |
| 127 | + for ver_rng in unaffected_version_ranges: |
| 128 | + |
| 129 | + if i in ver_rng: |
| 130 | + |
| 131 | + safe_versions.add(i) |
| 132 | + |
| 133 | + return (safe_versions, all_versions-safe_versions) |
| 134 | + |
| 135 | + |
| 136 | +class rubyAPI: |
| 137 | + |
| 138 | + base_endpt = 'https://rubygems.org/api/v1/versions/{}.json' |
| 139 | + |
| 140 | + def __init__(self): |
| 141 | + self.client = requests.Session() |
| 142 | + self.cache = {} |
| 143 | + |
| 144 | + def call_api(self, pkg_name) -> List: |
| 145 | + end_pt = self.base_endpt.format(pkg_name) |
| 146 | + try: |
| 147 | + resp = self.client.get(end_pt) |
| 148 | + return resp.json() |
| 149 | + # this covers 404 alright |
| 150 | + except JSONDecodeError: |
| 151 | + return [] |
| 152 | + |
| 153 | + def get_all_version_of_package(self, pkg_name) -> Set[str]: |
| 154 | + all_versions = set() |
| 155 | + if self.cache.get(pkg_name): |
| 156 | + return self.cache.get(pkg_name) |
| 157 | + |
| 158 | + json_resp = self.call_api(pkg_name) |
| 159 | + for release in json_resp: |
| 160 | + all_versions.add(release['number']) |
| 161 | + self.cache[pkg_name] = all_versions |
| 162 | + return all_versions |
0 commit comments