Ruby importer rewrite

requirements.txt

@@ -51,3 +51,4 @@ traitlets==4.3.3
 wcwidth==0.1.7
 whitenoise==5.0.1
 zipp==0.6.0
+requests==2.23.0

vulnerabilities/import_runner.py

@@ -212,7 +212,6 @@ def _bulk_insert_impacted_and_resolved_packages(
 
     for advisory in batch:
         vuln = _advisory_to_vulnerability(advisory, vulnerabilities)
-        vulnerabilities.remove(vuln)  # minor optimization
 
         for impacted_purl in advisory.impacted_package_urls:
             # TODO Figure out when/how it happens that a package is missing from the dict and fix it

vulnerabilities/importers/__init__.py

@@ -26,3 +26,4 @@
 from vulnerabilities.importers.npm import NpmDataSource
 from vulnerabilities.importers.rust import RustDataSource
 from vulnerabilities.importers.safety_db import SafetyDbDataSource
+from vulnerabilities.importers.ruby import RubyDataSource

vulnerabilities/importers/ruby.py

@@ -1,76 +1,162 @@
-import urllib.request
-from urllib.error import HTTPError
-from zipfile import ZipFile
-from io import BytesIO
-import yaml
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from json import JSONDecodeError
+from typing import Set
+from typing import List
+
 from dephell_specifier import RangeSpecifier
-from urllib.request import urlopen
-
-RUBYSEC_DB_URL = 'https://github.com/rubysec/ruby-advisory-db/archive/master.zip'
-
-
-def rubygem_advisories(url, prefix='ruby-advisory-db-master/gems/'):
-    with urlopen(url) as response:
-        with ZipFile(BytesIO(response.read())) as zf:
-            for path in zf.namelist():
-                if path.startswith(prefix) and path.endswith('.yml'):
-                    yield yaml.safe_load(zf.open(path))
-
-
-def get_all_versions_of_package(package_name):
-    url_to_load = 'https://rubygems.org/api/v1/versions/' + package_name + '.yaml'
-    try:
-        page = urllib.request.urlopen(url_to_load)
-        package_history = yaml.safe_load(page)
-    except HTTPError:
-        return []
-    for version in package_history:
-        yield version['number']
-
-
-def get_patched_range(spec_list):
-    spec_list = [string.replace(' ', '') for string in spec_list]
-    for spec in spec_list:
-        if 'rc' in spec:
-            continue
-        yield RangeSpecifier(spec)
-
-
-def import_vulnerabilities():
-    vulnerability_package_dicts = []
-    for vulnerability in rubygem_advisories(RUBYSEC_DB_URL):
-
-        package_name = vulnerability.get(
-            'gem')
-
-        if not package_name:
-            continue
-
-        if 'cve' in vulnerability:
-            vulnerability_id = 'CVE-{}'.format(vulnerability['cve'])
-        else:
-            continue
-
-        advisory_url = vulnerability.get('url')
-        patched_version_ranges = list(
-            get_patched_range(
-                vulnerability.get('patched_versions', [])))
-        all_versions = set(get_all_versions_of_package(package_name))
-        unaffected_versions = set()
-
-        if patched_version_ranges:
-            for version in all_versions:
-                for spec in patched_version_ranges:
-                    if version in spec:
-                        unaffected_versions.add(version)
-                        break
-
-        affected_versions = all_versions - unaffected_versions
-        vulnerability_package_dicts.append({
-            'package_name': package_name,
-            'cve_id': vulnerability_id,
-            'fixed_versions': unaffected_versions,
-            'affected_versions': affected_versions,
-            'advisory': advisory_url
-        })
-    return vulnerability_package_dicts
+from dephell_specifier.range_specifier import InvalidSpecifier
+from packageurl import PackageURL
+import requests
+import yaml
+
+from vulnerabilities.data_source import Advisory
+from vulnerabilities.data_source import GitDataSource
+
+
+class RubyDataSource(GitDataSource):
+
+    def __enter__(self):
+        super(RubyDataSource, self).__enter__()
+
+        if not getattr(self, '_added_files', None):
+            self._added_files, self._updated_files = self.file_changes(
+                recursive=True, file_ext='yml', subdir='./gems')
+
+    def updated_advisories(self) -> Set[Advisory]:
+        files = self._updated_files
+        advisories = []
+        for f in files:
+            processed_data = self.process_file(f)
+            if processed_data:
+                advisories.append(processed_data)
+        return self.batch_advisories(advisories)
+
+    def added_advisories(self) -> Set[Advisory]:
+        files = self._added_files
+        advisories = []
+        for f in files:
+            processed_data = self.process_file(f)
+            if processed_data:
+                advisories.append(processed_data)
+        return self.batch_advisories(advisories)
+
+    def process_file(self, path) -> List[Advisory]:
+        with open(path) as f:
+            record = yaml.safe_load(f)
+            package_name = record.get(
+                'gem')
+
+            if not package_name:
+                return
+
+            if 'cve' in record:
+                cve_id = 'CVE-{}'.format(record['cve'])
+            else:
+                return
+
+            safe_version_ranges = record.get('patched_versions', [])
+            # this case happens when the advisory contain only 'patched_versions' field
+            # and it has value None(i.e it is empty :( ).
+            if not safe_version_ranges:
+                safe_version_ranges = []
+            safe_version_ranges += record.get('unaffected_versions', [])
+            safe_version_ranges = [i for i in safe_version_ranges if i]
+
+            if not getattr(self, 'pkg_manager_api', None):
+                self.pkg_manager_api = rubyAPI()
+            all_vers = self.pkg_manager_api.get_all_version_of_package(
+                package_name)
+            safe_versions, affected_versions = self.categorize_versions(
+                all_vers, safe_version_ranges)
+
+            impacted_purls = {
+                PackageURL(
+                    name=package_name,
+                    type='gem',
+                    version=version,
+                ) for version in affected_versions}
+
+            resolved_purls = {
+                PackageURL(
+                    name=package_name,
+                    type='gem',
+                    version=version,
+                ) for version in safe_versions}
+
+            return Advisory(
+                summary=record.get('description', ''),
+                impacted_package_urls=impacted_purls,
+                resolved_package_urls=resolved_purls,
+                reference_urls=[record.get('url', '')],
+                cve_id=cve_id
+            )
+
+    @staticmethod
+    def categorize_versions(all_versions, unaffected_version_ranges):
+
+        for id, elem in enumerate(unaffected_version_ranges):
+            try:
+                unaffected_version_ranges[id] = RangeSpecifier(
+                    elem.replace(' ', ''))
+            except InvalidSpecifier:
+                continue
+
+        safe_versions = set()
+        for i in all_versions:
+            for ver_rng in unaffected_version_ranges:
+
+                if i in ver_rng:
+
+                    safe_versions.add(i)
+
+        return (safe_versions, all_versions-safe_versions)
+
+
+class rubyAPI:
+
+    base_endpt = 'https://rubygems.org/api/v1/versions/{}.json'
+
+    def __init__(self):
+        self.client = requests.Session()
+        self.cache = {}
+
+    def call_api(self, pkg_name) -> List:
+        end_pt = self.base_endpt.format(pkg_name)
+        try:
+            resp = self.client.get(end_pt)
+            return resp.json()
+        # this covers 404 alright
+        except JSONDecodeError:
+            return []
+
+    def get_all_version_of_package(self, pkg_name) -> Set[str]:
+        all_versions = set()
+        if self.cache.get(pkg_name):
+            return self.cache.get(pkg_name)
+
+        json_resp = self.call_api(pkg_name)
+        for release in json_resp:
+            all_versions.add(release['number'])
+        self.cache[pkg_name] = all_versions
+        return all_versions

vulnerabilities/migrations/0008_ruby_importer.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+from django.db import migrations
+
+
+def add_ruby_importer(apps, _):
+    Importer = apps.get_model('vulnerabilities', 'Importer')
+
+    Importer.objects.create(
+        name='ruby',
+        license='',
+        last_run=None,
+        data_source='RubyDataSource',
+        data_source_cfg={
+            'repository_url': 'https://github.com/rubysec/ruby-advisory-db.git',
+        },
+    )
+
+
+def remove_ruby_importer(apps, _):
+    Importer = apps.get_model('vulnerabilities', 'Importer')
+    qs = Importer.objects.filter(name='ruby')
+    if qs:
+        qs[0].delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vulnerabilities', '0007_npm_importer'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_ruby_importer, remove_ruby_importer),
+    ]

vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125675.yml

@@ -0,0 +1,9 @@
+---
+gem: sidekiq
+osvdb: 125675
+url: https://github.com/mperham/sidekiq/pull/2422
+title: Sidekiq Gem for Ruby Multiple Unspecified CSRF
+date: 2015-07-06
+description: Sidekiq::Web lacks CSRF protection
+patched_versions:
+  - ">= 3.4.2"

vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125676.yml

@@ -0,0 +1,14 @@
+---
+gem: sidekiq
+osvdb: 125676
+url: https://github.com/mperham/sidekiq/issues/2330
+title: |
+  Sidekiq Gem for Ruby web/views/queue.erb CurrentMessagesInQueue Element
+  Reflected XSS
+date: 2015-06-04
+description: XSS via queue name in Sidekiq::Web
+patched_versions:
+  - ">= 3.4.0"
+related:
+  osvdb:
+    - 125677

vulnerabilities/tests/test_data/ruby/sidekiq/OSVDB-125678.yml

@@ -0,0 +1,9 @@
+---
+gem: sidekiq
+osvdb: 125678
+url: https://github.com/mperham/sidekiq/pull/2309
+title: Sidekiq Gem for Ruby web/views/queue.erb msg.display_class Element XSS
+date: 2015-04-21
+description: XSS via job arguments display class in Sidekiq::Web
+patched_versions:
+  - ">= 3.4.0"

vulnerabilities/tests/test_data/ruby/sinatra/CVE-2018-11627.yml

@@ -0,0 +1,15 @@
+---
+gem: sinatra
+cve: 2018-11627
+url: https://github.com/sinatra/sinatra/issues/1428
+title: XSS via the 400 Bad Request page
+date: 2018-05-31
+description: |
+  Sinatra before 2.0.2 has XSS via the 400 Bad Request page that occurs upon a params parser exception.
+
+cvss_v3: 6.1
+
+patched_versions:
+  - ">= 2.0.2"
+
+

vulnerabilities/tests/test_data/ruby/sinatra/CVE-2018-7212.yml

@@ -0,0 +1,19 @@
+---
+gem: sinatra
+cve: 2018-7212
+url: https://github.com/sinatra/sinatra/pull/1379
+date: 2018-01-09
+title: sinatra ruby gem path traversal via backslash characters on Windows
+description: |
+  An issue was discovered in rack-protection/lib/rack/protection/path_traversal.rb
+  in Sinatra 2.x before 2.0.1 on Windows. Path traversal is possible via backslash
+  characters.
+
+cvss_v3: 5.3
+cvss_v2: 5.0
+
+patched_versions:
+  - ">= 2.0.1"
+
+unaffected_versions:
+  - "<= 1.0.0"