Merge pull request #143 from sbs2001/develop

Add ruby gems advisory importer

Author: haikoschol

requirements.txt

@@ -1,20 +1,33 @@
-setuptools==36.5.0
-
+attrs==19.3.0
 beautifulsoup4==4.7.1
-lxml==4.3.3
-django==2.2.8
-djangorestframework==3.9.2
+dephell-specifier==0.2.1
+dj-database-url==0.4.2
+Django==2.2.8
 django-filter==2.2.0
+djangorestframework==3.9.2
+gunicorn==19.7.1
+importlib-metadata==1.3.0
+lxml==4.3.3
+more-itertools==8.0.2
 packageurl-python==0.8.7
-semantic-version==2.8.2
-
-# Tests
-pytest==5.3.2  
-pytest-django==3.7.0
+packaging==19.2
+pluggy==0.13.1
+psycopg2==2.8.4
+py==1.8.0
 pycodestyle==2.5.0
-
-# Deployment
-gunicorn==19.7.1
+pyparsing==2.4.5
+pytest==5.3.2
+pytest-dependency==0.4.0
+pytest-django==3.7.0
+pytest-mock==1.13.0
+pytz==2019.3
+PyYAML==5.3
+saneyaml==0.4
+semantic-version==2.8.2
+six==1.13.0
+soupsieve==1.9.5
+sqlparse==0.3.0
+tqdm==4.41.1
+wcwidth==0.1.7
 whitenoise==5.0.1
-dj_database_url==0.4.2
-psycopg2==2.8.4
+zipp==0.6.0

vulnerabilities/data_dump.py

@@ -144,14 +144,14 @@ def archlinux_dump(extract_data):
 
             for ap in affected_packages:
                 ImpactedPackage.objects.get_or_create(
-                        vulnerability=vulnerability,
-                        package=ap,
+                    vulnerability=vulnerability,
+                    package=ap,
                 )
 
             for fp in fixed_packages:
                 ResolvedPackage.objects.get_or_create(
-                        vulnerability=vulnerability,
-                        package=fp,
+                    vulnerability=vulnerability,
+                    package=fp,
                 )
 
 
@@ -190,3 +190,38 @@ def npm_dump(extract_data):
                     vulnerability=vulnerability,
                     package=package_fixed
                 )
+
+
+def ruby_dump(extract_data):
+    for package_data in extract_data:
+
+        vulnerability, _ = Vulnerability.objects.get_or_create(
+            cve_id=package_data['cve_id']
+        )
+
+        VulnerabilityReference.objects.get_or_create(
+            vulnerability=vulnerability,
+            url=package_data['advisory']
+        )
+
+        for version in package_data['affected_versions']:
+            affected_package = Package.objects.create(
+                name=package_data['package_name'],
+                type='gem',
+                version=version
+            )
+            ImpactedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=affected_package
+            )
+
+        for version in package_data['fixed_versions']:
+            unaffected_package = Package.objects.create(
+                name=package_data['package_name'],
+                type='gem',
+                version=version
+            )
+            ResolvedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=unaffected_package
+            )

vulnerabilities/management/commands/import.py

@@ -24,21 +24,25 @@
 from django.core.management.base import BaseCommand, CommandError
 
 from vulnerabilities import data_dump as dd
-from vulnerabilities.scraper import debian, ubuntu, archlinux, npm
+from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby
 
 IMPORTERS = {
+    'ruby': lambda: dd.ruby_dump(ruby.import_vulnerabilities()),
     'npm': lambda: dd.npm_dump(npm.scrape_vulnerabilities()),
     'debian': lambda: dd.debian_dump(debian.scrape_vulnerabilities()),
     'ubuntu': lambda: dd.ubuntu_dump(ubuntu.scrape_cves()),
-    'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities()),
+    'archlinux': lambda: dd.archlinux_dump(archlinux.scrape_vulnerabilities())
 }
 
 
 class Command(BaseCommand):
     help = 'Import vulnerability data'
 
     def add_arguments(self, parser):
-        parser.add_argument('--list', action='store_true', help='List available data sources')
+        parser.add_argument(
+            '--list',
+            action='store_true',
+            help='List available data sources')
 
         parser.add_argument('--all', action='store_true',
                             help='Import data from all available sources')
@@ -69,7 +73,8 @@ def validate_sources(self, sources):
             raise CommandError(f'Unknown data sources: {unknown}')
 
     def list_sources(self):
-        self.stdout.write('Vulnerability data can be imported from the following sources:')
+        self.stdout.write(
+            'Vulnerability data can be imported from the following sources:')
         self.stdout.write(', '.join(IMPORTERS.keys()))
 
     def import_data(self, sources):

vulnerabilities/scraper/ruby.py

@@ -0,0 +1,77 @@
+import os
+import urllib.request
+from urllib.error import HTTPError
+from zipfile import ZipFile
+from io import BytesIO
+import saneyaml
+from dephell_specifier import RangeSpecifier
+from urllib.request import urlopen
+
+RUBYSEC_DB_URL = 'https://github.com/rubysec/ruby-advisory-db/archive/master.zip'
+
+
+def rubygem_advisories(url, prefix='ruby-advisory-db-master/gems/'):
+    with urlopen(url) as response:
+        with ZipFile(BytesIO(response.read())) as zf:
+            for path in zf.namelist():
+                if path.startswith(prefix) and path.endswith('.yml'):
+                    yield saneyaml.load(zf.open(path))
+
+
+def get_all_versions_of_package(package_name):
+    url_to_load = 'https://rubygems.org/api/v1/versions/' + package_name + '.yaml'
+    try:
+        page = urllib.request.urlopen(url_to_load)
+        package_history = saneyaml.load(page)
+    except HTTPError:
+        return []
+    for version in package_history:
+        yield version['number']
+
+
+def get_patched_range(spec_list):
+    spec_list = [string.replace(' ', '') for string in spec_list]
+    for spec in spec_list:
+        if 'rc' in spec:
+            continue
+        yield RangeSpecifier(spec)
+
+
+def import_vulnerabilities():
+    vulnerability_package_dicts = []
+    for vulnerability in rubygem_advisories(RUBYSEC_DB_URL):
+
+        package_name = vulnerability.get(
+            'gem')
+
+        if not package_name:
+            continue
+
+        if 'cve' in vulnerability:
+            vulnerability_id = 'CVE-{}'.format(vulnerability['cve'])
+        else:
+            continue
+
+        advisory_url = vulnerability.get('url')
+        patched_version_ranges = list(
+            get_patched_range(
+                vulnerability.get('patched_versions', [])))
+        all_versions = set(get_all_versions_of_package(package_name))
+        unaffected_versions = set()
+
+        if patched_version_ranges:
+            for version in all_versions:
+                for spec in patched_version_ranges:
+                    if version in spec:
+                        unaffected_versions.add(version)
+                        break
+
+        affected_versions = all_versions - unaffected_versions
+        vulnerability_package_dicts.append({
+            'package_name': package_name,
+            'cve_id': vulnerability_id,
+            'fixed_versions': unaffected_versions,
+            'affected_versions': affected_versions,
+            'advisory': advisory_url
+        })
+    return vulnerability_package_dicts