Merge pull request #153 from sbs2001/develop

Add  importer for Python safetydb

Author: haikoschol

vulnerabilities/data_dump.py

@@ -262,3 +262,38 @@ def rust_dump(extract_data):
                 vulnerability=vulnerability,
                 package=unaffected_package
             )
+
+
+def safetydb_dump(extract_data):
+    for package_data in extract_data:
+        vulnerability, _ = Vulnerability.objects.get_or_create(
+            summary=package_data['description'],
+            cve_id=package_data['cve_id']
+        )
+
+        VulnerabilityReference.objects.get_or_create(
+            vulnerability=vulnerability,
+            reference_id=package_data['vuln_id']
+        )
+
+        for version in package_data['affected_versions']:
+            affected_package = Package.objects.create(
+                name=package_data['package_name'],
+                type='pypi',
+                version=version
+            )
+            ImpactedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=affected_package
+            )
+
+        for version in package_data['unaffected_versions']:
+            unaffected_package = Package.objects.create(
+                name=package_data['package_name'],
+                type='pypi',
+                version=version
+            )
+            ResolvedPackage.objects.create(
+                vulnerability=vulnerability,
+                package=unaffected_package
+            )

vulnerabilities/management/commands/import.py

@@ -24,9 +24,10 @@
 from django.core.management.base import BaseCommand, CommandError
 
 from vulnerabilities import data_dump as dd
-from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby, rust
+from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby, rust, safety_db
 
 IMPORTERS = {
+    'safetydb': lambda: safety_db.import_vulnerabilities(),
     'rust': lambda: dd.rust_dump(rust.import_vulnerabilities()),
     'ruby': lambda: dd.ruby_dump(ruby.import_vulnerabilities()),
     'npm': lambda: dd.npm_dump(npm.scrape_vulnerabilities()),

vulnerabilities/scraper/safety_db.py

@@ -0,0 +1,61 @@
+# Data Imported from https://github.com/pyupio/safety-db
+import json
+from urllib.error import HTTPError
+from urllib.request import urlopen
+from dephell_specifier import RangeSpecifier
+
+URL = 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json'
+
+
+def get_data_file():
+    with urlopen(URL) as file:
+        json_file = json.load(file)
+    return json_file
+
+
+def get_all_versions_of_package(package_name):
+    url = "https://pypi.org/pypi/{}/json".format(package_name)
+    releases = set()
+    try:
+        with urlopen(url) as response:
+            json_file = json.load(response)
+    except HTTPError:  # PyPi does not have data about this package, we skip these
+        return releases
+    for release in json_file['releases']:
+        releases.add(release)
+    return releases
+
+
+def import_vulnerabilities():
+    vulnerability_package_dicts = []
+    data = get_data_file()
+    cves = set()
+    for package_name in data:
+        all_package_versions = set(get_all_versions_of_package(package_name))
+        if len(all_package_versions) == 0:
+            # PyPi does not have data about this package, we skip these
+            continue
+        for advisory in data[package_name]:
+            description = advisory['advisory']
+            cve_id = advisory.get('cve')
+            vuln_id = advisory['id']
+            vuln_version_ranges = advisory['specs']
+            for vuln_version_range in vuln_version_ranges:
+                version_range = RangeSpecifier(vuln_version_range)
+                affected_versions = set()
+                for version in all_package_versions:
+                    if version in version_range:
+                        affected_versions.add(version)
+            unaffected_versions = all_package_versions - affected_versions
+            if cve_id not in cves:
+                # This data has duplicates hence the check
+                vulnerability_package_dicts.append({
+                    'package_name': package_name,
+                    'vuln_id': vuln_id,
+                    'affected_versions': affected_versions,
+                    'unaffected_versions': unaffected_versions,
+                    'cve_id': cve_id,
+                    'description': description
+                })
+                cves.add(cve_id)
+    return vulnerability_package_dicts