diff --git a/requirements.txt b/requirements.txt index dc77be3f8..621436e00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,3 +31,4 @@ tqdm==4.41.1 wcwidth==0.1.7 whitenoise==5.0.1 zipp==0.6.0 +pytoml==0.1.21 \ No newline at end of file diff --git a/vulnerabilities/data_dump.py b/vulnerabilities/data_dump.py index cc99f8dc8..831252c37 100644 --- a/vulnerabilities/data_dump.py +++ b/vulnerabilities/data_dump.py @@ -225,3 +225,40 @@ def ruby_dump(extract_data): vulnerability=vulnerability, package=unaffected_package ) + + +def rust_dump(extract_data): + + for package_data in extract_data: + + vulnerability, _ = Vulnerability.objects.get_or_create( + summary=package_data['description'] + ) + + VulnerabilityReference.objects.get_or_create( + vulnerability=vulnerability, + url=package_data['advisory'], + reference_id=package_data['vuln_id'] + ) + + for version in package_data['affected_versions']: + affected_package = Package.objects.create( + name=package_data['package_name'], + type='cargo', + version=version + ) + ImpactedPackage.objects.create( + vulnerability=vulnerability, + package=affected_package + ) + + for version in package_data['fixed_versions']: + unaffected_package = Package.objects.create( + name=package_data['package_name'], + type='cargo', + version=version + ) + ResolvedPackage.objects.create( + vulnerability=vulnerability, + package=unaffected_package + ) diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index f71528903..ebf59b89c 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -24,9 +24,10 @@ from django.core.management.base import BaseCommand, CommandError from vulnerabilities import data_dump as dd -from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby +from vulnerabilities.scraper import debian, ubuntu, archlinux, npm, ruby, rust IMPORTERS = { + 'rust': lambda: dd.rust_dump(rust.import_vulnerabilities()), 'ruby': lambda: dd.ruby_dump(ruby.import_vulnerabilities()), 'npm': lambda: dd.npm_dump(npm.scrape_vulnerabilities()), 'debian': lambda: dd.debian_dump(debian.scrape_vulnerabilities()), diff --git a/vulnerabilities/scraper/rust.py b/vulnerabilities/scraper/rust.py new file mode 100644 index 000000000..715f78ad3 --- /dev/null +++ b/vulnerabilities/scraper/rust.py @@ -0,0 +1,93 @@ +from io import BytesIO +import json + +from dephell_specifier import RangeSpecifier +import pytoml as toml +from urllib.request import urlopen +import urllib.request +from urllib.error import HTTPError +from zipfile import ZipFile + + +RUSTSEC_DB_URL = 'https://github.com/RustSec/advisory-db/archive/master.zip' + + +def rust_crate_advisories(url, prefix='advisory-db-master/crates/'): + with urlopen(url) as response: + with ZipFile(BytesIO(response.read())) as zf: + for path in zf.namelist(): + if path.startswith(prefix) and path.endswith('.toml'): + yield toml.load(zf.open(path)) + + +def all_versions_of_crate(crate): + info_url = "https://crates.io/api/v1/crates/{}".format(crate) + with urlopen(info_url) as info: + info = json.load(info) + for version_info in info['versions']: + yield version_info['num'] + + +def import_vulnerabilities(): + + vulnerability_package_dicts = [] + for advisory in rust_crate_advisories(RUSTSEC_DB_URL): + + affected_version_range_lists = list(advisory.get( + 'affected', {}).get('functions', {}).values()) + unaffected_version_range_list = advisory['advisory'].get( + 'unaffected_versions', []) + patched_version_range_list = advisory['advisory']['patched_versions'] + + # FIXME: Make distinction between unaffected and patched packages + # Check https://github.com/nexB/vulnerablecode/issues/144 + unaffected_version_range_list += patched_version_range_list + + have_unaffected_version_range = len(unaffected_version_range_list) != 0 + have_affected_version_range = len(affected_version_range_lists) != 0 + + if not (have_affected_version_range or have_unaffected_version_range): + continue + + vuln_id = advisory['advisory']['id'] + crate_name = advisory['advisory']['package'] + description = advisory['advisory']['description'] + reference = advisory['advisory'].get('url', '') + + all_versions = set(all_versions_of_crate(crate_name)) + affected_versions = set() + unaffected_versions = set() + + for version in all_versions: + categorised = False + for affected_version_range_list in affected_version_range_lists: + for affected_version_range in affected_version_range_list: + affected_version_range = RangeSpecifier( + affected_version_range) + if version in affected_version_range: + affected_versions.add(version) + categorised = True + break + + if have_unaffected_version_range and not categorised: + for unaffected_version_range in unaffected_version_range_list: + if version in RangeSpecifier(unaffected_version_range): + unaffected_versions.add(version) + break + + if not have_unaffected_version_range: + unaffected_versions = all_versions - affected_versions + + elif not have_affected_version_range: + affected_versions = all_versions - unaffected_versions + + vulnerability_package_dicts.append({ + 'package_name': crate_name, + 'vuln_id': vuln_id, + 'fixed_versions': unaffected_versions, + 'affected_versions': affected_versions, + 'advisory': reference, + 'description': description + }) + + return vulnerability_package_dicts