From 58aeb83594a659f0d3b9a5f72ae4cd584f375a72 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Mon, 13 Apr 2020 16:26:49 +0530 Subject: [PATCH 1/2] Add schema validation for debian importer Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/debian.py | 40 ++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/scraper/debian.py b/vulnerabilities/scraper/debian.py index 80edec456..c4a24f121 100644 --- a/vulnerabilities/scraper/debian.py +++ b/vulnerabilities/scraper/debian.py @@ -23,6 +23,7 @@ import json from urllib.request import urlopen +from schema import Regex, Or, Schema, Optional DEBIAN_TRACKER_URL = 'https://security-tracker.debian.org/tracker/data/json' @@ -33,8 +34,10 @@ def extract_vulnerabilities(debian_data, base_release='jessie'): package and vulnerability from a mapping of Debian vulnerabilities data. """ - package_vulnerabilities = [] + validate_schema(debian_data) + + package_vulnerabilities = [] for package_name, vulnerabilities in debian_data.items(): if not vulnerabilities or not package_name: continue @@ -68,6 +71,41 @@ def extract_vulnerabilities(debian_data, base_release='jessie'): return package_vulnerabilities +def validate_schema(advisory_dict): + + deb_versions = ['bullseye', 'buster', 'buster-security', 'sid', + 'stretch', 'stretch-security', 'jessie', 'jessie-security'] + scheme = { + str: + { + Or(Regex(r"CVE-\d+-\d+"), Regex(r"TEMP-.+-.+")): + { + 'releases': + { + Or(*deb_versions): + { + 'repositories': + { + Or(*deb_versions): str + }, + 'status': str, + 'urgency': str, + Optional('fixed_version'): str, + Optional(str): object, + } + + }, + Optional('description'): str, + Optional('debianbug'): int, + Optional(str): object, + } + } + + } + + Schema(scheme).validate(advisory_dict) + + def scrape_vulnerabilities(): """ Scrape debian' security tracker. From 1052f0254336f5632cedacfc513f81fe224301d9 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Mon, 13 Apr 2020 16:27:20 +0530 Subject: [PATCH 2/2] Add schema validation for archlinux importer Signed-off-by: Shivam Sandbhor --- vulnerabilities/scraper/archlinux.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/scraper/archlinux.py b/vulnerabilities/scraper/archlinux.py index 1b50ae3b7..69c6630f4 100644 --- a/vulnerabilities/scraper/archlinux.py +++ b/vulnerabilities/scraper/archlinux.py @@ -23,14 +23,36 @@ import json from urllib.request import urlopen - +from schema import Regex, Schema, Or ARCHLINUX_TRACKER_URL = 'https://security.archlinux.org/json' +def validate_schema(advisory_dict): + scheme = { + + 'advisories': list, + 'affected': str, + 'fixed': Or(None, str), + 'issues': [Regex(r'CVE-\d+-\d+')], + 'name': str, + 'packages': [str], + 'status': str, + 'ticket': object, + 'type': str, + 'severity': str, + + } + + Schema(scheme).validate(advisory_dict) + + def scrape_vulnerabilities(): """ Fetch and return data scraped from archlinux security tracker. """ json_content = urlopen(ARCHLINUX_TRACKER_URL).read() arch_data = json.loads(json_content) + for advisory in arch_data: + validate_schema(advisory) + return arch_data