From f6e1bca51b432c0a8cc3aa6a5a8eac13659da7bd Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Wed, 29 Jul 2020 19:58:26 +0530 Subject: [PATCH 1/2] Add etags to safetydb importer Signed-off-by: Shivam Sandbhor --- vulnerabilities/importer_yielder.py | 3 ++- vulnerabilities/importers/safety_db.py | 21 +++++++++++++++++++-- vulnerabilities/tests/test_safety_db.py | 1 + 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 9a60a6869..55a449f5e 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -67,7 +67,8 @@ 'last_run': None, 'data_source': 'SafetyDbDataSource', 'data_source_cfg': { - 'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json' # nopep8 + 'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json', # nopep8 + 'etags': {} }, }, { diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py index f18d40319..7873c0138 100755 --- a/vulnerabilities/importers/safety_db.py +++ b/vulnerabilities/importers/safety_db.py @@ -33,6 +33,7 @@ from typing import Tuple from urllib.error import HTTPError from urllib.request import urlopen +import requests from dephell_specifier import RangeSpecifier from packageurl import PackageURL @@ -67,6 +68,7 @@ def validate_schema(advisory_dict): @dataclasses.dataclass class SafetyDbConfiguration(DataSourceConfiguration): url: str + etags: dict class SafetyDbDataSource(DataSource): @@ -90,8 +92,11 @@ def set_api(self, packages): asyncio.run(self._versions.load_api(packages)) def _fetch(self) -> Mapping[str, Any]: - with urlopen(self.config.url) as response: - return json.load(response) + if self.create_etag(self.config.url): + with urlopen(self.config.url) as response: + return json.load(response) + + return [] def collect_packages(self): return {pkg for pkg in self._api_response} @@ -132,6 +137,18 @@ def updated_advisories(self) -> Set[Advisory]: return self.batch_advisories(advisories) + def create_etag(self, url): + etag = requests.head(url).headers.get('ETag') + if not etag: + # Kind of inaccurate to return True since etag is + # not created + return True + elif url in self.config.etags: + if self.config.etags[url] == etag: + return False + self.config.etags[url] = etag + return True + def categorize_versions( package_name: str, all_versions: Set[str], version_specs: Iterable[str], diff --git a/vulnerabilities/tests/test_safety_db.py b/vulnerabilities/tests/test_safety_db.py index befaa6d64..bfc7b957c 100644 --- a/vulnerabilities/tests/test_safety_db.py +++ b/vulnerabilities/tests/test_safety_db.py @@ -57,6 +57,7 @@ def setUpClass(cls) -> None: data_source='SafetyDbDataSource', data_source_cfg={ 'url': 'https://example.com', + 'etags': {} }, ) From 091d479f12bcb48f9b06532cc1b1e4ec08db6b37 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Wed, 29 Jul 2020 22:35:03 +0530 Subject: [PATCH 2/2] Leverage Etags in openssl importer Signed-off-by: Shivam Sandbhor --- vulnerabilities/importer_yielder.py | 4 ++- vulnerabilities/importers/openssl.py | 49 ++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 55a449f5e..e76ce5f4b 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -149,7 +149,9 @@ 'license': '', 'last_run': None, 'data_source': 'OpenSSLDataSource', - 'data_source_cfg': {}, + 'data_source_cfg': { + 'etags': {} + }, }, { 'name': 'ubuntu_usn', diff --git a/vulnerabilities/importers/openssl.py b/vulnerabilities/importers/openssl.py index ffc01ec15..00862d663 100644 --- a/vulnerabilities/importers/openssl.py +++ b/vulnerabilities/importers/openssl.py @@ -20,27 +20,58 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. +import dataclasses +import re from typing import Set +import xml.etree.ElementTree as ET from packageurl import PackageURL import requests -import re from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import Reference +from vulnerabilities.data_source import DataSourceConfiguration -import xml.etree.ElementTree as ET + +@dataclasses.dataclass +class OpenSSLDataSourceConfiguration(DataSourceConfiguration): + etags: dict class OpenSSLDataSource(DataSource): + CONFIG_CLASS = OpenSSLDataSourceConfiguration + + url = "https://www.openssl.org/news/vulnerabilities.xml" + def updated_advisories(self) -> Set[Advisory]: - raw_data = self.fetch() - advisories = self.to_advisories(raw_data) - return self.batch_advisories(advisories) + # Etags are like hashes of web responses. We maintain + # (url, etag) mappings in the DB. `create_etag` creates + # (url, etag) pair. If a (url, etag) already exists then the code + # skips processing the response further to avoid duplicate work + if self.create_etag(self.url): + raw_data = self.fetch() + advisories = self.to_advisories(raw_data) + return self.batch_advisories(advisories) + + return [] + + def create_etag(self, url): + etag = requests.head(url).headers.get("ETag") + if not etag: + # Kind of inaccurate to return True since etag is + # not created + return True + + elif url in self.config.etags: + if self.config.etags[url] == etag: + return False + + self.config.etags[url] = etag + return True def fetch(self): - return requests.get("https://www.openssl.org/news/vulnerabilities.xml").content + return requests.get(self.url).content @staticmethod def to_advisories(xml_response: str) -> Set[Advisory]: @@ -58,7 +89,11 @@ def to_advisories(xml_response: str) -> Set[Advisory]: for info in element: if info.tag == "cve": - cve_id = "CVE-" + info.attrib.get("name") + if info.attrib.get("name"): + cve_id = "CVE-" + info.attrib.get("name") + + else: + continue if info.tag == "affects": # Vulnerable package versions