Merge pull request #237 from sbs2001/add_etags

Save  etags to importers wherever possible

Author: sbs2001

vulnerabilities/importer_yielder.py

@@ -67,7 +67,8 @@
         'last_run': None,
         'data_source': 'SafetyDbDataSource',
         'data_source_cfg': {
-            'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json'  # nopep8
+            'url': 'https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json',  # nopep8
+            'etags': {}
         },
     },
     {
@@ -148,7 +149,9 @@
         'license': '',
         'last_run': None,
         'data_source': 'OpenSSLDataSource',
-        'data_source_cfg': {},
+        'data_source_cfg': {
+            'etags': {}
+        },
     },
     {
         'name': 'ubuntu_usn',

vulnerabilities/importers/openssl.py

@@ -20,27 +20,58 @@
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
+import dataclasses
+import re
 from typing import Set
+import xml.etree.ElementTree as ET
 
 from packageurl import PackageURL
 import requests
-import re
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
 from vulnerabilities.data_source import Reference
+from vulnerabilities.data_source import DataSourceConfiguration
 
-import xml.etree.ElementTree as ET
+
+@dataclasses.dataclass
+class OpenSSLDataSourceConfiguration(DataSourceConfiguration):
+    etags: dict
 
 
 class OpenSSLDataSource(DataSource):
+    CONFIG_CLASS = OpenSSLDataSourceConfiguration
+
+    url = "https://www.openssl.org/news/vulnerabilities.xml"
+
     def updated_advisories(self) -> Set[Advisory]:
-        raw_data = self.fetch()
-        advisories = self.to_advisories(raw_data)
-        return self.batch_advisories(advisories)
+        # Etags are like hashes of web responses. We maintain
+        # (url, etag) mappings in the DB. `create_etag`  creates
+        # (url, etag) pair. If a (url, etag) already exists then the code
+        # skips processing the response further to avoid duplicate work
+        if self.create_etag(self.url):
+            raw_data = self.fetch()
+            advisories = self.to_advisories(raw_data)
+            return self.batch_advisories(advisories)
+
+        return []
+
+    def create_etag(self, url):
+        etag = requests.head(url).headers.get("ETag")
+        if not etag:
+            # Kind of inaccurate to return True since etag is
+            # not created
+            return True
+
+        elif url in self.config.etags:
+            if self.config.etags[url] == etag:
+                return False
+
+        self.config.etags[url] = etag
+        return True
 
     def fetch(self):
-        return requests.get("https://www.openssl.org/news/vulnerabilities.xml").content
+        return requests.get(self.url).content
 
     @staticmethod
     def to_advisories(xml_response: str) -> Set[Advisory]:
@@ -58,7 +89,11 @@ def to_advisories(xml_response: str) -> Set[Advisory]:
                 for info in element:
 
                     if info.tag == "cve":
-                        cve_id = "CVE-" + info.attrib.get("name")
+                        if info.attrib.get("name"):
+                            cve_id = "CVE-" + info.attrib.get("name")
+
+                        else:
+                            continue
 
                     if info.tag == "affects":
                         # Vulnerable package versions

vulnerabilities/importers/safety_db.py

@@ -33,6 +33,7 @@
 from typing import Tuple
 from urllib.error import HTTPError
 from urllib.request import urlopen
+import requests
 
 from dephell_specifier import RangeSpecifier
 from packageurl import PackageURL
@@ -67,6 +68,7 @@ def validate_schema(advisory_dict):
 @dataclasses.dataclass
 class SafetyDbConfiguration(DataSourceConfiguration):
     url: str
+    etags: dict
 
 
 class SafetyDbDataSource(DataSource):
@@ -90,8 +92,11 @@ def set_api(self, packages):
         asyncio.run(self._versions.load_api(packages))
 
     def _fetch(self) -> Mapping[str, Any]:
-        with urlopen(self.config.url) as response:
-            return json.load(response)
+        if self.create_etag(self.config.url):
+            with urlopen(self.config.url) as response:
+                return json.load(response)
+
+        return []
 
     def collect_packages(self):
         return {pkg for pkg in self._api_response}
@@ -132,6 +137,18 @@ def updated_advisories(self) -> Set[Advisory]:
 
         return self.batch_advisories(advisories)
 
+    def create_etag(self, url):
+        etag = requests.head(url).headers.get('ETag')
+        if not etag:
+            # Kind of inaccurate to return True since etag is
+            # not created
+            return True
+        elif url in self.config.etags:
+            if self.config.etags[url] == etag:
+                return False
+        self.config.etags[url] = etag
+        return True
+
 
 def categorize_versions(
     package_name: str, all_versions: Set[str], version_specs: Iterable[str],

vulnerabilities/tests/test_safety_db.py

@@ -57,6 +57,7 @@ def setUpClass(cls) -> None:
             data_source='SafetyDbDataSource',
             data_source_cfg={
                 'url': 'https://example.com',
+                'etags': {}
             },
         )