Cleanup codebase and fix minor bugs and other improvements

 * Move the implementation of the methods for handling etags and different
   file  formats into a separate `helpers.py` module to prevent code duplication

 * Fix bugs in the msr2019 importer.

Author: sbs2001

vulnerabilities/helpers.py

@@ -0,0 +1,54 @@
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/vulnerablecode/
+# The VulnerableCode software is licensed under the Apache License version 2.0.
+# Data generated with VulnerableCode require an acknowledgment.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  VulnerableCode is a free software from nexB Inc. and others.
+#  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+
+import json
+import requests
+import yaml
+import pytoml as toml
+
+
+def load_yaml(path):
+    with open(path) as f:
+        return yaml.safe_load(f)
+
+
+def load_json(path):
+    with open(path) as f:
+        return json.load(f)
+
+
+def load_toml(path):
+    with open(path) as f:
+        return toml.load(f)
+
+
+def create_etag(data_src, url, etag_key):
+    etag = requests.head(url).headers.get(etag_key)
+    if not etag:
+        return True
+
+    elif url in data_src.config.etags:
+        if data_src.config.etags[url] == etag:
+            return False
+
+    data_src.config.etags[url] = etag
+    return True

vulnerabilities/import_runner.py

@@ -90,7 +90,7 @@ class ImportRunner:
     Efficiency:
         - Bulk inserts should be used whenever possible.
         - Checking whether a record already exists should be kept to a minimum
-          (the data source should know this instead).
+        (the data source should know this instead).
         - All update and select operations must use indexed columns.
     """
 

vulnerabilities/importer_yielder.py

@@ -188,7 +188,7 @@
         'last_run': None,
         'data_source': 'ProjectKBMSRDataSource',
         'data_source_cfg': {
-            'etag': {}
+            'etags': {}
         }
     },
     {
@@ -216,7 +216,7 @@
         'last_run': None,
         'data_source': 'NginxDataSource',
         'data_source_cfg': {
-            'etag': {}
+            'etags': {}
         },
     },
     {

vulnerabilities/importers/alpine_linux.py

@@ -20,6 +20,7 @@
 #  for any legal advice.
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+import logging
 from typing import Any
 from typing import Iterable
 from typing import List

vulnerabilities/importers/apache_httpd.py

@@ -20,7 +20,7 @@
 #  VulnerableCode is a free software tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-from dataclasses import dataclass
+import dataclasses
 from xml.etree import ElementTree
 
 import requests
@@ -29,9 +29,10 @@
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
 from vulnerabilities.data_source import DataSourceConfiguration
+from vulnerabilities.helpers import create_etag
 
 
-@dataclass
+@dataclasses.dataclass
 class ApacheHTTPDDataSourceConfiguration(DataSourceConfiguration):
     etags: dict
 
@@ -47,25 +48,13 @@ def updated_advisories(self):
         # (url, etag) pair. If a (url, etag) already exists then the code
         # skips processing the response further to avoid duplicate work
 
-        if self.create_etag(self.url):
+        if create_etag(data_src=self, url=self.url, etag_key="ETag"):
             data = fetch_xml(self.url)
             advisories = to_advisories(data)
             return self.batch_advisories(advisories)
 
         return []
 
-    def create_etag(self, url):
-        etag = requests.head(url).headers.get("ETag")
-        if not etag:
-            return True
-
-        elif url in self.config.etags:
-            if self.config.etags[url] == etag:
-                return False
-
-        self.config.etags[url] = etag
-        return True
-
 
 def to_advisories(data):
     advisories = []

vulnerabilities/importers/debian_oval.py

@@ -35,6 +35,7 @@
 
 from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration
 from vulnerabilities.package_managers import DebianVersionAPI
+from vulnerabilities.helpers import create_etag
 
 
 @dataclasses.dataclass
@@ -52,35 +53,24 @@ def __init__(self, *args, **kwargs):
         # we could avoid setting translations, and have it
         # set by default in the OvalParser, but we don't yet know
         # whether all OVAL providers use the same format
-        self.translations = {'less than': '<'}
+        self.translations = {"less than": "<"}
         self.pkg_manager_api = DebianVersionAPI()
 
     def _fetch(self):
-        base_url = 'https://www.debian.org/security/oval/'
-        file_name = 'oval-definitions-{}.xml'
+        base_url = "https://www.debian.org/security/oval/"
+        file_name = "oval-definitions-{}.xml"
         releases = self.config.releases
         for release in releases:
             file_url = base_url + file_name.format(release)
-            if not self.create_etag(file_url):
+            if not create_etag(data_src=self, url=file_url, etag_key="ETag"):
                 continue
+
             resp = requests.get(file_url).content
             yield (
-                {'type': 'deb', 'namespace': 'debian',
-                    'qualifiers': {'distro': release}
-                 },
-                ET.ElementTree(ET.fromstring(resp.decode('utf-8')))
+                {"type": "deb", "namespace": "debian", "qualifiers": {"distro": release}},
+                ET.ElementTree(ET.fromstring(resp.decode("utf-8"))),
             )
         return []
 
     def set_api(self, packages):
         asyncio.run(self.pkg_manager_api.load_api(packages))
-
-    def create_etag(self, url):
-        etag = requests.head(url).headers.get('ETag')
-        if not etag:
-            return True
-        elif url in self.config.etags:
-            if self.config.etags[url] == etag:
-                return False
-        self.config.etags[url] = etag
-        return True

vulnerabilities/importers/kaybee.py

@@ -20,13 +20,12 @@
 #  VulnerableCode is a free software tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
-import yaml
-
 from packageurl import PackageURL
 
 from vulnerabilities.data_source import GitDataSource
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import Reference
+from vulnerabilities.helpers import load_yaml
 
 
 class KaybeeDataSource(GitDataSource):
@@ -36,14 +35,11 @@ def __enter__(self):
             recursive=True,
             file_ext="yaml",
         )
-        print(self._added_files.union(self._updated_files))
 
     def updated_advisories(self):
         advisories = []
         for yaml_file in self._added_files.union(self._updated_files):
-            print(yaml_file)
             advisories.append(yaml_file_to_advisory(yaml_file))
-            print(advisories[-1])
 
         return self.batch_advisories(advisories)
 
@@ -76,9 +72,3 @@ def yaml_file_to_advisory(yaml_path):
         resolved_package_urls=resolved_packages,
         vuln_references=references,
     )
-
-
-# TODO refactor all such  commonly needed helpers into one single module
-def load_yaml(path):
-    with open(path) as f:
-        return yaml.safe_load(f)

vulnerabilities/importers/nginx.py

@@ -1,23 +1,23 @@
-# Copyright (c)  nexB Inc. and others. All rights reserved.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/vulnerablecode/
-# The VulnerableCode software is licensed under the Apache License version
+# The VulnerableCode software is licensed under the Apache License version 2.0.
 # Data generated with VulnerableCode require an acknowledgment.
 #
 # You may not use this software except in compliance with the License.
-# You may obtain a copy of the License at: http://apache.org/licenses/LICE
-# Unless required by applicable law or agreed to in writing, software dist
-# under the License is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES
-# CONDITIONS OF ANY KIND, either express or implied. See the License for t
-# specific language governing permissions and limitations under the Licens
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
 #
-# When you publish or redistribute any data created with VulnerableCode or
-# derivative work, you must accompany this data with the following acknowl
+# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode
+# derivative work, you must accompany this data with the following acknowledgment:
 #
-#  Generated with VulnerableCode and provided on an 'AS IS' BASIS, WITHOUT
-#  OR CONDITIONS OF ANY KIND, either express or implied. No content create
-#  VulnerableCode should be considered or used as legal advice. Consult an
+#  Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  VulnerableCode should be considered or used as legal advice. Consult an Attorney
 #  for any legal advice.
-#  VulnerableCode is a free software  from nexB Inc. and others.
+#  VulnerableCode is a free software tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 import asyncio
@@ -34,11 +34,12 @@
 from vulnerabilities.data_source import DataSourceConfiguration
 from vulnerabilities.data_source import Reference
 from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.helpers import create_etag
 
 
 @dataclasses.dataclass
 class NginxDataSourceConfiguration(DataSourceConfiguration):
-    etag: dict
+    etags: dict
 
 
 class NginxDataSource(DataSource):
@@ -57,24 +58,12 @@ def set_api(self):
 
     def updated_advisories(self):
         advisories = []
-        if self.create_etag():
+        if create_etag(data_src=self, url=self.url, etag_key="ETag"):
             self.set_api()
             data = requests.get(self.url).content
             advisories.extend(self.to_advisories(data))
         return self.batch_advisories(advisories)
 
-    def create_etag(self):
-        etag = requests.head(self.url).headers.get("ETag")
-        if not etag:
-            return True
-
-        elif self.url in self.config.etag:
-            if self.config.etag[self.url] == etag:
-                return False
-
-        self.config.etag[self.url] = etag
-        return True
-
     def to_advisories(self, data):
         advisories = []
         soup = BeautifulSoup(data)

vulnerabilities/importers/npm.py

@@ -1,5 +1,5 @@
 # Author: Navonil Das (@NavonilDas)
-# Copyright (c) 2017 nexB Inc. and others. All rights reserved.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # http://nexb.com and https://github.com/nexB/vulnerablecode/
 # The VulnerableCode software is licensed under the Apache License version 2.0.
 # Data generated with VulnerableCode require an acknowledgment.
@@ -18,11 +18,10 @@
 #  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
 #  VulnerableCode should be considered or used as legal advice. Consult an Attorney
 #  for any legal advice.
-#  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
+#  VulnerableCode is a free software from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 import asyncio
-import json
 from typing import Any
 from typing import List
 from typing import Mapping
@@ -40,6 +39,7 @@
 from vulnerabilities.data_source import GitDataSource
 from vulnerabilities.data_source import Reference
 from vulnerabilities.package_managers import NpmVersionAPI
+from vulnerabilities.helpers import load_json
 
 NPM_URL = "https://registry.npmjs.org{}"
 
@@ -121,7 +121,9 @@ def _versions_to_purls(package_name, versions):
 
 
 def categorize_versions(
-    all_versions: Set[str], aff_version_range: str, fixed_version_range: str,
+    all_versions: Set[str],
+    aff_version_range: str,
+    fixed_version_range: str,
 ) -> Tuple[Set[str], Set[str]]:
     """
     Seperate list of affected versions and unaffected versions from all versions
@@ -146,8 +148,3 @@ def categorize_versions(
             aff_ver.add(ver)
 
     return aff_ver, fix_ver
-
-
-def load_json(path):
-    with open(path) as f:
-        return json.load(f)

vulnerabilities/importers/nvd.py

@@ -32,6 +32,7 @@
 from vulnerabilities.data_source import DataSource
 from vulnerabilities.data_source import DataSourceConfiguration
 from vulnerabilities.data_source import Reference
+from vulnerabilities.helpers import create_etag
 
 
 @dataclasses.dataclass
@@ -49,13 +50,13 @@ class NVDDataSource(DataSource):
     def updated_advisories(self):
         current_year = date.today().year
         # NVD json feeds start from 2002.
-        for year in range(2002, current_year+1):
+        for year in range(2002, current_year + 1):
             download_url = BASE_URL.format(year)
             # Etags are like hashes of web responses. We maintain
             # (url, etag) mappings in the DB. `create_etag`  creates
             # (url, etag) pair. If a (url, etag) already exists then the code
             # skips processing the response further to avoid duplicate work
-            if self.create_etag(download_url):
+            if create_etag(data_src=self, url=download_url, etag_key="etag"):
                 data = self.fetch(download_url)
                 yield self.to_advisories(data)
 
@@ -130,15 +131,3 @@ def extract_cpes(cve_item):
             for cpe_data in node.get("cpe_match", []):
                 cpes.add(cpe_data["cpe23Uri"])
         return cpes
-
-    def create_etag(self, url):
-        etag = requests.head(url).headers.get("etag")
-        if not etag:
-            # Kind of inaccurate to return True since etag is
-            # not created
-            return True
-        elif url in self.config.etags:
-            if self.config.etags[url] == etag:
-                return False
-        self.config.etags[url] = etag
-        return True