Use class for returning versions partitioned by cutoff time

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

Author: sbs2001

pytest.ini

@@ -1,2 +1,4 @@
 [pytest]
-DJANGO_SETTINGS_MODULE = vulnerablecode.settings
+DJANGO_SETTINGS_MODULE = vulnerablecode.settings
+markers =
+    webtest

vulnerabilities/importers/apache_httpd.py

@@ -37,7 +37,6 @@
 from vulnerabilities.data_source import VulnerabilitySeverity
 from vulnerabilities.package_managers import GitHubTagsAPI
 from vulnerabilities.severity_systems import scoring_systems
-from vulnerabilities.helpers import create_etag
 from vulnerabilities.helpers import nearest_patched_package
 
 
@@ -106,7 +105,7 @@ def to_advisory(self, data):
             fixed_packages.extend(
                 [
                     PackageURL(type="apache", name="httpd", version=version)
-                    for version in self.version_api.get("apache/httpd")["valid"]
+                    for version in self.version_api.get("apache/httpd").valid_versions
                     if MavenVersion(version) in version_range
                 ]
             )
@@ -115,7 +114,7 @@ def to_advisory(self, data):
             affected_packages.extend(
                 [
                     PackageURL(type="apache", name="httpd", version=version)
-                    for version in self.version_api.get("apache/httpd")["valid"]
+                    for version in self.version_api.get("apache/httpd").valid_versions
                     if MavenVersion(version) in version_range
                 ]
             )

vulnerabilities/importers/apache_kafka.py

@@ -72,7 +72,7 @@ def to_advisory(self, advisory_page):
 
             fixed_packages = [
                 PackageURL(type="apache", name="kafka", version=version)
-                for version in self.version_api.get("apache/kafka")["valid"]
+                for version in self.version_api.get("apache/kafka").valid_versions
                 if any(
                     [
                         MavenVersion(version) in version_range
@@ -83,7 +83,7 @@ def to_advisory(self, advisory_page):
 
             affected_packages = [
                 PackageURL(type="apache", name="kafka", version=version)
-                for version in self.version_api.get("apache/kafka")["valid"]
+                for version in self.version_api.get("apache/kafka").valid_versions
                 if any(
                     [
                         MavenVersion(version) in version_range

vulnerabilities/importers/apache_tomcat.py

@@ -63,7 +63,7 @@ def updated_advisories(self):
 
     def fetch_pages(self):
         tomcat_major_versions = {
-            i[0] for i in self.version_api.get("org.apache.tomcat:tomcat")["valid"]
+            i[0] for i in self.version_api.get("org.apache.tomcat:tomcat").valid_versions
         }
         for version in tomcat_major_versions:
             page_url = self.base_url.format(version)
@@ -104,7 +104,9 @@ def to_advisories(self, apache_tomcat_advisory_html):
                             PackageURL(
                                 type="maven", namespace="apache", name="tomcat", version=version
                             )
-                            for version in self.version_api.get("org.apache.tomcat:tomcat")["valid"]
+                            for version in self.version_api.get(
+                                "org.apache.tomcat:tomcat"
+                            ).valid_versions
                             if MavenVersion(version) in version_range
                         ]
                     )

vulnerabilities/importers/elixir_security.py

@@ -83,7 +83,7 @@ def get_versions_for_pkg_from_range_list(self, version_range_list, pkg_name):
 
         safe_pkg_versions = []
         vuln_pkg_versions = []
-        all_version_list = self.pkg_manager_api.get(pkg_name)["valid"]
+        all_version_list = self.pkg_manager_api.get(pkg_name).valid_versions
         if not version_range_list:
             return [], all_version_list
         version_ranges = [

vulnerabilities/importers/github.py

@@ -209,7 +209,7 @@ def process_response(self) -> List[Advisory]:
                         aff_vers, unaff_vers = self.categorize_versions(
                             self.version_api.package_type,
                             aff_range,
-                            self.version_api.get(name, until=cutoff_time)["valid"],
+                            self.version_api.get(name, until=cutoff_time).valid_versions,
                         )
                         affected_purls = [
                             PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)

vulnerabilities/importers/istio.py

@@ -66,7 +66,7 @@ def get_pkg_versions_from_ranges(self, version_range_list, release_date):
         """Takes a list of version ranges(affected) of a package
         as parameter and returns a tuple of safe package versions and
         vulnerable package versions"""
-        all_version = self.version_api.get("istio/istio", release_date)["valid"]
+        all_version = self.version_api.get("istio/istio", release_date).valid_versions
         safe_pkg_versions = []
         vuln_pkg_versions = []
         version_ranges = [

vulnerabilities/importers/nginx.py

@@ -53,8 +53,10 @@ def set_api(self):
 
         # For some reason nginx tags it's releases are in the form of `release-1.2.3`
         # Chop off the `release-` part here.
-        for index, version in enumerate(self.version_api.cache["nginx/nginx"]["valid"]):
-            self.version_api.cache["nginx/nginx"]["valid"][index] = version.replace("release-", "")
+        for index, version in enumerate(self.version_api.cache["nginx/nginx"].valid_versions):
+            self.version_api.cache["nginx/nginx"].valid_versions[index] = version.replace(
+                "release-", ""
+            )
 
     def updated_advisories(self):
         advisories = []
@@ -135,7 +137,7 @@ def extract_fixed_pkgs(self, vuln_info):
             )
 
         valid_versions = find_valid_versions(
-            self.version_api.get("nginx/nginx")["valid"], version_ranges
+            self.version_api.get("nginx/nginx").valid_versions, version_ranges
         )
 
         return [
@@ -172,7 +174,7 @@ def extract_vuln_pkgs(self, vuln_info):
             )
 
         valid_versions = find_valid_versions(
-            self.version_api.get("nginx/nginx")["valid"], version_ranges
+            self.version_api.get("nginx/nginx").valid_versions, version_ranges
         )
         qualifiers = {}
         if windows_only:

vulnerabilities/importers/npm.py

@@ -88,7 +88,7 @@ def process_file(self, file) -> List[Advisory]:
         publish_date = parse(record["updated_at"])
         publish_date.replace(tzinfo=pytz.UTC)
 
-        all_versions = self.versions.get(package_name, until=publish_date)["valid"]
+        all_versions = self.versions.get(package_name, until=publish_date).valid_versions
         aff_range = record.get("vulnerable_versions")
         if not aff_range:
             aff_range = ""

vulnerabilities/importers/ruby.py

@@ -103,10 +103,7 @@ def process_file(self, path) -> List[Advisory]:
 
         if not getattr(self, "pkg_manager_api", None):
             self.pkg_manager_api = RubyVersionAPI()
-        all_vers = self.pkg_manager_api.get(package_name, until=publish_time)["valid"]
-        print(
-            f"Ignored {len(self.pkg_manager_api.get(package_name,until=publish_time)['new'])} versions"
-        )
+        all_vers = self.pkg_manager_api.get(package_name, until=publish_time).valid_versions
         safe_versions, affected_versions = self.categorize_versions(all_vers, safe_version_ranges)
 
         impacted_purls = [