Time travel to the date of advisory publish time when importing

pytest.ini

@@ -1,2 +1,4 @@
 [pytest]
-DJANGO_SETTINGS_MODULE = vulnerablecode.settings
+DJANGO_SETTINGS_MODULE = vulnerablecode.settings
+markers =
+    webtest

vulnerabilities/importers/apache_httpd.py

@@ -37,7 +37,6 @@
 from vulnerabilities.data_source import VulnerabilitySeverity
 from vulnerabilities.package_managers import GitHubTagsAPI
 from vulnerabilities.severity_systems import scoring_systems
-from vulnerabilities.helpers import create_etag
 from vulnerabilities.helpers import nearest_patched_package
 
 
@@ -106,7 +105,7 @@ def to_advisory(self, data):
             fixed_packages.extend(
                 [
                     PackageURL(type="apache", name="httpd", version=version)
-                    for version in self.version_api.get("apache/httpd")
+                    for version in self.version_api.get("apache/httpd").valid_versions
                     if MavenVersion(version) in version_range
                 ]
             )
@@ -115,7 +114,7 @@ def to_advisory(self, data):
             affected_packages.extend(
                 [
                     PackageURL(type="apache", name="httpd", version=version)
-                    for version in self.version_api.get("apache/httpd")
+                    for version in self.version_api.get("apache/httpd").valid_versions
                     if MavenVersion(version) in version_range
                 ]
             )

vulnerabilities/importers/apache_kafka.py

@@ -72,7 +72,7 @@ def to_advisory(self, advisory_page):
 
             fixed_packages = [
                 PackageURL(type="apache", name="kafka", version=version)
-                for version in self.version_api.get("apache/kafka")
+                for version in self.version_api.get("apache/kafka").valid_versions
                 if any(
                     [
                         MavenVersion(version) in version_range
@@ -83,7 +83,7 @@ def to_advisory(self, advisory_page):
 
             affected_packages = [
                 PackageURL(type="apache", name="kafka", version=version)
-                for version in self.version_api.get("apache/kafka")
+                for version in self.version_api.get("apache/kafka").valid_versions
                 if any(
                     [
                         MavenVersion(version) in version_range

vulnerabilities/importers/apache_tomcat.py

@@ -29,6 +29,7 @@
 from bs4 import BeautifulSoup
 from univers.version_specifier import VersionSpecifier
 from univers.versions import MavenVersion
+from univers.versions import SemverVersion
 from packageurl import PackageURL
 
 from vulnerabilities.data_source import Advisory
@@ -62,7 +63,11 @@ def updated_advisories(self):
         return self.batch_advisories(advisories)
 
     def fetch_pages(self):
-        tomcat_major_versions = {i[0] for i in self.version_api.get("org.apache.tomcat:tomcat")}
+        # Here Semver is used because it has notion of major, minor versions.
+        tomcat_major_versions = {
+            SemverVersion(i).value.major
+            for i in self.version_api.get("org.apache.tomcat:tomcat").valid_versions
+        }
         for version in tomcat_major_versions:
             page_url = self.base_url.format(version)
             if create_etag(self, page_url, "ETag"):
@@ -102,7 +107,9 @@ def to_advisories(self, apache_tomcat_advisory_html):
                             PackageURL(
                                 type="maven", namespace="apache", name="tomcat", version=version
                             )
-                            for version in self.version_api.get("org.apache.tomcat:tomcat")
+                            for version in self.version_api.get(
+                                "org.apache.tomcat:tomcat"
+                            ).valid_versions
                             if MavenVersion(version) in version_range
                         ]
                     )

vulnerabilities/importers/debian_oval.py

@@ -23,14 +23,8 @@
 
 import asyncio
 import dataclasses
-from typing import Iterable
-from typing import List
-from typing import Mapping
-from typing import Set
 import xml.etree.ElementTree as ET
 
-from aiohttp import ClientSession
-from aiohttp.client_exceptions import ClientResponseError, ServerDisconnectedError
 import requests
 
 from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration

vulnerabilities/importers/elixir_security.py

@@ -83,7 +83,7 @@ def get_versions_for_pkg_from_range_list(self, version_range_list, pkg_name):
 
         safe_pkg_versions = []
         vuln_pkg_versions = []
-        all_version_list = self.pkg_manager_api.get(pkg_name)
+        all_version_list = self.pkg_manager_api.get(pkg_name).valid_versions
         if not version_range_list:
             return [], all_version_list
         version_ranges = [

vulnerabilities/importers/github.py

@@ -23,7 +23,7 @@
 import asyncio
 import os
 import dataclasses
-import json
+from dateutil import parser as dateparser
 from typing import Set
 from typing import Tuple
 from typing import List
@@ -67,6 +67,7 @@
                     url
                 }
                 severity
+                publishedAt
                 }
                 package {
                 name
@@ -185,7 +186,6 @@ def fetch(self) -> Mapping[str, List[Mapping]]:
             end_cursor_exp = ""
 
             while True:
-
                 query_json = {"query": query % (ecosystem, end_cursor_exp)}
                 resp = requests.post(self.config.endpoint, headers=headers, json=query_json).json()
                 if resp.get("message") == "Bad credentials":
@@ -260,12 +260,16 @@ def process_response(self) -> List[Advisory]:
             for resp_page in self.advisories[ecosystem]:
                 for adv in resp_page["data"]["securityVulnerabilities"]["edges"]:
                     name = adv["node"]["package"]["name"]
-
+                    cutoff_time = dateparser.parse(adv["node"]["advisory"]["publishedAt"])
+                    affected_purls = []
+                    unaffected_purls = []
                     if self.process_name(ecosystem, name):
                         ns, pkg_name = self.process_name(ecosystem, name)
                         aff_range = adv["node"]["vulnerableVersionRange"]
                         aff_vers, unaff_vers = self.categorize_versions(
-                            self.version_api.package_type, aff_range, self.version_api.get(name)
+                            self.version_api.package_type,
+                            aff_range,
+                            self.version_api.get(name, until=cutoff_time).valid_versions,
                         )
                         affected_purls = [
                             PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
@@ -276,10 +280,6 @@ def process_response(self) -> List[Advisory]:
                             PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
                             for version in unaff_vers
                         ]
-                    else:
-                        affected_purls = []
-                        unaffected_purls = []
-
                     cve_ids = set()
                     references = self.extract_references(adv["node"]["advisory"]["references"])
                     vuln_desc = adv["node"]["advisory"]["summary"]

vulnerabilities/importers/istio.py

@@ -20,8 +20,9 @@
 #  VulnerableCode is a free software tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 import asyncio
+import pytz
 import re
-from typing import List
+from dateutil import parser
 from typing import Set
 
 import saneyaml
@@ -62,11 +63,11 @@ def updated_advisories(self) -> Set[Advisory]:
                 advisories.extend(processed_data)
         return self.batch_advisories(advisories)
 
-    def get_pkg_versions_from_ranges(self, version_range_list):
+    def get_pkg_versions_from_ranges(self, version_range_list, release_date):
         """Takes a list of version ranges(affected) of a package
         as parameter and returns a tuple of safe package versions and
         vulnerable package versions"""
-        all_version = self.version_api.get("istio/istio")
+        all_version = self.version_api.get("istio/istio", release_date).valid_versions
         safe_pkg_versions = []
         vuln_pkg_versions = []
         version_ranges = [
@@ -86,6 +87,7 @@ def process_file(self, path):
         advisories = []
 
         data = self.get_data_from_md(path)
+        release_date = parser.parse(data["publishdate"]).replace(tzinfo=pytz.UTC)
 
         releases = []
         if data.get("releases"):
@@ -128,7 +130,7 @@ def process_file(self, path):
                 data["release_ranges"] = []
 
             safe_pkg_versions, vuln_pkg_versions = self.get_pkg_versions_from_ranges(
-                data["release_ranges"]
+                data["release_ranges"], release_date
             )
 
             affected_packages = []

vulnerabilities/importers/nginx.py

@@ -53,8 +53,10 @@ def set_api(self):
 
         # For some reason nginx tags it's releases are in the form of `release-1.2.3`
         # Chop off the `release-` part here.
-        for index, version in enumerate(self.version_api.cache["nginx/nginx"]):
-            self.version_api.cache["nginx/nginx"][index] = version.replace("release-", "")
+        for index, version in enumerate(self.version_api.cache["nginx/nginx"].valid_versions):
+            self.version_api.cache["nginx/nginx"].valid_versions[index] = version.replace(
+                "release-", ""
+            )
 
     def updated_advisories(self):
         advisories = []
@@ -134,7 +136,9 @@ def extract_fixed_pkgs(self, vuln_info):
                 VersionSpecifier.from_scheme_version_spec_string("semver", "^" + rng[:-1])
             )
 
-        valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
+        valid_versions = find_valid_versions(
+            self.version_api.get("nginx/nginx").valid_versions, version_ranges
+        )
 
         return [
             PackageURL(type="generic", name="nginx", version=version) for version in valid_versions
@@ -169,7 +173,9 @@ def extract_vuln_pkgs(self, vuln_info):
                 )
             )
 
-        valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
+        valid_versions = find_valid_versions(
+            self.version_api.get("nginx/nginx").valid_versions, version_ranges
+        )
         qualifiers = {}
         if windows_only:
             qualifiers["os"] = "windows"

vulnerabilities/importers/npm.py

@@ -22,14 +22,11 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 import asyncio
-from typing import Any
+import pytz
 from typing import List
-from typing import Mapping
 from typing import Set
 from typing import Tuple
-from urllib.error import HTTPError
 from urllib.parse import quote
-from urllib.request import urlopen
 
 from dateutil.parser import parse
 from univers.version_specifier import VersionSpecifier
@@ -87,7 +84,11 @@ def process_file(self, file) -> List[Advisory]:
         record = load_json(file)
         advisories = []
         package_name = record["module_name"].strip()
-        all_versions = self.versions.get(package_name)
+
+        publish_date = parse(record["updated_at"])
+        publish_date.replace(tzinfo=pytz.UTC)
+
+        all_versions = self.versions.get(package_name, until=publish_date).valid_versions
         aff_range = record.get("vulnerable_versions")
         if not aff_range:
             aff_range = ""

vulnerabilities/importers/project_kb_msr2019.py

@@ -22,12 +22,10 @@
 
 import csv
 import dataclasses
-import re
 import urllib.request
 
 # Reading CSV file from  a url using `requests` is bit too complicated.
 # Use `urllib.request` for that purpose.
-from packageurl import PackageURL
 
 
 from vulnerabilities.data_source import Advisory

vulnerabilities/importers/ruby.py

@@ -23,6 +23,8 @@
 import asyncio
 from typing import Set
 from typing import List
+from dateutil.parser import parse
+from pytz import UTC
 
 from packageurl import PackageURL
 from univers.version_specifier import VersionSpecifier
@@ -90,6 +92,7 @@ def process_file(self, path) -> List[Advisory]:
         else:
             return
 
+        publish_time = parse(record["date"]).replace(tzinfo=UTC)
         safe_version_ranges = record.get("patched_versions", [])
         # this case happens when the advisory contain only 'patched_versions' field
         # and it has value None(i.e it is empty :( ).
@@ -100,7 +103,7 @@ def process_file(self, path) -> List[Advisory]:
 
         if not getattr(self, "pkg_manager_api", None):
             self.pkg_manager_api = RubyVersionAPI()
-        all_vers = self.pkg_manager_api.get(package_name)
+        all_vers = self.pkg_manager_api.get(package_name, until=publish_time).valid_versions
         safe_versions, affected_versions = self.categorize_versions(all_vers, safe_version_ranges)
 
         impacted_purls = [

vulnerabilities/importers/rust.py

@@ -26,8 +26,10 @@
 from typing import List
 from typing import Set
 from typing import Tuple
+from dateutil.parser import parse
 
 import toml
+import pytz
 from univers.version_specifier import VersionSpecifier
 from univers.versions import SemverVersion
 from packageurl import PackageURL
@@ -98,7 +100,8 @@ def _load_advisory(self, path: str) -> Optional[Advisory]:
         if advisory.get("url"):
             references.append(Reference(url=advisory["url"]))
 
-        all_versions = self.crates_api.get(crate_name)
+        publish_date = parse(advisory["date"]).replace(tzinfo=pytz.UTC)
+        all_versions = self.crates_api.get(crate_name, publish_date).valid_versions
 
         # FIXME: Avoid wildcard version ranges for now.
         # See https://github.com/RustSec/advisory-db/discussions/831

vulnerabilities/importers/safety_db.py

@@ -86,7 +86,7 @@ def updated_advisories(self) -> Set[Advisory]:
                 # Skip it. The 'cumin' entry is wrong
                 continue
 
-            all_package_versions = self.versions.get(package_name)
+            all_package_versions = self.versions.get(package_name).valid_versions
             if not len(all_package_versions):
                 # PyPi does not have data about this package, we skip these
                 continue

vulnerabilities/importers/ubuntu.py

@@ -25,19 +25,11 @@
 import bz2
 import dataclasses
 import logging
-from typing import Iterable
-from typing import List
-from typing import Mapping
-from typing import Set
 import xml.etree.ElementTree as ET
-
-from aiohttp import ClientSession
-from aiohttp.client_exceptions import ClientResponseError
 import requests
 
 from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration
 from vulnerabilities.package_managers import LaunchpadVersionAPI
-from vulnerabilities.helpers import create_etag
 
 logger = logging.getLogger(__name__)