Add basic implementation for time travel in imports

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

Author: sbs2001

vulnerabilities/importers/debian_oval.py

@@ -23,14 +23,8 @@
 
 import asyncio
 import dataclasses
-from typing import Iterable
-from typing import List
-from typing import Mapping
-from typing import Set
 import xml.etree.ElementTree as ET
 
-from aiohttp import ClientSession
-from aiohttp.client_exceptions import ClientResponseError, ServerDisconnectedError
 import requests
 
 from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration

vulnerabilities/importers/github.py

@@ -24,6 +24,8 @@
 import os
 import dataclasses
 import json
+from xml.etree.ElementTree import parse
+from dateutil import parser
 from typing import Set
 from typing import Tuple
 from typing import List
@@ -34,6 +36,7 @@
 from packageurl import PackageURL
 from univers.version_specifier import VersionSpecifier
 from univers.versions import version_class_by_package_type
+from univers.versions import InvalidVersion
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
@@ -67,6 +70,7 @@
                     url
                 }
                 severity
+                publishedAt
                 }
                 package {
                 name
@@ -122,7 +126,6 @@ def fetch(self) -> Mapping[str, List[Mapping]]:
             end_cursor_exp = ""
 
             while True:
-
                 query_json = {"query": query % (ecosystem, end_cursor_exp)}
                 resp = requests.post(self.config.endpoint, headers=headers, json=query_json).json()
                 if resp.get("message") == "Bad credentials":
@@ -197,12 +200,16 @@ def process_response(self) -> List[Advisory]:
             for resp_page in self.advisories[ecosystem]:
                 for adv in resp_page["data"]["securityVulnerabilities"]["edges"]:
                     name = adv["node"]["package"]["name"]
-
+                    cutoff_time = parser.parse(adv["node"]["advisory"]["publishedAt"])
+                    affected_purls = []
+                    unaffected_purls = []
                     if self.process_name(ecosystem, name):
                         ns, pkg_name = self.process_name(ecosystem, name)
                         aff_range = adv["node"]["vulnerableVersionRange"]
                         aff_vers, unaff_vers = self.categorize_versions(
-                            self.version_api.package_type, aff_range, self.version_api.get(name)
+                            self.version_api.package_type,
+                            aff_range,
+                            self.version_api.get(name, until=cutoff_time)["valid"],
                         )
                         affected_purls = [
                             PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
@@ -213,10 +220,6 @@ def process_response(self) -> List[Advisory]:
                             PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
                             for version in unaff_vers
                         ]
-                    else:
-                        affected_purls = []
-                        unaffected_purls = []
-
                     cve_ids = set()
                     references = self.extract_references(adv["node"]["advisory"]["references"])
                     vuln_desc = adv["node"]["advisory"]["summary"]
@@ -265,9 +268,12 @@ def categorize_versions(
         affected_versions = []
         unaffected_versions = []
         for version in all_versions:
-            if version_class(version) in version_range:
-                affected_versions.append(version)
-            else:
-                unaffected_versions.append(version)
+            try:
+                if version_class(version) in version_range:
+                    affected_versions.append(version)
+                else:
+                    unaffected_versions.append(version)
+            except InvalidVersion:
+                continue
 
         return (affected_versions, unaffected_versions)

vulnerabilities/importers/istio.py

@@ -21,7 +21,6 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 import asyncio
 import re
-from typing import List
 from typing import Set
 
 import saneyaml

vulnerabilities/importers/npm.py

@@ -22,14 +22,11 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 import asyncio
-from typing import Any
+import pytz
 from typing import List
-from typing import Mapping
 from typing import Set
 from typing import Tuple
-from urllib.error import HTTPError
 from urllib.parse import quote
-from urllib.request import urlopen
 
 from dateutil.parser import parse
 from univers.version_specifier import VersionSpecifier
@@ -87,7 +84,11 @@ def process_file(self, file) -> List[Advisory]:
         record = load_json(file)
         advisories = []
         package_name = record["module_name"].strip()
-        all_versions = self.versions.get(package_name)
+
+        publish_date = parse(record["updated_at"])
+        publish_date.replace(tzinfo=pytz.UTC)
+
+        all_versions = self.versions.get(package_name, until=publish_date)
         aff_range = record.get("vulnerable_versions")
         if not aff_range:
             aff_range = ""

vulnerabilities/importers/rust.py

@@ -26,8 +26,10 @@
 from typing import List
 from typing import Set
 from typing import Tuple
+from dateutil.parser import parse
 
 import toml
+import pytz
 from univers.version_specifier import VersionSpecifier
 from univers.versions import SemverVersion
 from packageurl import PackageURL
@@ -98,7 +100,8 @@ def _load_advisory(self, path: str) -> Optional[Advisory]:
         if advisory.get("url"):
             references.append(Reference(url=advisory["url"]))
 
-        all_versions = self.crates_api.get(crate_name)
+        publish_date = parse(advisory["date"]).replace(tzinfo=pytz.UTC)
+        all_versions = self.crates_api.get(crate_name, publish_date)["valid"]
 
         # FIXME: Avoid wildcard version ranges for now.
         # See https://github.com/RustSec/advisory-db/discussions/831

vulnerabilities/package_managers.py

@@ -21,6 +21,8 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 import asyncio
+from collections import namedtuple
+from dateutil import parser
 from json import JSONDecodeError
 from typing import Mapping
 from typing import Set
@@ -32,12 +34,22 @@
 from aiohttp.client_exceptions import ServerDisconnectedError
 
 
+Version = namedtuple("Version", field_names=["value", "release_date"])
+
+
 class VersionAPI:
     def __init__(self, cache: Mapping[str, Set[str]] = None):
         self.cache = cache or {}
 
-    def get(self, package_name: str) -> Set[str]:
-        return self.cache.get(package_name, set())
+    def get(self, package_name, until=None) -> Set[str]:
+        versions = {"new": set(), "valid": set()}
+        for version in self.cache.get(package_name, set()):
+            if until and version.release_date and version.release_date > until:
+                versions["new"].add(version.value)
+                continue
+            versions["valid"].add(version.value)
+
+        return versions
 
 
 def client_session():
@@ -97,7 +109,16 @@ async def fetch(self, pkg, session):
         try:
             response = await session.request(method="GET", url=url)
             response = await response.json()
-            versions = set(response["releases"])
+            for version in response["releases"]:
+                if response["releases"][version]:
+                    versions.add(
+                        Version(
+                            value=version,
+                            release_date=parser.parse(
+                                response["releases"][version][-1]["upload_time_iso_8601"]
+                            ),
+                        )
+                    )
         except ClientResponseError:
             # PYPI removed this package.
             # https://www.zdnet.com/article/twelve-malicious-python-libraries-found-and-removed-from-pypi/  # nopep8
@@ -121,7 +142,11 @@ async def fetch(self, pkg, session):
         response = await response.json()
         versions = set()
         for version_info in response["versions"]:
-            versions.add(version_info["num"])
+            versions.add(
+                Version(
+                    value=version_info["num"], release_date=parser.parse(version_info["updated_at"])
+                )
+            )
 
         self.cache[pkg] = versions
 
@@ -143,7 +168,11 @@ async def fetch(self, pkg, session):
             response = await session.request(method="GET", url=url)
             response = await response.json()
             for release in response:
-                versions.add(release["number"])
+                versions.add(
+                    Version(
+                        value=release["number"], release_date=parser.parse(release["created_at"])
+                    )
+                )
         except (ClientResponseError, JSONDecodeError):
             pass
 
@@ -166,7 +195,13 @@ async def fetch(self, pkg, session):
         try:
             response = await session.request(method="GET", url=url)
             response = await response.json()
-            versions = {v for v in response.get("versions", [])}
+            for version in response.get("versions", []):
+                release_date = response.get("time", {}).get(version)
+                if release_date:
+                    release_date = parser.parse(release_date)
+                    versions.add(Version(value=version, release_date=release_date))
+                else:
+                    versions.add(Version(value=version, release_date=None))
 
         except ClientResponseError:
             pass
@@ -294,7 +329,12 @@ def extract_versions(resp: dict) -> Set[str]:
         try:
             for entry_group in resp["items"]:
                 for entry in entry_group["items"]:
-                    all_versions.add(entry["catalogEntry"]["version"])
+                    all_versions.add(
+                        Version(
+                            value=entry["catalogEntry"]["version"],
+                            release_date=parser.parse(entry["catalogEntry"]["published"]),
+                        )
+                    )
         # FIXME: json response for YamlDotNet.Signed triggers this exception.
         # Some packages with many versions give a response of a list of endpoints.
         # In such cases rather, we should collect data from those endpoints.
@@ -327,15 +367,21 @@ def composer_url(pkg_name: str) -> str:
             vendor, name = pkg_name.split("/")
         except ValueError:
             # TODO Log this
-            return None
+            return
         return f"https://repo.packagist.org/p/{vendor}/{name}.json"
 
     @staticmethod
     def extract_versions(resp: dict, pkg_name: str) -> Set[str]:
-        all_versions = resp["packages"][pkg_name].keys()
-        all_versions = {
-            version.replace("v", "") for version in all_versions if "dev" not in version
-        }
+        all_versions = set()
+        for version in resp["packages"][pkg_name]:
+            if "dev" in version:
+                continue
+            all_versions.add(
+                Version(
+                    value=version.replace("v", ""),
+                    release_date=parser.parse(resp["packages"][pkg_name][version]["time"]),
+                )
+            )
         # This if statement ensures, that all_versions contains only released versions
         # See https://github.com/composer/composer/blob/44a4429978d1b3c6223277b875762b2930e83e8c/doc/articles/versions.md#tags  # nopep8
         # for explanation of removing 'v'