Merge pull request #467 from nexB/time_travel_advisory

Time travel to the date of advisory publish time when importing

Author: sbs2001

pytest.ini

@@ -1,2 +1,4 @@
 [pytest]
-DJANGO_SETTINGS_MODULE = vulnerablecode.settings
+DJANGO_SETTINGS_MODULE = vulnerablecode.settings
+markers =
+    webtest

vulnerabilities/importers/apache_httpd.py

@@ -37,7 +37,6 @@
 from vulnerabilities.data_source import VulnerabilitySeverity
 from vulnerabilities.package_managers import GitHubTagsAPI
 from vulnerabilities.severity_systems import scoring_systems
-from vulnerabilities.helpers import create_etag
 from vulnerabilities.helpers import nearest_patched_package
 
 
@@ -106,7 +105,7 @@ def to_advisory(self, data):
             fixed_packages.extend(
                 [
                     PackageURL(type="apache", name="httpd", version=version)
-                    for version in self.version_api.get("apache/httpd")
+                    for version in self.version_api.get("apache/httpd").valid_versions
                     if MavenVersion(version) in version_range
                 ]
             )
@@ -115,7 +114,7 @@ def to_advisory(self, data):
             affected_packages.extend(
                 [
                     PackageURL(type="apache", name="httpd", version=version)
-                    for version in self.version_api.get("apache/httpd")
+                    for version in self.version_api.get("apache/httpd").valid_versions
                     if MavenVersion(version) in version_range
                 ]
             )

vulnerabilities/importers/apache_kafka.py

@@ -72,7 +72,7 @@ def to_advisory(self, advisory_page):
 
             fixed_packages = [
                 PackageURL(type="apache", name="kafka", version=version)
-                for version in self.version_api.get("apache/kafka")
+                for version in self.version_api.get("apache/kafka").valid_versions
                 if any(
                     [
                         MavenVersion(version) in version_range
@@ -83,7 +83,7 @@ def to_advisory(self, advisory_page):
 
             affected_packages = [
                 PackageURL(type="apache", name="kafka", version=version)
-                for version in self.version_api.get("apache/kafka")
+                for version in self.version_api.get("apache/kafka").valid_versions
                 if any(
                     [
                         MavenVersion(version) in version_range

vulnerabilities/importers/apache_tomcat.py

@@ -29,6 +29,7 @@
 from bs4 import BeautifulSoup
 from univers.version_specifier import VersionSpecifier
 from univers.versions import MavenVersion
+from univers.versions import SemverVersion
 from packageurl import PackageURL
 
 from vulnerabilities.data_source import Advisory
@@ -62,7 +63,11 @@ def updated_advisories(self):
         return self.batch_advisories(advisories)
 
     def fetch_pages(self):
-        tomcat_major_versions = {i[0] for i in self.version_api.get("org.apache.tomcat:tomcat")}
+        # Here Semver is used because it has notion of major, minor versions.
+        tomcat_major_versions = {
+            SemverVersion(i).value.major
+            for i in self.version_api.get("org.apache.tomcat:tomcat").valid_versions
+        }
         for version in tomcat_major_versions:
             page_url = self.base_url.format(version)
             if create_etag(self, page_url, "ETag"):
@@ -102,7 +107,9 @@ def to_advisories(self, apache_tomcat_advisory_html):
                             PackageURL(
                                 type="maven", namespace="apache", name="tomcat", version=version
                             )
-                            for version in self.version_api.get("org.apache.tomcat:tomcat")
+                            for version in self.version_api.get(
+                                "org.apache.tomcat:tomcat"
+                            ).valid_versions
                             if MavenVersion(version) in version_range
                         ]
                     )

vulnerabilities/importers/debian_oval.py

@@ -23,14 +23,8 @@
 
 import asyncio
 import dataclasses
-from typing import Iterable
-from typing import List
-from typing import Mapping
-from typing import Set
 import xml.etree.ElementTree as ET
 
-from aiohttp import ClientSession
-from aiohttp.client_exceptions import ClientResponseError, ServerDisconnectedError
 import requests
 
 from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration

vulnerabilities/importers/elixir_security.py

@@ -83,7 +83,7 @@ def get_versions_for_pkg_from_range_list(self, version_range_list, pkg_name):
 
         safe_pkg_versions = []
         vuln_pkg_versions = []
-        all_version_list = self.pkg_manager_api.get(pkg_name)
+        all_version_list = self.pkg_manager_api.get(pkg_name).valid_versions
         if not version_range_list:
             return [], all_version_list
         version_ranges = [

vulnerabilities/importers/github.py

@@ -23,7 +23,7 @@
 import asyncio
 import os
 import dataclasses
-import json
+from dateutil import parser as dateparser
 from typing import Set
 from typing import Tuple
 from typing import List
@@ -67,6 +67,7 @@
                     url
                 }
                 severity
+                publishedAt
                 }
                 package {
                 name
@@ -185,7 +186,6 @@ def fetch(self) -> Mapping[str, List[Mapping]]:
             end_cursor_exp = ""
 
             while True:
-
                 query_json = {"query": query % (ecosystem, end_cursor_exp)}
                 resp = requests.post(self.config.endpoint, headers=headers, json=query_json).json()
                 if resp.get("message") == "Bad credentials":
@@ -260,12 +260,16 @@ def process_response(self) -> List[Advisory]:
             for resp_page in self.advisories[ecosystem]:
                 for adv in resp_page["data"]["securityVulnerabilities"]["edges"]:
                     name = adv["node"]["package"]["name"]
-
+                    cutoff_time = dateparser.parse(adv["node"]["advisory"]["publishedAt"])
+                    affected_purls = []
+                    unaffected_purls = []
                     if self.process_name(ecosystem, name):
                         ns, pkg_name = self.process_name(ecosystem, name)
                         aff_range = adv["node"]["vulnerableVersionRange"]
                         aff_vers, unaff_vers = self.categorize_versions(
-                            self.version_api.package_type, aff_range, self.version_api.get(name)
+                            self.version_api.package_type,
+                            aff_range,
+                            self.version_api.get(name, until=cutoff_time).valid_versions,
                         )
                         affected_purls = [
                             PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
@@ -276,10 +280,6 @@ def process_response(self) -> List[Advisory]:
                             PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
                             for version in unaff_vers
                         ]
-                    else:
-                        affected_purls = []
-                        unaffected_purls = []
-
                     cve_ids = set()
                     references = self.extract_references(adv["node"]["advisory"]["references"])
                     vuln_desc = adv["node"]["advisory"]["summary"]

vulnerabilities/importers/istio.py

@@ -20,8 +20,9 @@
 #  VulnerableCode is a free software tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 import asyncio
+import pytz
 import re
-from typing import List
+from dateutil import parser
 from typing import Set
 
 import saneyaml
@@ -62,11 +63,11 @@ def updated_advisories(self) -> Set[Advisory]:
                 advisories.extend(processed_data)
         return self.batch_advisories(advisories)
 
-    def get_pkg_versions_from_ranges(self, version_range_list):
+    def get_pkg_versions_from_ranges(self, version_range_list, release_date):
         """Takes a list of version ranges(affected) of a package
         as parameter and returns a tuple of safe package versions and
         vulnerable package versions"""
-        all_version = self.version_api.get("istio/istio")
+        all_version = self.version_api.get("istio/istio", release_date).valid_versions
         safe_pkg_versions = []
         vuln_pkg_versions = []
         version_ranges = [
@@ -86,6 +87,7 @@ def process_file(self, path):
         advisories = []
 
         data = self.get_data_from_md(path)
+        release_date = parser.parse(data["publishdate"]).replace(tzinfo=pytz.UTC)
 
         releases = []
         if data.get("releases"):
@@ -128,7 +130,7 @@ def process_file(self, path):
                 data["release_ranges"] = []
 
             safe_pkg_versions, vuln_pkg_versions = self.get_pkg_versions_from_ranges(
-                data["release_ranges"]
+                data["release_ranges"], release_date
             )
 
             affected_packages = []

vulnerabilities/importers/nginx.py

@@ -53,8 +53,10 @@ def set_api(self):
 
         # For some reason nginx tags it's releases are in the form of `release-1.2.3`
         # Chop off the `release-` part here.
-        for index, version in enumerate(self.version_api.cache["nginx/nginx"]):
-            self.version_api.cache["nginx/nginx"][index] = version.replace("release-", "")
+        for index, version in enumerate(self.version_api.cache["nginx/nginx"].valid_versions):
+            self.version_api.cache["nginx/nginx"].valid_versions[index] = version.replace(
+                "release-", ""
+            )
 
     def updated_advisories(self):
         advisories = []
@@ -134,7 +136,9 @@ def extract_fixed_pkgs(self, vuln_info):
                 VersionSpecifier.from_scheme_version_spec_string("semver", "^" + rng[:-1])
             )
 
-        valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
+        valid_versions = find_valid_versions(
+            self.version_api.get("nginx/nginx").valid_versions, version_ranges
+        )
 
         return [
             PackageURL(type="generic", name="nginx", version=version) for version in valid_versions
@@ -169,7 +173,9 @@ def extract_vuln_pkgs(self, vuln_info):
                 )
             )
 
-        valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
+        valid_versions = find_valid_versions(
+            self.version_api.get("nginx/nginx").valid_versions, version_ranges
+        )
         qualifiers = {}
         if windows_only:
             qualifiers["os"] = "windows"

vulnerabilities/importers/npm.py

@@ -22,14 +22,11 @@
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
 
 import asyncio
-from typing import Any
+import pytz
 from typing import List
-from typing import Mapping
 from typing import Set
 from typing import Tuple
-from urllib.error import HTTPError
 from urllib.parse import quote
-from urllib.request import urlopen
 
 from dateutil.parser import parse
 from univers.version_specifier import VersionSpecifier
@@ -87,7 +84,11 @@ def process_file(self, file) -> List[Advisory]:
         record = load_json(file)
         advisories = []
         package_name = record["module_name"].strip()
-        all_versions = self.versions.get(package_name)
+
+        publish_date = parse(record["updated_at"])
+        publish_date.replace(tzinfo=pytz.UTC)
+
+        all_versions = self.versions.get(package_name, until=publish_date).valid_versions
         aff_range = record.get("vulnerable_versions")
         if not aff_range:
             aff_range = ""