diff --git a/pytest.ini b/pytest.ini
index 8a0f512df..d64e30334 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,2 +1,4 @@
[pytest]
-DJANGO_SETTINGS_MODULE = vulnerablecode.settings
\ No newline at end of file
+DJANGO_SETTINGS_MODULE = vulnerablecode.settings
+markers =
+ webtest
\ No newline at end of file
diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py
index 0b6e87bea..ffb5288bf 100644
--- a/vulnerabilities/importers/apache_httpd.py
+++ b/vulnerabilities/importers/apache_httpd.py
@@ -37,7 +37,6 @@
from vulnerabilities.data_source import VulnerabilitySeverity
from vulnerabilities.package_managers import GitHubTagsAPI
from vulnerabilities.severity_systems import scoring_systems
-from vulnerabilities.helpers import create_etag
from vulnerabilities.helpers import nearest_patched_package
@@ -106,7 +105,7 @@ def to_advisory(self, data):
fixed_packages.extend(
[
PackageURL(type="apache", name="httpd", version=version)
- for version in self.version_api.get("apache/httpd")
+ for version in self.version_api.get("apache/httpd").valid_versions
if MavenVersion(version) in version_range
]
)
@@ -115,7 +114,7 @@ def to_advisory(self, data):
affected_packages.extend(
[
PackageURL(type="apache", name="httpd", version=version)
- for version in self.version_api.get("apache/httpd")
+ for version in self.version_api.get("apache/httpd").valid_versions
if MavenVersion(version) in version_range
]
)
diff --git a/vulnerabilities/importers/apache_kafka.py b/vulnerabilities/importers/apache_kafka.py
index 105f327f0..3f30d340a 100644
--- a/vulnerabilities/importers/apache_kafka.py
+++ b/vulnerabilities/importers/apache_kafka.py
@@ -72,7 +72,7 @@ def to_advisory(self, advisory_page):
fixed_packages = [
PackageURL(type="apache", name="kafka", version=version)
- for version in self.version_api.get("apache/kafka")
+ for version in self.version_api.get("apache/kafka").valid_versions
if any(
[
MavenVersion(version) in version_range
@@ -83,7 +83,7 @@ def to_advisory(self, advisory_page):
affected_packages = [
PackageURL(type="apache", name="kafka", version=version)
- for version in self.version_api.get("apache/kafka")
+ for version in self.version_api.get("apache/kafka").valid_versions
if any(
[
MavenVersion(version) in version_range
diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py
index 42a728840..afcc85fb8 100644
--- a/vulnerabilities/importers/apache_tomcat.py
+++ b/vulnerabilities/importers/apache_tomcat.py
@@ -29,6 +29,7 @@
from bs4 import BeautifulSoup
from univers.version_specifier import VersionSpecifier
from univers.versions import MavenVersion
+from univers.versions import SemverVersion
from packageurl import PackageURL
from vulnerabilities.data_source import Advisory
@@ -62,7 +63,11 @@ def updated_advisories(self):
return self.batch_advisories(advisories)
def fetch_pages(self):
- tomcat_major_versions = {i[0] for i in self.version_api.get("org.apache.tomcat:tomcat")}
+ # Here Semver is used because it has notion of major, minor versions.
+ tomcat_major_versions = {
+ SemverVersion(i).value.major
+ for i in self.version_api.get("org.apache.tomcat:tomcat").valid_versions
+ }
for version in tomcat_major_versions:
page_url = self.base_url.format(version)
if create_etag(self, page_url, "ETag"):
@@ -102,7 +107,9 @@ def to_advisories(self, apache_tomcat_advisory_html):
PackageURL(
type="maven", namespace="apache", name="tomcat", version=version
)
- for version in self.version_api.get("org.apache.tomcat:tomcat")
+ for version in self.version_api.get(
+ "org.apache.tomcat:tomcat"
+ ).valid_versions
if MavenVersion(version) in version_range
]
)
diff --git a/vulnerabilities/importers/debian_oval.py b/vulnerabilities/importers/debian_oval.py
index e73a1da26..50f9eb9ce 100644
--- a/vulnerabilities/importers/debian_oval.py
+++ b/vulnerabilities/importers/debian_oval.py
@@ -23,14 +23,8 @@
import asyncio
import dataclasses
-from typing import Iterable
-from typing import List
-from typing import Mapping
-from typing import Set
import xml.etree.ElementTree as ET
-from aiohttp import ClientSession
-from aiohttp.client_exceptions import ClientResponseError, ServerDisconnectedError
import requests
from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration
diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py
index fcb3e6c42..46ef933a8 100644
--- a/vulnerabilities/importers/elixir_security.py
+++ b/vulnerabilities/importers/elixir_security.py
@@ -83,7 +83,7 @@ def get_versions_for_pkg_from_range_list(self, version_range_list, pkg_name):
safe_pkg_versions = []
vuln_pkg_versions = []
- all_version_list = self.pkg_manager_api.get(pkg_name)
+ all_version_list = self.pkg_manager_api.get(pkg_name).valid_versions
if not version_range_list:
return [], all_version_list
version_ranges = [
diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py
index 6da7a5769..b1cee1fda 100644
--- a/vulnerabilities/importers/github.py
+++ b/vulnerabilities/importers/github.py
@@ -23,7 +23,7 @@
import asyncio
import os
import dataclasses
-import json
+from dateutil import parser as dateparser
from typing import Set
from typing import Tuple
from typing import List
@@ -67,6 +67,7 @@
url
}
severity
+ publishedAt
}
package {
name
@@ -185,7 +186,6 @@ def fetch(self) -> Mapping[str, List[Mapping]]:
end_cursor_exp = ""
while True:
-
query_json = {"query": query % (ecosystem, end_cursor_exp)}
resp = requests.post(self.config.endpoint, headers=headers, json=query_json).json()
if resp.get("message") == "Bad credentials":
@@ -260,12 +260,16 @@ def process_response(self) -> List[Advisory]:
for resp_page in self.advisories[ecosystem]:
for adv in resp_page["data"]["securityVulnerabilities"]["edges"]:
name = adv["node"]["package"]["name"]
-
+ cutoff_time = dateparser.parse(adv["node"]["advisory"]["publishedAt"])
+ affected_purls = []
+ unaffected_purls = []
if self.process_name(ecosystem, name):
ns, pkg_name = self.process_name(ecosystem, name)
aff_range = adv["node"]["vulnerableVersionRange"]
aff_vers, unaff_vers = self.categorize_versions(
- self.version_api.package_type, aff_range, self.version_api.get(name)
+ self.version_api.package_type,
+ aff_range,
+ self.version_api.get(name, until=cutoff_time).valid_versions,
)
affected_purls = [
PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
@@ -276,10 +280,6 @@ def process_response(self) -> List[Advisory]:
PackageURL(name=pkg_name, namespace=ns, version=version, type=pkg_type)
for version in unaff_vers
]
- else:
- affected_purls = []
- unaffected_purls = []
-
cve_ids = set()
references = self.extract_references(adv["node"]["advisory"]["references"])
vuln_desc = adv["node"]["advisory"]["summary"]
diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py
index 103044bc2..ca2717040 100644
--- a/vulnerabilities/importers/istio.py
+++ b/vulnerabilities/importers/istio.py
@@ -20,8 +20,9 @@
# VulnerableCode is a free software tool from nexB Inc. and others.
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
import asyncio
+import pytz
import re
-from typing import List
+from dateutil import parser
from typing import Set
import saneyaml
@@ -62,11 +63,11 @@ def updated_advisories(self) -> Set[Advisory]:
advisories.extend(processed_data)
return self.batch_advisories(advisories)
- def get_pkg_versions_from_ranges(self, version_range_list):
+ def get_pkg_versions_from_ranges(self, version_range_list, release_date):
"""Takes a list of version ranges(affected) of a package
as parameter and returns a tuple of safe package versions and
vulnerable package versions"""
- all_version = self.version_api.get("istio/istio")
+ all_version = self.version_api.get("istio/istio", release_date).valid_versions
safe_pkg_versions = []
vuln_pkg_versions = []
version_ranges = [
@@ -86,6 +87,7 @@ def process_file(self, path):
advisories = []
data = self.get_data_from_md(path)
+ release_date = parser.parse(data["publishdate"]).replace(tzinfo=pytz.UTC)
releases = []
if data.get("releases"):
@@ -128,7 +130,7 @@ def process_file(self, path):
data["release_ranges"] = []
safe_pkg_versions, vuln_pkg_versions = self.get_pkg_versions_from_ranges(
- data["release_ranges"]
+ data["release_ranges"], release_date
)
affected_packages = []
diff --git a/vulnerabilities/importers/nginx.py b/vulnerabilities/importers/nginx.py
index e5be6247c..77913398a 100644
--- a/vulnerabilities/importers/nginx.py
+++ b/vulnerabilities/importers/nginx.py
@@ -53,8 +53,10 @@ def set_api(self):
# For some reason nginx tags it's releases are in the form of `release-1.2.3`
# Chop off the `release-` part here.
- for index, version in enumerate(self.version_api.cache["nginx/nginx"]):
- self.version_api.cache["nginx/nginx"][index] = version.replace("release-", "")
+ for index, version in enumerate(self.version_api.cache["nginx/nginx"].valid_versions):
+ self.version_api.cache["nginx/nginx"].valid_versions[index] = version.replace(
+ "release-", ""
+ )
def updated_advisories(self):
advisories = []
@@ -134,7 +136,9 @@ def extract_fixed_pkgs(self, vuln_info):
VersionSpecifier.from_scheme_version_spec_string("semver", "^" + rng[:-1])
)
- valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
+ valid_versions = find_valid_versions(
+ self.version_api.get("nginx/nginx").valid_versions, version_ranges
+ )
return [
PackageURL(type="generic", name="nginx", version=version) for version in valid_versions
@@ -169,7 +173,9 @@ def extract_vuln_pkgs(self, vuln_info):
)
)
- valid_versions = find_valid_versions(self.version_api.get("nginx/nginx"), version_ranges)
+ valid_versions = find_valid_versions(
+ self.version_api.get("nginx/nginx").valid_versions, version_ranges
+ )
qualifiers = {}
if windows_only:
qualifiers["os"] = "windows"
diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py
index 76c6fbc28..e8bdf84ba 100644
--- a/vulnerabilities/importers/npm.py
+++ b/vulnerabilities/importers/npm.py
@@ -22,14 +22,11 @@
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
import asyncio
-from typing import Any
+import pytz
from typing import List
-from typing import Mapping
from typing import Set
from typing import Tuple
-from urllib.error import HTTPError
from urllib.parse import quote
-from urllib.request import urlopen
from dateutil.parser import parse
from univers.version_specifier import VersionSpecifier
@@ -87,7 +84,11 @@ def process_file(self, file) -> List[Advisory]:
record = load_json(file)
advisories = []
package_name = record["module_name"].strip()
- all_versions = self.versions.get(package_name)
+
+ publish_date = parse(record["updated_at"])
+ publish_date.replace(tzinfo=pytz.UTC)
+
+ all_versions = self.versions.get(package_name, until=publish_date).valid_versions
aff_range = record.get("vulnerable_versions")
if not aff_range:
aff_range = ""
diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py
index 8b97eccf1..897a0c4ff 100644
--- a/vulnerabilities/importers/project_kb_msr2019.py
+++ b/vulnerabilities/importers/project_kb_msr2019.py
@@ -22,12 +22,10 @@
import csv
import dataclasses
-import re
import urllib.request
# Reading CSV file from a url using `requests` is bit too complicated.
# Use `urllib.request` for that purpose.
-from packageurl import PackageURL
from vulnerabilities.data_source import Advisory
diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py
index a801a5f81..ad1e7dc59 100644
--- a/vulnerabilities/importers/ruby.py
+++ b/vulnerabilities/importers/ruby.py
@@ -23,6 +23,8 @@
import asyncio
from typing import Set
from typing import List
+from dateutil.parser import parse
+from pytz import UTC
from packageurl import PackageURL
from univers.version_specifier import VersionSpecifier
@@ -90,6 +92,7 @@ def process_file(self, path) -> List[Advisory]:
else:
return
+ publish_time = parse(record["date"]).replace(tzinfo=UTC)
safe_version_ranges = record.get("patched_versions", [])
# this case happens when the advisory contain only 'patched_versions' field
# and it has value None(i.e it is empty :( ).
@@ -100,7 +103,7 @@ def process_file(self, path) -> List[Advisory]:
if not getattr(self, "pkg_manager_api", None):
self.pkg_manager_api = RubyVersionAPI()
- all_vers = self.pkg_manager_api.get(package_name)
+ all_vers = self.pkg_manager_api.get(package_name, until=publish_time).valid_versions
safe_versions, affected_versions = self.categorize_versions(all_vers, safe_version_ranges)
impacted_purls = [
diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py
index 557ff7d06..32d5f07fb 100644
--- a/vulnerabilities/importers/rust.py
+++ b/vulnerabilities/importers/rust.py
@@ -26,8 +26,10 @@
from typing import List
from typing import Set
from typing import Tuple
+from dateutil.parser import parse
import toml
+import pytz
from univers.version_specifier import VersionSpecifier
from univers.versions import SemverVersion
from packageurl import PackageURL
@@ -98,7 +100,8 @@ def _load_advisory(self, path: str) -> Optional[Advisory]:
if advisory.get("url"):
references.append(Reference(url=advisory["url"]))
- all_versions = self.crates_api.get(crate_name)
+ publish_date = parse(advisory["date"]).replace(tzinfo=pytz.UTC)
+ all_versions = self.crates_api.get(crate_name, publish_date).valid_versions
# FIXME: Avoid wildcard version ranges for now.
# See https://github.com/RustSec/advisory-db/discussions/831
diff --git a/vulnerabilities/importers/safety_db.py b/vulnerabilities/importers/safety_db.py
index 9e0305439..e4f5d3cd2 100755
--- a/vulnerabilities/importers/safety_db.py
+++ b/vulnerabilities/importers/safety_db.py
@@ -86,7 +86,7 @@ def updated_advisories(self) -> Set[Advisory]:
# Skip it. The 'cumin' entry is wrong
continue
- all_package_versions = self.versions.get(package_name)
+ all_package_versions = self.versions.get(package_name).valid_versions
if not len(all_package_versions):
# PyPi does not have data about this package, we skip these
continue
diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py
index dadc4e15d..4124317c0 100644
--- a/vulnerabilities/importers/ubuntu.py
+++ b/vulnerabilities/importers/ubuntu.py
@@ -25,19 +25,11 @@
import bz2
import dataclasses
import logging
-from typing import Iterable
-from typing import List
-from typing import Mapping
-from typing import Set
import xml.etree.ElementTree as ET
-
-from aiohttp import ClientSession
-from aiohttp.client_exceptions import ClientResponseError
import requests
from vulnerabilities.data_source import OvalDataSource, DataSourceConfiguration
from vulnerabilities.package_managers import LaunchpadVersionAPI
-from vulnerabilities.helpers import create_etag
logger = logging.getLogger(__name__)
diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 501949201..399ac3c4a 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -21,23 +21,49 @@
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
import asyncio
+import dataclasses
+import pytz
+import xml.etree.ElementTree as ET
+from bs4 import BeautifulSoup
+from dateutil import parser as dateparser
from json import JSONDecodeError
from typing import Mapping
from typing import Set
from typing import List
-import xml.etree.ElementTree as ET
+from datetime import datetime
from aiohttp import ClientSession
from aiohttp.client_exceptions import ClientResponseError
from aiohttp.client_exceptions import ServerDisconnectedError
+@dataclasses.dataclass(frozen=True)
+class Version:
+ value: str
+ release_date: datetime = None
+
+
+@dataclasses.dataclass(frozen=True)
+class VersionResponse:
+ valid_versions: Set[str] = dataclasses.field(default_factory=set)
+ newer_versions: Set[str] = dataclasses.field(default_factory=set)
+
+
+@dataclasses.dataclass(frozen=True)
class VersionAPI:
def __init__(self, cache: Mapping[str, Set[str]] = None):
self.cache = cache or {}
- def get(self, package_name: str) -> Set[str]:
- return self.cache.get(package_name, set())
+ def get(self, package_name, until=None) -> Set[str]:
+ new_versions = set()
+ valid_versions = set()
+ for version in self.cache.get(package_name, set()):
+ if until and version.release_date and version.release_date > until:
+ new_versions.add(version.value)
+ continue
+ valid_versions.add(version.value)
+
+ return VersionResponse(valid_versions=valid_versions, newer_versions=new_versions)
def client_session():
@@ -97,7 +123,22 @@ async def fetch(self, pkg, session):
try:
response = await session.request(method="GET", url=url)
response = await response.json()
- versions = set(response["releases"])
+ for version, download_items in response["releases"].items():
+ if download_items:
+ latest_download_item = max(
+ download_items,
+ key=lambda download_item: dateparser.parse(
+ download_item["upload_time_iso_8601"]
+ ),
+ )
+ versions.add(
+ Version(
+ value=version,
+ release_date=dateparser.parse(
+ latest_download_item["upload_time_iso_8601"]
+ ),
+ )
+ )
except ClientResponseError:
# PYPI removed this package.
# https://www.zdnet.com/article/twelve-malicious-python-libraries-found-and-removed-from-pypi/ # nopep8
@@ -121,7 +162,12 @@ async def fetch(self, pkg, session):
response = await response.json()
versions = set()
for version_info in response["versions"]:
- versions.add(version_info["num"])
+ versions.add(
+ Version(
+ value=version_info["num"],
+ release_date=dateparser.parse(version_info["updated_at"]),
+ )
+ )
self.cache[pkg] = versions
@@ -143,7 +189,12 @@ async def fetch(self, pkg, session):
response = await session.request(method="GET", url=url)
response = await response.json()
for release in response:
- versions.add(release["number"])
+ versions.add(
+ Version(
+ value=release["number"],
+ release_date=dateparser.parse(release["created_at"]),
+ )
+ )
except (ClientResponseError, JSONDecodeError):
pass
@@ -166,7 +217,13 @@ async def fetch(self, pkg, session):
try:
response = await session.request(method="GET", url=url)
response = await response.json()
- versions = {v for v in response.get("versions", [])}
+ for version in response.get("versions", []):
+ release_date = response.get("time", {}).get(version)
+ if release_date:
+ release_date = dateparser.parse(release_date)
+ versions.add(Version(value=version, release_date=release_date))
+ else:
+ versions.add(Version(value=version, release_date=None))
except ClientResponseError:
pass
@@ -261,7 +318,7 @@ def extract_versions(xml_response: ET.ElementTree) -> Set[str]:
all_versions = set()
for child in xml_response.getroot().iter():
if child.tag == "version":
- all_versions.add(child.text)
+ all_versions.add(Version(child.text))
return all_versions
@@ -294,7 +351,12 @@ def extract_versions(resp: dict) -> Set[str]:
try:
for entry_group in resp["items"]:
for entry in entry_group["items"]:
- all_versions.add(entry["catalogEntry"]["version"])
+ all_versions.add(
+ Version(
+ value=entry["catalogEntry"]["version"],
+ release_date=dateparser.parse(entry["catalogEntry"]["published"]),
+ )
+ )
# FIXME: json response for YamlDotNet.Signed triggers this exception.
# Some packages with many versions give a response of a list of endpoints.
# In such cases rather, we should collect data from those endpoints.
@@ -327,18 +389,25 @@ def composer_url(pkg_name: str) -> str:
vendor, name = pkg_name.split("/")
except ValueError:
# TODO Log this
- return None
+ return
return f"https://repo.packagist.org/p/{vendor}/{name}.json"
@staticmethod
def extract_versions(resp: dict, pkg_name: str) -> Set[str]:
- all_versions = resp["packages"][pkg_name].keys()
- all_versions = {
- version.replace("v", "") for version in all_versions if "dev" not in version
- }
- # This if statement ensures, that all_versions contains only released versions
- # See https://github.com/composer/composer/blob/44a4429978d1b3c6223277b875762b2930e83e8c/doc/articles/versions.md#tags # nopep8
- # for explanation of removing 'v'
+ all_versions = set()
+ for version in resp["packages"][pkg_name]:
+ if "dev" in version:
+ continue
+
+ # This if statement ensures, that all_versions contains only released versions
+ # See https://github.com/composer/composer/blob/44a4429978d1b3c6223277b875762b2930e83e8c/doc/articles/versions.md#tags # nopep8
+ # for explanation of removing 'v'
+ all_versions.add(
+ Version(
+ value=version.lstrip("v"),
+ release_date=dateparser.parse(resp["packages"][pkg_name][version]["time"]),
+ )
+ )
return all_versions
@@ -347,22 +416,54 @@ class GitHubTagsAPI(VersionAPI):
package_type = "github"
async def load_api(self, repo_set):
- async with client_session() as session:
+ session = client_session()
+ async with session as session:
await asyncio.gather(
*[
- self.fetch(owner_repo.lower(), session)
+ self.fetch(owner_repo.lower())
for owner_repo in repo_set
if owner_repo.lower() not in self.cache
]
)
- async def fetch(self, owner_repo: str, session) -> None:
- # owner_repo is a string of format "{repo_owner}/{repo_name}"
- # Example value of owner_repo = "nexB/scancode-toolkit"
- endpoint = f"https://api.github.com/repos/{owner_repo}/git/refs/tags"
- resp = await session.request(method="GET", url=endpoint)
- resp = await resp.json()
- self.cache[owner_repo] = [release["ref"].split("/")[-1] for release in resp]
+ async def fetch(self, owner_repo: str, endpoint=None) -> None:
+ """
+ owner_repo is a string of format "{repo_owner}/{repo_name}"
+ Example value of owner_repo = "nexB/scancode-toolkit"
+ """
+ if owner_repo not in self.cache:
+ self.cache[owner_repo] = set()
+
+ if not endpoint:
+ endpoint = f"https://github.com/{owner_repo}/tags"
+ async with client_session() as session:
+ resp = await session.get(endpoint)
+ resp = await resp.read()
+
+ soup = BeautifulSoup(resp, features="lxml")
+ for release_entry in soup.find_all("div", {"class": "commit"}):
+ version = None
+ for links in release_entry.find_all("a"):
+ if f"/{owner_repo}/releases/tag/" in links["href"].lower():
+ prefix, _slash, version = links["href"].rpartition("/")
+ version = version.lstrip("v")
+ break
+
+ release_date = release_entry.find("relative-time")["datetime"]
+ self.cache[owner_repo].add(
+ Version(value=version, release_date=dateparser.parse(release_date))
+ )
+
+ url = None
+ pagination_links = soup.find("div", {"class": "paginate-container"}).find_all("a")
+ for link in pagination_links:
+ if link.text == "Next":
+ url = link["href"]
+ break
+
+ if url:
+ # FIXME: this could be asynced to improve performance
+ await self.fetch(owner_repo, url)
class HexVersionAPI(VersionAPI):
@@ -379,7 +480,12 @@ async def fetch(self, pkg, session):
response = await session.request(method="GET", url=url)
response = await response.json()
for release in response["releases"]:
- versions.add(release["version"])
+ versions.add(
+ Version(
+ value=release["version"],
+ release_date=dateparser.parse(release["inserted_at"]),
+ )
+ )
except (ClientResponseError, JSONDecodeError):
pass
diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py
index 384696646..47dcbef7e 100644
--- a/vulnerabilities/tests/test_apache_httpd.py
+++ b/vulnerabilities/tests/test_apache_httpd.py
@@ -31,6 +31,7 @@
from vulnerabilities.data_source import Advisory
from vulnerabilities.data_source import VulnerabilitySeverity
from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.package_managers import Version
from vulnerabilities.severity_systems import scoring_systems
from vulnerabilities.importers.apache_httpd import ApacheHTTPDDataSource
from vulnerabilities.helpers import AffectedPackage
@@ -44,7 +45,7 @@ class TestApacheHTTPDDataSource(TestCase):
def setUpClass(cls):
data_source_cfg = {"etags": {}}
cls.data_src = ApacheHTTPDDataSource(1, config=data_source_cfg)
- known_versions = ["1.3.2", "1.3.1", "1.3.0"]
+ known_versions = [Version("1.3.2"), Version("1.3.1"), Version("1.3.0")]
cls.data_src.version_api = GitHubTagsAPI(cache={"apache/httpd": known_versions})
with open(TEST_DATA) as f:
cls.data = json.load(f)
diff --git a/vulnerabilities/tests/test_apache_kafka.py b/vulnerabilities/tests/test_apache_kafka.py
index a2435b425..46869ed6c 100644
--- a/vulnerabilities/tests/test_apache_kafka.py
+++ b/vulnerabilities/tests/test_apache_kafka.py
@@ -29,6 +29,7 @@
from vulnerabilities.data_source import Advisory
from vulnerabilities.data_source import Reference
from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.package_managers import Version
from vulnerabilities.importers.apache_kafka import ApacheKafkaDataSource
from vulnerabilities.importers.apache_kafka import to_version_ranges
from vulnerabilities.helpers import AffectedPackage
@@ -63,7 +64,9 @@ def test_to_version_ranges(self):
def test_to_advisory(self):
data_source = ApacheKafkaDataSource(batch_size=1)
- data_source.version_api = GitHubTagsAPI(cache={"apache/kafka": ["2.1.2", "0.10.2.2"]})
+ data_source.version_api = GitHubTagsAPI(
+ cache={"apache/kafka": [Version("2.1.2"), Version("0.10.2.2")]}
+ )
expected_advisories = [
Advisory(
summary="In Apache Kafka versions between 0.11.0.0 and 2.1.0, it is possible to manually\n craft a Produce request which bypasses transaction/idempotent ACL validation.\n Only authenticated clients with Write permission on the respective topics are\n able to exploit this vulnerability. Users should upgrade to 2.1.1 or later\n where this vulnerability has been fixed.",
diff --git a/vulnerabilities/tests/test_apache_tomcat.py b/vulnerabilities/tests/test_apache_tomcat.py
index 290ca3652..d7419dedc 100644
--- a/vulnerabilities/tests/test_apache_tomcat.py
+++ b/vulnerabilities/tests/test_apache_tomcat.py
@@ -21,7 +21,6 @@
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
import os
-from unittest.mock import MagicMock
from unittest.mock import patch
from unittest import TestCase
@@ -31,6 +30,8 @@
from vulnerabilities.data_source import Reference
from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource
from vulnerabilities.helpers import AffectedPackage
+from vulnerabilities.package_managers import Version
+from vulnerabilities.package_managers import MavenVersionAPI
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_tomcat", "security-9.html")
@@ -40,11 +41,19 @@ class TestApacheTomcatDataSource(TestCase):
@classmethod
def setUpClass(cls):
data_source_cfg = {"etags": {}}
- mock_api = {"org.apache.tomcat:tomcat": ["9.0.0.M1", "9.0.0.M2", "8.0.0.M1", "6.0.0M2"]}
+ mock_api = MavenVersionAPI(
+ cache={
+ "org.apache.tomcat:tomcat": [
+ Version("9.0.0.M1"),
+ Version("9.0.0.M2"),
+ Version("8.0.0.M1"),
+ Version("6.0.0M2"),
+ ]
+ }
+ )
with patch("vulnerabilities.importers.apache_tomcat.MavenVersionAPI"):
with patch("vulnerabilities.importers.apache_tomcat.asyncio"):
cls.data_src = ApacheTomcatDataSource(1, config=data_source_cfg)
-
cls.data_src.version_api = mock_api
def test_to_advisories(self):
diff --git a/vulnerabilities/tests/test_data/github_api/response.json b/vulnerabilities/tests/test_data/github_api/response.json
index a1fb7e9ea..d890394fe 100644
--- a/vulnerabilities/tests/test_data/github_api/response.json
+++ b/vulnerabilities/tests/test_data/github_api/response.json
@@ -21,7 +21,8 @@
"url":"https://github.com/advisories/GHSA-qcxh-w3j9-58qr"
}
],
- "severity": "MODERATE"
+ "severity": "MODERATE",
+ "publishedAt": "2021-05-24T18:12:20Z"
},
"package": {
"name": "org.apache.tomcat.embed:tomcat-embed-core"
@@ -48,7 +49,8 @@
"url":"https://github.com/advisories/GHSA-qcxh-w3j9-58qr"
}
],
- "severity": "HIGH"
+ "severity": "HIGH",
+ "publishedAt": "2021-05-24T18:12:20Z"
},
"package": {
"name": "org.apache.tomcat.embed:tomcat-embed-core"
@@ -75,7 +77,8 @@
"url":"https://github.com/advisories/GHSA-c9hw-wf7x-jp9j"
}
],
- "severity": "LOW"
+ "severity": "LOW",
+ "publishedAt": "2021-05-24T18:12:20Z"
},
"package": {
"name": "org.apache.tomcat.embed:tomcat-embed-core"
@@ -102,7 +105,8 @@
"url":"https://github.com/advisories/GHSA-c9hw-wf7x-jp9j"
}
],
- "severity": "MODERATE"
+ "severity": "MODERATE",
+ "publishedAt": "2021-05-24T18:12:20Z"
},
"package": {
"name": "org.apache.tomcat.embed:tomcat-embed-core"
@@ -129,7 +133,8 @@
"url":"https://github.com/advisories/GHSA-c9hw-wf7x-jp9j"
}
],
- "severity": "LOW"
+ "severity": "LOW",
+ "publishedAt": "2021-05-24T18:12:20Z"
},
"package": {
"name": "org.apache.tomcat.embed:tomcat-embed-core"
diff --git a/vulnerabilities/tests/test_data/maven_api/easygcm.html b/vulnerabilities/tests/test_data/maven_api/easygcm.html
new file mode 100644
index 000000000..280faf6a2
--- /dev/null
+++ b/vulnerabilities/tests/test_data/maven_api/easygcm.html
@@ -0,0 +1,31 @@
+
+
+
+ Central Repository: eu/inloop/easygcm
+
+
+
+
+
+
+
+
+ ../
+1.2.2/ 2014-12-22 10:29 -
+1.2.3/ 2014-12-22 10:53 -
+1.3.0/ 2015-03-12 15:20 -
+maven-metadata.xml 2015-03-12 15:22 385
+maven-metadata.xml.md5 2015-03-12 15:22 32
+maven-metadata.xml.sha1 2015-03-12 15:22 40
+
+
+
+
+
+
\ No newline at end of file
diff --git a/vulnerabilities/tests/test_data/npm.zip b/vulnerabilities/tests/test_data/npm.zip
index 37d86e7f1..fdb957a32 100644
Binary files a/vulnerabilities/tests/test_data/npm.zip and b/vulnerabilities/tests/test_data/npm.zip differ
diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py
index 774a2209f..68022b17c 100644
--- a/vulnerabilities/tests/test_elixir_security.py
+++ b/vulnerabilities/tests/test_elixir_security.py
@@ -30,6 +30,7 @@
from vulnerabilities.data_source import Reference
from vulnerabilities.importers.elixir_security import ElixirSecurityDataSource
from vulnerabilities.package_managers import HexVersionAPI
+from vulnerabilities.package_managers import Version
from vulnerabilities.helpers import AffectedPackage
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -45,17 +46,17 @@ def setUpClass(cls):
cls.data_src.pkg_manager_api = HexVersionAPI(
{
"coherence": [
- "0.5.2",
- "0.5.1",
- "0.5.0",
- "0.4.0",
- "0.3.1",
- "0.3.0",
- "0.2.0",
- "0.1.3",
- "0.1.2",
- "0.1.1",
- "0.1.0",
+ Version("0.5.2"),
+ Version("0.5.1"),
+ Version("0.5.0"),
+ Version("0.4.0"),
+ Version("0.3.1"),
+ Version("0.3.0"),
+ Version("0.2.0"),
+ Version("0.1.3"),
+ Version("0.1.2"),
+ Version("0.1.1"),
+ Version("0.1.0"),
]
}
)
diff --git a/vulnerabilities/tests/test_github.py b/vulnerabilities/tests/test_github.py
index 3306b75a5..c186711aa 100644
--- a/vulnerabilities/tests/test_github.py
+++ b/vulnerabilities/tests/test_github.py
@@ -26,17 +26,14 @@
from unittest.mock import patch
from unittest.mock import MagicMock
from unittest.mock import call
-import xml.etree.ElementTree as ET
-from collections import OrderedDict
-from requests.models import Response
from packageurl import PackageURL
from vulnerabilities.data_source import Advisory
from vulnerabilities.data_source import Reference
from vulnerabilities.data_source import VulnerabilitySeverity
from vulnerabilities.importers.github import GitHubAPIDataSource
-from vulnerabilities.package_managers import MavenVersionAPI
+from vulnerabilities.package_managers import MavenVersionAPI, Version
from vulnerabilities.package_managers import NugetVersionAPI
from vulnerabilities.package_managers import ComposerVersionAPI
from vulnerabilities.severity_systems import ScoringSystem
@@ -295,12 +292,14 @@ def test_process_response(self):
),
]
- mock_version_api = MagicMock()
- mock_version_api.package_type = "maven"
- mock_version_api.get = lambda x: {"1.2.0", "9.0.2"}
+ mock_version_api = MavenVersionAPI(
+ cache={
+ "org.apache.tomcat.embed:tomcat-embed-core": {Version("1.2.0"), Version("9.0.2")}
+ }
+ )
with patch(
"vulnerabilities.importers.github.MavenVersionAPI", return_value=mock_version_api
- ): # nopep8
+ ):
with patch("vulnerabilities.importers.github.GitHubAPIDataSource.set_api"):
found_advisories = self.data_src.process_response()
diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py
index f17dc64ed..986830edc 100644
--- a/vulnerabilities/tests/test_istio.py
+++ b/vulnerabilities/tests/test_istio.py
@@ -29,6 +29,7 @@
from vulnerabilities.data_source import Advisory, Reference
from vulnerabilities.importers.istio import IstioDataSource
from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.package_managers import Version
from vulnerabilities.helpers import AffectedPackage
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -44,16 +45,16 @@ def setUpClass(cls):
cls.data_src.version_api = GitHubTagsAPI(
{
"istio/istio": [
- "1.0.0",
- "1.1.0",
- "1.1.1",
- "1.1.17",
- "1.2.1",
- "1.2.7",
- "1.3.0",
- "1.3.1",
- "1.3.2",
- "1.9.1",
+ Version(value="1.0.0"),
+ Version(value="1.1.0"),
+ Version(value="1.1.1"),
+ Version(value="1.1.17"),
+ Version(value="1.2.1"),
+ Version(value="1.2.7"),
+ Version(value="1.3.0"),
+ Version(value="1.3.1"),
+ Version(value="1.3.2"),
+ Version(value="1.9.1"),
]
}
)
diff --git a/vulnerabilities/tests/test_nginx.py b/vulnerabilities/tests/test_nginx.py
index 38551a46b..a62e009f7 100644
--- a/vulnerabilities/tests/test_nginx.py
+++ b/vulnerabilities/tests/test_nginx.py
@@ -27,9 +27,9 @@
from packageurl import PackageURL
from vulnerabilities.data_source import Advisory
-from vulnerabilities.data_source import Reference
from vulnerabilities.importers.nginx import NginxDataSource
from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.package_managers import Version
from vulnerabilities.helpers import AffectedPackage
@@ -45,11 +45,17 @@ def setUpClass(cls):
data_source_cfg = {"etags": {}}
cls.data_src = NginxDataSource(1, config=data_source_cfg)
cls.data_src.version_api = GitHubTagsAPI(
- cache={"nginx/nginx": {"1.2.3", "1.7.0", "1.3.9", "0.7.52"}}
+ cache={
+ "nginx/nginx": {
+ Version("1.2.3"),
+ Version("1.7.0"),
+ Version("1.3.9"),
+ Version("0.7.52"),
+ }
+ }
)
def test_to_advisories(self):
- # expected_advisories = [Advisory(summary='An error log data are not sanitized', vulnerability_id='CVE-2009-4487', affected_packages=[], references=[]), Advisory(summary='Directory traversal vulnerability', vulnerability_id='CVE-2009-3898', affected_packages=[AffectedPackage(vulnerable_package=PackageURL(type='generic', namespace=None, name='nginx', version='0.7.52', qualifiers={}, subpath=None), patched_package=None)], references=[]), Advisory(summary='Stack-based buffer overflow with specially crafted request', vulnerability_id='CVE-2013-2028', affected_packages=[AffectedPackage(vulnerable_package=PackageURL(type='generic', namespace=None, name='nginx', version='1.3.9', qualifiers={}, subpath=None), patched_package=PackageURL(type='generic', namespace=None, name='nginx', version='1.7.0', qualifiers={}, subpath=None))], references=[]), Advisory(summary='The renegotiation vulnerability in SSL protocol', vulnerability_id='CVE-2009-3555', affected_packages=[AffectedPackage(vulnerable_package=PackageURL(type='generic', namespace=None, name='nginx', version='0.7.52', qualifiers={}, subpath=None), patched_package=None)], references=[]), Advisory(summary='Vulnerabilities with Windows directory aliases', vulnerability_id='CVE-2011-4963', affected_packages=[AffectedPackage(vulnerable_package=PackageURL(type='generic', namespace=None, name='nginx', version='0.7.52', qualifiers={'os': 'windows'}, subpath=None), patched_package=PackageURL(type='generic', namespace=None, name='nginx', version='1.2.3', qualifiers={}, subpath=None)), AffectedPackage(vulnerable_package=PackageURL(type='generic', namespace=None, name='nginx', version='1.2.3', qualifiers={'os': 'windows'}, subpath=None), patched_package=PackageURL(type='generic', namespace=None, name='nginx', version='1.3.9', qualifiers={}, subpath=None))], references=[]), Advisory(summary='Vulnerabilities with invalid UTF-8 sequence on Windows', vulnerability_id='CVE-2010-2266', affected_packages=[AffectedPackage(vulnerable_package=PackageURL(type='generic', namespace=None, name='nginx', version='0.7.52', qualifiers={'os': 'windows'}, subpath=None), patched_package=None)], references=[])]
expected_advisories = [
Advisory(
summary="An error log data are not sanitized",
diff --git a/vulnerabilities/tests/test_npm.py b/vulnerabilities/tests/test_npm.py
index 953cb1d96..6deab261d 100644
--- a/vulnerabilities/tests/test_npm.py
+++ b/vulnerabilities/tests/test_npm.py
@@ -20,7 +20,6 @@
# for any legal advice.
# VulnerableCode is a free software code scanning tool from nexB Inc. and others.
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
-import json
import os
import shutil
import tempfile
@@ -32,6 +31,7 @@
from vulnerabilities import models
from vulnerabilities.import_runner import ImportRunner
from vulnerabilities.package_managers import NpmVersionAPI
+from vulnerabilities.package_managers import Version
from vulnerabilities.importers.npm import categorize_versions
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -40,9 +40,15 @@
MOCK_VERSION_API = NpmVersionAPI(
cache={
- "jquery": {"3.4.0", "3.8.0"},
- "kerberos": {"0.5.8", "1.2.0"},
- "@hapi/subtext": {"3.7.0", "4.1.1", "6.1.3", "7.0.0", "7.0.5"},
+ "jquery": {Version("3.4.0"), Version("3.8.0")},
+ "kerberos": {Version("0.5.8"), Version("1.2.0")},
+ "@hapi/subtext": {
+ Version("3.7.0"),
+ Version("4.1.1"),
+ Version("6.1.3"),
+ Version("7.0.0"),
+ Version("7.0.5"),
+ },
}
)
@@ -67,7 +73,7 @@ def setUpClass(cls) -> None:
data_source="NpmDataSource",
data_source_cfg={
"repository_url": "https://example.git",
- "working_directory": os.path.join(cls.tempdir, "npm_test"),
+ "working_directory": os.path.join(cls.tempdir, "npm/npm_test"),
"create_working_directory": False,
"remove_working_directory": False,
},
diff --git a/vulnerabilities/tests/test_package_managers.py b/vulnerabilities/tests/test_package_managers.py
index a74a46907..28c7cb3ab 100644
--- a/vulnerabilities/tests/test_package_managers.py
+++ b/vulnerabilities/tests/test_package_managers.py
@@ -21,17 +21,21 @@
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
import asyncio
-import os
import json
-from unittest import TestCase
-from unittest.mock import patch
-from unittest.mock import MagicMock, AsyncMock
+import os
import xml.etree.ElementTree as ET
-from aiohttp import test_utils
+from datetime import datetime
+from bs4 import BeautifulSoup
+from dateutil.tz import tzlocal
+from pytz import UTC
+from unittest import TestCase
+from unittest.mock import AsyncMock
from vulnerabilities.package_managers import ComposerVersionAPI
from vulnerabilities.package_managers import MavenVersionAPI
from vulnerabilities.package_managers import NugetVersionAPI
+from vulnerabilities.package_managers import Version
+from vulnerabilities.package_managers import VersionResponse
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_DATA = os.path.join(BASE_DIR, "test_data")
@@ -62,72 +66,270 @@ def setUpClass(cls):
cls.response = json.load(f)
cls.expected_versions = {
- "9.5.3",
- "8.7.30",
- "9.3.1",
- "9.5.1",
- "9.5.11",
- "9.5.6",
- "8.7.18",
- "8.7.15",
- "9.4.0",
- "9.5.7",
- "8.7.21",
- "9.5.12",
- "9.5.14",
- "8.7.27",
- "8.7.17",
- "8.7.9",
- "10.4.3",
- "10.0.0",
- "10.1.0",
- "9.5.13",
- "9.5.5",
- "8.7.22",
- "8.7.10",
- "8.7.24",
- "8.7.13",
- "8.7.14",
- "8.7.19",
- "9.5.17",
- "9.3.2",
- "9.5.15",
- "8.7.8",
- "9.3.3",
- "8.7.32",
- "10.4.0",
- "10.4.1",
- "9.5.18",
- "9.1.0",
- "9.5.19",
- "9.5.2",
- "8.7.26",
- "8.7.20",
- "10.2.0",
- "8.7.31",
- "8.7.11",
- "9.2.1",
- "8.7.25",
- "9.5.10",
- "10.2.2",
- "10.4.2",
- "9.5.9",
- "9.2.0",
- "9.3.0",
- "9.5.16",
- "10.3.0",
- "8.7.7",
- "10.4.4",
- "8.7.12",
- "8.7.29",
- "10.2.1",
- "9.5.8",
- "9.5.4",
- "9.5.0",
- "8.7.28",
- "8.7.23",
- "9.0.0",
- "8.7.16",
+ Version(
+ value="8.7.10",
+ release_date=datetime(2018, 2, 6, 10, 46, 2, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.11",
+ release_date=datetime(2018, 3, 13, 12, 44, 45, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.12",
+ release_date=datetime(2018, 3, 22, 11, 35, 42, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.13",
+ release_date=datetime(2018, 4, 17, 8, 15, 46, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.14",
+ release_date=datetime(2018, 5, 22, 13, 51, 9, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.15",
+ release_date=datetime(2018, 5, 23, 11, 31, 21, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.16",
+ release_date=datetime(2018, 6, 11, 17, 18, 14, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.17",
+ release_date=datetime(2018, 7, 12, 11, 29, 19, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.18",
+ release_date=datetime(2018, 7, 31, 8, 15, 29, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.19",
+ release_date=datetime(2018, 8, 21, 7, 23, 21, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.21",
+ release_date=datetime(2018, 12, 11, 12, 40, 12, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.20",
+ release_date=datetime(2018, 10, 30, 10, 39, 51, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.22",
+ release_date=datetime(2018, 12, 14, 7, 43, 50, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.23",
+ release_date=datetime(2019, 1, 22, 10, 10, 2, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.24",
+ release_date=datetime(2019, 1, 22, 15, 25, 55, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.25",
+ release_date=datetime(2019, 5, 7, 10, 5, 55, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.26",
+ release_date=datetime(2019, 5, 15, 11, 24, 12, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.27",
+ release_date=datetime(2019, 6, 25, 8, 24, 21, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.28",
+ release_date=datetime(2019, 10, 15, 7, 21, 52, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.29",
+ release_date=datetime(2019, 10, 30, 21, 0, 45, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.30",
+ release_date=datetime(2019, 12, 17, 10, 49, 17, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.31",
+ release_date=datetime(2020, 2, 17, 23, 29, 16, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.7",
+ release_date=datetime(2017, 9, 19, 14, 22, 53, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.32",
+ release_date=datetime(2020, 3, 31, 8, 33, 3, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.8",
+ release_date=datetime(2017, 10, 10, 16, 8, 44, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="8.7.9",
+ release_date=datetime(2017, 12, 12, 16, 9, 50, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.0.0",
+ release_date=datetime(2017, 12, 12, 16, 48, 22, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.1.0",
+ release_date=datetime(2018, 1, 30, 15, 31, 12, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.2.0",
+ release_date=datetime(2018, 4, 9, 20, 51, 35, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.2.1",
+ release_date=datetime(2018, 5, 22, 13, 47, 11, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.3.0",
+ release_date=datetime(2018, 6, 11, 17, 14, 33, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.3.1",
+ release_date=datetime(2018, 7, 12, 11, 33, 12, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.3.2",
+ release_date=datetime(2018, 7, 12, 15, 51, 49, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.3.3",
+ release_date=datetime(2018, 7, 31, 8, 20, 17, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.0",
+ release_date=datetime(2018, 10, 2, 8, 10, 33, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.4.0",
+ release_date=datetime(2018, 9, 4, 12, 8, 20, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.1",
+ release_date=datetime(2018, 10, 30, 10, 45, 30, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.10",
+ release_date=datetime(2019, 10, 15, 7, 29, 55, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.11",
+ release_date=datetime(2019, 10, 30, 20, 46, 49, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.12",
+ release_date=datetime(2019, 12, 17, 10, 53, 45, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.13",
+ release_date=datetime(2019, 12, 17, 14, 17, 37, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.14",
+ release_date=datetime(2020, 2, 17, 23, 37, 2, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.15",
+ release_date=datetime(2020, 3, 31, 8, 40, 25, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.16",
+ release_date=datetime(2020, 4, 28, 9, 22, 14, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.17",
+ release_date=datetime(2020, 5, 12, 10, 36, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.18",
+ release_date=datetime(2020, 5, 19, 13, 10, 50, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.2",
+ release_date=datetime(2018, 12, 11, 12, 42, 55, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.19",
+ release_date=datetime(2020, 6, 9, 8, 44, 34, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.3",
+ release_date=datetime(2018, 12, 14, 7, 28, 48, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.4",
+ release_date=datetime(2019, 1, 22, 10, 12, 4, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.5",
+ release_date=datetime(2019, 3, 4, 20, 25, 8, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.6",
+ release_date=datetime(2019, 5, 7, 10, 16, 30, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.7",
+ release_date=datetime(2019, 5, 15, 11, 41, 51, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.8",
+ release_date=datetime(2019, 6, 25, 8, 28, 51, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="9.5.9",
+ release_date=datetime(2019, 8, 20, 9, 33, 35, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.0.0",
+ release_date=datetime(2019, 7, 23, 7, 6, 3, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.1.0",
+ release_date=datetime(2019, 10, 1, 8, 18, 18, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.2.0",
+ release_date=datetime(2019, 12, 3, 11, 16, 26, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.2.1",
+ release_date=datetime(2019, 12, 17, 11, 0, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.2.2",
+ release_date=datetime(2019, 12, 17, 11, 36, 14, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.3.0",
+ release_date=datetime(2020, 2, 25, 12, 50, 9, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.4.0",
+ release_date=datetime(2020, 4, 21, 8, 0, 15, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.4.1",
+ release_date=datetime(2020, 4, 28, 9, 7, 54, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.4.2",
+ release_date=datetime(2020, 5, 12, 10, 41, 40, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.4.4",
+ release_date=datetime(2020, 6, 9, 8, 56, 30, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="10.4.3",
+ release_date=datetime(2020, 5, 19, 13, 16, 31, tzinfo=tzlocal()),
+ ),
}
def test_composer_url(self):
@@ -142,10 +344,10 @@ def test_extract_versions(self):
def test_fetch(self):
- assert self.version_api.get("typo3/cms-core") == set()
+ assert self.version_api.get("typo3/cms-core") == VersionResponse()
client_session = MockClientSession(self.response)
asyncio.run(self.version_api.fetch("typo3/cms-core", client_session))
- assert self.version_api.get("typo3/cms-core") == self.expected_versions
+ assert self.version_api.cache["typo3/cms-core"] == self.expected_versions
class TestMavenVersionAPI(TestCase):
@@ -172,15 +374,15 @@ def test_artifact_url(self):
)
def test_extract_versions(self):
- expected_versions = {"1.2.2", "1.2.3", "1.3.0"}
+ expected_versions = {Version("1.2.2"), Version("1.2.3"), Version("1.3.0")}
assert expected_versions == self.version_api.extract_versions(self.response)
def test_fetch(self):
- assert self.version_api.get("org.apache:kafka") == set()
- expected = {"1.2.3", "1.3.0", "1.2.2"}
+ assert self.version_api.get("org.apache:kafka") == VersionResponse()
+ expected = {"1.2.2", "1.2.3", "1.3.0"}
client_session = MockClientSession(self.content)
asyncio.run(self.version_api.fetch("org.apache:kafka", client_session))
- assert self.version_api.get("org.apache:kafka") == expected
+ assert self.version_api.get("org.apache:kafka") == VersionResponse(valid_versions=expected)
class TestNugetVersionAPI(TestCase):
@@ -191,20 +393,62 @@ def setUpClass(cls):
cls.response = json.load(f)
cls.expected_versions = {
- "0.23.0",
- "0.24.0",
- "1.0.0",
- "1.0.1",
- "1.0.2",
- "2.0.0",
- "2.0.0-preview01",
- "2.6.0",
- "2.1.0",
- "2.2.0",
- "2.3.0",
- "2.4.0",
- "2.5.0",
- "2.7.0",
+ Version(
+ value="1.0.0",
+ release_date=datetime(2018, 9, 13, 8, 16, 0, 420000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="1.0.1",
+ release_date=datetime(2020, 1, 17, 15, 31, 41, 857000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="1.0.2",
+ release_date=datetime(2020, 4, 21, 12, 24, 53, 877000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="2.0.0-preview01",
+ release_date=datetime(2018, 1, 9, 17, 12, 20, 440000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="2.0.0",
+ release_date=datetime(2018, 9, 27, 13, 33, 15, 370000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="2.1.0",
+ release_date=datetime(2018, 10, 16, 6, 59, 44, 680000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="2.2.0",
+ release_date=datetime(2018, 11, 23, 8, 13, 8, 3000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="2.3.0",
+ release_date=datetime(2019, 6, 27, 14, 27, 31, 613000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="2.4.0",
+ release_date=datetime(2020, 1, 17, 15, 11, 5, 810000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="2.5.0",
+ release_date=datetime(2020, 3, 24, 14, 22, 39, 960000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="2.7.0",
+ release_date=datetime(2020, 4, 21, 12, 27, 36, 427000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="2.6.0",
+ release_date=datetime(2020, 3, 27, 11, 6, 27, 500000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="0.24.0",
+ release_date=datetime(2018, 3, 30, 7, 25, 18, 393000, tzinfo=tzlocal()),
+ ),
+ Version(
+ value="0.23.0",
+ release_date=datetime(2018, 1, 17, 9, 32, 59, 283000, tzinfo=tzlocal()),
+ ),
}
def test_nuget_url(self):
@@ -219,13 +463,30 @@ def test_extract_versions(self):
def test_fetch(self):
- assert self.version_api.get("Exfat.Ntfs") == set()
+ assert self.version_api.get("Exfat.Ntfs") == VersionResponse()
client_session = MockClientSession(self.response)
asyncio.run(self.version_api.fetch("Exfat.Ntfs", client_session))
- assert self.version_api.get("Exfat.Ntfs") == self.expected_versions
+ assert self.version_api.get("Exfat.Ntfs") == VersionResponse(
+ newer_versions=set(),
+ valid_versions={
+ "2.0.0",
+ "2.1.0",
+ "2.0.0-preview01",
+ "0.24.0",
+ "0.23.0",
+ "1.0.1",
+ "2.2.0",
+ "2.4.0",
+ "1.0.0",
+ "1.0.2",
+ "2.3.0",
+ "2.7.0",
+ "2.5.0",
+ "2.6.0",
+ },
+ )
# def test_load_to_api(self):
-
# assert self.version_api.get("Exfat.Ntfs") == set()
# mock_response = MagicMock()
diff --git a/vulnerabilities/tests/test_ruby.py b/vulnerabilities/tests/test_ruby.py
index 664f001ed..affcf8fa1 100644
--- a/vulnerabilities/tests/test_ruby.py
+++ b/vulnerabilities/tests/test_ruby.py
@@ -24,15 +24,14 @@
import pathlib
from unittest.mock import patch
from unittest import TestCase
-from collections import OrderedDict
from packageurl import PackageURL
from vulnerabilities.importers.ruby import RubyDataSource
-from vulnerabilities.data_source import GitDataSourceConfiguration
from vulnerabilities.data_source import Advisory
from vulnerabilities.data_source import Reference
from vulnerabilities.package_managers import RubyVersionAPI
+from vulnerabilities.package_managers import VersionResponse
from vulnerabilities.helpers import AffectedPackage
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -55,7 +54,9 @@ def setUpClass(cls):
@patch(
"vulnerabilities.package_managers.RubyVersionAPI.get",
- return_value={"1.0.0", "1.8.0", "2.0.3"},
+ return_value=VersionResponse(
+ valid_versions={"1.0.0", "1.8.0", "2.0.3"}, newer_versions=set()
+ ),
)
def test_process_file(self, mock_write):
expected_advisories = [
diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py
index f563ccc5e..9ec34c7b1 100644
--- a/vulnerabilities/tests/test_rust.py
+++ b/vulnerabilities/tests/test_rust.py
@@ -30,23 +30,27 @@
from vulnerabilities.importers.rust import categorize_versions
from vulnerabilities.importers.rust import get_advisory_data
from vulnerabilities.importers.rust import RustDataSource
+from vulnerabilities.package_managers import Version
+from vulnerabilities.package_managers import CratesVersionAPI
from vulnerabilities.helpers import AffectedPackage
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
TEST_DATA = os.path.join(BASE_DIR, "test_data/rust")
-MOCKED_CRATES_API_VERSIONS = {
- "bitvec": {"0.10.0", "0.12.0", "0.18.0"},
- "bumpalo": {"2.8.0", "3.0.1", "3.2.5"},
- "cbox": {"0.10.0", "0.12.0", "0.18.0"},
- "flatbuffers": {"0.3.0", "0.5.0", "0.6.5"},
- "hyper": {"0.10.0", "0.12.0", "0.13.0"},
- "byte_struct": {"0.6.1", "0.6.0", "1.0.0"},
-}
+MOCKED_CRATES_API_VERSIONS = CratesVersionAPI(
+ cache={
+ "bitvec": {Version("0.10.0"), Version("0.12.0"), Version("0.18.0")},
+ "bumpalo": {Version("2.8.0"), Version("3.0.1"), Version("3.2.5")},
+ "cbox": {Version("0.10.0"), Version("0.12.0"), Version("0.18.0")},
+ "flatbuffers": {Version("0.3.0"), Version("0.5.0"), Version("0.6.5")},
+ "hyper": {Version("0.10.0"), Version("0.12.0"), Version("0.13.0")},
+ "byte_struct": {Version("0.6.1"), Version("0.6.0"), Version("1.0.0")},
+ }
+)
def test_categorize_versions():
- flatbuffers_versions = MOCKED_CRATES_API_VERSIONS["flatbuffers"]
+ flatbuffers_versions = MOCKED_CRATES_API_VERSIONS.get("flatbuffers").valid_versions
unaffected_ranges = [VersionSpecifier.from_scheme_version_spec_string("semver", "< 0.4.0")]
affected_ranges = [
diff --git a/vulnerabilities/tests/test_safety_db.py b/vulnerabilities/tests/test_safety_db.py
index 20bbc87b2..ddc78f9a7 100644
--- a/vulnerabilities/tests/test_safety_db.py
+++ b/vulnerabilities/tests/test_safety_db.py
@@ -22,16 +22,16 @@
# Visit https://github.com/nexB/vulnerablecode/ for support and download.
import json
import os
-from unittest.mock import patch
from unittest import TestCase
from packageurl import PackageURL
-from vulnerabilities.importers.safety_db import PypiVersionAPI
from vulnerabilities.importers.safety_db import categorize_versions
from vulnerabilities.importers.safety_db import SafetyDbDataSource
from vulnerabilities.data_source import Advisory
from vulnerabilities.data_source import Reference
+from vulnerabilities.package_managers import PypiVersionAPI
+from vulnerabilities.package_managers import Version
from vulnerabilities.helpers import AffectedPackage
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -39,9 +39,16 @@
MOCK_VERSION_API = PypiVersionAPI(
cache={
- "ampache": {"2.0", "5.2.1"},
- "django": {"1.8", "1.4.19", "1.4.22", "1.5.1", "1.6.9", "1.8.14"},
- "zulip": {"2.0", "2.1.1", "2.1.2", "2.1.3"},
+ "ampache": {Version("2.0"), Version("5.2.1")},
+ "django": {
+ Version("1.8"),
+ Version("1.4.19"),
+ Version("1.4.22"),
+ Version("1.5.1"),
+ Version("1.6.9"),
+ Version("1.8.14"),
+ },
+ "zulip": {Version("2.0"), Version("2.1.1"), Version("2.1.2"), Version("2.1.3")},
}
)