diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 349563520..ff48a374a 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -483,7 +483,7 @@ def updated_advisories(self) -> List[Advisory]: except Exception: logger.error( f"Failed to get updated_advisories: {oval_file!r} " - "with {metadata!r}:\n" + traceback.format_exc() + f"with {metadata!r}:\n" + traceback.format_exc() ) continue @@ -538,7 +538,7 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis affected_version_range = VersionSpecifier.from_scheme_version_spec_string( version_scheme, affected_version_range ) - all_versions = self.pkg_manager_api.get(package_name) + all_versions = self.pkg_manager_api.get(package_name).valid_versions # FIXME: what is this 50 DB limit? that's too small for versions # FIXME: we should not drop data this way diff --git a/vulnerabilities/importers/nginx.py b/vulnerabilities/importers/nginx.py index 77913398a..c5372647d 100644 --- a/vulnerabilities/importers/nginx.py +++ b/vulnerabilities/importers/nginx.py @@ -34,6 +34,7 @@ from vulnerabilities.data_source import DataSourceConfiguration from vulnerabilities.data_source import Reference from vulnerabilities.package_managers import GitHubTagsAPI +from vulnerabilities.package_managers import Version from vulnerabilities.helpers import nearest_patched_package @@ -53,10 +54,14 @@ def set_api(self): # For some reason nginx tags it's releases are in the form of `release-1.2.3` # Chop off the `release-` part here. - for index, version in enumerate(self.version_api.cache["nginx/nginx"].valid_versions): - self.version_api.cache["nginx/nginx"].valid_versions[index] = version.replace( - "release-", "" + normalized_versions = set() + while self.version_api.cache["nginx/nginx"]: + version = self.version_api.cache["nginx/nginx"].pop() + normalized_version = Version( + version.value.replace("release-", ""), version.release_date ) + normalized_versions.add(normalized_version) + self.version_api.cache["nginx/nginx"] = normalized_versions def updated_advisories(self): advisories = [] diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index e827f26b7..ca875a832 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -98,7 +98,12 @@ async def fetch(self, pkg, session): self.cache[pkg] = {} break for release in resp_json["entries"]: - all_versions.add(release["source_package_version"].replace("0:", "")) + all_versions.add( + Version( + value=release["source_package_version"].replace("0:", ""), + release_date=release["date_published"], + ) + ) if resp_json.get("next_collection_link"): url = resp_json["next_collection_link"] else: @@ -233,7 +238,7 @@ async def fetch(self, pkg, session, retry_count=5): self.cache[pkg] = {} return for release in resp_json["versions"]: - all_versions.add(release["version"].replace("0:", "")) + all_versions.add(Version(value=release["version"].replace("0:", ""))) self.cache[pkg] = all_versions # TODO : Handle ServerDisconnectedError by using some sort of @@ -372,14 +377,14 @@ class GitHubTagsAPI(VersionAPI): package_type = "github" - async def fetch(self, owner_repo: str, session) -> None: + async def fetch(self, owner_repo: str, session, endpoint=None) -> None: """ owner_repo is a string of format "{repo_owner}/{repo_name}" Example value of owner_repo = "nexB/scancode-toolkit" """ self.cache[owner_repo] = set() - endpoint = f"https://github.com/{owner_repo}/tags" - + if not endpoint: + endpoint = f"https://github.com/{owner_repo}/tags" resp = await session.get(endpoint) resp = await resp.read() @@ -406,7 +411,7 @@ async def fetch(self, owner_repo: str, session) -> None: if url: # FIXME: this could be asynced to improve performance - await self.fetch(owner_repo, url) + await self.fetch(owner_repo, session, url) class HexVersionAPI(VersionAPI): diff --git a/vulnerabilities/tests/test_debian_oval.py b/vulnerabilities/tests/test_debian_oval.py index e7b298834..ffce735f5 100644 --- a/vulnerabilities/tests/test_debian_oval.py +++ b/vulnerabilities/tests/test_debian_oval.py @@ -6,6 +6,7 @@ from packageurl import PackageURL from vulnerabilities.importers.debian_oval import DebianOvalDataSource +from vulnerabilities.package_managers import VersionResponse from vulnerabilities.data_source import Advisory from vulnerabilities.helpers import AffectedPackage @@ -30,7 +31,9 @@ def setUpClass(cls): @patch( "vulnerabilities.importers.debian_oval.DebianVersionAPI.get", - return_value={"1.11.1+dfsg-5+deb7u1", "0.11.1+dfsg-5+deb7u1", "2.3.9"}, + return_value=VersionResponse( + valid_versions={"1.11.1+dfsg-5+deb7u1", "0.11.1+dfsg-5+deb7u1", "2.3.9"} + ), ) @patch("vulnerabilities.importers.debian_oval.DebianVersionAPI.load_api", new=mock) def test_get_data_from_xml_doc(self, mock_write): diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index 16d1f217b..23b10750f 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -10,6 +10,7 @@ from vulnerabilities.oval_parser import OvalParser from vulnerabilities.importers.ubuntu import UbuntuDataSource +from vulnerabilities.package_managers import VersionResponse from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import Reference from vulnerabilities.helpers import AffectedPackage @@ -182,7 +183,7 @@ def setUpClass(cls): @patch( "vulnerabilities.importers.ubuntu.LaunchpadVersionAPI.get", - return_value={"0.3.0", "0.2.0", "2.14-2"}, + return_value=VersionResponse(valid_versions={"0.3.0", "0.2.0", "2.14-2"}), ) @patch("vulnerabilities.importers.ubuntu.LaunchpadVersionAPI.load_api", new=mock) def test_get_data_from_xml_doc(self, mock_write):