Merge pull request #505 from Hritik14/importer-bugfix

Importers bugfix [nginx, debian_oval, ubuntu]

Author: Hritik14

vulnerabilities/data_source.py

@@ -483,7 +483,7 @@ def updated_advisories(self) -> List[Advisory]:
             except Exception:
                 logger.error(
                     f"Failed to get updated_advisories: {oval_file!r} "
-                    "with {metadata!r}:\n" + traceback.format_exc()
+                    f"with {metadata!r}:\n" + traceback.format_exc()
                 )
                 continue
 
@@ -538,7 +538,7 @@ def get_data_from_xml_doc(self, xml_doc: ET.ElementTree, pkg_metadata={}) -> Lis
                     affected_version_range = VersionSpecifier.from_scheme_version_spec_string(
                         version_scheme, affected_version_range
                     )
-                    all_versions = self.pkg_manager_api.get(package_name)
+                    all_versions = self.pkg_manager_api.get(package_name).valid_versions
 
                     # FIXME: what is this 50 DB limit? that's too small for versions
                     # FIXME: we should not drop data this way

vulnerabilities/importers/nginx.py

@@ -34,6 +34,7 @@
 from vulnerabilities.data_source import DataSourceConfiguration
 from vulnerabilities.data_source import Reference
 from vulnerabilities.package_managers import GitHubTagsAPI
+from vulnerabilities.package_managers import Version
 from vulnerabilities.helpers import nearest_patched_package
 
 
@@ -53,10 +54,14 @@ def set_api(self):
 
         # For some reason nginx tags it's releases are in the form of `release-1.2.3`
         # Chop off the `release-` part here.
-        for index, version in enumerate(self.version_api.cache["nginx/nginx"].valid_versions):
-            self.version_api.cache["nginx/nginx"].valid_versions[index] = version.replace(
-                "release-", ""
+        normalized_versions = set()
+        while self.version_api.cache["nginx/nginx"]:
+            version = self.version_api.cache["nginx/nginx"].pop()
+            normalized_version = Version(
+                version.value.replace("release-", ""), version.release_date
             )
+            normalized_versions.add(normalized_version)
+        self.version_api.cache["nginx/nginx"] = normalized_versions
 
     def updated_advisories(self):
         advisories = []

vulnerabilities/package_managers.py

@@ -98,7 +98,12 @@ async def fetch(self, pkg, session):
                     self.cache[pkg] = {}
                     break
                 for release in resp_json["entries"]:
-                    all_versions.add(release["source_package_version"].replace("0:", ""))
+                    all_versions.add(
+                        Version(
+                            value=release["source_package_version"].replace("0:", ""),
+                            release_date=release["date_published"],
+                        )
+                    )
                 if resp_json.get("next_collection_link"):
                     url = resp_json["next_collection_link"]
                 else:
@@ -233,7 +238,7 @@ async def fetch(self, pkg, session, retry_count=5):
                 self.cache[pkg] = {}
                 return
             for release in resp_json["versions"]:
-                all_versions.add(release["version"].replace("0:", ""))
+                all_versions.add(Version(value=release["version"].replace("0:", "")))
 
             self.cache[pkg] = all_versions
         # TODO : Handle ServerDisconnectedError by using some sort of
@@ -372,14 +377,14 @@ class GitHubTagsAPI(VersionAPI):
 
     package_type = "github"
 
-    async def fetch(self, owner_repo: str, session) -> None:
+    async def fetch(self, owner_repo: str, session, endpoint=None) -> None:
         """
         owner_repo is a string of format "{repo_owner}/{repo_name}"
         Example value of owner_repo = "nexB/scancode-toolkit"
         """
         self.cache[owner_repo] = set()
-        endpoint = f"https://github.com/{owner_repo}/tags"
-
+        if not endpoint:
+            endpoint = f"https://github.com/{owner_repo}/tags"
         resp = await session.get(endpoint)
         resp = await resp.read()
 
@@ -406,7 +411,7 @@ async def fetch(self, owner_repo: str, session) -> None:
 
         if url:
             # FIXME: this could be asynced to improve performance
-            await self.fetch(owner_repo, url)
+            await self.fetch(owner_repo, session, url)
 
 
 class HexVersionAPI(VersionAPI):

vulnerabilities/tests/test_debian_oval.py

@@ -6,6 +6,7 @@
 from packageurl import PackageURL
 
 from vulnerabilities.importers.debian_oval import DebianOvalDataSource
+from vulnerabilities.package_managers import VersionResponse
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.helpers import AffectedPackage
 
@@ -30,7 +31,9 @@ def setUpClass(cls):
 
     @patch(
         "vulnerabilities.importers.debian_oval.DebianVersionAPI.get",
-        return_value={"1.11.1+dfsg-5+deb7u1", "0.11.1+dfsg-5+deb7u1", "2.3.9"},
+        return_value=VersionResponse(
+            valid_versions={"1.11.1+dfsg-5+deb7u1", "0.11.1+dfsg-5+deb7u1", "2.3.9"}
+        ),
     )
     @patch("vulnerabilities.importers.debian_oval.DebianVersionAPI.load_api", new=mock)
     def test_get_data_from_xml_doc(self, mock_write):

vulnerabilities/tests/test_ubuntu.py

@@ -10,6 +10,7 @@
 
 from vulnerabilities.oval_parser import OvalParser
 from vulnerabilities.importers.ubuntu import UbuntuDataSource
+from vulnerabilities.package_managers import VersionResponse
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import Reference
 from vulnerabilities.helpers import AffectedPackage
@@ -182,7 +183,7 @@ def setUpClass(cls):
 
     @patch(
         "vulnerabilities.importers.ubuntu.LaunchpadVersionAPI.get",
-        return_value={"0.3.0", "0.2.0", "2.14-2"},
+        return_value=VersionResponse(valid_versions={"0.3.0", "0.2.0", "2.14-2"}),
     )
     @patch("vulnerabilities.importers.ubuntu.LaunchpadVersionAPI.load_api", new=mock)
     def test_get_data_from_xml_doc(self, mock_write):