Make review changes

Signed-off-by: Shivam Sandbhor <shivam@crowdsec.net>

Author: sbs2001

vulnerabilities/importers/apache_tomcat.py

@@ -29,6 +29,7 @@
 from bs4 import BeautifulSoup
 from univers.version_specifier import VersionSpecifier
 from univers.versions import MavenVersion
+from univers.versions import SemverVersion
 from packageurl import PackageURL
 
 from vulnerabilities.data_source import Advisory
@@ -62,8 +63,10 @@ def updated_advisories(self):
         return self.batch_advisories(advisories)
 
     def fetch_pages(self):
+        # Here Semver is used because it has notion of major, minor versions.
         tomcat_major_versions = {
-            i[0] for i in self.version_api.get("org.apache.tomcat:tomcat").valid_versions
+            SemverVersion(i).value.major
+            for i in self.version_api.get("org.apache.tomcat:tomcat").valid_versions
         }
         for version in tomcat_major_versions:
             page_url = self.base_url.format(version)

vulnerabilities/importers/github.py

@@ -23,9 +23,7 @@
 import asyncio
 import os
 import dataclasses
-import json
-from xml.etree.ElementTree import parse
-from dateutil import parser
+from dateutil import parser as dateparser
 from typing import Set
 from typing import Tuple
 from typing import List
@@ -36,7 +34,6 @@
 from packageurl import PackageURL
 from univers.version_specifier import VersionSpecifier
 from univers.versions import version_class_by_package_type
-from univers.versions import InvalidVersion
 
 from vulnerabilities.data_source import Advisory
 from vulnerabilities.data_source import DataSource
@@ -263,7 +260,7 @@ def process_response(self) -> List[Advisory]:
             for resp_page in self.advisories[ecosystem]:
                 for adv in resp_page["data"]["securityVulnerabilities"]["edges"]:
                     name = adv["node"]["package"]["name"]
-                    cutoff_time = parser.parse(adv["node"]["advisory"]["publishedAt"])
+                    cutoff_time = dateparser.parse(adv["node"]["advisory"]["publishedAt"])
                     affected_purls = []
                     unaffected_purls = []
                     if self.process_name(ecosystem, name):

vulnerabilities/importers/safety_db.py

@@ -86,7 +86,7 @@ def updated_advisories(self) -> Set[Advisory]:
                 # Skip it. The 'cumin' entry is wrong
                 continue
 
-            all_package_versions = self.versions.get(package_name)
+            all_package_versions = self.versions.get(package_name).valid_versions
             if not len(all_package_versions):
                 # PyPi does not have data about this package, we skip these
                 continue

vulnerabilities/package_managers.py

@@ -23,12 +23,13 @@
 import asyncio
 import dataclasses
 import pytz
+import xml.etree.ElementTree as ET
 from bs4 import BeautifulSoup
-from dateutil import parser
+from dateutil import parser as dateparser
 from json import JSONDecodeError
 from typing import Mapping
-from typing import Tuple
 from typing import Set
+from typing import List
 from datetime import datetime
 
 from aiohttp import ClientSession
@@ -122,13 +123,19 @@ async def fetch(self, pkg, session):
         try:
             response = await session.request(method="GET", url=url)
             response = await response.json()
-            for version in response["releases"]:
-                if response["releases"][version]:
+            for version, download_items in response["releases"].items():
+                if download_items:
+                    latest_download_item = max(
+                        download_items,
+                        key=lambda download_item: dateparser.parse(
+                            download_item["upload_time_iso_8601"]
+                        ),
+                    )
                     versions.add(
                         Version(
                             value=version,
-                            release_date=parser.parse(
-                                response["releases"][version][-1]["upload_time_iso_8601"]
+                            release_date=dateparser.parse(
+                                latest_download_item["upload_time_iso_8601"]
                             ),
                         )
                     )
@@ -157,7 +164,8 @@ async def fetch(self, pkg, session):
         for version_info in response["versions"]:
             versions.add(
                 Version(
-                    value=version_info["num"], release_date=parser.parse(version_info["updated_at"])
+                    value=version_info["num"],
+                    release_date=dateparser.parse(version_info["updated_at"]),
                 )
             )
 
@@ -183,7 +191,8 @@ async def fetch(self, pkg, session):
             for release in response:
                 versions.add(
                     Version(
-                        value=release["number"], release_date=parser.parse(release["created_at"])
+                        value=release["number"],
+                        release_date=dateparser.parse(release["created_at"]),
                     )
                 )
         except (ClientResponseError, JSONDecodeError):
@@ -211,7 +220,7 @@ async def fetch(self, pkg, session):
             for version in response.get("versions", []):
                 release_date = response.get("time", {}).get(version)
                 if release_date:
-                    release_date = parser.parse(release_date)
+                    release_date = dateparser.parse(release_date)
                     versions.add(Version(value=version, release_date=release_date))
                 else:
                     versions.add(Version(value=version, release_date=None))
@@ -265,15 +274,12 @@ class MavenVersionAPI(VersionAPI):
     async def load_api(self, pkg_set):
         async with client_session() as session:
             await asyncio.gather(
-                *[
-                    self.fetch(pkg, session)
-                    for pkg in pkg_set
-                    if pkg not in self.cache and "camel" not in pkg
-                ]
+                *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache]
             )
 
     async def fetch(self, pkg, session) -> None:
-        endpoint = self.artifact_url(pkg)
+        artifact_comps = pkg.split(":")
+        endpoint = self.artifact_url(artifact_comps)
         try:
             resp = await session.request(method="GET", url=endpoint)
             resp = await resp.read()
@@ -282,15 +288,11 @@ async def fetch(self, pkg, session) -> None:
             self.cache[pkg] = set()
             return
 
-        soup = BeautifulSoup(resp, features="lxml")
-        try:
-            self.cache[pkg] = self.extract_versions(soup)
-        except:
-            raise
+        xml_resp = ET.ElementTree(ET.fromstring(resp.decode("utf-8")))
+        self.cache[pkg] = self.extract_versions(xml_resp)
 
     @staticmethod
-    def artifact_url(pkg: str) -> str:
-        artifact_comps = pkg.split(":")
+    def artifact_url(artifact_comps: List[str]) -> str:
         base_url = "https://repo1.maven.org/maven2/{}"
         try:
             group_id, artifact_id = artifact_comps
@@ -306,35 +308,19 @@ def artifact_url(pkg: str) -> str:
                 raise
 
         group_url = group_id.replace(".", "/")
-        suffix = group_url + "/" + artifact_id + "/"
+        suffix = group_url + "/" + artifact_id + "/" + "maven-metadata.xml"
         endpoint = base_url.format(suffix)
 
         return endpoint
 
     @staticmethod
-    def extract_versions(soup: BeautifulSoup) -> Set[Version]:
-        pre_tag = soup.find("pre")
-        prev_tag = None
-        versions = set()
-        for i, atag in enumerate(pre_tag):
-            if atag.name == "a" and i != 0:
-                prev_tag = atag
-            elif prev_tag:
-                text_groups = atag.split()
-                if text_groups[-1] != "-":
-                    break
-                date = " ".join(text_groups[:-1])
-                if date != "-":
-                    versions.add(
-                        Version(
-                            value=prev_tag.text[:-1],
-                            release_date=parser.parse(date).replace(tzinfo=pytz.UTC),
-                        )
-                    )
-                else:
-                    versions.add(Version(value=prev_tag.text[:-1], release_date=None))
+    def extract_versions(xml_response: ET.ElementTree) -> Set[str]:
+        all_versions = set()
+        for child in xml_response.getroot().iter():
+            if child.tag == "version":
+                all_versions.add(Version(child.text))
 
-        return versions
+        return all_versions
 
 
 class NugetVersionAPI(VersionAPI):
@@ -368,7 +354,7 @@ def extract_versions(resp: dict) -> Set[str]:
                     all_versions.add(
                         Version(
                             value=entry["catalogEntry"]["version"],
-                            release_date=parser.parse(entry["catalogEntry"]["published"]),
+                            release_date=dateparser.parse(entry["catalogEntry"]["published"]),
                         )
                     )
         # FIXME: json response for YamlDotNet.Signed triggers this exception.
@@ -412,15 +398,16 @@ def extract_versions(resp: dict, pkg_name: str) -> Set[str]:
         for version in resp["packages"][pkg_name]:
             if "dev" in version:
                 continue
+
+            # This if statement ensures, that all_versions contains only released versions
+            # See https://github.com/composer/composer/blob/44a4429978d1b3c6223277b875762b2930e83e8c/doc/articles/versions.md#tags  # nopep8
+            # for explanation of removing 'v'
             all_versions.add(
                 Version(
-                    value=version.replace("v", ""),
-                    release_date=parser.parse(resp["packages"][pkg_name][version]["time"]),
+                    value=version.lstrip("v"),
+                    release_date=dateparser.parse(resp["packages"][pkg_name][version]["time"]),
                 )
             )
-        # This if statement ensures, that all_versions contains only released versions
-        # See https://github.com/composer/composer/blob/44a4429978d1b3c6223277b875762b2930e83e8c/doc/articles/versions.md#tags  # nopep8
-        # for explanation of removing 'v'
         return all_versions
 
 
@@ -440,8 +427,10 @@ async def load_api(self, repo_set):
             )
 
     async def fetch(self, owner_repo: str, endpoint=None) -> None:
-        # owner_repo is a string of format "{repo_owner}/{repo_name}"
-        # Example value of owner_repo = "nexB/scancode-toolkit"
+        """
+        owner_repo is a string of format "{repo_owner}/{repo_name}"
+        Example value of owner_repo = "nexB/scancode-toolkit"
+        """
         if owner_repo not in self.cache:
             self.cache[owner_repo] = set()
 
@@ -456,12 +445,13 @@ async def fetch(self, owner_repo: str, endpoint=None) -> None:
             version = None
             for links in release_entry.find_all("a"):
                 if f"/{owner_repo}/releases/tag/" in links["href"].lower():
-                    version = links["href"].split("/")[-1]
+                    prefix, _slash, version = links["href"].rpartition("/")
+                    version = version.lstrip("v")
                     break
 
             release_date = release_entry.find("relative-time")["datetime"]
             self.cache[owner_repo].add(
-                Version(value=version, release_date=parser.parse(release_date))
+                Version(value=version, release_date=dateparser.parse(release_date))
             )
 
         url = None
@@ -492,7 +482,8 @@ async def fetch(self, pkg, session):
             for release in response["releases"]:
                 versions.add(
                     Version(
-                        value=release["version"], release_date=parser.parse(release["inserted_at"])
+                        value=release["version"],
+                        release_date=dateparser.parse(release["inserted_at"]),
                     )
                 )
         except (ClientResponseError, JSONDecodeError):

vulnerabilities/tests/test_package_managers.py

@@ -23,6 +23,7 @@
 import asyncio
 import json
 import os
+import xml.etree.ElementTree as ET
 from datetime import datetime
 from bs4 import BeautifulSoup
 from dateutil.tz import tzlocal
@@ -353,40 +354,35 @@ class TestMavenVersionAPI(TestCase):
     @classmethod
     def setUpClass(cls):
         cls.version_api = MavenVersionAPI()
-        with open(os.path.join(TEST_DATA, "maven_api", "easygcm.html"), "rb") as f:
-            data = f.read()
-            cls.response = BeautifulSoup(data, features="lxml")
-            cls.content = data
+        with open(os.path.join(TEST_DATA, "maven_api", "maven-metadata.xml")) as f:
+            cls.response = ET.parse(f)
+
+        with open(os.path.join(TEST_DATA, "maven_api", "maven-metadata.xml"), "rb") as f:
+            cls.content = f.read()
 
     def test_artifact_url(self):
-        eg_pkg1 = "org.apache:kafka"
-        eg_pkg2 = "apple.msft.windows.mac.oss:exfat-ntfs"
+        eg_comps1 = ["org.apache", "kafka"]
+        eg_comps2 = ["apple.msft.windows.mac.oss", "exfat-ntfs"]
 
-        url1 = self.version_api.artifact_url(eg_pkg1)
-        url2 = self.version_api.artifact_url(eg_pkg2)
+        url1 = self.version_api.artifact_url(eg_comps1)
+        url2 = self.version_api.artifact_url(eg_comps2)
 
-        assert "https://repo1.maven.org/maven2/org/apache/kafka/" == url1
-        assert "https://repo1.maven.org/maven2/apple/msft/windows/mac/oss/exfat-ntfs/" == url2
+        assert "https://repo1.maven.org/maven2/org/apache/kafka/maven-metadata.xml" == url1
+        assert (
+            "https://repo1.maven.org/maven2"
+            "/apple/msft/windows/mac/oss/exfat-ntfs/maven-metadata.xml" == url2
+        )
 
     def test_extract_versions(self):
-        expected_versions = {
-            Version(value="1.3.0", release_date=datetime(2015, 3, 12, 15, 20, tzinfo=UTC)),
-            Version(value="1.2.3", release_date=datetime(2014, 12, 22, 10, 53, tzinfo=UTC)),
-            Version(value="1.2.2", release_date=datetime(2014, 12, 22, 10, 29, tzinfo=UTC)),
-        }
+        expected_versions = {Version("1.2.2"), Version("1.2.3"), Version("1.3.0")}
         assert expected_versions == self.version_api.extract_versions(self.response)
 
     def test_fetch(self):
         assert self.version_api.get("org.apache:kafka") == VersionResponse()
-        expected = {
-            Version(value="1.2.2", release_date=datetime(2014, 12, 22, 10, 29, tzinfo=UTC)),
-            Version(value="1.3.0", release_date=datetime(2015, 3, 12, 15, 20, tzinfo=UTC)),
-            Version(value="1.2.3", release_date=datetime(2014, 12, 22, 10, 53, tzinfo=UTC)),
-        }
-
+        expected = {"1.2.2", "1.2.3", "1.3.0"}
         client_session = MockClientSession(self.content)
         asyncio.run(self.version_api.fetch("org.apache:kafka", client_session))
-        assert self.version_api.cache["org.apache:kafka"] == expected
+        assert self.version_api.get("org.apache:kafka") == VersionResponse(valid_versions=expected)
 
 
 class TestNugetVersionAPI(TestCase):