diff --git a/requirements.txt b/requirements.txt index 57dc4ad21..c78787307 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ toml>=0.10.2 lxml>=4.6.3 gunicorn>=20.1.0 django-environ==0.4.5 +defusedxml==0.7.1 diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index ca875a832..f7fcb70a2 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -24,6 +24,7 @@ import xml.etree.ElementTree as ET from datetime import datetime from json import JSONDecodeError +from subprocess import check_output from typing import List from typing import Mapping from typing import Set @@ -377,41 +378,20 @@ class GitHubTagsAPI(VersionAPI): package_type = "github" - async def fetch(self, owner_repo: str, session, endpoint=None) -> None: + async def fetch(self, owner_repo: str, session) -> None: """ owner_repo is a string of format "{repo_owner}/{repo_name}" Example value of owner_repo = "nexB/scancode-toolkit" """ self.cache[owner_repo] = set() - if not endpoint: - endpoint = f"https://github.com/{owner_repo}/tags" - resp = await session.get(endpoint) - resp = await resp.read() - - soup = BeautifulSoup(resp, features="lxml") - for release_entry in soup.find_all("div", {"class": "commit"}): - version = None - for links in release_entry.find_all("a"): - if f"/{owner_repo}/releases/tag/" in links["href"].lower(): - prefix, _slash, version = links["href"].rpartition("/") - version = version.lstrip("v") - break - - release_date = release_entry.find("relative-time")["datetime"] - self.cache[owner_repo].add( - Version(value=version, release_date=dateparser.parse(release_date)) - ) - - url = None - pagination_links = soup.find("div", {"class": "paginate-container"}).find_all("a") - for link in pagination_links: - if link.text == "Next": - url = link["href"] - break - - if url: - # FIXME: this could be asynced to improve performance - await self.fetch(owner_repo, session, url) + endpoint = f"https://github.com/{owner_repo}" + + tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True) + elements = ET.fromstring(tags_xml) + for entry in elements.iter("entry"): + name = entry.find("name").text + release_date = dateparser.parse(entry.find("commit/date").text) + self.cache[owner_repo].add(Version(value=name, release_date=release_date)) class HexVersionAPI(VersionAPI):