Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,4 @@ toml>=0.10.2
lxml>=4.6.3
gunicorn>=20.1.0
django-environ==0.4.5
defusedxml==0.7.1
40 changes: 10 additions & 30 deletions vulnerabilities/package_managers.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import xml.etree.ElementTree as ET
from datetime import datetime
from json import JSONDecodeError
from subprocess import check_output
from typing import List
from typing import Mapping
from typing import Set
Expand Down Expand Up @@ -377,41 +378,20 @@ class GitHubTagsAPI(VersionAPI):

package_type = "github"

async def fetch(self, owner_repo: str, session, endpoint=None) -> None:
async def fetch(self, owner_repo: str, session) -> None:
"""
owner_repo is a string of format "{repo_owner}/{repo_name}"
Example value of owner_repo = "nexB/scancode-toolkit"
"""
self.cache[owner_repo] = set()
if not endpoint:
endpoint = f"https://github.com/{owner_repo}/tags"
resp = await session.get(endpoint)
resp = await resp.read()

soup = BeautifulSoup(resp, features="lxml")
for release_entry in soup.find_all("div", {"class": "commit"}):
version = None
for links in release_entry.find_all("a"):
if f"/{owner_repo}/releases/tag/" in links["href"].lower():
prefix, _slash, version = links["href"].rpartition("/")
version = version.lstrip("v")
break

release_date = release_entry.find("relative-time")["datetime"]
self.cache[owner_repo].add(
Version(value=version, release_date=dateparser.parse(release_date))
)

url = None
pagination_links = soup.find("div", {"class": "paginate-container"}).find_all("a")
for link in pagination_links:
if link.text == "Next":
url = link["href"]
break

if url:
# FIXME: this could be asynced to improve performance
await self.fetch(owner_repo, session, url)
endpoint = f"https://github.com/{owner_repo}"

tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True)
elements = ET.fromstring(tags_xml)
for entry in elements.iter("entry"):
name = entry.find("name").text
release_date = dateparser.parse(entry.find("commit/date").text)
self.cache[owner_repo].add(Version(value=name, release_date=release_date))


class HexVersionAPI(VersionAPI):
Expand Down