Skip to content

Commit 4192af6

Browse files
committed
Use svn to collects tags in GitHubTagsAPI
Surprisingly, GitHub allows svn requests to repositories. Now we can have all the tags with a single request. This is much more efficient and gentle to the APIs. Signed-off-by: Hritik Vijay <hritikxx8@gmail.com>
1 parent 7d3df72 commit 4192af6

2 files changed

Lines changed: 11 additions & 31 deletions

File tree

requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,4 @@ toml>=0.10.2
1717
lxml>=4.6.3
1818
gunicorn>=20.1.0
1919
django-environ==0.4.5
20+
defusedxml==0.7.1

vulnerabilities/package_managers.py

Lines changed: 10 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import xml.etree.ElementTree as ET
2525
from datetime import datetime
2626
from json import JSONDecodeError
27+
from subprocess import check_output
2728
from typing import List
2829
from typing import Mapping
2930
from typing import Set
@@ -377,42 +378,20 @@ class GitHubTagsAPI(VersionAPI):
377378

378379
package_type = "github"
379380

380-
async def fetch(self, owner_repo: str, session, endpoint=None) -> None:
381+
async def fetch(self, owner_repo: str, session) -> None:
381382
"""
382383
owner_repo is a string of format "{repo_owner}/{repo_name}"
383384
Example value of owner_repo = "nexB/scancode-toolkit"
384385
"""
385386
self.cache[owner_repo] = set()
386-
if not endpoint:
387-
endpoint = f"https://github.com/{owner_repo}/tags"
388-
resp = await session.get(endpoint)
389-
resp = await resp.read()
390-
391-
soup = BeautifulSoup(resp, features="lxml")
392-
for release_entry in soup.find_all("div", {"class": "commit"}):
393-
version = None
394-
for links in release_entry.find_all("a"):
395-
if f"/{owner_repo}/releases/tag/" in links["href"].lower():
396-
prefix, _slash, version = links["href"].rpartition("/")
397-
version = version.lstrip("v")
398-
break
399-
400-
release_date = release_entry.find("relative-time")["datetime"]
401-
self.cache[owner_repo].add(
402-
Version(value=version, release_date=dateparser.parse(release_date))
403-
)
404-
405-
url = None
406-
pagination_links = soup.find("div", {"class": "paginate-container"}).find_all("a")
407-
for link in pagination_links:
408-
if link.text == "Next":
409-
url = link["href"]
410-
break
411-
412-
if url:
413-
# FIXME: this could be asynced to improve performance
414-
await self.fetch(owner_repo, session, url)
415-
387+
endpoint = f"https://github.com/{owner_repo}"
388+
389+
tags_xml = check_output(['svn', 'ls', '--xml', f"{endpoint}/tags"], text=True)
390+
elements = ET.fromstring(tags_xml)
391+
for entry in elements.iter("entry"):
392+
name = entry.find("name").text
393+
release_date = dateparser.parse(entry.find("commit/date").text)
394+
self.cache[owner_repo].add(Version(value=name, release_date=release_date))
416395

417396
class HexVersionAPI(VersionAPI):
418397
async def fetch(self, pkg, session):

0 commit comments

Comments
 (0)