use github api to find github releases

previous version with svn uses blocking synchronous api and do not work well with proxies

the tag api do not contain the commit infos and thus we don't have the release date.
I think what we really want in vulnerable code is the list of actual releases, and not really the tags.

Signed-off-by: Pierre Tardy <pierre.tardy@renault.com>

Author: tardyp

vulnerabilities/package_managers.py

@@ -19,6 +19,7 @@
 #  for any legal advice.
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+import os
 import asyncio
 import dataclasses
 import xml.etree.ElementTree as ET
@@ -30,6 +31,7 @@
 from typing import Set
 
 from aiohttp import ClientSession
+import aiohttp
 from aiohttp.client_exceptions import ClientResponseError
 from aiohttp.client_exceptions import ServerDisconnectedError
 from bs4 import BeautifulSoup
@@ -378,20 +380,36 @@ class GitHubTagsAPI(VersionAPI):
 
     package_type = "github"
 
-    async def fetch(self, owner_repo: str, session) -> None:
+    def __init__(self, cache: Mapping[str, Set[Version]] = None):
+        self.gh_token = os.getenv("GH_TOKEN")
+        super().__init__(cache=cache)
+
+    async def fetch(self, owner_repo: str, session: aiohttp.ClientSession) -> None:
         """
         owner_repo is a string of format "{repo_owner}/{repo_name}"
         Example value of owner_repo = "nexB/scancode-toolkit"
         """
         self.cache[owner_repo] = set()
-        endpoint = f"https://github.com/{owner_repo}"
-
-        tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True)
-        elements = ET.fromstring(tags_xml)
-        for entry in elements.iter("entry"):
-            name = entry.find("name").text
-            release_date = dateparser.parse(entry.find("commit/date").text)
-            self.cache[owner_repo].add(Version(value=name, release_date=release_date))
+        if self.gh_token:
+            headers = {"Authorization": "token " + self.gh_token}
+            endpoint = f"https://api.github.com/repos/{owner_repo}/releases"
+            response = await session.get(endpoint)
+            resp_json = await response.json()
+
+            for entry in resp_json:
+                name = entry["tag_name"]
+                release_date = dateparser.parse(entry["created_at"])
+                self.cache[owner_repo].add(Version(value=name, release_date=release_date))
+        else:
+            # In case we don't have GH_TOKEN, we use the svn ls method to get the tags
+            # It allows to get all the information needed in one request without any rate limiting
+            endpoint = f"https://github.com/{owner_repo}"
+            tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True)
+            elements = ET.fromstring(tags_xml)
+            for entry in elements.iter("entry"):
+                name = entry.find("name").text
+                release_date = dateparser.parse(entry.find("commit/date").text)
+
 
 
 class HexVersionAPI(VersionAPI):

vulnerabilities/tests/test_data/github_api/release_response.json

@@ -0,0 +1,76 @@
+[
+  {
+    "url": "https://api.github.com/repos/nexB/vulnerablecode/releases/32748782",
+    "assets_url": "https://api.github.com/repos/nexB/vulnerablecode/releases/32748782/assets",
+    "upload_url": "https://uploads.github.com/repos/nexB/vulnerablecode/releases/32748782/assets{?name,label}",
+    "html_url": "https://github.com/nexB/vulnerablecode/releases/tag/v20.10",
+    "id": 32748782,
+    "author": {
+      "login": "pombredanne",
+      "id": 675997,
+      "node_id": "MDQ6VXNlcjY3NTk5Nw==",
+      "avatar_url": "https://avatars.githubusercontent.com/u/675997?v=4",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/pombredanne",
+      "html_url": "https://github.com/pombredanne",
+      "followers_url": "https://api.github.com/users/pombredanne/followers",
+      "following_url": "https://api.github.com/users/pombredanne/following{/other_user}",
+      "gists_url": "https://api.github.com/users/pombredanne/gists{/gist_id}",
+      "starred_url": "https://api.github.com/users/pombredanne/starred{/owner}{/repo}",
+      "subscriptions_url": "https://api.github.com/users/pombredanne/subscriptions",
+      "organizations_url": "https://api.github.com/users/pombredanne/orgs",
+      "repos_url": "https://api.github.com/users/pombredanne/repos",
+      "events_url": "https://api.github.com/users/pombredanne/events{/privacy}",
+      "received_events_url": "https://api.github.com/users/pombredanne/received_events",
+      "type": "User",
+      "site_admin": false
+    },
+    "node_id": "MDc6UmVsZWFzZTMyNzQ4Nzgy",
+    "tag_name": "v20.10",
+    "target_commitish": "main",
+    "name": "v20.10",
+    "draft": false,
+    "prerelease": false,
+    "created_at": "2020-09-28T12:31:16Z",
+    "published_at": "2020-10-19T10:46:17Z",
+    "assets": [
+      {
+        "url": "https://api.github.com/repos/nexB/vulnerablecode/releases/assets/27230021",
+        "id": 27230021,
+        "node_id": "MDEyOlJlbGVhc2VBc3NldDI3MjMwMDIx",
+        "name": "vulnerablecode-2020-10-19.json.xz",
+        "label": null,
+        "uploader": {
+          "login": "pombredanne",
+          "id": 675997,
+          "node_id": "MDQ6VXNlcjY3NTk5Nw==",
+          "avatar_url": "https://avatars.githubusercontent.com/u/675997?v=4",
+          "gravatar_id": "",
+          "url": "https://api.github.com/users/pombredanne",
+          "html_url": "https://github.com/pombredanne",
+          "followers_url": "https://api.github.com/users/pombredanne/followers",
+          "following_url": "https://api.github.com/users/pombredanne/following{/other_user}",
+          "gists_url": "https://api.github.com/users/pombredanne/gists{/gist_id}",
+          "starred_url": "https://api.github.com/users/pombredanne/starred{/owner}{/repo}",
+          "subscriptions_url": "https://api.github.com/users/pombredanne/subscriptions",
+          "organizations_url": "https://api.github.com/users/pombredanne/orgs",
+          "repos_url": "https://api.github.com/users/pombredanne/repos",
+          "events_url": "https://api.github.com/users/pombredanne/events{/privacy}",
+          "received_events_url": "https://api.github.com/users/pombredanne/received_events",
+          "type": "User",
+          "site_admin": false
+        },
+        "content_type": "application/x-xz",
+        "state": "uploaded",
+        "size": 13603356,
+        "download_count": 20,
+        "created_at": "2020-10-20T09:40:08Z",
+        "updated_at": "2020-10-20T09:40:25Z",
+        "browser_download_url": "https://github.com/nexB/vulnerablecode/releases/download/v20.10/vulnerablecode-2020-10-19.json.xz"
+      }
+    ],
+    "tarball_url": "https://api.github.com/repos/nexB/vulnerablecode/tarball/v20.10",
+    "zipball_url": "https://api.github.com/repos/nexB/vulnerablecode/zipball/v20.10",
+    "body": "This release comes with the new calver versioning scheme and an initial data dump.\r\n\r\nTo load the JSON data attached here:\r\n- extract it with `unxz vulnerablecode-2020-10-19.json.xz`\r\n- run `DJANGO_DEV=1 python manage.py loaddata vulnerablecode-2020-10-19.json`\r\n\r\nThe data import is not optimized yet and takes a long time."
+  }
+]

vulnerabilities/tests/test_package_managers.py

@@ -25,7 +25,6 @@
 import os
 import xml.etree.ElementTree as ET
 from datetime import datetime
-from bs4 import BeautifulSoup
 from dateutil.tz import tzlocal
 from pytz import UTC
 from unittest import TestCase
@@ -34,8 +33,10 @@
 from vulnerabilities.package_managers import ComposerVersionAPI
 from vulnerabilities.package_managers import MavenVersionAPI
 from vulnerabilities.package_managers import NugetVersionAPI
+from vulnerabilities.package_managers import GitHubTagsAPI
 from vulnerabilities.package_managers import Version
 from vulnerabilities.package_managers import VersionResponse
+from vulnerabilities.package_managers import client_session
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 TEST_DATA = os.path.join(BASE_DIR, "test_data")
@@ -51,6 +52,14 @@ async def request(self, *args, **kwargs):
         mock_response.read = self.read
         return mock_response
 
+    def get(self, *args, **kwargs):
+        kwargs["method"] = "get"
+        return self.request(*args, **kwargs)
+
+    def post(self, *args, **kwargs):
+        kwargs["method"] = "post"
+        return self.request(*args, **kwargs)
+
     async def json(self):
         return self.return_val
 
@@ -496,3 +505,25 @@ def test_fetch(self):
     #         self.version_api.load_to_api("Exfat.Ntfs")
 
     #     assert self.version_api.get("Exfat.Ntfs") == self.expected_versions
+
+
+class TestGitHubTagsAPI(TestCase):
+    def setUp(self) -> None:
+        self.version_api = GitHubTagsAPI()
+        with open(os.path.join(TEST_DATA, "github_api", "release_response.json")) as f:
+            self.response = json.load(f)
+        return super().setUp()
+
+    def test_fetch(self):
+        async def async_run():
+            session = MockClientSession(self.response)
+            await self.version_api.fetch("nexB/vulnerablecode", session)
+
+        asyncio.run(async_run())
+
+        assert self.version_api.get("nexB/vulnerablecode") == VersionResponse(
+            newer_versions=set(),
+            valid_versions={
+                "v20.10",
+            },
+        )