Fix github improver

vulnerabilities/helpers.py

@@ -178,7 +178,8 @@ def nearest_patched_package(
 
         affected_package_with_patched_package_objects.append(
             AffectedPackage(
-                vulnerable_package=vulnerable_package.purl, patched_package=patched_package.purl
+                vulnerable_package=vulnerable_package.purl,
+                patched_package=patched_package.purl if patched_package else None,
             )
         )
 

vulnerabilities/importers/github.py

@@ -527,11 +527,10 @@ def resolve_version_range(
     for package_version in package_versions or []:
         if package_version in ignorable_versions:
             continue
-        # Remove leading 'v'
-        if package_version.startswith("v") or package_version.startswith("V"):
-            package_version = package_version.replace("V", "").replace("v", "")
         # Remove whitespace
         package_version = package_version.replace(" ", "")
+        # Remove leading 'v'
+        package_version = package_version.lstrip("vV")
         try:
             version = affected_version_range.version_class(package_version)
         except Exception:

vulnerabilities/package_managers_2.py

@@ -13,6 +13,7 @@
 from dateutil import parser as dateparser
 from django.utils.dateparse import parse_datetime
 
+from vulnerabilities.helpers import get_item
 from vulnerabilities.package_managers import VersionResponse
 
 LOGGER = logging.getLogger(__name__)
@@ -31,7 +32,11 @@ class VersionResponse:
 
 
 def get_response(url, type="json"):
-    resp = requests.get(url=url)
+    try:
+        resp = requests.get(url=url)
+    except:
+        LOGGER.error(traceback.format_exc())
+        return None
     if not resp.status_code == 200:
         LOGGER.error(f"Error while fetching {url}: {resp.status_code}")
         return None
@@ -86,20 +91,23 @@ def fetch(self, pkg):
             self.cache[pkg] = versions
             return
 
-        for version, download_items in response["releases"].items() or {}:
+        releases = response.get("releases") or {}
+        for version, download_items in releases.items():
             if download_items:
                 latest_download_item = max(
                     download_items,
                     key=lambda download_item: dateparser.parse(
                         download_item["upload_time_iso_8601"]
-                        if "upload_time_iso_8601" in download_item
-                        else LOGGER.error(f"{download_item} has no upload_time_iso_8601")
-                    ),
+                    )
+                    if download_item.get("upload_time_iso_8601")
+                    else None,
                 )
                 versions.add(
                     LegacyVersion(
                         value=version,
-                        release_date=dateparser.parse(latest_download_item["upload_time_iso_8601"]),
+                        release_date=dateparser.parse(latest_download_item["upload_time_iso_8601"])
+                        if latest_download_item.get("upload_time_iso_8601")
+                        else None,
                     )
                 )
         self.cache[pkg] = versions
@@ -117,8 +125,13 @@ def fetch(self, pkg):
             self.cache[pkg] = versions
             return
         for release in response:
-            if release["number"] and release["published_at"]:
+            if release.get("published_at"):
                 release_date = dateparser.parse(release["published_at"])
+            elif release.get("created_at"):
+                release_date = dateparser.parse(release["created_at"])
+            else:
+                release_date = None
+            if release.get("number"):
                 versions.add(LegacyVersion(value=release["number"], release_date=release_date))
             else:
                 LOGGER.error(f"Failed to parse release {release}")
@@ -196,12 +209,16 @@ def nuget_url(pkg_name: str) -> str:
     @staticmethod
     def extract_versions(resp: dict) -> Set[LegacyVersion]:
         all_versions = set()
-        for entry_group in resp["items"] or []:
-            for entry in entry_group["items"] or []:
-                catalog_entry = entry["catalogEntry"] or {}
+        for entry_group in resp.get("items") or []:
+            for entry in entry_group.get("items") or []:
+                catalog_entry = entry.get("catalogEntry") or {}
                 version = catalog_entry.get("version")
-                release_date = dateparser.parse(catalog_entry.get("published"))
-                if version and release_date:
+                release_date = (
+                    dateparser.parse(catalog_entry["published"])
+                    if catalog_entry.get("published")
+                    else None
+                )
+                if version:
                     all_versions.add(
                         LegacyVersion(
                             value=version,
@@ -345,17 +362,18 @@ def composer_url(pkg_name: str) -> Optional[str]:
     @staticmethod
     def extract_versions(resp: dict, pkg_name: str) -> Set[LegacyVersion]:
         all_versions = set()
-        for version in resp["packages"][pkg_name]:
+        for version in get_item(resp, "packages", pkg_name) or []:
             if "dev" in version:
                 continue
 
             # This if statement ensures, that all_versions contains only released versions
             # See https://github.com/composer/composer/blob/44a4429978d1b3c6223277b875762b2930e83e8c/doc/articles/versions.md#tags  # nopep8
             # for explanation of removing 'v'
+            time = get_item(resp, "packages", pkg_name, version, "time")
             all_versions.add(
                 LegacyVersion(
                     value=version.lstrip("v"),
-                    release_date=dateparser.parse(resp["packages"][pkg_name][version]["time"]),
+                    release_date=dateparser.parse(time) if time else None,
                 )
             )
         return all_versions

vulnerabilities/tests/test_data/package_manager_data/gem.json

@@ -0,0 +1,54 @@
+[
+  {
+    "authors": "David Heinemeier Hansson",
+    "built_at": "2022-03-08T00:00:00.000Z",
+    "published_at": "2022-03-08T17:50:52.496Z",
+    "description": "Ruby on Rails is a full-stack web framework optimized for programmer happiness and sustainable productivity. It encourages beautiful code by favoring convention over configuration.",
+    "downloads_count": 295102,
+    "metadata": {
+      "changelog_uri": "https://github.com/rails/rails/releases/tag/v7.0.2.3",
+      "bug_tracker_uri": "https://github.com/rails/rails/issues",
+      "source_code_uri": "https://github.com/rails/rails/tree/v7.0.2.3",
+      "mailing_list_uri": "https://discuss.rubyonrails.org/c/rubyonrails-talk",
+      "documentation_uri": "https://api.rubyonrails.org/v7.0.2.3/",
+      "rubygems_mfa_required": true
+    },
+    "number": "7.0.2.3",
+    "summary": "Full-stack web application framework.",
+    "platform": "ruby",
+    "rubygems_version": ">= 1.8.11",
+    "ruby_version": ">= 2.7.0",
+    "prerelease": false,
+    "licenses": [
+      "MIT"
+    ],
+    "requirements": [],
+    "sha": "ee4e24075c72dec6e02e3fcddec86399c2b4eb0466efe4ccb5f78f96d3daa283"
+  },
+  {
+    "authors": "David Heinemeier Hansson",
+    "built_at": "2022-02-11T00:00:00.000Z",
+    "created_at": "2022-02-11T19:44:19.017Z",
+    "description": "Ruby on Rails is a full-stack web framework optimized for programmer happiness and sustainable productivity. It encourages beautiful code by favoring convention over configuration.",
+    "downloads_count": 347689,
+    "metadata": {
+      "changelog_uri": "https://github.com/rails/rails/releases/tag/v7.0.2.2",
+      "bug_tracker_uri": "https://github.com/rails/rails/issues",
+      "source_code_uri": "https://github.com/rails/rails/tree/v7.0.2.2",
+      "mailing_list_uri": "https://discuss.rubyonrails.org/c/rubyonrails-talk",
+      "documentation_uri": "https://api.rubyonrails.org/v7.0.2.2/",
+      "rubygems_mfa_required": true
+    },
+    "number": "7.0.2.2",
+    "summary": "Full-stack web application framework.",
+    "platform": "ruby",
+    "rubygems_version": ">= 1.8.11",
+    "ruby_version": ">= 2.7.0",
+    "prerelease": false,
+    "licenses": [
+      "MIT"
+    ],
+    "requirements": [],
+    "sha": "3393e21131e2120a42cf634416033e587b5dfdccdc84d1a2d2c176b847f6f17f"
+  }
+]

vulnerabilities/tests/test_helpers.py

@@ -19,7 +19,6 @@
 #  for any legal advice.
 #  VulnerableCode is a free software tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
-from unittest import result
 
 from packageurl import PackageURL
 
@@ -31,6 +30,7 @@ def test_nearest_patched_package():
 
     result = nearest_patched_package(
         vulnerable_packages=[
+            PackageURL(type="npm", name="foo", version="2.0.4"),
             PackageURL(type="npm", name="foo", version="2.0.0"),
             PackageURL(type="npm", name="foo", version="2.0.1"),
             PackageURL(type="npm", name="foo", version="1.9.8"),
@@ -66,4 +66,10 @@ def test_nearest_patched_package():
                 type="npm", namespace=None, name="foo", version="2.0.2", qualifiers={}, subpath=None
             ),
         ),
+        LegacyAffectedPackage(
+            vulnerable_package=PackageURL(
+                type="npm", namespace=None, name="foo", version="2.0.4", qualifiers={}, subpath=None
+            ),
+            patched_package=None,
+        ),
     ] == result

vulnerabilities/tests/test_package_managers_2.py

@@ -1,13 +1,16 @@
 import json
 import os
 from datetime import datetime
+from unittest import mock
 
 import pytest
 import pytz
 
 from vulnerabilities.package_managers_2 import GoproxyVersionAPI
 from vulnerabilities.package_managers_2 import LegacyVersion
 from vulnerabilities.package_managers_2 import NugetVersionAPI
+from vulnerabilities.package_managers_2 import PypiVersionAPI
+from vulnerabilities.package_managers_2 import RubyVersionAPI
 
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 TEST_DATA = os.path.join(BASE_DIR, "test_data", "package_manager_data")
@@ -69,3 +72,103 @@ def test_nuget_extract_version():
             value="3.5.1", release_date=datetime(2015, 1, 23, 1, 5, 44, 447000, tzinfo=pytz.UTC)
         ),
     }
+
+
+def test_nuget_extract_version_with_illformed_data():
+    assert NugetVersionAPI.extract_versions({"items": [{"items": [{"catalogEntry": {}}]}]}) == set()
+
+
+@mock.patch("vulnerabilities.package_managers_2.get_response")
+def test_pypi_fetch_data(mock_response):
+    pypi_api = PypiVersionAPI()
+    with open(os.path.join(TEST_DATA, "pypi.json"), "r") as f:
+        mock_response.return_value = json.load(f)
+    pypi_api.fetch("django")
+    assert pypi_api.cache == {
+        "django": {
+            LegacyVersion(
+                value="1.10.5",
+                release_date=datetime(2017, 1, 4, 19, 23, 0, 596664, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10.8",
+                release_date=datetime(2017, 9, 5, 15, 31, 58, 221021, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10rc1",
+                release_date=datetime(2016, 7, 18, 18, 5, 5, 503584, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10.4",
+                release_date=datetime(2016, 12, 1, 23, 46, 50, 215935, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10a1",
+                release_date=datetime(2016, 5, 20, 12, 24, 59, 952686, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10.3",
+                release_date=datetime(2016, 11, 1, 13, 57, 16, 55061, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10.1",
+                release_date=datetime(2016, 9, 1, 23, 18, 18, 672706, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10.2",
+                release_date=datetime(2016, 10, 1, 20, 5, 31, 330942, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10.7",
+                release_date=datetime(2017, 4, 4, 14, 27, 54, 235551, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10.6",
+                release_date=datetime(2017, 3, 1, 13, 37, 40, 243134, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.1.4",
+                release_date=datetime(2011, 2, 9, 4, 13, 7, 75, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10b1",
+                release_date=datetime(2016, 6, 22, 1, 15, 17, 267637, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.1.3",
+                release_date=datetime(2010, 12, 23, 5, 14, 23, 509436, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="1.10",
+                release_date=datetime(2016, 8, 1, 18, 32, 16, 280614, tzinfo=pytz.UTC),
+            ),
+        }
+    }
+
+
+@mock.patch("vulnerabilities.package_managers_2.get_response")
+def test_pypi_fetch_with_no_release(mock_response):
+    pypi_api = PypiVersionAPI()
+    mock_response.return_value = {"info": {}}
+    pypi_api.fetch("django")
+    assert pypi_api.cache == {"django": set()}
+
+
+@mock.patch("vulnerabilities.package_managers_2.get_response")
+def test_pypi_fetch_with_no_release(mock_response):
+    ruby_api = RubyVersionAPI()
+    with open(os.path.join(TEST_DATA, "gem.json"), "r") as f:
+        mock_response.return_value = json.load(f)
+    ruby_api.fetch("rails")
+    assert ruby_api.cache == {
+        "rails": {
+            LegacyVersion(
+                value="7.0.2.3",
+                release_date=datetime(2022, 3, 8, 17, 50, 52, 496000, tzinfo=pytz.UTC),
+            ),
+            LegacyVersion(
+                value="7.0.2.2",
+                release_date=datetime(2022, 2, 11, 19, 44, 19, 17000, tzinfo=pytz.UTC),
+            ),
+        }
+    }