diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index d857a72db..bcd447a48 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -178,7 +178,8 @@ def nearest_patched_package( affected_package_with_patched_package_objects.append( AffectedPackage( - vulnerable_package=vulnerable_package.purl, patched_package=patched_package.purl + vulnerable_package=vulnerable_package.purl, + patched_package=patched_package.purl if patched_package else None, ) ) diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py index f72950aa5..281585f03 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/importers/github.py @@ -527,11 +527,10 @@ def resolve_version_range( for package_version in package_versions or []: if package_version in ignorable_versions: continue - # Remove leading 'v' - if package_version.startswith("v") or package_version.startswith("V"): - package_version = package_version.replace("V", "").replace("v", "") # Remove whitespace package_version = package_version.replace(" ", "") + # Remove leading 'v' + package_version = package_version.lstrip("vV") try: version = affected_version_range.version_class(package_version) except Exception: diff --git a/vulnerabilities/package_managers_2.py b/vulnerabilities/package_managers_2.py index 3c207824f..68eee5e82 100644 --- a/vulnerabilities/package_managers_2.py +++ b/vulnerabilities/package_managers_2.py @@ -13,6 +13,7 @@ from dateutil import parser as dateparser from django.utils.dateparse import parse_datetime +from vulnerabilities.helpers import get_item from vulnerabilities.package_managers import VersionResponse LOGGER = logging.getLogger(__name__) @@ -31,7 +32,11 @@ class VersionResponse: def get_response(url, type="json"): - resp = requests.get(url=url) + try: + resp = requests.get(url=url) + except: + LOGGER.error(traceback.format_exc()) + return None if not resp.status_code == 200: LOGGER.error(f"Error while fetching {url}: {resp.status_code}") return None @@ -86,20 +91,23 @@ def fetch(self, pkg): self.cache[pkg] = versions return - for version, download_items in response["releases"].items() or {}: + releases = response.get("releases") or {} + for version, download_items in releases.items(): if download_items: latest_download_item = max( download_items, key=lambda download_item: dateparser.parse( download_item["upload_time_iso_8601"] - if "upload_time_iso_8601" in download_item - else LOGGER.error(f"{download_item} has no upload_time_iso_8601") - ), + ) + if download_item.get("upload_time_iso_8601") + else None, ) versions.add( LegacyVersion( value=version, - release_date=dateparser.parse(latest_download_item["upload_time_iso_8601"]), + release_date=dateparser.parse(latest_download_item["upload_time_iso_8601"]) + if latest_download_item.get("upload_time_iso_8601") + else None, ) ) self.cache[pkg] = versions @@ -117,8 +125,13 @@ def fetch(self, pkg): self.cache[pkg] = versions return for release in response: - if release["number"] and release["published_at"]: + if release.get("published_at"): release_date = dateparser.parse(release["published_at"]) + elif release.get("created_at"): + release_date = dateparser.parse(release["created_at"]) + else: + release_date = None + if release.get("number"): versions.add(LegacyVersion(value=release["number"], release_date=release_date)) else: LOGGER.error(f"Failed to parse release {release}") @@ -196,12 +209,16 @@ def nuget_url(pkg_name: str) -> str: @staticmethod def extract_versions(resp: dict) -> Set[LegacyVersion]: all_versions = set() - for entry_group in resp["items"] or []: - for entry in entry_group["items"] or []: - catalog_entry = entry["catalogEntry"] or {} + for entry_group in resp.get("items") or []: + for entry in entry_group.get("items") or []: + catalog_entry = entry.get("catalogEntry") or {} version = catalog_entry.get("version") - release_date = dateparser.parse(catalog_entry.get("published")) - if version and release_date: + release_date = ( + dateparser.parse(catalog_entry["published"]) + if catalog_entry.get("published") + else None + ) + if version: all_versions.add( LegacyVersion( value=version, @@ -345,17 +362,18 @@ def composer_url(pkg_name: str) -> Optional[str]: @staticmethod def extract_versions(resp: dict, pkg_name: str) -> Set[LegacyVersion]: all_versions = set() - for version in resp["packages"][pkg_name]: + for version in get_item(resp, "packages", pkg_name) or []: if "dev" in version: continue # This if statement ensures, that all_versions contains only released versions # See https://github.com/composer/composer/blob/44a4429978d1b3c6223277b875762b2930e83e8c/doc/articles/versions.md#tags # nopep8 # for explanation of removing 'v' + time = get_item(resp, "packages", pkg_name, version, "time") all_versions.add( LegacyVersion( value=version.lstrip("v"), - release_date=dateparser.parse(resp["packages"][pkg_name][version]["time"]), + release_date=dateparser.parse(time) if time else None, ) ) return all_versions diff --git a/vulnerabilities/tests/test_data/package_manager_data/gem.json b/vulnerabilities/tests/test_data/package_manager_data/gem.json new file mode 100644 index 000000000..ba49463f1 --- /dev/null +++ b/vulnerabilities/tests/test_data/package_manager_data/gem.json @@ -0,0 +1,54 @@ +[ + { + "authors": "David Heinemeier Hansson", + "built_at": "2022-03-08T00:00:00.000Z", + "published_at": "2022-03-08T17:50:52.496Z", + "description": "Ruby on Rails is a full-stack web framework optimized for programmer happiness and sustainable productivity. It encourages beautiful code by favoring convention over configuration.", + "downloads_count": 295102, + "metadata": { + "changelog_uri": "https://github.com/rails/rails/releases/tag/v7.0.2.3", + "bug_tracker_uri": "https://github.com/rails/rails/issues", + "source_code_uri": "https://github.com/rails/rails/tree/v7.0.2.3", + "mailing_list_uri": "https://discuss.rubyonrails.org/c/rubyonrails-talk", + "documentation_uri": "https://api.rubyonrails.org/v7.0.2.3/", + "rubygems_mfa_required": true + }, + "number": "7.0.2.3", + "summary": "Full-stack web application framework.", + "platform": "ruby", + "rubygems_version": ">= 1.8.11", + "ruby_version": ">= 2.7.0", + "prerelease": false, + "licenses": [ + "MIT" + ], + "requirements": [], + "sha": "ee4e24075c72dec6e02e3fcddec86399c2b4eb0466efe4ccb5f78f96d3daa283" + }, + { + "authors": "David Heinemeier Hansson", + "built_at": "2022-02-11T00:00:00.000Z", + "created_at": "2022-02-11T19:44:19.017Z", + "description": "Ruby on Rails is a full-stack web framework optimized for programmer happiness and sustainable productivity. It encourages beautiful code by favoring convention over configuration.", + "downloads_count": 347689, + "metadata": { + "changelog_uri": "https://github.com/rails/rails/releases/tag/v7.0.2.2", + "bug_tracker_uri": "https://github.com/rails/rails/issues", + "source_code_uri": "https://github.com/rails/rails/tree/v7.0.2.2", + "mailing_list_uri": "https://discuss.rubyonrails.org/c/rubyonrails-talk", + "documentation_uri": "https://api.rubyonrails.org/v7.0.2.2/", + "rubygems_mfa_required": true + }, + "number": "7.0.2.2", + "summary": "Full-stack web application framework.", + "platform": "ruby", + "rubygems_version": ">= 1.8.11", + "ruby_version": ">= 2.7.0", + "prerelease": false, + "licenses": [ + "MIT" + ], + "requirements": [], + "sha": "3393e21131e2120a42cf634416033e587b5dfdccdc84d1a2d2c176b847f6f17f" + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_helpers.py b/vulnerabilities/tests/test_helpers.py index 79e2cb371..78972b25d 100644 --- a/vulnerabilities/tests/test_helpers.py +++ b/vulnerabilities/tests/test_helpers.py @@ -19,7 +19,6 @@ # for any legal advice. # VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -from unittest import result from packageurl import PackageURL @@ -31,6 +30,7 @@ def test_nearest_patched_package(): result = nearest_patched_package( vulnerable_packages=[ + PackageURL(type="npm", name="foo", version="2.0.4"), PackageURL(type="npm", name="foo", version="2.0.0"), PackageURL(type="npm", name="foo", version="2.0.1"), PackageURL(type="npm", name="foo", version="1.9.8"), @@ -66,4 +66,10 @@ def test_nearest_patched_package(): type="npm", namespace=None, name="foo", version="2.0.2", qualifiers={}, subpath=None ), ), + LegacyAffectedPackage( + vulnerable_package=PackageURL( + type="npm", namespace=None, name="foo", version="2.0.4", qualifiers={}, subpath=None + ), + patched_package=None, + ), ] == result diff --git a/vulnerabilities/tests/test_package_managers_2.py b/vulnerabilities/tests/test_package_managers_2.py index 27cb7acb6..d0d569290 100644 --- a/vulnerabilities/tests/test_package_managers_2.py +++ b/vulnerabilities/tests/test_package_managers_2.py @@ -1,6 +1,7 @@ import json import os from datetime import datetime +from unittest import mock import pytest import pytz @@ -8,6 +9,8 @@ from vulnerabilities.package_managers_2 import GoproxyVersionAPI from vulnerabilities.package_managers_2 import LegacyVersion from vulnerabilities.package_managers_2 import NugetVersionAPI +from vulnerabilities.package_managers_2 import PypiVersionAPI +from vulnerabilities.package_managers_2 import RubyVersionAPI BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data", "package_manager_data") @@ -69,3 +72,103 @@ def test_nuget_extract_version(): value="3.5.1", release_date=datetime(2015, 1, 23, 1, 5, 44, 447000, tzinfo=pytz.UTC) ), } + + +def test_nuget_extract_version_with_illformed_data(): + assert NugetVersionAPI.extract_versions({"items": [{"items": [{"catalogEntry": {}}]}]}) == set() + + +@mock.patch("vulnerabilities.package_managers_2.get_response") +def test_pypi_fetch_data(mock_response): + pypi_api = PypiVersionAPI() + with open(os.path.join(TEST_DATA, "pypi.json"), "r") as f: + mock_response.return_value = json.load(f) + pypi_api.fetch("django") + assert pypi_api.cache == { + "django": { + LegacyVersion( + value="1.10.5", + release_date=datetime(2017, 1, 4, 19, 23, 0, 596664, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10.8", + release_date=datetime(2017, 9, 5, 15, 31, 58, 221021, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10rc1", + release_date=datetime(2016, 7, 18, 18, 5, 5, 503584, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10.4", + release_date=datetime(2016, 12, 1, 23, 46, 50, 215935, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10a1", + release_date=datetime(2016, 5, 20, 12, 24, 59, 952686, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10.3", + release_date=datetime(2016, 11, 1, 13, 57, 16, 55061, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10.1", + release_date=datetime(2016, 9, 1, 23, 18, 18, 672706, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10.2", + release_date=datetime(2016, 10, 1, 20, 5, 31, 330942, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10.7", + release_date=datetime(2017, 4, 4, 14, 27, 54, 235551, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10.6", + release_date=datetime(2017, 3, 1, 13, 37, 40, 243134, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.1.4", + release_date=datetime(2011, 2, 9, 4, 13, 7, 75, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10b1", + release_date=datetime(2016, 6, 22, 1, 15, 17, 267637, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.1.3", + release_date=datetime(2010, 12, 23, 5, 14, 23, 509436, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="1.10", + release_date=datetime(2016, 8, 1, 18, 32, 16, 280614, tzinfo=pytz.UTC), + ), + } + } + + +@mock.patch("vulnerabilities.package_managers_2.get_response") +def test_pypi_fetch_with_no_release(mock_response): + pypi_api = PypiVersionAPI() + mock_response.return_value = {"info": {}} + pypi_api.fetch("django") + assert pypi_api.cache == {"django": set()} + + +@mock.patch("vulnerabilities.package_managers_2.get_response") +def test_pypi_fetch_with_no_release(mock_response): + ruby_api = RubyVersionAPI() + with open(os.path.join(TEST_DATA, "gem.json"), "r") as f: + mock_response.return_value = json.load(f) + ruby_api.fetch("rails") + assert ruby_api.cache == { + "rails": { + LegacyVersion( + value="7.0.2.3", + release_date=datetime(2022, 3, 8, 17, 50, 52, 496000, tzinfo=pytz.UTC), + ), + LegacyVersion( + value="7.0.2.2", + release_date=datetime(2022, 2, 11, 19, 44, 19, 17000, tzinfo=pytz.UTC), + ), + } + }