Fix API

CHANGELOG.rst

@@ -1,6 +1,39 @@
 Release notes
 =============
 
+Version v34.0.0rc4
+-------------------
+
+- Drop migration for removing duplicated changelogs.
+
+
+Version v34.0.0rc3
+-------------------
+
+- Add resource URL to the vulnerability and package details view in the API serializers (#1423)
+- Add support for all osv ecosystems (#926)
+- Add RubyImporter to git_importer test_git_importer_clone (#799)
+- Remove duplicated changelogs (#1400)
+- Fix Encoding Type in Fireeye Importer (#1404)
+- Add license_url for GitHub Importer (#1392)
+- Add support for CVSS vectors display (#1312)
+
+
+Version v34.0.0rc2
+-------------------
+
+- We updated package-url models, WARNING: in next major version of 
+  vulnerablecode i.e v35.0.0 qualifiers will be of type ``string`` and not ``dict``.
+- We added changelog and dates on packages and vulnerabilities.
+- We fixed table borders in Vulnerability details UI #1356 (#1358)
+- We added robots.txt in views.
+- We fixed import runner's process_inferences (#1360)
+- We fixed debian OVAL importer (#1361)
+- We added graph model diagrams #977(#1350)
+- We added endpoint for purl lookup (#1359)
+- We fixed swagger API docs generation (#1366)
+- Fix issues https://github.com/nexB/vulnerablecode/issues/1385, https://github.com/nexB/vulnerablecode/issues/1387
+
 
 Version v34.0.0rc1
 -------------------

requirements.txt

@@ -16,7 +16,7 @@ cffi==1.15.0
 chardet==4.0.0
 charset-normalizer==2.0.12
 click==8.1.2
-cryptography==41.0.6
+cryptography==42.0.4
 decorator==5.1.1
 defusedxml==0.7.1
 distro==1.7.0
@@ -35,7 +35,7 @@ executing==0.8.3
 freezegun==1.2.1
 frozenlist==1.3.0
 gitdb==4.0.9
-GitPython==3.1.37
+GitPython==3.1.41
 gunicorn==20.1.0
 idna==3.3
 imagesize==1.3.0
@@ -44,7 +44,7 @@ iniconfig==1.1.1
 ipython==8.10.0
 isort==5.10.1
 jedi==0.18.1
-Jinja2==3.1.1
+Jinja2==3.1.3
 jsonschema==3.2.0
 license-expression==21.6.14
 lxml==4.9.1
@@ -106,7 +106,7 @@ toml==0.10.2
 tomli==2.0.1
 traitlets==5.1.1
 typing_extensions==4.1.1
-univers==30.10.0
+univers==30.11.0
 urllib3==1.26.18
 wcwidth==0.2.5
 websocket-client==0.59.0

setup.cfg

@@ -71,7 +71,7 @@ install_requires =
 
     #essentials
     packageurl-python>=0.10.5rc1
-    univers>=30.10.0
+    univers>=30.11.0
     license-expression>=21.6.14
 
     # file and data formats

vulnerabilities/api.py

@@ -20,6 +20,7 @@
 from rest_framework import viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
+from rest_framework.reverse import reverse
 from rest_framework.throttling import AnonRateThrottle
 from rest_framework.throttling import UserRateThrottle
 
@@ -48,7 +49,41 @@ class Meta:
         fields = ["reference_url", "reference_id", "scores", "url"]
 
 
-class MinimalPackageSerializer(serializers.HyperlinkedModelSerializer):
+class BaseResourceSerializer(serializers.HyperlinkedModelSerializer):
+    """
+    Base serializer containing common methods.
+    """
+
+    def get_fields(self):
+        fields = super().get_fields()
+        fields["resource_url"] = serializers.SerializerMethodField(method_name="get_resource_url")
+        return fields
+
+    def get_resource_url(self, instance):
+        """
+        Return the instance fully qualified URL including the schema and domain.
+
+        Usage:
+            resource_url = serializers.SerializerMethodField()
+        """
+        resource_url = instance.get_absolute_url()
+
+        if request := self.context.get("request", None):
+            return request.build_absolute_uri(location=resource_url)
+
+        return resource_url
+
+
+class PurlPackageSerializer(BaseResourceSerializer):
+
+    purl = serializers.CharField(source="package_url")
+
+    class Meta:
+        model = Package
+        fields = ["url", "purl", "is_vulnerable"]
+
+
+class MinimalPackageSerializer(PurlPackageSerializer):
     """
     Used for nesting inside vulnerability focused APIs.
     """
@@ -72,14 +107,12 @@ def get_vulnerability(self, vuln):
 
     affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities")
 
-    purl = serializers.CharField(source="package_url")
-
     class Meta:
         model = Package
         fields = ["url", "purl", "is_vulnerable", "affected_by_vulnerabilities"]
 
 
-class MinimalVulnerabilitySerializer(serializers.HyperlinkedModelSerializer):
+class MinimalVulnerabilitySerializer(BaseResourceSerializer):
     """
     Lookup vulnerabilities by aliases (such as a CVE).
     """
@@ -99,7 +132,7 @@ class Meta:
         fields = ["alias"]
 
 
-class VulnSerializerRefsAndSummary(serializers.HyperlinkedModelSerializer):
+class VulnSerializerRefsAndSummary(BaseResourceSerializer):
     """
     Lookup vulnerabilities references by aliases (such as a CVE).
     """
@@ -110,7 +143,7 @@ def to_representation(self, instance):
         data["aliases"] = aliases
         return data
 
-    fixed_packages = MinimalPackageSerializer(
+    fixed_packages = PurlPackageSerializer(
         many=True, source="filtered_fixed_packages", read_only=True
     )
 
@@ -141,24 +174,23 @@ def to_representation(self, instance):
         return representation
 
 
-class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer):
-    fixed_packages = MinimalPackageSerializer(
+class VulnerabilitySerializer(BaseResourceSerializer):
+    fixed_packages = PurlPackageSerializer(
         many=True, source="filtered_fixed_packages", read_only=True
     )
-    affected_packages = MinimalPackageSerializer(many=True, read_only=True)
+    affected_packages = PurlPackageSerializer(many=True, read_only=True)
 
     references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set")
     aliases = AliasSerializer(many=True, source="alias")
     weaknesses = WeaknessSerializer(many=True)
 
     def to_representation(self, instance):
-        representation = super().to_representation(instance)
+        data = super().to_representation(instance)
 
-        # Exclude None values from the weaknesses list
-        weaknesses = representation.get("weaknesses", [])
-        representation["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None]
+        weaknesses = data.get("weaknesses", [])
+        data["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None]
 
-        return representation
+        return data
 
     class Meta:
         model = Vulnerability
@@ -174,14 +206,15 @@ class Meta:
         ]
 
 
-class PackageSerializer(serializers.HyperlinkedModelSerializer):
+class PackageSerializer(BaseResourceSerializer):
     """
     Lookup software package using Package URLs
     """
 
     def to_representation(self, instance):
         data = super().to_representation(instance)
         data["qualifiers"] = normalize_qualifiers(data["qualifiers"], encode=False)
+
         return data
 
     next_non_vulnerable_version = serializers.SerializerMethodField("get_next_non_vulnerable")

vulnerabilities/importers/__init__.py

@@ -18,6 +18,7 @@
 from vulnerabilities.importers import fireeye
 from vulnerabilities.importers import gentoo
 from vulnerabilities.importers import github
+from vulnerabilities.importers import github_osv
 from vulnerabilities.importers import gitlab
 from vulnerabilities.importers import istio
 from vulnerabilities.importers import mozilla
@@ -32,6 +33,7 @@
 from vulnerabilities.importers import pysec
 from vulnerabilities.importers import redhat
 from vulnerabilities.importers import retiredotnet
+from vulnerabilities.importers import ruby
 from vulnerabilities.importers import suse_scores
 from vulnerabilities.importers import ubuntu
 from vulnerabilities.importers import ubuntu_usn
@@ -67,6 +69,8 @@
     fireeye.FireyeImporter,
     apache_kafka.ApacheKafkaImporter,
     oss_fuzz.OSSFuzzImporter,
+    ruby.RubyImporter,
+    github_osv.GithubOSVImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/fireeye.py

@@ -44,7 +44,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                 if Path(file).stem == "README":
                     continue
                 try:
-                    with open(file) as f:
+                    with open(file, encoding="utf-8-sig") as f:
                         yield parse_advisory_data(raw_data=f.read(), file=file, base_path=base_path)
                 except UnicodeError:
                     logger.error(f"Invalid file {file}")

vulnerabilities/importers/github.py

@@ -92,6 +92,7 @@
 class GitHubAPIImporter(Importer):
     spdx_license_expression = "CC-BY-4.0"
     importer_name = "GHSA Importer"
+    license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
 
     def advisory_data(self) -> Iterable[AdvisoryData]:
         for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items():

vulnerabilities/importers/github_osv.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+import logging
+from pathlib import Path
+from typing import Iterable
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import Importer
+from vulnerabilities.importers.osv import parse_advisory_data
+from vulnerabilities.utils import get_advisory_url
+
+logger = logging.getLogger(__name__)
+
+
+class GithubOSVImporter(Importer):
+    license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
+    spdx_license_expression = "CC-BY-4.0"
+    repo_url = "git+https://github.com/github/advisory-database/"
+    importer_name = "GithubOSV Importer"
+
+    def advisory_data(self) -> Iterable[AdvisoryData]:
+        supported_ecosystems = [
+            "pypi",
+            "npm",
+            "maven",
+            "golang",
+            "composer",
+            "hex",
+            "gem",
+            "nuget",
+            "cargo",
+        ]
+        try:
+            self.clone(repo_url=self.repo_url)
+            base_path = Path(self.vcs_response.dest_dir)
+            # filter out non-github-reviewed files and only keep the files end-with .json
+            advisory_dirs = base_path / "advisories/github-reviewed"
+            for file in advisory_dirs.glob("**/*.json"):
+                advisory_url = get_advisory_url(
+                    file=file,
+                    base_path=base_path,
+                    url="https://github.com/github/advisory-database/blob/main/",
+                )
+                with open(file) as f:
+                    raw_data = json.load(f)
+                yield parse_advisory_data(raw_data, supported_ecosystems, advisory_url)
+        finally:
+            if self.vcs_response:
+                self.vcs_response.delete()

vulnerabilities/importers/oss_fuzz.py

@@ -40,7 +40,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                         url="https://github.com/pypa/advisory-database/blob/main/",
                     )
                     yield parse_advisory_data(
-                        yaml_data, supported_ecosystem="oss-fuzz", advisory_url=advisory_url
+                        yaml_data, supported_ecosystems=["oss-fuzz"], advisory_url=advisory_url
                     )
         finally:
             if self.vcs_response: