Migrate alpine importer to importer improver model

Current alpine importer models need to be refactored to give AdvisoryData instead of Advisory, also add some validation to parse license expression and make affected_version_range optional

Add tests to test scraping of webpages and parsing of data

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

AUTHORS.rst

@@ -13,3 +13,4 @@ The following organizations or individuals have contributed to this repo:
 - Navonil Das @NavonilDas
 - Tushar Upadhyay @tushar912
 - Hritik Vijay @hritik14
+- Tushar Goel @TG1999

requirements.txt

@@ -8,7 +8,7 @@ django-widget-tweaks>=1.4.8
 packageurl-python>=0.9.4
 binaryornot>=0.4.4
 GitPython>=3.1.17
-univers>=30.0.0
+univers>=30.1.0
 saneyaml>=0.5.2
 beautifulsoup4>=4.9.3
 python-dateutil>=2.8.1
@@ -17,5 +17,6 @@ lxml>=4.6.4
 gunicorn>=20.1.0
 django-environ==0.4.5
 defusedxml==0.7.1
+license-expression>=21.6.14
 
 Markdown==3.3.4

vulnerabilities/importer.py

@@ -38,6 +38,7 @@
 from binaryornot.helpers import is_binary_string
 from git import DiffIndex
 from git import Repo
+from license_expression import Licensing
 from packageurl import PackageURL
 from univers.version_range import VersionRange
 from univers.versions import Version
@@ -117,22 +118,28 @@ class AffectedPackage:
     """
     Contains a range of affected versions and a fixed version of a given package
     The PackageURL supplied must *not* have a version
+    It must contain either `affected_version_range` or `fixed_version`
     """
 
     package: PackageURL
-    affected_version_range: VersionRange
+    affected_version_range: Optional[VersionRange] = None
     fixed_version: Optional[Version] = None
 
     def __post_init__(self):
         if self.package.version:
-            raise ValueError
+            raise ValueError("The PackageURL supplied must not have a version")
+        if not (self.affected_version_range or self.fixed_version):
+            raise ValueError(
+                "Affected Package should at least have either a fixed version or affected version range"
+            )
 
     def get_fixed_purl(self):
         """
         Return PackageURL corresponding to object's fixed_version
         """
-        fixed_version = self.fixed_version
-        fixed_purl = self.package._replace(version=str(fixed_version))
+        if not self.fixed_version:
+            raise ValueError("Affected package should have a fixed version")
+        fixed_purl = self.package._replace(version=str(self.fixed_version))
         return fixed_purl
 
     @classmethod
@@ -152,7 +159,8 @@ def merge(cls, affected_packages: Iterable):
         fixed_versions = set()
         purls = set()
         for pkg in affected_packages:
-            affected_version_ranges.add(pkg.affected_version_range)
+            if pkg.affected_version_range:
+                affected_version_ranges.add(pkg.affected_version_range)
             if pkg.fixed_version:
                 fixed_versions.add(pkg.fixed_version)
             purls.add(pkg.package)
@@ -164,9 +172,12 @@ def to_dict(self):
         """
         Return a serializable dict that can be converted back using self.from_dict
         """
+        affected_version_range = None
+        if self.affected_version_range:
+            affected_version_range = str(self.affected_version_range)
         return {
             "package": self.package.to_dict(),
-            "affected_version_range": str(self.affected_version_range),
+            "affected_version_range": affected_version_range,
             "fixed_version": str(self.fixed_version) if self.fixed_version else None,
         }
 
@@ -176,9 +187,13 @@ def from_dict(cls, affected_pkg: dict):
         Return an AffectedPackage object from dict generated by self.to_dict
         """
         package = PackageURL(**affected_pkg["package"])
-        affected_version_range = VersionRange.from_string(affected_pkg["affected_version_range"])
+        affected_version_range = None
+        if affected_pkg["affected_version_range"]:
+            affected_version_range = VersionRange.from_string(
+                affected_pkg["affected_version_range"]
+            )
         fixed_version = affected_pkg["fixed_version"]
-        if fixed_version:
+        if fixed_version and affected_version_range:
             # TODO: revisit after https://github.com/nexB/univers/issues/10
             fixed_version = affected_version_range.version_class(fixed_version)
 
@@ -203,7 +218,7 @@ class AdvisoryData:
     """
 
     aliases: List[str] = dataclasses.field(default_factory=list)
-    summary: str = None
+    summary: Optional[str] = None
     affected_packages: List[AffectedPackage] = dataclasses.field(default_factory=list)
     references: List[Reference] = dataclasses.field(default_factory=list)
     date_published: Optional[datetime.datetime] = None
@@ -217,17 +232,29 @@ class NoLicenseError(Exception):
     pass
 
 
+class InvalidSPDXLicense(Exception):
+    pass
+
+
 class Importer:
     """
     An Importer collects data from various upstreams and returns corresponding AdvisoryData objects
     in its advisory_data method.  Subclass this class to implement an importer
     """
 
     spdx_license_expression = ""
+    license_url = ""
 
     def __init__(self):
         if not self.spdx_license_expression:
             raise Exception(f"Cannot run importer {self!r} without a license")
+        licensing = Licensing()
+        try:
+            licensing.parse(self.spdx_license_expression)
+        except InvalidSPDXLicense as e:
+            raise ValueError(
+                f"{self.spdx_license_expression!r} is not a valid SPDX license expression"
+            ) from e
 
     @classproperty
     def qualified_name(cls):

vulnerabilities/importers/__init__.py

@@ -19,8 +19,9 @@
 #  for any legal advice.
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+from vulnerabilities.importers import alpine_linux
 from vulnerabilities.importers import nginx
 
-IMPORTERS_REGISTRY = [nginx.NginxImporter]
+IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}