Read PEP 639 License-Expression in PyPI #4431

Wheels and PKG-INFO that only set License-Expression were reduced
to license classifiers, so the declared SPDX expression was ignored.
Prefer License-Expression over the legacy License field when both
are present.

Signed-off-by: Brian Wing <bwing@anaconda.com>

Author: bwingconda

AUTHORS.rst

@@ -18,6 +18,7 @@ The following organizations or individuals have contributed to ScanCode:
 - Ayush @macayu17
 - Ayan Sinha Mahapatra @AyanSinhaMahapatra
 - Ayush Jain @aj4ayushjain
+- Brian Wing @bwingconda
 - Bruno Oliveira @nicoddemus
 - Carmen Bianca Bakker @carmenbianca
 - Chaitya Shah @Chaitya62

CHANGELOG.rst

@@ -4,6 +4,12 @@ Changelog
 Next release
 --------------
 
+- Collect PEP 639 ``License-Expression`` from PyPI METADATA, PKG-INFO and
+  pyproject.toml in ``packagedcode.pypi.get_declared_license``. Wheels that
+  only declare ``License-Expression: MIT`` (with no legacy ``License`` field)
+  were previously reduced to license classifiers, so the expression was
+  ignored. https://github.com/aboutcode-org/scancode-toolkit/issues/4431
+
 - Fix the optional ``licenses`` extra dependency typo to install
   ``licensedcode-data``.
   https://github.com/aboutcode-org/scancode-toolkit/pull/5056

src/packagedcode/pypi.py

@@ -1942,17 +1942,26 @@ def get_declared_license(metainfo):
     """
     Return a mapping of declared license information and license file name
     found in a ``metainfo`` package data mapping.
+
+    Collects PEP 639 ``License-Expression`` (and pyproject.toml
+    ``license-expression``) in addition to the legacy ``License`` field and
+    license classifiers. ``License-Expression`` is preferred when both it and
+    ``License`` are present: they are mutually exclusive in core metadata, and
+    the expression is the machine-readable declared license.
     """
     declared_license = {}
     # TODO: We should make the declared license as it is, this should be
     # updated in scancode to parse a pure string
     lic = get_attribute(metainfo, 'License')
+    license_expression = get_attribute(metainfo, 'License-Expression')
     license_file = get_attribute(metainfo, 'License-File')
     if not license_file and lic:
         if isinstance(lic, dict) and 'file' in lic.keys():
             license_file = lic.pop('file')
 
-    if lic and not lic == 'UNKNOWN':
+    if license_expression and not license_expression == 'UNKNOWN':
+        declared_license['license'] = license_expression
+    elif lic and not lic == 'UNKNOWN':
         if 'text' in lic:
             declared_license['license'] = lic.get('text')
         else:

tests/packagedcode/data/pypi/metadata/v23/PKG-INFO-expected.json

@@ -51,6 +51,28 @@
     "declared_license_expression": "mit",
     "declared_license_expression_spdx": "MIT",
     "license_detections": [
+      {
+        "license_expression": "mit",
+        "license_expression_spdx": "MIT",
+        "matches": [
+          {
+            "license_expression": "mit",
+            "license_expression_spdx": "MIT",
+            "from_file": null,
+            "start_line": 1,
+            "end_line": 1,
+            "matcher": "1-spdx-id",
+            "score": 100.0,
+            "matched_length": 1,
+            "match_coverage": 100.0,
+            "rule_relevance": 100,
+            "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06",
+            "rule_url": null,
+            "matched_text": "MIT"
+          }
+        ],
+        "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf"
+      },
       {
         "license_expression": "mit",
         "license_expression_spdx": "MIT",
@@ -77,7 +99,7 @@
     "other_license_expression": null,
     "other_license_expression_spdx": null,
     "other_license_detections": [],
-    "extracted_license_statement": "classifiers:\n  - 'License :: OSI Approved :: MIT License'\n",
+    "extracted_license_statement": "license: MIT\nclassifiers:\n  - 'License :: OSI Approved :: MIT License'\n",
     "notice_text": null,
     "source_packages": [],
     "file_references": [],

tests/packagedcode/test_pypi.py

@@ -260,7 +260,33 @@ def test_parse_with_unpacked_wheel_meta_v24_1(self):
         test_file = self.get_test_loc('pypi/unpacked_wheel/metadata-2.4/narwhals-1.29.0.dist-info/METADATA')
         package = pypi.PythonInstalledWheelMetadataFile.parse(test_file)
         expected_loc = self.get_test_loc('pypi/unpacked_wheel/metadata-2.4/narwhals-1.29.0.dist-info-expected.json')
-        self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)    
+        self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
+
+
+class TestPypiGetDeclaredLicense(PackageTester):
+    test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
+
+    def test_get_declared_license_from_license_expression(self):
+        declared, license_file = pypi.get_declared_license({
+            'License-Expression': 'MIT',
+        })
+        assert declared == {'license': 'MIT'}
+        assert license_file is None
+
+    def test_get_declared_license_from_pyproject_license_expression(self):
+        declared, license_file = pypi.get_declared_license({
+            'license-expression': 'Apache-2.0 AND MIT',
+        })
+        assert declared == {'license': 'Apache-2.0 AND MIT'}
+        assert license_file is None
+
+    def test_get_declared_license_prefers_license_expression_over_license(self):
+        declared, license_file = pypi.get_declared_license({
+            'License': 'MIT',
+            'License-Expression': 'BSD-3-Clause',
+        })
+        assert declared == {'license': 'BSD-3-Clause'}
+        assert license_file is None
 
 
 class TestPypiUnpackedSdist(PackageTester):