Read package manifests with utf-8-sig so a BOM does not drop the package

CHANGELOG.rst

@@ -4,6 +4,10 @@ Changelog
 Next release
 --------------
 
+- Read package manifests with the ``utf-8-sig`` codec so that a UTF-8 BOM does
+  not make the package disappear from the scan.
+  https://github.com/aboutcode-org/scancode-toolkit/pull/5321
+
 - Fix the optional ``licenses`` extra dependency typo to install
   ``licensedcode-data``.
   https://github.com/aboutcode-org/scancode-toolkit/pull/5056

src/packagedcode/about.py

@@ -52,7 +52,7 @@ def parse(cls, location, package_only=False):
         Yield one or more Package manifest objects given a file ``location`` pointing to a
         package archive, manifest or similar.
         """
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             package_data = saneyaml.load(loc.read())
 
         # About files can contain any purl and also have a namespace

src/packagedcode/bower.py

@@ -26,7 +26,7 @@ class BowerJsonHandler(models.DatafileHandler):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             package_data = json.load(loc)
 
         name = package_data.get('name')

src/packagedcode/chef.py

@@ -188,7 +188,7 @@ def parse(cls, location, package_only=False):
         Yield one or more Package manifest objects given a file ``location``
         pointing to a package archive, manifest or similar.
         """
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             package_data = json.load(loc)
         yield build_package(
             package_data=package_data,
@@ -207,7 +207,7 @@ class ChefMetadataRbHandler(BaseChefMetadataHandler):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             file_contents = loc.read()
 
         # we use a Pygments formatter for parsing lexed Ruby code

src/packagedcode/conan.py

@@ -167,7 +167,7 @@ def _parse(cls, conan_recipe, package_only=False):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding="utf-8") as loc:
+        with io.open(location, encoding="utf-8-sig") as loc:
             conan_recipe = loc.read()
 
         yield cls._parse(conan_recipe, package_only)
@@ -186,7 +186,7 @@ class ConanDataHandler(models.DatafileHandler):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding="utf-8") as loc:
+        with io.open(location, encoding="utf-8-sig") as loc:
             conan_data = loc.read()
 
         conan_data = saneyaml.load(conan_data)

src/packagedcode/conda.py

@@ -322,7 +322,7 @@ class CondaMetaJsonHandler(CondaBaseHandler):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             conda_metadata = json.load(loc)
 
         name = conda_metadata.get('name')
@@ -614,7 +614,7 @@ def get_meta_yaml_data(location):
     # FIXME: use Jinja to process these
     variables = get_variables(location)
     yaml_lines = []
-    with io.open(location, encoding='utf-8') as metayaml:
+    with io.open(location, encoding='utf-8-sig') as metayaml:
         for line in metayaml:
             if not line:
                 continue
@@ -656,7 +656,7 @@ def get_variables(location):
     {% set sha256 = "bc7512f2eef785b037d836f4cc6faded457ac277f75c6e34eccd12da7c85258f" %}
     """
     result = {}
-    with io.open(location, encoding='utf-8') as loc:
+    with io.open(location, encoding='utf-8-sig') as loc:
         for line in loc.readlines():
             if not line:
                 continue

src/packagedcode/cran.py

@@ -115,7 +115,7 @@ def get_cran_description(location):
     Parse a CRAN DESCRIPTION file as YAML and return a mapping of metadata.
     """
     yaml_lines = []
-    with io.open(location, encoding='utf-8') as loc:
+    with io.open(location, encoding='utf-8-sig') as loc:
         for line in loc.readlines():
             if not line:
                 continue

src/packagedcode/freebsd.py

@@ -113,7 +113,7 @@ def parse(cls, location, package_only=False):
         Yield one or more Package manifest objects given a file ``location`` pointing to a
         package archive, manifest or similar.
         """
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             yaml_data = saneyaml.load(loc)
 
         yield cls._parse(yaml_data, package_only)

src/packagedcode/go_mod.py

@@ -114,7 +114,7 @@ def parse_gomod(location):
         >>> assert p.group('ns_name') == ('github.com/davecgh/go-spew')
         >>> assert p.group('version') == ('v1.1.1')
     """
-    with io.open(location, encoding='utf-8', closefd=True) as data:
+    with io.open(location, encoding='utf-8-sig', closefd=True) as data:
         lines = data.readlines()
 
     gomods = GoModule()
@@ -229,7 +229,7 @@ def parse_gosum(location):
         >>> assert p.group('version') == ('v0.3.1')
         >>> assert p.group('checksum') == ('WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=')
     """
-    with io.open(location, encoding='utf-8', closefd=True) as data:
+    with io.open(location, encoding='utf-8-sig', closefd=True) as data:
         lines = data.readlines()
 
     gosums = []

src/packagedcode/godeps.py

@@ -138,7 +138,7 @@ def load(self, location):
         Load self from a location string or a file-like object containing a
         Godeps JSON.
         """
-        with io.open(location, encoding='utf-8') as godep:
+        with io.open(location, encoding='utf-8-sig') as godep:
             text = godep.read()
         return self.loads(text)
 

src/packagedcode/haxe.py

@@ -109,7 +109,7 @@ def parse(cls, location, package_only=False):
             "contributors": ["back2dos", "ncannasse", "jason", "Simn", "nadako", "andyli"]
         }
         """
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             json_data = json.load(loc)
 
         yield cls._parse(json_data, package_only)

src/packagedcode/npm.py

@@ -609,7 +609,7 @@ def _parse(cls, json_data, package_only=False):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             json_data = json.load(loc)
 
         yield cls._parse(json_data, package_only)
@@ -620,7 +620,7 @@ class BaseNpmLockHandler(BaseNpmHandler):
     @classmethod
     def parse(cls, location, package_only=False):
 
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             package_data = json.load(loc)
 
         # we have two formats: v1 and v2
@@ -1004,7 +1004,7 @@ def parse(cls, location, package_only=False):
 
         Yield a single PackageData
         """
-        with io.open(location, encoding='utf-8') as yl:
+        with io.open(location, encoding='utf-8-sig') as yl:
             yl_dependencies = yl.read().split('\n\n')
 
         dependencies_by_purl = {}

src/packagedcode/phpcomposer.py

@@ -66,7 +66,7 @@ def parse(cls, location, package_only=False):
         Note that this is NOT exactly the packagist.json format (all are closely
         related of course but have important (even if minor) differences.
         """
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             package_json = json.load(loc)
 
         yield build_package_data(package_json, package_only)
@@ -175,7 +175,7 @@ class PhpComposerLockHandler(BasePhpComposerHandler):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding='utf-8') as loc:
+        with io.open(location, encoding='utf-8-sig') as loc:
             package_data = json.load(loc)
 
         packages = [

src/packagedcode/readme.py

@@ -56,7 +56,7 @@ class ReadmeHandler(models.NonAssemblableDatafileHandler):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with open(location, encoding='utf-8') as loc:
+        with open(location, encoding='utf-8-sig') as loc:
             readme_manifest = loc.read()
 
         package_data = build_package(readme_manifest, package_only)

src/packagedcode/rpm.py

@@ -410,7 +410,7 @@ class RpmMarinerContainerManifestHandler(models.DatafileHandler):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding='utf-8') as data:
+        with io.open(location, encoding='utf-8-sig') as data:
             lines = data.readlines()
 
         for line in lines:

src/packagedcode/swift.py

@@ -203,7 +203,7 @@ def _parse(cls, swift_dependency_relation, package_only=False):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding="utf-8") as loc:
+        with io.open(location, encoding="utf-8-sig") as loc:
             swift_dependency_relation = json.load(loc)
 
         yield cls._parse(swift_dependency_relation, package_only)
@@ -242,7 +242,7 @@ def _parse(cls, swift_manifest, package_only=False):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding="utf-8") as loc:
+        with io.open(location, encoding="utf-8-sig") as loc:
             swift_manifest = json.load(loc)
 
         yield cls._parse(swift_manifest, package_only)
@@ -261,7 +261,7 @@ class SwiftPackageResolvedHandler(BaseSwiftDatafileHandler):
 
     @classmethod
     def parse(cls, location, package_only=False):
-        with io.open(location, encoding="utf-8") as loc:
+        with io.open(location, encoding="utf-8-sig") as loc:
             package_resolved = json.load(loc)
 
         resolved_doc_version = package_resolved.get("version")

tests/packagedcode/test_utf8_bom.py

@@ -0,0 +1,63 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import pytest
+
+from packagedcode import bower
+from packagedcode import haxe
+from packagedcode import npm
+from packagedcode import phpcomposer
+
+
+BOM = ''
+
+MANIFESTS = [
+    (npm.NpmPackageJsonHandler, 'package.json',
+     '{"name": "demo", "version": "1.0.0", "license": "MIT"}'),
+    (bower.BowerJsonHandler, 'bower.json',
+     '{"name": "demo", "version": "1.0.0", "license": "MIT"}'),
+    (phpcomposer.PhpComposerJsonHandler, 'composer.json',
+     '{"name": "acme/demo", "version": "1.0.0", "license": "MIT"}'),
+    (haxe.HaxelibJsonHandler, 'haxelib.json',
+     '{"name": "demo", "version": "1.0.0", "license": "MIT"}'),
+]
+
+
+@pytest.mark.parametrize(
+    'handler, filename, content',
+    MANIFESTS,
+    ids=[filename for _, filename, _ in MANIFESTS],
+)
+def test_manifest_with_a_utf8_bom_is_parsed(handler, filename, content, tmp_path):
+    """A BOM is legal in a package manifest and package managers accept one.
+
+    Python's utf-8 codec does not consume it, it decodes the three bytes to
+    U+FEFF and json.load then raises, so the whole package used to disappear
+    from the scan.
+    """
+    location = tmp_path / filename
+    location.write_text(BOM + content, encoding='utf-8')
+
+    packages = list(handler.parse(location=str(location)))
+
+    assert [(p.name, p.version) for p in packages] == [('demo', '1.0.0')]
+
+
+@pytest.mark.parametrize(
+    'handler, filename, content',
+    MANIFESTS,
+    ids=[filename for _, filename, _ in MANIFESTS],
+)
+def test_manifest_without_a_bom_is_unaffected(handler, filename, content, tmp_path):
+    location = tmp_path / filename
+    location.write_text(content, encoding='utf-8')
+
+    packages = list(handler.parse(location=str(location)))
+
+    assert [(p.name, p.version) for p in packages] == [('demo', '1.0.0')]