Merge pull request #582 from jkloetzke/modernize

Update Python version support and Github workflow

.gitattributes

@@ -0,0 +1 @@
+*.svg binary

.github/workflows/workflow.yaml

@@ -4,31 +4,33 @@ on: [push, pull_request]
 
 jobs:
     build-linux:
-        runs-on: ubuntu-20.04
+        runs-on: ubuntu-22.04
         timeout-minutes: 15
         strategy:
             matrix:
-                python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+                python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
             fail-fast: false
 
         steps:
             - name: Checkout
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
 
             - name: Set up Python ${{ matrix.python-version }}
-              uses: actions/setup-python@v2
+              uses: actions/setup-python@v5
               with:
                 python-version: ${{ matrix.python-version }}
+                cache: pip
 
             - name: Cache Jenkins integration test
-              uses: actions/cache@v3
+              uses: actions/cache@v4
               with:
                 path: test/integration/jenkins/cache
                 key: ${{ runner.os }}
 
             - name: Install dependencies
               run: |
                 python -m pip install --upgrade pip
+                python -m pip install --upgrade setuptools
                 pip install PyYAML coverage schema python-magic pyparsing sphinx wheel
                 sudo apt-get update
                 sudo apt-get install cvs
@@ -44,36 +46,35 @@ jobs:
                 python3 setup.py bdist_wheel --plat-name manylinux1_x86_64
 
             - name: Upload coverage to Codecov
-              # Coverage is not complete on Python <3.7 (see pym/bob/utils.py)
-              if: matrix.python-version != '3.6'
               uses: codecov/codecov-action@v4
               with:
                 token: ${{ secrets.CODECOV_TOKEN }}
 
             - name: Store the binary wheel
-              uses: actions/upload-artifact@v3
+              uses: actions/upload-artifact@v4
               with:
-                name: python-package-distributions
+                name: python-dist-linux-${{ matrix.python-version }}
                 path: dist
 
     build-windows:
         runs-on: windows-latest
         timeout-minutes: 20
         steps:
             - name: Checkout
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
 
-            - name: Set up Python 3.11
-              uses: actions/setup-python@v2
+            - name: Set up Python 3.12
+              uses: actions/setup-python@v5
               with:
-                python-version: "3.11"
+                python-version: "3.12"
+                cache: pip
 
             - name: Prepare MSYS2 environment
               run: |
                 C:\msys64\usr\bin\bash -l -c "pacman -Sy --needed --noconfirm parallel || true"
 
             - name: Cache Jenkins integration test
-              uses: actions/cache@v3
+              uses: actions/cache@v4
               with:
                 path: test/integration/jenkins/cache
                 key: ${{ runner.os }}
@@ -101,26 +102,28 @@ jobs:
         timeout-minutes: 5
         steps:
             - name: Checkout
-              uses: actions/checkout@v2
+              uses: actions/checkout@v4
 
-            - name: Set up Python 3.11
-              uses: actions/setup-python@v2
+            - name: Set up Python 3.12
+              uses: actions/setup-python@v5
               with:
-                python-version: "3.11"
+                python-version: "3.12"
+                cache: pip
 
             - name: Install dependencies
               run: |
                 python -m pip install --upgrade pip
+                python -m pip install --upgrade setuptools
                 pip install PyYAML schema python-magic pyparsing sphinx wheel
 
             - name: Build Python package
               run: |
                 python3 setup.py sdist
 
             - name: Store the source distribution
-              uses: actions/upload-artifact@v3
+              uses: actions/upload-artifact@v4
               with:
-                name: python-package-distributions
+                name: python-dist-sdist
                 path: dist
 
     publish:
@@ -134,9 +137,10 @@ jobs:
             - build-sdist
         steps:
             - name: Download all the dists
-              uses: actions/download-artifact@v3
+              uses: actions/download-artifact@v4
               with:
-                name: python-package-distributions
+                pattern: python-dist-*
+                merge-multiple: true
                 path: dist/
 
             - name: List

doc/installation.rst

@@ -4,7 +4,7 @@ Installation
 Dependencies
 ============
 
-Bob is built with Python3 (>=3.6). Some additional Python packages are
+Bob is built with Python3 (>=3.7). Some additional Python packages are
 required. They are installed automatically as dependencies.
 
 Apart from the Python dependencies additional run time dependencies could arise,

pym/bob/archive.py

@@ -24,7 +24,7 @@
 from .tty import stepAction, stepMessage, \
     SKIPPED, EXECUTED, WARNING, INFO, TRACE, ERROR, IMPORTANT
 from .utils import asHexStr, removePath, isWindows, sslNoVerifyContext, \
-    getBashPath
+    getBashPath, tarfileOpen
 from shlex import quote
 from tempfile import mkstemp, NamedTemporaryFile, TemporaryFile, gettempdir
 import argparse
@@ -38,6 +38,7 @@
 import io
 import os
 import os.path
+import shutil
 import signal
 import ssl
 import subprocess
@@ -145,16 +146,17 @@ def __extractPackage(self, tar, audit, content):
                     raise BuildError("File name encoding error while extracting '{}'".format(f.name),
                                      help="Your locale(7) probably does not (fully) support unicode.")
             elif f.name == "meta/audit.json.gz":
-                f.name = audit
-                tar.extract(f)
+                with tar.extractfile(f) as audit_src:
+                    with open(audit, 'wb') as audit_dst:
+                        shutil.copyfileobj(audit_src, audit_dst)
             elif f.name == "content" or f.name == "meta":
                 pass
             else:
                 raise BuildError("Binary artifact contained unknown file: " + f.name)
             f = tar.next()
 
     def _extract(self, fileobj, audit, content):
-        with tarfile.open(None, "r|*", fileobj=fileobj, errorlevel=1) as tar:
+        with tarfileOpen(None, "r|*", fileobj=fileobj, errorlevel=1) as tar:
             removePath(audit)
             removePath(content)
             os.makedirs(content)
@@ -163,8 +165,8 @@ def _extract(self, fileobj, audit, content):
     def _pack(self, name, fileobj, audit, content):
         pax = { 'bob-archive-vsn' : "1" }
         with gzip.open(name or fileobj, 'wb', 6) as gzf:
-            with tarfile.open(name, "w", fileobj=gzf,
-                              format=tarfile.PAX_FORMAT, pax_headers=pax) as tar:
+            with tarfileOpen(name, "w", fileobj=gzf,
+                             format=tarfile.PAX_FORMAT, pax_headers=pax) as tar:
                 tar.add(audit, "meta/" + os.path.basename(audit))
                 tar.add(content, arcname="content")
 

pym/bob/builder.py

@@ -1234,7 +1234,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                 # re-runs with "build-only" the dependent steps should
                 # trigger.
                 if BobState().getResultHash(prettySrcPath) is not None:
-                    oldCheckoutHash = datetime.datetime.utcnow()
+                    oldCheckoutHash = datetime.datetime.now()
                     BobState().setResultHash(prettySrcPath, oldCheckoutHash)
 
                 with stepExec(checkoutStep, "CHECKOUT",
@@ -1327,7 +1327,7 @@ async def _cookBuildStep(self, buildStep, depth, buildBuildId):
                 # content. If the execution fails we have nothing reliable
                 # left and we _must_ run it again.
                 BobState().delInputHashes(prettyBuildPath)
-                BobState().setResultHash(prettyBuildPath, datetime.datetime.utcnow())
+                BobState().setResultHash(prettyBuildPath, datetime.datetime.now())
                 # build it
                 with stepExec(buildStep, "BUILD", prettyBuildPath, INFO) as buildAction:
                     visibleAction = fullAction if fullAction.visible else buildAction
@@ -1578,7 +1578,7 @@ async def _cookPackageStep(self, packageStep, depth, packageBuildId):
             with stepExec(packageStep, "PACKAGE", prettyPackagePath, (ALWAYS, NORMAL)) as fullAction:
                 # invalidate result because folder will be cleared
                 BobState().delInputHashes(prettyPackagePath)
-                BobState().setResultHash(prettyPackagePath, datetime.datetime.utcnow())
+                BobState().setResultHash(prettyPackagePath, datetime.datetime.now())
                 with stepExec(packageStep, "PACKAGE", prettyPackagePath, INFO) as packageAction:
                     visibleAction = fullAction if fullAction.visible else packageAction
                     await self._runShell(packageStep, "package", visibleAction)

pym/bob/cmds/archive.py

@@ -5,7 +5,7 @@
 
 from ..audit import Audit
 from ..errors import BobError
-from ..utils import binStat, asHexStr, infixBinaryOp
+from ..utils import binStat, asHexStr, infixBinaryOp, tarfileOpen
 import argparse
 import gzip
 import json
@@ -118,7 +118,7 @@ def __scan(self, fileName, verbose):
 
             # read audit trail
             if verbose: print("scan", fileName)
-            with tarfile.open(fileName, errorlevel=1) as tar:
+            with tarfileOpen(fileName, errorlevel=1) as tar:
                 # validate
                 if tar.pax_headers.get('bob-archive-vsn') != "1":
                     print("Not a Bob archive:", fileName, "Ignored!")

pym/bob/develop/version.py

@@ -9,8 +9,8 @@
 if sys.version_info.major != 3:
     print("Bob requires Python 3")
     sys.exit(1)
-elif sys.version_info.minor < 6:
-    print("Bob requires at least Python 3.6")
+elif sys.version_info.minor < 7:
+    print("Bob requires at least Python 3.7")
     sys.exit(1)
 
 def getVersion():

pym/bob/scm/imp.py

@@ -6,15 +6,14 @@
 from ..errors import BuildError
 from ..stringparser import IfExpression
 from ..tty import stepAction, INFO, TRACE
-from ..utils import asHexStr, hashDirectory, emptyDirectory
+from ..utils import asHexStr, hashDirectory, emptyDirectory, tarfileOpen
 from .scm import Scm, ScmAudit
 import base64
 import io
 import os, os.path
 import schema
 import shutil
 import stat
-import tarfile
 
 def copyTree(src, dst, invoker):
     """Recursively copy directory tree.
@@ -84,7 +83,7 @@ def packTree(src):
 
     try:
         f = io.BytesIO()
-        with tarfile.open(fileobj=f, mode="w:xz") as tar:
+        with tarfileOpen(fileobj=f, mode="w:xz") as tar:
             # Special handling for MSYS. Symlinks fail if the target does not
             # exist and Python will fall back to create a copy on extraction.
             # To prevent this first everything *except* symlinks is archived
@@ -107,7 +106,7 @@ def filterSymlinks(ti):
 def unpackTree(data, dest):
     try:
         f = io.BytesIO(base64.b85decode(data))
-        with tarfile.open(fileobj=f, mode="r:xz") as tar:
+        with tarfileOpen(fileobj=f, mode="r:xz") as tar:
             tar.extractall(dest)
     except OSError as e:
         raise BuildError("Error unpacking files: {}".format(str(e)))

pym/bob/utils.py

@@ -17,6 +17,10 @@
 import sys
 import sysconfig
 
+# The stat.st_ino field is 128 bit since Python 3.12 on Windows...
+def maskIno(ino):
+    return (ino ^ (ino >> 64)) & 0xFFFFFFFFFFFFFFFF
+
 def hashString(string):
     h = hashlib.md5()
     h.update(string.encode("utf8"))
@@ -341,8 +345,8 @@ class DirHasher:
     ])
 
     class FileIndex:
-        SIGNATURE        = b'BOB1'
-        CACHE_ENTRY_FMT  = '=qqLQLQ20sH'
+        SIGNATURE        = b'BOB2'
+        CACHE_ENTRY_FMT  = '=qqQQLQ20sH'
         CACHE_ENTRY_SIZE = struct.calcsize(CACHE_ENTRY_FMT)
 
         class Stat:
@@ -422,7 +426,7 @@ def __writeEntry(self, name, st, digest):
                 else:
                     self.__outFile.write(DirHasher.FileIndex.SIGNATURE)
             self.__outFile.write(struct.pack(DirHasher.FileIndex.CACHE_ENTRY_FMT, st.st_ctime_ns,
-                st.st_mtime_ns, st.st_dev, st.st_ino, st.st_mode, st.st_size,
+                st.st_mtime_ns, st.st_dev, maskIno(st.st_ino), st.st_mode, st.st_size,
                 digest, len(name)))
             self.__outFile.write(name)
 
@@ -432,7 +436,7 @@ def __match(self, name, st):
             e = self.__current
             res = ((e.name == name) and (e.ctime == st.st_ctime_ns) and
                 (e.mtime == st.st_mtime_ns) and (e.dev == st.st_dev) and
-                (e.ino == st.st_ino) and (e.mode == st.st_mode) and
+                (e.ino == maskIno(st.st_ino)) and (e.mode == st.st_mode) and
                 (e.size == st.st_size))
             #if not res: print("Mismatch", e.name, name, e, st)
             return res
@@ -578,8 +582,8 @@ def hashPath(path, index=None, ignoreDirs=None):
 
 def binStat(path):
     st = os.stat(path)
-    return struct.pack('=qqLQLQ', st.st_ctime_ns, st.st_mtime_ns,
-                       st.st_dev, st.st_ino, st.st_mode, st.st_size)
+    return struct.pack('=qqQQLQ', st.st_ctime_ns, st.st_mtime_ns,
+                       st.st_dev, maskIno(st.st_ino), st.st_mode, st.st_size)
 
 
 # There are two "magic" modules with similar functionality. Find out which one we got and adapt.
@@ -809,3 +813,32 @@ def sslNoVerifyContext():
     context.check_hostname = False
     context.verify_mode = ssl.CERT_NONE
     return context
+
+# Python 3.12 tarfile compliance
+
+def _tarExtractFilter(member, path):
+    path = os.path.realpath(path)
+    name = member.name
+
+    # Strip absolute path names (GNU tar default)
+    if name.startswith(('/', os.sep)):
+        name = name.lstrip('/' + os.sep)
+        member = member.replace(name=name, deep=False)
+
+    # If still absolute (e.g. C:/...), bail out.
+    if os.path.isabs(name):
+        raise BuildError(f"Refusing to extract absolute path '{name}' from tar file.")
+
+    # Ensure we stay in the destination
+    full_name = os.path.realpath(os.path.join(path, name))
+    if os.path.commonpath([full_name, path]) != path:
+        raise BuildError(f"Refusing to extract '{name}' from tar file. File is outside of destination directory.")
+
+    return member
+
+def tarfileOpen(*args, **kwargs):
+    import tarfile
+    ret = tarfile.open(*args, **kwargs)
+    # Set a sensible extraction filter, used by Python 3.12 and later...
+    ret.extraction_filter = _tarExtractFilter
+    return ret

setup.py

@@ -129,7 +129,7 @@ class build(build_orig):
     cmdclass = cmdclass,
 
     # Our runtime dependencies
-    python_requires = '>=3.6',
+    python_requires = '>=3.7',
     install_requires = [
         'PyYAML',
         'schema',
@@ -144,8 +144,7 @@ class build(build_orig):
 
     # Installation time dependencies only needed by setup.py
     setup_requires = [
-        'setuptools_scm<7',   # automatically get package version
-                              # TODO: remove constraint when Python 3.7 is required
+        'setuptools_scm',   # automatically get package version
     ],
 
     # Provide executables

test/unit/mocks/jenkins_tests.py

@@ -21,6 +21,8 @@ def __init__(self, tgz):
     def __enter__(self):
         self.tmpdir = tempfile.TemporaryDirectory()
         with tarfile.open(self.tgz, "r:gz", errorlevel=1) as tar:
+            # Python >= 3.12 compatibility
+            tar.extraction_filter = (lambda member, path: member)
             f = tar.next()
             while f is not None:
                 if f.name.startswith("content/"):