From 94c7524ef4367c29716f2c5539127eb7e797d385 Mon Sep 17 00:00:00 2001 From: Nicolas Dumazet Date: Sat, 29 Aug 2026 16:47:52 +0200 Subject: [PATCH 1/4] fix(venv_shebang_rewriter): use a py_binary instead of a shell/coreutils script venv_shebang_rewriter.sh ran directly (ctx.actions.run) with no declared PATH, and used head/tail/chmod resolved from PATH. On systems without an FHS-style /bin:/usr/bin (e.g. NixOS), this failed with 'head: command not found'. Port it to a plain Python script and expose it as a py_binary, matching this project's own documented guidance (PyExecToolsInfo's exec_interpreter docs: 'prefer to define a py_binary instead ... this makes it much easier to setup the runtime environment'). This needs no new external dependency: the interpreter comes from this project's own Python toolchain, already used everywhere else in this repo. --- python/private/pypi/BUILD.bazel | 19 ++++++++-- python/private/pypi/venv_shebang_rewriter.py | 37 ++++++++++++++++++++ python/private/pypi/venv_shebang_rewriter.sh | 27 -------------- 3 files changed, 54 insertions(+), 29 deletions(-) create mode 100644 python/private/pypi/venv_shebang_rewriter.py delete mode 100755 python/private/pypi/venv_shebang_rewriter.sh diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 38ecabc50e..35286c5deb 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -13,6 +13,7 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//python:py_binary.bzl", "py_binary") package(default_visibility = ["//:__subpackages__"]) @@ -32,20 +33,34 @@ alias( visibility = ["//visibility:public"], ) +py_binary( + name = "venv_shebang_rewriter_py", + srcs = ["venv_shebang_rewriter.py"], + legacy_create_init = False, + main = "venv_shebang_rewriter.py", +) + alias( name = "venv_shebang_rewriter", actual = select({ "@platforms//os:windows": "venv_shebang_rewriter.ps1", - "//conditions:default": "venv_shebang_rewriter.sh", + "//conditions:default": "venv_shebang_rewriter_py", }), visibility = ["//visibility:public"], ) +py_binary( + name = "wheel_record_rewriter_py", + srcs = ["wheel_record_rewriter.py"], + legacy_create_init = False, + main = "wheel_record_rewriter.py", +) + alias( name = "wheel_record_rewriter", actual = select({ "@platforms//os:windows": "wheel_record_rewriter.ps1", - "//conditions:default": "wheel_record_rewriter.sh", + "//conditions:default": "wheel_record_rewriter_py", }), visibility = ["//visibility:public"], ) diff --git a/python/private/pypi/venv_shebang_rewriter.py b/python/private/pypi/venv_shebang_rewriter.py new file mode 100644 index 0000000000..ab921074a2 --- /dev/null +++ b/python/private/pypi/venv_shebang_rewriter.py @@ -0,0 +1,37 @@ +"""Rewrites a console_script wrapper's shebang into a batch/shell-Python polyglot.""" + +import os +import sys + + +def main(argv): + in_path, out_path, target_os = argv[1:4] + + with open(in_path, "rb") as in_file: + first_line = in_file.readline() + rest = in_file.read() + + with open(out_path, "wb") as out_file: + if target_os == "windows": + python_exe = b"pythonw.exe" if first_line.startswith(b"#!pythonw") else b"python.exe" + # A Batch-Python polyglot. Batch executes the first line and exits, + # while Python (via -x) ignores the first line and executes the rest. + out_file.write( + b'@setlocal enabledelayedexpansion & "%~dp0' + python_exe + + b'" -x "%~f0" %* & exit /b !ERRORLEVEL!\r\n', + ) + else: + out_file.write(b"#!/bin/sh\n") + # A Shell-Python polyglot. The shell executes the triple-quoted 'exec' + # command, re-running the script with python3 from the scripts directory. + # Python ignores the triple-quoted string and continues. + out_file.write(b"'''exec' \"$(dirname \"$0\")/python3\" \"$0\" \"$@\"\n' '''\n") + + out_file.write(rest) + + mode = os.stat(out_path).st_mode + os.chmod(out_path, mode | 0o111) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/python/private/pypi/venv_shebang_rewriter.sh b/python/private/pypi/venv_shebang_rewriter.sh deleted file mode 100755 index d4391d3352..0000000000 --- a/python/private/pypi/venv_shebang_rewriter.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -set -eu - -IN="$1" -OUT="$2" -TARGET_OS="$3" - -FIRST_LINE=$(head -n 1 "$IN") - -if [ "$TARGET_OS" = "windows" ]; then - case "$FIRST_LINE" in - "#!pythonw"*) PYTHON_EXE="pythonw.exe" ;; - *) PYTHON_EXE="python.exe" ;; - esac - # A Batch-Python polyglot. Batch executes the first line and exits, - # while Python (via -x) ignores the first line and executes the rest. - printf "@setlocal enabledelayedexpansion & \"%%~dp0$PYTHON_EXE\" -x \"%%~f0\" %%* & exit /b !ERRORLEVEL!\r\n" > "$OUT" -else - printf "#!/bin/sh\n" > "$OUT" - # A Shell-Python polyglot. The shell executes the triple-quoted 'exec' - # command, re-running the script with python3 from the scripts directory. - # Python ignores the triple-quoted string and continues. - printf "'''exec' \"\$(dirname \"\$0\")/python3\" \"\$0\" \"\$@\"\n' '''\n" >> "$OUT" -fi - -tail -n +2 "$IN" >> "$OUT" -chmod +x "$OUT" From 8994e234973c5316f3f9262f758df5cb7f817733 Mon Sep 17 00:00:00 2001 From: Nicolas Dumazet Date: Sat, 29 Aug 2026 16:48:02 +0200 Subject: [PATCH 2/4] fix(wheel_record_rewriter): use a py_binary instead of an awk script wheel_record_rewriter.sh ran directly (ctx.actions.run) with no declared PATH, and used awk resolved from PATH. On systems without an FHS-style /bin:/usr/bin (e.g. NixOS), this failed with 'awk: command not found'. Rather than translate the awk program into POSIX sh, port it to a plain Python script and expose it as a py_binary, same rationale and approach as venv_shebang_rewriter.py. --- python/private/pypi/wheel_record_rewriter.py | 69 ++++++++++++++++ python/private/pypi/wheel_record_rewriter.sh | 85 -------------------- 2 files changed, 69 insertions(+), 85 deletions(-) create mode 100644 python/private/pypi/wheel_record_rewriter.py delete mode 100755 python/private/pypi/wheel_record_rewriter.sh diff --git a/python/private/pypi/wheel_record_rewriter.py b/python/private/pypi/wheel_record_rewriter.py new file mode 100644 index 0000000000..4d8c66bd3c --- /dev/null +++ b/python/private/pypi/wheel_record_rewriter.py @@ -0,0 +1,69 @@ +"""Rewrites a wheel's RECORD file to reflect its final installed layout.""" + +import sys + + +def _rewrite(in_path, out_path, target_os, data_dir_basename, rewritten_scripts): + data_prefix = data_dir_basename + "/" + quoted_data_prefix = '"' + data_prefix + + if target_os == "windows": + data_repl = "../../" + headers_repl = "../../Include/" + platlib_repl = "" + purelib_repl = "" + scripts_repl = "../../Scripts/" + else: + data_repl = "../../../" + headers_repl = "../../../include/" + platlib_repl = "" + purelib_repl = "" + scripts_repl = "../../../bin/" + + with open(in_path) as in_file, open(out_path, "w") as out_file: + for raw_line in in_file: + line = raw_line.rstrip("\n") + + if line.startswith(quoted_data_prefix): + quote = '"' + rest = line[len(quoted_data_prefix):] + elif line.startswith(data_prefix): + quote = "" + rest = line[len(data_prefix):] + else: + out_file.write(line + "\n") + continue + + if rest.startswith("purelib/"): + out_file.write(quote + purelib_repl + rest[len("purelib/"):] + "\n") + elif rest.startswith("platlib/"): + out_file.write(quote + platlib_repl + rest[len("platlib/"):] + "\n") + elif rest.startswith("scripts/"): + entry = rest[len("scripts/"):] + if target_os == "windows": + if quote == '"': + idx = entry.index('"') + else: + idx = entry.index(",") + spath, suffix = entry[:idx], entry[idx:] + if spath in rewritten_scripts: + spath += ".bat" + out_file.write(quote + scripts_repl + spath + suffix + "\n") + else: + out_file.write(quote + scripts_repl + entry + "\n") + elif rest.startswith("headers/"): + out_file.write(quote + headers_repl + rest[len("headers/"):] + "\n") + elif rest.startswith("data/"): + out_file.write(quote + data_repl + rest[len("data/"):] + "\n") + else: + out_file.write(line + "\n") + + +def main(argv): + in_path, out_path, target_os, data_dir_basename = argv[1:5] + rewritten_scripts = set(argv[5:]) + _rewrite(in_path, out_path, target_os, data_dir_basename, rewritten_scripts) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/python/private/pypi/wheel_record_rewriter.sh b/python/private/pypi/wheel_record_rewriter.sh deleted file mode 100755 index 4d93f2a3ae..0000000000 --- a/python/private/pypi/wheel_record_rewriter.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh -set -eu - -IN="$1" -OUT="$2" -TARGET_OS="$3" -DATA_DIR_BASENAME="$4" -shift 4 - -DATA_PREFIX="${DATA_DIR_BASENAME}/" -QUOTED_DATA_PREFIX="\"${DATA_DIR_BASENAME}/" - -if [ "$TARGET_OS" = "windows" ]; then - DATA_REPL="../../" - HEADERS_REPL="../../Include/" - PLATLIB_REPL="" - PURELIB_REPL="" - SCRIPTS_REPL="../../Scripts/" -else - DATA_REPL="../../../" - HEADERS_REPL="../../../include/" - PLATLIB_REPL="" - PURELIB_REPL="" - SCRIPTS_REPL="../../../bin/" -fi - -awk -v data_prefix="$DATA_PREFIX" \ - -v quoted_data_prefix="$QUOTED_DATA_PREFIX" \ - -v data_repl="$DATA_REPL" \ - -v headers_repl="$HEADERS_REPL" \ - -v platlib_repl="$PLATLIB_REPL" \ - -v purelib_repl="$PURELIB_REPL" \ - -v scripts_repl="$SCRIPTS_REPL" \ - -v target_os="$TARGET_OS" ' -BEGIN { - for (i = 2; i < ARGC; i++) { - rewritten[ARGV[i]] = 1 - } - ARGC = 2 -} -{ - line = $0 - quote = "" - if (substr(line, 1, length(quoted_data_prefix)) == quoted_data_prefix) { - quote = "\"" - rest = substr(line, length(quoted_data_prefix) + 1) - } else if (substr(line, 1, length(data_prefix)) == data_prefix) { - rest = substr(line, length(data_prefix) + 1) - } else { - print line - next - } - - if (substr(rest, 1, 8) == "purelib/") { - print quote purelib_repl substr(rest, 9) - } else if (substr(rest, 1, 8) == "platlib/") { - print quote platlib_repl substr(rest, 9) - } else if (substr(rest, 1, 8) == "scripts/") { - entry = substr(rest, 9) - if (target_os == "windows") { - if (quote == "\"") { - idx = index(entry, "\"") - spath = substr(entry, 1, idx - 1) - suffix = substr(entry, idx) - } else { - idx = index(entry, ",") - spath = substr(entry, 1, idx - 1) - suffix = substr(entry, idx) - } - if (spath in rewritten) { - spath = spath ".bat" - } - print quote scripts_repl spath suffix - } else { - print quote scripts_repl entry - } - } else if (substr(rest, 1, 8) == "headers/") { - print quote headers_repl substr(rest, 9) - } else if (substr(rest, 1, 5) == "data/") { - print quote data_repl substr(rest, 6) - } else { - print line - } -} -' "$IN" "$@" > "$OUT" From 353e06b4468499bf77635af91bd62506260c8368 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 30 Aug 2026 18:34:34 -0700 Subject: [PATCH 3/4] fix(pypi): ensure UTF-8 encoding in RECORD rewriter and add news entry Use explicit UTF-8 encoding and CRLF stripping when rewriting RECORD files, format Python rewriter scripts, and add the missing release news entry for PR #4125. --- news/4125.fixed.md | 4 ++++ python/private/pypi/venv_shebang_rewriter.py | 13 +++++++---- python/private/pypi/wheel_record_rewriter.py | 24 ++++++++++++-------- 3 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 news/4125.fixed.md diff --git a/news/4125.fixed.md b/news/4125.fixed.md new file mode 100644 index 0000000000..b0231963c5 --- /dev/null +++ b/news/4125.fixed.md @@ -0,0 +1,4 @@ +(pypi) Ported internal venv shebang and wheel {obj}`RECORD` rewriters from +shell scripts to Python ({obj}`py_binary`) to eliminate dependencies on host +coreutils and `awk` in `PATH`, fixing wheel installation on non-FHS systems like +NixOS ([#4125](https://github.com/bazel-contrib/rules_python/pull/4125)). diff --git a/python/private/pypi/venv_shebang_rewriter.py b/python/private/pypi/venv_shebang_rewriter.py index ab921074a2..48e615bbf1 100644 --- a/python/private/pypi/venv_shebang_rewriter.py +++ b/python/private/pypi/venv_shebang_rewriter.py @@ -13,19 +13,24 @@ def main(argv): with open(out_path, "wb") as out_file: if target_os == "windows": - python_exe = b"pythonw.exe" if first_line.startswith(b"#!pythonw") else b"python.exe" + python_exe = ( + b"pythonw.exe" if first_line.startswith(b"#!pythonw") else b"python.exe" + ) # A Batch-Python polyglot. Batch executes the first line and exits, # while Python (via -x) ignores the first line and executes the rest. out_file.write( - b'@setlocal enabledelayedexpansion & "%~dp0' + python_exe + - b'" -x "%~f0" %* & exit /b !ERRORLEVEL!\r\n', + b'@setlocal enabledelayedexpansion & "%~dp0' + + python_exe + + b'" -x "%~f0" %* & exit /b !ERRORLEVEL!\r\n', ) else: out_file.write(b"#!/bin/sh\n") # A Shell-Python polyglot. The shell executes the triple-quoted 'exec' # command, re-running the script with python3 from the scripts directory. # Python ignores the triple-quoted string and continues. - out_file.write(b"'''exec' \"$(dirname \"$0\")/python3\" \"$0\" \"$@\"\n' '''\n") + out_file.write( + b"'''exec' \"$(dirname \"$0\")/python3\" \"$0\" \"$@\"\n' '''\n" + ) out_file.write(rest) diff --git a/python/private/pypi/wheel_record_rewriter.py b/python/private/pypi/wheel_record_rewriter.py index 4d8c66bd3c..8931b30360 100644 --- a/python/private/pypi/wheel_record_rewriter.py +++ b/python/private/pypi/wheel_record_rewriter.py @@ -20,26 +20,32 @@ def _rewrite(in_path, out_path, target_os, data_dir_basename, rewritten_scripts) purelib_repl = "" scripts_repl = "../../../bin/" - with open(in_path) as in_file, open(out_path, "w") as out_file: + # PEP 427 specifies archive filenames are UTF-8, and while the encoding of + # dist-info files isn't formally standardized in PEP 376, packaging tools + # like pip and importlib.metadata treat them as UTF-8 in practice. + # See: https://discuss.python.org/t/encoding-of-files-in-the-dist-info-directory/6734/4 + with open(in_path, encoding="utf-8") as in_file, open( + out_path, "w", encoding="utf-8", newline="\n" + ) as out_file: for raw_line in in_file: - line = raw_line.rstrip("\n") + line = raw_line.rstrip("\r\n") if line.startswith(quoted_data_prefix): quote = '"' - rest = line[len(quoted_data_prefix):] + rest = line[len(quoted_data_prefix) :] elif line.startswith(data_prefix): quote = "" - rest = line[len(data_prefix):] + rest = line[len(data_prefix) :] else: out_file.write(line + "\n") continue if rest.startswith("purelib/"): - out_file.write(quote + purelib_repl + rest[len("purelib/"):] + "\n") + out_file.write(quote + purelib_repl + rest[len("purelib/") :] + "\n") elif rest.startswith("platlib/"): - out_file.write(quote + platlib_repl + rest[len("platlib/"):] + "\n") + out_file.write(quote + platlib_repl + rest[len("platlib/") :] + "\n") elif rest.startswith("scripts/"): - entry = rest[len("scripts/"):] + entry = rest[len("scripts/") :] if target_os == "windows": if quote == '"': idx = entry.index('"') @@ -52,9 +58,9 @@ def _rewrite(in_path, out_path, target_os, data_dir_basename, rewritten_scripts) else: out_file.write(quote + scripts_repl + entry + "\n") elif rest.startswith("headers/"): - out_file.write(quote + headers_repl + rest[len("headers/"):] + "\n") + out_file.write(quote + headers_repl + rest[len("headers/") :] + "\n") elif rest.startswith("data/"): - out_file.write(quote + data_repl + rest[len("data/"):] + "\n") + out_file.write(quote + data_repl + rest[len("data/") :] + "\n") else: out_file.write(line + "\n") From 8e5a0b794c6e04862b42f58dc36fc52dfb3e2d81 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 30 Aug 2026 18:39:00 -0700 Subject: [PATCH 4/4] docs(news): refine news entry description for PR #4125 Focus news entry on user-visible behavior regarding venv mode on NixOS. --- news/4125.fixed.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/news/4125.fixed.md b/news/4125.fixed.md index b0231963c5..54c2096fe6 100644 --- a/news/4125.fixed.md +++ b/news/4125.fixed.md @@ -1,4 +1,3 @@ -(pypi) Ported internal venv shebang and wheel {obj}`RECORD` rewriters from -shell scripts to Python ({obj}`py_binary`) to eliminate dependencies on host -coreutils and `awk` in `PATH`, fixing wheel installation on non-FHS systems like -NixOS ([#4125](https://github.com/bazel-contrib/rules_python/pull/4125)). +(pypi) Venv mode now works on NixOS and no longer requires coreutils to process +wheel scripts and RECORD files +([#4125](https://github.com/bazel-contrib/rules_python/pull/4125)).