Merge upstream 3.12.3 release

Summary: This merges the upstream changes from 3.12.2 -> 3.12.3 into our fork. Merge-point in upstream is commit `f6650f9ad73359051f3e558c2431a109bc016664` (also pushed in cinder github as a `meta/3.12.3-base` tag). Merging required a few manual merge conflict resolutions, called out as diff comments, and summarized here: 1. `patchlevel.h` -- update the meta version patch 1. `Python/initconfig.c` -- fixed collision between the lazy imports env var and a new upstream env var 1. `Lib/test/test_property.py` -- fixed conflict between a test added upstream and the backported deletion of tests that were in the same place (D52014322) 1. `Tools/msi/bundle/bundle.wxs` -- customizations done in D53734154 conflicted with an upstream change Reviewed upstream changes to configure and Makefile to look for needed buckification changes: 1. upstream added thread sanitizer configure flag - buckification not affected 1. some changes in sbom generation - not relevant to buckification 1. some tests moved around - buckification not affected Reviewed added/removed files in case they need to be reflected in the buckification: ``` # (on the git merge commit) $ git diff --name-status HEAD^ HEAD | sort | less A .github/workflows/reusable-tsan.yml A Doc/c-api/hash.rst A Doc/howto/gdb_helpers.rst A Lib/test/libregrtest/tsan.py A Lib/test/test_doctest/decorator_mod.py A Lib/test/test_pydoc/__init__.py A Lib/test/test_tools/test_makefile.py A Lib/test/zipdir_backslash.zip A Misc/NEWS.d/3.12.3.rst A Misc/externals.spdx.json A Tools/scripts/sortperf.py A Tools/tsan/supressions.txt D Lib/test/sortperf.py ``` Reviewed By: fried Differential Revision: D55930999 fbshipit-source-id: d6bea295c376f955cf7df39d69dec60c61531c68
facebookincubator · Apr 11, 2024 · 23c541e · 23c541e
1 parent a0199fd
commit 23c541e
Show file tree

Hide file tree

Showing 453 changed files with 15,342 additions and 8,701 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -2,11 +2,11 @@ FROM docker.io/library/fedora:37
 
 ENV CC=clang
 
-ENV WASI_SDK_VERSION=19
+ENV WASI_SDK_VERSION=20
 ENV WASI_SDK_PATH=/opt/wasi-sdk
 
 ENV WASMTIME_HOME=/opt/wasmtime
-ENV WASMTIME_VERSION=7.0.0
+ENV WASMTIME_VERSION=18.0.2
 ENV WASMTIME_CPU_ARCH=x86_64
 
 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \

diff --git a/.editorconfig b/.editorconfig
@@ -1,6 +1,6 @@
 root = true
 
-[*.{py,c,cpp,h,rst,md,yml}]
+[*.{py,c,cpp,h,js,rst,md,yml}]
 trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = space
@@ -11,5 +11,5 @@ indent_size = 4
 [*.rst]
 indent_size = 3
 
-[*.yml]
+[*.{js,yml}]
 indent_size = 2
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -168,5 +168,6 @@ Lib/ast.py                    @isidentical
 **/*zipfile/_path/*           @jaraco
 
 # SBOM
+/Misc/externals.spdx.json     @sethmlarson
 /Misc/sbom.spdx.json          @sethmlarson
 /Tools/build/generate_sbom.py @sethmlarson
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -81,7 +81,7 @@ jobs:
       - name: Get a list of the changed documentation-related files
         if: github.event_name == 'pull_request'
         id: changed-docs-files
-        uses: Ana06/get-changed-files@v2.2.0
+        uses: Ana06/get-changed-files@v2.3.0
         with:
           filter: |
             Doc/**
@@ -109,7 +109,7 @@ jobs:
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
       - name: Install dependencies
         run: |
             sudo ./.github/workflows/posix-deps-apt.sh
@@ -138,7 +138,7 @@ jobs:
         if: ${{ failure() && steps.check.conclusion == 'failure' }}
         run: |
           make regen-abidump
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: Publish updated ABI files
         if: ${{ failure() && steps.check.conclusion == 'failure' }}
         with:
@@ -155,14 +155,16 @@ jobs:
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.x'
+      - name: Runner image version
+        run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
       - name: Restore config.cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: config.cache
-          key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
+          key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
       - name: Add ccache to PATH
@@ -190,7 +192,7 @@ jobs:
           # so we run it before regen-global-objects gets rum (in regen-all).
           make regen-deepfreeze
           make -j4 regen-all
-          make regen-stdlib-module-names
+          make regen-stdlib-module-names regen-sbom
       - name: Check for changes
         run: |
           git add -u
@@ -257,11 +259,13 @@ jobs:
       LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -273,7 +277,7 @@ jobs:
         echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
@@ -318,7 +322,7 @@ jobs:
         echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
@@ -340,11 +344,13 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -374,7 +380,7 @@ jobs:
         ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt
     - name: 'Restore Hypothesis database'
       id: cache-hypothesis-database
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ./hypothesis
         key: hypothesis-database-${{ github.head_ref || github.run_id }}
@@ -400,7 +406,7 @@ jobs:
           -x test_subprocess \
           -x test_signal \
           -x test_sysconfig
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       if: always()
       with:
         name: hypothesis-example-db
@@ -419,11 +425,13 @@ jobs:
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -439,7 +447,7 @@ jobs:
         echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
@@ -463,6 +471,15 @@ jobs:
     - name: Tests
       run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
 
+  build_tsan:
+    name: 'Thread sanitizer'
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    uses: ./.github/workflows/reusable-tsan.yml
+    with:
+      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      options: ./configure --config-cache --with-thread-sanitizer --with-pydebug
+
   all-required-green:  # This job does nothing and is only used for the branch protection
     name: All required checks pass
     if: always()
@@ -477,6 +494,7 @@ jobs:
     - build_windows
     - test_hypothesis
     - build_asan
+    - build_tsan
 
     runs-on: ubuntu-latest
 
@@ -505,6 +523,7 @@ jobs:
             build_ubuntu_ssltests,
             build_windows,
             build_asan,
+            build_tsan,
             '
             || ''
           }}

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -20,7 +20,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
-      - uses: pre-commit/[email protected].0
+      - uses: pre-commit/[email protected].1
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -31,7 +31,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
           cache: pip

diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml
@@ -13,12 +13,12 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 14
       - run: npm install mailgun.js form-data
       - name: Send notification
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         env:
           MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }}
         with:

diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml
@@ -23,7 +23,7 @@ jobs:
           - { project: 32, label: sprint }
 
     steps:
-      - uses: actions/add-to-project@v0.1.0
+      - uses: actions/add-to-project@v1.0.0
         with:
           project-url: https://github.com/orgs/python/projects/${{ matrix.project }}
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}

diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml
@@ -16,7 +16,7 @@ jobs:
     timeout-minutes: 10
 
     steps:
-      - uses: mheap/github-action-required-labels@v4
+      - uses: mheap/github-action-required-labels@v5
         with:
           mode: exactly
           count: 0

diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml
@@ -41,7 +41,7 @@ jobs:
         git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \
           --no-tags --prune --no-recurse-submodules
     - name: 'Set up Python'
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3'
         cache: 'pip'
@@ -72,7 +72,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: 'Set up Python'
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.11'  # known to work with Sphinx 4.2
         cache: 'pip'
@@ -89,7 +89,7 @@ jobs:
     timeout-minutes: 60
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }}

diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml
@@ -17,6 +17,7 @@ jobs:
       HOMEBREW_NO_ANALYTICS: 1
       HOMEBREW_NO_AUTO_UPDATE: 1
       HOMEBREW_NO_INSTALL_CLEANUP: 1
+      HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
       PYTHONSTRICTEXTENSIONBUILD: 1
     strategy:
       fail-fast: false
@@ -28,11 +29,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: config.cache
-        key: ${{ github.job }}-${{ matrix.os }}-${{ inputs.config_hash }}
+        key: ${{ github.job }}-${{ matrix.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Install Homebrew dependencies
       run: brew install pkg-config [email protected] xz gdbm tcl-tk
     - name: Configure CPython

diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml
@@ -0,0 +1,51 @@
+on:
+  workflow_call:
+    inputs:
+      config_hash:
+        required: true
+        type: string
+      options:
+        required: true
+        type: string
+
+jobs:
+  build_tsan_reusable:
+    name: 'Thread sanitizer'
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
+    - name: Restore config.cache
+      uses: actions/cache@v4
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
+    - name: Install Dependencies
+      run: |
+        sudo ./.github/workflows/posix-deps-apt.sh
+        sudo apt install -y clang
+        # Reduce ASLR to avoid TSAN crashing
+        sudo sysctl -w vm.mmap_rnd_bits=28
+    - name: TSAN Option Setup
+      run: |
+        echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV
+        echo "CC=clang" >> $GITHUB_ENV
+        echo "CXX=clang++" >> $GITHUB_ENV
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+      with:
+        save: ${{ github.event_name == 'push' }}
+        max-size: "200M"
+    - name: Configure CPython
+      run: ${{ inputs.options }}
+    - name: Build CPython
+      run: make -j4
+    - name: Display build info
+      run: make pythoninfo
+    - name: Tests
+      run: ./python -m test --tsan -j4
diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml
@@ -29,7 +29,7 @@ jobs:
         echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
     - name: 'Restore OpenSSL build'
       id: cache-openssl
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
         key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
@@ -52,11 +52,13 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
     - name: Restore config.cache
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
-        key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }}
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: ${{ inputs.options }}