Skip to content

Commit

Permalink
introduce cibuildwheel
Browse files Browse the repository at this point in the history
  • Loading branch information
@mh-northlander
mh-northlander committed Jan 10, 2025
1 parent 0809517 commit 240b9c0
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 197 deletions.
129 changes: 46 additions & 83 deletions .github/workflows/build-python-wheels.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,106 +10,69 @@ on:
branches: [ develop ]

jobs:
build-sdist:
name: Build source distribution
runs-on: ubuntu-latest
build-wheels:
name: build wheels using cibuildwheel
runs-on: ${{ format('{0}-latest', matrix.os) }}
strategy:
fail-fast: false
matrix:
include:
- os: "ubuntu"
platform: "linux"
arch: "x86_64"
- os: "macos"
arch: "universal2"
- os: "windows"
arch: "AMD64"
env:
build-sdist: ${{ matrix.os == 'ubuntu' && matrix.arch == 'x86_64' }}

steps:
- uses: actions/checkout@v4
- name: Setup python
- name: setup python for sdist and cibuildwheel
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools setuptools-rust build
- name: Build sdist
working-directory: ./python
run: bash build-sdist.sh
- uses: actions/upload-artifact@v4
with:
name: artifact-sdist
path: python/dist/*

build-linux-wheels:
name: Build Linux ${{ matrix.arch }} Python Wheels (+PGO if possible)
runs-on: ubuntu-latest
strategy:
matrix:
arch: ["amd64", "arm64"]
steps:
- uses: actions/checkout@v4
- name: Download dictionary
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools setuptools-rust build cibuildwheel
- name: download dictionary for PGO
if: ${{ matrix.os == 'ubuntu' }}
run: bash fetch_dictionary.sh "20220519" "core"
- name: Add aarch64/x86 target for macos
if: ${{ matrix.os == 'macos' }}
run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
- uses: actions/cache@v4
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-${{ matrix.arch }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: build in manylinux container (w/ pgo)
if: ${{ matrix.arch == 'amd64' }}
uses: ./.github/actions/run-in-manylinux
with:
arch: ${{ matrix.arch }}
script: python/build-wheels-manylinux-pgo.sh
- name: build in manylinux container (w/o pgo)
if: ${{ matrix.arch == 'arm64' }}
uses: ./.github/actions/run-in-manylinux
with:
arch: ${{ matrix.arch }}
script: python/build-wheels-manylinux.sh
- uses: actions/upload-artifact@v4
with:
name: artifact-manylinux-${{ matrix.arch }}
path: python/dist/*manylinux*

build-non-linux-wheels:
name: Build wheels on ${{ matrix.os }} / ${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-latest, macOS-latest]
python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ]
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

steps:
- uses: actions/checkout@v4

- uses: Quansight-Labs/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v4
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Install dependencies
run: |
python -m pip install -U pip
python -m pip install -U setuptools setuptools_rust build
- name: Add aarch64/x86 target for Rust
run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
if: startsWith(matrix.os, 'macOS')
- name: Build wheel
- name: build sdist
if: ${{ env.build-sdist == 'true' }}
working-directory: ./python
run: python -m build --wheel
run: bash build-sdist.sh
- name: cibuildwheel ${{ matrix.platform || matrix.os }}
env:
ARCHFLAGS: -arch x86_64 -arch arm64
MACOSX_DEPLOYMENT_TARGET: 10.12
# most configuration are in python/pyproject.toml
CIBW_PLATFORM: ${{ matrix.platform || matrix.os }}
CIBW_ARCHS: ${{ matrix.arch }}
run: cibuildwheel python/ --output-dir python/dist/

- uses: actions/upload-artifact@v4
with:
name: artifact-${{ matrix.os }}-${{ matrix.python-version }}
path: python/dist/*.whl
name: artifact-${{ matrix.os }}
path: |
./python/dist/*.whl
./python/dist/*.tar.gz
upload-to-pypi: # run only if all have succeeded
needs: [ build-sdist, build-non-linux-wheels, build-linux-wheels ]
needs: [ build-wheels ]
# https://stackoverflow.com/questions/58475748/github-actions-how-to-check-if-current-push-has-new-tag-is-new-release
if: startsWith(github.ref, 'refs/tags/v') # only for tags
runs-on: ubuntu-latest
Expand Down
159 changes: 45 additions & 114 deletions .github/workflows/python-upload-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,142 +5,80 @@ on:
branches: [ pre/v* ]

jobs:
build-sdist:
name: Build source distribution
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Setup python
uses: actions/setup-python@v5
with:
python-version: '3.11'

- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools setuptools-rust build packaging
- name: Modify version for TestPyPI upload
run: python ./python/modify_version_for_testpypi.py

- name: Build sdist
working-directory: ./python
run: bash build-sdist.sh

- uses: actions/upload-artifact@v4
with:
name: artifact-sdist
path: python/dist/*

build-linux-wheels:
name: Build Linux ${{ matrix.arch }} Python Wheels (+PGO if possible)
runs-on: ubuntu-latest
build-wheels:
name: build wheels using cibuildwheel
runs-on: ${{ format('{0}-latest', matrix.os) }}
strategy:
fail-fast: false
matrix:
arch: ["amd64", "arm64"]
include:
- os: "ubuntu"
platform: "linux"
arch: "x86_64"
- os: "macos"
arch: "universal2"
- os: "windows"
arch: "AMD64"
env:
build-sdist: ${{ matrix.os == 'ubuntu' && matrix.arch == 'x86_64' }}

steps:
- uses: actions/checkout@v4
- name: Download dictionary
run: bash fetch_dictionary.sh "20220519" "core"
- uses: actions/cache@v4
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-${{ matrix.arch }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Setup python
- name: setup python for sdist and cibuildwheel
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install dependencies
run: |
python -m pip install -U pip
python -m pip install -U packaging
- name: Modify version for TestPyPI upload
run: python ./python/modify_version_for_testpypi.py
- name: build in manylinux container (w/ pgo)
if: ${{ matrix.arch == 'amd64' }}
uses: ./.github/actions/run-in-manylinux
with:
arch: ${{ matrix.arch }}
script: python/build-wheels-manylinux-pgo.sh
- name: build in manylinux container (w/o pgo)
if: ${{ matrix.arch == 'arm64' }}
uses: ./.github/actions/run-in-manylinux
with:
arch: ${{ matrix.arch }}
script: python/build-wheels-manylinux.sh
- uses: actions/upload-artifact@v4
with:
name: artifact-manylinux-${{ matrix.arch }}
path: python/dist/*manylinux*

build-non-linux-wheels:
name: Build wheels on ${{ matrix.os }} / ${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-latest, macOS-latest]
python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ]

steps:
- uses: actions/checkout@v4

- uses: Quansight-Labs/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

python -m pip install --upgrade pip
python -m pip install --upgrade setuptools setuptools-rust build cibuildwheel packaging
- name: download dictionary for PGO
if: ${{ matrix.os == 'ubuntu' }}
run: bash fetch_dictionary.sh "20220519" "core"
- name: Add aarch64/x86 target for macos
if: ${{ matrix.os == 'macos' }}
run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
- uses: actions/cache@v4
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

- name: Install dependencies
run: |
python -m pip install -U pip
python -m pip install -U setuptools setuptools_rust build packaging
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

- name: Modify version for TestPyPI upload
run: python ./python/modify_version_for_testpypi.py

- name: Add aarch64/x86 target for Rust
run: rustup target add aarch64-apple-darwin x86_64-apple-darwin
if: startsWith(matrix.os, 'macOS')

- name: Build wheel
- name: build sdist
if: ${{ env.build-sdist == 'true' }}
working-directory: ./python
run: python -m build --wheel
run: bash build-sdist.sh
- name: cibuildwheel ${{ matrix.platform || matrix.os }}
env:
ARCHFLAGS: -arch x86_64 -arch arm64
MACOSX_DEPLOYMENT_TARGET: 10.12

# most configuration are in python/pyproject.toml
CIBW_PLATFORM: ${{ matrix.platform || matrix.os }}
CIBW_ARCHS: ${{ matrix.arch }}
run: cibuildwheel python/ --output-dir python/dist/

- uses: actions/upload-artifact@v4
with:
name: artifact-${{ matrix.os }}-${{ matrix.python-version }}
path: python/dist/*.whl
name: artifact-${{ matrix.os }}
path: |
./python/dist/*.whl
./python/dist/*.tar.gz
upload-to-testpypi: # run only if all have succeeded
needs: [ build-sdist, build-non-linux-wheels, build-linux-wheels ]
needs: [ build-wheels ]
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: artifact-*
path: dist/
merge-multiple: true

- name: List files to upload
run: ls -R dist/

- name: Publish distribution to Test PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
Expand All @@ -157,34 +95,27 @@ jobs:
python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t" ]
fail-fast: false
runs-on: ${{ matrix.os }}

steps:
- uses: actions/checkout@v4
- uses: Quansight-Labs/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Upgrade pip
run: python -m pip install --upgrade pip

- name: Show compatible tags
run: python -m pip debug --verbose

- name: Install our module from TestPyPi
run: python -m pip -vvvv install --pre -U -i https://test.pypi.org/simple/ SudachiPy

- name: Install dependencies
run: python -m pip install sudachidict_core tokenizers

- name: Run test
working-directory: ./python
run: python -m unittest

- name: Check that binary works (C mode)
run: |
sudachipy .github/data/input.txt -o result-c.txt
git diff --color=always --no-index -- result-c.txt .github/data/expected-c.txt
- name: Check that binary works (A mode)
run: |
sudachipy .github/data/input.txt -m A -o result-a.txt
Expand Down
Loading

0 comments on commit 240b9c0

Please sign in to comment.