From d6100ee060ba799ccd154b7008bb5ad1da18a0f3 Mon Sep 17 00:00:00 2001 From: Keming Date: Tue, 5 Nov 2024 16:56:14 +0800 Subject: [PATCH] chore: add CI lint and build test (#41) * chore: add CI lint and build test Signed-off-by: Keming * fix typo Signed-off-by: Keming * fix pgrx build dependencies Signed-off-by: Keming * use pgrx docker image to accelerate the dev-build/clippy/test Signed-off-by: Keming * fix pgrx docker image ubuntu 22.04 aarch64 Signed-off-by: Keming * address commments Signed-off-by: Keming * Update .github/workflows/style.yml Co-authored-by: usamoi * enable sccache Signed-off-by: Keming --------- Signed-off-by: Keming Co-authored-by: usamoi --- .github/workflows/rust.yml | 69 +++++++++++++++++++++++++++++++++++++ .github/workflows/style.yml | 37 ++++++++++++++++++++ .taplo.toml | 10 ++++++ Cargo.toml | 19 +++++----- bench/index.py | 2 +- docker/pgrx.Dockerfile | 62 +++++++++++++++++++++++++++++++++ 6 files changed, 189 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/rust.yml create mode 100644 .github/workflows/style.yml create mode 100644 .taplo.toml create mode 100644 docker/pgrx.Dockerfile diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..018139a --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,69 @@ +name: Rust + +on: + pull_request: + paths: + - '.github/workflows/rust.yml' + - 'src/**' + - 'Cargo.lock' + - 'Cargo.toml' + - '*.control' + - 'rust-toolchain.toml' + push: + branches: + - main + paths: + - '.github/workflows/rust.yml' + - 'src/**' + - 'Cargo.lock' + - 'Cargo.toml' + - '*.control' + - 'rust-toolchain.toml' + merge_group: + workflow_dispatch: + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + arch: ["x86_64", "aarch64"] + env: + PGRX_IMAGE: "kemingy/pgrx:0.12.8" + + steps: + - uses: actions/checkout@v4 + - name: Configure sccache + uses: actions/github-script@v7 + with: + script: | + const url = process.env.ACTIONS_CACHE_URL || ''; + const token = process.env.ACTIONS_RUNTIME_TOKEN || ''; + core.exportVariable( + 'CACHE_ENVS', + `-e CARGO_INCREMENTAL=0 -e SCCACHE_GHA_ENABLED=true -e RUSTC_WRAPPER=sccache -e ACTIONS_CACHE_URL=${url} -e ACTIONS_RUNTIME_TOKEN=${token}`, + ); + - name: Set up docker images and permissions + run: | + docker pull $PGRX_IMAGE + echo "Default user: $(id -u):$(id -g)" + sudo chown -R 1000:1000 . + + - name: Clippy + run: | + for v in {14..17}; do + docker run --rm -v .:/workspace $CACHE_ENVS $PGRX_IMAGE clippy --target ${{ matrix.arch }}-unknown-linux-gnu --features "pg$v" -- -D warnings + done + - name: Build + run: | + for v in {14..17}; do + docker run --rm -v .:/workspace $CACHE_ENVS $PGRX_IMAGE build --lib --target ${{ matrix.arch }}-unknown-linux-gnu --features "pg$v" + done + - name: Test + run: | + # pg agnostic tests + docker run --rm -v .:/workspace $CACHE_ENVS $PGRX_IMAGE test --no-fail-fast --target ${{ matrix.arch }}-unknown-linux-gnu --features pg17 diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml new file mode 100644 index 0000000..230d7d0 --- /dev/null +++ b/.github/workflows/style.yml @@ -0,0 +1,37 @@ +name: Style + +on: + pull_request: + branches: + - main + push: + branches: + - main + merge_group: + workflow_dispatch: + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + + - name: Typos + uses: crate-ci/typos@master + + - name: TOML lint + run: | + curl -fsSL https://github.com/tamasfe/taplo/releases/latest/download/taplo-full-linux-x86_64.gz | gzip -d - | install -m 755 /dev/stdin /usr/local/bin/taplo + taplo fmt --check + + - name: Cargo Lint + run: | + cargo fmt --check diff --git a/.taplo.toml b/.taplo.toml new file mode 100644 index 0000000..d9a9fda --- /dev/null +++ b/.taplo.toml @@ -0,0 +1,10 @@ +[formatting] +indent_string = " " + +[[rule]] +keys = ["dependencies", "*-denpendencies", "lints", "patch.*", "profile.*"] + +[rule.formatting] +reorder_keys = true +reorder_arrays = true +align_comments = true diff --git a/Cargo.toml b/Cargo.toml index f47b7b4..bd6643f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,20 +21,21 @@ pg16 = ["pgrx/pg16", "pgrx-catalog/pg16"] pg17 = ["pgrx/pg17", "pgrx-catalog/pg17"] [dependencies] +paste = "1" pgrx = { version = "=0.12.8", default-features = false, features = ["cshim"] } pgrx-catalog = "0.1.0" +rand = "0.8.5" +rand_chacha = "0.3.1" +rand_distr = "0.4.3" +serde = "1" +toml = "0.8.19" +validator = "0.18.1" + base = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" } common = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" } detect = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" } k_means = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" } quantization = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" } -paste = "1" -serde = "1" -toml = "0.8.19" -rand = "0.8.5" -rand_chacha = "0.3.1" -rand_distr = "0.4.3" -validator = "0.18.1" # lock algebra version forever so that the QR decomposition never changes for same input nalgebra = { version = "=0.33.0", default-features = false } @@ -48,12 +49,12 @@ rust.unused_lifetimes = "warn" rust.unused_qualifications = "warn" [profile.opt] +debug-assertions = false inherits = "dev" opt-level = 3 -debug-assertions = false overflow-checks = false [profile.release] -lto = "fat" codegen-units = 1 debug = true +lto = "fat" diff --git a/bench/index.py b/bench/index.py index 4fde97f..97b67a0 100644 --- a/bench/index.py +++ b/bench/index.py @@ -179,7 +179,7 @@ async def main(dataset): await add_embeddings(conn, args.name, args.dim, dataset["train"]) index_finish = asyncio.Event() - # Need a seperate connection for monitor process + # Need a separate connection for monitor process monitor_conn = await create_connection(url) monitor_task = monitor_index_build( monitor_conn, diff --git a/docker/pgrx.Dockerfile b/docker/pgrx.Dockerfile new file mode 100644 index 0000000..920817b --- /dev/null +++ b/docker/pgrx.Dockerfile @@ -0,0 +1,62 @@ +# CNPG only support Debian 12 (Bookworm) +FROM ubuntu:22.04 + +ARG PGRX_VERSION=0.12.8 +ARG SCCACHE_VERSION=0.8.2 + +ENV DEBIAN_FRONTEND=noninteractive \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + RUSTFLAGS="-Dwarnings" \ + RUST_BACKTRACE=1 \ + CARGO_TERM_COLOR=always + +RUN set -eux; \ + apt update; \ + apt install -y --no-install-recommends \ + curl \ + ca-certificates \ + build-essential \ + postgresql-common gnupg \ + crossbuild-essential-arm64 \ + qemu-user-static \ + libreadline-dev zlib1g-dev flex bison libxml2-dev libxslt-dev libssl-dev libxml2-utils xsltproc ccache pkg-config \ + clang + +# set up sccache +RUN set -ex; \ + curl -fsSL -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz; \ + tar -xzf sccache.tar.gz --strip-components=1; \ + rm sccache.tar.gz; \ + mv sccache /usr/local/bin/ + +RUN /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y +# install all the PostgresQL +RUN set -ex; \ + for v in $(seq 14 17); do \ + apt install -y --no-install-recommends postgresql-$v postgresql-server-dev-$v; \ + done; \ + rm -rf /var/lib/apt/lists/*; + +# create a non-root user (make it compatible with Ubuntu 24.04) +RUN useradd -u 1000 -U -m ubuntu +USER ubuntu +ENV PATH="$PATH:/home/ubuntu/.cargo/bin" +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain=none -y + +WORKDIR /workspace +COPY rust-toolchain.toml /workspace/rust-toolchain.toml +# ref: https://github.com/pgcentralfoundation/pgrx/blob/develop/docs/src/extension/build/cross-compile.md +RUN set -ex; \ + echo 'target.aarch64-unknown-linux-gnu.linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config.toml; \ + echo 'target.aarch64-unknown-linux-gnu.runner = ["qemu-aarch64-static", "-L", "/usr/aarch64-linux-gnu"]' >> ~/.cargo/config.toml +RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu + +RUN cargo install cargo-pgrx --locked --version=${PGRX_VERSION} + +RUN set -ex; \ + for v in $(seq 14 17); do \ + cargo pgrx init --pg$v=/usr/lib/postgresql/$v/bin/pg_config; \ + done; + +ENTRYPOINT [ "cargo" ]