chore: add CI lint and build test (#41)

* chore: add CI lint and build test

Signed-off-by: Keming <[email protected]>

* fix typo

Signed-off-by: Keming <[email protected]>

* fix pgrx build dependencies

Signed-off-by: Keming <[email protected]>

* use pgrx docker image to accelerate the dev-build/clippy/test

Signed-off-by: Keming <[email protected]>

* fix pgrx docker image ubuntu 22.04 aarch64

Signed-off-by: Keming <[email protected]>

* address commments

Signed-off-by: Keming <[email protected]>

* Update .github/workflows/style.yml

Co-authored-by: usamoi <[email protected]>

* enable sccache

Signed-off-by: Keming <[email protected]>

---------

Signed-off-by: Keming <[email protected]>
Co-authored-by: usamoi <[email protected]>

.github/workflows/rust.yml

@@ -0,0 +1,69 @@
+name: Rust
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/rust.yml'
+      - 'src/**'
+      - 'Cargo.lock'
+      - 'Cargo.toml'
+      - '*.control'
+      - 'rust-toolchain.toml'
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/rust.yml'
+      - 'src/**'
+      - 'Cargo.lock'
+      - 'Cargo.toml'
+      - '*.control'
+      - 'rust-toolchain.toml'
+  merge_group:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: ["x86_64", "aarch64"]
+    env:
+      PGRX_IMAGE: "kemingy/pgrx:0.12.8"
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure sccache
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const url = process.env.ACTIONS_CACHE_URL || '';
+            const token = process.env.ACTIONS_RUNTIME_TOKEN || '';
+            core.exportVariable(
+              'CACHE_ENVS',
+              `-e CARGO_INCREMENTAL=0 -e SCCACHE_GHA_ENABLED=true -e RUSTC_WRAPPER=sccache -e ACTIONS_CACHE_URL=${url} -e ACTIONS_RUNTIME_TOKEN=${token}`,
+            );
+      - name: Set up docker images and permissions
+        run: |
+          docker pull $PGRX_IMAGE
+          echo "Default user: $(id -u):$(id -g)"
+          sudo chown -R 1000:1000 .
+
+      - name: Clippy
+        run: |
+          for v in {14..17}; do
+            docker run --rm -v .:/workspace $CACHE_ENVS $PGRX_IMAGE clippy --target ${{ matrix.arch }}-unknown-linux-gnu --features "pg$v" -- -D warnings
+          done
+      - name: Build
+        run: |
+          for v in {14..17}; do
+            docker run --rm -v .:/workspace $CACHE_ENVS $PGRX_IMAGE build --lib --target ${{ matrix.arch }}-unknown-linux-gnu --features "pg$v"
+          done
+      - name: Test
+        run: |
+          # pg agnostic tests
+          docker run --rm -v .:/workspace $CACHE_ENVS $PGRX_IMAGE test --no-fail-fast --target ${{ matrix.arch }}-unknown-linux-gnu --features pg17

.github/workflows/style.yml

@@ -0,0 +1,37 @@
+name: Style
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+  merge_group:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Typos
+        uses: crate-ci/typos@master
+
+      - name: TOML lint
+        run: |
+          curl -fsSL https://github.com/tamasfe/taplo/releases/latest/download/taplo-full-linux-x86_64.gz | gzip -d - | install -m 755 /dev/stdin /usr/local/bin/taplo
+          taplo fmt --check
+
+      - name: Cargo Lint
+        run: |
+          cargo fmt --check

.taplo.toml

@@ -0,0 +1,10 @@
+[formatting]
+indent_string = "    "
+
+[[rule]]
+keys = ["dependencies", "*-denpendencies", "lints", "patch.*", "profile.*"]
+
+[rule.formatting]
+reorder_keys = true
+reorder_arrays = true
+align_comments = true

Cargo.toml

@@ -21,20 +21,21 @@ pg16 = ["pgrx/pg16", "pgrx-catalog/pg16"]
 pg17 = ["pgrx/pg17", "pgrx-catalog/pg17"]
 
 [dependencies]
+paste = "1"
 pgrx = { version = "=0.12.8", default-features = false, features = ["cshim"] }
 pgrx-catalog = "0.1.0"
+rand = "0.8.5"
+rand_chacha = "0.3.1"
+rand_distr = "0.4.3"
+serde = "1"
+toml = "0.8.19"
+validator = "0.18.1"
+
 base = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" }
 common = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" }
 detect = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" }
 k_means = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" }
 quantization = { git = "https://github.com/tensorchord/pgvecto.rs.git", branch = "rabbithole-2" }
-paste = "1"
-serde = "1"
-toml = "0.8.19"
-rand = "0.8.5"
-rand_chacha = "0.3.1"
-rand_distr = "0.4.3"
-validator = "0.18.1"
 
 # lock algebra version forever so that the QR decomposition never changes for same input
 nalgebra = { version = "=0.33.0", default-features = false }
@@ -48,12 +49,12 @@ rust.unused_lifetimes = "warn"
 rust.unused_qualifications = "warn"
 
 [profile.opt]
+debug-assertions = false
 inherits = "dev"
 opt-level = 3
-debug-assertions = false
 overflow-checks = false
 
 [profile.release]
-lto = "fat"
 codegen-units = 1
 debug = true
+lto = "fat"

bench/index.py

@@ -179,7 +179,7 @@ async def main(dataset):
     await add_embeddings(conn, args.name, args.dim, dataset["train"])
 
     index_finish = asyncio.Event()
-    # Need a seperate connection for monitor process
+    # Need a separate connection for monitor process
     monitor_conn = await create_connection(url)
     monitor_task = monitor_index_build(
         monitor_conn,

docker/pgrx.Dockerfile

@@ -0,0 +1,62 @@
+# CNPG only support Debian 12 (Bookworm)
+FROM ubuntu:22.04
+
+ARG PGRX_VERSION=0.12.8
+ARG SCCACHE_VERSION=0.8.2
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    LANG=en_US.UTF-8 \
+    LC_ALL=en_US.UTF-8 \
+    RUSTFLAGS="-Dwarnings" \
+    RUST_BACKTRACE=1 \
+    CARGO_TERM_COLOR=always
+
+RUN set -eux; \
+    apt update; \
+    apt install -y --no-install-recommends \
+        curl \
+        ca-certificates \
+        build-essential \
+        postgresql-common gnupg \
+        crossbuild-essential-arm64 \
+        qemu-user-static \
+        libreadline-dev zlib1g-dev flex bison libxml2-dev libxslt-dev libssl-dev libxml2-utils xsltproc ccache pkg-config \
+        clang
+
+# set up sccache
+RUN set -ex; \
+    curl -fsSL -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz; \
+    tar -xzf sccache.tar.gz --strip-components=1; \
+    rm sccache.tar.gz; \
+    mv sccache /usr/local/bin/
+
+RUN /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
+# install all the PostgresQL
+RUN set -ex; \
+    for v in $(seq 14 17); do \
+        apt install -y --no-install-recommends postgresql-$v postgresql-server-dev-$v; \
+    done; \
+    rm -rf /var/lib/apt/lists/*;
+
+# create a non-root user (make it compatible with Ubuntu 24.04)
+RUN useradd -u 1000 -U -m ubuntu
+USER ubuntu
+ENV PATH="$PATH:/home/ubuntu/.cargo/bin"
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain=none -y
+
+WORKDIR /workspace
+COPY rust-toolchain.toml /workspace/rust-toolchain.toml
+# ref: https://github.com/pgcentralfoundation/pgrx/blob/develop/docs/src/extension/build/cross-compile.md
+RUN set -ex; \
+    echo 'target.aarch64-unknown-linux-gnu.linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config.toml; \
+    echo 'target.aarch64-unknown-linux-gnu.runner = ["qemu-aarch64-static", "-L", "/usr/aarch64-linux-gnu"]' >> ~/.cargo/config.toml
+RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu
+
+RUN cargo install cargo-pgrx --locked --version=${PGRX_VERSION}
+
+RUN set -ex; \
+    for v in $(seq 14 17); do \
+        cargo pgrx init --pg$v=/usr/lib/postgresql/$v/bin/pg_config; \
+    done;
+
+ENTRYPOINT [ "cargo" ]