Run the geospatial benchmarks on every ADBC database #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The geospatial benchmark cases (benchmarks/geospatial) on every ADBC | |
| # database, against real ARCO-ERA5 and WeatherBench 2 data. Each case | |
| # checks its SQL answer against an xarray reference, so a cell passes | |
| # only if the database computed the right numbers. | |
| # | |
| # One job per database, in parallel; each starts only its own server. | |
| # Runs daily and by hand; results land in the job summary. | |
| name: geospatial on ADBC databases | |
| on: | |
| schedule: | |
| - cron: "0 7 * * *" | |
| pull_request: | |
| paths: | |
| - ".github/workflows/geospatial-adbc.yml" | |
| - "benchmarks/geospatial/_engines.py" | |
| - "tests/_adbc.py" | |
| workflow_dispatch: | |
| inputs: | |
| cases: | |
| description: Comma-separated cases | |
| default: 02_climatology,03_zonal_mean,04_anomaly,05_forecast_skill,06_zonal_vector | |
| jobs: | |
| geospatial: | |
| name: ${{ matrix.engine }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - engine: adbc-sqlite | |
| - engine: adbc-duckdb | |
| - engine: adbc-chdb | |
| - engine: adbc-datafusion | |
| - engine: adbc-postgresql | |
| image: postgres:18 | |
| port: 5432 | |
| uri: postgresql://postgres:xql@localhost:5432/postgres | |
| - engine: adbc-mysql | |
| image: mysql:8.4 | |
| port: 3306 | |
| uri: mysql://root:xql@127.0.0.1:3306/xql | |
| - engine: adbc-mariadb | |
| image: mariadb:11.4 | |
| port: 3306 | |
| uri: mysql://root:xql@127.0.0.1:3306/xql | |
| - engine: adbc-clickhouse | |
| image: clickhouse/clickhouse-server:26.8 | |
| port: 8123 | |
| uri: http://localhost:8123/?user=default&password=xql | |
| - engine: adbc-trino | |
| image: trinodb/trino:483 | |
| port: 8080 | |
| uri: http://ci@localhost:8080?catalog=memory&schema=default | |
| - engine: adbc-mssql | |
| image: mcr.microsoft.com/mssql/server:2022-latest | |
| port: 1433 | |
| uri: sqlserver://sa:XqlPassw0rd@localhost:1433?database=master | |
| - engine: adbc-flightsql | |
| image: gizmodata/gizmosql:latest | |
| port: 31337 | |
| uri: grpc://localhost:31337 | |
| services: | |
| # An empty image (the in-process databases) starts no container. | |
| # Each image reads only its own variables below. | |
| db: | |
| image: ${{ matrix.image }} | |
| ports: | |
| - ${{ matrix.port || 1 }}:${{ matrix.port || 1 }} | |
| env: | |
| POSTGRES_PASSWORD: xql | |
| MYSQL_ROOT_PASSWORD: xql | |
| MYSQL_DATABASE: xql | |
| MARIADB_ROOT_PASSWORD: xql | |
| MARIADB_DATABASE: xql | |
| CLICKHOUSE_PASSWORD: xql | |
| ACCEPT_EULA: "Y" | |
| MSSQL_SA_PASSWORD: XqlPassw0rd | |
| TLS_ENABLED: "0" | |
| GIZMOSQL_USERNAME: xql | |
| GIZMOSQL_PASSWORD: xql | |
| env: | |
| VIRTUAL_ENV: ${{ github.workspace }}/.venv | |
| XARRAY_SQL_TEST_FLIGHTSQL_USERNAME: xql | |
| XARRAY_SQL_TEST_FLIGHTSQL_PASSWORD: xql | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - name: Setup sccache | |
| uses: mozilla-actions/sccache-action@v0.0.9 | |
| - name: Configure sccache | |
| run: | | |
| echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV | |
| echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV | |
| - uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| - name: Install xarray_sql | |
| run: uv sync --dev --no-install-package xarray-sql | |
| - name: build rust | |
| run: uv run --no-project maturin develop --uv | |
| - name: Install ADBC drivers | |
| run: | | |
| uv pip install dbc adbc-driver-postgresql adbc-driver-flightsql psutil | |
| for driver in chdb datafusion mysql clickhouse trino mssql; do | |
| uv run --no-project dbc install "$driver" | |
| done | |
| - name: Point the tests' backend table at the server | |
| if: matrix.uri | |
| run: | | |
| name="${{ matrix.engine }}" | |
| name="${name#adbc-}" | |
| echo "XARRAY_SQL_TEST_${name^^}_URI=${{ matrix.uri }}" >> "$GITHUB_ENV" | |
| - name: Wait for the server | |
| if: matrix.uri | |
| run: | | |
| uv run --no-project python - <<'EOF' | |
| import os, sys, time | |
| sys.path.insert(0, os.getcwd()) | |
| from tests._adbc import BACKENDS | |
| name = "${{ matrix.engine }}".removeprefix("adbc-") | |
| backend = next(b for b in BACKENDS if b.name == name) | |
| for attempt in range(80): | |
| try: | |
| con = backend.connect() | |
| cur = con.cursor() | |
| cur.execute("SELECT 1") | |
| cur.fetchall() | |
| print(f"{name} is up") | |
| break | |
| except Exception as exc: | |
| print(f"waiting for {name}: {str(exc)[:120]}") | |
| time.sleep(3) | |
| else: | |
| sys.exit(f"{name} did not start") | |
| EOF | |
| - name: Run the geospatial cases | |
| run: | | |
| mkdir -p results | |
| uv run --no-project python benchmarks/geospatial/engine_suite.py \ | |
| --local --reps 1 \ | |
| --cases "${{ inputs.cases || '02_climatology,03_zonal_mean,04_anomaly,05_forecast_skill,06_zonal_vector' }}" \ | |
| --engines "${{ matrix.engine }}" \ | |
| --cell-timeout 2700 \ | |
| --out "results/${{ matrix.engine }}.json" \ | |
| --jsonl "results/${{ matrix.engine }}.jsonl" | |
| - name: Summarize | |
| if: always() | |
| run: | | |
| echo "## ${{ matrix.engine }}" >> "$GITHUB_STEP_SUMMARY" | |
| cat "results/${{ matrix.engine }}.md" >> "$GITHUB_STEP_SUMMARY" || true | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: geospatial-${{ matrix.engine }} | |
| path: results/ |