Add datafusion ci (#29)

* feat: add ci for datafusion producer and consumer tests
* chore: update substrait version in datafusion plans

.github/workflows/test.yml

@@ -4,9 +4,30 @@ on:
   push:
     branches:
       - main
-      - duckdb_producer_ci
 
 jobs:
+  datafusion_producer:
+    name: Run DataFusion producer tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Build & run
+        run: docker run --rm $(docker build -q --file ./ci/docker/datafusion_producer.Dockerfile .)
+  datafusion_consumer:
+    name: Run DataFusion consumer tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Build & run
+        run: docker run --rm $(docker build -q --file ./ci/docker/datafusion_consumer.Dockerfile .)
   duckdb_producer:
     name: Run DuckDB producer tests
     runs-on: ubuntu-latest

ci/docker/datafusion_consumer.Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu:22.04
+
+ENV PYTHONUNBUFFERED=1
+RUN apt-get update && apt-get install -y python3.10 && ln -sf python3 /usr/bin/python
+RUN apt install -y pip
+RUN pip install --upgrade pip setuptools pytest pytest-snapshot substrait pyarrow protobuf duckdb filelock datafusion ibis_substrait JPype1
+
+WORKDIR /substrait_consumer
+COPY . .
+
+CMD /usr/bin/python -mpytest -m consume_substrait_snapshot --consumer=DataFusionConsumer substrait_consumer/tests/functional/extension_functions

ci/docker/datafusion_producer.Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu:22.04
+
+ENV PYTHONUNBUFFERED=1
+RUN apt-get update && apt-get install -y python3.10 && ln -sf python3 /usr/bin/python
+RUN apt install -y pip
+RUN pip install --upgrade pip setuptools pytest pytest-snapshot substrait pyarrow protobuf duckdb filelock datafusion ibis_substrait JPype1
+
+WORKDIR /substrait_consumer
+COPY . .
+
+CMD /usr/bin/python -mpytest -m produce_substrait_snapshot --producer=DataFusionProducer substrait_consumer/tests/functional/extension_functions

substrait_consumer/conftest.py

@@ -5,7 +5,7 @@
 from filelock import FileLock
 
 from substrait_consumer.consumers import (
-    AceroConsumer, DatafusionConsumer, DuckDBConsumer)
+    AceroConsumer, DataFusionConsumer, DuckDBConsumer)
 from substrait_consumer.producers import (
     DataFusionProducer, DuckDBProducer, IbisProducer, IsthmusProducer)
 
@@ -64,7 +64,7 @@ def saveplan(request):
 
 
 PRODUCERS = [DataFusionProducer, DuckDBProducer, IbisProducer, IsthmusProducer]
-CONSUMERS = [DatafusionConsumer, DuckDBConsumer]
+CONSUMERS = [DataFusionConsumer, DuckDBConsumer]
 
 
 def _get_consumers():

substrait_consumer/consumers.py

@@ -158,7 +158,7 @@ def run_substrait_query(self, substrait_query: str) -> pa.Table:
         return result
 
 
-class DatafusionConsumer:
+class DataFusionConsumer:
     """
     Adapts the Datafusion Substrait consumer to the test framework.
     """