feat: merge entry scripts tests (#185)

This PR merges all entry test scripts of the tests that are wrappers
around the same functions in `common.py`. In particular, all tests using
`substrait_consumer_sql_test` are now being run from
`test_consumers.py`, those using `substrait_producer_sql_test` are being
called from `test_sql_producers.py`, and those using
`generate_snapshot_results` are being called from
`test_generate_results.py`. Since the tests are now run by different
scripts, the CI targets get adapted accordingly.

Signed-off-by: Ingo Müller <[email protected]>

.github/workflows/support_matrix.yml

@@ -56,8 +56,8 @@ jobs:
       run: |-
         pytest -v --tb=no \
             -m ${{ matrix.target }}_substrait_snapshot \
-            substrait_consumer/tests/functional/extension_functions/ \
-            substrait_consumer/functional/queries/ibis_expressions/ \
+            -k "function/ or ibis_expressions" \
+            substrait_consumer/ \
             --csv ${{ matrix.target }}r_pytest_output.csv --csv-delimiter ';' \
             --csv-columns 'id,status,properties_as_columns'
 

.github/workflows/test.yml

@@ -26,11 +26,8 @@ jobs:
     - name: Install Python dependencies
       run: pip install -r requirements.txt
 
-    - name: Run functions tests
-      run: pytest -v -m consume_substrait_snapshot --consumer=${{ matrix.consumer }} substrait_consumer/tests/functional/extension_functions
-
-    - name: Run relations tests
-      run: pytest -v -m consume_substrait_snapshot --consumer=${{ matrix.consumer }} substrait_consumer/tests/functional/relations
+    - name: Run tests for ${{ matrix.consumer }} consumer
+      run: pytest -v -m consume_substrait_snapshot --consumer=${{ matrix.consumer }} substrait_consumer/
 
   producers:
     name: Run producer tests
@@ -78,53 +75,9 @@ jobs:
         mkdir ./jars
         cp substrait-java/isthmus/build/libs/*all.jar ./jars
 
-    - name: Run functions tests
-      run: pytest -v -m produce_substrait_snapshot --producer=${{ matrix.producer }} substrait_consumer/tests/functional/extension_functions
-
-    - name: Run relations tests
-      run: pytest -v -m produce_substrait_snapshot --producer=${{ matrix.producer }} substrait_consumer/tests/functional/relations
+    - name: Run tests for ${{ matrix.producer }} producer
+      run: pytest -v -m produce_substrait_snapshot --producer=${{ matrix.producer }} substrait_consumer/
 
     - name: Run Ibis expression tests
       if: ${{ matrix.producer == 'ibis' }}
       run: pytest substrait_consumer/functional/queries/ibis_expressions/
-
-  integration:
-    name: Run integration tests
-    runs-on: ubuntu-22.04
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-      with:
-        submodules: recursive
-
-    - name: Install Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-        cache: 'pip'
-
-    - name: Install Python dependencies
-      run: pip install -r requirements.txt
-
-    - name: Install Java
-      uses: actions/setup-java@v4
-      with:
-        distribution: 'temurin'
-        java-version: '17'
-
-    - name: Setup Gradle
-      uses: gradle/actions/setup-gradle@v4
-      with:
-        gradle-version: 8.8
-
-    - name: Build Isthmus
-      working-directory: ./substrait-java/isthmus
-      run: ../gradlew shadowJar
-
-    - name: Copy Isthmus JARs
-      run: |-
-        mkdir ./jars
-        cp substrait-java/isthmus/build/libs/*all.jar ./jars
-
-    - name: Run integration tests
-      run: pytest -v substrait_consumer/tests/integration

substrait_consumer/tests/functional/extension_functions/test_consumers.py → substrait_consumer/test_consumers.py

@@ -4,23 +4,32 @@
 import pytest
 from pytest_snapshot.plugin import Snapshot
 
+from substrait_consumer.consumers.acero_consumer import AceroConsumer
 from substrait_consumer.consumers.datafusion_consumer import DataFusionConsumer
 from substrait_consumer.functional.utils import load_json
 from substrait_consumer.functional.common import substrait_consumer_sql_test
+from substrait_consumer.producers.isthmus_producer import IsthmusProducer
 
-CONFIG_DIR = Path(__file__).parent.parent.parent.parent / "testdata"
-FUNCTION_CONFIG_DIR = CONFIG_DIR / "function"
+CONFIG_DIR = Path(__file__).parent / "testdata"
 TEST_CASE_PATHS = list(
-    (path.relative_to(CONFIG_DIR),) for path in FUNCTION_CONFIG_DIR.rglob("*.json")
+    (path.relative_to(CONFIG_DIR),) for path in CONFIG_DIR.rglob("*.json")
 )
 IDS = (str(path[0]).removesuffix(".json") for path in TEST_CASE_PATHS)
 
 
 @pytest.fixture
 def mark_consumer_tests_as_xfail(request):
     """Marks a subset of tests as expected to be fail."""
+    producer = request.getfixturevalue("producer")
     consumer = request.getfixturevalue("consumer")
     path = request.getfixturevalue("path")
+    if isinstance(consumer, AceroConsumer):
+        if (
+            isinstance(producer, IsthmusProducer)
+        ) and "compute_within_aggregate" in str(path):
+            pytest.skip(
+                reason="'isthmus-acero-compute_within_aggregate' currently crashes"
+            )
     if isinstance(consumer, DataFusionConsumer):
         flaky_tests = [
             "arithmetic_decimal/scalar/add",

substrait_consumer/tests/functional/relations/test_generate_results.py → substrait_consumer/test_generate_results.py

@@ -7,10 +7,9 @@
 from substrait_consumer.functional.utils import load_json
 from substrait_consumer.functional.common import generate_snapshot_results
 
-CONFIG_DIR = Path(__file__).parent.parent.parent.parent / "testdata"
-RELATION_CONFIG_DIR = CONFIG_DIR / "relation"
+CONFIG_DIR = Path(__file__).parent / "testdata"
 TEST_CASE_PATHS = list(
-    (path.relative_to(CONFIG_DIR),) for path in RELATION_CONFIG_DIR.rglob("*.json")
+    (path.relative_to(CONFIG_DIR),) for path in CONFIG_DIR.rglob("*.json")
 )
 IDS = (str(path[0]).removesuffix(".json") for path in TEST_CASE_PATHS)
 

substrait_consumer/tests/functional/relations/test_sql_producers.py → substrait_consumer/test_sql_producers.py

@@ -7,10 +7,9 @@
 from substrait_consumer.functional.utils import load_json
 from substrait_consumer.functional.common import substrait_producer_sql_test
 
-CONFIG_DIR = Path(__file__).parent.parent.parent.parent / "testdata"
-RELATION_CONFIG_DIR = CONFIG_DIR / "relation"
+CONFIG_DIR = Path(__file__).parent / "testdata"
 TEST_CASE_PATHS = list(
-    (path.relative_to(CONFIG_DIR),) for path in RELATION_CONFIG_DIR.rglob("*.json")
+    (path.relative_to(CONFIG_DIR),) for path in CONFIG_DIR.rglob("*.json")
 )
 IDS = list(str(path[0]).removesuffix(".json") for path in TEST_CASE_PATHS)