From 9aa6cdfa765ac93797cfb197b759135df791ae19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingo=20M=C3=BCller?= Date: Tue, 17 Dec 2024 13:52:42 +0100 Subject: [PATCH] feat: express TPC-H plan generation with existing infra (#172) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR uses `substrait_producer_sql_test` from `common.py` to express the TPC-H test targets that regenerate the Substrait plans using Isthmus. This replaces test-specific code with generic code and is, thus, a steps towards reducing the complexity of the tests. As a consequence, the paths of some snapshots change; however, note in the corresponding git commit that all files are renamed without content changes. Signed-off-by: Ingo Müller --- substrait_consumer/functional/common.py | 2 + .../IsthmusProducer/q01-isthmus_outcome.txt | 1 + .../IsthmusProducer/q01_plan.json} | 0 .../IsthmusProducer/q02-isthmus_outcome.txt | 1 + .../IsthmusProducer/q02_plan.json} | 0 .../IsthmusProducer/q03-isthmus_outcome.txt | 1 + .../IsthmusProducer/q03_plan.json} | 0 .../IsthmusProducer/q04-isthmus_outcome.txt | 1 + .../IsthmusProducer/q04_plan.json} | 0 .../IsthmusProducer/q05-isthmus_outcome.txt | 1 + .../IsthmusProducer/q05_plan.json} | 0 .../IsthmusProducer/q06-isthmus_outcome.txt | 1 + .../IsthmusProducer/q06_plan.json} | 0 .../IsthmusProducer/q07-isthmus_outcome.txt | 1 + .../IsthmusProducer/q07_plan.json} | 0 .../IsthmusProducer/q08-isthmus_outcome.txt | 1 + .../IsthmusProducer/q08_plan.json} | 0 .../IsthmusProducer/q09-isthmus_outcome.txt | 1 + .../IsthmusProducer/q09_plan.json} | 0 .../IsthmusProducer/q10-isthmus_outcome.txt | 1 + .../IsthmusProducer/q10_plan.json} | 0 .../IsthmusProducer/q11-isthmus_outcome.txt | 1 + .../IsthmusProducer/q11_plan.json} | 0 .../IsthmusProducer/q12-isthmus_outcome.txt | 1 + .../IsthmusProducer/q12_plan.json} | 0 .../IsthmusProducer/q13-isthmus_outcome.txt | 1 + .../IsthmusProducer/q13_plan.json} | 0 .../IsthmusProducer/q14-isthmus_outcome.txt | 1 + .../IsthmusProducer/q14_plan.json} | 0 .../IsthmusProducer/q15-isthmus_outcome.txt | 1 + .../IsthmusProducer/q15_plan.json} | 0 .../IsthmusProducer/q16-isthmus_outcome.txt | 1 + .../IsthmusProducer/q16_plan.json} | 0 .../IsthmusProducer/q17-isthmus_outcome.txt | 1 + .../IsthmusProducer/q17_plan.json} | 0 .../IsthmusProducer/q18-isthmus_outcome.txt | 1 + .../IsthmusProducer/q18_plan.json} | 0 .../IsthmusProducer/q19-isthmus_outcome.txt | 1 + .../IsthmusProducer/q19_plan.json} | 0 .../IsthmusProducer/q20-isthmus_outcome.txt | 1 + .../IsthmusProducer/q20_plan.json} | 0 .../IsthmusProducer/q21-isthmus_outcome.txt | 1 + .../IsthmusProducer/q21_plan.json} | 0 .../IsthmusProducer/q22-isthmus_outcome.txt | 1 + .../IsthmusProducer/q22_plan.json} | 0 .../integration/test_isthmus_on_acero.py | 9 +++- .../integration/test_tpch_plans_valid.py | 43 +++++++------------ 47 files changed, 46 insertions(+), 30 deletions(-) create mode 100644 substrait_consumer/tests/functional/tpch/q01_snapshots/IsthmusProducer/q01-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_01_plan.json => functional/tpch/q01_snapshots/IsthmusProducer/q01_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q02_snapshots/IsthmusProducer/q02-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_02_plan.json => functional/tpch/q02_snapshots/IsthmusProducer/q02_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q03_snapshots/IsthmusProducer/q03-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_03_plan.json => functional/tpch/q03_snapshots/IsthmusProducer/q03_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q04_snapshots/IsthmusProducer/q04-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_04_plan.json => functional/tpch/q04_snapshots/IsthmusProducer/q04_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q05_snapshots/IsthmusProducer/q05-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_05_plan.json => functional/tpch/q05_snapshots/IsthmusProducer/q05_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q06_snapshots/IsthmusProducer/q06-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_06_plan.json => functional/tpch/q06_snapshots/IsthmusProducer/q06_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q07_snapshots/IsthmusProducer/q07-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_07_plan.json => functional/tpch/q07_snapshots/IsthmusProducer/q07_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q08_snapshots/IsthmusProducer/q08-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_08_plan.json => functional/tpch/q08_snapshots/IsthmusProducer/q08_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q09_snapshots/IsthmusProducer/q09-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_09_plan.json => functional/tpch/q09_snapshots/IsthmusProducer/q09_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q10_snapshots/IsthmusProducer/q10-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_10_plan.json => functional/tpch/q10_snapshots/IsthmusProducer/q10_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q11_snapshots/IsthmusProducer/q11-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_11_plan.json => functional/tpch/q11_snapshots/IsthmusProducer/q11_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q12_snapshots/IsthmusProducer/q12-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_12_plan.json => functional/tpch/q12_snapshots/IsthmusProducer/q12_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q13_snapshots/IsthmusProducer/q13-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_13_plan.json => functional/tpch/q13_snapshots/IsthmusProducer/q13_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q14_snapshots/IsthmusProducer/q14-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_14_plan.json => functional/tpch/q14_snapshots/IsthmusProducer/q14_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q15_snapshots/IsthmusProducer/q15-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_15_plan.json => functional/tpch/q15_snapshots/IsthmusProducer/q15_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q16_snapshots/IsthmusProducer/q16-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_16_plan.json => functional/tpch/q16_snapshots/IsthmusProducer/q16_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q17_snapshots/IsthmusProducer/q17-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_17_plan.json => functional/tpch/q17_snapshots/IsthmusProducer/q17_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q18_snapshots/IsthmusProducer/q18-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_18_plan.json => functional/tpch/q18_snapshots/IsthmusProducer/q18_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q19_snapshots/IsthmusProducer/q19-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_19_plan.json => functional/tpch/q19_snapshots/IsthmusProducer/q19_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q20_snapshots/IsthmusProducer/q20-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_20_plan.json => functional/tpch/q20_snapshots/IsthmusProducer/q20_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q21_snapshots/IsthmusProducer/q21-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_21_plan.json => functional/tpch/q21_snapshots/IsthmusProducer/q21_plan.json} (100%) create mode 100644 substrait_consumer/tests/functional/tpch/q22_snapshots/IsthmusProducer/q22-isthmus_outcome.txt rename substrait_consumer/tests/{integration/queries/tpch_substrait_plans/query_22_plan.json => functional/tpch/q22_snapshots/IsthmusProducer/q22_plan.json} (100%) diff --git a/substrait_consumer/functional/common.py b/substrait_consumer/functional/common.py index b03e03b9..8cfefefa 100644 --- a/substrait_consumer/functional/common.py +++ b/substrait_consumer/functional/common.py @@ -21,9 +21,11 @@ RELATION_SNAPSHOT_DIR = ( Path(__file__).parent.parent / "tests" / "functional" / "relations" ) +TPCH_SNAPSHOT_DIR = Path(__file__).parent.parent / "tests" / "functional" / "tpch" SNAPSHOT_DIR = { "function": FUNCTION_SNAPSHOT_DIR, "relation": RELATION_SNAPSHOT_DIR, + "tpch": TPCH_SNAPSHOT_DIR, } diff --git a/substrait_consumer/tests/functional/tpch/q01_snapshots/IsthmusProducer/q01-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q01_snapshots/IsthmusProducer/q01-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q01_snapshots/IsthmusProducer/q01-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_01_plan.json b/substrait_consumer/tests/functional/tpch/q01_snapshots/IsthmusProducer/q01_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_01_plan.json rename to substrait_consumer/tests/functional/tpch/q01_snapshots/IsthmusProducer/q01_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q02_snapshots/IsthmusProducer/q02-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q02_snapshots/IsthmusProducer/q02-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q02_snapshots/IsthmusProducer/q02-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_02_plan.json b/substrait_consumer/tests/functional/tpch/q02_snapshots/IsthmusProducer/q02_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_02_plan.json rename to substrait_consumer/tests/functional/tpch/q02_snapshots/IsthmusProducer/q02_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q03_snapshots/IsthmusProducer/q03-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q03_snapshots/IsthmusProducer/q03-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q03_snapshots/IsthmusProducer/q03-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_03_plan.json b/substrait_consumer/tests/functional/tpch/q03_snapshots/IsthmusProducer/q03_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_03_plan.json rename to substrait_consumer/tests/functional/tpch/q03_snapshots/IsthmusProducer/q03_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q04_snapshots/IsthmusProducer/q04-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q04_snapshots/IsthmusProducer/q04-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q04_snapshots/IsthmusProducer/q04-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_04_plan.json b/substrait_consumer/tests/functional/tpch/q04_snapshots/IsthmusProducer/q04_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_04_plan.json rename to substrait_consumer/tests/functional/tpch/q04_snapshots/IsthmusProducer/q04_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q05_snapshots/IsthmusProducer/q05-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q05_snapshots/IsthmusProducer/q05-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q05_snapshots/IsthmusProducer/q05-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_05_plan.json b/substrait_consumer/tests/functional/tpch/q05_snapshots/IsthmusProducer/q05_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_05_plan.json rename to substrait_consumer/tests/functional/tpch/q05_snapshots/IsthmusProducer/q05_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q06_snapshots/IsthmusProducer/q06-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q06_snapshots/IsthmusProducer/q06-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q06_snapshots/IsthmusProducer/q06-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_06_plan.json b/substrait_consumer/tests/functional/tpch/q06_snapshots/IsthmusProducer/q06_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_06_plan.json rename to substrait_consumer/tests/functional/tpch/q06_snapshots/IsthmusProducer/q06_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q07_snapshots/IsthmusProducer/q07-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q07_snapshots/IsthmusProducer/q07-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q07_snapshots/IsthmusProducer/q07-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_07_plan.json b/substrait_consumer/tests/functional/tpch/q07_snapshots/IsthmusProducer/q07_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_07_plan.json rename to substrait_consumer/tests/functional/tpch/q07_snapshots/IsthmusProducer/q07_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q08_snapshots/IsthmusProducer/q08-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q08_snapshots/IsthmusProducer/q08-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q08_snapshots/IsthmusProducer/q08-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_08_plan.json b/substrait_consumer/tests/functional/tpch/q08_snapshots/IsthmusProducer/q08_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_08_plan.json rename to substrait_consumer/tests/functional/tpch/q08_snapshots/IsthmusProducer/q08_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q09_snapshots/IsthmusProducer/q09-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q09_snapshots/IsthmusProducer/q09-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q09_snapshots/IsthmusProducer/q09-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_09_plan.json b/substrait_consumer/tests/functional/tpch/q09_snapshots/IsthmusProducer/q09_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_09_plan.json rename to substrait_consumer/tests/functional/tpch/q09_snapshots/IsthmusProducer/q09_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q10_snapshots/IsthmusProducer/q10-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q10_snapshots/IsthmusProducer/q10-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q10_snapshots/IsthmusProducer/q10-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_10_plan.json b/substrait_consumer/tests/functional/tpch/q10_snapshots/IsthmusProducer/q10_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_10_plan.json rename to substrait_consumer/tests/functional/tpch/q10_snapshots/IsthmusProducer/q10_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q11_snapshots/IsthmusProducer/q11-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q11_snapshots/IsthmusProducer/q11-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q11_snapshots/IsthmusProducer/q11-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_11_plan.json b/substrait_consumer/tests/functional/tpch/q11_snapshots/IsthmusProducer/q11_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_11_plan.json rename to substrait_consumer/tests/functional/tpch/q11_snapshots/IsthmusProducer/q11_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q12_snapshots/IsthmusProducer/q12-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q12_snapshots/IsthmusProducer/q12-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q12_snapshots/IsthmusProducer/q12-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_12_plan.json b/substrait_consumer/tests/functional/tpch/q12_snapshots/IsthmusProducer/q12_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_12_plan.json rename to substrait_consumer/tests/functional/tpch/q12_snapshots/IsthmusProducer/q12_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q13_snapshots/IsthmusProducer/q13-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q13_snapshots/IsthmusProducer/q13-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q13_snapshots/IsthmusProducer/q13-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_13_plan.json b/substrait_consumer/tests/functional/tpch/q13_snapshots/IsthmusProducer/q13_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_13_plan.json rename to substrait_consumer/tests/functional/tpch/q13_snapshots/IsthmusProducer/q13_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q14_snapshots/IsthmusProducer/q14-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q14_snapshots/IsthmusProducer/q14-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q14_snapshots/IsthmusProducer/q14-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_14_plan.json b/substrait_consumer/tests/functional/tpch/q14_snapshots/IsthmusProducer/q14_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_14_plan.json rename to substrait_consumer/tests/functional/tpch/q14_snapshots/IsthmusProducer/q14_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q15_snapshots/IsthmusProducer/q15-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q15_snapshots/IsthmusProducer/q15-isthmus_outcome.txt new file mode 100644 index 00000000..3f774b33 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q15_snapshots/IsthmusProducer/q15-isthmus_outcome.txt @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_15_plan.json b/substrait_consumer/tests/functional/tpch/q15_snapshots/IsthmusProducer/q15_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_15_plan.json rename to substrait_consumer/tests/functional/tpch/q15_snapshots/IsthmusProducer/q15_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q16_snapshots/IsthmusProducer/q16-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q16_snapshots/IsthmusProducer/q16-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q16_snapshots/IsthmusProducer/q16-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_16_plan.json b/substrait_consumer/tests/functional/tpch/q16_snapshots/IsthmusProducer/q16_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_16_plan.json rename to substrait_consumer/tests/functional/tpch/q16_snapshots/IsthmusProducer/q16_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q17_snapshots/IsthmusProducer/q17-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q17_snapshots/IsthmusProducer/q17-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q17_snapshots/IsthmusProducer/q17-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_17_plan.json b/substrait_consumer/tests/functional/tpch/q17_snapshots/IsthmusProducer/q17_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_17_plan.json rename to substrait_consumer/tests/functional/tpch/q17_snapshots/IsthmusProducer/q17_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q18_snapshots/IsthmusProducer/q18-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q18_snapshots/IsthmusProducer/q18-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q18_snapshots/IsthmusProducer/q18-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_18_plan.json b/substrait_consumer/tests/functional/tpch/q18_snapshots/IsthmusProducer/q18_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_18_plan.json rename to substrait_consumer/tests/functional/tpch/q18_snapshots/IsthmusProducer/q18_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q19_snapshots/IsthmusProducer/q19-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q19_snapshots/IsthmusProducer/q19-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q19_snapshots/IsthmusProducer/q19-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_19_plan.json b/substrait_consumer/tests/functional/tpch/q19_snapshots/IsthmusProducer/q19_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_19_plan.json rename to substrait_consumer/tests/functional/tpch/q19_snapshots/IsthmusProducer/q19_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q20_snapshots/IsthmusProducer/q20-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q20_snapshots/IsthmusProducer/q20-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q20_snapshots/IsthmusProducer/q20-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_20_plan.json b/substrait_consumer/tests/functional/tpch/q20_snapshots/IsthmusProducer/q20_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_20_plan.json rename to substrait_consumer/tests/functional/tpch/q20_snapshots/IsthmusProducer/q20_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q21_snapshots/IsthmusProducer/q21-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q21_snapshots/IsthmusProducer/q21-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q21_snapshots/IsthmusProducer/q21-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_21_plan.json b/substrait_consumer/tests/functional/tpch/q21_snapshots/IsthmusProducer/q21_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_21_plan.json rename to substrait_consumer/tests/functional/tpch/q21_snapshots/IsthmusProducer/q21_plan.json diff --git a/substrait_consumer/tests/functional/tpch/q22_snapshots/IsthmusProducer/q22-isthmus_outcome.txt b/substrait_consumer/tests/functional/tpch/q22_snapshots/IsthmusProducer/q22-isthmus_outcome.txt new file mode 100644 index 00000000..4791ed55 --- /dev/null +++ b/substrait_consumer/tests/functional/tpch/q22_snapshots/IsthmusProducer/q22-isthmus_outcome.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_22_plan.json b/substrait_consumer/tests/functional/tpch/q22_snapshots/IsthmusProducer/q22_plan.json similarity index 100% rename from substrait_consumer/tests/integration/queries/tpch_substrait_plans/query_22_plan.json rename to substrait_consumer/tests/functional/tpch/q22_snapshots/IsthmusProducer/q22_plan.json diff --git a/substrait_consumer/tests/integration/test_isthmus_on_acero.py b/substrait_consumer/tests/integration/test_isthmus_on_acero.py index 70690edf..12182dcc 100644 --- a/substrait_consumer/tests/integration/test_isthmus_on_acero.py +++ b/substrait_consumer/tests/integration/test_isthmus_on_acero.py @@ -8,7 +8,7 @@ from substrait_consumer.functional.utils import load_json from substrait_consumer.producers.duckdb_producer import DuckDBProducer -PLAN_DIR = Path(__file__).parent / "queries" / "tpch_substrait_plans" +PLAN_DIR = Path(__file__).parent.parent / "functional" / "tpch" CONFIG_DIR = Path(__file__).parent.parent / "integration" TPCH_CONFIG_DIR = CONFIG_DIR / "tpch" @@ -68,7 +68,12 @@ def test_isthmus_substrait_plan( outcome_path = f"query_{tpch_num:02d}_outcome.txt" # Load Isthmus plan from file. - substrait_plan_path = PLAN_DIR / f"query_{tpch_num:02d}_plan.json" + substrait_plan_path = ( + PLAN_DIR + / f"q{tpch_num:02d}_snapshots" + / "IsthmusProducer" + / f"q{tpch_num:02d}_plan.json" + ) with open(substrait_plan_path, "r") as f: proto_bytes = f.read() diff --git a/substrait_consumer/tests/integration/test_tpch_plans_valid.py b/substrait_consumer/tests/integration/test_tpch_plans_valid.py index 96584c4e..b191912e 100644 --- a/substrait_consumer/tests/integration/test_tpch_plans_valid.py +++ b/substrait_consumer/tests/integration/test_tpch_plans_valid.py @@ -4,15 +4,11 @@ import pytest from pytest_snapshot.plugin import Snapshot -from substrait_consumer.functional.common import check_match +from substrait_consumer.functional.common import substrait_producer_sql_test from substrait_consumer.functional.utils import load_json from substrait_consumer.producers.duckdb_producer import DuckDBProducer from substrait_consumer.producers.isthmus_producer import IsthmusProducer -PLAN_SNAPSHOT_DIR = ( - Path(__file__).parent / "queries" / "tpch_substrait_plans" -) - CONFIG_DIR = Path(__file__).parent.parent / "integration" TPCH_CONFIG_DIR = CONFIG_DIR / "tpch" TEST_CASE_PATHS = list( @@ -23,38 +19,29 @@ @pytest.mark.parametrize(["path"], TEST_CASE_PATHS, ids=IDS) @pytest.mark.usefixtures("prepare_tpch_parquet_data") +@pytest.mark.produce_substrait_snapshot def test_isthmus_substrait_plan_generation( path: Path, snapshot: Snapshot, + record_property, db_con: duckdb.DuckDBPyConnection, -) -> None: - """ - Generate the substrait plans using Isthmus. - """ +): test_case = load_json(CONFIG_DIR / path) - test_name = test_case["test_name"] local_files = test_case["local_files"] named_tables = test_case["named_tables"] - sql_query, supported_producers = test_case["sql_query"] - - assert "isthmus" in supported_producers - - tpch_num = test_name.split("_")[-1].zfill(2) - snapshot.snapshot_dir = PLAN_SNAPSHOT_DIR - + sql_query = test_case["sql_query"] producer = IsthmusProducer() - producer.setup(db_con, local_files, named_tables) - - try: - substrait_query = producer.produce_substrait(sql_query) - except BaseException as e: - snapshot.assert_match(str(type(e)), f"query_{tpch_num}_outcome.txt") - return - - match_result = check_match( - snapshot, str(substrait_query), f"query_{tpch_num}_plan.json" + substrait_producer_sql_test( + path, + snapshot, + record_property, + db_con, + local_files, + named_tables, + sql_query, + producer, + validate=False, ) - snapshot.assert_match(str(match_result), f"query_{tpch_num}_outcome.txt") @pytest.mark.parametrize(["path"], TEST_CASE_PATHS, ids=IDS)