diff --git a/tests/functional/adapter/test_icerberg_format.py b/tests/functional/adapter/test_iceberg_format.py similarity index 73% rename from tests/functional/adapter/test_icerberg_format.py rename to tests/functional/adapter/test_iceberg_format.py index 7bfcf18..79aeb44 100644 --- a/tests/functional/adapter/test_icerberg_format.py +++ b/tests/functional/adapter/test_iceberg_format.py @@ -14,25 +14,21 @@ import pytest -from dbt.tests.util import ( - run_dbt, - check_result_nodes_by_name, - relation_from_name, - check_relation_types, - check_relations_equal, -) - -from tests.functional.adapter.test_basic import ( - TestSimpleMaterializationsImpala, - TestIncrementalImpala, -) - from dbt.tests.adapter.basic.files import ( model_base, model_incremental, base_view_sql, schema_base_yml, ) +from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations +from dbt.tests.adapter.basic.test_incremental import BaseIncremental + +from dbt.tests.util import ( + run_dbt, + check_result_nodes_by_name, + relation_from_name, + check_relation_types, +) def is_iceberg_table(project, tableName): @@ -60,6 +56,18 @@ def is_iceberg_table(project, tableName): + model_base ) +iceberg_base_v2_table_sql = ( + """ +{{ + config( + materialized="table", + table_type="iceberg", + tbl_properties="('format-version'='2')", + ) +}}""".strip() + + model_base +) + iceberg_base_materialized_var_sql = ( """ {{ @@ -73,7 +81,7 @@ def is_iceberg_table(project, tableName): incremental_iceberg_sql = ( """ - {{ +{{ config( materialized="incremental", table_type="iceberg" @@ -83,6 +91,19 @@ def is_iceberg_table(project, tableName): + model_incremental ) +incremental_iceberg_v2_sql = ( + """ +{{ + config( + materialized="incremental", + table_type="iceberg", + tbl_properties="('format-version'='2')", + ) +}} +""".strip() + + model_incremental +) + incremental_partition_iceberg_sql = """ {{ @@ -103,7 +124,22 @@ def is_iceberg_table(project, tableName): config( materialized="incremental", partition_by=["id_partition1", "id_partition2"], - table_type="iceberg" + table_type="iceberg", + ) +}} +select *, id as id_partition1, id as id_partition2 from {{ source('raw', 'seed') }} +{% if is_incremental() %} + where id > (select max(id) from {{ this }}) +{% endif %} +""".strip() + +incremental_multiple_partition_iceberg_v2_sql = """ + {{ + config( + materialized="incremental", + partition_by=["id_partition1", "id_partition2"], + table_type="iceberg", + tbl_properties="('format-version'='2')", ) }} select *, id as id_partition1, id as id_partition2 from {{ source('raw', 'seed') }} @@ -128,10 +164,27 @@ def is_iceberg_table(project, tableName): """.strip() +insertoverwrite_iceberg_v2_sql = """ + {{ + config( + materialized="incremental", + incremental_strategy="insert_overwrite", + partition_by="id_partition1", + table_type="iceberg", + tbl_properties="('format-version'='2')", + ) +}} +select *, id as id_partition1 from {{ source('raw', 'seed') }} +{% if is_incremental() %} + where id > (select max(id) from {{ this }}) +{% endif %} +""".strip() + + # For iceberg table formats, check_relations_equal util is not working as expected # Impala upstream issue: https://issues.apache.org/jira/browse/IMPALA-12097 # Hence removing this check from unit tests -class TestSimpleMaterializationsIcebergFormatImpala(TestSimpleMaterializationsImpala): +class TestSimpleMaterializationsIcebergImpala(BaseSimpleMaterializations): @pytest.fixture(scope="class") def models(self): return { @@ -211,7 +264,18 @@ def test_base(self, project): check_relation_types(project.adapter, expected) -class TestIncrementalIcebergFormatImpala(TestIncrementalImpala): +class TestSimpleMaterializationsIcebergV2Impala(TestSimpleMaterializationsIcebergImpala): + @pytest.fixture(scope="class") + def models(self): + return { + "view_model.sql": base_view_sql, + "table_model.sql": iceberg_base_v2_table_sql, + "swappable.sql": iceberg_base_materialized_var_sql, + "schema.yml": schema_base_yml, + } + + +class TestIncrementalIcebergImpala(BaseIncremental): @pytest.fixture(scope="class") def models(self): return { @@ -268,7 +332,16 @@ def test_incremental(self, project): assert len(catalog.sources) == 1 -class TestIncrementalPartitionIcebergFormatImpala(TestIncrementalIcebergFormatImpala): +class TestIncrementalIcebergV2Impala(TestIncrementalIcebergImpala): + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_test_model.sql": incremental_iceberg_v2_sql, + "schema.yml": schema_base_yml, + } + + +class TestIncrementalPartitionIcebergImpala(TestIncrementalIcebergImpala): @pytest.fixture(scope="class") def models(self): return { @@ -277,7 +350,7 @@ def models(self): } -class TestIncrementalMultiplePartitionIcebergFormatImpala(TestIncrementalIcebergFormatImpala): +class TestIncrementalMultiplePartitionIcebergImpala(TestIncrementalIcebergImpala): @pytest.fixture(scope="class") def models(self): return { @@ -286,10 +359,28 @@ def models(self): } -class TestInsertoverwriteIcebergFormatImpala(TestIncrementalIcebergFormatImpala): +class TestIncrementalMultiplePartitionIcebergV2Impala(TestIncrementalIcebergImpala): + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_test_model.sql": incremental_multiple_partition_iceberg_v2_sql, + "schema.yml": schema_base_yml, + } + + +class TestInsertoverwriteIcebergImpala(TestIncrementalIcebergImpala): @pytest.fixture(scope="class") def models(self): return { "incremental_test_model.sql": insertoverwrite_iceberg_sql, "schema.yml": schema_base_yml, } + + +class TestInsertoverwriteIcebergV2Impala(TestIncrementalIcebergImpala): + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_test_model.sql": insertoverwrite_iceberg_v2_sql, + "schema.yml": schema_base_yml, + }