From 81b713b968820027233080ea47c2fe32ec67eca1 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 16 Apr 2024 14:04:07 -0500 Subject: [PATCH 01/17] initial push of adding cache functions for materialized views --- .../macros/materializations/models/materialized_view.sql | 2 ++ plugins/postgres/dbt/include/postgres/macros/adapters.sql | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view.sql index 6dc30bf9a9a..ded11c93855 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view.sql @@ -109,6 +109,8 @@ {{ build_sql }} {% endcall %} + {{ adapter.rename_relation(existing_relation, target_relation) }} + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} diff --git a/plugins/postgres/dbt/include/postgres/macros/adapters.sql b/plugins/postgres/dbt/include/postgres/macros/adapters.sql index ee864e9b7c5..294443be272 100644 --- a/plugins/postgres/dbt/include/postgres/macros/adapters.sql +++ b/plugins/postgres/dbt/include/postgres/macros/adapters.sql @@ -39,7 +39,7 @@ on {{ relation }} {% if index_config.type -%} using {{ index_config.type }} {%- endif %} - ({{ comma_separated_columns }}); + ({{ comma_separated_columns }}) {%- endmacro %} {% macro postgres__create_schema(relation) -%} From 86d07e4b5910ab96ef7462ab1275334532f3a16e Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 16 Apr 2024 15:57:36 -0500 Subject: [PATCH 02/17] update --- .../macros/materializations/models/materialized_view.sql | 2 -- .../global_project/macros/relations/materialized_view/drop.sql | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/dbt/include/global_project/macros/materializations/models/materialized_view.sql b/core/dbt/include/global_project/macros/materializations/models/materialized_view.sql index ded11c93855..6dc30bf9a9a 100644 --- a/core/dbt/include/global_project/macros/materializations/models/materialized_view.sql +++ b/core/dbt/include/global_project/macros/materializations/models/materialized_view.sql @@ -109,8 +109,6 @@ {{ build_sql }} {% endcall %} - {{ adapter.rename_relation(existing_relation, target_relation) }} - {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql index e60e1dc24d0..1d0d1fa11ef 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql @@ -6,6 +6,7 @@ actually executes the drop, and `get_drop_sql`, which returns the template. {% macro drop_materialized_view(relation) -%} {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} + {{ adapter.cache_dropped(relation) }} {%- endmacro %} From 9f400fd804e1b45073b504b202d84c5c2e83af75 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 16 Apr 2024 16:01:01 -0500 Subject: [PATCH 03/17] add cache_renamed --- .../macros/relations/materialized_view/rename.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql index abd5babf68e..0ffe9ab4777 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql @@ -1,5 +1,9 @@ {% macro get_rename_materialized_view_sql(relation, new_name) %} + {%- set to_relation = adapter.get_relation(database=relation.database, schema=relation.schema, identifier=new_name) %} {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}} + {%- if relation is not none and to_relation is not none -%} + {{ adapter.cache_renamed(from_relation=relation, to_relation=to_relation) }} + {%- endif -%} {% endmacro %} From 342d54b0576df1c060987e4f2424bb2a87ef89de Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 17 Apr 2024 11:56:11 -0500 Subject: [PATCH 04/17] remove current attempt of add cache_renamed --- .../macros/relations/materialized_view/rename.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql index 0ffe9ab4777..abd5babf68e 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql @@ -1,9 +1,5 @@ {% macro get_rename_materialized_view_sql(relation, new_name) %} - {%- set to_relation = adapter.get_relation(database=relation.database, schema=relation.schema, identifier=new_name) %} {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}} - {%- if relation is not none and to_relation is not none -%} - {{ adapter.cache_renamed(from_relation=relation, to_relation=to_relation) }} - {%- endif -%} {% endmacro %} From 51231a6c7dbfa37d8884f4a6d9eeeb0308a28ad2 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Wed, 17 Apr 2024 15:44:12 -0500 Subject: [PATCH 05/17] add checks for add, and drop to see if relation exists in dispatch macro --- .../macros/relations/materialized_view/create.sql | 3 +++ .../macros/relations/materialized_view/drop.sql | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql index a0f2614c9ca..5f78c5a4119 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql @@ -1,4 +1,7 @@ {% macro get_create_materialized_view_as_sql(relation, sql) -%} + {% if relation is not none %} + {{ adapter.cache_added(relation) }} + {% endif %} {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}} {%- endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql index 1d0d1fa11ef..28f53550fab 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql @@ -5,8 +5,10 @@ actually executes the drop, and `get_drop_sql`, which returns the template. */ #} {% macro drop_materialized_view(relation) -%} + {% if relation is not none %} + {{ adapter.cache_dropped(relation) }} + {% endif %} {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} - {{ adapter.cache_dropped(relation) }} {%- endmacro %} From 28e1f47d7cf926eb25ed4f48d1a3bb0e23fcd4b6 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Thu, 18 Apr 2024 13:19:35 -0500 Subject: [PATCH 06/17] move calls from distpach macro into the default call/postgres specific versions incase we are not catching the dispatch as a layer of call or in drop case of it being a explicit return call --- .../macros/relations/materialized_view/create.sql | 4 ---- .../macros/relations/materialized_view/drop.sql | 4 +--- .../postgres/macros/relations/materialized_view/create.sql | 4 ++++ .../postgres/macros/relations/materialized_view/drop.sql | 1 + 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql index 5f78c5a4119..e8bca49a8c9 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql @@ -1,11 +1,7 @@ {% macro get_create_materialized_view_as_sql(relation, sql) -%} - {% if relation is not none %} - {{ adapter.cache_added(relation) }} - {% endif %} {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}} {%- endmacro %} - {% macro default__get_create_materialized_view_as_sql(relation, sql) -%} {{ exceptions.raise_compiler_error( "`get_create_materialized_view_as_sql` has not been implemented for this adapter." diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql index 28f53550fab..6c9b3a8b76c 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql @@ -5,13 +5,11 @@ actually executes the drop, and `get_drop_sql`, which returns the template. */ #} {% macro drop_materialized_view(relation) -%} - {% if relation is not none %} - {{ adapter.cache_dropped(relation) }} - {% endif %} {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} {%- endmacro %} {% macro default__drop_materialized_view(relation) -%} drop materialized view if exists {{ relation }} cascade + {{ adapter.cache_dropped(relation) }} {%- endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql index 17e5cb06434..6d18459f872 100644 --- a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql @@ -1,6 +1,10 @@ {% macro postgres__get_create_materialized_view_as_sql(relation, sql) %} create materialized view if not exists {{ relation }} as {{ sql }}; + {% if execute %} + {{ adapter.cache_added(relation) }} + {% endif %} + {% for _index_dict in config.get('indexes', []) -%} {{- get_create_index_sql(relation, _index_dict) -}} {%- endfor -%} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/drop.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/drop.sql index 2263bb652b2..521b31d98c0 100644 --- a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/drop.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/drop.sql @@ -1,3 +1,4 @@ {% macro postgres__drop_materialized_view(relation) -%} drop materialized view if exists {{ relation }} cascade + {{ adapter.cache_dropped(relation) }} {%- endmacro %} From 5ed2d74a6f00542673616e60475ee76a5ffe0062 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Thu, 18 Apr 2024 14:52:10 -0500 Subject: [PATCH 07/17] split up relation into parts to build up to_relation --- .../macros/relations/materialized_view/rename.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql index 293ec9d1e12..f5bc4574405 100644 --- a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql @@ -1,3 +1,11 @@ {% macro postgres__get_rename_materialized_view_sql(relation, new_name) %} + {% if relation is not none %} + {% set database = relation.database %} + {% set schema = relation.schema %} + {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %} + {% if to_relation is not none %} + {{ adapter.rename_relation(from_relation=relation, to_relation=to_relation) }} + {% endif %} + {% endif %} alter materialized view {{ relation }} rename to {{ new_name }} {% endmacro %} From 7e56792c564f7cf0c8a88d82764a850245ef6068 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 19 Apr 2024 00:45:23 -0500 Subject: [PATCH 08/17] move cache_dropped back to the dispatch original dispatch macro --- .../global_project/macros/relations/materialized_view/drop.sql | 2 +- .../postgres/macros/relations/materialized_view/drop.sql | 1 - .../postgres/macros/relations/materialized_view/rename.sql | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql index 6c9b3a8b76c..a0771735ca9 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql @@ -5,11 +5,11 @@ actually executes the drop, and `get_drop_sql`, which returns the template. */ #} {% macro drop_materialized_view(relation) -%} + {{ adapter.cache_dropped(relation) }} {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} {%- endmacro %} {% macro default__drop_materialized_view(relation) -%} drop materialized view if exists {{ relation }} cascade - {{ adapter.cache_dropped(relation) }} {%- endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/drop.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/drop.sql index 521b31d98c0..2263bb652b2 100644 --- a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/drop.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/drop.sql @@ -1,4 +1,3 @@ {% macro postgres__drop_materialized_view(relation) -%} drop materialized view if exists {{ relation }} cascade - {{ adapter.cache_dropped(relation) }} {%- endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql index f5bc4574405..14703a038fd 100644 --- a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql @@ -3,6 +3,7 @@ {% set database = relation.database %} {% set schema = relation.schema %} {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %} + {{ debug() }} {% if to_relation is not none %} {{ adapter.rename_relation(from_relation=relation, to_relation=to_relation) }} {% endif %} From d6f61dcb24de1598b2cf46428542f03b994aefb0 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 19 Apr 2024 00:52:02 -0500 Subject: [PATCH 09/17] move cache_dropped back to the dispatch original dispatch macro --- .../postgres/macros/relations/materialized_view/rename.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql index 14703a038fd..8980d9220c2 100644 --- a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql @@ -2,8 +2,7 @@ {% if relation is not none %} {% set database = relation.database %} {% set schema = relation.schema %} - {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %} - {{ debug() }} + {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %}} {% if to_relation is not none %} {{ adapter.rename_relation(from_relation=relation, to_relation=to_relation) }} {% endif %} From 70049f5b94dcd686e687046add022afb2151b615 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 19 Apr 2024 14:28:43 -0500 Subject: [PATCH 10/17] readd removed space between macros --- .../macros/relations/materialized_view/create.sql | 1 + .../macros/relations/materialized_view/rename.sql | 8 ++++++++ .../macros/relations/materialized_view/rename.sql | 8 -------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql index e8bca49a8c9..a0f2614c9ca 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql @@ -2,6 +2,7 @@ {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}} {%- endmacro %} + {% macro default__get_create_materialized_view_as_sql(relation, sql) -%} {{ exceptions.raise_compiler_error( "`get_create_materialized_view_as_sql` has not been implemented for this adapter." diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql index abd5babf68e..5a9406c3ee0 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql @@ -1,4 +1,12 @@ {% macro get_rename_materialized_view_sql(relation, new_name) %} + {% if relation is not none %} + {% set database = relation.database %} + {% set schema = relation.schema %} + {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %}} + {% if to_relation is not none %} + {{ adapter.rename_relation(from_relation=relation, to_relation=to_relation) }} + {% endif %} + {% endif %} {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}} {% endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql index 8980d9220c2..293ec9d1e12 100644 --- a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/rename.sql @@ -1,11 +1,3 @@ {% macro postgres__get_rename_materialized_view_sql(relation, new_name) %} - {% if relation is not none %} - {% set database = relation.database %} - {% set schema = relation.schema %} - {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %}} - {% if to_relation is not none %} - {{ adapter.rename_relation(from_relation=relation, to_relation=to_relation) }} - {% endif %} - {% endif %} alter materialized view {{ relation }} rename to {{ new_name }} {% endmacro %} From 64c9cabd81960cd7e0bf2fc98b17ff38fd31a037 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 19 Apr 2024 14:30:48 -0500 Subject: [PATCH 11/17] move cache_added back to distpach macro --- .../macros/relations/materialized_view/create.sql | 3 +++ .../postgres/macros/relations/materialized_view/create.sql | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql index a0f2614c9ca..4d116b3b4b7 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql @@ -1,4 +1,7 @@ {% macro get_create_materialized_view_as_sql(relation, sql) -%} + {% if execute %} + {{ adapter.cache_added(relation) }} + {% endif %} {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}} {%- endmacro %} diff --git a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql index 6d18459f872..17e5cb06434 100644 --- a/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql +++ b/plugins/postgres/dbt/include/postgres/macros/relations/materialized_view/create.sql @@ -1,10 +1,6 @@ {% macro postgres__get_create_materialized_view_as_sql(relation, sql) %} create materialized view if not exists {{ relation }} as {{ sql }}; - {% if execute %} - {{ adapter.cache_added(relation) }} - {% endif %} - {% for _index_dict in config.get('indexes', []) -%} {{- get_create_index_sql(relation, _index_dict) -}} {%- endfor -%} From 39f88feb9ff977279e6ec82c25c2af9bfa3eb7de Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 19 Apr 2024 14:32:32 -0500 Subject: [PATCH 12/17] remove extra curly brace --- .../macros/relations/materialized_view/rename.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql index 5a9406c3ee0..7ac8d25ae21 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql @@ -2,7 +2,7 @@ {% if relation is not none %} {% set database = relation.database %} {% set schema = relation.schema %} - {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %}} + {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %} {% if to_relation is not none %} {{ adapter.rename_relation(from_relation=relation, to_relation=to_relation) }} {% endif %} From 5f0f049b2e56680c3074486056c8480b1de77a66 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 19 Apr 2024 14:35:13 -0500 Subject: [PATCH 13/17] revert get_relation call back get cache_renamed --- .../macros/relations/materialized_view/rename.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql index 7ac8d25ae21..b5ea2807743 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql @@ -1,10 +1,10 @@ {% macro get_rename_materialized_view_sql(relation, new_name) %} + {% set database = relation.database %} + {% set schema = relation.schema %} + {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %} {% if relation is not none %} - {% set database = relation.database %} - {% set schema = relation.schema %} - {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %} {% if to_relation is not none %} - {{ adapter.rename_relation(from_relation=relation, to_relation=to_relation) }} + {{ adapter.cache_renamed(from_relation=relation, to_relation=to_relation) }} {% endif %} {% endif %} {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}} From 96bd884cb233b7defebacad8d6d9f830ca1a9c99 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 23 Apr 2024 11:26:26 -0500 Subject: [PATCH 14/17] move of cache_renamed to get_rename_sql --- .../macros/relations/materialized_view/rename.sql | 8 -------- .../include/global_project/macros/relations/rename.sql | 8 ++++++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql index b5ea2807743..abd5babf68e 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/rename.sql @@ -1,12 +1,4 @@ {% macro get_rename_materialized_view_sql(relation, new_name) %} - {% set database = relation.database %} - {% set schema = relation.schema %} - {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %} - {% if relation is not none %} - {% if to_relation is not none %} - {{ adapter.cache_renamed(from_relation=relation, to_relation=to_relation) }} - {% endif %} - {% endif %} {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}} {% endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/rename.sql b/core/dbt/include/global_project/macros/relations/rename.sql index d7f3a72e29a..087d0912f83 100644 --- a/core/dbt/include/global_project/macros/relations/rename.sql +++ b/core/dbt/include/global_project/macros/relations/rename.sql @@ -1,4 +1,12 @@ {%- macro get_rename_sql(relation, new_name) -%} + {% set database = relation.database %} + {% set schema = relation.schema %} + {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %} + {% if relation is not none %} + {% if to_relation is not none %} + {{ adapter.cache_renamed(from_relation=relation, to_relation=to_relation) }} + {% endif %} + {% endif %} {{- log('Applying RENAME to: ' ~ relation) -}} {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}} {%- endmacro -%} From 8352986bed25ede92c4b990ba0868b8db5f48f6f Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 23 Apr 2024 17:22:29 -0500 Subject: [PATCH 15/17] remove updated cache_changes and remove Relation.MaterializedView from renabmeable (takes us back to previous functionality) --- .../macros/relations/materialized_view/create.sql | 3 --- .../macros/relations/materialized_view/drop.sql | 1 - .../include/global_project/macros/relations/rename.sql | 8 -------- plugins/postgres/dbt/adapters/postgres/relation.py | 1 - 4 files changed, 13 deletions(-) diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql index 4d116b3b4b7..a0f2614c9ca 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/create.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/create.sql @@ -1,7 +1,4 @@ {% macro get_create_materialized_view_as_sql(relation, sql) -%} - {% if execute %} - {{ adapter.cache_added(relation) }} - {% endif %} {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}} {%- endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql index a0771735ca9..e60e1dc24d0 100644 --- a/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql +++ b/core/dbt/include/global_project/macros/relations/materialized_view/drop.sql @@ -5,7 +5,6 @@ actually executes the drop, and `get_drop_sql`, which returns the template. */ #} {% macro drop_materialized_view(relation) -%} - {{ adapter.cache_dropped(relation) }} {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }} {%- endmacro %} diff --git a/core/dbt/include/global_project/macros/relations/rename.sql b/core/dbt/include/global_project/macros/relations/rename.sql index 087d0912f83..d7f3a72e29a 100644 --- a/core/dbt/include/global_project/macros/relations/rename.sql +++ b/core/dbt/include/global_project/macros/relations/rename.sql @@ -1,12 +1,4 @@ {%- macro get_rename_sql(relation, new_name) -%} - {% set database = relation.database %} - {% set schema = relation.schema %} - {% set to_relation = adapter.get_relation(database=database, schema=schema, identifier=new_name) %} - {% if relation is not none %} - {% if to_relation is not none %} - {{ adapter.cache_renamed(from_relation=relation, to_relation=to_relation) }} - {% endif %} - {% endif %} {{- log('Applying RENAME to: ' ~ relation) -}} {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}} {%- endmacro -%} diff --git a/plugins/postgres/dbt/adapters/postgres/relation.py b/plugins/postgres/dbt/adapters/postgres/relation.py index e722f05bedc..544f7419848 100644 --- a/plugins/postgres/dbt/adapters/postgres/relation.py +++ b/plugins/postgres/dbt/adapters/postgres/relation.py @@ -26,7 +26,6 @@ class PostgresRelation(BaseRelation): { RelationType.View, RelationType.Table, - RelationType.MaterializedView, } ) ) From ceff6f20b034425933a9330b11e3c03dae3f94d3 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 23 Apr 2024 17:24:25 -0500 Subject: [PATCH 16/17] add changelog --- .changes/unreleased/Fixes-20240423-172407.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240423-172407.yaml diff --git a/.changes/unreleased/Fixes-20240423-172407.yaml b/.changes/unreleased/Fixes-20240423-172407.yaml new file mode 100644 index 00000000000..14914e3f48c --- /dev/null +++ b/.changes/unreleased/Fixes-20240423-172407.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: remove materialized views from renambeable relation and remove a quote +time: 2024-04-23T17:24:07.249421-05:00 +custom: + Author: McKnight-42 + Issue: "127" From ad2c97698e1a1b93e4666310632b6cd0c60b473c Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 23 Apr 2024 17:56:48 -0500 Subject: [PATCH 17/17] add simple test case, and doc string --- .../test_posgres_materialized_view.py | 65 +++++++++++++++++++ tests/unit/test_renamed_relations.py | 1 - 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 tests/functional/materializations/materialized_view_tests/test_posgres_materialized_view.py diff --git a/tests/functional/materializations/materialized_view_tests/test_posgres_materialized_view.py b/tests/functional/materializations/materialized_view_tests/test_posgres_materialized_view.py new file mode 100644 index 00000000000..e1f4378ba53 --- /dev/null +++ b/tests/functional/materializations/materialized_view_tests/test_posgres_materialized_view.py @@ -0,0 +1,65 @@ +import pytest +from dbt.tests.util import run_dbt + +SEED = """ +order_id,customer_id,total_amount,order_date +1,101,50.00,2024-04-01 +2,102,75.00,2024-04-02 +3,103,100.00,2024-04-03 +4,101,30.00,2024-04-04 +5,104,45.00,2024-04-05 +""".strip() + +ORDERS = """ +-- models/orders.sql +{{ + config( + materialized='materialized_view' + ) +}} + +SELECT + order_id, + customer_id, + total_amount, + order_date +FROM + {{ ref('source_orders') }} +""" + +PRODUCT_SALES = """ +{{ + config( + materialized='materialized_view' + ) +}} + +SELECT + order_id, + SUM(total_amount) AS total_sales_amount +FROM + {{ ref('orders') }} +GROUP BY + order_id +""" + + +class TestPostgresTestRefreshMaterializedView: + """ + this test addresses a issue in postgres around materialized views, + and renaming against a model who has dependent models that are also materialized views + related pr: https://github.com/dbt-labs/dbt-core/pull/9959 + """ + + @pytest.fixture(scope="class") + def models(self): + yield {"orders.sql": ORDERS, "product_sales.sql": PRODUCT_SALES} + + @pytest.fixture(scope="class") + def seeds(self): + yield {"source_orders.csv": SEED} + + def test_postgres_refresh_dependent_naterialized_views(self, project): + run_dbt(["seed"]) + run_dbt(["run", "--full-refresh"]) + run_dbt(["run", "--full-refresh"]) diff --git a/tests/unit/test_renamed_relations.py b/tests/unit/test_renamed_relations.py index 20f2453e089..e2efac8ba76 100644 --- a/tests/unit/test_renamed_relations.py +++ b/tests/unit/test_renamed_relations.py @@ -13,6 +13,5 @@ def test_renameable_relation(): { RelationType.View, RelationType.Table, - RelationType.MaterializedView, } )