From ad0fded91c12dee1e44820cf167b8f939e69cb7e Mon Sep 17 00:00:00 2001 From: Kyle Burke Date: Thu, 4 Apr 2024 17:42:45 -0400 Subject: [PATCH 1/8] Add email domain column to snowflake_external.yml and update create_external_table.sql and snowflake.yml --- .../plugins/snowflake/snowflake_external.yml | 5 +++++ .../plugins/snowflake/create_external_table.sql | 17 +++++++++++++++-- sample_sources/snowflake.yml | 12 ++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/integration_tests/models/plugins/snowflake/snowflake_external.yml b/integration_tests/models/plugins/snowflake/snowflake_external.yml index 195cca9d..7e5c7920 100644 --- a/integration_tests/models/plugins/snowflake/snowflake_external.yml +++ b/integration_tests/models/plugins/snowflake/snowflake_external.yml @@ -20,6 +20,11 @@ sources: data_type: varchar(64) - name: email data_type: varchar(64) + - name: email domain + data_type: varchar(64) + quote: true + identifier: email + expression: split_part(value:email::VARCHAR, '@', 2) tests: &equal-to-the-people - dbt_utils.equality: compare_model: ref('people') diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index f48b87a7..8babec42 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -27,9 +27,22 @@ {%- if not infer_schema -%} {%- for column in columns %} {%- set column_quoted = adapter.quote(column.name) if column.quote else column.name %} + {%- set column_identifier -%} + {%- if 'identifier' in column and column.quote -%} + {{adapter.quote(column.identifier)}} + {%- elif 'identifier' in column -%} + {{column.identifier}} + {%- else -%} + {{column_quoted}} + {%- endif -%} + {%- endset %} {%- set col_expression -%} - {%- set col_id = 'value:c' ~ loop.index if is_csv else 'value:' ~ column_quoted -%} - (case when is_null_value({{col_id}}) or lower({{col_id}}) = 'null' then null else {{col_id}} end) + {%- if column.expression -%} + {{column.expression}} + {%- else -%} + {%- set col_id = 'value:c' ~ loop.index if is_csv else 'value:' ~ column_identifier -%} + (case when is_null_value({{col_id}}) or lower({{col_id}}) = 'null' then null else {{col_id}} end) + {%- endif -%} {%- endset %} {{column_quoted}} {{column.data_type}} as ({{col_expression}}::{{column.data_type}}) {{- ',' if not loop.last -}} diff --git a/sample_sources/snowflake.yml b/sample_sources/snowflake.yml index 035f5335..cdd47eb9 100644 --- a/sample_sources/snowflake.yml +++ b/sample_sources/snowflake.yml @@ -32,6 +32,18 @@ sources: - name: etl_tstamp data_type: timestamp description: "Timestamp event began ETL" + - name: etl timestamp + # Use double-quoted identifiers for name and identifier + quote: true + # Specifying identifier lets us rename etl_tstamp to "etl timestamp" + identifier: etl_tstamp + data_type: timestamp + description: "Timestamp event began ETL with a double quoted identifier" + - name: etl_date + data_type: date + description: "Date event began ETL" + # Expressions can manipulate the variant value prior to casting to data_type. + expression: TRY_TO_DATE(VALUE:etl_tstamp::VARCHAR, 'YYYYMMDD') - name: contexts data_type: variant description: "Contexts attached to event by Tracker" From 7e5b2f93c7df010fb65ec62e4866874d50204d29 Mon Sep 17 00:00:00 2001 From: Anders Date: Thu, 11 Apr 2024 21:54:35 -0400 Subject: [PATCH 2/8] Update integration_tests/models/plugins/snowflake/snowflake_external.yml --- .../models/plugins/snowflake/snowflake_external.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/models/plugins/snowflake/snowflake_external.yml b/integration_tests/models/plugins/snowflake/snowflake_external.yml index c52e7b5e..f1a62165 100644 --- a/integration_tests/models/plugins/snowflake/snowflake_external.yml +++ b/integration_tests/models/plugins/snowflake/snowflake_external.yml @@ -25,6 +25,7 @@ sources: quote: true identifier: email expression: split_part(value:email::VARCHAR, '@', 2) + tests: &equal-to-the-people - dbt_utils.equality: compare_model: ref('people') compare_columns: From 9173c20703ed873691f0d7442019b2be5ab5fe14 Mon Sep 17 00:00:00 2001 From: Anders Date: Thu, 11 Apr 2024 21:55:24 -0400 Subject: [PATCH 3/8] Update integration_tests/models/plugins/snowflake/snowflake_external.yml --- .../models/plugins/snowflake/snowflake_external.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/models/plugins/snowflake/snowflake_external.yml b/integration_tests/models/plugins/snowflake/snowflake_external.yml index f1a62165..6054c2ef 100644 --- a/integration_tests/models/plugins/snowflake/snowflake_external.yml +++ b/integration_tests/models/plugins/snowflake/snowflake_external.yml @@ -25,7 +25,7 @@ sources: quote: true identifier: email expression: split_part(value:email::VARCHAR, '@', 2) - tests: &equal-to-the-people + tests: &equal-to-the-people - dbt_utils.equality: compare_model: ref('people') compare_columns: From 00a8f5cf83c6cb5d8b1e26c7e96461a16cb63705 Mon Sep 17 00:00:00 2001 From: Kyle Burke Date: Tue, 16 Apr 2024 11:36:51 -0400 Subject: [PATCH 4/8] Update snowflake_external.yml to include email_domain column --- .../models/plugins/snowflake/snowflake_external.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/models/plugins/snowflake/snowflake_external.yml b/integration_tests/models/plugins/snowflake/snowflake_external.yml index 7e5c7920..0ec0be75 100644 --- a/integration_tests/models/plugins/snowflake/snowflake_external.yml +++ b/integration_tests/models/plugins/snowflake/snowflake_external.yml @@ -20,7 +20,7 @@ sources: data_type: varchar(64) - name: email data_type: varchar(64) - - name: email domain + - name: email_domain data_type: varchar(64) quote: true identifier: email From 19a334994ff8bd375d3675b992eb8f5e1eb80a78 Mon Sep 17 00:00:00 2001 From: Kyle Burke Date: Tue, 16 Apr 2024 11:51:29 -0400 Subject: [PATCH 5/8] Add email_domain column to people_expression.sql and snowflake_external.yml --- .../plugins/snowflake/people_expression.sql | 4 +++ .../plugins/snowflake/snowflake_external.yml | 34 +++++++++++++++---- 2 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 integration_tests/models/plugins/snowflake/people_expression.sql diff --git a/integration_tests/models/plugins/snowflake/people_expression.sql b/integration_tests/models/plugins/snowflake/people_expression.sql new file mode 100644 index 00000000..677d119f --- /dev/null +++ b/integration_tests/models/plugins/snowflake/people_expression.sql @@ -0,0 +1,4 @@ +SELECT + {{ dbt_utils.star(from=ref('people')) }}, + split_part(email, '@', 2) as email_domain +FROM {{ ref('people') }} \ No newline at end of file diff --git a/integration_tests/models/plugins/snowflake/snowflake_external.yml b/integration_tests/models/plugins/snowflake/snowflake_external.yml index 513ae55d..9645790b 100644 --- a/integration_tests/models/plugins/snowflake/snowflake_external.yml +++ b/integration_tests/models/plugins/snowflake/snowflake_external.yml @@ -20,11 +20,6 @@ sources: data_type: varchar(64) - name: email data_type: varchar(64) - - name: email_domain - data_type: varchar(64) - quote: true - identifier: email - expression: split_part(value:email::VARCHAR, '@', 2) tests: &equal-to-the-people - dbt_utils.equality: compare_model: ref('people') @@ -178,4 +173,31 @@ sources: - id - first_name - last_name - - email_alias \ No newline at end of file + - email_alias + + # test for column expression + - name: people_csv_expression + external: *csv-people + columns: + - name: id + data_type: int + - name: first_name + data_type: varchar(64) + - name: last_name + data_type: varchar(64) + - name: email + data_type: varchar(64) + - name: email_domain + data_type: varchar(64) + quote: true + identifier: email + expression: split_part(value:email::VARCHAR, '@', 2) + tests: + - dbt_utils.equality: + compare_model: ref('people_expression') + compare_columns: + - id + - first_name + - last_name + - email + - email_domain \ No newline at end of file From 100d23e037bbf4d1a729aca4e8d1e2e9f93c2d9e Mon Sep 17 00:00:00 2001 From: Kyle Burke Date: Tue, 16 Apr 2024 13:36:56 -0400 Subject: [PATCH 6/8] Update snowflake_external.yml to use people_json_expression instead of people_csv_expression --- .../plugins/snowflake/snowflake_external.yml | 6 +++--- .../snowflake/create_external_table.sql | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/integration_tests/models/plugins/snowflake/snowflake_external.yml b/integration_tests/models/plugins/snowflake/snowflake_external.yml index 9645790b..651513dc 100644 --- a/integration_tests/models/plugins/snowflake/snowflake_external.yml +++ b/integration_tests/models/plugins/snowflake/snowflake_external.yml @@ -176,8 +176,8 @@ sources: - email_alias # test for column expression - - name: people_csv_expression - external: *csv-people + - name: people_json_expression + external: *json-people columns: - name: id data_type: int @@ -189,8 +189,8 @@ sources: data_type: varchar(64) - name: email_domain data_type: varchar(64) + alias: EMAIL_DOMAIN quote: true - identifier: email expression: split_part(value:email::VARCHAR, '@', 2) tests: - dbt_utils.equality: diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index a0a03d26..8edbc56a 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -26,14 +26,14 @@ {%- endfor -%}{%- endif -%} {%- if not infer_schema -%} {%- for column in columns %} - {%- set column_alias = column.alias if column.alias else column.name %} - {%- set column_alias_quoted = adapter.quote(column_alias) if column.quote else column_alias %} + {# {%- set column_alias = column.alias if column.alias else column.name %} + {%- set column_alias_quoted = adapter.quote(column_alias) if column.quote else column_alias %} #} {%- set column_quoted = adapter.quote(column.name) if column.quote else column.name %} - {%- set column_identifier -%} - {%- if 'identifier' in column and column.quote -%} - {{adapter.quote(column.identifier)}} - {%- elif 'identifier' in column -%} - {{column.identifier}} + {%- set column_alias -%} + {%- if 'alias' in column and column.quote -%} + {{adapter.quote(column.alias)}} + {%- elif 'alias' in column -%} + {{column.alias}} {%- else -%} {{column_quoted}} {%- endif -%} @@ -42,11 +42,11 @@ {%- if column.expression -%} {{column.expression}} {%- else -%} - {%- set col_id = 'value:c' ~ loop.index if is_csv else 'value:' ~ column_identifier -%} + {%- set col_id = 'value:c' ~ loop.index if is_csv else 'value:' ~ column_alias -%} (case when is_null_value({{col_id}}) or lower({{col_id}}) = 'null' then null else {{col_id}} end) {%- endif -%} {%- endset %} - {{column_alias_quoted}} {{column.data_type}} as ({{col_expression}}::{{column.data_type}}) + {{column_alias}} {{column.data_type}} as ({{col_expression}}::{{column.data_type}}) {{- ',' if not loop.last -}} {% endfor %} {% else %} From 3590e0543c3d773fecbae907afcff90211e4bafb Mon Sep 17 00:00:00 2001 From: Kyle Burke Date: Tue, 16 Apr 2024 13:40:35 -0400 Subject: [PATCH 7/8] Update snowflake.yml to rename etl timestamp to etl_timestamp --- sample_sources/snowflake.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample_sources/snowflake.yml b/sample_sources/snowflake.yml index 3f294516..5498c319 100644 --- a/sample_sources/snowflake.yml +++ b/sample_sources/snowflake.yml @@ -35,8 +35,8 @@ sources: - name: etl timestamp # Use double-quoted identifiers for name and identifier quote: true - # Specifying identifier lets us rename etl_tstamp to "etl timestamp" - identifier: etl_tstamp + # Specifying alias lets us rename etl timestamp to "etl_timestamp" + alias: etl_timestamp data_type: timestamp description: "Timestamp event began ETL with a double quoted identifier" - name: etl_date From 0b0447274f108f8fc89c76050eb6b439e5b7ff86 Mon Sep 17 00:00:00 2001 From: Kyle Burke Date: Tue, 16 Apr 2024 13:54:14 -0400 Subject: [PATCH 8/8] Refactor create_external_table.sql in snowflake plugin --- macros/plugins/snowflake/create_external_table.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index 8edbc56a..40328961 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -26,8 +26,6 @@ {%- endfor -%}{%- endif -%} {%- if not infer_schema -%} {%- for column in columns %} - {# {%- set column_alias = column.alias if column.alias else column.name %} - {%- set column_alias_quoted = adapter.quote(column_alias) if column.quote else column_alias %} #} {%- set column_quoted = adapter.quote(column.name) if column.quote else column.name %} {%- set column_alias -%} {%- if 'alias' in column and column.quote -%}