Merge branch 'main' into jtmcn/external-tables/snf-copied-at

integration_tests/models/plugins/bigquery/bigquery_external.yml

@@ -16,6 +16,7 @@ sources:
         columns: &cols-of-the-people
           - name: id
             data_type: int64
+            description: id_of_the_person
           - name: first_name
             data_type: string
           - name: last_name

integration_tests/models/plugins/snowflake/snowflake_external.yml

@@ -20,6 +20,9 @@ sources:
             data_type: varchar(64)
           - name: email
             data_type: varchar(64)
+          - name: email
+            alias: email_alias
+            data_type: varchar(64)          
         tests: &equal-to-the-people
           - dbt_utils.equality:
               compare_model: ref('people')
@@ -28,6 +31,7 @@ sources:
                 - first_name
                 - last_name
                 - email
+                - email_alias
 
       - name: people_csv_partitioned
         external:

macros/common/stage_external_sources.sql

@@ -66,6 +66,9 @@
 
         {% endfor %}
 
+        {% set update_columns = dbt_external_tables.update_external_table_columns(node) %}
+        {{ update_columns }}
+
     {% endfor %}
 
 {% endmacro %}

macros/common/update_external_table_columns.sql

@@ -0,0 +1,7 @@
+{% macro update_external_table_columns(source_node) %}
+    {{ return(adapter.dispatch('update_external_table_columns', 'dbt_external_tables')(source_node)) }}
+{% endmacro %}
+
+{% macro default__update_external_table_columns(source_node) %}
+
+{% endmacro %}

macros/plugins/bigquery/create_external_table.sql

@@ -1,5 +1,4 @@
 {% macro bigquery__create_external_table(source_node) %}
-
     {%- set columns = source_node.columns.values() -%}
     {%- set external = source_node.external -%}
     {%- set partitions = external.partitions -%}

macros/plugins/bigquery/update_external_table_columns.sql

@@ -0,0 +1,5 @@
+{% macro bigquery__update_external_table_columns(source_node) %}
+    {%- set columns = source_node.columns -%}
+    {%- set relation = source(source_node.source_name, source_node.name) -%}
+    {%- do adapter.update_columns(relation, columns) -%}
+{% endmacro %}

macros/plugins/snowflake/create_external_table.sql

@@ -26,12 +26,14 @@
         {%- endfor -%}{%- endif -%}
         {%- if not infer_schema -%}
             {%- for column in columns %}
+                {%- set column_alias = column.alias if column.alias else column.name %}
+                {%- set column_alias_quoted = adapter.quote(column_alias) if column.quote else column_alias %}
                 {%- set column_quoted = adapter.quote(column.name) if column.quote else column.name %}
                 {%- set col_expression -%}
                     {%- set col_id = 'value:c' ~ loop.index if is_csv else 'value:' ~ column_quoted -%}
                     (case when is_null_value({{col_id}}) or lower({{col_id}}) = 'null' then null else {{col_id}} end)
                 {%- endset %}
-                {{column_quoted}} {{column.data_type}} as ({{col_expression}}::{{column.data_type}})
+                {{column_alias_quoted}} {{column.data_type}} as ({{col_expression}}::{{column.data_type}})
                 {{- ',' if not loop.last -}}
             {% endfor %}
         {% else %}
@@ -51,6 +53,9 @@
     {% if external.auto_refresh in (true, false) -%}
       auto_refresh = {{external.auto_refresh}}
     {%- endif %}
+    {% if external.aws_sns_topic -%}
+      aws_sns_topic = '{{external.aws_sns_topic}}'
+    {%- endif %}
     {% if external.table_format | lower == "delta" %}
       refresh_on_create = false
     {% endif %}

sample_sources/snowflake.yml

@@ -84,4 +84,12 @@ sources:
           - name: id
             description: this is an id
           - name: name
-            description: and this is a name
+            description: and this is a name
+
+      - name: aws_sns_refresh_tbl
+        description: "External table using AWS SNS for auto-refresh"
+        external:
+          location: "@stage"                  # reference an existing external stage
+          file_format: "( type = csv )"   
+          # auto_refresh is assumed, setting to false is not supported
+          aws_sns_topic: "arn:aws:sns:us-east-1:123456789012:my_topic" # SNS topic ARN