diff --git a/macros/external/create_external_table.sql b/macros/external/create_external_table.sql index 4a73d3ed..783fcf04 100644 --- a/macros/external/create_external_table.sql +++ b/macros/external/create_external_table.sql @@ -82,9 +82,10 @@ {% endfor %} ) {% if partitions -%} partition by ({{partitions|map(attribute='name')|join(', ')}}) {%- endif %} - {% if external.location -%} location = {{external.location}} {%- endif %} {# stage #} + location = {{external.location}} {# stage #} {% if external.auto_refresh -%} auto_refresh = {{external.auto_refresh}} {%- endif %} - {% if external.file_format -%} file_format = {{external.file_format}} {%- endif %} + {% if external.pattern -%} pattern = {{external.pattern}} {%- endif %} + file_format = {{external.file_format}} {% endmacro %} {% macro bigquery__create_external_table(source_node) %} diff --git a/macros/external/stage_external_sources.sql b/macros/external/stage_external_sources.sql index 29391b98..a0b67b1f 100644 --- a/macros/external/stage_external_sources.sql +++ b/macros/external/stage_external_sources.sql @@ -4,7 +4,8 @@ {% if node.resource_type == 'source' and node.external.location != none %} - {%- do log('Staging external table ' ~ node.source_name ~ '.' ~ node.name, info = true) -%} + {% set ts = modules.datetime.datetime.now().strftime('%H:%M:%S') %} + {%- do log(ts ~ ' + Staging external table ' ~ node.source_name ~ '.' ~ node.name, info = true) -%} {%- set run_queue = [] -%} @@ -15,7 +16,7 @@ {%- do run_queue.append(create_external_table(node)) -%} - {%- if node.external.partitions and target.type != 'spark' -%} + {%- if node.external.partitions and target.type == 'redshift' -%} {%- set run_queue = run_queue + refresh_external_table(node).split(';') -%} {%- endif -%} @@ -25,8 +26,8 @@ {{ q }} {% endcall %} - {% set status = load_result('runner')['status'] %} {% set ts = modules.datetime.datetime.now().strftime('%H:%M:%S') %} + {% set status = load_result('runner')['status'] %} {% set msg = ts ~ ' + (' ~ loop.index ~ ') ' ~ status %} {% do log(msg, info = true) %}