Bug/total queries (#140)

* bug/total-queries

* update seed

* update macro and namings

* update yml

* update models, changelog, readme

* regen docs

* update yml

CHANGELOG.md

@@ -1,3 +1,15 @@
+# dbt_fivetran_log v1.10.0
+[PR #140](https://github.com/fivetran/dbt_fivetran_log/pull/140) includes the following updates:
+
+## Breaking Changes
+> A `--full-refresh` is recommended after upgrading to ensure historical records in incremental models are refreshed.
+- Updated the `fivetran_log_json_parse` macro for Redshift to return `NULL` instead of an empty string when a JSON path is not found. This resolves errors caused by casting empty strings to integers in Redshift.
+- Standardized the `message_data` field from the `LOG` source, in which JSON key names can appear in both camelCase (e.g., `{"totalQueries":5}`) and snake_case (e.g., `{"total_queries":5}`) formats, depending on the Fivetran connector version. The `fivetran_platform__audit_table` and `fivetran_platform__connector_daily_events` models now convert all key names to snake_case for consistency.
+- These changes are considered breaking because the standardization of key names (e.g., `totalQueries` to `total_queries`) may impact downstream reporting by including previously ignored values.
+
+## Under the Hood (Maintainers Only)
+- Enhanced seed data for integration testing to include the different spellings and ensure compatibility with Redshift.
+
 # dbt_fivetran_log v1.9.1
 [PR #138](https://github.com/fivetran/dbt_fivetran_log/pull/138) includes the following updates:
 

README.md

@@ -70,7 +70,7 @@ Include the following Fivetran Platform package version range in your `packages.
 ```yaml
 packages:
   - package: fivetran/fivetran_log
-    version: [">=1.9.0", "<1.10.0"]
+    version: [">=1.10.0", "<1.11.0"]
 ```
 
 > Note that although the source connector is now "Fivetran Platform", the package retains the old name of "fivetran_log".

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'fivetran_log'
-version: '1.9.1'
+version: '1.10.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '1.9.1'
+version: '1.10.0'
 
 config-version: 2
 profile: 'integration_tests'

integration_tests/seeds/log.csv

@@ -31,4 +31,4 @@ V3H62Gj5c6nB8bY+OdxMHvp/hNs=,2021-12-10 14:26:05.907,2021-12-10 20:30:53.778,thi
 jkLxnPUfQ/9mHaWCmxUQEhul8ZA=,2023-11-09 11:31:31.579000,2023-11-09 15:55:12.560000,protestations_mourned,INFO,"{""actor"":""[email protected]"",""properties"":{""endpoint"":""************"",""apiKey"":""************"",""customEvents"":[],""syncMode"":""AllEvents"",""isE2ETest"":false,""events"":[],""customEventSyncMode"":""AllEvents""},""id"":""iterable""}",create_connector,,
 R7UqnKYn6OT04HkUcPNjXA95qqI=,2021-12-10 16:26:29.719,2021-12-10 20:30:53.878,intrinsic_departed,INFO,says actor but not a json,status,,456abc
 D7UqnKYn6OT04HkUcPNjXA95qqp=,2024-11-18 16:26:29.719,2024-11-19 20:30:53.878,aft_gleeful,INFO,"{""total_queries"":15,""total_rows"":4810}",extract_summary,,456abc
-h7UqnKYn6OT04HkUcPNjXA95qlp=,2024-11-19 16:26:29.719,2024-11-19 20:30:53.878,aft_gleeful,INFO,"{""total_queries"":5,""total_rows"":4810}",extract_summary,,456abc
+h7UqnKYn6OT04HkUcPNjXA95qlp=,2024-11-19 16:26:29.719,2024-11-19 20:30:53.878,aft_gleeful,INFO,"{""totalQueries"":5,""total_rows"":4810}",extract_summary,,456abc

macros/fivetran_log_json_parse.sql

@@ -18,10 +18,12 @@
 
 {% macro redshift__fivetran_log_json_parse(string, string_path) %}
 
-  json_extract_path_text(
-    {{ string }}, 
-    {%- for s in string_path -%}'{{ s }}'{%- if not loop.last -%},{%- endif -%}{%- endfor -%}, 
-    true ) -- this flag sets null_if_invalid=true
+  nullif(
+    json_extract_path_text(
+      {{ string }}, 
+      {%- for s in string_path -%}'{{ s }}'{%- if not loop.last -%},{%- endif -%}{%- endfor -%}, 
+      true ) -- this flag sets null_if_invalid=true
+    , '')
 
 {% endmacro %}
 

models/fivetran_platform__audit_table.sql

@@ -12,7 +12,11 @@
 
 with base as (
 
-    select *
+    select
+        connector_id,
+        created_at,
+        event_subtype, 
+        replace(message_data, 'operationType', 'operation_type') as message_data
     from {{ ref('stg_fivetran_platform__log') }}
     where event_subtype in ('sync_start', 'sync_end', 'write_to_table_start', 'write_to_table_end', 'records_modified')
 
@@ -37,7 +41,7 @@ sync_log as (
         *,
         {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['table']) }} as table_name,
         {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['schema']) }} as schema_name,
-        {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['operationType']) }} as operation_type,
+        {{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['operation_type']) }} as operation_type,
         cast ({{ fivetran_log.fivetran_log_json_parse(string='message_data', string_path=['count']) }} as {{ dbt.type_bigint() }}) as row_count
     from base
     where event_subtype = 'records_modified'

models/fivetran_platform__connector_daily_events.sql

@@ -14,7 +14,7 @@ log_events as (
         connector_id,
         cast( {{ dbt.date_trunc('day', 'created_at') }} as date) as date_day,
         event_subtype,
-        message_data
+        replace(message_data, 'totalQueries', 'total_queries') as message_data
 
     from {{ ref('stg_fivetran_platform__log') }}