Refactor pytest unit tests to dbt unit tests

.gitignore

@@ -2,6 +2,7 @@
 target/
 dbt_packages/
 logs/
+package-lock.yml
 
 google-cloud-sdk/
 unit_tests/.env

README.md

@@ -304,7 +304,26 @@ gcloud auth application-default login --scopes=https://www.googleapis.com/auth/b
 ```
 # Unit Testing
 
-This package uses `pytest` as a method of unit testing individual models. More details can be found in the [unit_tests/README.md](unit_tests) folder.
+The dbt-ga4 package treats each model and macro as a 'unit' of code. If we fix the input to each unit, we can test that we received the expected output.
+
+This package currently uses a combination of dbt unit tests and `pytest` as a method of unit testing individual models. The remaining `pytest` unit test will be refactored to a dbt unit test when possible - progress on the bug preventing that work can be tracked [here](https://github.com/dbt-labs/dbt-core/issues/10353).
+
+### dbt unit tests
+
+dbt's documentation on unit tests can be found [here](https://docs.getdbt.com/docs/build/unit-tests). Unit tests are performed the same way other types of dbt tests are executed.
+
+Execute a specific test:
+```
+dbt test -s <test_name>
+```
+Execute all tests configured for a model:
+```
+dbt test -s <model_name>
+```
+
+### pytest
+
+More details on using `pytest` for unit testing can be found in the [unit_tests/README.md](unit_tests) folder.
 
 # Overriding Default Channel Groupings
 

models/staging/stg_ga4__client_key_first_last_events.yml

@@ -7,4 +7,20 @@ models:
       - name: client_key
         description: Hashed combination of user_pseudo_id and stream_id
         tests: 
-          - unique
+          - unique
+unit_tests:
+  - name: test_stg_ga4__client_key_first_last_events
+    description: Test pulling the first and last event per client key
+    model: stg_ga4__client_key_first_last_events
+    given:
+      - input: ref('stg_ga4__events')
+        format: csv
+        rows: |
+          stream_id,client_key,event_key,event_timestamp
+          1,IX+OyYJBgjwqML19GB/XIQ==,H06dLW6OhNJJ6SoEPFsSyg==,1661339279816517
+          1,IX+OyYJBgjwqML19GB/XIQ==,gt1SoAtrxDv33uDGwVeMVA==,1661339279816518
+    expect:
+      format: csv
+      rows: |
+        client_key,first_event,last_event
+        IX+OyYJBgjwqML19GB/XIQ==,H06dLW6OhNJJ6SoEPFsSyg==,gt1SoAtrxDv33uDGwVeMVA==

models/staging/stg_ga4__client_key_first_last_pageviews.yml

@@ -7,4 +7,20 @@ models:
       - name: client_key
         description: Hashed combination of user_pseudo_id and stream_id
         tests:
-          - unique
+          - unique
+unit_tests:
+  - name: test_stg_ga4__client_key_first_last_pageviews
+    description: Test pulling the first and last page view per client key
+    model: stg_ga4__client_key_first_last_pageviews
+    given:
+      - input: ref('stg_ga4__event_page_view')
+        format: csv
+        rows: |
+          stream_id,client_key,event_key,event_timestamp,page_location
+          1,IX+OyYJBgjwqML19GB/XIQ==,H06dLW6OhNJJ6SoEPFsSyg==,1661339279816517,A
+          1,IX+OyYJBgjwqML19GB/XIQ==,gt1SoAtrxDv33uDGwVeMVA==,1661339279816518,B
+    expect:
+      format: csv
+      rows: |
+        client_key,first_page_view_event_key,last_page_view_event_key,first_page_location,last_page_location
+        IX+OyYJBgjwqML19GB/XIQ==,H06dLW6OhNJJ6SoEPFsSyg==,gt1SoAtrxDv33uDGwVeMVA==,A,B

models/staging/stg_ga4__derived_session_properties.yml

@@ -8,4 +8,39 @@ models:
     columns:
       - name: session_key
         tests:
-          - unique  
+          - unique
+# unit_tests:
+#   - name: test_derived_session_properties
+#     description: Test whether a derived property is successfully retrieved from multiple event payloads
+#     model: stg_ga4__derived_session_properties
+#     given:
+#       - input: ref('stg_ga4__events')
+#         format: sql
+#         rows: |
+#           select 
+#           'AAA' as session_key
+#           , 1617691790431476 as event_timestamp
+#           , 'first_visit' as event_name
+#           , ARRAY[STRUCT('my_param' as key, STRUCT(1 as int_value) as value)] as event_params
+#           , ARRAY[STRUCT('my_property' as key, STRUCT('value1' as string_value) as value)] as user_properties
+#           union all
+#           select
+#           'AAA' as session_key
+#           , 1617691790431477 as event_timestamp
+#           , 'first_visit' as event_name
+#           , ARRAY[STRUCT('my_param' as key, STRUCT(2 as int_value) as value)] as event_params
+#           , ARRAY[] as user_properties
+#           union all
+#           select 
+#           'BBB' as session_key
+#           , 1617691790431477 as event_timestamp
+#           , 'first_visit' as event_name
+#           , ARRAY[STRUCT('my_param' as key, STRUCT(1 as int_value) as value)] as event_params
+#           , ARRAY[STRUCT('my_property' as key, STRUCT('value2' as string_value) as value)] as user_properties
+#     expect:
+#       format: dict
+#       rows:
+#         - {session_key: AAA, my_derived_property: 2, my_derived_property2: value1}
+#         - {session_key: BBB, my_derived_property: 1, my_derived_property2: value2}
+#     overrides:
+#       vars: {derived_session_properties: [{event_parameter: 'my_param',session_property_name: 'my_derived_property',value_type: 'int_value'},{user_property: 'my_property',session_property_name: 'my_derived_property2',value_type: 'string_value'}]}

models/staging/stg_ga4__derived_user_properties.yml

@@ -7,4 +7,36 @@ models:
       - name: client_key
         description: Hashed combination of user_pseudo_id and stream_id
         tests:
-          - unique  
+          - unique  
+# unit_tests:
+#   - name: test_derived_user_properties
+#     description: Test whether a derived user property is successfully retrieved from multiple event payloads
+#     model: stg_ga4__derived_user_properties
+#     given:
+#       - input: ref('stg_ga4__events')
+#         format: sql
+#         rows: |
+#           select 
+#           'AAA' as client_key
+#           , 1617691790431476 as event_timestamp
+#           , 'first_visit' as event_name
+#           , ARRAY[STRUCT('my_param' as key, STRUCT(1 as int_value) as value)] as event_params
+#           union all
+#           select
+#           'AAA' as client_key
+#           , 1617691790431477 as event_timestamp
+#           , 'first_visit' as event_name
+#           , ARRAY[STRUCT('my_param' as key, STRUCT(2 as int_value) as value)] as event_params
+#           union all
+#           select 
+#           'BBB' as client_key
+#           , 1617691790431477 as event_timestamp
+#           , 'first_visit' as event_name
+#           , ARRAY[STRUCT('my_param' as key, STRUCT(1 as int_value) as value)] as event_params
+#     expect:
+#       format: dict
+#       rows:
+#         - {client_key: AAA, my_derived_property: 2}
+#         - {client_key: BBB, my_derived_property: 1}
+#     overrides:
+#       vars: {derived_user_properties: [{event_parameter: 'my_param',user_property_name: 'my_derived_property',value_type: 'int_value'}]}

models/staging/stg_ga4__event_to_query_string_params.yml

@@ -3,4 +3,23 @@ version: 2
 models:  
   - name: stg_ga4__event_to_query_string_params
     description: This model pivots the query string parameters contained within the event's page_location field to become rows. Each row is a single parameter/value combination contained in a single event's query string.
-
+unit_tests:
+  - name: test_stg_ga4__event_to_query_string_params
+    description: Test whether event query strings are flattened for each query string parameter
+    model: stg_ga4__event_to_query_string_params
+    given:
+      - input: ref('stg_ga4__events')
+        format: csv
+        rows: |
+          event_key,page_query_string
+          aaa,param1=value1&param2=value2
+          bbb,param1
+          ccc,param1=
+    expect:
+      format: csv
+      rows: |
+        event_key,param,value
+        aaa,param1,value1
+        aaa,param2,value2
+        bbb,param1,
+        ccc,param1,