diff --git a/projects/demo/models/intermediate/users/int_users.sql b/projects/demo/models/intermediate/users/int_users.sql index d36e468..2dbfe97 100644 --- a/projects/demo/models/intermediate/users/int_users.sql +++ b/projects/demo/models/intermediate/users/int_users.sql @@ -45,7 +45,11 @@ SELECT decidim_users.admin_terms_accepted_at, decidim_users.blocked, decidim_users.blocked_at, - (CASE WHEN decidim_users.confirmed_at IS NULL THEN false ELSE true END) AS confirmed, + (CASE WHEN decidim_users.confirmed_at IS NULL THEN false ELSE true END) AS confirmed, + (CASE WHEN decidim_users.spam_report_timestamp IS NULL THEN false ELSE true END) AS spam, + decidim_users.spam_probability, + DATE(decidim_users.spam_report_timestamp) AS spam_reported_at, decidim_users.extended_data + FROM {{ ref ("stg_decidim_users")}} as decidim_users WHERE type LIKE 'Decidim::User' diff --git a/projects/demo/models/marts/users/all_users.sql b/projects/demo/models/marts/users/all_users.sql index bdae2e1..4225f8d 100644 --- a/projects/demo/models/marts/users/all_users.sql +++ b/projects/demo/models/marts/users/all_users.sql @@ -97,6 +97,9 @@ SELECT COALESCE(forms_answers.has_answered_survey, 0)::boolean AS has_answered_survey, decidim_users.confirmed, CONCAT('https://', decidim_organizations.host, '/profiles/', decidim_users.nickname, '/activity') AS url, + decidim_users.spam, + decidim_users.spam_probability, + decidim_users.spam_reported_at, decidim_users.extended_data FROM {{ ref("int_users") }} AS decidim_users LEFT JOIN followings ON followings.decidim_user_id = decidim_users.id diff --git a/projects/demo/models/marts/users/schema.yml b/projects/demo/models/marts/users/schema.yml index 3beb5e8..7198f20 100644 --- a/projects/demo/models/marts/users/schema.yml +++ b/projects/demo/models/marts/users/schema.yml @@ -95,6 +95,21 @@ models: column_name: id to: ref('int_users') field: id + - name: all_users + schema: "{{ env_var('TARGET_SCHEMA') }}" + columns: + - name: spam + description: "Indicates if the user has been reported or not" + - name: spam_probability + description: "When a user has been reported as spam, gives the coefficient of spam likelihood, comprised between 0 and 1" + data_tests: + - dbt_utils.accepted_range: + min_value: 0 + max_value: 1 + config: + where: "spam IS NOT null" + - name: spam_reported_at + description: "When a user has been reported as spam, gives the report date" unit_tests: - name: deleted_users_check @@ -109,7 +124,7 @@ unit_tests: expect: format: dict rows: - - {id: 2} + - {id: 2} # - name: test_url_concatenation # description: "Check URL concatenation logic" # model: all_users diff --git a/projects/demo/models/staging/decidim/stg_decidim_users.sql b/projects/demo/models/staging/decidim/stg_decidim_users.sql index 7321616..5cd0004 100644 --- a/projects/demo/models/staging/decidim/stg_decidim_users.sql +++ b/projects/demo/models/staging/decidim/stg_decidim_users.sql @@ -36,7 +36,10 @@ renamed AS ( blocked, blocked_at, (CASE WHEN confirmed_at IS NULL THEN false ELSE true END) AS "confirmed", + (extended_data::jsonb->'spam_detection'->>'spam_probability')::float as spam_probability, + extended_data::jsonb->'spam_detection'->> 'reported_at' as spam_report_timestamp, extended_data + FROM source ) SELECT * FROM renamed