feat(add) : adding spam report colums

OpenSourcePolitics · Jan 14, 2025 · 897fb68 · 897fb68
1 parent d333ee7
commit 897fb68
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 2 deletions.
diff --git a/projects/demo/models/intermediate/users/int_users.sql b/projects/demo/models/intermediate/users/int_users.sql
@@ -45,7 +45,11 @@ SELECT
     decidim_users.admin_terms_accepted_at,
     decidim_users.blocked,
     decidim_users.blocked_at,
-    (CASE WHEN decidim_users.confirmed_at IS NULL THEN false ELSE true END) AS confirmed,
+    (CASE WHEN decidim_users.confirmed_at IS NULL THEN false ELSE true END) AS confirmed, 
+    (CASE WHEN decidim_users.spam_report_timestamp IS NULL THEN false ELSE true END) AS spam, 
+    decidim_users.spam_probability,
+    DATE(decidim_users.spam_report_timestamp) AS spam_reported_at,
     decidim_users.extended_data
+
 FROM {{ ref ("stg_decidim_users")}} as decidim_users
     WHERE type LIKE 'Decidim::User'
diff --git a/projects/demo/models/marts/users/all_users.sql b/projects/demo/models/marts/users/all_users.sql
@@ -97,6 +97,9 @@ SELECT
     COALESCE(forms_answers.has_answered_survey, 0)::boolean AS has_answered_survey,
     decidim_users.confirmed,
     CONCAT('https://', decidim_organizations.host, '/profiles/', decidim_users.nickname, '/activity') AS url,
+    decidim_users.spam,
+    decidim_users.spam_probability,
+    decidim_users.spam_reported_at,
     decidim_users.extended_data
 FROM {{ ref("int_users") }} AS decidim_users
 LEFT JOIN followings ON followings.decidim_user_id = decidim_users.id

diff --git a/projects/demo/models/marts/users/schema.yml b/projects/demo/models/marts/users/schema.yml
@@ -95,6 +95,21 @@ models:
           column_name: id
           to: ref('int_users')
           field: id
+  - name: all_users
+    schema: "{{ env_var('TARGET_SCHEMA') }}"
+    columns:
+      - name: spam
+        description: "Indicates if the user has been reported or not"
+      - name: spam_probability
+        description: "When a user has been reported as spam, gives the coefficient of spam likelihood, comprised between 0 and 1"
+        data_tests: 
+          - dbt_utils.accepted_range:
+              min_value: 0
+              max_value: 1
+              config:
+                where: "spam IS NOT null"  
+      - name: spam_reported_at
+        description: "When a user has been reported as spam, gives the report date"
 
 unit_tests:
   - name: deleted_users_check
@@ -109,7 +124,7 @@ unit_tests:
     expect:
       format: dict
       rows:
-          - {id: 2}
+          - {id: 2} 
 #  - name: test_url_concatenation
 #    description: "Check URL concatenation logic"
 #    model: all_users

diff --git a/projects/demo/models/staging/decidim/stg_decidim_users.sql b/projects/demo/models/staging/decidim/stg_decidim_users.sql
@@ -36,7 +36,10 @@ renamed AS (
         blocked,
         blocked_at,
         (CASE WHEN confirmed_at IS NULL THEN false ELSE true END) AS "confirmed",
+        (extended_data::jsonb->'spam_detection'->>'spam_probability')::float as spam_probability,
+	    extended_data::jsonb->'spam_detection'->> 'reported_at' as spam_report_timestamp,	
         extended_data
+
     FROM source
 )
 SELECT * FROM renamed