Merge pull request #22 from OpenSourcePolitics/add_proposal_states

Integration of custom_proposals states for decidim-app 2.4

projects/app_dev_k8s/dbt_project.yml

@@ -15,7 +15,7 @@ model-paths: ["models"]
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
 seed-paths: ["seeds"]
-macro-paths: ["macros"]
+macro-paths: ["../demo/macros"]
 snapshot-paths: ["snapshots"]
 
 clean-targets:         # directories to be removed by `dbt clean`

projects/app_dev_k8s/packages.yml

@@ -1,4 +1,5 @@
 packages:
   - local: ../demo
+  - local: ../packages/decidim_app/proposal_states
   - package: dbt-labs/dbt_utils
     version: 1.2.0

projects/demo/models/intermediate/proposals/int_proposals.sql

@@ -13,7 +13,7 @@ SELECT
     decidim_proposals.decidim_scope_id,
     decidim_proposals.created_at,
     decidim_proposals.published_at,
-    COALESCE(NULLIF(decidim_proposals.state, NULL), 'Pas d''état') AS state,
+    COALESCE(decidim_proposals.state,'Pas d''état') AS state,
     (CASE
         WHEN decidim_proposals.state = 'evaluating' THEN 'En cours d''évaluation'
         WHEN decidim_proposals.state = 'rejected' THEN 'Rejetée'

projects/demo/models/intermediate/proposals/schema.yml

@@ -44,8 +44,8 @@ unit_tests:
     expect:
       format: dict
       rows:
-        - {id: 1, state: 'Pas d''état', translated_state: 'Pas d''état'}
-        - {id: 2, state: 'rejected', translated_state: 'Rejetée'}
+        - {id: 1, state: 'Pas d''état'}
+        - {id: 2, state: 'rejected'}
   - name: test_translated_state
     description: "Test that the state column is correctly translated"
     model: int_proposals
@@ -56,12 +56,14 @@ unit_tests:
           - {id: 1, published_at: '2024-08-12', state: 'evaluating'}
           - {id: 2, published_at: '2024-08-12', state: 'rejected'}
           - {id: 3, published_at: '2024-08-12', state: 'accepted'}
+          - {id: 4, published_at: '2024-08-12', state:  NULL}
     expect:
       format: dict
       rows:
         - {id: 1, translated_state: 'En cours d''évaluation'}
         - {id: 2, translated_state: 'Rejetée'}
         - {id: 3, translated_state: 'Acceptée'}
+        - {id: 4, translated_state: 'Pas d''état'}
   - name: test_default_address
     description: "Test that the address column correctly defaults to 'Pas d'adresse' if empty"
     model: int_proposals

projects/demo/models/marts/proposals/proposals.sql

@@ -50,7 +50,7 @@ proposals AS (
         decidim_proposals.endorsements_count,
         COALESCE(votes.votes_count,0) AS votes_count
     FROM {{ ref("int_proposals")}} AS decidim_proposals
-    JOIN {{ ref("components")}} AS decidim_components ON decidim_components.id = decidim_component_id
+    JOIN {{ ref("components")}} AS decidim_components ON decidim_components.id = decidim_proposals.decidim_component_id
     LEFT JOIN coauthorships ON decidim_proposals.id = coauthorships.coauthorable_id
     LEFT JOIN {{ ref("stg_decidim_moderations")}} AS decidim_moderations
         ON decidim_moderations.decidim_reportable_id = decidim_proposals.id

projects/demo/models/marts/proposals/schema.yml

@@ -7,6 +7,9 @@ models:
     columns:
       - name: id
         description: "Unique identifier for the proposal"
+        tests:
+          - unique
+          - not_null
       - name: decidim_participatory_space_id
         description: "Identifier for the participatory space where the proposal was created"
       - name: decidim_participatory_space_slug

projects/lyon/dbt_project.yml

@@ -16,7 +16,7 @@ model-paths: ["models"]
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
 seed-paths: ["seeds"]
-macro-paths: ["macros"]
+macro-paths: ["../demo/macros"]
 snapshot-paths: ["snapshots"]
 
 clean-targets:         # directories to be removed by `dbt clean`

projects/lyon/packages.yml

@@ -1,4 +1,5 @@
 packages:
   - local: ../demo
+  - local: ../packages/decidim_app/proposal_states
   - package: dbt-labs/dbt_utils
     version: 1.2.0

projects/packages/decidim_app/proposal_states/dbt_project.yml

@@ -0,0 +1,34 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'proposal_states'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'proposal_states'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+  # Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{ config(...) }}` macro.
+models:
+  proposal_states:
+    +materialized: table

projects/packages/decidim_app/proposal_states/models/intermediate/int_proposals_with_state.sql

@@ -0,0 +1,22 @@
+{{ config(
+    indexes=[
+      {'columns': ['id'], 'type': 'btree'},
+    ]
+)}}
+
+SELECT
+    decidim_proposals.id,
+    decidim_proposals.title,
+    decidim_proposals.body,
+    decidim_proposals.resource_type,
+    decidim_proposals.decidim_component_id,
+    decidim_proposals.decidim_scope_id,
+    decidim_proposals.created_at,
+    decidim_proposals.published_at,
+    decidim_proposals.state,
+    decidim_proposals.decidim_proposals_proposal_state_id, 
+    decidim_proposals.comments_count,
+    decidim_proposals.endorsements_count,
+    COALESCE(NULLIF(decidim_proposals.address,''),'Pas d''adresse') AS address
+FROM {{ ref ("stg_decidim_proposals_with_state")}} AS decidim_proposals
+WHERE published_at IS NOT NULL

projects/packages/decidim_app/proposal_states/models/marts/proposals_with_state.sql

@@ -0,0 +1,70 @@
+{{ config(
+    indexes=[
+      {'columns': ['id'], 'type': 'btree'},
+      {'columns': ['resource_type'], 'type': 'btree'}
+    ]
+)}}
+
+WITH coauthorships AS (
+    SELECT 
+        array_agg(decidim_users.id) AS authors_ids,
+        decidim_coauthorships.coauthorable_id 
+    FROM {{ ref("int_users")}} AS decidim_users 
+    JOIN {{ ref("stg_decidim_coauthorships")}} AS decidim_coauthorships on decidim_users.id = decidim_coauthorships.decidim_author_id
+    WHERE coauthorable_type = 'Decidim::Proposals::Proposal'
+    GROUP BY coauthorable_id
+), 
+categorizations AS (
+    {{ categorizations_filter('Decidim::Proposals::Proposal') }}
+),
+votes AS (
+    SELECT
+        decidim_proposal_id,
+        COUNT(id) AS votes_count
+    FROM {{ ref("stg_decidim_proposals_votes")}}
+   GROUP BY decidim_proposal_id
+),
+proposals AS (
+    SELECT
+        decidim_proposals.id,
+        decidim_components.ps_id AS decidim_participatory_space_id,
+        decidim_components.ps_slug AS decidim_participatory_space_slug,
+        decidim_scopes.name AS decidim_scope_name,
+        decidim_proposals.title,
+        decidim_proposals.body,
+        decidim_proposals.resource_type,
+        concat('https://', decidim_components.organization_host, '/', decidim_components.ps_space_type_slug, '/', decidim_components.ps_slug, '/f/', decidim_proposals.decidim_component_id, '/proposals/', decidim_proposals.id) AS url,
+        decidim_proposals.decidim_component_id,
+        decidim_proposals.created_at,
+        decidim_proposals.published_at,
+        decidim_proposals.state AS legacy_state,
+        coauthorships.authors_ids,
+        COALESCE(coauthorships.authors_ids[1], -1) AS first_author_id,
+        decidim_proposals.address,
+        categorizations.categories,
+        COALESCE(categorizations.categories[1], 'Sans catégorie') AS first_category,
+        categorizations.sub_categories,
+        COALESCE(categorizations.sub_categories[1], 'Sans sous-catégorie') AS first_sub_category,
+        decidim_proposals.comments_count,
+        decidim_proposals.endorsements_count,
+        COALESCE(votes.votes_count,0) AS votes_count,
+        decidim_proposals_proposal_states.id AS custom_state_id, 
+        decidim_proposals_proposal_states.token AS state, 
+        decidim_proposals_proposal_states.title AS custom_state,
+        decidim_proposals_proposal_states.description,
+        decidim_proposals_proposal_states.proposals_count
+    FROM {{ ref("int_proposals_with_state")}} AS decidim_proposals
+    JOIN {{ ref("components")}} AS decidim_components ON decidim_components.id = decidim_proposals.decidim_component_id
+    LEFT JOIN coauthorships ON decidim_proposals.id = coauthorships.coauthorable_id
+    LEFT JOIN {{ ref("stg_decidim_moderations")}} AS decidim_moderations
+        ON decidim_moderations.decidim_reportable_id = decidim_proposals.id
+        AND decidim_moderations.decidim_reportable_type = 'Decidim::Proposals::Proposal'
+    LEFT JOIN {{ ref("int_scopes")}} AS decidim_scopes ON decidim_scopes.id = decidim_proposals.decidim_scope_id
+    LEFT JOIN votes ON decidim_proposals.id = votes.decidim_proposal_id
+    LEFT JOIN categorizations ON categorizations.categorizable_id = decidim_proposals.id
+    JOIN {{ ref("stg_decidim_proposals_custom_states")}} AS decidim_proposals_proposal_states ON decidim_proposals_proposal_states.id = decidim_proposals.decidim_proposals_proposal_state_id
+    WHERE decidim_moderations.hidden_at IS NULL
+    AND decidim_proposals.published_at IS NOT NULL
+)
+
+SELECT * FROM proposals

projects/packages/decidim_app/proposal_states/models/marts/schema.yml

@@ -0,0 +1,49 @@
+unit_tests:
+  - name: test_custom_state_unicity
+    description: "Test that a single proposal cannot be assigned to 2 different custom states "
+    model: proposals_with_state
+    given:
+      - input: ref('int_users')
+        format: dict
+        rows:
+          - {}
+      - input: ref('stg_decidim_coauthorships')
+        format: dict
+        rows:
+          - {}
+      - input: ref('stg_decidim_proposals_votes')
+        format: dict
+        rows:
+          - {}
+      - input: ref('components')
+        format: dict
+        rows:
+          - {id: 100}
+      - input: ref('stg_decidim_moderations')
+        format: dict
+        rows:
+          - {}
+      - input: ref('int_scopes')
+        format: dict
+        rows:
+          - {}
+      - input: ref('categorizations')
+        format: dict
+        rows:
+          - {}
+      - input: ref('int_proposals_with_state')
+        format: dict
+        rows:
+          - {id: 41, published_at: '2023-01-01', decidim_component_id: 100, decidim_proposals_proposal_state_id: 45}
+          - {id: 42, published_at: '2023-01-02', decidim_component_id: 100, decidim_proposals_proposal_state_id: 46}
+      - input: ref('stg_decidim_proposals_custom_states')
+        format: dict
+        rows:
+          - {id: 45, token: 'withdrawn', title: 'Retirée'}
+          - {id: 46, token: 'accepted', title: 'Acceptée'}
+    expect:
+      format: dict
+      rows:
+        - {id: 41, state: 'withdrawn', custom_state: 'Retirée', custom_state_id: 45}
+        - {id: 42, state: 'accepted', custom_state: 'Acceptée', custom_state_id: 46}
+

projects/packages/decidim_app/proposal_states/models/staging/decidim/proposals/schema.yml

@@ -0,0 +1,9 @@
+version: 2
+
+sources:
+  - name: decidim
+    database: "{{ env_var('DBNAME') }}"
+    schema: public  
+    tables:
+      - name: decidim_proposals_proposals
+      - name: decidim_proposals_proposal_states

projects/packages/decidim_app/proposal_states/models/staging/decidim/proposals/stg_decidim_proposals_custom_states.sql

@@ -0,0 +1,8 @@
+SELECT
+    id,
+    token,
+	title::jsonb->>'fr' AS title,
+	description,
+	decidim_component_id,
+	proposals_count
+FROM {{ source('decidim', 'decidim_proposals_proposal_states') }} 

projects/packages/decidim_app/proposal_states/models/staging/decidim/proposals/stg_decidim_proposals_with_state.sql

@@ -0,0 +1,16 @@
+SELECT
+    id,
+    regexp_replace(title::jsonb->>'fr', E'(<[^>]+>)|(&[a-z]+;)', '', 'gi') as title,
+    regexp_replace(body::jsonb->>'fr', E'(<[^>]+>)|(&[a-z]+;)', '', 'gi') as body,
+    'Decidim::Proposals::Proposal' as resource_type,
+    decidim_component_id,
+    decidim_scope_id,
+    created_at,
+    published_at,
+    state,
+    comments_count,
+    endorsements_count,
+    address, 
+    decidim_proposals_proposal_state_id
+FROM {{ source('decidim', 'decidim_proposals_proposals') }}
+