Merge branch 'master' into fix/omniauth--extra-user-fields--verified-…

…email

.github/workflows/ci_cd.yml

@@ -89,7 +89,7 @@ jobs:
           ruby-version: ${{ env.RUBY_VERSION }}
           bundler-cache: true
       - run: |
-          sudo apt install libu2f-udev
+          sudo apt install libu2f-udev imagemagick
           wget --no-verbose -O /tmp/chrome.deb https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${{env.CHROME_VERSION}}-1_amd64.deb
           sudo dpkg -i /tmp/chrome.deb
           rm /tmp/chrome.deb
@@ -163,7 +163,7 @@ jobs:
           ruby-version: ${{ env.RUBY_VERSION }}
           bundler-cache: true
       - run: |
-          sudo apt install libu2f-udev
+          sudo apt install libu2f-udev imagemagick
           wget --no-verbose -O /tmp/chrome.deb https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${{env.CHROME_VERSION}}-1_amd64.deb
           sudo dpkg -i /tmp/chrome.deb
           rm /tmp/chrome.deb

Gemfile.lock

@@ -1152,7 +1152,7 @@ GEM
       activesupport
     wisper (2.0.1)
     wisper-rspec (1.1.0)
-    wkhtmltopdf-binary (0.12.6.6)
+    wkhtmltopdf-binary (0.12.6.8)
     xpath (3.2.0)
       nokogiri (~> 1.8)
     zeitwerk (2.6.15)
@@ -1235,4 +1235,4 @@ RUBY VERSION
    ruby 3.0.6p216
 
 BUNDLED WITH
-   2.5.22
+   2.5.10

app/controllers/concerns/decidim/budgets/orderable.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module Decidim
+  module Budgets
+    # Common logic to sorting resources
+    module Orderable
+      extend ActiveSupport::Concern
+
+      included do
+        include Decidim::Orderable
+
+        private
+
+        # Available orders based on enabled settings
+        def available_orders
+          @available_orders ||= [default_order] + possible_orders.excluding(default_order)
+        end
+
+        def possible_orders
+          @possible_orders ||= begin
+            available_orders = []
+            available_orders << "random" if voting_open? || !votes_are_visible?
+            available_orders << "most_voted" if votes_are_visible?
+            available_orders += %w(alphabetical highest_cost lowest_cost)
+            available_orders
+          end
+        end
+
+        def default_order
+          @default_order ||= fetch_default_order
+        end
+
+        def fetch_default_order
+          default_order = current_settings.default_sort_order.presence || component_settings.default_sort_order
+          return order_by_default if default_order == "default"
+
+          possible_orders.include?(default_order) ? default_order : order_by_default
+        end
+
+        def order_by_default
+          voting_open? || !votes_are_visible? ? "random" : "most_voted"
+        end
+
+        def votes_are_visible?
+          current_settings.show_votes?
+        end
+
+        def reorder(projects)
+          case order
+          when "alphabetical"
+            reorder_alphabetically(projects)
+          when "highest_cost"
+            reorder_by_highest_cost(projects)
+          when "lowest_cost"
+            reorder_by_lowest_cost(projects)
+          when "most_voted"
+            reorder_by_most_voted(projects)
+          when "random"
+            reorder_randomly(projects)
+          else
+            projects
+          end
+        end
+
+        def reorder_alphabetically(projects)
+          projects.ordered_ids(
+            projects.sort_by { |project| project.title[I18n.locale.to_s] || "" }.map(&:id)
+          )
+        end
+
+        def reorder_by_highest_cost(projects)
+          projects.order(budget_amount: :desc)
+        end
+
+        def reorder_by_lowest_cost(projects)
+          projects.order(budget_amount: :asc)
+        end
+
+        def reorder_by_most_voted(projects)
+          return projects unless votes_are_visible?
+
+          ids = projects.sort_by(&:confirmed_orders_count).map(&:id).reverse
+          projects.ordered_ids(ids)
+        end
+
+        def reorder_randomly(projects)
+          projects.order_randomly(random_seed)
+        end
+      end
+    end
+  end
+end

app/controllers/decidim/system/dashboard_controller.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Decidim
+  module System
+    class DashboardController < Decidim::System::ApplicationController
+      before_action :check_organizations_presence
+
+      def show
+        @organizations = Organization.all
+        @db_size = db_size
+      end
+
+      def check_organizations_presence
+        return if Organization.exists?
+
+        redirect_to new_organization_path
+      end
+
+      private
+
+      def db_size
+        dbname = ActiveRecord::Base.connection.current_database
+        sql = "SELECT pg_size_pretty(pg_database_size('#{dbname}'));"
+        ActiveRecord::Base.connection.execute(sql)[0]["pg_size_pretty"]
+      end
+    end
+  end
+end

app/jobs/clear_duplicated_half_signup_users_job.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+class ClearDuplicatedHalfSignupUsersJob < ApplicationJob
+  include Decidim::Logging
+
+  def perform
+    @dup_decidim_users_count = 0
+    @dup_half_signup_count = 0
+
+    log! "Start clearing half signup accounts..."
+    if duplicated_phone_numbers.blank?
+      log! "No duplicated phone numbers found"
+      return
+    end
+
+    log! "Found #{duplicated_phone_numbers.count} duplicated phone number to cleanup"
+    duplicated_phone_numbers.each do |phone_info|
+      phone_number, phone_country = phone_info
+      users = Decidim::User.where(phone_number: phone_number, phone_country: phone_country)
+
+      clear_data users
+    end
+
+    log! "Total distinct numbers to clear : #{duplicated_phone_numbers.size}"
+    log! "Half signup users archived : #{@dup_half_signup_count}"
+    log! "Decidim users account updated : #{@dup_decidim_users_count}"
+    log! "Total accounts modified : #{@dup_half_signup_count + @dup_decidim_users_count}"
+    log! "Terminated !"
+  end
+
+  private
+
+  def duplicated_phone_numbers
+    @duplicated_phone_numbers ||= Decidim::User
+                                  .where.not(phone_number: [nil, ""])
+                                  .where.not(phone_country: [nil, ""])
+                                  .group(:phone_number, :phone_country)
+                                  .having("count(*) > 1")
+                                  .pluck(:phone_number, :phone_country)
+  end
+
+  def clear_data(users)
+    decidim_user_dup_accounts = []
+
+    users.each do |user|
+      if user.email.include?("quick_auth")
+        @dup_half_signup_count += 1
+        soft_delete_user(user, delete_reason)
+      else
+        @dup_decidim_users_count += 1
+        decidim_user_dup_accounts << user
+      end
+    end
+
+    return if decidim_user_dup_accounts.blank?
+    # The unique user might be a user without email, if so, it should be cleared
+    return if decidim_user_dup_accounts.size <= 1 && decidim_user_dup_accounts.first.email.present?
+
+    # if there is multiple decidim user accounts, clear all phone number for these accounts
+    decidim_user_dup_accounts.each do |decidim_user|
+      clear_account_phone_number(decidim_user)
+    end
+  end
+
+  def soft_delete_user(user, reason)
+    return unless user.email&.include?("quick_auth")
+
+    email = user.email
+    phone = user.phone_number
+    user.extended_data = user.extended_data.merge({
+                                                    half_signup: {
+                                                      email: email,
+                                                      phone_number: phone,
+                                                      phone_country: user.phone_country
+                                                    }
+                                                  })
+
+    user.phone_number = nil
+    user.phone_country = nil
+
+    form = Decidim::DeleteAccountForm.from_params(delete_reason: reason)
+    Decidim::DestroyAccount.call(user, form) do
+      on(:ok) do
+        log!("User (ID/#{user.id} email/#{email} phone/#{obfuscate_phone_number(phone)}) has been deleted")
+      end
+      on(:invalid) do
+        log!("User (ID/#{user.id} email/#{email} phone/#{obfuscate_phone_number(phone)}) cannot be deleted: #{form.errors.full_messages}")
+      end
+    end
+  end
+
+  def clear_account_phone_number(user)
+    phone_number = user.phone_number
+    Decidim::User.transaction do
+      user.extended_data = user.extended_data.merge({
+                                                      half_signup: {
+                                                        phone_number: user.phone_number,
+                                                        phone_country: user.phone_country
+                                                      }
+                                                    })
+
+      user.phone_number = nil
+      user.phone_country = nil
+      user.save(validate: false)
+    end
+
+    log! "User (ID/#{user.id} phone/#{obfuscate_phone_number(phone_number)} email/#{user.email}) has been cleaned"
+  end
+
+  def obfuscate_phone_number(phone_number)
+    return "No phone number" if phone_number.blank?
+
+    visible_prefix = phone_number[0..1]
+    visible_suffix = phone_number[-2..]
+    obfuscated_middle = "*" * (phone_number.length - 4)
+
+    visible_prefix + obfuscated_middle + visible_suffix
+  end
+
+  def current_date
+    Date.current.strftime "%Y-%m-%d"
+  end
+
+  def delete_reason
+    "HalfSignup duplicated account (#{current_date})"
+  end
+end

app/jobs/concerns/decidim/logging.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Logging
+    private
+
+    def log!(msg, level = :warn)
+      msg = "(#{self.class})> #{msg}"
+
+      case level
+      when :info
+        Rails.logger.info msg
+        stdout_logger.info msg unless Rails.env.test?
+      else
+        Rails.logger.warn msg
+        stdout_logger.warn msg unless Rails.env.test?
+      end
+    end
+
+    def stdout_logger
+      @stdout_logger ||= Logger.new($stdout)
+    end
+  end
+end

app/jobs/decidim/papertrail_versions_job.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Decidim
+  class PapertrailVersionsJob < ApplicationJob
+    queue_as :default
+
+    include Decidim::Logging
+
+    def perform(ret = nil)
+      ret = retention(ret)
+
+      log! "Cleaning versions in database..."
+      log! "Cleaning item_types : #{item_types.join(", ")}"
+
+      total = 0
+      PaperTrail::Version.where(item_type: item_types).where("created_at <= ?", ret).in_batches(of: 5000) do |versions|
+        total += versions.size
+        versions.destroy_all
+      end
+
+      log! "#{total} versions removed"
+    end
+
+    private
+
+    def retention(ret)
+      return ret if ret.present? && ret.is_a?(Time)
+
+      ret = Rails.application.secrets.dig(:decidim, :database, :versions, :clean, :retention)
+      ret.months.ago
+    end
+
+    # Exhaustive list of item_types to remove from versions table
+    def item_types
+      @item_types ||= %w(
+        Decidim::Accountability::TimelineEntry
+        Decidim::Accountability::Result
+        Decidim::Attachment
+        Decidim::AttachmentCollection
+        Decidim::Blogs::Post
+        Decidim::Budgets::Project
+        Decidim::Comments::Comment
+        Decidim::Conferences::MediaLink
+        Decidim::Conferences::Partner
+        Decidim::Debates::Debate
+        Decidim::Categorization
+        Decidim::Categorization
+        Decidim::Forms::Questionnaire
+        Decidim::UserBaseEntity
+      )
+    end
+  end
+end