fix: Deduplicate accounts (#653)

app/jobs/clear_duplicated_half_signup_users_job.rb

@@ -1,93 +1,110 @@
 # frozen_string_literal: true
 
-# rubocop:disable Rails/Output
 class ClearDuplicatedHalfSignupUsersJob < ApplicationJob
   include Decidim::Logging
 
   def perform
-    duplicated_numbers = find_duplicated_phone_numbers
-    return puts("No duplicated phone numbers found") if duplicated_numbers.empty?
+    @dup_decidim_users_count = 0
+    @dup_half_signup_count = 0
 
-    alerts = []
+    log! "Start clearing half signup accounts..."
+    if duplicated_phone_numbers.blank?
+      log! "No duplicated phone numbers found"
+      return
+    end
 
-    duplicated_numbers.each do |phone_info|
+    log! "Found #{duplicated_phone_numbers.count} duplicated phone number to cleanup"
+    duplicated_phone_numbers.each do |phone_info|
       phone_number, phone_country = phone_info
+      users = Decidim::User.where(phone_number: phone_number, phone_country: phone_country)
 
-      users_with_phone = Decidim::User.where(phone_number: phone_number, phone_country: phone_country)
-      quick_auth_users = users_with_phone.select { |user| user.email.include?("quick_auth") }
-      other_users = users_with_phone.reject { |user| user.email.include?("quick_auth") }
-
-      quick_auth_users.each { |user| soft_delete_user(user, "Duplicated account") }
-
-      alerts << generate_alert_message(phone_number, other_users) if other_users.map(&:email).uniq.size > 1
+      clear_data users
     end
-    display_alerts(alerts)
+
+    log! "Total distinct numbers to clear : #{duplicated_phone_numbers.size}"
+    log! "Half signup users archived : #{@dup_half_signup_count}"
+    log! "Decidim users account updated : #{@dup_decidim_users_count}"
+    log! "Total accounts modified : #{@dup_half_signup_count + @dup_decidim_users_count}"
+    log! "Terminated !"
   end
 
   private
 
-  def find_duplicated_phone_numbers
-    Decidim::User
-      .where.not(phone_number: [nil, ""])
-      .where.not(phone_country: [nil, ""])
-      .group(:phone_number, :phone_country)
-      .having("count(*) > 1")
-      .pluck(:phone_number, :phone_country)
+  def duplicated_phone_numbers
+    @duplicated_phone_numbers ||= Decidim::User
+                                  .where.not(phone_number: [nil, ""])
+                                  .where.not(phone_country: [nil, ""])
+                                  .group(:phone_number, :phone_country)
+                                  .having("count(*) > 1")
+                                  .pluck(:phone_number, :phone_country)
   end
 
-  def soft_delete_user(user, reason)
-    if user.email.include?("quick_auth")
-      puts "\n---- Processing duplicated phone number: #{obfuscate_phone_number(user.phone_number)} ----\n"
-
-      user.update(phone_number: nil, phone_country: nil)
-
-      form = Decidim::DeleteAccountForm.from_params(delete_reason: reason)
-      previous_email = user.email
-      Decidim::DestroyAccount.call(user, form) do
-        on(:ok) do
-          log!("User #{user.id} (#{previous_email}) has been deleted", :info)
-          puts("User #{user.id} (#{previous_email}) has been deleted")
-          save_deleted_user_email(user, previous_email)
-        end
-        on(:invalid) do
-          log!("Failed to delete user #{user.id} (#{user.email}): #{form.errors.full_messages}", :warn)
-          puts("Failed to delete user #{user.id} (#{user.email}): #{form.errors.full_messages}")
-        end
+  def clear_data(users)
+    decidim_user_dup_accounts = []
+
+    users.each do |user|
+      if user.email.include?("quick_auth")
+        @dup_half_signup_count += 1
+        soft_delete_user(user, delete_reason)
+      else
+        @dup_decidim_users_count += 1
+        decidim_user_dup_accounts << user
       end
-    else
-      log!("Not a Quick Auth account, skipping deletion", :info)
-      puts("Not a Quick Auth account, skipping deletion")
     end
-  end
-
-  def save_deleted_user_email(user, email)
-    user.extended_data["deleted_user_email"] = email
-    user.save
-  end
-
-  def generate_alert_message(phone_number, users)
-    obfuscated_number = obfuscate_phone_number(phone_number)
-    emails = users.map { |user| "#{user.id} - #{user.email}" }
-    email_pairs = emails.each_slice(2).map { |pair| pair.join(" | ") }
-
-    <<~MSG
-
-      ALERT: Duplicated Phone Number Detected : #{obfuscated_number}
 
-      Users with this number:
+    return if decidim_user_dup_accounts.blank?
+    # The unique user might be a user without email, if so, it should be cleared
+    return if decidim_user_dup_accounts.size <= 1 && decidim_user_dup_accounts.first.email.present?
 
-      #{email_pairs.join("\n")}
-    MSG
+    # if there is multiple decidim user accounts, clear all phone number for these accounts
+    decidim_user_dup_accounts.each do |decidim_user|
+      clear_account_phone_number(decidim_user)
+    end
   end
 
-  def display_alerts(alerts)
-    return if alerts.empty?
+  def soft_delete_user(user, reason)
+    return unless user.email&.include?("quick_auth")
+
+    email = user.email
+    phone = user.phone_number
+    user.extended_data = user.extended_data.merge({
+                                                    half_signup: {
+                                                      email: email,
+                                                      phone_number: phone,
+                                                      phone_country: user.phone_country
+                                                    }
+                                                  })
+
+    user.phone_number = nil
+    user.phone_country = nil
+
+    form = Decidim::DeleteAccountForm.from_params(delete_reason: reason)
+    Decidim::DestroyAccount.call(user, form) do
+      on(:ok) do
+        log!("User (ID/#{user.id} email/#{email} phone/#{obfuscate_phone_number(phone)}) has been deleted")
+      end
+      on(:invalid) do
+        log!("User (ID/#{user.id} email/#{email} phone/#{obfuscate_phone_number(phone)}) cannot be deleted: #{form.errors.full_messages}")
+      end
+    end
+  end
 
-    puts "\n---- ALERTS ----"
-    alerts.each do |alert|
-      puts alert
-      log!(alert, :warn)
+  def clear_account_phone_number(user)
+    phone_number = user.phone_number
+    Decidim::User.transaction do
+      user.extended_data = user.extended_data.merge({
+                                                      half_signup: {
+                                                        phone_number: user.phone_number,
+                                                        phone_country: user.phone_country
+                                                      }
+                                                    })
+
+      user.phone_number = nil
+      user.phone_country = nil
+      user.save(validate: false)
     end
+
+    log! "User (ID/#{user.id} phone/#{obfuscate_phone_number(phone_number)} email/#{user.email}) has been cleaned"
   end
 
   def obfuscate_phone_number(phone_number)
@@ -99,5 +116,12 @@ def obfuscate_phone_number(phone_number)
 
     visible_prefix + obfuscated_middle + visible_suffix
   end
+
+  def current_date
+    Date.current.strftime "%Y-%m-%d"
+  end
+
+  def delete_reason
+    "HalfSignup duplicated account (#{current_date})"
+  end
 end
-# rubocop:enable Rails/Output

app/jobs/concerns/decidim/logging.rb

@@ -5,13 +5,20 @@ module Logging
     private
 
     def log!(msg, level = :warn)
-      msg = "(#{self.class}) #{Time.current.strftime("%d-%m-%Y %H:%M")}> #{msg}"
+      msg = "(#{self.class})> #{msg}"
+
       case level
       when :info
         Rails.logger.info msg
+        stdout_logger.info msg unless Rails.env.test?
       else
         Rails.logger.warn msg
+        stdout_logger.warn msg unless Rails.env.test?
       end
     end
+
+    def stdout_logger
+      @stdout_logger ||= Logger.new($stdout)
+    end
   end
 end

app/jobs/decidim/papertrail_versions_job.rb

@@ -18,7 +18,7 @@ def perform(ret = nil)
         versions.destroy_all
       end
 
-      log! "#{total} versions have been removed"
+      log! "#{total} versions removed"
     end
 
     private

lib/tasks/clear_duplicated_users.rake

@@ -3,6 +3,8 @@
 namespace :decidim do
   desc "Clear duplicated users with the same phone_numbers in the database"
   task clear_duplicated_users: :environment do
+    include Decidim::Logging
+
     ClearDuplicatedHalfSignupUsersJob.perform_now
   end
 end

lib/tasks/db.rake

@@ -49,5 +49,35 @@ namespace :decidim do
         puts "(decidim:db:versions:clean) #{Time.current.strftime("%d-%m-%Y %H:%M:%S")}> Job delayed to Sidekiq."
       end
     end
+
+    namespace :restore do
+      desc "Clear database dump to work with localhost"
+      task local: :environment do
+        puts "(decidim:db:restore:local) #{Time.current.strftime("%d-%m-%Y %H:%M:%S")}> Modifying Organization settings..."
+        organizations = Decidim::Organization.all.pluck(:id, :name, :host)
+
+        if organizations.blank?
+          puts "(decidim:db:restore:local) #{Time.current.strftime("%d-%m-%Y %H:%M:%S")}> No existing organizations..."
+          puts "(decidim:db:restore:local) #{Time.current.strftime("%d-%m-%Y %H:%M:%S")}> Terminating"
+          return
+        elsif organizations.size == 1
+          organization = Decidim::Organization.first
+        else
+          organizations.each do |org|
+            puts "#{org.id}) #{org.name} - #{org.host}"
+          end
+          puts "Select the organization ID: "
+          org_id = $stdin.gets
+          organization = Decidim::Organization.find(org_id)
+        end
+
+        organization.host = "localhost"
+        organization.smtp_settings = {}
+        organization.omniauth_settings = {}
+        organization.save(validate: false)
+
+        puts "(decidim:db:restore:local) #{Time.current.strftime("%d-%m-%Y %H:%M:%S")}> Changes done..."
+      end
+    end
   end
 end