feat: Clear minio s3 bucket (#612)

* feat(Docker): Add minio service

* feat(rake): Add new tasks to cleanup s3 bucket

* fix: Add S3 purge rake task

* fix: S3 Bucket endpoint for docker local

* fix(rake): Active storage clear orphans job

* fix(sidekiq): Add sidekiq configuration

* fix: Logger for active_storage.rake job

* fix: Prevent duplicated ActiveRecord Query

app/jobs/active_storage_clear_orphans_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class ActiveStorageClearOrphansJob < ApplicationJob
+  include ActionView::Helpers::NumberHelper
+  queue_as :default
+
+  def perform(**args)
+    limit = args[:limit] || 10_000
+    Rails.logger.info "Looking for orphan blobs in S3... (limit: #{limit})"
+    objects = ActiveStorage::Blob.service.bucket.objects
+    Rails.logger.info "Total files: #{objects.size}"
+
+    current_iteration = 0
+    sum = 0
+    orphans_count = 0
+    objects.each do |obj|
+      break if current_iteration >= limit
+
+      current_iteration += 1
+      next if ActiveStorage::Blob.exists?(key: obj.key)
+
+      sum += delete_object(obj)
+      orphans_count += 1
+    end
+
+    Rails.logger.info "Size: #{number_to_human_size(sum)} in #{orphans_count} files"
+    Rails.logger.info "Configuration limit is #{limit} files"
+    Rails.logger.info "Terminated task... "
+  end
+
+  private
+
+  def delete_object(obj)
+    Rails.logger.info "Removing orphan: #{obj.key}"
+    size = obj.size
+    obj.delete
+    size
+  end
+end

config/sidekiq.yml

@@ -63,6 +63,10 @@
       cron: '0 1 * * *'
       class: NotifyProgressInitiatives
       queue: initiatives
+    ActiveStorageClearOrphans:
+      cron: '30 6 1 9 0' # Run at 06:30AM on 1st September
+      class: ActiveStorageClearOrphansJob
+      queue: default
     CleanAdminLogs:
       cron: "0 9 0 * * *"
       class: Decidim::Cleaner::CleanAdminLogsJob

config/storage.yml

@@ -8,7 +8,7 @@ local:
 
 scaleway:
   service: S3
-  endpoint: https://<%= Rails.application.secrets.dig(:scaleway, :endpoint) %>
+  endpoint: <%= Rails.application.secrets.dig(:scaleway, :bucket_name) == "localhost" ? "http" : "https" %>://<%= Rails.application.secrets.dig(:scaleway, :endpoint) %>
   access_key_id: <%= Rails.application.secrets.dig(:scaleway, :id) %>
   secret_access_key: <%= Rails.application.secrets.dig(:scaleway, :token) %>
   region: fr-par

docker-compose.local.yml

@@ -1,20 +1,41 @@
 services:
+  minio:
+    container_name: minio
+    image: "bitnami/minio:latest"
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      - MINIO_DEFAULT_BUCKETS=localhost
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+    networks:
+      - minio_network
+    volumes:
+      - 'minio:/bitnami/minio/data'
+
   database:
     image: postgres
     volumes:
       - pg-data:/var/lib/postgresql/data
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
+    networks:
+      - minio_network
   memcached:
     image: memcached
     ports:
       - "11211:11211"
+    networks:
+      - minio_network
   redis:
     image: redis
     ports:
       - "6379:6379"
     volumes:
       - redis-data:/var/lib/redis/data
+    networks:
+      - minio_network
   sidekiq:
     image: decidim-app:latest
     command: [ "bundle", "exec", "sidekiq", "-C", "config/sidekiq.yml" ]
@@ -26,6 +47,7 @@ services:
       - MEMCACHE_SERVERS=memcached:11211
       - RAILS_SERVE_STATIC_FILES=true
       - RAILS_LOG_TO_STDOUT=true
+      - RAILS_LOG_LEVEL=debug
       - ASSET_HOST=localhost:3000
       - FORCE_SSL=1
       - ENABLE_LETTER_OPENER=1
@@ -41,13 +63,19 @@ services:
       - GEOCODER_LOOKUP_API_KEY=${GEOCODER_LOOKUP_API_KEY}
       - DEFAULT_LOCALE=${DEFAULT_LOCALE}
       - AVAILABLE_LOCALES=${AVAILABLE_LOCALES}
+      - OBJECTSTORE_S3_HOST=minio:9000
+      - SCALEWAY_BUCKET_NAME=localhost
+      - SCALEWAY_ID=minioadmin
+      - SCALEWAY_TOKEN=minioadmin
     depends_on:
       - app
     volumes:
       - shared-volume:/app
     links:
       - database
       - redis
+    networks:
+      - minio_network
   app:
     image: decidim-app:latest
     environment:
@@ -58,6 +86,7 @@ services:
       - MEMCACHE_SERVERS=memcached:11211
       - RAILS_SERVE_STATIC_FILES=true
       - RAILS_LOG_TO_STDOUT=true
+      - RAILS_LOG_LEVEL=debug
       - ASSET_HOST=localhost:3000
       - FORCE_SSL=1
       - ENABLE_LETTER_OPENER=1
@@ -73,6 +102,10 @@ services:
       - GEOCODER_LOOKUP_API_KEY=${GEOCODER_LOOKUP_API_KEY}
       - DEFAULT_LOCALE=${DEFAULT_LOCALE}
       - AVAILABLE_LOCALES=${AVAILABLE_LOCALES}
+      - OBJECTSTORE_S3_HOST=minio:9000
+      - SCALEWAY_BUCKET_NAME=localhost
+      - SCALEWAY_ID=minioadmin
+      - SCALEWAY_TOKEN=minioadmin
     volumes:
       - shared-volume:/app
     ports:
@@ -81,8 +114,16 @@ services:
       - database
       - redis
       - memcached
+    networks:
+      - minio_network
+
+networks:
+  minio_network:
+    driver: bridge
 
 volumes:
   shared-volume: { }
   pg-data: { }
-  redis-data: { }
+  redis-data: { }
+  minio:
+    driver: local

lib/tasks/active_storage.rake

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+namespace :active_storage do
+  namespace :purge do
+    desc "Purge orphan blobs in databa"
+    task blobs: :environment do
+      Rails.logger.info "Looking for blobs without attachments in database..."
+      blobs = ActiveStorage::Blob.where.not(id: ActiveStorage::Attachment.select(:blob_id))
+
+      if blobs.count.zero?
+        Rails.logger.info "Database is clean !"
+        Rails.logger.info "Terminating task..."
+      else
+        Rails.logger.info "Found #{blobs.count} orphan blobs !"
+        blobs.each(&:purge)
+        Rails.logger.info "Task terminated !"
+      end
+    end
+
+    desc "Purge orphan blobs in S3"
+    task s3: :environment do
+      limit = ENV.fetch("S3_LIMIT", "10000").to_i
+      ActiveStorageClearOrphansJob.perform_later(limit: limit)
+    end
+  end
+end