diff --git a/app/jobs/active_storage_clear_orphans_job.rb b/app/jobs/active_storage_clear_orphans_job.rb new file mode 100644 index 0000000000..efa7554111 --- /dev/null +++ b/app/jobs/active_storage_clear_orphans_job.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class ActiveStorageClearOrphansJob < ApplicationJob + include ActionView::Helpers::NumberHelper + queue_as :default + + def perform(**args) + limit = args[:limit] || 10_000 + Rails.logger.info "Looking for orphan blobs in S3... (limit: #{limit})" + objects = ActiveStorage::Blob.service.bucket.objects + Rails.logger.info "Total files: #{objects.size}" + + current_iteration = 0 + sum = 0 + orphans_count = 0 + objects.each do |obj| + break if current_iteration >= limit + + current_iteration += 1 + next if ActiveStorage::Blob.exists?(key: obj.key) + + sum += delete_object(obj) + orphans_count += 1 + end + + Rails.logger.info "Size: #{number_to_human_size(sum)} in #{orphans_count} files" + Rails.logger.info "Configuration limit is #{limit} files" + Rails.logger.info "Terminated task... " + end + + private + + def delete_object(obj) + Rails.logger.info "Removing orphan: #{obj.key}" + size = obj.size + obj.delete + size + end +end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 0d311832d8..06a488dcc0 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -63,6 +63,10 @@ cron: '0 1 * * *' class: NotifyProgressInitiatives queue: initiatives + ActiveStorageClearOrphans: + cron: '30 6 1 9 0' # Run at 06:30AM on 1st September + class: ActiveStorageClearOrphansJob + queue: default CleanAdminLogs: cron: "0 9 0 * * *" class: Decidim::Cleaner::CleanAdminLogsJob diff --git a/config/storage.yml b/config/storage.yml index 54e2f21c1d..552b5fbcb6 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -8,7 +8,7 @@ local: scaleway: service: S3 - endpoint: https://<%= Rails.application.secrets.dig(:scaleway, :endpoint) %> + endpoint: <%= Rails.application.secrets.dig(:scaleway, :bucket_name) == "localhost" ? "http" : "https" %>://<%= Rails.application.secrets.dig(:scaleway, :endpoint) %> access_key_id: <%= Rails.application.secrets.dig(:scaleway, :id) %> secret_access_key: <%= Rails.application.secrets.dig(:scaleway, :token) %> region: fr-par diff --git a/docker-compose.local.yml b/docker-compose.local.yml index c1e5ab0fcd..0a5cb54a6f 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -1,20 +1,41 @@ services: + minio: + container_name: minio + image: "bitnami/minio:latest" + ports: + - "9000:9000" + - "9001:9001" + environment: + - MINIO_DEFAULT_BUCKETS=localhost + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + networks: + - minio_network + volumes: + - 'minio:/bitnami/minio/data' + database: image: postgres volumes: - pg-data:/var/lib/postgresql/data environment: - POSTGRES_HOST_AUTH_METHOD=trust + networks: + - minio_network memcached: image: memcached ports: - "11211:11211" + networks: + - minio_network redis: image: redis ports: - "6379:6379" volumes: - redis-data:/var/lib/redis/data + networks: + - minio_network sidekiq: image: decidim-app:latest command: [ "bundle", "exec", "sidekiq", "-C", "config/sidekiq.yml" ] @@ -26,6 +47,7 @@ services: - MEMCACHE_SERVERS=memcached:11211 - RAILS_SERVE_STATIC_FILES=true - RAILS_LOG_TO_STDOUT=true + - RAILS_LOG_LEVEL=debug - ASSET_HOST=localhost:3000 - FORCE_SSL=1 - ENABLE_LETTER_OPENER=1 @@ -41,6 +63,10 @@ services: - GEOCODER_LOOKUP_API_KEY=${GEOCODER_LOOKUP_API_KEY} - DEFAULT_LOCALE=${DEFAULT_LOCALE} - AVAILABLE_LOCALES=${AVAILABLE_LOCALES} + - OBJECTSTORE_S3_HOST=minio:9000 + - SCALEWAY_BUCKET_NAME=localhost + - SCALEWAY_ID=minioadmin + - SCALEWAY_TOKEN=minioadmin depends_on: - app volumes: @@ -48,6 +74,8 @@ services: links: - database - redis + networks: + - minio_network app: image: decidim-app:latest environment: @@ -58,6 +86,7 @@ services: - MEMCACHE_SERVERS=memcached:11211 - RAILS_SERVE_STATIC_FILES=true - RAILS_LOG_TO_STDOUT=true + - RAILS_LOG_LEVEL=debug - ASSET_HOST=localhost:3000 - FORCE_SSL=1 - ENABLE_LETTER_OPENER=1 @@ -73,6 +102,10 @@ services: - GEOCODER_LOOKUP_API_KEY=${GEOCODER_LOOKUP_API_KEY} - DEFAULT_LOCALE=${DEFAULT_LOCALE} - AVAILABLE_LOCALES=${AVAILABLE_LOCALES} + - OBJECTSTORE_S3_HOST=minio:9000 + - SCALEWAY_BUCKET_NAME=localhost + - SCALEWAY_ID=minioadmin + - SCALEWAY_TOKEN=minioadmin volumes: - shared-volume:/app ports: @@ -81,8 +114,16 @@ services: - database - redis - memcached + networks: + - minio_network + +networks: + minio_network: + driver: bridge volumes: shared-volume: { } pg-data: { } - redis-data: { } \ No newline at end of file + redis-data: { } + minio: + driver: local diff --git a/lib/tasks/active_storage.rake b/lib/tasks/active_storage.rake new file mode 100644 index 0000000000..ef696ccb31 --- /dev/null +++ b/lib/tasks/active_storage.rake @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +namespace :active_storage do + namespace :purge do + desc "Purge orphan blobs in databa" + task blobs: :environment do + Rails.logger.info "Looking for blobs without attachments in database..." + blobs = ActiveStorage::Blob.where.not(id: ActiveStorage::Attachment.select(:blob_id)) + + if blobs.count.zero? + Rails.logger.info "Database is clean !" + Rails.logger.info "Terminating task..." + else + Rails.logger.info "Found #{blobs.count} orphan blobs !" + blobs.each(&:purge) + Rails.logger.info "Task terminated !" + end + end + + desc "Purge orphan blobs in S3" + task s3: :environment do + limit = ENV.fetch("S3_LIMIT", "10000").to_i + ActiveStorageClearOrphansJob.perform_later(limit: limit) + end + end +end