Add improved CLI interface for removing remote media (#8411)

./bin/tootctl media remove --days 7 --background

Make the old rake task point to it

diff --git a/app/workers/maintenance/destroy_media_worker.rb b/app/workers/maintenance/destroy_media_worker.rb
index 5f052983be6b42e5432cf474e7dad053fcf1b6fe..cde33d6d79a53df409cdb49c72ff7ea10e3e9351 100644 (file)
--- a/app/workers/maintenance/destroy_media_worker.rb
+++ b/app/workers/maintenance/destroy_media_worker.rb
@@ -6,7 +6,7 @@ class Maintenance::DestroyMediaWorker
   sidekiq_options queue: 'pull'
 
   def perform(media_attachment_id)
-    media = MediaAttachment.find(media_attachment_id)
+    media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id)
     media.destroy
   rescue ActiveRecord::RecordNotFound
     true

diff --git a/app/workers/maintenance/redownload_account_media_worker.rb b/app/workers/maintenance/redownload_account_media_worker.rb
index fc26815f25e87f6bc2705b8a343a1f2d5b58b333..6afbe6e19cef35261697b5e92be3e1d438e36717 100644 (file)
--- a/app/workers/maintenance/redownload_account_media_worker.rb
+++ b/app/workers/maintenance/redownload_account_media_worker.rb
@@ -6,7 +6,7 @@ class Maintenance::RedownloadAccountMediaWorker
   sidekiq_options queue: 'pull', retry: false
 
   def perform(account_id)
-    account = Account.find(account_id)
+    account = account_id.is_a?(Account) ? account_id : Account.find(account_id)
     account.reset_avatar!
     account.reset_header!
     account.save

diff --git a/app/workers/maintenance/uncache_media_worker.rb b/app/workers/maintenance/uncache_media_worker.rb
index 2d1a670a76ce6e9e4d5b3f77aabbbed34bd8d8b7..4bc62ef755c30e712420a494fd71a82fcf623b09 100644 (file)
--- a/app/workers/maintenance/uncache_media_worker.rb
+++ b/app/workers/maintenance/uncache_media_worker.rb
@@ -6,7 +6,7 @@ class Maintenance::UncacheMediaWorker
   sidekiq_options queue: 'pull'
 
   def perform(media_attachment_id)
-    media = MediaAttachment.find(media_attachment_id)
+    media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id)
 
     return if media.file.blank?
 

diff --git a/bin/tootctl b/bin/tootctl
new file mode 100755 (executable)
index 0000000..2fe0252
--- /dev/null
+++ b/bin/tootctl
@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../config/application', __dir__)
+require_relative '../lib/cli'
+Mastodon::CLI.start(ARGV)

diff --git a/lib/cli.rb b/lib/cli.rb
new file mode 100644 (file)
index 0000000..7e82806
--- /dev/null
+++ b/lib/cli.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'thor'
+require_relative 'mastodon/media_cli'
+
+module Mastodon
+  class CLI < Thor
+    desc 'media SUBCOMMAND ...ARGS', 'manage media files'
+    subcommand 'media', Mastodon::MediaCLI
+  end
+end

diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb
new file mode 100644 (file)
index 0000000..cc6ad07
--- /dev/null
+++ b/lib/mastodon/media_cli.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+
+# rubocop:disable Rails/Output
+
+module Mastodon
+  class MediaCLI < Thor
+    option :days, type: :numeric, default: 7
+    option :background, type: :boolean, default: false
+    desc 'remove', 'remove remote media files'
+    long_desc <<-DESC
+      Removes locally cached copies of media attachments from other servers.
+
+      The --days option specifies how old media attachments have to be before
+      they are removed. It defaults to 7 days.
+
+      With the --background option, instead of deleting the files sequentially,
+      they will be queued into Sidekiq and the command will exit as soon as
+      possible. In Sidekiq they will be processed with higher concurrency, but
+      it may impact other operations of the Mastodon server, and it may overload
+      the underlying file storage.
+    DESC
+    def remove
+      time_ago = options[:days].days.ago
+      queued   = 0
+
+      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
+        if options[:background]
+          queued += media_attachments.size
+          Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
+        else
+          media_attachments.each do |m|
+            Maintenance::UncacheMediaWorker.new.perform(m)
+            print '.'
+          end
+        end
+      end
+
+      puts
+      puts "Scheduled the deletion of #{queued} media attachments" if options[:background]
+    end
+  end
+end
+
+# rubocop:enable Rails/Output

diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake
index 191ce634c441911bf040e7d80018ca316c87d569..9ce39335deb43a726a817f374954c24484fa1ab3 100644 (file)
--- a/lib/tasks/mastodon.rake
+++ b/lib/tasks/mastodon.rake
@@ -512,14 +512,9 @@ namespace :mastodon do
 
     desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)'
     task remove_remote: :environment do
-      time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago
-      nb_media_attachments = 0
-
-      MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments|
-        nb_media_attachments += media_attachments.length
-        Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
-      end
-      puts "Scheduled the deletion of #{nb_media_attachments} media attachments"
+      require_relative '../mastodon/media_cli'
+      cli = Mastodon::MediaCLI.new([], days: ENV['NUM_DAYS'] || 7)
+      cli.invoke(:remove)
     end
 
     desc 'Set unknown attachment type for remote-only attachments'