Add parallelization to `tootctl search deploy` (#12051)

* Add parallel gem

* Modify parallel option in tootctl search deploy

* Add paralell option to tootctl search deploy

* Change 1 to false

* Clean up

* Rename --parallel to --processes

diff --git a/Gemfile b/Gemfile
index 7ed1a4e6bb78eb007e68b5903144a4a8716ca37e..f6ee3b52e5953fe7a6caafe99cafe3f087563703 100644 (file)
--- a/Gemfile
+++ b/Gemfile
@@ -67,6 +67,7 @@ gem 'oj', '~> 3.9'
 gem 'ostatus2', '~> 2.0'
 gem 'ox', '~> 2.11'
 gem 'parslet'
+gem 'parallel', '~> 1.17'
 gem 'posix-spawn', git: 'https://github.com/rtomayko/posix-spawn', ref: '58465d2e213991f8afb13b984854a49fcdcc980c'
 gem 'pundit', '~> 2.1'
 gem 'premailer-rails'

diff --git a/Gemfile.lock b/Gemfile.lock
index d708b34c145a6ebd970f1f8b1cd6a606a9070796..b94647a20a20d4a9892e066bf506b34db843aab4 100644 (file)
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -748,6 +748,7 @@ DEPENDENCIES
   ox (~> 2.11)
   paperclip (~> 6.0)
   paperclip-av-transcoder (~> 0.6)
+  parallel (~> 1.17)
   parallel_tests (~> 2.29)
   parslet
   pg (~> 1.1)

diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb
index 42ad93f1edbfa271a7a9cd091a4848adcbc1165a..8bd5f9543ffe38092477c9f840b687c54170e9f5 100644 (file)
--- a/lib/mastodon/search_cli.rb
+++ b/lib/mastodon/search_cli.rb
@@ -6,6 +6,7 @@ require_relative 'cli_helper'
 
 module Mastodon
   class SearchCLI < Thor
+    option :processes, default: 2, aliases: [:p]
     desc 'deploy', 'Create or update an ElasticSearch index and populate it'
     long_desc <<~LONG_DESC
       If ElasticSearch is empty, this command will create the necessary indices
@@ -13,10 +14,28 @@ module Mastodon
 
       This command will also upgrade indices if the underlying schema has been
       changed since the last run.
+
+      With the --processes option, parallelize execution of the command. The
+      default is 2. If "auto" is specified, the number is automatically
+      derived from available CPUs.
     LONG_DESC
     def deploy
-      processed = Chewy::RakeHelper.upgrade
-      Chewy::RakeHelper.sync(except: processed)
+      processed = Chewy::RakeHelper.upgrade(parallel: processes)
+      Chewy::RakeHelper.sync(except: processed, parallel: processes)
+    end
+
+    private
+
+    def processes
+      return true if options[:processes] == 'auto'
+
+      num = options[:processes].to_i
+
+      if num < 2
+        nil
+      else
+        num
+      end
     end
   end
 end