Reduce number of items in feeds, optimize regeneration worker slightly,

make regeneration worker unique, (only schedule/execute once at a time)

diff --git a/Gemfile b/Gemfile
index cb98241311b27d8f78afdcf41ec7a035a05494d8..41c636904b09547490c6cb7f45d2276f0166f3c3 100644 (file)
--- a/Gemfile
+++ b/Gemfile
@@ -46,6 +46,8 @@ gem 'will_paginate'
 gem 'rack-attack'
 gem 'rack-cors', require: 'rack/cors'
 gem 'sidekiq'
+gem 'sidekiq-unique-jobs'
+gem 'sidekiq-merger'
 gem 'rails-settings-cached'
 gem 'simple-navigation'
 gem 'statsd-instrument'

diff --git a/Gemfile.lock b/Gemfile.lock
index 6e3115249db684184471cb380f584e844acd0cd2..27de1bee085874c07d242da3c75032dfd7265abe 100644 (file)
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -387,6 +387,13 @@ GEM
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
       redis (~> 3.2, >= 3.2.1)
+    sidekiq-merger (0.0.11)
+      activesupport (>= 3.2, < 6)
+      concurrent-ruby (~> 1.0)
+      sidekiq (>= 3.4, < 5)
+    sidekiq-unique-jobs (4.0.18)
+      sidekiq (>= 2.6)
+      thor
     simple-navigation (4.0.3)
       activesupport (>= 2.3.2)
     simple_form (3.2.1)
@@ -510,6 +517,8 @@ DEPENDENCIES
   sass-rails (~> 5.0)
   sdoc (~> 0.4.0)
   sidekiq
+  sidekiq-merger
+  sidekiq-unique-jobs
   simple-navigation
   simple_form
   simplecov

diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 919bc3df90532d466ff4c5c8ab533b4f18fdc21c..a2efcce10729fc7daf6a79d66f0b922e9eac8c58 100644 (file)
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -5,7 +5,7 @@ require 'singleton'
 class FeedManager
   include Singleton
 
-  MAX_ITEMS = 800
+  MAX_ITEMS = 400
 
   def key(type, id)
     "feed:#{type}:#{id}"
@@ -50,9 +50,9 @@ class FeedManager
 
   def merge_into_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
-    query        = from_account.statuses.limit(MAX_ITEMS)
+    query        = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
 
-    if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS
+    if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
       oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
       query = query.where('id > ?', oldest_home_score)
     end

diff --git a/app/services/precompute_feed_service.rb b/app/services/precompute_feed_service.rb
index e1ec56e8d3bde09b11795cc197ea6b277f017829..a57c401d04cb33bcd41447312dbef403288d8e65 100644 (file)
--- a/app/services/precompute_feed_service.rb
+++ b/app/services/precompute_feed_service.rb
@@ -5,9 +5,11 @@ class PrecomputeFeedService < BaseService
   # @param [Symbol] type :home or :mentions
   # @param [Account] account
   def call(_, account)
-    Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS).each do |status|
-      next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
-      redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
+    redis.pipelined do
+      Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS / 4).each do |status|
+        next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
+        redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
+      end
     end
   end
 

diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 82665b581f91a06c55731cac35cca17f9fbd7f77..da8b845f62b83a8ddcd0ede11fcc75c104dd7cd2 100644 (file)
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -3,7 +3,7 @@
 class RegenerationWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull', backtrace: true
+  sidekiq_options queue: 'pull', backtrace: true, unique: :until_executed
 
   def perform(account_id, _ = :home)
     PrecomputeFeedService.new.call(:home, Account.find(account_id))