Fixes/do not override timestamps (#7336)

* Revert "Fixes/do not override timestamps (#7331)"

This reverts commit 581a5c9d29ef2a12f46b67a1097a9ad6df1c6953.

* Document Snowflake ID corner-case a bit more

Snowflake IDs are used for two purposes: making object identifiers harder to
guess and ensuring they are in chronological order. For this reason, they
are based on the `created_at` attribute of the object.

Unfortunately, inserting items with older snowflakes IDs will break the
assumption of consumers of the paging APIs that new items will always have
a greater identifier than the last seen one.

* Add `override_timestamps` virtual attribute to not correlate snowflake ID with created_at

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 5b97a6208d2e9a3b104eed858b43f2477401fbfc..84d4b175206f46534a916c624e6bace9c8a78271 100644 (file)
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -78,9 +78,11 @@ class ActivityPub::Activity
     notify_about_reblog(status) if reblog_of_local_account?(status)
     notify_about_mentions(status)
 
-    # Only continue if the status is supposed to have
-    # arrived in real-time
-    return unless status.within_realtime_window?
+    # Only continue if the status is supposed to have arrived in real-time.
+    # Note that if @options[:override_timestamps] isn't set, the status
+    # may have a lower snowflake id than other existing statuses, potentially
+    # "hiding" it from paginated API calls
+    return unless @options[:override_timestamps] || status.within_realtime_window?
 
     distribute_to_followers(status)
   end

diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 8840a450cb761ab971af88d55b6f817c6003a7d0..7e146ea8c43d79f20ed6e52a4ad082b36a99268c 100644 (file)
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -16,6 +16,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
       reblog: original_status,
       uri: @json['id'],
       created_at: @json['published'],
+      override_timestamps: @options[:override_timestamps],
       visibility: original_status.visibility
     )
 

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index edee2691ffcada501f4f1ad779e738fdb4d99f44..8d17a4ebe2e8efb99be6e50b84a8a03ec1c3d7c2 100644 (file)
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -48,6 +48,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       language: detected_language,
       spoiler_text: @object['summary'] || '',
       created_at: @object['published'],
+      override_timestamps: @options[:override_timestamps],
       reply: @object['inReplyTo'].present?,
       sensitive: @object['sensitive'] || false,
       visibility: visibility_from_audience,

diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index a24a0093c35e6def74197d8f72181c4766907b09..1e7f47029944aa81fc9e8c67c62def1b8934b47b 100644 (file)
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -40,6 +40,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
         text: content,
         spoiler_text: content_warning,
         created_at: published,
+        override_timestamps: @options[:override_timestamps],
         reply: thread?,
         language: content_language,
         visibility: visibility_scope,
@@ -61,7 +62,14 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
     Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
 
     LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
-    DistributionWorker.perform_async(status.id) if status.within_realtime_window?
+
+    # Only continue if the status is supposed to have arrived in real-time.
+    # Note that if @options[:override_timestamps] isn't set, the status
+    # may have a lower snowflake id than other existing statuses, potentially
+    # "hiding" it from paginated API calls
+    return status unless @options[:override_timestamps] || status.within_realtime_window?
+
+    DistributionWorker.perform_async(status.id)
 
     status
   end

diff --git a/app/models/status.rb b/app/models/status.rb
index fbb1f89aa0e58c6ad691f4f60b4d444acf833ec0..853e75b43fa5214d247fb56497aa56a2df64c1ba 100644 (file)
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -31,6 +31,10 @@ class Status < ApplicationRecord
   include Cacheable
   include StatusThreadingConcern
 
+  # If `override_timestamps` is set at creation time, Snowflake ID creation
+  # will be based on current time instead of `created_at`
+  attr_accessor :override_timestamps
+
   update_index('statuses#status', :proper) if Chewy.enabled?
 
   enum visibility: [:public, :unlisted, :private, :direct], _suffix: :visibility

diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb
index bb9adf64bde9cccb11c293900b68d7acf1a5eb8e..0e2e0edddba98b56db5ab48e7056e5e147d672ab 100644 (file)
--- a/app/workers/activitypub/processing_worker.rb
+++ b/app/workers/activitypub/processing_worker.rb
@@ -6,6 +6,6 @@ class ActivityPub::ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id))
+    ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true)
   end
 end

diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 5df404bcc9b7bb3e653bbfb20d22482b9457c398..978c3aba26f2d0238fe0266868ca91ff6fc84d54 100644 (file)
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -6,6 +6,6 @@ class ProcessingWorker
   sidekiq_options backtrace: true
 
   def perform(account_id, body)
-    ProcessFeedService.new.call(body, Account.find(account_id))
+    ProcessFeedService.new.call(body, Account.find(account_id), override_timestamps: true)
   end
 end

diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb
index 219e323d487b88d07c737734613ac724c0cb8b3a..9e5bc7383c1f546548eaaf1502d5a3782a9f4ab2 100644 (file)
--- a/lib/mastodon/snowflake.rb
+++ b/lib/mastodon/snowflake.rb
@@ -7,7 +7,7 @@ module Mastodon::Snowflake
     def self.around_create(record)
       now = Time.now.utc
 
-      if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at
+      if record.created_at.nil? || record.created_at >= now || record.created_at == record.updated_at || record.override_timestamps
         yield
       else
         record.id = Mastodon::Snowflake.id_at(record.created_at)