]> cat aescling's git repositories - mastodon.git/commitdiff
Merge branch 'main' into glitch-soc/merge-upstream
authorClaire <claire.github-309c@sitedethib.com>
Wed, 19 Jan 2022 22:19:00 +0000 (23:19 +0100)
committerClaire <claire.github-309c@sitedethib.com>
Wed, 19 Jan 2022 22:52:48 +0000 (23:52 +0100)
Conflicts:
- `app/lib/activitypub/activity/create.rb`:
  Upstream refactored how `Create` activities are handled and how values are
  extracted from `Create`d objects. This conflicted with how glitch-soc
  supported the `directMessage` flag to explicitly distinguish between
  limited and direct messages.
  Ported glitch-soc's changes to latest upstream changes.
- `app/services/fan_out_on_write_service.rb`:
  Upstream largely refactored that file and changed some of the logic.
  This conflicted with glitch-soc's handling of the direct timeline and
  the options to allow replies and boosts in public feeds.
  Ported those glitch-soc changes on top of latest upstream changes.
- `app/services/process_mentions_service.rb`:
  Upstream refactored to move mention-related ActivityPub deliveries to
  `ActivityPub::DeliveryWorker`, while glitch-soc contained an extra check
  to not send local-only toots to remote mentioned users.
  Took upstream's version, as the check is not needed anymore, since it is
  performed at the `ActivityPub::DeliveryWorker` call site already.
- `app/workers/feed_insert_worker.rb`:
  Upstream added support for `update` toot events, while glitch-soc had
  support for an extra timeline support, `direct`.
  Ported upstream changes and extended them to the `direct` timeline.

Additional changes:
- `app/lib/activitypub/parser/status_parser.rb`:
  Added code to handle the `directMessage` flag and take it into account
  to compute visibility.
- `app/lib/feed_manager.rb`:
  Extended upstream's support of `update` toot events to glitch-soc's
  `direct` timeline.

12 files changed:
1  2 
app/javascript/styles/mastodon/components.scss
app/lib/activitypub/activity/create.rb
app/lib/activitypub/parser/status_parser.rb
app/lib/feed_manager.rb
app/models/status.rb
app/serializers/activitypub/note_serializer.rb
app/serializers/rest/status_serializer.rb
app/services/fan_out_on_write_service.rb
app/services/remove_status_service.rb
app/workers/feed_insert_worker.rb
config/routes.rb
db/schema.rb

index c50ddf8d5a3a4790b634ac67f89c546a9e7519fe,a861c34bc3fd992bc8c09a3eb05e509f38ff8d84..9e93cac64e41b221ef04b1419daa3db011ab3477
@@@ -132,29 -152,19 +152,19 @@@ class ActivityPub::Activity::Create < A
        # If there is at least one silent mention, then the status can be considered
        # as a limited-audience status, and not strictly a direct message, but only
        # if we considered a direct message in the first place
-       next unless @params[:visibility] == :direct && direct_message.nil?
-       @params[:visibility] = :limited
 -      @params[:visibility] = :limited if @params[:visibility] == :direct
++      @params[:visibility] = :limited if @params[:visibility] == :direct && !@object['directMessage']
      end
  
-     # If the payload was delivered to a specific inbox, the inbox owner must have
-     # access to it, unless they already have access to it anyway
-     return if @options[:delivered_to_account_id].nil? || @mentions.any? { |mention| mention.account_id == @options[:delivered_to_account_id] }
-     @mentions << Mention.new(account_id: @options[:delivered_to_account_id], silent: true)
-     return unless @params[:visibility] == :direct && direct_message.nil?
-     @params[:visibility] = :limited
+     # Accounts that are tagged but are not in the audience are not
+     # supposed to be notified explicitly
+     @silenced_account_ids = @mentions.map(&:account_id) - accounts_in_audience.map(&:id)
    end
  
    def postprocess_audience_and_deliver
      return if @status.mentions.find_by(account_id: @options[:delivered_to_account_id])
  
-     delivered_to_account = Account.find(@options[:delivered_to_account_id])
      @status.mentions.create(account: delivered_to_account, silent: true)
-     @status.update(visibility: :limited) if @status.direct_visibility? && direct_message.nil?
 -    @status.update(visibility: :limited) if @status.direct_visibility?
++    @status.update(visibility: :limited) if @status.direct_visibility? && !@object['directMessage']
  
      return unless delivered_to_account.following?(@account)
  
index 0000000000000000000000000000000000000000,3ba154d01551faaf0a17b4f2dffebaca099e3df1..75b8f3d5c3f870eca92a0fe35027267a7c0ff019
mode 000000,100644..100644
--- /dev/null
@@@ -1,0 -1,118 +1,124 @@@
+ # frozen_string_literal: true
+ class ActivityPub::Parser::StatusParser
+   include JsonLdHelper
+   # @param [Hash] json
+   # @param [Hash] magic_values
+   # @option magic_values [String] :followers_collection
+   def initialize(json, magic_values = {})
+     @json         = json
+     @object       = json['object'] || json
+     @magic_values = magic_values
+   end
+   def uri
+     id = @object['id']
+     if id&.start_with?('bear:')
+       Addressable::URI.parse(id).query_values['u']
+     else
+       id
+     end
+   rescue Addressable::URI::InvalidURIError
+     id
+   end
+   def url
+     url_to_href(@object['url'], 'text/html') if @object['url'].present?
+   end
+   def text
+     if @object['content'].present?
+       @object['content']
+     elsif content_language_map?
+       @object['contentMap'].values.first
+     end
+   end
+   def spoiler_text
+     if @object['summary'].present?
+       @object['summary']
+     elsif summary_language_map?
+       @object['summaryMap'].values.first
+     end
+   end
+   def title
+     if @object['name'].present?
+       @object['name']
+     elsif name_language_map?
+       @object['nameMap'].values.first
+     end
+   end
+   def created_at
+     @object['published']&.to_datetime
+   rescue ArgumentError
+     nil
+   end
+   def edited_at
+     @object['updated']&.to_datetime
+   rescue ArgumentError
+     nil
+   end
+   def reply
+     @object['inReplyTo'].present?
+   end
+   def sensitive
+     @object['sensitive']
+   end
+   def visibility
+     if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) }
+       :public
+     elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) }
+       :unlisted
+     elsif audience_to.include?(@magic_values[:followers_collection])
+       :private
++    elsif direct_message == false
++      :limited
+     else
+       :direct
+     end
+   end
+   def language
+     if content_language_map?
+       @object['contentMap'].keys.first
+     elsif name_language_map?
+       @object['nameMap'].keys.first
+     elsif summary_language_map?
+       @object['summaryMap'].keys.first
+     end
+   end
++  def direct_message
++    @object['directMessage']
++  end
++
+   private
+   def audience_to
+     as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) }
+   end
+   def audience_cc
+     as_array(@object['cc'] || @json['cc']).map { |x| value_or_id(x) }
+   end
+   def summary_language_map?
+     @object['summaryMap'].is_a?(Hash) && !@object['summaryMap'].empty?
+   end
+   def content_language_map?
+     @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty?
+   end
+   def name_language_map?
+     @object['nameMap'].is_a?(Hash) && !@object['nameMap'].empty?
+   end
+ end
index d57508ef9cc52dbd5a36186da37d38c8cb693f33,c4dd9d00ff3680923a1c4ea6326995592b69c5b5..12732532732ab277abd8ceea0af498be8bdba520
@@@ -98,29 -100,6 +102,29 @@@ class FeedManage
      true
    end
  
-   def push_to_direct(account, status)
 +  # Add a status to a linear direct message feed and send a streaming API update
 +  # @param [Account] account
 +  # @param [Status] status
 +  # @return [Boolean]
-     PushUpdateWorker.perform_async(account.id, status.id, "timeline:direct:#{account.id}")
++  def push_to_direct(account, status, update: false)
 +    return false unless add_to_feed(:direct, account.id, status)
 +
 +    trim(:direct, account.id)
-   def unpush_from_direct(account, status)
++    PushUpdateWorker.perform_async(account.id, status.id, "timeline:direct:#{account.id}") unless update
 +    true
 +  end
 +
 +  # Remove a status from a linear direct message feed and send a streaming API update
 +  # @param [List] list
 +  # @param [Status] status
 +  # @return [Boolean]
-     redis.publish("timeline:direct:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
++  def unpush_from_direct(account, status, update: false)
 +    return false unless remove_from_feed(:direct, account.id, status)
 +
++    redis.publish("timeline:direct:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update
 +    true
 +  end
 +
    # Fill a home feed with an account's statuses
    # @param [Account] from_account
    # @param [Account] into_account
index ef9e6d817d60a2043a62243dcfb0ff1bb7beb515,3358d6891b815069e746d6520414356829159a59..d57026354e3876578b76b55533b06215b7c00f27
  #  account_id             :bigint(8)        not null
  #  application_id         :bigint(8)
  #  in_reply_to_account_id :bigint(8)
 +#  local_only             :boolean
 +#  full_status_text       :text             default(""), not null
  #  poll_id                :bigint(8)
 +#  content_type           :string
  #  deleted_at             :datetime
+ #  edited_at              :datetime
  #
  
  class Status < ApplicationRecord
index e08c537b0cfbb8d29bcaadf701e976e0b3b748d3,12dabc65a08ad961ad6db87003633c49234e3255..aa552a7248856b949a0b7bcc0f6d6253371efb16
@@@ -11,9 -11,8 +11,10 @@@ class ActivityPub::NoteSerializer < Act
  
    attribute :content
    attribute :content_map, if: :language?
+   attribute :updated, if: :edited?
  
 +  attribute :direct_message, if: :non_public?
 +
    has_many :media_attachments, key: :attachment
    has_many :virtual_tags, key: :tag
  
index 6fa98ce12b2a29803c934c6d63a2a68567441ae7,f62f78a790f64076245cd2baf41bf378d69d5795..169a2411d97c59dd24b46a93c7cbbdfd76c7e5f4
@@@ -3,49 -3,71 +3,73 @@@
  class FanOutOnWriteService < BaseService
    # Push a status into home and mentions feeds
    # @param [Status] status
-   def call(status)
-     raise Mastodon::RaceConditionError if status.visibility.nil?
-     deliver_to_self(status) if status.account.local?
-     if status.direct_visibility?
-       deliver_to_mentioned_followers(status)
-       deliver_to_direct_timelines(status)
-       deliver_to_own_conversation(status)
-     elsif status.limited_visibility?
-       deliver_to_mentioned_followers(status)
-     else
-       deliver_to_followers(status)
-       deliver_to_lists(status)
-     end
+   # @param [Hash] options
+   # @option options [Boolean] update
+   # @option options [Array<Integer>] silenced_account_ids
+   def call(status, options = {})
+     @status    = status
+     @account   = status.account
+     @options   = options
+     check_race_condition!
+     fan_out_to_local_recipients!
+     fan_out_to_public_streams! if broadcastable?
+   end
  
-     return if status.account.silenced? || !status.public_visibility?
-     return if status.reblog? && !Setting.show_reblogs_in_public_timelines
+   private
  
-     render_anonymous_payload(status)
+   def check_race_condition!
+     # I don't know why but at some point we had an issue where
+     # this service was being executed with status objects
+     # that had a null visibility - which should not be possible
+     # since the column in the database is not nullable.
+     #
+     # This check re-queues the service to be run at a later time
+     # with the full object, if something like it occurs
  
-     deliver_to_hashtags(status)
+     raise Mastodon::RaceConditionError if @status.visibility.nil?
+   end
  
-     return if status.reply? && status.in_reply_to_account_id != status.account_id && !Setting.show_replies_in_public_timelines
+   def fan_out_to_local_recipients!
+     deliver_to_self!
+     notify_mentioned_accounts!
  
-     deliver_to_public(status)
-     deliver_to_media(status) if status.media_attachments.any?
+     case @status.visibility.to_sym
+     when :public, :unlisted, :private
+       deliver_to_all_followers!
+       deliver_to_lists!
+     when :limited
+       deliver_to_mentioned_followers!
+     else
+       deliver_to_mentioned_followers!
+       deliver_to_conversation!
++      deliver_to_direct_timelines!
+     end
    end
  
-   private
+   def fan_out_to_public_streams!
+     broadcast_to_hashtag_streams!
+     broadcast_to_public_streams!
+   end
  
-   def deliver_to_self(status)
-     Rails.logger.debug "Delivering status #{status.id} to author"
-     FeedManager.instance.push_to_home(status.account, status)
-     FeedManager.instance.push_to_direct(status.account, status) if status.direct_visibility?
+   def deliver_to_self!
+     FeedManager.instance.push_to_home(@account, @status, update: update?) if @account.local?
++    FeedManager.instance.push_to_direct(@account, @status, update: update?) if @account.local? && @status.direct_visibility?
    end
  
-   def deliver_to_followers(status)
-     Rails.logger.debug "Delivering status #{status.id} to followers"
+   def notify_mentioned_accounts!
+     @status.active_mentions.where.not(id: @options[:silenced_account_ids] || []).joins(:account).merge(Account.local).select(:id, :account_id).reorder(nil).find_in_batches do |mentions|
+       LocalNotificationWorker.push_bulk(mentions) do |mention|
+         [mention.account_id, mention.id, 'Mention', :mention]
+       end
+     end
+   end
  
-     status.account.followers_for_local_distribution.select(:id).reorder(nil).find_in_batches do |followers|
+   def deliver_to_all_followers!
+     @account.followers_for_local_distribution.select(:id).reorder(nil).find_in_batches do |followers|
        FeedInsertWorker.push_bulk(followers) do |follower|
-         [status.id, follower.id, :home]
+         [@status.id, follower.id, :home, update: update?]
        end
      end
    end
      end
    end
  
-   def render_anonymous_payload(status)
-     @payload = InlineRenderer.render(status, nil, :status)
-     @payload = Oj.dump(event: :update, payload: @payload)
++  def deliver_to_direct_timelines!
++    FeedInsertWorker.push_bulk(@status.mentions.includes(:account).map(&:account).select { |mentioned_account| mentioned_account.local? }) do |account|
++      [@status.id, account.id, :direct, update: update?]
++    end
 +  end
 +
-   def deliver_to_hashtags(status)
-     Rails.logger.debug "Delivering status #{status.id} to hashtags"
-     status.tags.pluck(:name).each do |hashtag|
-       Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", @payload)
-       Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", @payload) if status.local?
+   def broadcast_to_hashtag_streams!
+     @status.tags.pluck(:name).each do |hashtag|
+       Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", anonymous_payload)
+       Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", anonymous_payload) if @status.local?
      end
    end
  
-   def deliver_to_public(status)
-     Rails.logger.debug "Delivering status #{status.id} to public timeline"
+   def broadcast_to_public_streams!
 -    return if @status.reply? && @status.in_reply_to_account_id != @account.id
++    return if @status.reply? && @status.in_reply_to_account_id != @account.id && !Setting.show_replies_in_public_timelines
  
-     Redis.current.publish('timeline:public', @payload)
-     if status.local?
-       Redis.current.publish('timeline:public:local', @payload)
-     else
-       Redis.current.publish('timeline:public:remote', @payload)
+     Redis.current.publish('timeline:public', anonymous_payload)
+     Redis.current.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', anonymous_payload)
+     if @status.media_attachments.any?
+       Redis.current.publish('timeline:public:media', anonymous_payload)
+       Redis.current.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', anonymous_payload)
      end
    end
  
-   def deliver_to_media(status)
-     Rails.logger.debug "Delivering status #{status.id} to media timeline"
-     Redis.current.publish('timeline:public:media', @payload)
-     if status.local?
-       Redis.current.publish('timeline:public:local:media', @payload)
-     else
-       Redis.current.publish('timeline:public:remote:media', @payload)
-     end
+   def deliver_to_conversation!
+     AccountConversation.add_status(@account, @status) unless update?
    end
  
-   def deliver_to_direct_timelines(status)
-     Rails.logger.debug "Delivering status #{status.id} to direct timelines"
+   def anonymous_payload
+     @anonymous_payload ||= Oj.dump(
+       event: update? ? :'status.update' : :update,
+       payload: InlineRenderer.render(@status, nil, :status)
+     )
+   end
  
-     FeedInsertWorker.push_bulk(status.mentions.includes(:account).map(&:account).select { |mentioned_account| mentioned_account.local? }) do |account|
-       [status.id, account.id, :direct]
-     end
+   def update?
+     @is_update
    end
  
-   def deliver_to_own_conversation(status)
-     AccountConversation.add_status(status.account, status)
+   def broadcastable?
 -    @status.public_visibility? && !@status.reblog? && !@account.silenced?
++    @status.public_visibility? && !@account.silenced? && (!@status.reblog? || Setting.show_reblogs_in_public_timelines)
    end
  end
Simple merge
index 45e6bb88dc9e4d81f299b0a56ef14c2838d83b5a,0122be95d906ec6064d78931bdba73c91cee9c75..043e508854e96743fac517bb705c83059cd56c53
@@@ -51,11 -50,18 +54,22 @@@ class FeedInsertWorke
    def perform_push
      case @type
      when :home
-       FeedManager.instance.push_to_home(@follower, @status)
+       FeedManager.instance.push_to_home(@follower, @status, update: update?)
      when :list
-       FeedManager.instance.push_to_list(@list, @status)
+       FeedManager.instance.push_to_list(@list, @status, update: update?)
 +    when :direct
-       FeedManager.instance.push_to_direct(@account, @status)
++      FeedManager.instance.push_to_direct(@account, @status, update: update?)
+     end
+   end
+   def perform_unpush
+     case @type
+     when :home
+       FeedManager.instance.unpush_from_home(@follower, @status, update: true)
+     when :list
+       FeedManager.instance.unpush_from_list(@list, @status, update: true)
++    when :direct
++      FeedManager.instance.unpush_from_direct(@account, @status, update: true)
      end
    end
  
Simple merge
diff --cc db/schema.rb
index 7b5a301ffe7d6f4ae4163ee436ed600a6a4b06cf,4e0f76dcdb4e78b31e352bb5b5929ab93d095389..1c07a1a493c38d36f182aeb50cd07c47e91350cf
@@@ -852,10 -864,9 +864,11 @@@ ActiveRecord::Schema.define(version: 20
      t.bigint "account_id", null: false
      t.bigint "application_id"
      t.bigint "in_reply_to_account_id"
 +    t.boolean "local_only"
      t.bigint "poll_id"
 +    t.string "content_type"
      t.datetime "deleted_at"
+     t.datetime "edited_at"
      t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
      t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)"
      t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"