From: Claire Date: Wed, 19 Jan 2022 22:19:00 +0000 (+0100) Subject: Merge branch 'main' into glitch-soc/merge-upstream X-Git-Url: https://git.xn--scling-oua.cat.family/?a=commitdiff_plain;h=fe89554a5423da2bbc8c337cb8f1c43d1631c9bb;p=mastodon.git Merge branch 'main' into glitch-soc/merge-upstream Conflicts: - `app/lib/activitypub/activity/create.rb`: Upstream refactored how `Create` activities are handled and how values are extracted from `Create`d objects. This conflicted with how glitch-soc supported the `directMessage` flag to explicitly distinguish between limited and direct messages. Ported glitch-soc's changes to latest upstream changes. - `app/services/fan_out_on_write_service.rb`: Upstream largely refactored that file and changed some of the logic. This conflicted with glitch-soc's handling of the direct timeline and the options to allow replies and boosts in public feeds. Ported those glitch-soc changes on top of latest upstream changes. - `app/services/process_mentions_service.rb`: Upstream refactored to move mention-related ActivityPub deliveries to `ActivityPub::DeliveryWorker`, while glitch-soc contained an extra check to not send local-only toots to remote mentioned users. Took upstream's version, as the check is not needed anymore, since it is performed at the `ActivityPub::DeliveryWorker` call site already. - `app/workers/feed_insert_worker.rb`: Upstream added support for `update` toot events, while glitch-soc had support for an extra timeline support, `direct`. Ported upstream changes and extended them to the `direct` timeline. Additional changes: - `app/lib/activitypub/parser/status_parser.rb`: Added code to handle the `directMessage` flag and take it into account to compute visibility. - `app/lib/feed_manager.rb`: Extended upstream's support of `update` toot events to glitch-soc's `direct` timeline. --- fe89554a5423da2bbc8c337cb8f1c43d1631c9bb diff --cc app/lib/activitypub/activity/create.rb index c50ddf8d5,a861c34bc..9e93cac64 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@@ -132,29 -152,19 +152,19 @@@ class ActivityPub::Activity::Create < A # If there is at least one silent mention, then the status can be considered # as a limited-audience status, and not strictly a direct message, but only # if we considered a direct message in the first place - next unless @params[:visibility] == :direct && direct_message.nil? - - @params[:visibility] = :limited - @params[:visibility] = :limited if @params[:visibility] == :direct ++ @params[:visibility] = :limited if @params[:visibility] == :direct && !@object['directMessage'] end - # If the payload was delivered to a specific inbox, the inbox owner must have - # access to it, unless they already have access to it anyway - return if @options[:delivered_to_account_id].nil? || @mentions.any? { |mention| mention.account_id == @options[:delivered_to_account_id] } - - @mentions << Mention.new(account_id: @options[:delivered_to_account_id], silent: true) - - return unless @params[:visibility] == :direct && direct_message.nil? - - @params[:visibility] = :limited + # Accounts that are tagged but are not in the audience are not + # supposed to be notified explicitly + @silenced_account_ids = @mentions.map(&:account_id) - accounts_in_audience.map(&:id) end def postprocess_audience_and_deliver return if @status.mentions.find_by(account_id: @options[:delivered_to_account_id]) - delivered_to_account = Account.find(@options[:delivered_to_account_id]) - @status.mentions.create(account: delivered_to_account, silent: true) - @status.update(visibility: :limited) if @status.direct_visibility? && direct_message.nil? - @status.update(visibility: :limited) if @status.direct_visibility? ++ @status.update(visibility: :limited) if @status.direct_visibility? && !@object['directMessage'] return unless delivered_to_account.following?(@account) diff --cc app/lib/activitypub/parser/status_parser.rb index 000000000,3ba154d01..75b8f3d5c mode 000000,100644..100644 --- a/app/lib/activitypub/parser/status_parser.rb +++ b/app/lib/activitypub/parser/status_parser.rb @@@ -1,0 -1,118 +1,124 @@@ + # frozen_string_literal: true + + class ActivityPub::Parser::StatusParser + include JsonLdHelper + + # @param [Hash] json + # @param [Hash] magic_values + # @option magic_values [String] :followers_collection + def initialize(json, magic_values = {}) + @json = json + @object = json['object'] || json + @magic_values = magic_values + end + + def uri + id = @object['id'] + + if id&.start_with?('bear:') + Addressable::URI.parse(id).query_values['u'] + else + id + end + rescue Addressable::URI::InvalidURIError + id + end + + def url + url_to_href(@object['url'], 'text/html') if @object['url'].present? + end + + def text + if @object['content'].present? + @object['content'] + elsif content_language_map? + @object['contentMap'].values.first + end + end + + def spoiler_text + if @object['summary'].present? + @object['summary'] + elsif summary_language_map? + @object['summaryMap'].values.first + end + end + + def title + if @object['name'].present? + @object['name'] + elsif name_language_map? + @object['nameMap'].values.first + end + end + + def created_at + @object['published']&.to_datetime + rescue ArgumentError + nil + end + + def edited_at + @object['updated']&.to_datetime + rescue ArgumentError + nil + end + + def reply + @object['inReplyTo'].present? + end + + def sensitive + @object['sensitive'] + end + + def visibility + if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) } + :public + elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) } + :unlisted + elsif audience_to.include?(@magic_values[:followers_collection]) + :private ++ elsif direct_message == false ++ :limited + else + :direct + end + end + + def language + if content_language_map? + @object['contentMap'].keys.first + elsif name_language_map? + @object['nameMap'].keys.first + elsif summary_language_map? + @object['summaryMap'].keys.first + end + end + ++ def direct_message ++ @object['directMessage'] ++ end ++ + private + + def audience_to + as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) } + end + + def audience_cc + as_array(@object['cc'] || @json['cc']).map { |x| value_or_id(x) } + end + + def summary_language_map? + @object['summaryMap'].is_a?(Hash) && !@object['summaryMap'].empty? + end + + def content_language_map? + @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty? + end + + def name_language_map? + @object['nameMap'].is_a?(Hash) && !@object['nameMap'].empty? + end + end diff --cc app/lib/feed_manager.rb index d57508ef9,c4dd9d00f..127325327 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@@ -98,29 -100,6 +102,29 @@@ class FeedManage true end + # Add a status to a linear direct message feed and send a streaming API update + # @param [Account] account + # @param [Status] status + # @return [Boolean] - def push_to_direct(account, status) ++ def push_to_direct(account, status, update: false) + return false unless add_to_feed(:direct, account.id, status) + + trim(:direct, account.id) - PushUpdateWorker.perform_async(account.id, status.id, "timeline:direct:#{account.id}") ++ PushUpdateWorker.perform_async(account.id, status.id, "timeline:direct:#{account.id}") unless update + true + end + + # Remove a status from a linear direct message feed and send a streaming API update + # @param [List] list + # @param [Status] status + # @return [Boolean] - def unpush_from_direct(account, status) ++ def unpush_from_direct(account, status, update: false) + return false unless remove_from_feed(:direct, account.id, status) + - redis.publish("timeline:direct:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) ++ redis.publish("timeline:direct:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update + true + end + # Fill a home feed with an account's statuses # @param [Account] from_account # @param [Account] into_account diff --cc app/models/status.rb index ef9e6d817,3358d6891..d57026354 --- a/app/models/status.rb +++ b/app/models/status.rb @@@ -21,11 -21,9 +21,12 @@@ # account_id :bigint(8) not null # application_id :bigint(8) # in_reply_to_account_id :bigint(8) +# local_only :boolean +# full_status_text :text default(""), not null # poll_id :bigint(8) +# content_type :string # deleted_at :datetime + # edited_at :datetime # class Status < ApplicationRecord diff --cc app/serializers/activitypub/note_serializer.rb index e08c537b0,12dabc65a..aa552a724 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@@ -11,9 -11,8 +11,10 @@@ class ActivityPub::NoteSerializer < Act attribute :content attribute :content_map, if: :language? + attribute :updated, if: :edited? + attribute :direct_message, if: :non_public? + has_many :media_attachments, key: :attachment has_many :virtual_tags, key: :tag diff --cc app/services/fan_out_on_write_service.rb index 6fa98ce12,f62f78a79..169a2411d --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@@ -3,49 -3,71 +3,73 @@@ class FanOutOnWriteService < BaseService # Push a status into home and mentions feeds # @param [Status] status - def call(status) - raise Mastodon::RaceConditionError if status.visibility.nil? - - deliver_to_self(status) if status.account.local? - - if status.direct_visibility? - deliver_to_mentioned_followers(status) - deliver_to_direct_timelines(status) - deliver_to_own_conversation(status) - elsif status.limited_visibility? - deliver_to_mentioned_followers(status) - else - deliver_to_followers(status) - deliver_to_lists(status) - end + # @param [Hash] options + # @option options [Boolean] update + # @option options [Array] silenced_account_ids + def call(status, options = {}) + @status = status + @account = status.account + @options = options + + check_race_condition! + + fan_out_to_local_recipients! + fan_out_to_public_streams! if broadcastable? + end - return if status.account.silenced? || !status.public_visibility? - return if status.reblog? && !Setting.show_reblogs_in_public_timelines + private - render_anonymous_payload(status) + def check_race_condition! + # I don't know why but at some point we had an issue where + # this service was being executed with status objects + # that had a null visibility - which should not be possible + # since the column in the database is not nullable. + # + # This check re-queues the service to be run at a later time + # with the full object, if something like it occurs - deliver_to_hashtags(status) + raise Mastodon::RaceConditionError if @status.visibility.nil? + end - return if status.reply? && status.in_reply_to_account_id != status.account_id && !Setting.show_replies_in_public_timelines + def fan_out_to_local_recipients! + deliver_to_self! + notify_mentioned_accounts! - deliver_to_public(status) - deliver_to_media(status) if status.media_attachments.any? + case @status.visibility.to_sym + when :public, :unlisted, :private + deliver_to_all_followers! + deliver_to_lists! + when :limited + deliver_to_mentioned_followers! + else + deliver_to_mentioned_followers! + deliver_to_conversation! ++ deliver_to_direct_timelines! + end end - private + def fan_out_to_public_streams! + broadcast_to_hashtag_streams! + broadcast_to_public_streams! + end - def deliver_to_self(status) - Rails.logger.debug "Delivering status #{status.id} to author" - FeedManager.instance.push_to_home(status.account, status) - FeedManager.instance.push_to_direct(status.account, status) if status.direct_visibility? + def deliver_to_self! + FeedManager.instance.push_to_home(@account, @status, update: update?) if @account.local? ++ FeedManager.instance.push_to_direct(@account, @status, update: update?) if @account.local? && @status.direct_visibility? end - def deliver_to_followers(status) - Rails.logger.debug "Delivering status #{status.id} to followers" + def notify_mentioned_accounts! + @status.active_mentions.where.not(id: @options[:silenced_account_ids] || []).joins(:account).merge(Account.local).select(:id, :account_id).reorder(nil).find_in_batches do |mentions| + LocalNotificationWorker.push_bulk(mentions) do |mention| + [mention.account_id, mention.id, 'Mention', :mention] + end + end + end - status.account.followers_for_local_distribution.select(:id).reorder(nil).find_in_batches do |followers| + def deliver_to_all_followers! + @account.followers_for_local_distribution.select(:id).reorder(nil).find_in_batches do |followers| FeedInsertWorker.push_bulk(followers) do |follower| - [status.id, follower.id, :home] + [@status.id, follower.id, :home, update: update?] end end end @@@ -70,51 -88,41 +90,47 @@@ end end - def render_anonymous_payload(status) - @payload = InlineRenderer.render(status, nil, :status) - @payload = Oj.dump(event: :update, payload: @payload) ++ def deliver_to_direct_timelines! ++ FeedInsertWorker.push_bulk(@status.mentions.includes(:account).map(&:account).select { |mentioned_account| mentioned_account.local? }) do |account| ++ [@status.id, account.id, :direct, update: update?] ++ end + end + - def deliver_to_hashtags(status) - Rails.logger.debug "Delivering status #{status.id} to hashtags" - - status.tags.pluck(:name).each do |hashtag| - Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", @payload) - Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", @payload) if status.local? + def broadcast_to_hashtag_streams! + @status.tags.pluck(:name).each do |hashtag| + Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", anonymous_payload) + Redis.current.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", anonymous_payload) if @status.local? end end - def deliver_to_public(status) - Rails.logger.debug "Delivering status #{status.id} to public timeline" + def broadcast_to_public_streams! - return if @status.reply? && @status.in_reply_to_account_id != @account.id ++ return if @status.reply? && @status.in_reply_to_account_id != @account.id && !Setting.show_replies_in_public_timelines - Redis.current.publish('timeline:public', @payload) - if status.local? - Redis.current.publish('timeline:public:local', @payload) - else - Redis.current.publish('timeline:public:remote', @payload) + Redis.current.publish('timeline:public', anonymous_payload) + Redis.current.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', anonymous_payload) + + if @status.media_attachments.any? + Redis.current.publish('timeline:public:media', anonymous_payload) + Redis.current.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', anonymous_payload) end end - def deliver_to_media(status) - Rails.logger.debug "Delivering status #{status.id} to media timeline" - - Redis.current.publish('timeline:public:media', @payload) - if status.local? - Redis.current.publish('timeline:public:local:media', @payload) - else - Redis.current.publish('timeline:public:remote:media', @payload) - end + def deliver_to_conversation! + AccountConversation.add_status(@account, @status) unless update? end - def deliver_to_direct_timelines(status) - Rails.logger.debug "Delivering status #{status.id} to direct timelines" + def anonymous_payload + @anonymous_payload ||= Oj.dump( + event: update? ? :'status.update' : :update, + payload: InlineRenderer.render(@status, nil, :status) + ) + end - FeedInsertWorker.push_bulk(status.mentions.includes(:account).map(&:account).select { |mentioned_account| mentioned_account.local? }) do |account| - [status.id, account.id, :direct] - end + def update? + @is_update end - def deliver_to_own_conversation(status) - AccountConversation.add_status(status.account, status) + def broadcastable? - @status.public_visibility? && !@status.reblog? && !@account.silenced? ++ @status.public_visibility? && !@account.silenced? && (!@status.reblog? || Setting.show_reblogs_in_public_timelines) end end diff --cc app/workers/feed_insert_worker.rb index 45e6bb88d,0122be95d..043e50885 --- a/app/workers/feed_insert_worker.rb +++ b/app/workers/feed_insert_worker.rb @@@ -51,11 -50,18 +54,22 @@@ class FeedInsertWorke def perform_push case @type when :home - FeedManager.instance.push_to_home(@follower, @status) + FeedManager.instance.push_to_home(@follower, @status, update: update?) when :list - FeedManager.instance.push_to_list(@list, @status) + FeedManager.instance.push_to_list(@list, @status, update: update?) + when :direct - FeedManager.instance.push_to_direct(@account, @status) ++ FeedManager.instance.push_to_direct(@account, @status, update: update?) + end + end + + def perform_unpush + case @type + when :home + FeedManager.instance.unpush_from_home(@follower, @status, update: true) + when :list + FeedManager.instance.unpush_from_list(@list, @status, update: true) ++ when :direct ++ FeedManager.instance.unpush_from_direct(@account, @status, update: true) end end diff --cc db/schema.rb index 7b5a301ff,4e0f76dcd..1c07a1a49 --- a/db/schema.rb +++ b/db/schema.rb @@@ -852,10 -864,9 +864,11 @@@ ActiveRecord::Schema.define(version: 20 t.bigint "account_id", null: false t.bigint "application_id" t.bigint "in_reply_to_account_id" + t.boolean "local_only" t.bigint "poll_id" + t.string "content_type" t.datetime "deleted_at" + t.datetime "edited_at" t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)" t.index ["deleted_at"], name: "index_statuses_on_deleted_at", where: "(deleted_at IS NOT NULL)" t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"