@saml_enabled = ENV['SAML_ENABLED'] == 'true'
@pam_enabled = ENV['PAM_ENABLED'] == 'true'
@hidden_service = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true'
- @trending_hashtags = TrendingTags.get(7)
+ @trending_hashtags = TrendingTags.get(10, filtered: false)
@profile_directory = Setting.profile_directory
@timeline_preview = Setting.timeline_preview
@spam_check_enabled = Setting.spam_check_enabled
class TagsController < BaseController
before_action :set_tags, only: :index
before_action :set_tag, except: :index
- before_action :set_filter_params
def index
authorize :tag, :index?
end
- def hide
- authorize @tag, :hide?
- @tag.account_tag_stat.update!(hidden: true)
- redirect_to admin_tags_path(@filter_params)
+ def show
+ authorize @tag, :show?
end
- def unhide
- authorize @tag, :unhide?
- @tag.account_tag_stat.update!(hidden: false)
- redirect_to admin_tags_path(@filter_params)
+ def update
+ authorize @tag, :update?
+
+ if @tag.update(tag_params.merge(reviewed_at: Time.now.utc))
+ redirect_to admin_tag_path(@tag.id)
+ else
+ render :show
+ end
end
private
def set_tags
- @tags = Tag.discoverable
- @tags.merge!(Tag.hidden) if filter_params[:hidden]
+ @tags = filtered_tags.page(params[:page])
end
def set_tag
@tag = Tag.find(params[:id])
end
- def set_filter_params
- @filter_params = filter_params.to_hash.symbolize_keys
+ def filtered_tags
+ scope = Tag
+ scope = scope.discoverable if filter_params[:context] == 'directory'
+ scope = scope.reviewed if filter_params[:review] == 'reviewed'
+ scope = scope.pending_review if filter_params[:review] == 'pending_review'
+ scope.reorder(score: :desc)
end
def filter_params
- params.permit(:hidden)
+ params.slice(:context, :review).permit(:context, :review)
+ end
+
+ def tag_params
+ params.require(:tag).permit(:name, :trendable, :usable, :listable)
end
end
end
--- /dev/null
+# frozen_string_literal: true
+
+class Api::V1::TrendsController < Api::BaseController
+ before_action :set_tags
+
+ respond_to :json
+
+ def index
+ render json: @tags, each_serializer: REST::TagSerializer
+ end
+
+ private
+
+ def set_tags
+ @tags = TrendingTags.get(limit_param(10))
+ end
+end
:setting_advanced_layout,
:setting_use_blurhash,
:setting_use_pending_items,
- notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account),
+ notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag),
interactions: %i(must_be_follower must_be_following must_be_following_dm)
)
end
REPORT_FILTERS = %i(resolved account_id target_account_id).freeze
INVITE_FILTER = %i(available expired).freeze
CUSTOM_EMOJI_FILTERS = %i(local remote by_domain shortcode).freeze
- TAGS_FILTERS = %i(hidden).freeze
+ TAGS_FILTERS = %i(context review).freeze
INSTANCES_FILTERS = %i(limited by_domain).freeze
FOLLOWERS_FILTERS = %i(relationship status by_domain activity order).freeze
FILTERS = ACCOUNT_FILTERS + REPORT_FILTERS + INVITE_FILTER + CUSTOM_EMOJI_FILTERS + TAGS_FILTERS + INSTANCES_FILTERS + FOLLOWERS_FILTERS
def filter_link_to(text, link_to_params, link_class_params = link_to_params)
- new_url = filtered_url_for(link_to_params)
+ new_url = filtered_url_for(link_to_params)
new_class = filtered_url_for(link_class_params)
+
link_to text, new_url, class: filter_link_class(new_class)
end
mail to: @me.user_email, subject: I18n.t('admin_mailer.new_pending_account.subject', instance: @instance, username: @account.username)
end
end
+
+ def new_trending_tag(recipient, tag)
+ @tag = tag
+ @me = recipient
+ @instance = Rails.configuration.x.local_domain
+
+ locale_for_account(@me) do
+ mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name)
+ end
+ end
end
class ApplicationRecord < ActiveRecord::Base
self.abstract_class = true
+
include Remotable
+
+ def boolean_with_default(key, default_value)
+ value = attributes[key]
+
+ if value.nil?
+ default_value
+ else
+ value
+ end
+ end
end
#
# Table name: tags
#
-# id :bigint(8) not null, primary key
-# name :string default(""), not null
-# created_at :datetime not null
-# updated_at :datetime not null
-# score :integer
+# id :bigint(8) not null, primary key
+# name :string default(""), not null
+# created_at :datetime not null
+# updated_at :datetime not null
+# score :integer
+# usable :boolean
+# trendable :boolean
+# listable :boolean
+# reviewed_at :datetime
+# requested_review_at :datetime
#
class Tag < ApplicationRecord
HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+ validate :validate_name_change, if: -> { !new_record? && name_changed? }
- scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
- scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
+ scope :reviewed, -> { where.not(reviewed_at: nil) }
+ scope :pending_review, -> { where(reviewed_at: nil).where.not(requested_review_at: nil) }
+ scope :discoverable, -> { where.not(listable: false).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
delegate :accounts_count,
:accounts_count=,
:increment_count!,
:decrement_count!,
- :hidden?,
to: :account_tag_stat
after_save :save_account_tag_stat
name
end
+ def usable
+ boolean_with_default('usable', true)
+ end
+
+ alias usable? usable
+
+ def listable
+ boolean_with_default('listable', true)
+ end
+
+ alias listable? listable
+
+ def trendable
+ boolean_with_default('trendable', false)
+ end
+
+ alias trendable? trendable
+
+ def requires_review?
+ reviewed_at.nil?
+ end
+
+ def reviewed?
+ reviewed_at.present?
+ end
+
+ def requested_review?
+ requested_review_at.present?
+ end
+
+ def trending?
+ TrendingTags.trending?(self)
+ end
+
def history
days = []
return unless account_tag_stat&.changed?
account_tag_stat.save
end
+
+ def validate_name_change
+ errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
+ end
end
include Redisable
def record_use!(tag, account, at_time = Time.now.utc)
- return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot?
+ return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
increment_historical_use!(tag.id, at_time)
increment_unique_use!(tag.id, account.id, at_time)
- increment_vote!(tag.id, at_time)
+ increment_vote!(tag, at_time)
end
- def get(limit)
- key = "#{KEY}:#{Time.now.utc.beginning_of_day.to_i}"
- tag_ids = redis.zrevrange(key, 0, limit - 1).map(&:to_i)
- tags = Tag.where(id: tag_ids).to_a.each_with_object({}) { |tag, h| h[tag.id] = tag }
+ def get(limit, filtered: true)
+ tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, limit - 1).map(&:to_i)
+
+ tags = Tag.where(id: tag_ids)
+ tags = tags.where(trendable: true) if filtered
+ tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
+
tag_ids.map { |tag_id| tags[tag_id] }.compact
end
+ def trending?(tag)
+ rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
+ rank.present? && rank <= 10
+ end
+
private
def increment_historical_use!(tag_id, at_time)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
- def increment_vote!(tag_id, at_time)
+ def increment_vote!(tag, at_time)
key = "#{KEY}:#{at_time.beginning_of_day.to_i}"
- expected = redis.pfcount("activity:tags:#{tag_id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+ expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
expected = 1.0 if expected.zero?
- observed = redis.pfcount("activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+ observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
if expected > observed || observed < THRESHOLD
- redis.zrem(key, tag_id.to_s)
+ redis.zrem(key, tag.id)
else
- score = ((observed - expected)**2) / expected
- added = redis.zadd(key, score, tag_id.to_s)
- bump_tag_score!(tag_id) if added
+ score = ((observed - expected)**2) / expected
+ old_rank = redis.zrevrank(key, tag.id)
+
+ redis.zadd(key, score, tag.id)
+ request_review!(tag) if (old_rank.nil? || old_rank > 10) && redis.zrevrank(key, tag.id) <= 10 && !tag.trendable? && tag.requires_review? && !tag.requested_review?
end
redis.expire(key, EXPIRE_TRENDS_AFTER)
end
- def bump_tag_score!(tag_id)
- Tag.where(id: tag_id).update_all('score = COALESCE(score, 0) + 1')
- end
-
- def disallowed_hashtags
- return @disallowed_hashtags if defined?(@disallowed_hashtags)
-
- @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
- @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
- @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+ def request_review!(tag)
+ User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
end
end
end
settings.notification_emails['pending_account']
end
+ def allows_trending_tag_emails?
+ settings.notification_emails['trending_tag']
+ end
+
def hides_network?
@hides_network ||= settings.hide_network
end
staff?
end
- def hide?
+ def show?
staff?
end
- def unhide?
+ def update?
staff?
end
end
def validate(status)
return unless status.local? && !status.reblog?
- @status = status
- tags = select_tags
-
- status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size)) unless tags.empty?
- end
-
- private
-
- def select_tags
- tags = Extractor.extract_hashtags(@status.text)
- tags.keep_if { |tag| disallowed_hashtags.include? tag.downcase }
- end
-
- def disallowed_hashtags
- return @disallowed_hashtags if @disallowed_hashtags
-
- @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
- @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
- @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+ disallowed_hashtags = Tag.matching_name(Extractor.extract_hashtags(status.text)).reject(&:usable?)
+ status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_hashtags.map(&:name).join(', '), count: disallowed_hashtags.size)) unless disallowed_hashtags.empty?
end
end
%ul
- @trending_hashtags.each do |tag|
%li
- = link_to "##{tag.name}", web_url("timelines/tag/#{tag.name}")
+ = link_to content_tag(:span, "##{tag.name}", class: !tag.trendable? && !tag.reviewed? ? 'warning-hint' : (!tag.trendable? ? 'negative-hint' : nil)), admin_tag_path(tag.id)
%span.pull-right= number_with_delimiter(tag.history[0][:accounts].to_i)
-%tr
- %td
- = link_to explore_hashtag_path(tag) do
+.directory__tag
+ = link_to admin_tag_path(tag.id) do
+ %h4
= fa_icon 'hashtag'
= tag.name
- %td
- = t('directories.people', count: tag.accounts_count)
- %td
- - if tag.hidden?
- = table_link_to 'eye', t('admin.tags.unhide'), unhide_admin_tag_path(tag.id, **@filter_params), method: :post
- - else
- = table_link_to 'eye-slash', t('admin.tags.hide'), hide_admin_tag_path(tag.id, **@filter_params), method: :post
+
+ %small
+ = t('admin.tags.in_directory', count: tag.accounts_count)
+ •
+ = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
+
+ - if tag.trending?
+ = fa_icon 'fire fw'
+ = t('admin.tags.trending_right_now')
+
+ .trends__item__current= number_to_human tag.history.first[:uses], strip_insignificant_zeros: true
.filters
.filter-subset
- %strong= t('admin.reports.status')
+ %strong= t('admin.tags.context')
%ul
- %li= filter_link_to t('admin.tags.visible'), hidden: nil
- %li= filter_link_to t('admin.tags.hidden'), hidden: '1'
+ %li= filter_link_to t('generic.all'), context: nil
+ %li= filter_link_to t('admin.tags.directory'), context: 'directory'
-.table-wrapper
- %table.table
- %thead
- %tr
- %th= t('admin.tags.name')
- %th= t('admin.tags.accounts')
- %th
- %tbody
- = render @tags
+ .filter-subset
+ %strong= t('admin.tags.review')
+ %ul
+ %li= filter_link_to t('generic.all'), review: nil
+ %li= filter_link_to t('admin.tags.reviewed'), review: 'reviewed'
+ %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), review: 'pending_review'
+
+%hr.spacer/
+
+= render @tags
+= paginate @tags
--- /dev/null
+- content_for :page_title do
+ = "##{@tag.name}"
+
+= simple_form_for @tag, url: admin_tag_path(@tag.id) do |f|
+ = render 'shared/error_messages', object: @tag
+
+ .fields-group
+ = f.input :name, wrapper: :with_block_label
+
+ .fields-group
+ = f.input :usable, as: :boolean, wrapper: :with_label
+ = f.input :trendable, as: :boolean, wrapper: :with_label
+ = f.input :listable, as: :boolean, wrapper: :with_label
+
+ .actions
+ = f.button :button, t('generic.save_changes'), type: :submit
--- /dev/null
+<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
+
+<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %>
+
+<%= raw t('application_mailer.view')%> <%= admin_tags_url(review: 'pending_review') %>
- if current_user.staff?
= ff.input :report, as: :boolean, wrapper: :with_label
= ff.input :pending_account, as: :boolean, wrapper: :with_label
+ = ff.input :trending_tag, as: :boolean, wrapper: :with_label
.fields-group
= f.simple_fields_for :notification_emails, hash_to_object(current_user.settings.notification_emails) do |ff|
title: Account statuses
with_media: With media
tags:
- accounts: Accounts
- hidden: Hidden
- hide: Hide from directory
- name: Hashtag
+ context: Context
+ directory: In directory
+ in_directory: "%{count} in directory"
+ review: Review status
+ reviewed: Reviewed
title: Hashtags
- unhide: Show in directory
- visible: Visible
+ trending_right_now: Trending right now
+ unique_uses_today: "%{count} posting today"
title: Administration
warning_presets:
add_new: Add new
body: "%{reporter} has reported %{target}"
body_remote: Someone from %{domain} has reported %{target}
subject: New report for %{instance} (#%{id})
+ new_trending_tag:
+ body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
+ subject: New hashtag up for review on %{instance} (#%{name})
appearance:
advanced_web_interface: Advanced web interface
advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.'
pinned: Pinned toot
reblogged: boosted
sensitive_content: Sensitive content
+ tags:
+ does_not_match_previous_name: does not match the previous name
terms:
body_html: |
<h2>Privacy Policy</h2>
text: This will help us review your application
sessions:
otp: 'Enter the two-factor code generated by your phone app or use one of your recovery codes:'
+ tag:
+ name: You can only change the casing of the letters, for example, to make it more readable
user:
chosen_languages: When checked, only toots in selected languages will be displayed in public timelines
labels:
pending_account: Send e-mail when a new account needs review
reblog: Send e-mail when someone boosts your status
report: Send e-mail when a new report is submitted
+ trending_tag: Send e-mail when an unreviewed hashtag is trending
+ tag:
+ listable: Allow this hashtag to appear on the profile directory
+ trendable: Allow this hashtag to appear under trends
+ usable: Allow toots to use this hashtag
'no': 'No'
recommended: Recommended
required:
s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
- s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
+ s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
end
end
resources :account_moderation_notes, only: [:create, :destroy]
-
- resources :tags, only: [:index] do
- member do
- post :hide
- post :unhide
- end
- end
+ resources :tags, only: [:index, :show, :update]
end
get '/admin', to: redirect('/admin/dashboard', status: 302)
resources :mutes, only: [:index]
resources :favourites, only: [:index]
resources :reports, only: [:create]
+ resources :trends, only: [:index]
resources :filters, only: [:index, :create, :show, :update, :destroy]
resources :endorsements, only: [:index]
digest: true
report: true
pending_account: true
+ trending_tag: true
interactions:
must_be_follower: false
must_be_following: false
--- /dev/null
+class AddCapabilitiesToTags < ActiveRecord::Migration[5.2]
+ def change
+ add_column :tags, :usable, :boolean
+ add_column :tags, :trendable, :boolean
+ add_column :tags, :listable, :boolean
+ add_column :tags, :reviewed_at, :datetime
+ add_column :tags, :requested_review_at, :datetime
+ end
+end
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2019_07_29_185330) do
+ActiveRecord::Schema.define(version: 2019_08_05_123746) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.integer "score"
+ t.boolean "usable"
+ t.boolean "trendable"
+ t.boolean "listable"
+ t.datetime "reviewed_at"
+ t.datetime "requested_review_at"
t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
end
end
describe 'GET #index' do
- before do
- account_tag_stat = Fabricate(:tag).account_tag_stat
- account_tag_stat.update(hidden: hidden, accounts_count: 1)
- get :index, params: { hidden: hidden }
- end
-
- context 'with hidden tags' do
- let(:hidden) { true }
-
- it 'returns status 200' do
- expect(response).to have_http_status(200)
- end
- end
-
- context 'without hidden tags' do
- let(:hidden) { false }
-
- it 'returns status 200' do
- expect(response).to have_http_status(200)
- end
- end
- end
-
- describe 'POST #hide' do
- let(:tag) { Fabricate(:tag) }
+ let!(:tag) { Fabricate(:tag) }
before do
- tag.account_tag_stat.update(hidden: false)
- post :hide, params: { id: tag.id }
- end
-
- it 'hides tag' do
- tag.reload
- expect(tag).to be_hidden
- end
-
- it 'redirects to admin_tags_path' do
- expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
- end
- end
-
- describe 'POST #unhide' do
- let(:tag) { Fabricate(:tag) }
-
- before do
- tag.account_tag_stat.update(hidden: true)
- post :unhide, params: { id: tag.id }
- end
-
- it 'unhides tag' do
- tag.reload
- expect(tag).not_to be_hidden
+ get :index
end
- it 'redirects to admin_tags_path' do
- expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
+ it 'returns status 200' do
+ expect(response).to have_http_status(200)
end
end
end
let(:admin) { Fabricate(:user, admin: true).account }
let(:john) { Fabricate(:user).account }
- permissions :index?, :hide?, :unhide? do
+ permissions :index?, :show?, :update? do
context 'staff?' do
it 'permits' do
expect(subject).to permit(admin, Tag)
require 'rails_helper'
RSpec.describe DisallowedHashtagsValidator, type: :validator do
+ let(:disallowed_tags) { [] }
+
describe '#validate' do
before do
- allow_any_instance_of(described_class).to receive(:select_tags) { tags }
+ disallowed_tags.each { |name| Fabricate(:tag, name: name, usable: false) }
described_class.new.validate(status)
end
- let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: '') }
+ let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: disallowed_tags.map { |x| '#' + x }.join(' ')) }
let(:errors) { double(add: nil) }
- context 'unless status.local? && !status.reblog?' do
+ context 'for a remote reblog' do
let(:local) { false }
let(:reblog) { true }
- it 'not calls errors.add' do
+ it 'does not add errors' do
expect(errors).not_to have_received(:add).with(:text, any_args)
end
end
- context 'status.local? && !status.reblog?' do
+ context 'for a local original status' do
let(:local) { true }
let(:reblog) { false }
- context 'tags.empty?' do
- let(:tags) { [] }
+ context 'when does not contain any disallowed hashtags' do
+ let(:disallowed_tags) { [] }
- it 'not calls errors.add' do
+ it 'does not add errors' do
expect(errors).not_to have_received(:add).with(:text, any_args)
end
end
- context '!tags.empty?' do
- let(:tags) { %w(a b c) }
+ context 'when contains disallowed hashtags' do
+ let(:disallowed_tags) { %w(a b c) }
- it 'calls errors.add' do
+ it 'adds an error' do
expect(errors).to have_received(:add)
- .with(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size))
+ .with(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_tags.join(', '), count: disallowed_tags.size))
end
end
end
cat aescling's git repositories / mastodon.git / commitdiff