#
# Table name: accounts
#
-# id :bigint(8) not null, primary key
-# username :string default(""), not null
-# domain :string
-# secret :string default(""), not null
-# private_key :text
-# public_key :text default(""), not null
-# remote_url :string default(""), not null
-# salmon_url :string default(""), not null
-# hub_url :string default(""), not null
-# created_at :datetime not null
-# updated_at :datetime not null
-# note :text default(""), not null
-# display_name :string default(""), not null
-# uri :string default(""), not null
-# url :string
-# avatar_file_name :string
-# avatar_content_type :string
-# avatar_file_size :integer
-# avatar_updated_at :datetime
-# header_file_name :string
-# header_content_type :string
-# header_file_size :integer
-# header_updated_at :datetime
-# avatar_remote_url :string
-# subscription_expires_at :datetime
-# locked :boolean default(FALSE), not null
-# header_remote_url :string default(""), not null
-# last_webfingered_at :datetime
-# inbox_url :string default(""), not null
-# outbox_url :string default(""), not null
-# shared_inbox_url :string default(""), not null
-# followers_url :string default(""), not null
-# protocol :integer default("ostatus"), not null
-# memorial :boolean default(FALSE), not null
-# moved_to_account_id :bigint(8)
-# featured_collection_url :string
-# fields :jsonb
-# actor_type :string
-# discoverable :boolean
-# also_known_as :string is an Array
-# silenced_at :datetime
-# suspended_at :datetime
-# trust_level :integer
-# hide_collections :boolean
+# id :bigint(8) not null, primary key
+# username :string default(""), not null
+# domain :string
+# secret :string default(""), not null
+# private_key :text
+# public_key :text default(""), not null
+# remote_url :string default(""), not null
+# salmon_url :string default(""), not null
+# hub_url :string default(""), not null
+# created_at :datetime not null
+# updated_at :datetime not null
+# note :text default(""), not null
+# display_name :string default(""), not null
+# uri :string default(""), not null
+# url :string
+# avatar_file_name :string
+# avatar_content_type :string
+# avatar_file_size :integer
+# avatar_updated_at :datetime
+# header_file_name :string
+# header_content_type :string
+# header_file_size :integer
+# header_updated_at :datetime
+# avatar_remote_url :string
+# subscription_expires_at :datetime
+# locked :boolean default(FALSE), not null
+# header_remote_url :string default(""), not null
+# last_webfingered_at :datetime
+# inbox_url :string default(""), not null
+# outbox_url :string default(""), not null
+# shared_inbox_url :string default(""), not null
+# followers_url :string default(""), not null
+# protocol :integer default("ostatus"), not null
+# memorial :boolean default(FALSE), not null
+# moved_to_account_id :bigint(8)
+# featured_collection_url :string
+# fields :jsonb
+# actor_type :string
+# discoverable :boolean
+# also_known_as :string is an Array
+# silenced_at :datetime
+# suspended_at :datetime
+# trust_level :integer
+# hide_collections :boolean
+# avatar_storage_schema_version :integer
+# header_storage_schema_version :integer
#
class Account < ApplicationRecord
#
# Table name: custom_emojis
#
-# id :bigint(8) not null, primary key
-# shortcode :string default(""), not null
-# domain :string
-# image_file_name :string
-# image_content_type :string
-# image_file_size :integer
-# image_updated_at :datetime
-# created_at :datetime not null
-# updated_at :datetime not null
-# disabled :boolean default(FALSE), not null
-# uri :string
-# image_remote_url :string
-# visible_in_picker :boolean default(TRUE), not null
-# category_id :bigint(8)
+# id :bigint(8) not null, primary key
+# shortcode :string default(""), not null
+# domain :string
+# image_file_name :string
+# image_content_type :string
+# image_file_size :integer
+# image_updated_at :datetime
+# created_at :datetime not null
+# updated_at :datetime not null
+# disabled :boolean default(FALSE), not null
+# uri :string
+# image_remote_url :string
+# visible_in_picker :boolean default(TRUE), not null
+# category_id :bigint(8)
+# image_storage_schema_version :integer
#
class CustomEmoji < ApplicationRecord
#
# Table name: media_attachments
#
-# id :bigint(8) not null, primary key
-# status_id :bigint(8)
-# file_file_name :string
-# file_content_type :string
-# file_file_size :integer
-# file_updated_at :datetime
-# remote_url :string default(""), not null
-# created_at :datetime not null
-# updated_at :datetime not null
-# shortcode :string
-# type :integer default("image"), not null
-# file_meta :json
-# account_id :bigint(8)
-# description :text
-# scheduled_status_id :bigint(8)
-# blurhash :string
-# processing :integer
+# id :bigint(8) not null, primary key
+# status_id :bigint(8)
+# file_file_name :string
+# file_content_type :string
+# file_file_size :integer
+# file_updated_at :datetime
+# remote_url :string default(""), not null
+# created_at :datetime not null
+# updated_at :datetime not null
+# shortcode :string
+# type :integer default("image"), not null
+# file_meta :json
+# account_id :bigint(8)
+# description :text
+# scheduled_status_id :bigint(8)
+# blurhash :string
+# processing :integer
+# file_storage_schema_version :integer
#
class MediaAttachment < ApplicationRecord
#
# Table name: preview_cards
#
-# id :bigint(8) not null, primary key
-# url :string default(""), not null
-# title :string default(""), not null
-# description :string default(""), not null
-# image_file_name :string
-# image_content_type :string
-# image_file_size :integer
-# image_updated_at :datetime
-# type :integer default("link"), not null
-# html :text default(""), not null
-# author_name :string default(""), not null
-# author_url :string default(""), not null
-# provider_name :string default(""), not null
-# provider_url :string default(""), not null
-# width :integer default(0), not null
-# height :integer default(0), not null
-# created_at :datetime not null
-# updated_at :datetime not null
-# embed_url :string default(""), not null
+# id :bigint(8) not null, primary key
+# url :string default(""), not null
+# title :string default(""), not null
+# description :string default(""), not null
+# image_file_name :string
+# image_content_type :string
+# image_file_size :integer
+# image_updated_at :datetime
+# type :integer default("link"), not null
+# html :text default(""), not null
+# author_name :string default(""), not null
+# author_url :string default(""), not null
+# provider_name :string default(""), not null
+# provider_url :string default(""), not null
+# width :integer default(0), not null
+# height :integer default(0), not null
+# created_at :datetime not null
+# updated_at :datetime not null
+# embed_url :string default(""), not null
+# image_storage_schema_version :integer
#
class PreviewCard < ApplicationRecord
before_save :extract_dimensions, if: :link?
+ def local?
+ false
+ end
+
def missing_image?
width.present? && height.present? && image_file_name.blank?
end
end
end
+Paperclip.interpolates :path_prefix do |attachment, style|
+ if attachment.storage_schema_version >= 1 && attachment.instance.respond_to?(:local?) && !attachment.instance.local?
+ 'cache' + File::SEPARATOR
+ else
+ ''
+ end
+end
+
+Paperclip.interpolates :url_prefix do |attachment, style|
+ if attachment.storage_schema_version >= 1 && attachment.instance.respond_to?(:local?) && !attachment.instance.local?
+ 'cache/'
+ else
+ ''
+ end
+end
+
Paperclip::Attachment.default_options.merge!(
use_timestamp: false,
- path: ':class/:attachment/:id_partition/:style/:filename',
+ path: ':url_prefix:class/:attachment/:id_partition/:style/:filename',
storage: :fog
)
Paperclip::Attachment.default_options.merge!(
storage: :filesystem,
use_timestamp: true,
- path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':class', ':attachment', ':id_partition', ':style', ':filename'),
- url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:class/:attachment/:id_partition/:style/:filename',
+ path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':path_prefix:class', ':attachment', ':id_partition', ':style', ':filename'),
+ url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:url_prefix:class/:attachment/:id_partition/:style/:filename',
)
end
--- /dev/null
+class AddStorageSchemaVersion < ActiveRecord::Migration[5.2]
+ def change
+ add_column :preview_cards, :image_storage_schema_version, :integer
+ add_column :accounts, :avatar_storage_schema_version, :integer
+ add_column :accounts, :header_storage_schema_version, :integer
+ add_column :media_attachments, :file_storage_schema_version, :integer
+ add_column :custom_emojis, :image_storage_schema_version, :integer
+ end
+end
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2020_04_07_202420) do
+ActiveRecord::Schema.define(version: 2020_04_17_125749) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
t.datetime "suspended_at"
t.integer "trust_level"
t.boolean "hide_collections"
+ t.integer "avatar_storage_schema_version"
+ t.integer "header_storage_schema_version"
t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
t.string "image_remote_url"
t.boolean "visible_in_picker", default: true, null: false
t.bigint "category_id"
+ t.integer "image_storage_schema_version"
t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true
end
t.bigint "scheduled_status_id"
t.string "blurhash"
t.integer "processing"
+ t.integer "file_storage_schema_version"
t.index ["account_id"], name: "index_media_attachments_on_account_id"
t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id"
t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "embed_url", default: "", null: false
+ t.integer "image_storage_schema_version"
t.index ["url"], name: "index_preview_cards_on_url", unique: true
end
require_relative 'mastodon/domains_cli'
require_relative 'mastodon/preview_cards_cli'
require_relative 'mastodon/cache_cli'
+require_relative 'mastodon/upgrade_cli'
require_relative 'mastodon/version'
module Mastodon
desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
subcommand 'cache', Mastodon::CacheCLI
+ desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities'
+ subcommand 'upgrade', Mastodon::UpgradeCLI
+
option :dry_run, type: :boolean
desc 'self-destruct', 'Erase the server from the federation'
long_desc <<~LONG_DESC
module Mastodon
module CLIHelper
+ def dry_run?
+ options[:dry_run]
+ end
+
def create_progress_bar(total = nil)
ProgressBar.create(total: total, format: '%c/%u |%b%i| %e')
end
record_map = preload_records_from_mixed_objects(objects)
objects.each do |object|
- path_segments = object.key.split('/')
+ path_segments = object.key.split('/')
+ path_segments.delete('cache')
+
model_name = path_segments.first.classify
attachment_name = path_segments[1].singularize
record_id = path_segments[2..-2].join.to_i
Find.find(File.join(*[root_path, prefix].compact)) do |path|
next if File.directory?(path)
- key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
- path_segments = key.split(File::SEPARATOR)
+ key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
+
+ path_segments = key.split(File::SEPARATOR)
+ path_segments.delete('cache')
+
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
attachment_name = path_segments[1].singularize
desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
def lookup(url)
- path = Addressable::URI.parse(url).path
+ path = Addressable::URI.parse(url).path
+
path_segments = path.split('/')[2..-1]
- model_name = path_segments.first.classify
- record_id = path_segments[2..-2].join.to_i
+ path_segments.delete('cache')
+
+ model_name = path_segments.first.classify
+ record_id = path_segments[2..-2].join.to_i
unless PRELOAD_MODEL_WHITELIST.include?(model_name)
say("Cannot find corresponding model: #{model_name}", :red)
preload_map = Hash.new { |hash, key| hash[key] = [] }
objects.map do |object|
- segments = object.key.split('/')
+ segments = object.key.split('/')
+ segments.delete('cache')
+
model_name = segments.first.classify
record_id = segments[2..-2].join.to_i
--- /dev/null
+# frozen_string_literal: true
+
+require_relative '../../config/boot'
+require_relative '../../config/environment'
+require_relative 'cli_helper'
+
+module Mastodon
+ class UpgradeCLI < Thor
+ include CLIHelper
+
+ def self.exit_on_failure?
+ true
+ end
+
+ CURRENT_STORAGE_SCHEMA_VERSION = 1
+
+ option :dry_run, type: :boolean, default: false
+ option :verbose, type: :boolean, default: false, aliases: [:v]
+ desc 'storage-schema', 'Upgrade storage schema of various file attachments to the latest version'
+ long_desc <<~LONG_DESC
+ Iterates over every file attachment of every record and, if its storage schema is outdated, performs the
+ necessary upgrade to the latest one. In practice this means e.g. moving files to different directories.
+
+ Will most likely take a long time.
+ LONG_DESC
+ def storage_schema
+ progress = create_progress_bar(nil)
+ dry_run = dry_run? ? ' (DRY RUN)' : ''
+ records = 0
+
+ klasses = [
+ Account,
+ CustomEmoji,
+ MediaAttachment,
+ PreviewCard,
+ ]
+
+ klasses.each do |klass|
+ attachment_names = klass.attachment_definitions.keys
+
+ klass.find_each do |record|
+ attachment_names.each do |attachment_name|
+ attachment = record.public_send(attachment_name)
+
+ next if attachment.blank? || attachment.storage_schema_version >= CURRENT_STORAGE_SCHEMA_VERSION
+
+ attachment.styles.each_key do |style|
+ case Paperclip::Attachment.default_options[:storage]
+ when :s3
+ upgrade_storage_s3(progress, attachment, style)
+ when :fog
+ upgrade_storage_fog(progress, attachment, style)
+ when :filesystem
+ upgrade_storage_filesystem(progress, attachment, style)
+ end
+
+ progress.increment
+ end
+
+ attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
+ end
+
+ if record.changed?
+ record.save unless dry_run?
+ records += 1
+ end
+ end
+ end
+
+ progress.total = progress.progress
+ progress.finish
+
+ say("Upgraded storage schema of #{records} records#{dry_run}", :green, true)
+ end
+
+ private
+
+ def upgrade_storage_s3(progress, attachment, style)
+ previous_storage_schema_version = attachment.storage_schema_version
+ object = attachment.s3_object(style)
+
+ attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
+
+ upgraded_path = attachment.path(style)
+
+ if upgraded_path != object.key && object.exists?
+ progress.log("Moving #{object.key} to #{upgraded_path}") if options[:verbose]
+
+ begin
+ object.move_to(upgraded_path) unless dry_run?
+ rescue => e
+ progress.log(pastel.red("Error processing #{object.key}: #{e}"))
+ end
+ end
+
+ # Because we move files style-by-style, it's important to restore
+ # previous version at the end. The upgrade will be recorded after
+ # all styles are updated
+ attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
+ end
+
+ def upgrade_storage_fog(_progress, _attachment, _style)
+ say('The fog storage driver is not supported for this operation at this time', :red)
+ exit(1)
+ end
+
+ def upgrade_storage_filesystem(progress, attachment, style)
+ previous_storage_schema_version = attachment.storage_schema_version
+ previous_path = attachment.path(style)
+
+ attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
+
+ upgraded_path = attachment.path(style)
+
+ if upgraded_path != previous_path && File.exist?(previous_path)
+ progress.log("Moving #{previous_path} to #{upgraded_path}") if options[:verbose]
+
+ begin
+ unless dry_run?
+ FileUtils.mkdir_p(File.dirname(upgraded_path))
+ FileUtils.mv(previous_path, upgraded_path)
+
+ begin
+ FileUtils.rmdir(previous_path, parents: true)
+ rescue Errno::ENOTEMPTY
+ # OK
+ end
+ end
+ rescue => e
+ progress.log(pastel.red("Error processing #{previous_path}: #{e}"))
+
+ unless dry_run?
+ begin
+ FileUtils.rmdir(upgraded_path, parents: true)
+ rescue Errno::ENOTEMPTY
+ # OK
+ end
+ end
+ end
+ end
+
+ # Because we move files style-by-style, it's important to restore
+ # previous version at the end. The upgrade will be recorded after
+ # all styles are updated
+ attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
+ end
+ end
+end
end
end
+ def storage_schema_version
+ instance_read(:storage_schema_version) || 0
+ end
+
+ def assign_attributes
+ super
+ instance_write(:storage_schema_version, 1)
+ end
+
def variant?(other_filename)
return true if original_filename == other_filename
return false if original_filename.nil?
cat aescling's git repositories / mastodon.git / commitdiff