def fetch_resource_without_id_validation(uri)
build_request(uri).perform do |response|
- response.code == 200 ? body_to_json(response.to_s) : nil
+ response.code == 200 ? body_to_json(response.body_with_limit) : nil
end
end
class NotPermittedError < Error; end
class ValidationError < Error; end
class HostValidationError < ValidationError; end
+ class LengthValidationError < ValidationError; end
class RaceConditionError < Error; end
class UnexpectedResponseError < Error
else
Request.new(:get, url).perform do |res|
raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html'
- Nokogiri::HTML(res.to_s)
+ Nokogiri::HTML(res.body_with_limit)
end
end
end
begin
- yield response
+ yield response.extend(ClientLimit)
ensure
http_client.close
end
@http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2)
end
+ module ClientLimit
+ def body_with_limit(limit = 1.megabyte)
+ raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
+
+ if charset.nil?
+ encoding = Encoding::BINARY
+ else
+ begin
+ encoding = Encoding.find(charset)
+ rescue ArgumentError
+ encoding = Encoding::BINARY
+ end
+ end
+
+ contents = String.new(encoding: encoding)
+
+ while (chunk = readpartial)
+ contents << chunk
+ chunk.clear
+
+ raise Mastodon::LengthValidationError if contents.bytesize > limit
+ end
+
+ contents
+ end
+ end
+
class Socket < TCPSocket
class << self
def open(host, *args)
end
end
- private_constant :Socket
+ private_constant :ClientLimit, :Socket
end
include AccountHeader
include AccountInteractions
include Attachmentable
- include Remotable
include Paginable
enum protocol: [:ostatus, :activitypub]
class ApplicationRecord < ActiveRecord::Base
self.abstract_class = true
+ include Remotable
end
extend ActiveSupport::Concern
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
+ LIMIT = 2.megabytes
class_methods do
def avatar_styles(file)
# Avatar upload
has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES
- validates_attachment_size :avatar, less_than: 2.megabytes
+ validates_attachment_size :avatar, less_than: LIMIT
+ remotable_attachment :avatar, LIMIT
end
def avatar_original_url
extend ActiveSupport::Concern
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
+ LIMIT = 2.megabytes
class_methods do
def header_styles(file)
# Header upload
has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES
- validates_attachment_size :header, less_than: 2.megabytes
+ validates_attachment_size :header, less_than: LIMIT
+ remotable_attachment :header, LIMIT
end
def header_original_url
module Remotable
extend ActiveSupport::Concern
- included do
- attachment_definitions.each_key do |attachment_name|
+ class_methods do
+ def remotable_attachment(attachment_name, limit)
attribute_name = "#{attachment_name}_remote_url".to_sym
method_name = "#{attribute_name}=".to_sym
alt_method_name = "reset_#{attachment_name}!".to_sym
File.extname(filename)
end
- send("#{attachment_name}=", StringIO.new(response.to_s))
+ send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit)))
send("#{attachment_name}_file_name=", basename + extname)
self[attribute_name] = url if has_attribute?(attribute_name)
#
class CustomEmoji < ApplicationRecord
+ LIMIT = 50.kilobytes
+
SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}'
SCAN_RE = /(?<=[^[:alnum:]:]|\n|^)
has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }
- validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { in: 0..50.kilobytes }
+ validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { less_than: LIMIT }
validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
scope :local, -> { where(domain: nil) }
scope :remote, -> { where.not(domain: nil) }
scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
- include Remotable
+ remotable_attachment :image, LIMIT
def local?
domain.nil?
},
}.freeze
+ LIMIT = 8.megabytes
+
belongs_to :account, inverse_of: :media_attachments, optional: true
belongs_to :status, inverse_of: :media_attachments, optional: true
processors: ->(f) { file_processors f },
convert_options: { all: '-quality 90 -strip' }
- include Remotable
-
validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES
- validates_attachment_size :file, less_than: 8.megabytes
+ validates_attachment_size :file, less_than: LIMIT
+ remotable_attachment :file, LIMIT
validates :account, presence: true
validates :description, length: { maximum: 420 }, if: :local?
class PreviewCard < ApplicationRecord
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
+ LIMIT = 1.megabytes
self.inheritance_column = false
has_attached_file :image, styles: { original: { geometry: '400x400>', file_geometry_parser: FastGeometryParser } }, convert_options: { all: '-quality 80 -strip' }
include Attachmentable
- include Remotable
validates :url, presence: true, uniqueness: true
validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES
- validates_attachment_size :image, less_than: 1.megabytes
+ validates_attachment_size :image, less_than: LIMIT
+ remotable_attachment :image, LIMIT
before_save :extract_dimensions, if: :link?
return nil if response.code != 200
if response.mime_type == 'application/atom+xml'
- [@url, { prefetched_body: response.to_s }, :ostatus]
+ [@url, { prefetched_body: response.body_with_limit }, :ostatus]
elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type)
- json = body_to_json(response.to_s)
+ body = response.body_with_limit
+ json = body_to_json(body)
if supported_context?(json) && json['type'] == 'Person' && json['inbox'].present?
- [json['id'], { prefetched_body: response.to_s, id: true }, :activitypub]
+ [json['id'], { prefetched_body: body, id: true }, :activitypub]
elsif supported_context?(json) && json['type'] == 'Note'
- [json['id'], { prefetched_body: response.to_s, id: true }, :activitypub]
+ [json['id'], { prefetched_body: body, id: true }, :activitypub]
else
@unsupported_activity = true
nil
end
def process_html(response)
- page = Nokogiri::HTML(response.to_s)
+ page = Nokogiri::HTML(response.body_with_limit)
json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' }
Request.new(:get, @url).perform do |res|
if res.code == 200 && res.mime_type == 'text/html'
- @html = res.to_s
+ @html = res.body_with_limit
@html_charset = res.charset
else
@html = nil
@atom_body = Request.new(:get, atom_url).perform do |response|
raise Mastodon::UnexpectedResponseError, response unless response.code == 200
- response.to_s
+ response.body_with_limit
end
end
def callback_get_with_params
Request.new(:get, subscription.callback_url, params: callback_params).perform do |response|
- @callback_response_body = response.body.to_s
+ @callback_response_body = response.body_with_limit
end
end
# frozen_string_literal: true
require 'rails_helper'
+require 'securerandom'
describe Request do
subject { Request.new(:get, 'http://example.com') }
expect_any_instance_of(HTTP::Client).to receive(:close)
expect { |block| subject.perform &block }.to yield_control
end
+
+ it 'returns response which implements body_with_limit' do
+ subject.perform do |response|
+ expect(response).to respond_to :body_with_limit
+ end
+ end
end
context 'with private host' do
end
end
end
+
+ describe "response's body_with_limit method" do
+ it 'rejects body more than 1 megabyte by default' do
+ stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes))
+ expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
+ end
+
+ it 'accepts body less than 1 megabyte by default' do
+ stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes))
+ expect { subject.perform { |response| response.body_with_limit } }.not_to raise_error
+ end
+
+ it 'rejects body by given size' do
+ stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes))
+ expect { subject.perform { |response| response.body_with_limit(1.kilobyte) } }.to raise_error Mastodon::LengthValidationError
+ end
+
+ it 'rejects too large chunked body' do
+ stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Transfer-Encoding' => 'chunked' })
+ expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
+ end
+
+ it 'rejects too large monolithic body' do
+ stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Content-Length' => 2.megabytes })
+ expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
+ end
+
+ it 'uses binary encoding if Content-Type does not tell encoding' do
+ stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html' })
+ expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY
+ end
+
+ it 'uses binary encoding if Content-Type tells unknown encoding' do
+ stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=unknown' })
+ expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY
+ end
+
+ it 'uses encoding specified by Content-Type' do
+ stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=UTF-8' })
+ expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::UTF_8
+ end
+ end
end
context 'Remotable module is included' do
before do
- class Foo; include Remotable; end
+ class Foo
+ include Remotable
+ remotable_attachment :hoge, 1.kilobyte
+ end
end
let(:attribute_name) { "#{hoge}_remote_url".to_sym }
cat aescling's git repositories / mastodon.git / commitdiff