Validate HTTP response length while receiving (#6891)

to_s method of HTTP::Response keeps blocking while it receives the whole
content, no matter how it is big. This means it may waste time to receive
unacceptably large files. It may also consume memory and disk in the
process. This solves the inefficency by checking response length while
receiving.
This commit is contained in:
Akihiko Odaki 2018-03-26 21:02:10 +09:00 committed by Eugen Rochko
parent 18965cb0e6
commit 40e5d2303b
18 changed files with 115 additions and 26 deletions

View file

@ -61,7 +61,7 @@ module JsonLdHelper
def fetch_resource_without_id_validation(uri) def fetch_resource_without_id_validation(uri)
build_request(uri).perform do |response| build_request(uri).perform do |response|
response.code == 200 ? body_to_json(response.to_s) : nil response.code == 200 ? body_to_json(response.body_with_limit) : nil
end end
end end

View file

@ -5,6 +5,7 @@ module Mastodon
class NotPermittedError < Error; end class NotPermittedError < Error; end
class ValidationError < Error; end class ValidationError < Error; end
class HostValidationError < ValidationError; end class HostValidationError < ValidationError; end
class LengthValidationError < ValidationError; end
class RaceConditionError < Error; end class RaceConditionError < Error; end
class UnexpectedResponseError < Error class UnexpectedResponseError < Error

View file

@ -18,7 +18,7 @@ class ProviderDiscovery < OEmbed::ProviderDiscovery
else else
Request.new(:get, url).perform do |res| Request.new(:get, url).perform do |res|
raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html' raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html'
Nokogiri::HTML(res.to_s) Nokogiri::HTML(res.body_with_limit)
end end
end end

View file

@ -40,7 +40,7 @@ class Request
end end
begin begin
yield response yield response.extend(ClientLimit)
ensure ensure
http_client.close http_client.close
end end
@ -99,6 +99,33 @@ class Request
@http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2) @http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2)
end end
module ClientLimit
def body_with_limit(limit = 1.megabyte)
raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
if charset.nil?
encoding = Encoding::BINARY
else
begin
encoding = Encoding.find(charset)
rescue ArgumentError
encoding = Encoding::BINARY
end
end
contents = String.new(encoding: encoding)
while (chunk = readpartial)
contents << chunk
chunk.clear
raise Mastodon::LengthValidationError if contents.bytesize > limit
end
contents
end
end
class Socket < TCPSocket class Socket < TCPSocket
class << self class << self
def open(host, *args) def open(host, *args)
@ -118,5 +145,5 @@ class Request
end end
end end
private_constant :Socket private_constant :ClientLimit, :Socket
end end

View file

@ -55,7 +55,6 @@ class Account < ApplicationRecord
include AccountHeader include AccountHeader
include AccountInteractions include AccountInteractions
include Attachmentable include Attachmentable
include Remotable
include Paginable include Paginable
enum protocol: [:ostatus, :activitypub] enum protocol: [:ostatus, :activitypub]

View file

@ -2,4 +2,5 @@
class ApplicationRecord < ActiveRecord::Base class ApplicationRecord < ActiveRecord::Base
self.abstract_class = true self.abstract_class = true
include Remotable
end end

View file

@ -4,6 +4,7 @@ module AccountAvatar
extend ActiveSupport::Concern extend ActiveSupport::Concern
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
LIMIT = 2.megabytes
class_methods do class_methods do
def avatar_styles(file) def avatar_styles(file)
@ -19,7 +20,8 @@ module AccountAvatar
# Avatar upload # Avatar upload
has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail] has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES
validates_attachment_size :avatar, less_than: 2.megabytes validates_attachment_size :avatar, less_than: LIMIT
remotable_attachment :avatar, LIMIT
end end
def avatar_original_url def avatar_original_url

View file

@ -4,6 +4,7 @@ module AccountHeader
extend ActiveSupport::Concern extend ActiveSupport::Concern
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
LIMIT = 2.megabytes
class_methods do class_methods do
def header_styles(file) def header_styles(file)
@ -19,7 +20,8 @@ module AccountHeader
# Header upload # Header upload
has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail] has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES
validates_attachment_size :header, less_than: 2.megabytes validates_attachment_size :header, less_than: LIMIT
remotable_attachment :header, LIMIT
end end
def header_original_url def header_original_url

View file

@ -3,8 +3,8 @@
module Remotable module Remotable
extend ActiveSupport::Concern extend ActiveSupport::Concern
included do class_methods do
attachment_definitions.each_key do |attachment_name| def remotable_attachment(attachment_name, limit)
attribute_name = "#{attachment_name}_remote_url".to_sym attribute_name = "#{attachment_name}_remote_url".to_sym
method_name = "#{attribute_name}=".to_sym method_name = "#{attribute_name}=".to_sym
alt_method_name = "reset_#{attachment_name}!".to_sym alt_method_name = "reset_#{attachment_name}!".to_sym
@ -33,7 +33,7 @@ module Remotable
File.extname(filename) File.extname(filename)
end end
send("#{attachment_name}=", StringIO.new(response.to_s)) send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit)))
send("#{attachment_name}_file_name=", basename + extname) send("#{attachment_name}_file_name=", basename + extname)
self[attribute_name] = url if has_attribute?(attribute_name) self[attribute_name] = url if has_attribute?(attribute_name)

View file

@ -19,6 +19,8 @@
# #
class CustomEmoji < ApplicationRecord class CustomEmoji < ApplicationRecord
LIMIT = 50.kilobytes
SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}' SHORTCODE_RE_FRAGMENT = '[a-zA-Z0-9_]{2,}'
SCAN_RE = /(?<=[^[:alnum:]:]|\n|^) SCAN_RE = /(?<=[^[:alnum:]:]|\n|^)
@ -29,14 +31,14 @@ class CustomEmoji < ApplicationRecord
has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } } has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }
validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { in: 0..50.kilobytes } validates_attachment :image, content_type: { content_type: 'image/png' }, presence: true, size: { less_than: LIMIT }
validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 } validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
scope :local, -> { where(domain: nil) } scope :local, -> { where(domain: nil) }
scope :remote, -> { where.not(domain: nil) } scope :remote, -> { where.not(domain: nil) }
scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) } scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) }
include Remotable remotable_attachment :image, LIMIT
def local? def local?
domain.nil? domain.nil?

View file

@ -56,6 +56,8 @@ class MediaAttachment < ApplicationRecord
}, },
}.freeze }.freeze
LIMIT = 8.megabytes
belongs_to :account, inverse_of: :media_attachments, optional: true belongs_to :account, inverse_of: :media_attachments, optional: true
belongs_to :status, inverse_of: :media_attachments, optional: true belongs_to :status, inverse_of: :media_attachments, optional: true
@ -64,10 +66,9 @@ class MediaAttachment < ApplicationRecord
processors: ->(f) { file_processors f }, processors: ->(f) { file_processors f },
convert_options: { all: '-quality 90 -strip' } convert_options: { all: '-quality 90 -strip' }
include Remotable
validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES
validates_attachment_size :file, less_than: 8.megabytes validates_attachment_size :file, less_than: LIMIT
remotable_attachment :file, LIMIT
validates :account, presence: true validates :account, presence: true
validates :description, length: { maximum: 420 }, if: :local? validates :description, length: { maximum: 420 }, if: :local?

View file

@ -26,6 +26,7 @@
class PreviewCard < ApplicationRecord class PreviewCard < ApplicationRecord
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
LIMIT = 1.megabytes
self.inheritance_column = false self.inheritance_column = false
@ -36,11 +37,11 @@ class PreviewCard < ApplicationRecord
has_attached_file :image, styles: { original: { geometry: '400x400>', file_geometry_parser: FastGeometryParser } }, convert_options: { all: '-quality 80 -strip' } has_attached_file :image, styles: { original: { geometry: '400x400>', file_geometry_parser: FastGeometryParser } }, convert_options: { all: '-quality 80 -strip' }
include Attachmentable include Attachmentable
include Remotable
validates :url, presence: true, uniqueness: true validates :url, presence: true, uniqueness: true
validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES
validates_attachment_size :image, less_than: 1.megabytes validates_attachment_size :image, less_than: LIMIT
remotable_attachment :image, LIMIT
before_save :extract_dimensions, if: :link? before_save :extract_dimensions, if: :link?

View file

@ -38,13 +38,14 @@ class FetchAtomService < BaseService
return nil if response.code != 200 return nil if response.code != 200
if response.mime_type == 'application/atom+xml' if response.mime_type == 'application/atom+xml'
[@url, { prefetched_body: response.to_s }, :ostatus] [@url, { prefetched_body: response.body_with_limit }, :ostatus]
elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type) elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type)
json = body_to_json(response.to_s) body = response.body_with_limit
json = body_to_json(body)
if supported_context?(json) && json['type'] == 'Person' && json['inbox'].present? if supported_context?(json) && json['type'] == 'Person' && json['inbox'].present?
[json['id'], { prefetched_body: response.to_s, id: true }, :activitypub] [json['id'], { prefetched_body: body, id: true }, :activitypub]
elsif supported_context?(json) && json['type'] == 'Note' elsif supported_context?(json) && json['type'] == 'Note'
[json['id'], { prefetched_body: response.to_s, id: true }, :activitypub] [json['id'], { prefetched_body: body, id: true }, :activitypub]
else else
@unsupported_activity = true @unsupported_activity = true
nil nil
@ -61,7 +62,7 @@ class FetchAtomService < BaseService
end end
def process_html(response) def process_html(response)
page = Nokogiri::HTML(response.to_s) page = Nokogiri::HTML(response.body_with_limit)
json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) }
atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' } atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' }

View file

@ -45,7 +45,7 @@ class FetchLinkCardService < BaseService
Request.new(:get, @url).perform do |res| Request.new(:get, @url).perform do |res|
if res.code == 200 && res.mime_type == 'text/html' if res.code == 200 && res.mime_type == 'text/html'
@html = res.to_s @html = res.body_with_limit
@html_charset = res.charset @html_charset = res.charset
else else
@html = nil @html = nil

View file

@ -181,7 +181,7 @@ class ResolveAccountService < BaseService
@atom_body = Request.new(:get, atom_url).perform do |response| @atom_body = Request.new(:get, atom_url).perform do |response|
raise Mastodon::UnexpectedResponseError, response unless response.code == 200 raise Mastodon::UnexpectedResponseError, response unless response.code == 200
response.to_s response.body_with_limit
end end
end end

View file

@ -57,7 +57,7 @@ class Pubsubhubbub::ConfirmationWorker
def callback_get_with_params def callback_get_with_params
Request.new(:get, subscription.callback_url, params: callback_params).perform do |response| Request.new(:get, subscription.callback_url, params: callback_params).perform do |response|
@callback_response_body = response.body.to_s @callback_response_body = response.body_with_limit
end end
end end

View file

@ -1,6 +1,7 @@
# frozen_string_literal: true # frozen_string_literal: true
require 'rails_helper' require 'rails_helper'
require 'securerandom'
describe Request do describe Request do
subject { Request.new(:get, 'http://example.com') } subject { Request.new(:get, 'http://example.com') }
@ -64,6 +65,12 @@ describe Request do
expect_any_instance_of(HTTP::Client).to receive(:close) expect_any_instance_of(HTTP::Client).to receive(:close)
expect { |block| subject.perform &block }.to yield_control expect { |block| subject.perform &block }.to yield_control
end end
it 'returns response which implements body_with_limit' do
subject.perform do |response|
expect(response).to respond_to :body_with_limit
end
end
end end
context 'with private host' do context 'with private host' do
@ -81,4 +88,46 @@ describe Request do
end end
end end
end end
describe "response's body_with_limit method" do
it 'rejects body more than 1 megabyte by default' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes))
expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
end
it 'accepts body less than 1 megabyte by default' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes))
expect { subject.perform { |response| response.body_with_limit } }.not_to raise_error
end
it 'rejects body by given size' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.kilobytes))
expect { subject.perform { |response| response.body_with_limit(1.kilobyte) } }.to raise_error Mastodon::LengthValidationError
end
it 'rejects too large chunked body' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Transfer-Encoding' => 'chunked' })
expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
end
it 'rejects too large monolithic body' do
stub_request(:any, 'http://example.com').to_return(body: SecureRandom.random_bytes(2.megabytes), headers: { 'Content-Length' => 2.megabytes })
expect { subject.perform { |response| response.body_with_limit } }.to raise_error Mastodon::LengthValidationError
end
it 'uses binary encoding if Content-Type does not tell encoding' do
stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html' })
expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY
end
it 'uses binary encoding if Content-Type tells unknown encoding' do
stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=unknown' })
expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::BINARY
end
it 'uses encoding specified by Content-Type' do
stub_request(:any, 'http://example.com').to_return(body: '', headers: { 'Content-Type' => 'text/html; charset=UTF-8' })
expect(subject.perform { |response| response.body_with_limit.encoding }).to eq Encoding::UTF_8
end
end
end end

View file

@ -29,7 +29,10 @@ RSpec.describe Remotable do
context 'Remotable module is included' do context 'Remotable module is included' do
before do before do
class Foo; include Remotable; end class Foo
include Remotable
remotable_attachment :hoge, 1.kilobyte
end
end end
let(:attribute_name) { "#{hoge}_remote_url".to_sym } let(:attribute_name) { "#{hoge}_remote_url".to_sym }