require 'net/http' require 'json' module Preview class HttpFetcher class FetchFailed < StandardError; end class FetchTimeout < FetchFailed; end class ResponseTooLarge < FetchFailed; end MAX_REDIRECTS = 5 DEFAULT_MAX_BYTES = 5.megabytes Response = Data.define(:body, :content_type, :url) def self.fetch(raw_url, max_bytes: DEFAULT_MAX_BYTES, redirects: MAX_REDIRECTS, allowed_hosts: nil) uri, addresses = UrlSafety.validate(raw_url) if allowed_hosts && !allowed_hosts.include?(uri.host.downcase) raise FetchFailed, '許可されてゐない redirect 先です.' end response = request(uri, addresses.first, max_bytes) if response.is_a?(Net::HTTPRedirection) location = response['location'] if redirects.zero? log_failure(:redirect_limit, url: uri.to_s, redirects:, location:, content_type: response['content-type'], content_length: response['content-length']) raise FetchFailed, 'redirect が多すぎます.' end if location.blank? log_failure(:blank_redirect_location, url: uri.to_s, redirects:, content_type: response['content-type'], content_length: response['content-length']) raise FetchFailed, 'redirect 先が不正です.' end redirect_url = begin URI.join(uri, location).to_s rescue URI::InvalidURIError => e log_failure(:invalid_redirect_location, url: uri.to_s, redirects:, location:, error: e.class.name, message: e.message) raise FetchFailed, 'redirect 先が不正です.' end return fetch(redirect_url, max_bytes:, redirects: redirects - 1, allowed_hosts:) end unless response.is_a?(Net::HTTPSuccess) log_failure(:http_status, url: uri.to_s, code: response.code, content_type: response['content-type'], content_length: response['content-length']) raise FetchFailed, "外部サーバーが HTTP #{ response.code } を返しました." end Response.new(response.body, response['content-type'].to_s, uri.to_s) rescue Net::OpenTimeout, Net::ReadTimeout, Timeout::Error => e log_failure(:timeout, url: uri&.to_s || raw_url, error: e.class.name, message: e.message) raise FetchTimeout, e.message rescue SocketError, SystemCallError, OpenSSL::SSL::SSLError, EOFError => e log_failure(:network_error, url: uri&.to_s || raw_url, error: e.class.name, message: e.message) raise FetchFailed, e.message end def self.request(uri, ip_address, max_bytes) http = Net::HTTP.new(uri.host, uri.port) http.ipaddr = ip_address http.use_ssl = uri.scheme == 'https' http.open_timeout = 5 http.read_timeout = 8 http.write_timeout = 5 request = Net::HTTP::Get.new(uri) request['User-Agent'] = 'BTRC-Hub thumbnail preview' request['Accept'] = 'text/html,image/*;q=0.9,*/*;q=0.1' http.request(request) do |response| length = response['content-length'].to_i if length > max_bytes log_failure(:response_too_large, url: uri.to_s, content_type: response['content-type'], content_length: response['content-length'], max_bytes:) raise ResponseTooLarge, '外部データが大きすぎます.' end body = +'' response.read_body do |chunk| body << chunk next unless body.bytesize > max_bytes log_failure(:response_too_large, url: uri.to_s, content_type: response['content-type'], content_length: response['content-length'], bytes_read: body.bytesize, max_bytes:) raise ResponseTooLarge, '外部データが大きすぎます.' end response.instance_variable_set(:@body, body) response.instance_variable_set(:@read, true) return response end end def self.log_failure(reason, **payload) Rails.logger.warn("preview_http_fetcher_failure #{ { reason:, **payload }.to_json }") end private_class_method :request, :log_failure end end