diff --git a/backend/app/controllers/preview_controller.rb b/backend/app/controllers/preview_controller.rb index 5486838..38c884d 100644 --- a/backend/app/controllers/preview_controller.rb +++ b/backend/app/controllers/preview_controller.rb @@ -1,50 +1,52 @@ class PreviewController < ApplicationController + before_action :require_member! + def title - # TODO: # 既知サイトなら決まったフォーマットで title 取得するやぅに. - return head :unauthorized unless current_user + return render_bad_request('URL は必須です.') if params[:url].blank? - url = params[:url] - return render_bad_request('URL は必須です.') unless url.present? - - unless url.start_with?(/http(s)?:\/\//) - url = 'http://' + url - end - - html = URI.open(url, open_timeout: 5, read_timeout: 5).read - doc = Nokogiri::HTML.parse(html) - title = doc.at('title')&.text&.strip - - render json: { title: title } - rescue => e - render_bad_request(e.message) + render json: { title: Preview::ThumbnailFetcher.title(params[:url]) } + rescue Preview::UrlSafety::UnsafeUrl => e + render_preview_error(e.message, :unprocessable_entity) + rescue Preview::HttpFetcher::FetchTimeout => e + render_preview_error(e.message, :gateway_timeout) + rescue Preview::HttpFetcher::FetchFailed => e + render_preview_error(e.message, :bad_gateway) end def thumbnail - # TODO: 既知ドメインであれば指定のアドレスからサムネールを取得するやぅにする. + return render_bad_request('URL は必須です.') if params[:url].blank? + image = MiniMagick::Image.read(Preview::ThumbnailFetcher.fetch(params[:url])) + image.auto_orient + image.resize '180x180>' + image.format 'png' + send_data image.to_blob, type: 'image/png', disposition: 'inline' + rescue Preview::UrlSafety::UnsafeUrl => e + render_preview_error(e.message, :unprocessable_entity) + rescue Preview::Screenshot::Busy => e + render_preview_error(e.message, :too_many_requests) + rescue Preview::Screenshot::TimedOut, Preview::HttpFetcher::FetchTimeout => e + render_preview_error(e.message, :gateway_timeout) + rescue Preview::HttpFetcher::FetchFailed => e + render_preview_error(e.message, :bad_gateway) + rescue Preview::Screenshot::Unavailable, MiniMagick::Error => e + render_preview_error(e.message, :unprocessable_entity) + end + + private + + def require_member! return head :unauthorized unless current_user + return if current_user.gte_member? - url = params[:url] - return render_bad_request('URL は必須です.') if url.blank? + head :forbidden + end - unless url.start_with?(/http(s)?:\/\//) - url = 'http://' + url - end - - path = Rails.root.join('tmp', "thumb_#{ SecureRandom.hex }.png") - system("node #{ Rails.root }/lib/screenshot.js #{ Shellwords.escape(url) } #{ path }") - - if File.exist?(path) - image = MiniMagick::Image.open(path) - image.resize '180x180' - File.delete(path) rescue nil - send_file image.path, type: 'image/png', disposition: 'inline' - else - render json: { type: 'internal_server_error', - message: 'サムネールを生成できませんでした.', - errors: { }, - base_errors: ['サムネールを生成できませんでした.'] }, - status: :internal_server_error - end + def render_preview_error(message, status) + render json: { type: status.to_s, + message:, + errors: { }, + base_errors: [message] }, + status: end end diff --git a/backend/app/services/preview/html_metadata_extractor.rb b/backend/app/services/preview/html_metadata_extractor.rb new file mode 100644 index 0000000..480450d --- /dev/null +++ b/backend/app/services/preview/html_metadata_extractor.rb @@ -0,0 +1,21 @@ +module Preview + class HtmlMetadataExtractor + IMAGE_SELECTORS = [ + 'meta[property="og:image"]', + 'meta[name="twitter:image"]', + 'meta[name="thumbnail"]' + ].freeze + + def self.extract(response) + document = Nokogiri::HTML.parse(response.body) + image_url = IMAGE_SELECTORS.filter_map { + document.at_css(_1)&.[]('content')&.strip.presence + }.first + + { title: document.at_css('title')&.text&.strip, + image_url: image_url ? URI.join(response.url, image_url).to_s : nil } + rescue URI::InvalidURIError + { title: document.at_css('title')&.text&.strip, image_url: nil } + end + end +end diff --git a/backend/app/services/preview/http_fetcher.rb b/backend/app/services/preview/http_fetcher.rb new file mode 100644 index 0000000..0978bdf --- /dev/null +++ b/backend/app/services/preview/http_fetcher.rb @@ -0,0 +1,69 @@ +require 'net/http' + +module Preview + class HttpFetcher + class FetchFailed < StandardError; end + class FetchTimeout < FetchFailed; end + class ResponseTooLarge < FetchFailed; end + + MAX_REDIRECTS = 5 + DEFAULT_MAX_BYTES = 5.megabytes + + Response = Data.define(:body, :content_type, :url) + + def self.fetch(raw_url, max_bytes: DEFAULT_MAX_BYTES, redirects: MAX_REDIRECTS) + uri, addresses = UrlSafety.validate(raw_url) + response = request(uri, addresses.first, max_bytes) + + if response.is_a?(Net::HTTPRedirection) + raise FetchFailed, 'redirect が多すぎます.' if redirects.zero? + + location = response['location'] + raise FetchFailed, 'redirect 先が不正です.' if location.blank? + + return fetch(URI.join(uri, location).to_s, + max_bytes:, + redirects: redirects - 1) + end + + unless response.is_a?(Net::HTTPSuccess) + raise FetchFailed, "外部サーバーが HTTP #{ response.code } を返しました." + end + + Response.new(response.body, response['content-type'].to_s, uri.to_s) + rescue Net::OpenTimeout, Net::ReadTimeout, Timeout::Error => e + raise FetchTimeout, e.message + rescue SocketError, SystemCallError, OpenSSL::SSL::SSLError, EOFError => e + raise FetchFailed, e.message + end + + def self.request(uri, ip_address, max_bytes) + http = Net::HTTP.new(uri.host, uri.port) + http.ipaddr = ip_address + http.use_ssl = uri.scheme == 'https' + http.open_timeout = 5 + http.read_timeout = 8 + http.write_timeout = 5 + + request = Net::HTTP::Get.new(uri) + request['User-Agent'] = 'BTRC-Hub thumbnail preview' + request['Accept'] = 'text/html,image/*;q=0.9,*/*;q=0.1' + + http.request(request) do |response| + length = response['content-length'].to_i + raise ResponseTooLarge, '外部データが大きすぎます.' if length > max_bytes + + body = +'' + response.read_body do |chunk| + body << chunk + raise ResponseTooLarge, '外部データが大きすぎます.' if body.bytesize > max_bytes + end + response.instance_variable_set(:@body, body) + response.instance_variable_set(:@read, true) + return response + end + end + + private_class_method :request + end +end diff --git a/backend/app/services/preview/known_site_extractor.rb b/backend/app/services/preview/known_site_extractor.rb new file mode 100644 index 0000000..3319c3f --- /dev/null +++ b/backend/app/services/preview/known_site_extractor.rb @@ -0,0 +1,32 @@ +module Preview + class KnownSiteExtractor + def self.thumbnail_url(uri) + youtube_thumbnail(uri) || niconico_thumbnail(uri) + end + + def self.youtube_thumbnail(uri) + id = + case uri.host&.downcase + when 'youtu.be' + uri.path.split('/').reject(&:blank?).first + when 'www.youtube.com', 'youtube.com', 'm.youtube.com' + uri.path == '/watch' ? URI.decode_www_form(uri.query.to_s).to_h['v'] : nil + end + return unless id&.match?(/\A[A-Za-z0-9_-]{6,20}\z/) + + "https://i.ytimg.com/vi/#{ id }/hqdefault.jpg" + end + + def self.niconico_thumbnail(uri) + return unless ['www.nicovideo.jp', 'nicovideo.jp', 'nico.ms'].include?(uri.host&.downcase) + + id = uri.path[%r{/(?:watch/)?(sm\d+)\z}, 1] + return unless id + + numeric_id = id.delete_prefix('sm') + "https://nicovideo.cdn.nimg.jp/thumbnails/#{ numeric_id }/#{ numeric_id }.L" + end + + private_class_method :youtube_thumbnail, :niconico_thumbnail + end +end diff --git a/backend/app/services/preview/screenshot.rb b/backend/app/services/preview/screenshot.rb new file mode 100644 index 0000000..76707c0 --- /dev/null +++ b/backend/app/services/preview/screenshot.rb @@ -0,0 +1,64 @@ +module Preview + class Screenshot + class Unavailable < StandardError; end + class Busy < Unavailable; end + class TimedOut < Unavailable; end + + LIMIT = ENV.fetch('PREVIEW_SCREENSHOT_CONCURRENCY', 2).to_i.clamp(1, 10) + TIMEOUT = ENV.fetch('PREVIEW_SCREENSHOT_TIMEOUT', 20).to_i.clamp(5, 60) + @mutex = Mutex.new + @active = 0 + + def self.capture(url) + acquire! + tempfile = nil + pid = nil + + begin + tempfile = Tempfile.new(['preview-thumbnail', '.png']) + tempfile.close + status = nil + pid = Process.spawn( + 'node', Rails.root.join('lib/screenshot.js').to_s, + url, tempfile.path, + out: File::NULL, err: File::NULL, pgroup: true) + Timeout.timeout(TIMEOUT) do + _finished_pid, status = Process.wait2(pid) + end + raise Unavailable, 'スクリーンショットを生成できませんでした.' unless status.success? + raise Unavailable, 'スクリーンショットを生成できませんでした.' unless File.size?(tempfile.path) + + File.binread(tempfile.path) + rescue Timeout::Error + terminate(pid) + raise TimedOut, 'スクリーンショット生成が timeout しました.' + ensure + tempfile&.unlink + release! + end + end + + def self.acquire! + @mutex.synchronize do + raise Busy, 'スクリーンショット生成が混雑してゐます.' if @active >= LIMIT + + @active += 1 + end + end + + def self.release! + @mutex.synchronize { @active -= 1 } + end + + def self.terminate(pid) + return unless pid + + Process.kill('TERM', -pid) + Process.wait(pid) + rescue Errno::ESRCH, Errno::ECHILD + nil + end + + private_class_method :acquire!, :release!, :terminate + end +end diff --git a/backend/app/services/preview/thumbnail_fetcher.rb b/backend/app/services/preview/thumbnail_fetcher.rb new file mode 100644 index 0000000..6ee1cc9 --- /dev/null +++ b/backend/app/services/preview/thumbnail_fetcher.rb @@ -0,0 +1,37 @@ +module Preview + class ThumbnailFetcher + class GenerationFailed < StandardError; end + + def self.fetch(raw_url) + uri, = UrlSafety.validate(raw_url) + page = nil + + known_url = KnownSiteExtractor.thumbnail_url(uri) + image = fetch_image(known_url) if known_url + + unless image + page = HttpFetcher.fetch(uri.to_s) + metadata = HtmlMetadataExtractor.extract(page) + image = fetch_image(metadata[:image_url]) if metadata[:image_url] + end + + image || Screenshot.capture(uri.to_s) + end + + def self.title(raw_url) + uri, = UrlSafety.validate(raw_url) + HtmlMetadataExtractor.extract(HttpFetcher.fetch(uri.to_s))[:title] + end + + def self.fetch_image(url) + response = HttpFetcher.fetch(url) + return unless response.content_type.downcase.start_with?('image/') + + response.body + rescue HttpFetcher::FetchFailed + nil + end + + private_class_method :fetch_image + end +end diff --git a/backend/app/services/preview/url_safety.rb b/backend/app/services/preview/url_safety.rb new file mode 100644 index 0000000..2b8c4b9 --- /dev/null +++ b/backend/app/services/preview/url_safety.rb @@ -0,0 +1,53 @@ +require 'resolv' +require 'ipaddr' +require 'uri' + +module Preview + class UrlSafety + class UnsafeUrl < StandardError; end + + FORBIDDEN_NETWORKS = [ + '0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8', + '169.254.0.0/16', '172.16.0.0/12', '192.0.0.0/24', + '192.0.2.0/24', '192.168.0.0/16', '198.18.0.0/15', + '198.51.100.0/24', '203.0.113.0/24', '224.0.0.0/4', + '240.0.0.0/4', '::/128', '::1/128', 'fc00::/7', 'fe80::/10', + 'ff00::/8', '2001:db8::/32', '::ffff:0:0/96' + ].map { IPAddr.new(_1) }.freeze + + def self.validate(raw_url) + value = raw_url.to_s.strip + if value.match?(/\A[a-z][a-z0-9+\-.]*:/i) + unless value.match?(/\Ahttps?:\/\//i) + raise UnsafeUrl, 'http または https の URL を指定してください.' + end + else + value = "https://#{ value }" + end + uri = URI.parse(value) + + unless ['http', 'https'].include?(uri.scheme&.downcase) && uri.host.present? + raise UnsafeUrl, 'http または https の URL を指定してください.' + end + raise UnsafeUrl, 'userinfo つき URL は使用できません.' if uri.userinfo.present? + + addresses = Resolv.getaddresses(uri.host) + raise UnsafeUrl, 'URL のホストを解決できません.' if addresses.empty? + + parsed_addresses = addresses.map { IPAddr.new(_1) } + if parsed_addresses.any? { |address| forbidden?(address) } + raise UnsafeUrl, '安全でない接続先は使用できません.' + end + + [uri, parsed_addresses.map(&:to_s)] + rescue URI::InvalidURIError, IPAddr::InvalidAddressError + raise UnsafeUrl, 'URL が不正です.' + end + + def self.forbidden?(address) + FORBIDDEN_NETWORKS.any? { _1.include?(address) } + end + + private_class_method :forbidden? + end +end diff --git a/backend/lib/screenshot.js b/backend/lib/screenshot.js index cef1da0..357bc7f 100644 --- a/backend/lib/screenshot.js +++ b/backend/lib/screenshot.js @@ -1,18 +1,60 @@ const puppeteer = require ('puppeteer') -const fs = require ('fs') +const dns = require ('dns').promises +const net = require ('net') +const forbiddenNetworks = [ + ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], + ['127.0.0.0', 8], ['169.254.0.0', 16], ['172.16.0.0', 12], + ['192.0.0.0', 24], ['192.0.2.0', 24], ['192.168.0.0', 16], + ['198.18.0.0', 15], ['198.51.100.0', 24], ['203.0.113.0', 24], + ['224.0.0.0', 4], ['240.0.0.0', 4], ['::', 128], ['::1', 128], + ['fc00::', 7], ['fe80::', 10], ['ff00::', 8], ['2001:db8::', 32], + ['::ffff:0:0', 96]] + +const blockList = new net.BlockList () +forbiddenNetworks.forEach (([address, prefix]) => { + blockList.addSubnet (address, prefix, net.isIPv6 (address) ? 'ipv6' : 'ipv4') +}) + + +const safeUrl = async rawUrl => { + const url = new URL (rawUrl) + if (!['http:', 'https:'].includes (url.protocol) || url.username || url.password) + return false + + const addresses = await dns.lookup (url.hostname, { all: true, verbatim: true }) + return addresses.length > 0 && addresses.every ( + ({ address, family }) => !blockList.check (address, family === 6 ? 'ipv6' : 'ipv4')) +} void (async () => { const url = process.argv[2] const output = process.argv[3] + if (!(await safeUrl (url))) + throw new Error ('Unsafe page URL') const browser = await puppeteer.launch ({ args: ['--no-sandbox', '--disable-setuid-sandbox'] }) - const page = await browser.newPage () - await page.setViewport ({ width: 960, height: 960 }) - await page.goto (url, { waitUntil: 'networkidle2', timeout: 15000 }) - - await page.screenshot ({ path: output }) - await browser.close () -}) () + try + { + const page = await browser.newPage () + await page.setRequestInterception (true) + page.on ('request', request => { + void safeUrl (request.url ()) + .then (safe => safe ? request.continue () : request.abort ()) + .catch (() => request.abort ()) + }) + await page.setViewport ({ width: 960, height: 960 }) + await page.goto (url, { waitUntil: 'domcontentloaded', timeout: 12000 }) + await new Promise (resolve => setTimeout (resolve, 1000)) + await page.screenshot ({ path: output }) + } + finally + { + await browser.close () + } +}) ().catch (error => { + console.error (error) + process.exitCode = 1 +})