サムネ取得 API 見直し (#398) (#402)

Reviewed-on: #402
Co-authored-by: miteruzo <miteruzo@naver.com>
Co-committed-by: miteruzo <miteruzo@naver.com>

backend/app/controllers/preview_controller.rb

@@ -1,50 +1,57 @@
 class PreviewController < ApplicationController
+  before_action :require_member!
+
   def title
-    # TODO: # 既知サイトなら決まったフォーマットで title 取得するやぅに．
-    return head :unauthorized unless current_user
+    return render_bad_request('URL は必須です．') if params[:url].blank?
 
-    url = params[:url]
-    return render_bad_request('URL は必須です．') unless url.present?
-
-    unless url.start_with?(/http(s)?:\/\//)
-      url = 'http://' + url
-    end
-
-    html = URI.open(url, open_timeout: 5, read_timeout: 5).read
-    doc = Nokogiri::HTML.parse(html)
-    title = doc.at('title')&.text&.strip
-
-    render json: { title: title }
-  rescue => e
+    render json: { title: Preview::ThumbnailFetcher.title(params[:url]) }
+  rescue Preview::UrlSafety::UnsafeUrl => e
     render_bad_request(e.message)
+  rescue Preview::HttpFetcher::FetchTimeout => e
+    render_preview_error(e.message, :gateway_timeout)
+  rescue Preview::HttpFetcher::ResponseTooLarge => e
+    render_preview_error(e.message, :payload_too_large)
+  rescue Preview::HttpFetcher::FetchFailed => e
+    render_preview_error(e.message, :bad_gateway)
   end
 
   def thumbnail
-    # TODO: 既知ドメインであれば指定のアドレスからサムネールを取得するやぅにする．
+    return render_bad_request('URL は必須です．') if params[:url].blank?
 
+    image = MiniMagick::Image.read(Preview::ThumbnailFetcher.fetch(params[:url]))
+    image.auto_orient
+    image.resize '180x180>'
+    image.format 'png'
+    width, height = image.dimensions
+    raise Preview::ThumbnailFetcher::GenerationFailed, 'サムネール画像の変換に失敗しました．' if width > 180 || height > 180
+
+    send_data image.to_blob, type: 'image/png', disposition: 'inline'
+  rescue Preview::UrlSafety::UnsafeUrl => e
+    render_bad_request(e.message)
+  rescue Preview::HttpFetcher::FetchTimeout => e
+    render_preview_error(e.message, :gateway_timeout)
+  rescue Preview::HttpFetcher::ResponseTooLarge => e
+    render_preview_error(e.message, :payload_too_large)
+  rescue Preview::HttpFetcher::FetchFailed => e
+    render_preview_error(e.message, :bad_gateway)
+  rescue Preview::ThumbnailFetcher::GenerationFailed, MiniMagick::Error => e
+    render_unprocessable_entity(e.message)
+  end
+
+  private
+
+  def require_member!
     return head :unauthorized unless current_user
+    return if current_user.gte_member?
 
-    url = params[:url]
-    return render_bad_request('URL は必須です．') if url.blank?
+    head :forbidden
+  end
 
-    unless url.start_with?(/http(s)?:\/\//)
-      url = 'http://' + url
-    end
-
-    path = Rails.root.join('tmp', "thumb_#{ SecureRandom.hex }.png")
-    system("node #{ Rails.root }/lib/screenshot.js #{ Shellwords.escape(url) } #{ path }")
-
-    if File.exist?(path)
-      image = MiniMagick::Image.open(path)
-      image.resize '180x180'
-      File.delete(path) rescue nil
-      send_file image.path, type: 'image/png', disposition: 'inline'
-    else
-      render json: { type: 'internal_server_error',
-                     message: 'サムネールを生成できませんでした．',
-                     errors: { },
-                     base_errors: ['サムネールを生成できませんでした．'] },
-             status: :internal_server_error
-    end
+  def render_preview_error(message, status)
+    render json: { type: status.to_s,
+                   message:,
+                   errors: { },
+                   base_errors: [message] },
+           status:
   end
 end

backend/app/services/preview/html_metadata_extractor.rb

@@ -0,0 +1,21 @@
+module Preview
+  class HtmlMetadataExtractor
+    IMAGE_SELECTORS = [
+      'meta[property="og:image"]',
+      'meta[name="twitter:image"]',
+      'meta[name="thumbnail"]'
+    ].freeze
+
+    def self.extract(response)
+      document = Nokogiri::HTML.parse(response.body)
+      image_url = IMAGE_SELECTORS.filter_map {
+        document.at_css(_1)&.[]('content')&.strip.presence
+      }.first
+
+      { title: document.at_css('title')&.text&.strip,
+        image_url: image_url ? URI.join(response.url, image_url).to_s : nil }
+    rescue URI::InvalidURIError
+      { title: document.at_css('title')&.text&.strip, image_url: nil }
+    end
+  end
+end

backend/app/services/preview/http_fetcher.rb

@@ -0,0 +1,122 @@
+require 'net/http'
+require 'json'
+
+module Preview
+  class HttpFetcher
+    class FetchFailed < StandardError; end
+    class FetchTimeout < FetchFailed; end
+    class ResponseTooLarge < FetchFailed; end
+
+    MAX_REDIRECTS = 5
+    DEFAULT_MAX_BYTES = 5.megabytes
+
+    Response = Data.define(:body, :content_type, :url)
+
+    def self.fetch(raw_url, max_bytes: DEFAULT_MAX_BYTES, redirects: MAX_REDIRECTS)
+      uri, addresses = UrlSafety.validate(raw_url)
+      response = request(uri, addresses.first, max_bytes)
+
+      if response.is_a?(Net::HTTPRedirection)
+        location = response['location']
+        if redirects.zero?
+          log_failure(:redirect_limit,
+                      url: uri.to_s,
+                      redirects:,
+                      location:,
+                      content_type: response['content-type'],
+                      content_length: response['content-length'])
+          raise FetchFailed, 'redirect が多すぎます．'
+        end
+
+        if location.blank?
+          log_failure(:blank_redirect_location,
+                      url: uri.to_s,
+                      redirects:,
+                      content_type: response['content-type'],
+                      content_length: response['content-length'])
+          raise FetchFailed, 'redirect 先が不正です．'
+        end
+
+        redirect_url =
+            begin
+              URI.join(uri, location).to_s
+            rescue URI::InvalidURIError => e
+              log_failure(:invalid_redirect_location,
+                          url: uri.to_s,
+                          redirects:,
+                          location:,
+                          error: e.class.name,
+                          message: e.message)
+              raise FetchFailed, 'redirect 先が不正です．'
+            end
+
+        return fetch(redirect_url, max_bytes:, redirects: redirects - 1)
+      end
+
+      unless response.is_a?(Net::HTTPSuccess)
+        log_failure(:http_status,
+                    url: uri.to_s,
+                    code: response.code,
+                    content_type: response['content-type'],
+                    content_length: response['content-length'])
+        raise FetchFailed, "外部サーバーが HTTP #{ response.code } を返しました．"
+      end
+
+      Response.new(response.body, response['content-type'].to_s, uri.to_s)
+    rescue Net::OpenTimeout, Net::ReadTimeout, Timeout::Error => e
+      log_failure(:timeout, url: uri&.to_s || raw_url, error: e.class.name, message: e.message)
+      raise FetchTimeout, e.message
+    rescue SocketError, SystemCallError, OpenSSL::SSL::SSLError, EOFError => e
+      log_failure(:network_error, url: uri&.to_s || raw_url, error: e.class.name, message: e.message)
+      raise FetchFailed, e.message
+    end
+
+    def self.request(uri, ip_address, max_bytes)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.ipaddr = ip_address
+      http.use_ssl = uri.scheme == 'https'
+      http.open_timeout = 5
+      http.read_timeout = 8
+      http.write_timeout = 5
+
+      request = Net::HTTP::Get.new(uri)
+      request['User-Agent'] = 'BTRC-Hub thumbnail preview'
+      request['Accept'] = 'text/html,image/*;q=0.9,*/*;q=0.1'
+
+      http.request(request) do |response|
+        length = response['content-length'].to_i
+        if length > max_bytes
+          log_failure(:response_too_large,
+                      url: uri.to_s,
+                      content_type: response['content-type'],
+                      content_length: response['content-length'],
+                      max_bytes:)
+          raise ResponseTooLarge, '外部データが大きすぎます．'
+        end
+
+        body = +''
+        response.read_body do |chunk|
+          body << chunk
+          next unless body.bytesize > max_bytes
+
+          log_failure(:response_too_large,
+                      url: uri.to_s,
+                      content_type: response['content-type'],
+                      content_length: response['content-length'],
+                      bytes_read: body.bytesize,
+                      max_bytes:)
+          raise ResponseTooLarge, '外部データが大きすぎます．'
+        end
+        response.instance_variable_set(:@body, body)
+        response.instance_variable_set(:@read, true)
+        return response
+      end
+    end
+
+    def self.log_failure(reason, **payload)
+      Rails.logger.warn("preview_http_fetcher_failure #{ { reason:, **payload }.to_json }")
+    end
+
+    private_class_method :request, :log_failure
+  end
+end

backend/app/services/preview/known_site_extractor.rb

@@ -0,0 +1,31 @@
+module Preview
+  class KnownSiteExtractor
+    def self.thumbnail_url(uri)
+      youtube_thumbnail(uri)
+    end
+
+    def self.niconico_video_id(uri)
+      case uri.host&.downcase
+      when 'www.nicovideo.jp', 'nicovideo.jp'
+        uri.path[%r{\A/watch/(sm\d+)\z}, 1]
+      when 'nico.ms'
+        uri.path[%r{\A/(sm\d+)\z}, 1]
+      end
+    end
+
+    def self.youtube_thumbnail(uri)
+      id =
+          case uri.host&.downcase
+          when 'youtu.be'
+            uri.path.split('/').reject(&:blank?).first
+          when 'www.youtube.com', 'youtube.com', 'm.youtube.com'
+            uri.path == '/watch' ? URI.decode_www_form(uri.query.to_s).to_h['v'] : nil
+          end
+      return unless id&.match?(/\A[A-Za-z0-9_-]{6,20}\z/)
+
+      "https://i.ytimg.com/vi/#{ id }/hqdefault.jpg"
+    end
+
+    private_class_method :youtube_thumbnail
+  end
+end

backend/app/services/preview/thumbnail_fetcher.rb

@@ -0,0 +1,95 @@
+module Preview
+  class ThumbnailFetcher
+    class GenerationFailed < StandardError; end
+    ALLOWED_IMAGE_CONTENT_TYPES = [
+      'image/jpeg', 'image/png', 'image/gif', 'image/webp'
+    ].freeze
+    HTML_MAX_BYTES = 1.megabyte
+    NICONICO_XML_MAX_BYTES = 256.kilobytes
+
+    def self.fetch(raw_url)
+      uri, = UrlSafety.validate(raw_url)
+
+      known_url = KnownSiteExtractor.thumbnail_url(uri)
+      image = fetch_image_or_nil(known_url) if known_url
+      return image if image
+
+      niconico_url = niconico_thumbnail_url(uri)
+      image = fetch_image_or_nil(niconico_url) if niconico_url
+      return image if image
+
+      page = HttpFetcher.fetch(uri.to_s, max_bytes: HTML_MAX_BYTES)
+      metadata = HtmlMetadataExtractor.extract(page)
+      raise GenerationFailed, 'サムネール画像が見つかりませんでした．' if metadata[:image_url].blank?
+
+      fetch_image!(metadata[:image_url])
+    end
+
+    def self.title(raw_url)
+      uri, = UrlSafety.validate(raw_url)
+      HtmlMetadataExtractor.extract(
+          HttpFetcher.fetch(uri.to_s, max_bytes: HTML_MAX_BYTES))[:title]
+    end
+
+    def self.fetch_image_or_nil(url)
+      return nil if url.blank?
+
+      response = HttpFetcher.fetch(url)
+      return nil unless allowed_image_content_type?(response.content_type)
+
+      response.body
+    rescue HttpFetcher::FetchTimeout
+      raise
+    rescue HttpFetcher::FetchFailed
+      nil
+    end
+
+    def self.fetch_image!(url)
+      response = HttpFetcher.fetch(url)
+      unless allowed_image_content_type?(response.content_type)
+        raise GenerationFailed, 'サムネール画像が見つかりませんでした．'
+      end
+
+      response.body
+    rescue HttpFetcher::FetchTimeout
+      raise
+    rescue HttpFetcher::ResponseTooLarge
+      raise
+    rescue HttpFetcher::FetchFailed
+      raise GenerationFailed, 'サムネール画像を取得できませんでした．'
+    end
+
+    def self.niconico_thumbnail_url(uri)
+      video_id = KnownSiteExtractor.niconico_video_id(uri)
+      return nil if video_id.blank?
+
+      response = HttpFetcher.fetch("https://ext.nicovideo.jp/api/getthumbinfo/#{ video_id }",
+                                   max_bytes: NICONICO_XML_MAX_BYTES)
+      xml = Nokogiri::XML(response.body)
+      return nil unless xml.at_xpath('/nicovideo_thumb_response/@status')&.value == 'ok'
+
+      xml.at_xpath('//thumbnail_url')&.text&.strip.presence
+    rescue HttpFetcher::FetchFailed, HttpFetcher::FetchTimeout => e
+      Rails.logger.info("preview_niconico_getthumbinfo_fallback #{ { url: uri.to_s,
+                                                                     video_id:,
+                                                                     error: e.class.name,
+                                                                     message: e.message }.to_json }")
+      nil
+    rescue Nokogiri::XML::SyntaxError => e
+      Rails.logger.info("preview_niconico_getthumbinfo_fallback #{ { url: uri.to_s,
+                                                                     video_id:,
+                                                                     error: e.class.name,
+                                                                     message: e.message }.to_json }")
+      nil
+    end
+
+    def self.allowed_image_content_type?(content_type)
+      mime_type = content_type.to_s.split(';', 2).first.downcase.strip
+      ALLOWED_IMAGE_CONTENT_TYPES.include?(mime_type)
+    end
+
+    private_class_method :fetch_image_or_nil, :fetch_image!,
+                         :niconico_thumbnail_url,
+                         :allowed_image_content_type?
+  end
+end

backend/app/services/preview/url_safety.rb

@@ -0,0 +1,55 @@
+require 'resolv'
+require 'ipaddr'
+require 'uri'
+
+module Preview
+  class UrlSafety
+    class UnsafeUrl < StandardError; end
+
+    FORBIDDEN_NETWORKS = [
+      '0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8',
+      '169.254.0.0/16', '172.16.0.0/12', '192.0.0.0/24',
+      '192.0.2.0/24', '192.168.0.0/16', '198.18.0.0/15',
+      '198.51.100.0/24', '203.0.113.0/24', '224.0.0.0/4',
+      '240.0.0.0/4', '::/128', '::1/128', 'fc00::/7', 'fe80::/10',
+      'ff00::/8', '2001:db8::/32', '::ffff:0:0/96'
+    ].map { IPAddr.new(_1) }.freeze
+
+    def self.validate(raw_url)
+      value = raw_url.to_s.strip
+      if value.match?(/\A[a-z][a-z0-9+\-.]*:/i)
+        unless value.match?(/\Ahttps?:\/\//i)
+          raise UnsafeUrl, 'http または https の URL を指定してください．'
+        end
+      else
+        value = "http://#{ value }"
+      end
+      uri = URI.parse(value)
+
+      unless ['http', 'https'].include?(uri.scheme&.downcase) && uri.host.present?
+        raise UnsafeUrl, 'http または https の URL を指定してください．'
+      end
+      raise UnsafeUrl, 'userinfo つき URL は使用できません．' if uri.userinfo.present?
+
+      addresses = Resolv.getaddresses(uri.host)
+      raise UnsafeUrl, 'URL のホストを解決できません．' if addresses.empty?
+
+      parsed_addresses = addresses.map { IPAddr.new(_1) }
+      if parsed_addresses.any? { |address| forbidden?(address) }
+        raise UnsafeUrl, '安全でない接続先は使用できません．'
+      end
+
+      [uri, parsed_addresses.map(&:to_s)]
+    rescue Resolv::ResolvError
+      raise UnsafeUrl, 'URL のホストを解決できません．'
+    rescue URI::InvalidURIError, IPAddr::InvalidAddressError
+      raise UnsafeUrl, 'URL が不正です．'
+    end
+
+    def self.forbidden?(address)
+      FORBIDDEN_NETWORKS.any? { _1.include?(address) }
+    end
+
+    private_class_method :forbidden?
+  end
+end

backend/lib/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "lib",
-  "version": "1.0.0",
-  "main": "screenshot.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "keywords": [],
-  "author": "",
-  "license": "ISC",
-  "description": "",
-  "dependencies": {
-    "puppeteer": "^24.10.0"
-  }
-}

backend/lib/screenshot.js

@@ -1,18 +0,0 @@
-const puppeteer = require ('puppeteer')
-const fs = require ('fs')
-
-
-void (async () => {
-  const url = process.argv[2]
-  const output = process.argv[3]
-
-  const browser = await puppeteer.launch ({
-      args: ['--no-sandbox', '--disable-setuid-sandbox'] })
-
-  const page = await browser.newPage ()
-  await page.setViewport ({ width: 960, height: 960 })
-  await page.goto (url, { waitUntil: 'networkidle2', timeout: 15000 })
-
-  await page.screenshot ({ path: output })
-  await browser.close ()
-}) ()

backend/spec/requests/preview_spec.rb

@@ -1,28 +1,46 @@
-require "rails_helper"
+require 'rails_helper'
 
 
-RSpec.describe "Preview", type: :request do
-  describe "GET /preview/title" do
-    it "401 unless logged in" do
+RSpec.describe 'Preview', type: :request do
+  describe 'GET /preview/title' do
+    it '401 unless logged in' do
       sign_out
-      get "/preview/title", params: { url: "example.com" }
+      get '/preview/title', params: { url: 'example.com' }
       expect(response).to have_http_status(:unauthorized)
     end
 
-    it "400 when url blank" do
-      sign_in_as(create(:user))
-      get "/preview/title", params: { url: "" }
+    it '403 when logged in as guest' do
+      sign_in_as(create(:user, :guest))
+      get '/preview/title', params: { url: 'example.com' }
+      expect(response).to have_http_status(:forbidden)
+    end
+
+    it '400 when url blank' do
+      sign_in_as(create(:user, :member))
+      get '/preview/title', params: { url: '' }
       expect(response).to have_http_status(:bad_request)
     end
 
-    it "returns parsed title (stubbing URI.open)" do
-      sign_in_as(create(:user))
-      fake_html = "<html><head><title>  Hello  </title></head></html>"
-      allow(URI).to receive(:open).and_return(StringIO.new(fake_html))
+    it 'returns parsed title' do
+      sign_in_as(create(:user, :member))
+      allow(Preview::ThumbnailFetcher)
+          .to receive(:title)
+          .with('example.com')
+          .and_return('Hello')
 
-      get "/preview/title", params: { url: "example.com" }
+      get '/preview/title', params: { url: 'example.com' }
       expect(response).to have_http_status(:ok)
-      expect(json["title"]).to eq("Hello")
+      expect(json['title']).to eq('Hello')
+    end
+
+    it '413 when fetched response is too large' do
+      sign_in_as(create(:user, :member))
+      allow(Preview::ThumbnailFetcher)
+          .to receive(:title)
+          .and_raise(Preview::HttpFetcher::ResponseTooLarge, '外部データが大きすぎます．')
+
+      get '/preview/title', params: { url: 'example.com' }
+      expect(response).to have_http_status(:payload_too_large)
     end
   end
 end

backend/spec/services/preview/http_fetcher_spec.rb

@@ -0,0 +1,23 @@
+require 'rails_helper'
+
+RSpec.describe Preview::HttpFetcher do
+  describe '.fetch' do
+    it 'raises FetchFailed when redirect location is invalid' do
+      redirect = Net::HTTPFound.new('1.1', '302', 'Found')
+      redirect['location'] = 'http://[invalid'
+
+      allow(Preview::UrlSafety).to receive(:validate)
+          .with('https://example.com/page')
+          .and_return([URI.parse('https://example.com/page'), ['203.0.113.10']])
+      allow(described_class).to receive(:request)
+          .and_return(redirect)
+      allow(Rails.logger).to receive(:warn)
+
+      expect {
+        described_class.fetch('https://example.com/page')
+      }.to raise_error(Preview::HttpFetcher::FetchFailed)
+      expect(Rails.logger).to have_received(:warn)
+          .with(/invalid_redirect_location/)
+    end
+  end
+end

backend/spec/services/preview/thumbnail_fetcher_spec.rb

@@ -0,0 +1,51 @@
+require 'rails_helper'
+
+RSpec.describe Preview::ThumbnailFetcher do
+  describe '.fetch' do
+    it 'rejects svg thumbnails' do
+      page = Preview::HttpFetcher::Response.new(
+          '<meta property="og:image" content="https://example.com/thumb.svg">',
+          'text/html',
+          'https://example.com/page')
+      svg = Preview::HttpFetcher::Response.new(
+          '<svg></svg>',
+          'image/svg+xml',
+          'https://example.com/thumb.svg')
+
+      allow(Preview::UrlSafety).to receive(:validate)
+          .and_return([URI.parse('https://example.com/page'), ['203.0.113.10']])
+      allow(Preview::HttpFetcher).to receive(:fetch)
+          .with('https://example.com/page', max_bytes: described_class::HTML_MAX_BYTES)
+          .and_return(page)
+      allow(Preview::HttpFetcher).to receive(:fetch)
+          .with('https://example.com/thumb.svg')
+          .and_return(svg)
+
+      expect {
+        described_class.fetch('https://example.com/page')
+      }.to raise_error(Preview::ThumbnailFetcher::GenerationFailed)
+    end
+
+    it 'accepts allowed image content type with parameters' do
+      page = Preview::HttpFetcher::Response.new(
+          '<meta property="og:image" content="https://example.com/thumb.jpg">',
+          'text/html',
+          'https://example.com/page')
+      image = Preview::HttpFetcher::Response.new(
+          'jpeg-bytes',
+          'image/jpeg; charset=binary',
+          'https://example.com/thumb.jpg')
+
+      allow(Preview::UrlSafety).to receive(:validate)
+          .and_return([URI.parse('https://example.com/page'), ['203.0.113.10']])
+      allow(Preview::HttpFetcher).to receive(:fetch)
+          .with('https://example.com/page', max_bytes: described_class::HTML_MAX_BYTES)
+          .and_return(page)
+      allow(Preview::HttpFetcher).to receive(:fetch)
+          .with('https://example.com/thumb.jpg')
+          .and_return(image)
+
+      expect(described_class.fetch('https://example.com/page')).to eq('jpeg-bytes')
+    end
+  end
+end

backend/spec/services/preview/url_safety_spec.rb

@@ -0,0 +1,15 @@
+require 'rails_helper'
+
+RSpec.describe Preview::UrlSafety do
+  describe '.validate' do
+    it 'raises UnsafeUrl when DNS resolution fails' do
+      allow(Resolv).to receive(:getaddresses)
+          .with('missing.example')
+          .and_raise(Resolv::ResolvError)
+
+      expect {
+        described_class.validate('https://missing.example')
+      }.to raise_error(Preview::UrlSafety::UnsafeUrl)
+    end
+  end
+end