#398

backend/app/controllers/preview_controller.rb

@@ -9,6 +9,8 @@ class PreviewController < ApplicationController
     render_bad_request(e.message)
   rescue Preview::HttpFetcher::FetchTimeout => e
     render_preview_error(e.message, :gateway_timeout)
+  rescue Preview::HttpFetcher::ResponseTooLarge => e
+    render_preview_error(e.message, :payload_too_large)
   rescue Preview::HttpFetcher::FetchFailed => e
     render_preview_error(e.message, :bad_gateway)
   end
@@ -28,6 +30,8 @@ class PreviewController < ApplicationController
     render_bad_request(e.message)
   rescue Preview::HttpFetcher::FetchTimeout => e
     render_preview_error(e.message, :gateway_timeout)
+  rescue Preview::HttpFetcher::ResponseTooLarge => e
+    render_preview_error(e.message, :payload_too_large)
   rescue Preview::HttpFetcher::FetchFailed => e
     render_preview_error(e.message, :bad_gateway)
   rescue Preview::ThumbnailFetcher::GenerationFailed, MiniMagick::Error => e

backend/app/services/preview/http_fetcher.rb

@@ -1,4 +1,5 @@
 require 'net/http'
+require 'json'
 
 module Preview
   class HttpFetcher
@@ -16,9 +17,17 @@ module Preview
       response = request(uri, addresses.first, max_bytes)
 
       if response.is_a?(Net::HTTPRedirection)
-        raise FetchFailed, 'redirect が多すぎます．' if redirects.zero?
-
         location = response['location']
+        if redirects.zero?
+          log_failure(:redirect_limit,
+                      url: uri.to_s,
+                      redirects:,
+                      location:,
+                      content_type: response['content-type'],
+                      content_length: response['content-length'])
+          raise FetchFailed, 'redirect が多すぎます．'
+        end
+
         raise FetchFailed, 'redirect 先が不正です．' if location.blank?
 
         return fetch(URI.join(uri, location).to_s,
@@ -27,13 +36,20 @@ module Preview
       end
 
       unless response.is_a?(Net::HTTPSuccess)
+        log_failure(:http_status,
+                    url: uri.to_s,
+                    code: response.code,
+                    content_type: response['content-type'],
+                    content_length: response['content-length'])
         raise FetchFailed, "外部サーバーが HTTP #{ response.code } を返しました．"
       end
 
       Response.new(response.body, response['content-type'].to_s, uri.to_s)
     rescue Net::OpenTimeout, Net::ReadTimeout, Timeout::Error => e
+      log_failure(:timeout, url: uri&.to_s || raw_url, error: e.class.name, message: e.message)
       raise FetchTimeout, e.message
     rescue SocketError, SystemCallError, OpenSSL::SSL::SSLError, EOFError => e
+      log_failure(:network_error, url: uri&.to_s || raw_url, error: e.class.name, message: e.message)
       raise FetchFailed, e.message
     end
 
@@ -51,12 +67,27 @@ module Preview
 
       http.request(request) do |response|
         length = response['content-length'].to_i
-        raise ResponseTooLarge, '外部データが大きすぎます．' if length > max_bytes
+        if length > max_bytes
+          log_failure(:response_too_large,
+                      url: uri.to_s,
+                      content_type: response['content-type'],
+                      content_length: response['content-length'],
+                      max_bytes:)
+          raise ResponseTooLarge, '外部データが大きすぎます．'
+        end
 
         body = +''
         response.read_body do |chunk|
           body << chunk
-          raise ResponseTooLarge, '外部データが大きすぎます．' if body.bytesize > max_bytes
+          next unless body.bytesize > max_bytes
+
+          log_failure(:response_too_large,
+                      url: uri.to_s,
+                      content_type: response['content-type'],
+                      content_length: response['content-length'],
+                      bytes_read: body.bytesize,
+                      max_bytes:)
+          raise ResponseTooLarge, '外部データが大きすぎます．'
         end
         response.instance_variable_set(:@body, body)
         response.instance_variable_set(:@read, true)
@@ -64,6 +95,10 @@ module Preview
       end
     end
 
-    private_class_method :request
+    def self.log_failure(reason, **payload)
+      Rails.logger.warn("preview_http_fetcher_failure #{ { reason:, **payload }.to_json }")
+    end
+
+    private_class_method :request, :log_failure
   end
 end

backend/app/services/preview/known_site_extractor.rb

@@ -1,7 +1,16 @@
 module Preview
   class KnownSiteExtractor
     def self.thumbnail_url(uri)
-      youtube_thumbnail(uri) || niconico_thumbnail(uri)
+      youtube_thumbnail(uri)
+    end
+
+    def self.niconico_video_id(uri)
+      case uri.host&.downcase
+      when 'www.nicovideo.jp', 'nicovideo.jp'
+        uri.path[%r{\A/watch/(sm\d+)\z}, 1]
+      when 'nico.ms'
+        uri.path[%r{\A/(sm\d+)\z}, 1]
+      end
     end
 
     def self.youtube_thumbnail(uri)
@@ -17,16 +26,6 @@ module Preview
       "https://i.ytimg.com/vi/#{ id }/hqdefault.jpg"
     end
 
-    def self.niconico_thumbnail(uri)
+    private_class_method :youtube_thumbnail
-      return unless ['www.nicovideo.jp', 'nicovideo.jp', 'nico.ms'].include?(uri.host&.downcase)
-
-      id = uri.path[%r{/(?:watch/)?(sm\d+)\z}, 1]
-      return unless id
-
-      numeric_id = id.delete_prefix('sm')
-      "https://nicovideo.cdn.nimg.jp/thumbnails/#{ numeric_id }/#{ numeric_id }.L"
-    end
-
-    private_class_method :youtube_thumbnail, :niconico_thumbnail
   end
 end

backend/app/services/preview/thumbnail_fetcher.rb

@@ -1,26 +1,47 @@
 module Preview
   class ThumbnailFetcher
     class GenerationFailed < StandardError; end
+    HTML_MAX_BYTES = 1.megabyte
+    NICONICO_XML_MAX_BYTES = 256.kilobytes
 
     def self.fetch(raw_url)
       uri, = UrlSafety.validate(raw_url)
 
       known_url = KnownSiteExtractor.thumbnail_url(uri)
-      return fetch_image(known_url) if known_url
+      image = fetch_image_or_nil(known_url) if known_url
+      return image if image
 
-      page = HttpFetcher.fetch(uri.to_s)
+      niconico_url = niconico_thumbnail_url(uri)
+      image = fetch_image_or_nil(niconico_url) if niconico_url
+      return image if image
+
+      page = HttpFetcher.fetch(uri.to_s, max_bytes: HTML_MAX_BYTES)
       metadata = HtmlMetadataExtractor.extract(page)
       raise GenerationFailed, 'サムネール画像が見つかりませんでした．' if metadata[:image_url].blank?
 
-      fetch_image(metadata[:image_url])
+      fetch_image!(metadata[:image_url])
     end
 
     def self.title(raw_url)
       uri, = UrlSafety.validate(raw_url)
-      HtmlMetadataExtractor.extract(HttpFetcher.fetch(uri.to_s))[:title]
+      HtmlMetadataExtractor.extract(
+          HttpFetcher.fetch(uri.to_s, max_bytes: HTML_MAX_BYTES))[:title]
     end
 
-    def self.fetch_image(url)
+    def self.fetch_image_or_nil(url)
+      return nil if url.blank?
+
+      response = HttpFetcher.fetch(url)
+      return nil unless response.content_type.downcase.start_with?('image/')
+
+      response.body
+    rescue HttpFetcher::FetchTimeout
+      raise
+    rescue HttpFetcher::FetchFailed
+      nil
+    end
+
+    def self.fetch_image!(url)
       response = HttpFetcher.fetch(url)
       unless response.content_type.downcase.start_with?('image/')
         raise GenerationFailed, 'サムネール画像が見つかりませんでした．'
@@ -29,10 +50,37 @@ module Preview
       response.body
     rescue HttpFetcher::FetchTimeout
       raise
+    rescue HttpFetcher::ResponseTooLarge
+      raise
     rescue HttpFetcher::FetchFailed
       raise GenerationFailed, 'サムネール画像を取得できませんでした．'
     end
 
-    private_class_method :fetch_image
+    def self.niconico_thumbnail_url(uri)
+      video_id = KnownSiteExtractor.niconico_video_id(uri)
+      return nil if video_id.blank?
+
+      response = HttpFetcher.fetch("https://ext.nicovideo.jp/api/getthumbinfo/#{ video_id }",
+                                   max_bytes: NICONICO_XML_MAX_BYTES)
+      xml = Nokogiri::XML(response.body)
+      return nil unless xml.at_xpath('/nicovideo_thumb_response/@status')&.value == 'ok'
+
+      xml.at_xpath('//thumbnail_url')&.text&.strip.presence
+    rescue HttpFetcher::FetchFailed, HttpFetcher::FetchTimeout => e
+      Rails.logger.info("preview_niconico_getthumbinfo_fallback #{ { url: uri.to_s,
+                                                                     video_id:,
+                                                                     error: e.class.name,
+                                                                     message: e.message }.to_json }")
+      nil
+    rescue Nokogiri::XML::SyntaxError => e
+      Rails.logger.info("preview_niconico_getthumbinfo_fallback #{ { url: uri.to_s,
+                                                                     video_id:,
+                                                                     error: e.class.name,
+                                                                     message: e.message }.to_json }")
+      nil
+    end
+
+    private_class_method :fetch_image_or_nil, :fetch_image!,
+                         :niconico_thumbnail_url
   end
 end

backend/spec/requests/preview_spec.rb

@@ -1,28 +1,46 @@
-require "rails_helper"
+require 'rails_helper'
 
 
-RSpec.describe "Preview", type: :request do
+RSpec.describe 'Preview', type: :request do
-  describe "GET /preview/title" do
+  describe 'GET /preview/title' do
-    it "401 unless logged in" do
+    it '401 unless logged in' do
       sign_out
-      get "/preview/title", params: { url: "example.com" }
+      get '/preview/title', params: { url: 'example.com' }
       expect(response).to have_http_status(:unauthorized)
     end
 
-    it "400 when url blank" do
+    it '403 when logged in as guest' do
-      sign_in_as(create(:user))
+      sign_in_as(create(:user, :guest))
-      get "/preview/title", params: { url: "" }
+      get '/preview/title', params: { url: 'example.com' }
+      expect(response).to have_http_status(:forbidden)
+    end
+
+    it '400 when url blank' do
+      sign_in_as(create(:user, :member))
+      get '/preview/title', params: { url: '' }
       expect(response).to have_http_status(:bad_request)
     end
 
-    it "returns parsed title (stubbing URI.open)" do
+    it 'returns parsed title' do
-      sign_in_as(create(:user))
+      sign_in_as(create(:user, :member))
-      fake_html = "<html><head><title>  Hello  </title></head></html>"
+      allow(Preview::ThumbnailFetcher)
-      allow(URI).to receive(:open).and_return(StringIO.new(fake_html))
+          .to receive(:title)
+          .with('example.com')
+          .and_return('Hello')
 
-      get "/preview/title", params: { url: "example.com" }
+      get '/preview/title', params: { url: 'example.com' }
       expect(response).to have_http_status(:ok)
-      expect(json["title"]).to eq("Hello")
+      expect(json['title']).to eq('Hello')
+    end
+
+    it '413 when fetched response is too large' do
+      sign_in_as(create(:user, :member))
+      allow(Preview::ThumbnailFetcher)
+          .to receive(:title)
+          .and_raise(Preview::HttpFetcher::ResponseTooLarge, '外部データが大きすぎます．')
+
+      get '/preview/title', params: { url: 'example.com' }
+      expect(response).to have_http_status(:payload_too_large)
     end
   end
 end