From 3a8f7d8d8982930c2041e02a5ba479eaf8025bd9 Mon Sep 17 00:00:00 2001 From: miteruzo Date: Fri, 10 Jul 2026 00:34:45 +0900 Subject: [PATCH] #398 --- backend/app/controllers/preview_controller.rb | 4 ++ backend/app/services/preview/http_fetcher.rb | 45 ++++++++++++-- .../services/preview/known_site_extractor.rb | 23 ++++--- .../app/services/preview/thumbnail_fetcher.rb | 60 +++++++++++++++++-- 4 files changed, 109 insertions(+), 23 deletions(-) diff --git a/backend/app/controllers/preview_controller.rb b/backend/app/controllers/preview_controller.rb index dd6458e..4829e2b 100644 --- a/backend/app/controllers/preview_controller.rb +++ b/backend/app/controllers/preview_controller.rb @@ -9,6 +9,8 @@ class PreviewController < ApplicationController render_bad_request(e.message) rescue Preview::HttpFetcher::FetchTimeout => e render_preview_error(e.message, :gateway_timeout) + rescue Preview::HttpFetcher::ResponseTooLarge => e + render_preview_error(e.message, :payload_too_large) rescue Preview::HttpFetcher::FetchFailed => e render_preview_error(e.message, :bad_gateway) end @@ -28,6 +30,8 @@ class PreviewController < ApplicationController render_bad_request(e.message) rescue Preview::HttpFetcher::FetchTimeout => e render_preview_error(e.message, :gateway_timeout) + rescue Preview::HttpFetcher::ResponseTooLarge => e + render_preview_error(e.message, :payload_too_large) rescue Preview::HttpFetcher::FetchFailed => e render_preview_error(e.message, :bad_gateway) rescue Preview::ThumbnailFetcher::GenerationFailed, MiniMagick::Error => e diff --git a/backend/app/services/preview/http_fetcher.rb b/backend/app/services/preview/http_fetcher.rb index 0978bdf..b88f91a 100644 --- a/backend/app/services/preview/http_fetcher.rb +++ b/backend/app/services/preview/http_fetcher.rb @@ -1,4 +1,5 @@ require 'net/http' +require 'json' module Preview class HttpFetcher @@ -16,9 +17,17 @@ module Preview response = request(uri, addresses.first, max_bytes) if response.is_a?(Net::HTTPRedirection) - raise FetchFailed, 'redirect が多すぎます．' if redirects.zero? - location = response['location'] + if redirects.zero? + log_failure(:redirect_limit, + url: uri.to_s, + redirects:, + location:, + content_type: response['content-type'], + content_length: response['content-length']) + raise FetchFailed, 'redirect が多すぎます．' + end + raise FetchFailed, 'redirect 先が不正です．' if location.blank? return fetch(URI.join(uri, location).to_s, @@ -27,13 +36,20 @@ module Preview end unless response.is_a?(Net::HTTPSuccess) + log_failure(:http_status, + url: uri.to_s, + code: response.code, + content_type: response['content-type'], + content_length: response['content-length']) raise FetchFailed, "外部サーバーが HTTP #{ response.code } を返しました．" end Response.new(response.body, response['content-type'].to_s, uri.to_s) rescue Net::OpenTimeout, Net::ReadTimeout, Timeout::Error => e + log_failure(:timeout, url: uri&.to_s || raw_url, error: e.class.name, message: e.message) raise FetchTimeout, e.message rescue SocketError, SystemCallError, OpenSSL::SSL::SSLError, EOFError => e + log_failure(:network_error, url: uri&.to_s || raw_url, error: e.class.name, message: e.message) raise FetchFailed, e.message end @@ -51,12 +67,27 @@ module Preview http.request(request) do |response| length = response['content-length'].to_i - raise ResponseTooLarge, '外部データが大きすぎます．' if length > max_bytes + if length > max_bytes + log_failure(:response_too_large, + url: uri.to_s, + content_type: response['content-type'], + content_length: response['content-length'], + max_bytes:) + raise ResponseTooLarge, '外部データが大きすぎます．' + end body = +'' response.read_body do |chunk| body << chunk - raise ResponseTooLarge, '外部データが大きすぎます．' if body.bytesize > max_bytes + next unless body.bytesize > max_bytes + + log_failure(:response_too_large, + url: uri.to_s, + content_type: response['content-type'], + content_length: response['content-length'], + bytes_read: body.bytesize, + max_bytes:) + raise ResponseTooLarge, '外部データが大きすぎます．' end response.instance_variable_set(:@body, body) response.instance_variable_set(:@read, true) @@ -64,6 +95,10 @@ module Preview end end - private_class_method :request + def self.log_failure(reason, **payload) + Rails.logger.warn("preview_http_fetcher_failure #{ { reason:, **payload }.to_json }") + end + + private_class_method :request, :log_failure end end diff --git a/backend/app/services/preview/known_site_extractor.rb b/backend/app/services/preview/known_site_extractor.rb index 3319c3f..b36629c 100644 --- a/backend/app/services/preview/known_site_extractor.rb +++ b/backend/app/services/preview/known_site_extractor.rb @@ -1,7 +1,16 @@ module Preview class KnownSiteExtractor def self.thumbnail_url(uri) - youtube_thumbnail(uri) || niconico_thumbnail(uri) + youtube_thumbnail(uri) + end + + def self.niconico_video_id(uri) + case uri.host&.downcase + when 'www.nicovideo.jp', 'nicovideo.jp' + uri.path[%r{\A/watch/(sm\d+)\z}, 1] + when 'nico.ms' + uri.path[%r{\A/(sm\d+)\z}, 1] + end end def self.youtube_thumbnail(uri) @@ -17,16 +26,6 @@ module Preview "https://i.ytimg.com/vi/#{ id }/hqdefault.jpg" end - def self.niconico_thumbnail(uri) - return unless ['www.nicovideo.jp', 'nicovideo.jp', 'nico.ms'].include?(uri.host&.downcase) - - id = uri.path[%r{/(?:watch/)?(sm\d+)\z}, 1] - return unless id - - numeric_id = id.delete_prefix('sm') - "https://nicovideo.cdn.nimg.jp/thumbnails/#{ numeric_id }/#{ numeric_id }.L" - end - - private_class_method :youtube_thumbnail, :niconico_thumbnail + private_class_method :youtube_thumbnail end end diff --git a/backend/app/services/preview/thumbnail_fetcher.rb b/backend/app/services/preview/thumbnail_fetcher.rb index ba3f3cd..4df99ad 100644 --- a/backend/app/services/preview/thumbnail_fetcher.rb +++ b/backend/app/services/preview/thumbnail_fetcher.rb @@ -1,26 +1,47 @@ module Preview class ThumbnailFetcher class GenerationFailed < StandardError; end + HTML_MAX_BYTES = 1.megabyte + NICONICO_XML_MAX_BYTES = 256.kilobytes def self.fetch(raw_url) uri, = UrlSafety.validate(raw_url) known_url = KnownSiteExtractor.thumbnail_url(uri) - return fetch_image(known_url) if known_url + image = fetch_image_or_nil(known_url) if known_url + return image if image - page = HttpFetcher.fetch(uri.to_s) + niconico_url = niconico_thumbnail_url(uri) + image = fetch_image_or_nil(niconico_url) if niconico_url + return image if image + + page = HttpFetcher.fetch(uri.to_s, max_bytes: HTML_MAX_BYTES) metadata = HtmlMetadataExtractor.extract(page) raise GenerationFailed, 'サムネール画像が見つかりませんでした．' if metadata[:image_url].blank? - fetch_image(metadata[:image_url]) + fetch_image!(metadata[:image_url]) end def self.title(raw_url) uri, = UrlSafety.validate(raw_url) - HtmlMetadataExtractor.extract(HttpFetcher.fetch(uri.to_s))[:title] + HtmlMetadataExtractor.extract( + HttpFetcher.fetch(uri.to_s, max_bytes: HTML_MAX_BYTES))[:title] end - def self.fetch_image(url) + def self.fetch_image_or_nil(url) + return nil if url.blank? + + response = HttpFetcher.fetch(url) + return nil unless response.content_type.downcase.start_with?('image/') + + response.body + rescue HttpFetcher::FetchTimeout + raise + rescue HttpFetcher::FetchFailed + nil + end + + def self.fetch_image!(url) response = HttpFetcher.fetch(url) unless response.content_type.downcase.start_with?('image/') raise GenerationFailed, 'サムネール画像が見つかりませんでした．' @@ -29,10 +50,37 @@ module Preview response.body rescue HttpFetcher::FetchTimeout raise + rescue HttpFetcher::ResponseTooLarge + raise rescue HttpFetcher::FetchFailed raise GenerationFailed, 'サムネール画像を取得できませんでした．' end - private_class_method :fetch_image + def self.niconico_thumbnail_url(uri) + video_id = KnownSiteExtractor.niconico_video_id(uri) + return nil if video_id.blank? + + response = HttpFetcher.fetch("https://ext.nicovideo.jp/api/getthumbinfo/#{ video_id }", + max_bytes: NICONICO_XML_MAX_BYTES) + xml = Nokogiri::XML(response.body) + return nil unless xml.at_xpath('/nicovideo_thumb_response/@status')&.value == 'ok' + + xml.at_xpath('//thumbnail_url')&.text&.strip.presence + rescue HttpFetcher::FetchFailed, HttpFetcher::FetchTimeout => e + Rails.logger.info("preview_niconico_getthumbinfo_fallback #{ { url: uri.to_s, + video_id:, + error: e.class.name, + message: e.message }.to_json }") + nil + rescue Nokogiri::XML::SyntaxError => e + Rails.logger.info("preview_niconico_getthumbinfo_fallback #{ { url: uri.to_s, + video_id:, + error: e.class.name, + message: e.message }.to_json }") + nil + end + + private_class_method :fetch_image_or_nil, :fetch_image!, + :niconico_thumbnail_url end end