require 'timeout' class PostImportPreviewer FIELDS = [ 'title', 'thumbnail_base', 'original_created_from', 'original_created_before', 'duration', 'tags', 'parent_post_ids'].freeze FETCH_WARNING_FIELDS = ['url', 'title', 'thumbnail_base'].freeze EXISTING_SKIP_WARNING = '既存投稿のためスキップします.'.freeze TITLE_FETCH_WARNING = 'タイトルを取得できませんでした.'.freeze THUMBNAIL_FETCH_WARNING = 'サムネールを取得できませんでした.'.freeze METADATA_FETCH_WARNING = 'メタデータを取得できませんでした.'.freeze def preview_rows rows:, fetch_metadata: true, metadata_cache: { } prepared_rows = rows.map { prepare_row(_1) } url_counts = prepared_rows.filter_map { _1[:normal_url] }.tally existing_posts = Post.where(url: prepared_rows.map { _1[:normal_url] }.compact.uniq).index_by(&:url) existing_parent_ids = preload_parent_ids(prepared_rows) preload_metadata!( prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts) known_tags = preload_known_tags( prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts) prepared_rows.map { |row| preview_row(row, fetch_metadata:, metadata_cache:, existing_posts:, url_counts:, known_tags:, existing_parent_ids:) } end def normalised_url value PostUrlNormaliser.normalise(value) end private def prepare_row row source = row.symbolize_keys url = source[:url].to_s.strip normal_url = normalised_url(url) source.merge(url_text: url, normal_url:) end def preview_row row, fetch_metadata:, metadata_cache:, existing_posts:, url_counts:, known_tags:, existing_parent_ids: attributes = initial_attributes(row) provenance = initial_provenance(row) tag_sources = initial_tag_sources(row, attributes, provenance) field_warnings = initial_field_warnings(row) base_warnings = initial_base_warnings(row) url = row[:url_text] provenance['url'] = 'manual' normal_url = row[:normal_url] url_for_metadata = normal_url || url existing_post = normal_url.present? ? existing_posts[normal_url] : nil validation_errors = {} validation_errors[:url] = ['URL が不正です.'] if normal_url.blank? if normal_url.present? && url_counts[normal_url].to_i > 1 validation_errors[:url] = ['URL が重複しています.'] end if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata clear_automatic_values!(attributes, provenance, tag_sources) clear_fetch_warnings!(field_warnings) end if validation_errors.blank? && normal_url.present? && existing_post add_field_warning!(field_warnings, 'url', EXISTING_SKIP_WARNING) attributes['tags'] = merged_tags(tag_sources, provenance['tags']) warnings_present = field_warnings.values.any?(&:present?) || base_warnings.present? return { source_row: row[:source_row], url: normal_url, attributes:, provenance:, tag_sources:, metadata_url: url_for_metadata, skip_reason: 'existing', existing_post_id: existing_post.id, field_warnings:, base_warnings:, validation_errors:, status: warnings_present ? 'warning' : 'ready' } end should_fetch = validation_errors.blank? should_fetch &&= should_fetch_metadata?(fetch_metadata, row[:source_row].to_i) if should_fetch clear_fetch_warnings!(field_warnings) metadata = metadata_for(url_for_metadata, metadata_cache) apply_metadata!(attributes, provenance, tag_sources, metadata[:data]) apply_fetch_warnings!(field_warnings, metadata[:warnings]) end attributes['url'] = url validate_basic_data(attributes, validation_errors) validate_preview_tags(merged_tags(tag_sources, provenance['tags']), validation_errors, field_warnings, known_tags) validate_parents(attributes['parent_post_ids'], validation_errors, existing_parent_ids) attributes.delete('url') attributes['tags'] = merged_tags(tag_sources, provenance['tags']) warnings_present = field_warnings.values.any?(&:present?) || base_warnings.present? { source_row: row[:source_row], url: normal_url || url, attributes:, provenance:, tag_sources:, metadata_url: url_for_metadata, skip_reason: nil, existing_post_id: nil, field_warnings:, base_warnings:, validation_errors:, status: validation_errors.present? ? 'error' : (warnings_present ? 'warning' : 'ready') } end def should_fetch_metadata? fetch_metadata, source_row case fetch_metadata when true then true when Integer then fetch_metadata == source_row when false, nil then false else raise ArgumentError, 'メタデータ取得対象が不正です.' end end def initial_attributes row attributes = row[:attributes]&.stringify_keys || { } FIELDS.to_h { |field| [field, attributes[field].to_s] } end def initial_provenance row provenance = row[:provenance]&.stringify_keys || { } FIELDS.to_h { |field| [field, provenance[field].presence || 'automatic'] } .merge('url' => 'manual') end def initial_tag_sources row, attributes, provenance sources = row[:tag_sources]&.stringify_keys || { 'automatic' => '', 'manual' => '' } sources['automatic'] = sources['automatic'].to_s sources['manual'] = provenance['tags'] == 'manual' ? attributes['tags'].to_s : sources['manual'].to_s sources end def initial_field_warnings row (row[:field_warnings] || { }) .stringify_keys .transform_values { |value| Array(value).map(&:to_s) } end def initial_base_warnings row Array(row[:base_warnings]).map(&:to_s) end def clear_automatic_values! attributes, provenance, tag_sources ['title', 'thumbnail_base', 'original_created_from', 'original_created_before', 'duration'].each do |field| attributes[field] = '' if provenance[field] == 'automatic' end tag_sources['automatic'] = '' attributes['tags'] = merged_tags(tag_sources, provenance['tags']) end def clear_fetch_warnings! field_warnings FETCH_WARNING_FIELDS.each do |field| field_warnings.delete(field) end end def metadata_for url, cache cache[url] ||= fetch_metadata(url) end def fetch_metadata url return { data: { }, warnings: { 'url' => ['URL が空です.'] } } if url.blank? data = PostMetadataFetcher.fetch(url).stringify_keys.compact warnings = { } add_field_warning!(warnings, 'title', TITLE_FETCH_WARNING) if data['title'].blank? if data['thumbnail_base'].blank? add_field_warning!(warnings, 'thumbnail_base', THUMBNAIL_FETCH_WARNING) end { data:, warnings: } rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e Rails.logger.info( "post_import_metadata_fetch_failure "\ "#{ { error: e.class.name, message: e.message }.to_json }") { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } } end def preload_metadata! prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts urls = prepared_rows.filter_map { |row| next unless row[:normal_url].present? next unless should_fetch_metadata?(fetch_metadata, row[:source_row].to_i) next if url_counts[row[:normal_url]].to_i > 1 next if existing_posts.key?(row[:normal_url]) row[:normal_url] }.uniq urls = urls.reject { metadata_cache.key?(_1) } return if urls.empty? url_queue = Queue.new result_queue = Queue.new urls.each { url_queue << _1 } workers = [urls.length, 4].min.times.map { Thread.new { Rails.application.executor.wrap do loop do url = url_queue.pop(true) result_queue << [url, safe_fetch_metadata(url)] rescue ThreadError break end end } } Timeout.timeout(15) { workers.each(&:join) } rescue Timeout::Error workers&.each(&:kill) ensure workers&.each(&:join) while result_queue&.size.to_i.positive? url, result = result_queue.pop metadata_cache[url] = result end urls&.each do |url| metadata_cache[url] ||= { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } } end end def safe_fetch_metadata url fetch_metadata(url) rescue StandardError => e Rails.logger.error( "post_import_metadata_fetch_unexpected_failure "\ "#{ { error: e.class.name, message: e.message }.to_json }") { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } } end def preload_known_tags prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts names = prepared_rows.flat_map { |row| attributes = initial_attributes(row) provenance = initial_provenance(row) tag_sources = initial_tag_sources(row, attributes, provenance) if metadata_url_changed?(row) clear_automatic_values!(attributes, provenance, tag_sources) end if should_apply_metadata_to_row?(row, fetch_metadata, existing_posts, url_counts) metadata = metadata_for(row[:normal_url], metadata_cache) apply_metadata!(attributes, provenance, tag_sources, metadata[:data]) end preview_tag_names(merged_tags(tag_sources, provenance['tags'])) }.compact.uniq return { } if names.empty? Tag.joins(:tag_name) .where(tag_names: { name: names }) .includes(:tag_name) .to_a .index_by(&:name) end def preload_parent_ids prepared_rows ids = prepared_rows.flat_map { |row| attributes = initial_attributes(row) preview_parent_ids(attributes['parent_post_ids']) }.uniq return { } if ids.empty? Post.where(id: ids).pluck(:id).to_h { [_1, true] } end def metadata_url_changed? row row[:metadata_url].present? && row[:metadata_url] != (row[:normal_url] || row[:url_text]) end def should_apply_metadata_to_row? row, fetch_metadata, existing_posts, url_counts normal_url = row[:normal_url] return false if normal_url.blank? return false if url_counts[normal_url].to_i > 1 return false if existing_posts.key?(normal_url) should_fetch_metadata?(fetch_metadata, row[:source_row].to_i) end def apply_metadata! attributes, provenance, tag_sources, metadata metadata.each do |field, value| if field == 'tags' next if provenance['tags'] == 'manual' tag_sources['automatic'] = value.to_s attributes['tags'] = merged_tags(tag_sources) next end next unless provenance[field] == 'automatic' attributes[field] = value provenance[field] = 'automatic' end end def apply_fetch_warnings! field_warnings, warnings warnings.each do |field, values| values.each do |value| add_field_warning!(field_warnings, field, value) end end end def add_field_warning! field_warnings, field, message field_warnings[field] ||= [] field_warnings[field] << message unless field_warnings[field].include?(message) end def merged_tags sources, origin = nil return sources['manual'].to_s if origin == 'manual' sources['automatic'].to_s end def preview_tag_names raw names = raw.to_s.split return [] if names.empty? if names.any? { _1.downcase.start_with?('nico:') } return [] end names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first } rescue Tag::SectionLiteralParseError [] end def validate_preview_tags raw, errors, field_warnings, known_tags names = raw.to_s.split return if names.empty? if names.any? { _1.downcase.start_with?('nico:') } errors[:tags] = ['ニコニコ・タグは直接指定できません.'] return end parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first } existing = parsed.filter_map { known_tags[_1] } deprecated = existing.select(&:deprecated?).map(&:name) errors[:tags] = ["廃止済みタグがあります: #{ deprecated.join(' ') }"] if deprecated.present? known = existing.reject(&:deprecated?).map(&:name) new_tags = parsed.uniq - known if new_tags.present? add_field_warning!(field_warnings, 'tags', "新規タグを作成します: #{ new_tags.join(' ') }") end rescue Tag::SectionLiteralParseError errors[:tags] = ['タグ区間の記法が不正です.'] end def validate_basic_data attributes, errors post = Post.new(title: attributes['title'].presence, url: attributes['url'], thumbnail_base: attributes['thumbnail_base'].presence, original_created_from: attributes['original_created_from'].presence, original_created_before: attributes['original_created_before'].presence, video_ms: parse_duration(attributes['duration'], errors)) post.valid? post.errors.each do |error| next if error.attribute == :url && error.type == :taken (errors[error.attribute] ||= []) << error.message end end def parse_duration value, errors return nil if value.blank? value.is_a?(Numeric) ? value.to_i : Tag.time_to_ms!(value.to_s, tag_name: '動画時間') rescue Tag::SectionLiteralParseError errors[:video_ms] = ['動画時間の記法が不正です.'] nil end def preview_parent_ids raw raw.to_s.split.map { Integer(_1, exception: false) }.compact end def validate_parents raw, errors, existing_parent_ids ids = raw.to_s.split.map { Integer(_1, exception: false) } return if ids.compact.empty? && raw.to_s.blank? if ids.any? { _1.nil? || _1 <= 0 } errors[:parent_post_ids] = ['親投稿 Id. が不正です.'] return end if ids.uniq.any? { !existing_parent_ids[_1] } errors[:parent_post_ids] = ['存在しない親投稿 Id. があります.'] end end end