Reviewed-on: #413 Co-authored-by: miteruzo <miteruzo@naver.com> Co-committed-by: miteruzo <miteruzo@naver.com>
このコミットはPull リクエスト #413 でマージされました．
このコミットが含まれているのは:
@@ -0,0 +1,522 @@
|
||||
require 'time'
|
||||
require 'timeout'
|
||||
|
||||
class PostImportPreviewer
|
||||
FIELDS = [
|
||||
'title',
|
||||
'thumbnail_base',
|
||||
'original_created_from',
|
||||
'original_created_before',
|
||||
'video_ms',
|
||||
'duration',
|
||||
'tags',
|
||||
'parent_post_ids'].freeze
|
||||
FETCH_WARNING_FIELDS = ['url', 'title', 'thumbnail_base'].freeze
|
||||
TITLE_FETCH_WARNING = 'タイトルを取得できませんでした．'.freeze
|
||||
THUMBNAIL_FETCH_WARNING = 'サムネールを取得できませんでした．'.freeze
|
||||
METADATA_FETCH_WARNING = '自動取得に失敗しました．'.freeze
|
||||
|
||||
def preview_rows rows:, fetch_metadata: true, metadata_cache: { }
|
||||
prepared_rows = rows.map { prepare_row(_1) }
|
||||
url_counts = prepared_rows.filter_map { _1[:normal_url] }.tally
|
||||
existing_posts =
|
||||
Post.where(url: prepared_rows.map { _1[:normal_url] }.compact.uniq).index_by(&:url)
|
||||
existing_parent_ids = preload_parent_ids(prepared_rows)
|
||||
preload_metadata!(
|
||||
prepared_rows,
|
||||
fetch_metadata,
|
||||
metadata_cache,
|
||||
existing_posts,
|
||||
url_counts)
|
||||
known_tags =
|
||||
preload_known_tags(
|
||||
prepared_rows,
|
||||
fetch_metadata,
|
||||
metadata_cache,
|
||||
existing_posts,
|
||||
url_counts)
|
||||
prepared_rows.map { |row|
|
||||
preview_row(row,
|
||||
fetch_metadata:,
|
||||
metadata_cache:,
|
||||
existing_posts:,
|
||||
url_counts:,
|
||||
known_tags:,
|
||||
existing_parent_ids:)
|
||||
}
|
||||
end
|
||||
|
||||
def normalised_url value
|
||||
PostUrlNormaliser.normalise(value)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def prepare_row row
|
||||
source = row.symbolize_keys
|
||||
url = source[:url].to_s.strip
|
||||
normal_url = normalised_url(url)
|
||||
source.merge(url_text: url, normal_url:, url_error: validate_url_safety(normal_url))
|
||||
end
|
||||
|
||||
def preview_row row,
|
||||
fetch_metadata:,
|
||||
metadata_cache:,
|
||||
existing_posts:,
|
||||
url_counts:,
|
||||
known_tags:,
|
||||
existing_parent_ids:
|
||||
attributes = initial_attributes(row)
|
||||
provenance = initial_provenance(row)
|
||||
tag_sources = initial_tag_sources(row, attributes, provenance)
|
||||
field_warnings = initial_field_warnings(row)
|
||||
base_warnings = initial_base_warnings(row)
|
||||
url = row[:url_text]
|
||||
provenance['url'] = 'manual'
|
||||
normal_url = row[:normal_url]
|
||||
url_for_metadata = normal_url || url
|
||||
existing_post = normal_url.present? ? existing_posts[normal_url] : nil
|
||||
|
||||
validation_errors = {}
|
||||
validation_errors[:url] = ['URL が不正です．'] if normal_url.blank?
|
||||
if row[:url_error].present?
|
||||
validation_errors[:url] = [row[:url_error]]
|
||||
end
|
||||
if normal_url.present? && url_counts[normal_url].to_i > 1
|
||||
validation_errors[:url] = ['URL が重複しています．']
|
||||
end
|
||||
|
||||
if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
|
||||
clear_automatic_values!(attributes, provenance, tag_sources)
|
||||
field_warnings = { }
|
||||
base_warnings = [ ]
|
||||
end
|
||||
|
||||
if validation_errors.blank? && normal_url.present? && existing_post
|
||||
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
|
||||
warnings_present = field_warnings.values.any?(&:present?) || base_warnings.present?
|
||||
return { source_row: row[:source_row],
|
||||
url: normal_url,
|
||||
attributes:,
|
||||
provenance:,
|
||||
tag_sources:,
|
||||
metadata_url: url_for_metadata,
|
||||
skip_reason: 'existing',
|
||||
existing_post_id: existing_post.id,
|
||||
field_warnings:,
|
||||
base_warnings:,
|
||||
validation_errors:,
|
||||
status: warnings_present ? 'warning' : 'ready' }
|
||||
end
|
||||
|
||||
should_fetch = validation_errors.blank?
|
||||
should_fetch &&= should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
|
||||
if should_fetch
|
||||
clear_fetch_warnings!(field_warnings)
|
||||
metadata = metadata_for(url_for_metadata, metadata_cache)
|
||||
if metadata[:validation_errors].present?
|
||||
validation_errors.merge!(metadata[:validation_errors])
|
||||
else
|
||||
apply_metadata!(attributes, provenance, tag_sources, metadata[:data])
|
||||
apply_fetch_warnings!(field_warnings, metadata[:warnings])
|
||||
end
|
||||
end
|
||||
|
||||
attributes['url'] = url
|
||||
validate_basic_data(attributes, validation_errors)
|
||||
validate_preview_tags(merged_tags(tag_sources, provenance['tags']),
|
||||
validation_errors,
|
||||
known_tags)
|
||||
validate_parents(attributes['parent_post_ids'], validation_errors, existing_parent_ids)
|
||||
attributes.delete('url')
|
||||
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
|
||||
|
||||
warnings_present = field_warnings.values.any?(&:present?) || base_warnings.present?
|
||||
{ source_row: row[:source_row],
|
||||
url: validation_errors[:url].present? ? url : (normal_url || url),
|
||||
attributes:,
|
||||
provenance:,
|
||||
tag_sources:,
|
||||
metadata_url: url_for_metadata,
|
||||
skip_reason: nil,
|
||||
existing_post_id: nil,
|
||||
field_warnings:,
|
||||
base_warnings:,
|
||||
validation_errors:,
|
||||
status: validation_errors.present? ? 'error' : (warnings_present ? 'warning' : 'ready') }
|
||||
end
|
||||
|
||||
def should_fetch_metadata? fetch_metadata, source_row
|
||||
case fetch_metadata
|
||||
when true then true
|
||||
when Integer then fetch_metadata == source_row
|
||||
when false, nil then false
|
||||
else raise ArgumentError, '取得対象が不正です．'
|
||||
end
|
||||
end
|
||||
|
||||
def initial_attributes row
|
||||
attributes = row[:attributes]&.stringify_keys || { }
|
||||
FIELDS.to_h { |field|
|
||||
value = attributes[field]
|
||||
normalised =
|
||||
if field == 'duration'
|
||||
normalise_duration_attribute(value)
|
||||
elsif field == 'video_ms'
|
||||
value.nil? ? '' : value.to_s
|
||||
else
|
||||
value.to_s
|
||||
end
|
||||
[field, normalised] }
|
||||
end
|
||||
|
||||
def initial_provenance row
|
||||
provenance = row[:provenance]&.stringify_keys || { }
|
||||
FIELDS.to_h { |field| [field, provenance[field].presence || 'automatic'] }
|
||||
.merge('url' => 'manual')
|
||||
end
|
||||
|
||||
def initial_tag_sources row, attributes, provenance
|
||||
sources = row[:tag_sources]&.stringify_keys || { 'automatic' => '', 'manual' => '' }
|
||||
sources['automatic'] = sources['automatic'].to_s
|
||||
sources['manual'] =
|
||||
provenance['tags'] == 'manual' ? attributes['tags'].to_s : sources['manual'].to_s
|
||||
sources
|
||||
end
|
||||
|
||||
def initial_field_warnings row
|
||||
(row[:field_warnings] || { })
|
||||
.stringify_keys
|
||||
.transform_values { |value| Array(value).map(&:to_s) }
|
||||
end
|
||||
|
||||
def initial_base_warnings row
|
||||
Array(row[:base_warnings]).map(&:to_s)
|
||||
end
|
||||
|
||||
def clear_automatic_values! attributes, provenance, tag_sources
|
||||
['title', 'thumbnail_base', 'original_created_from',
|
||||
'original_created_before', 'video_ms', 'duration'].each do |field|
|
||||
attributes[field] = '' if provenance[field] == 'automatic'
|
||||
end
|
||||
tag_sources['automatic'] = ''
|
||||
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
|
||||
end
|
||||
|
||||
def clear_fetch_warnings! field_warnings
|
||||
FETCH_WARNING_FIELDS.each do |field|
|
||||
field_warnings.delete(field)
|
||||
end
|
||||
end
|
||||
|
||||
def metadata_for url, cache
|
||||
cache[url] ||= fetch_metadata(url)
|
||||
end
|
||||
|
||||
def fetch_metadata url
|
||||
return { data: { }, warnings: { 'url' => ['URL が空です．'] },
|
||||
validation_errors: { } } if url.blank?
|
||||
|
||||
data = sanitise_metadata(PostMetadataFetcher.fetch(url).stringify_keys.compact)
|
||||
warnings = { }
|
||||
add_field_warning!(warnings, 'title', TITLE_FETCH_WARNING) if data['title'].blank?
|
||||
if data['thumbnail_base'].blank?
|
||||
add_field_warning!(warnings, 'thumbnail_base', THUMBNAIL_FETCH_WARNING)
|
||||
end
|
||||
{ data:, warnings:, validation_errors: { } }
|
||||
rescue Preview::UrlSafety::UnsafeUrl => e
|
||||
payload = { error: e.class.name, message: e.message }
|
||||
Rails.logger.info(
|
||||
"post_import_metadata_fetch_unsafe_url #{ payload.to_json }")
|
||||
{ data: { }, warnings: { }, validation_errors: { url: [e.message] } }
|
||||
rescue Preview::HttpFetcher::FetchFailed,
|
||||
Preview::HttpFetcher::ResponseTooLarge => e
|
||||
payload = { error: e.class.name, message: e.message }
|
||||
Rails.logger.info(
|
||||
"post_import_metadata_fetch_failure #{ payload.to_json }")
|
||||
{ data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] }, validation_errors: { } }
|
||||
end
|
||||
|
||||
def sanitise_metadata metadata
|
||||
{
|
||||
'title' => sanitise_metadata_title(metadata['title']),
|
||||
'thumbnail_base' => sanitise_metadata_url(metadata['thumbnail_base']),
|
||||
'original_created_from' => sanitise_metadata_time(metadata['original_created_from']),
|
||||
'original_created_before' => sanitise_metadata_time(metadata['original_created_before']),
|
||||
'duration' => sanitise_metadata_duration(metadata['duration']),
|
||||
'tags' => metadata['tags'].to_s.presence }.compact
|
||||
end
|
||||
|
||||
def sanitise_metadata_title value
|
||||
value.is_a?(String) ? value.presence : nil
|
||||
end
|
||||
|
||||
def sanitise_metadata_url value
|
||||
return nil unless value.is_a?(String)
|
||||
|
||||
stripped = value.strip
|
||||
return nil if stripped.blank?
|
||||
|
||||
uri = URI.parse(stripped)
|
||||
return nil unless uri.is_a?(URI::HTTP) && uri.host.present?
|
||||
|
||||
stripped
|
||||
rescue URI::InvalidURIError
|
||||
nil
|
||||
end
|
||||
|
||||
def sanitise_metadata_time value
|
||||
return nil unless value.is_a?(String)
|
||||
|
||||
Time.iso8601(value).in_time_zone.change(sec: 0, nsec: 0).iso8601
|
||||
rescue ArgumentError, TypeError
|
||||
nil
|
||||
end
|
||||
|
||||
def sanitise_metadata_duration value
|
||||
return nil unless value.is_a?(String)
|
||||
|
||||
value.presence
|
||||
end
|
||||
|
||||
def normalise_duration_attribute value
|
||||
return '' if value.nil?
|
||||
return value if value.is_a?(String)
|
||||
|
||||
milliseconds = Integer(value, exception: false)
|
||||
return value.to_s if milliseconds.nil? || milliseconds <= 0
|
||||
|
||||
seconds_string = (milliseconds / 1_000.0).to_s
|
||||
seconds_string.end_with?('.0') ? seconds_string.delete_suffix('.0') : seconds_string
|
||||
end
|
||||
|
||||
def preload_metadata! prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts
|
||||
urls = prepared_rows.filter_map { |row|
|
||||
next unless row[:normal_url].present?
|
||||
next if row[:url_error].present?
|
||||
next unless should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
|
||||
next if url_counts[row[:normal_url]].to_i > 1
|
||||
next if existing_posts.key?(row[:normal_url])
|
||||
|
||||
row[:normal_url]
|
||||
}.uniq
|
||||
urls = urls.reject { metadata_cache.key?(_1) }
|
||||
return if urls.empty?
|
||||
|
||||
url_queue = Queue.new
|
||||
result_queue = Queue.new
|
||||
urls.each { url_queue << _1 }
|
||||
workers = [urls.length, 4].min.times.map {
|
||||
Thread.new {
|
||||
Rails.application.executor.wrap do
|
||||
loop do
|
||||
url = url_queue.pop(true)
|
||||
result_queue << [url, safe_fetch_metadata(url)]
|
||||
rescue ThreadError
|
||||
break
|
||||
end
|
||||
end
|
||||
}
|
||||
}
|
||||
Timeout.timeout(15) { workers.each(&:join) }
|
||||
rescue Timeout::Error
|
||||
workers&.each(&:kill)
|
||||
ensure
|
||||
workers&.each(&:join)
|
||||
while result_queue&.size.to_i.positive?
|
||||
url, result = result_queue.pop
|
||||
metadata_cache[url] = result
|
||||
end
|
||||
urls&.each do |url|
|
||||
metadata_cache[url] ||= { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } }
|
||||
end
|
||||
end
|
||||
|
||||
def safe_fetch_metadata url
|
||||
fetch_metadata(url)
|
||||
rescue Preview::UrlSafety::UnsafeUrl => e
|
||||
payload = { error: e.class.name, message: e.message }
|
||||
Rails.logger.info(
|
||||
"post_import_metadata_fetch_unsafe_url #{ payload.to_json }")
|
||||
{ data: { }, warnings: { }, validation_errors: { url: [e.message] } }
|
||||
rescue StandardError => e
|
||||
payload = { error: e.class.name, message: e.message }
|
||||
Rails.logger.error(
|
||||
"post_import_metadata_fetch_unexpected_failure #{ payload.to_json }")
|
||||
{ data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] }, validation_errors: { } }
|
||||
end
|
||||
|
||||
def preload_known_tags prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts
|
||||
names = prepared_rows.flat_map { |row|
|
||||
attributes = initial_attributes(row)
|
||||
provenance = initial_provenance(row)
|
||||
tag_sources = initial_tag_sources(row, attributes, provenance)
|
||||
if metadata_url_changed?(row)
|
||||
clear_automatic_values!(attributes, provenance, tag_sources)
|
||||
end
|
||||
if should_apply_metadata_to_row?(row, fetch_metadata, existing_posts, url_counts)
|
||||
metadata = metadata_for(row[:normal_url], metadata_cache)
|
||||
apply_metadata!(attributes, provenance, tag_sources, metadata[:data])
|
||||
end
|
||||
preview_tag_names(merged_tags(tag_sources, provenance['tags']))
|
||||
}.compact.uniq
|
||||
return { } if names.empty?
|
||||
|
||||
Tag.joins(:tag_name)
|
||||
.where(tag_names: { name: names })
|
||||
.includes(:tag_name)
|
||||
.to_a
|
||||
.index_by(&:name)
|
||||
end
|
||||
|
||||
def preload_parent_ids prepared_rows
|
||||
ids = prepared_rows.flat_map { |row|
|
||||
attributes = initial_attributes(row)
|
||||
preview_parent_ids(attributes['parent_post_ids'])
|
||||
}.uniq
|
||||
return { } if ids.empty?
|
||||
|
||||
Post.where(id: ids).pluck(:id).to_h { [_1, true] }
|
||||
end
|
||||
|
||||
def metadata_url_changed? row
|
||||
row[:metadata_url].present? && row[:metadata_url] != (row[:normal_url] || row[:url_text])
|
||||
end
|
||||
|
||||
def should_apply_metadata_to_row? row, fetch_metadata, existing_posts, url_counts
|
||||
normal_url = row[:normal_url]
|
||||
return false if normal_url.blank?
|
||||
return false if row[:url_error].present?
|
||||
return false if url_counts[normal_url].to_i > 1
|
||||
return false if existing_posts.key?(normal_url)
|
||||
|
||||
should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
|
||||
end
|
||||
|
||||
def validate_url_safety normal_url
|
||||
return nil if normal_url.blank?
|
||||
|
||||
Preview::UrlSafety.validate(normal_url)
|
||||
nil
|
||||
rescue Preview::UrlSafety::UnsafeUrl => e
|
||||
e.message
|
||||
end
|
||||
|
||||
def apply_metadata! attributes, provenance, tag_sources, metadata
|
||||
metadata.each do |field, value|
|
||||
if field == 'tags'
|
||||
next if provenance['tags'] == 'manual'
|
||||
|
||||
tag_sources['automatic'] = value.to_s
|
||||
attributes['tags'] = merged_tags(tag_sources)
|
||||
next
|
||||
end
|
||||
next unless provenance[field] == 'automatic'
|
||||
|
||||
attributes[field] = value
|
||||
provenance[field] = 'automatic'
|
||||
end
|
||||
end
|
||||
|
||||
def apply_fetch_warnings! field_warnings, warnings
|
||||
warnings.each do |field, values|
|
||||
values.each do |value|
|
||||
add_field_warning!(field_warnings, field, value)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def add_field_warning! field_warnings, field, message
|
||||
field_warnings[field] ||= []
|
||||
field_warnings[field] << message unless field_warnings[field].include?(message)
|
||||
end
|
||||
|
||||
def merged_tags sources, origin = nil
|
||||
return sources['manual'].to_s if origin == 'manual'
|
||||
|
||||
sources['automatic'].to_s
|
||||
end
|
||||
|
||||
def preview_tag_names raw
|
||||
names = raw.to_s.split
|
||||
return [] if names.empty?
|
||||
if names.any? { _1.downcase.start_with?('nico:') }
|
||||
return []
|
||||
end
|
||||
|
||||
names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
|
||||
rescue Tag::SectionLiteralParseError
|
||||
[]
|
||||
end
|
||||
|
||||
def validate_preview_tags raw, errors, known_tags
|
||||
names = raw.to_s.split
|
||||
return if names.empty?
|
||||
if names.any? { _1.downcase.start_with?('nico:') }
|
||||
errors[:tags] = ['ニコニコ・タグは直接指定できません．']
|
||||
return
|
||||
end
|
||||
|
||||
parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
|
||||
existing = parsed.filter_map { known_tags[_1] }
|
||||
deprecated = existing.select(&:deprecated?).map(&:name)
|
||||
errors[:tags] = ["廃止済みタグがあります: #{ deprecated.join(' ') }"] if deprecated.present?
|
||||
rescue Tag::SectionLiteralParseError
|
||||
errors[:tags] = ['タグ区間の記法が不正です．']
|
||||
end
|
||||
|
||||
def validate_basic_data attributes, errors
|
||||
post = Post.new(title: attributes['title'].presence,
|
||||
url: attributes['url'],
|
||||
thumbnail_base: attributes['thumbnail_base'].presence,
|
||||
original_created_from: attributes['original_created_from'].presence,
|
||||
original_created_before: attributes['original_created_before'].presence,
|
||||
video_ms: parse_video_ms(attributes, errors))
|
||||
post.valid?
|
||||
post.errors.each do |error|
|
||||
next if error.attribute == :url && error.type == :taken
|
||||
|
||||
(errors[error.attribute] ||= []) << error.message
|
||||
end
|
||||
end
|
||||
|
||||
def parse_video_ms attributes, errors
|
||||
return nil unless attributes['tags'].to_s.split.include?('動画')
|
||||
|
||||
video_ms = attributes['video_ms']
|
||||
if video_ms.present?
|
||||
value = Integer(video_ms, exception: false)
|
||||
if value&.positive?
|
||||
return value
|
||||
end
|
||||
|
||||
errors[:video_ms] = ['動画時間の記法が不正です．']
|
||||
return nil
|
||||
end
|
||||
|
||||
value = attributes['duration']
|
||||
return nil if value.blank?
|
||||
|
||||
Tag.time_to_ms!(value.to_s, tag_name: '動画時間')
|
||||
rescue Tag::SectionLiteralParseError
|
||||
errors[:video_ms] = ['動画時間の記法が不正です．']
|
||||
nil
|
||||
end
|
||||
|
||||
def preview_parent_ids raw
|
||||
raw.to_s.split.map { Integer(_1, exception: false) }.compact
|
||||
end
|
||||
|
||||
def validate_parents raw, errors, existing_parent_ids
|
||||
ids = raw.to_s.split.map { Integer(_1, exception: false) }
|
||||
return if ids.compact.empty? && raw.to_s.blank?
|
||||
|
||||
if ids.any? { _1.nil? || _1 <= 0 }
|
||||
errors[:parent_post_ids] = ['親投稿 Id. が不正です．']
|
||||
return
|
||||
end
|
||||
if ids.uniq.any? { !existing_parent_ids[_1] }
|
||||
errors[:parent_post_ids] = ['存在しない親投稿 Id. があります．']
|
||||
end
|
||||
end
|
||||
end
|
||||
新しい課題から参照
ユーザをブロックする