miteruzo/btrc-hub
1
0
フォーク 0
コード 課題 61 Pull リクエスト 1 リリース Wiki アクティビティ

#399

ソースを参照
このコミットが含まれているのは:
みてるぞ
2026-07-13 00:58:02 +09:00
親 aa96ec95d1
コミット ac82adc6b3
12個のファイルの変更，789行の追加，256行の削除
Patchファイルをダウンロード Diffファイルをダウンロード すべてのファイルを展開 すべてのファイルを折り畳む
backend/app/controllers/post_imports_controller.rb
+6 -7
ファイルの表示
@@ -2,7 +2,7 @@ class PostImportsController < ApplicationController
before_action :require_member!
def preview
rows = PostImportPreviewer.new(existing: params[:existing]).preview_rows(
rows = PostImportPreviewer.new.preview_rows(
rows: PostImportUrlListParser.parse(params[:source]))
render json: { rows: }
rescue ArgumentError => e
@@ -13,18 +13,17 @@ class PostImportsController < ApplicationController
rows = normalised_import_rows(allow_warning_fields: true)
changed_row = Integer(params[:changed_row], exception: false)
result =
PostImportPreviewer.new(existing: params[:existing])
.preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {})
PostImportPreviewer.new.preview_rows(rows:,
fetch_metadata: changed_row,
metadata_cache: { })
render json: { rows: result }
rescue ArgumentError => e
render_bad_request(e.message)
end
def create
result = PostImportRunner.new(
actor: current_user,
rows: normalised_import_rows,
existing: params[:existing]).run
result = PostImportRunner.new(actor: current_user,
rows: normalised_import_rows).run
render json: result, status: result[:created].positive? ? :created : :ok
rescue ArgumentError => e
render_bad_request(e.message)
backend/app/services/post_import_previewer.rb
+79 -40
ファイルの表示
@@ -8,9 +8,11 @@ class PostImportPreviewer
'tags',
'parent_post_ids',
].freeze
def initialize existing: 'skip'
@existing = existing == 'error' ? 'error' : 'skip'
end
FETCH_WARNING_FIELDS = ['url', 'title', 'thumbnail_base'].freeze
EXISTING_SKIP_WARNING = '既存投稿のためスキップします．'.freeze
TITLE_FETCH_WARNING = 'タイトルを取得できませんでした．'.freeze
THUMBNAIL_FETCH_WARNING = 'サムネールを取得できませんでした．'.freeze
METADATA_FETCH_WARNING = 'メタデータを取得できませんでした．'.freeze
def preview_rows rows:, fetch_metadata: true, metadata_cache: { }
urls = {}
@@ -30,48 +32,60 @@ class PostImportPreviewer
attributes = initial_attributes(row)
provenance = initial_provenance(row)
tag_sources = initial_tag_sources(row, attributes, provenance)
field_warnings = initial_field_warnings(row)
base_warnings = initial_base_warnings(row)
url = row[:url].to_s.strip
provenance['url'] = 'manual'
normal_url = normalised_url(url)
url_for_metadata = normal_url || url
existing_post = normal_url.present? && Post.exists?(url: normal_url)
validation_warnings = []
validation_errors = {}
validation_errors[:url] = ['URL が不正です．'] if normal_url.blank?
validation_errors[:url] = ['URL が重複しています．'] if normal_url.present? && urls.key?(normal_url)
urls[normal_url] = true if normal_url.present?
if normal_url.present? && existing_post
if @existing == 'error'
validation_errors[:url] = ['既存投稿です．']
else
validation_warnings << '既存投稿のためスキップします．'
end
end
if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
clear_automatic_values!(attributes, provenance, tag_sources)
clear_fetch_warnings!(field_warnings)
end
if normal_url.present? && existing_post
add_field_warning!(field_warnings, 'url', EXISTING_SKIP_WARNING)
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
warnings_present = field_warnings.values.any?(&:present?) || base_warnings.present?
return {
source_row: row[:source_row],
url: normal_url,
attributes:,
provenance:,
tag_sources:,
metadata_url: url_for_metadata,
field_warnings:,
base_warnings:,
validation_errors:,
status: validation_errors.present? ? 'error' : (warnings_present ? 'warning' : 'ready'),
}
end
fetch_warnings = Array(row[:fetch_warnings]).dup
should_fetch = should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
if should_fetch && !(existing_post && @existing == 'skip')
metadata, fetch_warnings =
metadata_for(url_for_metadata, metadata_cache, fetch_warnings)
apply_metadata!(attributes, provenance, tag_sources, metadata)
if should_fetch
clear_fetch_warnings!(field_warnings)
metadata = metadata_for(url_for_metadata, metadata_cache)
apply_metadata!(attributes, provenance, tag_sources, metadata[:data])
apply_fetch_warnings!(field_warnings, metadata[:warnings])
end
attributes['url'] = url
validate_basic_data(attributes, validation_errors)
validate_preview_tags(
merged_tags(tag_sources, provenance['tags']),
validation_errors,
validation_warnings,
)
validate_preview_tags(merged_tags(tag_sources, provenance['tags']),
validation_errors,
field_warnings)
validate_parents(attributes['parent_post_ids'], validation_errors)
attributes.delete('url')
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
warnings_present = field_warnings.values.any?(&:present?) || base_warnings.present?
{
source_row: row[:source_row],
url: normal_url || url,
@@ -79,14 +93,10 @@ class PostImportPreviewer
provenance:,
tag_sources:,
metadata_url: url_for_metadata,
fetch_warnings:,
validation_warnings:,
warnings: fetch_warnings + validation_warnings,
field_warnings:,
base_warnings:,
validation_errors:,
status: validation_errors.present? \
? 'error' \
: ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
existing: existing_post,
status: validation_errors.present? ? 'error' : (warnings_present ? 'warning' : 'ready'),
}
end
@@ -100,12 +110,12 @@ class PostImportPreviewer
end
def initial_attributes row
attributes = row[:attributes]&.stringify_keys || {}
attributes = row[:attributes]&.stringify_keys || { }
FIELDS.to_h { |field| [field, attributes[field].to_s] }
end
def initial_provenance row
provenance = row[:provenance]&.stringify_keys || {}
provenance = row[:provenance]&.stringify_keys || { }
FIELDS.to_h { |field| [field, provenance[field].presence || 'automatic'] }
.merge('url' => 'manual')
end
@@ -118,6 +128,14 @@ class PostImportPreviewer
sources
end
def initial_field_warnings row
(row[:field_warnings] || { }).stringify_keys.transform_values { |value| Array(value).map(&:to_s) }
end
def initial_base_warnings row
Array(row[:base_warnings]).map(&:to_s)
end
def clear_automatic_values! attributes, provenance, tag_sources
['title', 'thumbnail_base', 'original_created_from',
'original_created_before', 'duration'].each do |field|
@@ -127,18 +145,24 @@ class PostImportPreviewer
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
end
def metadata_for url, cache, previous_warnings
def clear_fetch_warnings! field_warnings
FETCH_WARNING_FIELDS.each do |field|
field_warnings.delete(field)
end
end
def metadata_for url, cache
cache[url] ||= fetch_metadata(url)
end
def fetch_metadata url
return [{}, ['URL が空です．']] if url.blank?
return { data: { }, warnings: { 'url' => ['URL が空です．'] } } if url.blank?
data = PostMetadataFetcher.fetch(url).stringify_keys
warnings = []
warnings << 'タイトルを取得できませんでした．' if data['title'].blank?
warnings << 'サムネールを取得できませんでした．' if data['thumbnail_base'].blank?
[data.compact, warnings]
data = PostMetadataFetcher.fetch(url).stringify_keys.compact
warnings = { }
add_field_warning!(warnings, 'title', TITLE_FETCH_WARNING) if data['title'].blank?
add_field_warning!(warnings, 'thumbnail_base', THUMBNAIL_FETCH_WARNING) if data['thumbnail_base'].blank?
{ data:, warnings: }
rescue Preview::UrlSafety::UnsafeUrl,
Preview::HttpFetcher::FetchFailed,
Preview::HttpFetcher::ResponseTooLarge => e
@@ -146,7 +170,7 @@ class PostImportPreviewer
"post_import_metadata_fetch_failure "\
"#{ { error: e.class.name, message: e.message }.to_json }",
)
[{}, ['メタデータを取得できませんでした．']]
{ data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } }
end
def apply_metadata! attributes, provenance, tag_sources, metadata
@@ -165,13 +189,26 @@ class PostImportPreviewer
end
end
def apply_fetch_warnings! field_warnings, warnings
warnings.each do |field, values|
values.each do |value|
add_field_warning!(field_warnings, field, value)
end
end
end
def add_field_warning! field_warnings, field, message
field_warnings[field] ||= []
field_warnings[field] << message unless field_warnings[field].include?(message)
end
def merged_tags sources, origin = nil
return sources['manual'].to_s if origin == 'manual'
sources['automatic'].to_s
end
def validate_preview_tags raw, errors, warnings
def validate_preview_tags raw, errors, field_warnings
names = raw.to_s.split
return if names.empty?
if names.any? { _1.downcase.start_with?('nico:') }
@@ -185,7 +222,9 @@ class PostImportPreviewer
errors[:tags] = ["廃止済みタグがあります: #{ deprecated.join(' ') }"] if deprecated.present?
known = existing.reject(&:deprecated?).map(&:name)
new_tags = parsed.uniq - known
warnings << "新規タグを作成します: #{ new_tags.join(' ') }" if new_tags.present?
if new_tags.present?
add_field_warning!(field_warnings, 'tags', "新規タグを作成します: #{ new_tags.join(' ') }")
end
rescue Tag::SectionLiteralParseError
errors[:tags] = ['タグ区間の記法が不正です．']
end
backend/app/services/post_import_row_normaliser.rb
+18 -8
ファイルの表示
@@ -64,10 +64,10 @@ class PostImportRowNormaliser
return keys unless allow_warning_fields
keys + [
{ fetch_warnings: [] },
{ fetchWarnings: [] },
{ validation_warnings: [] },
{ validationWarnings: [] },
{ field_warnings: {} },
{ fieldWarnings: {} },
{ base_warnings: [] },
{ baseWarnings: [] },
]
end
private_class_method :permitted_keys
@@ -141,10 +141,12 @@ class PostImportRowNormaliser
def self.normalise_warning_values! normalised, allow_warning_fields:
return unless allow_warning_fields
%w[fetch_warnings validation_warnings].each do |field|
values = normalised[field]
next if values.nil?
field_warnings = normalised['field_warnings']
unless field_warnings.nil? || field_warnings.is_a?(Hash)
raise ArgumentError, '警告の形式が不正です．'
end
field_warnings&.each do |key, values|
raise ArgumentError, '警告の形式が不正です．' unless ATTRIBUTE_FIELDS.include?(key) || key == 'url'
unless values.is_a?(Array) && values.all? { _1.is_a?(String) }
raise ArgumentError, '警告の形式が不正です．'
end
@@ -152,6 +154,14 @@ class PostImportRowNormaliser
raise ArgumentError, '警告が大きすぎます．'
end
end
base_warnings = normalised['base_warnings']
return if base_warnings.nil?
unless base_warnings.is_a?(Array) && base_warnings.all? { _1.is_a?(String) }
raise ArgumentError, '警告の形式が不正です．'
end
raise ArgumentError, '警告が大きすぎます．' if base_warnings.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES }
end
private_class_method :normalise_warning_values!
backend/app/services/post_import_runner.rb
+3 -5
ファイルの表示
@@ -1,8 +1,7 @@
class PostImportRunner
def initialize actor:, rows:, existing: 'skip'
def initialize actor:, rows:
@actor = actor
@rows = rows
@existing = existing == 'error' ? 'error' : 'skip'
end
def run
@@ -21,8 +20,7 @@ class PostImportRunner
def run_row row
attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore }
preview = PostImportPreviewer.new(existing: @existing)
.preview_rows(
preview = PostImportPreviewer.new.preview_rows(
rows: [{
source_row: row['source_row'],
url: row['url'],
@@ -31,7 +29,7 @@ class PostImportRunner
tag_sources: row['tag_sources'],
}],
fetch_metadata: false).first
if @existing == 'skip' && preview[:existing]
if Array(preview.dig(:field_warnings, 'url')).include?(PostImportPreviewer::EXISTING_SKIP_WARNING)
return row.slice('source_row').merge(status: 'skipped')
end
backend/app/services/post_import_url_list_parser.rb
+1 -1
ファイルの表示
@@ -11,7 +11,7 @@ class PostImportUrlListParser
url = line.strip
next if url.blank?
raise ArgumentError, 'URL が長すぎます．' if url.bytesize > MAX_URL_BYTES
raise ArgumentError, "#{ index + 1 } 行目: URL が長すぎます．" if url.bytesize > MAX_URL_BYTES
{ source_row: index + 1, url: }
}