diff --git a/backend/app/controllers/post_imports_controller.rb b/backend/app/controllers/post_imports_controller.rb deleted file mode 100644 index 9f3a6c8..0000000 --- a/backend/app/controllers/post_imports_controller.rb +++ /dev/null @@ -1,49 +0,0 @@ -class PostImportsController < ApplicationController - before_action :require_member! - - def preview - rows = PostImportPreviewer.new.preview_rows( - rows: PostImportUrlListParser.parse(params[:source])) - render json: { rows: } - rescue ArgumentError => e - render_bad_request e.message - end - - def validate - rows = normalised_import_rows allow_warning_fields: true - changed_row = - if params[:changed_row].to_s == 'all' - true - else - Integer(params[:changed_row], exception: false) - end - result = - PostImportPreviewer.new.preview_rows(rows:, - fetch_metadata: changed_row, - metadata_cache: { }) - render json: { rows: result } - rescue ArgumentError => e - render_bad_request e.message - end - - def create - result = PostImportRunner.new(actor: current_user, - rows: normalised_import_rows).run - render json: result, status: result[:created].positive? ? :created : :ok - rescue ArgumentError => e - render_bad_request e.message - end - - private - - def require_member! - return head :unauthorized unless current_user - return if current_user.gte_member? - - head :forbidden - end - - def normalised_import_rows allow_warning_fields: false - PostImportRowNormaliser.normalise!(params[:rows], allow_warning_fields:) - end -end diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index 5bfbdb1..c4b9854 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -114,6 +114,37 @@ class PostsController < ApplicationController render json: PostRepr.base(post, current_user) end + def preview + return head :unauthorized unless current_user + return head :forbidden unless current_user.gte_member? + return render_bad_request('URL は必須です.') if params[:url].blank? + + preview = PostImportPreviewer.new.preview_rows( + rows: [{ + source_row: 1, + url: params[:url].to_s, + attributes: { }, + provenance: { }, + tag_sources: { } }], + fetch_metadata: true).first + return render_validation_error fields: preview[:validation_errors] if preview[:validation_errors].present? + + render json: { + url: preview[:url], + title: preview[:attributes]['title'], + thumbnail_base: preview[:attributes]['thumbnail_base'], + tags: preview[:attributes]['tags'], + original_created_from: preview[:attributes]['original_created_from'], + original_created_before: preview[:attributes]['original_created_before'], + duration: preview[:attributes]['duration'], + video_ms: preview[:attributes]['video_ms'], + field_warnings: preview[:field_warnings], + base_warnings: preview[:base_warnings], + existing_post: compact_post(preview[:existing_post_id]) } + rescue ArgumentError => e + render_bad_request e.message + end + def show post = Post @@ -142,17 +173,21 @@ class PostsController < ApplicationController return head :unauthorized unless current_user return head :forbidden unless current_user.gte_member? + if bool?(:dry) + preflight = PostCreatePreflight.new( + attributes: post_create_attributes, + thumbnail: params[:thumbnail]).run + return render json: preflight + end + post = PostCreator.new(actor: current_user, - attributes: { title: params[:title], url: params[:url], - thumbnail: params[:thumbnail], tags: params[:tags], - original_created_from: params[:original_created_from], - original_created_before: params[:original_created_before], - parent_post_ids: parse_parent_post_ids, - video_ms: params[:video_ms], - duration: params[:duration] }).create! + attributes: post_create_attributes.merge( + thumbnail: params[:thumbnail])).create! post.reload render json: PostRepr.base(post), status: :created + rescue PostCreatePreflight::ValidationFailed => e + render_validation_error fields: e.fields, base: e.base_errors rescue Tag::NicoTagNormalisationError render_validation_error fields: { tags: 'ニコニコ・タグは直接指定できません.' } rescue Tag::DeprecatedTagNormalisationError @@ -161,6 +196,8 @@ class PostsController < ApplicationController render_validation_error fields: { tags: ['タグ区間の記法が不正です.'] } rescue PostCreator::VideoMsParseError render_validation_error fields: { video_ms: ['動画時間の記法が不正です.'] } + rescue Post::RemoteThumbnailFetchFailed + render_validation_error fields: { thumbnail_base: ['サムネイル画像の取得に失敗しました.'] } rescue MiniMagick::Error render_validation_error fields: { thumbnail: ['サムネイル画像の変換に失敗しました.'] } rescue ArgumentError => e @@ -169,6 +206,22 @@ class PostsController < ApplicationController render_post_form_record_invalid e.record end + def bulk + return head :unauthorized unless current_user + return head :forbidden unless current_user.gte_member? + return head :unsupported_media_type unless request.content_mime_type == Mime[:multipart_form] + + result = PostBulkCreator.new( + actor: current_user, + posts: parse_bulk_posts_manifest, + thumbnails: parse_bulk_thumbnails).run + render json: result + rescue JSON::ParserError + render_bad_request 'posts manifest の JSON が不正です.' + rescue ArgumentError => e + render_validation_error base: [e.message] + end + def viewed return head :unauthorized unless current_user @@ -463,6 +516,63 @@ class PostsController < ApplicationController }.uniq end + def post_create_attributes + { title: params[:title], + url: params[:url], + thumbnail_base: params[:thumbnail_base], + tags: params[:tags], + original_created_from: params[:original_created_from], + original_created_before: params[:original_created_before], + parent_post_ids: parse_parent_post_ids, + video_ms: params[:video_ms], + duration: params[:duration] } + end + + def parse_bulk_posts_manifest + manifest = params[:posts] + raise ArgumentError, 'posts は必須です.' if manifest.blank? + + posts = JSON.parse(manifest) + raise ArgumentError, 'posts は配列で指定してください.' unless posts.is_a?(Array) + raise ArgumentError, '投稿件数は 1 件以上必要です.' if posts.empty? + raise ArgumentError, '投稿件数が多すぎます.' if posts.length > 100 + raise ArgumentError, 'posts 要素の形式が不正です.' unless posts.all? { _1.is_a?(Hash) } + + posts + end + + def parse_bulk_thumbnails + thumbnails = { } + post_count = Array(JSON.parse(params[:posts].presence || '[]')).length + + params.to_unsafe_h.each do |key, value| + match = key.match(/\Athumbnails\[(\d+)\]\z/) + next if match.nil? + + index = Integer(match[1], 10) + raise ArgumentError, 'thumbnail index が範囲外です.' if index.negative? || index >= post_count + + thumbnails[index] = value + end + + thumbnails + end + + def compact_post post_id + return nil if post_id.blank? + + post = Post.with_attached_thumbnail.find_by(id: post_id) + return nil if post.nil? + + { id: post.id, + title: post.title, + url: post.url, + thumbnail_url: + if post.thumbnail.attached? + rails_storage_proxy_url(post.thumbnail, only_path: false) + end } + end + def sync_parent_posts! post, parent_post_ids if parent_post_ids.include?(post.id) post.errors.add :parent_post_ids, '自分自身を親投稿にはできません.' diff --git a/backend/app/controllers/preview_controller.rb b/backend/app/controllers/preview_controller.rb index 80e4f7a..4829e2b 100644 --- a/backend/app/controllers/preview_controller.rb +++ b/backend/app/controllers/preview_controller.rb @@ -38,27 +38,6 @@ class PreviewController < ApplicationController render_unprocessable_entity(e.message) end - def image - return render_bad_request('URL は必須です.') if params[:url].blank? - - attachment = Post.resized_thumbnail_attachment( - StringIO.new(Preview::ThumbnailFetcher.fetch_image_response(params[:url]).body)) - attachment[:io].rewind - send_data attachment[:io].read, - type: attachment[:content_type], - disposition: 'inline' - rescue Preview::UrlSafety::UnsafeUrl => e - render_bad_request(e.message) - rescue Preview::HttpFetcher::FetchTimeout => e - render_preview_error(e.message, :gateway_timeout) - rescue Preview::HttpFetcher::ResponseTooLarge => e - render_preview_error(e.message, :payload_too_large) - rescue Preview::HttpFetcher::FetchFailed => e - render_preview_error(e.message, :bad_gateway) - rescue Preview::ThumbnailFetcher::GenerationFailed, MiniMagick::Error => e - render_unprocessable_entity(e.message) - end - private def require_member! diff --git a/backend/app/models/post.rb b/backend/app/models/post.rb index 3f58072..1190311 100644 --- a/backend/app/models/post.rb +++ b/backend/app/models/post.rb @@ -27,6 +27,18 @@ class Post < ApplicationRecord upload.rewind end + def self.remote_thumbnail_attachment(raw_url) + response = Preview::ThumbnailFetcher.fetch_image_response(raw_url) + resized_thumbnail_attachment(StringIO.new(response.body)) + rescue Preview::UrlSafety::UnsafeUrl, + Preview::ThumbnailFetcher::GenerationFailed, + Preview::HttpFetcher::FetchFailed, + Preview::HttpFetcher::FetchTimeout, + Preview::HttpFetcher::ResponseTooLarge, + MiniMagick::Error => e + raise RemoteThumbnailFetchFailed, e.message + end + belongs_to :uploaded_user, class_name: 'User', optional: true has_many :post_tags, dependent: :destroy, inverse_of: :post @@ -157,17 +169,7 @@ class Post < ApplicationRecord end def attach_thumbnail_from_url! raw_url - response = Preview::ThumbnailFetcher.fetch_image_response(raw_url) - thumbnail.attach( - self.class.resized_thumbnail_attachment( - StringIO.new(response.body))) - rescue Preview::UrlSafety::UnsafeUrl, - Preview::ThumbnailFetcher::GenerationFailed, - Preview::HttpFetcher::FetchFailed, - Preview::HttpFetcher::FetchTimeout, - Preview::HttpFetcher::ResponseTooLarge, - MiniMagick::Error => e - raise RemoteThumbnailFetchFailed, e.message + thumbnail.attach(self.class.remote_thumbnail_attachment(raw_url)) end private diff --git a/backend/app/services/post_bulk_creator.rb b/backend/app/services/post_bulk_creator.rb new file mode 100644 index 0000000..35b2454 --- /dev/null +++ b/backend/app/services/post_bulk_creator.rb @@ -0,0 +1,150 @@ +class PostBulkCreator + def initialize actor:, posts:, thumbnails: + @actor = actor + @posts = posts + @thumbnails = thumbnails + end + + def run + results = @posts.each_with_index.map { |attributes, index| + create_row(attributes, index) + } + + { results: } + end + + private + + def create_row attributes, index + preflight = + PostCreatePreflight.new( + attributes: attributes, + thumbnail: thumbnail_for(index, attributes)).run + if preflight[:existing_post].present? + return { + status: 'skipped', + existing_post: preflight[:existing_post] } + end + + post = PostCreator.new( + actor: @actor, + attributes: normalised_attributes(attributes, preflight, index)).create! + result = { + status: 'created', + post: { id: post.id } } + result[:field_warnings] = preflight[:field_warnings] if preflight[:field_warnings].present? + result[:base_warnings] = preflight[:base_warnings] if preflight[:base_warnings].present? + result + rescue PostCreatePreflight::ValidationFailed => e + { + status: 'failed', + recoverable: true, + errors: e.fields, + base_errors: e.base_errors } + rescue ActiveRecord::RecordInvalid => e + existing_post = existing_post_for_race(attributes, e.record) + if existing_post.present? + return { + status: 'skipped', + existing_post: existing_post } + end + + { + status: 'failed', + recoverable: true, + errors: e.record.errors.to_hash, + base_errors: e.record.errors[:base] } + rescue ActiveRecord::RecordNotUnique => e + raise unless e.message.include?('index_posts_on_url') + + existing_post = existing_post_for_race(attributes) + raise if existing_post.nil? + + { + status: 'skipped', + existing_post: existing_post } + rescue Tag::NicoTagNormalisationError + { + status: 'failed', + recoverable: true, + errors: { tags: ['ニコニコ・タグは直接指定できません.'] }, + base_errors: [] } + rescue Tag::DeprecatedTagNormalisationError + { + status: 'failed', + recoverable: true, + errors: { tags: ['廃止済みタグは付与できません.'] }, + base_errors: [] } + rescue PostCreator::VideoMsParseError + { + status: 'failed', + recoverable: true, + errors: { video_ms: ['動画時間の記法が不正です.'] }, + base_errors: [] } + rescue Post::RemoteThumbnailFetchFailed + { + status: 'failed', + recoverable: true, + errors: { thumbnail_base: ['サムネイル画像の取得に失敗しました.'] }, + base_errors: [] } + rescue ArgumentError => e + { + status: 'failed', + recoverable: true, + errors: { base: [e.message] }, + base_errors: [] } + rescue StandardError => e + Rails.logger.error( + "post_bulk_creator_failure #{ { error: e.class.name, + message: e.message }.to_json }") + { + status: 'failed', + recoverable: false, + errors: { base: ['登録中にエラーが発生しました.'] }, + base_errors: [] } + end + + def normalised_attributes attributes, preflight, index + { + url: preflight[:url], + title: preflight[:title], + thumbnail_base: preflight[:thumbnail_base], + thumbnail: thumbnail_for(index, attributes), + tags: preflight[:tags], + parent_post_ids: preflight[:parent_post_ids], + original_created_from: preflight[:original_created_from], + original_created_before: preflight[:original_created_before], + duration: preflight[:duration], + video_ms: preflight[:video_ms] } + end + + def thumbnail_for index, attributes + return nil if attributes['thumbnail_base'].present? || attributes[:thumbnail_base].present? + + @thumbnails[index] + end + + def existing_post_for_race attributes, record = nil + return nil if record.present? && !(record.errors.of_kind?(:url, :taken)) + + normal_url = PostUrlNormaliser.normalise(attributes['url'] || attributes[:url]) + return nil if normal_url.blank? + + compact_existing_post(Post.with_attached_thumbnail.find_by(url: normal_url)) + end + + def compact_existing_post post + return nil if post.nil? + + { + id: post.id, + title: post.title, + url: post.url, + thumbnail_url: + if post.thumbnail.attached? + Rails.application.routes.url_helpers.rails_storage_proxy_url( + post.thumbnail, + only_path: false) + end } + end +end diff --git a/backend/app/services/post_create_preflight.rb b/backend/app/services/post_create_preflight.rb new file mode 100644 index 0000000..1ea24f2 --- /dev/null +++ b/backend/app/services/post_create_preflight.rb @@ -0,0 +1,104 @@ +class PostCreatePreflight + class ValidationFailed < StandardError + attr_reader :fields, :base_errors + + def initialize fields: { }, base_errors: [] + super('入力内容を確認してください.') + @fields = fields + @base_errors = base_errors + end + end + + def initialize attributes:, thumbnail: nil + @attributes = attributes.symbolize_keys + @thumbnail = thumbnail + end + + def run + preview = PostImportPreviewer.new.preview_rows( + rows: [preview_row], + fetch_metadata: false).first + + validate_thumbnail_upload! + + if preview[:validation_errors].present? + raise ValidationFailed.new(fields: preview[:validation_errors]) + end + + { + url: preview[:url], + title: preview[:attributes]['title'], + thumbnail_base: preview[:attributes]['thumbnail_base'], + tags: preview[:attributes]['tags'], + parent_post_ids: preview[:attributes]['parent_post_ids'], + original_created_from: preview[:attributes]['original_created_from'], + original_created_before: preview[:attributes]['original_created_before'], + duration: preview[:attributes]['duration'], + video_ms: preview[:attributes]['video_ms'], + field_warnings: preview[:field_warnings], + base_warnings: preview[:base_warnings], + existing_post: existing_post_compact(preview[:existing_post_id]) } + end + + private + + def preview_row + { + source_row: 1, + url: @attributes[:url].to_s, + attributes: { + 'title' => @attributes[:title].to_s, + 'thumbnail_base' => @attributes[:thumbnail_base].to_s, + 'original_created_from' => @attributes[:original_created_from].to_s, + 'original_created_before' => @attributes[:original_created_before].to_s, + 'duration' => @attributes[:duration].to_s, + 'video_ms' => @attributes[:video_ms], + 'tags' => @attributes[:tags].to_s, + 'parent_post_ids' => parent_post_ids_text }, + provenance: { + 'url' => 'manual', + 'title' => 'manual', + 'thumbnail_base' => 'manual', + 'original_created_from' => 'manual', + 'original_created_before' => 'manual', + 'duration' => 'manual', + 'video_ms' => 'manual', + 'tags' => 'manual', + 'parent_post_ids' => 'manual' }, + tag_sources: { + 'automatic' => '', + 'manual' => @attributes[:tags].to_s } } + end + + def parent_post_ids_text + Array(@attributes[:parent_post_ids]).flat_map { _1.to_s.split }.join(' ') + end + + def validate_thumbnail_upload! + return if @attributes[:thumbnail_base].present? + return if @thumbnail.blank? + + attachment = Post.resized_thumbnail_attachment(@thumbnail) + attachment[:io].close if attachment[:io].respond_to?(:close) + rescue MiniMagick::Error + raise ValidationFailed.new(fields: { thumbnail: ['サムネイル画像の変換に失敗しました.'] }) + end + + def existing_post_compact post_id + return nil if post_id.blank? + + post = Post.with_attached_thumbnail.find_by(id: post_id) + return nil if post.nil? + + { + id: post.id, + title: post.title, + url: post.url, + thumbnail_url: + if post.thumbnail.attached? + Rails.application.routes.url_helpers.rails_storage_proxy_url( + post.thumbnail, + only_path: false) + end } + end +end diff --git a/backend/app/services/post_creator.rb b/backend/app/services/post_creator.rb index 6d96be8..5da1a07 100644 --- a/backend/app/services/post_creator.rb +++ b/backend/app/services/post_creator.rb @@ -10,16 +10,17 @@ class PostCreator end def create! + thumbnail_attachment = prepare_thumbnail_attachment post = Post.new(title: @attributes[:title].presence, url: @attributes[:url], thumbnail_base: @attributes[:thumbnail_base].presence, uploaded_user: @actor, original_created_from: @attributes[:original_created_from].presence, original_created_before: @attributes[:original_created_before].presence) - attach_thumbnail!(post) ApplicationRecord.transaction do post.save! + post.thumbnail.attach(thumbnail_attachment) if thumbnail_attachment.present? Tag.normalise_tags!(tag_names, deny_deprecated: true, with_sections: true) => { tags:, sections: } TagVersioning.record_tag_snapshots!(tags, created_by_user: @actor) @@ -36,19 +37,10 @@ class PostCreator private - def attach_thumbnail! post - thumbnail = @attributes[:thumbnail] - if thumbnail.present? - post.thumbnail.attach(Post.resized_thumbnail_attachment(thumbnail)) - return - end - - thumbnail_base = post.thumbnail_base - return if thumbnail_base.blank? - - post.attach_thumbnail_from_url!(thumbnail_base) - rescue Post::RemoteThumbnailFetchFailed => e - @field_warnings[:thumbnail_base] = [e.message] + def prepare_thumbnail_attachment + PostThumbnailAttachmentBuilder.build( + thumbnail: @attributes[:thumbnail], + thumbnail_base: @attributes[:thumbnail_base].presence) end def tag_names = @attributes[:tags].to_s.split diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index 7837896..f0ccc20 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -7,6 +7,7 @@ class PostImportPreviewer 'thumbnail_base', 'original_created_from', 'original_created_before', + 'video_ms', 'duration', 'tags', 'parent_post_ids'].freeze @@ -162,6 +163,8 @@ class PostImportPreviewer normalised = if field == 'duration' normalise_duration_attribute(value) + elsif field == 'video_ms' + value.nil? ? '' : value.to_s else value.to_s end @@ -194,7 +197,7 @@ class PostImportPreviewer def clear_automatic_values! attributes, provenance, tag_sources ['title', 'thumbnail_base', 'original_created_from', - 'original_created_before', 'duration'].each do |field| + 'original_created_before', 'video_ms', 'duration'].each do |field| attributes[field] = '' if provenance[field] == 'automatic' end tag_sources['automatic'] = '' @@ -466,7 +469,7 @@ class PostImportPreviewer thumbnail_base: attributes['thumbnail_base'].presence, original_created_from: attributes['original_created_from'].presence, original_created_before: attributes['original_created_before'].presence, - video_ms: parse_duration(attributes['duration'], errors)) + video_ms: parse_video_ms(attributes, errors)) post.valid? post.errors.each do |error| next if error.attribute == :url && error.type == :taken @@ -475,7 +478,19 @@ class PostImportPreviewer end end - def parse_duration value, errors + def parse_video_ms attributes, errors + video_ms = attributes['video_ms'] + if video_ms.present? + value = Integer(video_ms, exception: false) + if value&.positive? + return value + end + + errors[:video_ms] = ['動画時間の記法が不正です.'] + return nil + end + + value = attributes['duration'] return nil if value.blank? Tag.time_to_ms!(value.to_s, tag_name: '動画時間') diff --git a/backend/app/services/post_import_row_normaliser.rb b/backend/app/services/post_import_row_normaliser.rb deleted file mode 100644 index 7362d3b..0000000 --- a/backend/app/services/post_import_row_normaliser.rb +++ /dev/null @@ -1,190 +0,0 @@ -class PostImportRowNormaliser - ORIGINS = ['automatic', 'manual'].freeze - STRING_FIELDS = [ - 'title', - 'thumbnail_base', - 'original_created_from', - 'original_created_before', - 'duration', - 'tags', - 'parent_post_ids'].freeze - FLEXIBLE_FIELDS = ['video_ms'].freeze - ATTRIBUTE_FIELDS = (STRING_FIELDS + FLEXIBLE_FIELDS).freeze - - def self.normalise! rows, allow_warning_fields: false - raise ArgumentError, '取込行の形式が不正です.' unless rows.is_a?(Array) - raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportUrlListParser::MAX_ROWS - - normalised_rows = rows.map { normalise_row!(_1, allow_warning_fields:) } - source_rows = normalised_rows.map { _1['source_row'] } - raise ArgumentError, '元行番号が重複しています.' if source_rows.uniq.length != source_rows.length - if normalised_rows.sum { row_bytesize(_1) } > PostImportUrlListParser::MAX_BYTES - raise ArgumentError, '取込データが大きすぎます.' - end - - normalised_rows - end - - def self.normalise_row! row, allow_warning_fields: - unless row.is_a?(Hash) || row.is_a?(ActionController::Parameters) - raise ArgumentError, '取込行の形式が不正です.' - end - - parameters = - row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row) - permitted = parameters.permit(*permitted_keys(allow_warning_fields)) - normalised = permitted.to_h.deep_transform_keys { _1.to_s.underscore } - - normalised['source_row'] = normalise_source_row!(normalised['source_row']) - normalise_url!(normalised['url']) - normalise_metadata_url!(normalised['metadata_url']) - normalise_attributes!(normalised.fetch('attributes', { })) - normalise_provenance!(normalised.fetch('provenance', { })) - normalise_tag_sources!(normalised['tag_sources']) - normalise_warning_values!(normalised, allow_warning_fields:) - if row_bytesize(normalised) > PostImportUrlListParser::MAX_BYTES - raise ArgumentError, '取込行が大きすぎます.' - end - - normalised - end - - def self.permitted_keys allow_warning_fields - keys = [ - :source_row, - :sourceRow, - :url, - :metadata_url, - :metadataUrl, - { attributes: {} }, - { provenance: {} }, - { tag_sources: {} }, - { tagSources: {} }] - return keys unless allow_warning_fields - - keys + [ - { field_warnings: {} }, - { fieldWarnings: {} }, - { base_warnings: [] }, - { baseWarnings: [] }] - end - private_class_method :permitted_keys - - def self.normalise_source_row! value - source_row = Integer(value, exception: false) - raise ArgumentError, '元行番号が不正です.' if source_row.nil? || source_row <= 0 - - source_row - end - private_class_method :normalise_source_row! - - def self.normalise_url! value - unless value.is_a?(String) - raise ArgumentError, 'URL の形式が不正です.' - end - if value.bytesize > PostImportUrlListParser::MAX_URL_BYTES - raise ArgumentError, 'URL が長すぎます.' - end - end - private_class_method :normalise_url! - - def self.normalise_metadata_url! value - raise ArgumentError, 'metadata_url の形式が不正です.' unless value.nil? || value.is_a?(String) - if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES - raise ArgumentError, 'metadata_url が長すぎます.' - end - end - private_class_method :normalise_metadata_url! - - def self.normalise_attributes! attributes - raise ArgumentError, 'attributes の形式が不正です.' unless attributes.is_a?(Hash) - raise ArgumentError, '取込項目が不正です.' unless (attributes.keys - ATTRIBUTE_FIELDS).empty? - - attributes.each do |key, value| - case key - when *STRING_FIELDS - raise ArgumentError, '取込項目の型が不正です.' unless value.nil? || value.is_a?(String) - when *FLEXIBLE_FIELDS - unless value.nil? || value.is_a?(String) || value.is_a?(Numeric) - raise ArgumentError, '取込項目の型が不正です.' - end - end - if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES - raise ArgumentError, '取込項目が大きすぎます.' - end - end - end - private_class_method :normalise_attributes! - - def self.normalise_provenance! provenance - unless provenance.is_a?(Hash) - raise ArgumentError, 'provenance の形式が不正です.' - end - - allowed = ATTRIBUTE_FIELDS + ['url'] - unless (provenance.keys - allowed).empty? - raise ArgumentError, '値の由来が不正です.' - end - unless provenance.values.all? { ORIGINS.include?(_1) } - raise ArgumentError, '値の由来が不正です.' - end - end - private_class_method :normalise_provenance! - - def self.normalise_tag_sources! tag_sources - return if tag_sources.nil? - - unless tag_sources.is_a?(Hash) - raise ArgumentError, 'タグ由来の形式が不正です.' - end - unless (tag_sources.keys - ORIGINS).empty? - raise ArgumentError, 'タグ由来の形式が不正です.' - end - unless tag_sources.values.all? { _1.is_a?(String) } - raise ArgumentError, 'タグ由来の形式が不正です.' - end - if tag_sources.values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES } - raise ArgumentError, 'タグ由来が大きすぎます.' - end - if tag_sources.values.sum(&:bytesize) > PostImportUrlListParser::MAX_URL_BYTES - raise ArgumentError, 'タグ由来が大きすぎます.' - end - end - private_class_method :normalise_tag_sources! - - def self.normalise_warning_values! normalised, allow_warning_fields: - return unless allow_warning_fields - - field_warnings = normalised['field_warnings'] - unless field_warnings.nil? || field_warnings.is_a?(Hash) - raise ArgumentError, '警告の形式が不正です.' - end - field_warnings&.each do |key, values| - unless ATTRIBUTE_FIELDS.include?(key) || key == 'url' - raise ArgumentError, '警告の形式が不正です.' - end - unless values.is_a?(Array) && values.all? { _1.is_a?(String) } - raise ArgumentError, '警告の形式が不正です.' - end - if values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES } - raise ArgumentError, '警告が大きすぎます.' - end - end - - base_warnings = normalised['base_warnings'] - return if base_warnings.nil? - - unless base_warnings.is_a?(Array) && base_warnings.all? { _1.is_a?(String) } - raise ArgumentError, '警告の形式が不正です.' - end - if base_warnings.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES } - raise ArgumentError, '警告が大きすぎます.' - end - end - private_class_method :normalise_warning_values! - - def self.row_bytesize row - row.to_json.bytesize - end - private_class_method :row_bytesize -end diff --git a/backend/app/services/post_import_runner.rb b/backend/app/services/post_import_runner.rb deleted file mode 100644 index 1f53c54..0000000 --- a/backend/app/services/post_import_runner.rb +++ /dev/null @@ -1,109 +0,0 @@ -class PostImportRunner - def initialize actor:, rows: - @actor = actor - @rows = rows - end - - def run - normalised_rows = PostImportRowNormaliser.normalise!(@rows) - previews = PostImportPreviewer.new.preview_rows(rows: normalised_rows, - fetch_metadata: false) - preview_map = previews.index_by { _1[:source_row] } - - results = normalised_rows.map do |row| - run_row(row, preview_map.fetch(row['source_row'])) - end - - { created: results.count { _1[:status] == 'created' }, - skipped: results.count { _1[:status] == 'skipped' }, - failed: results.count { _1[:status] == 'failed' }, - rows: results } - end - - private - - def run_row row, preview - attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore } - return { source_row: row['source_row'], - status: 'failed', - errors: preview[:validation_errors], - recoverable: true } if preview[:validation_errors].present? - if preview[:skip_reason] == 'existing' - return { source_row: row['source_row'], - status: 'skipped', - existing_post_id: preview[:existing_post_id] } - end - - attributes['tags'] = preview[:attributes]['tags'] - attributes['url'] = row['url'] - creator = PostCreator.new(actor: @actor, attributes:) - post = creator.create! - result = { source_row: row['source_row'], status: 'created', post: PostRepr.base(post) } - if creator.field_warnings.present? - result[:field_warnings] = creator.field_warnings - end - result - rescue ActiveRecord::RecordInvalid => e - existing_post = existing_post_for_race(row, e.record) - if existing_post - return { source_row: row['source_row'], - status: 'skipped', - existing_post_id: existing_post.id } - end - - { source_row: row['source_row'], - status: 'failed', - errors: e.record.errors.to_hash, - recoverable: true } - rescue ActiveRecord::RecordNotUnique => e - raise unless url_record_not_unique?(e) - - existing_post = existing_post_for_race(row) - raise unless existing_post - - { source_row: row['source_row'], - status: 'skipped', - existing_post_id: existing_post.id } - rescue Tag::NicoTagNormalisationError - { source_row: row['source_row'], - status: 'failed', - errors: { tags: ['ニコニコ・タグは直接指定できません.'] }, - recoverable: true } - rescue Tag::DeprecatedTagNormalisationError - { source_row: row['source_row'], - status: 'failed', - errors: { tags: ['廃止済みタグは付与できません.'] }, - recoverable: true } - rescue PostCreator::VideoMsParseError - { source_row: row['source_row'], - status: 'failed', - errors: { duration: ['動画時間の記法が不正です.'] }, - recoverable: true } - rescue ArgumentError - { source_row: row['source_row'], - status: 'failed', - errors: { base: ['入力値が不正です.'] }, - recoverable: true } - rescue StandardError => e - Rails.logger.error("post_import_runner_failure #{ { error: e.class.name, - message: e.message }.to_json }") - { source_row: row['source_row'], - status: 'failed', - errors: { base: ['登録中にエラーが発生しました.'] } } - end - - def existing_post_for_race row, record = nil - if record && !(record.errors.of_kind?(:url, :taken)) - return nil - end - - normal_url = PostUrlNormaliser.normalise(row['url']) - return nil if normal_url.blank? - - Post.find_by(url: normal_url) - end - - def url_record_not_unique? error - error.message.include?('index_posts_on_url') - end -end diff --git a/backend/app/services/post_thumbnail_attachment_builder.rb b/backend/app/services/post_thumbnail_attachment_builder.rb new file mode 100644 index 0000000..ad71c91 --- /dev/null +++ b/backend/app/services/post_thumbnail_attachment_builder.rb @@ -0,0 +1,11 @@ +class PostThumbnailAttachmentBuilder + def self.build thumbnail:, thumbnail_base: + if thumbnail_base.present? + return Post.remote_thumbnail_attachment(thumbnail_base) + end + + return nil if thumbnail.blank? + + Post.resized_thumbnail_attachment(thumbnail) + end +end diff --git a/backend/config/routes.rb b/backend/config/routes.rb index cd87bec..d347ecb 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -31,13 +31,6 @@ Rails.application.routes.draw do scope :preview, controller: :preview do get :title get :thumbnail - get :image - end - - scope 'posts/import', controller: :post_imports do - post :preview - post :validate - post '', action: :create end resources :wiki_pages, path: 'wiki', only: [:index, :show, :create, :update] do @@ -59,6 +52,8 @@ Rails.application.routes.draw do resources :posts, only: [:index, :show, :create, :update] do collection do + get :preview + post :bulk get :random get :changes get :versions, to: 'post_versions#index' diff --git a/frontend/src/components/PostOriginalCreatedTimeField.tsx b/frontend/src/components/PostOriginalCreatedTimeField.tsx index d0cac09..51e9f61 100644 --- a/frontend/src/components/PostOriginalCreatedTimeField.tsx +++ b/frontend/src/components/PostOriginalCreatedTimeField.tsx @@ -24,10 +24,10 @@ const PostOriginalCreatedTimeField: FC = ( {({ describedBy, invalid }) => ( <> -
-
+
+
= (
-
-
+
+
= ( - { url, alt = 'サムネール', className = 'h-16 w-16' }, + { url, + alt = 'サムネール', + className = 'h-16 w-16', + referrerPolicy }, ) => { const [failed, setFailed] = useState (false) @@ -32,6 +36,7 @@ const PostThumbnailPreview: FC = ( {alt} setFailed (true)}/>) } diff --git a/frontend/src/components/posts/import/PostImportThumbnailPreview.tsx b/frontend/src/components/posts/import/PostImportThumbnailPreview.tsx index 04caff7..3077db6 100644 --- a/frontend/src/components/posts/import/PostImportThumbnailPreview.tsx +++ b/frontend/src/components/posts/import/PostImportThumbnailPreview.tsx @@ -1,7 +1,4 @@ -import { useEffect, useRef, useState } from 'react' - import PostThumbnailPreview from '@/components/posts/PostThumbnailPreview' -import { apiGet } from '@/lib/api' import type { FC } from 'react' @@ -15,61 +12,11 @@ const PostImportThumbnailPreview: FC = ( { url, alt = 'サムネール', className = 'h-16 w-16' }, -) => { - const [previewUrl, setPreviewUrl] = useState ('') - const previewUrlRef = useRef ('') - - useEffect (() => { - if (previewUrlRef.current) - { - URL.revokeObjectURL (previewUrlRef.current) - previewUrlRef.current = '' - } - setPreviewUrl ('') - - if (!(url)) - return - - const controller = new AbortController () - - const loadPreview = async () => { - try - { - const blob = await apiGet ('/preview/image', { - params: { url }, - signal: controller.signal, - responseType: 'blob' }) - if (controller.signal.aborted) - return - - const nextPreviewUrl = URL.createObjectURL (blob) - previewUrlRef.current = nextPreviewUrl - setPreviewUrl (nextPreviewUrl) - } - catch - { - if (!(controller.signal.aborted)) - setPreviewUrl ('') - } - } - - void loadPreview () - - return () => { - controller.abort () - if (previewUrlRef.current) - { - URL.revokeObjectURL (previewUrlRef.current) - previewUrlRef.current = '' - } - } - }, [url]) - - return ( - ) -} +) => ( + ) export default PostImportThumbnailPreview diff --git a/frontend/src/lib/postImportStorage.ts b/frontend/src/lib/postImportStorage.ts index 00e3c89..9e9ae17 100644 --- a/frontend/src/lib/postImportStorage.ts +++ b/frontend/src/lib/postImportStorage.ts @@ -6,11 +6,12 @@ import type { PostImportOrigin, PostImportSkipReason, StorageErrorHandler } from '@/lib/postImportTypes' -const SESSION_VERSION = 2 +const SESSION_VERSION = 3 const SESSION_PREFIX = 'post-import-session:' -const CURRENT_SESSION_KEY = `${ SESSION_PREFIX }current` const SOURCE_DRAFT_KEY = 'post-import-source-draft' const SESSION_MAX_AGE_MS = 24 * 60 * 60 * 1000 +const SESSION_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i const ATTRIBUTE_KEYS = [ 'title', 'thumbnailBase', @@ -29,7 +30,21 @@ const isPlainObject = (value: unknown): value is Record => typeof value === 'object' && value != null && !(Array.isArray (value)) -const sessionKey = (): string => CURRENT_SESSION_KEY +export const validatePostImportSessionId = (value: unknown): string | null => { + if (typeof value !== 'string') + return null + if (value.length > 36 || !(SESSION_ID_PATTERN.test (value))) + return null + + return value.toLowerCase () +} + + +export const generatePostImportSessionId = (): string => + crypto.randomUUID () + + +const sessionKey = (sessionId: string): string => `${ SESSION_PREFIX }${ sessionId }` const readStorage = ( @@ -283,9 +298,9 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { } -const isExpiredSession = (savedAt: string): boolean => { - const value = Date.parse (savedAt) - return Number.isNaN (value) || Date.now () - value > SESSION_MAX_AGE_MS +const isExpiredSession = (expiresAt: string): boolean => { + const value = Date.parse (expiresAt) + return Number.isNaN (value) || value <= Date.now () } @@ -303,20 +318,25 @@ export const cleanupExpiredPostImportSessions = ( if (key == null || !(key.startsWith (SESSION_PREFIX))) continue - const raw = sessionStorage.getItem (key) - if (raw == null) - continue - - if (key !== CURRENT_SESSION_KEY) + const sessionId = validatePostImportSessionId (key.slice (SESSION_PREFIX.length)) + if (sessionId == null) { sessionStorage.removeItem (key) --i continue } + + const raw = sessionStorage.getItem (key) + if (raw == null) + continue + try { - const value = JSON.parse (raw) as { savedAt?: string } - if (typeof value.savedAt !== 'string' || isExpiredSession (value.savedAt)) + const value = JSON.parse (raw) as { expiresAt?: string + rows?: unknown[] } + if (typeof value.expiresAt !== 'string' + || isExpiredSession (value.expiresAt) + || !(Array.isArray (value.rows))) { sessionStorage.removeItem (key) --i @@ -370,38 +390,33 @@ export const clearPostImportSourceDraft = ( export const savePostImportSession = ( - sessionOrLegacyId: string | Omit, - sessionOrOnError?: Omit | StorageErrorHandler, + sessionId: string, + session: Omit, onError?: StorageErrorHandler, ): boolean => { - const session = - typeof sessionOrLegacyId === 'string' - ? sessionOrOnError as Omit - : sessionOrLegacyId - const errorHandler = - typeof sessionOrLegacyId === 'string' - ? onError - : sessionOrOnError as StorageErrorHandler | undefined + const validatedId = validatePostImportSessionId (sessionId) + if (validatedId == null) + return false return writeStorage ( - sessionKey (), + sessionKey (validatedId), JSON.stringify ({ ...session, version: SESSION_VERSION, - savedAt: new Date ().toISOString () }), - errorHandler) + expiresAt: new Date (Date.now () + SESSION_MAX_AGE_MS).toISOString () }), + onError) } export const loadPostImportSession = ( - legacyIdOrOnError?: string | StorageErrorHandler, + sessionId: string, onError?: StorageErrorHandler, ): PostImportSession | null => { - const errorHandler = - typeof legacyIdOrOnError === 'string' - ? onError - : legacyIdOrOnError - const raw = readStorage (sessionKey (), errorHandler) + const validatedId = validatePostImportSessionId (sessionId) + if (validatedId == null) + return null + + const raw = readStorage (sessionKey (validatedId), onError) if (raw == null) return null @@ -410,9 +425,9 @@ export const loadPostImportSession = ( const value = JSON.parse (raw) as Partial if (value.version !== SESSION_VERSION || !(Array.isArray (value.rows))) return null - if (typeof value.savedAt !== 'string' || isExpiredSession (value.savedAt)) + if (typeof value.expiresAt !== 'string' || isExpiredSession (value.expiresAt)) { - removeStorage (sessionKey (), errorHandler) + removeStorage (sessionKey (validatedId), onError) return null } @@ -422,7 +437,7 @@ export const loadPostImportSession = ( return { version: SESSION_VERSION, - savedAt: value.savedAt, + expiresAt: value.expiresAt, source: typeof value.source === 'string' ? value.source : '', rows: rows as PostImportRow[], repairMode: value.repairMode === 'failed' ? 'failed' : 'all' } @@ -435,7 +450,12 @@ export const loadPostImportSession = ( export const clearPostImportSession = ( + sessionId: string, onError?: StorageErrorHandler, ) => { - removeStorage (sessionKey (), onError) + const validatedId = validatePostImportSessionId (sessionId) + if (validatedId == null) + return + + removeStorage (sessionKey (validatedId), onError) } diff --git a/frontend/src/lib/postImportTypes.ts b/frontend/src/lib/postImportTypes.ts index 632a945..e707085 100644 --- a/frontend/src/lib/postImportTypes.ts +++ b/frontend/src/lib/postImportTypes.ts @@ -35,7 +35,9 @@ export type PostImportRow = { resetSnapshot: PostImportResetSnapshot createdPostId?: number importStatus?: PostImportStatus - recoverable?: boolean } + recoverable?: boolean + thumbnailFile?: File + thumbnailObjectUrl?: string } export type PostImportResultRow = | { @@ -61,11 +63,11 @@ export type PostImportResultRow = recoverable?: boolean } export type PostImportSession = { - version: number - savedAt: string - source: string - rows: PostImportRow[] - repairMode: PostImportRepairMode } + version: number + expiresAt: string + source: string + rows: PostImportRow[] + repairMode: PostImportRepairMode } export type PostImportSourceIssue = { sourceRow: number diff --git a/frontend/src/lib/postNewQueryState.ts b/frontend/src/lib/postNewQueryState.ts index c07a3cb..413fd16 100644 --- a/frontend/src/lib/postNewQueryState.ts +++ b/frontend/src/lib/postNewQueryState.ts @@ -1,4 +1,5 @@ import { resultRepairMode } from '@/lib/postImportRows' +import { validatePostImportSessionId } from '@/lib/postImportStorage' import type { PostImportOrigin, @@ -12,6 +13,7 @@ type QueryRowState = { thumbnail_base: string original_created_from: string original_created_before: string + video_ms: string duration: string tags: string parent_post_ids: string @@ -37,6 +39,7 @@ type EncodedRowState = { h?: string f?: string b?: string + x?: string d?: string g?: string p?: string @@ -58,8 +61,8 @@ export type ParsedPostNewState = { shortcut: boolean } export type SerialisedPostNewState = { - path: string - rows: PostImportRow[] } + path: string + complete: boolean } const BASIC_KEYS = [ 'url', @@ -67,6 +70,7 @@ const BASIC_KEYS = [ 'thumbnail_base', 'original_created_from', 'original_created_before', + 'video_ms', 'duration', 'tags', 'parent_post_ids'] as const @@ -77,7 +81,7 @@ const STATE_KEYS = [ 'import_status', 'created_post_id', 'recoverable'] as const -const SINGLE_ROW_KEYS = [...BASIC_KEYS, ...STATE_KEYS] as const +const SINGLE_ROW_KEYS = [...BASIC_KEYS, ...STATE_KEYS, 'session_id'] as const const MAX_REGULAR_URL_LENGTH = 768 const MAX_COMPRESSED_STATE_LENGTH = 767 const COMPRESSED_STATE_PREFIX = 'z1.' @@ -241,6 +245,7 @@ const baseRowState = (row: PostImportRow): QueryRowState => ({ thumbnail_base: String (row.attributes.thumbnailBase ?? ''), original_created_from: String (row.attributes.originalCreatedFrom ?? ''), original_created_before: String (row.attributes.originalCreatedBefore ?? ''), + video_ms: String (row.attributes.videoMs ?? ''), duration: String (row.attributes.duration ?? ''), tags: String (row.attributes.tags ?? ''), parent_post_ids: String (row.attributes.parentPostIds ?? ''), @@ -263,6 +268,7 @@ const buildRow = ( thumbnailBase: manual.has ('thumbnailBase') ? 'manual' : 'automatic', originalCreatedFrom: manual.has ('originalCreatedFrom') ? 'manual' : 'automatic', originalCreatedBefore: manual.has ('originalCreatedBefore') ? 'manual' : 'automatic', + videoMs: 'automatic', duration: 'automatic', tags: manual.has ('tags') ? 'manual' : 'automatic', parentPostIds: manual.has ('parentPostIds') ? 'manual' : 'automatic' } @@ -274,6 +280,7 @@ const buildRow = ( thumbnailBase: state.thumbnail_base, originalCreatedFrom: state.original_created_from, originalCreatedBefore: state.original_created_before, + videoMs: state.video_ms, duration: state.duration, tags: state.tags, parentPostIds: state.parent_post_ids } @@ -356,6 +363,7 @@ const parseEncodedRow = ( ['h', 'thumbnail_base'], ['f', 'original_created_from'], ['b', 'original_created_before'], + ['x', 'video_ms'], ['d', 'duration'], ['g', 'tags'], ['p', 'parent_post_ids']] as const @@ -382,6 +390,7 @@ const parseEncodedRow = ( thumbnail_base: parsedStrings.thumbnail_base, original_created_from: parsedStrings.original_created_from, original_created_before: parsedStrings.original_created_before, + video_ms: parsedStrings.video_ms, duration: parsedStrings.duration, tags: parsedStrings.tags, parent_post_ids: parsedStrings.parent_post_ids, @@ -399,7 +408,8 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { if (url == null || url === '') return null - const hasOnlyUrl = Array.from (params.keys ()).every (key => key === 'url') + const hasOnlyUrl = Array.from (params.keys ()).every ( + key => key === 'url' || key === 'session_id') if (hasOnlyUrl) { const row = buildRow (1, { @@ -408,6 +418,7 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { thumbnail_base: '', original_created_from: '', original_created_before: '', + video_ms: '', duration: '', tags: '', parent_post_ids: '', @@ -444,6 +455,7 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { thumbnail_base: params.get ('thumbnail_base') ?? '', original_created_from: params.get ('original_created_from') ?? '', original_created_before: params.get ('original_created_before') ?? '', + video_ms: params.get ('video_ms') ?? '', duration: params.get ('duration') ?? '', tags: params.get ('tags') ?? '', parent_post_ids: params.get ('parent_post_ids') ?? '', @@ -546,6 +558,7 @@ const regularQuery = (rows: PostImportRow[]): URLSearchParams => { params.set ('thumbnail_base', String (row?.attributes.thumbnailBase ?? '')) params.set ('original_created_from', String (row?.attributes.originalCreatedFrom ?? '')) params.set ('original_created_before', String (row?.attributes.originalCreatedBefore ?? '')) + params.set ('video_ms', String (row?.attributes.videoMs ?? '')) params.set ('duration', String (row?.attributes.duration ?? '')) params.set ('tags', String (row?.attributes.tags ?? '')) params.set ('parent_post_ids', String (row?.attributes.parentPostIds ?? '')) @@ -589,6 +602,8 @@ const encodeRowState = ( encoded.f = base.original_created_from if (base.original_created_before !== '') encoded.b = base.original_created_before + if (base.video_ms !== '') + encoded.x = base.video_ms if (base.duration !== '') encoded.d = base.duration if (base.tags !== '') @@ -626,6 +641,24 @@ export const hasPostNewReviewState = (search: string): boolean => { } +export const parsePostNewSessionId = (search: string): string | null => + validatePostImportSessionId ( + new URLSearchParams (search).get ('session_id')) + + +export const appendPostNewSessionId = ( + path: string, + sessionId: string, +): string => { + const validatedId = validatePostImportSessionId (sessionId) + if (validatedId == null) + return path + + const separator = path.includes ('?') ? '&' : '?' + return `${ path }${ separator }session_id=${ validatedId }` +} + + export const parsePostNewState = async ( search: string, ): Promise => { @@ -661,7 +694,7 @@ const serialiseCompressedRows = async ( if (payload.length <= MAX_COMPRESSED_STATE_LENGTH) return { path: `/posts/new?state=${ state }`, - rows: nextRows } + complete: count === rows.length } } } catch @@ -680,7 +713,7 @@ export const serialisePostNewState = async ( if (regularPath.length < MAX_REGULAR_URL_LENGTH) return { path: regularPath, - rows } + complete: true } return await serialiseCompressedRows (rows) } diff --git a/frontend/src/pages/posts/PostImportResultPage.tsx b/frontend/src/pages/posts/PostImportResultPage.tsx deleted file mode 100644 index 7521565..0000000 --- a/frontend/src/pages/posts/PostImportResultPage.tsx +++ /dev/null @@ -1,544 +0,0 @@ -import { useEffect, useMemo, useRef, useState } from 'react' -import { Helmet } from 'react-helmet-async' -import { useNavigate, useParams } from 'react-router-dom' - -import FieldError from '@/components/common/FieldError' -import FieldWarning from '@/components/common/FieldWarning' -import PageTitle from '@/components/common/PageTitle' -import PrefetchLink from '@/components/PrefetchLink' -import MainArea from '@/components/layout/MainArea' -import PostImportRowForm from '@/components/posts/import/PostImportRowForm' -import PostImportStatusBadge from '@/components/posts/import/PostImportStatusBadge' -import { displayPostImportStatus } from '@/components/posts/import/postImportRowStatus' -import { Button } from '@/components/ui/button' -import { toast } from '@/components/ui/use-toast' -import { SITE_TITLE } from '@/config' -import { apiPost } from '@/lib/api' -import useDialogue from '@/lib/dialogues/useDialogue' -import { canEditContent } from '@/lib/users' -import { clearPostImportSourceDraft, - buildNextEditedRow, - canEditResultRow, - canRetryResultRow, - hasExactSourceRows, - initialisePreviewRows, - loadPostImportSession, - mergeImportResults, - mergeValidatedImportRow, - replaceImportRow, - resultRepairMode, - resultRowMessages, - resultRowWarnings, - resultSummaryCounts, - retryImportRow, - savePostImportSession } from '@/lib/postImportSession' -import { originalCreatedAtString } from '@/lib/utils' -import Forbidden from '@/pages/Forbidden' - -import type { FC } from 'react' - -import type { PostImportRowDraft } from '@/components/posts/import/PostImportRowForm' -import type { PostImportResultRow, PostImportRow } from '@/lib/postImportSession' -import type { PostImportSession } from '@/lib/postImportSession' -import type { User } from '@/types' - -type Props = { user: User | null } - - -const PostImportResultPage: FC = ({ user }) => { - const editable = canEditContent (user) - const dialogue = useDialogue () - const navigate = useNavigate () - const { sessionId } = useParams () - - const [session, setSession] = useState (null) - const [missing, setMissing] = useState (false) - const [loadingRow, setLoadingRow] = useState (null) - const [, setEditingRow] = useState (null) - const sessionRef = useRef (null) - - useEffect (() => { - if (sessionId == null) - return - - const loaded = loadPostImportSession (sessionId, message => - toast ({ title: '取込状態を復元できませんでした', description: message })) - setSession (loaded) - setMissing (loaded == null) - }, [sessionId]) - - useEffect (() => { - if (sessionId == null || session == null) - return - - savePostImportSession (sessionId, session, message => - toast ({ title: '取込状態を保存できませんでした', description: message })) - }, [session, sessionId]) - - useEffect (() => { - sessionRef.current = session - }, [session]) - - const counts = useMemo ( - () => resultSummaryCounts (session?.rows ?? []), - [session]) - const busy = loadingRow != null - - const persistSession = (nextSession: PostImportSession) => { - if (sessionId == null) - return - - savePostImportSession (sessionId, nextSession, message => - toast ({ title: '取込状態を保存できませんでした', description: message })) - } - - const saveDraft = async ( - row: PostImportRow, - { draft, resetRequested }: { - draft: PostImportRowDraft - resetRequested: boolean }, - ): Promise<{ saved: boolean - row: PostImportRow | null }> => { - const currentSession = sessionRef.current - if (currentSession == null) - return { saved: false, row: null } - if (!(canEditResultRow (row))) - return { saved: false, row: null } - - const baseRow = - resetRequested - ? { ...row, - url: row.resetSnapshot.url, - attributes: { ...row.resetSnapshot.attributes }, - provenance: { ...row.resetSnapshot.provenance }, - tagSources: { ...row.resetSnapshot.tagSources }, - fieldWarnings: Object.fromEntries ( - Object.entries (row.resetSnapshot.fieldWarnings) - .map (([key, values]) => [key, [...values]])), - baseWarnings: [...row.resetSnapshot.baseWarnings], - metadataUrl: row.resetSnapshot.metadataUrl } - : row - const urlChanged = draft.url !== baseRow.url - const nextRow = buildNextEditedRow (baseRow, draft, urlChanged) - - const nextRows = currentSession.rows.map (row => - row.sourceRow === baseRow.sourceRow - ? nextRow - : row) - try - { - const validated = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/validate', { - rows: - nextRows - .filter (row => row.importStatus !== 'created') - .map (row => ({ sourceRow: row.sourceRow, - url: row.url, - attributes: row.attributes, - provenance: row.provenance, - tagSources: row.tagSources, - metadataUrl: row.metadataUrl })), - changed_row: urlChanged ? baseRow.sourceRow : -1 }) - const validatedRows = initialisePreviewRows (validated.rows) - const target = validatedRows.find ( - validatedRow => validatedRow.sourceRow === baseRow.sourceRow) - const latestSession = sessionRef.current - if (latestSession == null) - return { saved: false, row: null } - if (target == null) - { - const restoredRows = replaceImportRow (latestSession.rows, row) - const restoredSession = { - ...latestSession, - rows: restoredRows, - repairMode: resultRepairMode (restoredRows) } - sessionRef.current = restoredSession - setSession (restoredSession) - toast ({ title: '行の再検証結果が不完全でした' }) - return { saved: false, row: null } - } - const editedRows = replaceImportRow (latestSession.rows, nextRow) - const rows = mergeValidatedImportRow (editedRows, target) - const nextSession = { - ...latestSession, - rows, - repairMode: resultRepairMode (rows) } - sessionRef.current = nextSession - setSession (nextSession) - persistSession (nextSession) - const mergedTarget = - rows.find (mergedRow => mergedRow.sourceRow === baseRow.sourceRow) - ?? target - if (Object.keys (mergedTarget.validationErrors).length > 0) - return { saved: false, row: mergedTarget } - return { saved: true, row: null } - } - catch - { - toast ({ title: '行の再検証に失敗しました' }) - return { saved: false, row: null } - } - } - - const openEditingDialogue = async (row: PostImportRow) => { - const saveRowDraft = ( - { draft, resetRequested }: { - draft: PostImportRowDraft - resetRequested: boolean }, - ) => - saveDraft (row, { draft, resetRequested }) - - await dialogue.form ({ - title: '投稿を編輯', - description: `投稿 ${ row.sourceRow } の内容を確認し、必要な項目を編輯してください.`, - cancelText: '取消', - size: 'large', - body: controls => ( - ) }) - } - - const editRow = async (row: PostImportRow) => { - setEditingRow (row) - try - { - await openEditingDialogue (row) - } - finally - { - setEditingRow (current => - current?.sourceRow === row.sourceRow - ? null - : current) - } - } - - const retry = async (sourceRow: number) => { - if (session == null || sessionId == null || loadingRow != null) - return - - const initialSession = sessionRef.current - if (initialSession == null) - return - - setLoadingRow (sourceRow) - const originalRow = initialSession.rows.find (row => row.sourceRow === sourceRow) - if (originalRow == null) - { - setLoadingRow (null) - return - } - try - { - const pendingRows = retryImportRow (initialSession.rows, sourceRow) - const pendingSession = { ...initialSession, rows: pendingRows } - sessionRef.current = pendingSession - setSession (pendingSession) - persistSession (pendingSession) - const requestedRows = pendingRows.filter (row => row.importStatus !== 'created') - const validated = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/validate', { - rows: requestedRows.map (row => ({ - sourceRow: row.sourceRow, - url: row.url, - attributes: row.attributes, - provenance: row.provenance, - tagSources: row.tagSources, - metadataUrl: row.metadataUrl })), - changed_row: -1 }) - const validatedRows = initialisePreviewRows (validated.rows) - if (!(hasExactSourceRows (requestedRows.map (row => row.sourceRow), validatedRows))) - { - const latest = sessionRef.current ?? pendingSession - const restoredRows = replaceImportRow (latest.rows, originalRow) - const restoredSession = { - ...latest, - rows: restoredRows, - repairMode: resultRepairMode (restoredRows) } - sessionRef.current = restoredSession - setSession (restoredSession) - persistSession (restoredSession) - toast ({ title: '再検証結果が不完全でした' }) - return - } - const validatedTarget = validatedRows.find ( - row => row.sourceRow === sourceRow) - if (validatedTarget == null) - { - const latest = sessionRef.current ?? pendingSession - const restoredRows = replaceImportRow (latest.rows, originalRow) - const restoredSession = { - ...latest, - rows: restoredRows, - repairMode: resultRepairMode (restoredRows) } - sessionRef.current = restoredSession - setSession (restoredSession) - persistSession (restoredSession) - toast ({ title: '再検証結果が不完全でした' }) - return - } - const latestAfterValidate = sessionRef.current ?? pendingSession - const mergedValidatedRows = mergeValidatedImportRow ( - latestAfterValidate.rows, - validatedTarget) - const nextSession = { - ...latestAfterValidate, - rows: mergedValidatedRows, - repairMode: resultRepairMode (mergedValidatedRows) } - sessionRef.current = nextSession - setSession (nextSession) - persistSession (nextSession) - const target = mergedValidatedRows.find (row => row.sourceRow === sourceRow) - if (target == null) - { - const restoredRows = replaceImportRow (latestAfterValidate.rows, originalRow) - const restoredSession = { - ...latestAfterValidate, - rows: restoredRows, - repairMode: resultRepairMode (restoredRows) } - sessionRef.current = restoredSession - setSession (restoredSession) - persistSession (restoredSession) - toast ({ title: '再検証結果が不完全でした' }) - return - } - if (Object.keys (target.validationErrors ?? { }).length > 0) - { - const saved = savePostImportSession (sessionId, nextSession, message => - toast ({ title: '取込状態を保存できませんでした', description: message })) - if (saved) - void editRow (target) - return - } - - const result = await apiPost<{ - created: number - skipped: number - failed: number - rows: PostImportResultRow[] }> ('/posts/import', { - rows: [{ - sourceRow: target.sourceRow, - url: target.url, - attributes: target.attributes, - provenance: target.provenance, - tagSources: target.tagSources, - metadataUrl: target.metadataUrl }] }) - if (!(hasExactSourceRows ([sourceRow], result.rows))) - { - const latest = sessionRef.current ?? nextSession - const restoredRows = replaceImportRow (latest.rows, originalRow) - const restoredSession = { - ...latest, - rows: restoredRows, - repairMode: resultRepairMode (restoredRows) } - sessionRef.current = restoredSession - setSession (restoredSession) - persistSession (restoredSession) - toast ({ title: '登録結果が不完全でした' }) - return - } - const latestAfterImport = sessionRef.current ?? nextSession - const mergedRows = mergeImportResults (latestAfterImport.rows, result.rows) - const recoverableRows = result.rows.filter (row => - row.status === 'failed' - && row.recoverable - && Object.keys (row.errors ?? { }).length > 0) - const nextRows = mergedRows.map ((row): PostImportRow => { - const recoverable = recoverableRows.find ( - recoverableRow => recoverableRow.sourceRow === row.sourceRow) - if (recoverable == null) - return row - return { - ...row, - importStatus: 'pending', - recoverable: true, - validationErrors: recoverable.errors ?? { }, - importErrors: undefined } - }) - const recoverableTarget = nextRows.find (row => row.sourceRow === sourceRow) - const resultSession = { - ...nextSession, - rows: nextRows, - repairMode: resultRepairMode (nextRows) } - sessionRef.current = resultSession - setSession (resultSession) - persistSession (resultSession) - if (recoverableTarget != null - && Object.keys (recoverableTarget.validationErrors).length > 0) - { - const saved = savePostImportSession (sessionId, resultSession, message => - toast ({ title: '取込状態を保存できませんでした', description: message })) - if (saved) - void editRow (recoverableTarget) - return - } - } - catch - { - const latest = sessionRef.current - if (latest != null) - { - const restoredRows = replaceImportRow (latest.rows, originalRow) - const restoredSession = { - ...latest, - rows: restoredRows, - repairMode: resultRepairMode (restoredRows) } - sessionRef.current = restoredSession - setSession (restoredSession) - persistSession (restoredSession) - } - toast ({ title: '再試行に失敗しました' }) - } - finally - { - setLoadingRow (null) - } - } - - const openRepair = (sourceRow: number) => { - const currentSession = sessionRef.current - if (currentSession == null || sessionId == null || loadingRow != null) - return - - const nextSession = { ...currentSession, repairMode: 'failed' as const } - sessionRef.current = nextSession - setSession (nextSession) - const saved = savePostImportSession (sessionId, nextSession, message => - toast ({ title: '取込状態を保存できませんでした', description: message })) - if (!(saved)) - return - const row = nextSession.rows.find (rowValue => rowValue.sourceRow === sourceRow) - if (row != null) - void editRow (row) - } - - if (!(editable)) - return - - if (missing || sessionId == null || session == null) - { - return ( - -
- 投稿インポート結果 - - -
-
) - } - - return ( - - - {`投稿インポート結果 | ${ SITE_TITLE }`} - - -
- 登録結果 - -
- 登録成功 {counts.created}件 スキップ {counts.skipped}件 失敗 {counts.failed}件 -
- -
- {session.rows.map (row => { - const displayStatus = displayPostImportStatus (row) - const canEdit = canEditResultRow (row) - const canRetry = canRetryResultRow (row) - return ( -
-
-
-
- 行 {row.sourceRow} - {displayStatus != null && } -
-
- {String (row.attributes.title ?? '') || row.url} -
-
- {row.url} -
-
- {String (row.attributes.tags ?? '') || 'タグなし'} -
-
- {originalCreatedAtString ( - row.attributes.originalCreatedFrom?.toString () ?? null, - row.attributes.originalCreatedBefore?.toString () ?? null)} - {row.attributes.duration ? ` / ${ row.attributes.duration }` : ''} -
- - -
- -
- {(row.createdPostId != null || row.existingPostId != null) && ( - )} - {canEdit && ( - )} - {canRetry && ( - )} -
-
-
)})} -
- -
- - -
-
-
) -} - -export default PostImportResultPage diff --git a/frontend/src/pages/posts/PostImportReviewPage.tsx b/frontend/src/pages/posts/PostImportReviewPage.tsx index ef24c8a..b2616ad 100644 --- a/frontend/src/pages/posts/PostImportReviewPage.tsx +++ b/frontend/src/pages/posts/PostImportReviewPage.tsx @@ -13,33 +13,35 @@ import PostImportRowSummary from '@/components/posts/import/PostImportRowSummary import { Button } from '@/components/ui/button' import { toast } from '@/components/ui/use-toast' import { SITE_TITLE } from '@/config' -import { apiPost } from '@/lib/api' +import { apiGet, apiPost, isApiError } from '@/lib/api' import useDialogue from '@/lib/dialogues/useDialogue' import { fetchPost } from '@/lib/posts' import { + appendPostNewSessionId, parsePostNewState, + parsePostNewSessionId, serialisePostNewState, } from '@/lib/postNewQueryState' import { buildNextEditedRow, canEditReviewRow, canRetryResultRow, + clearPostImportSession, clearPostImportSourceDraft, - hasExactSourceRows, + generatePostImportSessionId, initialisePreviewRows, isCompletedReviewRow, isExistingSkipRow, isNonRecoverableFailedRow, mergeImportResults, - mergeValidatedImportRow, - mergeValidatedImportRows, processableImportRows, replaceImportRow, resultRepairMode, resultRowMessages, reviewSummaryCounts, retryImportRow, - validatableImportRows, + savePostImportSession, + loadPostImportSession, } from '@/lib/postImportSession' import { postsKeys } from '@/lib/queryKeys' import { useClientBehaviourSettings } from '@/lib/useClientBehaviourSettings' @@ -58,6 +60,29 @@ import type { Post, User } from '@/types' type Props = { user: User | null } +type PreviewResponse = { + url: string + title?: string + thumbnailBase?: string + originalCreatedFrom?: string + originalCreatedBefore?: string + duration?: string + videoMs?: number + tags?: string + fieldWarnings?: Record + baseWarnings?: string[] + existingPost?: { id: number } | null } + +type BulkApiRow = { + status: 'created' | 'skipped' | 'failed' + post?: { id: number } + existingPost?: { id: number } | null + fieldWarnings?: Record + baseWarnings?: string[] + errors?: Record + baseErrors?: string[] + recoverable?: boolean } + type ExistingSkippedRowsProps = { open: boolean rows: PostImportRow[] } @@ -70,13 +95,122 @@ const isRepairRow = (row: PostImportRow): boolean => const buildSession = (rows: PostImportRow[]): PostImportSession => ({ - version: 2, - savedAt: new Date ().toISOString (), + version: 3, + expiresAt: '', source: rows.map (row => row.url).join ('\n'), rows, repairMode: resultRepairMode (rows) }) +const buildDryRunFormData = (row: PostImportRow): FormData => { + const formData = new FormData () + formData.append ('url', row.url) + formData.append ('title', String (row.attributes.title ?? '')) + formData.append ('thumbnail_base', String (row.attributes.thumbnailBase ?? '')) + formData.append ('tags', String (row.attributes.tags ?? '')) + formData.append ('parent_post_ids', String (row.attributes.parentPostIds ?? '')) + formData.append ( + 'original_created_from', + String (row.attributes.originalCreatedFrom ?? '')) + formData.append ( + 'original_created_before', + String (row.attributes.originalCreatedBefore ?? '')) + formData.append ('video_ms', String (row.attributes.videoMs ?? '')) + formData.append ('duration', String (row.attributes.duration ?? '')) + return formData +} + + +const mergeDryRunRow = ( + currentRow: PostImportRow, + result: PreviewResponse, +): PostImportRow => + initialisePreviewRows ([{ + ...currentRow, + url: result.url, + attributes: { + ...currentRow.attributes, + title: result.title ?? '', + thumbnailBase: result.thumbnailBase ?? '', + originalCreatedFrom: result.originalCreatedFrom ?? '', + originalCreatedBefore: result.originalCreatedBefore ?? '', + duration: result.duration ?? '', + videoMs: result.videoMs ?? '', + tags: result.tags ?? '', + parentPostIds: String (currentRow.attributes.parentPostIds ?? '') }, + fieldWarnings: result.fieldWarnings ?? { }, + baseWarnings: result.baseWarnings ?? [], + validationErrors: { }, + importErrors: undefined, + status: + Object.keys (result.fieldWarnings ?? { }).length > 0 + || (result.baseWarnings?.length ?? 0) > 0 + ? 'warning' + : 'ready', + skipReason: result.existingPost?.id != null ? 'existing' : undefined, + existingPostId: result.existingPost?.id, + importStatus: + currentRow.importStatus === 'created' + ? 'created' + : 'pending', + recoverable: undefined }])[0] + + +const indexedBulkResults = ( + rows: PostImportRow[], + results: BulkApiRow[], +): PostImportResultRow[] => + rows.map ((row, index) => { + const result = results[index] + if (result?.status === 'created' && result.post != null) + { + return { + sourceRow: row.sourceRow, + status: 'created', + post: result.post, + fieldWarnings: result.fieldWarnings, + baseWarnings: result.baseWarnings, + errors: result.errors } + } + if (result?.status === 'skipped' && result.existingPost != null) + { + return { + sourceRow: row.sourceRow, + status: 'skipped', + existingPostId: result.existingPost.id, + fieldWarnings: result.fieldWarnings, + baseWarnings: result.baseWarnings, + errors: result.errors } + } + return { + sourceRow: row.sourceRow, + status: 'failed', + fieldWarnings: result?.fieldWarnings, + baseWarnings: result?.baseWarnings, + errors: result?.errors, + recoverable: result?.recoverable } + }) + + +const buildBulkFormData = (rows: PostImportRow[]): FormData => { + const formData = new FormData () + formData.append ( + 'posts', + JSON.stringify ( + rows.map (row => ({ + url: row.url, + title: row.attributes.title ?? '', + thumbnail_base: row.attributes.thumbnailBase ?? '', + tags: row.attributes.tags ?? '', + parent_post_ids: row.attributes.parentPostIds ?? '', + original_created_from: row.attributes.originalCreatedFrom ?? '', + original_created_before: row.attributes.originalCreatedBefore ?? '', + video_ms: row.attributes.videoMs ?? '', + duration: row.attributes.duration ?? '' })))) + return formData +} + + const ExistingSkippedRows: FC = ({ open, rows }) => { const existingPostIds = useMemo ( () => @@ -157,12 +291,16 @@ const PostImportReviewPage: FC = ({ user }) => { const [editingRow, setEditingRow] = useState (null) const [showExistingRows, setShowExistingRows] = useState (false) const sessionRef = useRef (null) + const sessionIdRef = useRef (null) const persistedSearchRef = useRef (null) const persistSequenceRef = useRef (0) const persistSession = async ( nextSession: PostImportSession, ): Promise => { + const sessionId = + sessionIdRef.current ?? generatePostImportSessionId () + sessionIdRef.current = sessionId const sequence = ++persistSequenceRef.current const serialised = await serialisePostNewState (nextSession.rows) if (serialised == null) @@ -170,16 +308,25 @@ const PostImportReviewPage: FC = ({ user }) => { if (persistSequenceRef.current !== sequence) return sessionRef.current - const persistedSession = buildSession (serialised.rows) - persistedSearchRef.current = - (new URL (serialised.path, window.location.origin)).search + savePostImportSession (sessionId, nextSession) + const loadedSession = loadPostImportSession (sessionId) + if (!(serialised.complete) + && loadedSession?.rows.length !== nextSession.rows.length) + return null + + const persistedSession = buildSession (nextSession.rows) + const path = appendPostNewSessionId (serialised.path, sessionId) + persistedSearchRef.current = (new URL (path, window.location.origin)).search sessionRef.current = persistedSession setSession (persistedSession) - navigate (serialised.path, { replace: true }) + navigate (path, { replace: true }) return persistedSession } const finishImport = () => { + const sessionId = sessionIdRef.current + if (sessionId != null) + clearPostImportSession (sessionId) clearPostImportSourceDraft (message => toast ({ title: '入力内容を削除できませんでした', description: message })) navigate ('/posts') @@ -202,7 +349,10 @@ const PostImportReviewPage: FC = ({ user }) => { return } - const loaded = buildSession (parsed.rows) + const parsedSessionId = parsePostNewSessionId (location.search) + const sessionId = parsedSessionId ?? generatePostImportSessionId () + sessionIdRef.current = sessionId + const loaded = loadPostImportSession (sessionId) ?? buildSession (parsed.rows) persistedSearchRef.current = location.search sessionRef.current = loaded setSession (loaded) @@ -211,11 +361,71 @@ const PostImportReviewPage: FC = ({ user }) => { { try { - const preview = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/preview', { - source: parsed.source }) + const preview = await apiGet ('/posts/preview', { + params: { url: parsed.source } }) if (!(active)) return - const rows = initialisePreviewRows (preview.rows) + const rows = initialisePreviewRows ([{ + sourceRow: 1, + url: preview.url, + attributes: { + title: preview.title ?? '', + thumbnailBase: preview.thumbnailBase ?? '', + originalCreatedFrom: preview.originalCreatedFrom ?? '', + originalCreatedBefore: preview.originalCreatedBefore ?? '', + duration: preview.duration ?? '', + videoMs: preview.videoMs ?? '', + tags: preview.tags ?? '', + parentPostIds: '' }, + fieldWarnings: preview.fieldWarnings ?? { }, + baseWarnings: preview.baseWarnings ?? [], + validationErrors: { }, + provenance: { + url: 'manual', + title: 'automatic', + thumbnailBase: 'automatic', + originalCreatedFrom: 'automatic', + originalCreatedBefore: 'automatic', + duration: 'automatic', + videoMs: 'automatic', + tags: 'automatic', + parentPostIds: 'automatic' }, + tagSources: { + automatic: preview.tags ?? '', + manual: '' }, + status: + Object.keys (preview.fieldWarnings ?? { }).length > 0 + || (preview.baseWarnings?.length ?? 0) > 0 + ? 'warning' + : 'ready', + skipReason: preview.existingPost?.id != null ? 'existing' : undefined, + existingPostId: preview.existingPost?.id, + resetSnapshot: { + url: preview.url, + attributes: { + title: preview.title ?? '', + thumbnailBase: preview.thumbnailBase ?? '', + originalCreatedFrom: preview.originalCreatedFrom ?? '', + originalCreatedBefore: preview.originalCreatedBefore ?? '', + duration: preview.duration ?? '', + videoMs: preview.videoMs ?? '', + tags: preview.tags ?? '', + parentPostIds: '' }, + provenance: { + url: 'manual', + title: 'automatic', + thumbnailBase: 'automatic', + originalCreatedFrom: 'automatic', + originalCreatedBefore: 'automatic', + duration: 'automatic', + videoMs: 'automatic', + tags: 'automatic', + parentPostIds: 'automatic' }, + tagSources: { + automatic: preview.tags ?? '', + manual: '' }, + fieldWarnings: preview.fieldWarnings ?? { }, + baseWarnings: preview.baseWarnings ?? [] } }]) const persisted = await persistSession (buildSession (rows)) if (persisted == null) return @@ -228,37 +438,10 @@ const PostImportReviewPage: FC = ({ user }) => { return } - const requestedRows = validatableImportRows (parsed.rows) - if (requestedRows.length === 0) - return - - try - { - const validated = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/validate', { - rows: requestedRows.map (row => ({ - sourceRow: row.sourceRow, - url: row.url, - attributes: row.attributes, - provenance: row.provenance, - tagSources: row.tagSources, - metadataUrl: row.metadataUrl })), - changed_row: 'all' }) - if (!(active)) - return - - const validatedRows = initialisePreviewRows (validated.rows) - if (!(hasExactSourceRows (requestedRows.map (row => row.sourceRow), validatedRows))) - return - - const latest = sessionRef.current ?? loaded - const rows = mergeValidatedImportRows (latest.rows, validatedRows) - const persisted = await persistSession (buildSession (rows)) - if (persisted == null) - return - } - catch - { - } + if (parsedSessionId == null) + { + await persistSession (loaded) + } } void hydrate () @@ -354,51 +537,46 @@ const PostImportReviewPage: FC = ({ user }) => { : row const urlChanged = draft.url !== baseRow.url const nextRow = buildNextEditedRow (baseRow, draft, urlChanged) - const nextRows = currentSession.rows.map (currentRow => - currentRow.sourceRow === baseRow.sourceRow - ? nextRow - : currentRow) try { - const validated = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/validate', { - rows: - validatableImportRows (nextRows) - .map (currentRow => ({ - sourceRow: currentRow.sourceRow, - url: currentRow.url, - attributes: currentRow.attributes, - provenance: currentRow.provenance, - tagSources: currentRow.tagSources, - metadataUrl: currentRow.metadataUrl })), - changed_row: urlChanged ? baseRow.sourceRow : -1 }) - const validatedRows = initialisePreviewRows (validated.rows) - const target = validatedRows.find ( - validatedRow => validatedRow.sourceRow === baseRow.sourceRow) + const dryRun = await apiPost ( + '/posts?dry=1', + buildDryRunFormData (nextRow)) const latestSession = sessionRef.current if (latestSession == null) return { saved: false, row: null } - if (target == null) - { - const restoredRows = replaceImportRow (latestSession.rows, row) - await persistSession (buildSession (restoredRows)) - toast ({ title: '行の再検証結果が不完全でした' }) - return { saved: false, row: null } - } - const editedRows = replaceImportRow (latestSession.rows, nextRow) - const mergedRows = mergeValidatedImportRow (editedRows, target) + const mergedRow = mergeDryRunRow (nextRow, dryRun) + const mergedRows = replaceImportRow (latestSession.rows, mergedRow) const nextSession = await persistSession (buildSession (mergedRows)) if (nextSession == null) return { saved: false, row: null } - const mergedTarget = - nextSession.rows.find (mergedRow => mergedRow.sourceRow === baseRow.sourceRow) - ?? target + const mergedTarget = nextSession.rows.find ( + currentRow => currentRow.sourceRow === baseRow.sourceRow) + if (mergedTarget == null) + return { saved: false, row: null } if (Object.keys (mergedTarget.validationErrors).length > 0) return { saved: false, row: mergedTarget } return { saved: true, row: null } } - catch + catch (requestError) { + if (isApiError<{ + errors?: Record + baseErrors?: string[] + }> (requestError) + && requestError.response?.status === 422) + { + const errorRow = { + ...nextRow, + validationErrors: requestError.response.data.errors ?? { }, + baseWarnings: [], + fieldWarnings: { }, + status: 'error' as const } + setMessageRow (errorRow) + return { saved: false, row: errorRow } + } + toast ({ title: '行の再検証に失敗しました' }) return { saved: false, row: null } } @@ -461,81 +639,26 @@ const PostImportReviewPage: FC = ({ user }) => { if (pendingSession == null) return - const requestedRows = validatableImportRows (pendingSession.rows) - const validated = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/validate', { - rows: requestedRows.map (row => ({ - sourceRow: row.sourceRow, - url: row.url, - attributes: row.attributes, - provenance: row.provenance, - tagSources: row.tagSources, - metadataUrl: row.metadataUrl })), - changed_row: -1 }) - const validatedRows = initialisePreviewRows (validated.rows) - if (!(hasExactSourceRows (requestedRows.map (row => row.sourceRow), validatedRows))) - { - const latest = sessionRef.current ?? pendingSession - const restoredRows = replaceImportRow (latest.rows, originalRow) - await persistSession (buildSession (restoredRows)) - toast ({ title: '再検証結果が不完全でした' }) - return - } - - const validatedTarget = validatedRows.find (row => row.sourceRow === sourceRow) - if (validatedTarget == null) - { - const latest = sessionRef.current ?? pendingSession - const restoredRows = replaceImportRow (latest.rows, originalRow) - await persistSession (buildSession (restoredRows)) - toast ({ title: '再検証結果が不完全でした' }) - return - } - - const latestAfterValidate = sessionRef.current ?? pendingSession - const mergedValidatedRows = mergeValidatedImportRow ( - latestAfterValidate.rows, - validatedTarget) - const nextSession = await persistSession (buildSession (mergedValidatedRows)) - if (nextSession == null) - return - const target = nextSession.rows.find (row => row.sourceRow === sourceRow) + const target = pendingSession.rows.find (row => row.sourceRow === sourceRow) if (target == null) - { - const restoredRows = replaceImportRow (latestAfterValidate.rows, originalRow) - await persistSession (buildSession (restoredRows)) - toast ({ title: '再検証結果が不完全でした' }) - return - } - if (Object.keys (target.validationErrors ?? { }).length > 0) - { - editRow (target) - return - } + return - const result = await apiPost<{ - created: number - skipped: number - failed: number - rows: PostImportResultRow[] }> ('/posts/import', { - rows: [{ - sourceRow: target.sourceRow, - url: target.url, - attributes: target.attributes, - provenance: target.provenance, - tagSources: target.tagSources, - metadataUrl: target.metadataUrl }] }) - if (!(hasExactSourceRows ([sourceRow], result.rows))) + const result = await apiPost<{ results: BulkApiRow[] }>( + '/posts/bulk', + buildBulkFormData ([target])) + if (result.results.length !== 1) { - const latest = sessionRef.current ?? nextSession + const latest = sessionRef.current ?? pendingSession const restoredRows = replaceImportRow (latest.rows, originalRow) await persistSession (buildSession (restoredRows)) toast ({ title: '登録結果が不完全でした' }) return } - const latestAfterImport = sessionRef.current ?? nextSession - const mergedRows = mergeImportResults (latestAfterImport.rows, result.rows) - const recoverableRows = result.rows.filter (row => + const indexedResults = indexedBulkResults ([target], result.results) + const latestAfterImport = sessionRef.current ?? pendingSession + const mergedRows = mergeImportResults (latestAfterImport.rows, indexedResults) + const recoverableRows = indexedResults.filter (row => row.status === 'failed' && row.recoverable && Object.keys (row.errors ?? { }).length > 0) @@ -579,66 +702,26 @@ const PostImportReviewPage: FC = ({ user }) => { setLoading (true) try { - const validatableRows = validatableImportRows (currentSession.rows) - if (validatableRows.length === 0) + const processingRows = processableImportRows (currentSession.rows) + if (processingRows.length === 0) { finishImport () return } - const validated = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/validate', { - rows: - validatableRows.map (row => ({ - sourceRow: row.sourceRow, - url: row.url, - attributes: row.attributes, - provenance: row.provenance, - tagSources: row.tagSources, - metadataUrl: row.metadataUrl })), - changed_row: -1 }) - const validatedRows = initialisePreviewRows (validated.rows) - const expectedSourceRows = validatableRows.map (row => row.sourceRow) - if (!(hasExactSourceRows (expectedSourceRows, validatedRows))) - { - toast ({ title: '再検証結果が不完全でした' }) - return - } - const latestAfterValidate = sessionRef.current ?? currentSession - const mergedRows = mergeValidatedImportRows (latestAfterValidate.rows, validatedRows) - const persistedValidated = await persistSession (buildSession (mergedRows)) - if (persistedValidated == null) - return - const currentRows = persistedValidated.rows - const firstInvalid = currentRows.find ( - row => Object.keys (row.validationErrors).length > 0) - if (firstInvalid != null) - { - editRow (firstInvalid) - return - } - - const result = await apiPost<{ - created: number - skipped: number - failed: number - rows: PostImportResultRow[] }> ('/posts/import', { - rows: processableImportRows (currentRows).map (row => ({ - sourceRow: row.sourceRow, - url: row.url, - attributes: row.attributes, - provenance: row.provenance, - tagSources: row.tagSources, - metadataUrl: row.metadataUrl })) }) - const expectedImportRows = processableImportRows (currentRows).map (row => row.sourceRow) - if (!(hasExactSourceRows (expectedImportRows, result.rows))) + const result = await apiPost<{ results: BulkApiRow[] }>( + '/posts/bulk', + buildBulkFormData (processingRows)) + if (result.results.length !== processingRows.length) { toast ({ title: '登録結果が不完全でした' }) return } - const latestAfterImport = sessionRef.current ?? buildSession (currentRows) - const mergedResults = mergeImportResults (latestAfterImport.rows, result.rows) - const recoverableRows = result.rows.filter (row => + const indexedResults = indexedBulkResults (processingRows, result.results) + const latestAfterImport = sessionRef.current ?? buildSession (currentSession.rows) + const mergedResults = mergeImportResults (latestAfterImport.rows, indexedResults) + const recoverableRows = indexedResults.filter (row => row.status === 'failed' && row.recoverable && Object.keys (row.errors ?? { }).length > 0) diff --git a/frontend/src/pages/posts/PostImportSourcePage.tsx b/frontend/src/pages/posts/PostImportSourcePage.tsx index 45afab1..eb0a72d 100644 --- a/frontend/src/pages/posts/PostImportSourcePage.tsx +++ b/frontend/src/pages/posts/PostImportSourcePage.tsx @@ -11,13 +11,20 @@ import MainArea from '@/components/layout/MainArea' import { Button } from '@/components/ui/button' import { toast } from '@/components/ui/use-toast' import { SITE_TITLE } from '@/config' -import { apiPost, isApiError } from '@/lib/api' -import { serialisePostNewState } from '@/lib/postNewQueryState' +import { apiGet, isApiError } from '@/lib/api' +import { + appendPostNewSessionId, + serialisePostNewState, +} from '@/lib/postNewQueryState' import { canEditContent } from '@/lib/users' import { countImportSourceLines, + generatePostImportSessionId, cleanupExpiredPostImportSessions, initialisePreviewRows, + loadPostImportSession, loadPostImportSourceDraft, + resultRepairMode, + savePostImportSession, savePostImportSourceDraft, validateImportSource } from '@/lib/postImportSession' import Forbidden from '@/pages/Forbidden' @@ -29,6 +36,19 @@ import type { User } from '@/types' type Props = { user: User | null } +type PreviewResponse = { + url: string + title?: string + thumbnailBase?: string + originalCreatedFrom?: string + originalCreatedBefore?: string + duration?: string + videoMs?: number + tags?: string + fieldWarnings?: Record + baseWarnings?: string[] + existingPost?: { id: number } | null } + const MAX_ROWS = 100 const SOURCE_ERROR_ID = 'post-import-source-error' const SOURCE_ISSUES_ID = 'post-import-source-issues' @@ -44,6 +64,96 @@ const urlIssuesFromRows = (rows: PostImportRow[], source: string) => { } +const previewRowFromResult = ( + sourceRow: number, + result: PreviewResponse, +): PostImportRow => { + const fieldWarnings = result.fieldWarnings ?? { } + const baseWarnings = result.baseWarnings ?? [] + const row: PostImportRow = { + sourceRow, + url: result.url, + attributes: { + title: result.title ?? '', + thumbnailBase: result.thumbnailBase ?? '', + originalCreatedFrom: result.originalCreatedFrom ?? '', + originalCreatedBefore: result.originalCreatedBefore ?? '', + duration: result.duration ?? '', + videoMs: result.videoMs ?? '', + tags: result.tags ?? '', + parentPostIds: '' }, + fieldWarnings, + baseWarnings, + validationErrors: { }, + provenance: { + url: 'manual', + title: 'automatic', + thumbnailBase: 'automatic', + originalCreatedFrom: 'automatic', + originalCreatedBefore: 'automatic', + duration: 'automatic', + videoMs: 'automatic', + tags: 'automatic', + parentPostIds: 'automatic' }, + tagSources: { + automatic: result.tags ?? '', + manual: '' }, + status: + Object.keys (fieldWarnings).length > 0 || baseWarnings.length > 0 + ? 'warning' + : 'ready', + skipReason: result.existingPost?.id != null ? 'existing' : undefined, + existingPostId: result.existingPost?.id, + resetSnapshot: { + url: result.url, + attributes: { + title: result.title ?? '', + thumbnailBase: result.thumbnailBase ?? '', + originalCreatedFrom: result.originalCreatedFrom ?? '', + originalCreatedBefore: result.originalCreatedBefore ?? '', + duration: result.duration ?? '', + videoMs: result.videoMs ?? '', + tags: result.tags ?? '', + parentPostIds: '' }, + provenance: { + url: 'manual', + title: 'automatic', + thumbnailBase: 'automatic', + originalCreatedFrom: 'automatic', + originalCreatedBefore: 'automatic', + duration: 'automatic', + videoMs: 'automatic', + tags: 'automatic', + parentPostIds: 'automatic' }, + tagSources: { + automatic: result.tags ?? '', + manual: '' }, + fieldWarnings, + baseWarnings } } + return row +} + + +const fetchPreviewRows = async (urls: string[]): Promise => { + const results: PostImportRow[] = new Array (urls.length) + let nextIndex = 0 + const worker = async () => { + while (nextIndex < urls.length) + { + const index = nextIndex + ++nextIndex + const result = await apiGet ('/posts/preview', { + params: { url: urls[index] } }) + results[index] = previewRowFromResult (index + 1, result) + } + } + + await Promise.all ( + Array.from ({ length: Math.min (4, urls.length) }, () => worker ())) + return results +} + + const PostImportSourcePage: FC = ({ user }) => { const editable = canEditContent (user) const navigate = useNavigate () @@ -114,17 +224,30 @@ const PostImportSourcePage: FC = ({ user }) => { setSourceError (null) try { - const data = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/preview', { source }) - const urlIssues = urlIssuesFromRows (data.rows, source) + const urls = source.split (/\r\n|\n|\r/).map (line => line.trim ()).filter (line => line !== '') + const rows = await fetchPreviewRows (urls) + const urlIssues = urlIssuesFromRows (rows, source) if (urlIssues.length > 0) { setSourceIssues (urlIssues) return } - const nextRows = initialisePreviewRows (data.rows) + const nextRows = initialisePreviewRows (rows) const serialised = await serialisePostNewState (nextRows) - if (serialised != null) - navigate (serialised.path) + if (serialised == null) + return + + const sessionId = generatePostImportSessionId () + const session = { + source, + rows: nextRows, + repairMode: resultRepairMode (nextRows) } + savePostImportSession (sessionId, session) + const sessionLoaded = loadPostImportSession (sessionId) + if (!(serialised.complete) && sessionLoaded?.rows.length !== nextRows.length) + return + + navigate (appendPostNewSessionId (serialised.path, sessionId)) } catch (requestError) {