From 9eca6709340123fa146c2598b12f2d4121341287 Mon Sep 17 00:00:00 2001 From: miteruzo Date: Sat, 18 Jul 2026 01:12:55 +0900 Subject: [PATCH] #399 --- backend/app/controllers/posts_controller.rb | 14 +- backend/app/services/post_bulk_creator.rb | 63 ++-- backend/app/services/post_create_plan.rb | 150 +++++++-- backend/app/services/post_create_preflight.rb | 11 +- backend/app/services/post_creator.rb | 66 +++- .../post_thumbnail_upload_validator.rb | 7 + backend/config/routes.rb | 2 +- frontend/src/lib/postImportRows.ts | 2 + .../src/pages/posts/PostImportReviewPage.tsx | 225 ++++++------- .../src/pages/posts/PostImportSourcePage.tsx | 297 ++++-------------- 10 files changed, 435 insertions(+), 402 deletions(-) diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index 2ff9175..cb5850c 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -115,7 +115,7 @@ class PostsController < ApplicationController render json: PostRepr.base(post, current_user) end - def preview + def metadata return head :unauthorized unless current_user return head :forbidden unless current_user.gte_member? return render_bad_request('URL は必須です．') if params[:url].blank? @@ -128,6 +128,7 @@ class PostsController < ApplicationController provenance: { }, tag_sources: { } }], fetch_metadata: true).first + return render_validation_error fields: preview[:validation_errors] if preview[:validation_errors].present? render json: { url: preview[:url], @@ -140,7 +141,6 @@ class PostsController < ApplicationController video_ms: preview[:attributes]['video_ms'], field_warnings: preview[:field_warnings], base_warnings: preview[:base_warnings], - validation_errors: preview[:validation_errors], existing_post: compact_post(preview[:existing_post_id]) } rescue ArgumentError => e render_bad_request e.message @@ -178,6 +178,11 @@ class PostsController < ApplicationController attributes: post_create_attributes, thumbnail: params[:thumbnail]).run return render json: dry_run_json(preflight) if bool?(:dry) + if preflight[:existing_post].present? + post = Post.new(url: preflight[:url]) + post.errors.add :url, :taken + return render_post_form_record_invalid post + end post = PostCreator.new(actor: current_user, attributes: post_create_attributes.merge( @@ -191,7 +196,10 @@ class PostsController < ApplicationController :original_created_before, :duration, :video_ms, - :normalised_tags, + :direct_tag_specs, + :default_tag_specs, + :snapshot_tag_specs, + :post_tag_specs, :tag_sections, :normalised_parent_post_ids).symbolize_keys).merge( thumbnail: params[:thumbnail])).create! diff --git a/backend/app/services/post_bulk_creator.rb b/backend/app/services/post_bulk_creator.rb index 6bd333e..9865506 100644 --- a/backend/app/services/post_bulk_creator.rb +++ b/backend/app/services/post_bulk_creator.rb @@ -12,30 +12,32 @@ class PostBulkCreator workers = Array.new(2) do Thread.new do - ActiveRecord::Base.connection_pool.with_connection do - actor = User.find(@actor_id) - loop do - index = nil - begin - index = mutex.synchronize do - current = next_index - next_index += 1 - current - end - break if index >= @posts.length + Rails.application.executor.wrap do + ActiveRecord::Base.connection_pool.with_connection do + actor = User.find(@actor_id) + loop do + index = nil + begin + index = mutex.synchronize do + current = next_index + next_index += 1 + current + end + break if index >= @posts.length - attributes = @posts[index] - results[index] = create_row(actor, attributes, index) - rescue StandardError => e - Rails.logger.error( - "post_bulk_creator_worker_failure #{ { error: e.class.name, - message: e.message, - index: }.to_json }") - results[index] = { - status: 'failed', - recoverable: false, - errors: { base: ['登録中にエラーが発生しました．'] }, - base_errors: [] } + attributes = @posts[index] + results[index] = create_row(actor, attributes, index) + rescue StandardError => e + Rails.logger.error( + "post_bulk_creator_worker_failure #{ { error: e.class.name, + message: e.message, + index: }.to_json }") + results[index] = { + status: 'failed', + recoverable: false, + errors: { base: ['登録中にエラーが発生しました．'] }, + base_errors: [] } + end end end end @@ -43,6 +45,16 @@ class PostBulkCreator end workers.each(&:join) + results.each_index do |index| + next if results[index].present? + + results[index] = { + status: 'failed', + recoverable: false, + errors: { base: ['登録中にエラーが発生しました．'] }, + base_errors: [] } + end + { results: } end @@ -156,7 +168,10 @@ class PostBulkCreator original_created_before: preflight[:original_created_before], duration: preflight[:duration], video_ms: preflight[:video_ms], - normalised_tags: preflight[:normalised_tags], + direct_tag_specs: preflight[:direct_tag_specs], + default_tag_specs: preflight[:default_tag_specs], + snapshot_tag_specs: preflight[:snapshot_tag_specs], + post_tag_specs: preflight[:post_tag_specs], tag_sections: preflight[:tag_sections], normalised_parent_post_ids: preflight[:normalised_parent_post_ids] } end diff --git a/backend/app/services/post_create_plan.rb b/backend/app/services/post_create_plan.rb index 4a295b0..8cd09bc 100644 --- a/backend/app/services/post_create_plan.rb +++ b/backend/app/services/post_create_plan.rb @@ -1,16 +1,20 @@ class PostCreatePlan + VIDEO_TAG_NAME = '動画'.freeze + TAGME_TAG_NAME = 'タグ希望'.freeze + NO_DEERJIKIST_TAG_NAME = 'ニジラー情報不詳'.freeze + def initialize attributes: @attributes = attributes.symbolize_keys + @existing_tags_by_name = nil end def build! - Tag.normalise_tags!(tag_names, with_tagme: false, deny_deprecated: true, - with_sections: true) => - { tags:, sections: } - - tags = Tag.expand_parent_tags(tags).reject(&:deprecated?) - video_ms = normalise_video_ms(tags) - validate_video_sections!(video_ms, sections) + direct_tag_specs, tag_sections = parse_direct_tag_specs + default_tag_specs = build_default_tag_specs(direct_tag_specs) + snapshot_tag_specs = merge_tag_specs(direct_tag_specs + default_tag_specs) + post_tag_specs = expand_parent_tag_specs(snapshot_tag_specs) + video_ms = normalise_video_ms(snapshot_tag_specs) + validate_video_sections!(video_ms, tag_sections) parent_post_ids = normalise_parent_post_ids validate_parent_post_ids!(parent_post_ids) @@ -20,12 +24,15 @@ class PostCreatePlan thumbnail_base: @attributes[:thumbnail_base].presence, original_created_from: @attributes[:original_created_from].presence, original_created_before: @attributes[:original_created_before].presence, - tags: serialised_tags(tags, sections), + tags: serialised_tags(direct_tag_specs, tag_sections), duration: @attributes[:duration].to_s, video_ms: video_ms, parent_post_ids: parent_post_ids.join(' '), - normalised_tags: tags, - tag_sections: sections, + direct_tag_specs: direct_tag_specs, + default_tag_specs: default_tag_specs, + snapshot_tag_specs: snapshot_tag_specs, + post_tag_specs: post_tag_specs, + tag_sections: tag_sections, normalised_parent_post_ids: parent_post_ids } end @@ -33,6 +40,112 @@ class PostCreatePlan def tag_names = @attributes[:tags].to_s.split + def parse_direct_tag_specs + tag_sections = { } + direct_tag_specs = [] + + tag_names.each do |raw_name| + tag_name, category, sections = parse_raw_tag_name(raw_name) + existing_tag = existing_tags_by_name[tag_name] + raise Tag::NicoTagNormalisationError if existing_tag&.nico? + raise Tag::DeprecatedTagNormalisationError, [existing_tag.name] if existing_tag&.deprecated? + + direct_tag_specs << { + name: tag_name, + category: (category || existing_tag&.category || 'general').to_sym } + if sections.present? + tag_sections[tag_name] ||= [] + tag_sections[tag_name].concat(sections) + tag_sections[tag_name] = Tag.merge_section_ranges(tag_sections[tag_name]) + tag_sections.delete(tag_name) if tag_sections[tag_name] == [[0, nil]] + end + end + + [merge_tag_specs(direct_tag_specs), tag_sections] + end + + def parse_raw_tag_name raw_name + name = raw_name.to_s + prefix, category = + Tag::CATEGORY_PREFIXES.find { + name.downcase.start_with?(_1[0]) + } || ['', nil] + name = name.sub(/\A#{ prefix }/i, '') + + sections = [] + while (match = name.match(/\A(\S*?)\[([^\[\]\s]*)-([^\[\]\s]*)\](\S*)\z/)) + name = "#{ match[1] }#{ match[4] }" + next if match[2].empty? && match[3].empty? + + sections << Tag.normalise_section_range!( + begin_raw: match[2], + end_raw: match[3], + tag_name: name) + end + raise Tag::SectionLiteralParseError.new(raw_name, raw_name) if name.include?('[') || name.include?(']') + + [TagName.canonicalise(name).first, category&.to_sym, sections] + end + + def build_default_tag_specs direct_tag_specs + default_tag_specs = [] + if direct_tag_specs.length < 10 && direct_tag_specs.none? { _1[:name] == TAGME_TAG_NAME } + default_tag_specs << { + name: TAGME_TAG_NAME, + category: :meta } + end + if direct_tag_specs.none? { deerjikist_tag_spec?(_1) } + default_tag_specs << { + name: NO_DEERJIKIST_TAG_NAME, + category: :meta } + end + + default_tag_specs + end + + def expand_parent_tag_specs snapshot_tag_specs + existing_snapshot_tags = snapshot_tag_specs.filter_map { existing_tags_by_name[_1[:name]] } + expanded_parent_specs = + Tag.expand_parent_tags(existing_snapshot_tags) + .reject(&:deprecated?) + .map { |tag| + { + name: tag.name, + category: tag.category.to_sym } } + merge_tag_specs(snapshot_tag_specs + expanded_parent_specs) + end + + def merge_tag_specs specs + specs.each_with_object({ }) do |spec, merged| + merged[spec[:name]] = + if merged.key?(spec[:name]) && merged[spec[:name]][:category] != :general + merged[spec[:name]] + else + { + name: spec[:name], + category: spec[:category] } + end + end.values.sort_by { _1[:name] } + end + + def existing_tags_by_name + @existing_tags_by_name ||= begin + names = tag_names.map { canonical_tag_name_without_sections(_1) }.uniq + Tag.joins(:tag_name).where(tag_names: { name: names }).index_by(&:name) + end + end + + def canonical_tag_name_without_sections raw_name + name, = parse_raw_tag_name(raw_name) + name + end + + def deerjikist_tag_spec? spec + return true if spec[:category] == :deerjikist + + existing_tags_by_name[spec[:name]]&.deerjikist? + end + def normalise_parent_post_ids Array(@attributes[:parent_post_ids]).flat_map { _1.to_s.split }.map { |token| id = Integer(token, exception: false) @@ -47,14 +160,14 @@ class PostCreatePlan raise ArgumentError, "存在しない親投稿 Id. があります: #{ missing.join(' ') }" if missing.present? end - def serialised_tags tags, sections - tags.uniq(&:id).map { |tag| - "#{ tag.name }#{ sections[tag.id].to_a.map { Post.section_literal(_1) }.join }" + def serialised_tags direct_tag_specs, tag_sections + direct_tag_specs.map { |spec| + "#{ spec[:name] }#{ tag_sections[spec[:name]].to_a.map { Post.section_literal(_1) }.join }" }.sort.join(' ') end - def normalise_video_ms tags - return nil unless tags.any? { _1.id == Tag.video.id } + def normalise_video_ms snapshot_tag_specs + return nil unless snapshot_tag_specs.any? { _1[:name] == VIDEO_TAG_NAME } video_ms = @attributes[:video_ms] if video_ms.present? @@ -75,17 +188,18 @@ class PostCreatePlan raise PostCreator::VideoMsParseError end - def validate_video_sections! video_ms, sections + def validate_video_sections! video_ms, tag_sections return unless video_ms - sections.each_value do |ranges| + tag_sections.each_value do |ranges| ranges.each do |begin_ms, end_ms| - post = Post.new if begin_ms >= video_ms + post = Post.new post.errors.add :video_ms, 'タグ区間の開始が動画時間以上です．' raise ActiveRecord::RecordInvalid, post end if end_ms && end_ms > video_ms + post = Post.new post.errors.add :video_ms, 'タグ区間の終端が動画時間を超えてゐます．' raise ActiveRecord::RecordInvalid, post end diff --git a/backend/app/services/post_create_preflight.rb b/backend/app/services/post_create_preflight.rb index 8c15f1a..1ef7f4e 100644 --- a/backend/app/services/post_create_preflight.rb +++ b/backend/app/services/post_create_preflight.rb @@ -62,7 +62,10 @@ class PostCreatePreflight original_created_before: plan[:original_created_before], duration: plan[:duration], video_ms: plan[:video_ms], - normalised_tags: plan[:normalised_tags], + direct_tag_specs: plan[:direct_tag_specs], + default_tag_specs: plan[:default_tag_specs], + snapshot_tag_specs: plan[:snapshot_tag_specs], + post_tag_specs: plan[:post_tag_specs], tag_sections: plan[:tag_sections], normalised_parent_post_ids: plan[:normalised_parent_post_ids], field_warnings: final_field_warnings(preview[:field_warnings] || { }), @@ -126,8 +129,8 @@ class PostCreatePreflight thumbnail_warnings = (thumbnail_warnings + ['サムネールなし']).uniq end - { - **field_warnings, - 'thumbnail_base' => thumbnail_warnings } + next_warnings = field_warnings.except('thumbnail_base') + next_warnings['thumbnail_base'] = thumbnail_warnings if thumbnail_warnings.present? + next_warnings end end diff --git a/backend/app/services/post_creator.rb b/backend/app/services/post_creator.rb index 4912a88..8078991 100644 --- a/backend/app/services/post_creator.rb +++ b/backend/app/services/post_creator.rb @@ -21,12 +21,13 @@ class PostCreator ApplicationRecord.transaction do post.save! post.thumbnail.attach(thumbnail_attachment) if thumbnail_attachment.present? - tags = planned_tags + snapshot_tags = planned_snapshot_tags + post_tags = planned_post_tags sections = planned_sections - TagVersioning.record_tag_snapshots!(tags, created_by_user: @actor) + TagVersioning.record_tag_snapshots!(snapshot_tags, created_by_user: @actor) post.video_ms = planned_video_ms post.save! - sync_post_tags!(post, tags, sections) + sync_post_tags!(post, post_tags, sections) sync_parent_posts!(post, planned_parent_post_ids) PostVersionRecorder.record!(post:, event_type: :create, created_by_user: @actor) end @@ -44,7 +45,9 @@ class PostCreator thumbnail_base: @attributes[:thumbnail_base].presence) end - def planned_tags = planned_create_attributes[:normalised_tags] + def planned_snapshot_tags = planned_create_attributes[:snapshot_tags] + + def planned_post_tags = planned_create_attributes[:post_tags] def planned_sections = planned_create_attributes[:tag_sections] @@ -56,18 +59,65 @@ class PostCreator def planned_create_attributes @planned_create_attributes ||= begin - if @attributes.key?(:normalised_tags) + if @attributes.key?(:snapshot_tag_specs) + snapshot_tags = materialise_tags(@attributes[:snapshot_tag_specs] || []) + post_tags = materialise_tags(@attributes[:post_tag_specs] || []) { - normalised_tags: @attributes[:normalised_tags], - tag_sections: @attributes[:tag_sections] || { }, + snapshot_tags: snapshot_tags, + post_tags: post_tags, + tag_sections: materialise_sections( + @attributes[:tag_sections] || { }, + snapshot_tags, + post_tags), normalised_parent_post_ids: @attributes[:normalised_parent_post_ids] || [], video_ms: @attributes[:video_ms] } else - PostCreatePlan.new(attributes: @attributes).build! + build_materialised_plan end end end + def build_materialised_plan + plan = PostCreatePlan.new(attributes: @attributes).build! + snapshot_tags = materialise_tags(plan[:snapshot_tag_specs] || []) + post_tags = materialise_tags(plan[:post_tag_specs] || []) + { + snapshot_tags: snapshot_tags, + post_tags: post_tags, + tag_sections: materialise_sections( + plan[:tag_sections] || { }, + snapshot_tags, + post_tags), + normalised_parent_post_ids: plan[:normalised_parent_post_ids] || [], + video_ms: plan[:video_ms] } + end + + def materialise_tags specs + Array(specs).each_with_object({ }) do |spec, tags| + name = spec[:name] || spec['name'] + category = spec[:category] || spec['category'] + next if name.blank? || category.blank? + + tag = Tag.find_or_create_by_tag_name!(name, category:) + tag.update!(category:) if tag.category.to_sym != category.to_sym + tags[name] ||= tag + end.values + end + + def materialise_sections sections_by_name, snapshot_tags, post_tags + tags_by_name = post_tags.index_by(&:name) + snapshot_tags.each do |tag| + tags_by_name[tag.name] ||= tag + end + + sections_by_name.each_with_object({ }) do |(tag_name, ranges), sections| + tag = tags_by_name[tag_name.to_s] + next if tag.nil? + + sections[tag.id] = Array(ranges).map { |range| [range[0], range[1]] } + end + end + def sync_post_tags! post, desired_tags, sections desired_ids = desired_tags.map(&:id).to_set current_ids = post.tags.pluck(:id).to_set diff --git a/backend/app/services/post_thumbnail_upload_validator.rb b/backend/app/services/post_thumbnail_upload_validator.rb index 9e78e6c..fc2b14b 100644 --- a/backend/app/services/post_thumbnail_upload_validator.rb +++ b/backend/app/services/post_thumbnail_upload_validator.rb @@ -1,5 +1,6 @@ class PostThumbnailUploadValidator MAX_THUMBNAIL_BYTES = 20 * 1024 * 1024 + ALLOWED_CONTENT_TYPES = Preview::ThumbnailFetcher::ALLOWED_IMAGE_CONTENT_TYPES.freeze class InvalidUpload < StandardError; end @@ -7,6 +8,7 @@ class PostThumbnailUploadValidator return if thumbnail.blank? raise InvalidUpload, 'thumbnail upload が不正です．' unless thumbnail.is_a?(ActionDispatch::Http::UploadedFile) raise InvalidUpload, 'thumbnail file size が大きすぎます．' if thumbnail.size > MAX_THUMBNAIL_BYTES + raise InvalidUpload, 'サムネイル画像の形式が不正です．' unless allowed_content_type?(thumbnail.content_type) attachment = Post.resized_thumbnail_attachment(thumbnail) attachment[:io].close if attachment[:io].respond_to?(:close) @@ -15,4 +17,9 @@ class PostThumbnailUploadValidator ensure thumbnail&.rewind if thumbnail.respond_to?(:rewind) end + + def self.allowed_content_type? content_type + mime_type = content_type.to_s.split(';', 2).first.to_s.downcase.strip + ALLOWED_CONTENT_TYPES.include?(mime_type) + end end diff --git a/backend/config/routes.rb b/backend/config/routes.rb index d347ecb..30cc5bd 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -52,7 +52,7 @@ Rails.application.routes.draw do resources :posts, only: [:index, :show, :create, :update] do collection do - get :preview + get :metadata post :bulk get :random get :changes diff --git a/frontend/src/lib/postImportRows.ts b/frontend/src/lib/postImportRows.ts index 3755efe..24642e9 100644 --- a/frontend/src/lib/postImportRows.ts +++ b/frontend/src/lib/postImportRows.ts @@ -90,6 +90,8 @@ export const applyThumbnailWarnings = (rows: PostImportRow[]): PostImportRow[] = export const processableImportRows = (rows: PostImportRow[]): PostImportRow[] => validatableImportRows (rows).filter (row => { + if (row.metadataUrl == null) + return false if (row.importStatus === 'created') return false if (row.importStatus === 'skipped') diff --git a/frontend/src/pages/posts/PostImportReviewPage.tsx b/frontend/src/pages/posts/PostImportReviewPage.tsx index d7eac2a..b3e71ce 100644 --- a/frontend/src/pages/posts/PostImportReviewPage.tsx +++ b/frontend/src/pages/posts/PostImportReviewPage.tsx @@ -30,7 +30,6 @@ import { clearPostImportSourceDraft, generatePostImportSessionId, hasThumbnailBaseValue, - initialisePreviewRows, isCompletedReviewRow, isExistingSkipRow, isNonRecoverableFailedRow, @@ -62,7 +61,7 @@ import type { User } from '@/types' type Props = { user: User | null } -type PreviewResponse = { +type PostMetadataResponse = { url: string title?: string thumbnailBase?: string @@ -105,6 +104,47 @@ const isRepairRow = (row: PostImportRow): boolean => || (row.importStatus === 'pending' && Object.keys (row.validationErrors).length > 0)) +const DUPLICATE_URL_MESSAGE = 'URL が重複しています．' + +const applyDuplicateUrlErrors = ( + rows: PostImportRow[], +): PostImportRow[] => { + const counts = rows.reduce> ((result, row) => { + if (row.url !== '') + result[row.url] = (result[row.url] ?? 0) + 1 + return result + }, { }) + + return rows.map (row => { + const urlErrors = (row.validationErrors.url ?? []).filter ( + message => message !== DUPLICATE_URL_MESSAGE) + if ((counts[row.url] ?? 0) < 2) + { + const nextValidationErrors = { + ...row.validationErrors, + url: urlErrors } + const hasErrors = Object.values (nextValidationErrors).some ( + messages => messages.length > 0) + return urlErrors.length === (row.validationErrors.url ?? []).length + ? row + : { + ...row, + validationErrors: nextValidationErrors, + status: hasErrors ? 'error' : row.status } + } + + return { + ...row, + skipReason: row.skipReason === 'manual' ? 'manual' : undefined, + existingPostId: undefined, + existingPost: undefined, + validationErrors: { + ...row.validationErrors, + url: [...new Set ([...urlErrors, DUPLICATE_URL_MESSAGE])] }, + status: 'error' } + }) +} + const buildSession = (rows: PostImportRow[]): PostImportSession => ({ version: 3, @@ -137,9 +177,12 @@ const buildDryRunFormData = (row: PostImportRow): FormData => { const mergeDryRunRow = ( currentRow: PostImportRow, - result: PreviewResponse, -): PostImportRow => - applyThumbnailWarning ({ + result: PostMetadataResponse, +): PostImportRow => { + const hasWarnings = + Object.values (result.fieldWarnings ?? { }).some (messages => messages.length > 0) + || (result.baseWarnings?.length ?? 0) > 0 + return applyThumbnailWarning ({ ...currentRow, url: result.url, attributes: { @@ -157,8 +200,7 @@ const mergeDryRunRow = ( validationErrors: { }, importErrors: undefined, status: - Object.keys (result.fieldWarnings ?? { }).length > 0 - || (result.baseWarnings?.length ?? 0) > 0 + hasWarnings ? 'warning' : 'ready', skipReason: result.existingPost?.id != null ? 'existing' : undefined, @@ -169,10 +211,11 @@ const mergeDryRunRow = ( ? 'created' : 'pending', recoverable: undefined }) +} const buildPreviewResetSnapshot = ( - preview: PreviewResponse, + preview: PostMetadataResponse, ): PostImportRow['resetSnapshot'] => ({ url: preview.url, attributes: { @@ -246,7 +289,7 @@ const indexedBulkResults = ( const mergePreviewRow = ( currentRow: PostImportRow, - preview: PreviewResponse, + preview: PostMetadataResponse, ): PostImportRow => { const fieldWarnings = preview.fieldWarnings ?? { } const baseWarnings = preview.baseWarnings ?? [] @@ -405,7 +448,7 @@ const PostImportReviewPage: FC = ({ user }) => { if (serialised == null) return null if (persistSequenceRef.current !== sequence) - return sessionRef.current + return null const saved = savePostImportSession (sessionId, nextSession) const loadedSession = loadPostImportSession (sessionId) @@ -447,8 +490,23 @@ const PostImportReviewPage: FC = ({ user }) => { } const refreshRows = async (baseSession: PostImportSession) => { + setLoading (true) let nextIndex = 0 - const previews = new Map () + const previewErrors = new Map> () + let workingRows = [...(sessionRef.current ?? baseSession).rows] + + const mergeWorkingRow = ( + sourceRow: number, + updater: (row: PostImportRow) => PostImportRow, + ) => { + workingRows = workingRows.map (row => + row.sourceRow === sourceRow + ? updater (row) + : row) + const nextSession = buildSession (workingRows) + sessionRef.current = nextSession + setSession (nextSession) + } const worker = async () => { while (active) { @@ -461,18 +519,19 @@ const PostImportReviewPage: FC = ({ user }) => { if (baseRow.skipReason === 'manual' || baseRow.importStatus === 'created' || baseRow.importStatus === 'skipped' - || isNonRecoverableFailedRow (baseRow)) + || isNonRecoverableFailedRow (baseRow) + || baseRow.metadataUrl != null) continue const controller = new AbortController () controllers.add (controller) try { - const preview = await apiGet ('/posts/preview', { + const preview = await apiGet ('/posts/metadata', { params: { url: baseRow.url }, signal: controller.signal }) if (!(active) || previewSequenceRef.current !== previewSequence) return - previews.set (baseRow.sourceRow, preview) + mergeWorkingRow (baseRow.sourceRow, row => mergePreviewRow (row, preview)) } catch (requestError) { @@ -480,8 +539,25 @@ const PostImportReviewPage: FC = ({ user }) => { return if (!(active) || previewSequenceRef.current !== previewSequence) return - if (!(isApiError (requestError))) + if (!(isApiError<{ + errors?: Record + baseErrors?: string[] + }> (requestError))) continue + if (requestError.response?.status === 422) + { + const rowErrors = { + ...(requestError.response.data.errors ?? { }), + ...(requestError.response.data.baseErrors?.length + ? { + base: requestError.response.data.baseErrors } + : { }) } + previewErrors.set (baseRow.sourceRow, rowErrors) + mergeWorkingRow (baseRow.sourceRow, row => ({ + ...row, + validationErrors: rowErrors, + status: 'error' })) + } } finally { @@ -493,20 +569,22 @@ const PostImportReviewPage: FC = ({ user }) => { await Promise.all ( Array.from ({ length: Math.min (4, baseSession.rows.length) }, () => worker ())) if (!(active) || previewSequenceRef.current !== previewSequence) - return + { + setLoading (false) + return + } - const latestSession = sessionRef.current ?? baseSession - const nextRows = latestSession.rows.map (row => { - const preview = previews.get (row.sourceRow) - if (preview == null - || row.skipReason === 'manual' - || row.importStatus === 'created' - || row.importStatus === 'skipped' - || isNonRecoverableFailedRow (row)) + const nextRows = applyDuplicateUrlErrors (workingRows.map (row => { + const rowErrors = previewErrors.get (row.sourceRow) + if (rowErrors == null) return row - return mergePreviewRow (row, preview) - }) + return { + ...row, + validationErrors: rowErrors, + status: 'error' as const } + })) await persistSession (buildSession (nextRows)) + setLoading (false) } const hydrate = async () => { @@ -528,89 +606,6 @@ const PostImportReviewPage: FC = ({ user }) => { sessionRef.current = loaded setSession (loaded) - if (loadedSession == null && parsed?.shortcut) - { - try - { - const preview = await apiGet ('/posts/preview', { - params: { url: parsed.source } }) - if (!(active)) - return - const rows = [applyThumbnailWarning (initialisePreviewRows ([{ - sourceRow: 1, - url: preview.url, - attributes: { - title: preview.title ?? '', - thumbnailBase: preview.thumbnailBase ?? '', - originalCreatedFrom: preview.originalCreatedFrom ?? '', - originalCreatedBefore: preview.originalCreatedBefore ?? '', - duration: preview.duration ?? '', - videoMs: preview.videoMs ?? '', - tags: preview.tags ?? '', - parentPostIds: String (preview.parentPostIds ?? '') }, - fieldWarnings: preview.fieldWarnings ?? { }, - baseWarnings: preview.baseWarnings ?? [], - validationErrors: { }, - provenance: { - url: 'manual', - title: 'automatic', - thumbnailBase: 'automatic', - originalCreatedFrom: 'automatic', - originalCreatedBefore: 'automatic', - duration: 'automatic', - videoMs: 'automatic', - tags: 'automatic', - parentPostIds: 'automatic' }, - tagSources: { - automatic: preview.tags ?? '', - manual: '' }, - status: - Object.keys (preview.fieldWarnings ?? { }).length > 0 - || (preview.baseWarnings?.length ?? 0) > 0 - ? 'warning' - : 'ready', - skipReason: preview.existingPost?.id != null ? 'existing' : undefined, - existingPostId: preview.existingPost?.id, - existingPost: preview.existingPost ?? undefined, - resetSnapshot: { - url: preview.url, - attributes: { - title: preview.title ?? '', - thumbnailBase: preview.thumbnailBase ?? '', - originalCreatedFrom: preview.originalCreatedFrom ?? '', - originalCreatedBefore: preview.originalCreatedBefore ?? '', - duration: preview.duration ?? '', - videoMs: preview.videoMs ?? '', - tags: preview.tags ?? '', - parentPostIds: String (preview.parentPostIds ?? '') }, - provenance: { - url: 'manual', - title: 'automatic', - thumbnailBase: 'automatic', - originalCreatedFrom: 'automatic', - originalCreatedBefore: 'automatic', - duration: 'automatic', - videoMs: 'automatic', - tags: 'automatic', - parentPostIds: 'automatic' }, - tagSources: { - automatic: preview.tags ?? '', - manual: '' }, - fieldWarnings: preview.fieldWarnings ?? { }, - baseWarnings: preview.baseWarnings ?? [], - metadataUrl: preview.url } }])[0])] - const persisted = await persistSession (buildSession (rows)) - if (persisted == null) - return - } - catch - { - if (active) - navigate ('/posts/new', { replace: true }) - } - return - } - void refreshRows (loaded) } @@ -649,6 +644,9 @@ const PostImportReviewPage: FC = ({ user }) => { const existingRows = sortedRows.filter (row => isExistingSkipRow (row)) const reviewRows = sortedRows.filter (row => !(isExistingSkipRow (row))) const busy = loading || loadingRow != null + const canSubmit = + rows.every (row => isCompletedReviewRow (row)) + || processableImportRows (rows).length > 0 const toggleManualSkip = async (sourceRow: number, checked: boolean) => { if (busy) @@ -709,10 +707,10 @@ const PostImportReviewPage: FC = ({ user }) => { urlChanged ? mergePreviewRow ( nextRow, - await apiGet ('/posts/preview', { + await apiGet ('/posts/metadata', { params: { url: nextRow.url } })) : nextRow - const dryRun = await apiPost ( + const dryRun = await apiPost ( '/posts?dry=1', buildDryRunFormData (candidateRow)) const latestSession = sessionRef.current @@ -971,6 +969,7 @@ const PostImportReviewPage: FC = ({ user }) => { = ({ user }) => { const PostImportFooter = ( { loading, + canSubmit, creatableCount, manualSkippedCount, existingSkippedCount, pendingOrErrorCount, onBack, onSubmit }: { loading: boolean + canSubmit: boolean creatableCount: number manualSkippedCount: number existingSkippedCount: number @@ -1013,7 +1014,7 @@ const PostImportFooter = ( diff --git a/frontend/src/pages/posts/PostImportSourcePage.tsx b/frontend/src/pages/posts/PostImportSourcePage.tsx index fc9840b..f0b91d9 100644 --- a/frontend/src/pages/posts/PostImportSourcePage.tsx +++ b/frontend/src/pages/posts/PostImportSourcePage.tsx @@ -11,7 +11,6 @@ import MainArea from '@/components/layout/MainArea' import { Button } from '@/components/ui/button' import { toast } from '@/components/ui/use-toast' import { SITE_TITLE } from '@/config' -import { apiGet, isApiError } from '@/lib/api' import { appendPostNewSessionId, serialisePostNewState, @@ -37,191 +36,72 @@ import type { User } from '@/types' type Props = { user: User | null } -type PreviewResponse = { - url: string - title?: string - thumbnailBase?: string - originalCreatedFrom?: string - originalCreatedBefore?: string - duration?: string - videoMs?: number - tags?: string - fieldWarnings?: Record - baseWarnings?: string[] - validationErrors?: Record - existingPost?: { - id: number - title: string - url: string - thumbnailUrl?: string } | null } - const MAX_ROWS = 100 const SOURCE_ERROR_ID = 'post-import-source-error' const SOURCE_ISSUES_ID = 'post-import-source-issues' -const urlIssuesFromRows = (rows: PostImportRow[], source: string) => { - const sourceLines = source.split (/\r\n|\n|\r/) - - return rows.flatMap (row => - (row.validationErrors.url ?? []).map (message => ({ - sourceRow: row.sourceRow, - message, - url: sourceLines[row.sourceRow - 1]?.trim () ?? row.url }))) -} - - -const previewRowFromResult = ( - sourceRow: number, - result: PreviewResponse, -): PostImportRow => { - const fieldWarnings = result.fieldWarnings ?? { } - const baseWarnings = result.baseWarnings ?? [] - const validationErrors = result.validationErrors ?? { } - const hasWarnings = - Object.values (fieldWarnings).some (messages => messages.length > 0) - || baseWarnings.length > 0 - const hasErrors = - Object.values (validationErrors).some (messages => messages.length > 0) - const row: PostImportRow = { - sourceRow, - url: result.url, - attributes: { - title: result.title ?? '', - thumbnailBase: result.thumbnailBase ?? '', - originalCreatedFrom: result.originalCreatedFrom ?? '', - originalCreatedBefore: result.originalCreatedBefore ?? '', - duration: result.duration ?? '', - videoMs: result.videoMs ?? '', - tags: result.tags ?? '', - parentPostIds: '' }, - fieldWarnings, - baseWarnings, - validationErrors, - provenance: { - url: 'manual', - title: 'automatic', - thumbnailBase: 'automatic', - originalCreatedFrom: 'automatic', - originalCreatedBefore: 'automatic', - duration: 'automatic', - videoMs: 'automatic', - tags: 'automatic', - parentPostIds: 'automatic' }, - tagSources: { - automatic: result.tags ?? '', - manual: '' }, - status: - hasErrors - ? 'error' - : hasWarnings - ? 'warning' - : 'ready', - skipReason: result.existingPost?.id != null ? 'existing' : undefined, - existingPostId: result.existingPost?.id, - existingPost: result.existingPost ?? undefined, - resetSnapshot: { - url: result.url, - attributes: { - title: result.title ?? '', - thumbnailBase: result.thumbnailBase ?? '', - originalCreatedFrom: result.originalCreatedFrom ?? '', - originalCreatedBefore: result.originalCreatedBefore ?? '', - duration: result.duration ?? '', - videoMs: result.videoMs ?? '', - tags: result.tags ?? '', - parentPostIds: '' }, - provenance: { - url: 'manual', - title: 'automatic', - thumbnailBase: 'automatic', - originalCreatedFrom: 'automatic', - originalCreatedBefore: 'automatic', - duration: 'automatic', - videoMs: 'automatic', - tags: 'automatic', - parentPostIds: 'automatic' }, - tagSources: { - automatic: result.tags ?? '', - manual: '' }, - fieldWarnings, - baseWarnings } } - return row -} - - -const fetchPreviewRows = async ( - rows: Array<{ sourceRow: number - url: string }>, - signal: AbortSignal, -): Promise<{ - rows: PostImportRow[] - issues: Array<{ sourceRow: number - message: string - url: string }> -}> => { - const results: Array = new Array (rows.length).fill (null) - const issues: Array<{ sourceRow: number - message: string - url: string }> = [] - let nextIndex = 0 - - const previewIssues = ( - sourceRow: number, - url: string, - requestError: unknown, - ): Array<{ sourceRow: number - message: string - url: string }> => { - if (!(isApiError<{ - message?: string - errors?: Record - baseErrors?: string[] - }> (requestError))) - { - return [{ - sourceRow, - message: '入力を確認してください．', - url }] - } - - const fieldMessages = Object.values (requestError.response?.data?.errors ?? { }).flat () - const baseMessages = requestError.response?.data?.baseErrors ?? [] - const messages = [...new Set ([...fieldMessages, ...baseMessages])] - return (messages.length > 0 ? messages : [requestError.response?.data?.message ?? '入力を確認してください．']) - .map (message => ({ - sourceRow, - message, - url })) - } - - const worker = async () => { - while (nextIndex < rows.length) - { - const index = nextIndex - ++nextIndex - const row = rows[index] - try - { - const result = await apiGet ('/posts/preview', { - params: { url: row.url }, - signal }) - results[index] = previewRowFromResult (row.sourceRow, result) - } - catch (requestError) - { - if (signal.aborted) - return - issues.push (...previewIssues (row.sourceRow, row.url, requestError)) - } - } - } - - await Promise.all ( - Array.from ({ length: Math.min (4, rows.length) }, () => worker ())) - return { - rows: results.filter ((row): row is PostImportRow => row != null), - issues } -} +const buildInitialRows = (source: string): PostImportRow[] => + source + .split (/\r\n|\n|\r/) + .map ((line, index) => ({ + sourceRow: index + 1, + url: line.trim () })) + .filter (row => row.url !== '') + .map (row => ({ + sourceRow: row.sourceRow, + url: row.url, + attributes: { + title: '', + thumbnailBase: '', + originalCreatedFrom: '', + originalCreatedBefore: '', + duration: '', + videoMs: '', + tags: '', + parentPostIds: '' }, + fieldWarnings: { }, + baseWarnings: [], + validationErrors: { }, + provenance: { + url: 'manual', + title: 'automatic', + thumbnailBase: 'automatic', + originalCreatedFrom: 'automatic', + originalCreatedBefore: 'automatic', + duration: 'automatic', + videoMs: 'automatic', + tags: 'automatic', + parentPostIds: 'automatic' }, + tagSources: { + automatic: '', + manual: '' }, + status: 'ready' as const, + resetSnapshot: { + url: row.url, + attributes: { + title: '', + thumbnailBase: '', + originalCreatedFrom: '', + originalCreatedBefore: '', + duration: '', + videoMs: '', + tags: '', + parentPostIds: '' }, + provenance: { + url: 'manual', + title: 'automatic', + thumbnailBase: 'automatic', + originalCreatedFrom: 'automatic', + originalCreatedBefore: 'automatic', + duration: 'automatic', + videoMs: 'automatic', + tags: 'automatic', + parentPostIds: 'automatic' }, + tagSources: { + automatic: '', + manual: '' }, + fieldWarnings: { }, + baseWarnings: [] } })) const PostImportSourcePage: FC = ({ user }) => { @@ -233,7 +113,6 @@ const PostImportSourcePage: FC = ({ user }) => { const [sourceError, setSourceError] = useState (null) const saveTimer = useRef (null) const editedRef = useRef (false) - const previewController = useRef (null) const lineCount = countImportSourceLines (source) const messages = sourceError != null ? [sourceError] : [] @@ -274,10 +153,6 @@ const PostImportSourcePage: FC = ({ user }) => { } }, [source]) - useEffect (() => () => { - previewController.current?.abort () - }, []) - const preview = async () => { if (lineCount === 0) { @@ -299,42 +174,7 @@ const PostImportSourcePage: FC = ({ user }) => { setSourceError (null) try { - previewController.current?.abort () - const controller = new AbortController () - previewController.current = controller - const previewed = await fetchPreviewRows ( - source - .split (/\r\n|\n|\r/) - .map ((line, index) => ({ - sourceRow: index + 1, - url: line.trim () })) - .filter (row => row.url !== ''), - controller.signal) - if (previewed.issues.length > 0) - { - setSourceIssues (previewed.issues) - return - } - const urlIssues = urlIssuesFromRows (previewed.rows, source) - const duplicateIssues = Object.entries ( - previewed.rows.reduce> ((acc, row) => { - acc[row.url] ||= [] - acc[row.url].push (row) - return acc - }, { })) - .flatMap (([, rows]) => - rows.length < 2 - ? [] - : rows.map (row => ({ - sourceRow: row.sourceRow, - message: 'URL が重複しています．', - url: row.url }))) - if (urlIssues.length > 0 || duplicateIssues.length > 0) - { - setSourceIssues ([...urlIssues, ...duplicateIssues]) - return - } - const nextRows = initialisePreviewRows (previewed.rows) + const nextRows = initialisePreviewRows (buildInitialRows (source)) const serialised = await serialisePostNewState (nextRows) if (serialised == null) return @@ -355,16 +195,9 @@ const PostImportSourcePage: FC = ({ user }) => { navigate (appendPostNewSessionId (serialised.path, sessionId)) } - catch (requestError) + catch { - const message = - isApiError<{ message?: string, baseErrors?: string[] }> (requestError) - ? (requestError.response?.data?.message - ?? requestError.response?.data?.baseErrors?.[0]) - : undefined - setSourceError (message ?? '入力を確認してください．') - toast ({ title: '投稿情報の取得に失敗しました', - description: message ?? '入力を確認してください．' }) + return } finally {