From 025f49cbcb80f823d3dd09761bc35dd9ae6dcd1b Mon Sep 17 00:00:00 2001 From: miteruzo Date: Wed, 15 Jul 2026 20:51:24 +0900 Subject: [PATCH] #399 --- backend/app/services/post_import_previewer.rb | 7 +++- .../services/post_import_row_normaliser.rb | 36 +++++++++++++----- backend/app/services/post_import_runner.rb | 30 ++++++++++++--- backend/app/services/post_metadata_fetcher.rb | 15 ++++++-- .../posts/import/PostImportRowDialog.tsx | 4 +- frontend/src/lib/postImportRows.ts | 6 ++- frontend/src/lib/postImportStorage.ts | 11 ++++++ frontend/src/lib/postImportTypes.ts | 5 ++- .../src/pages/posts/PostImportResultPage.tsx | 34 +++++++++++++++-- .../src/pages/posts/PostImportReviewPage.tsx | 38 ++++++++++++++++++- .../src/pages/posts/PostImportSourcePage.tsx | 4 +- 11 files changed, 157 insertions(+), 33 deletions(-) diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index c2bd9fc..c85cf8f 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -1,3 +1,4 @@ +require 'time' require 'timeout' class PostImportPreviewer @@ -86,7 +87,8 @@ class PostImportPreviewer if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata clear_automatic_values!(attributes, provenance, tag_sources) - clear_fetch_warnings!(field_warnings) + field_warnings = { } + base_warnings = [ ] end if validation_errors.blank? && normal_url.present? && existing_post @@ -248,7 +250,8 @@ class PostImportPreviewer def sanitise_metadata_time value return nil unless value.is_a?(String) - Time.zone.parse(value)&.iso8601 + time = Time.iso8601(value) + time.nsec.zero? ? time.iso8601 : time.iso8601(9) rescue ArgumentError, TypeError nil end diff --git a/backend/app/services/post_import_row_normaliser.rb b/backend/app/services/post_import_row_normaliser.rb index f03c17e..2dc7051 100644 --- a/backend/app/services/post_import_row_normaliser.rb +++ b/backend/app/services/post_import_row_normaliser.rb @@ -78,8 +78,12 @@ class PostImportRowNormaliser private_class_method :normalise_source_row! def self.normalise_url! value - raise ArgumentError, 'URL の形式が不正です．' unless value.is_a?(String) - raise ArgumentError, 'URL が長すぎます．' if value.bytesize > PostImportUrlListParser::MAX_URL_BYTES + unless value.is_a?(String) + raise ArgumentError, 'URL の形式が不正です．' + end + if value.bytesize > PostImportUrlListParser::MAX_URL_BYTES + raise ArgumentError, 'URL が長すぎます．' + end end private_class_method :normalise_url! @@ -112,20 +116,32 @@ class PostImportRowNormaliser private_class_method :normalise_attributes! def self.normalise_provenance! provenance - raise ArgumentError, 'provenance の形式が不正です．' unless provenance.is_a?(Hash) + unless provenance.is_a?(Hash) + raise ArgumentError, 'provenance の形式が不正です．' + end allowed = ATTRIBUTE_FIELDS + ['url'] - raise ArgumentError, '値の由来が不正です．' unless (provenance.keys - allowed).empty? - raise ArgumentError, '値の由来が不正です．' unless provenance.values.all? { ORIGINS.include?(_1) } + unless (provenance.keys - allowed).empty? + raise ArgumentError, '値の由来が不正です．' + end + unless provenance.values.all? { ORIGINS.include?(_1) } + raise ArgumentError, '値の由来が不正です．' + end end private_class_method :normalise_provenance! def self.normalise_tag_sources! tag_sources return if tag_sources.nil? - raise ArgumentError, 'タグ由来の形式が不正です．' unless tag_sources.is_a?(Hash) - raise ArgumentError, 'タグ由来の形式が不正です．' unless (tag_sources.keys - ORIGINS).empty? - raise ArgumentError, 'タグ由来の形式が不正です．' unless tag_sources.values.all? { _1.is_a?(String) } + unless tag_sources.is_a?(Hash) + raise ArgumentError, 'タグ由来の形式が不正です．' + end + unless (tag_sources.keys - ORIGINS).empty? + raise ArgumentError, 'タグ由来の形式が不正です．' + end + unless tag_sources.values.all? { _1.is_a?(String) } + raise ArgumentError, 'タグ由来の形式が不正です．' + end if tag_sources.values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES } raise ArgumentError, 'タグ由来が大きすぎます．' end @@ -143,7 +159,9 @@ class PostImportRowNormaliser raise ArgumentError, '警告の形式が不正です．' end field_warnings&.each do |key, values| - raise ArgumentError, '警告の形式が不正です．' unless ATTRIBUTE_FIELDS.include?(key) || key == 'url' + unless ATTRIBUTE_FIELDS.include?(key) || key == 'url' + raise ArgumentError, '警告の形式が不正です．' + end unless values.is_a?(Array) && values.all? { _1.is_a?(String) } raise ArgumentError, '警告の形式が不正です．' end diff --git a/backend/app/services/post_import_runner.rb b/backend/app/services/post_import_runner.rb index 4646c1c..0e4f69d 100644 --- a/backend/app/services/post_import_runner.rb +++ b/backend/app/services/post_import_runner.rb @@ -26,7 +26,8 @@ class PostImportRunner attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore } return { source_row: row['source_row'], status: 'failed', - errors: preview[:validation_errors] } if preview[:validation_errors].present? + errors: preview[:validation_errors], + recoverable: true } if preview[:validation_errors].present? if preview[:skip_reason] == 'existing' return { source_row: row['source_row'], status: 'skipped', @@ -45,8 +46,13 @@ class PostImportRunner existing_post_id: existing_post.id } end - { source_row: row['source_row'], status: 'failed', errors: e.record.errors.to_hash } - rescue ActiveRecord::RecordNotUnique + { source_row: row['source_row'], + status: 'failed', + errors: e.record.errors.to_hash, + recoverable: true } + rescue ActiveRecord::RecordNotUnique => e + raise unless url_record_not_unique?(e) + existing_post = existing_post_for_race(row) raise unless existing_post @@ -56,15 +62,23 @@ class PostImportRunner rescue Tag::NicoTagNormalisationError { source_row: row['source_row'], status: 'failed', - errors: { tags: ['ニコニコ・タグは直接指定できません．'] } } + errors: { tags: ['ニコニコ・タグは直接指定できません．'] }, + recoverable: true } rescue Tag::DeprecatedTagNormalisationError { source_row: row['source_row'], status: 'failed', - errors: { tags: ['廃止済みタグは付与できません．'] } } + errors: { tags: ['廃止済みタグは付与できません．'] }, + recoverable: true } + rescue PostCreator::VideoMsParseError + { source_row: row['source_row'], + status: 'failed', + errors: { duration: ['動画時間の記法が不正です．'] }, + recoverable: true } rescue ArgumentError { source_row: row['source_row'], status: 'failed', - errors: { base: ['入力値が不正です．'] } } + errors: { base: ['入力値が不正です．'] }, + recoverable: true } rescue StandardError => e Rails.logger.error("post_import_runner_failure #{ { error: e.class.name, message: e.message }.to_json }") @@ -83,4 +97,8 @@ class PostImportRunner Post.find_by(url: normal_url) end + + def url_record_not_unique? error + error.message.include?('index_posts_on_url') + end end diff --git a/backend/app/services/post_metadata_fetcher.rb b/backend/app/services/post_metadata_fetcher.rb index 23cb12f..e8e1493 100644 --- a/backend/app/services/post_metadata_fetcher.rb +++ b/backend/app/services/post_metadata_fetcher.rb @@ -1,3 +1,5 @@ +require 'time' + class PostMetadataFetcher TIMESTAMP_PATTERN = Regexp.new( @@ -26,8 +28,8 @@ class PostMetadataFetcher { title: metadata[:title], thumbnail_base: Preview::KnownSiteExtractor.thumbnail_url(uri) || metadata[:image_url], - original_created_from: created_range&.first&.iso8601, - original_created_before: created_range&.last&.iso8601, + original_created_from: serialise_time(created_range&.first), + original_created_before: serialise_time(created_range&.last), duration: duration&.to_f&.then { _1.positive? ? (_1 * 1_000).round : nil }, tags: platform_tags.join(' ') } end @@ -114,9 +116,16 @@ class PostMetadataFetcher Rational(value.to_i, 10**value.length) end + def self.serialise_time value + return nil if value.nil? + + value.nsec.zero? ? value.iso8601 : value.iso8601(9) + end + private_class_method :platform_tags, :original_created_range, :parse_timestamp_range, :parse_offset, - :fractional_seconds + :fractional_seconds, + :serialise_time end diff --git a/frontend/src/components/posts/import/PostImportRowDialog.tsx b/frontend/src/components/posts/import/PostImportRowDialog.tsx index f2f5c8f..3f8368e 100644 --- a/frontend/src/components/posts/import/PostImportRowDialog.tsx +++ b/frontend/src/components/posts/import/PostImportRowDialog.tsx @@ -73,7 +73,7 @@ const PostImportRowDialog: FC = ( const [resetRequested, setResetRequested] = useState (false) useEffect (() => { - if (open && row) + if (open && row != null) { setDraft (buildDraft (row)) setResetRequested (false) @@ -90,7 +90,7 @@ const PostImportRowDialog: FC = ( value: Draft[Key], ) => { setDraft (current => - current ? { ...current, [key]: value } : current) + current != null ? { ...current, [key]: value } : current) } const save = async () => { diff --git a/frontend/src/lib/postImportRows.ts b/frontend/src/lib/postImportRows.ts index 8defbd0..2e25853 100644 --- a/frontend/src/lib/postImportRows.ts +++ b/frontend/src/lib/postImportRows.ts @@ -14,6 +14,9 @@ const buildResetSnapshot = (row: PostImportRow) => ({ tagSources: { automatic: row.tagSources?.automatic ?? '', manual: row.tagSources?.manual ?? '' }, + fieldWarnings: Object.fromEntries ( + Object.entries (row.fieldWarnings).map (([key, values]) => [key, [...values]])), + baseWarnings: [...row.baseWarnings], metadataUrl: row.metadataUrl }) @@ -73,8 +76,7 @@ export const mergeValidatedImportRows = ( if ( previous.importStatus === 'created' || previous.importStatus === 'skipped' - || previous.importStatus === 'failed' - ) + || previous.importStatus === 'failed') return previous const row = validatedMap.get (previous.sourceRow) diff --git a/frontend/src/lib/postImportStorage.ts b/frontend/src/lib/postImportStorage.ts index ed923ff..30fba58 100644 --- a/frontend/src/lib/postImportStorage.ts +++ b/frontend/src/lib/postImportStorage.ts @@ -21,6 +21,7 @@ const ATTRIBUTE_KEYS = [ 'parentPostIds'] as const const PROVENANCE_KEYS = [...ATTRIBUTE_KEYS, 'url'] as const const TAG_SOURCE_KEYS = ['automatic', 'manual'] as const +const WARNING_KEYS = [...ATTRIBUTE_KEYS, 'url'] as const const isPlainObject = (value: unknown): value is Record => @@ -162,6 +163,14 @@ const sanitiseResetSnapshot = (value: unknown): PostImportResetSnapshot | null = return null if (!(Object.values (value.tagSources).every (_1 => typeof _1 === 'string'))) return null + const fieldWarnings = ensureStringListRecord (value.fieldWarnings) + if (fieldWarnings == null) + return null + if (!(hasOnlyKeys (fieldWarnings, WARNING_KEYS))) + return null + if (!(Array.isArray (value.baseWarnings)) + || !(value.baseWarnings.every (_1 => typeof _1 === 'string'))) + return null if (value.metadataUrl != null && typeof value.metadataUrl !== 'string') return null @@ -170,6 +179,8 @@ const sanitiseResetSnapshot = (value: unknown): PostImportResetSnapshot | null = attributes: value.attributes as Record, provenance: value.provenance as Record, tagSources: value.tagSources as Record, + fieldWarnings, + baseWarnings: value.baseWarnings, metadataUrl: value.metadataUrl as string | undefined } } diff --git a/frontend/src/lib/postImportTypes.ts b/frontend/src/lib/postImportTypes.ts index 7305e74..a138a49 100644 --- a/frontend/src/lib/postImportTypes.ts +++ b/frontend/src/lib/postImportTypes.ts @@ -14,6 +14,8 @@ export type PostImportResetSnapshot = { attributes: Record provenance: Record tagSources: Record + fieldWarnings: Record + baseWarnings: string[] metadataUrl?: string } export type PostImportRow = { @@ -48,7 +50,8 @@ export type PostImportResultRow = | { sourceRow: number status: 'failed' - errors?: Record } + errors?: Record + recoverable?: boolean } export type PostImportSession = { version: number diff --git a/frontend/src/pages/posts/PostImportResultPage.tsx b/frontend/src/pages/posts/PostImportResultPage.tsx index 395ba29..d5f29b0 100644 --- a/frontend/src/pages/posts/PostImportResultPage.tsx +++ b/frontend/src/pages/posts/PostImportResultPage.tsx @@ -120,10 +120,36 @@ const PostImportResultPage: FC = ({ user }) => { provenance: target.provenance, tagSources: target.tagSources, metadataUrl: target.metadataUrl }] }) - setSession (current => - current - ? { ...current, rows: mergeImportResults (current.rows, result.rows) } - : current) + const mergedRows = mergeImportResults (nextSession.rows, result.rows) + const recoverableRows = result.rows.filter (row => + row.status === 'failed' + && row.recoverable + && Object.keys (row.errors ?? { }).length > 0) + const nextRows = mergedRows.map (row => { + const recoverable = recoverableRows.find (_1 => _1.sourceRow === row.sourceRow) + if (recoverable == null) + return row + return { + ...row, + importStatus: 'pending', + validationErrors: recoverable.errors ?? { }, + importErrors: undefined } + }) + const recoverableTarget = nextRows.find (_1 => _1.sourceRow === sourceRow) + const resultSession = { + ...nextSession, + rows: nextRows, + repairMode: recoverableTarget == null ? 'all' as const : 'failed' as const } + setSession (resultSession) + if (recoverableTarget != null + && Object.keys (recoverableTarget.validationErrors).length > 0) + { + const saved = savePostImportSession (sessionId, resultSession, message => + toast ({ title: '取込状態を保存できませんでした', description: message })) + if (saved) + navigate (`/posts/import/${ sessionId }/review?edit=${ sourceRow }`) + return + } } catch { diff --git a/frontend/src/pages/posts/PostImportReviewPage.tsx b/frontend/src/pages/posts/PostImportReviewPage.tsx index 28fac34..f21821d 100644 --- a/frontend/src/pages/posts/PostImportReviewPage.tsx +++ b/frontend/src/pages/posts/PostImportReviewPage.tsx @@ -115,7 +115,7 @@ const PostImportReviewPage: FC = ({ user }) => { const updateSessionRows = (nextRows: PostImportRow[]) => setSession (current => - current ? { ...current, rows: nextRows } : current) + current != null ? { ...current, rows: nextRows } : current) const saveDraft = async ( { draft, resetRequested, resetSnapshot }: { @@ -136,6 +136,10 @@ const PostImportReviewPage: FC = ({ user }) => { attributes: { ...resetSnapshot.attributes }, provenance: { ...resetSnapshot.provenance }, tagSources: { ...resetSnapshot.tagSources }, + fieldWarnings: Object.fromEntries ( + Object.entries (resetSnapshot.fieldWarnings) + .map (([key, values]) => [key, [...values]])), + baseWarnings: [...resetSnapshot.baseWarnings], metadataUrl: resetSnapshot.metadataUrl } : editingRow const urlChanged = draft.url !== baseRow.url @@ -223,7 +227,37 @@ const PostImportReviewPage: FC = ({ user }) => { provenance: row.provenance, tagSources: row.tagSources, metadataUrl: row.metadataUrl })) }) - const nextRows = mergeImportResults (mergedRows, result.rows) + const mergedResults = mergeImportResults (mergedRows, result.rows) + const recoverableRows = result.rows.filter (row => + row.status === 'failed' + && row.recoverable + && Object.keys (row.errors ?? { }).length > 0) + const nextRows = mergedResults.map (row => { + const recoverable = recoverableRows.find (_1 => _1.sourceRow === row.sourceRow) + if (recoverable == null) + return row + return { + ...row, + importStatus: 'pending', + validationErrors: recoverable.errors ?? { }, + importErrors: undefined } + }) + const firstRecoverable = nextRows.find (row => + Object.keys (row.validationErrors).length > 0 + && row.importStatus !== 'created') + if (firstRecoverable != null) + { + const repairSession = { ...session, + rows: nextRows, + repairMode: 'failed' as const } + setSession (repairSession) + setDialogMessageRow (firstRecoverable) + const saved = savePostImportSession (sessionId, repairSession, message => + toast ({ title: '取込状態を保存できませんでした', description: message })) + if (saved) + setSearchParams ({ edit: String (firstRecoverable.sourceRow) }) + return + } const nextSession = { ...session, rows: nextRows, repairMode: 'all' as const } setSession (nextSession) const saved = savePostImportSession (sessionId, nextSession, message => diff --git a/frontend/src/pages/posts/PostImportSourcePage.tsx b/frontend/src/pages/posts/PostImportSourcePage.tsx index 86574e5..db07579 100644 --- a/frontend/src/pages/posts/PostImportSourcePage.tsx +++ b/frontend/src/pages/posts/PostImportSourcePage.tsx @@ -56,9 +56,9 @@ const PostImportSourcePage: FC = ({ user }) => { const editedRef = useRef (false) const lineCount = countImportSourceLines (source) - const messages = sourceError ? [sourceError] : [] + const messages = sourceError != null ? [sourceError] : [] const sourceDescribedBy = [ - sourceError ? SOURCE_ERROR_ID : null, + sourceError != null ? SOURCE_ERROR_ID : null, sourceIssues.length > 0 ? SOURCE_ISSUES_ID : null] .filter (_1 => _1 != null) .join (' ')