From 58828597d708df02ba1f2035c81bcc19f29104b7 Mon Sep 17 00:00:00 2001 From: miteruzo Date: Wed, 15 Jul 2026 08:03:25 +0900 Subject: [PATCH] #399 --- backend/app/services/post_import_previewer.rb | 73 ++++++++++++++----- backend/app/services/post_import_runner.rb | 22 +++++- .../src/components/common/FieldWarning.tsx | 2 +- frontend/src/lib/postImportRows.ts | 28 ++++++- frontend/src/lib/postImportStorage.ts | 26 +++++++ frontend/src/lib/postImportTypes.ts | 9 ++- .../src/pages/posts/PostImportSourcePage.tsx | 29 ++++++-- 7 files changed, 156 insertions(+), 33 deletions(-) diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index bdd0a6a..7052f96 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -20,7 +20,6 @@ class PostImportPreviewer url_counts = prepared_rows.filter_map { _1[:normal_url] }.tally existing_posts = Post.where(url: prepared_rows.map { _1[:normal_url] }.compact.uniq).index_by(&:url) - known_tags = preload_known_tags(prepared_rows) existing_parent_ids = preload_parent_ids(prepared_rows) preload_metadata!( prepared_rows, @@ -28,6 +27,13 @@ class PostImportPreviewer metadata_cache, existing_posts, url_counts) + known_tags = + preload_known_tags( + prepared_rows, + fetch_metadata, + metadata_cache, + existing_posts, + url_counts) prepared_rows.map { |row| preview_row(row, fetch_metadata:, @@ -96,14 +102,7 @@ class PostImportPreviewer field_warnings:, base_warnings:, validation_errors:, - status: - if validation_errors.present? - 'error' - elsif warnings_present - 'warning' - else - 'ready' - end } + status: warnings_present ? 'warning' : 'ready' } end should_fetch = validation_errors.blank? @@ -228,31 +227,56 @@ class PostImportPreviewer urls = urls.reject { metadata_cache.key?(_1) } return if urls.empty? - queue = Queue.new - urls.each { queue << _1 } + url_queue = Queue.new + result_queue = Queue.new + urls.each { url_queue << _1 } workers = [urls.length, 4].min.times.map { Thread.new { - loop do - url = queue.pop(true) - metadata_cache[url] = fetch_metadata(url) - rescue ThreadError - break + Rails.application.executor.wrap do + loop do + url = url_queue.pop(true) + result_queue << [url, safe_fetch_metadata(url)] + rescue ThreadError + break + end end } } Timeout.timeout(15) { workers.each(&:join) } rescue Timeout::Error workers&.each(&:kill) - urls.each do |url| + ensure + workers&.each(&:join) + while result_queue&.size.to_i.positive? + url, result = result_queue.pop + metadata_cache[url] = result + end + urls&.each do |url| metadata_cache[url] ||= { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } } end end - def preload_known_tags prepared_rows + def safe_fetch_metadata url + fetch_metadata(url) + rescue StandardError => e + Rails.logger.error( + "post_import_metadata_fetch_unexpected_failure "\ + "#{ { error: e.class.name, message: e.message }.to_json }") + { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } } + end + + def preload_known_tags prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts names = prepared_rows.flat_map { |row| attributes = initial_attributes(row) provenance = initial_provenance(row) tag_sources = initial_tag_sources(row, attributes, provenance) + if metadata_url_changed?(row) + clear_automatic_values!(attributes, provenance, tag_sources) + end + if should_apply_metadata_to_row?(row, fetch_metadata, existing_posts, url_counts) + metadata = metadata_for(row[:normal_url], metadata_cache) + apply_metadata!(attributes, provenance, tag_sources, metadata[:data]) + end preview_tag_names(merged_tags(tag_sources, provenance['tags'])) }.compact.uniq return { } if names.empty? @@ -274,6 +298,19 @@ class PostImportPreviewer Post.where(id: ids).pluck(:id).to_h { [_1, true] } end + def metadata_url_changed? row + row[:metadata_url].present? && row[:metadata_url] != (row[:normal_url] || row[:url_text]) + end + + def should_apply_metadata_to_row? row, fetch_metadata, existing_posts, url_counts + normal_url = row[:normal_url] + return false if normal_url.blank? + return false if url_counts[normal_url].to_i > 1 + return false if existing_posts.key?(normal_url) + + should_fetch_metadata?(fetch_metadata, row[:source_row].to_i) + end + def apply_metadata! attributes, provenance, tag_sources, metadata metadata.each do |field, value| if field == 'tags' diff --git a/backend/app/services/post_import_runner.rb b/backend/app/services/post_import_runner.rb index 86a10de..44564dd 100644 --- a/backend/app/services/post_import_runner.rb +++ b/backend/app/services/post_import_runner.rb @@ -27,13 +27,24 @@ class PostImportRunner return { source_row: row['source_row'], status: 'failed', errors: preview[:validation_errors] } if preview[:validation_errors].present? - return row.slice('source_row').merge(status: 'skipped') if preview[:skip_reason] == 'existing' + if preview[:skip_reason] == 'existing' + return { source_row: row['source_row'], + status: 'skipped', + existing_post_id: preview[:existing_post_id] } + end attributes['tags'] = preview[:attributes]['tags'] attributes['url'] = row['url'] post = PostCreator.new(actor: @actor, attributes:).create! { source_row: row['source_row'], status: 'created', post: PostRepr.base(post) } rescue ActiveRecord::RecordInvalid => e + existing_post = existing_post_for_race(row, e.record) + if existing_post + return { source_row: row['source_row'], + status: 'skipped', + existing_post_id: existing_post.id } + end + { source_row: row['source_row'], status: 'failed', errors: e.record.errors.to_hash } rescue Tag::NicoTagNormalisationError { source_row: row['source_row'], @@ -54,4 +65,13 @@ class PostImportRunner status: 'failed', errors: { base: ['登録中にエラーが発生しました．'] } } end + + def existing_post_for_race row, record + return nil unless record.errors.of_kind?(:url, :taken) + + normal_url = PostUrlNormaliser.normalise(row['url']) + return nil if normal_url.blank? + + Post.find_by(url: normal_url) + end end diff --git a/frontend/src/components/common/FieldWarning.tsx b/frontend/src/components/common/FieldWarning.tsx index 6f815ba..1f42ed0 100644 --- a/frontend/src/components/common/FieldWarning.tsx +++ b/frontend/src/components/common/FieldWarning.tsx @@ -5,7 +5,7 @@ type Props = { id?: string export const FieldWarning: FC = ({ id, messages }: Props) => { - if (!(messages) || messages.length === 0) + if (messages == null || messages.length === 0) return null return ( diff --git a/frontend/src/lib/postImportRows.ts b/frontend/src/lib/postImportRows.ts index 12bb8f9..f73bfd7 100644 --- a/frontend/src/lib/postImportRows.ts +++ b/frontend/src/lib/postImportRows.ts @@ -175,13 +175,33 @@ export const mergeImportResults = ( const resultMap = new Map (results.map (row => [row.sourceRow, row])) return rows.map (row => { const result = resultMap.get (row.sourceRow) - return result - ? { + if (result == null) + return row + + switch (result.status) + { + case 'created': + return { ...row, - importStatus: result.status, + importStatus: 'created', createdPostId: result.post?.id, + existingPostId: undefined, importErrors: result.errors } - : row + case 'skipped': + return { + ...row, + importStatus: 'skipped', + createdPostId: undefined, + existingPostId: result.existingPostId, + importErrors: result.errors } + case 'failed': + return { + ...row, + importStatus: 'failed', + createdPostId: undefined, + existingPostId: undefined, + importErrors: result.errors } + } }) } diff --git a/frontend/src/lib/postImportStorage.ts b/frontend/src/lib/postImportStorage.ts index 6f626c5..da42709 100644 --- a/frontend/src/lib/postImportStorage.ts +++ b/frontend/src/lib/postImportStorage.ts @@ -9,6 +9,17 @@ const SESSION_VERSION = 2 const SESSION_PREFIX = 'post-import-session:' const SOURCE_DRAFT_KEY = 'post-import-source-draft' const SESSION_MAX_AGE_MS = 24 * 60 * 60 * 1000 +const ATTRIBUTE_KEYS = [ + 'title', + 'thumbnailBase', + 'originalCreatedFrom', + 'originalCreatedBefore', + 'duration', + 'videoMs', + 'tags', + 'parentPostIds'] as const +const PROVENANCE_KEYS = [...ATTRIBUTE_KEYS, 'url'] as const +const TAG_SOURCE_KEYS = ['automatic', 'manual'] as const const isPlainObject = (value: unknown): value is Record => @@ -120,6 +131,12 @@ const isValidSkipReason = ( const isPositiveInteger = (value: unknown): value is number => Number.isInteger (value) && Number (value) > 0 +const hasOnlyKeys = ( + value: Record, + allowedKeys: readonly string[], +): boolean => + Object.keys (value).every (key => allowedKeys.includes (key)) + const sanitiseRow = (value: unknown): PostImportRow | null => { if (!(isPlainObject (value))) @@ -163,6 +180,13 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { return null const provenanceEntries = Object.entries (value.provenance) + if (!(hasOnlyKeys (value.attributes, ATTRIBUTE_KEYS))) + return null + if (!(Object.values (value.attributes).every (entry => + typeof entry === 'string' || typeof entry === 'number'))) + return null + if (!(hasOnlyKeys (value.provenance, PROVENANCE_KEYS))) + return null if (!(provenanceEntries.every (([, origin]) => isValidOrigin (origin)))) return null @@ -170,6 +194,8 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { { if (!(isPlainObject (value.tagSources))) return null + if (!(hasOnlyKeys (value.tagSources, TAG_SOURCE_KEYS))) + return null if (!(Object.values (value.tagSources).every (_1 => typeof _1 === 'string'))) return null } diff --git a/frontend/src/lib/postImportTypes.ts b/frontend/src/lib/postImportTypes.ts index e09f9e9..7b5b4b9 100644 --- a/frontend/src/lib/postImportTypes.ts +++ b/frontend/src/lib/postImportTypes.ts @@ -27,10 +27,11 @@ export type PostImportRow = { importStatus?: PostImportStatus } export type PostImportResultRow = { - sourceRow: number - status: PostImportResultStatus - post?: { id: number } - errors?: Record } + sourceRow: number + status: PostImportResultStatus + post?: { id: number } + existingPostId?: number + errors?: Record } export type PostImportSession = { version: number diff --git a/frontend/src/pages/posts/PostImportSourcePage.tsx b/frontend/src/pages/posts/PostImportSourcePage.tsx index 0cba73a..ad95f90 100644 --- a/frontend/src/pages/posts/PostImportSourcePage.tsx +++ b/frontend/src/pages/posts/PostImportSourcePage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useRef, useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { Helmet } from 'react-helmet-async' import { useNavigate } from 'react-router-dom' @@ -32,18 +32,23 @@ const MAX_ROWS = 100 const SOURCE_ERROR_ID = 'post-import-source-error' const SOURCE_ISSUES_ID = 'post-import-source-issues' +const urlIssuesFromRows = (rows: PostImportRow[]) => + rows.flatMap (row => + (row.validationErrors.url ?? []).map (message => ({ + sourceRow: row.sourceRow, + message, + url: row.url }))) + const PostImportSourcePage: FC = ({ user }) => { const editable = canEditContent (user) const navigate = useNavigate () - const draft = useMemo (() => loadPostImportSourceDraft (message => - toast ({ title: '保存済み入力を復元できませんでした', description: message })), []) - - const [source, setSource] = useState (draft.source) + const [source, setSource] = useState ('') const [loading, setLoading] = useState (false) const [sourceIssues, setSourceIssues] = useState> ([]) const [sourceError, setSourceError] = useState (null) const saveTimer = useRef (null) + const editedRef = useRef (false) const lineCount = countImportSourceLines (source) const messages = sourceError ? [sourceError] : [] @@ -56,6 +61,13 @@ const PostImportSourcePage: FC = ({ user }) => { useEffect (() => { cleanupExpiredPostImportSessions (message => toast ({ title: '保存済みデータを整理できませんでした', description: message })) + + const draft = loadPostImportSourceDraft (message => + toast ({ title: '保存済み入力を復元できませんでした', description: message })) + if (!(editedRef.current)) + { + setSource (current => current === '' ? draft.source : current) + } }, []) useEffect (() => { @@ -94,6 +106,12 @@ const PostImportSourcePage: FC = ({ user }) => { { const data = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/preview', { source }) + const urlIssues = urlIssuesFromRows (data.rows) + if (urlIssues.length > 0) + { + setSourceIssues (urlIssues) + return + } const sessionId = createPostImportSessionId () const saved = savePostImportSession (sessionId, { source, @@ -154,6 +172,7 @@ const PostImportSourcePage: FC = ({ user }) => { description: message })) }} onChange={ev => { + editedRef.current = true setSource (ev.target.value) setSourceError (null) setSourceIssues ([])