From 95520811338a904f9b6f4210d80cf083c2a346eb Mon Sep 17 00:00:00 2001 From: miteruzo Date: Sat, 18 Jul 2026 02:08:20 +0900 Subject: [PATCH] #399 --- backend/app/controllers/posts_controller.rb | 74 +++-- backend/app/models/post.rb | 73 ++++- .../app/representations/post_compact_repr.rb | 2 +- .../post_thumbnail_upload_validator.rb | 4 +- .../app/services/preview/thumbnail_fetcher.rb | 2 +- .../posts/import/PostImportStatusBadge.tsx | 2 + .../posts/import/postImportRowStatus.ts | 16 +- frontend/src/lib/postImportRows.ts | 38 ++- .../src/pages/posts/PostImportReviewPage.tsx | 291 +++++++++++------- 9 files changed, 351 insertions(+), 151 deletions(-) diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index cb5850c..69bbc41 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -120,30 +120,64 @@ class PostsController < ApplicationController return head :forbidden unless current_user.gte_member? return render_bad_request('URL は必須です．') if params[:url].blank? - preview = PostImportPreviewer.new.preview_rows( - rows: [{ - source_row: 1, - url: params[:url].to_s, - attributes: { }, - provenance: { }, - tag_sources: { } }], - fetch_metadata: true).first - return render_validation_error fields: preview[:validation_errors] if preview[:validation_errors].present? + normal_url = PostUrlNormaliser.normalise(params[:url].to_s) + return render_validation_error(fields: { url: ['URL が不正です．'] }) if normal_url.blank? + + Preview::UrlSafety.validate(normal_url) + existing_post = Post.with_attached_thumbnail.find_by(url: normal_url) + if existing_post.present? + return render json: { + url: normal_url, + title: nil, + thumbnail_base: nil, + tags: nil, + original_created_from: nil, + original_created_before: nil, + duration: nil, + video_ms: nil, + field_warnings: { }, + base_warnings: [], + existing_post: compact_post(existing_post.id) } + end + + metadata = PostMetadataFetcher.fetch(normal_url) + field_warnings = { } + field_warnings[:title] = ['タイトルを取得できませんでした．'] if metadata[:title].blank? + if metadata[:thumbnail_base].blank? + field_warnings[:thumbnail_base] = ['サムネールを取得できませんでした．'] + end render json: { - url: preview[:url], - title: preview[:attributes]['title'], - thumbnail_base: preview[:attributes]['thumbnail_base'], - tags: preview[:attributes]['tags'], - original_created_from: preview[:attributes]['original_created_from'], - original_created_before: preview[:attributes]['original_created_before'], - duration: preview[:attributes]['duration'], - video_ms: preview[:attributes]['video_ms'], - field_warnings: preview[:field_warnings], - base_warnings: preview[:base_warnings], - existing_post: compact_post(preview[:existing_post_id]) } + url: normal_url, + title: metadata[:title], + thumbnail_base: metadata[:thumbnail_base], + tags: metadata[:tags], + original_created_from: metadata[:original_created_from], + original_created_before: metadata[:original_created_before], + duration: metadata[:duration], + video_ms: metadata[:video_ms], + field_warnings: field_warnings, + base_warnings: [], + existing_post: nil } rescue ArgumentError => e render_bad_request e.message + rescue Preview::UrlSafety::UnsafeUrl => e + render_validation_error fields: { url: [e.message] } + rescue Preview::HttpFetcher::FetchFailed, + Preview::HttpFetcher::FetchTimeout, + Preview::HttpFetcher::ResponseTooLarge + render json: { + url: normal_url, + title: nil, + thumbnail_base: nil, + tags: nil, + original_created_from: nil, + original_created_before: nil, + duration: nil, + video_ms: nil, + field_warnings: { url: ['自動取得に失敗しました．'] }, + base_warnings: [], + existing_post: nil } end def show diff --git a/backend/app/models/post.rb b/backend/app/models/post.rb index 589961f..9597c40 100644 --- a/backend/app/models/post.rb +++ b/backend/app/models/post.rb @@ -3,6 +3,7 @@ class Post < ApplicationRecord require 'mini_magick' require 'nokogiri' require 'stringio' + require 'timeout' class RemoteThumbnailFetchFailed < StandardError; end @@ -15,6 +16,7 @@ class Post < ApplicationRecord REMOTE_SVG_CONTENT_TYPE = 'image/svg+xml'.freeze MAX_SVG_DIMENSION = 4_096 MAX_SVG_PIXELS = 16_777_216 + THUMBNAIL_PROCESS_TIMEOUT = 5.seconds def self.resized_thumbnail_attachment(upload, content_type: nil) upload.rewind @@ -216,15 +218,37 @@ class Post < ApplicationRecord end def self.image_for_thumbnail_upload(bytes, content_type: nil) - return MiniMagick::Image.read(bytes) unless svg_content_type?(content_type) + return decode_raster_thumbnail(bytes) unless svg_content_type?(content_type) || svg_document_bytes?(bytes) - MiniMagick::Image.read(sanitised_svg_bytes(bytes)) + decode_svg_thumbnail(bytes) end def self.svg_content_type?(content_type) content_type.to_s.split(';', 2).first.to_s.downcase.strip == REMOTE_SVG_CONTENT_TYPE end + def self.svg_document_bytes?(bytes) + document = Nokogiri::XML( + bytes, + nil, + nil, + Nokogiri::XML::ParseOptions::STRICT | + Nokogiri::XML::ParseOptions::NONET) + document.root&.name == 'svg' + rescue Nokogiri::XML::SyntaxError + false + end + + def self.decode_raster_thumbnail(bytes) + Timeout.timeout(THUMBNAIL_PROCESS_TIMEOUT) { MiniMagick::Image.read(bytes) } + end + + def self.decode_svg_thumbnail(bytes) + Timeout.timeout(THUMBNAIL_PROCESS_TIMEOUT) do + MiniMagick::Image.read(sanitised_svg_bytes(bytes)) + end + end + def self.sanitised_svg_bytes(bytes) parse_options = Nokogiri::XML::ParseOptions::STRICT | @@ -256,20 +280,44 @@ class Post < ApplicationRecord name = node.name.to_s.downcase next true if name == 'script' || name == 'foreignobject' + next style_contains_disallowed_urls?(node.text.to_s) if name == 'style' node.attribute_nodes.any? do |attribute| attribute_name = attribute.name.to_s.downcase attribute_value = attribute.value.to_s attribute_name.start_with?('on') || - attribute_name == 'href' || - attribute_name == 'xlink:href' || - attribute_name == 'src' || - (attribute_name == 'style' && attribute_value.match?(/url\s*\(/i)) || - attribute_value.match?(/\A\s*(?:https?:|data:|\/\/)/i) + external_svg_reference?(attribute_name, attribute_value) end end end + def self.external_svg_reference?(attribute_name, attribute_value) + return external_svg_url?(attribute_value) if ['href', 'xlink:href', 'src'].include?(attribute_name) + return style_contains_disallowed_urls?(attribute_value) if attribute_name == 'style' + return svg_url_function_disallowed?(attribute_value) if attribute_value.match?(/url\s*\(/i) + + false + end + + def self.external_svg_url?(value) + stripped = value.to_s.strip + return false if stripped.blank? || stripped.start_with?('#') + + stripped.match?(/\A(?:https?:|data:|\/\/)/i) + end + + def self.style_contains_disallowed_urls?(value) + text = value.to_s + text.match?(/@import/i) || svg_url_function_disallowed?(text) + end + + def self.svg_url_function_disallowed?(value) + value.to_s.scan(/url\s*\(([^)]*)\)/i).flatten.any? do |entry| + reference = entry.to_s.strip.delete_prefix("'").delete_prefix('"').delete_suffix("'").delete_suffix('"') + reference.present? && !(reference.start_with?('#')) + end + end + def self.svg_dimensions(root) width = svg_length_to_pixels(root['width']) height = svg_length_to_pixels(root['height']) @@ -287,15 +335,24 @@ class Post < ApplicationRecord matched = /\A([0-9]+(?:\.[0-9]+)?)(px)?\z/i.match(value.to_s.strip) return nil if matched == nil - Float(matched[1]) + pixels = Float(matched[1]) + return nil unless pixels.finite? && pixels.positive? + + pixels rescue ArgumentError nil end private_class_method :image_for_thumbnail_upload, :svg_content_type?, + :decode_raster_thumbnail, + :decode_svg_thumbnail, :sanitised_svg_bytes, :svg_uses_disallowed_features?, + :external_svg_reference?, + :external_svg_url?, + :style_contains_disallowed_urls?, + :svg_url_function_disallowed?, :svg_dimensions, :svg_length_to_pixels diff --git a/backend/app/representations/post_compact_repr.rb b/backend/app/representations/post_compact_repr.rb index 06f88f7..f1a0797 100644 --- a/backend/app/representations/post_compact_repr.rb +++ b/backend/app/representations/post_compact_repr.rb @@ -20,7 +20,7 @@ module PostCompactRepr Rails.application.routes.url_helpers.rails_storage_proxy_url( post.thumbnail, only_path: false) - rescue + rescue ActionController::UrlGenerationError, ArgumentError, URI::InvalidURIError nil end end diff --git a/backend/app/services/post_thumbnail_upload_validator.rb b/backend/app/services/post_thumbnail_upload_validator.rb index 74d3650..37e752a 100644 --- a/backend/app/services/post_thumbnail_upload_validator.rb +++ b/backend/app/services/post_thumbnail_upload_validator.rb @@ -11,8 +11,10 @@ class PostThumbnailUploadValidator end raise InvalidUpload, 'thumbnail file size が大きすぎます．' if thumbnail.size > MAX_THUMBNAIL_BYTES raise InvalidUpload, 'サムネイル画像の形式が不正です．' unless allowed_content_type?(thumbnail.content_type) + raise InvalidUpload, 'サムネイル画像の形式が不正です．' if Post.svg_document_bytes?(thumbnail.read) + thumbnail.rewind - attachment = Post.resized_thumbnail_attachment(thumbnail) + attachment = Post.resized_thumbnail_attachment(thumbnail, content_type: thumbnail.content_type) attachment[:io].close if attachment[:io].respond_to?(:close) rescue MiniMagick::Error raise InvalidUpload, 'サムネイル画像の変換に失敗しました．' diff --git a/backend/app/services/preview/thumbnail_fetcher.rb b/backend/app/services/preview/thumbnail_fetcher.rb index b552b17..57ff16e 100644 --- a/backend/app/services/preview/thumbnail_fetcher.rb +++ b/backend/app/services/preview/thumbnail_fetcher.rb @@ -32,7 +32,7 @@ module Preview def self.fetch_image_response(raw_url) uri, = UrlSafety.validate(raw_url) response = HttpFetcher.fetch(uri.to_s) - unless allowed_remote_image_content_type?(response.content_type) + unless allowed_remote_image_content_type?(response.content_type) || Post.svg_document_bytes?(response.body) raise GenerationFailed, 'サムネール画像が見つかりませんでした．' end diff --git a/frontend/src/components/posts/import/PostImportStatusBadge.tsx b/frontend/src/components/posts/import/PostImportStatusBadge.tsx index 1aea1a6..6c66990 100644 --- a/frontend/src/components/posts/import/PostImportStatusBadge.tsx +++ b/frontend/src/components/posts/import/PostImportStatusBadge.tsx @@ -10,6 +10,7 @@ type Props = { const LABELS: Record = { ready: '登録可能', + error: '登録不可', warning: '警告', skipped: 'スキップ', created: '登録済み', @@ -17,6 +18,7 @@ const LABELS: Record = { const TONES: Record = { ready: 'success', + error: 'warning', warning: 'warning', skipped: 'neutral', created: 'success', diff --git a/frontend/src/components/posts/import/postImportRowStatus.ts b/frontend/src/components/posts/import/postImportRowStatus.ts index 3f99f8c..0e7ed4a 100644 --- a/frontend/src/components/posts/import/postImportRowStatus.ts +++ b/frontend/src/components/posts/import/postImportRowStatus.ts @@ -2,6 +2,7 @@ import type { PostImportRow } from '@/lib/postImportSession' export type PostImportDisplayStatus = 'ready' + | 'error' | 'skipped' | 'warning' | 'created' @@ -18,13 +19,16 @@ export const displayPostImportStatus = ( ): PostImportDisplayStatus | null => row.status === 'pending' ? null + : (row.importStatus === 'failed') + ? 'failed' : (row.skipReason != null || row.importStatus === 'skipped') ? 'skipped' : (row.importStatus === 'created') ? 'created' - : ((Object.keys (row.validationErrors ?? { }).length > 0 - || row.importStatus === 'failed') - ? 'failed' - : ((hasWarnings (row) || row.status === 'warning') - ? 'warning' - : 'ready')) + : (row.status === 'error') + ? 'error' + : (Object.values (row.validationErrors ?? { }).some (messages => messages.length > 0)) + ? 'error' + : ((hasWarnings (row) || row.status === 'warning') + ? 'warning' + : 'ready') diff --git a/frontend/src/lib/postImportRows.ts b/frontend/src/lib/postImportRows.ts index 24642e9..6de6236 100644 --- a/frontend/src/lib/postImportRows.ts +++ b/frontend/src/lib/postImportRows.ts @@ -22,8 +22,21 @@ export const isManualSkipRow = (row: PostImportRow): boolean => const hasSkipReason = (row: PostImportRow): boolean => row.skipReason != null +export const compactMessageRecord = ( + messages: Record, +): Record => + Object.fromEntries ( + Object.entries (messages).filter (([, values]) => values.length > 0)) + + +export const hasErrorMessages = ( + messages: Record, +): boolean => + Object.values (messages).some (values => values.length > 0) + + const hasValidationErrors = (row: PostImportRow): boolean => - Object.keys (row.validationErrors ?? { }).length > 0 + hasErrorMessages (row.validationErrors ?? { }) const isRecoverableRow = (row: PostImportRow): boolean => row.recoverable === true @@ -79,9 +92,9 @@ const thumbnailWarnings = (row: PostImportRow): string[] => { export const applyThumbnailWarning = (row: PostImportRow): PostImportRow => ({ ...row, - fieldWarnings: { + fieldWarnings: compactMessageRecord ({ ...row.fieldWarnings, - thumbnailBase: thumbnailWarnings (row) } }) + thumbnailBase: thumbnailWarnings (row) }) }) export const applyThumbnailWarnings = (rows: PostImportRow[]): PostImportRow[] => @@ -90,7 +103,7 @@ export const applyThumbnailWarnings = (rows: PostImportRow[]): PostImportRow[] = export const processableImportRows = (rows: PostImportRow[]): PostImportRow[] => validatableImportRows (rows).filter (row => { - if (row.metadataUrl == null) + if (row.status === 'pending') return false if (row.importStatus === 'created') return false @@ -290,7 +303,7 @@ export const mergeValidatedImportRows = ( return previous const fieldWarnings = - Object.keys (row.fieldWarnings).length > 0 || row.metadataUrl !== previous.metadataUrl + hasErrorMessages (row.fieldWarnings) || row.metadataUrl !== previous.metadataUrl ? { ...row.fieldWarnings } : { ...previous.fieldWarnings } for (const [field, origin] of Object.entries (row.provenance)) @@ -345,9 +358,10 @@ export const mergeImportResults = ( createdPostId: result.post.id, existingPostId: undefined, existingPost: undefined, - fieldWarnings: result.fieldWarnings ?? row.fieldWarnings, + fieldWarnings: compactMessageRecord ( + result.fieldWarnings ?? row.fieldWarnings), baseWarnings: result.baseWarnings ?? row.baseWarnings, - importErrors: result.errors } + importErrors: compactMessageRecord (result.errors ?? { }) } case 'skipped': return { ...row, @@ -357,9 +371,10 @@ export const mergeImportResults = ( createdPostId: undefined, existingPostId: result.existingPostId, existingPost: result.existingPost ?? row.existingPost, - fieldWarnings: result.fieldWarnings ?? row.fieldWarnings, + fieldWarnings: compactMessageRecord ( + result.fieldWarnings ?? row.fieldWarnings), baseWarnings: result.baseWarnings ?? row.baseWarnings, - importErrors: result.errors } + importErrors: compactMessageRecord (result.errors ?? { }) } case 'failed': return { ...row, @@ -369,9 +384,10 @@ export const mergeImportResults = ( createdPostId: undefined, existingPostId: undefined, existingPost: undefined, - fieldWarnings: result.fieldWarnings ?? row.fieldWarnings, + fieldWarnings: compactMessageRecord ( + result.fieldWarnings ?? row.fieldWarnings), baseWarnings: result.baseWarnings ?? row.baseWarnings, - importErrors: result.errors } + importErrors: compactMessageRecord (result.errors ?? { }) } } }) } diff --git a/frontend/src/pages/posts/PostImportReviewPage.tsx b/frontend/src/pages/posts/PostImportReviewPage.tsx index e02f410..15d82f5 100644 --- a/frontend/src/pages/posts/PostImportReviewPage.tsx +++ b/frontend/src/pages/posts/PostImportReviewPage.tsx @@ -30,8 +30,10 @@ import { clearPostImportSourceDraft, generatePostImportSessionId, hasThumbnailBaseValue, + hasErrorMessages, isCompletedReviewRow, isExistingSkipRow, + isManualSkipRow, isNonRecoverableFailedRow, mergeImportResults, processableImportRows, @@ -41,7 +43,7 @@ import { reviewSummaryCounts, retryImportRow, loadPostImportSession, - serialisedPostImportRowsEqual, + compactMessageRecord, savePostImportSession, } from '@/lib/postImportSession' import { useClientBehaviourSettings } from '@/lib/useClientBehaviourSettings' @@ -102,15 +104,48 @@ const isRepairRow = (row: PostImportRow): boolean => row.recoverable === true && (row.importStatus === 'failed' || (row.importStatus === 'pending' - && Object.keys (row.validationErrors).length > 0)) + && hasErrorMessages (row.validationErrors))) const DUPLICATE_URL_MESSAGE = 'URL が重複しています．' +const rowMetadataPending = (row: PostImportRow): boolean => + row.status === 'pending' + +const rowOperationBusy = ( + row: PostImportRow, + submitting: boolean, + loadingRow: number | null, +): boolean => + submitting + || loadingRow === row.sourceRow + || rowMetadataPending (row) + + +const rowSkipBusy = ( + row: PostImportRow, + submitting: boolean, + loadingRow: number | null, +): boolean => + submitting + || loadingRow === row.sourceRow + || (rowMetadataPending (row) && !(isManualSkipRow (row))) + +const shouldFetchMetadata = (row: PostImportRow): boolean => + row.status === 'pending' + && row.skipReason !== 'manual' + && row.importStatus !== 'created' + && row.importStatus !== 'skipped' + && !(isNonRecoverableFailedRow (row)) + && row.metadataUrl == null + const applyDuplicateUrlErrors = ( rows: PostImportRow[], ): PostImportRow[] => { const counts = rows.reduce> ((result, row) => { - if (row.url !== '') + if (row.url !== '' + && !(isManualSkipRow (row)) + && row.importStatus !== 'created' + && row.importStatus !== 'skipped') result[row.url] = (result[row.url] ?? 0) + 1 return result }, { }) @@ -120,27 +155,30 @@ const applyDuplicateUrlErrors = ( message => message !== DUPLICATE_URL_MESSAGE) if ((counts[row.url] ?? 0) < 2) { - const nextValidationErrors = { + const nextValidationErrors = compactMessageRecord ({ ...row.validationErrors, - url: urlErrors } - const hasErrors = Object.values (nextValidationErrors).some ( - messages => messages.length > 0) + url: urlErrors }) + const hasErrors = hasErrorMessages (nextValidationErrors) return urlErrors.length === (row.validationErrors.url ?? []).length ? row : { ...row, validationErrors: nextValidationErrors, + skipReason: + row.skipReason === 'manual' + ? 'manual' + : row.existingPostId != null + ? 'existing' + : undefined, status: hasErrors ? 'error' : row.status } } return { ...row, skipReason: row.skipReason === 'manual' ? 'manual' : undefined, - existingPostId: undefined, - existingPost: undefined, - validationErrors: { + validationErrors: compactMessageRecord ({ ...row.validationErrors, - url: [...new Set ([...urlErrors, DUPLICATE_URL_MESSAGE])] }, + url: [...new Set ([...urlErrors, DUPLICATE_URL_MESSAGE])] }), status: 'error' } }) } @@ -179,8 +217,9 @@ const mergeDryRunRow = ( currentRow: PostImportRow, result: PostMetadataResponse, ): PostImportRow => { + const fieldWarnings = compactMessageRecord (result.fieldWarnings ?? { }) const hasWarnings = - Object.values (result.fieldWarnings ?? { }).some (messages => messages.length > 0) + Object.values (fieldWarnings).some (messages => messages.length > 0) || (result.baseWarnings?.length ?? 0) > 0 return applyThumbnailWarning ({ ...currentRow, @@ -195,7 +234,7 @@ const mergeDryRunRow = ( videoMs: result.videoMs ?? '', tags: result.tags ?? '', parentPostIds: String (result.parentPostIds ?? currentRow.attributes.parentPostIds ?? '') }, - fieldWarnings: result.fieldWarnings ?? { }, + fieldWarnings, baseWarnings: result.baseWarnings ?? [], validationErrors: { }, importErrors: undefined, @@ -291,9 +330,9 @@ const mergePreviewRow = ( currentRow: PostImportRow, preview: PostMetadataResponse, ): PostImportRow => { - const fieldWarnings = preview.fieldWarnings ?? { } + const fieldWarnings = compactMessageRecord (preview.fieldWarnings ?? { }) const baseWarnings = preview.baseWarnings ?? [] - const validationErrors = preview.validationErrors ?? { } + const validationErrors = compactMessageRecord (preview.validationErrors ?? { }) const metadataChanged = currentRow.metadataUrl !== preview.url const hasWarnings = Object.values (fieldWarnings).some (messages => messages.length > 0) @@ -438,7 +477,20 @@ const PostImportReviewPage: FC = ({ user }) => { const persistSequenceRef = useRef (0) const previewSequenceRef = useRef (0) - const persistSession = async ( + const commitSessionState = ( + nextSession: PostImportSession, + ): PostImportSession => { + const sessionId = + sessionIdRef.current ?? generatePostImportSessionId () + sessionIdRef.current = sessionId + const persistedSession = buildSession (nextSession.rows) + savePostImportSession (sessionId, persistedSession) + sessionRef.current = persistedSession + setSession (persistedSession) + return persistedSession + } + + const syncSessionUrl = async ( nextSession: PostImportSession, ): Promise => { const sessionId = @@ -451,26 +503,21 @@ const PostImportReviewPage: FC = ({ user }) => { if (persistSequenceRef.current !== sequence) return null - const saved = savePostImportSession (sessionId, nextSession) - const loadedSession = loadPostImportSession (sessionId) - const canRecoverFromUrlOnly = serialised.complete - if (!(canRecoverFromUrlOnly) - && (!(saved)) - && !(serialisedPostImportRowsEqual (loadedSession?.rows ?? [], nextSession.rows))) - return null - if (saved - && !(serialisedPostImportRowsEqual (loadedSession?.rows ?? [], nextSession.rows))) - return null + const persistedSession = commitSessionState (nextSession) - const persistedSession = buildSession (nextSession.rows) const path = appendPostNewSessionId (serialised.path, sessionId) persistedSearchRef.current = (new URL (path, window.location.origin)).search - sessionRef.current = persistedSession - setSession (persistedSession) navigate (path, { replace: true }) return persistedSession } + const currentRowBySource = ( + sourceRow: number, + fallback?: PostImportSession, + ): PostImportRow | null => + (sessionRef.current?.rows ?? fallback?.rows ?? []).find ( + row => row.sourceRow === sourceRow) ?? null + const finishImport = () => { const sessionId = sessionIdRef.current if (sessionId != null) @@ -485,7 +532,9 @@ const PostImportReviewPage: FC = ({ user }) => { const previewSequence = ++previewSequenceRef.current const controllers = new Set () - if (sessionRef.current != null && persistedSearchRef.current === location.search) + if (sessionRef.current != null + && persistedSearchRef.current === location.search + && !(sessionRef.current.rows.some (row => shouldFetchMetadata (row)))) return () => { previewSequenceRef.current = previewSequence } @@ -496,20 +545,16 @@ const PostImportReviewPage: FC = ({ user }) => { try { let nextIndex = 0 - const previewErrors = new Map> () - let workingRows = [...(sessionRef.current ?? baseSession).rows] - - const mergeWorkingRow = ( + const mergeCurrentRow = ( sourceRow: number, updater: (row: PostImportRow) => PostImportRow, ) => { - workingRows = workingRows.map (row => + const latestRows = sessionRef.current?.rows ?? baseSession.rows + const nextRows = applyDuplicateUrlErrors (latestRows.map (row => row.sourceRow === sourceRow ? updater (row) - : row) - const nextSession = buildSession (workingRows) - sessionRef.current = nextSession - setSession (nextSession) + : row)) + commitSessionState (buildSession (nextRows)) } const worker = async () => { while (active) @@ -520,22 +565,25 @@ const PostImportReviewPage: FC = ({ user }) => { return const baseRow = baseSession.rows[index] - if (baseRow.skipReason === 'manual' - || baseRow.importStatus === 'created' - || baseRow.importStatus === 'skipped' - || isNonRecoverableFailedRow (baseRow) - || baseRow.metadataUrl != null) + const currentRow = currentRowBySource (baseRow.sourceRow, baseSession) + if (currentRow == null || !(shouldFetchMetadata (currentRow))) continue const controller = new AbortController () controllers.add (controller) try { const preview = await apiGet ('/posts/metadata', { - params: { url: baseRow.url }, + params: { url: currentRow.url }, signal: controller.signal }) if (!(active) || previewSequenceRef.current !== previewSequence) return - mergeWorkingRow (baseRow.sourceRow, row => mergePreviewRow (row, preview)) + mergeCurrentRow (currentRow.sourceRow, row => + row.importStatus === 'created' + || row.importStatus === 'skipped' + || isNonRecoverableFailedRow (row) + || row.skipReason === 'manual' + ? row + : mergePreviewRow (row, preview)) } catch (requestError) { @@ -547,21 +595,29 @@ const PostImportReviewPage: FC = ({ user }) => { errors?: Record baseErrors?: string[] }> (requestError))) - continue + { + mergeCurrentRow (currentRow.sourceRow, row => ({ + ...row, + status: 'error' })) + continue + } if (requestError.response?.status === 422) { - const rowErrors = { + const rowErrors = compactMessageRecord ({ ...(requestError.response.data.errors ?? { }), ...(requestError.response.data.baseErrors?.length ? { base: requestError.response.data.baseErrors } - : { }) } - previewErrors.set (baseRow.sourceRow, rowErrors) - mergeWorkingRow (baseRow.sourceRow, row => ({ + : { }) }) + mergeCurrentRow (currentRow.sourceRow, row => ({ ...row, validationErrors: rowErrors, status: 'error' })) } + else + mergeCurrentRow (currentRow.sourceRow, row => ({ + ...row, + status: 'error' })) } finally { @@ -575,20 +631,13 @@ const PostImportReviewPage: FC = ({ user }) => { if (!(active) || previewSequenceRef.current !== previewSequence) return - const nextRows = applyDuplicateUrlErrors (workingRows.map (row => { - const rowErrors = previewErrors.get (row.sourceRow) - if (rowErrors == null) - return row - return { - ...row, - validationErrors: rowErrors, - status: 'error' as const } - })) - await persistSession (buildSession (nextRows)) + const latestRows = sessionRef.current?.rows ?? baseSession.rows + const nextRows = applyDuplicateUrlErrors (latestRows) + await syncSessionUrl (buildSession (nextRows)) } finally { - if (active) + if (active && previewSequenceRef.current === previewSequence) setMetadataLoading (false) } } @@ -649,18 +698,19 @@ const PostImportReviewPage: FC = ({ user }) => { : rows const existingRows = sortedRows.filter (row => isExistingSkipRow (row)) const reviewRows = sortedRows.filter (row => !(isExistingSkipRow (row))) - const busy = metadataLoading || submitting || loadingRow != null const canSubmit = rows.every (row => isCompletedReviewRow (row)) || processableImportRows (rows).length > 0 const toggleManualSkip = async (sourceRow: number, checked: boolean) => { - if (busy) - return - const currentSession = sessionRef.current if (currentSession == null) return + const currentRow = currentRowBySource (sourceRow, currentSession) + if (currentRow == null) + return + if (rowSkipBusy (currentRow, submitting, loadingRow)) + return const nextRows = currentSession.rows.map (row => { if (row.sourceRow !== sourceRow) @@ -674,7 +724,13 @@ const PostImportReviewPage: FC = ({ user }) => { existingPostId: checked ? undefined : row.existingPostId, existingPost: checked ? undefined : row.existingPost } }) - await persistSession (buildSession (nextRows)) + const nextSession = buildSession (applyDuplicateUrlErrors (nextRows)) + if (metadataLoading) + { + commitSessionState (nextSession) + return + } + await syncSessionUrl (nextSession) } const saveDraft = async ( @@ -687,23 +743,26 @@ const PostImportReviewPage: FC = ({ user }) => { const currentSession = sessionRef.current if (currentSession == null) return { saved: false, row: null } - if (!(canEditReviewRow (row))) + const currentRow = currentRowBySource (row.sourceRow, currentSession) + if (currentRow == null) + return { saved: false, row: null } + if (!(canEditReviewRow (currentRow))) return { saved: false, row: null } const baseRow = resetRequested - ? { ...row, - url: row.resetSnapshot.url, - attributes: { ...row.resetSnapshot.attributes }, - provenance: { ...row.resetSnapshot.provenance }, - tagSources: { ...row.resetSnapshot.tagSources }, + ? { ...currentRow, + url: currentRow.resetSnapshot.url, + attributes: { ...currentRow.resetSnapshot.attributes }, + provenance: { ...currentRow.resetSnapshot.provenance }, + tagSources: { ...currentRow.resetSnapshot.tagSources }, fieldWarnings: Object.fromEntries ( - Object.entries (row.resetSnapshot.fieldWarnings) + Object.entries (currentRow.resetSnapshot.fieldWarnings) .map (([key, values]) => [key, [...values]])), - baseWarnings: [...row.resetSnapshot.baseWarnings], - metadataUrl: row.resetSnapshot.metadataUrl, + baseWarnings: [...currentRow.resetSnapshot.baseWarnings], + metadataUrl: currentRow.resetSnapshot.metadataUrl, thumbnailFile: undefined } - : row + : currentRow const urlChanged = draft.url !== baseRow.url const nextRow = buildNextEditedRow (baseRow, draft, urlChanged) let candidateRow = nextRow @@ -724,15 +783,19 @@ const PostImportReviewPage: FC = ({ user }) => { return { saved: false, row: null } const mergedRow = mergeDryRunRow (candidateRow, dryRun) - const mergedRows = replaceImportRow (latestSession.rows, mergedRow) - const nextSession = await persistSession (buildSession (mergedRows)) + const mergedRows = applyDuplicateUrlErrors ( + replaceImportRow (latestSession.rows, mergedRow)) + const nextSession = + metadataLoading + ? commitSessionState (buildSession (mergedRows)) + : await syncSessionUrl (buildSession (mergedRows)) if (nextSession == null) return { saved: false, row: null } const mergedTarget = nextSession.rows.find ( currentRow => currentRow.sourceRow === baseRow.sourceRow) if (mergedTarget == null) return { saved: false, row: null } - if (Object.keys (mergedTarget.validationErrors).length > 0) + if (hasErrorMessages (mergedTarget.validationErrors)) return { saved: false, row: mergedTarget } return { saved: true, row: null } } @@ -746,7 +809,12 @@ const PostImportReviewPage: FC = ({ user }) => { { const errorRow = { ...candidateRow, - validationErrors: requestError.response.data.errors ?? { }, + validationErrors: compactMessageRecord ({ + ...(requestError.response.data.errors ?? { }), + ...(requestError.response.data.baseErrors?.length + ? { + base: requestError.response.data.baseErrors } + : { }) }), baseWarnings: [], fieldWarnings: { }, status: 'error' as const } @@ -778,39 +846,48 @@ const PostImportReviewPage: FC = ({ user }) => { } const editRow = async (row: PostImportRow) => { - setEditingRow (row) + const currentRow = sessionRef.current?.rows.find ( + current => current.sourceRow === row.sourceRow) + if (currentRow == null) + return + if (!(canEditReviewRow (currentRow))) + return + if (rowOperationBusy (currentRow, submitting, loadingRow)) + return + + setEditingRow (currentRow) try { - await openEditingDialogue (row) + await openEditingDialogue (currentRow) } finally { setEditingRow (current => - current?.sourceRow === row.sourceRow + current?.sourceRow === currentRow.sourceRow ? null : current) } } const retry = async (sourceRow: number) => { - if (busy) - return - const initialSession = sessionRef.current if (initialSession == null) return - setLoadingRow (sourceRow) const originalRow = initialSession.rows.find (row => row.sourceRow === sourceRow) if (originalRow == null) - { - setLoadingRow (null) - return - } + return + if (rowOperationBusy (originalRow, submitting, loadingRow)) + return + + setLoadingRow (sourceRow) try { const pendingRows = retryImportRow (initialSession.rows, sourceRow) - const pendingSession = await persistSession (buildSession (pendingRows)) + const pendingSession = + metadataLoading + ? commitSessionState (buildSession (pendingRows)) + : await syncSessionUrl (buildSession (pendingRows)) if (pendingSession == null) return @@ -825,16 +902,20 @@ const PostImportReviewPage: FC = ({ user }) => { { const latest = sessionRef.current ?? pendingSession const restoredRows = replaceImportRow (latest.rows, originalRow) - await persistSession (buildSession (restoredRows)) + if (metadataLoading) + commitSessionState (buildSession (restoredRows)) + else + await syncSessionUrl (buildSession (restoredRows)) toast ({ title: '登録結果が不完全でした' }) return } const indexedResults = indexedBulkResults ([target], result.results) const latestAfterImport = sessionRef.current ?? pendingSession - const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults) + const nextRows = applyDuplicateUrlErrors ( + mergeImportResults (latestAfterImport.rows, indexedResults)) .map (row => applyThumbnailWarning (row)) - const persisted = await persistSession (buildSession (nextRows)) + const persisted = await syncSessionUrl (buildSession (nextRows)) if (persisted != null && persisted.rows.every (row => isCompletedReviewRow (row))) finishImport () @@ -843,7 +924,10 @@ const PostImportReviewPage: FC = ({ user }) => { { const latest = sessionRef.current ?? initialSession const restoredRows = replaceImportRow (latest.rows, originalRow) - await persistSession (buildSession (restoredRows)) + if (metadataLoading) + commitSessionState (buildSession (restoredRows)) + else + await syncSessionUrl (buildSession (restoredRows)) toast ({ title: '再試行に失敗しました' }) } finally @@ -854,7 +938,7 @@ const PostImportReviewPage: FC = ({ user }) => { const submit = async () => { const currentSession = sessionRef.current - if (currentSession == null || busy) + if (currentSession == null || metadataLoading || submitting || loadingRow != null) return if (currentSession.rows.every (row => isCompletedReviewRow (row))) { @@ -880,9 +964,10 @@ const PostImportReviewPage: FC = ({ user }) => { const indexedResults = indexedBulkResults (processingRows, result.results) const latestAfterImport = sessionRef.current ?? buildSession (currentSession.rows) - const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults) + const nextRows = applyDuplicateUrlErrors ( + mergeImportResults (latestAfterImport.rows, indexedResults)) .map (row => applyThumbnailWarning (row)) - const persisted = await persistSession (buildSession (nextRows)) + const persisted = await syncSessionUrl (buildSession (nextRows)) if (persisted != null && persisted.rows.every (row => isCompletedReviewRow (row))) finishImport () @@ -958,10 +1043,10 @@ const PostImportReviewPage: FC = ({ user }) => { row={row} displayNumber={index + 1} showSkipToggle={true} - editDisabled={busy} - retryDisabled={busy} + editDisabled={rowOperationBusy (row, submitting, loadingRow)} + retryDisabled={rowOperationBusy (row, submitting, loadingRow)} skipDisabled={ - busy + rowSkipBusy (row, submitting, loadingRow) || isExistingSkipRow (row) || row.importStatus === 'created'} onEdit={() => editRow (row)}