diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index c4b9854..19886fb 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -1,5 +1,7 @@ class PostsController < ApplicationController Event = Struct.new(:post, :tag, :user, :change_type, :timestamp, keyword_init: true) + MAX_BULK_THUMBNAIL_BYTES = 20 * 1024 * 1024 + MAX_BULK_REQUEST_BYTES = 40 * 1024 * 1024 class VideoMsParseError < ArgumentError ; @@ -210,11 +212,10 @@ class PostsController < ApplicationController return head :unauthorized unless current_user return head :forbidden unless current_user.gte_member? return head :unsupported_media_type unless request.content_mime_type == Mime[:multipart_form] - - result = PostBulkCreator.new( - actor: current_user, - posts: parse_bulk_posts_manifest, - thumbnails: parse_bulk_thumbnails).run + return head :payload_too_large if request.content_length.to_i > MAX_BULK_REQUEST_BYTES + posts = parse_bulk_posts_manifest + thumbnails = parse_bulk_thumbnails(posts.length) + result = PostBulkCreator.new(actor: current_user, posts:, thumbnails:).run render json: result rescue JSON::ParserError render_bad_request 'posts manifest の JSON が不正です．' @@ -541,16 +542,22 @@ class PostsController < ApplicationController posts end - def parse_bulk_thumbnails + def parse_bulk_thumbnails post_count thumbnails = { } - post_count = Array(JSON.parse(params[:posts].presence || '[]')).length + raw = params[:thumbnails] + return thumbnails if raw.blank? + raise ArgumentError, 'thumbnail key が不正です．' unless raw.respond_to?(:to_unsafe_h) - params.to_unsafe_h.each do |key, value| - match = key.match(/\Athumbnails\[(\d+)\]\z/) - next if match.nil? + raw.to_unsafe_h.each do |key, value| + raise ArgumentError, 'thumbnail key が不正です．' unless key.to_s.match?(/\A\d+\z/) - index = Integer(match[1], 10) + index = Integer(key, 10) raise ArgumentError, 'thumbnail index が範囲外です．' if index.negative? || index >= post_count + raise ArgumentError, 'thumbnail index が重複しています．' if thumbnails.key?(index) + unless value.is_a?(ActionDispatch::Http::UploadedFile) + raise ArgumentError, 'thumbnail upload が不正です．' + end + raise ArgumentError, 'thumbnail file size が大きすぎます．' if value.size > MAX_BULK_THUMBNAIL_BYTES thumbnails[index] = value end diff --git a/backend/app/models/post.rb b/backend/app/models/post.rb index 1190311..0c7fc2a 100644 --- a/backend/app/models/post.rb +++ b/backend/app/models/post.rb @@ -15,9 +15,7 @@ class Post < ApplicationRecord def self.resized_thumbnail_attachment(upload) upload.rewind image = MiniMagick::Image.read(upload.read) - image.resize '180x180^' - image.gravity 'Center' - image.extent '180x180' + image.resize '180x180' image.format 'jpg' { io: StringIO.new(image.to_blob), diff --git a/backend/app/services/post_creator.rb b/backend/app/services/post_creator.rb index 5da1a07..8f7d937 100644 --- a/backend/app/services/post_creator.rb +++ b/backend/app/services/post_creator.rb @@ -33,6 +33,9 @@ class PostCreator PostVersionRecorder.record!(post:, event_type: :create, created_by_user: @actor) end post + rescue StandardError + post&.thumbnail&.purge if post&.thumbnail&.attached? + raise end private diff --git a/frontend/src/components/posts/import/PostImportRowForm.tsx b/frontend/src/components/posts/import/PostImportRowForm.tsx index b3487a6..ede8764 100644 --- a/frontend/src/components/posts/import/PostImportRowForm.tsx +++ b/frontend/src/components/posts/import/PostImportRowForm.tsx @@ -28,7 +28,8 @@ const buildDraft = (row: PostImportRow): Draft => ({ originalCreatedFrom: String (row.attributes.originalCreatedFrom ?? ''), originalCreatedBefore: String (row.attributes.originalCreatedBefore ?? ''), tags: String (row.attributes.tags ?? ''), - parentPostIds: String (row.attributes.parentPostIds ?? '') }) + parentPostIds: String (row.attributes.parentPostIds ?? ''), + thumbnailFile: row.thumbnailFile }) const buildResetDraft = (row: PostImportRow): Draft => ({ url: row.resetSnapshot.url, @@ -37,7 +38,8 @@ const buildResetDraft = (row: PostImportRow): Draft => ({ originalCreatedFrom: String (row.resetSnapshot.attributes.originalCreatedFrom ?? ''), originalCreatedBefore: String (row.resetSnapshot.attributes.originalCreatedBefore ?? ''), tags: String (row.resetSnapshot.attributes.tags ?? ''), - parentPostIds: String (row.resetSnapshot.attributes.parentPostIds ?? '') }) + parentPostIds: String (row.resetSnapshot.attributes.parentPostIds ?? ''), + thumbnailFile: undefined }) const groupedMessages = (...values: (string[] | undefined)[]): string[] => [...new Set (values.flatMap (value => value ?? []))] @@ -64,6 +66,13 @@ const sameTagSources = ( (current?.automatic ?? '') === reset.automatic && (current?.manual ?? '') === reset.manual +const sameWarnings = ( + current: PostImportRow, + reset: PostImportRow['resetSnapshot'], +): boolean => + JSON.stringify (current.fieldWarnings) === JSON.stringify (reset.fieldWarnings) + && JSON.stringify (current.baseWarnings) === JSON.stringify (reset.baseWarnings) + const PostImportRowForm: FC = ( { row, @@ -76,6 +85,8 @@ const PostImportRowForm: FC = ( const [resetRequested, setResetRequested] = useState (false) const [committedThumbnailBase, setCommittedThumbnailBase] = useState ( () => String (row.attributes.thumbnailBase ?? '')) + const [thumbnailObjectUrl, setThumbnailObjectUrl] = useState ( + () => row.thumbnailObjectUrl) useEffect (() => { const nextDraft = buildDraft (row) @@ -83,8 +94,14 @@ const PostImportRowForm: FC = ( setMessageRow (null) setResetRequested (false) setCommittedThumbnailBase (String (row.attributes.thumbnailBase ?? '')) + setThumbnailObjectUrl (row.thumbnailObjectUrl) }, [row]) + useEffect (() => () => { + if (thumbnailObjectUrl != null) + URL.revokeObjectURL (thumbnailObjectUrl) + }, [thumbnailObjectUrl]) + const displayRow = messageRow ?? row const resetDraft = useMemo ( () => buildResetDraft (row), @@ -94,7 +111,8 @@ const PostImportRowForm: FC = ( || (sameDraft (draft, resetDraft) && sameProvenance (row.provenance, row.resetSnapshot.provenance) && sameTagSources (row.tagSources, row.resetSnapshot.tagSources) - && row.metadataUrl === row.resetSnapshot.metadataUrl) + && row.metadataUrl === row.resetSnapshot.metadataUrl + && sameWarnings (displayRow, row.resetSnapshot)) const update = ( key: Key, @@ -122,8 +140,11 @@ const PostImportRowForm: FC = ( setResetRequested (true) setMessageRow (null) setCommittedThumbnailBase (resetDraft.thumbnailBase) + if (thumbnailObjectUrl != null) + URL.revokeObjectURL (thumbnailObjectUrl) + setThumbnailObjectUrl (undefined) return false - }, [controls, resetDisabled, resetDraft]) + }, [controls, resetDisabled, resetDraft, thumbnailObjectUrl]) const save = useCallback (async (): Promise => { setSaving (true) @@ -163,7 +184,7 @@ const PostImportRowForm: FC = (
@@ -191,7 +212,34 @@ const PostImportRowForm: FC = ( if (draft.thumbnailBase !== committedThumbnailBase) setCommittedThumbnailBase (draft.thumbnailBase) }} - onChange={value => update ('thumbnailBase', value)}/> + onChange={value => { + if (value && thumbnailObjectUrl != null) + { + URL.revokeObjectURL (thumbnailObjectUrl) + setThumbnailObjectUrl (undefined) + update ('thumbnailFile', undefined) + } + update ('thumbnailBase', value) + }}/> + {!(draft.thumbnailBase) && ( + { + const file = event.target.files?.[0] + if (thumbnailObjectUrl != null) + URL.revokeObjectURL (thumbnailObjectUrl) + if (file == null) + { + setThumbnailObjectUrl (undefined) + update ('thumbnailFile', undefined) + return + } + const objectUrl = URL.createObjectURL (file) + setThumbnailObjectUrl (objectUrl) + update ('thumbnailFile', file) + }}/>)} } core={{ title: { diff --git a/frontend/src/lib/postImportRows.ts b/frontend/src/lib/postImportRows.ts index 25e0c58..cb9a6a6 100644 --- a/frontend/src/lib/postImportRows.ts +++ b/frontend/src/lib/postImportRows.ts @@ -2,6 +2,8 @@ import type { PostImportEditableDraft, PostImportResultRow, PostImportRow } from '@/lib/postImportTypes' +const THUMBNAIL_MISSING_WARNING = 'サムネールなし' + export const isExistingSkipRow = (row: PostImportRow): boolean => row.skipReason === 'existing' @@ -55,6 +57,29 @@ const buildResetSnapshot = (row: PostImportRow) => ({ baseWarnings: [...row.baseWarnings], metadataUrl: row.metadataUrl }) +const deduped = (values: string[]): string[] => + [...new Set (values)] + + +const thumbnailWarnings = (row: PostImportRow): string[] => { + const current = row.fieldWarnings.thumbnailBase ?? [] + const others = current.filter (message => message !== THUMBNAIL_MISSING_WARNING) + return row.attributes.thumbnailBase || row.thumbnailFile != null + ? others + : deduped ([...others, THUMBNAIL_MISSING_WARNING]) +} + + +export const applyThumbnailWarning = (row: PostImportRow): PostImportRow => ({ + ...row, + fieldWarnings: { + ...row.fieldWarnings, + thumbnailBase: thumbnailWarnings (row) } }) + + +export const applyThumbnailWarnings = (rows: PostImportRow[]): PostImportRow[] => + rows.map (row => applyThumbnailWarning (row)) + export const processableImportRows = (rows: PostImportRow[]): PostImportRow[] => validatableImportRows (rows).filter (row => { @@ -182,6 +207,7 @@ export const buildNextEditedRow = ( ...editingRow, url: draft.url, attributes: nextAttributes, + thumbnailFile: draft.thumbnailFile, provenance: { ...nextProvenance, url: urlChanged ? 'manual' : (editingRow.provenance.url ?? 'manual') }, @@ -258,6 +284,7 @@ export const mergeValidatedImportRows = ( tagSources: row.tagSources, skipReason: row.skipReason, existingPostId: row.existingPostId, + existingPost: row.existingPost, fieldWarnings, baseWarnings: row.baseWarnings.length > 0 || row.metadataUrl !== previous.metadataUrl @@ -294,6 +321,7 @@ export const mergeImportResults = ( skipReason: undefined, createdPostId: result.post.id, existingPostId: undefined, + existingPost: undefined, fieldWarnings: result.fieldWarnings ?? row.fieldWarnings, baseWarnings: result.baseWarnings ?? row.baseWarnings, importErrors: result.errors } @@ -305,6 +333,7 @@ export const mergeImportResults = ( skipReason: 'existing', createdPostId: undefined, existingPostId: result.existingPostId, + existingPost: result.existingPost ?? row.existingPost, fieldWarnings: result.fieldWarnings ?? row.fieldWarnings, baseWarnings: result.baseWarnings ?? row.baseWarnings, importErrors: result.errors } @@ -316,6 +345,7 @@ export const mergeImportResults = ( skipReason: undefined, createdPostId: undefined, existingPostId: undefined, + existingPost: undefined, fieldWarnings: result.fieldWarnings ?? row.fieldWarnings, baseWarnings: result.baseWarnings ?? row.baseWarnings, importErrors: result.errors } @@ -336,6 +366,7 @@ export const retryImportRow = ( : row) export const initialisePreviewRows = (rows: PostImportRow[]): PostImportRow[] => - rows.map (row => ({ - ...row, - resetSnapshot: buildResetSnapshot (row) })) + applyThumbnailWarnings ( + rows.map (row => ({ + ...row, + resetSnapshot: buildResetSnapshot (row) }))) diff --git a/frontend/src/lib/postImportStorage.ts b/frontend/src/lib/postImportStorage.ts index 9e9ae17..d67f8e1 100644 --- a/frontend/src/lib/postImportStorage.ts +++ b/frontend/src/lib/postImportStorage.ts @@ -1,4 +1,5 @@ import type { PostImportOrigin, + PostImportExistingPost, PostImportResetSnapshot, PostImportRow, PostImportSession, @@ -200,6 +201,25 @@ const sanitiseResetSnapshot = (value: unknown): PostImportResetSnapshot | null = metadataUrl: value.metadataUrl as string | undefined } } +const sanitiseExistingPost = (value: unknown): PostImportExistingPost | undefined => { + if (value == null) + return undefined + if (!(isPlainObject (value))) + return undefined + if (!(isPositiveInteger (value.id))) + return undefined + if (typeof value.title !== 'string' || typeof value.url !== 'string') + return undefined + if (value.thumbnailUrl != null && typeof value.thumbnailUrl !== 'string') + return undefined + + return { + id: Number (value.id), + title: value.title, + url: value.url, + thumbnailUrl: value.thumbnailUrl as string | undefined } +} + const sanitiseRow = (value: unknown): PostImportRow | null => { if (!(isPlainObject (value))) @@ -289,6 +309,7 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { skipReason: value.skipReason ?? undefined, existingPostId: isPositiveInteger (value.existingPostId) ? Number (value.existingPostId) : undefined, + existingPost: sanitiseExistingPost (value.existingPost), metadataUrl: typeof value.metadataUrl === 'string' ? value.metadataUrl : undefined, resetSnapshot, createdPostId: @@ -297,6 +318,26 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { recoverable: value.recoverable === true ? true : undefined } } +const sanitiseSession = (value: unknown): PostImportSession | null => { + if (!(isPlainObject (value))) + return null + if (value.version !== SESSION_VERSION || !(Array.isArray (value.rows))) + return null + if (typeof value.expiresAt !== 'string' || isExpiredSession (value.expiresAt)) + return null + + const rows = value.rows.map (sanitiseRow) + if (rows.some (row => row == null)) + return null + + return { + version: SESSION_VERSION, + expiresAt: value.expiresAt, + source: typeof value.source === 'string' ? value.source : '', + rows: rows as PostImportRow[], + repairMode: value.repairMode === 'failed' ? 'failed' : 'all' } +} + const isExpiredSession = (expiresAt: string): boolean => { const value = Date.parse (expiresAt) @@ -332,11 +373,7 @@ export const cleanupExpiredPostImportSessions = ( try { - const value = JSON.parse (raw) as { expiresAt?: string - rows?: unknown[] } - if (typeof value.expiresAt !== 'string' - || isExpiredSession (value.expiresAt) - || !(Array.isArray (value.rows))) + if (sanitiseSession (JSON.parse (raw)) == null) { sessionStorage.removeItem (key) --i @@ -422,25 +459,13 @@ export const loadPostImportSession = ( try { - const value = JSON.parse (raw) as Partial - if (value.version !== SESSION_VERSION || !(Array.isArray (value.rows))) - return null - if (typeof value.expiresAt !== 'string' || isExpiredSession (value.expiresAt)) + const session = sanitiseSession (JSON.parse (raw)) + if (session == null) { removeStorage (sessionKey (validatedId), onError) return null } - - const rows = value.rows.map (sanitiseRow) - if (rows.some (row => row == null)) - return null - - return { - version: SESSION_VERSION, - expiresAt: value.expiresAt, - source: typeof value.source === 'string' ? value.source : '', - rows: rows as PostImportRow[], - repairMode: value.repairMode === 'failed' ? 'failed' : 'all' } + return session } catch { diff --git a/frontend/src/lib/postImportTypes.ts b/frontend/src/lib/postImportTypes.ts index e707085..9969ec9 100644 --- a/frontend/src/lib/postImportTypes.ts +++ b/frontend/src/lib/postImportTypes.ts @@ -18,6 +18,12 @@ export type PostImportResetSnapshot = { baseWarnings: string[] metadataUrl?: string } +export type PostImportExistingPost = { + id: number + title: string + url: string + thumbnailUrl?: string } + export type PostImportRow = { sourceRow: number url: string @@ -31,6 +37,7 @@ export type PostImportRow = { status: 'ready' | 'warning' | 'error' skipReason?: PostImportSkipReason existingPostId?: number + existingPost?: PostImportExistingPost metadataUrl?: string resetSnapshot: PostImportResetSnapshot createdPostId?: number @@ -51,6 +58,7 @@ export type PostImportResultRow = sourceRow: number status: 'skipped' existingPostId: number + existingPost?: PostImportExistingPost fieldWarnings?: Record baseWarnings?: string[] errors?: Record } @@ -81,6 +89,7 @@ export type PostImportEditableDraft = { originalCreatedFrom: string originalCreatedBefore: string tags: string - parentPostIds: string } + parentPostIds: string + thumbnailFile?: File } export type StorageErrorHandler = (message: string) => void diff --git a/frontend/src/lib/postNewQueryState.ts b/frontend/src/lib/postNewQueryState.ts index 413fd16..f7c5ac8 100644 --- a/frontend/src/lib/postNewQueryState.ts +++ b/frontend/src/lib/postNewQueryState.ts @@ -9,6 +9,7 @@ import type { } from '@/lib/postImportTypes' type QueryRowState = { + source_row: number title: string thumbnail_base: string original_created_from: string @@ -34,6 +35,7 @@ type EncodedSkip = 'e' | 'm' type EncodedImportStatus = 'p' | 'c' | 's' | 'f' type EncodedRowState = { + n?: number u?: string t?: string h?: string @@ -241,6 +243,7 @@ const manualFields = (row: PostImportRow): string[] => const baseRowState = (row: PostImportRow): QueryRowState => ({ + source_row: row.sourceRow, title: String (row.attributes.title ?? ''), thumbnail_base: String (row.attributes.thumbnailBase ?? ''), original_created_from: String (row.attributes.originalCreatedFrom ?? ''), @@ -257,10 +260,7 @@ const baseRowState = (row: PostImportRow): QueryRowState => ({ recoverable: row.recoverable ?? null }) -const buildRow = ( - sourceRow: number, - state: FullRowState, -): PostImportRow => { +const buildRow = (state: FullRowState): PostImportRow => { const manual = new Set (state.manual) const provenance: Record = { url: 'manual', @@ -286,7 +286,7 @@ const buildRow = ( parentPostIds: state.parent_post_ids } return { - sourceRow, + sourceRow: state.source_row, url: state.url, attributes, fieldWarnings: { }, @@ -336,6 +336,7 @@ const parseEncodedRow = ( const existingPostId = row['e'] const createdPostId = row['c'] const recoverable = row['r'] + const sourceRow = row['n'] if (requireUrl && typeof url !== 'string') return null @@ -357,6 +358,11 @@ const parseEncodedRow = ( return null if (recoverable != null && typeof recoverable !== 'boolean') return null + if (sourceRow != null + && !(typeof sourceRow === 'number' + && Number.isInteger (sourceRow) + && sourceRow > 0)) + return null const stringFields = [ ['t', 'title'], @@ -385,6 +391,10 @@ const parseEncodedRow = ( } return { + source_row: + typeof sourceRow === 'number' + ? sourceRow + : 1, url: typeof url === 'string' ? url : '', title: parsedStrings.title, thumbnail_base: parsedStrings.thumbnail_base, @@ -412,7 +422,8 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { key => key === 'url' || key === 'session_id') if (hasOnlyUrl) { - const row = buildRow (1, { + const row = buildRow ({ + source_row: 1, url, title: '', thumbnail_base: '', @@ -449,7 +460,8 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { if (importStatus != null && !(isValidImportStatus (importStatus))) return null - const row = buildRow (1, { + const row = buildRow ({ + source_row: 1, url, title: params.get ('title') ?? '', thumbnail_base: params.get ('thumbnail_base') ?? '', @@ -508,7 +520,7 @@ const parseMetaRows = ( if (fullRows.some (row => row == null)) return null - const rows = fullRows.map ((row, index) => buildRow (index + 1, row as FullRowState)) + const rows = fullRows.map (row => buildRow (row as FullRowState)) return { rows, source: urls.join ('\n'), @@ -539,7 +551,7 @@ const parseFullState = async (stateValue: string): Promise row == null)) return null - const rows = fullRows.map ((row, index) => buildRow (index + 1, row as FullRowState)) + const rows = fullRows.map (row => buildRow (row as FullRowState)) return { rows, source: rows.map (row => row.url).join ('\n'), @@ -594,6 +606,7 @@ const encodeRowState = ( const base = baseRowState (row) const encoded: EncodedRowState = includeUrl ? { u: row.url } : { } + encoded.n = base.source_row if (base.title !== '') encoded.t = base.title if (base.thumbnail_base !== '') @@ -684,19 +697,19 @@ const serialiseCompressedRows = async ( rows: PostImportRow[], ): Promise => { try - { - for (let count = rows.length; count > 0; --count) - { - const nextRows = rows.slice (0, count) - const encoded = encodeStateRows (nextRows) - const state = await encodeCompressedState (JSON.stringify (encoded)) - const payload = state.slice (COMPRESSED_STATE_PREFIX.length) - if (payload.length <= MAX_COMPRESSED_STATE_LENGTH) - return { - path: `/posts/new?state=${ state }`, - complete: count === rows.length } - } - } + { + for (let count = rows.length; count > 0; --count) + { + const nextRows = rows.slice (0, count) + const encoded = encodeStateRows (nextRows) + const state = await encodeCompressedState (JSON.stringify (encoded)) + const path = `/posts/new?state=${ state }` + if (path.length <= MAX_COMPRESSED_STATE_LENGTH) + return { + path, + complete: count === rows.length } + } + } catch { } diff --git a/frontend/src/pages/posts/PostImportReviewPage.tsx b/frontend/src/pages/posts/PostImportReviewPage.tsx index b2616ad..81f1b8b 100644 --- a/frontend/src/pages/posts/PostImportReviewPage.tsx +++ b/frontend/src/pages/posts/PostImportReviewPage.tsx @@ -1,7 +1,7 @@ -import { useQueries } from '@tanstack/react-query' import { AnimatePresence, motion } from 'framer-motion' import { useEffect, useMemo, useRef, useState } from 'react' import { Helmet } from 'react-helmet-async' +import { ChevronRight } from 'lucide-react' import { useLocation, useNavigate } from 'react-router-dom' import PageTitle from '@/components/common/PageTitle' @@ -15,7 +15,6 @@ import { toast } from '@/components/ui/use-toast' import { SITE_TITLE } from '@/config' import { apiGet, apiPost, isApiError } from '@/lib/api' import useDialogue from '@/lib/dialogues/useDialogue' -import { fetchPost } from '@/lib/posts' import { appendPostNewSessionId, parsePostNewState, @@ -23,6 +22,7 @@ import { serialisePostNewState, } from '@/lib/postNewQueryState' import { + applyThumbnailWarning, buildNextEditedRow, canEditReviewRow, canRetryResultRow, @@ -40,11 +40,11 @@ import { resultRowMessages, reviewSummaryCounts, retryImportRow, - savePostImportSession, loadPostImportSession, + savePostImportSession, } from '@/lib/postImportSession' -import { postsKeys } from '@/lib/queryKeys' import { useClientBehaviourSettings } from '@/lib/useClientBehaviourSettings' +import { cn } from '@/lib/utils' import { canEditContent } from '@/lib/users' import Forbidden from '@/pages/Forbidden' @@ -56,7 +56,7 @@ import type { PostImportRow, PostImportSession, } from '@/lib/postImportSession' -import type { Post, User } from '@/types' +import type { User } from '@/types' type Props = { user: User | null } @@ -71,12 +71,20 @@ type PreviewResponse = { tags?: string fieldWarnings?: Record baseWarnings?: string[] - existingPost?: { id: number } | null } + existingPost?: { + id: number + title: string + url: string + thumbnailUrl?: string } | null } type BulkApiRow = { status: 'created' | 'skipped' | 'failed' post?: { id: number } - existingPost?: { id: number } | null + existingPost?: { + id: number + title: string + url: string + thumbnailUrl?: string } | null fieldWarnings?: Record baseWarnings?: string[] errors?: Record @@ -84,7 +92,7 @@ type BulkApiRow = { recoverable?: boolean } type ExistingSkippedRowsProps = { - open: boolean + id: string rows: PostImportRow[] } const isRepairRow = (row: PostImportRow): boolean => @@ -117,6 +125,8 @@ const buildDryRunFormData = (row: PostImportRow): FormData => { String (row.attributes.originalCreatedBefore ?? '')) formData.append ('video_ms', String (row.attributes.videoMs ?? '')) formData.append ('duration', String (row.attributes.duration ?? '')) + if (!(row.attributes.thumbnailBase) && row.thumbnailFile != null) + formData.append ('thumbnail', row.thumbnailFile) return formData } @@ -125,7 +135,8 @@ const mergeDryRunRow = ( currentRow: PostImportRow, result: PreviewResponse, ): PostImportRow => - initialisePreviewRows ([{ + applyThumbnailWarning ( + initialisePreviewRows ([{ ...currentRow, url: result.url, attributes: { @@ -149,11 +160,12 @@ const mergeDryRunRow = ( : 'ready', skipReason: result.existingPost?.id != null ? 'existing' : undefined, existingPostId: result.existingPost?.id, + existingPost: result.existingPost ?? undefined, importStatus: currentRow.importStatus === 'created' ? 'created' : 'pending', - recoverable: undefined }])[0] + recoverable: undefined }])[0]) const indexedBulkResults = ( @@ -178,6 +190,7 @@ const indexedBulkResults = ( sourceRow: row.sourceRow, status: 'skipped', existingPostId: result.existingPost.id, + existingPost: result.existingPost, fieldWarnings: result.fieldWarnings, baseWarnings: result.baseWarnings, errors: result.errors } @@ -191,6 +204,81 @@ const indexedBulkResults = ( recoverable: result?.recoverable } }) +const serialisableRowJson = (row: PostImportRow): string => + JSON.stringify ({ + url: row.url, + sourceRow: row.sourceRow, + attributes: row.attributes, + fieldWarnings: row.fieldWarnings, + baseWarnings: row.baseWarnings, + validationErrors: row.validationErrors, + importErrors: row.importErrors, + provenance: row.provenance, + tagSources: row.tagSources, + status: row.status, + skipReason: row.skipReason, + existingPostId: row.existingPostId, + existingPost: row.existingPost, + metadataUrl: row.metadataUrl, + resetSnapshot: row.resetSnapshot, + createdPostId: row.createdPostId, + importStatus: row.importStatus, + recoverable: row.recoverable }) + +const serialisableRowsEqual = ( + left: PostImportRow[], + right: PostImportRow[], +): boolean => + left.length === right.length + && left.every ((row, index) => + serialisableRowJson (row) === serialisableRowJson (right[index])) + +const mergePreviewRow = ( + currentRow: PostImportRow, + preview: PreviewResponse, +): PostImportRow => { + const nextRow: PostImportRow = { + ...currentRow, + url: preview.url, + fieldWarnings: preview.fieldWarnings ?? { }, + baseWarnings: preview.baseWarnings ?? [], + existingPostId: preview.existingPost?.id, + existingPost: preview.existingPost ?? undefined, + skipReason: + preview.existingPost?.id != null + ? 'existing' + : currentRow.skipReason === 'manual' + ? 'manual' + : undefined, + status: + Object.keys (preview.fieldWarnings ?? { }).length > 0 + || (preview.baseWarnings?.length ?? 0) > 0 + ? 'warning' + : 'ready' } + + if (currentRow.provenance.title !== 'manual') + nextRow.attributes.title = preview.title ?? '' + if (currentRow.provenance.thumbnailBase !== 'manual') + nextRow.attributes.thumbnailBase = preview.thumbnailBase ?? '' + if (currentRow.provenance.originalCreatedFrom !== 'manual') + nextRow.attributes.originalCreatedFrom = preview.originalCreatedFrom ?? '' + if (currentRow.provenance.originalCreatedBefore !== 'manual') + nextRow.attributes.originalCreatedBefore = preview.originalCreatedBefore ?? '' + if (currentRow.provenance.tags !== 'manual') + { + nextRow.attributes.tags = preview.tags ?? '' + nextRow.tagSources = { + automatic: preview.tags ?? '', + manual: currentRow.tagSources?.manual ?? '' } + } + if (currentRow.provenance.videoMs !== 'manual') + nextRow.attributes.videoMs = preview.videoMs ?? '' + if (currentRow.provenance.duration !== 'manual') + nextRow.attributes.duration = preview.duration ?? '' + + return applyThumbnailWarning (nextRow) +} + const buildBulkFormData = (rows: PostImportRow[]): FormData => { const formData = new FormData () @@ -207,58 +295,36 @@ const buildBulkFormData = (rows: PostImportRow[]): FormData => { original_created_before: row.attributes.originalCreatedBefore ?? '', video_ms: row.attributes.videoMs ?? '', duration: row.attributes.duration ?? '' })))) + rows.forEach ((row, index) => { + if (!(row.attributes.thumbnailBase) && row.thumbnailFile != null) + formData.append (`thumbnails[${ index }]`, row.thumbnailFile) + }) return formData } -const ExistingSkippedRows: FC = ({ open, rows }) => { - const existingPostIds = useMemo ( - () => - open - ? Array.from ( - new Set ( - rows - .map (row => row.existingPostId) - .filter ((postId): postId is number => postId != null), - ), - ) - : [], - [open, rows], - ) - const posts = useQueries ({ - queries: existingPostIds.map (postId => ({ - enabled: open, - queryKey: postsKeys.show (String (postId)), - queryFn: () => fetchPost (String (postId)) })), - }) - const postsById = new Map ( - posts - .map (query => query.data) - .filter ((post): post is Post => post != null) - .map (post => [post.id, post]), - ) +const ExistingSkippedRows: FC = ({ id, rows }) => { + const posts = rows + .map (row => row.existingPost) + .filter ((post): post is NonNullable => post != null) return ( -
+
- {existingPostIds.map (postId => { - const post = postsById.get (postId) - if (post == null) - return
- + {posts.map (post => { return (
- {post.title ?? ''} + {post.title}
{post.url} @@ -294,6 +360,7 @@ const PostImportReviewPage: FC = ({ user }) => { const sessionIdRef = useRef (null) const persistedSearchRef = useRef (null) const persistSequenceRef = useRef (0) + const previewSequenceRef = useRef (0) const persistSession = async ( nextSession: PostImportSession, @@ -308,10 +375,15 @@ const PostImportReviewPage: FC = ({ user }) => { if (persistSequenceRef.current !== sequence) return sessionRef.current - savePostImportSession (sessionId, nextSession) + const saved = savePostImportSession (sessionId, nextSession) const loadedSession = loadPostImportSession (sessionId) - if (!(serialised.complete) - && loadedSession?.rows.length !== nextSession.rows.length) + const canRecoverFromUrlOnly = serialised.complete + if (!(canRecoverFromUrlOnly) + && (!(saved)) + && !(serialisableRowsEqual (loadedSession?.rows ?? [], nextSession.rows))) + return null + if (saved + && !(serialisableRowsEqual (loadedSession?.rows ?? [], nextSession.rows))) return null const persistedSession = buildSession (nextSession.rows) @@ -334,11 +406,80 @@ const PostImportReviewPage: FC = ({ user }) => { useEffect (() => { let active = true + const previewSequence = ++previewSequenceRef.current + const controllers = new Set () if (sessionRef.current != null && persistedSearchRef.current === location.search) return () => { + previewSequenceRef.current = previewSequence } + const refreshRows = async (baseSession: PostImportSession) => { + let nextIndex = 0 + const worker = async () => { + while (active) + { + const index = nextIndex + ++nextIndex + if (index >= baseSession.rows.length) + return + + const baseRow = baseSession.rows[index] + const controller = new AbortController () + controllers.add (controller) + try + { + const preview = await apiGet ('/posts/preview', { + params: { url: baseRow.url }, + signal: controller.signal }) + if (!(active) || previewSequenceRef.current !== previewSequence) + return + + const latestSession = sessionRef.current ?? baseSession + const currentRow = latestSession.rows.find ( + row => row.sourceRow === baseRow.sourceRow) + if (currentRow == null) + continue + const mergedRow = mergePreviewRow (currentRow, preview) + const nextRows = replaceImportRow (latestSession.rows, mergedRow) + const nextSession = buildSession (nextRows) + sessionRef.current = nextSession + setSession (nextSession) + } + catch (requestError) + { + if (controller.signal.aborted) + return + if (!(active) || previewSequenceRef.current !== previewSequence) + return + const latestSession = sessionRef.current ?? baseSession + const currentRow = latestSession.rows.find ( + row => row.sourceRow === baseRow.sourceRow) + if (currentRow == null) + continue + const nextRows = replaceImportRow ( + latestSession.rows, + applyThumbnailWarning ({ + ...currentRow, + existingPostId: undefined, + existingPost: undefined })) + const nextSession = buildSession (nextRows) + sessionRef.current = nextSession + setSession (nextSession) + if (!(isApiError (requestError))) + continue + } + finally + { + controllers.delete (controller) + } + } + } + + await Promise.all ( + Array.from ({ length: Math.min (4, baseSession.rows.length) }, () => worker ())) + } + const hydrate = async () => { const parsed = await parsePostNewState (location.search) if (!(active)) @@ -365,7 +506,7 @@ const PostImportReviewPage: FC = ({ user }) => { params: { url: parsed.source } }) if (!(active)) return - const rows = initialisePreviewRows ([{ + const rows = [applyThumbnailWarning (initialisePreviewRows ([{ sourceRow: 1, url: preview.url, attributes: { @@ -400,6 +541,7 @@ const PostImportReviewPage: FC = ({ user }) => { : 'ready', skipReason: preview.existingPost?.id != null ? 'existing' : undefined, existingPostId: preview.existingPost?.id, + existingPost: preview.existingPost ?? undefined, resetSnapshot: { url: preview.url, attributes: { @@ -425,7 +567,7 @@ const PostImportReviewPage: FC = ({ user }) => { automatic: preview.tags ?? '', manual: '' }, fieldWarnings: preview.fieldWarnings ?? { }, - baseWarnings: preview.baseWarnings ?? [] } }]) + baseWarnings: preview.baseWarnings ?? [] } }])[0])] const persisted = await persistSession (buildSession (rows)) if (persisted == null) return @@ -442,12 +584,14 @@ const PostImportReviewPage: FC = ({ user }) => { { await persistSession (loaded) } + void refreshRows (loaded) } void hydrate () return () => { active = false + controllers.forEach (controller => controller.abort ()) } }, [location.search, navigate]) @@ -456,13 +600,6 @@ const PostImportReviewPage: FC = ({ user }) => { sessionRef.current = session }, [session]) - useEffect (() => { - if (session == null || session.rows.length === 0) - return - if (session.rows.every (row => isCompletedReviewRow (row))) - finishImport () - }, [session]) - useEffect (() => { if (editingRow == null || session?.repairMode !== 'failed') return @@ -504,7 +641,8 @@ const PostImportReviewPage: FC = ({ user }) => { return { ...row, skipReason: checked ? 'manual' : undefined, - existingPostId: checked ? undefined : row.existingPostId } + existingPostId: checked ? undefined : row.existingPostId, + existingPost: checked ? undefined : row.existingPost } }) await persistSession (buildSession (nextRows)) } @@ -533,7 +671,9 @@ const PostImportReviewPage: FC = ({ user }) => { Object.entries (row.resetSnapshot.fieldWarnings) .map (([key, values]) => [key, [...values]])), baseWarnings: [...row.resetSnapshot.baseWarnings], - metadataUrl: row.resetSnapshot.metadataUrl } + metadataUrl: row.resetSnapshot.metadataUrl, + thumbnailFile: undefined, + thumbnailObjectUrl: undefined } : row const urlChanged = draft.url !== baseRow.url const nextRow = buildNextEditedRow (baseRow, draft, urlChanged) @@ -573,8 +713,7 @@ const PostImportReviewPage: FC = ({ user }) => { baseWarnings: [], fieldWarnings: { }, status: 'error' as const } - setMessageRow (errorRow) - return { saved: false, row: errorRow } + return { saved: false, row: applyThumbnailWarning (errorRow) } } toast ({ title: '行の再検証に失敗しました' }) @@ -666,15 +805,17 @@ const PostImportReviewPage: FC = ({ user }) => { const recoverable = recoverableRows.find ( failedRow => failedRow.sourceRow === row.sourceRow) if (recoverable == null) - return row - return { + return applyThumbnailWarning (row) + return applyThumbnailWarning ({ ...row, importStatus: 'pending', recoverable: true, validationErrors: recoverable.errors ?? { }, - importErrors: undefined } + importErrors: undefined }) }) - await persistSession (buildSession (nextRows)) + const persisted = await persistSession (buildSession (nextRows)) + if (persisted != null && processableImportRows (persisted.rows).length === 0) + finishImport () } catch { @@ -729,15 +870,17 @@ const PostImportReviewPage: FC = ({ user }) => { const recoverable = recoverableRows.find ( failedRow => failedRow.sourceRow === row.sourceRow) if (recoverable == null) - return row - return { + return applyThumbnailWarning (row) + return applyThumbnailWarning ({ ...row, importStatus: 'pending', recoverable: true, validationErrors: recoverable.errors ?? { }, - importErrors: undefined } + importErrors: undefined }) }) - await persistSession (buildSession (nextRows)) + const persisted = await persistSession (buildSession (nextRows)) + if (persisted != null && processableImportRows (persisted.rows).length === 0) + finishImport () } catch { @@ -769,8 +912,19 @@ const PostImportReviewPage: FC = ({ user }) => {
@@ -785,7 +939,9 @@ const PostImportReviewPage: FC = ({ user }) => { exit={{ height: 0, opacity: 0 }} transition={existingRowsTransition} className="overflow-hidden"> - + )}
)} @@ -848,7 +1004,7 @@ const PostImportFooter = ( 作成対象 {creatableCount}件 手動スキップ {manualSkippedCount}件 既存投稿による自動スキップ {existingSkippedCount}件 - error／未処理 {pendingOrErrorCount}件 + 登録不可／未処理 {pendingOrErrorCount}件