From 0ae41b62665bdd9a7f17984c08377068f44bd43b Mon Sep 17 00:00:00 2001 From: miteruzo Date: Fri, 17 Jul 2026 23:14:37 +0900 Subject: [PATCH] #399 --- backend/app/controllers/posts_controller.rb | 31 ++-- .../app/representations/post_compact_repr.rb | 26 ++++ backend/app/services/post_bulk_creator.rb | 32 ++--- backend/app/services/post_create_preflight.rb | 13 +- backend/app/services/post_import_previewer.rb | 10 +- .../posts/PostCreationDataFields.tsx | 8 +- .../components/posts/PostDurationField.tsx | 3 +- .../components/posts/PostThumbnailPreview.tsx | 26 +++- .../posts/import/PostImportRowForm.tsx | 83 ++++++----- .../posts/import/PostImportRowSummary.tsx | 17 ++- .../posts/import/PostImportStatusBadge.tsx | 8 +- .../import/PostImportThumbnailPreview.tsx | 3 + .../posts/import/postImportRowStatus.ts | 22 +-- frontend/src/lib/postImportRows.ts | 27 +++- frontend/src/lib/postImportStorage.ts | 133 +++++++++++++++++- frontend/src/lib/postImportTypes.ts | 4 +- frontend/src/lib/postNewQueryState.ts | 8 +- .../src/pages/posts/PostImportReviewPage.tsx | 111 +++++++-------- .../src/pages/posts/PostImportSourcePage.tsx | 106 ++++++++------ 19 files changed, 455 insertions(+), 216 deletions(-) create mode 100644 backend/app/representations/post_compact_repr.rb diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index 19886fb..bfc5dad 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -175,15 +175,23 @@ class PostsController < ApplicationController return head :unauthorized unless current_user return head :forbidden unless current_user.gte_member? - if bool?(:dry) - preflight = PostCreatePreflight.new( - attributes: post_create_attributes, - thumbnail: params[:thumbnail]).run - return render json: preflight - end + preflight = PostCreatePreflight.new( + attributes: post_create_attributes, + thumbnail: params[:thumbnail]).run + return render json: preflight if bool?(:dry) post = PostCreator.new(actor: current_user, attributes: post_create_attributes.merge( + preflight.slice( + :url, + :title, + :thumbnail_base, + :tags, + :parent_post_ids, + :original_created_from, + :original_created_before, + :duration, + :video_ms).symbolize_keys).merge( thumbnail: params[:thumbnail])).create! post.reload @@ -532,6 +540,7 @@ class PostsController < ApplicationController def parse_bulk_posts_manifest manifest = params[:posts] raise ArgumentError, 'posts は必須です．' if manifest.blank? + raise ArgumentError, 'posts は JSON 文字列で指定してください．' unless manifest.is_a?(String) posts = JSON.parse(manifest) raise ArgumentError, 'posts は配列で指定してください．' unless posts.is_a?(Array) @@ -569,15 +578,7 @@ class PostsController < ApplicationController return nil if post_id.blank? post = Post.with_attached_thumbnail.find_by(id: post_id) - return nil if post.nil? - - { id: post.id, - title: post.title, - url: post.url, - thumbnail_url: - if post.thumbnail.attached? - rails_storage_proxy_url(post.thumbnail, only_path: false) - end } + PostCompactRepr.base(post) end def sync_parent_posts! post, parent_post_ids diff --git a/backend/app/representations/post_compact_repr.rb b/backend/app/representations/post_compact_repr.rb new file mode 100644 index 0000000..06f88f7 --- /dev/null +++ b/backend/app/representations/post_compact_repr.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + + +module PostCompactRepr + module_function + + def base post + return nil if post.nil? + + { + id: post.id, + title: post.title, + url: post.url, + thumbnail_url: thumbnail_url(post) } + end + + def thumbnail_url post + return nil unless post.thumbnail.attached? + + Rails.application.routes.url_helpers.rails_storage_proxy_url( + post.thumbnail, + only_path: false) + rescue + nil + end +end diff --git a/backend/app/services/post_bulk_creator.rb b/backend/app/services/post_bulk_creator.rb index 35b2454..4c75752 100644 --- a/backend/app/services/post_bulk_creator.rb +++ b/backend/app/services/post_bulk_creator.rb @@ -55,14 +55,21 @@ class PostBulkCreator errors: e.record.errors.to_hash, base_errors: e.record.errors[:base] } rescue ActiveRecord::RecordNotUnique => e - raise unless e.message.include?('index_posts_on_url') - - existing_post = existing_post_for_race(attributes) - raise if existing_post.nil? + if e.message.include?('index_posts_on_url') + existing_post = existing_post_for_race(attributes) + return { + status: 'skipped', + existing_post: existing_post } if existing_post.present? + end + Rails.logger.error( + "post_bulk_creator_record_not_unique #{ { error: e.class.name, + message: e.message }.to_json }") { - status: 'skipped', - existing_post: existing_post } + status: 'failed', + recoverable: false, + errors: { base: ['登録中にエラーが発生しました．'] }, + base_errors: [] } rescue Tag::NicoTagNormalisationError { status: 'failed', @@ -134,17 +141,6 @@ class PostBulkCreator end def compact_existing_post post - return nil if post.nil? - - { - id: post.id, - title: post.title, - url: post.url, - thumbnail_url: - if post.thumbnail.attached? - Rails.application.routes.url_helpers.rails_storage_proxy_url( - post.thumbnail, - only_path: false) - end } + PostCompactRepr.base(post) end end diff --git a/backend/app/services/post_create_preflight.rb b/backend/app/services/post_create_preflight.rb index 1ea24f2..fe84b94 100644 --- a/backend/app/services/post_create_preflight.rb +++ b/backend/app/services/post_create_preflight.rb @@ -88,17 +88,6 @@ class PostCreatePreflight return nil if post_id.blank? post = Post.with_attached_thumbnail.find_by(id: post_id) - return nil if post.nil? - - { - id: post.id, - title: post.title, - url: post.url, - thumbnail_url: - if post.thumbnail.attached? - Rails.application.routes.url_helpers.rails_storage_proxy_url( - post.thumbnail, - only_path: false) - end } + PostCompactRepr.base(post) end end diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index f0ccc20..cbe15f8 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -255,7 +255,15 @@ class PostImportPreviewer def sanitise_metadata_url value return nil unless value.is_a?(String) - PostUrlNormaliser.normalise(value) + stripped = value.strip + return nil if stripped.blank? + + uri = URI.parse(stripped) + return nil unless uri.is_a?(URI::HTTP) && uri.host.present? + + stripped + rescue URI::InvalidURIError + nil end def sanitise_metadata_time value diff --git a/frontend/src/components/posts/PostCreationDataFields.tsx b/frontend/src/components/posts/PostCreationDataFields.tsx index b53ae54..7348c5d 100644 --- a/frontend/src/components/posts/PostCreationDataFields.tsx +++ b/frontend/src/components/posts/PostCreationDataFields.tsx @@ -17,13 +17,15 @@ type Props = { type?: string placeholder?: string } thumbnailField: ReactNode - core: PostCoreDataFieldsProps } + core: PostCoreDataFieldsProps + extraFields?: ReactNode } const PostCreationDataFields: FC = ( { url, thumbnailField, - core }, + core, + extraFields }, ) => ( <> = ( {thumbnailField} + + {extraFields} ) export default PostCreationDataFields diff --git a/frontend/src/components/posts/PostDurationField.tsx b/frontend/src/components/posts/PostDurationField.tsx index 972a290..5dcda0b 100644 --- a/frontend/src/components/posts/PostDurationField.tsx +++ b/frontend/src/components/posts/PostDurationField.tsx @@ -21,8 +21,7 @@ const PostDurationField: FC = ( onChange={onChange} errors={errors} disabled={disabled} - type="text" - placeholder="例: 2 / 2.5 / 1:23"/> + type="text"/> ) export default PostDurationField diff --git a/frontend/src/components/posts/PostThumbnailPreview.tsx b/frontend/src/components/posts/PostThumbnailPreview.tsx index 66e64dd..5383f13 100644 --- a/frontend/src/components/posts/PostThumbnailPreview.tsx +++ b/frontend/src/components/posts/PostThumbnailPreview.tsx @@ -6,6 +6,7 @@ import type { FC } from 'react' type Props = { url: string + file?: File alt?: string className?: string referrerPolicy?: 'no-referrer' } @@ -13,17 +14,36 @@ type Props = { const PostThumbnailPreview: FC = ( { url, + file, alt = 'サムネール', className = 'h-16 w-16', referrerPolicy }, ) => { const [failed, setFailed] = useState (false) + const [fileUrl, setFileUrl] = useState (null) useEffect (() => { setFailed (false) - }, [url]) + }, [file, url]) - if (!(url) || failed) + useEffect (() => { + if (file == null) + { + setFileUrl (null) + return + } + + const nextUrl = URL.createObjectURL (file) + setFileUrl (nextUrl) + + return () => { + URL.revokeObjectURL (nextUrl) + } + }, [file]) + + const resolvedUrl = url.trim () !== '' ? url : (fileUrl ?? '') + + if (resolvedUrl === '' || failed) { return (
= ( return ( {alt} } +const THUMBNAIL_MISSING_WARNING = 'サムネールなし' + const buildDraft = (row: PostImportRow): Draft => ({ url: row.url, title: String (row.attributes.title ?? ''), @@ -29,6 +33,7 @@ const buildDraft = (row: PostImportRow): Draft => ({ originalCreatedBefore: String (row.attributes.originalCreatedBefore ?? ''), tags: String (row.attributes.tags ?? ''), parentPostIds: String (row.attributes.parentPostIds ?? ''), + duration: String (row.attributes.duration ?? ''), thumbnailFile: row.thumbnailFile }) const buildResetDraft = (row: PostImportRow): Draft => ({ @@ -39,6 +44,7 @@ const buildResetDraft = (row: PostImportRow): Draft => ({ originalCreatedBefore: String (row.resetSnapshot.attributes.originalCreatedBefore ?? ''), tags: String (row.resetSnapshot.attributes.tags ?? ''), parentPostIds: String (row.resetSnapshot.attributes.parentPostIds ?? ''), + duration: String (row.resetSnapshot.attributes.duration ?? ''), thumbnailFile: undefined }) const groupedMessages = (...values: (string[] | undefined)[]): string[] => @@ -52,6 +58,8 @@ const sameDraft = (left: Draft, right: Draft): boolean => && left.originalCreatedBefore === right.originalCreatedBefore && left.tags === right.tags && left.parentPostIds === right.parentPostIds + && left.duration === right.duration + && left.thumbnailFile === right.thumbnailFile const sameProvenance = ( current: PostImportRow['provenance'], @@ -73,6 +81,17 @@ const sameWarnings = ( JSON.stringify (current.fieldWarnings) === JSON.stringify (reset.fieldWarnings) && JSON.stringify (current.baseWarnings) === JSON.stringify (reset.baseWarnings) +const thumbnailWarnings = ( + messages: string[] | undefined, + thumbnailBase: string, + thumbnailFile: File | undefined, +): string[] => { + const others = (messages ?? []).filter (message => message !== THUMBNAIL_MISSING_WARNING) + return hasThumbnailBaseValue (thumbnailBase) || thumbnailFile != null + ? others + : [...new Set ([...others, THUMBNAIL_MISSING_WARNING])] +} + const PostImportRowForm: FC = ( { row, @@ -85,8 +104,6 @@ const PostImportRowForm: FC = ( const [resetRequested, setResetRequested] = useState (false) const [committedThumbnailBase, setCommittedThumbnailBase] = useState ( () => String (row.attributes.thumbnailBase ?? '')) - const [thumbnailObjectUrl, setThumbnailObjectUrl] = useState ( - () => row.thumbnailObjectUrl) useEffect (() => { const nextDraft = buildDraft (row) @@ -94,18 +111,17 @@ const PostImportRowForm: FC = ( setMessageRow (null) setResetRequested (false) setCommittedThumbnailBase (String (row.attributes.thumbnailBase ?? '')) - setThumbnailObjectUrl (row.thumbnailObjectUrl) }, [row]) - useEffect (() => () => { - if (thumbnailObjectUrl != null) - URL.revokeObjectURL (thumbnailObjectUrl) - }, [thumbnailObjectUrl]) - const displayRow = messageRow ?? row const resetDraft = useMemo ( () => buildResetDraft (row), [row]) + const durationVisible = hasVideoTag (draft.tags) + const currentThumbnailWarnings = thumbnailWarnings ( + displayRow.fieldWarnings.thumbnailBase, + draft.thumbnailBase, + draft.thumbnailFile) const resetDisabled = saving || (sameDraft (draft, resetDraft) @@ -129,7 +145,6 @@ const PostImportRowForm: FC = ( const confirmed = await controls.confirm ({ title: '変更をリセットしますか?', - description: '現在の URL に対する自動取得直後の内容へ戻します．', confirmText: 'リセット', cancelText: '取消', variant: 'danger' }) @@ -140,11 +155,8 @@ const PostImportRowForm: FC = ( setResetRequested (true) setMessageRow (null) setCommittedThumbnailBase (resetDraft.thumbnailBase) - if (thumbnailObjectUrl != null) - URL.revokeObjectURL (thumbnailObjectUrl) - setThumbnailObjectUrl (undefined) return false - }, [controls, resetDisabled, resetDraft, thumbnailObjectUrl]) + }, [controls, resetDisabled, resetDraft]) const save = useCallback (async (): Promise => { setSaving (true) @@ -184,7 +196,11 @@ const PostImportRowForm: FC = (
@@ -204,40 +220,22 @@ const PostImportRowForm: FC = ( label="サムネール" value={draft.thumbnailBase} disabled={saving} - warnings={displayRow.fieldWarnings.thumbnailBase} + warnings={currentThumbnailWarnings} errors={groupedMessages ( displayRow.validationErrors.thumbnailBase, displayRow.importErrors?.thumbnailBase)} onBlur={() => { - if (draft.thumbnailBase !== committedThumbnailBase) + if (draft.thumbnailBase.trim () !== committedThumbnailBase.trim ()) setCommittedThumbnailBase (draft.thumbnailBase) }} - onChange={value => { - if (value && thumbnailObjectUrl != null) - { - URL.revokeObjectURL (thumbnailObjectUrl) - setThumbnailObjectUrl (undefined) - update ('thumbnailFile', undefined) - } - update ('thumbnailBase', value) - }}/> - {!(draft.thumbnailBase) && ( + onChange={value => update ('thumbnailBase', value)}/> + {!(hasThumbnailBaseValue (draft.thumbnailBase)) && ( { const file = event.target.files?.[0] - if (thumbnailObjectUrl != null) - URL.revokeObjectURL (thumbnailObjectUrl) - if (file == null) - { - setThumbnailObjectUrl (undefined) - update ('thumbnailFile', undefined) - return - } - const objectUrl = URL.createObjectURL (file) - setThumbnailObjectUrl (objectUrl) update ('thumbnailFile', file) }}/>)} } @@ -283,7 +281,18 @@ const PostImportRowForm: FC = ( disabled: saving, errors: groupedMessages ( displayRow.validationErrors.parentPostIds, - displayRow.importErrors?.parentPostIds) } }}/> + displayRow.importErrors?.parentPostIds) } }} + extraFields={ + durationVisible + ? ( + update ('duration', value)} + disabled={saving} + errors={groupedMessages ( + displayRow.validationErrors.videoMs, + displayRow.importErrors?.videoMs)}/>) + : null}/> diff --git a/frontend/src/components/posts/import/PostImportRowSummary.tsx b/frontend/src/components/posts/import/PostImportRowSummary.tsx index 2b3cfab..e355134 100644 --- a/frontend/src/components/posts/import/PostImportRowSummary.tsx +++ b/frontend/src/components/posts/import/PostImportRowSummary.tsx @@ -3,6 +3,7 @@ import { Button } from '@/components/ui/button' import PostImportThumbnailPreview from '@/components/posts/import/PostImportThumbnailPreview' import PostImportStatusBadge from '@/components/posts/import/PostImportStatusBadge' import { displayPostImportStatus } from '@/components/posts/import/postImportRowStatus' +import { hasVideoTag } from '@/lib/postImportSession' import { canEditReviewRow, canRetryResultRow } from '@/lib/postImportSession' import { cn, originalCreatedAtString } from '@/lib/utils' @@ -54,6 +55,8 @@ const PostImportRowSummary: FC = ( const retryAllowed = onRetry != null && canRetryResultRow (row) const skipChecked = row.skipReason === 'manual' const rowNumber = displayNumber ?? row.sourceRow + const duration = String (row.attributes.duration ?? '') + const showDuration = hasVideoTag (row.attributes.tags) && duration !== '' const skipControl = showSkipToggle ? (
- {String (row.attributes.title ?? '') || '（タイトル未取得）'} + {String (row.attributes.title ?? '')}
{row.url} @@ -93,6 +97,10 @@ const PostImportRowSummary: FC = (
{summaryDate (row)}
+ {showDuration && ( +
+ 動画時間 {duration} +
)} {warning && (
{warning} @@ -133,10 +141,11 @@ const PostImportRowSummary: FC = (
- {String (row.attributes.title ?? '') || '（タイトル未取得）'} + {String (row.attributes.title ?? '')}
{row.url} @@ -151,6 +160,10 @@ const PostImportRowSummary: FC = (
{summaryDate (row)}
+ {showDuration && ( +
+ 動画時間 {duration} +
)} {warning && (
{warning} diff --git a/frontend/src/components/posts/import/PostImportStatusBadge.tsx b/frontend/src/components/posts/import/PostImportStatusBadge.tsx index f2b0b39..1aea1a6 100644 --- a/frontend/src/components/posts/import/PostImportStatusBadge.tsx +++ b/frontend/src/components/posts/import/PostImportStatusBadge.tsx @@ -11,12 +11,16 @@ type Props = { const LABELS: Record = { ready: '登録可能', warning: '警告', - skipped: 'スキップ' } + skipped: 'スキップ', + created: '登録済み', + failed: '登録失敗' } const TONES: Record = { ready: 'success', warning: 'warning', - skipped: 'neutral' } + skipped: 'neutral', + created: 'success', + failed: 'warning' } const PostImportStatusBadge: FC = ({ value }) => ( diff --git a/frontend/src/components/posts/import/PostImportThumbnailPreview.tsx b/frontend/src/components/posts/import/PostImportThumbnailPreview.tsx index 3077db6..6afad5f 100644 --- a/frontend/src/components/posts/import/PostImportThumbnailPreview.tsx +++ b/frontend/src/components/posts/import/PostImportThumbnailPreview.tsx @@ -4,17 +4,20 @@ import type { FC } from 'react' type Props = { url: string + file?: File alt?: string className?: string } const PostImportThumbnailPreview: FC = ( { url, + file, alt = 'サムネール', className = 'h-16 w-16' }, ) => ( ) diff --git a/frontend/src/components/posts/import/postImportRowStatus.ts b/frontend/src/components/posts/import/postImportRowStatus.ts index 7979aa4..4e4af51 100644 --- a/frontend/src/components/posts/import/postImportRowStatus.ts +++ b/frontend/src/components/posts/import/postImportRowStatus.ts @@ -1,6 +1,11 @@ import type { PostImportRow } from '@/lib/postImportSession' -export type PostImportDisplayStatus = 'ready' | 'skipped' | 'warning' +export type PostImportDisplayStatus = + 'ready' + | 'skipped' + | 'warning' + | 'created' + | 'failed' export type PostImportBadgeValue = PostImportDisplayStatus @@ -13,10 +18,11 @@ export const displayPostImportStatus = ( ): PostImportDisplayStatus | null => (row.skipReason != null || row.importStatus === 'skipped') ? 'skipped' - : ((Object.keys (row.validationErrors ?? { }).length > 0 - || row.importStatus === 'failed' - || row.importStatus === 'created') - ? null - : ((hasWarnings (row) || row.status === 'warning') - ? 'warning' - : 'ready')) + : (row.importStatus === 'created') + ? 'created' + : ((Object.keys (row.validationErrors ?? { }).length > 0 + || row.importStatus === 'failed') + ? 'failed' + : ((hasWarnings (row) || row.status === 'warning') + ? 'warning' + : 'ready')) diff --git a/frontend/src/lib/postImportRows.ts b/frontend/src/lib/postImportRows.ts index cb9a6a6..3755efe 100644 --- a/frontend/src/lib/postImportRows.ts +++ b/frontend/src/lib/postImportRows.ts @@ -4,6 +4,13 @@ import type { PostImportEditableDraft, const THUMBNAIL_MISSING_WARNING = 'サムネールなし' +export const hasThumbnailBaseValue = (value: unknown): boolean => + typeof value === 'string' && value.trim () !== '' + +export const hasVideoTag = (value: unknown): boolean => + typeof value === 'string' + && value.split (/\s+/).includes ('動画') + export const isExistingSkipRow = (row: PostImportRow): boolean => row.skipReason === 'existing' @@ -64,7 +71,7 @@ const deduped = (values: string[]): string[] => const thumbnailWarnings = (row: PostImportRow): string[] => { const current = row.fieldWarnings.thumbnailBase ?? [] const others = current.filter (message => message !== THUMBNAIL_MISSING_WARNING) - return row.attributes.thumbnailBase || row.thumbnailFile != null + return hasThumbnailBaseValue (row.attributes.thumbnailBase) || row.thumbnailFile != null ? others : deduped ([...others, THUMBNAIL_MISSING_WARNING]) } @@ -168,6 +175,14 @@ export const resultRowWarnings = (row: PostImportRow): string[] => ...Object.values (row.fieldWarnings ?? { }).flat (), ...row.baseWarnings])] + +const isManualChange = ( + current: unknown, + next: string, +): boolean => + next !== String (current ?? '') + + export const buildNextEditedRow = ( editingRow: PostImportRow, draft: PostImportEditableDraft, @@ -183,16 +198,22 @@ export const buildNextEditedRow = ( ['thumbnailBase', draft.thumbnailBase], ['originalCreatedFrom', draft.originalCreatedFrom], ['originalCreatedBefore', draft.originalCreatedBefore], + ['duration', draft.duration], ['parentPostIds', draft.parentPostIds]] as const draftFields.forEach (([field, value]) => { nextAttributes[field] = value nextProvenance[field] = - value !== String (editingRow.attributes[field] ?? '') + isManualChange (editingRow.attributes[field], value) ? 'manual' : (editingRow.provenance[field] ?? 'automatic') }) + if (nextProvenance.duration === 'manual') + { + nextAttributes.videoMs = '' + nextProvenance.videoMs = 'manual' + } nextAttributes.tags = draft.tags - if (draft.tags !== String (editingRow.attributes.tags ?? '')) + if (isManualChange (editingRow.attributes.tags, draft.tags)) { nextProvenance.tags = 'manual' nextTagSources.manual = draft.tags diff --git a/frontend/src/lib/postImportStorage.ts b/frontend/src/lib/postImportStorage.ts index d67f8e1..c91531b 100644 --- a/frontend/src/lib/postImportStorage.ts +++ b/frontend/src/lib/postImportStorage.ts @@ -25,6 +25,31 @@ const ATTRIBUTE_KEYS = [ const PROVENANCE_KEYS = [...ATTRIBUTE_KEYS, 'url'] as const const TAG_SOURCE_KEYS = ['automatic', 'manual'] as const const WARNING_KEYS = [...ATTRIBUTE_KEYS, 'url'] as const +const ROW_KEYS = [ + 'sourceRow', + 'url', + 'attributes', + 'fieldWarnings', + 'baseWarnings', + 'validationErrors', + 'importErrors', + 'provenance', + 'tagSources', + 'status', + 'skipReason', + 'existingPostId', + 'existingPost', + 'metadataUrl', + 'resetSnapshot', + 'createdPostId', + 'importStatus', + 'recoverable'] as const +const SESSION_KEYS = [ + 'version', + 'expiresAt', + 'source', + 'rows', + 'repairMode'] as const const isPlainObject = (value: unknown): value is Record => @@ -220,10 +245,110 @@ const sanitiseExistingPost = (value: unknown): PostImportExistingPost | undefine thumbnailUrl: value.thumbnailUrl as string | undefined } } +const recalculateThumbnailWarning = (row: PostImportRow): PostImportRow => { + const others = (row.fieldWarnings.thumbnailBase ?? []).filter ( + key => key !== 'サムネールなし') + const thumbnailBasePresent = + typeof row.attributes.thumbnailBase === 'string' + && row.attributes.thumbnailBase.trim () !== '' + + return { + ...row, + fieldWarnings: { + ...row.fieldWarnings, + thumbnailBase: + thumbnailBasePresent + ? others + : [...new Set ([...others, 'サムネールなし'])] } } +} + +const serialiseExistingPost = (value: PostImportExistingPost | undefined) => + value == null + ? undefined + : { + id: value.id, + title: value.title, + url: value.url, + thumbnailUrl: value.thumbnailUrl } + +const serialiseResetSnapshot = (value: PostImportResetSnapshot) => ({ + url: value.url, + attributes: Object.fromEntries ( + Object.entries (value.attributes).map (([key, entry]) => [key, entry])), + provenance: Object.fromEntries ( + Object.entries (value.provenance).map (([key, entry]) => [key, entry])), + tagSources: { + automatic: value.tagSources.automatic, + manual: value.tagSources.manual }, + fieldWarnings: Object.fromEntries ( + Object.entries (value.fieldWarnings).map (([key, entry]) => [key, [...entry]])), + baseWarnings: [...value.baseWarnings], + metadataUrl: value.metadataUrl }) + +export const serialisePostImportRow = (row: PostImportRow) => { + const thumbnailBaseWarnings = (() => { + const others = (row.fieldWarnings.thumbnailBase ?? []).filter ( + message => message !== 'サムネールなし') + const thumbnailBasePresent = + typeof row.attributes.thumbnailBase === 'string' + && row.attributes.thumbnailBase.trim () !== '' + return thumbnailBasePresent + ? others + : [...new Set ([...others, 'サムネールなし'])] + }) () + + return { + sourceRow: row.sourceRow, + url: row.url, + attributes: Object.fromEntries ( + Object.entries (row.attributes).map (([key, entry]) => [key, entry])), + fieldWarnings: Object.fromEntries ( + Object.entries ({ + ...row.fieldWarnings, + thumbnailBase: thumbnailBaseWarnings }).map (([key, entry]) => [key, [...entry]])), + baseWarnings: [...row.baseWarnings], + validationErrors: Object.fromEntries ( + Object.entries (row.validationErrors).map (([key, entry]) => [key, [...entry]])), + importErrors: + row.importErrors == null + ? undefined + : Object.fromEntries ( + Object.entries (row.importErrors).map (([key, entry]) => [key, [...entry]])), + provenance: Object.fromEntries ( + Object.entries (row.provenance).map (([key, entry]) => [key, entry])), + tagSources: + row.tagSources == null + ? undefined + : { + automatic: row.tagSources.automatic, + manual: row.tagSources.manual }, + status: row.status, + skipReason: row.skipReason, + existingPostId: row.existingPostId, + existingPost: serialiseExistingPost (row.existingPost), + metadataUrl: row.metadataUrl, + resetSnapshot: serialiseResetSnapshot (row.resetSnapshot), + createdPostId: row.createdPostId, + importStatus: row.importStatus, + recoverable: row.recoverable } +} + +export const serialisePostImportRows = (rows: PostImportRow[]) => + rows.map (row => serialisePostImportRow (row)) + +export const serialisedPostImportRowsEqual = ( + left: PostImportRow[], + right: PostImportRow[], +): boolean => + JSON.stringify (serialisePostImportRows (left)) + === JSON.stringify (serialisePostImportRows (right)) + const sanitiseRow = (value: unknown): PostImportRow | null => { if (!(isPlainObject (value))) return null + if (!(hasOnlyKeys (value, ROW_KEYS))) + return null if (!(Number.isInteger (value.sourceRow)) || Number (value.sourceRow) <= 0) return null if (typeof value.url !== 'string') @@ -321,6 +446,8 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { const sanitiseSession = (value: unknown): PostImportSession | null => { if (!(isPlainObject (value))) return null + if (!(hasOnlyKeys (value, SESSION_KEYS))) + return null if (value.version !== SESSION_VERSION || !(Array.isArray (value.rows))) return null if (typeof value.expiresAt !== 'string' || isExpiredSession (value.expiresAt)) @@ -334,7 +461,7 @@ const sanitiseSession = (value: unknown): PostImportSession | null => { version: SESSION_VERSION, expiresAt: value.expiresAt, source: typeof value.source === 'string' ? value.source : '', - rows: rows as PostImportRow[], + rows: (rows as PostImportRow[]).map (row => recalculateThumbnailWarning (row)), repairMode: value.repairMode === 'failed' ? 'failed' : 'all' } } @@ -438,7 +565,9 @@ export const savePostImportSession = ( return writeStorage ( sessionKey (validatedId), JSON.stringify ({ - ...session, + source: session.source, + rows: serialisePostImportRows (session.rows), + repairMode: session.repairMode, version: SESSION_VERSION, expiresAt: new Date (Date.now () + SESSION_MAX_AGE_MS).toISOString () }), onError) diff --git a/frontend/src/lib/postImportTypes.ts b/frontend/src/lib/postImportTypes.ts index 9969ec9..baf9423 100644 --- a/frontend/src/lib/postImportTypes.ts +++ b/frontend/src/lib/postImportTypes.ts @@ -43,8 +43,7 @@ export type PostImportRow = { createdPostId?: number importStatus?: PostImportStatus recoverable?: boolean - thumbnailFile?: File - thumbnailObjectUrl?: string } + thumbnailFile?: File } export type PostImportResultRow = | { @@ -88,6 +87,7 @@ export type PostImportEditableDraft = { thumbnailBase: string originalCreatedFrom: string originalCreatedBefore: string + duration: string tags: string parentPostIds: string thumbnailFile?: File } diff --git a/frontend/src/lib/postNewQueryState.ts b/frontend/src/lib/postNewQueryState.ts index f7c5ac8..1d358cf 100644 --- a/frontend/src/lib/postNewQueryState.ts +++ b/frontend/src/lib/postNewQueryState.ts @@ -237,6 +237,7 @@ const manualFields = (row: PostImportRow): string[] => 'thumbnailBase', 'originalCreatedFrom', 'originalCreatedBefore', + 'duration', 'tags', 'parentPostIds'] .filter (field => row.provenance[field] === 'manual') @@ -268,8 +269,8 @@ const buildRow = (state: FullRowState): PostImportRow => { thumbnailBase: manual.has ('thumbnailBase') ? 'manual' : 'automatic', originalCreatedFrom: manual.has ('originalCreatedFrom') ? 'manual' : 'automatic', originalCreatedBefore: manual.has ('originalCreatedBefore') ? 'manual' : 'automatic', - videoMs: 'automatic', - duration: 'automatic', + videoMs: manual.has ('duration') ? 'manual' : 'automatic', + duration: manual.has ('duration') ? 'manual' : 'automatic', tags: manual.has ('tags') ? 'manual' : 'automatic', parentPostIds: manual.has ('parentPostIds') ? 'manual' : 'automatic' } const tagSources = provenance.tags === 'manual' @@ -280,7 +281,7 @@ const buildRow = (state: FullRowState): PostImportRow => { thumbnailBase: state.thumbnail_base, originalCreatedFrom: state.original_created_from, originalCreatedBefore: state.original_created_before, - videoMs: state.video_ms, + videoMs: manual.has ('duration') ? '' : state.video_ms, duration: state.duration, tags: state.tags, parentPostIds: state.parent_post_ids } @@ -651,6 +652,7 @@ export const hasPostNewReviewState = (search: string): boolean => { || params.has ('urls') || params.has ('meta') || params.has ('url') + || parsePostNewSessionId (search) != null } diff --git a/frontend/src/pages/posts/PostImportReviewPage.tsx b/frontend/src/pages/posts/PostImportReviewPage.tsx index 3bfb736..3aa2e6f 100644 --- a/frontend/src/pages/posts/PostImportReviewPage.tsx +++ b/frontend/src/pages/posts/PostImportReviewPage.tsx @@ -29,6 +29,7 @@ import { clearPostImportSession, clearPostImportSourceDraft, generatePostImportSessionId, + hasThumbnailBaseValue, initialisePreviewRows, isCompletedReviewRow, isExistingSkipRow, @@ -41,6 +42,7 @@ import { reviewSummaryCounts, retryImportRow, loadPostImportSession, + serialisedPostImportRowsEqual, savePostImportSession, } from '@/lib/postImportSession' import { useClientBehaviourSettings } from '@/lib/useClientBehaviourSettings' @@ -125,7 +127,7 @@ const buildDryRunFormData = (row: PostImportRow): FormData => { String (row.attributes.originalCreatedBefore ?? '')) formData.append ('video_ms', String (row.attributes.videoMs ?? '')) formData.append ('duration', String (row.attributes.duration ?? '')) - if (!(row.attributes.thumbnailBase) && row.thumbnailFile != null) + if (!(hasThumbnailBaseValue (row.attributes.thumbnailBase)) && row.thumbnailFile != null) formData.append ('thumbnail', row.thumbnailFile) return formData } @@ -174,6 +176,12 @@ const indexedBulkResults = ( ): PostImportResultRow[] => rows.map ((row, index) => { const result = results[index] + const errors = + result?.baseErrors?.length + ? { + ...(result.errors ?? { }), + base: [...new Set ([...(result.errors?.base ?? []), ...result.baseErrors])] } + : result?.errors if (result?.status === 'created' && result.post != null) { return { @@ -182,7 +190,7 @@ const indexedBulkResults = ( post: result.post, fieldWarnings: result.fieldWarnings, baseWarnings: result.baseWarnings, - errors: result.errors } + errors } } if (result?.status === 'skipped' && result.existingPost != null) { @@ -193,52 +201,28 @@ const indexedBulkResults = ( existingPost: result.existingPost, fieldWarnings: result.fieldWarnings, baseWarnings: result.baseWarnings, - errors: result.errors } + errors } } return { sourceRow: row.sourceRow, status: 'failed', fieldWarnings: result?.fieldWarnings, baseWarnings: result?.baseWarnings, - errors: result?.errors, + errors, recoverable: result?.recoverable } }) -const serialisableRowJson = (row: PostImportRow): string => - JSON.stringify ({ - url: row.url, - sourceRow: row.sourceRow, - attributes: row.attributes, - fieldWarnings: row.fieldWarnings, - baseWarnings: row.baseWarnings, - validationErrors: row.validationErrors, - importErrors: row.importErrors, - provenance: row.provenance, - tagSources: row.tagSources, - status: row.status, - skipReason: row.skipReason, - existingPostId: row.existingPostId, - existingPost: row.existingPost, - metadataUrl: row.metadataUrl, - resetSnapshot: row.resetSnapshot, - createdPostId: row.createdPostId, - importStatus: row.importStatus, - recoverable: row.recoverable }) - -const serialisableRowsEqual = ( - left: PostImportRow[], - right: PostImportRow[], -): boolean => - left.length === right.length - && left.every ((row, index) => - serialisableRowJson (row) === serialisableRowJson (right[index])) - const mergePreviewRow = ( currentRow: PostImportRow, preview: PreviewResponse, ): PostImportRow => { const nextRow: PostImportRow = { ...currentRow, + attributes: { ...currentRow.attributes }, + provenance: { ...currentRow.provenance }, + tagSources: { + automatic: currentRow.tagSources?.automatic ?? '', + manual: currentRow.tagSources?.manual ?? '' }, url: preview.url, fieldWarnings: preview.fieldWarnings ?? { }, baseWarnings: preview.baseWarnings ?? [], @@ -267,9 +251,7 @@ const mergePreviewRow = ( if (currentRow.provenance.tags !== 'manual') { nextRow.attributes.tags = preview.tags ?? '' - nextRow.tagSources = { - automatic: preview.tags ?? '', - manual: currentRow.tagSources?.manual ?? '' } + nextRow.tagSources.automatic = preview.tags ?? '' } if (currentRow.provenance.videoMs !== 'manual') nextRow.attributes.videoMs = preview.videoMs ?? '' @@ -296,7 +278,7 @@ const buildBulkFormData = (rows: PostImportRow[]): FormData => { video_ms: row.attributes.videoMs ?? '', duration: row.attributes.duration ?? '' })))) rows.forEach ((row, index) => { - if (!(row.attributes.thumbnailBase) && row.thumbnailFile != null) + if (!(hasThumbnailBaseValue (row.attributes.thumbnailBase)) && row.thumbnailFile != null) formData.append (`thumbnails[${ index }]`, row.thumbnailFile) }) return formData @@ -380,10 +362,10 @@ const PostImportReviewPage: FC = ({ user }) => { const canRecoverFromUrlOnly = serialised.complete if (!(canRecoverFromUrlOnly) && (!(saved)) - && !(serialisableRowsEqual (loadedSession?.rows ?? [], nextSession.rows))) + && !(serialisedPostImportRowsEqual (loadedSession?.rows ?? [], nextSession.rows))) return null if (saved - && !(serialisableRowsEqual (loadedSession?.rows ?? [], nextSession.rows))) + && !(serialisedPostImportRowsEqual (loadedSession?.rows ?? [], nextSession.rows))) return null const persistedSession = buildSession (nextSession.rows) @@ -425,6 +407,11 @@ const PostImportReviewPage: FC = ({ user }) => { return const baseRow = baseSession.rows[index] + if (baseRow.skipReason === 'manual' + || baseRow.importStatus === 'created' + || baseRow.importStatus === 'skipped' + || isNonRecoverableFailedRow (baseRow)) + continue const controller = new AbortController () controllers.add (controller) try @@ -441,10 +428,8 @@ const PostImportReviewPage: FC = ({ user }) => { if (currentRow == null) continue const mergedRow = mergePreviewRow (currentRow, preview) - const nextRows = replaceImportRow (latestSession.rows, mergedRow) - const nextSession = buildSession (nextRows) - sessionRef.current = nextSession - setSession (nextSession) + await persistSession (buildSession ( + replaceImportRow (latestSession.rows, mergedRow))) } catch (requestError) { @@ -457,15 +442,6 @@ const PostImportReviewPage: FC = ({ user }) => { row => row.sourceRow === baseRow.sourceRow) if (currentRow == null) continue - const nextRows = replaceImportRow ( - latestSession.rows, - applyThumbnailWarning ({ - ...currentRow, - existingPostId: undefined, - existingPost: undefined })) - const nextSession = buildSession (nextRows) - sessionRef.current = nextSession - setSession (nextSession) if (!(isApiError (requestError))) continue } @@ -481,24 +457,25 @@ const PostImportReviewPage: FC = ({ user }) => { } const hydrate = async () => { + const parsedSessionId = parsePostNewSessionId (location.search) + const sessionId = parsedSessionId ?? generatePostImportSessionId () + const loadedSession = loadPostImportSession (sessionId) const parsed = await parsePostNewState (location.search) if (!(active)) return - if (parsed == null) + if (parsed == null && loadedSession == null) { navigate ('/posts/new', { replace: true }) return } - const parsedSessionId = parsePostNewSessionId (location.search) - const sessionId = parsedSessionId ?? generatePostImportSessionId () sessionIdRef.current = sessionId - const loaded = loadPostImportSession (sessionId) ?? buildSession (parsed.rows) + const loaded = loadedSession ?? buildSession (parsed?.rows ?? []) persistedSearchRef.current = location.search sessionRef.current = loaded setSession (loaded) - if (parsed.shortcut) + if (loadedSession == null && parsed?.shortcut) { try { @@ -580,7 +557,7 @@ const PostImportReviewPage: FC = ({ user }) => { return } - if (parsedSessionId == null) + if (loadedSession == null) { await persistSession (loaded) } @@ -672,21 +649,28 @@ const PostImportReviewPage: FC = ({ user }) => { .map (([key, values]) => [key, [...values]])), baseWarnings: [...row.resetSnapshot.baseWarnings], metadataUrl: row.resetSnapshot.metadataUrl, - thumbnailFile: undefined, - thumbnailObjectUrl: undefined } + thumbnailFile: undefined } : row const urlChanged = draft.url !== baseRow.url const nextRow = buildNextEditedRow (baseRow, draft, urlChanged) + let candidateRow = nextRow try { + candidateRow = + urlChanged + ? mergePreviewRow ( + nextRow, + await apiGet ('/posts/preview', { + params: { url: nextRow.url } })) + : nextRow const dryRun = await apiPost ( '/posts?dry=1', - buildDryRunFormData (nextRow)) + buildDryRunFormData (candidateRow)) const latestSession = sessionRef.current if (latestSession == null) return { saved: false, row: null } - const mergedRow = mergeDryRunRow (nextRow, dryRun) + const mergedRow = mergeDryRunRow (candidateRow, dryRun) const mergedRows = replaceImportRow (latestSession.rows, mergedRow) const nextSession = await persistSession (buildSession (mergedRows)) if (nextSession == null) @@ -708,7 +692,7 @@ const PostImportReviewPage: FC = ({ user }) => { && requestError.response?.status === 422) { const errorRow = { - ...nextRow, + ...candidateRow, validationErrors: requestError.response.data.errors ?? { }, baseWarnings: [], fieldWarnings: { }, @@ -731,7 +715,6 @@ const PostImportReviewPage: FC = ({ user }) => { await dialogue.form ({ title: '投稿を編輯', - description: `投稿 ${ row.sourceRow } の内容を確認し、必要な項目を編輯してください．`, cancelText: '取消', size: 'large', body: controls => ( diff --git a/frontend/src/pages/posts/PostImportSourcePage.tsx b/frontend/src/pages/posts/PostImportSourcePage.tsx index 59d4fdc..098e944 100644 --- a/frontend/src/pages/posts/PostImportSourcePage.tsx +++ b/frontend/src/pages/posts/PostImportSourcePage.tsx @@ -24,6 +24,7 @@ import { countImportSourceLines, loadPostImportSession, loadPostImportSourceDraft, resultRepairMode, + serialisedPostImportRowsEqual, savePostImportSession, savePostImportSourceDraft, validateImportSource } from '@/lib/postImportSession' @@ -143,8 +144,16 @@ const fetchPreviewRows = async ( rows: Array<{ sourceRow: number url: string }>, signal: AbortSignal, -): Promise => { - const results: PostImportRow[] = new Array (rows.length) +): Promise<{ + rows: PostImportRow[] + issues: Array<{ sourceRow: number + message: string + url: string }> +}> => { + const results: Array = new Array (rows.length).fill (null) + const issues: Array<{ sourceRow: number + message: string + url: string }> = [] let nextIndex = 0 const worker = async () => { while (nextIndex < rows.length) @@ -152,44 +161,43 @@ const fetchPreviewRows = async ( const index = nextIndex ++nextIndex const row = rows[index] - const result = await apiGet ('/posts/preview', { - params: { url: row.url }, - signal }) - results[index] = previewRowFromResult (row.sourceRow, result) + try + { + const result = await apiGet ('/posts/preview', { + params: { url: row.url }, + signal }) + results[index] = previewRowFromResult (row.sourceRow, result) + } + catch (requestError) + { + if (signal.aborted) + return + + const apiMessage = + isApiError<{ + message?: string + errors?: Record + baseErrors?: string[] + }> (requestError) + ? (requestError.response?.data?.errors?.url?.[0] + ?? requestError.response?.data?.message + ?? requestError.response?.data?.baseErrors?.[0]) + : undefined + issues.push ({ + sourceRow: row.sourceRow, + message: apiMessage ?? '入力を確認してください．', + url: row.url }) + } } } await Promise.all ( Array.from ({ length: Math.min (4, rows.length) }, () => worker ())) - return results + return { + rows: results.filter ((row): row is PostImportRow => row != null), + issues } } -const serialisableRowJson = (row: PostImportRow): string => - JSON.stringify ({ - sourceRow: row.sourceRow, - url: row.url, - attributes: row.attributes, - fieldWarnings: row.fieldWarnings, - baseWarnings: row.baseWarnings, - validationErrors: row.validationErrors, - provenance: row.provenance, - tagSources: row.tagSources, - status: row.status, - skipReason: row.skipReason, - existingPostId: row.existingPostId, - existingPost: row.existingPost, - resetSnapshot: row.resetSnapshot, - importStatus: row.importStatus, - recoverable: row.recoverable }) - -const sameRows = ( - left: PostImportRow[], - right: PostImportRow[], -): boolean => - left.length === right.length - && left.every ((row, index) => - serialisableRowJson (row) === serialisableRowJson (right[index])) - const PostImportSourcePage: FC = ({ user }) => { const editable = canEditContent (user) @@ -269,7 +277,7 @@ const PostImportSourcePage: FC = ({ user }) => { previewController.current?.abort () const controller = new AbortController () previewController.current = controller - const rows = await fetchPreviewRows ( + const previewed = await fetchPreviewRows ( source .split (/\r\n|\n|\r/) .map ((line, index) => ({ @@ -277,13 +285,31 @@ const PostImportSourcePage: FC = ({ user }) => { url: line.trim () })) .filter (row => row.url !== ''), controller.signal) - const urlIssues = urlIssuesFromRows (rows, source) - if (urlIssues.length > 0) + if (previewed.issues.length > 0) { - setSourceIssues (urlIssues) + setSourceIssues (previewed.issues) return } - const nextRows = initialisePreviewRows (rows) + const urlIssues = urlIssuesFromRows (previewed.rows, source) + const duplicateIssues = Object.entries ( + previewed.rows.reduce> ((acc, row) => { + acc[row.url] ||= [] + acc[row.url].push (row) + return acc + }, { })) + .flatMap (([, rows]) => + rows.length < 2 + ? [] + : rows.map (row => ({ + sourceRow: row.sourceRow, + message: 'URL が重複しています．', + url: row.url }))) + if (urlIssues.length > 0 || duplicateIssues.length > 0) + { + setSourceIssues ([...urlIssues, ...duplicateIssues]) + return + } + const nextRows = initialisePreviewRows (previewed.rows) const serialised = await serialisePostNewState (nextRows) if (serialised == null) return @@ -297,9 +323,9 @@ const PostImportSourcePage: FC = ({ user }) => { const sessionLoaded = loadPostImportSession (sessionId) if (!(serialised.complete) && (!(saved)) - && !(sameRows (sessionLoaded?.rows ?? [], nextRows))) + && !(serialisedPostImportRowsEqual (sessionLoaded?.rows ?? [], nextRows))) return - if (saved && !(sameRows (sessionLoaded?.rows ?? [], nextRows))) + if (saved && !(serialisedPostImportRowsEqual (sessionLoaded?.rows ?? [], nextRows))) return navigate (appendPostNewSessionId (serialised.path, sessionId))