From 6d037192c448df305cfa502236c9d565b33d6d27 Mon Sep 17 00:00:00 2001 From: miteruzo Date: Tue, 14 Jul 2026 19:12:23 +0900 Subject: [PATCH] #399 --- backend/app/controllers/posts_controller.rb | 3 +- backend/app/services/post_import_previewer.rb | 44 ++++++++++--------- backend/app/services/post_import_runner.rb | 5 +-- backend/app/services/post_metadata_fetcher.rb | 9 ++-- backend/app/services/post_url_normaliser.rb | 2 +- backend/app/services/preview/http_fetcher.rb | 4 +- frontend/src/lib/postImportSession.ts | 38 +++++++++++----- .../src/pages/posts/PostImportResultPage.tsx | 40 +++++++++++++---- frontend/src/pages/posts/PostNewPage.tsx | 35 ++++++--------- 9 files changed, 108 insertions(+), 72 deletions(-) diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index 9da8633..5bfbdb1 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -148,7 +148,8 @@ class PostsController < ApplicationController original_created_from: params[:original_created_from], original_created_before: params[:original_created_before], parent_post_ids: parse_parent_post_ids, - video_ms: params[:video_ms], duration: params[:duration] }).create! + video_ms: params[:video_ms], + duration: params[:duration] }).create! post.reload render json: PostRepr.base(post), status: :created diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index ea319b9..898485c 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -54,20 +54,25 @@ class PostImportPreviewer add_field_warning!(field_warnings, 'url', EXISTING_SKIP_WARNING) attributes['tags'] = merged_tags(tag_sources, provenance['tags']) warnings_present = field_warnings.values.any?(&:present?) || base_warnings.present? - return { - source_row: row[:source_row], - url: normal_url, - attributes:, - provenance:, - tag_sources:, - metadata_url: url_for_metadata, - skip_reason: 'existing', - existing_post_id: existing_post.id, - field_warnings:, - base_warnings:, - validation_errors:, - status: validation_errors.present? ? 'error' : (warnings_present ? 'warning' : 'ready'), - } + return { source_row: row[:source_row], + url: normal_url, + attributes:, + provenance:, + tag_sources:, + metadata_url: url_for_metadata, + skip_reason: 'existing', + existing_post_id: existing_post.id, + field_warnings:, + base_warnings:, + validation_errors:, + status: + if validation_errors.present? + 'error' + elsif warnings_present + 'warning' + else + 'ready' + end } end should_fetch = should_fetch_metadata?(fetch_metadata, row[:source_row].to_i) @@ -88,8 +93,7 @@ class PostImportPreviewer attributes['tags'] = merged_tags(tag_sources, provenance['tags']) warnings_present = field_warnings.values.any?(&:present?) || base_warnings.present? - { - source_row: row[:source_row], + { source_row: row[:source_row], url: normal_url || url, attributes:, provenance:, @@ -100,8 +104,7 @@ class PostImportPreviewer field_warnings:, base_warnings:, validation_errors:, - status: validation_errors.present? ? 'error' : (warnings_present ? 'warning' : 'ready'), - } + status: validation_errors.present? ? 'error' : (warnings_present ? 'warning' : 'ready') } end def should_fetch_metadata? fetch_metadata, source_row @@ -175,9 +178,8 @@ class PostImportPreviewer Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e Rails.logger.info( - "post_import_metadata_fetch_failure "\ - "#{ { error: e.class.name, message: e.message }.to_json }", - ) + "post_import_metadata_fetch_failure "\ + "#{ { error: e.class.name, message: e.message }.to_json }") { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } } end diff --git a/backend/app/services/post_import_runner.rb b/backend/app/services/post_import_runner.rb index 7ba824d..86a10de 100644 --- a/backend/app/services/post_import_runner.rb +++ b/backend/app/services/post_import_runner.rb @@ -24,11 +24,10 @@ class PostImportRunner def run_row row, preview attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore } - return row.slice('source_row').merge(status: 'skipped') if preview[:skip_reason] == 'existing' - return { source_row: row['source_row'], status: 'failed', errors: preview[:validation_errors] } if preview[:validation_errors].present? + return row.slice('source_row').merge(status: 'skipped') if preview[:skip_reason] == 'existing' attributes['tags'] = preview[:attributes]['tags'] attributes['url'] = row['url'] @@ -44,7 +43,7 @@ class PostImportRunner { source_row: row['source_row'], status: 'failed', errors: { tags: ['廃止済みタグは付与できません．'] } } - rescue ArgumentError, PostCreator::VideoMsParseError + rescue ArgumentError { source_row: row['source_row'], status: 'failed', errors: { base: ['入力値が不正です．'] } } diff --git a/backend/app/services/post_metadata_fetcher.rb b/backend/app/services/post_metadata_fetcher.rb index 6a0d78f..83fd7d5 100644 --- a/backend/app/services/post_metadata_fetcher.rb +++ b/backend/app/services/post_metadata_fetcher.rb @@ -2,9 +2,8 @@ class PostMetadataFetcher def self.fetch raw_url uri, = Preview::UrlSafety.validate(raw_url) response = Preview::HttpFetcher.fetch( - uri.to_s, - max_bytes: Preview::ThumbnailFetcher::HTML_MAX_BYTES, - ) + uri.to_s, + max_bytes: Preview::ThumbnailFetcher::HTML_MAX_BYTES) metadata = Preview::HtmlMetadataExtractor.extract(response) document = Nokogiri::HTML.parse(response.body) content = lambda { |name| @@ -19,7 +18,8 @@ class PostMetadataFetcher created_range = original_created_range(published) platform_tags = platform_tags(uri) { title: metadata[:title], - thumbnail_base: Preview::KnownSiteExtractor.thumbnail_url(uri) || metadata[:image_url], + thumbnail_base: + Preview::KnownSiteExtractor.thumbnail_url(uri) || metadata[:image_url], original_created_from: created_range&.first&.iso8601, original_created_before: created_range&.last&.iso8601, duration: duration&.to_f&.then { _1.positive? ? (_1 * 1_000).round : nil }, @@ -56,5 +56,6 @@ class PostMetadataFetcher rescue ArgumentError, TypeError nil end + private_class_method :platform_tags, :original_created_range end diff --git a/backend/app/services/post_url_normaliser.rb b/backend/app/services/post_url_normaliser.rb index 09aa5be..a8d4251 100644 --- a/backend/app/services/post_url_normaliser.rb +++ b/backend/app/services/post_url_normaliser.rb @@ -6,7 +6,7 @@ class PostUrlNormaliser uri.host = uri.host.downcase uri.path = uri.path.sub(/\/\z/, '') if uri.path.present? - uri.to_s + PostUrlSanitisationRule.sanitise(uri.to_s) rescue URI::InvalidURIError nil end diff --git a/backend/app/services/preview/http_fetcher.rb b/backend/app/services/preview/http_fetcher.rb index 8a4aecf..6ce1c8f 100644 --- a/backend/app/services/preview/http_fetcher.rb +++ b/backend/app/services/preview/http_fetcher.rb @@ -12,7 +12,9 @@ module Preview Response = Data.define(:body, :content_type, :url) - def self.fetch(raw_url, max_bytes: DEFAULT_MAX_BYTES, redirects: MAX_REDIRECTS, + def self.fetch(raw_url, + max_bytes: DEFAULT_MAX_BYTES, + redirects: MAX_REDIRECTS, allowed_hosts: nil) uri, addresses = UrlSafety.validate(raw_url) if allowed_hosts && !allowed_hosts.include?(uri.host.downcase) diff --git a/frontend/src/lib/postImportSession.ts b/frontend/src/lib/postImportSession.ts index 763ad40..a3a3433 100644 --- a/frontend/src/lib/postImportSession.ts +++ b/frontend/src/lib/postImportSession.ts @@ -193,6 +193,9 @@ const isValidSkipReason = ( ): value is PostImportSkipReason => value === 'existing' +const isPositiveInteger = (value: unknown): value is number => + Number.isInteger (value) && Number (value) > 0 + const sanitiseRow = (value: unknown): PostImportRow | null => { if (!(isPlainObject (value))) @@ -211,6 +214,12 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { return null if (value.skipReason != null && !(isValidSkipReason (value.skipReason))) return null + if (value.skipReason === 'existing' && !(isPositiveInteger (value.existingPostId))) + return null + if (value.skipReason !== 'existing' && value.existingPostId != null) + return null + if (value.createdPostId != null && !(isPositiveInteger (value.createdPostId))) + return null const validationErrors = ensureStringListRecord (value.validationErrors) const fieldWarnings = ensureStringListRecord (value.fieldWarnings) @@ -250,13 +259,13 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { provenance: value.provenance as Record, tagSources: value.tagSources as Record | undefined, status: value.status, - skipReason: value.skipReason, - existingPostId: - Number.isInteger (value.existingPostId) ? Number (value.existingPostId) : undefined, - metadataUrl: typeof value.metadataUrl === 'string' ? value.metadataUrl : undefined, - createdPostId: - Number.isInteger (value.createdPostId) ? Number (value.createdPostId) : undefined, - importStatus: value.importStatus } + skipReason: value.skipReason, + existingPostId: + isPositiveInteger (value.existingPostId) ? Number (value.existingPostId) : undefined, + metadataUrl: typeof value.metadataUrl === 'string' ? value.metadataUrl : undefined, + createdPostId: + isPositiveInteger (value.createdPostId) ? Number (value.createdPostId) : undefined, + importStatus: value.importStatus } } @@ -605,6 +614,9 @@ export const validateImportSource = ( const hasSkipReason = (row: PostImportRow): boolean => row.skipReason === 'existing' +const hasValidationErrors = (row: PostImportRow): boolean => + Object.keys (row.validationErrors ?? { }).length > 0 + export const submittableImportRows = (rows: PostImportRow[]): PostImportRow[] => rows.filter (row => { @@ -612,10 +624,12 @@ export const submittableImportRows = (rows: PostImportRow[]): PostImportRow[] => return false if (row.importStatus === 'skipped') return false - if (hasSkipReason (row)) + if (hasSkipReason (row) && !(hasValidationErrors (row))) return false if (row.importStatus === 'failed') return false + if (hasValidationErrors (row)) + return false return row.importStatus == null || row.importStatus === 'pending' }) @@ -624,9 +638,11 @@ export const reviewSummaryCounts = (rows: PostImportRow[]) => ({ total: rows.length, submittable: rows.filter (row => submittableImportRows ([row]).length > 0 - && Object.keys (row.validationErrors ?? { }).length === 0).length, - invalid: rows.filter (row => Object.keys (row.validationErrors ?? { }).length > 0).length, + && !(hasValidationErrors (row))).length, + invalid: rows.filter (row => hasValidationErrors (row)).length, skipPlanned: rows.filter (row => - hasSkipReason (row) && row.importStatus !== 'created').length, + hasSkipReason (row) + && !(hasValidationErrors (row)) + && row.importStatus !== 'created').length, created: rows.filter (row => row.importStatus === 'created').length, failed: rows.filter (row => row.importStatus === 'failed').length }) diff --git a/frontend/src/pages/posts/PostImportResultPage.tsx b/frontend/src/pages/posts/PostImportResultPage.tsx index 1eb823d..4407e3d 100644 --- a/frontend/src/pages/posts/PostImportResultPage.tsx +++ b/frontend/src/pages/posts/PostImportResultPage.tsx @@ -16,9 +16,11 @@ import { canEditContent } from '@/lib/users' import { clearPostImportSourceDraft, loadPostImportSession, mergeImportResults, + mergeValidatedImportRows, retryImportRow, savePostImportSession, type PostImportResultRow, + type PostImportRow, type PostImportSession } from '@/lib/postImportSession' import Forbidden from '@/pages/Forbidden' @@ -92,9 +94,31 @@ const PostImportResultPage: FC = ({ user }) => { { const pendingRows = retryImportRow (session.rows, sourceRow) setSession ({ ...session, rows: pendingRows }) - const target = pendingRows.find (_1 => _1.sourceRow === sourceRow) + const validated = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/validate', { + rows: pendingRows + .filter (row => row.importStatus !== 'created') + .map (row => ({ + sourceRow: row.sourceRow, + url: row.url, + attributes: row.attributes, + provenance: row.provenance, + tagSources: row.tagSources, + metadataUrl: row.metadataUrl })), + changed_row: -1 }) + const validatedRows = mergeValidatedImportRows (pendingRows, validated.rows).map (row => + row.sourceRow === sourceRow + && Object.keys (row.validationErrors ?? { }).length > 0 + ? { ...row, importStatus: 'failed' as const } + : row) + setSession (current => + current + ? { ...current, rows: validatedRows } + : current) + const target = validatedRows.find (_1 => _1.sourceRow === sourceRow) if (target == null) return + if (Object.keys (target.validationErrors ?? { }).length > 0) + return const result = await apiPost<{ created: number @@ -125,15 +149,15 @@ const PostImportResultPage: FC = ({ user }) => { } const openRepair = (sourceRow: number) => { + if (session == null || sessionId == null) + return + const nextSession = { ...session, repairMode: 'failed' as const } setSession (nextSession) - if (sessionId) - { - const saved = savePostImportSession (sessionId, nextSession, message => - toast ({ title: '取込状態を保存できませんでした', description: message })) - if (!(saved)) - return - } + const saved = savePostImportSession (sessionId, nextSession, message => + toast ({ title: '取込状態を保存できませんでした', description: message })) + if (!(saved)) + return navigate (`/posts/import/${ sessionId }/review?edit=${ sourceRow }`) } diff --git a/frontend/src/pages/posts/PostNewPage.tsx b/frontend/src/pages/posts/PostNewPage.tsx index 7576f09..47303e5 100644 --- a/frontend/src/pages/posts/PostNewPage.tsx +++ b/frontend/src/pages/posts/PostNewPage.tsx @@ -24,8 +24,7 @@ import type { User } from '@/types' type Props = { user: User | null } -type PostFormField = - 'url' +type PostFormField = 'url' | 'title' | 'tags' | 'parentPostIds' @@ -42,10 +41,8 @@ const PostNewPage: FC = ({ user }) => { const { baseErrors, fieldErrors, clearValidationErrors, applyValidationError } = useValidationErrors () - const [originalCreatedBefore, setOriginalCreatedBefore] = - useState (null) - const [originalCreatedFrom, setOriginalCreatedFrom] = - useState (null) + const [originalCreatedBefore, setOriginalCreatedBefore] = useState (null) + const [originalCreatedFrom, setOriginalCreatedFrom] = useState (null) const [parentPostIds, setParentPostIds] = useState ('') const [tags, setTags] = useState ('') const [duration, setDuration] = useState ('') @@ -57,11 +54,9 @@ const PostNewPage: FC = ({ user }) => { const [url, setURL] = useState ('') const thumbnailPreviewRef = useRef ('') - const videoFlg = - useMemo (() => - tags.split (/\s+/).some ( - tag => tag.replace (/\[.*\]$/, '') === '動画'), - [tags]) + const videoFlg = useMemo (() => + tags.split (/\s+/).some ( + tag => tag.replace (/\[.*\]$/, '') === '動画'), [tags]) const handleSubmit = async () => { clearValidationErrors () @@ -82,8 +77,7 @@ const PostNewPage: FC = ({ user }) => { try { - await apiPost ('/posts', formData, - { headers: { 'Content-Type': 'multipart/form-data' } }) + await apiPost ('/posts', formData, { headers: { 'Content-Type': 'multipart/form-data' } }) toast ({ title: '投稿成功!' }) navigate ('/posts') } @@ -98,8 +92,7 @@ const PostNewPage: FC = ({ user }) => { setTitleLoading (true) try { - const data = await apiGet<{ title: string }> ('/preview/title', - { params: { url } }) + const data = await apiGet<{ title: string }> ('/preview/title', { params: { url } }) setTitle (data.title || '') } finally @@ -116,13 +109,11 @@ const PostNewPage: FC = ({ user }) => { URL.revokeObjectURL (thumbnailPreviewRef.current) try { - const data = await apiGet ('/preview/thumbnail', - { params: { url }, - responseType: 'blob' }) + const data = await apiGet ('/preview/thumbnail', { params: { url }, + responseType: 'blob' }) const imageURL = URL.createObjectURL (data) setThumbnailPreview (imageURL) - setThumbnailFile (new File ([data], - 'thumbnail.png', + setThumbnailFile (new File ([data], 'thumbnail.png', { type: data.type || 'image/png' })) } finally @@ -176,7 +167,7 @@ const PostNewPage: FC = ({ user }) => { @@ -193,7 +184,7 @@ const PostNewPage: FC = ({ user }) => {