このコミットが含まれているのは:
2026-07-18 02:08:20 +09:00
コミット 9552081133
9個のファイルの変更351行の追加151行の削除
+54 -20
ファイルの表示
@@ -120,30 +120,64 @@ class PostsController < ApplicationController
return head :forbidden unless current_user.gte_member? return head :forbidden unless current_user.gte_member?
return render_bad_request('URL は必須です.') if params[:url].blank? return render_bad_request('URL は必須です.') if params[:url].blank?
preview = PostImportPreviewer.new.preview_rows( normal_url = PostUrlNormaliser.normalise(params[:url].to_s)
rows: [{ return render_validation_error(fields: { url: ['URL が不正です.'] }) if normal_url.blank?
source_row: 1,
url: params[:url].to_s, Preview::UrlSafety.validate(normal_url)
attributes: { }, existing_post = Post.with_attached_thumbnail.find_by(url: normal_url)
provenance: { }, if existing_post.present?
tag_sources: { } }], return render json: {
fetch_metadata: true).first url: normal_url,
return render_validation_error fields: preview[:validation_errors] if preview[:validation_errors].present? title: nil,
thumbnail_base: nil,
tags: nil,
original_created_from: nil,
original_created_before: nil,
duration: nil,
video_ms: nil,
field_warnings: { },
base_warnings: [],
existing_post: compact_post(existing_post.id) }
end
metadata = PostMetadataFetcher.fetch(normal_url)
field_warnings = { }
field_warnings[:title] = ['タイトルを取得できませんでした.'] if metadata[:title].blank?
if metadata[:thumbnail_base].blank?
field_warnings[:thumbnail_base] = ['サムネールを取得できませんでした.']
end
render json: { render json: {
url: preview[:url], url: normal_url,
title: preview[:attributes]['title'], title: metadata[:title],
thumbnail_base: preview[:attributes]['thumbnail_base'], thumbnail_base: metadata[:thumbnail_base],
tags: preview[:attributes]['tags'], tags: metadata[:tags],
original_created_from: preview[:attributes]['original_created_from'], original_created_from: metadata[:original_created_from],
original_created_before: preview[:attributes]['original_created_before'], original_created_before: metadata[:original_created_before],
duration: preview[:attributes]['duration'], duration: metadata[:duration],
video_ms: preview[:attributes]['video_ms'], video_ms: metadata[:video_ms],
field_warnings: preview[:field_warnings], field_warnings: field_warnings,
base_warnings: preview[:base_warnings], base_warnings: [],
existing_post: compact_post(preview[:existing_post_id]) } existing_post: nil }
rescue ArgumentError => e rescue ArgumentError => e
render_bad_request e.message render_bad_request e.message
rescue Preview::UrlSafety::UnsafeUrl => e
render_validation_error fields: { url: [e.message] }
rescue Preview::HttpFetcher::FetchFailed,
Preview::HttpFetcher::FetchTimeout,
Preview::HttpFetcher::ResponseTooLarge
render json: {
url: normal_url,
title: nil,
thumbnail_base: nil,
tags: nil,
original_created_from: nil,
original_created_before: nil,
duration: nil,
video_ms: nil,
field_warnings: { url: ['自動取得に失敗しました.'] },
base_warnings: [],
existing_post: nil }
end end
def show def show
+65 -8
ファイルの表示
@@ -3,6 +3,7 @@ class Post < ApplicationRecord
require 'mini_magick' require 'mini_magick'
require 'nokogiri' require 'nokogiri'
require 'stringio' require 'stringio'
require 'timeout'
class RemoteThumbnailFetchFailed < StandardError; end class RemoteThumbnailFetchFailed < StandardError; end
@@ -15,6 +16,7 @@ class Post < ApplicationRecord
REMOTE_SVG_CONTENT_TYPE = 'image/svg+xml'.freeze REMOTE_SVG_CONTENT_TYPE = 'image/svg+xml'.freeze
MAX_SVG_DIMENSION = 4_096 MAX_SVG_DIMENSION = 4_096
MAX_SVG_PIXELS = 16_777_216 MAX_SVG_PIXELS = 16_777_216
THUMBNAIL_PROCESS_TIMEOUT = 5.seconds
def self.resized_thumbnail_attachment(upload, content_type: nil) def self.resized_thumbnail_attachment(upload, content_type: nil)
upload.rewind upload.rewind
@@ -216,15 +218,37 @@ class Post < ApplicationRecord
end end
def self.image_for_thumbnail_upload(bytes, content_type: nil) def self.image_for_thumbnail_upload(bytes, content_type: nil)
return MiniMagick::Image.read(bytes) unless svg_content_type?(content_type) return decode_raster_thumbnail(bytes) unless svg_content_type?(content_type) || svg_document_bytes?(bytes)
MiniMagick::Image.read(sanitised_svg_bytes(bytes)) decode_svg_thumbnail(bytes)
end end
def self.svg_content_type?(content_type) def self.svg_content_type?(content_type)
content_type.to_s.split(';', 2).first.to_s.downcase.strip == REMOTE_SVG_CONTENT_TYPE content_type.to_s.split(';', 2).first.to_s.downcase.strip == REMOTE_SVG_CONTENT_TYPE
end end
def self.svg_document_bytes?(bytes)
document = Nokogiri::XML(
bytes,
nil,
nil,
Nokogiri::XML::ParseOptions::STRICT |
Nokogiri::XML::ParseOptions::NONET)
document.root&.name == 'svg'
rescue Nokogiri::XML::SyntaxError
false
end
def self.decode_raster_thumbnail(bytes)
Timeout.timeout(THUMBNAIL_PROCESS_TIMEOUT) { MiniMagick::Image.read(bytes) }
end
def self.decode_svg_thumbnail(bytes)
Timeout.timeout(THUMBNAIL_PROCESS_TIMEOUT) do
MiniMagick::Image.read(sanitised_svg_bytes(bytes))
end
end
def self.sanitised_svg_bytes(bytes) def self.sanitised_svg_bytes(bytes)
parse_options = parse_options =
Nokogiri::XML::ParseOptions::STRICT | Nokogiri::XML::ParseOptions::STRICT |
@@ -256,20 +280,44 @@ class Post < ApplicationRecord
name = node.name.to_s.downcase name = node.name.to_s.downcase
next true if name == 'script' || name == 'foreignobject' next true if name == 'script' || name == 'foreignobject'
next style_contains_disallowed_urls?(node.text.to_s) if name == 'style'
node.attribute_nodes.any? do |attribute| node.attribute_nodes.any? do |attribute|
attribute_name = attribute.name.to_s.downcase attribute_name = attribute.name.to_s.downcase
attribute_value = attribute.value.to_s attribute_value = attribute.value.to_s
attribute_name.start_with?('on') || attribute_name.start_with?('on') ||
attribute_name == 'href' || external_svg_reference?(attribute_name, attribute_value)
attribute_name == 'xlink:href' ||
attribute_name == 'src' ||
(attribute_name == 'style' && attribute_value.match?(/url\s*\(/i)) ||
attribute_value.match?(/\A\s*(?:https?:|data:|\/\/)/i)
end end
end end
end end
def self.external_svg_reference?(attribute_name, attribute_value)
return external_svg_url?(attribute_value) if ['href', 'xlink:href', 'src'].include?(attribute_name)
return style_contains_disallowed_urls?(attribute_value) if attribute_name == 'style'
return svg_url_function_disallowed?(attribute_value) if attribute_value.match?(/url\s*\(/i)
false
end
def self.external_svg_url?(value)
stripped = value.to_s.strip
return false if stripped.blank? || stripped.start_with?('#')
stripped.match?(/\A(?:https?:|data:|\/\/)/i)
end
def self.style_contains_disallowed_urls?(value)
text = value.to_s
text.match?(/@import/i) || svg_url_function_disallowed?(text)
end
def self.svg_url_function_disallowed?(value)
value.to_s.scan(/url\s*\(([^)]*)\)/i).flatten.any? do |entry|
reference = entry.to_s.strip.delete_prefix("'").delete_prefix('"').delete_suffix("'").delete_suffix('"')
reference.present? && !(reference.start_with?('#'))
end
end
def self.svg_dimensions(root) def self.svg_dimensions(root)
width = svg_length_to_pixels(root['width']) width = svg_length_to_pixels(root['width'])
height = svg_length_to_pixels(root['height']) height = svg_length_to_pixels(root['height'])
@@ -287,15 +335,24 @@ class Post < ApplicationRecord
matched = /\A([0-9]+(?:\.[0-9]+)?)(px)?\z/i.match(value.to_s.strip) matched = /\A([0-9]+(?:\.[0-9]+)?)(px)?\z/i.match(value.to_s.strip)
return nil if matched == nil return nil if matched == nil
Float(matched[1]) pixels = Float(matched[1])
return nil unless pixels.finite? && pixels.positive?
pixels
rescue ArgumentError rescue ArgumentError
nil nil
end end
private_class_method :image_for_thumbnail_upload, private_class_method :image_for_thumbnail_upload,
:svg_content_type?, :svg_content_type?,
:decode_raster_thumbnail,
:decode_svg_thumbnail,
:sanitised_svg_bytes, :sanitised_svg_bytes,
:svg_uses_disallowed_features?, :svg_uses_disallowed_features?,
:external_svg_reference?,
:external_svg_url?,
:style_contains_disallowed_urls?,
:svg_url_function_disallowed?,
:svg_dimensions, :svg_dimensions,
:svg_length_to_pixels :svg_length_to_pixels
+1 -1
ファイルの表示
@@ -20,7 +20,7 @@ module PostCompactRepr
Rails.application.routes.url_helpers.rails_storage_proxy_url( Rails.application.routes.url_helpers.rails_storage_proxy_url(
post.thumbnail, post.thumbnail,
only_path: false) only_path: false)
rescue rescue ActionController::UrlGenerationError, ArgumentError, URI::InvalidURIError
nil nil
end end
end end
+3 -1
ファイルの表示
@@ -11,8 +11,10 @@ class PostThumbnailUploadValidator
end end
raise InvalidUpload, 'thumbnail file size が大きすぎます.' if thumbnail.size > MAX_THUMBNAIL_BYTES raise InvalidUpload, 'thumbnail file size が大きすぎます.' if thumbnail.size > MAX_THUMBNAIL_BYTES
raise InvalidUpload, 'サムネイル画像の形式が不正です.' unless allowed_content_type?(thumbnail.content_type) raise InvalidUpload, 'サムネイル画像の形式が不正です.' unless allowed_content_type?(thumbnail.content_type)
raise InvalidUpload, 'サムネイル画像の形式が不正です.' if Post.svg_document_bytes?(thumbnail.read)
thumbnail.rewind
attachment = Post.resized_thumbnail_attachment(thumbnail) attachment = Post.resized_thumbnail_attachment(thumbnail, content_type: thumbnail.content_type)
attachment[:io].close if attachment[:io].respond_to?(:close) attachment[:io].close if attachment[:io].respond_to?(:close)
rescue MiniMagick::Error rescue MiniMagick::Error
raise InvalidUpload, 'サムネイル画像の変換に失敗しました.' raise InvalidUpload, 'サムネイル画像の変換に失敗しました.'
+1 -1
ファイルの表示
@@ -32,7 +32,7 @@ module Preview
def self.fetch_image_response(raw_url) def self.fetch_image_response(raw_url)
uri, = UrlSafety.validate(raw_url) uri, = UrlSafety.validate(raw_url)
response = HttpFetcher.fetch(uri.to_s) response = HttpFetcher.fetch(uri.to_s)
unless allowed_remote_image_content_type?(response.content_type) unless allowed_remote_image_content_type?(response.content_type) || Post.svg_document_bytes?(response.body)
raise GenerationFailed, 'サムネール画像が見つかりませんでした.' raise GenerationFailed, 'サムネール画像が見つかりませんでした.'
end end
+2
ファイルの表示
@@ -10,6 +10,7 @@ type Props = {
const LABELS: Record<PostImportBadgeValue, string> = { const LABELS: Record<PostImportBadgeValue, string> = {
ready: '登録可能', ready: '登録可能',
error: '登録不可',
warning: '警告', warning: '警告',
skipped: 'スキップ', skipped: 'スキップ',
created: '登録済み', created: '登録済み',
@@ -17,6 +18,7 @@ const LABELS: Record<PostImportBadgeValue, string> = {
const TONES: Record<PostImportBadgeValue, StatusBadgeTone> = { const TONES: Record<PostImportBadgeValue, StatusBadgeTone> = {
ready: 'success', ready: 'success',
error: 'warning',
warning: 'warning', warning: 'warning',
skipped: 'neutral', skipped: 'neutral',
created: 'success', created: 'success',
+10 -6
ファイルの表示
@@ -2,6 +2,7 @@ import type { PostImportRow } from '@/lib/postImportSession'
export type PostImportDisplayStatus = export type PostImportDisplayStatus =
'ready' 'ready'
| 'error'
| 'skipped' | 'skipped'
| 'warning' | 'warning'
| 'created' | 'created'
@@ -18,13 +19,16 @@ export const displayPostImportStatus = (
): PostImportDisplayStatus | null => ): PostImportDisplayStatus | null =>
row.status === 'pending' row.status === 'pending'
? null ? null
: (row.importStatus === 'failed')
? 'failed'
: (row.skipReason != null || row.importStatus === 'skipped') : (row.skipReason != null || row.importStatus === 'skipped')
? 'skipped' ? 'skipped'
: (row.importStatus === 'created') : (row.importStatus === 'created')
? 'created' ? 'created'
: ((Object.keys (row.validationErrors ?? { }).length > 0 : (row.status === 'error')
|| row.importStatus === 'failed') ? 'error'
? 'failed' : (Object.values (row.validationErrors ?? { }).some (messages => messages.length > 0))
: ((hasWarnings (row) || row.status === 'warning') ? 'error'
? 'warning' : ((hasWarnings (row) || row.status === 'warning')
: 'ready')) ? 'warning'
: 'ready')
+27 -11
ファイルの表示
@@ -22,8 +22,21 @@ export const isManualSkipRow = (row: PostImportRow): boolean =>
const hasSkipReason = (row: PostImportRow): boolean => const hasSkipReason = (row: PostImportRow): boolean =>
row.skipReason != null row.skipReason != null
export const compactMessageRecord = (
messages: Record<string, string[]>,
): Record<string, string[]> =>
Object.fromEntries (
Object.entries (messages).filter (([, values]) => values.length > 0))
export const hasErrorMessages = (
messages: Record<string, string[]>,
): boolean =>
Object.values (messages).some (values => values.length > 0)
const hasValidationErrors = (row: PostImportRow): boolean => const hasValidationErrors = (row: PostImportRow): boolean =>
Object.keys (row.validationErrors ?? { }).length > 0 hasErrorMessages (row.validationErrors ?? { })
const isRecoverableRow = (row: PostImportRow): boolean => const isRecoverableRow = (row: PostImportRow): boolean =>
row.recoverable === true row.recoverable === true
@@ -79,9 +92,9 @@ const thumbnailWarnings = (row: PostImportRow): string[] => {
export const applyThumbnailWarning = (row: PostImportRow): PostImportRow => ({ export const applyThumbnailWarning = (row: PostImportRow): PostImportRow => ({
...row, ...row,
fieldWarnings: { fieldWarnings: compactMessageRecord ({
...row.fieldWarnings, ...row.fieldWarnings,
thumbnailBase: thumbnailWarnings (row) } }) thumbnailBase: thumbnailWarnings (row) }) })
export const applyThumbnailWarnings = (rows: PostImportRow[]): PostImportRow[] => export const applyThumbnailWarnings = (rows: PostImportRow[]): PostImportRow[] =>
@@ -90,7 +103,7 @@ export const applyThumbnailWarnings = (rows: PostImportRow[]): PostImportRow[] =
export const processableImportRows = (rows: PostImportRow[]): PostImportRow[] => export const processableImportRows = (rows: PostImportRow[]): PostImportRow[] =>
validatableImportRows (rows).filter (row => { validatableImportRows (rows).filter (row => {
if (row.metadataUrl == null) if (row.status === 'pending')
return false return false
if (row.importStatus === 'created') if (row.importStatus === 'created')
return false return false
@@ -290,7 +303,7 @@ export const mergeValidatedImportRows = (
return previous return previous
const fieldWarnings = const fieldWarnings =
Object.keys (row.fieldWarnings).length > 0 || row.metadataUrl !== previous.metadataUrl hasErrorMessages (row.fieldWarnings) || row.metadataUrl !== previous.metadataUrl
? { ...row.fieldWarnings } ? { ...row.fieldWarnings }
: { ...previous.fieldWarnings } : { ...previous.fieldWarnings }
for (const [field, origin] of Object.entries (row.provenance)) for (const [field, origin] of Object.entries (row.provenance))
@@ -345,9 +358,10 @@ export const mergeImportResults = (
createdPostId: result.post.id, createdPostId: result.post.id,
existingPostId: undefined, existingPostId: undefined,
existingPost: undefined, existingPost: undefined,
fieldWarnings: result.fieldWarnings ?? row.fieldWarnings, fieldWarnings: compactMessageRecord (
result.fieldWarnings ?? row.fieldWarnings),
baseWarnings: result.baseWarnings ?? row.baseWarnings, baseWarnings: result.baseWarnings ?? row.baseWarnings,
importErrors: result.errors } importErrors: compactMessageRecord (result.errors ?? { }) }
case 'skipped': case 'skipped':
return { return {
...row, ...row,
@@ -357,9 +371,10 @@ export const mergeImportResults = (
createdPostId: undefined, createdPostId: undefined,
existingPostId: result.existingPostId, existingPostId: result.existingPostId,
existingPost: result.existingPost ?? row.existingPost, existingPost: result.existingPost ?? row.existingPost,
fieldWarnings: result.fieldWarnings ?? row.fieldWarnings, fieldWarnings: compactMessageRecord (
result.fieldWarnings ?? row.fieldWarnings),
baseWarnings: result.baseWarnings ?? row.baseWarnings, baseWarnings: result.baseWarnings ?? row.baseWarnings,
importErrors: result.errors } importErrors: compactMessageRecord (result.errors ?? { }) }
case 'failed': case 'failed':
return { return {
...row, ...row,
@@ -369,9 +384,10 @@ export const mergeImportResults = (
createdPostId: undefined, createdPostId: undefined,
existingPostId: undefined, existingPostId: undefined,
existingPost: undefined, existingPost: undefined,
fieldWarnings: result.fieldWarnings ?? row.fieldWarnings, fieldWarnings: compactMessageRecord (
result.fieldWarnings ?? row.fieldWarnings),
baseWarnings: result.baseWarnings ?? row.baseWarnings, baseWarnings: result.baseWarnings ?? row.baseWarnings,
importErrors: result.errors } importErrors: compactMessageRecord (result.errors ?? { }) }
} }
}) })
} }
+188 -103
ファイルの表示
@@ -30,8 +30,10 @@ import {
clearPostImportSourceDraft, clearPostImportSourceDraft,
generatePostImportSessionId, generatePostImportSessionId,
hasThumbnailBaseValue, hasThumbnailBaseValue,
hasErrorMessages,
isCompletedReviewRow, isCompletedReviewRow,
isExistingSkipRow, isExistingSkipRow,
isManualSkipRow,
isNonRecoverableFailedRow, isNonRecoverableFailedRow,
mergeImportResults, mergeImportResults,
processableImportRows, processableImportRows,
@@ -41,7 +43,7 @@ import {
reviewSummaryCounts, reviewSummaryCounts,
retryImportRow, retryImportRow,
loadPostImportSession, loadPostImportSession,
serialisedPostImportRowsEqual, compactMessageRecord,
savePostImportSession, savePostImportSession,
} from '@/lib/postImportSession' } from '@/lib/postImportSession'
import { useClientBehaviourSettings } from '@/lib/useClientBehaviourSettings' import { useClientBehaviourSettings } from '@/lib/useClientBehaviourSettings'
@@ -102,15 +104,48 @@ const isRepairRow = (row: PostImportRow): boolean =>
row.recoverable === true row.recoverable === true
&& (row.importStatus === 'failed' && (row.importStatus === 'failed'
|| (row.importStatus === 'pending' || (row.importStatus === 'pending'
&& Object.keys (row.validationErrors).length > 0)) && hasErrorMessages (row.validationErrors)))
const DUPLICATE_URL_MESSAGE = 'URL が重複しています.' const DUPLICATE_URL_MESSAGE = 'URL が重複しています.'
const rowMetadataPending = (row: PostImportRow): boolean =>
row.status === 'pending'
const rowOperationBusy = (
row: PostImportRow,
submitting: boolean,
loadingRow: number | null,
): boolean =>
submitting
|| loadingRow === row.sourceRow
|| rowMetadataPending (row)
const rowSkipBusy = (
row: PostImportRow,
submitting: boolean,
loadingRow: number | null,
): boolean =>
submitting
|| loadingRow === row.sourceRow
|| (rowMetadataPending (row) && !(isManualSkipRow (row)))
const shouldFetchMetadata = (row: PostImportRow): boolean =>
row.status === 'pending'
&& row.skipReason !== 'manual'
&& row.importStatus !== 'created'
&& row.importStatus !== 'skipped'
&& !(isNonRecoverableFailedRow (row))
&& row.metadataUrl == null
const applyDuplicateUrlErrors = ( const applyDuplicateUrlErrors = (
rows: PostImportRow[], rows: PostImportRow[],
): PostImportRow[] => { ): PostImportRow[] => {
const counts = rows.reduce<Record<string, number>> ((result, row) => { const counts = rows.reduce<Record<string, number>> ((result, row) => {
if (row.url !== '') if (row.url !== ''
&& !(isManualSkipRow (row))
&& row.importStatus !== 'created'
&& row.importStatus !== 'skipped')
result[row.url] = (result[row.url] ?? 0) + 1 result[row.url] = (result[row.url] ?? 0) + 1
return result return result
}, { }) }, { })
@@ -120,27 +155,30 @@ const applyDuplicateUrlErrors = (
message => message !== DUPLICATE_URL_MESSAGE) message => message !== DUPLICATE_URL_MESSAGE)
if ((counts[row.url] ?? 0) < 2) if ((counts[row.url] ?? 0) < 2)
{ {
const nextValidationErrors = { const nextValidationErrors = compactMessageRecord ({
...row.validationErrors, ...row.validationErrors,
url: urlErrors } url: urlErrors })
const hasErrors = Object.values (nextValidationErrors).some ( const hasErrors = hasErrorMessages (nextValidationErrors)
messages => messages.length > 0)
return urlErrors.length === (row.validationErrors.url ?? []).length return urlErrors.length === (row.validationErrors.url ?? []).length
? row ? row
: { : {
...row, ...row,
validationErrors: nextValidationErrors, validationErrors: nextValidationErrors,
skipReason:
row.skipReason === 'manual'
? 'manual'
: row.existingPostId != null
? 'existing'
: undefined,
status: hasErrors ? 'error' : row.status } status: hasErrors ? 'error' : row.status }
} }
return { return {
...row, ...row,
skipReason: row.skipReason === 'manual' ? 'manual' : undefined, skipReason: row.skipReason === 'manual' ? 'manual' : undefined,
existingPostId: undefined, validationErrors: compactMessageRecord ({
existingPost: undefined,
validationErrors: {
...row.validationErrors, ...row.validationErrors,
url: [...new Set ([...urlErrors, DUPLICATE_URL_MESSAGE])] }, url: [...new Set ([...urlErrors, DUPLICATE_URL_MESSAGE])] }),
status: 'error' } status: 'error' }
}) })
} }
@@ -179,8 +217,9 @@ const mergeDryRunRow = (
currentRow: PostImportRow, currentRow: PostImportRow,
result: PostMetadataResponse, result: PostMetadataResponse,
): PostImportRow => { ): PostImportRow => {
const fieldWarnings = compactMessageRecord (result.fieldWarnings ?? { })
const hasWarnings = const hasWarnings =
Object.values (result.fieldWarnings ?? { }).some (messages => messages.length > 0) Object.values (fieldWarnings).some (messages => messages.length > 0)
|| (result.baseWarnings?.length ?? 0) > 0 || (result.baseWarnings?.length ?? 0) > 0
return applyThumbnailWarning ({ return applyThumbnailWarning ({
...currentRow, ...currentRow,
@@ -195,7 +234,7 @@ const mergeDryRunRow = (
videoMs: result.videoMs ?? '', videoMs: result.videoMs ?? '',
tags: result.tags ?? '', tags: result.tags ?? '',
parentPostIds: String (result.parentPostIds ?? currentRow.attributes.parentPostIds ?? '') }, parentPostIds: String (result.parentPostIds ?? currentRow.attributes.parentPostIds ?? '') },
fieldWarnings: result.fieldWarnings ?? { }, fieldWarnings,
baseWarnings: result.baseWarnings ?? [], baseWarnings: result.baseWarnings ?? [],
validationErrors: { }, validationErrors: { },
importErrors: undefined, importErrors: undefined,
@@ -291,9 +330,9 @@ const mergePreviewRow = (
currentRow: PostImportRow, currentRow: PostImportRow,
preview: PostMetadataResponse, preview: PostMetadataResponse,
): PostImportRow => { ): PostImportRow => {
const fieldWarnings = preview.fieldWarnings ?? { } const fieldWarnings = compactMessageRecord (preview.fieldWarnings ?? { })
const baseWarnings = preview.baseWarnings ?? [] const baseWarnings = preview.baseWarnings ?? []
const validationErrors = preview.validationErrors ?? { } const validationErrors = compactMessageRecord (preview.validationErrors ?? { })
const metadataChanged = currentRow.metadataUrl !== preview.url const metadataChanged = currentRow.metadataUrl !== preview.url
const hasWarnings = const hasWarnings =
Object.values (fieldWarnings).some (messages => messages.length > 0) Object.values (fieldWarnings).some (messages => messages.length > 0)
@@ -438,7 +477,20 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const persistSequenceRef = useRef (0) const persistSequenceRef = useRef (0)
const previewSequenceRef = useRef (0) const previewSequenceRef = useRef (0)
const persistSession = async ( const commitSessionState = (
nextSession: PostImportSession,
): PostImportSession => {
const sessionId =
sessionIdRef.current ?? generatePostImportSessionId ()
sessionIdRef.current = sessionId
const persistedSession = buildSession (nextSession.rows)
savePostImportSession (sessionId, persistedSession)
sessionRef.current = persistedSession
setSession (persistedSession)
return persistedSession
}
const syncSessionUrl = async (
nextSession: PostImportSession, nextSession: PostImportSession,
): Promise<PostImportSession | null> => { ): Promise<PostImportSession | null> => {
const sessionId = const sessionId =
@@ -451,26 +503,21 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
if (persistSequenceRef.current !== sequence) if (persistSequenceRef.current !== sequence)
return null return null
const saved = savePostImportSession (sessionId, nextSession) const persistedSession = commitSessionState (nextSession)
const loadedSession = loadPostImportSession (sessionId)
const canRecoverFromUrlOnly = serialised.complete
if (!(canRecoverFromUrlOnly)
&& (!(saved))
&& !(serialisedPostImportRowsEqual (loadedSession?.rows ?? [], nextSession.rows)))
return null
if (saved
&& !(serialisedPostImportRowsEqual (loadedSession?.rows ?? [], nextSession.rows)))
return null
const persistedSession = buildSession (nextSession.rows)
const path = appendPostNewSessionId (serialised.path, sessionId) const path = appendPostNewSessionId (serialised.path, sessionId)
persistedSearchRef.current = (new URL (path, window.location.origin)).search persistedSearchRef.current = (new URL (path, window.location.origin)).search
sessionRef.current = persistedSession
setSession (persistedSession)
navigate (path, { replace: true }) navigate (path, { replace: true })
return persistedSession return persistedSession
} }
const currentRowBySource = (
sourceRow: number,
fallback?: PostImportSession,
): PostImportRow | null =>
(sessionRef.current?.rows ?? fallback?.rows ?? []).find (
row => row.sourceRow === sourceRow) ?? null
const finishImport = () => { const finishImport = () => {
const sessionId = sessionIdRef.current const sessionId = sessionIdRef.current
if (sessionId != null) if (sessionId != null)
@@ -485,7 +532,9 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const previewSequence = ++previewSequenceRef.current const previewSequence = ++previewSequenceRef.current
const controllers = new Set<AbortController> () const controllers = new Set<AbortController> ()
if (sessionRef.current != null && persistedSearchRef.current === location.search) if (sessionRef.current != null
&& persistedSearchRef.current === location.search
&& !(sessionRef.current.rows.some (row => shouldFetchMetadata (row))))
return () => { return () => {
previewSequenceRef.current = previewSequence previewSequenceRef.current = previewSequence
} }
@@ -496,20 +545,16 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
try try
{ {
let nextIndex = 0 let nextIndex = 0
const previewErrors = new Map<number, Record<string, string[]>> () const mergeCurrentRow = (
let workingRows = [...(sessionRef.current ?? baseSession).rows]
const mergeWorkingRow = (
sourceRow: number, sourceRow: number,
updater: (row: PostImportRow) => PostImportRow, updater: (row: PostImportRow) => PostImportRow,
) => { ) => {
workingRows = workingRows.map (row => const latestRows = sessionRef.current?.rows ?? baseSession.rows
const nextRows = applyDuplicateUrlErrors (latestRows.map (row =>
row.sourceRow === sourceRow row.sourceRow === sourceRow
? updater (row) ? updater (row)
: row) : row))
const nextSession = buildSession (workingRows) commitSessionState (buildSession (nextRows))
sessionRef.current = nextSession
setSession (nextSession)
} }
const worker = async () => { const worker = async () => {
while (active) while (active)
@@ -520,22 +565,25 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
return return
const baseRow = baseSession.rows[index] const baseRow = baseSession.rows[index]
if (baseRow.skipReason === 'manual' const currentRow = currentRowBySource (baseRow.sourceRow, baseSession)
|| baseRow.importStatus === 'created' if (currentRow == null || !(shouldFetchMetadata (currentRow)))
|| baseRow.importStatus === 'skipped'
|| isNonRecoverableFailedRow (baseRow)
|| baseRow.metadataUrl != null)
continue continue
const controller = new AbortController () const controller = new AbortController ()
controllers.add (controller) controllers.add (controller)
try try
{ {
const preview = await apiGet<PostMetadataResponse> ('/posts/metadata', { const preview = await apiGet<PostMetadataResponse> ('/posts/metadata', {
params: { url: baseRow.url }, params: { url: currentRow.url },
signal: controller.signal }) signal: controller.signal })
if (!(active) || previewSequenceRef.current !== previewSequence) if (!(active) || previewSequenceRef.current !== previewSequence)
return return
mergeWorkingRow (baseRow.sourceRow, row => mergePreviewRow (row, preview)) mergeCurrentRow (currentRow.sourceRow, row =>
row.importStatus === 'created'
|| row.importStatus === 'skipped'
|| isNonRecoverableFailedRow (row)
|| row.skipReason === 'manual'
? row
: mergePreviewRow (row, preview))
} }
catch (requestError) catch (requestError)
{ {
@@ -547,21 +595,29 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
errors?: Record<string, string[]> errors?: Record<string, string[]>
baseErrors?: string[] baseErrors?: string[]
}> (requestError))) }> (requestError)))
continue {
mergeCurrentRow (currentRow.sourceRow, row => ({
...row,
status: 'error' }))
continue
}
if (requestError.response?.status === 422) if (requestError.response?.status === 422)
{ {
const rowErrors = { const rowErrors = compactMessageRecord ({
...(requestError.response.data.errors ?? { }), ...(requestError.response.data.errors ?? { }),
...(requestError.response.data.baseErrors?.length ...(requestError.response.data.baseErrors?.length
? { ? {
base: requestError.response.data.baseErrors } base: requestError.response.data.baseErrors }
: { }) } : { }) })
previewErrors.set (baseRow.sourceRow, rowErrors) mergeCurrentRow (currentRow.sourceRow, row => ({
mergeWorkingRow (baseRow.sourceRow, row => ({
...row, ...row,
validationErrors: rowErrors, validationErrors: rowErrors,
status: 'error' })) status: 'error' }))
} }
else
mergeCurrentRow (currentRow.sourceRow, row => ({
...row,
status: 'error' }))
} }
finally finally
{ {
@@ -575,20 +631,13 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
if (!(active) || previewSequenceRef.current !== previewSequence) if (!(active) || previewSequenceRef.current !== previewSequence)
return return
const nextRows = applyDuplicateUrlErrors (workingRows.map (row => { const latestRows = sessionRef.current?.rows ?? baseSession.rows
const rowErrors = previewErrors.get (row.sourceRow) const nextRows = applyDuplicateUrlErrors (latestRows)
if (rowErrors == null) await syncSessionUrl (buildSession (nextRows))
return row
return {
...row,
validationErrors: rowErrors,
status: 'error' as const }
}))
await persistSession (buildSession (nextRows))
} }
finally finally
{ {
if (active) if (active && previewSequenceRef.current === previewSequence)
setMetadataLoading (false) setMetadataLoading (false)
} }
} }
@@ -649,18 +698,19 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
: rows : rows
const existingRows = sortedRows.filter (row => isExistingSkipRow (row)) const existingRows = sortedRows.filter (row => isExistingSkipRow (row))
const reviewRows = sortedRows.filter (row => !(isExistingSkipRow (row))) const reviewRows = sortedRows.filter (row => !(isExistingSkipRow (row)))
const busy = metadataLoading || submitting || loadingRow != null
const canSubmit = const canSubmit =
rows.every (row => isCompletedReviewRow (row)) rows.every (row => isCompletedReviewRow (row))
|| processableImportRows (rows).length > 0 || processableImportRows (rows).length > 0
const toggleManualSkip = async (sourceRow: number, checked: boolean) => { const toggleManualSkip = async (sourceRow: number, checked: boolean) => {
if (busy)
return
const currentSession = sessionRef.current const currentSession = sessionRef.current
if (currentSession == null) if (currentSession == null)
return return
const currentRow = currentRowBySource (sourceRow, currentSession)
if (currentRow == null)
return
if (rowSkipBusy (currentRow, submitting, loadingRow))
return
const nextRows = currentSession.rows.map (row => { const nextRows = currentSession.rows.map (row => {
if (row.sourceRow !== sourceRow) if (row.sourceRow !== sourceRow)
@@ -674,7 +724,13 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
existingPostId: checked ? undefined : row.existingPostId, existingPostId: checked ? undefined : row.existingPostId,
existingPost: checked ? undefined : row.existingPost } existingPost: checked ? undefined : row.existingPost }
}) })
await persistSession (buildSession (nextRows)) const nextSession = buildSession (applyDuplicateUrlErrors (nextRows))
if (metadataLoading)
{
commitSessionState (nextSession)
return
}
await syncSessionUrl (nextSession)
} }
const saveDraft = async ( const saveDraft = async (
@@ -687,23 +743,26 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const currentSession = sessionRef.current const currentSession = sessionRef.current
if (currentSession == null) if (currentSession == null)
return { saved: false, row: null } return { saved: false, row: null }
if (!(canEditReviewRow (row))) const currentRow = currentRowBySource (row.sourceRow, currentSession)
if (currentRow == null)
return { saved: false, row: null }
if (!(canEditReviewRow (currentRow)))
return { saved: false, row: null } return { saved: false, row: null }
const baseRow = const baseRow =
resetRequested resetRequested
? { ...row, ? { ...currentRow,
url: row.resetSnapshot.url, url: currentRow.resetSnapshot.url,
attributes: { ...row.resetSnapshot.attributes }, attributes: { ...currentRow.resetSnapshot.attributes },
provenance: { ...row.resetSnapshot.provenance }, provenance: { ...currentRow.resetSnapshot.provenance },
tagSources: { ...row.resetSnapshot.tagSources }, tagSources: { ...currentRow.resetSnapshot.tagSources },
fieldWarnings: Object.fromEntries ( fieldWarnings: Object.fromEntries (
Object.entries (row.resetSnapshot.fieldWarnings) Object.entries (currentRow.resetSnapshot.fieldWarnings)
.map (([key, values]) => [key, [...values]])), .map (([key, values]) => [key, [...values]])),
baseWarnings: [...row.resetSnapshot.baseWarnings], baseWarnings: [...currentRow.resetSnapshot.baseWarnings],
metadataUrl: row.resetSnapshot.metadataUrl, metadataUrl: currentRow.resetSnapshot.metadataUrl,
thumbnailFile: undefined } thumbnailFile: undefined }
: row : currentRow
const urlChanged = draft.url !== baseRow.url const urlChanged = draft.url !== baseRow.url
const nextRow = buildNextEditedRow (baseRow, draft, urlChanged) const nextRow = buildNextEditedRow (baseRow, draft, urlChanged)
let candidateRow = nextRow let candidateRow = nextRow
@@ -724,15 +783,19 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
return { saved: false, row: null } return { saved: false, row: null }
const mergedRow = mergeDryRunRow (candidateRow, dryRun) const mergedRow = mergeDryRunRow (candidateRow, dryRun)
const mergedRows = replaceImportRow (latestSession.rows, mergedRow) const mergedRows = applyDuplicateUrlErrors (
const nextSession = await persistSession (buildSession (mergedRows)) replaceImportRow (latestSession.rows, mergedRow))
const nextSession =
metadataLoading
? commitSessionState (buildSession (mergedRows))
: await syncSessionUrl (buildSession (mergedRows))
if (nextSession == null) if (nextSession == null)
return { saved: false, row: null } return { saved: false, row: null }
const mergedTarget = nextSession.rows.find ( const mergedTarget = nextSession.rows.find (
currentRow => currentRow.sourceRow === baseRow.sourceRow) currentRow => currentRow.sourceRow === baseRow.sourceRow)
if (mergedTarget == null) if (mergedTarget == null)
return { saved: false, row: null } return { saved: false, row: null }
if (Object.keys (mergedTarget.validationErrors).length > 0) if (hasErrorMessages (mergedTarget.validationErrors))
return { saved: false, row: mergedTarget } return { saved: false, row: mergedTarget }
return { saved: true, row: null } return { saved: true, row: null }
} }
@@ -746,7 +809,12 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
{ {
const errorRow = { const errorRow = {
...candidateRow, ...candidateRow,
validationErrors: requestError.response.data.errors ?? { }, validationErrors: compactMessageRecord ({
...(requestError.response.data.errors ?? { }),
...(requestError.response.data.baseErrors?.length
? {
base: requestError.response.data.baseErrors }
: { }) }),
baseWarnings: [], baseWarnings: [],
fieldWarnings: { }, fieldWarnings: { },
status: 'error' as const } status: 'error' as const }
@@ -778,39 +846,48 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
} }
const editRow = async (row: PostImportRow) => { const editRow = async (row: PostImportRow) => {
setEditingRow (row) const currentRow = sessionRef.current?.rows.find (
current => current.sourceRow === row.sourceRow)
if (currentRow == null)
return
if (!(canEditReviewRow (currentRow)))
return
if (rowOperationBusy (currentRow, submitting, loadingRow))
return
setEditingRow (currentRow)
try try
{ {
await openEditingDialogue (row) await openEditingDialogue (currentRow)
} }
finally finally
{ {
setEditingRow (current => setEditingRow (current =>
current?.sourceRow === row.sourceRow current?.sourceRow === currentRow.sourceRow
? null ? null
: current) : current)
} }
} }
const retry = async (sourceRow: number) => { const retry = async (sourceRow: number) => {
if (busy)
return
const initialSession = sessionRef.current const initialSession = sessionRef.current
if (initialSession == null) if (initialSession == null)
return return
setLoadingRow (sourceRow)
const originalRow = initialSession.rows.find (row => row.sourceRow === sourceRow) const originalRow = initialSession.rows.find (row => row.sourceRow === sourceRow)
if (originalRow == null) if (originalRow == null)
{ return
setLoadingRow (null) if (rowOperationBusy (originalRow, submitting, loadingRow))
return return
}
setLoadingRow (sourceRow)
try try
{ {
const pendingRows = retryImportRow (initialSession.rows, sourceRow) const pendingRows = retryImportRow (initialSession.rows, sourceRow)
const pendingSession = await persistSession (buildSession (pendingRows)) const pendingSession =
metadataLoading
? commitSessionState (buildSession (pendingRows))
: await syncSessionUrl (buildSession (pendingRows))
if (pendingSession == null) if (pendingSession == null)
return return
@@ -825,16 +902,20 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
{ {
const latest = sessionRef.current ?? pendingSession const latest = sessionRef.current ?? pendingSession
const restoredRows = replaceImportRow (latest.rows, originalRow) const restoredRows = replaceImportRow (latest.rows, originalRow)
await persistSession (buildSession (restoredRows)) if (metadataLoading)
commitSessionState (buildSession (restoredRows))
else
await syncSessionUrl (buildSession (restoredRows))
toast ({ title: '登録結果が不完全でした' }) toast ({ title: '登録結果が不完全でした' })
return return
} }
const indexedResults = indexedBulkResults ([target], result.results) const indexedResults = indexedBulkResults ([target], result.results)
const latestAfterImport = sessionRef.current ?? pendingSession const latestAfterImport = sessionRef.current ?? pendingSession
const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults) const nextRows = applyDuplicateUrlErrors (
mergeImportResults (latestAfterImport.rows, indexedResults))
.map (row => applyThumbnailWarning (row)) .map (row => applyThumbnailWarning (row))
const persisted = await persistSession (buildSession (nextRows)) const persisted = await syncSessionUrl (buildSession (nextRows))
if (persisted != null if (persisted != null
&& persisted.rows.every (row => isCompletedReviewRow (row))) && persisted.rows.every (row => isCompletedReviewRow (row)))
finishImport () finishImport ()
@@ -843,7 +924,10 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
{ {
const latest = sessionRef.current ?? initialSession const latest = sessionRef.current ?? initialSession
const restoredRows = replaceImportRow (latest.rows, originalRow) const restoredRows = replaceImportRow (latest.rows, originalRow)
await persistSession (buildSession (restoredRows)) if (metadataLoading)
commitSessionState (buildSession (restoredRows))
else
await syncSessionUrl (buildSession (restoredRows))
toast ({ title: '再試行に失敗しました' }) toast ({ title: '再試行に失敗しました' })
} }
finally finally
@@ -854,7 +938,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const submit = async () => { const submit = async () => {
const currentSession = sessionRef.current const currentSession = sessionRef.current
if (currentSession == null || busy) if (currentSession == null || metadataLoading || submitting || loadingRow != null)
return return
if (currentSession.rows.every (row => isCompletedReviewRow (row))) if (currentSession.rows.every (row => isCompletedReviewRow (row)))
{ {
@@ -880,9 +964,10 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const indexedResults = indexedBulkResults (processingRows, result.results) const indexedResults = indexedBulkResults (processingRows, result.results)
const latestAfterImport = sessionRef.current ?? buildSession (currentSession.rows) const latestAfterImport = sessionRef.current ?? buildSession (currentSession.rows)
const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults) const nextRows = applyDuplicateUrlErrors (
mergeImportResults (latestAfterImport.rows, indexedResults))
.map (row => applyThumbnailWarning (row)) .map (row => applyThumbnailWarning (row))
const persisted = await persistSession (buildSession (nextRows)) const persisted = await syncSessionUrl (buildSession (nextRows))
if (persisted != null if (persisted != null
&& persisted.rows.every (row => isCompletedReviewRow (row))) && persisted.rows.every (row => isCompletedReviewRow (row)))
finishImport () finishImport ()
@@ -958,10 +1043,10 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
row={row} row={row}
displayNumber={index + 1} displayNumber={index + 1}
showSkipToggle={true} showSkipToggle={true}
editDisabled={busy} editDisabled={rowOperationBusy (row, submitting, loadingRow)}
retryDisabled={busy} retryDisabled={rowOperationBusy (row, submitting, loadingRow)}
skipDisabled={ skipDisabled={
busy rowSkipBusy (row, submitting, loadingRow)
|| isExistingSkipRow (row) || isExistingSkipRow (row)
|| row.importStatus === 'created'} || row.importStatus === 'created'}
onEdit={() => editRow (row)} onEdit={() => editRow (row)}