このコミットが含まれているのは:
@@ -120,30 +120,64 @@ class PostsController < ApplicationController
|
||||
return head :forbidden unless current_user.gte_member?
|
||||
return render_bad_request('URL は必須です.') if params[:url].blank?
|
||||
|
||||
preview = PostImportPreviewer.new.preview_rows(
|
||||
rows: [{
|
||||
source_row: 1,
|
||||
url: params[:url].to_s,
|
||||
attributes: { },
|
||||
provenance: { },
|
||||
tag_sources: { } }],
|
||||
fetch_metadata: true).first
|
||||
return render_validation_error fields: preview[:validation_errors] if preview[:validation_errors].present?
|
||||
normal_url = PostUrlNormaliser.normalise(params[:url].to_s)
|
||||
return render_validation_error(fields: { url: ['URL が不正です.'] }) if normal_url.blank?
|
||||
|
||||
Preview::UrlSafety.validate(normal_url)
|
||||
existing_post = Post.with_attached_thumbnail.find_by(url: normal_url)
|
||||
if existing_post.present?
|
||||
return render json: {
|
||||
url: normal_url,
|
||||
title: nil,
|
||||
thumbnail_base: nil,
|
||||
tags: nil,
|
||||
original_created_from: nil,
|
||||
original_created_before: nil,
|
||||
duration: nil,
|
||||
video_ms: nil,
|
||||
field_warnings: { },
|
||||
base_warnings: [],
|
||||
existing_post: compact_post(existing_post.id) }
|
||||
end
|
||||
|
||||
metadata = PostMetadataFetcher.fetch(normal_url)
|
||||
field_warnings = { }
|
||||
field_warnings[:title] = ['タイトルを取得できませんでした.'] if metadata[:title].blank?
|
||||
if metadata[:thumbnail_base].blank?
|
||||
field_warnings[:thumbnail_base] = ['サムネールを取得できませんでした.']
|
||||
end
|
||||
|
||||
render json: {
|
||||
url: preview[:url],
|
||||
title: preview[:attributes]['title'],
|
||||
thumbnail_base: preview[:attributes]['thumbnail_base'],
|
||||
tags: preview[:attributes]['tags'],
|
||||
original_created_from: preview[:attributes]['original_created_from'],
|
||||
original_created_before: preview[:attributes]['original_created_before'],
|
||||
duration: preview[:attributes]['duration'],
|
||||
video_ms: preview[:attributes]['video_ms'],
|
||||
field_warnings: preview[:field_warnings],
|
||||
base_warnings: preview[:base_warnings],
|
||||
existing_post: compact_post(preview[:existing_post_id]) }
|
||||
url: normal_url,
|
||||
title: metadata[:title],
|
||||
thumbnail_base: metadata[:thumbnail_base],
|
||||
tags: metadata[:tags],
|
||||
original_created_from: metadata[:original_created_from],
|
||||
original_created_before: metadata[:original_created_before],
|
||||
duration: metadata[:duration],
|
||||
video_ms: metadata[:video_ms],
|
||||
field_warnings: field_warnings,
|
||||
base_warnings: [],
|
||||
existing_post: nil }
|
||||
rescue ArgumentError => e
|
||||
render_bad_request e.message
|
||||
rescue Preview::UrlSafety::UnsafeUrl => e
|
||||
render_validation_error fields: { url: [e.message] }
|
||||
rescue Preview::HttpFetcher::FetchFailed,
|
||||
Preview::HttpFetcher::FetchTimeout,
|
||||
Preview::HttpFetcher::ResponseTooLarge
|
||||
render json: {
|
||||
url: normal_url,
|
||||
title: nil,
|
||||
thumbnail_base: nil,
|
||||
tags: nil,
|
||||
original_created_from: nil,
|
||||
original_created_before: nil,
|
||||
duration: nil,
|
||||
video_ms: nil,
|
||||
field_warnings: { url: ['自動取得に失敗しました.'] },
|
||||
base_warnings: [],
|
||||
existing_post: nil }
|
||||
end
|
||||
|
||||
def show
|
||||
|
||||
@@ -3,6 +3,7 @@ class Post < ApplicationRecord
|
||||
require 'mini_magick'
|
||||
require 'nokogiri'
|
||||
require 'stringio'
|
||||
require 'timeout'
|
||||
|
||||
class RemoteThumbnailFetchFailed < StandardError; end
|
||||
|
||||
@@ -15,6 +16,7 @@ class Post < ApplicationRecord
|
||||
REMOTE_SVG_CONTENT_TYPE = 'image/svg+xml'.freeze
|
||||
MAX_SVG_DIMENSION = 4_096
|
||||
MAX_SVG_PIXELS = 16_777_216
|
||||
THUMBNAIL_PROCESS_TIMEOUT = 5.seconds
|
||||
|
||||
def self.resized_thumbnail_attachment(upload, content_type: nil)
|
||||
upload.rewind
|
||||
@@ -216,15 +218,37 @@ class Post < ApplicationRecord
|
||||
end
|
||||
|
||||
def self.image_for_thumbnail_upload(bytes, content_type: nil)
|
||||
return MiniMagick::Image.read(bytes) unless svg_content_type?(content_type)
|
||||
return decode_raster_thumbnail(bytes) unless svg_content_type?(content_type) || svg_document_bytes?(bytes)
|
||||
|
||||
MiniMagick::Image.read(sanitised_svg_bytes(bytes))
|
||||
decode_svg_thumbnail(bytes)
|
||||
end
|
||||
|
||||
def self.svg_content_type?(content_type)
|
||||
content_type.to_s.split(';', 2).first.to_s.downcase.strip == REMOTE_SVG_CONTENT_TYPE
|
||||
end
|
||||
|
||||
def self.svg_document_bytes?(bytes)
|
||||
document = Nokogiri::XML(
|
||||
bytes,
|
||||
nil,
|
||||
nil,
|
||||
Nokogiri::XML::ParseOptions::STRICT |
|
||||
Nokogiri::XML::ParseOptions::NONET)
|
||||
document.root&.name == 'svg'
|
||||
rescue Nokogiri::XML::SyntaxError
|
||||
false
|
||||
end
|
||||
|
||||
def self.decode_raster_thumbnail(bytes)
|
||||
Timeout.timeout(THUMBNAIL_PROCESS_TIMEOUT) { MiniMagick::Image.read(bytes) }
|
||||
end
|
||||
|
||||
def self.decode_svg_thumbnail(bytes)
|
||||
Timeout.timeout(THUMBNAIL_PROCESS_TIMEOUT) do
|
||||
MiniMagick::Image.read(sanitised_svg_bytes(bytes))
|
||||
end
|
||||
end
|
||||
|
||||
def self.sanitised_svg_bytes(bytes)
|
||||
parse_options =
|
||||
Nokogiri::XML::ParseOptions::STRICT |
|
||||
@@ -256,20 +280,44 @@ class Post < ApplicationRecord
|
||||
|
||||
name = node.name.to_s.downcase
|
||||
next true if name == 'script' || name == 'foreignobject'
|
||||
next style_contains_disallowed_urls?(node.text.to_s) if name == 'style'
|
||||
|
||||
node.attribute_nodes.any? do |attribute|
|
||||
attribute_name = attribute.name.to_s.downcase
|
||||
attribute_value = attribute.value.to_s
|
||||
attribute_name.start_with?('on') ||
|
||||
attribute_name == 'href' ||
|
||||
attribute_name == 'xlink:href' ||
|
||||
attribute_name == 'src' ||
|
||||
(attribute_name == 'style' && attribute_value.match?(/url\s*\(/i)) ||
|
||||
attribute_value.match?(/\A\s*(?:https?:|data:|\/\/)/i)
|
||||
external_svg_reference?(attribute_name, attribute_value)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def self.external_svg_reference?(attribute_name, attribute_value)
|
||||
return external_svg_url?(attribute_value) if ['href', 'xlink:href', 'src'].include?(attribute_name)
|
||||
return style_contains_disallowed_urls?(attribute_value) if attribute_name == 'style'
|
||||
return svg_url_function_disallowed?(attribute_value) if attribute_value.match?(/url\s*\(/i)
|
||||
|
||||
false
|
||||
end
|
||||
|
||||
def self.external_svg_url?(value)
|
||||
stripped = value.to_s.strip
|
||||
return false if stripped.blank? || stripped.start_with?('#')
|
||||
|
||||
stripped.match?(/\A(?:https?:|data:|\/\/)/i)
|
||||
end
|
||||
|
||||
def self.style_contains_disallowed_urls?(value)
|
||||
text = value.to_s
|
||||
text.match?(/@import/i) || svg_url_function_disallowed?(text)
|
||||
end
|
||||
|
||||
def self.svg_url_function_disallowed?(value)
|
||||
value.to_s.scan(/url\s*\(([^)]*)\)/i).flatten.any? do |entry|
|
||||
reference = entry.to_s.strip.delete_prefix("'").delete_prefix('"').delete_suffix("'").delete_suffix('"')
|
||||
reference.present? && !(reference.start_with?('#'))
|
||||
end
|
||||
end
|
||||
|
||||
def self.svg_dimensions(root)
|
||||
width = svg_length_to_pixels(root['width'])
|
||||
height = svg_length_to_pixels(root['height'])
|
||||
@@ -287,15 +335,24 @@ class Post < ApplicationRecord
|
||||
matched = /\A([0-9]+(?:\.[0-9]+)?)(px)?\z/i.match(value.to_s.strip)
|
||||
return nil if matched == nil
|
||||
|
||||
Float(matched[1])
|
||||
pixels = Float(matched[1])
|
||||
return nil unless pixels.finite? && pixels.positive?
|
||||
|
||||
pixels
|
||||
rescue ArgumentError
|
||||
nil
|
||||
end
|
||||
|
||||
private_class_method :image_for_thumbnail_upload,
|
||||
:svg_content_type?,
|
||||
:decode_raster_thumbnail,
|
||||
:decode_svg_thumbnail,
|
||||
:sanitised_svg_bytes,
|
||||
:svg_uses_disallowed_features?,
|
||||
:external_svg_reference?,
|
||||
:external_svg_url?,
|
||||
:style_contains_disallowed_urls?,
|
||||
:svg_url_function_disallowed?,
|
||||
:svg_dimensions,
|
||||
:svg_length_to_pixels
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ module PostCompactRepr
|
||||
Rails.application.routes.url_helpers.rails_storage_proxy_url(
|
||||
post.thumbnail,
|
||||
only_path: false)
|
||||
rescue
|
||||
rescue ActionController::UrlGenerationError, ArgumentError, URI::InvalidURIError
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
@@ -11,8 +11,10 @@ class PostThumbnailUploadValidator
|
||||
end
|
||||
raise InvalidUpload, 'thumbnail file size が大きすぎます.' if thumbnail.size > MAX_THUMBNAIL_BYTES
|
||||
raise InvalidUpload, 'サムネイル画像の形式が不正です.' unless allowed_content_type?(thumbnail.content_type)
|
||||
raise InvalidUpload, 'サムネイル画像の形式が不正です.' if Post.svg_document_bytes?(thumbnail.read)
|
||||
thumbnail.rewind
|
||||
|
||||
attachment = Post.resized_thumbnail_attachment(thumbnail)
|
||||
attachment = Post.resized_thumbnail_attachment(thumbnail, content_type: thumbnail.content_type)
|
||||
attachment[:io].close if attachment[:io].respond_to?(:close)
|
||||
rescue MiniMagick::Error
|
||||
raise InvalidUpload, 'サムネイル画像の変換に失敗しました.'
|
||||
|
||||
@@ -32,7 +32,7 @@ module Preview
|
||||
def self.fetch_image_response(raw_url)
|
||||
uri, = UrlSafety.validate(raw_url)
|
||||
response = HttpFetcher.fetch(uri.to_s)
|
||||
unless allowed_remote_image_content_type?(response.content_type)
|
||||
unless allowed_remote_image_content_type?(response.content_type) || Post.svg_document_bytes?(response.body)
|
||||
raise GenerationFailed, 'サムネール画像が見つかりませんでした.'
|
||||
end
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ type Props = {
|
||||
|
||||
const LABELS: Record<PostImportBadgeValue, string> = {
|
||||
ready: '登録可能',
|
||||
error: '登録不可',
|
||||
warning: '警告',
|
||||
skipped: 'スキップ',
|
||||
created: '登録済み',
|
||||
@@ -17,6 +18,7 @@ const LABELS: Record<PostImportBadgeValue, string> = {
|
||||
|
||||
const TONES: Record<PostImportBadgeValue, StatusBadgeTone> = {
|
||||
ready: 'success',
|
||||
error: 'warning',
|
||||
warning: 'warning',
|
||||
skipped: 'neutral',
|
||||
created: 'success',
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { PostImportRow } from '@/lib/postImportSession'
|
||||
|
||||
export type PostImportDisplayStatus =
|
||||
'ready'
|
||||
| 'error'
|
||||
| 'skipped'
|
||||
| 'warning'
|
||||
| 'created'
|
||||
@@ -18,13 +19,16 @@ export const displayPostImportStatus = (
|
||||
): PostImportDisplayStatus | null =>
|
||||
row.status === 'pending'
|
||||
? null
|
||||
: (row.importStatus === 'failed')
|
||||
? 'failed'
|
||||
: (row.skipReason != null || row.importStatus === 'skipped')
|
||||
? 'skipped'
|
||||
: (row.importStatus === 'created')
|
||||
? 'created'
|
||||
: ((Object.keys (row.validationErrors ?? { }).length > 0
|
||||
|| row.importStatus === 'failed')
|
||||
? 'failed'
|
||||
: (row.status === 'error')
|
||||
? 'error'
|
||||
: (Object.values (row.validationErrors ?? { }).some (messages => messages.length > 0))
|
||||
? 'error'
|
||||
: ((hasWarnings (row) || row.status === 'warning')
|
||||
? 'warning'
|
||||
: 'ready'))
|
||||
: 'ready')
|
||||
|
||||
@@ -22,8 +22,21 @@ export const isManualSkipRow = (row: PostImportRow): boolean =>
|
||||
const hasSkipReason = (row: PostImportRow): boolean =>
|
||||
row.skipReason != null
|
||||
|
||||
export const compactMessageRecord = (
|
||||
messages: Record<string, string[]>,
|
||||
): Record<string, string[]> =>
|
||||
Object.fromEntries (
|
||||
Object.entries (messages).filter (([, values]) => values.length > 0))
|
||||
|
||||
|
||||
export const hasErrorMessages = (
|
||||
messages: Record<string, string[]>,
|
||||
): boolean =>
|
||||
Object.values (messages).some (values => values.length > 0)
|
||||
|
||||
|
||||
const hasValidationErrors = (row: PostImportRow): boolean =>
|
||||
Object.keys (row.validationErrors ?? { }).length > 0
|
||||
hasErrorMessages (row.validationErrors ?? { })
|
||||
|
||||
const isRecoverableRow = (row: PostImportRow): boolean =>
|
||||
row.recoverable === true
|
||||
@@ -79,9 +92,9 @@ const thumbnailWarnings = (row: PostImportRow): string[] => {
|
||||
|
||||
export const applyThumbnailWarning = (row: PostImportRow): PostImportRow => ({
|
||||
...row,
|
||||
fieldWarnings: {
|
||||
fieldWarnings: compactMessageRecord ({
|
||||
...row.fieldWarnings,
|
||||
thumbnailBase: thumbnailWarnings (row) } })
|
||||
thumbnailBase: thumbnailWarnings (row) }) })
|
||||
|
||||
|
||||
export const applyThumbnailWarnings = (rows: PostImportRow[]): PostImportRow[] =>
|
||||
@@ -90,7 +103,7 @@ export const applyThumbnailWarnings = (rows: PostImportRow[]): PostImportRow[] =
|
||||
|
||||
export const processableImportRows = (rows: PostImportRow[]): PostImportRow[] =>
|
||||
validatableImportRows (rows).filter (row => {
|
||||
if (row.metadataUrl == null)
|
||||
if (row.status === 'pending')
|
||||
return false
|
||||
if (row.importStatus === 'created')
|
||||
return false
|
||||
@@ -290,7 +303,7 @@ export const mergeValidatedImportRows = (
|
||||
return previous
|
||||
|
||||
const fieldWarnings =
|
||||
Object.keys (row.fieldWarnings).length > 0 || row.metadataUrl !== previous.metadataUrl
|
||||
hasErrorMessages (row.fieldWarnings) || row.metadataUrl !== previous.metadataUrl
|
||||
? { ...row.fieldWarnings }
|
||||
: { ...previous.fieldWarnings }
|
||||
for (const [field, origin] of Object.entries (row.provenance))
|
||||
@@ -345,9 +358,10 @@ export const mergeImportResults = (
|
||||
createdPostId: result.post.id,
|
||||
existingPostId: undefined,
|
||||
existingPost: undefined,
|
||||
fieldWarnings: result.fieldWarnings ?? row.fieldWarnings,
|
||||
fieldWarnings: compactMessageRecord (
|
||||
result.fieldWarnings ?? row.fieldWarnings),
|
||||
baseWarnings: result.baseWarnings ?? row.baseWarnings,
|
||||
importErrors: result.errors }
|
||||
importErrors: compactMessageRecord (result.errors ?? { }) }
|
||||
case 'skipped':
|
||||
return {
|
||||
...row,
|
||||
@@ -357,9 +371,10 @@ export const mergeImportResults = (
|
||||
createdPostId: undefined,
|
||||
existingPostId: result.existingPostId,
|
||||
existingPost: result.existingPost ?? row.existingPost,
|
||||
fieldWarnings: result.fieldWarnings ?? row.fieldWarnings,
|
||||
fieldWarnings: compactMessageRecord (
|
||||
result.fieldWarnings ?? row.fieldWarnings),
|
||||
baseWarnings: result.baseWarnings ?? row.baseWarnings,
|
||||
importErrors: result.errors }
|
||||
importErrors: compactMessageRecord (result.errors ?? { }) }
|
||||
case 'failed':
|
||||
return {
|
||||
...row,
|
||||
@@ -369,9 +384,10 @@ export const mergeImportResults = (
|
||||
createdPostId: undefined,
|
||||
existingPostId: undefined,
|
||||
existingPost: undefined,
|
||||
fieldWarnings: result.fieldWarnings ?? row.fieldWarnings,
|
||||
fieldWarnings: compactMessageRecord (
|
||||
result.fieldWarnings ?? row.fieldWarnings),
|
||||
baseWarnings: result.baseWarnings ?? row.baseWarnings,
|
||||
importErrors: result.errors }
|
||||
importErrors: compactMessageRecord (result.errors ?? { }) }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -30,8 +30,10 @@ import {
|
||||
clearPostImportSourceDraft,
|
||||
generatePostImportSessionId,
|
||||
hasThumbnailBaseValue,
|
||||
hasErrorMessages,
|
||||
isCompletedReviewRow,
|
||||
isExistingSkipRow,
|
||||
isManualSkipRow,
|
||||
isNonRecoverableFailedRow,
|
||||
mergeImportResults,
|
||||
processableImportRows,
|
||||
@@ -41,7 +43,7 @@ import {
|
||||
reviewSummaryCounts,
|
||||
retryImportRow,
|
||||
loadPostImportSession,
|
||||
serialisedPostImportRowsEqual,
|
||||
compactMessageRecord,
|
||||
savePostImportSession,
|
||||
} from '@/lib/postImportSession'
|
||||
import { useClientBehaviourSettings } from '@/lib/useClientBehaviourSettings'
|
||||
@@ -102,15 +104,48 @@ const isRepairRow = (row: PostImportRow): boolean =>
|
||||
row.recoverable === true
|
||||
&& (row.importStatus === 'failed'
|
||||
|| (row.importStatus === 'pending'
|
||||
&& Object.keys (row.validationErrors).length > 0))
|
||||
&& hasErrorMessages (row.validationErrors)))
|
||||
|
||||
const DUPLICATE_URL_MESSAGE = 'URL が重複しています.'
|
||||
|
||||
const rowMetadataPending = (row: PostImportRow): boolean =>
|
||||
row.status === 'pending'
|
||||
|
||||
const rowOperationBusy = (
|
||||
row: PostImportRow,
|
||||
submitting: boolean,
|
||||
loadingRow: number | null,
|
||||
): boolean =>
|
||||
submitting
|
||||
|| loadingRow === row.sourceRow
|
||||
|| rowMetadataPending (row)
|
||||
|
||||
|
||||
const rowSkipBusy = (
|
||||
row: PostImportRow,
|
||||
submitting: boolean,
|
||||
loadingRow: number | null,
|
||||
): boolean =>
|
||||
submitting
|
||||
|| loadingRow === row.sourceRow
|
||||
|| (rowMetadataPending (row) && !(isManualSkipRow (row)))
|
||||
|
||||
const shouldFetchMetadata = (row: PostImportRow): boolean =>
|
||||
row.status === 'pending'
|
||||
&& row.skipReason !== 'manual'
|
||||
&& row.importStatus !== 'created'
|
||||
&& row.importStatus !== 'skipped'
|
||||
&& !(isNonRecoverableFailedRow (row))
|
||||
&& row.metadataUrl == null
|
||||
|
||||
const applyDuplicateUrlErrors = (
|
||||
rows: PostImportRow[],
|
||||
): PostImportRow[] => {
|
||||
const counts = rows.reduce<Record<string, number>> ((result, row) => {
|
||||
if (row.url !== '')
|
||||
if (row.url !== ''
|
||||
&& !(isManualSkipRow (row))
|
||||
&& row.importStatus !== 'created'
|
||||
&& row.importStatus !== 'skipped')
|
||||
result[row.url] = (result[row.url] ?? 0) + 1
|
||||
return result
|
||||
}, { })
|
||||
@@ -120,27 +155,30 @@ const applyDuplicateUrlErrors = (
|
||||
message => message !== DUPLICATE_URL_MESSAGE)
|
||||
if ((counts[row.url] ?? 0) < 2)
|
||||
{
|
||||
const nextValidationErrors = {
|
||||
const nextValidationErrors = compactMessageRecord ({
|
||||
...row.validationErrors,
|
||||
url: urlErrors }
|
||||
const hasErrors = Object.values (nextValidationErrors).some (
|
||||
messages => messages.length > 0)
|
||||
url: urlErrors })
|
||||
const hasErrors = hasErrorMessages (nextValidationErrors)
|
||||
return urlErrors.length === (row.validationErrors.url ?? []).length
|
||||
? row
|
||||
: {
|
||||
...row,
|
||||
validationErrors: nextValidationErrors,
|
||||
skipReason:
|
||||
row.skipReason === 'manual'
|
||||
? 'manual'
|
||||
: row.existingPostId != null
|
||||
? 'existing'
|
||||
: undefined,
|
||||
status: hasErrors ? 'error' : row.status }
|
||||
}
|
||||
|
||||
return {
|
||||
...row,
|
||||
skipReason: row.skipReason === 'manual' ? 'manual' : undefined,
|
||||
existingPostId: undefined,
|
||||
existingPost: undefined,
|
||||
validationErrors: {
|
||||
validationErrors: compactMessageRecord ({
|
||||
...row.validationErrors,
|
||||
url: [...new Set ([...urlErrors, DUPLICATE_URL_MESSAGE])] },
|
||||
url: [...new Set ([...urlErrors, DUPLICATE_URL_MESSAGE])] }),
|
||||
status: 'error' }
|
||||
})
|
||||
}
|
||||
@@ -179,8 +217,9 @@ const mergeDryRunRow = (
|
||||
currentRow: PostImportRow,
|
||||
result: PostMetadataResponse,
|
||||
): PostImportRow => {
|
||||
const fieldWarnings = compactMessageRecord (result.fieldWarnings ?? { })
|
||||
const hasWarnings =
|
||||
Object.values (result.fieldWarnings ?? { }).some (messages => messages.length > 0)
|
||||
Object.values (fieldWarnings).some (messages => messages.length > 0)
|
||||
|| (result.baseWarnings?.length ?? 0) > 0
|
||||
return applyThumbnailWarning ({
|
||||
...currentRow,
|
||||
@@ -195,7 +234,7 @@ const mergeDryRunRow = (
|
||||
videoMs: result.videoMs ?? '',
|
||||
tags: result.tags ?? '',
|
||||
parentPostIds: String (result.parentPostIds ?? currentRow.attributes.parentPostIds ?? '') },
|
||||
fieldWarnings: result.fieldWarnings ?? { },
|
||||
fieldWarnings,
|
||||
baseWarnings: result.baseWarnings ?? [],
|
||||
validationErrors: { },
|
||||
importErrors: undefined,
|
||||
@@ -291,9 +330,9 @@ const mergePreviewRow = (
|
||||
currentRow: PostImportRow,
|
||||
preview: PostMetadataResponse,
|
||||
): PostImportRow => {
|
||||
const fieldWarnings = preview.fieldWarnings ?? { }
|
||||
const fieldWarnings = compactMessageRecord (preview.fieldWarnings ?? { })
|
||||
const baseWarnings = preview.baseWarnings ?? []
|
||||
const validationErrors = preview.validationErrors ?? { }
|
||||
const validationErrors = compactMessageRecord (preview.validationErrors ?? { })
|
||||
const metadataChanged = currentRow.metadataUrl !== preview.url
|
||||
const hasWarnings =
|
||||
Object.values (fieldWarnings).some (messages => messages.length > 0)
|
||||
@@ -438,7 +477,20 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
const persistSequenceRef = useRef (0)
|
||||
const previewSequenceRef = useRef (0)
|
||||
|
||||
const persistSession = async (
|
||||
const commitSessionState = (
|
||||
nextSession: PostImportSession,
|
||||
): PostImportSession => {
|
||||
const sessionId =
|
||||
sessionIdRef.current ?? generatePostImportSessionId ()
|
||||
sessionIdRef.current = sessionId
|
||||
const persistedSession = buildSession (nextSession.rows)
|
||||
savePostImportSession (sessionId, persistedSession)
|
||||
sessionRef.current = persistedSession
|
||||
setSession (persistedSession)
|
||||
return persistedSession
|
||||
}
|
||||
|
||||
const syncSessionUrl = async (
|
||||
nextSession: PostImportSession,
|
||||
): Promise<PostImportSession | null> => {
|
||||
const sessionId =
|
||||
@@ -451,26 +503,21 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
if (persistSequenceRef.current !== sequence)
|
||||
return null
|
||||
|
||||
const saved = savePostImportSession (sessionId, nextSession)
|
||||
const loadedSession = loadPostImportSession (sessionId)
|
||||
const canRecoverFromUrlOnly = serialised.complete
|
||||
if (!(canRecoverFromUrlOnly)
|
||||
&& (!(saved))
|
||||
&& !(serialisedPostImportRowsEqual (loadedSession?.rows ?? [], nextSession.rows)))
|
||||
return null
|
||||
if (saved
|
||||
&& !(serialisedPostImportRowsEqual (loadedSession?.rows ?? [], nextSession.rows)))
|
||||
return null
|
||||
const persistedSession = commitSessionState (nextSession)
|
||||
|
||||
const persistedSession = buildSession (nextSession.rows)
|
||||
const path = appendPostNewSessionId (serialised.path, sessionId)
|
||||
persistedSearchRef.current = (new URL (path, window.location.origin)).search
|
||||
sessionRef.current = persistedSession
|
||||
setSession (persistedSession)
|
||||
navigate (path, { replace: true })
|
||||
return persistedSession
|
||||
}
|
||||
|
||||
const currentRowBySource = (
|
||||
sourceRow: number,
|
||||
fallback?: PostImportSession,
|
||||
): PostImportRow | null =>
|
||||
(sessionRef.current?.rows ?? fallback?.rows ?? []).find (
|
||||
row => row.sourceRow === sourceRow) ?? null
|
||||
|
||||
const finishImport = () => {
|
||||
const sessionId = sessionIdRef.current
|
||||
if (sessionId != null)
|
||||
@@ -485,7 +532,9 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
const previewSequence = ++previewSequenceRef.current
|
||||
const controllers = new Set<AbortController> ()
|
||||
|
||||
if (sessionRef.current != null && persistedSearchRef.current === location.search)
|
||||
if (sessionRef.current != null
|
||||
&& persistedSearchRef.current === location.search
|
||||
&& !(sessionRef.current.rows.some (row => shouldFetchMetadata (row))))
|
||||
return () => {
|
||||
previewSequenceRef.current = previewSequence
|
||||
}
|
||||
@@ -496,20 +545,16 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
try
|
||||
{
|
||||
let nextIndex = 0
|
||||
const previewErrors = new Map<number, Record<string, string[]>> ()
|
||||
let workingRows = [...(sessionRef.current ?? baseSession).rows]
|
||||
|
||||
const mergeWorkingRow = (
|
||||
const mergeCurrentRow = (
|
||||
sourceRow: number,
|
||||
updater: (row: PostImportRow) => PostImportRow,
|
||||
) => {
|
||||
workingRows = workingRows.map (row =>
|
||||
const latestRows = sessionRef.current?.rows ?? baseSession.rows
|
||||
const nextRows = applyDuplicateUrlErrors (latestRows.map (row =>
|
||||
row.sourceRow === sourceRow
|
||||
? updater (row)
|
||||
: row)
|
||||
const nextSession = buildSession (workingRows)
|
||||
sessionRef.current = nextSession
|
||||
setSession (nextSession)
|
||||
: row))
|
||||
commitSessionState (buildSession (nextRows))
|
||||
}
|
||||
const worker = async () => {
|
||||
while (active)
|
||||
@@ -520,22 +565,25 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
return
|
||||
|
||||
const baseRow = baseSession.rows[index]
|
||||
if (baseRow.skipReason === 'manual'
|
||||
|| baseRow.importStatus === 'created'
|
||||
|| baseRow.importStatus === 'skipped'
|
||||
|| isNonRecoverableFailedRow (baseRow)
|
||||
|| baseRow.metadataUrl != null)
|
||||
const currentRow = currentRowBySource (baseRow.sourceRow, baseSession)
|
||||
if (currentRow == null || !(shouldFetchMetadata (currentRow)))
|
||||
continue
|
||||
const controller = new AbortController ()
|
||||
controllers.add (controller)
|
||||
try
|
||||
{
|
||||
const preview = await apiGet<PostMetadataResponse> ('/posts/metadata', {
|
||||
params: { url: baseRow.url },
|
||||
params: { url: currentRow.url },
|
||||
signal: controller.signal })
|
||||
if (!(active) || previewSequenceRef.current !== previewSequence)
|
||||
return
|
||||
mergeWorkingRow (baseRow.sourceRow, row => mergePreviewRow (row, preview))
|
||||
mergeCurrentRow (currentRow.sourceRow, row =>
|
||||
row.importStatus === 'created'
|
||||
|| row.importStatus === 'skipped'
|
||||
|| isNonRecoverableFailedRow (row)
|
||||
|| row.skipReason === 'manual'
|
||||
? row
|
||||
: mergePreviewRow (row, preview))
|
||||
}
|
||||
catch (requestError)
|
||||
{
|
||||
@@ -547,21 +595,29 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
errors?: Record<string, string[]>
|
||||
baseErrors?: string[]
|
||||
}> (requestError)))
|
||||
{
|
||||
mergeCurrentRow (currentRow.sourceRow, row => ({
|
||||
...row,
|
||||
status: 'error' }))
|
||||
continue
|
||||
}
|
||||
if (requestError.response?.status === 422)
|
||||
{
|
||||
const rowErrors = {
|
||||
const rowErrors = compactMessageRecord ({
|
||||
...(requestError.response.data.errors ?? { }),
|
||||
...(requestError.response.data.baseErrors?.length
|
||||
? {
|
||||
base: requestError.response.data.baseErrors }
|
||||
: { }) }
|
||||
previewErrors.set (baseRow.sourceRow, rowErrors)
|
||||
mergeWorkingRow (baseRow.sourceRow, row => ({
|
||||
: { }) })
|
||||
mergeCurrentRow (currentRow.sourceRow, row => ({
|
||||
...row,
|
||||
validationErrors: rowErrors,
|
||||
status: 'error' }))
|
||||
}
|
||||
else
|
||||
mergeCurrentRow (currentRow.sourceRow, row => ({
|
||||
...row,
|
||||
status: 'error' }))
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -575,20 +631,13 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
if (!(active) || previewSequenceRef.current !== previewSequence)
|
||||
return
|
||||
|
||||
const nextRows = applyDuplicateUrlErrors (workingRows.map (row => {
|
||||
const rowErrors = previewErrors.get (row.sourceRow)
|
||||
if (rowErrors == null)
|
||||
return row
|
||||
return {
|
||||
...row,
|
||||
validationErrors: rowErrors,
|
||||
status: 'error' as const }
|
||||
}))
|
||||
await persistSession (buildSession (nextRows))
|
||||
const latestRows = sessionRef.current?.rows ?? baseSession.rows
|
||||
const nextRows = applyDuplicateUrlErrors (latestRows)
|
||||
await syncSessionUrl (buildSession (nextRows))
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (active)
|
||||
if (active && previewSequenceRef.current === previewSequence)
|
||||
setMetadataLoading (false)
|
||||
}
|
||||
}
|
||||
@@ -649,18 +698,19 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
: rows
|
||||
const existingRows = sortedRows.filter (row => isExistingSkipRow (row))
|
||||
const reviewRows = sortedRows.filter (row => !(isExistingSkipRow (row)))
|
||||
const busy = metadataLoading || submitting || loadingRow != null
|
||||
const canSubmit =
|
||||
rows.every (row => isCompletedReviewRow (row))
|
||||
|| processableImportRows (rows).length > 0
|
||||
|
||||
const toggleManualSkip = async (sourceRow: number, checked: boolean) => {
|
||||
if (busy)
|
||||
return
|
||||
|
||||
const currentSession = sessionRef.current
|
||||
if (currentSession == null)
|
||||
return
|
||||
const currentRow = currentRowBySource (sourceRow, currentSession)
|
||||
if (currentRow == null)
|
||||
return
|
||||
if (rowSkipBusy (currentRow, submitting, loadingRow))
|
||||
return
|
||||
|
||||
const nextRows = currentSession.rows.map (row => {
|
||||
if (row.sourceRow !== sourceRow)
|
||||
@@ -674,7 +724,13 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
existingPostId: checked ? undefined : row.existingPostId,
|
||||
existingPost: checked ? undefined : row.existingPost }
|
||||
})
|
||||
await persistSession (buildSession (nextRows))
|
||||
const nextSession = buildSession (applyDuplicateUrlErrors (nextRows))
|
||||
if (metadataLoading)
|
||||
{
|
||||
commitSessionState (nextSession)
|
||||
return
|
||||
}
|
||||
await syncSessionUrl (nextSession)
|
||||
}
|
||||
|
||||
const saveDraft = async (
|
||||
@@ -687,23 +743,26 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
const currentSession = sessionRef.current
|
||||
if (currentSession == null)
|
||||
return { saved: false, row: null }
|
||||
if (!(canEditReviewRow (row)))
|
||||
const currentRow = currentRowBySource (row.sourceRow, currentSession)
|
||||
if (currentRow == null)
|
||||
return { saved: false, row: null }
|
||||
if (!(canEditReviewRow (currentRow)))
|
||||
return { saved: false, row: null }
|
||||
|
||||
const baseRow =
|
||||
resetRequested
|
||||
? { ...row,
|
||||
url: row.resetSnapshot.url,
|
||||
attributes: { ...row.resetSnapshot.attributes },
|
||||
provenance: { ...row.resetSnapshot.provenance },
|
||||
tagSources: { ...row.resetSnapshot.tagSources },
|
||||
? { ...currentRow,
|
||||
url: currentRow.resetSnapshot.url,
|
||||
attributes: { ...currentRow.resetSnapshot.attributes },
|
||||
provenance: { ...currentRow.resetSnapshot.provenance },
|
||||
tagSources: { ...currentRow.resetSnapshot.tagSources },
|
||||
fieldWarnings: Object.fromEntries (
|
||||
Object.entries (row.resetSnapshot.fieldWarnings)
|
||||
Object.entries (currentRow.resetSnapshot.fieldWarnings)
|
||||
.map (([key, values]) => [key, [...values]])),
|
||||
baseWarnings: [...row.resetSnapshot.baseWarnings],
|
||||
metadataUrl: row.resetSnapshot.metadataUrl,
|
||||
baseWarnings: [...currentRow.resetSnapshot.baseWarnings],
|
||||
metadataUrl: currentRow.resetSnapshot.metadataUrl,
|
||||
thumbnailFile: undefined }
|
||||
: row
|
||||
: currentRow
|
||||
const urlChanged = draft.url !== baseRow.url
|
||||
const nextRow = buildNextEditedRow (baseRow, draft, urlChanged)
|
||||
let candidateRow = nextRow
|
||||
@@ -724,15 +783,19 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
return { saved: false, row: null }
|
||||
|
||||
const mergedRow = mergeDryRunRow (candidateRow, dryRun)
|
||||
const mergedRows = replaceImportRow (latestSession.rows, mergedRow)
|
||||
const nextSession = await persistSession (buildSession (mergedRows))
|
||||
const mergedRows = applyDuplicateUrlErrors (
|
||||
replaceImportRow (latestSession.rows, mergedRow))
|
||||
const nextSession =
|
||||
metadataLoading
|
||||
? commitSessionState (buildSession (mergedRows))
|
||||
: await syncSessionUrl (buildSession (mergedRows))
|
||||
if (nextSession == null)
|
||||
return { saved: false, row: null }
|
||||
const mergedTarget = nextSession.rows.find (
|
||||
currentRow => currentRow.sourceRow === baseRow.sourceRow)
|
||||
if (mergedTarget == null)
|
||||
return { saved: false, row: null }
|
||||
if (Object.keys (mergedTarget.validationErrors).length > 0)
|
||||
if (hasErrorMessages (mergedTarget.validationErrors))
|
||||
return { saved: false, row: mergedTarget }
|
||||
return { saved: true, row: null }
|
||||
}
|
||||
@@ -746,7 +809,12 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
{
|
||||
const errorRow = {
|
||||
...candidateRow,
|
||||
validationErrors: requestError.response.data.errors ?? { },
|
||||
validationErrors: compactMessageRecord ({
|
||||
...(requestError.response.data.errors ?? { }),
|
||||
...(requestError.response.data.baseErrors?.length
|
||||
? {
|
||||
base: requestError.response.data.baseErrors }
|
||||
: { }) }),
|
||||
baseWarnings: [],
|
||||
fieldWarnings: { },
|
||||
status: 'error' as const }
|
||||
@@ -778,39 +846,48 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
}
|
||||
|
||||
const editRow = async (row: PostImportRow) => {
|
||||
setEditingRow (row)
|
||||
const currentRow = sessionRef.current?.rows.find (
|
||||
current => current.sourceRow === row.sourceRow)
|
||||
if (currentRow == null)
|
||||
return
|
||||
if (!(canEditReviewRow (currentRow)))
|
||||
return
|
||||
if (rowOperationBusy (currentRow, submitting, loadingRow))
|
||||
return
|
||||
|
||||
setEditingRow (currentRow)
|
||||
try
|
||||
{
|
||||
await openEditingDialogue (row)
|
||||
await openEditingDialogue (currentRow)
|
||||
}
|
||||
finally
|
||||
{
|
||||
setEditingRow (current =>
|
||||
current?.sourceRow === row.sourceRow
|
||||
current?.sourceRow === currentRow.sourceRow
|
||||
? null
|
||||
: current)
|
||||
}
|
||||
}
|
||||
|
||||
const retry = async (sourceRow: number) => {
|
||||
if (busy)
|
||||
return
|
||||
|
||||
const initialSession = sessionRef.current
|
||||
if (initialSession == null)
|
||||
return
|
||||
|
||||
setLoadingRow (sourceRow)
|
||||
const originalRow = initialSession.rows.find (row => row.sourceRow === sourceRow)
|
||||
if (originalRow == null)
|
||||
{
|
||||
setLoadingRow (null)
|
||||
return
|
||||
}
|
||||
if (rowOperationBusy (originalRow, submitting, loadingRow))
|
||||
return
|
||||
|
||||
setLoadingRow (sourceRow)
|
||||
try
|
||||
{
|
||||
const pendingRows = retryImportRow (initialSession.rows, sourceRow)
|
||||
const pendingSession = await persistSession (buildSession (pendingRows))
|
||||
const pendingSession =
|
||||
metadataLoading
|
||||
? commitSessionState (buildSession (pendingRows))
|
||||
: await syncSessionUrl (buildSession (pendingRows))
|
||||
if (pendingSession == null)
|
||||
return
|
||||
|
||||
@@ -825,16 +902,20 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
{
|
||||
const latest = sessionRef.current ?? pendingSession
|
||||
const restoredRows = replaceImportRow (latest.rows, originalRow)
|
||||
await persistSession (buildSession (restoredRows))
|
||||
if (metadataLoading)
|
||||
commitSessionState (buildSession (restoredRows))
|
||||
else
|
||||
await syncSessionUrl (buildSession (restoredRows))
|
||||
toast ({ title: '登録結果が不完全でした' })
|
||||
return
|
||||
}
|
||||
|
||||
const indexedResults = indexedBulkResults ([target], result.results)
|
||||
const latestAfterImport = sessionRef.current ?? pendingSession
|
||||
const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults)
|
||||
const nextRows = applyDuplicateUrlErrors (
|
||||
mergeImportResults (latestAfterImport.rows, indexedResults))
|
||||
.map (row => applyThumbnailWarning (row))
|
||||
const persisted = await persistSession (buildSession (nextRows))
|
||||
const persisted = await syncSessionUrl (buildSession (nextRows))
|
||||
if (persisted != null
|
||||
&& persisted.rows.every (row => isCompletedReviewRow (row)))
|
||||
finishImport ()
|
||||
@@ -843,7 +924,10 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
{
|
||||
const latest = sessionRef.current ?? initialSession
|
||||
const restoredRows = replaceImportRow (latest.rows, originalRow)
|
||||
await persistSession (buildSession (restoredRows))
|
||||
if (metadataLoading)
|
||||
commitSessionState (buildSession (restoredRows))
|
||||
else
|
||||
await syncSessionUrl (buildSession (restoredRows))
|
||||
toast ({ title: '再試行に失敗しました' })
|
||||
}
|
||||
finally
|
||||
@@ -854,7 +938,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
|
||||
const submit = async () => {
|
||||
const currentSession = sessionRef.current
|
||||
if (currentSession == null || busy)
|
||||
if (currentSession == null || metadataLoading || submitting || loadingRow != null)
|
||||
return
|
||||
if (currentSession.rows.every (row => isCompletedReviewRow (row)))
|
||||
{
|
||||
@@ -880,9 +964,10 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
|
||||
const indexedResults = indexedBulkResults (processingRows, result.results)
|
||||
const latestAfterImport = sessionRef.current ?? buildSession (currentSession.rows)
|
||||
const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults)
|
||||
const nextRows = applyDuplicateUrlErrors (
|
||||
mergeImportResults (latestAfterImport.rows, indexedResults))
|
||||
.map (row => applyThumbnailWarning (row))
|
||||
const persisted = await persistSession (buildSession (nextRows))
|
||||
const persisted = await syncSessionUrl (buildSession (nextRows))
|
||||
if (persisted != null
|
||||
&& persisted.rows.every (row => isCompletedReviewRow (row)))
|
||||
finishImport ()
|
||||
@@ -958,10 +1043,10 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
|
||||
row={row}
|
||||
displayNumber={index + 1}
|
||||
showSkipToggle={true}
|
||||
editDisabled={busy}
|
||||
retryDisabled={busy}
|
||||
editDisabled={rowOperationBusy (row, submitting, loadingRow)}
|
||||
retryDisabled={rowOperationBusy (row, submitting, loadingRow)}
|
||||
skipDisabled={
|
||||
busy
|
||||
rowSkipBusy (row, submitting, loadingRow)
|
||||
|| isExistingSkipRow (row)
|
||||
|| row.importStatus === 'created'}
|
||||
onEdit={() => editRow (row)}
|
||||
|
||||
新しい課題から参照
ユーザをブロックする