diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index 5ef7e5e..bdd0a6a 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -1,3 +1,5 @@ +require 'timeout' + class PostImportPreviewer FIELDS = [ 'title', @@ -6,8 +8,7 @@ class PostImportPreviewer 'original_created_before', 'duration', 'tags', - 'parent_post_ids', - ].freeze + 'parent_post_ids'].freeze FETCH_WARNING_FIELDS = ['url', 'title', 'thumbnail_base'].freeze EXISTING_SKIP_WARNING = '既存投稿のためスキップします．'.freeze TITLE_FETCH_WARNING = 'タイトルを取得できませんでした．'.freeze @@ -15,15 +16,26 @@ class PostImportPreviewer METADATA_FETCH_WARNING = 'メタデータを取得できませんでした．'.freeze def preview_rows rows:, fetch_metadata: true, metadata_cache: { } - urls = {} prepared_rows = rows.map { prepare_row(_1) } + url_counts = prepared_rows.filter_map { _1[:normal_url] }.tally existing_posts = Post.where(url: prepared_rows.map { _1[:normal_url] }.compact.uniq).index_by(&:url) + known_tags = preload_known_tags(prepared_rows) + existing_parent_ids = preload_parent_ids(prepared_rows) + preload_metadata!( + prepared_rows, + fetch_metadata, + metadata_cache, + existing_posts, + url_counts) prepared_rows.map { |row| - preview_row(row, urls:, + preview_row(row, fetch_metadata:, metadata_cache:, - existing_posts:) + existing_posts:, + url_counts:, + known_tags:, + existing_parent_ids:) } end @@ -40,7 +52,13 @@ class PostImportPreviewer source.merge(url_text: url, normal_url:) end - def preview_row row, urls:, fetch_metadata:, metadata_cache:, existing_posts: + def preview_row row, + fetch_metadata:, + metadata_cache:, + existing_posts:, + url_counts:, + known_tags:, + existing_parent_ids: attributes = initial_attributes(row) provenance = initial_provenance(row) tag_sources = initial_tag_sources(row, attributes, provenance) @@ -54,10 +72,9 @@ class PostImportPreviewer validation_errors = {} validation_errors[:url] = ['URL が不正です．'] if normal_url.blank? - if normal_url.present? && urls.key?(normal_url) + if normal_url.present? && url_counts[normal_url].to_i > 1 validation_errors[:url] = ['URL が重複しています．'] end - urls[normal_url] = true if normal_url.present? if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata clear_automatic_values!(attributes, provenance, tag_sources) @@ -89,7 +106,8 @@ class PostImportPreviewer end } end - should_fetch = should_fetch_metadata?(fetch_metadata, row[:source_row].to_i) + should_fetch = validation_errors.blank? + should_fetch &&= should_fetch_metadata?(fetch_metadata, row[:source_row].to_i) if should_fetch clear_fetch_warnings!(field_warnings) metadata = metadata_for(url_for_metadata, metadata_cache) @@ -101,8 +119,9 @@ class PostImportPreviewer validate_basic_data(attributes, validation_errors) validate_preview_tags(merged_tags(tag_sources, provenance['tags']), validation_errors, - field_warnings) - validate_parents(attributes['parent_post_ids'], validation_errors) + field_warnings, + known_tags) + validate_parents(attributes['parent_post_ids'], validation_errors, existing_parent_ids) attributes.delete('url') attributes['tags'] = merged_tags(tag_sources, provenance['tags']) @@ -197,6 +216,64 @@ class PostImportPreviewer { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } } end + def preload_metadata! prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts + urls = prepared_rows.filter_map { |row| + next unless row[:normal_url].present? + next unless should_fetch_metadata?(fetch_metadata, row[:source_row].to_i) + next if url_counts[row[:normal_url]].to_i > 1 + next if existing_posts.key?(row[:normal_url]) + + row[:normal_url] + }.uniq + urls = urls.reject { metadata_cache.key?(_1) } + return if urls.empty? + + queue = Queue.new + urls.each { queue << _1 } + workers = [urls.length, 4].min.times.map { + Thread.new { + loop do + url = queue.pop(true) + metadata_cache[url] = fetch_metadata(url) + rescue ThreadError + break + end + } + } + Timeout.timeout(15) { workers.each(&:join) } + rescue Timeout::Error + workers&.each(&:kill) + urls.each do |url| + metadata_cache[url] ||= { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } } + end + end + + def preload_known_tags prepared_rows + names = prepared_rows.flat_map { |row| + attributes = initial_attributes(row) + provenance = initial_provenance(row) + tag_sources = initial_tag_sources(row, attributes, provenance) + preview_tag_names(merged_tags(tag_sources, provenance['tags'])) + }.compact.uniq + return { } if names.empty? + + Tag.joins(:tag_name) + .where(tag_names: { name: names }) + .includes(:tag_name) + .to_a + .index_by(&:name) + end + + def preload_parent_ids prepared_rows + ids = prepared_rows.flat_map { |row| + attributes = initial_attributes(row) + preview_parent_ids(attributes['parent_post_ids']) + }.uniq + return { } if ids.empty? + + Post.where(id: ids).pluck(:id).to_h { [_1, true] } + end + def apply_metadata! attributes, provenance, tag_sources, metadata metadata.each do |field, value| if field == 'tags' @@ -232,7 +309,19 @@ class PostImportPreviewer sources['automatic'].to_s end - def validate_preview_tags raw, errors, field_warnings + def preview_tag_names raw + names = raw.to_s.split + return [] if names.empty? + if names.any? { _1.downcase.start_with?('nico:') } + return [] + end + + names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first } + rescue Tag::SectionLiteralParseError + [] + end + + def validate_preview_tags raw, errors, field_warnings, known_tags names = raw.to_s.split return if names.empty? if names.any? { _1.downcase.start_with?('nico:') } @@ -241,7 +330,7 @@ class PostImportPreviewer end parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first } - existing = Tag.joins(:tag_name).where(tag_names: { name: parsed }).includes(:tag_name).to_a + existing = parsed.filter_map { known_tags[_1] } deprecated = existing.select(&:deprecated?).map(&:name) errors[:tags] = ["廃止済みタグがあります: #{ deprecated.join(' ') }"] if deprecated.present? known = existing.reject(&:deprecated?).map(&:name) @@ -277,7 +366,11 @@ class PostImportPreviewer nil end - def validate_parents raw, errors + def preview_parent_ids raw + raw.to_s.split.map { Integer(_1, exception: false) }.compact + end + + def validate_parents raw, errors, existing_parent_ids ids = raw.to_s.split.map { Integer(_1, exception: false) } return if ids.compact.empty? && raw.to_s.blank? @@ -285,10 +378,8 @@ class PostImportPreviewer errors[:parent_post_ids] = ['親投稿 Id. が不正です．'] return end - if Post.where(id: ids).count != ids.uniq.length + if ids.uniq.any? { !existing_parent_ids[_1] } errors[:parent_post_ids] = ['存在しない親投稿 Id. があります．'] end end - - private :prepare_row end diff --git a/backend/app/services/post_import_row_normaliser.rb b/backend/app/services/post_import_row_normaliser.rb index fa2b7be..f03c17e 100644 --- a/backend/app/services/post_import_row_normaliser.rb +++ b/backend/app/services/post_import_row_normaliser.rb @@ -6,8 +6,7 @@ class PostImportRowNormaliser 'original_created_from', 'original_created_before', 'tags', - 'parent_post_ids', - ].freeze + 'parent_post_ids'].freeze FLEXIBLE_FIELDS = ['duration', 'video_ms'].freeze ATTRIBUTE_FIELDS = (STRING_FIELDS + FLEXIBLE_FIELDS).freeze @@ -59,16 +58,14 @@ class PostImportRowNormaliser { attributes: {} }, { provenance: {} }, { tag_sources: {} }, - { tagSources: {} }, - ] + { tagSources: {} }] return keys unless allow_warning_fields keys + [ { field_warnings: {} }, { fieldWarnings: {} }, { base_warnings: [] }, - { baseWarnings: [] }, - ] + { baseWarnings: [] }] end private_class_method :permitted_keys diff --git a/backend/app/services/post_metadata_fetcher.rb b/backend/app/services/post_metadata_fetcher.rb index d7a3baa..23cb12f 100644 --- a/backend/app/services/post_metadata_fetcher.rb +++ b/backend/app/services/post_metadata_fetcher.rb @@ -1,8 +1,9 @@ class PostMetadataFetcher TIMESTAMP_PATTERN = - /\A(\d{4})-(\d{2})-(\d{2})T(\d{2})/ \ - /(?::(\d{2})(?::(\d{2})(?:\.(\d+))?)?)?/ \ - /(Z|[+-]\d{2}:?\d{2})?\z/ + Regexp.new( + '\A(\d{4})-(\d{2})-(\d{2})T(\d{2})' \ + '(?::(\d{2})(?::(\d{2})(?:\.(\d+))?)?)?' \ + '(Z|[+-]\d{2}:?\d{2})?\z') def self.fetch raw_url uri, = Preview::UrlSafety.validate(raw_url) @@ -80,9 +81,14 @@ class PostMetadataFetcher minute = match[5]&.to_i || 0 second = match[6]&.to_i || 0 fraction = match[7] - offset = parse_offset(match[8]) - from = Time.new(year, month, day, hour, minute, second + fractional_seconds(fraction), offset) - .in_time_zone + offset = match[8] + whole_second = second + fractional_seconds(fraction) + from = + if offset.present? + Time.new(year, month, day, hour, minute, whole_second, parse_offset(offset)).in_time_zone + else + Time.zone.local(year, month, day, hour, minute, whole_second) + end before = if fraction.present? from + (10**(-fraction.length)) @@ -97,7 +103,6 @@ class PostMetadataFetcher end def self.parse_offset value - return Time.zone.formatted_offset if value.blank? return '+00:00' if value == 'Z' value.match?(/\A[+-]\d{2}:\d{2}\z/) ? value : "#{ value[0, 3] }:#{ value[3, 2] }" diff --git a/frontend/src/components/posts/import/PostImportRowDialog.tsx b/frontend/src/components/posts/import/PostImportRowDialog.tsx index 7472e5b..4224f66 100644 --- a/frontend/src/components/posts/import/PostImportRowDialog.tsx +++ b/frontend/src/components/posts/import/PostImportRowDialog.tsx @@ -54,16 +54,18 @@ const originalCreatedOrigin = ( row: PostImportRow, originalDraft: Draft, draft: Draft, -): PostImportOrigin => - originalDraft.originalCreatedFrom !== draft.originalCreatedFrom - || originalDraft.originalCreatedBefore !== draft.originalCreatedBefore - ? 'manual' - : ( +): PostImportOrigin => { + const changed = + originalDraft.originalCreatedFrom !== draft.originalCreatedFrom + || originalDraft.originalCreatedBefore !== draft.originalCreatedBefore + if (changed) + return 'manual' + + const manualOrigin = originOf (row, 'originalCreatedFrom') === 'manual' || originOf (row, 'originalCreatedBefore') === 'manual' - ? 'manual' - : 'automatic' - ) + return manualOrigin ? 'manual' : 'automatic' +} const buildDraft = (row: PostImportRow): Draft => ({ url: row.url, @@ -91,7 +93,7 @@ const PostImportRowDialog: FC = ( setDraft (buildDraft (row)) }, [open, row]) - if (!(row) || !(draft)) + if (row == null || draft == null) return null const originalDraft = buildDraft (row) diff --git a/frontend/src/lib/postImportRows.ts b/frontend/src/lib/postImportRows.ts new file mode 100644 index 0000000..12bb8f9 --- /dev/null +++ b/frontend/src/lib/postImportRows.ts @@ -0,0 +1,196 @@ +import type { PostImportResultRow, + PostImportRow } from '@/lib/postImportTypes' + +const hasSkipReason = (row: PostImportRow): boolean => + row.skipReason === 'existing' + +const hasValidationErrors = (row: PostImportRow): boolean => + Object.keys (row.validationErrors ?? { }).length > 0 + +const rowChanged = ( + previous: PostImportRow, + next: PostImportRow, +): boolean => + previous.url !== next.url + || JSON.stringify (previous.attributes) !== JSON.stringify (next.attributes) + || JSON.stringify (previous.provenance) !== JSON.stringify (next.provenance) + || JSON.stringify (previous.tagSources ?? { }) !== JSON.stringify (next.tagSources ?? { }) + + +export const processableImportRows = (rows: PostImportRow[]): PostImportRow[] => + rows.filter (row => { + if (row.importStatus === 'created') + return false + if (row.importStatus === 'skipped') + return false + if (row.importStatus === 'failed') + return false + if (hasValidationErrors (row)) + return false + return row.importStatus == null || row.importStatus === 'pending' + }) + +export const creatableImportRows = (rows: PostImportRow[]): PostImportRow[] => + processableImportRows (rows).filter (row => !(hasSkipReason (row))) + + +export const reviewSummaryCounts = (rows: PostImportRow[]) => ({ + total: rows.length, + submittable: rows.filter (row => + creatableImportRows ([row]).length > 0 + && !(hasValidationErrors (row))).length, + invalid: rows.filter (row => hasValidationErrors (row)).length, + skipPlanned: rows.filter (row => + hasSkipReason (row) + && !(hasValidationErrors (row)) + && row.importStatus !== 'created').length, + created: rows.filter (row => row.importStatus === 'created').length, + failed: rows.filter (row => row.importStatus === 'failed').length }) + + +export const resultSummaryCounts = (rows: PostImportRow[]) => + rows.reduce ( + (counts, row) => { + if (hasValidationErrors (row)) + { + ++counts.invalid + return counts + } + if (row.importStatus === 'created') + { + ++counts.created + return counts + } + if (row.importStatus === 'skipped') + { + ++counts.skipped + return counts + } + if (row.importStatus === 'failed') + ++counts.failed + return counts + }, + { created: 0, skipped: 0, failed: 0, invalid: 0 }) + + +export const mergeValidatedImportRows = ( + current: PostImportRow[], + validated: PostImportRow[], +): PostImportRow[] => { + const validatedMap = new Map (validated.map (row => [row.sourceRow, row])) + return current.map (previous => { + if (previous.importStatus === 'created') + return previous + + const row = validatedMap.get (previous.sourceRow) + if (row == null) + return previous + + const fieldWarnings = + Object.keys (row.fieldWarnings).length > 0 || row.metadataUrl !== previous.metadataUrl + ? { ...row.fieldWarnings } + : { ...previous.fieldWarnings } + for (const [field, origin] of Object.entries (row.provenance)) + { + if (origin === 'manual') + delete fieldWarnings[field] + } + + return { + ...previous, + url: row.url, + attributes: row.attributes, + provenance: row.provenance, + tagSources: row.tagSources, + skipReason: row.skipReason, + existingPostId: row.existingPostId, + fieldWarnings, + baseWarnings: + row.baseWarnings.length > 0 || row.metadataUrl !== previous.metadataUrl + ? row.baseWarnings + : previous.baseWarnings, + validationErrors: row.validationErrors, + status: row.status, + metadataUrl: row.metadataUrl } + }) +} + + +export const mergePreviewImportRows = ( + current: PostImportRow[], + preview: PostImportRow[], +): PostImportRow[] => { + const currentMap = new Map (current.map (row => [row.sourceRow, row])) + return preview.map (row => { + const previous = currentMap.get (row.sourceRow) + if (previous == null) + return row + if (previous.importStatus === 'created') + return previous + + const mergedAttributes = { ...row.attributes } + const mergedProvenance = { ...row.provenance } + + for (const [field, origin] of Object.entries (previous.provenance)) + { + if (origin !== 'manual' || field === 'url') + continue + if (field in previous.attributes) + mergedAttributes[field] = previous.attributes[field] + mergedProvenance[field] = 'manual' + } + + const mergedTagSources = + previous.provenance.tags === 'manual' + ? { + automatic: row.tagSources?.automatic ?? '', + manual: previous.tagSources?.manual ?? String (previous.attributes.tags ?? '') } + : row.tagSources + + const nextRow = { + ...row, + attributes: mergedAttributes, + provenance: mergedProvenance, + tagSources: mergedTagSources, + skipReason: row.skipReason, + existingPostId: row.existingPostId, + createdPostId: previous.createdPostId, + importStatus: previous.importStatus, + importErrors: previous.importErrors } + + if (rowChanged (previous, nextRow)) + { + nextRow.importStatus = 'pending' + nextRow.importErrors = undefined + } + return nextRow + }) +} + + +export const mergeImportResults = ( + rows: PostImportRow[], + results: PostImportResultRow[], +): PostImportRow[] => { + const resultMap = new Map (results.map (row => [row.sourceRow, row])) + return rows.map (row => { + const result = resultMap.get (row.sourceRow) + return result + ? { + ...row, + importStatus: result.status, + createdPostId: result.post?.id, + importErrors: result.errors } + : row + }) +} + + +export const retryImportRow = ( + rows: PostImportRow[], + sourceRow: number, +): PostImportRow[] => + rows.map (row => + row.sourceRow === sourceRow && row.importStatus === 'failed' + ? { ...row, importStatus: 'pending', importErrors: undefined } + : row) diff --git a/frontend/src/lib/postImportSession.ts b/frontend/src/lib/postImportSession.ts index e855e89..d43c394 100644 --- a/frontend/src/lib/postImportSession.ts +++ b/frontend/src/lib/postImportSession.ts @@ -1,653 +1,4 @@ -export type PostImportOrigin = 'automatic' | 'manual' -export type PostImportRepairMode = 'all' | 'failed' -export type PostImportStatus = - 'pending' - | 'created' - | 'skipped' - | 'failed' -export type PostImportSkipReason = 'existing' -export type PostImportResultStatus = 'created' | 'skipped' | 'failed' -export type PostImportAttributeValue = string | number - -export type PostImportRow = { - sourceRow: number - url: string - attributes: Record - fieldWarnings: Record - baseWarnings: string[] - validationErrors: Record - importErrors?: Record - provenance: Record - tagSources?: Record - status: 'ready' | 'warning' | 'error' - skipReason?: PostImportSkipReason - existingPostId?: number - metadataUrl?: string - createdPostId?: number - importStatus?: PostImportStatus } - -export type PostImportResultRow = { - sourceRow: number - status: PostImportResultStatus - post?: { id: number } - errors?: Record } - -export type PostImportSession = { - version: number - savedAt: string - source: string - rows: PostImportRow[] - repairMode: PostImportRepairMode } - -export type PostImportSourceIssue = { - sourceRow: number - message: string - url: string } - -type StorageErrorHandler = (message: string) => void - -const SESSION_VERSION = 2 -const SESSION_PREFIX = 'post-import-session:' -const SOURCE_DRAFT_KEY = 'post-import-source-draft' -const SESSION_MAX_AGE_MS = 24 * 60 * 60 * 1000 -const MAX_ROWS = 100 -const MAX_URL_BYTES = 20 * 1024 - - -const isPlainObject = (value: unknown): value is Record => - typeof value === 'object' && value != null && !(Array.isArray (value)) - - -const sessionKey = (sessionId: string): string => `${ SESSION_PREFIX }${ sessionId }` - - -const readStorage = ( - key: string, - onError?: StorageErrorHandler, -): string | null => { - if (typeof window === 'undefined') - return null - - try - { - return sessionStorage.getItem (key) - } - catch - { - onError?.('保存済みデータを読み込めませんでした．') - return null - } -} - - -const writeStorage = ( - key: string, - value: string, - onError?: StorageErrorHandler, -): boolean => { - if (typeof window === 'undefined') - return false - - try - { - sessionStorage.setItem (key, value) - return true - } - catch - { - onError?.('ブラウザへ保存できませんでした．') - return false - } -} - - -const removeStorage = ( - key: string, - onError?: StorageErrorHandler, -) => { - if (typeof window === 'undefined') - return - - try - { - sessionStorage.removeItem (key) - } - catch - { - onError?.('保存済みデータを削除できませんでした．') - } -} - - -const truncateUrl = (value: string): string => - value.length > 120 ? `${ value.slice (0, 117) }…` : value - - -const bytesize = (value: string): number => - new TextEncoder ().encode (value).length - - -const normaliseImportUrl = (value: string): string | null => { - const trimmed = value.trim () - if (!(trimmed)) - return null - - try - { - const url = new URL (trimmed) - if (!(url.protocol === 'http:' || url.protocol === 'https:')) - return null - if (!(url.host)) - return null - - url.hostname = url.hostname.toLowerCase () - if (url.pathname.endsWith ('/')) - url.pathname = url.pathname.replace (/\/+$/, '') - return url.toString () - } - catch - { - return null - } -} - - -const ensureStringListRecord = (value: unknown): Record | null => { - if (!(isPlainObject (value))) - return null - - const result: Record = { } - for (const [key, entry] of Object.entries (value)) - { - if (!(Array.isArray (entry)) || !(entry.every (_1 => typeof _1 === 'string'))) - return null - result[key] = entry - } - return result -} - - -const isValidStatus = ( - value: unknown, -): value is PostImportRow['status'] => - value === 'ready' || value === 'warning' || value === 'error' - - -const isValidImportStatus = ( - value: unknown, -): value is PostImportStatus => - value === 'pending' - || value === 'created' - || value === 'skipped' - || value === 'failed' - - -const isValidOrigin = ( - value: unknown, -): value is PostImportOrigin => - value === 'automatic' || value === 'manual' - - -const isValidSkipReason = ( - value: unknown, -): value is PostImportSkipReason => - value === 'existing' - -const isPositiveInteger = (value: unknown): value is number => - Number.isInteger (value) && Number (value) > 0 - - -const sanitiseRow = (value: unknown): PostImportRow | null => { - if (!(isPlainObject (value))) - return null - if (!(Number.isInteger (value.sourceRow)) || Number (value.sourceRow) <= 0) - return null - if (typeof value.url !== 'string') - return null - if (!(isPlainObject (value.attributes))) - return null - if (!(isPlainObject (value.provenance))) - return null - if (!(isValidStatus (value.status))) - return null - if (value.importStatus != null && !(isValidImportStatus (value.importStatus))) - return null - if (value.skipReason != null && !(isValidSkipReason (value.skipReason))) - return null - if (value.skipReason === 'existing' && !(isPositiveInteger (value.existingPostId))) - return null - if (value.skipReason !== 'existing' && value.existingPostId != null) - return null - if (value.importStatus === 'created' && !(isPositiveInteger (value.createdPostId))) - return null - if (value.importStatus !== 'created' && value.createdPostId != null) - return null - if (value.createdPostId != null && !(isPositiveInteger (value.createdPostId))) - return null - - const validationErrors = ensureStringListRecord (value.validationErrors) - const fieldWarnings = ensureStringListRecord (value.fieldWarnings) - if (validationErrors == null || fieldWarnings == null) - return null - - const importErrors = - value.importErrors == null - ? undefined - : ensureStringListRecord (value.importErrors) - if (value.importErrors != null && importErrors == null) - return null - if (!(Array.isArray (value.baseWarnings)) - || !(value.baseWarnings.every (_1 => typeof _1 === 'string'))) - return null - - const provenanceEntries = Object.entries (value.provenance) - if (!(provenanceEntries.every (([, origin]) => isValidOrigin (origin)))) - return null - - if (value.tagSources != null) - { - if (!(isPlainObject (value.tagSources))) - return null - if (!(Object.values (value.tagSources).every (_1 => typeof _1 === 'string'))) - return null - } - - return { - sourceRow: Number (value.sourceRow), - url: value.url, - attributes: value.attributes as Record, - fieldWarnings, - baseWarnings: value.baseWarnings, - validationErrors, - importErrors, - provenance: value.provenance as Record, - tagSources: value.tagSources as Record | undefined, - status: value.status, - skipReason: value.skipReason, - existingPostId: - isPositiveInteger (value.existingPostId) ? Number (value.existingPostId) : undefined, - metadataUrl: typeof value.metadataUrl === 'string' ? value.metadataUrl : undefined, - createdPostId: - isPositiveInteger (value.createdPostId) ? Number (value.createdPostId) : undefined, - importStatus: value.importStatus } -} - - -const isExpiredSession = (savedAt: string): boolean => { - const value = Date.parse (savedAt) - return Number.isNaN (value) || Date.now () - value > SESSION_MAX_AGE_MS -} - - -export const createPostImportSessionId = (): string => - typeof crypto !== 'undefined' && 'randomUUID' in crypto - ? crypto.randomUUID () - : `${ Date.now () }-${ Math.random ().toString (36).slice (2) }` - - -export const cleanupExpiredPostImportSessions = ( - onError?: StorageErrorHandler, -) => { - if (typeof window === 'undefined') - return - - try - { - for (let i = 0; i < sessionStorage.length; ++i) - { - const key = sessionStorage.key (i) - if (key == null || !(key.startsWith (SESSION_PREFIX))) - continue - - const raw = sessionStorage.getItem (key) - if (raw == null) - continue - try - { - const value = JSON.parse (raw) as { savedAt?: string } - if (typeof value.savedAt !== 'string' || isExpiredSession (value.savedAt)) - { - sessionStorage.removeItem (key) - --i - } - } - catch - { - sessionStorage.removeItem (key) - --i - } - } - } - catch - { - onError?.('保存済みデータを整理できませんでした．') - } -} - - -export const loadPostImportSourceDraft = ( - onError?: StorageErrorHandler, -): { source: string } => { - const raw = readStorage (SOURCE_DRAFT_KEY, onError) - if (raw == null) - return { source: '' } - - try - { - const value = JSON.parse (raw) as { source?: string } - return { source: typeof value.source === 'string' ? value.source : '' } - } - catch - { - return { source: '' } - } -} - - -export const savePostImportSourceDraft = ( - source: string, - onError?: StorageErrorHandler, -): boolean => - writeStorage (SOURCE_DRAFT_KEY, JSON.stringify ({ source }), onError) - - -export const clearPostImportSourceDraft = ( - onError?: StorageErrorHandler, -) => { - removeStorage (SOURCE_DRAFT_KEY, onError) -} - - -export const savePostImportSession = ( - sessionId: string, - session: Omit, - onError?: StorageErrorHandler, -): boolean => - writeStorage ( - sessionKey (sessionId), - JSON.stringify ({ - ...session, - version: SESSION_VERSION, - savedAt: new Date ().toISOString () }), - onError, - ) - - -export const loadPostImportSession = ( - sessionId: string, - onError?: StorageErrorHandler, -): PostImportSession | null => { - const raw = readStorage (sessionKey (sessionId), onError) - if (raw == null) - return null - - try - { - const value = JSON.parse (raw) as Partial - if (value.version !== SESSION_VERSION || !(Array.isArray (value.rows))) - return null - if (typeof value.savedAt !== 'string' || isExpiredSession (value.savedAt)) - { - removeStorage (sessionKey (sessionId), onError) - return null - } - - const rows = value.rows.map (sanitiseRow) - if (rows.some (_1 => _1 == null)) - return null - - return { - version: SESSION_VERSION, - savedAt: value.savedAt, - source: typeof value.source === 'string' ? value.source : '', - rows: rows as PostImportRow[], - repairMode: value.repairMode === 'failed' ? 'failed' : 'all' } - } - catch - { - return null - } -} - - -const rowChanged = ( - previous: PostImportRow, - next: PostImportRow, -): boolean => - previous.url !== next.url - || JSON.stringify (previous.attributes) !== JSON.stringify (next.attributes) - || JSON.stringify (previous.provenance) !== JSON.stringify (next.provenance) - || JSON.stringify (previous.tagSources ?? { }) !== JSON.stringify (next.tagSources ?? { }) - - -export const mergeValidatedImportRows = ( - current: PostImportRow[], - validated: PostImportRow[], -): PostImportRow[] => { - const validatedMap = new Map (validated.map (row => [row.sourceRow, row])) - return current.map (previous => { - if (previous.importStatus === 'created') - return previous - - const row = validatedMap.get (previous.sourceRow) - if (row == null) - return previous - - const fieldWarnings = - Object.keys (row.fieldWarnings).length > 0 || row.metadataUrl !== previous.metadataUrl - ? { ...row.fieldWarnings } - : { ...previous.fieldWarnings } - for (const [field, origin] of Object.entries (row.provenance)) - { - if (origin === 'manual') - delete fieldWarnings[field] - } - - return { - ...previous, - url: row.url, - attributes: row.attributes, - provenance: row.provenance, - tagSources: row.tagSources, - skipReason: row.skipReason, - existingPostId: row.existingPostId, - fieldWarnings, - baseWarnings: - row.baseWarnings.length > 0 || row.metadataUrl !== previous.metadataUrl - ? row.baseWarnings - : previous.baseWarnings, - validationErrors: row.validationErrors, - status: row.status, - metadataUrl: row.metadataUrl } - }) -} - - -export const mergePreviewImportRows = ( - current: PostImportRow[], - preview: PostImportRow[], -): PostImportRow[] => { - const currentMap = new Map (current.map (row => [row.sourceRow, row])) - return preview.map (row => { - const previous = currentMap.get (row.sourceRow) - if (previous == null) - return row - if (previous.importStatus === 'created') - return previous - - const mergedAttributes = { ...row.attributes } - const mergedProvenance = { ...row.provenance } - for (const [field, origin] of Object.entries (previous.provenance)) - { - if (origin !== 'manual' || field === 'url') - continue - if (field in previous.attributes) - mergedAttributes[field] = previous.attributes[field] - mergedProvenance[field] = 'manual' - } - - const mergedTagSources = - previous.provenance.tags === 'manual' - ? { - automatic: row.tagSources?.automatic ?? '', - manual: previous.tagSources?.manual ?? String (previous.attributes.tags ?? '') } - : row.tagSources - - const nextRow = { - ...row, - attributes: mergedAttributes, - provenance: mergedProvenance, - tagSources: mergedTagSources, - skipReason: row.skipReason, - existingPostId: row.existingPostId, - createdPostId: previous.createdPostId, - importStatus: previous.importStatus, - importErrors: previous.importErrors } - - if (rowChanged (previous, nextRow)) - { - nextRow.importStatus = 'pending' - nextRow.importErrors = undefined - } - return nextRow - }) -} - - -export const mergeImportResults = ( - rows: PostImportRow[], - results: PostImportResultRow[], -): PostImportRow[] => { - const resultMap = new Map (results.map (row => [row.sourceRow, row])) - return rows.map (row => { - const result = resultMap.get (row.sourceRow) - return result - ? { - ...row, - importStatus: result.status, - createdPostId: result.post?.id, - importErrors: result.errors } - : row - }) -} - - -export const retryImportRow = ( - rows: PostImportRow[], - sourceRow: number, -): PostImportRow[] => - rows.map (row => - row.sourceRow === sourceRow && row.importStatus === 'failed' - ? { ...row, importStatus: 'pending', importErrors: undefined } - : row) - - -export const countImportSourceLines = (source: string): number => - source - .split (/\r\n|\n|\r/) - .map (_1 => _1.trim ()) - .filter (_1 => _1 !== '') - .length - - -export const validateImportSource = ( - source: string, -): PostImportSourceIssue[] => { - const lines = source.split (/\r\n|\n|\r/) - const issues: PostImportSourceIssue[] = [] - const seen = new Map () - let count = 0 - - lines.forEach ((rawLine, index) => { - const value = rawLine.trim () - if (!(value)) - return - - ++count - const sourceRow = index + 1 - const displayUrl = truncateUrl (value) - const normalised = normaliseImportUrl (value) - if (count > MAX_ROWS) - { - issues.push ({ - sourceRow, - message: `取込件数は ${ MAX_ROWS } 件までです．`, - url: displayUrl }) - return - } - if (!(value.startsWith ('http://') || value.startsWith ('https://'))) - { - issues.push ({ - sourceRow, - message: 'HTTP または HTTPS の URL ではありません．', - url: displayUrl }) - return - } - if (bytesize (value) > MAX_URL_BYTES) - { - issues.push ({ - sourceRow, - message: 'URL が長すぎます．', - url: displayUrl }) - return - } - if (normalised == null) - { - issues.push ({ - sourceRow, - message: 'URL の形式が不正です．', - url: displayUrl }) - return - } - const duplicateRow = seen.get (normalised) - if (duplicateRow != null) - { - issues.push ({ - sourceRow, - message: `${ duplicateRow } 行目と同じ URL です．`, - url: displayUrl }) - return - } - seen.set (normalised, sourceRow) - }) - - return issues -} - - -const hasSkipReason = (row: PostImportRow): boolean => - row.skipReason === 'existing' - -const hasValidationErrors = (row: PostImportRow): boolean => - Object.keys (row.validationErrors ?? { }).length > 0 - - -export const processableImportRows = (rows: PostImportRow[]): PostImportRow[] => - rows.filter (row => { - if (row.importStatus === 'created') - return false - if (row.importStatus === 'skipped') - return false - if (row.importStatus === 'failed') - return false - if (hasValidationErrors (row)) - return false - return row.importStatus == null || row.importStatus === 'pending' - }) - -export const creatableImportRows = (rows: PostImportRow[]): PostImportRow[] => - processableImportRows (rows).filter (row => !(hasSkipReason (row))) - - -export const reviewSummaryCounts = (rows: PostImportRow[]) => ({ - total: rows.length, - submittable: rows.filter (row => - creatableImportRows ([row]).length > 0 - && !(hasValidationErrors (row))).length, - invalid: rows.filter (row => hasValidationErrors (row)).length, - skipPlanned: rows.filter (row => - hasSkipReason (row) - && !(hasValidationErrors (row)) - && row.importStatus !== 'created').length, - created: rows.filter (row => row.importStatus === 'created').length, - failed: rows.filter (row => row.importStatus === 'failed').length }) +export * from '@/lib/postImportTypes' +export * from '@/lib/postImportStorage' +export * from '@/lib/postImportSourceValidation' +export * from '@/lib/postImportRows' diff --git a/frontend/src/lib/postImportSourceValidation.ts b/frontend/src/lib/postImportSourceValidation.ts new file mode 100644 index 0000000..63cb796 --- /dev/null +++ b/frontend/src/lib/postImportSourceValidation.ts @@ -0,0 +1,110 @@ +import type { PostImportSourceIssue } from '@/lib/postImportTypes' + +const MAX_ROWS = 100 +const MAX_URL_BYTES = 20 * 1024 + + +const truncateUrl = (value: string): string => + value.length > 120 ? `${ value.slice (0, 117) }…` : value + + +const bytesize = (value: string): number => + new TextEncoder ().encode (value).length + + +const normaliseImportUrl = (value: string): string | null => { + const trimmed = value.trim () + if (!(trimmed)) + return null + + try + { + const url = new URL (trimmed) + if (!(url.protocol === 'http:' || url.protocol === 'https:')) + return null + if (!(url.host)) + return null + + url.hostname = url.hostname.toLowerCase () + if (url.pathname.endsWith ('/')) + url.pathname = url.pathname.replace (/\/+$/, '') + return url.toString () + } + catch + { + return null + } +} + + +export const countImportSourceLines = (source: string): number => + source + .split (/\r\n|\n|\r/) + .map (_1 => _1.trim ()) + .filter (_1 => _1 !== '') + .length + + +export const validateImportSource = ( + source: string, +): PostImportSourceIssue[] => { + const lines = source.split (/\r\n|\n|\r/) + const issues: PostImportSourceIssue[] = [] + const seen = new Map () + let count = 0 + + lines.forEach ((rawLine, index) => { + const value = rawLine.trim () + if (!(value)) + return + + ++count + const sourceRow = index + 1 + const displayUrl = truncateUrl (value) + const normalised = normaliseImportUrl (value) + if (count > MAX_ROWS) + { + issues.push ({ + sourceRow, + message: `取込件数は ${ MAX_ROWS } 件までです．`, + url: displayUrl }) + return + } + if (!(value.startsWith ('http://') || value.startsWith ('https://'))) + { + issues.push ({ + sourceRow, + message: 'HTTP または HTTPS の URL ではありません．', + url: displayUrl }) + return + } + if (bytesize (value) > MAX_URL_BYTES) + { + issues.push ({ + sourceRow, + message: 'URL が長すぎます．', + url: displayUrl }) + return + } + if (normalised == null) + { + issues.push ({ + sourceRow, + message: 'URL の形式が不正です．', + url: displayUrl }) + return + } + const duplicateRow = seen.get (normalised) + if (duplicateRow != null) + { + issues.push ({ + sourceRow, + message: `${ duplicateRow } 行目と同じ URL です．`, + url: displayUrl }) + return + } + seen.set (normalised, sourceRow) + }) + + return issues +} diff --git a/frontend/src/lib/postImportStorage.ts b/frontend/src/lib/postImportStorage.ts new file mode 100644 index 0000000..6f626c5 --- /dev/null +++ b/frontend/src/lib/postImportStorage.ts @@ -0,0 +1,331 @@ +import type { PostImportOrigin, + PostImportRow, + PostImportSession, + PostImportStatus, + PostImportSkipReason, + StorageErrorHandler } from '@/lib/postImportTypes' + +const SESSION_VERSION = 2 +const SESSION_PREFIX = 'post-import-session:' +const SOURCE_DRAFT_KEY = 'post-import-source-draft' +const SESSION_MAX_AGE_MS = 24 * 60 * 60 * 1000 + + +const isPlainObject = (value: unknown): value is Record => + typeof value === 'object' && value != null && !(Array.isArray (value)) + + +const sessionKey = (sessionId: string): string => `${ SESSION_PREFIX }${ sessionId }` + + +const readStorage = ( + key: string, + onError?: StorageErrorHandler, +): string | null => { + if (typeof window === 'undefined') + return null + + try + { + return sessionStorage.getItem (key) + } + catch + { + onError?.('保存済みデータを読み込めませんでした．') + return null + } +} + + +const writeStorage = ( + key: string, + value: string, + onError?: StorageErrorHandler, +): boolean => { + if (typeof window === 'undefined') + return false + + try + { + sessionStorage.setItem (key, value) + return true + } + catch + { + onError?.('ブラウザへ保存できませんでした．') + return false + } +} + + +const removeStorage = ( + key: string, + onError?: StorageErrorHandler, +) => { + if (typeof window === 'undefined') + return + + try + { + sessionStorage.removeItem (key) + } + catch + { + onError?.('保存済みデータを削除できませんでした．') + } +} + + +const ensureStringListRecord = (value: unknown): Record | null => { + if (!(isPlainObject (value))) + return null + + const result: Record = { } + for (const [key, entry] of Object.entries (value)) + { + if (!(Array.isArray (entry)) || !(entry.every (_1 => typeof _1 === 'string'))) + return null + result[key] = entry + } + return result +} + + +const isValidStatus = ( + value: unknown, +): value is PostImportRow['status'] => + value === 'ready' || value === 'warning' || value === 'error' + + +const isValidImportStatus = ( + value: unknown, +): value is PostImportStatus => + value === 'pending' + || value === 'created' + || value === 'skipped' + || value === 'failed' + + +const isValidOrigin = ( + value: unknown, +): value is PostImportOrigin => + value === 'automatic' || value === 'manual' + + +const isValidSkipReason = ( + value: unknown, +): value is PostImportSkipReason => + value === 'existing' + +const isPositiveInteger = (value: unknown): value is number => + Number.isInteger (value) && Number (value) > 0 + + +const sanitiseRow = (value: unknown): PostImportRow | null => { + if (!(isPlainObject (value))) + return null + if (!(Number.isInteger (value.sourceRow)) || Number (value.sourceRow) <= 0) + return null + if (typeof value.url !== 'string') + return null + if (!(isPlainObject (value.attributes))) + return null + if (!(isPlainObject (value.provenance))) + return null + if (!(isValidStatus (value.status))) + return null + if (value.importStatus != null && !(isValidImportStatus (value.importStatus))) + return null + if (value.skipReason != null && !(isValidSkipReason (value.skipReason))) + return null + if (value.skipReason === 'existing' && !(isPositiveInteger (value.existingPostId))) + return null + if (value.skipReason !== 'existing' && value.existingPostId != null) + return null + if (value.importStatus === 'created' && !(isPositiveInteger (value.createdPostId))) + return null + if (value.importStatus !== 'created' && value.createdPostId != null) + return null + + const validationErrors = ensureStringListRecord (value.validationErrors) + const fieldWarnings = ensureStringListRecord (value.fieldWarnings) + if (validationErrors == null || fieldWarnings == null) + return null + + const importErrors = + value.importErrors == null + ? undefined + : ensureStringListRecord (value.importErrors) + if (value.importErrors != null && importErrors == null) + return null + if (!(Array.isArray (value.baseWarnings)) + || !(value.baseWarnings.every (_1 => typeof _1 === 'string'))) + return null + + const provenanceEntries = Object.entries (value.provenance) + if (!(provenanceEntries.every (([, origin]) => isValidOrigin (origin)))) + return null + + if (value.tagSources != null) + { + if (!(isPlainObject (value.tagSources))) + return null + if (!(Object.values (value.tagSources).every (_1 => typeof _1 === 'string'))) + return null + } + + return { + sourceRow: Number (value.sourceRow), + url: value.url, + attributes: value.attributes as Record, + fieldWarnings, + baseWarnings: value.baseWarnings, + validationErrors, + importErrors, + provenance: value.provenance as Record, + tagSources: value.tagSources as Record | undefined, + status: value.status, + skipReason: value.skipReason, + existingPostId: + isPositiveInteger (value.existingPostId) ? Number (value.existingPostId) : undefined, + metadataUrl: typeof value.metadataUrl === 'string' ? value.metadataUrl : undefined, + createdPostId: + isPositiveInteger (value.createdPostId) ? Number (value.createdPostId) : undefined, + importStatus: value.importStatus } +} + + +const isExpiredSession = (savedAt: string): boolean => { + const value = Date.parse (savedAt) + return Number.isNaN (value) || Date.now () - value > SESSION_MAX_AGE_MS +} + + +export const createPostImportSessionId = (): string => + typeof crypto !== 'undefined' && 'randomUUID' in crypto + ? crypto.randomUUID () + : `${ Date.now () }-${ Math.random ().toString (36).slice (2) }` + + +export const cleanupExpiredPostImportSessions = ( + onError?: StorageErrorHandler, +) => { + if (typeof window === 'undefined') + return + + try + { + for (let i = 0; i < sessionStorage.length; ++i) + { + const key = sessionStorage.key (i) + if (key == null || !(key.startsWith (SESSION_PREFIX))) + continue + + const raw = sessionStorage.getItem (key) + if (raw == null) + continue + try + { + const value = JSON.parse (raw) as { savedAt?: string } + if (typeof value.savedAt !== 'string' || isExpiredSession (value.savedAt)) + { + sessionStorage.removeItem (key) + --i + } + } + catch + { + sessionStorage.removeItem (key) + --i + } + } + } + catch + { + onError?.('保存済みデータを整理できませんでした．') + } +} + + +export const loadPostImportSourceDraft = ( + onError?: StorageErrorHandler, +): { source: string } => { + const raw = readStorage (SOURCE_DRAFT_KEY, onError) + if (raw == null) + return { source: '' } + + try + { + const value = JSON.parse (raw) as { source?: string } + return { source: typeof value.source === 'string' ? value.source : '' } + } + catch + { + return { source: '' } + } +} + + +export const savePostImportSourceDraft = ( + source: string, + onError?: StorageErrorHandler, +): boolean => + writeStorage (SOURCE_DRAFT_KEY, JSON.stringify ({ source }), onError) + + +export const clearPostImportSourceDraft = ( + onError?: StorageErrorHandler, +) => { + removeStorage (SOURCE_DRAFT_KEY, onError) +} + + +export const savePostImportSession = ( + sessionId: string, + session: Omit, + onError?: StorageErrorHandler, +): boolean => + writeStorage ( + sessionKey (sessionId), + JSON.stringify ({ + ...session, + version: SESSION_VERSION, + savedAt: new Date ().toISOString () }), + onError) + + +export const loadPostImportSession = ( + sessionId: string, + onError?: StorageErrorHandler, +): PostImportSession | null => { + const raw = readStorage (sessionKey (sessionId), onError) + if (raw == null) + return null + + try + { + const value = JSON.parse (raw) as Partial + if (value.version !== SESSION_VERSION || !(Array.isArray (value.rows))) + return null + if (typeof value.savedAt !== 'string' || isExpiredSession (value.savedAt)) + { + removeStorage (sessionKey (sessionId), onError) + return null + } + + const rows = value.rows.map (sanitiseRow) + if (rows.some (_1 => _1 == null)) + return null + + return { + version: SESSION_VERSION, + savedAt: value.savedAt, + source: typeof value.source === 'string' ? value.source : '', + rows: rows as PostImportRow[], + repairMode: value.repairMode === 'failed' ? 'failed' : 'all' } + } + catch + { + return null + } +} diff --git a/frontend/src/lib/postImportTypes.ts b/frontend/src/lib/postImportTypes.ts new file mode 100644 index 0000000..e09f9e9 --- /dev/null +++ b/frontend/src/lib/postImportTypes.ts @@ -0,0 +1,47 @@ +export type PostImportOrigin = 'automatic' | 'manual' +export type PostImportRepairMode = 'all' | 'failed' +export type PostImportStatus = + 'pending' + | 'created' + | 'skipped' + | 'failed' +export type PostImportSkipReason = 'existing' +export type PostImportResultStatus = 'created' | 'skipped' | 'failed' +export type PostImportAttributeValue = string | number + +export type PostImportRow = { + sourceRow: number + url: string + attributes: Record + fieldWarnings: Record + baseWarnings: string[] + validationErrors: Record + importErrors?: Record + provenance: Record + tagSources?: Record + status: 'ready' | 'warning' | 'error' + skipReason?: PostImportSkipReason + existingPostId?: number + metadataUrl?: string + createdPostId?: number + importStatus?: PostImportStatus } + +export type PostImportResultRow = { + sourceRow: number + status: PostImportResultStatus + post?: { id: number } + errors?: Record } + +export type PostImportSession = { + version: number + savedAt: string + source: string + rows: PostImportRow[] + repairMode: PostImportRepairMode } + +export type PostImportSourceIssue = { + sourceRow: number + message: string + url: string } + +export type StorageErrorHandler = (message: string) => void diff --git a/frontend/src/pages/posts/PostImportResultPage.tsx b/frontend/src/pages/posts/PostImportResultPage.tsx index adf844f..08d026c 100644 --- a/frontend/src/pages/posts/PostImportResultPage.tsx +++ b/frontend/src/pages/posts/PostImportResultPage.tsx @@ -18,6 +18,7 @@ import { clearPostImportSourceDraft, loadPostImportSession, mergeImportResults, mergeValidatedImportRows, + resultSummaryCounts, retryImportRow, savePostImportSession, type PostImportResultRow, @@ -92,13 +93,9 @@ const PostImportResultPage: FC = ({ user }) => { toast ({ title: '取込状態を保存できませんでした', description: message })) }, [session, sessionId]) - const counts = useMemo (() => ({ - created: session?.rows.filter (_1 => _1.importStatus === 'created').length ?? 0, - skipped: session?.rows.filter (_1 => _1.importStatus === 'skipped').length ?? 0, - failed: session?.rows.filter (_1 => _1.importStatus === 'failed').length ?? 0, - invalid: - session?.rows.filter (_1 => effectivePostImportStatus (_1) === 'error').length ?? 0, - }), [session]) + const counts = useMemo ( + () => resultSummaryCounts (session?.rows ?? []), + [session]) const retry = async (sourceRow: number) => { if (session == null) @@ -120,11 +117,7 @@ const PostImportResultPage: FC = ({ user }) => { tagSources: row.tagSources, metadataUrl: row.metadataUrl })), changed_row: -1 }) - const validatedRows = mergeValidatedImportRows (pendingRows, validated.rows).map (row => - row.sourceRow === sourceRow - && Object.keys (row.validationErrors ?? { }).length > 0 - ? { ...row, importStatus: 'failed' as const } - : row) + const validatedRows = mergeValidatedImportRows (pendingRows, validated.rows) setSession (current => current ? { ...current, rows: validatedRows } @@ -238,20 +231,21 @@ const PostImportResultPage: FC = ({ user }) => {
- {row.createdPostId && ( + {(row.createdPostId != null || row.existingPostId != null) && ( )} - {status === 'error' && ( + {(status === 'error' || row.importStatus === 'failed') && ( )} - {row.importStatus === 'failed' && ( + {row.importStatus === 'failed' && status !== 'error' && ( <>