#399

backend/app/services/post_import_previewer.rb

@@ -20,7 +20,6 @@ class PostImportPreviewer
     url_counts = prepared_rows.filter_map { _1[:normal_url] }.tally
     existing_posts =
         Post.where(url: prepared_rows.map { _1[:normal_url] }.compact.uniq).index_by(&:url)
-    known_tags = preload_known_tags(prepared_rows)
     existing_parent_ids = preload_parent_ids(prepared_rows)
     preload_metadata!(
         prepared_rows,
@@ -28,6 +27,13 @@ class PostImportPreviewer
         metadata_cache,
         existing_posts,
         url_counts)
+    known_tags =
+        preload_known_tags(
+            prepared_rows,
+            fetch_metadata,
+            metadata_cache,
+            existing_posts,
+            url_counts)
     prepared_rows.map { |row|
       preview_row(row,
                   fetch_metadata:,
@@ -96,14 +102,7 @@ class PostImportPreviewer
                field_warnings:,
                base_warnings:,
                validation_errors:,
-               status:
+               status: warnings_present ? 'warning' : 'ready' }
-                   if validation_errors.present?
-                     'error'
-                   elsif warnings_present
-                     'warning'
-                   else
-                     'ready'
-                   end }
     end
 
     should_fetch = validation_errors.blank?
@@ -228,31 +227,56 @@ class PostImportPreviewer
     urls = urls.reject { metadata_cache.key?(_1) }
     return if urls.empty?
 
-    queue = Queue.new
+    url_queue = Queue.new
-    urls.each { queue << _1 }
+    result_queue = Queue.new
+    urls.each { url_queue << _1 }
     workers = [urls.length, 4].min.times.map {
       Thread.new {
-        loop do
+        Rails.application.executor.wrap do
-          url = queue.pop(true)
+          loop do
-          metadata_cache[url] = fetch_metadata(url)
+            url = url_queue.pop(true)
-        rescue ThreadError
+            result_queue << [url, safe_fetch_metadata(url)]
-          break
+          rescue ThreadError
+            break
+          end
         end
       }
     }
     Timeout.timeout(15) { workers.each(&:join) }
   rescue Timeout::Error
     workers&.each(&:kill)
-    urls.each do |url|
+  ensure
+    workers&.each(&:join)
+    while result_queue&.size.to_i.positive?
+      url, result = result_queue.pop
+      metadata_cache[url] = result
+    end
+    urls&.each do |url|
       metadata_cache[url] ||= { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } }
     end
   end
 
-  def preload_known_tags prepared_rows
+  def safe_fetch_metadata url
+    fetch_metadata(url)
+  rescue StandardError => e
+    Rails.logger.error(
+        "post_import_metadata_fetch_unexpected_failure "\
+        "#{ { error: e.class.name, message: e.message }.to_json }")
+    { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } }
+  end
+
+  def preload_known_tags prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts
     names = prepared_rows.flat_map { |row|
       attributes = initial_attributes(row)
       provenance = initial_provenance(row)
       tag_sources = initial_tag_sources(row, attributes, provenance)
+      if metadata_url_changed?(row)
+        clear_automatic_values!(attributes, provenance, tag_sources)
+      end
+      if should_apply_metadata_to_row?(row, fetch_metadata, existing_posts, url_counts)
+        metadata = metadata_for(row[:normal_url], metadata_cache)
+        apply_metadata!(attributes, provenance, tag_sources, metadata[:data])
+      end
       preview_tag_names(merged_tags(tag_sources, provenance['tags']))
     }.compact.uniq
     return { } if names.empty?
@@ -274,6 +298,19 @@ class PostImportPreviewer
     Post.where(id: ids).pluck(:id).to_h { [_1, true] }
   end
 
+  def metadata_url_changed? row
+    row[:metadata_url].present? && row[:metadata_url] != (row[:normal_url] || row[:url_text])
+  end
+
+  def should_apply_metadata_to_row? row, fetch_metadata, existing_posts, url_counts
+    normal_url = row[:normal_url]
+    return false if normal_url.blank?
+    return false if url_counts[normal_url].to_i > 1
+    return false if existing_posts.key?(normal_url)
+
+    should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
+  end
+
   def apply_metadata! attributes, provenance, tag_sources, metadata
     metadata.each do |field, value|
       if field == 'tags'

backend/app/services/post_import_runner.rb

@@ -27,13 +27,24 @@ class PostImportRunner
     return { source_row: row['source_row'],
              status: 'failed',
              errors: preview[:validation_errors] } if preview[:validation_errors].present?
-    return row.slice('source_row').merge(status: 'skipped') if preview[:skip_reason] == 'existing'
+    if preview[:skip_reason] == 'existing'
+      return { source_row: row['source_row'],
+               status: 'skipped',
+               existing_post_id: preview[:existing_post_id] }
+    end
 
     attributes['tags'] = preview[:attributes]['tags']
     attributes['url'] = row['url']
     post = PostCreator.new(actor: @actor, attributes:).create!
     { source_row: row['source_row'], status: 'created', post: PostRepr.base(post) }
   rescue ActiveRecord::RecordInvalid => e
+    existing_post = existing_post_for_race(row, e.record)
+    if existing_post
+      return { source_row: row['source_row'],
+               status: 'skipped',
+               existing_post_id: existing_post.id }
+    end
+
     { source_row: row['source_row'], status: 'failed', errors: e.record.errors.to_hash }
   rescue Tag::NicoTagNormalisationError
     { source_row: row['source_row'],
@@ -54,4 +65,13 @@ class PostImportRunner
       status: 'failed',
       errors: { base: ['登録中にエラーが発生しました．'] } }
   end
+
+  def existing_post_for_race row, record
+    return nil unless record.errors.of_kind?(:url, :taken)
+
+    normal_url = PostUrlNormaliser.normalise(row['url'])
+    return nil if normal_url.blank?
+
+    Post.find_by(url: normal_url)
+  end
 end

frontend/src/components/common/FieldWarning.tsx

@@ -5,7 +5,7 @@ type Props = { id?:        string
 
 
 export const FieldWarning: FC<Props> = ({ id, messages }: Props) => {
-  if (!(messages) || messages.length === 0)
+  if (messages == null || messages.length === 0)
     return null
 
   return (

frontend/src/lib/postImportRows.ts

@@ -175,13 +175,33 @@ export const mergeImportResults = (
     const resultMap = new Map (results.map (row => [row.sourceRow, row]))
     return rows.map (row => {
 	const result = resultMap.get (row.sourceRow)
-	return result
+	if (result == null)
-	    ? {
+	  return row
+
+	switch (result.status)
+	  {
+	  case 'created':
+	    return {
 		...row,
-		importStatus: result.status,
+		importStatus: 'created',
 		createdPostId: result.post?.id,
+		existingPostId: undefined,
 		importErrors: result.errors }
-	    : row
+	  case 'skipped':
+	    return {
+		...row,
+		importStatus: 'skipped',
+		createdPostId: undefined,
+		existingPostId: result.existingPostId,
+		importErrors: result.errors }
+	  case 'failed':
+	    return {
+		...row,
+		importStatus: 'failed',
+		createdPostId: undefined,
+		existingPostId: undefined,
+		importErrors: result.errors }
+	  }
     })
 }
 

frontend/src/lib/postImportStorage.ts

@@ -9,6 +9,17 @@ const SESSION_VERSION = 2
 const SESSION_PREFIX = 'post-import-session:'
 const SOURCE_DRAFT_KEY = 'post-import-source-draft'
 const SESSION_MAX_AGE_MS = 24 * 60 * 60 * 1000
+const ATTRIBUTE_KEYS = [
+    'title',
+    'thumbnailBase',
+    'originalCreatedFrom',
+    'originalCreatedBefore',
+    'duration',
+    'videoMs',
+    'tags',
+    'parentPostIds'] as const
+const PROVENANCE_KEYS = [...ATTRIBUTE_KEYS, 'url'] as const
+const TAG_SOURCE_KEYS = ['automatic', 'manual'] as const
 
 
 const isPlainObject = (value: unknown): value is Record<string, unknown> =>
@@ -120,6 +131,12 @@ const isValidSkipReason = (
 const isPositiveInteger = (value: unknown): value is number =>
     Number.isInteger (value) && Number (value) > 0
 
+const hasOnlyKeys = (
+    value:        Record<string, unknown>,
+    allowedKeys:  readonly string[],
+): boolean =>
+    Object.keys (value).every (key => allowedKeys.includes (key))
+
 
 const sanitiseRow = (value: unknown): PostImportRow | null => {
     if (!(isPlainObject (value)))
@@ -163,6 +180,13 @@ const sanitiseRow = (value: unknown): PostImportRow | null => {
       return null
 
     const provenanceEntries = Object.entries (value.provenance)
+    if (!(hasOnlyKeys (value.attributes, ATTRIBUTE_KEYS)))
+      return null
+    if (!(Object.values (value.attributes).every (entry =>
+	typeof entry === 'string' || typeof entry === 'number')))
+      return null
+    if (!(hasOnlyKeys (value.provenance, PROVENANCE_KEYS)))
+      return null
     if (!(provenanceEntries.every (([, origin]) => isValidOrigin (origin))))
       return null
 
@@ -170,6 +194,8 @@ const sanitiseRow = (value: unknown): PostImportRow | null => {
       {
 	if (!(isPlainObject (value.tagSources)))
 	  return null
+	if (!(hasOnlyKeys (value.tagSources, TAG_SOURCE_KEYS)))
+	  return null
 	if (!(Object.values (value.tagSources).every (_1 => typeof _1 === 'string')))
 	  return null
       }

frontend/src/lib/postImportTypes.ts

@@ -27,10 +27,11 @@ export type PostImportRow = {
     importStatus?:     PostImportStatus }
 
 export type PostImportResultRow = {
-    sourceRow: number
+    sourceRow:      number
-    status:    PostImportResultStatus
+    status:         PostImportResultStatus
-    post?:     { id: number }
+    post?:          { id: number }
-    errors?:   Record<string, string[]> }
+    existingPostId?: number
+    errors?:        Record<string, string[]> }
 
 export type PostImportSession = {
     version:    number

frontend/src/pages/posts/PostImportSourcePage.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useMemo, useRef, useState } from 'react'
+import { useEffect, useRef, useState } from 'react'
 import { Helmet } from 'react-helmet-async'
 import { useNavigate } from 'react-router-dom'
 
@@ -32,18 +32,23 @@ const MAX_ROWS = 100
 const SOURCE_ERROR_ID = 'post-import-source-error'
 const SOURCE_ISSUES_ID = 'post-import-source-issues'
 
+const urlIssuesFromRows = (rows: PostImportRow[]) =>
+    rows.flatMap (row =>
+	(row.validationErrors.url ?? []).map (message => ({
+	    sourceRow: row.sourceRow,
+	    message,
+	    url: row.url })))
+
 
 const PostImportSourcePage: FC<Props> = ({ user }) => {
     const editable = canEditContent (user)
     const navigate = useNavigate ()
-    const draft = useMemo (() => loadPostImportSourceDraft (message =>
+    const [source, setSource] = useState ('')
-	toast ({ title: '保存済み入力を復元できませんでした', description: message })), [])
-
-    const [source, setSource] = useState (draft.source)
     const [loading, setLoading] = useState (false)
     const [sourceIssues, setSourceIssues] = useState<ReturnType<typeof validateImportSource>> ([])
     const [sourceError, setSourceError] = useState<string | null> (null)
     const saveTimer = useRef<number | null> (null)
+    const editedRef = useRef (false)
 
     const lineCount = countImportSourceLines (source)
     const messages = sourceError ? [sourceError] : []
@@ -56,6 +61,13 @@ const PostImportSourcePage: FC<Props> = ({ user }) => {
     useEffect (() => {
 	cleanupExpiredPostImportSessions (message =>
 	    toast ({ title: '保存済みデータを整理できませんでした', description: message }))
+
+	const draft = loadPostImportSourceDraft (message =>
+	    toast ({ title: '保存済み入力を復元できませんでした', description: message }))
+	if (!(editedRef.current))
+	  {
+	    setSource (current => current === '' ? draft.source : current)
+	  }
     }, [])
 
     useEffect (() => {
@@ -94,6 +106,12 @@ const PostImportSourcePage: FC<Props> = ({ user }) => {
 	{
 	    const data = await apiPost<{ rows: PostImportRow[] }> ('/posts/import/preview', {
 		source })
+	    const urlIssues = urlIssuesFromRows (data.rows)
+	    if (urlIssues.length > 0)
+	      {
+		setSourceIssues (urlIssues)
+		return
+	      }
 	    const sessionId = createPostImportSessionId ()
 	    const saved = savePostImportSession (sessionId, {
 		source,
@@ -154,6 +172,7 @@ const PostImportSourcePage: FC<Props> = ({ user }) => {
 				      description: message }))
 			  }}
 			  onChange={ev => {
+			      editedRef.current = true
 			      setSource (ev.target.value)
 			      setSourceError (null)
 			      setSourceIssues ([])