#399

backend/app/services/post_import_previewer.rb

@@ -20,7 +20,6 @@ class PostImportPreviewer
     url_counts = prepared_rows.filter_map { _1[:normal_url] }.tally
     existing_posts =
         Post.where(url: prepared_rows.map { _1[:normal_url] }.compact.uniq).index_by(&:url)
-    known_tags = preload_known_tags(prepared_rows)
     existing_parent_ids = preload_parent_ids(prepared_rows)
     preload_metadata!(
         prepared_rows,
@@ -28,6 +27,13 @@ class PostImportPreviewer
         metadata_cache,
         existing_posts,
         url_counts)
+    known_tags =
+        preload_known_tags(
+            prepared_rows,
+            fetch_metadata,
+            metadata_cache,
+            existing_posts,
+            url_counts)
     prepared_rows.map { |row|
       preview_row(row,
                   fetch_metadata:,
@@ -96,14 +102,7 @@ class PostImportPreviewer
                field_warnings:,
                base_warnings:,
                validation_errors:,
-               status:
-                   if validation_errors.present?
-                     'error'
-                   elsif warnings_present
-                     'warning'
-                   else
-                     'ready'
-                   end }
+               status: warnings_present ? 'warning' : 'ready' }
     end
 
     should_fetch = validation_errors.blank?
@@ -228,31 +227,56 @@ class PostImportPreviewer
     urls = urls.reject { metadata_cache.key?(_1) }
     return if urls.empty?
 
-    queue = Queue.new
-    urls.each { queue << _1 }
+    url_queue = Queue.new
+    result_queue = Queue.new
+    urls.each { url_queue << _1 }
     workers = [urls.length, 4].min.times.map {
       Thread.new {
-        loop do
-          url = queue.pop(true)
-          metadata_cache[url] = fetch_metadata(url)
-        rescue ThreadError
-          break
+        Rails.application.executor.wrap do
+          loop do
+            url = url_queue.pop(true)
+            result_queue << [url, safe_fetch_metadata(url)]
+          rescue ThreadError
+            break
+          end
         end
       }
     }
     Timeout.timeout(15) { workers.each(&:join) }
   rescue Timeout::Error
     workers&.each(&:kill)
-    urls.each do |url|
+  ensure
+    workers&.each(&:join)
+    while result_queue&.size.to_i.positive?
+      url, result = result_queue.pop
+      metadata_cache[url] = result
+    end
+    urls&.each do |url|
       metadata_cache[url] ||= { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } }
     end
   end
 
-  def preload_known_tags prepared_rows
+  def safe_fetch_metadata url
+    fetch_metadata(url)
+  rescue StandardError => e
+    Rails.logger.error(
+        "post_import_metadata_fetch_unexpected_failure "\
+        "#{ { error: e.class.name, message: e.message }.to_json }")
+    { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } }
+  end
+
+  def preload_known_tags prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts
     names = prepared_rows.flat_map { |row|
       attributes = initial_attributes(row)
       provenance = initial_provenance(row)
       tag_sources = initial_tag_sources(row, attributes, provenance)
+      if metadata_url_changed?(row)
+        clear_automatic_values!(attributes, provenance, tag_sources)
+      end
+      if should_apply_metadata_to_row?(row, fetch_metadata, existing_posts, url_counts)
+        metadata = metadata_for(row[:normal_url], metadata_cache)
+        apply_metadata!(attributes, provenance, tag_sources, metadata[:data])
+      end
       preview_tag_names(merged_tags(tag_sources, provenance['tags']))
     }.compact.uniq
     return { } if names.empty?
@@ -274,6 +298,19 @@ class PostImportPreviewer
     Post.where(id: ids).pluck(:id).to_h { [_1, true] }
   end
 
+  def metadata_url_changed? row
+    row[:metadata_url].present? && row[:metadata_url] != (row[:normal_url] || row[:url_text])
+  end
+
+  def should_apply_metadata_to_row? row, fetch_metadata, existing_posts, url_counts
+    normal_url = row[:normal_url]
+    return false if normal_url.blank?
+    return false if url_counts[normal_url].to_i > 1
+    return false if existing_posts.key?(normal_url)
+
+    should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
+  end
+
   def apply_metadata! attributes, provenance, tag_sources, metadata
     metadata.each do |field, value|
       if field == 'tags'

backend/app/services/post_import_runner.rb

@@ -27,13 +27,24 @@ class PostImportRunner
     return { source_row: row['source_row'],
              status: 'failed',
              errors: preview[:validation_errors] } if preview[:validation_errors].present?
-    return row.slice('source_row').merge(status: 'skipped') if preview[:skip_reason] == 'existing'
+    if preview[:skip_reason] == 'existing'
+      return { source_row: row['source_row'],
+               status: 'skipped',
+               existing_post_id: preview[:existing_post_id] }
+    end
 
     attributes['tags'] = preview[:attributes]['tags']
     attributes['url'] = row['url']
     post = PostCreator.new(actor: @actor, attributes:).create!
     { source_row: row['source_row'], status: 'created', post: PostRepr.base(post) }
   rescue ActiveRecord::RecordInvalid => e
+    existing_post = existing_post_for_race(row, e.record)
+    if existing_post
+      return { source_row: row['source_row'],
+               status: 'skipped',
+               existing_post_id: existing_post.id }
+    end
+
     { source_row: row['source_row'], status: 'failed', errors: e.record.errors.to_hash }
   rescue Tag::NicoTagNormalisationError
     { source_row: row['source_row'],
@@ -54,4 +65,13 @@ class PostImportRunner
       status: 'failed',
       errors: { base: ['登録中にエラーが発生しました．'] } }
   end
+
+  def existing_post_for_race row, record
+    return nil unless record.errors.of_kind?(:url, :taken)
+
+    normal_url = PostUrlNormaliser.normalise(row['url'])
+    return nil if normal_url.blank?
+
+    Post.find_by(url: normal_url)
+  end
 end