#399

backend/app/services/post_import_previewer.rb

@@ -1,3 +1,5 @@
+require 'timeout'
+
 class PostImportPreviewer
   FIELDS = [
     'title',
@@ -6,8 +8,7 @@ class PostImportPreviewer
     'original_created_before',
     'duration',
     'tags',
-    'parent_post_ids',
-  ].freeze
+    'parent_post_ids'].freeze
   FETCH_WARNING_FIELDS = ['url', 'title', 'thumbnail_base'].freeze
   EXISTING_SKIP_WARNING = '既存投稿のためスキップします．'.freeze
   TITLE_FETCH_WARNING = 'タイトルを取得できませんでした．'.freeze
@@ -15,15 +16,26 @@ class PostImportPreviewer
   METADATA_FETCH_WARNING = 'メタデータを取得できませんでした．'.freeze
 
   def preview_rows rows:, fetch_metadata: true, metadata_cache: { }
-    urls = {}
     prepared_rows = rows.map { prepare_row(_1) }
+    url_counts = prepared_rows.filter_map { _1[:normal_url] }.tally
     existing_posts =
         Post.where(url: prepared_rows.map { _1[:normal_url] }.compact.uniq).index_by(&:url)
+    known_tags = preload_known_tags(prepared_rows)
+    existing_parent_ids = preload_parent_ids(prepared_rows)
+    preload_metadata!(
+        prepared_rows,
+        fetch_metadata,
+        metadata_cache,
+        existing_posts,
+        url_counts)
     prepared_rows.map { |row|
-      preview_row(row, urls:,
+      preview_row(row,
                   fetch_metadata:,
                   metadata_cache:,
-                  existing_posts:)
+                  existing_posts:,
+                  url_counts:,
+                  known_tags:,
+                  existing_parent_ids:)
     }
   end
 
@@ -40,7 +52,13 @@ class PostImportPreviewer
     source.merge(url_text: url, normal_url:)
   end
 
-  def preview_row row, urls:, fetch_metadata:, metadata_cache:, existing_posts:
+  def preview_row row,
+                  fetch_metadata:,
+                  metadata_cache:,
+                  existing_posts:,
+                  url_counts:,
+                  known_tags:,
+                  existing_parent_ids:
     attributes = initial_attributes(row)
     provenance = initial_provenance(row)
     tag_sources = initial_tag_sources(row, attributes, provenance)
@@ -54,10 +72,9 @@ class PostImportPreviewer
 
     validation_errors = {}
     validation_errors[:url] = ['URL が不正です．'] if normal_url.blank?
-    if normal_url.present? && urls.key?(normal_url)
+    if normal_url.present? && url_counts[normal_url].to_i > 1
       validation_errors[:url] = ['URL が重複しています．']
     end
-    urls[normal_url] = true if normal_url.present?
 
     if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
       clear_automatic_values!(attributes, provenance, tag_sources)
@@ -89,7 +106,8 @@ class PostImportPreviewer
                    end }
     end
 
-    should_fetch = should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
+    should_fetch = validation_errors.blank?
+    should_fetch &&= should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
     if should_fetch
       clear_fetch_warnings!(field_warnings)
       metadata = metadata_for(url_for_metadata, metadata_cache)
@@ -101,8 +119,9 @@ class PostImportPreviewer
     validate_basic_data(attributes, validation_errors)
     validate_preview_tags(merged_tags(tag_sources, provenance['tags']),
                           validation_errors,
-                          field_warnings)
-    validate_parents(attributes['parent_post_ids'], validation_errors)
+                          field_warnings,
+                          known_tags)
+    validate_parents(attributes['parent_post_ids'], validation_errors, existing_parent_ids)
     attributes.delete('url')
     attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
 
@@ -197,6 +216,64 @@ class PostImportPreviewer
     { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } }
   end
 
+  def preload_metadata! prepared_rows, fetch_metadata, metadata_cache, existing_posts, url_counts
+    urls = prepared_rows.filter_map { |row|
+      next unless row[:normal_url].present?
+      next unless should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
+      next if url_counts[row[:normal_url]].to_i > 1
+      next if existing_posts.key?(row[:normal_url])
+
+      row[:normal_url]
+    }.uniq
+    urls = urls.reject { metadata_cache.key?(_1) }
+    return if urls.empty?
+
+    queue = Queue.new
+    urls.each { queue << _1 }
+    workers = [urls.length, 4].min.times.map {
+      Thread.new {
+        loop do
+          url = queue.pop(true)
+          metadata_cache[url] = fetch_metadata(url)
+        rescue ThreadError
+          break
+        end
+      }
+    }
+    Timeout.timeout(15) { workers.each(&:join) }
+  rescue Timeout::Error
+    workers&.each(&:kill)
+    urls.each do |url|
+      metadata_cache[url] ||= { data: { }, warnings: { 'url' => [METADATA_FETCH_WARNING] } }
+    end
+  end
+
+  def preload_known_tags prepared_rows
+    names = prepared_rows.flat_map { |row|
+      attributes = initial_attributes(row)
+      provenance = initial_provenance(row)
+      tag_sources = initial_tag_sources(row, attributes, provenance)
+      preview_tag_names(merged_tags(tag_sources, provenance['tags']))
+    }.compact.uniq
+    return { } if names.empty?
+
+    Tag.joins(:tag_name)
+      .where(tag_names: { name: names })
+      .includes(:tag_name)
+      .to_a
+      .index_by(&:name)
+  end
+
+  def preload_parent_ids prepared_rows
+    ids = prepared_rows.flat_map { |row|
+      attributes = initial_attributes(row)
+      preview_parent_ids(attributes['parent_post_ids'])
+    }.uniq
+    return { } if ids.empty?
+
+    Post.where(id: ids).pluck(:id).to_h { [_1, true] }
+  end
+
   def apply_metadata! attributes, provenance, tag_sources, metadata
     metadata.each do |field, value|
       if field == 'tags'
@@ -232,7 +309,19 @@ class PostImportPreviewer
     sources['automatic'].to_s
   end
 
-  def validate_preview_tags raw, errors, field_warnings
+  def preview_tag_names raw
+    names = raw.to_s.split
+    return [] if names.empty?
+    if names.any? { _1.downcase.start_with?('nico:') }
+      return []
+    end
+
+    names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
+  rescue Tag::SectionLiteralParseError
+    []
+  end
+
+  def validate_preview_tags raw, errors, field_warnings, known_tags
     names = raw.to_s.split
     return if names.empty?
     if names.any? { _1.downcase.start_with?('nico:') }
@@ -241,7 +330,7 @@ class PostImportPreviewer
     end
 
     parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
-    existing = Tag.joins(:tag_name).where(tag_names: { name: parsed }).includes(:tag_name).to_a
+    existing = parsed.filter_map { known_tags[_1] }
     deprecated = existing.select(&:deprecated?).map(&:name)
     errors[:tags] = ["廃止済みタグがあります: #{ deprecated.join(' ') }"] if deprecated.present?
     known = existing.reject(&:deprecated?).map(&:name)
@@ -277,7 +366,11 @@ class PostImportPreviewer
     nil
   end
 
-  def validate_parents raw, errors
+  def preview_parent_ids raw
+    raw.to_s.split.map { Integer(_1, exception: false) }.compact
+  end
+
+  def validate_parents raw, errors, existing_parent_ids
     ids = raw.to_s.split.map { Integer(_1, exception: false) }
     return if ids.compact.empty? && raw.to_s.blank?
 
@@ -285,10 +378,8 @@ class PostImportPreviewer
       errors[:parent_post_ids] = ['親投稿 Id. が不正です．']
       return
     end
-    if Post.where(id: ids).count != ids.uniq.length
+    if ids.uniq.any? { !existing_parent_ids[_1] }
       errors[:parent_post_ids] = ['存在しない親投稿 Id. があります．']
     end
   end
-
-  private :prepare_row
 end

backend/app/services/post_import_row_normaliser.rb

@@ -6,8 +6,7 @@ class PostImportRowNormaliser
     'original_created_from',
     'original_created_before',
     'tags',
-    'parent_post_ids',
-  ].freeze
+    'parent_post_ids'].freeze
   FLEXIBLE_FIELDS = ['duration', 'video_ms'].freeze
   ATTRIBUTE_FIELDS = (STRING_FIELDS + FLEXIBLE_FIELDS).freeze
 
@@ -59,16 +58,14 @@ class PostImportRowNormaliser
       { attributes: {} },
       { provenance: {} },
       { tag_sources: {} },
-      { tagSources: {} },
-    ]
+      { tagSources: {} }]
     return keys unless allow_warning_fields
 
     keys + [
       { field_warnings: {} },
       { fieldWarnings: {} },
       { base_warnings: [] },
-      { baseWarnings: [] },
-    ]
+      { baseWarnings: [] }]
   end
   private_class_method :permitted_keys
 

backend/app/services/post_metadata_fetcher.rb

@@ -1,8 +1,9 @@
 class PostMetadataFetcher
   TIMESTAMP_PATTERN =
-      /\A(\d{4})-(\d{2})-(\d{2})T(\d{2})/ \
-      /(?::(\d{2})(?::(\d{2})(?:\.(\d+))?)?)?/ \
-      /(Z|[+-]\d{2}:?\d{2})?\z/
+      Regexp.new(
+          '\A(\d{4})-(\d{2})-(\d{2})T(\d{2})' \
+          '(?::(\d{2})(?::(\d{2})(?:\.(\d+))?)?)?' \
+          '(Z|[+-]\d{2}:?\d{2})?\z')
 
   def self.fetch raw_url
     uri, = Preview::UrlSafety.validate(raw_url)
@@ -80,9 +81,14 @@ class PostMetadataFetcher
     minute = match[5]&.to_i || 0
     second = match[6]&.to_i || 0
     fraction = match[7]
-    offset = parse_offset(match[8])
-    from = Time.new(year, month, day, hour, minute, second + fractional_seconds(fraction), offset)
-              .in_time_zone
+    offset = match[8]
+    whole_second = second + fractional_seconds(fraction)
+    from =
+        if offset.present?
+          Time.new(year, month, day, hour, minute, whole_second, parse_offset(offset)).in_time_zone
+        else
+          Time.zone.local(year, month, day, hour, minute, whole_second)
+        end
     before =
         if fraction.present?
           from + (10**(-fraction.length))
@@ -97,7 +103,6 @@ class PostMetadataFetcher
   end
 
   def self.parse_offset value
-    return Time.zone.formatted_offset if value.blank?
     return '+00:00' if value == 'Z'
 
     value.match?(/\A[+-]\d{2}:\d{2}\z/) ? value : "#{ value[0, 3] }:#{ value[3, 2] }"