#399

backend/app/controllers/post_imports_controller.rb

@@ -1,7 +1,7 @@
 class PostImportsController < ApplicationController
   before_action :require_member!
 
-  def preview
+  def parse
     source =
         if params[:format] == 'google_sheets'
           PostImportGoogleSheetsFetcher.fetch!(params[:source], rate_key: current_user.id)
@@ -11,6 +11,13 @@ class PostImportsController < ApplicationController
     format = params[:format] == 'google_sheets' ? 'csv' : params[:format]
     parsed = PostImportSourceParser.new(source:, format:,
                                         has_header: params[:has_header], json_path: params[:json_path]).parse
+    render json: parsed
+  rescue ArgumentError => e
+    render_bad_request(e.message)
+  end
+
+  def preview
+    parsed = parsed_params
     url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil)
     return render_bad_request('URL 列を選択してください．') if url_column.nil?
 
@@ -21,9 +28,23 @@ class PostImportsController < ApplicationController
     render_bad_request(e.message)
   end
 
+  def validate
+    rows = validate_rows_params
+    changed_row = Integer(params[:changed_row], exception: false)
+    result = PostImportPreviewer.new(parsed: { columns: [], rows: [] }, url_column: 0,
+                                     mappings: {}, existing: params[:existing]).preview_rows(
+                                         rows:, fetch_metadata: changed_row,
+                                         metadata_cache: { })
+    render json: { rows: result }
+  rescue ArgumentError => e
+    render_bad_request(e.message)
+  end
+
   def create
-    render json: PostImportRunner.new(actor: current_user, rows: params[:rows].to_a).run,
-           status: :created
+    result = PostImportRunner.new(actor: current_user, rows: params[:rows].to_a).run
+    render json: result, status: result[:created].positive? ? :created : :ok
+  rescue ArgumentError => e
+    render_bad_request(e.message)
   end
 
   private
@@ -34,4 +55,37 @@ class PostImportsController < ApplicationController
 
     head :forbidden
   end
+
+  def parsed_params
+    value = params.require(:parsed).to_unsafe_h.deep_transform_keys { _1.to_s.underscore }
+    columns = Array(value['columns'])
+    rows = Array(value['rows'])
+    raise ArgumentError, '解析結果の形式が不正です．' unless columns.all?(Hash) && rows.all?(Hash)
+
+    { columns: columns.map { |column| column.slice('key', 'label') },
+      rows: rows.map { |row| { source_row: row['source_row'], values: Array(row['values']) } } }
+  end
+
+  def validate_rows_params
+    rows = Array(params[:rows])
+    raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS
+
+    rows.map do |row|
+      value = ActionController::Parameters.new(row).permit(:source_row, :sourceRow, :url,
+                                                            attributes: {}, provenance: {},
+                                                            fetch_warnings: [], validation_warnings: [])
+      normalised = value.to_h.deep_transform_keys { _1.to_s.underscore }
+      attributes = normalised.fetch('attributes', { })
+      provenance = normalised.fetch('provenance', { })
+      allowed = PostImportPreviewer::FIELDS
+      unless attributes.is_a?(Hash) && provenance.is_a?(Hash) &&
+             (attributes.keys - allowed).empty? &&
+             (provenance.keys - (allowed + ['url'])).empty?
+        raise ArgumentError, '行の形式が不正です．'
+      end
+      raise ArgumentError, 'セルが大きすぎます．' if [normalised['url'], *attributes.values].any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
+
+      normalised.symbolize_keys
+    end
+  end
 end

backend/app/services/post_import_google_sheets_fetcher.rb

@@ -54,6 +54,6 @@ class PostImportGoogleSheetsFetcher
 
   def csv? response
     type = response.content_type.to_s.downcase
-    type.include?('text/csv') || response.body.valid_encoding?
+    type.include?('text/csv') || type.include?('application/csv')
   end
 end

backend/app/services/post_import_previewer.rb

@@ -9,30 +9,55 @@ class PostImportPreviewer
   end
 
   def preview
+    preview_rows(rows: @parsed[:rows])
+  end
+
+  def preview_rows rows:, fetch_metadata: true, metadata_cache: { }
     urls = { }
-    @parsed[:rows].map do |row|
-      values = row[:values]
-      attributes = FIELDS.to_h { |field| [field, mapped_value(field, values)] }
-      url = values[@url_column].to_s.strip
-      metadata, warnings = fetch_metadata(url)
-      attributes = metadata.merge(attributes) { |_key, automatic, input| input.presence || automatic }
+    rows.map do |row|
+      row = row.symbolize_keys
+      values = row[:values] || []
+      attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field| [field, mapped_value(field, values)] }
+      provenance = row[:provenance]&.stringify_keys || mapped_provenance(attributes)
+      url = row[:url].presence || values[@url_column].to_s.strip
+      provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped'
+      attributes['url'] = url
+      fetch_warnings = Array(row[:fetch_warnings]).dup
+      should_fetch = fetch_metadata == true || fetch_metadata.to_i == row[:source_row].to_i
+      metadata, fetch_warnings = metadata_for(url, should_fetch, metadata_cache, fetch_warnings)
+      metadata.each do |field, value|
+        next unless provenance[field] == 'automatic' || attributes[field].blank?
+
+        attributes[field] = value
+        provenance[field] = 'automatic'
+      end
       post = Post.new(url:)
       post.valid?
       normal_url = post.url
       errors = post.errors.to_hash.transform_values(&:dup)
+      validation_warnings = []
       errors[:url] = ['URL が重複しています．'] if normal_url.present? && urls.key?(normal_url)
       urls[normal_url] = true if normal_url.present?
       if normal_url.present? && Post.exists?(url: normal_url)
-        @existing == 'error' ? errors[:url] = ['既存投稿です．'] : warnings << '既存投稿のためスキップします．'
+        @existing == 'error' ? errors[:url] = ['既存投稿です．'] : validation_warnings << '既存投稿のためスキップします．'
       end
-      validate_preview_tags(attributes['tags'], errors)
+      validate_basic_data(attributes, errors)
+      validate_preview_tags(attributes['tags'], errors, validation_warnings)
       validate_parents(attributes['parent_post_ids'], errors)
-      { source_row: row[:source_row], url: normal_url || url, attributes:, warnings:, errors:,
-        status: errors.present? ? 'error' : (warnings.present? ? 'warning' : 'ready'),
+      attributes.delete('url')
+      { source_row: row[:source_row], url: normal_url || url, attributes:, provenance:,
+        fetch_warnings:, validation_warnings:, warnings: fetch_warnings + validation_warnings, errors:,
+        status: errors.present? ? 'error' : ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
         existing: normal_url.present? && Post.exists?(url: normal_url) }
     end
   end
 
+  def metadata_for url, fetch_metadata, cache, previous_warnings
+    return [{ }, previous_warnings] unless fetch_metadata
+
+    cache[url] ||= fetch_metadata(url)
+  end
+
   private
 
   def mapped_value field, values
@@ -43,33 +68,71 @@ class PostImportPreviewer
     Array(mapping['columns']).filter_map { |index| values[index.to_i].to_s.presence }.join(field == 'tags' ? ' ' : '')
   end
 
+  def mapped_provenance attributes
+    attributes.to_h.to_h { |field, value|
+      mapping = @mappings[field.to_s]
+      origin =
+          if value.present? && mapping&.[]('kind') == 'fixed'
+            'fixed'
+          elsif value.present?
+            'mapped'
+          else
+            'automatic'
+          end
+      [field.to_s, origin]
+    }
+  end
+
   def fetch_metadata url
     return [{ }, ['URL が空です．']] if url.blank?
 
-    data = PostMetadataFetcher.fetch(url)
+    data = PostMetadataFetcher.fetch(url).stringify_keys
     warnings = []
-    warnings << 'タイトルを取得できませんでした．' if data[:title].blank?
-    warnings << 'サムネールを取得できませんでした．' if data[:thumbnail_base].blank?
+    warnings << 'タイトルを取得できませんでした．' if data['title'].blank?
+    warnings << 'サムネールを取得できませんでした．' if data['thumbnail_base'].blank?
     [data.compact, warnings]
   rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e
     [{ }, ["メタデータを取得できませんでした: #{ e.message }"]]
   end
 
-  def validate_preview_tags raw, errors
+  def validate_preview_tags raw, errors, warnings
     names = raw.to_s.split
     return if names.empty?
     if names.any? { _1.downcase.start_with?('nico:') }
       errors[:tags] = ['ニコニコ・タグは直接指定できません．']
       return
     end
-    canonical = names.map { TagName.canonicalise(_1.sub(/\[.*\]\z/, '')).first }
-    if Tag.joins(:tag_name).where(tag_names: { name: canonical }, deprecated_at: nil).count < canonical.uniq.length
-      errors[:tags] = ['存在しないタグ又は廃止済みタグがあります．']
-    end
+    parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
+    existing = Tag.joins(:tag_name).where(tag_names: { name: parsed }).includes(:tag_name).to_a
+    deprecated = existing.select(&:deprecated?).map(&:name)
+    errors[:tags] = ["廃止済みタグがあります: #{ deprecated.join(' ') }"] if deprecated.present?
+    known = existing.reject(&:deprecated?).map(&:name)
+    new_tags = parsed.uniq - known
+    warnings << "新規タグを作成します: #{ new_tags.join(' ') }" if new_tags.present?
   rescue Tag::SectionLiteralParseError
     errors[:tags] = ['タグ区間の記法が不正です．']
   end
 
+  def validate_basic_data attributes, errors
+    post = Post.new(title: attributes['title'].presence,
+                    url: attributes['url'],
+                    thumbnail_base: attributes['thumbnail_base'].presence,
+                    original_created_from: attributes['original_created_from'].presence,
+                    original_created_before: attributes['original_created_before'].presence,
+                    video_ms: parse_duration(attributes['duration'], errors))
+    post.valid?
+    post.errors.each { |error| (errors[error.attribute] ||= []) << error.message }
+  end
+
+  def parse_duration value, errors
+    return nil if value.blank?
+
+    value.is_a?(Numeric) ? value.to_i : Tag.time_to_ms!(value.to_s, tag_name: '動画時間')
+  rescue Tag::SectionLiteralParseError
+    errors[:video_ms] = ['動画時間の記法が不正です．']
+    nil
+  end
+
   def validate_parents raw, errors
     ids = raw.to_s.split.map { Integer(_1, exception: false) }
     return if ids.compact.empty? && raw.to_s.blank?

backend/app/services/post_import_runner.rb

@@ -1,11 +1,15 @@
 class PostImportRunner
+  MAX_ROWS = PostImportSourceParser::MAX_ROWS
+  ATTRIBUTE_KEYS = %w[title thumbnail_base original_created_from original_created_before duration tags parent_post_ids video_ms].freeze
   def initialize actor:, rows:
     @actor = actor
     @rows = rows
   end
 
   def run
-    results = @rows.map { |row| run_row(row.to_h.stringify_keys) }
+    raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if @rows.length > MAX_ROWS
+
+    results = @rows.map { |row| run_row(row.to_h.deep_transform_keys { _1.to_s.underscore }) }
     { created: results.count { _1[:status] == 'created' }, skipped: results.count { _1[:status] == 'skipped' },
       failed: results.count { _1[:status] == 'failed' }, rows: results }
   end
@@ -15,10 +19,20 @@ class PostImportRunner
   def run_row row
     return row.slice('source_row').merge(status: 'skipped') if row['status'] == 'skipped' || row['existing']
     attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore }
+    raise ArgumentError, '取込項目が不正です．' unless (attributes.keys - ATTRIBUTE_KEYS).empty?
     attributes['url'] = row['url']
     post = PostCreator.new(actor: @actor, attributes:).create!
     { source_row: row['source_row'], status: 'created', post: PostRepr.base(post) }
+  rescue ActiveRecord::RecordInvalid => e
+    { source_row: row['source_row'], status: 'failed', errors: e.record.errors.to_hash }
+  rescue Tag::NicoTagNormalisationError
+    { source_row: row['source_row'], status: 'failed', errors: { tags: ['ニコニコ・タグは直接指定できません．'] } }
+  rescue Tag::DeprecatedTagNormalisationError
+    { source_row: row['source_row'], status: 'failed', errors: { tags: ['廃止済みタグは付与できません．'] } }
+  rescue ArgumentError, PostCreator::VideoMsParseError
+    { source_row: row['source_row'], status: 'failed', errors: { base: ['入力値が不正です．'] } }
   rescue StandardError => e
-    { source_row: row['source_row'], status: 'failed', errors: { base: [e.message] } }
+    Rails.logger.error("post_import_runner_failure #{ { error: e.class.name, message: e.message }.to_json }")
+    { source_row: row['source_row'], status: 'failed', errors: { base: ['登録中にエラーが発生しました．'] } }
   end
 end

backend/app/services/post_metadata_fetcher.rb

@@ -6,9 +6,22 @@ class PostMetadataFetcher
     document = Nokogiri::HTML.parse(response.body)
     content = -> name { document.at_css("meta[property='#{ name }'], meta[name='#{ name }']")&.[]('content')&.strip.presence }
     duration = content.call('og:video:duration') || content.call('video:duration')
+    published = content.call('article:published_time') || content.call('date')
+    published_at = Time.zone.parse(published.to_s) if published.present?
+    platform_tags = platform_tags(uri)
     { title: metadata[:title],
       thumbnail_base: Preview::KnownSiteExtractor.thumbnail_url(uri) || metadata[:image_url],
-      original_created_from: content.call('article:published_time') || content.call('date'),
-      duration: duration&.to_f&.then { _1.positive? ? (_1 * 1_000).round : nil } }
+      original_created_from: published_at&.iso8601,
+      original_created_before: published_at && (published_at + 1.minute).iso8601,
+      duration: duration&.to_f&.then { _1.positive? ? (_1 * 1_000).round : nil },
+      tags: platform_tags.join(' ') }
   end
+
+  def self.platform_tags uri
+    return ['動画', 'YouTube'] if Preview::KnownSiteExtractor.thumbnail_url(uri)
+    return ['動画', 'ニコニコ'] if Preview::KnownSiteExtractor.niconico_video_id(uri)
+
+    []
+  end
+  private_class_method :platform_tags
 end