#399

backend/app/controllers/post_imports_controller.rb

@@ -0,0 +1,37 @@
+class PostImportsController < ApplicationController
+  before_action :require_member!
+
+  def preview
+    source =
+        if params[:format] == 'google_sheets'
+          PostImportGoogleSheetsFetcher.fetch!(params[:source], rate_key: current_user.id)
+        else
+          params[:source]
+        end
+    format = params[:format] == 'google_sheets' ? 'csv' : params[:format]
+    parsed = PostImportSourceParser.new(source:, format:,
+                                        has_header: params[:has_header], json_path: params[:json_path]).parse
+    url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil)
+    return render_bad_request('URL 列を選択してください．') if url_column.nil?
+
+    rows = PostImportPreviewer.new(parsed:, url_column:, mappings: params[:mappings],
+                                   existing: params[:existing]).preview
+    render json: parsed.slice(:columns).merge(rows:)
+  rescue ArgumentError => e
+    render_bad_request(e.message)
+  end
+
+  def create
+    render json: PostImportRunner.new(actor: current_user, rows: params[:rows].to_a).run,
+           status: :created
+  end
+
+  private
+
+  def require_member!
+    return head :unauthorized unless current_user
+    return if current_user.gte_member?
+
+    head :forbidden
+  end
+end

backend/app/controllers/posts_controller.rb

@@ -142,37 +142,13 @@ class PostsController < ApplicationController
     return head :unauthorized unless current_user
     return head :forbidden unless current_user.gte_member?
 
-    # TODO: サイトに応じて thumbnail_base 設定
-    title = params[:title].presence
-    url = params[:url]
-    thumbnail = params[:thumbnail]
-    tag_names = params[:tags].to_s.split
-    original_created_from = params[:original_created_from]
-    original_created_before = params[:original_created_before]
-    parent_post_ids = parse_parent_post_ids
-    resized_thumbnail = thumbnail.present? ? Post.resized_thumbnail_attachment(thumbnail) : nil
-
-    post = Post.new(title:, url:, thumbnail_base: nil, uploaded_user: current_user,
-                    original_created_from:, original_created_before:)
-    post.thumbnail.attach(resized_thumbnail) if resized_thumbnail
-
-    ApplicationRecord.transaction do
-      post.save!
-
-      Tag.normalise_tags!(tag_names, deny_deprecated: true, with_sections: true) =>
-          { tags:, sections: }
-      TagVersioning.record_tag_snapshots!(tags, created_by_user: current_user)
-
-      tags = Tag.expand_parent_tags(tags).reject(&:deprecated?)
-      post.video_ms = normalise_video_ms(tags)
-      validate_video_sections!(post.video_ms, sections)
-      post.save!
-      sync_post_tags!(post, tags, sections)
-
-      sync_parent_posts!(post, parent_post_ids)
-
-      PostVersionRecorder.record!(post:, event_type: :create, created_by_user: current_user)
-    end
+    post = PostCreator.new(actor: current_user,
+                           attributes: { title: params[:title], url: params[:url],
+                                         thumbnail: params[:thumbnail], tags: params[:tags],
+                                         original_created_from: params[:original_created_from],
+                                         original_created_before: params[:original_created_before],
+                                         parent_post_ids: parse_parent_post_ids,
+                                         video_ms: params[:video_ms], duration: params[:duration] }).create!
 
     post.reload
     render json: PostRepr.base(post), status: :created
@@ -182,7 +158,7 @@ class PostsController < ApplicationController
     render_unprocessable_entity '廃止済みタグは付与できません．', field: :tags
   rescue Tag::SectionLiteralParseError
     render_validation_error fields: { tags: ['タグ区間の記法が不正です．'] }
-  rescue VideoMsParseError
+  rescue PostCreator::VideoMsParseError
     render_validation_error fields: { video_ms: ['動画時間の記法が不正です．'] }
   rescue MiniMagick::Error
     render_validation_error fields: { thumbnail: ['サムネイル画像の変換に失敗しました．'] }

backend/app/services/post_creator.rb

@@ -0,0 +1,112 @@
+class PostCreator
+  class VideoMsParseError < ArgumentError; end
+
+  def initialize actor:, attributes:
+    @actor = actor
+    @attributes = attributes.symbolize_keys
+  end
+
+  def create!
+    post = Post.new(title: @attributes[:title].presence,
+                    url: @attributes[:url],
+                    thumbnail_base: @attributes[:thumbnail_base].presence,
+                    uploaded_user: @actor,
+                    original_created_from: @attributes[:original_created_from].presence,
+                    original_created_before: @attributes[:original_created_before].presence)
+    thumbnail = @attributes[:thumbnail]
+    post.thumbnail.attach(Post.resized_thumbnail_attachment(thumbnail)) if thumbnail.present?
+
+    ApplicationRecord.transaction do
+      post.save!
+      Tag.normalise_tags!(tag_names, deny_deprecated: true, with_sections: true) =>
+          { tags:, sections: }
+      TagVersioning.record_tag_snapshots!(tags, created_by_user: @actor)
+      tags = Tag.expand_parent_tags(tags).reject(&:deprecated?)
+      post.video_ms = normalise_video_ms(tags)
+      validate_video_sections!(post.video_ms, sections)
+      post.save!
+      sync_post_tags!(post, tags, sections)
+      sync_parent_posts!(post, parent_post_ids)
+      PostVersionRecorder.record!(post:, event_type: :create, created_by_user: @actor)
+    end
+    post
+  end
+
+  private
+
+  def tag_names = @attributes[:tags].to_s.split
+
+  def parent_post_ids
+    Array(@attributes[:parent_post_ids]).flat_map { _1.to_s.split }.map { |token|
+      id = Integer(token, exception: false)
+      raise ArgumentError, "親投稿 Id. が不正です: #{ token }" if id.nil? || id <= 0
+
+      id
+    }.uniq
+  end
+
+  def normalise_video_ms tags
+    return nil unless tags.any? { _1.id == Tag.video.id }
+
+    video_ms = @attributes[:video_ms]
+    if video_ms.present?
+      value = Integer(video_ms, exception: false)
+      raise VideoMsParseError unless value&.positive?
+
+      return value
+    end
+    duration = @attributes[:duration]
+    return nil if duration.blank?
+
+    return duration.to_i if duration.is_a?(Numeric) && duration.to_i.positive?
+
+    value = Tag.time_to_ms!(duration.to_s, tag_name: '動画時間')
+    raise VideoMsParseError unless value.positive?
+
+    value
+  rescue Tag::SectionLiteralParseError
+    raise VideoMsParseError
+  end
+
+  def validate_video_sections! video_ms, sections
+    return unless video_ms
+
+    sections.each_value do |ranges|
+      ranges.each do |begin_ms, end_ms|
+        next if begin_ms < video_ms && (!end_ms || end_ms <= video_ms)
+
+        post = Post.new
+        post.errors.add :video_ms, 'タグ区間が動画時間の範囲外です．'
+        raise ActiveRecord::RecordInvalid, post
+      end
+    end
+  end
+
+  def sync_post_tags! post, desired_tags, sections
+    desired_ids = desired_tags.map(&:id).to_set
+    current_ids = post.tags.pluck(:id).to_set
+    Tag.where(id: desired_ids - current_ids).find_each do |tag|
+      PostTag.create_or_find_by!(post:, tag:, created_user: @actor)
+    end
+    PostTagSection.where(post_id: post.id).destroy_all
+    sections.each do |tag_id, ranges|
+      ranges.each { |begin_ms, end_ms| PostTagSection.create!(post_id: post.id, tag_id:, begin_ms:, end_ms:) }
+    end
+    PostTag.where(post_id: post.id, tag_id: (current_ids - desired_ids).to_a).kept.find_each do |post_tag|
+      post_tag.discard_by!(@actor)
+    end
+  end
+
+  def sync_parent_posts! post, ids
+    if ids.include?(post.id)
+      post.errors.add :parent_post_ids, '自分自身を親投稿にはできません．'
+      raise ActiveRecord::RecordInvalid, post
+    end
+    missing = ids - Post.where(id: ids).pluck(:id)
+    if missing.present?
+      post.errors.add :parent_post_ids, "存在しない親投稿 Id. があります: #{ missing.join(' ') }"
+      raise ActiveRecord::RecordInvalid, post
+    end
+    ids.each { |parent_post_id| PostImplication.create_or_find_by!(post:, parent_post_id:) }
+  end
+end

backend/app/services/post_import_google_sheets_fetcher.rb

@@ -0,0 +1,59 @@
+class PostImportGoogleSheetsFetcher
+  HOST = 'docs.google.com'.freeze
+  PUBLICATION_ID = /\A[A-Za-z0-9_-]+\z/
+  GID = /\A\d+\z/
+  RATE_LIMIT = 10
+  RATE_WINDOW = 1.minute
+
+  def self.fetch! raw_url, rate_key:
+    new(raw_url, rate_key:).fetch!
+  end
+
+  def initialize raw_url, rate_key:
+    @raw_url = raw_url.to_s
+    @rate_key = rate_key.to_s
+  end
+
+  def fetch!
+    publication_id, gid = parse!
+    throttle!
+    csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub?gid=#{ gid }&single=true&output=csv"
+    response = Preview::HttpFetcher.fetch(csv_url,
+                                          max_bytes: PostImportSourceParser::MAX_BYTES,
+                                          allowed_hosts: [HOST])
+    raise ArgumentError, 'スプレッドシートを CSV として取得できませんでした．' unless csv?(response)
+
+    response.body
+  rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed => e
+    raise ArgumentError, "スプレッドシートを取得できませんでした．ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください: #{ e.message }"
+  end
+
+  private
+
+  def parse!
+    uri = URI.parse(@raw_url)
+    unless uri.scheme == 'https' && uri.host&.downcase == HOST && uri.query.blank?
+      raise ArgumentError, '公開 Google スプレッドシート URL を指定してください．'
+    end
+    match = uri.path.match(%r{\A/spreadsheets/d/e/([^/]+)/pubhtml\z})
+    gid = URI.decode_www_form(uri.fragment.to_s).to_h['gid']
+    unless match && PUBLICATION_ID.match?(match[1]) && GID.match?(gid.to_s)
+      raise ArgumentError, '公開 Google スプレッドシート URL の publication_id 又は gid が不正です．'
+    end
+
+    [match[1], gid]
+  rescue URI::InvalidURIError
+    raise ArgumentError, '公開 Google スプレッドシート URL が不正です．'
+  end
+
+  def throttle!
+    key = "post-import-google-sheets:#{ @rate_key }"
+    count = Rails.cache.increment(key, 1, expires_in: RATE_WINDOW, initial: 0)
+    raise ArgumentError, 'スプレッドシート取得の回数が多すぎます．しばらくしてから再試行してください．' if count > RATE_LIMIT
+  end
+
+  def csv? response
+    type = response.content_type.to_s.downcase
+    type.include?('text/csv') || response.body.valid_encoding?
+  end
+end

backend/app/services/post_import_previewer.rb

@@ -0,0 +1,79 @@
+class PostImportPreviewer
+  FIELDS = %w[title thumbnail_base original_created_from original_created_before duration tags parent_post_ids].freeze
+
+  def initialize parsed:, url_column:, mappings: { }, existing: 'skip'
+    @parsed = parsed
+    @url_column = url_column.to_i
+    @mappings = mappings.to_h.stringify_keys
+    @existing = existing == 'error' ? 'error' : 'skip'
+  end
+
+  def preview
+    urls = { }
+    @parsed[:rows].map do |row|
+      values = row[:values]
+      attributes = FIELDS.to_h { |field| [field, mapped_value(field, values)] }
+      url = values[@url_column].to_s.strip
+      metadata, warnings = fetch_metadata(url)
+      attributes = metadata.merge(attributes) { |_key, automatic, input| input.presence || automatic }
+      post = Post.new(url:)
+      post.valid?
+      normal_url = post.url
+      errors = post.errors.to_hash.transform_values(&:dup)
+      errors[:url] = ['URL が重複しています．'] if normal_url.present? && urls.key?(normal_url)
+      urls[normal_url] = true if normal_url.present?
+      if normal_url.present? && Post.exists?(url: normal_url)
+        @existing == 'error' ? errors[:url] = ['既存投稿です．'] : warnings << '既存投稿のためスキップします．'
+      end
+      validate_preview_tags(attributes['tags'], errors)
+      validate_parents(attributes['parent_post_ids'], errors)
+      { source_row: row[:source_row], url: normal_url || url, attributes:, warnings:, errors:,
+        status: errors.present? ? 'error' : (warnings.present? ? 'warning' : 'ready'),
+        existing: normal_url.present? && Post.exists?(url: normal_url) }
+    end
+  end
+
+  private
+
+  def mapped_value field, values
+    mapping = @mappings[field]
+    return '' if mapping.blank?
+    return mapping['value'].to_s if mapping['kind'] == 'fixed'
+
+    Array(mapping['columns']).filter_map { |index| values[index.to_i].to_s.presence }.join(field == 'tags' ? ' ' : '')
+  end
+
+  def fetch_metadata url
+    return [{ }, ['URL が空です．']] if url.blank?
+
+    data = PostMetadataFetcher.fetch(url)
+    warnings = []
+    warnings << 'タイトルを取得できませんでした．' if data[:title].blank?
+    warnings << 'サムネールを取得できませんでした．' if data[:thumbnail_base].blank?
+    [data.compact, warnings]
+  rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e
+    [{ }, ["メタデータを取得できませんでした: #{ e.message }"]]
+  end
+
+  def validate_preview_tags raw, errors
+    names = raw.to_s.split
+    return if names.empty?
+    if names.any? { _1.downcase.start_with?('nico:') }
+      errors[:tags] = ['ニコニコ・タグは直接指定できません．']
+      return
+    end
+    canonical = names.map { TagName.canonicalise(_1.sub(/\[.*\]\z/, '')).first }
+    if Tag.joins(:tag_name).where(tag_names: { name: canonical }, deprecated_at: nil).count < canonical.uniq.length
+      errors[:tags] = ['存在しないタグ又は廃止済みタグがあります．']
+    end
+  rescue Tag::SectionLiteralParseError
+    errors[:tags] = ['タグ区間の記法が不正です．']
+  end
+
+  def validate_parents raw, errors
+    ids = raw.to_s.split.map { Integer(_1, exception: false) }
+    return if ids.compact.empty? && raw.to_s.blank?
+    errors[:parent_post_ids] = ['親投稿 Id. が不正です．'] and return if ids.any? { _1.nil? || _1 <= 0 }
+    errors[:parent_post_ids] = ['存在しない親投稿 Id. があります．'] if Post.where(id: ids).count != ids.uniq.length
+  end
+end

backend/app/services/post_import_runner.rb

@@ -0,0 +1,24 @@
+class PostImportRunner
+  def initialize actor:, rows:
+    @actor = actor
+    @rows = rows
+  end
+
+  def run
+    results = @rows.map { |row| run_row(row.to_h.stringify_keys) }
+    { created: results.count { _1[:status] == 'created' }, skipped: results.count { _1[:status] == 'skipped' },
+      failed: results.count { _1[:status] == 'failed' }, rows: results }
+  end
+
+  private
+
+  def run_row row
+    return row.slice('source_row').merge(status: 'skipped') if row['status'] == 'skipped' || row['existing']
+    attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore }
+    attributes['url'] = row['url']
+    post = PostCreator.new(actor: @actor, attributes:).create!
+    { source_row: row['source_row'], status: 'created', post: PostRepr.base(post) }
+  rescue StandardError => e
+    { source_row: row['source_row'], status: 'failed', errors: { base: [e.message] } }
+  end
+end

backend/app/services/post_import_source_parser.rb

@@ -0,0 +1,56 @@
+require 'csv'
+
+class PostImportSourceParser
+  MAX_ROWS = 100
+  MAX_BYTES = 1.megabyte
+  MAX_CELL_BYTES = 20.kilobytes
+
+  def initialize source:, format:, has_header: false, json_path: nil
+    @source = source.to_s
+    @format = format.to_s
+    @has_header = ActiveModel::Type::Boolean.new.cast(has_header)
+    @json_path = json_path.to_s
+  end
+
+  def parse
+    raise ArgumentError, '入力が大きすぎます．' if @source.bytesize > MAX_BYTES
+
+    @format == 'json' ? parse_json : parse_delimited
+  end
+
+  private
+
+  def parse_delimited
+    separator = @format == 'csv' ? ',' : "\t"
+    table = CSV.parse(@source, col_sep: separator).map { |row| row.map { _1.to_s.strip } }
+    raise ArgumentError, '入力が空です．' if table.empty?
+
+    headers = @has_header ? table.shift : nil
+    ensure_limits!(table)
+    width = table.map(&:length).max.to_i
+    { columns: (0...width).map { |index| { key: index.to_s,
+                                          label: "#{ ('A'.ord + index).chr }：#{ headers&.[](index).presence || '列' }" } },
+      rows: table.each_with_index.map { |values, index| { source_row: index + (@has_header ? 2 : 1), values: } } }
+  rescue CSV::MalformedCSVError => e
+    raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }"
+  end
+
+  def parse_json
+    value = JSON.parse(@source, max_nesting: 20)
+    @json_path.split('.').reject(&:blank?).each { |part| value = value.is_a?(Array) ? value[Integer(part)] : value.fetch(part) }
+    raise ArgumentError, 'JSON の投稿候補は配列にしてください．' unless value.is_a?(Array)
+
+    ensure_limits!(value)
+    keys = value.grep(Hash).flat_map(&:keys).map(&:to_s).uniq
+    { columns: keys.each_with_index.map { |key, index| { key:, label: "#{ ('A'.ord + index).chr }：#{ key }" } },
+      rows: value.each_with_index.map { |item, index| { source_row: index + 1,
+                                                         values: keys.map { item.is_a?(Hash) ? item[_1] || item[_1.to_sym] : nil } } } }
+  rescue JSON::ParserError, KeyError, ArgumentError => e
+    raise ArgumentError, "JSON の形式が不正です: #{ e.message }"
+  end
+
+  def ensure_limits! rows
+    raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if rows.length > MAX_ROWS
+    raise ArgumentError, 'セルが大きすぎます．' if rows.flatten.any? { _1.to_s.bytesize > MAX_CELL_BYTES }
+  end
+end

backend/app/services/post_metadata_fetcher.rb

@@ -0,0 +1,14 @@
+class PostMetadataFetcher
+  def self.fetch raw_url
+    uri, = Preview::UrlSafety.validate(raw_url)
+    response = Preview::HttpFetcher.fetch(uri.to_s, max_bytes: Preview::ThumbnailFetcher::HTML_MAX_BYTES)
+    metadata = Preview::HtmlMetadataExtractor.extract(response)
+    document = Nokogiri::HTML.parse(response.body)
+    content = -> name { document.at_css("meta[property='#{ name }'], meta[name='#{ name }']")&.[]('content')&.strip.presence }
+    duration = content.call('og:video:duration') || content.call('video:duration')
+    { title: metadata[:title],
+      thumbnail_base: Preview::KnownSiteExtractor.thumbnail_url(uri) || metadata[:image_url],
+      original_created_from: content.call('article:published_time') || content.call('date'),
+      duration: duration&.to_f&.then { _1.positive? ? (_1 * 1_000).round : nil } }
+  end
+end

backend/app/services/preview/http_fetcher.rb

@@ -12,8 +12,12 @@ module Preview
 
     Response = Data.define(:body, :content_type, :url)
 
-    def self.fetch(raw_url, max_bytes: DEFAULT_MAX_BYTES, redirects: MAX_REDIRECTS)
+    def self.fetch(raw_url, max_bytes: DEFAULT_MAX_BYTES, redirects: MAX_REDIRECTS,
+                   allowed_hosts: nil)
       uri, addresses = UrlSafety.validate(raw_url)
+      if allowed_hosts && !allowed_hosts.include?(uri.host.downcase)
+        raise FetchFailed, '許可されてゐない redirect 先です．'
+      end
       response = request(uri, addresses.first, max_bytes)
 
       if response.is_a?(Net::HTTPRedirection)
@@ -50,7 +54,7 @@ module Preview
               raise FetchFailed, 'redirect 先が不正です．'
             end
 
-        return fetch(redirect_url, max_bytes:, redirects: redirects - 1)
+        return fetch(redirect_url, max_bytes:, redirects: redirects - 1, allowed_hosts:)
       end
 
       unless response.is_a?(Net::HTTPSuccess)

backend/config/routes.rb

@@ -33,6 +33,11 @@ Rails.application.routes.draw do
     get :thumbnail
   end
 
+  scope 'posts/import', controller: :post_imports do
+    post :preview
+    post '', action: :create
+  end
+
   resources :wiki_pages, path: 'wiki', only: [:index, :show, :create, :update] do
     collection do
       get :search