このコミットが含まれているのは:
@@ -0,0 +1,37 @@
|
||||
class PostImportsController < ApplicationController
|
||||
before_action :require_member!
|
||||
|
||||
def preview
|
||||
source =
|
||||
if params[:format] == 'google_sheets'
|
||||
PostImportGoogleSheetsFetcher.fetch!(params[:source], rate_key: current_user.id)
|
||||
else
|
||||
params[:source]
|
||||
end
|
||||
format = params[:format] == 'google_sheets' ? 'csv' : params[:format]
|
||||
parsed = PostImportSourceParser.new(source:, format:,
|
||||
has_header: params[:has_header], json_path: params[:json_path]).parse
|
||||
url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil)
|
||||
return render_bad_request('URL 列を選択してください．') if url_column.nil?
|
||||
|
||||
rows = PostImportPreviewer.new(parsed:, url_column:, mappings: params[:mappings],
|
||||
existing: params[:existing]).preview
|
||||
render json: parsed.slice(:columns).merge(rows:)
|
||||
rescue ArgumentError => e
|
||||
render_bad_request(e.message)
|
||||
end
|
||||
|
||||
def create
|
||||
render json: PostImportRunner.new(actor: current_user, rows: params[:rows].to_a).run,
|
||||
status: :created
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def require_member!
|
||||
return head :unauthorized unless current_user
|
||||
return if current_user.gte_member?
|
||||
|
||||
head :forbidden
|
||||
end
|
||||
end
|
||||
@@ -142,37 +142,13 @@ class PostsController < ApplicationController
|
||||
return head :unauthorized unless current_user
|
||||
return head :forbidden unless current_user.gte_member?
|
||||
|
||||
# TODO: サイトに応じて thumbnail_base 設定
|
||||
title = params[:title].presence
|
||||
url = params[:url]
|
||||
thumbnail = params[:thumbnail]
|
||||
tag_names = params[:tags].to_s.split
|
||||
original_created_from = params[:original_created_from]
|
||||
original_created_before = params[:original_created_before]
|
||||
parent_post_ids = parse_parent_post_ids
|
||||
resized_thumbnail = thumbnail.present? ? Post.resized_thumbnail_attachment(thumbnail) : nil
|
||||
|
||||
post = Post.new(title:, url:, thumbnail_base: nil, uploaded_user: current_user,
|
||||
original_created_from:, original_created_before:)
|
||||
post.thumbnail.attach(resized_thumbnail) if resized_thumbnail
|
||||
|
||||
ApplicationRecord.transaction do
|
||||
post.save!
|
||||
|
||||
Tag.normalise_tags!(tag_names, deny_deprecated: true, with_sections: true) =>
|
||||
{ tags:, sections: }
|
||||
TagVersioning.record_tag_snapshots!(tags, created_by_user: current_user)
|
||||
|
||||
tags = Tag.expand_parent_tags(tags).reject(&:deprecated?)
|
||||
post.video_ms = normalise_video_ms(tags)
|
||||
validate_video_sections!(post.video_ms, sections)
|
||||
post.save!
|
||||
sync_post_tags!(post, tags, sections)
|
||||
|
||||
sync_parent_posts!(post, parent_post_ids)
|
||||
|
||||
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: current_user)
|
||||
end
|
||||
post = PostCreator.new(actor: current_user,
|
||||
attributes: { title: params[:title], url: params[:url],
|
||||
thumbnail: params[:thumbnail], tags: params[:tags],
|
||||
original_created_from: params[:original_created_from],
|
||||
original_created_before: params[:original_created_before],
|
||||
parent_post_ids: parse_parent_post_ids,
|
||||
video_ms: params[:video_ms], duration: params[:duration] }).create!
|
||||
|
||||
post.reload
|
||||
render json: PostRepr.base(post), status: :created
|
||||
@@ -182,7 +158,7 @@ class PostsController < ApplicationController
|
||||
render_unprocessable_entity '廃止済みタグは付与できません．', field: :tags
|
||||
rescue Tag::SectionLiteralParseError
|
||||
render_validation_error fields: { tags: ['タグ区間の記法が不正です．'] }
|
||||
rescue VideoMsParseError
|
||||
rescue PostCreator::VideoMsParseError
|
||||
render_validation_error fields: { video_ms: ['動画時間の記法が不正です．'] }
|
||||
rescue MiniMagick::Error
|
||||
render_validation_error fields: { thumbnail: ['サムネイル画像の変換に失敗しました．'] }
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
class PostCreator
|
||||
class VideoMsParseError < ArgumentError; end
|
||||
|
||||
def initialize actor:, attributes:
|
||||
@actor = actor
|
||||
@attributes = attributes.symbolize_keys
|
||||
end
|
||||
|
||||
def create!
|
||||
post = Post.new(title: @attributes[:title].presence,
|
||||
url: @attributes[:url],
|
||||
thumbnail_base: @attributes[:thumbnail_base].presence,
|
||||
uploaded_user: @actor,
|
||||
original_created_from: @attributes[:original_created_from].presence,
|
||||
original_created_before: @attributes[:original_created_before].presence)
|
||||
thumbnail = @attributes[:thumbnail]
|
||||
post.thumbnail.attach(Post.resized_thumbnail_attachment(thumbnail)) if thumbnail.present?
|
||||
|
||||
ApplicationRecord.transaction do
|
||||
post.save!
|
||||
Tag.normalise_tags!(tag_names, deny_deprecated: true, with_sections: true) =>
|
||||
{ tags:, sections: }
|
||||
TagVersioning.record_tag_snapshots!(tags, created_by_user: @actor)
|
||||
tags = Tag.expand_parent_tags(tags).reject(&:deprecated?)
|
||||
post.video_ms = normalise_video_ms(tags)
|
||||
validate_video_sections!(post.video_ms, sections)
|
||||
post.save!
|
||||
sync_post_tags!(post, tags, sections)
|
||||
sync_parent_posts!(post, parent_post_ids)
|
||||
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: @actor)
|
||||
end
|
||||
post
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def tag_names = @attributes[:tags].to_s.split
|
||||
|
||||
def parent_post_ids
|
||||
Array(@attributes[:parent_post_ids]).flat_map { _1.to_s.split }.map { |token|
|
||||
id = Integer(token, exception: false)
|
||||
raise ArgumentError, "親投稿 Id. が不正です: #{ token }" if id.nil? || id <= 0
|
||||
|
||||
id
|
||||
}.uniq
|
||||
end
|
||||
|
||||
def normalise_video_ms tags
|
||||
return nil unless tags.any? { _1.id == Tag.video.id }
|
||||
|
||||
video_ms = @attributes[:video_ms]
|
||||
if video_ms.present?
|
||||
value = Integer(video_ms, exception: false)
|
||||
raise VideoMsParseError unless value&.positive?
|
||||
|
||||
return value
|
||||
end
|
||||
duration = @attributes[:duration]
|
||||
return nil if duration.blank?
|
||||
|
||||
return duration.to_i if duration.is_a?(Numeric) && duration.to_i.positive?
|
||||
|
||||
value = Tag.time_to_ms!(duration.to_s, tag_name: '動画時間')
|
||||
raise VideoMsParseError unless value.positive?
|
||||
|
||||
value
|
||||
rescue Tag::SectionLiteralParseError
|
||||
raise VideoMsParseError
|
||||
end
|
||||
|
||||
def validate_video_sections! video_ms, sections
|
||||
return unless video_ms
|
||||
|
||||
sections.each_value do |ranges|
|
||||
ranges.each do |begin_ms, end_ms|
|
||||
next if begin_ms < video_ms && (!end_ms || end_ms <= video_ms)
|
||||
|
||||
post = Post.new
|
||||
post.errors.add :video_ms, 'タグ区間が動画時間の範囲外です．'
|
||||
raise ActiveRecord::RecordInvalid, post
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def sync_post_tags! post, desired_tags, sections
|
||||
desired_ids = desired_tags.map(&:id).to_set
|
||||
current_ids = post.tags.pluck(:id).to_set
|
||||
Tag.where(id: desired_ids - current_ids).find_each do |tag|
|
||||
PostTag.create_or_find_by!(post:, tag:, created_user: @actor)
|
||||
end
|
||||
PostTagSection.where(post_id: post.id).destroy_all
|
||||
sections.each do |tag_id, ranges|
|
||||
ranges.each { |begin_ms, end_ms| PostTagSection.create!(post_id: post.id, tag_id:, begin_ms:, end_ms:) }
|
||||
end
|
||||
PostTag.where(post_id: post.id, tag_id: (current_ids - desired_ids).to_a).kept.find_each do |post_tag|
|
||||
post_tag.discard_by!(@actor)
|
||||
end
|
||||
end
|
||||
|
||||
def sync_parent_posts! post, ids
|
||||
if ids.include?(post.id)
|
||||
post.errors.add :parent_post_ids, '自分自身を親投稿にはできません．'
|
||||
raise ActiveRecord::RecordInvalid, post
|
||||
end
|
||||
missing = ids - Post.where(id: ids).pluck(:id)
|
||||
if missing.present?
|
||||
post.errors.add :parent_post_ids, "存在しない親投稿 Id. があります: #{ missing.join(' ') }"
|
||||
raise ActiveRecord::RecordInvalid, post
|
||||
end
|
||||
ids.each { |parent_post_id| PostImplication.create_or_find_by!(post:, parent_post_id:) }
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,59 @@
|
||||
class PostImportGoogleSheetsFetcher
|
||||
HOST = 'docs.google.com'.freeze
|
||||
PUBLICATION_ID = /\A[A-Za-z0-9_-]+\z/
|
||||
GID = /\A\d+\z/
|
||||
RATE_LIMIT = 10
|
||||
RATE_WINDOW = 1.minute
|
||||
|
||||
def self.fetch! raw_url, rate_key:
|
||||
new(raw_url, rate_key:).fetch!
|
||||
end
|
||||
|
||||
def initialize raw_url, rate_key:
|
||||
@raw_url = raw_url.to_s
|
||||
@rate_key = rate_key.to_s
|
||||
end
|
||||
|
||||
def fetch!
|
||||
publication_id, gid = parse!
|
||||
throttle!
|
||||
csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub?gid=#{ gid }&single=true&output=csv"
|
||||
response = Preview::HttpFetcher.fetch(csv_url,
|
||||
max_bytes: PostImportSourceParser::MAX_BYTES,
|
||||
allowed_hosts: [HOST])
|
||||
raise ArgumentError, 'スプレッドシートを CSV として取得できませんでした．' unless csv?(response)
|
||||
|
||||
response.body
|
||||
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed => e
|
||||
raise ArgumentError, "スプレッドシートを取得できませんでした．ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください: #{ e.message }"
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def parse!
|
||||
uri = URI.parse(@raw_url)
|
||||
unless uri.scheme == 'https' && uri.host&.downcase == HOST && uri.query.blank?
|
||||
raise ArgumentError, '公開 Google スプレッドシート URL を指定してください．'
|
||||
end
|
||||
match = uri.path.match(%r{\A/spreadsheets/d/e/([^/]+)/pubhtml\z})
|
||||
gid = URI.decode_www_form(uri.fragment.to_s).to_h['gid']
|
||||
unless match && PUBLICATION_ID.match?(match[1]) && GID.match?(gid.to_s)
|
||||
raise ArgumentError, '公開 Google スプレッドシート URL の publication_id 又は gid が不正です．'
|
||||
end
|
||||
|
||||
[match[1], gid]
|
||||
rescue URI::InvalidURIError
|
||||
raise ArgumentError, '公開 Google スプレッドシート URL が不正です．'
|
||||
end
|
||||
|
||||
def throttle!
|
||||
key = "post-import-google-sheets:#{ @rate_key }"
|
||||
count = Rails.cache.increment(key, 1, expires_in: RATE_WINDOW, initial: 0)
|
||||
raise ArgumentError, 'スプレッドシート取得の回数が多すぎます．しばらくしてから再試行してください．' if count > RATE_LIMIT
|
||||
end
|
||||
|
||||
def csv? response
|
||||
type = response.content_type.to_s.downcase
|
||||
type.include?('text/csv') || response.body.valid_encoding?
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,79 @@
|
||||
class PostImportPreviewer
|
||||
FIELDS = %w[title thumbnail_base original_created_from original_created_before duration tags parent_post_ids].freeze
|
||||
|
||||
def initialize parsed:, url_column:, mappings: { }, existing: 'skip'
|
||||
@parsed = parsed
|
||||
@url_column = url_column.to_i
|
||||
@mappings = mappings.to_h.stringify_keys
|
||||
@existing = existing == 'error' ? 'error' : 'skip'
|
||||
end
|
||||
|
||||
def preview
|
||||
urls = { }
|
||||
@parsed[:rows].map do |row|
|
||||
values = row[:values]
|
||||
attributes = FIELDS.to_h { |field| [field, mapped_value(field, values)] }
|
||||
url = values[@url_column].to_s.strip
|
||||
metadata, warnings = fetch_metadata(url)
|
||||
attributes = metadata.merge(attributes) { |_key, automatic, input| input.presence || automatic }
|
||||
post = Post.new(url:)
|
||||
post.valid?
|
||||
normal_url = post.url
|
||||
errors = post.errors.to_hash.transform_values(&:dup)
|
||||
errors[:url] = ['URL が重複しています．'] if normal_url.present? && urls.key?(normal_url)
|
||||
urls[normal_url] = true if normal_url.present?
|
||||
if normal_url.present? && Post.exists?(url: normal_url)
|
||||
@existing == 'error' ? errors[:url] = ['既存投稿です．'] : warnings << '既存投稿のためスキップします．'
|
||||
end
|
||||
validate_preview_tags(attributes['tags'], errors)
|
||||
validate_parents(attributes['parent_post_ids'], errors)
|
||||
{ source_row: row[:source_row], url: normal_url || url, attributes:, warnings:, errors:,
|
||||
status: errors.present? ? 'error' : (warnings.present? ? 'warning' : 'ready'),
|
||||
existing: normal_url.present? && Post.exists?(url: normal_url) }
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def mapped_value field, values
|
||||
mapping = @mappings[field]
|
||||
return '' if mapping.blank?
|
||||
return mapping['value'].to_s if mapping['kind'] == 'fixed'
|
||||
|
||||
Array(mapping['columns']).filter_map { |index| values[index.to_i].to_s.presence }.join(field == 'tags' ? ' ' : '')
|
||||
end
|
||||
|
||||
def fetch_metadata url
|
||||
return [{ }, ['URL が空です．']] if url.blank?
|
||||
|
||||
data = PostMetadataFetcher.fetch(url)
|
||||
warnings = []
|
||||
warnings << 'タイトルを取得できませんでした．' if data[:title].blank?
|
||||
warnings << 'サムネールを取得できませんでした．' if data[:thumbnail_base].blank?
|
||||
[data.compact, warnings]
|
||||
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e
|
||||
[{ }, ["メタデータを取得できませんでした: #{ e.message }"]]
|
||||
end
|
||||
|
||||
def validate_preview_tags raw, errors
|
||||
names = raw.to_s.split
|
||||
return if names.empty?
|
||||
if names.any? { _1.downcase.start_with?('nico:') }
|
||||
errors[:tags] = ['ニコニコ・タグは直接指定できません．']
|
||||
return
|
||||
end
|
||||
canonical = names.map { TagName.canonicalise(_1.sub(/\[.*\]\z/, '')).first }
|
||||
if Tag.joins(:tag_name).where(tag_names: { name: canonical }, deprecated_at: nil).count < canonical.uniq.length
|
||||
errors[:tags] = ['存在しないタグ又は廃止済みタグがあります．']
|
||||
end
|
||||
rescue Tag::SectionLiteralParseError
|
||||
errors[:tags] = ['タグ区間の記法が不正です．']
|
||||
end
|
||||
|
||||
def validate_parents raw, errors
|
||||
ids = raw.to_s.split.map { Integer(_1, exception: false) }
|
||||
return if ids.compact.empty? && raw.to_s.blank?
|
||||
errors[:parent_post_ids] = ['親投稿 Id. が不正です．'] and return if ids.any? { _1.nil? || _1 <= 0 }
|
||||
errors[:parent_post_ids] = ['存在しない親投稿 Id. があります．'] if Post.where(id: ids).count != ids.uniq.length
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,24 @@
|
||||
class PostImportRunner
|
||||
def initialize actor:, rows:
|
||||
@actor = actor
|
||||
@rows = rows
|
||||
end
|
||||
|
||||
def run
|
||||
results = @rows.map { |row| run_row(row.to_h.stringify_keys) }
|
||||
{ created: results.count { _1[:status] == 'created' }, skipped: results.count { _1[:status] == 'skipped' },
|
||||
failed: results.count { _1[:status] == 'failed' }, rows: results }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def run_row row
|
||||
return row.slice('source_row').merge(status: 'skipped') if row['status'] == 'skipped' || row['existing']
|
||||
attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore }
|
||||
attributes['url'] = row['url']
|
||||
post = PostCreator.new(actor: @actor, attributes:).create!
|
||||
{ source_row: row['source_row'], status: 'created', post: PostRepr.base(post) }
|
||||
rescue StandardError => e
|
||||
{ source_row: row['source_row'], status: 'failed', errors: { base: [e.message] } }
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,56 @@
|
||||
require 'csv'
|
||||
|
||||
class PostImportSourceParser
|
||||
MAX_ROWS = 100
|
||||
MAX_BYTES = 1.megabyte
|
||||
MAX_CELL_BYTES = 20.kilobytes
|
||||
|
||||
def initialize source:, format:, has_header: false, json_path: nil
|
||||
@source = source.to_s
|
||||
@format = format.to_s
|
||||
@has_header = ActiveModel::Type::Boolean.new.cast(has_header)
|
||||
@json_path = json_path.to_s
|
||||
end
|
||||
|
||||
def parse
|
||||
raise ArgumentError, '入力が大きすぎます．' if @source.bytesize > MAX_BYTES
|
||||
|
||||
@format == 'json' ? parse_json : parse_delimited
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def parse_delimited
|
||||
separator = @format == 'csv' ? ',' : "\t"
|
||||
table = CSV.parse(@source, col_sep: separator).map { |row| row.map { _1.to_s.strip } }
|
||||
raise ArgumentError, '入力が空です．' if table.empty?
|
||||
|
||||
headers = @has_header ? table.shift : nil
|
||||
ensure_limits!(table)
|
||||
width = table.map(&:length).max.to_i
|
||||
{ columns: (0...width).map { |index| { key: index.to_s,
|
||||
label: "#{ ('A'.ord + index).chr }：#{ headers&.[](index).presence || '列' }" } },
|
||||
rows: table.each_with_index.map { |values, index| { source_row: index + (@has_header ? 2 : 1), values: } } }
|
||||
rescue CSV::MalformedCSVError => e
|
||||
raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }"
|
||||
end
|
||||
|
||||
def parse_json
|
||||
value = JSON.parse(@source, max_nesting: 20)
|
||||
@json_path.split('.').reject(&:blank?).each { |part| value = value.is_a?(Array) ? value[Integer(part)] : value.fetch(part) }
|
||||
raise ArgumentError, 'JSON の投稿候補は配列にしてください．' unless value.is_a?(Array)
|
||||
|
||||
ensure_limits!(value)
|
||||
keys = value.grep(Hash).flat_map(&:keys).map(&:to_s).uniq
|
||||
{ columns: keys.each_with_index.map { |key, index| { key:, label: "#{ ('A'.ord + index).chr }：#{ key }" } },
|
||||
rows: value.each_with_index.map { |item, index| { source_row: index + 1,
|
||||
values: keys.map { item.is_a?(Hash) ? item[_1] || item[_1.to_sym] : nil } } } }
|
||||
rescue JSON::ParserError, KeyError, ArgumentError => e
|
||||
raise ArgumentError, "JSON の形式が不正です: #{ e.message }"
|
||||
end
|
||||
|
||||
def ensure_limits! rows
|
||||
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if rows.length > MAX_ROWS
|
||||
raise ArgumentError, 'セルが大きすぎます．' if rows.flatten.any? { _1.to_s.bytesize > MAX_CELL_BYTES }
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,14 @@
|
||||
class PostMetadataFetcher
|
||||
def self.fetch raw_url
|
||||
uri, = Preview::UrlSafety.validate(raw_url)
|
||||
response = Preview::HttpFetcher.fetch(uri.to_s, max_bytes: Preview::ThumbnailFetcher::HTML_MAX_BYTES)
|
||||
metadata = Preview::HtmlMetadataExtractor.extract(response)
|
||||
document = Nokogiri::HTML.parse(response.body)
|
||||
content = -> name { document.at_css("meta[property='#{ name }'], meta[name='#{ name }']")&.[]('content')&.strip.presence }
|
||||
duration = content.call('og:video:duration') || content.call('video:duration')
|
||||
{ title: metadata[:title],
|
||||
thumbnail_base: Preview::KnownSiteExtractor.thumbnail_url(uri) || metadata[:image_url],
|
||||
original_created_from: content.call('article:published_time') || content.call('date'),
|
||||
duration: duration&.to_f&.then { _1.positive? ? (_1 * 1_000).round : nil } }
|
||||
end
|
||||
end
|
||||
@@ -12,8 +12,12 @@ module Preview
|
||||
|
||||
Response = Data.define(:body, :content_type, :url)
|
||||
|
||||
def self.fetch(raw_url, max_bytes: DEFAULT_MAX_BYTES, redirects: MAX_REDIRECTS)
|
||||
def self.fetch(raw_url, max_bytes: DEFAULT_MAX_BYTES, redirects: MAX_REDIRECTS,
|
||||
allowed_hosts: nil)
|
||||
uri, addresses = UrlSafety.validate(raw_url)
|
||||
if allowed_hosts && !allowed_hosts.include?(uri.host.downcase)
|
||||
raise FetchFailed, '許可されてゐない redirect 先です．'
|
||||
end
|
||||
response = request(uri, addresses.first, max_bytes)
|
||||
|
||||
if response.is_a?(Net::HTTPRedirection)
|
||||
@@ -50,7 +54,7 @@ module Preview
|
||||
raise FetchFailed, 'redirect 先が不正です．'
|
||||
end
|
||||
|
||||
return fetch(redirect_url, max_bytes:, redirects: redirects - 1)
|
||||
return fetch(redirect_url, max_bytes:, redirects: redirects - 1, allowed_hosts:)
|
||||
end
|
||||
|
||||
unless response.is_a?(Net::HTTPSuccess)
|
||||
|
||||
@@ -33,6 +33,11 @@ Rails.application.routes.draw do
|
||||
get :thumbnail
|
||||
end
|
||||
|
||||
scope 'posts/import', controller: :post_imports do
|
||||
post :preview
|
||||
post '', action: :create
|
||||
end
|
||||
|
||||
resources :wiki_pages, path: 'wiki', only: [:index, :show, :create, :update] do
|
||||
collection do
|
||||
get :search
|
||||
|
||||
@@ -43,6 +43,7 @@ import PostDetailPage from '@/pages/posts/PostDetailPage'
|
||||
import PostHistoryPage from '@/pages/posts/PostHistoryPage'
|
||||
import PostListPage from '@/pages/posts/PostListPage'
|
||||
import PostNewPage from '@/pages/posts/PostNewPage'
|
||||
import PostImportPage from '@/pages/posts/PostImportPage'
|
||||
import PostSearchPage from '@/pages/posts/PostSearchPage'
|
||||
import ServiceUnavailable from '@/pages/ServiceUnavailable'
|
||||
import SettingPage from '@/pages/users/SettingPage'
|
||||
@@ -75,6 +76,7 @@ const RouteTransitionWrapper = ({ animationMode, user, setUser }: {
|
||||
<Route path="/" element={<Navigate to="/posts" replace/>}/>
|
||||
<Route path="/posts" element={<PostListPage/>}/>
|
||||
<Route path="/posts/new" element={<PostNewPage user={user}/>}/>
|
||||
<Route path="/posts/import" element={<PostImportPage user={user}/>}/>
|
||||
<Route path="/posts/search" element={<PostSearchPage/>}/>
|
||||
<Route path="/posts/:id" element={<PostDetailRoute user={user}/>}/>
|
||||
<Route path="/posts/changes" element={<PostHistoryPage/>}/>
|
||||
@@ -113,6 +115,7 @@ const RouteTransitionWrapper = ({ animationMode, user, setUser }: {
|
||||
<Route path="/" element={<Navigate to="/posts" replace/>}/>
|
||||
<Route path="/posts" element={<PostListPage/>}/>
|
||||
<Route path="/posts/new" element={<PostNewPage user={user}/>}/>
|
||||
<Route path="/posts/import" element={<PostImportPage user={user}/>}/>
|
||||
<Route path="/posts/search" element={<PostSearchPage/>}/>
|
||||
<Route path="/posts/:id" element={<PostDetailRoute user={user}/>}/>
|
||||
<Route path="/posts/changes" element={<PostHistoryPage/>}/>
|
||||
|
||||
@@ -43,6 +43,7 @@ export const menuOutline = (
|
||||
{ name: '一覧', to: '/posts' },
|
||||
{ name: '検索', to: '/posts/search' },
|
||||
{ name: '追加', to: '/posts/new' },
|
||||
{ name: 'インポート', to: '/posts/import' },
|
||||
{ name: '全体履歴', to: '/posts/changes' },
|
||||
{ name: 'ヘルプ', to: '/wiki/ヘルプ:広場' }] },
|
||||
{ name: 'タグ', to: '/tags', subMenu: [
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import { useState } from 'react'
|
||||
import { Helmet } from 'react-helmet-async'
|
||||
|
||||
import FieldError from '@/components/common/FieldError'
|
||||
import Form from '@/components/common/Form'
|
||||
import FormField from '@/components/common/FormField'
|
||||
import PageTitle from '@/components/common/PageTitle'
|
||||
import MainArea from '@/components/layout/MainArea'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { toast } from '@/components/ui/use-toast'
|
||||
import { SITE_TITLE } from '@/config'
|
||||
import { apiPost } from '@/lib/api'
|
||||
import { canEditContent } from '@/lib/users'
|
||||
import { inputClass } from '@/lib/utils'
|
||||
import Forbidden from '@/pages/Forbidden'
|
||||
|
||||
import type { FC } from 'react'
|
||||
|
||||
import type { User } from '@/types'
|
||||
|
||||
type Props = { user: User | null }
|
||||
type Column = { key: string, label: string }
|
||||
type ImportRow = {
|
||||
sourceRow: number
|
||||
url: string
|
||||
attributes: Record<string, string>
|
||||
warnings: string[]
|
||||
errors: Record<string, string[]>
|
||||
status: 'ready' | 'warning' | 'error'
|
||||
existing: boolean }
|
||||
|
||||
|
||||
const detectFormat = (source: string): 'csv' | 'tsv' =>
|
||||
source.includes ('\t') ? 'tsv' : (source.includes (',') ? 'csv' : 'tsv')
|
||||
|
||||
const publicGoogleSheetURL = (source: string): boolean =>
|
||||
/^https:\/\/docs\.google\.com\/spreadsheets\/d\/e\/[A-Za-z0-9_-]+\/pubhtml#gid=\d+$/.test (
|
||||
source.trim ())
|
||||
|
||||
|
||||
const PostImportPage: FC<Props> = ({ user }) => {
|
||||
const [source, setSource] = useState ('')
|
||||
const [format, setFormat] = useState<'csv' | 'tsv' | 'json' | 'google_sheets'> ('tsv')
|
||||
const [hasHeader, setHasHeader] = useState (false)
|
||||
const [jsonPath, setJsonPath] = useState ('')
|
||||
const [urlColumn, setUrlColumn] = useState ('0')
|
||||
const [columns, setColumns] = useState<Column[]> ([])
|
||||
const [rows, setRows] = useState<ImportRow[]> ([])
|
||||
const [mappings, setMappings] = useState<Record<string, { columns: string[] }>> ({ })
|
||||
const [existing, setExisting] = useState<'skip' | 'error'> ('skip')
|
||||
const [loading, setLoading] = useState (false)
|
||||
const editable = canEditContent (user)
|
||||
|
||||
const preview = async () => {
|
||||
setLoading (true)
|
||||
try
|
||||
{
|
||||
const data = await apiPost<{ columns: Column[], rows: ImportRow[] }> ('/posts/import/preview', {
|
||||
source,
|
||||
format,
|
||||
has_header: hasHeader,
|
||||
json_path: jsonPath,
|
||||
url_column: urlColumn,
|
||||
mappings,
|
||||
existing })
|
||||
setColumns (data.columns)
|
||||
setRows (data.rows)
|
||||
}
|
||||
catch
|
||||
{
|
||||
toast ({ title: '読込みに失敗しました', description: '入力形式を確認してください。' })
|
||||
}
|
||||
finally
|
||||
{
|
||||
setLoading (false)
|
||||
}
|
||||
}
|
||||
|
||||
const updateAttribute = (index: number, field: string, value: string) => {
|
||||
setRows (current => current.map ((row, rowIndex) =>
|
||||
rowIndex === index ? { ...row, attributes: { ...row.attributes, [field]: value } } : row))
|
||||
}
|
||||
|
||||
const submit = async () => {
|
||||
const targets = rows.filter (row => row.status !== 'error' && !(row.existing && existing === 'skip'))
|
||||
if (targets.length === 0)
|
||||
return
|
||||
setLoading (true)
|
||||
try
|
||||
{
|
||||
const result = await apiPost<{ created: number, skipped: number, failed: number }> ('/posts/import',
|
||||
{ rows: targets })
|
||||
toast ({ title: `登録完了: ${result.created} 件`,
|
||||
description: `スキップ ${result.skipped} 件、失敗 ${result.failed} 件` })
|
||||
}
|
||||
catch
|
||||
{
|
||||
toast ({ title: '登録に失敗しました' })
|
||||
}
|
||||
finally
|
||||
{
|
||||
setLoading (false)
|
||||
}
|
||||
}
|
||||
|
||||
if (!(editable))
|
||||
return <Forbidden/>
|
||||
|
||||
return (
|
||||
<MainArea>
|
||||
<Helmet><title>{`投稿インポート | ${SITE_TITLE}`}</title></Helmet>
|
||||
<PageTitle>投稿インポート</PageTitle>
|
||||
{rows.length === 0 ? (
|
||||
<Form>
|
||||
<p className="text-sm text-neutral-600 dark:text-neutral-300">
|
||||
URL を一行に一件ずつ貼り付けるか、CSV・TSV・JSON を入力してください。
|
||||
</p>
|
||||
{publicGoogleSheetURL (source) && (
|
||||
<fieldset className="space-y-1 rounded border p-3 dark:border-neutral-700">
|
||||
<legend>公開 Google スプレッドシート URL</legend>
|
||||
<label className="mr-3"><input type="radio" checked={format === 'google_sheets'}
|
||||
onChange={() => setFormat ('google_sheets')}/>シート内データを読み込む</label>
|
||||
<label><input type="radio" checked={format !== 'google_sheets'}
|
||||
onChange={() => setFormat ('tsv')}/>URL 自体を投稿として扱う</label>
|
||||
</fieldset>)}
|
||||
<FormField label="入力形式">
|
||||
{() => <select value={format} onChange={ev => setFormat (ev.target.value as typeof format)}
|
||||
className={inputClass (false)}>
|
||||
<option value="tsv">テキスト・TSV</option><option value="csv">CSV</option>
|
||||
<option value="json">JSON</option>
|
||||
<option value="google_sheets">公開 Google スプレッドシート</option>
|
||||
</select>}
|
||||
</FormField>
|
||||
<FormField label="入力データ">
|
||||
{() => <textarea value={source} rows={12} className={inputClass (false)}
|
||||
onChange={ev => { setSource (ev.target.value); if (format !== 'json' && format !== 'google_sheets') setFormat (detectFormat (ev.target.value)) }}/>}
|
||||
</FormField>
|
||||
<input type="file" accept=".txt,.csv,.tsv,.json" onChange={ev => {
|
||||
const file = ev.target.files?.[0]
|
||||
if (file) file.text ().then (text => { setSource (text); setFormat (file.name.endsWith ('.json') ? 'json' : detectFormat (text)) })
|
||||
}}/>
|
||||
{format !== 'json' && <label className="flex gap-2"><input type="checkbox" checked={hasHeader}
|
||||
onChange={ev => setHasHeader (ev.target.checked)}/>見出し行を使用する</label>}
|
||||
{format === 'json' && <FormField label="投稿候補の位置（例: data.posts）">
|
||||
{() => <input value={jsonPath} onChange={ev => setJsonPath (ev.target.value)} className={inputClass (false)}/>}
|
||||
</FormField>}
|
||||
{format === 'google_sheets' && <p className="text-sm text-neutral-600 dark:text-neutral-300">
|
||||
非公開シートには対応していません。ウェブ公開 URL を使用するか、CSV を保存して選択、又は内容を貼り付けてください。
|
||||
</p>}
|
||||
<Button type="button" onClick={preview} disabled={loading || !(source)}>読込み・確認</Button>
|
||||
</Form>) : (
|
||||
<div className="space-y-4">
|
||||
<div className="flex flex-wrap gap-3 rounded border p-3 dark:border-neutral-700">
|
||||
<label>URL 列 <select value={urlColumn} onChange={ev => setUrlColumn (ev.target.value)}>
|
||||
{columns.map ((column, index) => <option key={column.key} value={index}>{column.label}</option>)}
|
||||
</select></label>
|
||||
<label>既存投稿 <select value={existing} onChange={ev => setExisting (ev.target.value as typeof existing)}>
|
||||
<option value="skip">スキップ</option><option value="error">エラー</option>
|
||||
</select></label>
|
||||
{['title', 'thumbnail_base', 'original_created_from', 'original_created_before', 'duration', 'tags', 'parent_post_ids'].map (field =>
|
||||
<label key={field}>{field}<select value={mappings[field]?.columns?.[0] ?? ''}
|
||||
onChange={ev => setMappings (current => ({ ...current, [field]: { columns: ev.target.value === '' ? [] : [ev.target.value] } }))}>
|
||||
<option value="">指定なし</option>{columns.map ((column, index) => <option key={column.key} value={index}>{column.label}</option>)}
|
||||
</select></label>)}
|
||||
<Button type="button" variant="outline" onClick={preview} disabled={loading}>再読込み</Button>
|
||||
</div>
|
||||
<div className="space-y-3 md:hidden">{rows.map ((row, index) => <RowCard key={row.sourceRow} row={row} index={index} update={updateAttribute}/>)}</div>
|
||||
<div className="hidden overflow-x-auto md:block"><table className="w-full text-sm"><thead><tr><th>行</th><th>URL</th><th>タイトル</th><th>サムネール</th><th>日時</th><th>動画時間</th><th>タグ</th><th>親投稿</th><th>状態</th></tr></thead>
|
||||
<tbody>{rows.map ((row, index) => <tr key={row.sourceRow} className="border-t dark:border-neutral-700"><td>{row.sourceRow}</td><td><input value={row.url} onChange={ev => { const next = [...rows]; next[index] = { ...row, url: ev.target.value }; setRows (next) }}/></td><td><input value={row.attributes.title ?? ''} onChange={ev => updateAttribute (index, 'title', ev.target.value)}/></td><td>{row.attributes.thumbnailBase && <img src={row.attributes.thumbnailBase} alt="サムネール" className="h-12"/>}</td><td>{row.attributes.originalCreatedFrom ?? ''}</td><td>{row.attributes.duration ?? ''}</td><td><input value={row.attributes.tags ?? ''} onChange={ev => updateAttribute (index, 'tags', ev.target.value)}/></td><td><input value={row.attributes.parentPostIds ?? ''} onChange={ev => updateAttribute (index, 'parent_post_ids', ev.target.value)}/></td><td>{row.status}<FieldError messages={[...row.warnings, ...Object.values (row.errors).flat ()]}/></td></tr>)}</tbody></table></div>
|
||||
<Button type="button" onClick={submit} disabled={loading}>有効な投稿を一括登録</Button>
|
||||
</div>)}
|
||||
</MainArea>)
|
||||
}
|
||||
|
||||
|
||||
const RowCard = ({ row, index, update }: { row: ImportRow, index: number, update: (index: number, field: string, value: string) => void }) => (
|
||||
<article className="space-y-2 rounded border bg-white p-3 text-neutral-900 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-100">
|
||||
<p>行 {row.sourceRow}・{row.status}</p><input value={row.url} readOnly className={inputClass (false)}/>
|
||||
<label>タイトル<input value={row.attributes.title ?? ''} onChange={ev => update (index, 'title', ev.target.value)} className={inputClass (false)}/></label>
|
||||
<label>タグ<input value={row.attributes.tags ?? ''} onChange={ev => update (index, 'tags', ev.target.value)} className={inputClass (false)}/></label>
|
||||
<FieldError messages={[...row.warnings, ...Object.values (row.errors).flat ()]}/>
|
||||
</article>)
|
||||
|
||||
export default PostImportPage
|
||||
@@ -49,6 +49,7 @@ const PostNewPage: FC<Props> = ({ user }) => {
|
||||
const [url, setURL] = useState ('')
|
||||
|
||||
const thumbnailPreviewRef = useRef ('')
|
||||
const urlRef = useRef ('')
|
||||
const videoFlg =
|
||||
useMemo (() => tags.split (/\s+/).some (tag => tag.replace (/\[.*\]$/, '') === '動画'),
|
||||
[tags])
|
||||
@@ -84,11 +85,13 @@ const PostNewPage: FC<Props> = ({ user }) => {
|
||||
}
|
||||
|
||||
const fetchTitle = useCallback (async () => {
|
||||
setTitleLoading (true)
|
||||
const requestedURL = url
|
||||
setTitleLoading (true)
|
||||
try
|
||||
{
|
||||
const data = await apiGet<{ title: string }> ('/preview/title', { params: { url } })
|
||||
setTitle (data.title || '')
|
||||
if (requestedURL === urlRef.current)
|
||||
setTitle (current => current || data.title || '')
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -97,6 +100,7 @@ const PostNewPage: FC<Props> = ({ user }) => {
|
||||
}, [url])
|
||||
|
||||
const fetchThumbnail = useCallback (async () => {
|
||||
const requestedURL = url
|
||||
setThumbnailPreview ('')
|
||||
setThumbnailFile (null)
|
||||
setThumbnailLoading (true)
|
||||
@@ -107,10 +111,13 @@ const PostNewPage: FC<Props> = ({ user }) => {
|
||||
const data = await apiGet<Blob> ('/preview/thumbnail',
|
||||
{ params: { url }, responseType: 'blob' })
|
||||
const imageURL = URL.createObjectURL (data)
|
||||
setThumbnailPreview (imageURL)
|
||||
setThumbnailFile (new File ([data],
|
||||
'thumbnail.png',
|
||||
{ type: data.type || 'image/png' }))
|
||||
if (requestedURL === urlRef.current)
|
||||
{
|
||||
setThumbnailPreview (current => current || imageURL)
|
||||
setThumbnailFile (current => current || new File ([data],
|
||||
'thumbnail.png',
|
||||
{ type: data.type || 'image/png' }))
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -122,6 +129,20 @@ const PostNewPage: FC<Props> = ({ user }) => {
|
||||
thumbnailPreviewRef.current = thumbnailPreview
|
||||
}, [thumbnailPreview])
|
||||
|
||||
useEffect (() => {
|
||||
urlRef.current = url
|
||||
}, [url])
|
||||
|
||||
useEffect (() => {
|
||||
if (!(url))
|
||||
return
|
||||
const timer = window.setTimeout (() => {
|
||||
fetchTitle ().catch (() => undefined)
|
||||
fetchThumbnail ().catch (() => undefined)
|
||||
}, 500)
|
||||
return () => window.clearTimeout (timer)
|
||||
}, [fetchThumbnail, fetchTitle, url])
|
||||
|
||||
if (!(editable))
|
||||
return <Forbidden/>
|
||||
|
||||
@@ -159,7 +180,7 @@ const PostNewPage: FC<Props> = ({ user }) => {
|
||||
onChange={ev => setTitle (ev.target.value)}
|
||||
disabled={titleLoading}/>
|
||||
<div className="flex flex-wrap items-center gap-2 text-sm">
|
||||
<span>必要なタイミングで URL から取得できます．</span>
|
||||
<span>URL 入力後に自動取得します．</span>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
@@ -176,7 +197,7 @@ const PostNewPage: FC<Props> = ({ user }) => {
|
||||
{({ describedBy, invalid }) => (
|
||||
<>
|
||||
<div className="mb-2 flex flex-wrap items-center gap-2 text-sm">
|
||||
<span>必要なタイミングで URL から取得できます．</span>
|
||||
<span>URL 入力後に自動取得します．</span>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
|
||||
新しい課題から参照
ユーザをブロックする