このコミットが含まれているのは:
2026-07-12 13:14:36 +09:00
コミット df17f20907
9個のファイルの変更304行の追加942行の削除
+5 -121
ファイルの表示
@@ -1,44 +1,10 @@
class PostImportsController < ApplicationController
MAPPING_PROPERTIES = ['kind', 'value', 'columns'].freeze
MAPPING_KINDS = ['columns', 'fixed'].freeze
before_action :require_member!
def parse
source =
if params[:format] == 'google_sheets'
PostImportGoogleSheetsFetcher.fetch!(params[:source],
rate_key: current_user.id)
else
params[:source]
end
format = params[:format] == 'google_sheets' ? 'csv' : params[:format]
parsed = PostImportSourceParser.new(
source:,
format:,
has_header: params[:has_header],
json_path: params[:json_path]).parse
render json: parsed
rescue ArgumentError => e
render_bad_request(e.message)
end
def preview
parsed = parsed_params
raw_url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil)
return render_bad_request('URL 列を選択してください.') if raw_url_column.nil?
url_column = Integer(raw_url_column, exception: false)
if url_column.nil? || url_column.negative? || url_column >= parsed[:columns].length
return render_bad_request('URL 列が不正です.')
end
rows = PostImportPreviewer.new(
parsed:,
url_column:,
mappings: mappings_params(parsed[:columns].length),
existing: params[:existing]).preview
render json: parsed.slice(:columns).merge(rows:)
rows = PostImportPreviewer.new(existing: params[:existing]).preview_rows(
rows: PostImportUrlListParser.parse(params[:source]))
render json: { rows: }
rescue ArgumentError => e
render_bad_request(e.message)
end
@@ -47,11 +13,7 @@ class PostImportsController < ApplicationController
rows = normalised_import_rows(allow_warning_fields: true)
changed_row = Integer(params[:changed_row], exception: false)
result =
PostImportPreviewer.new(
parsed: { columns: [], rows: [] },
url_column: 0,
mappings: {},
existing: params[:existing])
PostImportPreviewer.new(existing: params[:existing])
.preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {})
render json: { rows: result }
rescue ArgumentError => e
@@ -77,85 +39,7 @@ class PostImportsController < ApplicationController
head :forbidden
end
def parsed_params
value = params.require(:parsed).to_unsafe_h.deep_transform_keys { _1.to_s.underscore }
columns = Array(value['columns'])
rows = Array(value['rows'])
unless columns.all?(Hash) && rows.all?(Hash)
raise ArgumentError, '解析結果の形式が不正です.'
end
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportSourceParser::MAX_ROWS
raise ArgumentError, '列数が多すぎます.' if columns.length > 50
raise ArgumentError, '解析結果のセルが不正です.' unless rows.all? { |row|
values = Array(row['values'])
values.length <= columns.length && values.all? { |value|
value.is_a?(String) && value.bytesize <= PostImportSourceParser::MAX_CELL_BYTES
}
}
if rows.sum { Array(_1['values']).sum(&:bytesize) } > PostImportSourceParser::MAX_BYTES
raise ArgumentError, '解析結果が大きすぎます.'
end
parsed_rows = rows.map do |row|
source_row = Integer(row['source_row'], exception: false)
raise ArgumentError, '元行番号が不正です.' if source_row.nil? || source_row <= 0
{ source_row:, values: Array(row['values']) }
end
if parsed_rows.map { _1[:source_row] }.uniq.length != parsed_rows.length
raise ArgumentError, '元行番号が重複しています.'
end
{ columns: columns.map { |column| column.slice('key', 'label') }, rows: parsed_rows }
end
def mappings_params column_count
mappings = params[:mappings]
return { } if mappings.blank?
raise ArgumentError, '列割当ての形式が不正です.' unless mappings.is_a?(ActionController::Parameters)
raw = mappings.to_unsafe_h
unknown_fields = raw.keys - PostImportPreviewer::FIELDS
raise ArgumentError, '列割当て先が不正です.' if unknown_fields.present?
raise ArgumentError, '列割当てが大きすぎます.' if raw.to_json.bytesize > PostImportSourceParser::MAX_BYTES
permitted =
mappings.permit(*PostImportPreviewer::FIELDS.map { |field|
{ field => [:kind, :value, { columns: [] }] }
}).to_h
raw.each_with_object({ }) do |(field, value), result|
raise ArgumentError, '列割当ての形式が不正です.' unless value.is_a?(Hash)
unless (value.keys - MAPPING_PROPERTIES).empty?
raise ArgumentError, '列割当ての項目が不正です.'
end
value = permitted.fetch(field)
kind = value['kind'].presence || 'columns'
raise ArgumentError, '列割当ての種別が不正です.' unless kind.in?(MAPPING_KINDS)
columns = value['columns']
raise ArgumentError, '列番号が不正です.' unless (
columns.nil? || (columns.is_a?(Array) && columns.all? { |index|
parsed = Integer(index, exception: false)
parsed && parsed >= 0 && parsed < column_count
})
)
parsed_columns = Array(columns).map { Integer(_1) }
raise ArgumentError, '列番号が重複しています.' if parsed_columns.uniq.length != parsed_columns.length
raise ArgumentError, '固定値が不正です.' if kind == 'fixed' && !value['value'].is_a?(String)
if value['value'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, '固定値が大きすぎます.'
end
result[field] = {
'kind' => kind,
'value' => value['value'].to_s,
'columns' => parsed_columns }
end
end
def normalised_import_rows allow_warning_fields: false
rows = params[:rows]
PostImportRowNormaliser.normalise!(rows, allow_warning_fields:)
PostImportRowNormaliser.normalise!(params[:rows], allow_warning_fields:)
end
end
-67
ファイルの表示
@@ -1,67 +0,0 @@
class PostImportGoogleSheetsFetcher
HOST = 'docs.google.com'.freeze
PUBLICATION_ID = /\A[A-Za-z0-9_-]+\z/
GID = /\A\d+\z/
RATE_LIMIT = 10
RATE_WINDOW = 1.minute
def self.fetch! raw_url, rate_key:
new(raw_url, rate_key:).fetch!
end
def initialize raw_url, rate_key:
@raw_url = raw_url.to_s
@rate_key = rate_key.to_s
end
def fetch!
publication_id, gid = parse!
throttle!
csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub" \
"?gid=#{ gid }&single=true&output=csv"
response = Preview::HttpFetcher.fetch(csv_url,
max_bytes: PostImportSourceParser::MAX_BYTES,
allowed_hosts: [HOST])
raise ArgumentError, 'スプレッドシートを CSV として取得できませんでした.' unless csv?(response)
response.body
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed,
Preview::HttpFetcher::ResponseTooLarge => e
Rails.logger.info(
'post_import_google_sheets_fetch_failure '\
"#{ { error: e.class.name, message: e.message }.to_json }",
)
raise ArgumentError, 'スプレッドシートを取得できませんでした.ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください.'
end
private
def parse!
uri = URI.parse(@raw_url)
unless uri.scheme == 'https' && uri.host&.downcase == HOST && uri.query.blank?
raise ArgumentError, '公開 Google スプレッドシート URL を指定してください.'
end
match = uri.path.match(%r{\A/spreadsheets/d/e/([^/]+)/pubhtml\z})
gid = URI.decode_www_form(uri.fragment.to_s).to_h['gid']
unless match && PUBLICATION_ID.match?(match[1]) && GID.match?(gid.to_s)
raise ArgumentError, '公開 Google スプレッドシート URL の publication_id 又は gid が不正です.'
end
[match[1], gid]
rescue URI::InvalidURIError
raise ArgumentError, '公開 Google スプレッドシート URL が不正です.'
end
def throttle!
key = "post-import-google-sheets:#{ @rate_key }"
count = Rails.cache.increment(key, 1, expires_in: RATE_WINDOW, initial: 0)
if count > RATE_LIMIT
raise ArgumentError, 'スプレッドシート取得の回数が多すぎます.しばらくしてから再試行してください.'
end
end
def csv? response
type = response.content_type.to_s.downcase
type.include?('text/csv') || type.include?('application/csv')
end
end
+97 -119
ファイルの表示
@@ -8,96 +8,70 @@ class PostImportPreviewer
'tags',
'parent_post_ids',
].freeze
def initialize parsed:, url_column:, mappings: { }, existing: 'skip'
@parsed = parsed
@url_column = url_column.to_i
@mappings = mappings.stringify_keys
def initialize existing: 'skip'
@existing = existing == 'error' ? 'error' : 'skip'
end
def preview
preview_rows(rows: @parsed[:rows])
end
def preview_rows rows:, fetch_metadata: true, metadata_cache: { }
urls = { }
rows.map do |row|
row = row.symbolize_keys
values = row[:values] || []
attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field|
[field, mapped_value(field, values)]
}
provenance = row[:provenance]&.stringify_keys || mapped_provenance(attributes)
tag_sources =
row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance)
tag_sources['manual'] = attributes['tags'].to_s if provenance['tags'] == 'manual'
url = row[:url].presence || values[@url_column].to_s.strip
provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped'
url_for_metadata = normalised_url(url) || url
existing_post = normalised_url(url).present? && Post.exists?(url: normalised_url(url))
if existing_post && @existing == 'skip'
next {
source_row: row[:source_row],
url: normalised_url(url) || url,
attributes:,
provenance:,
tag_sources:,
metadata_url: url_for_metadata,
fetch_warnings: [],
validation_warnings: ['既存投稿のためスキップします.'],
warnings: ['既存投稿のためスキップします.'],
validation_errors: { },
status: 'warning',
existing: true,
urls = {}
rows.map { |row|
preview_row(row, urls:, fetch_metadata:, metadata_cache:)
}
end
url_changed = row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
clear_automatic_values!(attributes, provenance, tag_sources) if url_changed
attributes['url'] = url
fetch_warnings = Array(row[:fetch_warnings]).dup
should_fetch =
case fetch_metadata
when true then true
when Integer then fetch_metadata == row[:source_row].to_i
when false, nil then false
else raise ArgumentError, 'メタデータ取得対象が不正です.'
end
metadata, fetch_warnings =
metadata_for(url_for_metadata, should_fetch, metadata_cache, fetch_warnings)
metadata.each do |field, value|
if field == 'tags' && provenance[field] != 'manual'
tag_sources['automatic'] = value.to_s
attributes[field] = merged_tags(tag_sources)
next
end
next unless provenance[field] == 'automatic' || attributes[field].blank?
attributes[field] = value
provenance[field] = 'automatic'
def normalised_url value
PostUrlNormaliser.normalise(value)
end
private
def preview_row row, urls:, fetch_metadata:, metadata_cache:
row = row.symbolize_keys
attributes = initial_attributes(row)
provenance = initial_provenance(row)
tag_sources = initial_tag_sources(row, attributes, provenance)
url = row[:url].to_s.strip
provenance['url'] = 'manual'
normal_url = normalised_url(url)
errors = { }
errors[:url] = ['URL が不正です.'] if normal_url.blank?
url_for_metadata = normal_url || url
existing_post = normal_url.present? && Post.exists?(url: normal_url)
validation_warnings = []
errors[:url] = ['URL が重複しています.'] if normal_url.present? && urls.key?(normal_url)
validation_errors = {}
validation_errors[:url] = ['URL が不正です.'] if normal_url.blank?
validation_errors[:url] = ['URL が重複しています.'] if normal_url.present? && urls.key?(normal_url)
urls[normal_url] = true if normal_url.present?
if normal_url.present? && existing_post
if @existing == 'error'
errors[:url] = ['既存投稿です.']
validation_errors[:url] = ['既存投稿です.']
else
validation_warnings << '既存投稿のためスキップします.'
end
end
validate_basic_data(attributes, errors)
if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
clear_automatic_values!(attributes, provenance, tag_sources)
end
fetch_warnings = Array(row[:fetch_warnings]).dup
should_fetch = should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
if should_fetch && !(existing_post && @existing == 'skip')
metadata, fetch_warnings =
metadata_for(url_for_metadata, metadata_cache, fetch_warnings)
apply_metadata!(attributes, provenance, tag_sources, metadata)
end
attributes['url'] = url
validate_basic_data(attributes, validation_errors)
validate_preview_tags(
merged_tags(tag_sources, provenance['tags']),
errors,
validation_errors,
validation_warnings,
)
validate_parents(attributes['parent_post_ids'], errors)
validate_parents(attributes['parent_post_ids'], validation_errors)
attributes.delete('url')
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
{
source_row: row[:source_row],
url: normal_url || url,
@@ -108,64 +82,40 @@ class PostImportPreviewer
fetch_warnings:,
validation_warnings:,
warnings: fetch_warnings + validation_warnings,
validation_errors: errors,
status: errors.present? \
validation_errors:,
status: validation_errors.present? \
? 'error' \
: ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
existing: normal_url.present? && Post.exists?(url: normal_url),
}
end
end
def metadata_for url, fetch_metadata, cache, previous_warnings
return [{ }, previous_warnings] unless fetch_metadata
cache[url] ||= fetch_metadata(url)
end
def normalised_url value
PostUrlNormaliser.normalise(value)
end
private
def mapped_value field, values
mapping = @mappings[field]
return '' if mapping.blank?
return mapping['value'].to_s if mapping['kind'] == 'fixed'
Array(mapping['columns']).filter_map { |index|
values[index.to_i].to_s.presence
}.join(field == 'tags' ? ' ' : '')
end
def mapped_provenance attributes
attributes.to_h.to_h { |field, value|
mapping = @mappings[field.to_s]
origin =
if value.present? && mapping&.[]('kind') == 'fixed'
'fixed'
elsif value.present?
'mapped'
else
'automatic'
end
[field.to_s, origin]
existing: existing_post,
}
end
def initial_tag_sources attributes, provenance
origin = provenance['tags'] == 'fixed' ? 'fixed' : 'mapped'
{ 'automatic' => '', 'mapped' => origin == 'mapped' ? attributes['tags'].to_s : '',
'fixed' => origin == 'fixed' ? attributes['tags'].to_s : '', 'manual' => '' }
def should_fetch_metadata? fetch_metadata, source_row
case fetch_metadata
when true then true
when Integer then fetch_metadata == source_row
when false, nil then false
else raise ArgumentError, 'メタデータ取得対象が不正です.'
end
end
def merged_tags sources, origin = nil
return sources['manual'].to_s if origin == 'manual'
def initial_attributes row
attributes = row[:attributes]&.stringify_keys || {}
FIELDS.to_h { |field| [field, attributes[field].to_s] }
end
['automatic', 'mapped', 'fixed', 'manual'].flat_map {
sources[_1].to_s.split
}.uniq.join(' ')
def initial_provenance row
provenance = row[:provenance]&.stringify_keys || {}
FIELDS.to_h { |field| [field, provenance[field].presence || 'automatic'] }
.merge('url' => 'manual')
end
def initial_tag_sources row, attributes, provenance
sources = row[:tag_sources]&.stringify_keys || { 'automatic' => '', 'manual' => '' }
sources['automatic'] = sources['automatic'].to_s
sources['manual'] =
provenance['tags'] == 'manual' ? attributes['tags'].to_s : sources['manual'].to_s
sources
end
def clear_automatic_values! attributes, provenance, tag_sources
@@ -174,10 +124,15 @@ class PostImportPreviewer
attributes[field] = '' if provenance[field] == 'automatic'
end
tag_sources['automatic'] = ''
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
end
def metadata_for url, cache, previous_warnings
cache[url] ||= fetch_metadata(url)
end
def fetch_metadata url
return [{ }, ['URL が空です.']] if url.blank?
return [{}, ['URL が空です.']] if url.blank?
data = PostMetadataFetcher.fetch(url).stringify_keys
warnings = []
@@ -191,7 +146,29 @@ class PostImportPreviewer
"post_import_metadata_fetch_failure "\
"#{ { error: e.class.name, message: e.message }.to_json }",
)
[{ }, ['メタデータを取得できませんでした.']]
[{}, ['メタデータを取得できませんでした.']]
end
def apply_metadata! attributes, provenance, tag_sources, metadata
metadata.each do |field, value|
if field == 'tags'
next if provenance['tags'] == 'manual'
tag_sources['automatic'] = value.to_s
attributes['tags'] = merged_tags(tag_sources)
next
end
next unless provenance[field] == 'automatic' || attributes[field].blank?
attributes[field] = value
provenance[field] = 'automatic'
end
end
def merged_tags sources, origin = nil
return sources['manual'].to_s if origin == 'manual'
sources['automatic'].to_s
end
def validate_preview_tags raw, errors, warnings
@@ -201,6 +178,7 @@ class PostImportPreviewer
errors[:tags] = ['ニコニコ・タグは直接指定できません.']
return
end
parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
existing = Tag.joins(:tag_name).where(tag_names: { name: parsed }).includes(:tag_name).to_a
deprecated = existing.select(&:deprecated?).map(&:name)
+10 -10
ファイルの表示
@@ -1,5 +1,5 @@
class PostImportRowNormaliser
ORIGINS = ['automatic', 'mapped', 'fixed', 'manual'].freeze
ORIGINS = ['automatic', 'manual'].freeze
STRING_FIELDS = [
'title',
'thumbnail_base',
@@ -13,12 +13,12 @@ class PostImportRowNormaliser
def self.normalise! rows, allow_warning_fields: false
raise ArgumentError, '取込行の形式が不正です.' unless rows.is_a?(Array)
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportSourceParser::MAX_ROWS
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportUrlListParser::MAX_ROWS
normalised_rows = rows.map { normalise_row!(_1, allow_warning_fields:) }
source_rows = normalised_rows.map { _1['source_row'] }
raise ArgumentError, '元行番号が重複しています.' if source_rows.uniq.length != source_rows.length
if normalised_rows.sum { row_bytesize(_1) } > PostImportSourceParser::MAX_BYTES
if normalised_rows.sum { row_bytesize(_1) } > PostImportUrlListParser::MAX_BYTES
raise ArgumentError, '取込データが大きすぎます.'
end
@@ -42,7 +42,7 @@ class PostImportRowNormaliser
normalise_provenance!(normalised.fetch('provenance', { }))
normalise_tag_sources!(normalised['tag_sources'])
normalise_warning_values!(normalised, allow_warning_fields:)
if row_bytesize(normalised) > PostImportSourceParser::MAX_BYTES
if row_bytesize(normalised) > PostImportUrlListParser::MAX_BYTES
raise ArgumentError, '取込行が大きすぎます.'
end
@@ -82,13 +82,13 @@ class PostImportRowNormaliser
def self.normalise_url! value
raise ArgumentError, 'URL の形式が不正です.' unless value.is_a?(String)
raise ArgumentError, 'URL が長すぎます.' if value.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, 'URL が長すぎます.' if value.bytesize > PostImportUrlListParser::MAX_URL_BYTES
end
private_class_method :normalise_url!
def self.normalise_metadata_url! value
raise ArgumentError, 'metadata_url の形式が不正です.' unless value.nil? || value.is_a?(String)
if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES
raise ArgumentError, 'metadata_url が長すぎます.'
end
end
@@ -107,7 +107,7 @@ class PostImportRowNormaliser
raise ArgumentError, '取込項目の型が不正です.'
end
end
if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES
raise ArgumentError, '取込項目が大きすぎます.'
end
end
@@ -129,10 +129,10 @@ class PostImportRowNormaliser
raise ArgumentError, 'タグ由来の形式が不正です.' unless tag_sources.is_a?(Hash)
raise ArgumentError, 'タグ由来の形式が不正です.' unless (tag_sources.keys - ORIGINS).empty?
raise ArgumentError, 'タグ由来の形式が不正です.' unless tag_sources.values.all? { _1.is_a?(String) }
if tag_sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
if tag_sources.values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES }
raise ArgumentError, 'タグ由来が大きすぎます.'
end
if tag_sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES
if tag_sources.values.sum(&:bytesize) > PostImportUrlListParser::MAX_URL_BYTES
raise ArgumentError, 'タグ由来が大きすぎます.'
end
end
@@ -148,7 +148,7 @@ class PostImportRowNormaliser
unless values.is_a?(Array) && values.all? { _1.is_a?(String) }
raise ArgumentError, '警告の形式が不正です.'
end
if values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
if values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES }
raise ArgumentError, '警告が大きすぎます.'
end
end
+5 -9
ファイルの表示
@@ -21,11 +21,7 @@ class PostImportRunner
def run_row row
attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore }
preview = PostImportPreviewer.new(
parsed: { columns: [], rows: [] },
url_column: 0,
mappings: {},
existing: @existing)
preview = PostImportPreviewer.new(existing: @existing)
.preview_rows(
rows: [{
source_row: row['source_row'],
@@ -35,6 +31,10 @@ class PostImportRunner
tag_sources: row['tag_sources'],
}],
fetch_metadata: false).first
if @existing == 'skip' && preview[:existing]
return row.slice('source_row').merge(status: 'skipped')
end
if preview[:validation_errors].present?
return {
source_row: row['source_row'],
@@ -43,10 +43,6 @@ class PostImportRunner
}
end
if @existing == 'skip' && preview[:existing]
return row.slice('source_row').merge(status: 'skipped')
end
attributes['tags'] = preview[:attributes]['tags']
attributes['url'] = row['url']
post = PostCreator.new(actor: @actor, attributes:).create!
-118
ファイルの表示
@@ -1,118 +0,0 @@
require 'csv'
class PostImportSourceParser
MAX_ROWS = 100
MAX_BYTES = 1.megabyte
MAX_CELL_BYTES = 20.kilobytes
MAX_COLUMNS = 50
def initialize source:, format:, has_header: false, json_path: nil
@source = source.to_s
@format = format.to_s
@has_header = ActiveModel::Type::Boolean.new.cast(has_header)
@json_path = json_path.to_s
end
def parse
raise ArgumentError, '入力が大きすぎます.' if @source.bytesize > MAX_BYTES
@format == 'json' ? parse_json : parse_delimited
end
private
def parse_delimited
separator = @format == 'csv' ? ',' : "\t"
table = CSV.parse(@source, col_sep: separator).map { |row| row.map { _1.to_s.strip } }
raise ArgumentError, '入力が空です.' if table.empty?
headers = @has_header ? table.shift : nil
ensure_limits!(table)
width = table.map(&:length).max.to_i
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです." if width > MAX_COLUMNS
{
columns: (0...width).map { |index|
{
key: index.to_s,
label: "#{ column_name(index) }#{ headers&.[](index).presence || '列' }",
}
},
rows: table.each_with_index.map { |values, index|
{ source_row: index + (@has_header ? 2 : 1), values: }
},
}
rescue CSV::MalformedCSVError => e
raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }"
end
def parse_json
value = JSON.parse(@source, max_nesting: 20)
@json_path.split('.').reject(&:blank?).each do |part|
value =
case value
when Array
index = Integer(part, exception: false)
raise ArgumentError if index.nil? || index.negative? || index >= value.length
value[index]
when Hash
raise ArgumentError unless value.key?(part) || value.key?(part.to_sym)
value.key?(part) ? value[part] : value[part.to_sym]
else
raise ArgumentError
end
end
raise ArgumentError, 'JSON の投稿候補は配列にしてください.' unless value.is_a?(Array)
ensure_limits!(value)
unless value.all?(Hash)
invalid = value.each_with_index.filter_map { |item, index|
index + 1 unless item.is_a?(Hash)
}
raise ArgumentError, "JSON の投稿候補は object の配列にしてください: 行 #{ invalid.join(', ') }"
end
keys = value.flat_map(&:keys).map(&:to_s).uniq
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです." if keys.length > MAX_COLUMNS
{
columns: keys.each_with_index.map { |key, index|
{ key:, label: "#{ column_name(index) }#{ key }" }
},
rows: value.each_with_index.map { |item, index|
{
source_row: index + 1,
values: keys.map { |key|
json_value = item.key?(key) ? item[key] : item[key.to_sym]
json_cell(json_value)
},
}
},
}
rescue JSON::ParserError, KeyError, ArgumentError => e
raise ArgumentError, "JSON の形式が不正です: #{ e.message }"
end
def ensure_limits! rows
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです." if rows.length > MAX_ROWS
raise ArgumentError, 'セルが大きすぎます.' if rows.flatten.any? { _1.to_s.bytesize > MAX_CELL_BYTES }
end
def json_cell value
case value
when nil then ''
when String, Numeric, TrueClass, FalseClass then value.to_s
when Array, Hash then JSON.generate(value)
else raise ArgumentError, 'JSON のセル値が不正です.'
end
end
def column_name index
name = +''
loop do
name.prepend((65 + (index % 26)).chr)
index = (index / 26) - 1
break if index < 0
end
name
end
end
+23
ファイルの表示
@@ -0,0 +1,23 @@
class PostImportUrlListParser
MAX_ROWS = 100
MAX_BYTES = 1.megabyte
MAX_URL_BYTES = 20.kilobytes
def self.parse source
raw = source.to_s
raise ArgumentError, '入力が大きすぎます.' if raw.bytesize > MAX_BYTES
rows = raw.split(/\r\n|\n|\r/).each_with_index.filter_map { |line, index|
url = line.strip
next if url.blank?
raise ArgumentError, 'URL が長すぎます.' if url.bytesize > MAX_URL_BYTES
{ source_row: index + 1, url: }
}
raise ArgumentError, 'URL を入力してください.' if rows.empty?
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです." if rows.length > MAX_ROWS
rows
end
end
-1
ファイルの表示
@@ -34,7 +34,6 @@ Rails.application.routes.draw do
end
scope 'posts/import', controller: :post_imports do
post :parse
post :preview
post :validate
post '', action: :create
+48 -381
ファイルの表示
@@ -21,28 +21,17 @@ import type { User } from '@/types'
type Props = { user: User | null }
type Column = {
key: string
label: string }
type ParsedRow = {
sourceRow: number
values: string[] }
type ParsedImport = {
columns: Column[]
rows: ParsedRow[] }
type Origin = 'automatic' | 'mapped' | 'fixed' | 'manual'
type Origin = 'automatic' | 'manual'
type Phase = 'source' | 'preview' | 'result'
type RepairMode = 'all' | 'failed'
type ImportStatus = 'pending' | 'created' | 'skipped' | 'failed'
type ImportResultStatus = 'created' | 'skipped' | 'failed'
type Phase = 'source' | 'mapping' | 'preview' | 'result'
type AttributeValue = string | number
type ImportRow = {
sourceRow: number
url: string
attributes: Record<string, string>
attributes: Record<string, AttributeValue>
warnings: string[]
validationErrors: Record<string, string[]>
importErrors?: Record<string, string[]>
@@ -60,41 +49,13 @@ type ImportResultRow = {
post?: { id: number }
errors?: Record<string, string[]> }
type Mapping = { columns: string[] }
type RepairMode = 'all' | 'failed'
const MAPPING_FIELDS = [
'title',
'thumbnail_base',
'original_created_from',
'original_created_before',
'duration',
'tags',
'parent_post_ids'] as const
const RESULT_STATUSES: ImportResultStatus[] = ['created', 'skipped', 'failed']
const MERGE_FIELDS = ['url', 'attributes', 'provenance', 'tagSources'] as const
const ORIGIN_LABELS: Record<Origin, string> = {
automatic: '自動取得',
mapped: '入力データ',
fixed: '固定値',
manual: '手修正' }
const detectFormat = (source: string): 'csv' | 'tsv' =>
source.includes ('\t')
? 'tsv'
: source.includes (',')
? 'csv'
: 'tsv'
const publicGoogleSheetURL = (source: string): boolean =>
/^https:\/\/docs\.google\.com\/spreadsheets\/d\/e\/[A-Za-z0-9_-]+\/pubhtml#gid=\d+$/.test (
source.trim ())
const mergeValidatedRows = (
current: ImportRow[],
validated: ImportRow[],
@@ -118,111 +79,18 @@ const mergeValidatedRows = (
!(current.some (_1 => _1.sourceRow === row.sourceRow))))
const mergePreviewRows = (
current: ImportRow[],
previewed: ImportRow[],
): ImportRow[] => {
const currentMap = new Map (current.map (row => [row.sourceRow, row]))
const mergedRows = previewed.map (previewedRow => {
const previous = currentMap.get (previewedRow.sourceRow)
if (previous == null)
return previewedRow
if (previous.importStatus === 'created')
return previous
const merged = {
...previewedRow,
attributes: { ...previewedRow.attributes },
provenance: { ...previewedRow.provenance },
tagSources: { ...(previewedRow.tagSources ?? { }) } }
Object.entries (previous.provenance).forEach (([field, origin]) => {
if (origin !== 'manual')
return
if (field === 'url')
{
merged.url = previous.url
merged.metadataUrl = previous.metadataUrl
merged.provenance.url = 'manual'
return
}
if (field === 'tags')
{
merged.attributes.tags = previous.attributes.tags ?? ''
merged.provenance.tags = 'manual'
merged.tagSources = {
...merged.tagSources,
manual: previous.tagSources?.manual ?? previous.attributes.tags ?? '' }
return
}
merged.attributes[field] = previous.attributes[field] ?? ''
merged.provenance[field] = 'manual'
})
const changed = MERGE_FIELDS.some (field =>
JSON.stringify (previous[field]) !== JSON.stringify (merged[field]))
return {
...merged,
importStatus: changed ? 'pending' : previous.importStatus,
createdPostId: previous.createdPostId,
importErrors: changed ? undefined : previous.importErrors }
})
return mergedRows.concat (
current.filter (row =>
row.importStatus === 'created'
&& !(previewed.some (_1 => _1.sourceRow === row.sourceRow))))
}
const rowMessages = (row: ImportRow): string[] => [
...row.warnings,
...Object.values (row.validationErrors ?? { }).flat (),
...Object.values (row.importErrors ?? { }).flat ()]
const sampleURLCount = (rows: ParsedRow[], index: number): number =>
rows.filter (row => /^https?:\/\//.test (row.values[index] ?? '')).length
const renderMappingSelect = (
columns: Column[],
field: string,
value: string,
update: (field: string, value: string) => void,
) => (
<label key={field}>
{field}
<select
value={value}
onChange={ev => update (field, ev.target.value)}>
<option value=""></option>
{columns.map ((column, index) => (
<option key={column.key} value={index}>
{column.label}
</option>))}
</select>
</label>)
const PostImportPage: FC<Props> = ({ user }) => {
const editable = canEditContent (user)
const [source, setSource] = useState ('')
const [phase, setPhase] = useState<Phase> ('source')
const [format, setFormat] =
useState<'csv' | 'tsv' | 'json' | 'google_sheets'> ('tsv')
const [hasHeader, setHasHeader] = useState (false)
const [jsonPath, setJsonPath] = useState ('')
const [urlColumn, setUrlColumn] = useState ('')
const [parsed, setParsed] = useState<ParsedImport | null> (null)
const [columns, setColumns] = useState<Column[]> ([])
const [rows, setRows] = useState<ImportRow[]> ([])
const [mappings, setMappings] = useState<Record<string, Mapping>> ({ })
const [existing, setExisting] = useState<'skip' | 'error'> ('skip')
const [loading, setLoading] = useState (false)
const [repairMode, setRepairMode] = useState<RepairMode> ('all')
@@ -230,64 +98,16 @@ const PostImportPage: FC<Props> = ({ user }) => {
const validationGeneration = useRef (0)
const rowRefs = useRef<Record<number, HTMLElement | null>> ({ })
const parseSource = async () => {
setLoading (true)
try
{
const data = await apiPost<ParsedImport> ('/posts/import/parse', {
source,
format,
has_header: hasHeader,
json_path: jsonPath,
existing })
setColumns (data.columns)
setParsed (data)
setMappings ({ })
setRows ([])
setRepairMode ('all')
if (data.columns.length === 1)
{
setUrlColumn ('0')
}
else
{
const candidate = data.columns
.map ((_, index) => ({
index,
count: sampleURLCount (data.rows, index) }))
.sort ((a, b) => b.count - a.count)[0]
setUrlColumn (candidate?.count ? String (candidate.index) : '')
}
setPhase ('mapping')
}
catch
{
toast ({
title: '読込みに失敗しました',
description: '入力形式を確認してください.' })
}
finally
{
setLoading (false)
}
}
const preview = async () => {
if (!parsed || urlColumn === '')
return
setLoading (true)
try
{
const data = await apiPost<{ rows: ImportRow[] }> ('/posts/import/preview', {
parsed: { columns: parsed.columns, rows: parsed.rows },
url_column: urlColumn,
mappings,
source,
existing })
const mergedRows = mergePreviewRows (rows, data.rows)
setRows (mergedRows)
setRows (data.rows)
setRepairMode ('all')
setPhase ('preview')
await validateRows (mergedRows, -1)
}
catch (error)
{
@@ -297,7 +117,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
?? error.response?.data?.baseErrors?.[0]
: undefined
toast ({
title: '投稿プレビューに失敗しました',
title: '投稿情報の取得に失敗しました',
description: message ?? '入力を確認してください.' })
}
finally
@@ -309,6 +129,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
const validateRows = async (
nextRows: ImportRow[],
changedRow: number,
existingValue = existing,
) => {
const generation = ++validationGeneration.current
setLoading (true)
@@ -316,7 +137,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
{
const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', {
rows: nextRows.filter (row => row.importStatus !== 'created'),
existing,
existing: existingValue,
changed_row: changedRow })
if (generation === validationGeneration.current)
setRows (current => mergeValidatedRows (current, validated.rows))
@@ -337,7 +158,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
const next = {
...row,
attributes: { ...row.attributes, [field]: value },
provenance: { ...row.provenance, [field]: 'manual' },
provenance: { ...row.provenance, [field]: 'manual' as const },
importStatus: 'pending' as const,
importErrors: undefined }
const nextRows = rows.map ((currentRow, rowIndex) =>
@@ -358,7 +179,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
const next = {
...row,
url: value,
provenance: { ...row.provenance, url: 'manual' },
provenance: { ...row.provenance, url: 'manual' as const },
importStatus: 'pending' as const,
importErrors: undefined }
const nextRows = rows.map ((currentRow, rowIndex) =>
@@ -386,22 +207,26 @@ const PostImportPage: FC<Props> = ({ user }) => {
setRows (pendingRows)
try
{
const generation = ++validationGeneration.current
setLoading (true)
const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', {
rows: pendingRows.filter (row => row.importStatus !== 'created'),
existing: value,
changed_row: -1 })
if (generation === validationGeneration.current)
setRows (current => mergeValidatedRows (current, validated.rows))
await validateRows (pendingRows, -1, value)
}
finally
catch
{
if (validationGeneration.current > 0)
setLoading (false)
toast ({ title: '既存投稿の再検証に失敗しました' })
}
}
const retryRow = (sourceRow: number) =>
setRows (current => current.map (row =>
row.sourceRow === sourceRow && row.importStatus === 'failed'
? { ...row, importStatus: 'pending', importErrors: undefined }
: row))
const retryRowFromResult = (sourceRow: number) => {
retryRow (sourceRow)
setRepairMode ('failed')
setPhase ('preview')
}
const submit = async () => {
const targets = rows.filter (row =>
row.importStatus == null || row.importStatus === 'pending')
@@ -454,23 +279,6 @@ const PostImportPage: FC<Props> = ({ user }) => {
}
}
const updateMapping = (field: string, value: string) =>
setMappings (current => ({
...current,
[field]: { columns: value === '' ? [] : [value] } }))
const retryRow = (sourceRow: number) =>
setRows (current => current.map (row =>
row.sourceRow === sourceRow && row.importStatus === 'failed'
? { ...row, importStatus: 'pending', importErrors: undefined }
: row))
const retryRowFromResult = (sourceRow: number) => {
retryRow (sourceRow)
setRepairMode ('failed')
setPhase ('preview')
}
const resultRows = rows.filter (row =>
row.importStatus != null && RESULT_STATUSES.includes (row.importStatus))
const failedCount = rows.filter (row => row.importStatus === 'failed').length
@@ -508,165 +316,36 @@ const PostImportPage: FC<Props> = ({ user }) => {
? (
<Form>
<p className="text-sm text-neutral-600 dark:text-neutral-300">
URL
CSVTSVJSON
URL
</p>
{publicGoogleSheetURL (source) && (
<fieldset className="space-y-1 rounded border p-3 dark:border-neutral-700">
<legend> Google URL</legend>
<label className="mr-3">
<input
type="radio"
checked={format === 'google_sheets'}
onChange={() => setFormat ('google_sheets')}/>
</label>
<label>
<input
type="radio"
checked={format !== 'google_sheets'}
onChange={() => setFormat ('tsv')}/>
URL 稿
</label>
</fieldset>)}
<FormField label="入力形式">
{() => (
<select
value={format}
onChange={ev => setFormat (ev.target.value as typeof format)}
className={inputClass (false)}>
<option value="tsv">TSV</option>
<option value="csv">CSV</option>
<option value="json">JSON</option>
<option value="google_sheets">
Google
</option>
</select>)}
</FormField>
<FormField label="入力データ">
<FormField label="URL リスト">
{() => (
<textarea
value={source}
rows={12}
className={inputClass (false)}
onChange={ev => {
setSource (ev.target.value)
if (format !== 'json' && format !== 'google_sheets')
setFormat (detectFormat (ev.target.value))
}}/>)}
onChange={ev => setSource (ev.target.value)}/>)}
</FormField>
<input
type="file"
accept=".txt,.csv,.tsv,.json"
onChange={ev => {
const file = ev.target.files?.[0]
if (file)
{
file.text ().then (text => {
setSource (text)
setFormat (
file.name.endsWith ('.json')
? 'json'
: detectFormat (text))
})
}
}}/>
{format !== 'json' && (
<label className="flex gap-2">
<input
type="checkbox"
checked={hasHeader}
onChange={ev => setHasHeader (ev.target.checked)}/>
使
</label>)}
{format === 'json' && (
<FormField label="投稿候補の位置(例: data.posts)">
{() => (
<input
value={jsonPath}
onChange={ev => setJsonPath (ev.target.value)}
className={inputClass (false)}/>)}
</FormField>)}
{format === 'google_sheets' && (
<p className="text-sm text-neutral-600 dark:text-neutral-300">
URL 使CSV
</p>)}
<Button
type="button"
onClick={parseSource}
disabled={loading || !(source)}>
</Button>
</Form>)
: phase === 'mapping' && parsed
? (
<Form>
<PageTitle></PageTitle>
<p>URL 稿</p>
<label>
URL
稿
<select
value={urlColumn}
onChange={ev => setUrlColumn (ev.target.value)}>
<option value=""></option>
{columns.map ((column, index) => (
<option key={column.key} value={index}>
{column.label}
</option>))}
value={existing}
onChange={ev => setExisting (ev.target.value as typeof existing)}>
<option value="skip"></option>
<option value="error"></option>
</select>
</label>
{MAPPING_FIELDS.map (field =>
renderMappingSelect (columns,
field,
mappings[field]?.columns?.[0] ?? '',
updateMapping))}
<div className="overflow-x-auto">
<table className="text-sm">
<thead>
<tr>
{columns.map (column => (
<th key={column.key}>{column.label}</th>))}
</tr>
</thead>
<tbody>
{parsed.rows.slice (0, 5).map (row => (
<tr key={row.sourceRow}>
{row.values.map ((value, index) => (
<td key={index}>{value}</td>))}
</tr>))}
</tbody>
</table>
</div>
<Button
type="button"
variant="outline"
onClick={() => setPhase ('source')}>
</Button>
<Button
type="button"
onClick={preview}
disabled={loading || urlColumn === ''}>
稿
disabled={loading || !(source.trim ())}>
稿
</Button>
</Form>)
: phase === 'preview'
? (
<div className="space-y-4">
<div className="flex flex-wrap gap-3 rounded border p-3 dark:border-neutral-700">
<label>
URL
<select
value={urlColumn}
onChange={ev => setUrlColumn (ev.target.value)}>
{columns.map ((column, index) => (
<option key={column.key} value={index}>
{column.label}
</option>))}
</select>
</label>
<label>
稿
<select
@@ -676,18 +355,6 @@ const PostImportPage: FC<Props> = ({ user }) => {
<option value="error"></option>
</select>
</label>
{MAPPING_FIELDS.map (field =>
renderMappingSelect (columns,
field,
mappings[field]?.columns?.[0] ?? '',
updateMapping))}
<Button
type="button"
variant="outline"
onClick={preview}
disabled={loading}>
</Button>
</div>
{repairMode === 'failed' && (
<p className="text-sm text-neutral-600 dark:text-neutral-300">
@@ -711,9 +378,9 @@ const PostImportPage: FC<Props> = ({ user }) => {
variant="outline"
onClick={() => {
setRepairMode ('all')
setPhase ('mapping')
setPhase ('source')
}}>
URL
</Button>
<Button
type="button"
@@ -773,7 +440,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
setRepairMode ('all')
setPhase ('source')
}}>
URL
</Button>
</div>)}
</MainArea>)
@@ -814,23 +481,23 @@ const RowCard = (
onCommit={value => updateURL (index, value)}/>
<EditableValue
label="タイトル"
value={row.attributes.title ?? ''}
value={String (row.attributes.title ?? '')}
origin={row.provenance.title}
onCommit={value => update (index, 'title', value)}/>
<EditableValue
label="サムネール基底 URL"
value={row.attributes.thumbnailBase ?? ''}
value={String (row.attributes.thumbnailBase ?? '')}
origin={row.provenance.thumbnailBase}
onCommit={value => update (index, 'thumbnailBase', value)}/>
<ThumbnailPreview url={row.attributes.thumbnailBase ?? ''}/>
<ThumbnailPreview url={String (row.attributes.thumbnailBase ?? '')}/>
<EditableValue
label="作成日時 From"
value={row.attributes.originalCreatedFrom ?? ''}
value={String (row.attributes.originalCreatedFrom ?? '')}
origin={row.provenance.originalCreatedFrom}
onCommit={value => update (index, 'originalCreatedFrom', value)}/>
<EditableValue
label="作成日時 Before"
value={row.attributes.originalCreatedBefore ?? ''}
value={String (row.attributes.originalCreatedBefore ?? '')}
origin={row.provenance.originalCreatedBefore}
onCommit={value => update (index, 'originalCreatedBefore', value)}/>
<EditableValue
@@ -840,12 +507,12 @@ const RowCard = (
onCommit={value => update (index, 'duration', value)}/>
<EditableValue
label="タグ"
value={row.attributes.tags ?? ''}
value={String (row.attributes.tags ?? '')}
origin={row.provenance.tags}
onCommit={value => update (index, 'tags', value)}/>
<EditableValue
label="親投稿"
value={row.attributes.parentPostIds ?? ''}
value={String (row.attributes.parentPostIds ?? '')}
origin={row.provenance.parentPostIds}
onCommit={value => update (index, 'parentPostIds', value)}/>
{row.importStatus === 'failed' && (