コミットを比較
5 コミット
440d3d38be
...
f636d2a177
| 作成者 | SHA1 | 日付 | |
|---|---|---|---|
| f636d2a177 | |||
| 10dc776313 | |||
| a9e16735f8 | |||
| 440a6c9961 | |||
| 4535a9d260 |
@@ -79,7 +79,7 @@ class PostImportsController < ApplicationController
|
|||||||
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportSourceParser::MAX_ROWS
|
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportSourceParser::MAX_ROWS
|
||||||
|
|
||||||
rows.map do |row|
|
rows.map do |row|
|
||||||
value = ActionController::Parameters.new(row).permit(:source_row, :sourceRow, :url,
|
value = ActionController::Parameters.new(row).permit(:source_row, :sourceRow, :url, :metadata_url, :metadataUrl,
|
||||||
attributes: {}, provenance: {}, tag_sources: {},
|
attributes: {}, provenance: {}, tag_sources: {},
|
||||||
fetch_warnings: [], validation_warnings: [])
|
fetch_warnings: [], validation_warnings: [])
|
||||||
normalised = value.to_h.deep_transform_keys { _1.to_s.underscore }
|
normalised = value.to_h.deep_transform_keys { _1.to_s.underscore }
|
||||||
@@ -100,6 +100,12 @@ class PostImportsController < ApplicationController
|
|||||||
(normalised['tag_sources'].keys - ['automatic', 'mapped', 'fixed', 'manual']).present?)
|
(normalised['tag_sources'].keys - ['automatic', 'mapped', 'fixed', 'manual']).present?)
|
||||||
raise ArgumentError, 'タグ由来の形式が不正です.'
|
raise ArgumentError, 'タグ由来の形式が不正です.'
|
||||||
end
|
end
|
||||||
|
if normalised['tag_sources'].present? &&
|
||||||
|
(!normalised['tag_sources'].values.all?(String) ||
|
||||||
|
normalised['tag_sources'].values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } ||
|
||||||
|
normalised['tag_sources'].values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES)
|
||||||
|
raise ArgumentError, 'タグ由来が大きすぎます.'
|
||||||
|
end
|
||||||
|
|
||||||
normalised.symbolize_keys
|
normalised.symbolize_keys
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -25,7 +25,8 @@ class PostImportGoogleSheetsFetcher
|
|||||||
|
|
||||||
response.body
|
response.body
|
||||||
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed => e
|
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed => e
|
||||||
raise ArgumentError, "スプレッドシートを取得できませんでした.ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください: #{ e.message }"
|
Rails.logger.info("post_import_google_sheets_fetch_failure #{ { error: e.class.name, message: e.message }.to_json }")
|
||||||
|
raise ArgumentError, 'スプレッドシートを取得できませんでした.ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください.'
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|||||||
@@ -22,11 +22,20 @@ class PostImportPreviewer
|
|||||||
tag_sources = row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance)
|
tag_sources = row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance)
|
||||||
tag_sources['manual'] = attributes['tags'].to_s if provenance['tags'] == 'manual'
|
tag_sources['manual'] = attributes['tags'].to_s if provenance['tags'] == 'manual'
|
||||||
url = row[:url].presence || values[@url_column].to_s.strip
|
url = row[:url].presence || values[@url_column].to_s.strip
|
||||||
|
url_for_metadata = normalised_url(url) || url
|
||||||
|
url_changed = row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
|
||||||
|
clear_automatic_values!(attributes, provenance, tag_sources) if url_changed
|
||||||
provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped'
|
provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped'
|
||||||
attributes['url'] = url
|
attributes['url'] = url
|
||||||
fetch_warnings = Array(row[:fetch_warnings]).dup
|
fetch_warnings = Array(row[:fetch_warnings]).dup
|
||||||
should_fetch = fetch_metadata == true || fetch_metadata.to_i == row[:source_row].to_i
|
should_fetch =
|
||||||
metadata, fetch_warnings = metadata_for(url, should_fetch, metadata_cache, fetch_warnings)
|
case fetch_metadata
|
||||||
|
when true then true
|
||||||
|
when Integer then fetch_metadata == row[:source_row].to_i
|
||||||
|
when false, nil then false
|
||||||
|
else raise ArgumentError, 'メタデータ取得対象が不正です.'
|
||||||
|
end
|
||||||
|
metadata, fetch_warnings = metadata_for(url_for_metadata, should_fetch, metadata_cache, fetch_warnings)
|
||||||
metadata.each do |field, value|
|
metadata.each do |field, value|
|
||||||
if field == 'tags' && provenance[field] != 'manual'
|
if field == 'tags' && provenance[field] != 'manual'
|
||||||
tag_sources['automatic'] = value.to_s
|
tag_sources['automatic'] = value.to_s
|
||||||
@@ -49,12 +58,14 @@ class PostImportPreviewer
|
|||||||
@existing == 'error' ? errors[:url] = ['既存投稿です.'] : validation_warnings << '既存投稿のためスキップします.'
|
@existing == 'error' ? errors[:url] = ['既存投稿です.'] : validation_warnings << '既存投稿のためスキップします.'
|
||||||
end
|
end
|
||||||
validate_basic_data(attributes, errors)
|
validate_basic_data(attributes, errors)
|
||||||
validate_preview_tags(merged_tags(tag_sources), errors, validation_warnings)
|
validate_preview_tags(merged_tags(tag_sources, provenance['tags']), errors, validation_warnings)
|
||||||
validate_parents(attributes['parent_post_ids'], errors)
|
validate_parents(attributes['parent_post_ids'], errors)
|
||||||
attributes.delete('url')
|
attributes.delete('url')
|
||||||
attributes['tags'] = merged_tags(tag_sources)
|
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
|
||||||
{ source_row: row[:source_row], url: normal_url || url, attributes:, provenance:, tag_sources:,
|
{ source_row: row[:source_row], url: normal_url || url, attributes:, provenance:, tag_sources:,
|
||||||
fetch_warnings:, validation_warnings:, warnings: fetch_warnings + validation_warnings, errors:,
|
metadata_url: url_for_metadata,
|
||||||
|
fetch_warnings:, validation_warnings:, warnings: fetch_warnings + validation_warnings,
|
||||||
|
validation_errors: errors,
|
||||||
status: errors.present? ? 'error' : ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
|
status: errors.present? ? 'error' : ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
|
||||||
existing: normal_url.present? && Post.exists?(url: normal_url) }
|
existing: normal_url.present? && Post.exists?(url: normal_url) }
|
||||||
end
|
end
|
||||||
@@ -66,6 +77,12 @@ class PostImportPreviewer
|
|||||||
cache[url] ||= fetch_metadata(url)
|
cache[url] ||= fetch_metadata(url)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def normalised_url value
|
||||||
|
post = Post.new(url: value)
|
||||||
|
post.valid?
|
||||||
|
post.url if post.errors[:url].empty?
|
||||||
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def mapped_value field, values
|
def mapped_value field, values
|
||||||
@@ -97,10 +114,19 @@ class PostImportPreviewer
|
|||||||
'fixed' => origin == 'fixed' ? attributes['tags'].to_s : '', 'manual' => '' }
|
'fixed' => origin == 'fixed' ? attributes['tags'].to_s : '', 'manual' => '' }
|
||||||
end
|
end
|
||||||
|
|
||||||
def merged_tags sources
|
def merged_tags sources, origin = nil
|
||||||
|
return sources['manual'].to_s if origin == 'manual'
|
||||||
|
|
||||||
%w[automatic mapped fixed manual].flat_map { sources[_1].to_s.split }.uniq.join(' ')
|
%w[automatic mapped fixed manual].flat_map { sources[_1].to_s.split }.uniq.join(' ')
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def clear_automatic_values! attributes, provenance, tag_sources
|
||||||
|
%w[title thumbnail_base original_created_from original_created_before duration].each do |field|
|
||||||
|
attributes[field] = '' if provenance[field] == 'automatic'
|
||||||
|
end
|
||||||
|
tag_sources['automatic'] = ''
|
||||||
|
end
|
||||||
|
|
||||||
def fetch_metadata url
|
def fetch_metadata url
|
||||||
return [{ }, ['URL が空です.']] if url.blank?
|
return [{ }, ['URL が空です.']] if url.blank?
|
||||||
|
|
||||||
@@ -110,7 +136,8 @@ class PostImportPreviewer
|
|||||||
warnings << 'サムネールを取得できませんでした.' if data['thumbnail_base'].blank?
|
warnings << 'サムネールを取得できませんでした.' if data['thumbnail_base'].blank?
|
||||||
[data.compact, warnings]
|
[data.compact, warnings]
|
||||||
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e
|
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e
|
||||||
[{ }, ["メタデータを取得できませんでした: #{ e.message }"]]
|
Rails.logger.info("post_import_metadata_fetch_failure #{ { error: e.class.name, message: e.message }.to_json }")
|
||||||
|
[{ }, ['メタデータを取得できませんでした.']]
|
||||||
end
|
end
|
||||||
|
|
||||||
def validate_preview_tags raw, errors, warnings
|
def validate_preview_tags raw, errors, warnings
|
||||||
|
|||||||
@@ -18,17 +18,22 @@ class PostImportRunner
|
|||||||
private
|
private
|
||||||
|
|
||||||
def run_row row
|
def run_row row
|
||||||
|
validate_row_structure!(row)
|
||||||
attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore }
|
attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore }
|
||||||
raise ArgumentError, '取込項目が不正です.' unless (attributes.keys - ATTRIBUTE_KEYS).empty?
|
raise ArgumentError, '取込項目が不正です.' unless (attributes.keys - ATTRIBUTE_KEYS).empty?
|
||||||
raise ArgumentError, '取込項目が大きすぎます.' if attributes.values.any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
|
raise ArgumentError, '取込項目が大きすぎます.' if attributes.values.any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
|
||||||
raise ArgumentError, 'URL が長すぎます.' if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
|
raise ArgumentError, 'URL が長すぎます.' if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
|
||||||
|
validate_tag_sources!(row['tag_sources'])
|
||||||
preview = PostImportPreviewer.new(parsed: { columns: [], rows: [] }, url_column: 0,
|
preview = PostImportPreviewer.new(parsed: { columns: [], rows: [] }, url_column: 0,
|
||||||
mappings: {}, existing: @existing).preview_rows(
|
mappings: {}, existing: @existing).preview_rows(
|
||||||
rows: [{ source_row: row['source_row'], url: row['url'],
|
rows: [{ source_row: row['source_row'], url: row['url'],
|
||||||
attributes:, provenance: row['provenance'],
|
attributes:, provenance: row['provenance'],
|
||||||
tag_sources: row['tag_sources'] }],
|
tag_sources: row['tag_sources'] }],
|
||||||
fetch_metadata: false).first
|
fetch_metadata: false).first
|
||||||
return { source_row: row['source_row'], status: 'failed', errors: preview[:errors] } if preview[:errors].present?
|
if preview[:validation_errors].present?
|
||||||
|
return { source_row: row['source_row'], status: 'failed',
|
||||||
|
errors: preview[:validation_errors] }
|
||||||
|
end
|
||||||
return row.slice('source_row').merge(status: 'skipped') if @existing == 'skip' && preview[:existing]
|
return row.slice('source_row').merge(status: 'skipped') if @existing == 'skip' && preview[:existing]
|
||||||
attributes['tags'] = preview[:attributes]['tags']
|
attributes['tags'] = preview[:attributes]['tags']
|
||||||
attributes['url'] = row['url']
|
attributes['url'] = row['url']
|
||||||
@@ -46,4 +51,26 @@ class PostImportRunner
|
|||||||
Rails.logger.error("post_import_runner_failure #{ { error: e.class.name, message: e.message }.to_json }")
|
Rails.logger.error("post_import_runner_failure #{ { error: e.class.name, message: e.message }.to_json }")
|
||||||
{ source_row: row['source_row'], status: 'failed', errors: { base: ['登録中にエラーが発生しました.'] } }
|
{ source_row: row['source_row'], status: 'failed', errors: { base: ['登録中にエラーが発生しました.'] } }
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def validate_tag_sources! sources
|
||||||
|
return if sources.blank?
|
||||||
|
|
||||||
|
sources = sources.to_h
|
||||||
|
raise ArgumentError unless (sources.keys - %w[automatic mapped fixed manual]).empty?
|
||||||
|
raise ArgumentError unless sources.values.all?(String)
|
||||||
|
raise ArgumentError if sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
|
||||||
|
raise ArgumentError if sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES
|
||||||
|
end
|
||||||
|
|
||||||
|
def validate_row_structure! row
|
||||||
|
provenance = row['provenance']
|
||||||
|
raise ArgumentError unless provenance.is_a?(Hash)
|
||||||
|
allowed = ATTRIBUTE_KEYS + ['url']
|
||||||
|
raise ArgumentError unless (provenance.keys - allowed).empty?
|
||||||
|
raise ArgumentError unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) }
|
||||||
|
metadata_url = row['metadata_url']
|
||||||
|
raise ArgumentError unless metadata_url.nil? || metadata_url.is_a?(String)
|
||||||
|
raise ArgumentError if metadata_url.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
|
||||||
|
raise ArgumentError if row.to_json.bytesize > PostImportSourceParser::MAX_BYTES
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ class PostImportSourceParser
|
|||||||
MAX_ROWS = 100
|
MAX_ROWS = 100
|
||||||
MAX_BYTES = 1.megabyte
|
MAX_BYTES = 1.megabyte
|
||||||
MAX_CELL_BYTES = 20.kilobytes
|
MAX_CELL_BYTES = 20.kilobytes
|
||||||
|
MAX_COLUMNS = 50
|
||||||
|
|
||||||
def initialize source:, format:, has_header: false, json_path: nil
|
def initialize source:, format:, has_header: false, json_path: nil
|
||||||
@source = source.to_s
|
@source = source.to_s
|
||||||
@@ -28,8 +29,9 @@ class PostImportSourceParser
|
|||||||
headers = @has_header ? table.shift : nil
|
headers = @has_header ? table.shift : nil
|
||||||
ensure_limits!(table)
|
ensure_limits!(table)
|
||||||
width = table.map(&:length).max.to_i
|
width = table.map(&:length).max.to_i
|
||||||
|
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです." if width > MAX_COLUMNS
|
||||||
{ columns: (0...width).map { |index| { key: index.to_s,
|
{ columns: (0...width).map { |index| { key: index.to_s,
|
||||||
label: "#{ ('A'.ord + index).chr }:#{ headers&.[](index).presence || '列' }" } },
|
label: "#{ column_name(index) }:#{ headers&.[](index).presence || '列' }" } },
|
||||||
rows: table.each_with_index.map { |values, index| { source_row: index + (@has_header ? 2 : 1), values: } } }
|
rows: table.each_with_index.map { |values, index| { source_row: index + (@has_header ? 2 : 1), values: } } }
|
||||||
rescue CSV::MalformedCSVError => e
|
rescue CSV::MalformedCSVError => e
|
||||||
raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }"
|
raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }"
|
||||||
@@ -41,10 +43,18 @@ class PostImportSourceParser
|
|||||||
raise ArgumentError, 'JSON の投稿候補は配列にしてください.' unless value.is_a?(Array)
|
raise ArgumentError, 'JSON の投稿候補は配列にしてください.' unless value.is_a?(Array)
|
||||||
|
|
||||||
ensure_limits!(value)
|
ensure_limits!(value)
|
||||||
keys = value.grep(Hash).flat_map(&:keys).map(&:to_s).uniq
|
unless value.all?(Hash)
|
||||||
{ columns: keys.each_with_index.map { |key, index| { key:, label: "#{ ('A'.ord + index).chr }:#{ key }" } },
|
invalid = value.each_with_index.filter_map { |item, index| index + 1 unless item.is_a?(Hash) }
|
||||||
|
raise ArgumentError, "JSON の投稿候補は object の配列にしてください: 行 #{ invalid.join(', ') }"
|
||||||
|
end
|
||||||
|
keys = value.flat_map(&:keys).map(&:to_s).uniq
|
||||||
|
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです." if keys.length > MAX_COLUMNS
|
||||||
|
{ columns: keys.each_with_index.map { |key, index| { key:, label: "#{ column_name(index) }:#{ key }" } },
|
||||||
rows: value.each_with_index.map { |item, index| { source_row: index + 1,
|
rows: value.each_with_index.map { |item, index| { source_row: index + 1,
|
||||||
values: keys.map { item.is_a?(Hash) ? item[_1] || item[_1.to_sym] : nil } } } }
|
values: keys.map { |key|
|
||||||
|
value = item.key?(key) ? item[key] : item[key.to_sym]
|
||||||
|
json_cell(value)
|
||||||
|
} } } }
|
||||||
rescue JSON::ParserError, KeyError, ArgumentError => e
|
rescue JSON::ParserError, KeyError, ArgumentError => e
|
||||||
raise ArgumentError, "JSON の形式が不正です: #{ e.message }"
|
raise ArgumentError, "JSON の形式が不正です: #{ e.message }"
|
||||||
end
|
end
|
||||||
@@ -53,4 +63,23 @@ class PostImportSourceParser
|
|||||||
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです." if rows.length > MAX_ROWS
|
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです." if rows.length > MAX_ROWS
|
||||||
raise ArgumentError, 'セルが大きすぎます.' if rows.flatten.any? { _1.to_s.bytesize > MAX_CELL_BYTES }
|
raise ArgumentError, 'セルが大きすぎます.' if rows.flatten.any? { _1.to_s.bytesize > MAX_CELL_BYTES }
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def json_cell value
|
||||||
|
case value
|
||||||
|
when nil then ''
|
||||||
|
when String, Numeric, TrueClass, FalseClass then value.to_s
|
||||||
|
when Array, Hash then JSON.generate(value)
|
||||||
|
else raise ArgumentError, 'JSON のセル値が不正です.'
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def column_name index
|
||||||
|
name = +''
|
||||||
|
loop do
|
||||||
|
name.prepend((65 + (index % 26)).chr)
|
||||||
|
index = (index / 26) - 1
|
||||||
|
break if index < 0
|
||||||
|
end
|
||||||
|
name
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|||||||
@@ -37,7 +37,8 @@ class PostMetadataFetcher
|
|||||||
when /\A\d{4}-\d{2}-\d{2}T\d{2}#{ zone }\z/ then from + 1.hour
|
when /\A\d{4}-\d{2}-\d{2}T\d{2}#{ zone }\z/ then from + 1.hour
|
||||||
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}#{ zone }\z/ then from + 1.minute
|
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}#{ zone }\z/ then from + 1.minute
|
||||||
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}#{ zone }\z/ then from + 1.second
|
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}#{ zone }\z/ then from + 1.second
|
||||||
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+#{ zone }\z/ then from + 0.001
|
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.(\d+)#{ zone }\z/
|
||||||
|
from + (10**(-Regexp.last_match(1).length))
|
||||||
else return nil
|
else return nil
|
||||||
end
|
end
|
||||||
[from, before]
|
[from, before]
|
||||||
|
|||||||
@@ -26,14 +26,18 @@ type ImportRow = {
|
|||||||
url: string
|
url: string
|
||||||
attributes: Record<string, string>
|
attributes: Record<string, string>
|
||||||
warnings: string[]
|
warnings: string[]
|
||||||
errors: Record<string, string[]>
|
validationErrors: Record<string, string[]>
|
||||||
|
importErrors?: Record<string, string[]>
|
||||||
provenance: Record<string, 'automatic' | 'mapped' | 'fixed' | 'manual'>
|
provenance: Record<string, 'automatic' | 'mapped' | 'fixed' | 'manual'>
|
||||||
tagSources?: Record<'automatic' | 'mapped' | 'fixed' | 'manual', string>
|
tagSources?: Record<'automatic' | 'mapped' | 'fixed' | 'manual', string>
|
||||||
status: 'ready' | 'warning' | 'error'
|
status: 'ready' | 'warning' | 'error'
|
||||||
existing: boolean
|
existing: boolean
|
||||||
|
metadataUrl?: string
|
||||||
|
createdPostId?: number
|
||||||
importStatus?: 'pending' | 'created' | 'skipped' | 'failed' }
|
importStatus?: 'pending' | 'created' | 'skipped' | 'failed' }
|
||||||
type ParsedImport = { columns: Column[], rows: { sourceRow: number, values: string[] }[] }
|
type ParsedImport = { columns: Column[], rows: { sourceRow: number, values: string[] }[] }
|
||||||
type Phase = 'source' | 'mapping' | 'preview' | 'result'
|
type Phase = 'source' | 'mapping' | 'preview' | 'result'
|
||||||
|
type ImportResultStatus = 'created' | 'skipped' | 'failed'
|
||||||
|
|
||||||
|
|
||||||
const detectFormat = (source: string): 'csv' | 'tsv' =>
|
const detectFormat = (source: string): 'csv' | 'tsv' =>
|
||||||
@@ -54,7 +58,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
const [parsed, setParsed] = useState<ParsedImport | null> (null)
|
const [parsed, setParsed] = useState<ParsedImport | null> (null)
|
||||||
const [columns, setColumns] = useState<Column[]> ([])
|
const [columns, setColumns] = useState<Column[]> ([])
|
||||||
const [rows, setRows] = useState<ImportRow[]> ([])
|
const [rows, setRows] = useState<ImportRow[]> ([])
|
||||||
const [results, setResults] = useState<{ sourceRow: number, status: string, post?: { id: number }, errors?: Record<string, string[]> }[]> ([])
|
const [results, setResults] = useState<{ sourceRow: number, status: ImportResultStatus, post?: { id: number }, errors?: Record<string, string[]> }[]> ([])
|
||||||
const [mappings, setMappings] = useState<Record<string, { columns: string[] }>> ({ })
|
const [mappings, setMappings] = useState<Record<string, { columns: string[] }>> ({ })
|
||||||
const [existing, setExisting] = useState<'skip' | 'error'> ('skip')
|
const [existing, setExisting] = useState<'skip' | 'error'> ('skip')
|
||||||
const [loading, setLoading] = useState (false)
|
const [loading, setLoading] = useState (false)
|
||||||
@@ -73,6 +77,10 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
existing })
|
existing })
|
||||||
setColumns (data.columns)
|
setColumns (data.columns)
|
||||||
setParsed (data)
|
setParsed (data)
|
||||||
|
setMappings ({ })
|
||||||
|
setRows ([])
|
||||||
|
setResults ([])
|
||||||
|
setUrlColumn ('')
|
||||||
if (data.columns.length === 1)
|
if (data.columns.length === 1)
|
||||||
setUrlColumn ('0')
|
setUrlColumn ('0')
|
||||||
else
|
else
|
||||||
@@ -106,7 +114,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
url_column: urlColumn,
|
url_column: urlColumn,
|
||||||
mappings,
|
mappings,
|
||||||
existing })
|
existing })
|
||||||
setRows (data.rows)
|
setRows (current => mergeValidatedRows (current, data.rows))
|
||||||
setPhase ('preview')
|
setPhase ('preview')
|
||||||
}
|
}
|
||||||
catch
|
catch
|
||||||
@@ -119,11 +127,25 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const mergeValidatedRows = (current: ImportRow[], validated: ImportRow[]): ImportRow[] =>
|
||||||
|
current.map (previous => {
|
||||||
|
if (previous.importStatus === 'created')
|
||||||
|
return previous
|
||||||
|
const row = validated.find (_1 => _1.sourceRow === previous.sourceRow)
|
||||||
|
return row ? { ...row,
|
||||||
|
importStatus: previous.importStatus,
|
||||||
|
createdPostId: previous.createdPostId,
|
||||||
|
importErrors: previous.importErrors,
|
||||||
|
validationErrors: row.validationErrors ?? { } } : previous
|
||||||
|
}).concat (validated.filter (row => !current.some (_1 => _1.sourceRow === row.sourceRow)))
|
||||||
|
|
||||||
const updateAttribute = async (index: number, field: string, value: string) => {
|
const updateAttribute = async (index: number, field: string, value: string) => {
|
||||||
const row = rows[index]
|
const row = rows[index]
|
||||||
const next = { ...row,
|
const next = { ...row,
|
||||||
attributes: { ...row.attributes, [field]: value },
|
attributes: { ...row.attributes, [field]: value },
|
||||||
provenance: { ...row.provenance, [field]: 'manual' } }
|
provenance: { ...row.provenance, [field]: 'manual' },
|
||||||
|
importStatus: 'pending' as const,
|
||||||
|
importErrors: undefined }
|
||||||
const nextRows = rows.map ((currentRow, rowIndex) => rowIndex === index ? next : currentRow)
|
const nextRows = rows.map ((currentRow, rowIndex) => rowIndex === index ? next : currentRow)
|
||||||
setRows (nextRows)
|
setRows (nextRows)
|
||||||
const generation = ++validationGeneration.current
|
const generation = ++validationGeneration.current
|
||||||
@@ -131,10 +153,10 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate',
|
const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate',
|
||||||
{ rows: nextRows, existing,
|
{ rows: nextRows.filter (row => row.importStatus !== 'created'), existing,
|
||||||
changed_row: -1 })
|
changed_row: -1 })
|
||||||
if (generation === validationGeneration.current)
|
if (generation === validationGeneration.current)
|
||||||
setRows (validated.rows)
|
setRows (current => mergeValidatedRows (current, validated.rows))
|
||||||
}
|
}
|
||||||
catch
|
catch
|
||||||
{
|
{
|
||||||
@@ -149,7 +171,8 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
|
|
||||||
const updateURL = async (index: number, value: string) => {
|
const updateURL = async (index: number, value: string) => {
|
||||||
const row = rows[index]
|
const row = rows[index]
|
||||||
const next = { ...row, url: value, provenance: { ...row.provenance, url: 'manual' } }
|
const next = { ...row, url: value, provenance: { ...row.provenance, url: 'manual' },
|
||||||
|
importStatus: 'pending' as const, importErrors: undefined }
|
||||||
const nextRows = rows.map ((currentRow, rowIndex) => rowIndex === index ? next : currentRow)
|
const nextRows = rows.map ((currentRow, rowIndex) => rowIndex === index ? next : currentRow)
|
||||||
setRows (nextRows)
|
setRows (nextRows)
|
||||||
const generation = ++validationGeneration.current
|
const generation = ++validationGeneration.current
|
||||||
@@ -157,10 +180,10 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate',
|
const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate',
|
||||||
{ rows: nextRows, existing,
|
{ rows: nextRows.filter (row => row.importStatus !== 'created'), existing,
|
||||||
changed_row: row.sourceRow })
|
changed_row: row.sourceRow })
|
||||||
if (generation === validationGeneration.current)
|
if (generation === validationGeneration.current)
|
||||||
setRows (validated.rows)
|
setRows (current => mergeValidatedRows (current, validated.rows))
|
||||||
}
|
}
|
||||||
catch
|
catch
|
||||||
{
|
{
|
||||||
@@ -173,6 +196,27 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const updateExisting = async (value: 'skip' | 'error') => {
|
||||||
|
setExisting (value)
|
||||||
|
const pendingRows = rows.map (row => row.importStatus === 'created' ? row :
|
||||||
|
{ ...row, importStatus: 'pending' as const, importErrors: undefined })
|
||||||
|
setRows (pendingRows)
|
||||||
|
const generation = ++validationGeneration.current
|
||||||
|
setLoading (true)
|
||||||
|
try
|
||||||
|
{
|
||||||
|
const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate',
|
||||||
|
{ rows: pendingRows.filter (row => row.importStatus !== 'created'), existing: value, changed_row: -1 })
|
||||||
|
if (generation === validationGeneration.current)
|
||||||
|
setRows (current => mergeValidatedRows (current, validated.rows))
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (generation === validationGeneration.current)
|
||||||
|
setLoading (false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const submit = async () => {
|
const submit = async () => {
|
||||||
const targets = rows.filter (row => (row as ImportRow & { importStatus?: string }).importStatus !== 'created')
|
const targets = rows.filter (row => (row as ImportRow & { importStatus?: string }).importStatus !== 'created')
|
||||||
if (targets.length === 0)
|
if (targets.length === 0)
|
||||||
@@ -181,14 +225,19 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
const result = await apiPost<{ created: number, skipped: number, failed: number,
|
const result = await apiPost<{ created: number, skipped: number, failed: number,
|
||||||
rows: { sourceRow: number, status: string, post?: { id: number }, errors?: Record<string, string[]> }[] }> ('/posts/import', { rows: targets, existing })
|
rows: { sourceRow: number, status: ImportResultStatus, post?: { id: number }, errors?: Record<string, string[]> }[] }> ('/posts/import', { rows: targets, existing })
|
||||||
|
if (result.rows.some (row => !(['created', 'skipped', 'failed'] as string[]).includes (row.status)))
|
||||||
|
throw new Error ('不正な登録結果です。')
|
||||||
toast ({ title: `登録完了: ${result.created} 件`,
|
toast ({ title: `登録完了: ${result.created} 件`,
|
||||||
description: `スキップ ${result.skipped} 件、失敗 ${result.failed} 件` })
|
description: `スキップ ${result.skipped} 件、失敗 ${result.failed} 件` })
|
||||||
setPhase ('result')
|
setPhase ('result')
|
||||||
setResults (result.rows)
|
setResults (result.rows)
|
||||||
setRows (current => current.map (row => {
|
setRows (current => current.map (row => {
|
||||||
const resultRow = result.rows.find (_1 => _1.sourceRow === row.sourceRow)
|
const resultRow = result.rows.find (_1 => _1.sourceRow === row.sourceRow)
|
||||||
return resultRow ? { ...row, importStatus: resultRow.status } : row
|
return resultRow ? { ...row, importStatus: resultRow.status,
|
||||||
|
createdPostId: resultRow.post?.id,
|
||||||
|
importErrors: resultRow.errors,
|
||||||
|
validationErrors: row.validationErrors } : row
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
catch
|
catch
|
||||||
@@ -269,7 +318,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
<label>URL 列 <select value={urlColumn} onChange={ev => setUrlColumn (ev.target.value)}>
|
<label>URL 列 <select value={urlColumn} onChange={ev => setUrlColumn (ev.target.value)}>
|
||||||
{columns.map ((column, index) => <option key={column.key} value={index}>{column.label}</option>)}
|
{columns.map ((column, index) => <option key={column.key} value={index}>{column.label}</option>)}
|
||||||
</select></label>
|
</select></label>
|
||||||
<label>既存投稿 <select value={existing} onChange={ev => setExisting (ev.target.value as typeof existing)}>
|
<label>既存投稿 <select value={existing} onChange={ev => updateExisting (ev.target.value as typeof existing)}>
|
||||||
<option value="skip">スキップ</option><option value="error">エラー</option>
|
<option value="skip">スキップ</option><option value="error">エラー</option>
|
||||||
</select></label>
|
</select></label>
|
||||||
{['title', 'thumbnail_base', 'original_created_from', 'original_created_before', 'duration', 'tags', 'parent_post_ids'].map (field =>
|
{['title', 'thumbnail_base', 'original_created_from', 'original_created_before', 'duration', 'tags', 'parent_post_ids'].map (field =>
|
||||||
@@ -286,6 +335,7 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
<div className="space-y-3"><p>インポート処理が完了しました。</p>
|
<div className="space-y-3"><p>インポート処理が完了しました。</p>
|
||||||
{results.map (result => <div key={result.sourceRow} className="rounded border p-2 dark:border-neutral-700">行 {result.sourceRow}:{result.status} {result.post && <PrefetchLink to={`/posts/${result.post.id}`}>投稿を開く</PrefetchLink>}<FieldError messages={Object.values (result.errors ?? { }).flat ()}/></div>)}
|
{results.map (result => <div key={result.sourceRow} className="rounded border p-2 dark:border-neutral-700">行 {result.sourceRow}:{result.status} {result.post && <PrefetchLink to={`/posts/${result.post.id}`}>投稿を開く</PrefetchLink>}<FieldError messages={Object.values (result.errors ?? { }).flat ()}/></div>)}
|
||||||
<Button type="button" onClick={() => setPhase ('preview')}>結果を確認</Button>
|
<Button type="button" onClick={() => setPhase ('preview')}>結果を確認</Button>
|
||||||
|
<Button type="button" variant="outline" onClick={() => setPhase ('preview')}>失敗行を修正</Button>
|
||||||
<Button type="button" variant="outline" onClick={() => setPhase ('source')}>新しい入力元を選ぶ</Button>
|
<Button type="button" variant="outline" onClick={() => setPhase ('source')}>新しい入力元を選ぶ</Button>
|
||||||
</div>)}
|
</div>)}
|
||||||
</MainArea>)
|
</MainArea>)
|
||||||
@@ -295,16 +345,20 @@ const PostImportPage: FC<Props> = ({ user }) => {
|
|||||||
const RowCard = ({ row, index, update, updateURL }: { row: ImportRow, index: number, update: (index: number, field: string, value: string) => void, updateURL: (index: number, value: string) => void }) => (
|
const RowCard = ({ row, index, update, updateURL }: { row: ImportRow, index: number, update: (index: number, field: string, value: string) => void, updateURL: (index: number, value: string) => void }) => (
|
||||||
<article className="space-y-2 rounded border bg-white p-3 text-neutral-900 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-100">
|
<article className="space-y-2 rounded border bg-white p-3 text-neutral-900 dark:border-neutral-700 dark:bg-neutral-900 dark:text-neutral-100">
|
||||||
<p>行 {row.sourceRow}・{row.status}</p>
|
<p>行 {row.sourceRow}・{row.status}</p>
|
||||||
|
{row.importStatus === 'created' ? <p className="rounded bg-green-100 p-2 dark:bg-green-900">登録済み。この行は編集・再登録できません。{row.createdPostId && <PrefetchLink to={`/posts/${row.createdPostId}`}>投稿を開く</PrefetchLink>}</p> : <>
|
||||||
<EditableValue label="URL" value={row.url} origin={row.provenance.url} onCommit={value => updateURL (index, value)}/>
|
<EditableValue label="URL" value={row.url} origin={row.provenance.url} onCommit={value => updateURL (index, value)}/>
|
||||||
<EditableValue label="タイトル" value={row.attributes.title ?? ''} origin={row.provenance.title} onCommit={value => update (index, 'title', value)}/>
|
<EditableValue label="タイトル" value={row.attributes.title ?? ''} origin={row.provenance.title} onCommit={value => update (index, 'title', value)}/>
|
||||||
<EditableValue label="サムネール基底 URL" value={row.attributes.thumbnailBase ?? ''} origin={row.provenance.thumbnailBase} onCommit={value => update (index, 'thumbnailBase', value)}/>
|
<EditableValue label="サムネール基底 URL" value={row.attributes.thumbnailBase ?? ''} origin={row.provenance.thumbnailBase} onCommit={value => update (index, 'thumbnailBase', value)}/>
|
||||||
{row.attributes.thumbnailBase && <img src={row.attributes.thumbnailBase} alt="サムネール" className="max-h-32" onError={ev => { ev.currentTarget.hidden = true }}/>}
|
<ThumbnailPreview url={row.attributes.thumbnailBase ?? ''}/>
|
||||||
<EditableValue label="作成日時 From" value={row.attributes.originalCreatedFrom ?? ''} origin={row.provenance.originalCreatedFrom} onCommit={value => update (index, 'originalCreatedFrom', value)}/>
|
<EditableValue label="作成日時 From" value={row.attributes.originalCreatedFrom ?? ''} origin={row.provenance.originalCreatedFrom} onCommit={value => update (index, 'originalCreatedFrom', value)}/>
|
||||||
<EditableValue label="作成日時 Before" value={row.attributes.originalCreatedBefore ?? ''} origin={row.provenance.originalCreatedBefore} onCommit={value => update (index, 'originalCreatedBefore', value)}/>
|
<EditableValue label="作成日時 Before" value={row.attributes.originalCreatedBefore ?? ''} origin={row.provenance.originalCreatedBefore} onCommit={value => update (index, 'originalCreatedBefore', value)}/>
|
||||||
<EditableValue label="動画時間" value={String (row.attributes.duration ?? '')} origin={row.provenance.duration} onCommit={value => update (index, 'duration', value)}/>
|
<EditableValue label="動画時間" value={String (row.attributes.duration ?? '')} origin={row.provenance.duration} onCommit={value => update (index, 'duration', value)}/>
|
||||||
<EditableValue label="タグ" value={row.attributes.tags ?? ''} origin={row.provenance.tags} onCommit={value => update (index, 'tags', value)}/>
|
<EditableValue label="タグ" value={row.attributes.tags ?? ''} origin={row.provenance.tags} onCommit={value => update (index, 'tags', value)}/>
|
||||||
<EditableValue label="親投稿" value={row.attributes.parentPostIds ?? ''} origin={row.provenance.parentPostIds} onCommit={value => update (index, 'parentPostIds', value)}/>
|
<EditableValue label="親投稿" value={row.attributes.parentPostIds ?? ''} origin={row.provenance.parentPostIds} onCommit={value => update (index, 'parentPostIds', value)}/>
|
||||||
<FieldError messages={[...row.warnings, ...Object.values (row.errors).flat ()]}/>
|
</>}
|
||||||
|
<FieldError messages={[...row.warnings,
|
||||||
|
...Object.values (row.validationErrors ?? { }).flat (),
|
||||||
|
...Object.values (row.importErrors ?? { }).flat ()]}/>
|
||||||
</article>)
|
</article>)
|
||||||
|
|
||||||
const EditableValue = ({ label, value, origin, onCommit }: { label: string, value: string, origin: string | undefined, onCommit: (value: string) => void }) => {
|
const EditableValue = ({ label, value, origin, onCommit }: { label: string, value: string, origin: string | undefined, onCommit: (value: string) => void }) => {
|
||||||
@@ -324,4 +378,14 @@ const EditableValue = ({ label, value, origin, onCommit }: { label: string, valu
|
|||||||
</div>
|
</div>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ThumbnailPreview = ({ url }: { url: string }) => {
|
||||||
|
const [failed, setFailed] = useState (false)
|
||||||
|
useEffect (() => setFailed (false), [url])
|
||||||
|
if (!url)
|
||||||
|
return <p className="text-sm">サムネール未指定</p>
|
||||||
|
if (failed)
|
||||||
|
return <p className="text-sm text-red-600 dark:text-red-300">サムネールを表示できません</p>
|
||||||
|
return <img src={url} alt="サムネール" className="max-h-32" onError={() => setFailed (true)}/>
|
||||||
|
}
|
||||||
|
|
||||||
export default PostImportPage
|
export default PostImportPage
|
||||||
|
|||||||
新しい課題から参照
ユーザをブロックする