このコミットが含まれているのは:
2026-07-12 13:14:36 +09:00
コミット df17f20907
9個のファイルの変更304行の追加942行の削除
+6 -122
ファイルの表示
@@ -1,44 +1,10 @@
class PostImportsController < ApplicationController
MAPPING_PROPERTIES = ['kind', 'value', 'columns'].freeze
MAPPING_KINDS = ['columns', 'fixed'].freeze
before_action :require_member!
def parse
source =
if params[:format] == 'google_sheets'
PostImportGoogleSheetsFetcher.fetch!(params[:source],
rate_key: current_user.id)
else
params[:source]
end
format = params[:format] == 'google_sheets' ? 'csv' : params[:format]
parsed = PostImportSourceParser.new(
source:,
format:,
has_header: params[:has_header],
json_path: params[:json_path]).parse
render json: parsed
rescue ArgumentError => e
render_bad_request(e.message)
end
def preview
parsed = parsed_params
raw_url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil)
return render_bad_request('URL 列を選択してください.') if raw_url_column.nil?
url_column = Integer(raw_url_column, exception: false)
if url_column.nil? || url_column.negative? || url_column >= parsed[:columns].length
return render_bad_request('URL 列が不正です.')
end
rows = PostImportPreviewer.new(
parsed:,
url_column:,
mappings: mappings_params(parsed[:columns].length),
existing: params[:existing]).preview
render json: parsed.slice(:columns).merge(rows:)
rows = PostImportPreviewer.new(existing: params[:existing]).preview_rows(
rows: PostImportUrlListParser.parse(params[:source]))
render json: { rows: }
rescue ArgumentError => e
render_bad_request(e.message)
end
@@ -47,12 +13,8 @@ class PostImportsController < ApplicationController
rows = normalised_import_rows(allow_warning_fields: true)
changed_row = Integer(params[:changed_row], exception: false)
result =
PostImportPreviewer.new(
parsed: { columns: [], rows: [] },
url_column: 0,
mappings: {},
existing: params[:existing])
.preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {})
PostImportPreviewer.new(existing: params[:existing])
.preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {})
render json: { rows: result }
rescue ArgumentError => e
render_bad_request(e.message)
@@ -77,85 +39,7 @@ class PostImportsController < ApplicationController
head :forbidden
end
def parsed_params
value = params.require(:parsed).to_unsafe_h.deep_transform_keys { _1.to_s.underscore }
columns = Array(value['columns'])
rows = Array(value['rows'])
unless columns.all?(Hash) && rows.all?(Hash)
raise ArgumentError, '解析結果の形式が不正です.'
end
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportSourceParser::MAX_ROWS
raise ArgumentError, '列数が多すぎます.' if columns.length > 50
raise ArgumentError, '解析結果のセルが不正です.' unless rows.all? { |row|
values = Array(row['values'])
values.length <= columns.length && values.all? { |value|
value.is_a?(String) && value.bytesize <= PostImportSourceParser::MAX_CELL_BYTES
}
}
if rows.sum { Array(_1['values']).sum(&:bytesize) } > PostImportSourceParser::MAX_BYTES
raise ArgumentError, '解析結果が大きすぎます.'
end
parsed_rows = rows.map do |row|
source_row = Integer(row['source_row'], exception: false)
raise ArgumentError, '元行番号が不正です.' if source_row.nil? || source_row <= 0
{ source_row:, values: Array(row['values']) }
end
if parsed_rows.map { _1[:source_row] }.uniq.length != parsed_rows.length
raise ArgumentError, '元行番号が重複しています.'
end
{ columns: columns.map { |column| column.slice('key', 'label') }, rows: parsed_rows }
end
def mappings_params column_count
mappings = params[:mappings]
return { } if mappings.blank?
raise ArgumentError, '列割当ての形式が不正です.' unless mappings.is_a?(ActionController::Parameters)
raw = mappings.to_unsafe_h
unknown_fields = raw.keys - PostImportPreviewer::FIELDS
raise ArgumentError, '列割当て先が不正です.' if unknown_fields.present?
raise ArgumentError, '列割当てが大きすぎます.' if raw.to_json.bytesize > PostImportSourceParser::MAX_BYTES
permitted =
mappings.permit(*PostImportPreviewer::FIELDS.map { |field|
{ field => [:kind, :value, { columns: [] }] }
}).to_h
raw.each_with_object({ }) do |(field, value), result|
raise ArgumentError, '列割当ての形式が不正です.' unless value.is_a?(Hash)
unless (value.keys - MAPPING_PROPERTIES).empty?
raise ArgumentError, '列割当ての項目が不正です.'
end
value = permitted.fetch(field)
kind = value['kind'].presence || 'columns'
raise ArgumentError, '列割当ての種別が不正です.' unless kind.in?(MAPPING_KINDS)
columns = value['columns']
raise ArgumentError, '列番号が不正です.' unless (
columns.nil? || (columns.is_a?(Array) && columns.all? { |index|
parsed = Integer(index, exception: false)
parsed && parsed >= 0 && parsed < column_count
})
)
parsed_columns = Array(columns).map { Integer(_1) }
raise ArgumentError, '列番号が重複しています.' if parsed_columns.uniq.length != parsed_columns.length
raise ArgumentError, '固定値が不正です.' if kind == 'fixed' && !value['value'].is_a?(String)
if value['value'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, '固定値が大きすぎます.'
end
result[field] = {
'kind' => kind,
'value' => value['value'].to_s,
'columns' => parsed_columns }
end
end
def normalised_import_rows allow_warning_fields: false
rows = params[:rows]
PostImportRowNormaliser.normalise!(rows, allow_warning_fields:)
PostImportRowNormaliser.normalise!(params[:rows], allow_warning_fields:)
end
end
-67
ファイルの表示
@@ -1,67 +0,0 @@
class PostImportGoogleSheetsFetcher
HOST = 'docs.google.com'.freeze
PUBLICATION_ID = /\A[A-Za-z0-9_-]+\z/
GID = /\A\d+\z/
RATE_LIMIT = 10
RATE_WINDOW = 1.minute
def self.fetch! raw_url, rate_key:
new(raw_url, rate_key:).fetch!
end
def initialize raw_url, rate_key:
@raw_url = raw_url.to_s
@rate_key = rate_key.to_s
end
def fetch!
publication_id, gid = parse!
throttle!
csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub" \
"?gid=#{ gid }&single=true&output=csv"
response = Preview::HttpFetcher.fetch(csv_url,
max_bytes: PostImportSourceParser::MAX_BYTES,
allowed_hosts: [HOST])
raise ArgumentError, 'スプレッドシートを CSV として取得できませんでした.' unless csv?(response)
response.body
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed,
Preview::HttpFetcher::ResponseTooLarge => e
Rails.logger.info(
'post_import_google_sheets_fetch_failure '\
"#{ { error: e.class.name, message: e.message }.to_json }",
)
raise ArgumentError, 'スプレッドシートを取得できませんでした.ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください.'
end
private
def parse!
uri = URI.parse(@raw_url)
unless uri.scheme == 'https' && uri.host&.downcase == HOST && uri.query.blank?
raise ArgumentError, '公開 Google スプレッドシート URL を指定してください.'
end
match = uri.path.match(%r{\A/spreadsheets/d/e/([^/]+)/pubhtml\z})
gid = URI.decode_www_form(uri.fragment.to_s).to_h['gid']
unless match && PUBLICATION_ID.match?(match[1]) && GID.match?(gid.to_s)
raise ArgumentError, '公開 Google スプレッドシート URL の publication_id 又は gid が不正です.'
end
[match[1], gid]
rescue URI::InvalidURIError
raise ArgumentError, '公開 Google スプレッドシート URL が不正です.'
end
def throttle!
key = "post-import-google-sheets:#{ @rate_key }"
count = Rails.cache.increment(key, 1, expires_in: RATE_WINDOW, initial: 0)
if count > RATE_LIMIT
raise ArgumentError, 'スプレッドシート取得の回数が多すぎます.しばらくしてから再試行してください.'
end
end
def csv? response
type = response.content_type.to_s.downcase
type.include?('text/csv') || type.include?('application/csv')
end
end
+118 -140
ファイルの表示
@@ -8,119 +8,15 @@ class PostImportPreviewer
'tags',
'parent_post_ids',
].freeze
def initialize parsed:, url_column:, mappings: { }, existing: 'skip'
@parsed = parsed
@url_column = url_column.to_i
@mappings = mappings.stringify_keys
def initialize existing: 'skip'
@existing = existing == 'error' ? 'error' : 'skip'
end
def preview
preview_rows(rows: @parsed[:rows])
end
def preview_rows rows:, fetch_metadata: true, metadata_cache: { }
urls = { }
rows.map do |row|
row = row.symbolize_keys
values = row[:values] || []
attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field|
[field, mapped_value(field, values)]
}
provenance = row[:provenance]&.stringify_keys || mapped_provenance(attributes)
tag_sources =
row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance)
tag_sources['manual'] = attributes['tags'].to_s if provenance['tags'] == 'manual'
url = row[:url].presence || values[@url_column].to_s.strip
provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped'
url_for_metadata = normalised_url(url) || url
existing_post = normalised_url(url).present? && Post.exists?(url: normalised_url(url))
if existing_post && @existing == 'skip'
next {
source_row: row[:source_row],
url: normalised_url(url) || url,
attributes:,
provenance:,
tag_sources:,
metadata_url: url_for_metadata,
fetch_warnings: [],
validation_warnings: ['既存投稿のためスキップします.'],
warnings: ['既存投稿のためスキップします.'],
validation_errors: { },
status: 'warning',
existing: true,
}
end
url_changed = row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
clear_automatic_values!(attributes, provenance, tag_sources) if url_changed
attributes['url'] = url
fetch_warnings = Array(row[:fetch_warnings]).dup
should_fetch =
case fetch_metadata
when true then true
when Integer then fetch_metadata == row[:source_row].to_i
when false, nil then false
else raise ArgumentError, 'メタデータ取得対象が不正です.'
end
metadata, fetch_warnings =
metadata_for(url_for_metadata, should_fetch, metadata_cache, fetch_warnings)
metadata.each do |field, value|
if field == 'tags' && provenance[field] != 'manual'
tag_sources['automatic'] = value.to_s
attributes[field] = merged_tags(tag_sources)
next
end
next unless provenance[field] == 'automatic' || attributes[field].blank?
attributes[field] = value
provenance[field] = 'automatic'
end
normal_url = normalised_url(url)
errors = { }
errors[:url] = ['URL が不正です.'] if normal_url.blank?
validation_warnings = []
errors[:url] = ['URL が重複しています.'] if normal_url.present? && urls.key?(normal_url)
urls[normal_url] = true if normal_url.present?
if normal_url.present? && existing_post
if @existing == 'error'
errors[:url] = ['既存投稿です.']
else
validation_warnings << '既存投稿のためスキップします.'
end
end
validate_basic_data(attributes, errors)
validate_preview_tags(
merged_tags(tag_sources, provenance['tags']),
errors,
validation_warnings,
)
validate_parents(attributes['parent_post_ids'], errors)
attributes.delete('url')
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
{
source_row: row[:source_row],
url: normal_url || url,
attributes:,
provenance:,
tag_sources:,
metadata_url: url_for_metadata,
fetch_warnings:,
validation_warnings:,
warnings: fetch_warnings + validation_warnings,
validation_errors: errors,
status: errors.present? \
? 'error' \
: ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
existing: normal_url.present? && Post.exists?(url: normal_url),
}
end
end
def metadata_for url, fetch_metadata, cache, previous_warnings
return [{ }, previous_warnings] unless fetch_metadata
cache[url] ||= fetch_metadata(url)
urls = {}
rows.map { |row|
preview_row(row, urls:, fetch_metadata:, metadata_cache:)
}
end
def normalised_url value
@@ -129,43 +25,97 @@ class PostImportPreviewer
private
def mapped_value field, values
mapping = @mappings[field]
return '' if mapping.blank?
return mapping['value'].to_s if mapping['kind'] == 'fixed'
def preview_row row, urls:, fetch_metadata:, metadata_cache:
row = row.symbolize_keys
attributes = initial_attributes(row)
provenance = initial_provenance(row)
tag_sources = initial_tag_sources(row, attributes, provenance)
url = row[:url].to_s.strip
provenance['url'] = 'manual'
normal_url = normalised_url(url)
url_for_metadata = normal_url || url
existing_post = normal_url.present? && Post.exists?(url: normal_url)
Array(mapping['columns']).filter_map { |index|
values[index.to_i].to_s.presence
}.join(field == 'tags' ? ' ' : '')
end
validation_warnings = []
validation_errors = {}
validation_errors[:url] = ['URL が不正です.'] if normal_url.blank?
validation_errors[:url] = ['URL が重複しています.'] if normal_url.present? && urls.key?(normal_url)
urls[normal_url] = true if normal_url.present?
if normal_url.present? && existing_post
if @existing == 'error'
validation_errors[:url] = ['既存投稿です.']
else
validation_warnings << '既存投稿のためスキップします.'
end
end
def mapped_provenance attributes
attributes.to_h.to_h { |field, value|
mapping = @mappings[field.to_s]
origin =
if value.present? && mapping&.[]('kind') == 'fixed'
'fixed'
elsif value.present?
'mapped'
else
'automatic'
end
[field.to_s, origin]
if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
clear_automatic_values!(attributes, provenance, tag_sources)
end
fetch_warnings = Array(row[:fetch_warnings]).dup
should_fetch = should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
if should_fetch && !(existing_post && @existing == 'skip')
metadata, fetch_warnings =
metadata_for(url_for_metadata, metadata_cache, fetch_warnings)
apply_metadata!(attributes, provenance, tag_sources, metadata)
end
attributes['url'] = url
validate_basic_data(attributes, validation_errors)
validate_preview_tags(
merged_tags(tag_sources, provenance['tags']),
validation_errors,
validation_warnings,
)
validate_parents(attributes['parent_post_ids'], validation_errors)
attributes.delete('url')
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
{
source_row: row[:source_row],
url: normal_url || url,
attributes:,
provenance:,
tag_sources:,
metadata_url: url_for_metadata,
fetch_warnings:,
validation_warnings:,
warnings: fetch_warnings + validation_warnings,
validation_errors:,
status: validation_errors.present? \
? 'error' \
: ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
existing: existing_post,
}
end
def initial_tag_sources attributes, provenance
origin = provenance['tags'] == 'fixed' ? 'fixed' : 'mapped'
{ 'automatic' => '', 'mapped' => origin == 'mapped' ? attributes['tags'].to_s : '',
'fixed' => origin == 'fixed' ? attributes['tags'].to_s : '', 'manual' => '' }
def should_fetch_metadata? fetch_metadata, source_row
case fetch_metadata
when true then true
when Integer then fetch_metadata == source_row
when false, nil then false
else raise ArgumentError, 'メタデータ取得対象が不正です.'
end
end
def merged_tags sources, origin = nil
return sources['manual'].to_s if origin == 'manual'
def initial_attributes row
attributes = row[:attributes]&.stringify_keys || {}
FIELDS.to_h { |field| [field, attributes[field].to_s] }
end
['automatic', 'mapped', 'fixed', 'manual'].flat_map {
sources[_1].to_s.split
}.uniq.join(' ')
def initial_provenance row
provenance = row[:provenance]&.stringify_keys || {}
FIELDS.to_h { |field| [field, provenance[field].presence || 'automatic'] }
.merge('url' => 'manual')
end
def initial_tag_sources row, attributes, provenance
sources = row[:tag_sources]&.stringify_keys || { 'automatic' => '', 'manual' => '' }
sources['automatic'] = sources['automatic'].to_s
sources['manual'] =
provenance['tags'] == 'manual' ? attributes['tags'].to_s : sources['manual'].to_s
sources
end
def clear_automatic_values! attributes, provenance, tag_sources
@@ -174,10 +124,15 @@ class PostImportPreviewer
attributes[field] = '' if provenance[field] == 'automatic'
end
tag_sources['automatic'] = ''
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
end
def metadata_for url, cache, previous_warnings
cache[url] ||= fetch_metadata(url)
end
def fetch_metadata url
return [{ }, ['URL が空です.']] if url.blank?
return [{}, ['URL が空です.']] if url.blank?
data = PostMetadataFetcher.fetch(url).stringify_keys
warnings = []
@@ -191,7 +146,29 @@ class PostImportPreviewer
"post_import_metadata_fetch_failure "\
"#{ { error: e.class.name, message: e.message }.to_json }",
)
[{ }, ['メタデータを取得できませんでした.']]
[{}, ['メタデータを取得できませんでした.']]
end
def apply_metadata! attributes, provenance, tag_sources, metadata
metadata.each do |field, value|
if field == 'tags'
next if provenance['tags'] == 'manual'
tag_sources['automatic'] = value.to_s
attributes['tags'] = merged_tags(tag_sources)
next
end
next unless provenance[field] == 'automatic' || attributes[field].blank?
attributes[field] = value
provenance[field] = 'automatic'
end
end
def merged_tags sources, origin = nil
return sources['manual'].to_s if origin == 'manual'
sources['automatic'].to_s
end
def validate_preview_tags raw, errors, warnings
@@ -201,6 +178,7 @@ class PostImportPreviewer
errors[:tags] = ['ニコニコ・タグは直接指定できません.']
return
end
parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
existing = Tag.joins(:tag_name).where(tag_names: { name: parsed }).includes(:tag_name).to_a
deprecated = existing.select(&:deprecated?).map(&:name)
+10 -10
ファイルの表示
@@ -1,5 +1,5 @@
class PostImportRowNormaliser
ORIGINS = ['automatic', 'mapped', 'fixed', 'manual'].freeze
ORIGINS = ['automatic', 'manual'].freeze
STRING_FIELDS = [
'title',
'thumbnail_base',
@@ -13,12 +13,12 @@ class PostImportRowNormaliser
def self.normalise! rows, allow_warning_fields: false
raise ArgumentError, '取込行の形式が不正です.' unless rows.is_a?(Array)
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportSourceParser::MAX_ROWS
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportUrlListParser::MAX_ROWS
normalised_rows = rows.map { normalise_row!(_1, allow_warning_fields:) }
source_rows = normalised_rows.map { _1['source_row'] }
raise ArgumentError, '元行番号が重複しています.' if source_rows.uniq.length != source_rows.length
if normalised_rows.sum { row_bytesize(_1) } > PostImportSourceParser::MAX_BYTES
if normalised_rows.sum { row_bytesize(_1) } > PostImportUrlListParser::MAX_BYTES
raise ArgumentError, '取込データが大きすぎます.'
end
@@ -42,7 +42,7 @@ class PostImportRowNormaliser
normalise_provenance!(normalised.fetch('provenance', { }))
normalise_tag_sources!(normalised['tag_sources'])
normalise_warning_values!(normalised, allow_warning_fields:)
if row_bytesize(normalised) > PostImportSourceParser::MAX_BYTES
if row_bytesize(normalised) > PostImportUrlListParser::MAX_BYTES
raise ArgumentError, '取込行が大きすぎます.'
end
@@ -82,13 +82,13 @@ class PostImportRowNormaliser
def self.normalise_url! value
raise ArgumentError, 'URL の形式が不正です.' unless value.is_a?(String)
raise ArgumentError, 'URL が長すぎます.' if value.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, 'URL が長すぎます.' if value.bytesize > PostImportUrlListParser::MAX_URL_BYTES
end
private_class_method :normalise_url!
def self.normalise_metadata_url! value
raise ArgumentError, 'metadata_url の形式が不正です.' unless value.nil? || value.is_a?(String)
if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES
raise ArgumentError, 'metadata_url が長すぎます.'
end
end
@@ -107,7 +107,7 @@ class PostImportRowNormaliser
raise ArgumentError, '取込項目の型が不正です.'
end
end
if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES
raise ArgumentError, '取込項目が大きすぎます.'
end
end
@@ -129,10 +129,10 @@ class PostImportRowNormaliser
raise ArgumentError, 'タグ由来の形式が不正です.' unless tag_sources.is_a?(Hash)
raise ArgumentError, 'タグ由来の形式が不正です.' unless (tag_sources.keys - ORIGINS).empty?
raise ArgumentError, 'タグ由来の形式が不正です.' unless tag_sources.values.all? { _1.is_a?(String) }
if tag_sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
if tag_sources.values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES }
raise ArgumentError, 'タグ由来が大きすぎます.'
end
if tag_sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES
if tag_sources.values.sum(&:bytesize) > PostImportUrlListParser::MAX_URL_BYTES
raise ArgumentError, 'タグ由来が大きすぎます.'
end
end
@@ -148,7 +148,7 @@ class PostImportRowNormaliser
unless values.is_a?(Array) && values.all? { _1.is_a?(String) }
raise ArgumentError, '警告の形式が不正です.'
end
if values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
if values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES }
raise ArgumentError, '警告が大きすぎます.'
end
end
+5 -9
ファイルの表示
@@ -21,11 +21,7 @@ class PostImportRunner
def run_row row
attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore }
preview = PostImportPreviewer.new(
parsed: { columns: [], rows: [] },
url_column: 0,
mappings: {},
existing: @existing)
preview = PostImportPreviewer.new(existing: @existing)
.preview_rows(
rows: [{
source_row: row['source_row'],
@@ -35,6 +31,10 @@ class PostImportRunner
tag_sources: row['tag_sources'],
}],
fetch_metadata: false).first
if @existing == 'skip' && preview[:existing]
return row.slice('source_row').merge(status: 'skipped')
end
if preview[:validation_errors].present?
return {
source_row: row['source_row'],
@@ -43,10 +43,6 @@ class PostImportRunner
}
end
if @existing == 'skip' && preview[:existing]
return row.slice('source_row').merge(status: 'skipped')
end
attributes['tags'] = preview[:attributes]['tags']
attributes['url'] = row['url']
post = PostCreator.new(actor: @actor, attributes:).create!
-118
ファイルの表示
@@ -1,118 +0,0 @@
require 'csv'
class PostImportSourceParser
MAX_ROWS = 100
MAX_BYTES = 1.megabyte
MAX_CELL_BYTES = 20.kilobytes
MAX_COLUMNS = 50
def initialize source:, format:, has_header: false, json_path: nil
@source = source.to_s
@format = format.to_s
@has_header = ActiveModel::Type::Boolean.new.cast(has_header)
@json_path = json_path.to_s
end
def parse
raise ArgumentError, '入力が大きすぎます.' if @source.bytesize > MAX_BYTES
@format == 'json' ? parse_json : parse_delimited
end
private
def parse_delimited
separator = @format == 'csv' ? ',' : "\t"
table = CSV.parse(@source, col_sep: separator).map { |row| row.map { _1.to_s.strip } }
raise ArgumentError, '入力が空です.' if table.empty?
headers = @has_header ? table.shift : nil
ensure_limits!(table)
width = table.map(&:length).max.to_i
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです." if width > MAX_COLUMNS
{
columns: (0...width).map { |index|
{
key: index.to_s,
label: "#{ column_name(index) }#{ headers&.[](index).presence || '列' }",
}
},
rows: table.each_with_index.map { |values, index|
{ source_row: index + (@has_header ? 2 : 1), values: }
},
}
rescue CSV::MalformedCSVError => e
raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }"
end
def parse_json
value = JSON.parse(@source, max_nesting: 20)
@json_path.split('.').reject(&:blank?).each do |part|
value =
case value
when Array
index = Integer(part, exception: false)
raise ArgumentError if index.nil? || index.negative? || index >= value.length
value[index]
when Hash
raise ArgumentError unless value.key?(part) || value.key?(part.to_sym)
value.key?(part) ? value[part] : value[part.to_sym]
else
raise ArgumentError
end
end
raise ArgumentError, 'JSON の投稿候補は配列にしてください.' unless value.is_a?(Array)
ensure_limits!(value)
unless value.all?(Hash)
invalid = value.each_with_index.filter_map { |item, index|
index + 1 unless item.is_a?(Hash)
}
raise ArgumentError, "JSON の投稿候補は object の配列にしてください: 行 #{ invalid.join(', ') }"
end
keys = value.flat_map(&:keys).map(&:to_s).uniq
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです." if keys.length > MAX_COLUMNS
{
columns: keys.each_with_index.map { |key, index|
{ key:, label: "#{ column_name(index) }#{ key }" }
},
rows: value.each_with_index.map { |item, index|
{
source_row: index + 1,
values: keys.map { |key|
json_value = item.key?(key) ? item[key] : item[key.to_sym]
json_cell(json_value)
},
}
},
}
rescue JSON::ParserError, KeyError, ArgumentError => e
raise ArgumentError, "JSON の形式が不正です: #{ e.message }"
end
def ensure_limits! rows
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです." if rows.length > MAX_ROWS
raise ArgumentError, 'セルが大きすぎます.' if rows.flatten.any? { _1.to_s.bytesize > MAX_CELL_BYTES }
end
def json_cell value
case value
when nil then ''
when String, Numeric, TrueClass, FalseClass then value.to_s
when Array, Hash then JSON.generate(value)
else raise ArgumentError, 'JSON のセル値が不正です.'
end
end
def column_name index
name = +''
loop do
name.prepend((65 + (index % 26)).chr)
index = (index / 26) - 1
break if index < 0
end
name
end
end
+23
ファイルの表示
@@ -0,0 +1,23 @@
class PostImportUrlListParser
MAX_ROWS = 100
MAX_BYTES = 1.megabyte
MAX_URL_BYTES = 20.kilobytes
def self.parse source
raw = source.to_s
raise ArgumentError, '入力が大きすぎます.' if raw.bytesize > MAX_BYTES
rows = raw.split(/\r\n|\n|\r/).each_with_index.filter_map { |line, index|
url = line.strip
next if url.blank?
raise ArgumentError, 'URL が長すぎます.' if url.bytesize > MAX_URL_BYTES
{ source_row: index + 1, url: }
}
raise ArgumentError, 'URL を入力してください.' if rows.empty?
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです." if rows.length > MAX_ROWS
rows
end
end