このコミットが含まれているのは:
@@ -1,44 +1,10 @@
|
||||
class PostImportsController < ApplicationController
|
||||
MAPPING_PROPERTIES = ['kind', 'value', 'columns'].freeze
|
||||
MAPPING_KINDS = ['columns', 'fixed'].freeze
|
||||
|
||||
before_action :require_member!
|
||||
|
||||
def parse
|
||||
source =
|
||||
if params[:format] == 'google_sheets'
|
||||
PostImportGoogleSheetsFetcher.fetch!(params[:source],
|
||||
rate_key: current_user.id)
|
||||
else
|
||||
params[:source]
|
||||
end
|
||||
format = params[:format] == 'google_sheets' ? 'csv' : params[:format]
|
||||
parsed = PostImportSourceParser.new(
|
||||
source:,
|
||||
format:,
|
||||
has_header: params[:has_header],
|
||||
json_path: params[:json_path]).parse
|
||||
render json: parsed
|
||||
rescue ArgumentError => e
|
||||
render_bad_request(e.message)
|
||||
end
|
||||
|
||||
def preview
|
||||
parsed = parsed_params
|
||||
raw_url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil)
|
||||
return render_bad_request('URL 列を選択してください.') if raw_url_column.nil?
|
||||
|
||||
url_column = Integer(raw_url_column, exception: false)
|
||||
if url_column.nil? || url_column.negative? || url_column >= parsed[:columns].length
|
||||
return render_bad_request('URL 列が不正です.')
|
||||
end
|
||||
|
||||
rows = PostImportPreviewer.new(
|
||||
parsed:,
|
||||
url_column:,
|
||||
mappings: mappings_params(parsed[:columns].length),
|
||||
existing: params[:existing]).preview
|
||||
render json: parsed.slice(:columns).merge(rows:)
|
||||
rows = PostImportPreviewer.new(existing: params[:existing]).preview_rows(
|
||||
rows: PostImportUrlListParser.parse(params[:source]))
|
||||
render json: { rows: }
|
||||
rescue ArgumentError => e
|
||||
render_bad_request(e.message)
|
||||
end
|
||||
@@ -47,12 +13,8 @@ class PostImportsController < ApplicationController
|
||||
rows = normalised_import_rows(allow_warning_fields: true)
|
||||
changed_row = Integer(params[:changed_row], exception: false)
|
||||
result =
|
||||
PostImportPreviewer.new(
|
||||
parsed: { columns: [], rows: [] },
|
||||
url_column: 0,
|
||||
mappings: {},
|
||||
existing: params[:existing])
|
||||
.preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {})
|
||||
PostImportPreviewer.new(existing: params[:existing])
|
||||
.preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {})
|
||||
render json: { rows: result }
|
||||
rescue ArgumentError => e
|
||||
render_bad_request(e.message)
|
||||
@@ -77,85 +39,7 @@ class PostImportsController < ApplicationController
|
||||
head :forbidden
|
||||
end
|
||||
|
||||
def parsed_params
|
||||
value = params.require(:parsed).to_unsafe_h.deep_transform_keys { _1.to_s.underscore }
|
||||
columns = Array(value['columns'])
|
||||
rows = Array(value['rows'])
|
||||
unless columns.all?(Hash) && rows.all?(Hash)
|
||||
raise ArgumentError, '解析結果の形式が不正です.'
|
||||
end
|
||||
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportSourceParser::MAX_ROWS
|
||||
raise ArgumentError, '列数が多すぎます.' if columns.length > 50
|
||||
raise ArgumentError, '解析結果のセルが不正です.' unless rows.all? { |row|
|
||||
values = Array(row['values'])
|
||||
values.length <= columns.length && values.all? { |value|
|
||||
value.is_a?(String) && value.bytesize <= PostImportSourceParser::MAX_CELL_BYTES
|
||||
}
|
||||
}
|
||||
if rows.sum { Array(_1['values']).sum(&:bytesize) } > PostImportSourceParser::MAX_BYTES
|
||||
raise ArgumentError, '解析結果が大きすぎます.'
|
||||
end
|
||||
|
||||
parsed_rows = rows.map do |row|
|
||||
source_row = Integer(row['source_row'], exception: false)
|
||||
raise ArgumentError, '元行番号が不正です.' if source_row.nil? || source_row <= 0
|
||||
|
||||
{ source_row:, values: Array(row['values']) }
|
||||
end
|
||||
if parsed_rows.map { _1[:source_row] }.uniq.length != parsed_rows.length
|
||||
raise ArgumentError, '元行番号が重複しています.'
|
||||
end
|
||||
|
||||
{ columns: columns.map { |column| column.slice('key', 'label') }, rows: parsed_rows }
|
||||
end
|
||||
|
||||
def mappings_params column_count
|
||||
mappings = params[:mappings]
|
||||
return { } if mappings.blank?
|
||||
raise ArgumentError, '列割当ての形式が不正です.' unless mappings.is_a?(ActionController::Parameters)
|
||||
|
||||
raw = mappings.to_unsafe_h
|
||||
unknown_fields = raw.keys - PostImportPreviewer::FIELDS
|
||||
raise ArgumentError, '列割当て先が不正です.' if unknown_fields.present?
|
||||
raise ArgumentError, '列割当てが大きすぎます.' if raw.to_json.bytesize > PostImportSourceParser::MAX_BYTES
|
||||
|
||||
permitted =
|
||||
mappings.permit(*PostImportPreviewer::FIELDS.map { |field|
|
||||
{ field => [:kind, :value, { columns: [] }] }
|
||||
}).to_h
|
||||
|
||||
raw.each_with_object({ }) do |(field, value), result|
|
||||
raise ArgumentError, '列割当ての形式が不正です.' unless value.is_a?(Hash)
|
||||
unless (value.keys - MAPPING_PROPERTIES).empty?
|
||||
raise ArgumentError, '列割当ての項目が不正です.'
|
||||
end
|
||||
|
||||
value = permitted.fetch(field)
|
||||
kind = value['kind'].presence || 'columns'
|
||||
raise ArgumentError, '列割当ての種別が不正です.' unless kind.in?(MAPPING_KINDS)
|
||||
columns = value['columns']
|
||||
raise ArgumentError, '列番号が不正です.' unless (
|
||||
columns.nil? || (columns.is_a?(Array) && columns.all? { |index|
|
||||
parsed = Integer(index, exception: false)
|
||||
parsed && parsed >= 0 && parsed < column_count
|
||||
})
|
||||
)
|
||||
parsed_columns = Array(columns).map { Integer(_1) }
|
||||
raise ArgumentError, '列番号が重複しています.' if parsed_columns.uniq.length != parsed_columns.length
|
||||
raise ArgumentError, '固定値が不正です.' if kind == 'fixed' && !value['value'].is_a?(String)
|
||||
if value['value'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
|
||||
raise ArgumentError, '固定値が大きすぎます.'
|
||||
end
|
||||
|
||||
result[field] = {
|
||||
'kind' => kind,
|
||||
'value' => value['value'].to_s,
|
||||
'columns' => parsed_columns }
|
||||
end
|
||||
end
|
||||
|
||||
def normalised_import_rows allow_warning_fields: false
|
||||
rows = params[:rows]
|
||||
PostImportRowNormaliser.normalise!(rows, allow_warning_fields:)
|
||||
PostImportRowNormaliser.normalise!(params[:rows], allow_warning_fields:)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
class PostImportGoogleSheetsFetcher
|
||||
HOST = 'docs.google.com'.freeze
|
||||
PUBLICATION_ID = /\A[A-Za-z0-9_-]+\z/
|
||||
GID = /\A\d+\z/
|
||||
RATE_LIMIT = 10
|
||||
RATE_WINDOW = 1.minute
|
||||
|
||||
def self.fetch! raw_url, rate_key:
|
||||
new(raw_url, rate_key:).fetch!
|
||||
end
|
||||
|
||||
def initialize raw_url, rate_key:
|
||||
@raw_url = raw_url.to_s
|
||||
@rate_key = rate_key.to_s
|
||||
end
|
||||
|
||||
def fetch!
|
||||
publication_id, gid = parse!
|
||||
throttle!
|
||||
csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub" \
|
||||
"?gid=#{ gid }&single=true&output=csv"
|
||||
response = Preview::HttpFetcher.fetch(csv_url,
|
||||
max_bytes: PostImportSourceParser::MAX_BYTES,
|
||||
allowed_hosts: [HOST])
|
||||
raise ArgumentError, 'スプレッドシートを CSV として取得できませんでした.' unless csv?(response)
|
||||
|
||||
response.body
|
||||
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed,
|
||||
Preview::HttpFetcher::ResponseTooLarge => e
|
||||
Rails.logger.info(
|
||||
'post_import_google_sheets_fetch_failure '\
|
||||
"#{ { error: e.class.name, message: e.message }.to_json }",
|
||||
)
|
||||
raise ArgumentError, 'スプレッドシートを取得できませんでした.ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください.'
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def parse!
|
||||
uri = URI.parse(@raw_url)
|
||||
unless uri.scheme == 'https' && uri.host&.downcase == HOST && uri.query.blank?
|
||||
raise ArgumentError, '公開 Google スプレッドシート URL を指定してください.'
|
||||
end
|
||||
match = uri.path.match(%r{\A/spreadsheets/d/e/([^/]+)/pubhtml\z})
|
||||
gid = URI.decode_www_form(uri.fragment.to_s).to_h['gid']
|
||||
unless match && PUBLICATION_ID.match?(match[1]) && GID.match?(gid.to_s)
|
||||
raise ArgumentError, '公開 Google スプレッドシート URL の publication_id 又は gid が不正です.'
|
||||
end
|
||||
|
||||
[match[1], gid]
|
||||
rescue URI::InvalidURIError
|
||||
raise ArgumentError, '公開 Google スプレッドシート URL が不正です.'
|
||||
end
|
||||
|
||||
def throttle!
|
||||
key = "post-import-google-sheets:#{ @rate_key }"
|
||||
count = Rails.cache.increment(key, 1, expires_in: RATE_WINDOW, initial: 0)
|
||||
if count > RATE_LIMIT
|
||||
raise ArgumentError, 'スプレッドシート取得の回数が多すぎます.しばらくしてから再試行してください.'
|
||||
end
|
||||
end
|
||||
|
||||
def csv? response
|
||||
type = response.content_type.to_s.downcase
|
||||
type.include?('text/csv') || type.include?('application/csv')
|
||||
end
|
||||
end
|
||||
@@ -8,119 +8,15 @@ class PostImportPreviewer
|
||||
'tags',
|
||||
'parent_post_ids',
|
||||
].freeze
|
||||
|
||||
def initialize parsed:, url_column:, mappings: { }, existing: 'skip'
|
||||
@parsed = parsed
|
||||
@url_column = url_column.to_i
|
||||
@mappings = mappings.stringify_keys
|
||||
def initialize existing: 'skip'
|
||||
@existing = existing == 'error' ? 'error' : 'skip'
|
||||
end
|
||||
|
||||
def preview
|
||||
preview_rows(rows: @parsed[:rows])
|
||||
end
|
||||
|
||||
def preview_rows rows:, fetch_metadata: true, metadata_cache: { }
|
||||
urls = { }
|
||||
rows.map do |row|
|
||||
row = row.symbolize_keys
|
||||
values = row[:values] || []
|
||||
attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field|
|
||||
[field, mapped_value(field, values)]
|
||||
}
|
||||
provenance = row[:provenance]&.stringify_keys || mapped_provenance(attributes)
|
||||
tag_sources =
|
||||
row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance)
|
||||
tag_sources['manual'] = attributes['tags'].to_s if provenance['tags'] == 'manual'
|
||||
url = row[:url].presence || values[@url_column].to_s.strip
|
||||
provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped'
|
||||
url_for_metadata = normalised_url(url) || url
|
||||
existing_post = normalised_url(url).present? && Post.exists?(url: normalised_url(url))
|
||||
if existing_post && @existing == 'skip'
|
||||
next {
|
||||
source_row: row[:source_row],
|
||||
url: normalised_url(url) || url,
|
||||
attributes:,
|
||||
provenance:,
|
||||
tag_sources:,
|
||||
metadata_url: url_for_metadata,
|
||||
fetch_warnings: [],
|
||||
validation_warnings: ['既存投稿のためスキップします.'],
|
||||
warnings: ['既存投稿のためスキップします.'],
|
||||
validation_errors: { },
|
||||
status: 'warning',
|
||||
existing: true,
|
||||
}
|
||||
end
|
||||
url_changed = row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
|
||||
clear_automatic_values!(attributes, provenance, tag_sources) if url_changed
|
||||
attributes['url'] = url
|
||||
fetch_warnings = Array(row[:fetch_warnings]).dup
|
||||
should_fetch =
|
||||
case fetch_metadata
|
||||
when true then true
|
||||
when Integer then fetch_metadata == row[:source_row].to_i
|
||||
when false, nil then false
|
||||
else raise ArgumentError, 'メタデータ取得対象が不正です.'
|
||||
end
|
||||
metadata, fetch_warnings =
|
||||
metadata_for(url_for_metadata, should_fetch, metadata_cache, fetch_warnings)
|
||||
metadata.each do |field, value|
|
||||
if field == 'tags' && provenance[field] != 'manual'
|
||||
tag_sources['automatic'] = value.to_s
|
||||
attributes[field] = merged_tags(tag_sources)
|
||||
next
|
||||
end
|
||||
next unless provenance[field] == 'automatic' || attributes[field].blank?
|
||||
|
||||
attributes[field] = value
|
||||
provenance[field] = 'automatic'
|
||||
end
|
||||
normal_url = normalised_url(url)
|
||||
errors = { }
|
||||
errors[:url] = ['URL が不正です.'] if normal_url.blank?
|
||||
validation_warnings = []
|
||||
errors[:url] = ['URL が重複しています.'] if normal_url.present? && urls.key?(normal_url)
|
||||
urls[normal_url] = true if normal_url.present?
|
||||
if normal_url.present? && existing_post
|
||||
if @existing == 'error'
|
||||
errors[:url] = ['既存投稿です.']
|
||||
else
|
||||
validation_warnings << '既存投稿のためスキップします.'
|
||||
end
|
||||
end
|
||||
validate_basic_data(attributes, errors)
|
||||
validate_preview_tags(
|
||||
merged_tags(tag_sources, provenance['tags']),
|
||||
errors,
|
||||
validation_warnings,
|
||||
)
|
||||
validate_parents(attributes['parent_post_ids'], errors)
|
||||
attributes.delete('url')
|
||||
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
|
||||
{
|
||||
source_row: row[:source_row],
|
||||
url: normal_url || url,
|
||||
attributes:,
|
||||
provenance:,
|
||||
tag_sources:,
|
||||
metadata_url: url_for_metadata,
|
||||
fetch_warnings:,
|
||||
validation_warnings:,
|
||||
warnings: fetch_warnings + validation_warnings,
|
||||
validation_errors: errors,
|
||||
status: errors.present? \
|
||||
? 'error' \
|
||||
: ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
|
||||
existing: normal_url.present? && Post.exists?(url: normal_url),
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
def metadata_for url, fetch_metadata, cache, previous_warnings
|
||||
return [{ }, previous_warnings] unless fetch_metadata
|
||||
|
||||
cache[url] ||= fetch_metadata(url)
|
||||
urls = {}
|
||||
rows.map { |row|
|
||||
preview_row(row, urls:, fetch_metadata:, metadata_cache:)
|
||||
}
|
||||
end
|
||||
|
||||
def normalised_url value
|
||||
@@ -129,43 +25,97 @@ class PostImportPreviewer
|
||||
|
||||
private
|
||||
|
||||
def mapped_value field, values
|
||||
mapping = @mappings[field]
|
||||
return '' if mapping.blank?
|
||||
return mapping['value'].to_s if mapping['kind'] == 'fixed'
|
||||
def preview_row row, urls:, fetch_metadata:, metadata_cache:
|
||||
row = row.symbolize_keys
|
||||
attributes = initial_attributes(row)
|
||||
provenance = initial_provenance(row)
|
||||
tag_sources = initial_tag_sources(row, attributes, provenance)
|
||||
url = row[:url].to_s.strip
|
||||
provenance['url'] = 'manual'
|
||||
normal_url = normalised_url(url)
|
||||
url_for_metadata = normal_url || url
|
||||
existing_post = normal_url.present? && Post.exists?(url: normal_url)
|
||||
|
||||
Array(mapping['columns']).filter_map { |index|
|
||||
values[index.to_i].to_s.presence
|
||||
}.join(field == 'tags' ? ' ' : '')
|
||||
end
|
||||
validation_warnings = []
|
||||
validation_errors = {}
|
||||
validation_errors[:url] = ['URL が不正です.'] if normal_url.blank?
|
||||
validation_errors[:url] = ['URL が重複しています.'] if normal_url.present? && urls.key?(normal_url)
|
||||
urls[normal_url] = true if normal_url.present?
|
||||
if normal_url.present? && existing_post
|
||||
if @existing == 'error'
|
||||
validation_errors[:url] = ['既存投稿です.']
|
||||
else
|
||||
validation_warnings << '既存投稿のためスキップします.'
|
||||
end
|
||||
end
|
||||
|
||||
def mapped_provenance attributes
|
||||
attributes.to_h.to_h { |field, value|
|
||||
mapping = @mappings[field.to_s]
|
||||
origin =
|
||||
if value.present? && mapping&.[]('kind') == 'fixed'
|
||||
'fixed'
|
||||
elsif value.present?
|
||||
'mapped'
|
||||
else
|
||||
'automatic'
|
||||
end
|
||||
[field.to_s, origin]
|
||||
if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
|
||||
clear_automatic_values!(attributes, provenance, tag_sources)
|
||||
end
|
||||
|
||||
fetch_warnings = Array(row[:fetch_warnings]).dup
|
||||
should_fetch = should_fetch_metadata?(fetch_metadata, row[:source_row].to_i)
|
||||
if should_fetch && !(existing_post && @existing == 'skip')
|
||||
metadata, fetch_warnings =
|
||||
metadata_for(url_for_metadata, metadata_cache, fetch_warnings)
|
||||
apply_metadata!(attributes, provenance, tag_sources, metadata)
|
||||
end
|
||||
|
||||
attributes['url'] = url
|
||||
validate_basic_data(attributes, validation_errors)
|
||||
validate_preview_tags(
|
||||
merged_tags(tag_sources, provenance['tags']),
|
||||
validation_errors,
|
||||
validation_warnings,
|
||||
)
|
||||
validate_parents(attributes['parent_post_ids'], validation_errors)
|
||||
attributes.delete('url')
|
||||
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
|
||||
|
||||
{
|
||||
source_row: row[:source_row],
|
||||
url: normal_url || url,
|
||||
attributes:,
|
||||
provenance:,
|
||||
tag_sources:,
|
||||
metadata_url: url_for_metadata,
|
||||
fetch_warnings:,
|
||||
validation_warnings:,
|
||||
warnings: fetch_warnings + validation_warnings,
|
||||
validation_errors:,
|
||||
status: validation_errors.present? \
|
||||
? 'error' \
|
||||
: ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
|
||||
existing: existing_post,
|
||||
}
|
||||
end
|
||||
|
||||
def initial_tag_sources attributes, provenance
|
||||
origin = provenance['tags'] == 'fixed' ? 'fixed' : 'mapped'
|
||||
{ 'automatic' => '', 'mapped' => origin == 'mapped' ? attributes['tags'].to_s : '',
|
||||
'fixed' => origin == 'fixed' ? attributes['tags'].to_s : '', 'manual' => '' }
|
||||
def should_fetch_metadata? fetch_metadata, source_row
|
||||
case fetch_metadata
|
||||
when true then true
|
||||
when Integer then fetch_metadata == source_row
|
||||
when false, nil then false
|
||||
else raise ArgumentError, 'メタデータ取得対象が不正です.'
|
||||
end
|
||||
end
|
||||
|
||||
def merged_tags sources, origin = nil
|
||||
return sources['manual'].to_s if origin == 'manual'
|
||||
def initial_attributes row
|
||||
attributes = row[:attributes]&.stringify_keys || {}
|
||||
FIELDS.to_h { |field| [field, attributes[field].to_s] }
|
||||
end
|
||||
|
||||
['automatic', 'mapped', 'fixed', 'manual'].flat_map {
|
||||
sources[_1].to_s.split
|
||||
}.uniq.join(' ')
|
||||
def initial_provenance row
|
||||
provenance = row[:provenance]&.stringify_keys || {}
|
||||
FIELDS.to_h { |field| [field, provenance[field].presence || 'automatic'] }
|
||||
.merge('url' => 'manual')
|
||||
end
|
||||
|
||||
def initial_tag_sources row, attributes, provenance
|
||||
sources = row[:tag_sources]&.stringify_keys || { 'automatic' => '', 'manual' => '' }
|
||||
sources['automatic'] = sources['automatic'].to_s
|
||||
sources['manual'] =
|
||||
provenance['tags'] == 'manual' ? attributes['tags'].to_s : sources['manual'].to_s
|
||||
sources
|
||||
end
|
||||
|
||||
def clear_automatic_values! attributes, provenance, tag_sources
|
||||
@@ -174,10 +124,15 @@ class PostImportPreviewer
|
||||
attributes[field] = '' if provenance[field] == 'automatic'
|
||||
end
|
||||
tag_sources['automatic'] = ''
|
||||
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
|
||||
end
|
||||
|
||||
def metadata_for url, cache, previous_warnings
|
||||
cache[url] ||= fetch_metadata(url)
|
||||
end
|
||||
|
||||
def fetch_metadata url
|
||||
return [{ }, ['URL が空です.']] if url.blank?
|
||||
return [{}, ['URL が空です.']] if url.blank?
|
||||
|
||||
data = PostMetadataFetcher.fetch(url).stringify_keys
|
||||
warnings = []
|
||||
@@ -191,7 +146,29 @@ class PostImportPreviewer
|
||||
"post_import_metadata_fetch_failure "\
|
||||
"#{ { error: e.class.name, message: e.message }.to_json }",
|
||||
)
|
||||
[{ }, ['メタデータを取得できませんでした.']]
|
||||
[{}, ['メタデータを取得できませんでした.']]
|
||||
end
|
||||
|
||||
def apply_metadata! attributes, provenance, tag_sources, metadata
|
||||
metadata.each do |field, value|
|
||||
if field == 'tags'
|
||||
next if provenance['tags'] == 'manual'
|
||||
|
||||
tag_sources['automatic'] = value.to_s
|
||||
attributes['tags'] = merged_tags(tag_sources)
|
||||
next
|
||||
end
|
||||
next unless provenance[field] == 'automatic' || attributes[field].blank?
|
||||
|
||||
attributes[field] = value
|
||||
provenance[field] = 'automatic'
|
||||
end
|
||||
end
|
||||
|
||||
def merged_tags sources, origin = nil
|
||||
return sources['manual'].to_s if origin == 'manual'
|
||||
|
||||
sources['automatic'].to_s
|
||||
end
|
||||
|
||||
def validate_preview_tags raw, errors, warnings
|
||||
@@ -201,6 +178,7 @@ class PostImportPreviewer
|
||||
errors[:tags] = ['ニコニコ・タグは直接指定できません.']
|
||||
return
|
||||
end
|
||||
|
||||
parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first }
|
||||
existing = Tag.joins(:tag_name).where(tag_names: { name: parsed }).includes(:tag_name).to_a
|
||||
deprecated = existing.select(&:deprecated?).map(&:name)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
class PostImportRowNormaliser
|
||||
ORIGINS = ['automatic', 'mapped', 'fixed', 'manual'].freeze
|
||||
ORIGINS = ['automatic', 'manual'].freeze
|
||||
STRING_FIELDS = [
|
||||
'title',
|
||||
'thumbnail_base',
|
||||
@@ -13,12 +13,12 @@ class PostImportRowNormaliser
|
||||
|
||||
def self.normalise! rows, allow_warning_fields: false
|
||||
raise ArgumentError, '取込行の形式が不正です.' unless rows.is_a?(Array)
|
||||
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportSourceParser::MAX_ROWS
|
||||
raise ArgumentError, '取込件数が多すぎます.' if rows.length > PostImportUrlListParser::MAX_ROWS
|
||||
|
||||
normalised_rows = rows.map { normalise_row!(_1, allow_warning_fields:) }
|
||||
source_rows = normalised_rows.map { _1['source_row'] }
|
||||
raise ArgumentError, '元行番号が重複しています.' if source_rows.uniq.length != source_rows.length
|
||||
if normalised_rows.sum { row_bytesize(_1) } > PostImportSourceParser::MAX_BYTES
|
||||
if normalised_rows.sum { row_bytesize(_1) } > PostImportUrlListParser::MAX_BYTES
|
||||
raise ArgumentError, '取込データが大きすぎます.'
|
||||
end
|
||||
|
||||
@@ -42,7 +42,7 @@ class PostImportRowNormaliser
|
||||
normalise_provenance!(normalised.fetch('provenance', { }))
|
||||
normalise_tag_sources!(normalised['tag_sources'])
|
||||
normalise_warning_values!(normalised, allow_warning_fields:)
|
||||
if row_bytesize(normalised) > PostImportSourceParser::MAX_BYTES
|
||||
if row_bytesize(normalised) > PostImportUrlListParser::MAX_BYTES
|
||||
raise ArgumentError, '取込行が大きすぎます.'
|
||||
end
|
||||
|
||||
@@ -82,13 +82,13 @@ class PostImportRowNormaliser
|
||||
|
||||
def self.normalise_url! value
|
||||
raise ArgumentError, 'URL の形式が不正です.' unless value.is_a?(String)
|
||||
raise ArgumentError, 'URL が長すぎます.' if value.bytesize > PostImportSourceParser::MAX_CELL_BYTES
|
||||
raise ArgumentError, 'URL が長すぎます.' if value.bytesize > PostImportUrlListParser::MAX_URL_BYTES
|
||||
end
|
||||
private_class_method :normalise_url!
|
||||
|
||||
def self.normalise_metadata_url! value
|
||||
raise ArgumentError, 'metadata_url の形式が不正です.' unless value.nil? || value.is_a?(String)
|
||||
if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
|
||||
if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES
|
||||
raise ArgumentError, 'metadata_url が長すぎます.'
|
||||
end
|
||||
end
|
||||
@@ -107,7 +107,7 @@ class PostImportRowNormaliser
|
||||
raise ArgumentError, '取込項目の型が不正です.'
|
||||
end
|
||||
end
|
||||
if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
|
||||
if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES
|
||||
raise ArgumentError, '取込項目が大きすぎます.'
|
||||
end
|
||||
end
|
||||
@@ -129,10 +129,10 @@ class PostImportRowNormaliser
|
||||
raise ArgumentError, 'タグ由来の形式が不正です.' unless tag_sources.is_a?(Hash)
|
||||
raise ArgumentError, 'タグ由来の形式が不正です.' unless (tag_sources.keys - ORIGINS).empty?
|
||||
raise ArgumentError, 'タグ由来の形式が不正です.' unless tag_sources.values.all? { _1.is_a?(String) }
|
||||
if tag_sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
|
||||
if tag_sources.values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES }
|
||||
raise ArgumentError, 'タグ由来が大きすぎます.'
|
||||
end
|
||||
if tag_sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES
|
||||
if tag_sources.values.sum(&:bytesize) > PostImportUrlListParser::MAX_URL_BYTES
|
||||
raise ArgumentError, 'タグ由来が大きすぎます.'
|
||||
end
|
||||
end
|
||||
@@ -148,7 +148,7 @@ class PostImportRowNormaliser
|
||||
unless values.is_a?(Array) && values.all? { _1.is_a?(String) }
|
||||
raise ArgumentError, '警告の形式が不正です.'
|
||||
end
|
||||
if values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
|
||||
if values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES }
|
||||
raise ArgumentError, '警告が大きすぎます.'
|
||||
end
|
||||
end
|
||||
|
||||
@@ -21,11 +21,7 @@ class PostImportRunner
|
||||
|
||||
def run_row row
|
||||
attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore }
|
||||
preview = PostImportPreviewer.new(
|
||||
parsed: { columns: [], rows: [] },
|
||||
url_column: 0,
|
||||
mappings: {},
|
||||
existing: @existing)
|
||||
preview = PostImportPreviewer.new(existing: @existing)
|
||||
.preview_rows(
|
||||
rows: [{
|
||||
source_row: row['source_row'],
|
||||
@@ -35,6 +31,10 @@ class PostImportRunner
|
||||
tag_sources: row['tag_sources'],
|
||||
}],
|
||||
fetch_metadata: false).first
|
||||
if @existing == 'skip' && preview[:existing]
|
||||
return row.slice('source_row').merge(status: 'skipped')
|
||||
end
|
||||
|
||||
if preview[:validation_errors].present?
|
||||
return {
|
||||
source_row: row['source_row'],
|
||||
@@ -43,10 +43,6 @@ class PostImportRunner
|
||||
}
|
||||
end
|
||||
|
||||
if @existing == 'skip' && preview[:existing]
|
||||
return row.slice('source_row').merge(status: 'skipped')
|
||||
end
|
||||
|
||||
attributes['tags'] = preview[:attributes]['tags']
|
||||
attributes['url'] = row['url']
|
||||
post = PostCreator.new(actor: @actor, attributes:).create!
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
require 'csv'
|
||||
|
||||
class PostImportSourceParser
|
||||
MAX_ROWS = 100
|
||||
MAX_BYTES = 1.megabyte
|
||||
MAX_CELL_BYTES = 20.kilobytes
|
||||
MAX_COLUMNS = 50
|
||||
|
||||
def initialize source:, format:, has_header: false, json_path: nil
|
||||
@source = source.to_s
|
||||
@format = format.to_s
|
||||
@has_header = ActiveModel::Type::Boolean.new.cast(has_header)
|
||||
@json_path = json_path.to_s
|
||||
end
|
||||
|
||||
def parse
|
||||
raise ArgumentError, '入力が大きすぎます.' if @source.bytesize > MAX_BYTES
|
||||
|
||||
@format == 'json' ? parse_json : parse_delimited
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def parse_delimited
|
||||
separator = @format == 'csv' ? ',' : "\t"
|
||||
table = CSV.parse(@source, col_sep: separator).map { |row| row.map { _1.to_s.strip } }
|
||||
raise ArgumentError, '入力が空です.' if table.empty?
|
||||
|
||||
headers = @has_header ? table.shift : nil
|
||||
ensure_limits!(table)
|
||||
width = table.map(&:length).max.to_i
|
||||
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです." if width > MAX_COLUMNS
|
||||
{
|
||||
columns: (0...width).map { |index|
|
||||
{
|
||||
key: index.to_s,
|
||||
label: "#{ column_name(index) }:#{ headers&.[](index).presence || '列' }",
|
||||
}
|
||||
},
|
||||
rows: table.each_with_index.map { |values, index|
|
||||
{ source_row: index + (@has_header ? 2 : 1), values: }
|
||||
},
|
||||
}
|
||||
rescue CSV::MalformedCSVError => e
|
||||
raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }"
|
||||
end
|
||||
|
||||
def parse_json
|
||||
value = JSON.parse(@source, max_nesting: 20)
|
||||
@json_path.split('.').reject(&:blank?).each do |part|
|
||||
value =
|
||||
case value
|
||||
when Array
|
||||
index = Integer(part, exception: false)
|
||||
raise ArgumentError if index.nil? || index.negative? || index >= value.length
|
||||
|
||||
value[index]
|
||||
when Hash
|
||||
raise ArgumentError unless value.key?(part) || value.key?(part.to_sym)
|
||||
|
||||
value.key?(part) ? value[part] : value[part.to_sym]
|
||||
else
|
||||
raise ArgumentError
|
||||
end
|
||||
end
|
||||
raise ArgumentError, 'JSON の投稿候補は配列にしてください.' unless value.is_a?(Array)
|
||||
|
||||
ensure_limits!(value)
|
||||
unless value.all?(Hash)
|
||||
invalid = value.each_with_index.filter_map { |item, index|
|
||||
index + 1 unless item.is_a?(Hash)
|
||||
}
|
||||
raise ArgumentError, "JSON の投稿候補は object の配列にしてください: 行 #{ invalid.join(', ') }"
|
||||
end
|
||||
keys = value.flat_map(&:keys).map(&:to_s).uniq
|
||||
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです." if keys.length > MAX_COLUMNS
|
||||
{
|
||||
columns: keys.each_with_index.map { |key, index|
|
||||
{ key:, label: "#{ column_name(index) }:#{ key }" }
|
||||
},
|
||||
rows: value.each_with_index.map { |item, index|
|
||||
{
|
||||
source_row: index + 1,
|
||||
values: keys.map { |key|
|
||||
json_value = item.key?(key) ? item[key] : item[key.to_sym]
|
||||
json_cell(json_value)
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
rescue JSON::ParserError, KeyError, ArgumentError => e
|
||||
raise ArgumentError, "JSON の形式が不正です: #{ e.message }"
|
||||
end
|
||||
|
||||
def ensure_limits! rows
|
||||
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです." if rows.length > MAX_ROWS
|
||||
raise ArgumentError, 'セルが大きすぎます.' if rows.flatten.any? { _1.to_s.bytesize > MAX_CELL_BYTES }
|
||||
end
|
||||
|
||||
def json_cell value
|
||||
case value
|
||||
when nil then ''
|
||||
when String, Numeric, TrueClass, FalseClass then value.to_s
|
||||
when Array, Hash then JSON.generate(value)
|
||||
else raise ArgumentError, 'JSON のセル値が不正です.'
|
||||
end
|
||||
end
|
||||
|
||||
def column_name index
|
||||
name = +''
|
||||
loop do
|
||||
name.prepend((65 + (index % 26)).chr)
|
||||
index = (index / 26) - 1
|
||||
break if index < 0
|
||||
end
|
||||
name
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,23 @@
|
||||
class PostImportUrlListParser
|
||||
MAX_ROWS = 100
|
||||
MAX_BYTES = 1.megabyte
|
||||
MAX_URL_BYTES = 20.kilobytes
|
||||
|
||||
def self.parse source
|
||||
raw = source.to_s
|
||||
raise ArgumentError, '入力が大きすぎます.' if raw.bytesize > MAX_BYTES
|
||||
|
||||
rows = raw.split(/\r\n|\n|\r/).each_with_index.filter_map { |line, index|
|
||||
url = line.strip
|
||||
next if url.blank?
|
||||
|
||||
raise ArgumentError, 'URL が長すぎます.' if url.bytesize > MAX_URL_BYTES
|
||||
|
||||
{ source_row: index + 1, url: }
|
||||
}
|
||||
raise ArgumentError, 'URL を入力してください.' if rows.empty?
|
||||
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです." if rows.length > MAX_ROWS
|
||||
|
||||
rows
|
||||
end
|
||||
end
|
||||
新しい課題から参照
ユーザをブロックする