miteruzo/btrc-hub
1
0
フォーク 0
コード 課題 61 Pull リクエスト 1 リリース Wiki アクティビティ

#399

ソースを参照
このコミットが含まれているのは:
みてるぞ
2026-07-12 12:23:53 +09:00
親 8970edc59f
コミット 7bcb76516c
12個のファイルの変更，1267行の追加，479行の削除
Patchファイルをダウンロード Diffファイルをダウンロード すべてのファイルを展開 すべてのファイルを折り畳む
backend/app/controllers/post_imports_controller.rb
+105 -45
ファイルの表示
@@ -1,16 +1,23 @@
class PostImportsController < ApplicationController
MAPPING_PROPERTIES = ['kind', 'value', 'columns'].freeze
MAPPING_KINDS = ['columns', 'fixed'].freeze
before_action :require_member!
def parse
source =
if params[:format] == 'google_sheets'
PostImportGoogleSheetsFetcher.fetch!(params[:source], rate_key: current_user.id)
PostImportGoogleSheetsFetcher.fetch!(params[:source],
rate_key: current_user.id)
else
params[:source]
end
format = params[:format] == 'google_sheets' ? 'csv' : params[:format]
parsed = PostImportSourceParser.new(source:, format:,
has_header: params[:has_header], json_path: params[:json_path]).parse
parsed = PostImportSourceParser.new(
source:,
format:,
has_header: params[:has_header],
json_path: params[:json_path]).parse
render json: parsed
rescue ArgumentError => e
render_bad_request(e.message)
@@ -20,11 +27,17 @@ class PostImportsController < ApplicationController
parsed = parsed_params
raw_url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil)
return render_bad_request('URL 列を選択してください．') if raw_url_column.nil?
url_column = Integer(raw_url_column, exception: false)
return render_bad_request('URL 列が不正です．') if url_column.nil? || url_column.negative? || url_column >= parsed[:columns].length
rows = PostImportPreviewer.new(parsed:, url_column:, mappings: mappings_params(parsed[:columns].length),
existing: params[:existing]).preview
url_column = Integer(raw_url_column, exception: false)
if url_column.nil? || url_column.negative? || url_column >= parsed[:columns].length
return render_bad_request('URL 列が不正です．')
end
rows = PostImportPreviewer.new(
parsed:,
url_column:,
mappings: mappings_params(parsed[:columns].length),
existing: params[:existing]).preview
render json: parsed.slice(:columns).merge(rows:)
rescue ArgumentError => e
render_bad_request(e.message)
@@ -33,18 +46,23 @@ class PostImportsController < ApplicationController
def validate
rows = validate_rows_params
changed_row = Integer(params[:changed_row], exception: false)
result = PostImportPreviewer.new(parsed: { columns: [], rows: [] }, url_column: 0,
mappings: {}, existing: params[:existing]).preview_rows(
rows:, fetch_metadata: changed_row,
metadata_cache: { })
result =
PostImportPreviewer.new(
parsed: { columns: [], rows: [] },
url_column: 0,
mappings: {},
existing: params[:existing])
.preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {})
render json: { rows: result }
rescue ArgumentError => e
render_bad_request(e.message)
end
def create
result = PostImportRunner.new(actor: current_user, rows: create_rows_params,
existing: params[:existing]).run
result = PostImportRunner.new(
actor: current_user,
rows: create_rows_params,
existing: params[:existing]).run
render json: result, status: result[:created].positive? ? :created : :ok
rescue ArgumentError => e
render_bad_request(e.message)
@@ -63,14 +81,20 @@ class PostImportsController < ApplicationController
value = params.require(:parsed).to_unsafe_h.deep_transform_keys { _1.to_s.underscore }
columns = Array(value['columns'])
rows = Array(value['rows'])
raise ArgumentError, '解析結果の形式が不正です．' unless columns.all?(Hash) && rows.all?(Hash)
unless columns.all?(Hash) && rows.all?(Hash)
raise ArgumentError, '解析結果の形式が不正です．'
end
raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS
raise ArgumentError, '列数が多すぎます．' if columns.length > 50
unless rows.all? { |row| Array(row['values']).length <= columns.length &&
Array(row['values']).all? { _1.is_a?(String) && _1.bytesize <= PostImportSourceParser::MAX_CELL_BYTES } }
raise ArgumentError, '解析結果のセルが不正です．'
raise ArgumentError, '解析結果のセルが不正です．' unless rows.all? { |row|
values = Array(row['values'])
values.length <= columns.length && values.all? { |value|
value.is_a?(String) && value.bytesize <= PostImportSourceParser::MAX_CELL_BYTES
}
}
if rows.sum { Array(_1['values']).sum(&:bytesize) } > PostImportSourceParser::MAX_BYTES
raise ArgumentError, '解析結果が大きすぎます．'
end
raise ArgumentError, '解析結果が大きすぎます．' if rows.sum { Array(_1['values']).sum(&:bytesize) } > PostImportSourceParser::MAX_BYTES
parsed_rows = rows.map do |row|
source_row = Integer(row['source_row'], exception: false)
@@ -78,7 +102,9 @@ class PostImportsController < ApplicationController
{ source_row:, values: Array(row['values']) }
end
raise ArgumentError, '元行番号が重複しています．' if parsed_rows.map { _1[:source_row] }.uniq.length != parsed_rows.length
if parsed_rows.map { _1[:source_row] }.uniq.length != parsed_rows.length
raise ArgumentError, '元行番号が重複しています．'
end
{ columns: columns.map { |column| column.slice('key', 'label') }, rows: parsed_rows }
end
@@ -92,41 +118,61 @@ class PostImportsController < ApplicationController
unknown_fields = raw.keys - PostImportPreviewer::FIELDS
raise ArgumentError, '列割当て先が不正です．' if unknown_fields.present?
permitted = mappings.permit(*PostImportPreviewer::FIELDS.map { |field|
{ field => [:kind, :value, { columns: [] }] }
}).to_h
permitted =
mappings.permit(*PostImportPreviewer::FIELDS.map { |field|
{ field => [:kind, :value, { columns: [] }] }
}).to_h
raw.each_with_object({ }) do |(field, value), result|
raise ArgumentError, '列割当ての形式が不正です．' unless value.is_a?(Hash)
raise ArgumentError, '列割当ての項目が不正です．' unless (value.keys - ['kind', 'value', 'columns']).empty?
unless (value.keys - MAPPING_PROPERTIES).empty?
raise ArgumentError, '列割当ての項目が不正です．'
end
value = permitted.fetch(field)
kind = value['kind'].presence || 'columns'
raise ArgumentError, '列割当ての種別が不正です．' unless kind.in?(['columns', 'fixed'])
raise ArgumentError, '列割当ての種別が不正です．' unless kind.in?(MAPPING_KINDS)
columns = value['columns']
unless columns.nil? || (columns.is_a?(Array) && columns.all? { |index|
parsed = Integer(index, exception: false)
parsed && parsed >= 0 && parsed < column_count
})
raise ArgumentError, '列番号が不正です．'
end
raise ArgumentError, '列番号が不正です．' unless (
columns.nil? || (columns.is_a?(Array) && columns.all? { |index|
parsed = Integer(index, exception: false)
parsed && parsed >= 0 && parsed < column_count
})
)
raise ArgumentError, '列番号が重複しています．' if Array(columns).uniq.length != Array(columns).length
raise ArgumentError, '固定値が不正です．' if kind == 'fixed' && !value['value'].is_a?(String)
result[field] = { 'kind' => kind, 'value' => value['value'].to_s,
'columns' => Array(columns).map { Integer(_1) } }
result[field] = {
'kind' => kind,
'value' => value['value'].to_s,
'columns' => Array(columns).map { Integer(_1) } }
end
end
def validate_rows_params
rows = Array(params[:rows])
rows = params[:rows]
raise ArgumentError, '取込件数が多すぎます．' unless rows.is_a?(Array)
raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS
normalised_rows = rows.map do |row|
value = ActionController::Parameters.new(row).permit(:source_row, :sourceRow, :url, :metadata_url, :metadataUrl,
attributes: {}, provenance: {}, tag_sources: {}, tagSources: {},
fetch_warnings: [], fetchWarnings: [],
validation_warnings: [], validationWarnings: [])
unless row.is_a?(Hash) || row.is_a?(ActionController::Parameters)
raise ArgumentError, '行の形式が不正です．'
end
value = ActionController::Parameters.new(row).permit(
:source_row,
:sourceRow,
:url,
:metadata_url,
:metadataUrl,
attributes: {},
provenance: {},
tag_sources: {},
tagSources: {},
fetch_warnings: [],
fetchWarnings: [],
validation_warnings: [],
validationWarnings: [])
normalised = value.to_h.deep_transform_keys { _1.to_s.underscore }
attributes = normalised.fetch('attributes', { })
provenance = normalised.fetch('provenance', { })
@@ -139,16 +185,21 @@ class PostImportsController < ApplicationController
unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) }
raise ArgumentError, '値の由来が不正です．'
end
raise ArgumentError, 'セルが大きすぎます．' if [normalised['url'], *attributes.values].any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
raise ArgumentError, 'セルが大きすぎます．' if [normalised['url'], *attributes.values].any? { |value|
value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
}
if normalised['tag_sources'].present? &&
(!normalised['tag_sources'].is_a?(Hash) ||
(normalised['tag_sources'].keys - ['automatic', 'mapped', 'fixed', 'manual']).present?)
raise ArgumentError, 'タグ由来の形式が不正です．'
end
if normalised['tag_sources'].present? &&
(!normalised['tag_sources'].values.all?(String) ||
normalised['tag_sources'].values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } ||
normalised['tag_sources'].values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES)
if (normalised['tag_sources'].present? &&
(!normalised['tag_sources'].values.all?(String) ||
normalised['tag_sources'].values.any? { |value|
value.bytesize > PostImportSourceParser::MAX_CELL_BYTES
} ||
(normalised['tag_sources'].values.sum(&:bytesize) >
PostImportSourceParser::MAX_CELL_BYTES)))
raise ArgumentError, 'タグ由来が大きすぎます．'
end
@@ -175,9 +226,18 @@ class PostImportsController < ApplicationController
raise ArgumentError, '取込行の形式が不正です．'
end
parameters = row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row)
parameters.permit(:source_row, :sourceRow, :url, :metadata_url, :metadataUrl,
attributes: {}, provenance: {}, tag_sources: {}, tagSources: {}).to_h
parameters =
row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row)
parameters.permit(
:source_row,
:sourceRow,
:url,
:metadata_url,
:metadataUrl,
attributes: {},
provenance: {},
tag_sources: {},
tagSources: {}).to_h
end
end
end
backend/app/services/post_creator.rb
+8 -2
ファイルの表示
@@ -90,9 +90,15 @@ class PostCreator
end
PostTagSection.where(post_id: post.id).destroy_all
sections.each do |tag_id, ranges|
ranges.each { |begin_ms, end_ms| PostTagSection.create!(post_id: post.id, tag_id:, begin_ms:, end_ms:) }
ranges.each do |begin_ms, end_ms|
PostTagSection.create!(post_id: post.id,
tag_id:,
begin_ms:,
end_ms:)
end
end
PostTag.where(post_id: post.id, tag_id: (current_ids - desired_ids).to_a).kept.find_each do |post_tag|
PostTag.where(post_id: post.id,
tag_id: (current_ids - desired_ids).to_a).kept.find_each do |post_tag|
post_tag.discard_by!(@actor)
end
end
backend/app/services/post_import_google_sheets_fetcher.rb
+9 -3
ファイルの表示
@@ -17,7 +17,8 @@ class PostImportGoogleSheetsFetcher
def fetch!
publication_id, gid = parse!
throttle!
csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub?gid=#{ gid }&single=true&output=csv"
csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub" \
"?gid=#{ gid }&single=true&output=csv"
response = Preview::HttpFetcher.fetch(csv_url,
max_bytes: PostImportSourceParser::MAX_BYTES,
allowed_hosts: [HOST])
@@ -26,7 +27,10 @@ class PostImportGoogleSheetsFetcher
response.body
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed,
Preview::HttpFetcher::ResponseTooLarge => e
Rails.logger.info("post_import_google_sheets_fetch_failure #{ { error: e.class.name, message: e.message }.to_json }")
Rails.logger.info(
'post_import_google_sheets_fetch_failure '\
"#{ { error: e.class.name, message: e.message }.to_json }",
)
raise ArgumentError, 'スプレッドシートを取得できませんでした．ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください．'
end
@@ -51,7 +55,9 @@ class PostImportGoogleSheetsFetcher
def throttle!
key = "post-import-google-sheets:#{ @rate_key }"
count = Rails.cache.increment(key, 1, expires_in: RATE_WINDOW, initial: 0)
raise ArgumentError, 'スプレッドシート取得の回数が多すぎます．しばらくしてから再試行してください．' if count > RATE_LIMIT
if count > RATE_LIMIT
raise ArgumentError, 'スプレッドシート取得の回数が多すぎます．しばらくしてから再試行してください．'
end
end
def csv? response
backend/app/services/post_import_previewer.rb
+73 -21
ファイルの表示
@@ -1,5 +1,13 @@
class PostImportPreviewer
FIELDS = %w[title thumbnail_base original_created_from original_created_before duration tags parent_post_ids].freeze
FIELDS = [
'title',
'thumbnail_base',
'original_created_from',
'original_created_before',
'duration',
'tags',
'parent_post_ids',
].freeze
def initialize parsed:, url_column:, mappings: { }, existing: 'skip'
@parsed = parsed
@@ -17,20 +25,32 @@ class PostImportPreviewer
rows.map do |row|
row = row.symbolize_keys
values = row[:values] || []
attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field| [field, mapped_value(field, values)] }
attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field|
[field, mapped_value(field, values)]
}
provenance = row[:provenance]&.stringify_keys || mapped_provenance(attributes)
tag_sources = row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance)
tag_sources =
row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance)
tag_sources['manual'] = attributes['tags'].to_s if provenance['tags'] == 'manual'
url = row[:url].presence || values[@url_column].to_s.strip
provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped'
url_for_metadata = normalised_url(url) || url
existing_post = normalised_url(url).present? && Post.exists?(url: normalised_url(url))
if existing_post && @existing == 'skip'
next { source_row: row[:source_row], url: normalised_url(url) || url, attributes:,
provenance:, tag_sources:, metadata_url: url_for_metadata,
fetch_warnings: [], validation_warnings: ['既存投稿のためスキップします．'],
warnings: ['既存投稿のためスキップします．'], validation_errors: { },
status: 'warning', existing: true }
next {
source_row: row[:source_row],
url: normalised_url(url) || url,
attributes:,
provenance:,
tag_sources:,
metadata_url: url_for_metadata,
fetch_warnings: [],
validation_warnings: ['既存投稿のためスキップします．'],
warnings: ['既存投稿のためスキップします．'],
validation_errors: { },
status: 'warning',
existing: true,
}
end
url_changed = row[:metadata_url].present? && row[:metadata_url] != url_for_metadata
clear_automatic_values!(attributes, provenance, tag_sources) if url_changed
@@ -43,7 +63,8 @@ class PostImportPreviewer
when false, nil then false
else raise ArgumentError, 'メタデータ取得対象が不正です．'
end
metadata, fetch_warnings = metadata_for(url_for_metadata, should_fetch, metadata_cache, fetch_warnings)
metadata, fetch_warnings =
metadata_for(url_for_metadata, should_fetch, metadata_cache, fetch_warnings)
metadata.each do |field, value|
if field == 'tags' && provenance[field] != 'manual'
tag_sources['automatic'] = value.to_s
@@ -62,19 +83,37 @@ class PostImportPreviewer
errors[:url] = ['URL が重複しています．'] if normal_url.present? && urls.key?(normal_url)
urls[normal_url] = true if normal_url.present?
if normal_url.present? && existing_post
@existing == 'error' ? errors[:url] = ['既存投稿です．'] : validation_warnings << '既存投稿のためスキップします．'
if @existing == 'error'
errors[:url] = ['既存投稿です．']
else
validation_warnings << '既存投稿のためスキップします．'
end
end
validate_basic_data(attributes, errors)
validate_preview_tags(merged_tags(tag_sources, provenance['tags']), errors, validation_warnings)
validate_preview_tags(
merged_tags(tag_sources, provenance['tags']),
errors,
validation_warnings,
)
validate_parents(attributes['parent_post_ids'], errors)
attributes.delete('url')
attributes['tags'] = merged_tags(tag_sources, provenance['tags'])
{ source_row: row[:source_row], url: normal_url || url, attributes:, provenance:, tag_sources:,
{
source_row: row[:source_row],
url: normal_url || url,
attributes:,
provenance:,
tag_sources:,
metadata_url: url_for_metadata,
fetch_warnings:, validation_warnings:, warnings: fetch_warnings + validation_warnings,
fetch_warnings:,
validation_warnings:,
warnings: fetch_warnings + validation_warnings,
validation_errors: errors,
status: errors.present? ? 'error' : ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
existing: normal_url.present? && Post.exists?(url: normal_url) }
status: errors.present? \
? 'error' \
: ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'),
existing: normal_url.present? && Post.exists?(url: normal_url),
}
end
end
@@ -95,7 +134,9 @@ class PostImportPreviewer
return '' if mapping.blank?
return mapping['value'].to_s if mapping['kind'] == 'fixed'
Array(mapping['columns']).filter_map { |index| values[index.to_i].to_s.presence }.join(field == 'tags' ? ' ' : '')
Array(mapping['columns']).filter_map { |index|
values[index.to_i].to_s.presence
}.join(field == 'tags' ? ' ' : '')
end
def mapped_provenance attributes
@@ -122,11 +163,14 @@ class PostImportPreviewer
def merged_tags sources, origin = nil
return sources['manual'].to_s if origin == 'manual'
%w[automatic mapped fixed manual].flat_map { sources[_1].to_s.split }.uniq.join(' ')
['automatic', 'mapped', 'fixed', 'manual'].flat_map {
sources[_1].to_s.split
}.uniq.join(' ')
end
def clear_automatic_values! attributes, provenance, tag_sources
%w[title thumbnail_base original_created_from original_created_before duration].each do |field|
['title', 'thumbnail_base', 'original_created_from',
'original_created_before', 'duration'].each do |field|
attributes[field] = '' if provenance[field] == 'automatic'
end
tag_sources['automatic'] = ''
@@ -140,8 +184,13 @@ class PostImportPreviewer
warnings << 'タイトルを取得できませんでした．' if data['title'].blank?
warnings << 'サムネールを取得できませんでした．' if data['thumbnail_base'].blank?
[data.compact, warnings]
rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e
Rails.logger.info("post_import_metadata_fetch_failure #{ { error: e.class.name, message: e.message }.to_json }")
rescue Preview::UrlSafety::UnsafeUrl,
Preview::HttpFetcher::FetchFailed,
Preview::HttpFetcher::ResponseTooLarge => e
Rails.logger.info(
"post_import_metadata_fetch_failure "\
"#{ { error: e.class.name, message: e.message }.to_json }",
)
[{ }, ['メタデータを取得できませんでした．']]
end
@@ -190,7 +239,10 @@ class PostImportPreviewer
def validate_parents raw, errors
ids = raw.to_s.split.map { Integer(_1, exception: false) }
return if ids.compact.empty? && raw.to_s.blank?
errors[:parent_post_ids] = ['親投稿 Id. が不正です．'] and return if ids.any? { _1.nil? || _1 <= 0 }
errors[:parent_post_ids] = ['存在しない親投稿 Id. があります．'] if Post.where(id: ids).count != ids.uniq.length
if Post.where(id: ids).count != ids.uniq.length
errors[:parent_post_ids] = ['存在しない親投稿 Id. があります．']
end
end
end
backend/app/services/post_import_runner.rb
+73 -25
ファイルの表示
@@ -1,6 +1,16 @@
class PostImportRunner
MAX_ROWS = PostImportSourceParser::MAX_ROWS
ATTRIBUTE_KEYS = %w[title thumbnail_base original_created_from original_created_before duration tags parent_post_ids video_ms].freeze
ATTRIBUTE_KEYS = [
'title',
'thumbnail_base',
'original_created_from',
'original_created_before',
'duration',
'tags',
'parent_post_ids',
'video_ms',
].freeze
def initialize actor:, rows:, existing: 'skip'
@actor = actor
@rows = rows
@@ -9,15 +19,21 @@ class PostImportRunner
def run
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if @rows.length > MAX_ROWS
raise ArgumentError, '取込データが大きすぎます．' if @rows.sum { |row| row.to_json.bytesize } > PostImportSourceParser::MAX_BYTES
if @rows.sum { |row| row.to_json.bytesize } > PostImportSourceParser::MAX_BYTES
raise ArgumentError, '取込データが大きすぎます．'
end
normalised_rows = @rows.map { |row| normalise_row(row) }
source_rows = normalised_rows.map { _1['source_row'] }
raise ArgumentError, '元行番号が重複しています．' if source_rows.uniq.length != source_rows.length
results = normalised_rows.map { |row| run_row(row) }
{ created: results.count { _1[:status] == 'created' }, skipped: results.count { _1[:status] == 'skipped' },
failed: results.count { _1[:status] == 'failed' }, rows: results }
{
created: results.count { _1[:status] == 'created' },
skipped: results.count { _1[:status] == 'skipped' },
failed: results.count { _1[:status] == 'failed' },
rows: results,
}
end
private
@@ -26,20 +42,40 @@ class PostImportRunner
validate_row_structure!(row)
attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore }
raise ArgumentError, '取込項目が不正です．' unless (attributes.keys - ATTRIBUTE_KEYS).empty?
raise ArgumentError, '取込項目が大きすぎます．' if attributes.values.any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
raise ArgumentError, 'URL が長すぎます．' if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
validate_tag_sources!(row['tag_sources'])
preview = PostImportPreviewer.new(parsed: { columns: [], rows: [] }, url_column: 0,
mappings: {}, existing: @existing).preview_rows(
rows: [{ source_row: row['source_row'], url: row['url'],
attributes:, provenance: row['provenance'],
tag_sources: row['tag_sources'] }],
fetch_metadata: false).first
if preview[:validation_errors].present?
return { source_row: row['source_row'], status: 'failed',
errors: preview[:validation_errors] }
if attributes.values.any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
raise ArgumentError, '取込項目が大きすぎます．'
end
return row.slice('source_row').merge(status: 'skipped') if @existing == 'skip' && preview[:existing]
if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, 'URL が長すぎます．'
end
validate_tag_sources!(row['tag_sources'])
preview = PostImportPreviewer.new(
parsed: { columns: [], rows: [] },
url_column: 0,
mappings: {},
existing: @existing,
).preview_rows(
rows: [{
source_row: row['source_row'],
url: row['url'],
attributes:,
provenance: row['provenance'],
tag_sources: row['tag_sources'],
}],
fetch_metadata: false,
).first
if preview[:validation_errors].present?
return {
source_row: row['source_row'],
status: 'failed',
errors: preview[:validation_errors],
}
end
if @existing == 'skip' && preview[:existing]
return row.slice('source_row').merge(status: 'skipped')
end
attributes['tags'] = preview[:attributes]['tags']
attributes['url'] = row['url']
post = PostCreator.new(actor: @actor, attributes:).create!
@@ -53,7 +89,9 @@ class PostImportRunner
rescue ArgumentError, PostCreator::VideoMsParseError
{ source_row: row['source_row'], status: 'failed', errors: { base: ['入力値が不正です．'] } }
rescue StandardError => e
Rails.logger.error("post_import_runner_failure #{ { error: e.class.name, message: e.message }.to_json }")
Rails.logger.error(
"post_import_runner_failure #{ { error: e.class.name, message: e.message }.to_json }",
)
{ source_row: row['source_row'], status: 'failed', errors: { base: ['登録中にエラーが発生しました．'] } }
end
@@ -61,9 +99,11 @@ class PostImportRunner
return if sources.blank?
sources = sources.to_h
raise ArgumentError unless (sources.keys - %w[automatic mapped fixed manual]).empty?
raise ArgumentError unless (sources.keys - ['automatic', 'mapped', 'fixed', 'manual']).empty?
raise ArgumentError unless sources.values.all?(String)
raise ArgumentError if sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
if sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
raise ArgumentError
end
raise ArgumentError if sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES
end
@@ -71,12 +111,22 @@ class PostImportRunner
raise ArgumentError unless row['source_row'].is_a?(Integer) && row['source_row'].positive?
raise ArgumentError unless row['url'].is_a?(String)
raise ArgumentError unless row['attributes'].is_a?(Hash)
raise ArgumentError unless row['attributes'].values.all? { _1.is_a?(String) || _1.is_a?(Numeric) || _1 == true || _1 == false || _1.nil? }
unless row['attributes'].values.all? { |value|
value.is_a?(String) ||
value.is_a?(Numeric) ||
value == true ||
value == false ||
value.nil?
}
raise ArgumentError
end
provenance = row['provenance']
raise ArgumentError unless provenance.is_a?(Hash)
allowed = ATTRIBUTE_KEYS + ['url']
raise ArgumentError unless (provenance.keys - allowed).empty?
raise ArgumentError unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) }
unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) }
raise ArgumentError
end
metadata_url = row['metadata_url']
raise ArgumentError unless metadata_url.nil? || metadata_url.is_a?(String)
raise ArgumentError if metadata_url.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
@@ -86,9 +136,7 @@ class PostImportRunner
end
def normalise_row row
unless row.is_a?(Hash)
raise ArgumentError, '取込行の形式が不正です．'
end
raise ArgumentError, '取込行の形式が不正です．' unless row.is_a?(Hash)
normalised = row.deep_transform_keys { _1.to_s.underscore }
source_row = Integer(normalised['source_row'], exception: false)
backend/app/services/post_import_source_parser.rb
+28 -10
ファイルの表示
@@ -30,9 +30,17 @@ class PostImportSourceParser
ensure_limits!(table)
width = table.map(&:length).max.to_i
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです．" if width > MAX_COLUMNS
{ columns: (0...width).map { |index| { key: index.to_s,
label: "#{ column_name(index) }：#{ headers&.[](index).presence || '列' }" } },
rows: table.each_with_index.map { |values, index| { source_row: index + (@has_header ? 2 : 1), values: } } }
{
columns: (0...width).map { |index|
{
key: index.to_s,
label: "#{ column_name(index) }：#{ headers&.[](index).presence || '列' }",
}
},
rows: table.each_with_index.map { |values, index|
{ source_row: index + (@has_header ? 2 : 1), values: }
},
}
rescue CSV::MalformedCSVError => e
raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }"
end
@@ -59,17 +67,27 @@ class PostImportSourceParser
ensure_limits!(value)
unless value.all?(Hash)
invalid = value.each_with_index.filter_map { |item, index| index + 1 unless item.is_a?(Hash) }
invalid = value.each_with_index.filter_map { |item, index|
index + 1 unless item.is_a?(Hash)
}
raise ArgumentError, "JSON の投稿候補は object の配列にしてください: 行 #{ invalid.join(', ') }"
end
keys = value.flat_map(&:keys).map(&:to_s).uniq
raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです．" if keys.length > MAX_COLUMNS
{ columns: keys.each_with_index.map { |key, index| { key:, label: "#{ column_name(index) }：#{ key }" } },
rows: value.each_with_index.map { |item, index| { source_row: index + 1,
values: keys.map { |key|
value = item.key?(key) ? item[key] : item[key.to_sym]
json_cell(value)
} } } }
{
columns: keys.each_with_index.map { |key, index|
{ key:, label: "#{ column_name(index) }：#{ key }" }
},
rows: value.each_with_index.map { |item, index|
{
source_row: index + 1,
values: keys.map { |key|
json_value = item.key?(key) ? item[key] : item[key.to_sym]
json_cell(json_value)
},
}
},
}
rescue JSON::ParserError, KeyError, ArgumentError => e
raise ArgumentError, "JSON の形式が不正です: #{ e.message }"
end
backend/app/services/post_metadata_fetcher.rb
+24 -13
ファイルの表示
@@ -1,10 +1,19 @@
class PostMetadataFetcher
def self.fetch raw_url
uri, = Preview::UrlSafety.validate(raw_url)
response = Preview::HttpFetcher.fetch(uri.to_s, max_bytes: Preview::ThumbnailFetcher::HTML_MAX_BYTES)
response = Preview::HttpFetcher.fetch(
uri.to_s,
max_bytes: Preview::ThumbnailFetcher::HTML_MAX_BYTES,
)
metadata = Preview::HtmlMetadataExtractor.extract(response)
document = Nokogiri::HTML.parse(response.body)
content = -> name { document.at_css("meta[property='#{ name }'], meta[name='#{ name }']")&.[]('content')&.strip.presence }
content = lambda { |name|
document
.at_css("meta[property='#{ name }'], meta[name='#{ name }']")
&.[]('content')
&.strip
&.presence
}
duration = content.call('og:video:duration') || content.call('video:duration')
published = content.call('article:published_time') || content.call('date')
created_range = original_created_range(published)
@@ -30,17 +39,19 @@ class PostMetadataFetcher
raw = value.to_s.strip
from = Time.zone.parse(raw)
zone = '(?:Z|[+-]\d{2}:?\d{2})?'
before = case raw
when /\A\d{4}\z/ then from + 1.year
when /\A\d{4}-\d{2}\z/ then from + 1.month
when /\A\d{4}-\d{2}-\d{2}\z/ then from + 1.day
when /\A\d{4}-\d{2}-\d{2}T\d{2}#{ zone }\z/ then from + 1.hour
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}#{ zone }\z/ then from + 1.minute
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}#{ zone }\z/ then from + 1.second
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.(\d+)#{ zone }\z/
from + (10**(-Regexp.last_match(1).length))
else return nil
end
before =
case raw
when /\A\d{4}\z/ then from + 1.year
when /\A\d{4}-\d{2}\z/ then from + 1.month
when /\A\d{4}-\d{2}-\d{2}\z/ then from + 1.day
when /\A\d{4}-\d{2}-\d{2}T\d{2}#{ zone }\z/ then from + 1.hour
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}#{ zone }\z/ then from + 1.minute
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}#{ zone }\z/ then from + 1.second
when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.(\d+)#{ zone }\z/
from + (10**(-Regexp.last_match(1).length))
else
return nil
end
[from, before]
rescue ArgumentError, TypeError
nil
backend/app/services/preview/http_fetcher.rb
+4 -1
ファイルの表示
@@ -71,7 +71,10 @@ module Preview
log_failure(:timeout, url: uri&.to_s || raw_url, error: e.class.name, message: e.message)
raise FetchTimeout, e.message
rescue SocketError, SystemCallError, OpenSSL::SSL::SSLError, EOFError => e
log_failure(:network_error, url: uri&.to_s || raw_url, error: e.class.name, message: e.message)
log_failure(:network_error,
url: uri&.to_s || raw_url,
error: e.class.name,
message: e.message)
raise FetchFailed, e.message
end