miteruzo/btrc-hub
1
0
フォーク 0
コード 課題 61 Pull リクエスト 1 リリース Wiki アクティビティ

#399

ソースを参照
このコミットが含まれているのは:
みてるぞ
2026-07-12 12:43:31 +09:00
親 7bcb76516c
コミット 769966648b
4個のファイルの変更，354行の追加，209行の削除
Patchファイルをダウンロード Diffファイルをダウンロード すべてのファイルを展開 すべてのファイルを折り畳む
backend/app/controllers/post_imports_controller.rb
+11 -93
ファイルの表示
@@ -44,7 +44,7 @@ class PostImportsController < ApplicationController
end
def validate
rows = validate_rows_params
rows = normalised_import_rows(allow_warning_fields: true)
changed_row = Integer(params[:changed_row], exception: false)
result =
PostImportPreviewer.new(
@@ -61,7 +61,7 @@ class PostImportsController < ApplicationController
def create
result = PostImportRunner.new(
actor: current_user,
rows: create_rows_params,
rows: normalised_import_rows,
existing: params[:existing]).run
render json: result, status: result[:created].positive? ? :created : :ok
rescue ArgumentError => e
@@ -117,6 +117,7 @@ class PostImportsController < ApplicationController
raw = mappings.to_unsafe_h
unknown_fields = raw.keys - PostImportPreviewer::FIELDS
raise ArgumentError, '列割当て先が不正です．' if unknown_fields.present?
raise ArgumentError, '列割当てが大きすぎます．' if raw.to_json.bytesize > PostImportSourceParser::MAX_BYTES
permitted =
mappings.permit(*PostImportPreviewer::FIELDS.map { |field|
@@ -139,105 +140,22 @@ class PostImportsController < ApplicationController
parsed && parsed >= 0 && parsed < column_count
})
)
raise ArgumentError, '列番号が重複しています．' if Array(columns).uniq.length != Array(columns).length
parsed_columns = Array(columns).map { Integer(_1) }
raise ArgumentError, '列番号が重複しています．' if parsed_columns.uniq.length != parsed_columns.length
raise ArgumentError, '固定値が不正です．' if kind == 'fixed' && !value['value'].is_a?(String)
if value['value'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, '固定値が大きすぎます．'
end
result[field] = {
'kind' => kind,
'value' => value['value'].to_s,
'columns' => Array(columns).map { Integer(_1) } }
'columns' => parsed_columns }
end
end
def validate_rows_params
def normalised_import_rows allow_warning_fields: false
rows = params[:rows]
raise ArgumentError, '取込件数が多すぎます．' unless rows.is_a?(Array)
raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS
normalised_rows = rows.map do |row|
unless row.is_a?(Hash) || row.is_a?(ActionController::Parameters)
raise ArgumentError, '行の形式が不正です．'
end
value = ActionController::Parameters.new(row).permit(
:source_row,
:sourceRow,
:url,
:metadata_url,
:metadataUrl,
attributes: {},
provenance: {},
tag_sources: {},
tagSources: {},
fetch_warnings: [],
fetchWarnings: [],
validation_warnings: [],
validationWarnings: [])
normalised = value.to_h.deep_transform_keys { _1.to_s.underscore }
attributes = normalised.fetch('attributes', { })
provenance = normalised.fetch('provenance', { })
allowed = PostImportPreviewer::FIELDS
unless attributes.is_a?(Hash) && provenance.is_a?(Hash) &&
(attributes.keys - allowed).empty? &&
(provenance.keys - (allowed + ['url'])).empty?
raise ArgumentError, '行の形式が不正です．'
end
unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) }
raise ArgumentError, '値の由来が不正です．'
end
raise ArgumentError, 'セルが大きすぎます．' if [normalised['url'], *attributes.values].any? { |value|
value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
}
if normalised['tag_sources'].present? &&
(!normalised['tag_sources'].is_a?(Hash) ||
(normalised['tag_sources'].keys - ['automatic', 'mapped', 'fixed', 'manual']).present?)
raise ArgumentError, 'タグ由来の形式が不正です．'
end
if (normalised['tag_sources'].present? &&
(!normalised['tag_sources'].values.all?(String) ||
normalised['tag_sources'].values.any? { |value|
value.bytesize > PostImportSourceParser::MAX_CELL_BYTES
} ||
(normalised['tag_sources'].values.sum(&:bytesize) >
PostImportSourceParser::MAX_CELL_BYTES)))
raise ArgumentError, 'タグ由来が大きすぎます．'
end
source_row = Integer(normalised['source_row'], exception: false)
raise ArgumentError, '元行番号が不正です．' if source_row.nil? || source_row <= 0
normalised['source_row'] = source_row
normalised.symbolize_keys
end
source_rows = normalised_rows.map { _1[:source_row] }
if source_rows.uniq.length != source_rows.length
raise ArgumentError, '元行番号が不正です．'
end
normalised_rows
end
def create_rows_params
rows = params[:rows]
raise ArgumentError, '取込行の形式が不正です．' unless rows.is_a?(Array)
rows.map do |row|
unless row.is_a?(Hash) || row.is_a?(ActionController::Parameters)
raise ArgumentError, '取込行の形式が不正です．'
end
parameters =
row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row)
parameters.permit(
:source_row,
:sourceRow,
:url,
:metadata_url,
:metadataUrl,
attributes: {},
provenance: {},
tag_sources: {},
tagSources: {}).to_h
end
PostImportRowNormaliser.normalise!(rows, allow_warning_fields:)
end
end
backend/app/services/post_import_row_normaliser.rb
+162
ファイルの表示
@@ -0,0 +1,162 @@
class PostImportRowNormaliser
ORIGINS = ['automatic', 'mapped', 'fixed', 'manual'].freeze
STRING_FIELDS = [
'title',
'thumbnail_base',
'original_created_from',
'original_created_before',
'tags',
'parent_post_ids',
].freeze
FLEXIBLE_FIELDS = ['duration', 'video_ms'].freeze
ATTRIBUTE_FIELDS = (STRING_FIELDS + FLEXIBLE_FIELDS).freeze
def self.normalise! rows, allow_warning_fields: false
raise ArgumentError, '取込行の形式が不正です．' unless rows.is_a?(Array)
raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS
normalised_rows = rows.map { normalise_row!(_1, allow_warning_fields:) }
source_rows = normalised_rows.map { _1['source_row'] }
raise ArgumentError, '元行番号が重複しています．' if source_rows.uniq.length != source_rows.length
if normalised_rows.sum { row_bytesize(_1) } > PostImportSourceParser::MAX_BYTES
raise ArgumentError, '取込データが大きすぎます．'
end
normalised_rows
end
def self.normalise_row! row, allow_warning_fields:
unless row.is_a?(Hash) || row.is_a?(ActionController::Parameters)
raise ArgumentError, '取込行の形式が不正です．'
end
parameters =
row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row)
permitted = parameters.permit(*permitted_keys(allow_warning_fields))
normalised = permitted.to_h.deep_transform_keys { _1.to_s.underscore }
normalised['source_row'] = normalise_source_row!(normalised['source_row'])
normalise_url!(normalised['url'])
normalise_metadata_url!(normalised['metadata_url'])
normalise_attributes!(normalised.fetch('attributes', { }))
normalise_provenance!(normalised.fetch('provenance', { }))
normalise_tag_sources!(normalised['tag_sources'])
normalise_warning_values!(normalised, allow_warning_fields:)
if row_bytesize(normalised) > PostImportSourceParser::MAX_BYTES
raise ArgumentError, '取込行が大きすぎます．'
end
normalised
end
def self.permitted_keys allow_warning_fields
keys = [
:source_row,
:sourceRow,
:url,
:metadata_url,
:metadataUrl,
{ attributes: {} },
{ provenance: {} },
{ tag_sources: {} },
{ tagSources: {} },
]
return keys unless allow_warning_fields
keys + [
{ fetch_warnings: [] },
{ fetchWarnings: [] },
{ validation_warnings: [] },
{ validationWarnings: [] },
]
end
private_class_method :permitted_keys
def self.normalise_source_row! value
source_row = Integer(value, exception: false)
raise ArgumentError, '元行番号が不正です．' if source_row.nil? || source_row <= 0
source_row
end
private_class_method :normalise_source_row!
def self.normalise_url! value
raise ArgumentError, 'URL の形式が不正です．' unless value.is_a?(String)
raise ArgumentError, 'URL が長すぎます．' if value.bytesize > PostImportSourceParser::MAX_CELL_BYTES
end
private_class_method :normalise_url!
def self.normalise_metadata_url! value
raise ArgumentError, 'metadata_url の形式が不正です．' unless value.nil? || value.is_a?(String)
if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, 'metadata_url が長すぎます．'
end
end
private_class_method :normalise_metadata_url!
def self.normalise_attributes! attributes
raise ArgumentError, 'attributes の形式が不正です．' unless attributes.is_a?(Hash)
raise ArgumentError, '取込項目が不正です．' unless (attributes.keys - ATTRIBUTE_FIELDS).empty?
attributes.each do |key, value|
case key
when *STRING_FIELDS
raise ArgumentError, '取込項目の型が不正です．' unless value.nil? || value.is_a?(String)
when *FLEXIBLE_FIELDS
unless value.nil? || value.is_a?(String) || value.is_a?(Numeric)
raise ArgumentError, '取込項目の型が不正です．'
end
end
if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, '取込項目が大きすぎます．'
end
end
end
private_class_method :normalise_attributes!
def self.normalise_provenance! provenance
raise ArgumentError, 'provenance の形式が不正です．' unless provenance.is_a?(Hash)
allowed = ATTRIBUTE_FIELDS + ['url']
raise ArgumentError, '値の由来が不正です．' unless (provenance.keys - allowed).empty?
raise ArgumentError, '値の由来が不正です．' unless provenance.values.all? { ORIGINS.include?(_1) }
end
private_class_method :normalise_provenance!
def self.normalise_tag_sources! tag_sources
return if tag_sources.nil?
raise ArgumentError, 'タグ由来の形式が不正です．' unless tag_sources.is_a?(Hash)
raise ArgumentError, 'タグ由来の形式が不正です．' unless (tag_sources.keys - ORIGINS).empty?
raise ArgumentError, 'タグ由来の形式が不正です．' unless tag_sources.values.all? { _1.is_a?(String) }
if tag_sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
raise ArgumentError, 'タグ由来が大きすぎます．'
end
if tag_sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, 'タグ由来が大きすぎます．'
end
end
private_class_method :normalise_tag_sources!
def self.normalise_warning_values! normalised, allow_warning_fields:
return unless allow_warning_fields
%w[fetch_warnings validation_warnings].each do |field|
values = normalised[field]
next if values.nil?
unless values.is_a?(Array) && values.all? { _1.is_a?(String) }
raise ArgumentError, '警告の形式が不正です．'
end
if values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
raise ArgumentError, '警告が大きすぎます．'
end
end
end
private_class_method :normalise_warning_values!
def self.row_bytesize row
row.to_json.bytesize
end
private_class_method :row_bytesize
end
backend/app/services/post_import_runner.rb
+15 -95
ファイルの表示
@@ -1,16 +1,4 @@
class PostImportRunner
MAX_ROWS = PostImportSourceParser::MAX_ROWS
ATTRIBUTE_KEYS = [
'title',
'thumbnail_base',
'original_created_from',
'original_created_before',
'duration',
'tags',
'parent_post_ids',
'video_ms',
].freeze
def initialize actor:, rows:, existing: 'skip'
@actor = actor
@rows = rows
@@ -18,14 +6,7 @@ class PostImportRunner
end
def run
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if @rows.length > MAX_ROWS
if @rows.sum { |row| row.to_json.bytesize } > PostImportSourceParser::MAX_BYTES
raise ArgumentError, '取込データが大きすぎます．'
end
normalised_rows = @rows.map { |row| normalise_row(row) }
source_rows = normalised_rows.map { _1['source_row'] }
raise ArgumentError, '元行番号が重複しています．' if source_rows.uniq.length != source_rows.length
normalised_rows = PostImportRowNormaliser.normalise!(@rows)
results = normalised_rows.map { |row| run_row(row) }
{
@@ -39,31 +20,21 @@ class PostImportRunner
private
def run_row row
validate_row_structure!(row)
attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore }
raise ArgumentError, '取込項目が不正です．' unless (attributes.keys - ATTRIBUTE_KEYS).empty?
if attributes.values.any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
raise ArgumentError, '取込項目が大きすぎます．'
end
if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError, 'URL が長すぎます．'
end
validate_tag_sources!(row['tag_sources'])
attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore }
preview = PostImportPreviewer.new(
parsed: { columns: [], rows: [] },
url_column: 0,
mappings: {},
existing: @existing,
).preview_rows(
rows: [{
source_row: row['source_row'],
url: row['url'],
attributes:,
provenance: row['provenance'],
tag_sources: row['tag_sources'],
}],
fetch_metadata: false,
).first
parsed: { columns: [], rows: [] },
url_column: 0,
mappings: {},
existing: @existing)
.preview_rows(
rows: [{
source_row: row['source_row'],
url: row['url'],
attributes:,
provenance: row['provenance'],
tag_sources: row['tag_sources'],
}],
fetch_metadata: false).first
if preview[:validation_errors].present?
return {
source_row: row['source_row'],
@@ -94,55 +65,4 @@ class PostImportRunner
)
{ source_row: row['source_row'], status: 'failed', errors: { base: ['登録中にエラーが発生しました．'] } }
end
def validate_tag_sources! sources
return if sources.blank?
sources = sources.to_h
raise ArgumentError unless (sources.keys - ['automatic', 'mapped', 'fixed', 'manual']).empty?
raise ArgumentError unless sources.values.all?(String)
if sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES }
raise ArgumentError
end
raise ArgumentError if sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES
end
def validate_row_structure! row
raise ArgumentError unless row['source_row'].is_a?(Integer) && row['source_row'].positive?
raise ArgumentError unless row['url'].is_a?(String)
raise ArgumentError unless row['attributes'].is_a?(Hash)
unless row['attributes'].values.all? { |value|
value.is_a?(String) ||
value.is_a?(Numeric) ||
value == true ||
value == false ||
value.nil?
}
raise ArgumentError
end
provenance = row['provenance']
raise ArgumentError unless provenance.is_a?(Hash)
allowed = ATTRIBUTE_KEYS + ['url']
raise ArgumentError unless (provenance.keys - allowed).empty?
unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) }
raise ArgumentError
end
metadata_url = row['metadata_url']
raise ArgumentError unless metadata_url.nil? || metadata_url.is_a?(String)
raise ArgumentError if metadata_url.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES
raise ArgumentError if row.to_json.bytesize > PostImportSourceParser::MAX_BYTES
tag_sources = row['tag_sources']
raise ArgumentError unless tag_sources.nil? || tag_sources.is_a?(Hash)
end
def normalise_row row
raise ArgumentError, '取込行の形式が不正です．' unless row.is_a?(Hash)
normalised = row.deep_transform_keys { _1.to_s.underscore }
source_row = Integer(normalised['source_row'], exception: false)
raise ArgumentError, '元行番号が不正です．' if source_row.nil? || source_row <= 0
normalised['source_row'] = source_row
normalised
end
end