diff --git a/backend/app/controllers/post_imports_controller.rb b/backend/app/controllers/post_imports_controller.rb index 87cd229..8831389 100644 --- a/backend/app/controllers/post_imports_controller.rb +++ b/backend/app/controllers/post_imports_controller.rb @@ -44,7 +44,7 @@ class PostImportsController < ApplicationController end def validate - rows = validate_rows_params + rows = normalised_import_rows(allow_warning_fields: true) changed_row = Integer(params[:changed_row], exception: false) result = PostImportPreviewer.new( @@ -61,7 +61,7 @@ class PostImportsController < ApplicationController def create result = PostImportRunner.new( actor: current_user, - rows: create_rows_params, + rows: normalised_import_rows, existing: params[:existing]).run render json: result, status: result[:created].positive? ? :created : :ok rescue ArgumentError => e @@ -117,6 +117,7 @@ class PostImportsController < ApplicationController raw = mappings.to_unsafe_h unknown_fields = raw.keys - PostImportPreviewer::FIELDS raise ArgumentError, '列割当て先が不正です．' if unknown_fields.present? + raise ArgumentError, '列割当てが大きすぎます．' if raw.to_json.bytesize > PostImportSourceParser::MAX_BYTES permitted = mappings.permit(*PostImportPreviewer::FIELDS.map { |field| @@ -139,105 +140,22 @@ class PostImportsController < ApplicationController parsed && parsed >= 0 && parsed < column_count }) ) - raise ArgumentError, '列番号が重複しています．' if Array(columns).uniq.length != Array(columns).length + parsed_columns = Array(columns).map { Integer(_1) } + raise ArgumentError, '列番号が重複しています．' if parsed_columns.uniq.length != parsed_columns.length raise ArgumentError, '固定値が不正です．' if kind == 'fixed' && !value['value'].is_a?(String) + if value['value'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES + raise ArgumentError, '固定値が大きすぎます．' + end result[field] = { 'kind' => kind, 'value' => value['value'].to_s, - 'columns' => Array(columns).map { Integer(_1) } } + 'columns' => parsed_columns } end end - def validate_rows_params + def normalised_import_rows allow_warning_fields: false rows = params[:rows] - raise ArgumentError, '取込件数が多すぎます．' unless rows.is_a?(Array) - raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS - - normalised_rows = rows.map do |row| - unless row.is_a?(Hash) || row.is_a?(ActionController::Parameters) - raise ArgumentError, '行の形式が不正です．' - end - - value = ActionController::Parameters.new(row).permit( - :source_row, - :sourceRow, - :url, - :metadata_url, - :metadataUrl, - attributes: {}, - provenance: {}, - tag_sources: {}, - tagSources: {}, - fetch_warnings: [], - fetchWarnings: [], - validation_warnings: [], - validationWarnings: []) - normalised = value.to_h.deep_transform_keys { _1.to_s.underscore } - attributes = normalised.fetch('attributes', { }) - provenance = normalised.fetch('provenance', { }) - allowed = PostImportPreviewer::FIELDS - unless attributes.is_a?(Hash) && provenance.is_a?(Hash) && - (attributes.keys - allowed).empty? && - (provenance.keys - (allowed + ['url'])).empty? - raise ArgumentError, '行の形式が不正です．' - end - unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) } - raise ArgumentError, '値の由来が不正です．' - end - raise ArgumentError, 'セルが大きすぎます．' if [normalised['url'], *attributes.values].any? { |value| - value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES - } - if normalised['tag_sources'].present? && - (!normalised['tag_sources'].is_a?(Hash) || - (normalised['tag_sources'].keys - ['automatic', 'mapped', 'fixed', 'manual']).present?) - raise ArgumentError, 'タグ由来の形式が不正です．' - end - if (normalised['tag_sources'].present? && - (!normalised['tag_sources'].values.all?(String) || - normalised['tag_sources'].values.any? { |value| - value.bytesize > PostImportSourceParser::MAX_CELL_BYTES - } || - (normalised['tag_sources'].values.sum(&:bytesize) > - PostImportSourceParser::MAX_CELL_BYTES))) - raise ArgumentError, 'タグ由来が大きすぎます．' - end - - source_row = Integer(normalised['source_row'], exception: false) - raise ArgumentError, '元行番号が不正です．' if source_row.nil? || source_row <= 0 - - normalised['source_row'] = source_row - normalised.symbolize_keys - end - source_rows = normalised_rows.map { _1[:source_row] } - if source_rows.uniq.length != source_rows.length - raise ArgumentError, '元行番号が不正です．' - end - - normalised_rows - end - - def create_rows_params - rows = params[:rows] - raise ArgumentError, '取込行の形式が不正です．' unless rows.is_a?(Array) - - rows.map do |row| - unless row.is_a?(Hash) || row.is_a?(ActionController::Parameters) - raise ArgumentError, '取込行の形式が不正です．' - end - - parameters = - row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row) - parameters.permit( - :source_row, - :sourceRow, - :url, - :metadata_url, - :metadataUrl, - attributes: {}, - provenance: {}, - tag_sources: {}, - tagSources: {}).to_h - end + PostImportRowNormaliser.normalise!(rows, allow_warning_fields:) end end diff --git a/backend/app/services/post_import_row_normaliser.rb b/backend/app/services/post_import_row_normaliser.rb new file mode 100644 index 0000000..05cbd26 --- /dev/null +++ b/backend/app/services/post_import_row_normaliser.rb @@ -0,0 +1,162 @@ +class PostImportRowNormaliser + ORIGINS = ['automatic', 'mapped', 'fixed', 'manual'].freeze + STRING_FIELDS = [ + 'title', + 'thumbnail_base', + 'original_created_from', + 'original_created_before', + 'tags', + 'parent_post_ids', + ].freeze + FLEXIBLE_FIELDS = ['duration', 'video_ms'].freeze + ATTRIBUTE_FIELDS = (STRING_FIELDS + FLEXIBLE_FIELDS).freeze + + def self.normalise! rows, allow_warning_fields: false + raise ArgumentError, '取込行の形式が不正です．' unless rows.is_a?(Array) + raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS + + normalised_rows = rows.map { normalise_row!(_1, allow_warning_fields:) } + source_rows = normalised_rows.map { _1['source_row'] } + raise ArgumentError, '元行番号が重複しています．' if source_rows.uniq.length != source_rows.length + if normalised_rows.sum { row_bytesize(_1) } > PostImportSourceParser::MAX_BYTES + raise ArgumentError, '取込データが大きすぎます．' + end + + normalised_rows + end + + def self.normalise_row! row, allow_warning_fields: + unless row.is_a?(Hash) || row.is_a?(ActionController::Parameters) + raise ArgumentError, '取込行の形式が不正です．' + end + + parameters = + row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row) + permitted = parameters.permit(*permitted_keys(allow_warning_fields)) + normalised = permitted.to_h.deep_transform_keys { _1.to_s.underscore } + + normalised['source_row'] = normalise_source_row!(normalised['source_row']) + normalise_url!(normalised['url']) + normalise_metadata_url!(normalised['metadata_url']) + normalise_attributes!(normalised.fetch('attributes', { })) + normalise_provenance!(normalised.fetch('provenance', { })) + normalise_tag_sources!(normalised['tag_sources']) + normalise_warning_values!(normalised, allow_warning_fields:) + if row_bytesize(normalised) > PostImportSourceParser::MAX_BYTES + raise ArgumentError, '取込行が大きすぎます．' + end + + normalised + end + + def self.permitted_keys allow_warning_fields + keys = [ + :source_row, + :sourceRow, + :url, + :metadata_url, + :metadataUrl, + { attributes: {} }, + { provenance: {} }, + { tag_sources: {} }, + { tagSources: {} }, + ] + return keys unless allow_warning_fields + + keys + [ + { fetch_warnings: [] }, + { fetchWarnings: [] }, + { validation_warnings: [] }, + { validationWarnings: [] }, + ] + end + private_class_method :permitted_keys + + def self.normalise_source_row! value + source_row = Integer(value, exception: false) + raise ArgumentError, '元行番号が不正です．' if source_row.nil? || source_row <= 0 + + source_row + end + private_class_method :normalise_source_row! + + def self.normalise_url! value + raise ArgumentError, 'URL の形式が不正です．' unless value.is_a?(String) + raise ArgumentError, 'URL が長すぎます．' if value.bytesize > PostImportSourceParser::MAX_CELL_BYTES + end + private_class_method :normalise_url! + + def self.normalise_metadata_url! value + raise ArgumentError, 'metadata_url の形式が不正です．' unless value.nil? || value.is_a?(String) + if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES + raise ArgumentError, 'metadata_url が長すぎます．' + end + end + private_class_method :normalise_metadata_url! + + def self.normalise_attributes! attributes + raise ArgumentError, 'attributes の形式が不正です．' unless attributes.is_a?(Hash) + raise ArgumentError, '取込項目が不正です．' unless (attributes.keys - ATTRIBUTE_FIELDS).empty? + + attributes.each do |key, value| + case key + when *STRING_FIELDS + raise ArgumentError, '取込項目の型が不正です．' unless value.nil? || value.is_a?(String) + when *FLEXIBLE_FIELDS + unless value.nil? || value.is_a?(String) || value.is_a?(Numeric) + raise ArgumentError, '取込項目の型が不正です．' + end + end + if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES + raise ArgumentError, '取込項目が大きすぎます．' + end + end + end + private_class_method :normalise_attributes! + + def self.normalise_provenance! provenance + raise ArgumentError, 'provenance の形式が不正です．' unless provenance.is_a?(Hash) + + allowed = ATTRIBUTE_FIELDS + ['url'] + raise ArgumentError, '値の由来が不正です．' unless (provenance.keys - allowed).empty? + raise ArgumentError, '値の由来が不正です．' unless provenance.values.all? { ORIGINS.include?(_1) } + end + private_class_method :normalise_provenance! + + def self.normalise_tag_sources! tag_sources + return if tag_sources.nil? + + raise ArgumentError, 'タグ由来の形式が不正です．' unless tag_sources.is_a?(Hash) + raise ArgumentError, 'タグ由来の形式が不正です．' unless (tag_sources.keys - ORIGINS).empty? + raise ArgumentError, 'タグ由来の形式が不正です．' unless tag_sources.values.all? { _1.is_a?(String) } + if tag_sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } + raise ArgumentError, 'タグ由来が大きすぎます．' + end + if tag_sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES + raise ArgumentError, 'タグ由来が大きすぎます．' + end + end + private_class_method :normalise_tag_sources! + + def self.normalise_warning_values! normalised, allow_warning_fields: + return unless allow_warning_fields + + %w[fetch_warnings validation_warnings].each do |field| + values = normalised[field] + next if values.nil? + + unless values.is_a?(Array) && values.all? { _1.is_a?(String) } + raise ArgumentError, '警告の形式が不正です．' + end + if values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } + raise ArgumentError, '警告が大きすぎます．' + end + end + end + private_class_method :normalise_warning_values! + + def self.row_bytesize row + row.to_json.bytesize + end + private_class_method :row_bytesize +end diff --git a/backend/app/services/post_import_runner.rb b/backend/app/services/post_import_runner.rb index 3644b18..3a241e6 100644 --- a/backend/app/services/post_import_runner.rb +++ b/backend/app/services/post_import_runner.rb @@ -1,16 +1,4 @@ class PostImportRunner - MAX_ROWS = PostImportSourceParser::MAX_ROWS - ATTRIBUTE_KEYS = [ - 'title', - 'thumbnail_base', - 'original_created_from', - 'original_created_before', - 'duration', - 'tags', - 'parent_post_ids', - 'video_ms', - ].freeze - def initialize actor:, rows:, existing: 'skip' @actor = actor @rows = rows @@ -18,14 +6,7 @@ class PostImportRunner end def run - raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if @rows.length > MAX_ROWS - if @rows.sum { |row| row.to_json.bytesize } > PostImportSourceParser::MAX_BYTES - raise ArgumentError, '取込データが大きすぎます．' - end - - normalised_rows = @rows.map { |row| normalise_row(row) } - source_rows = normalised_rows.map { _1['source_row'] } - raise ArgumentError, '元行番号が重複しています．' if source_rows.uniq.length != source_rows.length + normalised_rows = PostImportRowNormaliser.normalise!(@rows) results = normalised_rows.map { |row| run_row(row) } { @@ -39,31 +20,21 @@ class PostImportRunner private def run_row row - validate_row_structure!(row) - attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore } - raise ArgumentError, '取込項目が不正です．' unless (attributes.keys - ATTRIBUTE_KEYS).empty? - if attributes.values.any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES } - raise ArgumentError, '取込項目が大きすぎます．' - end - if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES - raise ArgumentError, 'URL が長すぎます．' - end - validate_tag_sources!(row['tag_sources']) + attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore } preview = PostImportPreviewer.new( - parsed: { columns: [], rows: [] }, - url_column: 0, - mappings: {}, - existing: @existing, - ).preview_rows( - rows: [{ - source_row: row['source_row'], - url: row['url'], - attributes:, - provenance: row['provenance'], - tag_sources: row['tag_sources'], - }], - fetch_metadata: false, - ).first + parsed: { columns: [], rows: [] }, + url_column: 0, + mappings: {}, + existing: @existing) + .preview_rows( + rows: [{ + source_row: row['source_row'], + url: row['url'], + attributes:, + provenance: row['provenance'], + tag_sources: row['tag_sources'], + }], + fetch_metadata: false).first if preview[:validation_errors].present? return { source_row: row['source_row'], @@ -94,55 +65,4 @@ class PostImportRunner ) { source_row: row['source_row'], status: 'failed', errors: { base: ['登録中にエラーが発生しました．'] } } end - - def validate_tag_sources! sources - return if sources.blank? - - sources = sources.to_h - raise ArgumentError unless (sources.keys - ['automatic', 'mapped', 'fixed', 'manual']).empty? - raise ArgumentError unless sources.values.all?(String) - if sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } - raise ArgumentError - end - raise ArgumentError if sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES - end - - def validate_row_structure! row - raise ArgumentError unless row['source_row'].is_a?(Integer) && row['source_row'].positive? - raise ArgumentError unless row['url'].is_a?(String) - raise ArgumentError unless row['attributes'].is_a?(Hash) - unless row['attributes'].values.all? { |value| - value.is_a?(String) || - value.is_a?(Numeric) || - value == true || - value == false || - value.nil? - } - raise ArgumentError - end - provenance = row['provenance'] - raise ArgumentError unless provenance.is_a?(Hash) - allowed = ATTRIBUTE_KEYS + ['url'] - raise ArgumentError unless (provenance.keys - allowed).empty? - unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) } - raise ArgumentError - end - metadata_url = row['metadata_url'] - raise ArgumentError unless metadata_url.nil? || metadata_url.is_a?(String) - raise ArgumentError if metadata_url.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES - raise ArgumentError if row.to_json.bytesize > PostImportSourceParser::MAX_BYTES - tag_sources = row['tag_sources'] - raise ArgumentError unless tag_sources.nil? || tag_sources.is_a?(Hash) - end - - def normalise_row row - raise ArgumentError, '取込行の形式が不正です．' unless row.is_a?(Hash) - - normalised = row.deep_transform_keys { _1.to_s.underscore } - source_row = Integer(normalised['source_row'], exception: false) - raise ArgumentError, '元行番号が不正です．' if source_row.nil? || source_row <= 0 - - normalised['source_row'] = source_row - normalised - end end diff --git a/frontend/src/pages/posts/PostImportPage.tsx b/frontend/src/pages/posts/PostImportPage.tsx index e79ef86..314ac8f 100644 --- a/frontend/src/pages/posts/PostImportPage.tsx +++ b/frontend/src/pages/posts/PostImportPage.tsx @@ -61,6 +61,7 @@ type ImportResultRow = { errors?: Record } type Mapping = { columns: string[] } +type RepairMode = 'all' | 'failed' const MAPPING_FIELDS = [ 'title', @@ -72,6 +73,7 @@ const MAPPING_FIELDS = [ 'parent_post_ids'] as const const RESULT_STATUSES: ImportResultStatus[] = ['created', 'skipped', 'failed'] +const MERGE_FIELDS = ['url', 'attributes', 'provenance', 'tagSources'] as const const ORIGIN_LABELS: Record = { automatic: '自動取得', @@ -116,6 +118,67 @@ const mergeValidatedRows = ( !(current.some (_1 => _1.sourceRow === row.sourceRow)))) +const mergePreviewRows = ( + current: ImportRow[], + previewed: ImportRow[], +): ImportRow[] => { + const currentMap = new Map (current.map (row => [row.sourceRow, row])) + + const mergedRows = previewed.map (previewedRow => { + const previous = currentMap.get (previewedRow.sourceRow) + if (previous == null) + return previewedRow + if (previous.importStatus === 'created') + return previous + + const merged = { + ...previewedRow, + attributes: { ...previewedRow.attributes }, + provenance: { ...previewedRow.provenance }, + tagSources: { ...(previewedRow.tagSources ?? { }) } } + + Object.entries (previous.provenance).forEach (([field, origin]) => { + if (origin !== 'manual') + return + + if (field === 'url') + { + merged.url = previous.url + merged.metadataUrl = previous.metadataUrl + merged.provenance.url = 'manual' + return + } + if (field === 'tags') + { + merged.attributes.tags = previous.attributes.tags ?? '' + merged.provenance.tags = 'manual' + merged.tagSources = { + ...merged.tagSources, + manual: previous.tagSources?.manual ?? previous.attributes.tags ?? '' } + return + } + + merged.attributes[field] = previous.attributes[field] ?? '' + merged.provenance[field] = 'manual' + }) + + const changed = MERGE_FIELDS.some (field => + JSON.stringify (previous[field]) !== JSON.stringify (merged[field])) + + return { + ...merged, + importStatus: changed ? 'pending' : previous.importStatus, + createdPostId: previous.createdPostId, + importErrors: changed ? undefined : previous.importErrors } + }) + + return mergedRows.concat ( + current.filter (row => + row.importStatus === 'created' + && !(previewed.some (_1 => _1.sourceRow === row.sourceRow)))) +} + + const rowMessages = (row: ImportRow): string[] => [ ...row.warnings, ...Object.values (row.validationErrors ?? { }).flat (), @@ -159,12 +222,13 @@ const PostImportPage: FC = ({ user }) => { const [parsed, setParsed] = useState (null) const [columns, setColumns] = useState ([]) const [rows, setRows] = useState ([]) - const [results, setResults] = useState ([]) const [mappings, setMappings] = useState> ({ }) const [existing, setExisting] = useState<'skip' | 'error'> ('skip') const [loading, setLoading] = useState (false) + const [repairMode, setRepairMode] = useState ('all') const validationGeneration = useRef (0) + const rowRefs = useRef> ({ }) const parseSource = async () => { setLoading (true) @@ -180,7 +244,7 @@ const PostImportPage: FC = ({ user }) => { setParsed (data) setMappings ({ }) setRows ([]) - setResults ([]) + setRepairMode ('all') if (data.columns.length === 1) { setUrlColumn ('0') @@ -220,8 +284,10 @@ const PostImportPage: FC = ({ user }) => { url_column: urlColumn, mappings, existing }) - setRows (current => mergeValidatedRows (current, data.rows)) + const mergedRows = mergePreviewRows (rows, data.rows) + setRows (mergedRows) setPhase ('preview') + await validateRows (mergedRows, -1) } catch (error) { @@ -364,8 +430,8 @@ const PostImportPage: FC = ({ user }) => { toast ({ title: `登録完了: ${ result.created } 件`, description: `スキップ ${ result.skipped } 件、失敗 ${ result.failed } 件` }) + setRepairMode ('all') setPhase ('result') - setResults (result.rows) setRows (current => current.map (row => { const resultRow = result.rows.find (_1 => _1.sourceRow === row.sourceRow) return resultRow @@ -393,6 +459,42 @@ const PostImportPage: FC = ({ user }) => { ...current, [field]: { columns: value === '' ? [] : [value] } })) + const retryRow = (sourceRow: number) => + setRows (current => current.map (row => + row.sourceRow === sourceRow && row.importStatus === 'failed' + ? { ...row, importStatus: 'pending', importErrors: undefined } + : row)) + + const retryRowFromResult = (sourceRow: number) => { + retryRow (sourceRow) + setRepairMode ('failed') + setPhase ('preview') + } + + const resultRows = rows.filter (row => + row.importStatus != null && RESULT_STATUSES.includes (row.importStatus)) + const failedCount = rows.filter (row => row.importStatus === 'failed').length + const previewRows = + repairMode === 'failed' + ? rows.filter (row => + row.importStatus == null + || row.importStatus === 'pending' + || row.importStatus === 'failed') + : rows + + useEffect (() => { + if (phase !== 'preview' || repairMode !== 'failed') + return + + const failedRow = previewRows.find (row => row.importStatus === 'failed') + if (failedRow == null) + return + + rowRefs.current[failedRow.sourceRow]?.scrollIntoView ({ + block: 'start', + behaviour: 'smooth' }) + }, [phase, previewRows, repairMode]) + if (!(editable)) return @@ -587,19 +689,30 @@ const PostImportPage: FC = ({ user }) => { 自動取得を更新 + {repairMode === 'failed' && ( +

+ 失敗 {failedCount} 件を表示しています． +

)}
- {rows.map ((row, index) => ( + {previewRows.map (row => ( _1.sourceRow === row.sourceRow)} + setRef={node => { + rowRefs.current[row.sourceRow] = node + }} update={updateAttribute} - updateURL={updateURL}/>))} + updateURL={updateURL} + retryRow={retryRow}/>))}
+ )} + ))} )} @@ -646,13 +781,16 @@ const PostImportPage: FC = ({ user }) => { const RowCard = ( - { row, index, update, updateURL }: { + { row, index, setRef, update, updateURL, retryRow }: { row: ImportRow index: number + setRef: (node: HTMLElement | null) => void update: (index: number, field: string, value: string) => void - updateURL: (index: number, value: string) => void }, + updateURL: (index: number, value: string) => void + retryRow: (sourceRow: number) => void }, ) => (

@@ -710,6 +848,13 @@ const RowCard = ( value={row.attributes.parentPostIds ?? ''} origin={row.provenance.parentPostIds} onCommit={value => update (index, 'parentPostIds', value)}/> + {row.importStatus === 'failed' && ( + )} )}

)