diff --git a/AGENTS.md b/AGENTS.md index 5917b9c..c73df4c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -114,6 +114,9 @@ npm run preview - Ruby: `render` 系メソッド呼び出しでは、keyword 引数付きでも括弧を書かない。 - Ruby: never put a line break immediately before `)`. - Ruby: do not use `%w` or `%i`. +- Ruby: TypeScript / TSX の associative syntax に近い感覚で読むこと。 + Hash の `}` は block の `}` ではない。call の `)` も function 定義の + parameter-list `)` ではない。delimiter の役割を見誤らないこと。 - In Ruby, when an `if` condition is split across multiple lines and combines clauses with `&&` or `||`, wrap the whole condition in parentheses. - Ruby hashes are not blocks; keep `}` on the same line as the final pair. @@ -127,6 +130,152 @@ npm run preview - Keep the first element on the same line as `[` by default. - If an array would exceed the line limit, break after `[` and indent elements by 4 spaces. + +### Ruby delimiter and wrapping rules + +- Ruby でも、closing delimiter は glyph ではなく syntax role で判定する。 + `}` が block close なのか hash close なのか、`)` が method call なのか + grouping なのかを区別してから配置すること。 +- Ruby の multi-line method call では、receiver / method と opening `(` を + 同じ行に保ち、closing `)` を単独行へ落とさない。 +- Ruby の multi-line hash literal / keyword-like argument hash では、opening + `{` を最初の pair と同じ行に置き、closing `}` を最後の pair と同じ行に + 置く。Prettier 的な縦開き・縦閉じをしない。 +- Ruby の `if` / `unless` / `case` 条件で、複数行に分けるだけで安易に + `if ... end` へ展開しない。局所の既存コードが modifier 形式ならそれに + そろえる。 +- Ruby の guard 条件は、1 行で収まるなら modifier 形式を優先する。2 行以上に + なるなら通常の block 形式へ切り替へてよい。 +- Ruby の method chain や call argument を折り返す際、call-site の `)` + 直前で行を空けたり、closing delimiter を block のやうに独立させない。 + +Bad: + +```rb +source = + if params[:format] == 'google_sheets' + PostImportGoogleSheetsFetcher.fetch!(params[:source], + rate_key: current_user.id) + else + params[:source] + end +parsed = PostImportSourceParser.new( + source:, + format:, + has_header: params[:has_header], + json_path: params[:json_path], +).parse +``` + +Good: + +```rb +source = + if params[:format] == 'google_sheets' + PostImportGoogleSheetsFetcher.fetch!(params[:source], + rate_key: current_user.id) + else + params[:source] + end +parsed = PostImportSourceParser.new( + source:, + format:, + has_header: params[:has_header], + json_path: params[:json_path]).parse +``` + +Bad: + +```rb +result[field] = { + 'kind' => kind, + 'value' => value['value'].to_s, + 'columns' => Array(columns).map { Integer(_1) }, +} +``` + +Good: + +```rb +result[field] = { + 'kind' => kind, + 'value' => value['value'].to_s, + 'columns' => Array(columns).map { Integer(_1) } } +``` + +Bad: + +```rb +unless rows.all? { |row| + values = Array(row['values']) + values.length <= columns.length && + values.all? { |value| + value.is_a?(String) && + value.bytesize <= PostImportSourceParser::MAX_CELL_BYTES + } + } + raise ArgumentError, '解析結果のセルが不正です．' +end +``` + +Good: + +```rb +raise ArgumentError, '解析結果のセルが不正です．' unless rows.all? { |row| + values = Array(row['values']) + values.length <= columns.length && values.all? { |value| + value.is_a?(String) && value.bytesize <= PostImportSourceParser::MAX_CELL_BYTES + } +} +``` + +Bad: + +```rb +if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES + raise ArgumentError, 'URL が長すぎます．' +end +``` + +Good: + +```rb +raise ArgumentError, 'URL が長すぎます．' if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES +``` + +Bad: + +```rb +parameters = row.is_a?(ActionController::Parameters) ? row : + ActionController::Parameters.new(row) +``` + +Good: + +```rb +parameters = + row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row) +``` + +Bad: + +```rb +PostTagSection.create!( + post_id: post.id, + tag_id:, + begin_ms:, + end_ms:, +) +``` + +Good: + +```rb +PostTagSection.create!(post_id: post.id, + tag_id:, + begin_ms:, + end_ms:) +``` - TypeScript and Python: use GNU-style spacing before parentheses where syntactically valid. - Never write Ruby, TypeScript, or TSX lines longer than 99 characters. diff --git a/backend/app/controllers/post_imports_controller.rb b/backend/app/controllers/post_imports_controller.rb index 855e2b4..87cd229 100644 --- a/backend/app/controllers/post_imports_controller.rb +++ b/backend/app/controllers/post_imports_controller.rb @@ -1,16 +1,23 @@ class PostImportsController < ApplicationController + MAPPING_PROPERTIES = ['kind', 'value', 'columns'].freeze + MAPPING_KINDS = ['columns', 'fixed'].freeze + before_action :require_member! def parse source = if params[:format] == 'google_sheets' - PostImportGoogleSheetsFetcher.fetch!(params[:source], rate_key: current_user.id) + PostImportGoogleSheetsFetcher.fetch!(params[:source], + rate_key: current_user.id) else params[:source] end format = params[:format] == 'google_sheets' ? 'csv' : params[:format] - parsed = PostImportSourceParser.new(source:, format:, - has_header: params[:has_header], json_path: params[:json_path]).parse + parsed = PostImportSourceParser.new( + source:, + format:, + has_header: params[:has_header], + json_path: params[:json_path]).parse render json: parsed rescue ArgumentError => e render_bad_request(e.message) @@ -20,11 +27,17 @@ class PostImportsController < ApplicationController parsed = parsed_params raw_url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil) return render_bad_request('URL 列を選択してください．') if raw_url_column.nil? - url_column = Integer(raw_url_column, exception: false) - return render_bad_request('URL 列が不正です．') if url_column.nil? || url_column.negative? || url_column >= parsed[:columns].length - rows = PostImportPreviewer.new(parsed:, url_column:, mappings: mappings_params(parsed[:columns].length), - existing: params[:existing]).preview + url_column = Integer(raw_url_column, exception: false) + if url_column.nil? || url_column.negative? || url_column >= parsed[:columns].length + return render_bad_request('URL 列が不正です．') + end + + rows = PostImportPreviewer.new( + parsed:, + url_column:, + mappings: mappings_params(parsed[:columns].length), + existing: params[:existing]).preview render json: parsed.slice(:columns).merge(rows:) rescue ArgumentError => e render_bad_request(e.message) @@ -33,18 +46,23 @@ class PostImportsController < ApplicationController def validate rows = validate_rows_params changed_row = Integer(params[:changed_row], exception: false) - result = PostImportPreviewer.new(parsed: { columns: [], rows: [] }, url_column: 0, - mappings: {}, existing: params[:existing]).preview_rows( - rows:, fetch_metadata: changed_row, - metadata_cache: { }) + result = + PostImportPreviewer.new( + parsed: { columns: [], rows: [] }, + url_column: 0, + mappings: {}, + existing: params[:existing]) + .preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {}) render json: { rows: result } rescue ArgumentError => e render_bad_request(e.message) end def create - result = PostImportRunner.new(actor: current_user, rows: create_rows_params, - existing: params[:existing]).run + result = PostImportRunner.new( + actor: current_user, + rows: create_rows_params, + existing: params[:existing]).run render json: result, status: result[:created].positive? ? :created : :ok rescue ArgumentError => e render_bad_request(e.message) @@ -63,14 +81,20 @@ class PostImportsController < ApplicationController value = params.require(:parsed).to_unsafe_h.deep_transform_keys { _1.to_s.underscore } columns = Array(value['columns']) rows = Array(value['rows']) - raise ArgumentError, '解析結果の形式が不正です．' unless columns.all?(Hash) && rows.all?(Hash) + unless columns.all?(Hash) && rows.all?(Hash) + raise ArgumentError, '解析結果の形式が不正です．' + end raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS raise ArgumentError, '列数が多すぎます．' if columns.length > 50 - unless rows.all? { |row| Array(row['values']).length <= columns.length && - Array(row['values']).all? { _1.is_a?(String) && _1.bytesize <= PostImportSourceParser::MAX_CELL_BYTES } } - raise ArgumentError, '解析結果のセルが不正です．' + raise ArgumentError, '解析結果のセルが不正です．' unless rows.all? { |row| + values = Array(row['values']) + values.length <= columns.length && values.all? { |value| + value.is_a?(String) && value.bytesize <= PostImportSourceParser::MAX_CELL_BYTES + } + } + if rows.sum { Array(_1['values']).sum(&:bytesize) } > PostImportSourceParser::MAX_BYTES + raise ArgumentError, '解析結果が大きすぎます．' end - raise ArgumentError, '解析結果が大きすぎます．' if rows.sum { Array(_1['values']).sum(&:bytesize) } > PostImportSourceParser::MAX_BYTES parsed_rows = rows.map do |row| source_row = Integer(row['source_row'], exception: false) @@ -78,7 +102,9 @@ class PostImportsController < ApplicationController { source_row:, values: Array(row['values']) } end - raise ArgumentError, '元行番号が重複しています．' if parsed_rows.map { _1[:source_row] }.uniq.length != parsed_rows.length + if parsed_rows.map { _1[:source_row] }.uniq.length != parsed_rows.length + raise ArgumentError, '元行番号が重複しています．' + end { columns: columns.map { |column| column.slice('key', 'label') }, rows: parsed_rows } end @@ -92,41 +118,61 @@ class PostImportsController < ApplicationController unknown_fields = raw.keys - PostImportPreviewer::FIELDS raise ArgumentError, '列割当て先が不正です．' if unknown_fields.present? - permitted = mappings.permit(*PostImportPreviewer::FIELDS.map { |field| - { field => [:kind, :value, { columns: [] }] } - }).to_h + permitted = + mappings.permit(*PostImportPreviewer::FIELDS.map { |field| + { field => [:kind, :value, { columns: [] }] } + }).to_h raw.each_with_object({ }) do |(field, value), result| raise ArgumentError, '列割当ての形式が不正です．' unless value.is_a?(Hash) - raise ArgumentError, '列割当ての項目が不正です．' unless (value.keys - ['kind', 'value', 'columns']).empty? + unless (value.keys - MAPPING_PROPERTIES).empty? + raise ArgumentError, '列割当ての項目が不正です．' + end value = permitted.fetch(field) kind = value['kind'].presence || 'columns' - raise ArgumentError, '列割当ての種別が不正です．' unless kind.in?(['columns', 'fixed']) + raise ArgumentError, '列割当ての種別が不正です．' unless kind.in?(MAPPING_KINDS) columns = value['columns'] - unless columns.nil? || (columns.is_a?(Array) && columns.all? { |index| - parsed = Integer(index, exception: false) - parsed && parsed >= 0 && parsed < column_count - }) - raise ArgumentError, '列番号が不正です．' - end + raise ArgumentError, '列番号が不正です．' unless ( + columns.nil? || (columns.is_a?(Array) && columns.all? { |index| + parsed = Integer(index, exception: false) + parsed && parsed >= 0 && parsed < column_count + }) + ) raise ArgumentError, '列番号が重複しています．' if Array(columns).uniq.length != Array(columns).length raise ArgumentError, '固定値が不正です．' if kind == 'fixed' && !value['value'].is_a?(String) - result[field] = { 'kind' => kind, 'value' => value['value'].to_s, - 'columns' => Array(columns).map { Integer(_1) } } + result[field] = { + 'kind' => kind, + 'value' => value['value'].to_s, + 'columns' => Array(columns).map { Integer(_1) } } end end def validate_rows_params - rows = Array(params[:rows]) + rows = params[:rows] + raise ArgumentError, '取込件数が多すぎます．' unless rows.is_a?(Array) raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS normalised_rows = rows.map do |row| - value = ActionController::Parameters.new(row).permit(:source_row, :sourceRow, :url, :metadata_url, :metadataUrl, - attributes: {}, provenance: {}, tag_sources: {}, tagSources: {}, - fetch_warnings: [], fetchWarnings: [], - validation_warnings: [], validationWarnings: []) + unless row.is_a?(Hash) || row.is_a?(ActionController::Parameters) + raise ArgumentError, '行の形式が不正です．' + end + + value = ActionController::Parameters.new(row).permit( + :source_row, + :sourceRow, + :url, + :metadata_url, + :metadataUrl, + attributes: {}, + provenance: {}, + tag_sources: {}, + tagSources: {}, + fetch_warnings: [], + fetchWarnings: [], + validation_warnings: [], + validationWarnings: []) normalised = value.to_h.deep_transform_keys { _1.to_s.underscore } attributes = normalised.fetch('attributes', { }) provenance = normalised.fetch('provenance', { }) @@ -139,16 +185,21 @@ class PostImportsController < ApplicationController unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) } raise ArgumentError, '値の由来が不正です．' end - raise ArgumentError, 'セルが大きすぎます．' if [normalised['url'], *attributes.values].any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES } + raise ArgumentError, 'セルが大きすぎます．' if [normalised['url'], *attributes.values].any? { |value| + value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES + } if normalised['tag_sources'].present? && (!normalised['tag_sources'].is_a?(Hash) || (normalised['tag_sources'].keys - ['automatic', 'mapped', 'fixed', 'manual']).present?) raise ArgumentError, 'タグ由来の形式が不正です．' end - if normalised['tag_sources'].present? && - (!normalised['tag_sources'].values.all?(String) || - normalised['tag_sources'].values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } || - normalised['tag_sources'].values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES) + if (normalised['tag_sources'].present? && + (!normalised['tag_sources'].values.all?(String) || + normalised['tag_sources'].values.any? { |value| + value.bytesize > PostImportSourceParser::MAX_CELL_BYTES + } || + (normalised['tag_sources'].values.sum(&:bytesize) > + PostImportSourceParser::MAX_CELL_BYTES))) raise ArgumentError, 'タグ由来が大きすぎます．' end @@ -175,9 +226,18 @@ class PostImportsController < ApplicationController raise ArgumentError, '取込行の形式が不正です．' end - parameters = row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row) - parameters.permit(:source_row, :sourceRow, :url, :metadata_url, :metadataUrl, - attributes: {}, provenance: {}, tag_sources: {}, tagSources: {}).to_h + parameters = + row.is_a?(ActionController::Parameters) ? row : ActionController::Parameters.new(row) + parameters.permit( + :source_row, + :sourceRow, + :url, + :metadata_url, + :metadataUrl, + attributes: {}, + provenance: {}, + tag_sources: {}, + tagSources: {}).to_h end end end diff --git a/backend/app/services/post_creator.rb b/backend/app/services/post_creator.rb index 816ef48..4955d0d 100644 --- a/backend/app/services/post_creator.rb +++ b/backend/app/services/post_creator.rb @@ -90,9 +90,15 @@ class PostCreator end PostTagSection.where(post_id: post.id).destroy_all sections.each do |tag_id, ranges| - ranges.each { |begin_ms, end_ms| PostTagSection.create!(post_id: post.id, tag_id:, begin_ms:, end_ms:) } + ranges.each do |begin_ms, end_ms| + PostTagSection.create!(post_id: post.id, + tag_id:, + begin_ms:, + end_ms:) + end end - PostTag.where(post_id: post.id, tag_id: (current_ids - desired_ids).to_a).kept.find_each do |post_tag| + PostTag.where(post_id: post.id, + tag_id: (current_ids - desired_ids).to_a).kept.find_each do |post_tag| post_tag.discard_by!(@actor) end end diff --git a/backend/app/services/post_import_google_sheets_fetcher.rb b/backend/app/services/post_import_google_sheets_fetcher.rb index 23f564f..2486612 100644 --- a/backend/app/services/post_import_google_sheets_fetcher.rb +++ b/backend/app/services/post_import_google_sheets_fetcher.rb @@ -17,7 +17,8 @@ class PostImportGoogleSheetsFetcher def fetch! publication_id, gid = parse! throttle! - csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub?gid=#{ gid }&single=true&output=csv" + csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub" \ + "?gid=#{ gid }&single=true&output=csv" response = Preview::HttpFetcher.fetch(csv_url, max_bytes: PostImportSourceParser::MAX_BYTES, allowed_hosts: [HOST]) @@ -26,7 +27,10 @@ class PostImportGoogleSheetsFetcher response.body rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e - Rails.logger.info("post_import_google_sheets_fetch_failure #{ { error: e.class.name, message: e.message }.to_json }") + Rails.logger.info( + 'post_import_google_sheets_fetch_failure '\ + "#{ { error: e.class.name, message: e.message }.to_json }", + ) raise ArgumentError, 'スプレッドシートを取得できませんでした．ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください．' end @@ -51,7 +55,9 @@ class PostImportGoogleSheetsFetcher def throttle! key = "post-import-google-sheets:#{ @rate_key }" count = Rails.cache.increment(key, 1, expires_in: RATE_WINDOW, initial: 0) - raise ArgumentError, 'スプレッドシート取得の回数が多すぎます．しばらくしてから再試行してください．' if count > RATE_LIMIT + if count > RATE_LIMIT + raise ArgumentError, 'スプレッドシート取得の回数が多すぎます．しばらくしてから再試行してください．' + end end def csv? response diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index 80fc5d7..8766d3e 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -1,5 +1,13 @@ class PostImportPreviewer - FIELDS = %w[title thumbnail_base original_created_from original_created_before duration tags parent_post_ids].freeze + FIELDS = [ + 'title', + 'thumbnail_base', + 'original_created_from', + 'original_created_before', + 'duration', + 'tags', + 'parent_post_ids', + ].freeze def initialize parsed:, url_column:, mappings: { }, existing: 'skip' @parsed = parsed @@ -17,20 +25,32 @@ class PostImportPreviewer rows.map do |row| row = row.symbolize_keys values = row[:values] || [] - attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field| [field, mapped_value(field, values)] } + attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field| + [field, mapped_value(field, values)] + } provenance = row[:provenance]&.stringify_keys || mapped_provenance(attributes) - tag_sources = row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance) + tag_sources = + row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance) tag_sources['manual'] = attributes['tags'].to_s if provenance['tags'] == 'manual' url = row[:url].presence || values[@url_column].to_s.strip provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped' url_for_metadata = normalised_url(url) || url existing_post = normalised_url(url).present? && Post.exists?(url: normalised_url(url)) if existing_post && @existing == 'skip' - next { source_row: row[:source_row], url: normalised_url(url) || url, attributes:, - provenance:, tag_sources:, metadata_url: url_for_metadata, - fetch_warnings: [], validation_warnings: ['既存投稿のためスキップします．'], - warnings: ['既存投稿のためスキップします．'], validation_errors: { }, - status: 'warning', existing: true } + next { + source_row: row[:source_row], + url: normalised_url(url) || url, + attributes:, + provenance:, + tag_sources:, + metadata_url: url_for_metadata, + fetch_warnings: [], + validation_warnings: ['既存投稿のためスキップします．'], + warnings: ['既存投稿のためスキップします．'], + validation_errors: { }, + status: 'warning', + existing: true, + } end url_changed = row[:metadata_url].present? && row[:metadata_url] != url_for_metadata clear_automatic_values!(attributes, provenance, tag_sources) if url_changed @@ -43,7 +63,8 @@ class PostImportPreviewer when false, nil then false else raise ArgumentError, 'メタデータ取得対象が不正です．' end - metadata, fetch_warnings = metadata_for(url_for_metadata, should_fetch, metadata_cache, fetch_warnings) + metadata, fetch_warnings = + metadata_for(url_for_metadata, should_fetch, metadata_cache, fetch_warnings) metadata.each do |field, value| if field == 'tags' && provenance[field] != 'manual' tag_sources['automatic'] = value.to_s @@ -62,19 +83,37 @@ class PostImportPreviewer errors[:url] = ['URL が重複しています．'] if normal_url.present? && urls.key?(normal_url) urls[normal_url] = true if normal_url.present? if normal_url.present? && existing_post - @existing == 'error' ? errors[:url] = ['既存投稿です．'] : validation_warnings << '既存投稿のためスキップします．' + if @existing == 'error' + errors[:url] = ['既存投稿です．'] + else + validation_warnings << '既存投稿のためスキップします．' + end end validate_basic_data(attributes, errors) - validate_preview_tags(merged_tags(tag_sources, provenance['tags']), errors, validation_warnings) + validate_preview_tags( + merged_tags(tag_sources, provenance['tags']), + errors, + validation_warnings, + ) validate_parents(attributes['parent_post_ids'], errors) attributes.delete('url') attributes['tags'] = merged_tags(tag_sources, provenance['tags']) - { source_row: row[:source_row], url: normal_url || url, attributes:, provenance:, tag_sources:, + { + source_row: row[:source_row], + url: normal_url || url, + attributes:, + provenance:, + tag_sources:, metadata_url: url_for_metadata, - fetch_warnings:, validation_warnings:, warnings: fetch_warnings + validation_warnings, + fetch_warnings:, + validation_warnings:, + warnings: fetch_warnings + validation_warnings, validation_errors: errors, - status: errors.present? ? 'error' : ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'), - existing: normal_url.present? && Post.exists?(url: normal_url) } + status: errors.present? \ + ? 'error' \ + : ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'), + existing: normal_url.present? && Post.exists?(url: normal_url), + } end end @@ -95,7 +134,9 @@ class PostImportPreviewer return '' if mapping.blank? return mapping['value'].to_s if mapping['kind'] == 'fixed' - Array(mapping['columns']).filter_map { |index| values[index.to_i].to_s.presence }.join(field == 'tags' ? ' ' : '') + Array(mapping['columns']).filter_map { |index| + values[index.to_i].to_s.presence + }.join(field == 'tags' ? ' ' : '') end def mapped_provenance attributes @@ -122,11 +163,14 @@ class PostImportPreviewer def merged_tags sources, origin = nil return sources['manual'].to_s if origin == 'manual' - %w[automatic mapped fixed manual].flat_map { sources[_1].to_s.split }.uniq.join(' ') + ['automatic', 'mapped', 'fixed', 'manual'].flat_map { + sources[_1].to_s.split + }.uniq.join(' ') end def clear_automatic_values! attributes, provenance, tag_sources - %w[title thumbnail_base original_created_from original_created_before duration].each do |field| + ['title', 'thumbnail_base', 'original_created_from', + 'original_created_before', 'duration'].each do |field| attributes[field] = '' if provenance[field] == 'automatic' end tag_sources['automatic'] = '' @@ -140,8 +184,13 @@ class PostImportPreviewer warnings << 'タイトルを取得できませんでした．' if data['title'].blank? warnings << 'サムネールを取得できませんでした．' if data['thumbnail_base'].blank? [data.compact, warnings] - rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e - Rails.logger.info("post_import_metadata_fetch_failure #{ { error: e.class.name, message: e.message }.to_json }") + rescue Preview::UrlSafety::UnsafeUrl, + Preview::HttpFetcher::FetchFailed, + Preview::HttpFetcher::ResponseTooLarge => e + Rails.logger.info( + "post_import_metadata_fetch_failure "\ + "#{ { error: e.class.name, message: e.message }.to_json }", + ) [{ }, ['メタデータを取得できませんでした．']] end @@ -190,7 +239,10 @@ class PostImportPreviewer def validate_parents raw, errors ids = raw.to_s.split.map { Integer(_1, exception: false) } return if ids.compact.empty? && raw.to_s.blank? + errors[:parent_post_ids] = ['親投稿 Id. が不正です．'] and return if ids.any? { _1.nil? || _1 <= 0 } - errors[:parent_post_ids] = ['存在しない親投稿 Id. があります．'] if Post.where(id: ids).count != ids.uniq.length + if Post.where(id: ids).count != ids.uniq.length + errors[:parent_post_ids] = ['存在しない親投稿 Id. があります．'] + end end end diff --git a/backend/app/services/post_import_runner.rb b/backend/app/services/post_import_runner.rb index 7205db2..3644b18 100644 --- a/backend/app/services/post_import_runner.rb +++ b/backend/app/services/post_import_runner.rb @@ -1,6 +1,16 @@ class PostImportRunner MAX_ROWS = PostImportSourceParser::MAX_ROWS - ATTRIBUTE_KEYS = %w[title thumbnail_base original_created_from original_created_before duration tags parent_post_ids video_ms].freeze + ATTRIBUTE_KEYS = [ + 'title', + 'thumbnail_base', + 'original_created_from', + 'original_created_before', + 'duration', + 'tags', + 'parent_post_ids', + 'video_ms', + ].freeze + def initialize actor:, rows:, existing: 'skip' @actor = actor @rows = rows @@ -9,15 +19,21 @@ class PostImportRunner def run raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if @rows.length > MAX_ROWS - raise ArgumentError, '取込データが大きすぎます．' if @rows.sum { |row| row.to_json.bytesize } > PostImportSourceParser::MAX_BYTES + if @rows.sum { |row| row.to_json.bytesize } > PostImportSourceParser::MAX_BYTES + raise ArgumentError, '取込データが大きすぎます．' + end normalised_rows = @rows.map { |row| normalise_row(row) } source_rows = normalised_rows.map { _1['source_row'] } raise ArgumentError, '元行番号が重複しています．' if source_rows.uniq.length != source_rows.length results = normalised_rows.map { |row| run_row(row) } - { created: results.count { _1[:status] == 'created' }, skipped: results.count { _1[:status] == 'skipped' }, - failed: results.count { _1[:status] == 'failed' }, rows: results } + { + created: results.count { _1[:status] == 'created' }, + skipped: results.count { _1[:status] == 'skipped' }, + failed: results.count { _1[:status] == 'failed' }, + rows: results, + } end private @@ -26,20 +42,40 @@ class PostImportRunner validate_row_structure!(row) attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore } raise ArgumentError, '取込項目が不正です．' unless (attributes.keys - ATTRIBUTE_KEYS).empty? - raise ArgumentError, '取込項目が大きすぎます．' if attributes.values.any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES } - raise ArgumentError, 'URL が長すぎます．' if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES - validate_tag_sources!(row['tag_sources']) - preview = PostImportPreviewer.new(parsed: { columns: [], rows: [] }, url_column: 0, - mappings: {}, existing: @existing).preview_rows( - rows: [{ source_row: row['source_row'], url: row['url'], - attributes:, provenance: row['provenance'], - tag_sources: row['tag_sources'] }], - fetch_metadata: false).first - if preview[:validation_errors].present? - return { source_row: row['source_row'], status: 'failed', - errors: preview[:validation_errors] } + if attributes.values.any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES } + raise ArgumentError, '取込項目が大きすぎます．' end - return row.slice('source_row').merge(status: 'skipped') if @existing == 'skip' && preview[:existing] + if row['url'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES + raise ArgumentError, 'URL が長すぎます．' + end + validate_tag_sources!(row['tag_sources']) + preview = PostImportPreviewer.new( + parsed: { columns: [], rows: [] }, + url_column: 0, + mappings: {}, + existing: @existing, + ).preview_rows( + rows: [{ + source_row: row['source_row'], + url: row['url'], + attributes:, + provenance: row['provenance'], + tag_sources: row['tag_sources'], + }], + fetch_metadata: false, + ).first + if preview[:validation_errors].present? + return { + source_row: row['source_row'], + status: 'failed', + errors: preview[:validation_errors], + } + end + + if @existing == 'skip' && preview[:existing] + return row.slice('source_row').merge(status: 'skipped') + end + attributes['tags'] = preview[:attributes]['tags'] attributes['url'] = row['url'] post = PostCreator.new(actor: @actor, attributes:).create! @@ -53,7 +89,9 @@ class PostImportRunner rescue ArgumentError, PostCreator::VideoMsParseError { source_row: row['source_row'], status: 'failed', errors: { base: ['入力値が不正です．'] } } rescue StandardError => e - Rails.logger.error("post_import_runner_failure #{ { error: e.class.name, message: e.message }.to_json }") + Rails.logger.error( + "post_import_runner_failure #{ { error: e.class.name, message: e.message }.to_json }", + ) { source_row: row['source_row'], status: 'failed', errors: { base: ['登録中にエラーが発生しました．'] } } end @@ -61,9 +99,11 @@ class PostImportRunner return if sources.blank? sources = sources.to_h - raise ArgumentError unless (sources.keys - %w[automatic mapped fixed manual]).empty? + raise ArgumentError unless (sources.keys - ['automatic', 'mapped', 'fixed', 'manual']).empty? raise ArgumentError unless sources.values.all?(String) - raise ArgumentError if sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } + if sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } + raise ArgumentError + end raise ArgumentError if sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES end @@ -71,12 +111,22 @@ class PostImportRunner raise ArgumentError unless row['source_row'].is_a?(Integer) && row['source_row'].positive? raise ArgumentError unless row['url'].is_a?(String) raise ArgumentError unless row['attributes'].is_a?(Hash) - raise ArgumentError unless row['attributes'].values.all? { _1.is_a?(String) || _1.is_a?(Numeric) || _1 == true || _1 == false || _1.nil? } + unless row['attributes'].values.all? { |value| + value.is_a?(String) || + value.is_a?(Numeric) || + value == true || + value == false || + value.nil? + } + raise ArgumentError + end provenance = row['provenance'] raise ArgumentError unless provenance.is_a?(Hash) allowed = ATTRIBUTE_KEYS + ['url'] raise ArgumentError unless (provenance.keys - allowed).empty? - raise ArgumentError unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) } + unless provenance.values.all? { _1.in?(['automatic', 'mapped', 'fixed', 'manual']) } + raise ArgumentError + end metadata_url = row['metadata_url'] raise ArgumentError unless metadata_url.nil? || metadata_url.is_a?(String) raise ArgumentError if metadata_url.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES @@ -86,9 +136,7 @@ class PostImportRunner end def normalise_row row - unless row.is_a?(Hash) - raise ArgumentError, '取込行の形式が不正です．' - end + raise ArgumentError, '取込行の形式が不正です．' unless row.is_a?(Hash) normalised = row.deep_transform_keys { _1.to_s.underscore } source_row = Integer(normalised['source_row'], exception: false) diff --git a/backend/app/services/post_import_source_parser.rb b/backend/app/services/post_import_source_parser.rb index 5fff5f1..1460e6c 100644 --- a/backend/app/services/post_import_source_parser.rb +++ b/backend/app/services/post_import_source_parser.rb @@ -30,9 +30,17 @@ class PostImportSourceParser ensure_limits!(table) width = table.map(&:length).max.to_i raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです．" if width > MAX_COLUMNS - { columns: (0...width).map { |index| { key: index.to_s, - label: "#{ column_name(index) }：#{ headers&.[](index).presence || '列' }" } }, - rows: table.each_with_index.map { |values, index| { source_row: index + (@has_header ? 2 : 1), values: } } } + { + columns: (0...width).map { |index| + { + key: index.to_s, + label: "#{ column_name(index) }：#{ headers&.[](index).presence || '列' }", + } + }, + rows: table.each_with_index.map { |values, index| + { source_row: index + (@has_header ? 2 : 1), values: } + }, + } rescue CSV::MalformedCSVError => e raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }" end @@ -59,17 +67,27 @@ class PostImportSourceParser ensure_limits!(value) unless value.all?(Hash) - invalid = value.each_with_index.filter_map { |item, index| index + 1 unless item.is_a?(Hash) } + invalid = value.each_with_index.filter_map { |item, index| + index + 1 unless item.is_a?(Hash) + } raise ArgumentError, "JSON の投稿候補は object の配列にしてください: 行 #{ invalid.join(', ') }" end keys = value.flat_map(&:keys).map(&:to_s).uniq raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです．" if keys.length > MAX_COLUMNS - { columns: keys.each_with_index.map { |key, index| { key:, label: "#{ column_name(index) }：#{ key }" } }, - rows: value.each_with_index.map { |item, index| { source_row: index + 1, - values: keys.map { |key| - value = item.key?(key) ? item[key] : item[key.to_sym] - json_cell(value) - } } } } + { + columns: keys.each_with_index.map { |key, index| + { key:, label: "#{ column_name(index) }：#{ key }" } + }, + rows: value.each_with_index.map { |item, index| + { + source_row: index + 1, + values: keys.map { |key| + json_value = item.key?(key) ? item[key] : item[key.to_sym] + json_cell(json_value) + }, + } + }, + } rescue JSON::ParserError, KeyError, ArgumentError => e raise ArgumentError, "JSON の形式が不正です: #{ e.message }" end diff --git a/backend/app/services/post_metadata_fetcher.rb b/backend/app/services/post_metadata_fetcher.rb index 09389ac..6a0d78f 100644 --- a/backend/app/services/post_metadata_fetcher.rb +++ b/backend/app/services/post_metadata_fetcher.rb @@ -1,10 +1,19 @@ class PostMetadataFetcher def self.fetch raw_url uri, = Preview::UrlSafety.validate(raw_url) - response = Preview::HttpFetcher.fetch(uri.to_s, max_bytes: Preview::ThumbnailFetcher::HTML_MAX_BYTES) + response = Preview::HttpFetcher.fetch( + uri.to_s, + max_bytes: Preview::ThumbnailFetcher::HTML_MAX_BYTES, + ) metadata = Preview::HtmlMetadataExtractor.extract(response) document = Nokogiri::HTML.parse(response.body) - content = -> name { document.at_css("meta[property='#{ name }'], meta[name='#{ name }']")&.[]('content')&.strip.presence } + content = lambda { |name| + document + .at_css("meta[property='#{ name }'], meta[name='#{ name }']") + &.[]('content') + &.strip + &.presence + } duration = content.call('og:video:duration') || content.call('video:duration') published = content.call('article:published_time') || content.call('date') created_range = original_created_range(published) @@ -30,17 +39,19 @@ class PostMetadataFetcher raw = value.to_s.strip from = Time.zone.parse(raw) zone = '(?:Z|[+-]\d{2}:?\d{2})?' - before = case raw - when /\A\d{4}\z/ then from + 1.year - when /\A\d{4}-\d{2}\z/ then from + 1.month - when /\A\d{4}-\d{2}-\d{2}\z/ then from + 1.day - when /\A\d{4}-\d{2}-\d{2}T\d{2}#{ zone }\z/ then from + 1.hour - when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}#{ zone }\z/ then from + 1.minute - when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}#{ zone }\z/ then from + 1.second - when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.(\d+)#{ zone }\z/ - from + (10**(-Regexp.last_match(1).length)) - else return nil - end + before = + case raw + when /\A\d{4}\z/ then from + 1.year + when /\A\d{4}-\d{2}\z/ then from + 1.month + when /\A\d{4}-\d{2}-\d{2}\z/ then from + 1.day + when /\A\d{4}-\d{2}-\d{2}T\d{2}#{ zone }\z/ then from + 1.hour + when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}#{ zone }\z/ then from + 1.minute + when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}#{ zone }\z/ then from + 1.second + when /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.(\d+)#{ zone }\z/ + from + (10**(-Regexp.last_match(1).length)) + else + return nil + end [from, before] rescue ArgumentError, TypeError nil diff --git a/backend/app/services/preview/http_fetcher.rb b/backend/app/services/preview/http_fetcher.rb index 0ca503e..8a4aecf 100644 --- a/backend/app/services/preview/http_fetcher.rb +++ b/backend/app/services/preview/http_fetcher.rb @@ -71,7 +71,10 @@ module Preview log_failure(:timeout, url: uri&.to_s || raw_url, error: e.class.name, message: e.message) raise FetchTimeout, e.message rescue SocketError, SystemCallError, OpenSSL::SSL::SSLError, EOFError => e - log_failure(:network_error, url: uri&.to_s || raw_url, error: e.class.name, message: e.message) + log_failure(:network_error, + url: uri&.to_s || raw_url, + error: e.class.name, + message: e.message) raise FetchFailed, e.message end diff --git a/frontend/src/components/TopNav.tsx b/frontend/src/components/TopNav.tsx index e2215af..1bf2c78 100644 --- a/frontend/src/components/TopNav.tsx +++ b/frontend/src/components/TopNav.tsx @@ -26,7 +26,7 @@ export const menuOutline = ( tag?: Tag | null material?: Material | null wikiId: number | null - user: User | null, + user: User | null pathName: string }, ): Menu => { const postCount = tag?.postCount ?? material?.tag?.postCount ?? 0 @@ -43,7 +43,7 @@ export const menuOutline = ( { name: '一覧', to: '/posts' }, { name: '検索', to: '/posts/search' }, { name: '追加', to: '/posts/new' }, - { name: 'インポート', to: '/posts/import' }, + { name: '取込', to: '/posts/import' }, { name: '全体履歴', to: '/posts/changes' }, { name: 'ヘルプ', to: '/wiki/ヘルプ:広場' }] }, { name: 'タグ', to: '/tags', subMenu: [ diff --git a/frontend/src/pages/posts/PostImportPage.tsx b/frontend/src/pages/posts/PostImportPage.tsx index b4346ef..e79ef86 100644 --- a/frontend/src/pages/posts/PostImportPage.tsx +++ b/frontend/src/pages/posts/PostImportPage.tsx @@ -2,10 +2,10 @@ import { useEffect, useRef, useState } from 'react' import { Helmet } from 'react-helmet-async' import FieldError from '@/components/common/FieldError' -import PrefetchLink from '@/components/PrefetchLink' import Form from '@/components/common/Form' import FormField from '@/components/common/FormField' import PageTitle from '@/components/common/PageTitle' +import PrefetchLink from '@/components/PrefetchLink' import MainArea from '@/components/layout/MainArea' import { Button } from '@/components/ui/button' import { toast } from '@/components/ui/use-toast' @@ -20,383 +20,805 @@ import type { FC } from 'react' import type { User } from '@/types' type Props = { user: User | null } -type Column = { key: string, label: string } -type ImportRow = { + +type Column = { + key: string + label: string } + +type ParsedRow = { sourceRow: number - url: string - attributes: Record - warnings: string[] - validationErrors: Record - importErrors?: Record - provenance: Record - tagSources?: Record<'automatic' | 'mapped' | 'fixed' | 'manual', string> - status: 'ready' | 'warning' | 'error' - existing: boolean - metadataUrl?: string - createdPostId?: number - importStatus?: 'pending' | 'created' | 'skipped' | 'failed' } -type ParsedImport = { columns: Column[], rows: { sourceRow: number, values: string[] }[] } -type Phase = 'source' | 'mapping' | 'preview' | 'result' + values: string[] } + +type ParsedImport = { + columns: Column[] + rows: ParsedRow[] } + +type Origin = 'automatic' | 'mapped' | 'fixed' | 'manual' + +type ImportStatus = 'pending' | 'created' | 'skipped' | 'failed' type ImportResultStatus = 'created' | 'skipped' | 'failed' +type Phase = 'source' | 'mapping' | 'preview' | 'result' + +type ImportRow = { + sourceRow: number + url: string + attributes: Record + warnings: string[] + validationErrors: Record + importErrors?: Record + provenance: Record + tagSources?: Record + status: 'ready' | 'warning' | 'error' + existing: boolean + metadataUrl?: string + createdPostId?: number + importStatus?: ImportStatus } + +type ImportResultRow = { + sourceRow: number + status: ImportResultStatus + post?: { id: number } + errors?: Record } + +type Mapping = { columns: string[] } + +const MAPPING_FIELDS = [ + 'title', + 'thumbnail_base', + 'original_created_from', + 'original_created_before', + 'duration', + 'tags', + 'parent_post_ids'] as const + +const RESULT_STATUSES: ImportResultStatus[] = ['created', 'skipped', 'failed'] + +const ORIGIN_LABELS: Record = { + automatic: '自動取得', + mapped: '入力データ', + fixed: '固定値', + manual: '手修正' } const detectFormat = (source: string): 'csv' | 'tsv' => - source.includes ('\t') ? 'tsv' : (source.includes (',') ? 'csv' : 'tsv') + source.includes ('\t') + ? 'tsv' + : source.includes (',') + ? 'csv' + : 'tsv' + const publicGoogleSheetURL = (source: string): boolean => /^https:\/\/docs\.google\.com\/spreadsheets\/d\/e\/[A-Za-z0-9_-]+\/pubhtml#gid=\d+$/.test ( - source.trim ()) + source.trim ()) + + +const mergeValidatedRows = ( + current: ImportRow[], + validated: ImportRow[], +): ImportRow[] => + current + .map (previous => { + if (previous.importStatus === 'created') + return previous + + const row = validated.find (_1 => _1.sourceRow === previous.sourceRow) + return row + ? { + ...row, + importStatus: previous.importStatus, + createdPostId: previous.createdPostId, + importErrors: previous.importErrors, + validationErrors: row.validationErrors ?? { } } + : previous + }) + .concat (validated.filter (row => + !(current.some (_1 => _1.sourceRow === row.sourceRow)))) + + +const rowMessages = (row: ImportRow): string[] => [ + ...row.warnings, + ...Object.values (row.validationErrors ?? { }).flat (), + ...Object.values (row.importErrors ?? { }).flat ()] + + +const sampleURLCount = (rows: ParsedRow[], index: number): number => + rows.filter (row => /^https?:\/\//.test (row.values[index] ?? '')).length + + +const renderMappingSelect = ( + columns: Column[], + field: string, + value: string, + update: (field: string, value: string) => void, +) => ( + ) const PostImportPage: FC = ({ user }) => { - const [source, setSource] = useState ('') - const [phase, setPhase] = useState ('source') - const [format, setFormat] = useState<'csv' | 'tsv' | 'json' | 'google_sheets'> ('tsv') - const [hasHeader, setHasHeader] = useState (false) - const [jsonPath, setJsonPath] = useState ('') - const [urlColumn, setUrlColumn] = useState ('') - const [parsed, setParsed] = useState (null) - const [columns, setColumns] = useState ([]) - const [rows, setRows] = useState ([]) - const [results, setResults] = useState<{ sourceRow: number, status: ImportResultStatus, post?: { id: number }, errors?: Record }[]> ([]) - const [mappings, setMappings] = useState> ({ }) - const [existing, setExisting] = useState<'skip' | 'error'> ('skip') - const [loading, setLoading] = useState (false) - const validationGeneration = useRef (0) - const editable = canEditContent (user) + const editable = canEditContent (user) - const parseSource = async () => { - setLoading (true) - try - { - const data = await apiPost ('/posts/import/parse', { - source, - format, - has_header: hasHeader, - json_path: jsonPath, - existing }) - setColumns (data.columns) - setParsed (data) - setMappings ({ }) - setRows ([]) - setResults ([]) - setUrlColumn ('') - if (data.columns.length === 1) - setUrlColumn ('0') - else - { - const candidate = data.columns.map ((_, index) => ({ index, - count: data.rows.filter (row => /^https?:\/\//.test (row.values[index] ?? '')).length })) - .sort ((a, b) => b.count - a.count)[0] - if (candidate?.count) - setUrlColumn (String (candidate.index)) - } - setPhase ('mapping') - } - catch - { - toast ({ title: '読込みに失敗しました', description: '入力形式を確認してください。' }) - } - finally - { - setLoading (false) - } - } + const [source, setSource] = useState ('') + const [phase, setPhase] = useState ('source') + const [format, setFormat] = + useState<'csv' | 'tsv' | 'json' | 'google_sheets'> ('tsv') + const [hasHeader, setHasHeader] = useState (false) + const [jsonPath, setJsonPath] = useState ('') + const [urlColumn, setUrlColumn] = useState ('') + const [parsed, setParsed] = useState (null) + const [columns, setColumns] = useState ([]) + const [rows, setRows] = useState ([]) + const [results, setResults] = useState ([]) + const [mappings, setMappings] = useState> ({ }) + const [existing, setExisting] = useState<'skip' | 'error'> ('skip') + const [loading, setLoading] = useState (false) - const preview = async () => { - if (!parsed || urlColumn === '') - return - setLoading (true) - try - { - const data = await apiPost<{ rows: ImportRow[] }> ('/posts/import/preview', { - parsed: { columns: parsed.columns, rows: parsed.rows }, - url_column: urlColumn, - mappings, - existing }) - setRows (current => mergeValidatedRows (current, data.rows)) - setPhase ('preview') - } - catch (error) - { - const message = - isApiError<{ message?: string, baseErrors?: string[] }> (error) - ? error.response?.data?.message ?? error.response?.data?.baseErrors?.[0] - : undefined - toast ({ title: '投稿プレビューに失敗しました', description: message ?? '入力を確認してください。' }) - } - finally - { - setLoading (false) - } - } + const validationGeneration = useRef (0) - const mergeValidatedRows = (current: ImportRow[], validated: ImportRow[]): ImportRow[] => - current.map (previous => { - if (previous.importStatus === 'created') - return previous - const row = validated.find (_1 => _1.sourceRow === previous.sourceRow) - return row ? { ...row, - importStatus: previous.importStatus, - createdPostId: previous.createdPostId, - importErrors: previous.importErrors, - validationErrors: row.validationErrors ?? { } } : previous - }).concat (validated.filter (row => !current.some (_1 => _1.sourceRow === row.sourceRow))) + const parseSource = async () => { + setLoading (true) + try + { + const data = await apiPost ('/posts/import/parse', { + source, + format, + has_header: hasHeader, + json_path: jsonPath, + existing }) + setColumns (data.columns) + setParsed (data) + setMappings ({ }) + setRows ([]) + setResults ([]) + if (data.columns.length === 1) + { + setUrlColumn ('0') + } + else + { + const candidate = data.columns + .map ((_, index) => ({ + index, + count: sampleURLCount (data.rows, index) })) + .sort ((a, b) => b.count - a.count)[0] + setUrlColumn (candidate?.count ? String (candidate.index) : '') + } + setPhase ('mapping') + } + catch + { + toast ({ + title: '読込みに失敗しました', + description: '入力形式を確認してください．' }) + } + finally + { + setLoading (false) + } + } - const updateAttribute = async (index: number, field: string, value: string) => { - const row = rows[index] - const next = { ...row, - attributes: { ...row.attributes, [field]: value }, - provenance: { ...row.provenance, [field]: 'manual' }, - importStatus: 'pending' as const, - importErrors: undefined } - const nextRows = rows.map ((currentRow, rowIndex) => rowIndex === index ? next : currentRow) - setRows (nextRows) - const generation = ++validationGeneration.current - setLoading (true) - try - { - const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', - { rows: nextRows.filter (row => row.importStatus !== 'created'), existing, - changed_row: -1 }) - if (generation === validationGeneration.current) - setRows (current => mergeValidatedRows (current, validated.rows)) - } - catch - { - toast ({ title: '行の再検証に失敗しました' }) - } - finally - { - if (generation === validationGeneration.current) - setLoading (false) - } - } + const preview = async () => { + if (!parsed || urlColumn === '') + return - const updateURL = async (index: number, value: string) => { - const row = rows[index] - const next = { ...row, url: value, provenance: { ...row.provenance, url: 'manual' }, - importStatus: 'pending' as const, importErrors: undefined } - const nextRows = rows.map ((currentRow, rowIndex) => rowIndex === index ? next : currentRow) - setRows (nextRows) - const generation = ++validationGeneration.current - setLoading (true) - try - { - const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', - { rows: nextRows.filter (row => row.importStatus !== 'created'), existing, - changed_row: row.sourceRow }) - if (generation === validationGeneration.current) - setRows (current => mergeValidatedRows (current, validated.rows)) + setLoading (true) + try + { + const data = await apiPost<{ rows: ImportRow[] }> ('/posts/import/preview', { + parsed: { columns: parsed.columns, rows: parsed.rows }, + url_column: urlColumn, + mappings, + existing }) + setRows (current => mergeValidatedRows (current, data.rows)) + setPhase ('preview') + } + catch (error) + { + const message = + isApiError<{ message?: string, baseErrors?: string[] }> (error) + ? error.response?.data?.message + ?? error.response?.data?.baseErrors?.[0] + : undefined + toast ({ + title: '投稿プレビューに失敗しました', + description: message ?? '入力を確認してください．' }) + } + finally + { + setLoading (false) + } } - catch - { - toast ({ title: 'URL の再検証に失敗しました' }) - } - finally - { - if (generation === validationGeneration.current) - setLoading (false) - } - } - const updateExisting = async (value: 'skip' | 'error') => { - setExisting (value) - const pendingRows = rows.map (row => row.importStatus === 'created' ? row : - { ...row, importStatus: 'pending' as const, importErrors: undefined }) - setRows (pendingRows) - const generation = ++validationGeneration.current - setLoading (true) - try - { - const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', - { rows: pendingRows.filter (row => row.importStatus !== 'created'), existing: value, changed_row: -1 }) - if (generation === validationGeneration.current) - setRows (current => mergeValidatedRows (current, validated.rows)) + const validateRows = async ( + nextRows: ImportRow[], + changedRow: number, + ) => { + const generation = ++validationGeneration.current + setLoading (true) + try + { + const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', { + rows: nextRows.filter (row => row.importStatus !== 'created'), + existing, + changed_row: changedRow }) + if (generation === validationGeneration.current) + setRows (current => mergeValidatedRows (current, validated.rows)) + } + finally + { + if (generation === validationGeneration.current) + setLoading (false) + } } - finally - { - if (generation === validationGeneration.current) - setLoading (false) - } - } - const submit = async () => { - const targets = rows.filter (row => row.importStatus == null || row.importStatus === 'pending') - if (targets.length === 0) - return - setLoading (true) - try - { - const result = await apiPost<{ created: number, skipped: number, failed: number, - rows: { sourceRow: number, status: ImportResultStatus, post?: { id: number }, errors?: Record }[] }> ('/posts/import', { - rows: targets.map (row => ({ sourceRow: row.sourceRow, - url: row.url, - attributes: row.attributes, - provenance: row.provenance, - tagSources: row.tagSources, - metadataUrl: row.metadataUrl })), - existing }) - if (result.rows.some (row => !(['created', 'skipped', 'failed'] as string[]).includes (row.status))) - throw new Error ('不正な登録結果です。') - toast ({ title: `登録完了: ${result.created} 件`, - description: `スキップ ${result.skipped} 件、失敗 ${result.failed} 件` }) - setPhase ('result') - setResults (result.rows) - setRows (current => current.map (row => { - const resultRow = result.rows.find (_1 => _1.sourceRow === row.sourceRow) - return resultRow ? { ...row, importStatus: resultRow.status, - createdPostId: resultRow.post?.id, - importErrors: resultRow.errors, - validationErrors: row.validationErrors } : row - })) + const updateAttribute = async ( + index: number, + field: string, + value: string, + ) => { + const row = rows[index] + const next = { + ...row, + attributes: { ...row.attributes, [field]: value }, + provenance: { ...row.provenance, [field]: 'manual' }, + importStatus: 'pending' as const, + importErrors: undefined } + const nextRows = rows.map ((currentRow, rowIndex) => + rowIndex === index ? next : currentRow) + setRows (nextRows) + try + { + await validateRows (nextRows, -1) + } + catch + { + toast ({ title: '行の再検証に失敗しました' }) + } } - catch - { - toast ({ title: '登録に失敗しました' }) - } - finally - { - setLoading (false) - } - } - if (!(editable)) - return + const updateURL = async (index: number, value: string) => { + const row = rows[index] + const next = { + ...row, + url: value, + provenance: { ...row.provenance, url: 'manual' }, + importStatus: 'pending' as const, + importErrors: undefined } + const nextRows = rows.map ((currentRow, rowIndex) => + rowIndex === index ? next : currentRow) + setRows (nextRows) + try + { + await validateRows (nextRows, row.sourceRow) + } + catch + { + toast ({ title: 'URL の再検証に失敗しました' }) + } + } - return ( - - {`投稿インポート | ${SITE_TITLE}`} - 投稿インポート - {phase === 'source' ? ( -
-

- URL を一行に一件ずつ貼り付けるか、CSV・TSV・JSON を入力してください。 -

- {publicGoogleSheetURL (source) && ( -
- 公開 Google スプレッドシート URL - - -
)} - - {() => } - - - {() =>