diff --git a/backend/app/controllers/post_imports_controller.rb b/backend/app/controllers/post_imports_controller.rb index 8831389..289eaed 100644 --- a/backend/app/controllers/post_imports_controller.rb +++ b/backend/app/controllers/post_imports_controller.rb @@ -1,44 +1,10 @@ class PostImportsController < ApplicationController - MAPPING_PROPERTIES = ['kind', 'value', 'columns'].freeze - MAPPING_KINDS = ['columns', 'fixed'].freeze - before_action :require_member! - def parse - source = - if params[:format] == 'google_sheets' - PostImportGoogleSheetsFetcher.fetch!(params[:source], - rate_key: current_user.id) - else - params[:source] - end - format = params[:format] == 'google_sheets' ? 'csv' : params[:format] - parsed = PostImportSourceParser.new( - source:, - format:, - has_header: params[:has_header], - json_path: params[:json_path]).parse - render json: parsed - rescue ArgumentError => e - render_bad_request(e.message) - end - def preview - parsed = parsed_params - raw_url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil) - return render_bad_request('URL 列を選択してください．') if raw_url_column.nil? - - url_column = Integer(raw_url_column, exception: false) - if url_column.nil? || url_column.negative? || url_column >= parsed[:columns].length - return render_bad_request('URL 列が不正です．') - end - - rows = PostImportPreviewer.new( - parsed:, - url_column:, - mappings: mappings_params(parsed[:columns].length), - existing: params[:existing]).preview - render json: parsed.slice(:columns).merge(rows:) + rows = PostImportPreviewer.new(existing: params[:existing]).preview_rows( + rows: PostImportUrlListParser.parse(params[:source])) + render json: { rows: } rescue ArgumentError => e render_bad_request(e.message) end @@ -47,12 +13,8 @@ class PostImportsController < ApplicationController rows = normalised_import_rows(allow_warning_fields: true) changed_row = Integer(params[:changed_row], exception: false) result = - PostImportPreviewer.new( - parsed: { columns: [], rows: [] }, - url_column: 0, - mappings: {}, - existing: params[:existing]) - .preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {}) + PostImportPreviewer.new(existing: params[:existing]) + .preview_rows(rows:, fetch_metadata: changed_row, metadata_cache: {}) render json: { rows: result } rescue ArgumentError => e render_bad_request(e.message) @@ -77,85 +39,7 @@ class PostImportsController < ApplicationController head :forbidden end - def parsed_params - value = params.require(:parsed).to_unsafe_h.deep_transform_keys { _1.to_s.underscore } - columns = Array(value['columns']) - rows = Array(value['rows']) - unless columns.all?(Hash) && rows.all?(Hash) - raise ArgumentError, '解析結果の形式が不正です．' - end - raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS - raise ArgumentError, '列数が多すぎます．' if columns.length > 50 - raise ArgumentError, '解析結果のセルが不正です．' unless rows.all? { |row| - values = Array(row['values']) - values.length <= columns.length && values.all? { |value| - value.is_a?(String) && value.bytesize <= PostImportSourceParser::MAX_CELL_BYTES - } - } - if rows.sum { Array(_1['values']).sum(&:bytesize) } > PostImportSourceParser::MAX_BYTES - raise ArgumentError, '解析結果が大きすぎます．' - end - - parsed_rows = rows.map do |row| - source_row = Integer(row['source_row'], exception: false) - raise ArgumentError, '元行番号が不正です．' if source_row.nil? || source_row <= 0 - - { source_row:, values: Array(row['values']) } - end - if parsed_rows.map { _1[:source_row] }.uniq.length != parsed_rows.length - raise ArgumentError, '元行番号が重複しています．' - end - - { columns: columns.map { |column| column.slice('key', 'label') }, rows: parsed_rows } - end - - def mappings_params column_count - mappings = params[:mappings] - return { } if mappings.blank? - raise ArgumentError, '列割当ての形式が不正です．' unless mappings.is_a?(ActionController::Parameters) - - raw = mappings.to_unsafe_h - unknown_fields = raw.keys - PostImportPreviewer::FIELDS - raise ArgumentError, '列割当て先が不正です．' if unknown_fields.present? - raise ArgumentError, '列割当てが大きすぎます．' if raw.to_json.bytesize > PostImportSourceParser::MAX_BYTES - - permitted = - mappings.permit(*PostImportPreviewer::FIELDS.map { |field| - { field => [:kind, :value, { columns: [] }] } - }).to_h - - raw.each_with_object({ }) do |(field, value), result| - raise ArgumentError, '列割当ての形式が不正です．' unless value.is_a?(Hash) - unless (value.keys - MAPPING_PROPERTIES).empty? - raise ArgumentError, '列割当ての項目が不正です．' - end - - value = permitted.fetch(field) - kind = value['kind'].presence || 'columns' - raise ArgumentError, '列割当ての種別が不正です．' unless kind.in?(MAPPING_KINDS) - columns = value['columns'] - raise ArgumentError, '列番号が不正です．' unless ( - columns.nil? || (columns.is_a?(Array) && columns.all? { |index| - parsed = Integer(index, exception: false) - parsed && parsed >= 0 && parsed < column_count - }) - ) - parsed_columns = Array(columns).map { Integer(_1) } - raise ArgumentError, '列番号が重複しています．' if parsed_columns.uniq.length != parsed_columns.length - raise ArgumentError, '固定値が不正です．' if kind == 'fixed' && !value['value'].is_a?(String) - if value['value'].to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES - raise ArgumentError, '固定値が大きすぎます．' - end - - result[field] = { - 'kind' => kind, - 'value' => value['value'].to_s, - 'columns' => parsed_columns } - end - end - def normalised_import_rows allow_warning_fields: false - rows = params[:rows] - PostImportRowNormaliser.normalise!(rows, allow_warning_fields:) + PostImportRowNormaliser.normalise!(params[:rows], allow_warning_fields:) end end diff --git a/backend/app/services/post_import_google_sheets_fetcher.rb b/backend/app/services/post_import_google_sheets_fetcher.rb deleted file mode 100644 index 2486612..0000000 --- a/backend/app/services/post_import_google_sheets_fetcher.rb +++ /dev/null @@ -1,67 +0,0 @@ -class PostImportGoogleSheetsFetcher - HOST = 'docs.google.com'.freeze - PUBLICATION_ID = /\A[A-Za-z0-9_-]+\z/ - GID = /\A\d+\z/ - RATE_LIMIT = 10 - RATE_WINDOW = 1.minute - - def self.fetch! raw_url, rate_key: - new(raw_url, rate_key:).fetch! - end - - def initialize raw_url, rate_key: - @raw_url = raw_url.to_s - @rate_key = rate_key.to_s - end - - def fetch! - publication_id, gid = parse! - throttle! - csv_url = "https://#{ HOST }/spreadsheets/d/e/#{ publication_id }/pub" \ - "?gid=#{ gid }&single=true&output=csv" - response = Preview::HttpFetcher.fetch(csv_url, - max_bytes: PostImportSourceParser::MAX_BYTES, - allowed_hosts: [HOST]) - raise ArgumentError, 'スプレッドシートを CSV として取得できませんでした．' unless csv?(response) - - response.body - rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, - Preview::HttpFetcher::ResponseTooLarge => e - Rails.logger.info( - 'post_import_google_sheets_fetch_failure '\ - "#{ { error: e.class.name, message: e.message }.to_json }", - ) - raise ArgumentError, 'スプレッドシートを取得できませんでした．ウェブ公開 URL を確認するか、ファイル保存又はコピー貼付けを使用してください．' - end - - private - - def parse! - uri = URI.parse(@raw_url) - unless uri.scheme == 'https' && uri.host&.downcase == HOST && uri.query.blank? - raise ArgumentError, '公開 Google スプレッドシート URL を指定してください．' - end - match = uri.path.match(%r{\A/spreadsheets/d/e/([^/]+)/pubhtml\z}) - gid = URI.decode_www_form(uri.fragment.to_s).to_h['gid'] - unless match && PUBLICATION_ID.match?(match[1]) && GID.match?(gid.to_s) - raise ArgumentError, '公開 Google スプレッドシート URL の publication_id 又は gid が不正です．' - end - - [match[1], gid] - rescue URI::InvalidURIError - raise ArgumentError, '公開 Google スプレッドシート URL が不正です．' - end - - def throttle! - key = "post-import-google-sheets:#{ @rate_key }" - count = Rails.cache.increment(key, 1, expires_in: RATE_WINDOW, initial: 0) - if count > RATE_LIMIT - raise ArgumentError, 'スプレッドシート取得の回数が多すぎます．しばらくしてから再試行してください．' - end - end - - def csv? response - type = response.content_type.to_s.downcase - type.include?('text/csv') || type.include?('application/csv') - end -end diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index 8766d3e..5397fac 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -8,119 +8,15 @@ class PostImportPreviewer 'tags', 'parent_post_ids', ].freeze - - def initialize parsed:, url_column:, mappings: { }, existing: 'skip' - @parsed = parsed - @url_column = url_column.to_i - @mappings = mappings.stringify_keys + def initialize existing: 'skip' @existing = existing == 'error' ? 'error' : 'skip' end - def preview - preview_rows(rows: @parsed[:rows]) - end - def preview_rows rows:, fetch_metadata: true, metadata_cache: { } - urls = { } - rows.map do |row| - row = row.symbolize_keys - values = row[:values] || [] - attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field| - [field, mapped_value(field, values)] - } - provenance = row[:provenance]&.stringify_keys || mapped_provenance(attributes) - tag_sources = - row[:tag_sources]&.stringify_keys || initial_tag_sources(attributes, provenance) - tag_sources['manual'] = attributes['tags'].to_s if provenance['tags'] == 'manual' - url = row[:url].presence || values[@url_column].to_s.strip - provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped' - url_for_metadata = normalised_url(url) || url - existing_post = normalised_url(url).present? && Post.exists?(url: normalised_url(url)) - if existing_post && @existing == 'skip' - next { - source_row: row[:source_row], - url: normalised_url(url) || url, - attributes:, - provenance:, - tag_sources:, - metadata_url: url_for_metadata, - fetch_warnings: [], - validation_warnings: ['既存投稿のためスキップします．'], - warnings: ['既存投稿のためスキップします．'], - validation_errors: { }, - status: 'warning', - existing: true, - } - end - url_changed = row[:metadata_url].present? && row[:metadata_url] != url_for_metadata - clear_automatic_values!(attributes, provenance, tag_sources) if url_changed - attributes['url'] = url - fetch_warnings = Array(row[:fetch_warnings]).dup - should_fetch = - case fetch_metadata - when true then true - when Integer then fetch_metadata == row[:source_row].to_i - when false, nil then false - else raise ArgumentError, 'メタデータ取得対象が不正です．' - end - metadata, fetch_warnings = - metadata_for(url_for_metadata, should_fetch, metadata_cache, fetch_warnings) - metadata.each do |field, value| - if field == 'tags' && provenance[field] != 'manual' - tag_sources['automatic'] = value.to_s - attributes[field] = merged_tags(tag_sources) - next - end - next unless provenance[field] == 'automatic' || attributes[field].blank? - - attributes[field] = value - provenance[field] = 'automatic' - end - normal_url = normalised_url(url) - errors = { } - errors[:url] = ['URL が不正です．'] if normal_url.blank? - validation_warnings = [] - errors[:url] = ['URL が重複しています．'] if normal_url.present? && urls.key?(normal_url) - urls[normal_url] = true if normal_url.present? - if normal_url.present? && existing_post - if @existing == 'error' - errors[:url] = ['既存投稿です．'] - else - validation_warnings << '既存投稿のためスキップします．' - end - end - validate_basic_data(attributes, errors) - validate_preview_tags( - merged_tags(tag_sources, provenance['tags']), - errors, - validation_warnings, - ) - validate_parents(attributes['parent_post_ids'], errors) - attributes.delete('url') - attributes['tags'] = merged_tags(tag_sources, provenance['tags']) - { - source_row: row[:source_row], - url: normal_url || url, - attributes:, - provenance:, - tag_sources:, - metadata_url: url_for_metadata, - fetch_warnings:, - validation_warnings:, - warnings: fetch_warnings + validation_warnings, - validation_errors: errors, - status: errors.present? \ - ? 'error' \ - : ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'), - existing: normal_url.present? && Post.exists?(url: normal_url), - } - end - end - - def metadata_for url, fetch_metadata, cache, previous_warnings - return [{ }, previous_warnings] unless fetch_metadata - - cache[url] ||= fetch_metadata(url) + urls = {} + rows.map { |row| + preview_row(row, urls:, fetch_metadata:, metadata_cache:) + } end def normalised_url value @@ -129,43 +25,97 @@ class PostImportPreviewer private - def mapped_value field, values - mapping = @mappings[field] - return '' if mapping.blank? - return mapping['value'].to_s if mapping['kind'] == 'fixed' + def preview_row row, urls:, fetch_metadata:, metadata_cache: + row = row.symbolize_keys + attributes = initial_attributes(row) + provenance = initial_provenance(row) + tag_sources = initial_tag_sources(row, attributes, provenance) + url = row[:url].to_s.strip + provenance['url'] = 'manual' + normal_url = normalised_url(url) + url_for_metadata = normal_url || url + existing_post = normal_url.present? && Post.exists?(url: normal_url) - Array(mapping['columns']).filter_map { |index| - values[index.to_i].to_s.presence - }.join(field == 'tags' ? ' ' : '') - end + validation_warnings = [] + validation_errors = {} + validation_errors[:url] = ['URL が不正です．'] if normal_url.blank? + validation_errors[:url] = ['URL が重複しています．'] if normal_url.present? && urls.key?(normal_url) + urls[normal_url] = true if normal_url.present? + if normal_url.present? && existing_post + if @existing == 'error' + validation_errors[:url] = ['既存投稿です．'] + else + validation_warnings << '既存投稿のためスキップします．' + end + end - def mapped_provenance attributes - attributes.to_h.to_h { |field, value| - mapping = @mappings[field.to_s] - origin = - if value.present? && mapping&.[]('kind') == 'fixed' - 'fixed' - elsif value.present? - 'mapped' - else - 'automatic' - end - [field.to_s, origin] + if row[:metadata_url].present? && row[:metadata_url] != url_for_metadata + clear_automatic_values!(attributes, provenance, tag_sources) + end + + fetch_warnings = Array(row[:fetch_warnings]).dup + should_fetch = should_fetch_metadata?(fetch_metadata, row[:source_row].to_i) + if should_fetch && !(existing_post && @existing == 'skip') + metadata, fetch_warnings = + metadata_for(url_for_metadata, metadata_cache, fetch_warnings) + apply_metadata!(attributes, provenance, tag_sources, metadata) + end + + attributes['url'] = url + validate_basic_data(attributes, validation_errors) + validate_preview_tags( + merged_tags(tag_sources, provenance['tags']), + validation_errors, + validation_warnings, + ) + validate_parents(attributes['parent_post_ids'], validation_errors) + attributes.delete('url') + attributes['tags'] = merged_tags(tag_sources, provenance['tags']) + + { + source_row: row[:source_row], + url: normal_url || url, + attributes:, + provenance:, + tag_sources:, + metadata_url: url_for_metadata, + fetch_warnings:, + validation_warnings:, + warnings: fetch_warnings + validation_warnings, + validation_errors:, + status: validation_errors.present? \ + ? 'error' \ + : ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'), + existing: existing_post, } end - def initial_tag_sources attributes, provenance - origin = provenance['tags'] == 'fixed' ? 'fixed' : 'mapped' - { 'automatic' => '', 'mapped' => origin == 'mapped' ? attributes['tags'].to_s : '', - 'fixed' => origin == 'fixed' ? attributes['tags'].to_s : '', 'manual' => '' } + def should_fetch_metadata? fetch_metadata, source_row + case fetch_metadata + when true then true + when Integer then fetch_metadata == source_row + when false, nil then false + else raise ArgumentError, 'メタデータ取得対象が不正です．' + end end - def merged_tags sources, origin = nil - return sources['manual'].to_s if origin == 'manual' + def initial_attributes row + attributes = row[:attributes]&.stringify_keys || {} + FIELDS.to_h { |field| [field, attributes[field].to_s] } + end - ['automatic', 'mapped', 'fixed', 'manual'].flat_map { - sources[_1].to_s.split - }.uniq.join(' ') + def initial_provenance row + provenance = row[:provenance]&.stringify_keys || {} + FIELDS.to_h { |field| [field, provenance[field].presence || 'automatic'] } + .merge('url' => 'manual') + end + + def initial_tag_sources row, attributes, provenance + sources = row[:tag_sources]&.stringify_keys || { 'automatic' => '', 'manual' => '' } + sources['automatic'] = sources['automatic'].to_s + sources['manual'] = + provenance['tags'] == 'manual' ? attributes['tags'].to_s : sources['manual'].to_s + sources end def clear_automatic_values! attributes, provenance, tag_sources @@ -174,10 +124,15 @@ class PostImportPreviewer attributes[field] = '' if provenance[field] == 'automatic' end tag_sources['automatic'] = '' + attributes['tags'] = merged_tags(tag_sources, provenance['tags']) + end + + def metadata_for url, cache, previous_warnings + cache[url] ||= fetch_metadata(url) end def fetch_metadata url - return [{ }, ['URL が空です．']] if url.blank? + return [{}, ['URL が空です．']] if url.blank? data = PostMetadataFetcher.fetch(url).stringify_keys warnings = [] @@ -191,7 +146,29 @@ class PostImportPreviewer "post_import_metadata_fetch_failure "\ "#{ { error: e.class.name, message: e.message }.to_json }", ) - [{ }, ['メタデータを取得できませんでした．']] + [{}, ['メタデータを取得できませんでした．']] + end + + def apply_metadata! attributes, provenance, tag_sources, metadata + metadata.each do |field, value| + if field == 'tags' + next if provenance['tags'] == 'manual' + + tag_sources['automatic'] = value.to_s + attributes['tags'] = merged_tags(tag_sources) + next + end + next unless provenance[field] == 'automatic' || attributes[field].blank? + + attributes[field] = value + provenance[field] = 'automatic' + end + end + + def merged_tags sources, origin = nil + return sources['manual'].to_s if origin == 'manual' + + sources['automatic'].to_s end def validate_preview_tags raw, errors, warnings @@ -201,6 +178,7 @@ class PostImportPreviewer errors[:tags] = ['ニコニコ・タグは直接指定できません．'] return end + parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first } existing = Tag.joins(:tag_name).where(tag_names: { name: parsed }).includes(:tag_name).to_a deprecated = existing.select(&:deprecated?).map(&:name) diff --git a/backend/app/services/post_import_row_normaliser.rb b/backend/app/services/post_import_row_normaliser.rb index 05cbd26..92a41db 100644 --- a/backend/app/services/post_import_row_normaliser.rb +++ b/backend/app/services/post_import_row_normaliser.rb @@ -1,5 +1,5 @@ class PostImportRowNormaliser - ORIGINS = ['automatic', 'mapped', 'fixed', 'manual'].freeze + ORIGINS = ['automatic', 'manual'].freeze STRING_FIELDS = [ 'title', 'thumbnail_base', @@ -13,12 +13,12 @@ class PostImportRowNormaliser def self.normalise! rows, allow_warning_fields: false raise ArgumentError, '取込行の形式が不正です．' unless rows.is_a?(Array) - raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS + raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportUrlListParser::MAX_ROWS normalised_rows = rows.map { normalise_row!(_1, allow_warning_fields:) } source_rows = normalised_rows.map { _1['source_row'] } raise ArgumentError, '元行番号が重複しています．' if source_rows.uniq.length != source_rows.length - if normalised_rows.sum { row_bytesize(_1) } > PostImportSourceParser::MAX_BYTES + if normalised_rows.sum { row_bytesize(_1) } > PostImportUrlListParser::MAX_BYTES raise ArgumentError, '取込データが大きすぎます．' end @@ -42,7 +42,7 @@ class PostImportRowNormaliser normalise_provenance!(normalised.fetch('provenance', { })) normalise_tag_sources!(normalised['tag_sources']) normalise_warning_values!(normalised, allow_warning_fields:) - if row_bytesize(normalised) > PostImportSourceParser::MAX_BYTES + if row_bytesize(normalised) > PostImportUrlListParser::MAX_BYTES raise ArgumentError, '取込行が大きすぎます．' end @@ -82,13 +82,13 @@ class PostImportRowNormaliser def self.normalise_url! value raise ArgumentError, 'URL の形式が不正です．' unless value.is_a?(String) - raise ArgumentError, 'URL が長すぎます．' if value.bytesize > PostImportSourceParser::MAX_CELL_BYTES + raise ArgumentError, 'URL が長すぎます．' if value.bytesize > PostImportUrlListParser::MAX_URL_BYTES end private_class_method :normalise_url! def self.normalise_metadata_url! value raise ArgumentError, 'metadata_url の形式が不正です．' unless value.nil? || value.is_a?(String) - if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES + if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES raise ArgumentError, 'metadata_url が長すぎます．' end end @@ -107,7 +107,7 @@ class PostImportRowNormaliser raise ArgumentError, '取込項目の型が不正です．' end end - if value.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES + if value.to_s.bytesize > PostImportUrlListParser::MAX_URL_BYTES raise ArgumentError, '取込項目が大きすぎます．' end end @@ -129,10 +129,10 @@ class PostImportRowNormaliser raise ArgumentError, 'タグ由来の形式が不正です．' unless tag_sources.is_a?(Hash) raise ArgumentError, 'タグ由来の形式が不正です．' unless (tag_sources.keys - ORIGINS).empty? raise ArgumentError, 'タグ由来の形式が不正です．' unless tag_sources.values.all? { _1.is_a?(String) } - if tag_sources.values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } + if tag_sources.values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES } raise ArgumentError, 'タグ由来が大きすぎます．' end - if tag_sources.values.sum(&:bytesize) > PostImportSourceParser::MAX_CELL_BYTES + if tag_sources.values.sum(&:bytesize) > PostImportUrlListParser::MAX_URL_BYTES raise ArgumentError, 'タグ由来が大きすぎます．' end end @@ -148,7 +148,7 @@ class PostImportRowNormaliser unless values.is_a?(Array) && values.all? { _1.is_a?(String) } raise ArgumentError, '警告の形式が不正です．' end - if values.any? { _1.bytesize > PostImportSourceParser::MAX_CELL_BYTES } + if values.any? { _1.bytesize > PostImportUrlListParser::MAX_URL_BYTES } raise ArgumentError, '警告が大きすぎます．' end end diff --git a/backend/app/services/post_import_runner.rb b/backend/app/services/post_import_runner.rb index 3a241e6..19def46 100644 --- a/backend/app/services/post_import_runner.rb +++ b/backend/app/services/post_import_runner.rb @@ -21,11 +21,7 @@ class PostImportRunner def run_row row attributes = row.fetch('attributes', { }).transform_keys { _1.to_s.underscore } - preview = PostImportPreviewer.new( - parsed: { columns: [], rows: [] }, - url_column: 0, - mappings: {}, - existing: @existing) + preview = PostImportPreviewer.new(existing: @existing) .preview_rows( rows: [{ source_row: row['source_row'], @@ -35,6 +31,10 @@ class PostImportRunner tag_sources: row['tag_sources'], }], fetch_metadata: false).first + if @existing == 'skip' && preview[:existing] + return row.slice('source_row').merge(status: 'skipped') + end + if preview[:validation_errors].present? return { source_row: row['source_row'], @@ -43,10 +43,6 @@ class PostImportRunner } end - if @existing == 'skip' && preview[:existing] - return row.slice('source_row').merge(status: 'skipped') - end - attributes['tags'] = preview[:attributes]['tags'] attributes['url'] = row['url'] post = PostCreator.new(actor: @actor, attributes:).create! diff --git a/backend/app/services/post_import_source_parser.rb b/backend/app/services/post_import_source_parser.rb deleted file mode 100644 index 1460e6c..0000000 --- a/backend/app/services/post_import_source_parser.rb +++ /dev/null @@ -1,118 +0,0 @@ -require 'csv' - -class PostImportSourceParser - MAX_ROWS = 100 - MAX_BYTES = 1.megabyte - MAX_CELL_BYTES = 20.kilobytes - MAX_COLUMNS = 50 - - def initialize source:, format:, has_header: false, json_path: nil - @source = source.to_s - @format = format.to_s - @has_header = ActiveModel::Type::Boolean.new.cast(has_header) - @json_path = json_path.to_s - end - - def parse - raise ArgumentError, '入力が大きすぎます．' if @source.bytesize > MAX_BYTES - - @format == 'json' ? parse_json : parse_delimited - end - - private - - def parse_delimited - separator = @format == 'csv' ? ',' : "\t" - table = CSV.parse(@source, col_sep: separator).map { |row| row.map { _1.to_s.strip } } - raise ArgumentError, '入力が空です．' if table.empty? - - headers = @has_header ? table.shift : nil - ensure_limits!(table) - width = table.map(&:length).max.to_i - raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです．" if width > MAX_COLUMNS - { - columns: (0...width).map { |index| - { - key: index.to_s, - label: "#{ column_name(index) }：#{ headers&.[](index).presence || '列' }", - } - }, - rows: table.each_with_index.map { |values, index| - { source_row: index + (@has_header ? 2 : 1), values: } - }, - } - rescue CSV::MalformedCSVError => e - raise ArgumentError, "CSV・TSV の形式が不正です: #{ e.message }" - end - - def parse_json - value = JSON.parse(@source, max_nesting: 20) - @json_path.split('.').reject(&:blank?).each do |part| - value = - case value - when Array - index = Integer(part, exception: false) - raise ArgumentError if index.nil? || index.negative? || index >= value.length - - value[index] - when Hash - raise ArgumentError unless value.key?(part) || value.key?(part.to_sym) - - value.key?(part) ? value[part] : value[part.to_sym] - else - raise ArgumentError - end - end - raise ArgumentError, 'JSON の投稿候補は配列にしてください．' unless value.is_a?(Array) - - ensure_limits!(value) - unless value.all?(Hash) - invalid = value.each_with_index.filter_map { |item, index| - index + 1 unless item.is_a?(Hash) - } - raise ArgumentError, "JSON の投稿候補は object の配列にしてください: 行 #{ invalid.join(', ') }" - end - keys = value.flat_map(&:keys).map(&:to_s).uniq - raise ArgumentError, "列数は #{ MAX_COLUMNS } 列までです．" if keys.length > MAX_COLUMNS - { - columns: keys.each_with_index.map { |key, index| - { key:, label: "#{ column_name(index) }：#{ key }" } - }, - rows: value.each_with_index.map { |item, index| - { - source_row: index + 1, - values: keys.map { |key| - json_value = item.key?(key) ? item[key] : item[key.to_sym] - json_cell(json_value) - }, - } - }, - } - rescue JSON::ParserError, KeyError, ArgumentError => e - raise ArgumentError, "JSON の形式が不正です: #{ e.message }" - end - - def ensure_limits! rows - raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if rows.length > MAX_ROWS - raise ArgumentError, 'セルが大きすぎます．' if rows.flatten.any? { _1.to_s.bytesize > MAX_CELL_BYTES } - end - - def json_cell value - case value - when nil then '' - when String, Numeric, TrueClass, FalseClass then value.to_s - when Array, Hash then JSON.generate(value) - else raise ArgumentError, 'JSON のセル値が不正です．' - end - end - - def column_name index - name = +'' - loop do - name.prepend((65 + (index % 26)).chr) - index = (index / 26) - 1 - break if index < 0 - end - name - end -end diff --git a/backend/app/services/post_import_url_list_parser.rb b/backend/app/services/post_import_url_list_parser.rb new file mode 100644 index 0000000..80b78a5 --- /dev/null +++ b/backend/app/services/post_import_url_list_parser.rb @@ -0,0 +1,23 @@ +class PostImportUrlListParser + MAX_ROWS = 100 + MAX_BYTES = 1.megabyte + MAX_URL_BYTES = 20.kilobytes + + def self.parse source + raw = source.to_s + raise ArgumentError, '入力が大きすぎます．' if raw.bytesize > MAX_BYTES + + rows = raw.split(/\r\n|\n|\r/).each_with_index.filter_map { |line, index| + url = line.strip + next if url.blank? + + raise ArgumentError, 'URL が長すぎます．' if url.bytesize > MAX_URL_BYTES + + { source_row: index + 1, url: } + } + raise ArgumentError, 'URL を入力してください．' if rows.empty? + raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if rows.length > MAX_ROWS + + rows + end +end diff --git a/backend/config/routes.rb b/backend/config/routes.rb index 46af675..2f3484d 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -34,7 +34,6 @@ Rails.application.routes.draw do end scope 'posts/import', controller: :post_imports do - post :parse post :preview post :validate post '', action: :create diff --git a/frontend/src/pages/posts/PostImportPage.tsx b/frontend/src/pages/posts/PostImportPage.tsx index 314ac8f..7b10a0a 100644 --- a/frontend/src/pages/posts/PostImportPage.tsx +++ b/frontend/src/pages/posts/PostImportPage.tsx @@ -21,28 +21,17 @@ import type { User } from '@/types' type Props = { user: User | null } -type Column = { - key: string - label: string } - -type ParsedRow = { - sourceRow: number - values: string[] } - -type ParsedImport = { - columns: Column[] - rows: ParsedRow[] } - -type Origin = 'automatic' | 'mapped' | 'fixed' | 'manual' - +type Origin = 'automatic' | 'manual' +type Phase = 'source' | 'preview' | 'result' +type RepairMode = 'all' | 'failed' type ImportStatus = 'pending' | 'created' | 'skipped' | 'failed' type ImportResultStatus = 'created' | 'skipped' | 'failed' -type Phase = 'source' | 'mapping' | 'preview' | 'result' +type AttributeValue = string | number type ImportRow = { sourceRow: number url: string - attributes: Record + attributes: Record warnings: string[] validationErrors: Record importErrors?: Record @@ -60,41 +49,13 @@ type ImportResultRow = { post?: { id: number } errors?: Record } -type Mapping = { columns: string[] } -type RepairMode = 'all' | 'failed' - -const MAPPING_FIELDS = [ - 'title', - 'thumbnail_base', - 'original_created_from', - 'original_created_before', - 'duration', - 'tags', - 'parent_post_ids'] as const - const RESULT_STATUSES: ImportResultStatus[] = ['created', 'skipped', 'failed'] -const MERGE_FIELDS = ['url', 'attributes', 'provenance', 'tagSources'] as const const ORIGIN_LABELS: Record = { automatic: '自動取得', - mapped: '入力データ', - fixed: '固定値', manual: '手修正' } -const detectFormat = (source: string): 'csv' | 'tsv' => - source.includes ('\t') - ? 'tsv' - : source.includes (',') - ? 'csv' - : 'tsv' - - -const publicGoogleSheetURL = (source: string): boolean => - /^https:\/\/docs\.google\.com\/spreadsheets\/d\/e\/[A-Za-z0-9_-]+\/pubhtml#gid=\d+$/.test ( - source.trim ()) - - const mergeValidatedRows = ( current: ImportRow[], validated: ImportRow[], @@ -118,111 +79,18 @@ const mergeValidatedRows = ( !(current.some (_1 => _1.sourceRow === row.sourceRow)))) -const mergePreviewRows = ( - current: ImportRow[], - previewed: ImportRow[], -): ImportRow[] => { - const currentMap = new Map (current.map (row => [row.sourceRow, row])) - - const mergedRows = previewed.map (previewedRow => { - const previous = currentMap.get (previewedRow.sourceRow) - if (previous == null) - return previewedRow - if (previous.importStatus === 'created') - return previous - - const merged = { - ...previewedRow, - attributes: { ...previewedRow.attributes }, - provenance: { ...previewedRow.provenance }, - tagSources: { ...(previewedRow.tagSources ?? { }) } } - - Object.entries (previous.provenance).forEach (([field, origin]) => { - if (origin !== 'manual') - return - - if (field === 'url') - { - merged.url = previous.url - merged.metadataUrl = previous.metadataUrl - merged.provenance.url = 'manual' - return - } - if (field === 'tags') - { - merged.attributes.tags = previous.attributes.tags ?? '' - merged.provenance.tags = 'manual' - merged.tagSources = { - ...merged.tagSources, - manual: previous.tagSources?.manual ?? previous.attributes.tags ?? '' } - return - } - - merged.attributes[field] = previous.attributes[field] ?? '' - merged.provenance[field] = 'manual' - }) - - const changed = MERGE_FIELDS.some (field => - JSON.stringify (previous[field]) !== JSON.stringify (merged[field])) - - return { - ...merged, - importStatus: changed ? 'pending' : previous.importStatus, - createdPostId: previous.createdPostId, - importErrors: changed ? undefined : previous.importErrors } - }) - - return mergedRows.concat ( - current.filter (row => - row.importStatus === 'created' - && !(previewed.some (_1 => _1.sourceRow === row.sourceRow)))) -} - - const rowMessages = (row: ImportRow): string[] => [ ...row.warnings, ...Object.values (row.validationErrors ?? { }).flat (), ...Object.values (row.importErrors ?? { }).flat ()] -const sampleURLCount = (rows: ParsedRow[], index: number): number => - rows.filter (row => /^https?:\/\//.test (row.values[index] ?? '')).length - - -const renderMappingSelect = ( - columns: Column[], - field: string, - value: string, - update: (field: string, value: string) => void, -) => ( - ) - - const PostImportPage: FC = ({ user }) => { const editable = canEditContent (user) const [source, setSource] = useState ('') const [phase, setPhase] = useState ('source') - const [format, setFormat] = - useState<'csv' | 'tsv' | 'json' | 'google_sheets'> ('tsv') - const [hasHeader, setHasHeader] = useState (false) - const [jsonPath, setJsonPath] = useState ('') - const [urlColumn, setUrlColumn] = useState ('') - const [parsed, setParsed] = useState (null) - const [columns, setColumns] = useState ([]) const [rows, setRows] = useState ([]) - const [mappings, setMappings] = useState> ({ }) const [existing, setExisting] = useState<'skip' | 'error'> ('skip') const [loading, setLoading] = useState (false) const [repairMode, setRepairMode] = useState ('all') @@ -230,64 +98,16 @@ const PostImportPage: FC = ({ user }) => { const validationGeneration = useRef (0) const rowRefs = useRef> ({ }) - const parseSource = async () => { - setLoading (true) - try - { - const data = await apiPost ('/posts/import/parse', { - source, - format, - has_header: hasHeader, - json_path: jsonPath, - existing }) - setColumns (data.columns) - setParsed (data) - setMappings ({ }) - setRows ([]) - setRepairMode ('all') - if (data.columns.length === 1) - { - setUrlColumn ('0') - } - else - { - const candidate = data.columns - .map ((_, index) => ({ - index, - count: sampleURLCount (data.rows, index) })) - .sort ((a, b) => b.count - a.count)[0] - setUrlColumn (candidate?.count ? String (candidate.index) : '') - } - setPhase ('mapping') - } - catch - { - toast ({ - title: '読込みに失敗しました', - description: '入力形式を確認してください．' }) - } - finally - { - setLoading (false) - } - } - const preview = async () => { - if (!parsed || urlColumn === '') - return - setLoading (true) try { const data = await apiPost<{ rows: ImportRow[] }> ('/posts/import/preview', { - parsed: { columns: parsed.columns, rows: parsed.rows }, - url_column: urlColumn, - mappings, + source, existing }) - const mergedRows = mergePreviewRows (rows, data.rows) - setRows (mergedRows) + setRows (data.rows) + setRepairMode ('all') setPhase ('preview') - await validateRows (mergedRows, -1) } catch (error) { @@ -297,7 +117,7 @@ const PostImportPage: FC = ({ user }) => { ?? error.response?.data?.baseErrors?.[0] : undefined toast ({ - title: '投稿プレビューに失敗しました', + title: '投稿情報の取得に失敗しました', description: message ?? '入力を確認してください．' }) } finally @@ -309,6 +129,7 @@ const PostImportPage: FC = ({ user }) => { const validateRows = async ( nextRows: ImportRow[], changedRow: number, + existingValue = existing, ) => { const generation = ++validationGeneration.current setLoading (true) @@ -316,7 +137,7 @@ const PostImportPage: FC = ({ user }) => { { const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', { rows: nextRows.filter (row => row.importStatus !== 'created'), - existing, + existing: existingValue, changed_row: changedRow }) if (generation === validationGeneration.current) setRows (current => mergeValidatedRows (current, validated.rows)) @@ -337,7 +158,7 @@ const PostImportPage: FC = ({ user }) => { const next = { ...row, attributes: { ...row.attributes, [field]: value }, - provenance: { ...row.provenance, [field]: 'manual' }, + provenance: { ...row.provenance, [field]: 'manual' as const }, importStatus: 'pending' as const, importErrors: undefined } const nextRows = rows.map ((currentRow, rowIndex) => @@ -358,7 +179,7 @@ const PostImportPage: FC = ({ user }) => { const next = { ...row, url: value, - provenance: { ...row.provenance, url: 'manual' }, + provenance: { ...row.provenance, url: 'manual' as const }, importStatus: 'pending' as const, importErrors: undefined } const nextRows = rows.map ((currentRow, rowIndex) => @@ -386,22 +207,26 @@ const PostImportPage: FC = ({ user }) => { setRows (pendingRows) try { - const generation = ++validationGeneration.current - setLoading (true) - const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', { - rows: pendingRows.filter (row => row.importStatus !== 'created'), - existing: value, - changed_row: -1 }) - if (generation === validationGeneration.current) - setRows (current => mergeValidatedRows (current, validated.rows)) + await validateRows (pendingRows, -1, value) } - finally + catch { - if (validationGeneration.current > 0) - setLoading (false) + toast ({ title: '既存投稿の再検証に失敗しました' }) } } + const retryRow = (sourceRow: number) => + setRows (current => current.map (row => + row.sourceRow === sourceRow && row.importStatus === 'failed' + ? { ...row, importStatus: 'pending', importErrors: undefined } + : row)) + + const retryRowFromResult = (sourceRow: number) => { + retryRow (sourceRow) + setRepairMode ('failed') + setPhase ('preview') + } + const submit = async () => { const targets = rows.filter (row => row.importStatus == null || row.importStatus === 'pending') @@ -454,23 +279,6 @@ const PostImportPage: FC = ({ user }) => { } } - const updateMapping = (field: string, value: string) => - setMappings (current => ({ - ...current, - [field]: { columns: value === '' ? [] : [value] } })) - - const retryRow = (sourceRow: number) => - setRows (current => current.map (row => - row.sourceRow === sourceRow && row.importStatus === 'failed' - ? { ...row, importStatus: 'pending', importErrors: undefined } - : row)) - - const retryRowFromResult = (sourceRow: number) => { - retryRow (sourceRow) - setRepairMode ('failed') - setPhase ('preview') - } - const resultRows = rows.filter (row => row.importStatus != null && RESULT_STATUSES.includes (row.importStatus)) const failedCount = rows.filter (row => row.importStatus === 'failed').length @@ -508,274 +316,133 @@ const PostImportPage: FC = ({ user }) => { ? (

- URL を一行に一件ずつ貼り付けるか、 - CSV・TSV・JSON を入力してください． + URL を一行に一件ずつ入力してください．

- {publicGoogleSheetURL (source) && ( -
- 公開 Google スプレッドシート URL - - -
)} - - {() => ( - )} - - + {() => (