From 08bf92ff794d20d5dd54abdc80770cda67abf742 Mon Sep 17 00:00:00 2001 From: miteruzo Date: Sat, 11 Jul 2026 21:45:53 +0900 Subject: [PATCH] #399 --- .../controllers/post_imports_controller.rb | 60 ++++++++- .../post_import_google_sheets_fetcher.rb | 2 +- backend/app/services/post_import_previewer.rb | 99 ++++++++++++--- backend/app/services/post_import_runner.rb | 18 ++- backend/app/services/post_metadata_fetcher.rb | 17 ++- backend/config/routes.rb | 2 + frontend/src/pages/posts/PostImportPage.tsx | 117 ++++++++++++++++-- 7 files changed, 276 insertions(+), 39 deletions(-) diff --git a/backend/app/controllers/post_imports_controller.rb b/backend/app/controllers/post_imports_controller.rb index 65ffba8..e6fd0f0 100644 --- a/backend/app/controllers/post_imports_controller.rb +++ b/backend/app/controllers/post_imports_controller.rb @@ -1,7 +1,7 @@ class PostImportsController < ApplicationController before_action :require_member! - def preview + def parse source = if params[:format] == 'google_sheets' PostImportGoogleSheetsFetcher.fetch!(params[:source], rate_key: current_user.id) @@ -11,6 +11,13 @@ class PostImportsController < ApplicationController format = params[:format] == 'google_sheets' ? 'csv' : params[:format] parsed = PostImportSourceParser.new(source:, format:, has_header: params[:has_header], json_path: params[:json_path]).parse + render json: parsed + rescue ArgumentError => e + render_bad_request(e.message) + end + + def preview + parsed = parsed_params url_column = params[:url_column].presence || (parsed[:columns].length == 1 ? 0 : nil) return render_bad_request('URL 列を選択してください．') if url_column.nil? @@ -21,9 +28,23 @@ class PostImportsController < ApplicationController render_bad_request(e.message) end + def validate + rows = validate_rows_params + changed_row = Integer(params[:changed_row], exception: false) + result = PostImportPreviewer.new(parsed: { columns: [], rows: [] }, url_column: 0, + mappings: {}, existing: params[:existing]).preview_rows( + rows:, fetch_metadata: changed_row, + metadata_cache: { }) + render json: { rows: result } + rescue ArgumentError => e + render_bad_request(e.message) + end + def create - render json: PostImportRunner.new(actor: current_user, rows: params[:rows].to_a).run, - status: :created + result = PostImportRunner.new(actor: current_user, rows: params[:rows].to_a).run + render json: result, status: result[:created].positive? ? :created : :ok + rescue ArgumentError => e + render_bad_request(e.message) end private @@ -34,4 +55,37 @@ class PostImportsController < ApplicationController head :forbidden end + + def parsed_params + value = params.require(:parsed).to_unsafe_h.deep_transform_keys { _1.to_s.underscore } + columns = Array(value['columns']) + rows = Array(value['rows']) + raise ArgumentError, '解析結果の形式が不正です．' unless columns.all?(Hash) && rows.all?(Hash) + + { columns: columns.map { |column| column.slice('key', 'label') }, + rows: rows.map { |row| { source_row: row['source_row'], values: Array(row['values']) } } } + end + + def validate_rows_params + rows = Array(params[:rows]) + raise ArgumentError, '取込件数が多すぎます．' if rows.length > PostImportSourceParser::MAX_ROWS + + rows.map do |row| + value = ActionController::Parameters.new(row).permit(:source_row, :sourceRow, :url, + attributes: {}, provenance: {}, + fetch_warnings: [], validation_warnings: []) + normalised = value.to_h.deep_transform_keys { _1.to_s.underscore } + attributes = normalised.fetch('attributes', { }) + provenance = normalised.fetch('provenance', { }) + allowed = PostImportPreviewer::FIELDS + unless attributes.is_a?(Hash) && provenance.is_a?(Hash) && + (attributes.keys - allowed).empty? && + (provenance.keys - (allowed + ['url'])).empty? + raise ArgumentError, '行の形式が不正です．' + end + raise ArgumentError, 'セルが大きすぎます．' if [normalised['url'], *attributes.values].any? { _1.to_s.bytesize > PostImportSourceParser::MAX_CELL_BYTES } + + normalised.symbolize_keys + end + end end diff --git a/backend/app/services/post_import_google_sheets_fetcher.rb b/backend/app/services/post_import_google_sheets_fetcher.rb index 6da506f..af6341b 100644 --- a/backend/app/services/post_import_google_sheets_fetcher.rb +++ b/backend/app/services/post_import_google_sheets_fetcher.rb @@ -54,6 +54,6 @@ class PostImportGoogleSheetsFetcher def csv? response type = response.content_type.to_s.downcase - type.include?('text/csv') || response.body.valid_encoding? + type.include?('text/csv') || type.include?('application/csv') end end diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index 6d4323e..107f432 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -9,30 +9,55 @@ class PostImportPreviewer end def preview + preview_rows(rows: @parsed[:rows]) + end + + def preview_rows rows:, fetch_metadata: true, metadata_cache: { } urls = { } - @parsed[:rows].map do |row| - values = row[:values] - attributes = FIELDS.to_h { |field| [field, mapped_value(field, values)] } - url = values[@url_column].to_s.strip - metadata, warnings = fetch_metadata(url) - attributes = metadata.merge(attributes) { |_key, automatic, input| input.presence || automatic } + rows.map do |row| + row = row.symbolize_keys + values = row[:values] || [] + attributes = row[:attributes]&.stringify_keys || FIELDS.to_h { |field| [field, mapped_value(field, values)] } + provenance = row[:provenance]&.stringify_keys || mapped_provenance(attributes) + url = row[:url].presence || values[@url_column].to_s.strip + provenance['url'] ||= row[:url].present? ? 'manual' : 'mapped' + attributes['url'] = url + fetch_warnings = Array(row[:fetch_warnings]).dup + should_fetch = fetch_metadata == true || fetch_metadata.to_i == row[:source_row].to_i + metadata, fetch_warnings = metadata_for(url, should_fetch, metadata_cache, fetch_warnings) + metadata.each do |field, value| + next unless provenance[field] == 'automatic' || attributes[field].blank? + + attributes[field] = value + provenance[field] = 'automatic' + end post = Post.new(url:) post.valid? normal_url = post.url errors = post.errors.to_hash.transform_values(&:dup) + validation_warnings = [] errors[:url] = ['URL が重複しています．'] if normal_url.present? && urls.key?(normal_url) urls[normal_url] = true if normal_url.present? if normal_url.present? && Post.exists?(url: normal_url) - @existing == 'error' ? errors[:url] = ['既存投稿です．'] : warnings << '既存投稿のためスキップします．' + @existing == 'error' ? errors[:url] = ['既存投稿です．'] : validation_warnings << '既存投稿のためスキップします．' end - validate_preview_tags(attributes['tags'], errors) + validate_basic_data(attributes, errors) + validate_preview_tags(attributes['tags'], errors, validation_warnings) validate_parents(attributes['parent_post_ids'], errors) - { source_row: row[:source_row], url: normal_url || url, attributes:, warnings:, errors:, - status: errors.present? ? 'error' : (warnings.present? ? 'warning' : 'ready'), + attributes.delete('url') + { source_row: row[:source_row], url: normal_url || url, attributes:, provenance:, + fetch_warnings:, validation_warnings:, warnings: fetch_warnings + validation_warnings, errors:, + status: errors.present? ? 'error' : ((fetch_warnings + validation_warnings).present? ? 'warning' : 'ready'), existing: normal_url.present? && Post.exists?(url: normal_url) } end end + def metadata_for url, fetch_metadata, cache, previous_warnings + return [{ }, previous_warnings] unless fetch_metadata + + cache[url] ||= fetch_metadata(url) + end + private def mapped_value field, values @@ -43,33 +68,71 @@ class PostImportPreviewer Array(mapping['columns']).filter_map { |index| values[index.to_i].to_s.presence }.join(field == 'tags' ? ' ' : '') end + def mapped_provenance attributes + attributes.to_h.to_h { |field, value| + mapping = @mappings[field.to_s] + origin = + if value.present? && mapping&.[]('kind') == 'fixed' + 'fixed' + elsif value.present? + 'mapped' + else + 'automatic' + end + [field.to_s, origin] + } + end + def fetch_metadata url return [{ }, ['URL が空です．']] if url.blank? - data = PostMetadataFetcher.fetch(url) + data = PostMetadataFetcher.fetch(url).stringify_keys warnings = [] - warnings << 'タイトルを取得できませんでした．' if data[:title].blank? - warnings << 'サムネールを取得できませんでした．' if data[:thumbnail_base].blank? + warnings << 'タイトルを取得できませんでした．' if data['title'].blank? + warnings << 'サムネールを取得できませんでした．' if data['thumbnail_base'].blank? [data.compact, warnings] rescue Preview::UrlSafety::UnsafeUrl, Preview::HttpFetcher::FetchFailed, Preview::HttpFetcher::ResponseTooLarge => e [{ }, ["メタデータを取得できませんでした: #{ e.message }"]] end - def validate_preview_tags raw, errors + def validate_preview_tags raw, errors, warnings names = raw.to_s.split return if names.empty? if names.any? { _1.downcase.start_with?('nico:') } errors[:tags] = ['ニコニコ・タグは直接指定できません．'] return end - canonical = names.map { TagName.canonicalise(_1.sub(/\[.*\]\z/, '')).first } - if Tag.joins(:tag_name).where(tag_names: { name: canonical }, deprecated_at: nil).count < canonical.uniq.length - errors[:tags] = ['存在しないタグ又は廃止済みタグがあります．'] - end + parsed = names.map { |name| TagName.canonicalise(name.sub(/\[.*\]\z/, '')).first } + existing = Tag.joins(:tag_name).where(tag_names: { name: parsed }).includes(:tag_name).to_a + deprecated = existing.select(&:deprecated?).map(&:name) + errors[:tags] = ["廃止済みタグがあります: #{ deprecated.join(' ') }"] if deprecated.present? + known = existing.reject(&:deprecated?).map(&:name) + new_tags = parsed.uniq - known + warnings << "新規タグを作成します: #{ new_tags.join(' ') }" if new_tags.present? rescue Tag::SectionLiteralParseError errors[:tags] = ['タグ区間の記法が不正です．'] end + def validate_basic_data attributes, errors + post = Post.new(title: attributes['title'].presence, + url: attributes['url'], + thumbnail_base: attributes['thumbnail_base'].presence, + original_created_from: attributes['original_created_from'].presence, + original_created_before: attributes['original_created_before'].presence, + video_ms: parse_duration(attributes['duration'], errors)) + post.valid? + post.errors.each { |error| (errors[error.attribute] ||= []) << error.message } + end + + def parse_duration value, errors + return nil if value.blank? + + value.is_a?(Numeric) ? value.to_i : Tag.time_to_ms!(value.to_s, tag_name: '動画時間') + rescue Tag::SectionLiteralParseError + errors[:video_ms] = ['動画時間の記法が不正です．'] + nil + end + def validate_parents raw, errors ids = raw.to_s.split.map { Integer(_1, exception: false) } return if ids.compact.empty? && raw.to_s.blank? diff --git a/backend/app/services/post_import_runner.rb b/backend/app/services/post_import_runner.rb index 69231da..6b59b90 100644 --- a/backend/app/services/post_import_runner.rb +++ b/backend/app/services/post_import_runner.rb @@ -1,11 +1,15 @@ class PostImportRunner + MAX_ROWS = PostImportSourceParser::MAX_ROWS + ATTRIBUTE_KEYS = %w[title thumbnail_base original_created_from original_created_before duration tags parent_post_ids video_ms].freeze def initialize actor:, rows: @actor = actor @rows = rows end def run - results = @rows.map { |row| run_row(row.to_h.stringify_keys) } + raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if @rows.length > MAX_ROWS + + results = @rows.map { |row| run_row(row.to_h.deep_transform_keys { _1.to_s.underscore }) } { created: results.count { _1[:status] == 'created' }, skipped: results.count { _1[:status] == 'skipped' }, failed: results.count { _1[:status] == 'failed' }, rows: results } end @@ -15,10 +19,20 @@ class PostImportRunner def run_row row return row.slice('source_row').merge(status: 'skipped') if row['status'] == 'skipped' || row['existing'] attributes = row.fetch('attributes', { }).to_h.transform_keys { _1.to_s.underscore } + raise ArgumentError, '取込項目が不正です．' unless (attributes.keys - ATTRIBUTE_KEYS).empty? attributes['url'] = row['url'] post = PostCreator.new(actor: @actor, attributes:).create! { source_row: row['source_row'], status: 'created', post: PostRepr.base(post) } + rescue ActiveRecord::RecordInvalid => e + { source_row: row['source_row'], status: 'failed', errors: e.record.errors.to_hash } + rescue Tag::NicoTagNormalisationError + { source_row: row['source_row'], status: 'failed', errors: { tags: ['ニコニコ・タグは直接指定できません．'] } } + rescue Tag::DeprecatedTagNormalisationError + { source_row: row['source_row'], status: 'failed', errors: { tags: ['廃止済みタグは付与できません．'] } } + rescue ArgumentError, PostCreator::VideoMsParseError + { source_row: row['source_row'], status: 'failed', errors: { base: ['入力値が不正です．'] } } rescue StandardError => e - { source_row: row['source_row'], status: 'failed', errors: { base: [e.message] } } + Rails.logger.error("post_import_runner_failure #{ { error: e.class.name, message: e.message }.to_json }") + { source_row: row['source_row'], status: 'failed', errors: { base: ['登録中にエラーが発生しました．'] } } end end diff --git a/backend/app/services/post_metadata_fetcher.rb b/backend/app/services/post_metadata_fetcher.rb index 4ea6acb..f4ec66d 100644 --- a/backend/app/services/post_metadata_fetcher.rb +++ b/backend/app/services/post_metadata_fetcher.rb @@ -6,9 +6,22 @@ class PostMetadataFetcher document = Nokogiri::HTML.parse(response.body) content = -> name { document.at_css("meta[property='#{ name }'], meta[name='#{ name }']")&.[]('content')&.strip.presence } duration = content.call('og:video:duration') || content.call('video:duration') + published = content.call('article:published_time') || content.call('date') + published_at = Time.zone.parse(published.to_s) if published.present? + platform_tags = platform_tags(uri) { title: metadata[:title], thumbnail_base: Preview::KnownSiteExtractor.thumbnail_url(uri) || metadata[:image_url], - original_created_from: content.call('article:published_time') || content.call('date'), - duration: duration&.to_f&.then { _1.positive? ? (_1 * 1_000).round : nil } } + original_created_from: published_at&.iso8601, + original_created_before: published_at && (published_at + 1.minute).iso8601, + duration: duration&.to_f&.then { _1.positive? ? (_1 * 1_000).round : nil }, + tags: platform_tags.join(' ') } end + + def self.platform_tags uri + return ['動画', 'YouTube'] if Preview::KnownSiteExtractor.thumbnail_url(uri) + return ['動画', 'ニコニコ'] if Preview::KnownSiteExtractor.niconico_video_id(uri) + + [] + end + private_class_method :platform_tags end diff --git a/backend/config/routes.rb b/backend/config/routes.rb index afa1f06..46af675 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -34,7 +34,9 @@ Rails.application.routes.draw do end scope 'posts/import', controller: :post_imports do + post :parse post :preview + post :validate post '', action: :create end diff --git a/frontend/src/pages/posts/PostImportPage.tsx b/frontend/src/pages/posts/PostImportPage.tsx index 5a4f19f..6043479 100644 --- a/frontend/src/pages/posts/PostImportPage.tsx +++ b/frontend/src/pages/posts/PostImportPage.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react' +import { useRef, useState } from 'react' import { Helmet } from 'react-helmet-async' import FieldError from '@/components/common/FieldError' @@ -26,8 +26,10 @@ type ImportRow = { attributes: Record warnings: string[] errors: Record + provenance: Record status: 'ready' | 'warning' | 'error' existing: boolean } +type ParsedImport = { columns: Column[], rows: { sourceRow: number, values: string[] }[] } const detectFormat = (source: string): 'csv' | 'tsv' => @@ -43,28 +45,30 @@ const PostImportPage: FC = ({ user }) => { const [format, setFormat] = useState<'csv' | 'tsv' | 'json' | 'google_sheets'> ('tsv') const [hasHeader, setHasHeader] = useState (false) const [jsonPath, setJsonPath] = useState ('') - const [urlColumn, setUrlColumn] = useState ('0') + const [urlColumn, setUrlColumn] = useState ('') + const [parsed, setParsed] = useState (null) const [columns, setColumns] = useState ([]) const [rows, setRows] = useState ([]) const [mappings, setMappings] = useState> ({ }) const [existing, setExisting] = useState<'skip' | 'error'> ('skip') const [loading, setLoading] = useState (false) + const validationGeneration = useRef (0) const editable = canEditContent (user) - const preview = async () => { + const parseSource = async () => { setLoading (true) try { - const data = await apiPost<{ columns: Column[], rows: ImportRow[] }> ('/posts/import/preview', { + const data = await apiPost ('/posts/import/parse', { source, format, has_header: hasHeader, json_path: jsonPath, - url_column: urlColumn, - mappings, existing }) setColumns (data.columns) - setRows (data.rows) + setParsed (data) + if (data.columns.length === 1) + setUrlColumn ('0') } catch { @@ -76,9 +80,81 @@ const PostImportPage: FC = ({ user }) => { } } - const updateAttribute = (index: number, field: string, value: string) => { - setRows (current => current.map ((row, rowIndex) => - rowIndex === index ? { ...row, attributes: { ...row.attributes, [field]: value } } : row)) + const preview = async () => { + if (!parsed || urlColumn === '') + return + setLoading (true) + try + { + const data = await apiPost<{ rows: ImportRow[] }> ('/posts/import/preview', { + parsed: { columns: parsed.columns, rows: parsed.rows }, + url_column: urlColumn, + mappings, + existing }) + setRows (data.rows) + } + catch + { + toast ({ title: '投稿プレビューに失敗しました' }) + } + finally + { + setLoading (false) + } + } + + const updateAttribute = async (index: number, field: string, value: string) => { + const row = rows[index] + const next = { ...row, + attributes: { ...row.attributes, [field]: value }, + provenance: { ...row.provenance, [field]: 'manual' } } + const nextRows = rows.map ((currentRow, rowIndex) => rowIndex === index ? next : currentRow) + setRows (nextRows) + const generation = ++validationGeneration.current + setLoading (true) + try + { + const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', + { rows: nextRows, existing, + changed_row: -1 }) + if (generation === validationGeneration.current) + setRows (validated.rows) + } + catch + { + toast ({ title: '行の再検証に失敗しました' }) + } + finally + { + if (generation === validationGeneration.current) + setLoading (false) + } + } + + const updateURL = async (index: number, value: string) => { + const row = rows[index] + const next = { ...row, url: value, provenance: { ...row.provenance, url: 'manual' } } + const nextRows = rows.map ((currentRow, rowIndex) => rowIndex === index ? next : currentRow) + setRows (nextRows) + const generation = ++validationGeneration.current + setLoading (true) + try + { + const validated = await apiPost<{ rows: ImportRow[] }> ('/posts/import/validate', + { rows: nextRows, existing, + changed_row: row.sourceRow }) + if (generation === validationGeneration.current) + setRows (validated.rows) + } + catch + { + toast ({ title: 'URL の再検証に失敗しました' }) + } + finally + { + if (generation === validationGeneration.current) + setLoading (false) + } } const submit = async () => { @@ -147,7 +223,22 @@ const PostImportPage: FC = ({ user }) => { {format === 'google_sheets' &&

非公開シートには対応していません。ウェブ公開 URL を使用するか、CSV を保存して選択、又は内容を貼り付けてください。

} - + + ) : parsed && rows.length === 0 ? ( +
+ 列の割当て +

URL 列と補助列を指定してから、投稿情報を取得します。

+ + {['title', 'thumbnail_base', 'original_created_from', 'original_created_before', 'duration', 'tags', 'parent_post_ids'].map (field => + )} +
) : (
@@ -162,11 +253,11 @@ const PostImportPage: FC = ({ user }) => { onChange={ev => setMappings (current => ({ ...current, [field]: { columns: ev.target.value === '' ? [] : [ev.target.value] } }))}> {columns.map ((column, index) => )} )} - +
{rows.map ((row, index) => )}
- {rows.map ((row, index) => )}
行URLタイトルサムネール日時動画時間タグ親投稿状態
{row.sourceRow} { const next = [...rows]; next[index] = { ...row, url: ev.target.value }; setRows (next) }}/> updateAttribute (index, 'title', ev.target.value)}/>{row.attributes.thumbnailBase && サムネール}{row.attributes.originalCreatedFrom ?? ''}{row.attributes.duration ?? ''} updateAttribute (index, 'tags', ev.target.value)}/> updateAttribute (index, 'parent_post_ids', ev.target.value)}/>{row.status}
+ {rows.map ((row, index) => {row.sourceRow} updateURL (index, ev.target.value)}/> updateAttribute (index, 'title', ev.target.value)}/> updateAttribute (index, 'thumbnailBase', ev.target.value)}/> updateAttribute (index, 'originalCreatedFrom', ev.target.value)}/> updateAttribute (index, 'originalCreatedBefore', ev.target.value)}/> updateAttribute (index, 'duration', ev.target.value)}/> updateAttribute (index, 'tags', ev.target.value)}/> updateAttribute (index, 'parentPostIds', ev.target.value)}/>{row.status})}
)} )