diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index bfc5dad..ac80b82 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -1,6 +1,5 @@ class PostsController < ApplicationController Event = Struct.new(:post, :tag, :user, :change_type, :timestamp, keyword_init: true) - MAX_BULK_THUMBNAIL_BYTES = 20 * 1024 * 1024 MAX_BULK_REQUEST_BYTES = 40 * 1024 * 1024 class VideoMsParseError < ArgumentError @@ -178,7 +177,7 @@ class PostsController < ApplicationController preflight = PostCreatePreflight.new( attributes: post_create_attributes, thumbnail: params[:thumbnail]).run - return render json: preflight if bool?(:dry) + return render json: dry_run_json(preflight) if bool?(:dry) post = PostCreator.new(actor: current_user, attributes: post_create_attributes.merge( @@ -191,7 +190,10 @@ class PostsController < ApplicationController :original_created_from, :original_created_before, :duration, - :video_ms).symbolize_keys).merge( + :video_ms, + :normalised_tags, + :tag_sections, + :normalised_parent_post_ids).symbolize_keys).merge( thumbnail: params[:thumbnail])).create! post.reload @@ -566,7 +568,6 @@ class PostsController < ApplicationController unless value.is_a?(ActionDispatch::Http::UploadedFile) raise ArgumentError, 'thumbnail upload が不正です．' end - raise ArgumentError, 'thumbnail file size が大きすぎます．' if value.size > MAX_BULK_THUMBNAIL_BYTES thumbnails[index] = value end @@ -581,6 +582,22 @@ class PostsController < ApplicationController PostCompactRepr.base(post) end + def dry_run_json preflight + preflight.slice( + :url, + :title, + :thumbnail_base, + :tags, + :parent_post_ids, + :original_created_from, + :original_created_before, + :duration, + :video_ms, + :field_warnings, + :base_warnings, + :existing_post) + end + def sync_parent_posts! post, parent_post_ids if parent_post_ids.include?(post.id) post.errors.add :parent_post_ids, '自分自身を親投稿にはできません．' diff --git a/backend/app/services/post_bulk_creator.rb b/backend/app/services/post_bulk_creator.rb index 4c75752..6bd333e 100644 --- a/backend/app/services/post_bulk_creator.rb +++ b/backend/app/services/post_bulk_creator.rb @@ -1,21 +1,54 @@ class PostBulkCreator def initialize actor:, posts:, thumbnails: - @actor = actor + @actor_id = actor.id @posts = posts @thumbnails = thumbnails end def run - results = @posts.each_with_index.map { |attributes, index| - create_row(attributes, index) - } + results = Array.new(@posts.length) + mutex = Mutex.new + next_index = 0 + + workers = Array.new(2) do + Thread.new do + ActiveRecord::Base.connection_pool.with_connection do + actor = User.find(@actor_id) + loop do + index = nil + begin + index = mutex.synchronize do + current = next_index + next_index += 1 + current + end + break if index >= @posts.length + + attributes = @posts[index] + results[index] = create_row(actor, attributes, index) + rescue StandardError => e + Rails.logger.error( + "post_bulk_creator_worker_failure #{ { error: e.class.name, + message: e.message, + index: }.to_json }") + results[index] = { + status: 'failed', + recoverable: false, + errors: { base: ['登録中にエラーが発生しました．'] }, + base_errors: [] } + end + end + end + end + end + workers.each(&:join) { results: } end private - def create_row attributes, index + def create_row actor, attributes, index preflight = PostCreatePreflight.new( attributes: attributes, @@ -27,7 +60,7 @@ class PostBulkCreator end post = PostCreator.new( - actor: @actor, + actor: actor, attributes: normalised_attributes(attributes, preflight, index)).create! result = { status: 'created', @@ -122,7 +155,10 @@ class PostBulkCreator original_created_from: preflight[:original_created_from], original_created_before: preflight[:original_created_before], duration: preflight[:duration], - video_ms: preflight[:video_ms] } + video_ms: preflight[:video_ms], + normalised_tags: preflight[:normalised_tags], + tag_sections: preflight[:tag_sections], + normalised_parent_post_ids: preflight[:normalised_parent_post_ids] } end def thumbnail_for index, attributes diff --git a/backend/app/services/post_create_plan.rb b/backend/app/services/post_create_plan.rb new file mode 100644 index 0000000..4a295b0 --- /dev/null +++ b/backend/app/services/post_create_plan.rb @@ -0,0 +1,95 @@ +class PostCreatePlan + def initialize attributes: + @attributes = attributes.symbolize_keys + end + + def build! + Tag.normalise_tags!(tag_names, with_tagme: false, deny_deprecated: true, + with_sections: true) => + { tags:, sections: } + + tags = Tag.expand_parent_tags(tags).reject(&:deprecated?) + video_ms = normalise_video_ms(tags) + validate_video_sections!(video_ms, sections) + parent_post_ids = normalise_parent_post_ids + validate_parent_post_ids!(parent_post_ids) + + { + url: @attributes[:url], + title: @attributes[:title].to_s, + thumbnail_base: @attributes[:thumbnail_base].presence, + original_created_from: @attributes[:original_created_from].presence, + original_created_before: @attributes[:original_created_before].presence, + tags: serialised_tags(tags, sections), + duration: @attributes[:duration].to_s, + video_ms: video_ms, + parent_post_ids: parent_post_ids.join(' '), + normalised_tags: tags, + tag_sections: sections, + normalised_parent_post_ids: parent_post_ids } + end + + private + + def tag_names = @attributes[:tags].to_s.split + + def normalise_parent_post_ids + Array(@attributes[:parent_post_ids]).flat_map { _1.to_s.split }.map { |token| + id = Integer(token, exception: false) + raise ArgumentError, "親投稿 Id. が不正です: #{ token }" if id.nil? || id <= 0 + + id + }.uniq.sort + end + + def validate_parent_post_ids! ids + missing = ids - Post.where(id: ids).pluck(:id) + raise ArgumentError, "存在しない親投稿 Id. があります: #{ missing.join(' ') }" if missing.present? + end + + def serialised_tags tags, sections + tags.uniq(&:id).map { |tag| + "#{ tag.name }#{ sections[tag.id].to_a.map { Post.section_literal(_1) }.join }" + }.sort.join(' ') + end + + def normalise_video_ms tags + return nil unless tags.any? { _1.id == Tag.video.id } + + video_ms = @attributes[:video_ms] + if video_ms.present? + value = Integer(video_ms, exception: false) + raise PostCreator::VideoMsParseError unless value&.positive? + + return value + end + + duration = @attributes[:duration] + return nil if duration.blank? + + value = Tag.time_to_ms!(duration.to_s, tag_name: '動画時間') + raise PostCreator::VideoMsParseError unless value.positive? + + value + rescue Tag::SectionLiteralParseError + raise PostCreator::VideoMsParseError + end + + def validate_video_sections! video_ms, sections + return unless video_ms + + sections.each_value do |ranges| + ranges.each do |begin_ms, end_ms| + post = Post.new + if begin_ms >= video_ms + post.errors.add :video_ms, 'タグ区間の開始が動画時間以上です．' + raise ActiveRecord::RecordInvalid, post + end + if end_ms && end_ms > video_ms + post.errors.add :video_ms, 'タグ区間の終端が動画時間を超えてゐます．' + raise ActiveRecord::RecordInvalid, post + end + end + end + end +end diff --git a/backend/app/services/post_create_preflight.rb b/backend/app/services/post_create_preflight.rb index fe84b94..8c15f1a 100644 --- a/backend/app/services/post_create_preflight.rb +++ b/backend/app/services/post_create_preflight.rb @@ -18,24 +18,54 @@ class PostCreatePreflight preview = PostImportPreviewer.new.preview_rows( rows: [preview_row], fetch_metadata: false).first - - validate_thumbnail_upload! + if preview[:existing_post_id].present? + return { + url: preview[:url], + title: preview[:attributes]['title'], + thumbnail_base: preview[:attributes]['thumbnail_base'], + tags: preview[:attributes]['tags'], + parent_post_ids: preview[:attributes]['parent_post_ids'], + original_created_from: preview[:attributes]['original_created_from'], + original_created_before: preview[:attributes]['original_created_before'], + duration: preview[:attributes]['duration'], + video_ms: preview[:attributes]['video_ms'], + field_warnings: final_field_warnings(preview[:field_warnings] || { }), + base_warnings: preview[:base_warnings], + existing_post: existing_post_compact(preview[:existing_post_id]) } + end if preview[:validation_errors].present? raise ValidationFailed.new(fields: preview[:validation_errors]) end + validate_thumbnail_upload! + + plan = PostCreatePlan.new( + attributes: { + url: preview[:url], + title: preview[:attributes]['title'], + thumbnail_base: preview[:attributes]['thumbnail_base'], + tags: preview[:attributes]['tags'], + parent_post_ids: preview[:attributes]['parent_post_ids'], + original_created_from: preview[:attributes]['original_created_from'], + original_created_before: preview[:attributes]['original_created_before'], + duration: preview[:attributes]['duration'], + video_ms: preview[:attributes]['video_ms'] }).build! + { - url: preview[:url], - title: preview[:attributes]['title'], - thumbnail_base: preview[:attributes]['thumbnail_base'], - tags: preview[:attributes]['tags'], - parent_post_ids: preview[:attributes]['parent_post_ids'], - original_created_from: preview[:attributes]['original_created_from'], - original_created_before: preview[:attributes]['original_created_before'], - duration: preview[:attributes]['duration'], - video_ms: preview[:attributes]['video_ms'], - field_warnings: preview[:field_warnings], + url: plan[:url], + title: plan[:title], + thumbnail_base: plan[:thumbnail_base], + tags: plan[:tags], + parent_post_ids: plan[:parent_post_ids], + original_created_from: plan[:original_created_from], + original_created_before: plan[:original_created_before], + duration: plan[:duration], + video_ms: plan[:video_ms], + normalised_tags: plan[:normalised_tags], + tag_sections: plan[:tag_sections], + normalised_parent_post_ids: plan[:normalised_parent_post_ids], + field_warnings: final_field_warnings(preview[:field_warnings] || { }), base_warnings: preview[:base_warnings], existing_post: existing_post_compact(preview[:existing_post_id]) } end @@ -78,10 +108,9 @@ class PostCreatePreflight return if @attributes[:thumbnail_base].present? return if @thumbnail.blank? - attachment = Post.resized_thumbnail_attachment(@thumbnail) - attachment[:io].close if attachment[:io].respond_to?(:close) - rescue MiniMagick::Error - raise ValidationFailed.new(fields: { thumbnail: ['サムネイル画像の変換に失敗しました．'] }) + PostThumbnailUploadValidator.validate!(@thumbnail) + rescue PostThumbnailUploadValidator::InvalidUpload => e + raise ValidationFailed.new(fields: { thumbnail: [e.message] }) end def existing_post_compact post_id @@ -90,4 +119,15 @@ class PostCreatePreflight post = Post.with_attached_thumbnail.find_by(id: post_id) PostCompactRepr.base(post) end + + def final_field_warnings field_warnings + thumbnail_warnings = (field_warnings['thumbnail_base'] || []).reject { _1 == 'サムネールなし' } + if @attributes[:thumbnail_base].blank? && @thumbnail.blank? + thumbnail_warnings = (thumbnail_warnings + ['サムネールなし']).uniq + end + + { + **field_warnings, + 'thumbnail_base' => thumbnail_warnings } + end end diff --git a/backend/app/services/post_creator.rb b/backend/app/services/post_creator.rb index 8f7d937..4912a88 100644 --- a/backend/app/services/post_creator.rb +++ b/backend/app/services/post_creator.rb @@ -21,15 +21,13 @@ class PostCreator ApplicationRecord.transaction do post.save! post.thumbnail.attach(thumbnail_attachment) if thumbnail_attachment.present? - Tag.normalise_tags!(tag_names, deny_deprecated: true, with_sections: true) => - { tags:, sections: } + tags = planned_tags + sections = planned_sections TagVersioning.record_tag_snapshots!(tags, created_by_user: @actor) - tags = Tag.expand_parent_tags(tags).reject(&:deprecated?) - post.video_ms = normalise_video_ms(tags) - validate_video_sections!(post.video_ms, sections) + post.video_ms = planned_video_ms post.save! sync_post_tags!(post, tags, sections) - sync_parent_posts!(post, parent_post_ids) + sync_parent_posts!(post, planned_parent_post_ids) PostVersionRecorder.record!(post:, event_type: :create, created_by_user: @actor) end post @@ -46,52 +44,26 @@ class PostCreator thumbnail_base: @attributes[:thumbnail_base].presence) end - def tag_names = @attributes[:tags].to_s.split + def planned_tags = planned_create_attributes[:normalised_tags] - def parent_post_ids - Array(@attributes[:parent_post_ids]).flat_map { _1.to_s.split }.map { |token| - id = Integer(token, exception: false) - raise ArgumentError, "親投稿 Id. が不正です: #{ token }" if id.nil? || id <= 0 + def planned_sections = planned_create_attributes[:tag_sections] - id - }.uniq + def planned_parent_post_ids = planned_create_attributes[:normalised_parent_post_ids] + + def planned_video_ms + planned_create_attributes[:video_ms] end - def normalise_video_ms tags - return nil unless tags.any? { _1.id == Tag.video.id } - - video_ms = @attributes[:video_ms] - if video_ms.present? - value = Integer(video_ms, exception: false) - raise VideoMsParseError unless value&.positive? - - return value - end - duration = @attributes[:duration] - return nil if duration.blank? - - value = Tag.time_to_ms!(duration.to_s, tag_name: '動画時間') - raise VideoMsParseError unless value.positive? - - value - rescue Tag::SectionLiteralParseError - raise VideoMsParseError - end - - def validate_video_sections! video_ms, sections - return unless video_ms - - sections.each_value do |ranges| - ranges.each do |begin_ms, end_ms| - post = Post.new - if begin_ms >= video_ms - post.errors.add :video_ms, 'タグ区間の開始が動画時間以上です．' - raise ActiveRecord::RecordInvalid, post - end - if end_ms && end_ms > video_ms - post.errors.add :video_ms, 'タグ区間の終端が動画時間を超えてゐます．' - raise ActiveRecord::RecordInvalid, post - end + def planned_create_attributes + @planned_create_attributes ||= begin + if @attributes.key?(:normalised_tags) + { + normalised_tags: @attributes[:normalised_tags], + tag_sections: @attributes[:tag_sections] || { }, + normalised_parent_post_ids: @attributes[:normalised_parent_post_ids] || [], + video_ms: @attributes[:video_ms] } + else + PostCreatePlan.new(attributes: @attributes).build! end end end diff --git a/backend/app/services/post_import_previewer.rb b/backend/app/services/post_import_previewer.rb index cbe15f8..1d349a4 100644 --- a/backend/app/services/post_import_previewer.rb +++ b/backend/app/services/post_import_previewer.rb @@ -487,6 +487,8 @@ class PostImportPreviewer end def parse_video_ms attributes, errors + return nil unless attributes['tags'].to_s.split.include?('動画') + video_ms = attributes['video_ms'] if video_ms.present? value = Integer(video_ms, exception: false) diff --git a/backend/app/services/post_import_url_list_parser.rb b/backend/app/services/post_import_url_list_parser.rb deleted file mode 100644 index 794cc61..0000000 --- a/backend/app/services/post_import_url_list_parser.rb +++ /dev/null @@ -1,25 +0,0 @@ -class PostImportUrlListParser - MAX_ROWS = 100 - MAX_BYTES = 1.megabyte - MAX_URL_BYTES = 20.kilobytes - - def self.parse source - raw = source.to_s - raise ArgumentError, '入力が大きすぎます．' if raw.bytesize > MAX_BYTES - - rows = raw.split(/\r\n|\n|\r/).each_with_index.filter_map { |line, index| - url = line.strip - next if url.blank? - - if url.bytesize > MAX_URL_BYTES - raise ArgumentError, "#{ index + 1 } 行目: URL が長すぎます．" - end - - { source_row: index + 1, url: } - } - raise ArgumentError, 'URL を入力してください．' if rows.empty? - raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです．" if rows.length > MAX_ROWS - - rows - end -end diff --git a/backend/app/services/post_thumbnail_upload_validator.rb b/backend/app/services/post_thumbnail_upload_validator.rb new file mode 100644 index 0000000..9e78e6c --- /dev/null +++ b/backend/app/services/post_thumbnail_upload_validator.rb @@ -0,0 +1,18 @@ +class PostThumbnailUploadValidator + MAX_THUMBNAIL_BYTES = 20 * 1024 * 1024 + + class InvalidUpload < StandardError; end + + def self.validate! thumbnail + return if thumbnail.blank? + raise InvalidUpload, 'thumbnail upload が不正です．' unless thumbnail.is_a?(ActionDispatch::Http::UploadedFile) + raise InvalidUpload, 'thumbnail file size が大きすぎます．' if thumbnail.size > MAX_THUMBNAIL_BYTES + + attachment = Post.resized_thumbnail_attachment(thumbnail) + attachment[:io].close if attachment[:io].respond_to?(:close) + rescue MiniMagick::Error + raise InvalidUpload, 'サムネイル画像の変換に失敗しました．' + ensure + thumbnail&.rewind if thumbnail.respond_to?(:rewind) + end +end diff --git a/frontend/src/lib/postImportStorage.ts b/frontend/src/lib/postImportStorage.ts index c91531b..b35aa3e 100644 --- a/frontend/src/lib/postImportStorage.ts +++ b/frontend/src/lib/postImportStorage.ts @@ -11,6 +11,9 @@ const SESSION_VERSION = 3 const SESSION_PREFIX = 'post-import-session:' const SOURCE_DRAFT_KEY = 'post-import-source-draft' const SESSION_MAX_AGE_MS = 24 * 60 * 60 * 1000 +const MAX_SESSION_JSON_BYTES = 262_144 +const MAX_SESSION_ROWS = 100 +const MAX_SOURCE_BYTES = 262_144 const SESSION_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i const ATTRIBUTE_KEYS = [ @@ -50,11 +53,15 @@ const SESSION_KEYS = [ 'source', 'rows', 'repairMode'] as const +const textEncoder = new TextEncoder () const isPlainObject = (value: unknown): value is Record => typeof value === 'object' && value != null && !(Array.isArray (value)) +const byteLength = (value: string): number => + textEncoder.encode (value).byteLength + export const validatePostImportSessionId = (value: unknown): string | null => { if (typeof value !== 'string') @@ -184,6 +191,14 @@ const hasOnlyKeys = ( const sanitiseResetSnapshot = (value: unknown): PostImportResetSnapshot | null => { if (!(isPlainObject (value))) return null + if (!(hasOnlyKeys (value, ['url', + 'attributes', + 'provenance', + 'tagSources', + 'fieldWarnings', + 'baseWarnings', + 'metadataUrl']))) + return null if (typeof value.url !== 'string') return null if (!(isPlainObject (value.attributes))) @@ -353,6 +368,8 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { return null if (typeof value.url !== 'string') return null + if (byteLength (value.url) > 20_480) + return null if (!(isPlainObject (value.attributes))) return null if (!(isPlainObject (value.provenance))) @@ -450,17 +467,23 @@ const sanitiseSession = (value: unknown): PostImportSession | null => { return null if (value.version !== SESSION_VERSION || !(Array.isArray (value.rows))) return null + if (value.rows.length === 0 || value.rows.length > MAX_SESSION_ROWS) + return null if (typeof value.expiresAt !== 'string' || isExpiredSession (value.expiresAt)) return null + if (typeof value.source !== 'string' || byteLength (value.source) > MAX_SOURCE_BYTES) + return null const rows = value.rows.map (sanitiseRow) if (rows.some (row => row == null)) return null + if ((new Set (rows.map (row => row?.sourceRow))).size !== rows.length) + return null return { version: SESSION_VERSION, expiresAt: value.expiresAt, - source: typeof value.source === 'string' ? value.source : '', + source: value.source, rows: (rows as PostImportRow[]).map (row => recalculateThumbnailWarning (row)), repairMode: value.repairMode === 'failed' ? 'failed' : 'all' } } @@ -497,6 +520,12 @@ export const cleanupExpiredPostImportSessions = ( const raw = sessionStorage.getItem (key) if (raw == null) continue + if (byteLength (raw) > MAX_SESSION_JSON_BYTES) + { + sessionStorage.removeItem (key) + --i + continue + } try { @@ -585,6 +614,11 @@ export const loadPostImportSession = ( const raw = readStorage (sessionKey (validatedId), onError) if (raw == null) return null + if (byteLength (raw) > MAX_SESSION_JSON_BYTES) + { + removeStorage (sessionKey (validatedId), onError) + return null + } try { @@ -598,6 +632,7 @@ export const loadPostImportSession = ( } catch { + removeStorage (sessionKey (validatedId), onError) return null } } diff --git a/frontend/src/lib/postNewQueryState.ts b/frontend/src/lib/postNewQueryState.ts index 1d358cf..44a531e 100644 --- a/frontend/src/lib/postNewQueryState.ts +++ b/frontend/src/lib/postNewQueryState.ts @@ -84,13 +84,55 @@ const STATE_KEYS = [ 'created_post_id', 'recoverable'] as const const SINGLE_ROW_KEYS = [...BASIC_KEYS, ...STATE_KEYS, 'session_id'] as const +const ENCODED_ROW_KEYS = [ + 'n', + 'u', + 't', + 'h', + 'f', + 'b', + 'x', + 'd', + 'g', + 'p', + 'm', + 's', + 'e', + 'i', + 'c', + 'r'] as const +const ALLOWED_MANUAL_FIELDS = [ + 'title', + 'thumbnailBase', + 'originalCreatedFrom', + 'originalCreatedBefore', + 'duration', + 'tags', + 'parentPostIds'] as const const MAX_REGULAR_URL_LENGTH = 768 const MAX_COMPRESSED_STATE_LENGTH = 767 const COMPRESSED_STATE_PREFIX = 'z1.' +const MAX_COMPRESSED_STATE_PARAM_LENGTH = 8_192 +const MAX_COMPRESSED_STATE_BYTES = 16_384 +const MAX_DECOMPRESSED_STATE_BYTES = 262_144 +const MAX_STATE_ROWS = 100 +const MAX_QUERY_STRING_BYTES = 20_480 const textEncoder = new TextEncoder () const textDecoder = new TextDecoder () +const isPlainObject = (value: unknown): value is Record => + typeof value === 'object' && value != null && !(Array.isArray (value)) + +const hasOnlyKeys = ( + value: Record, + allowedKeys: readonly string[], +): boolean => + Object.keys (value).every (key => allowedKeys.includes (key)) + +const byteLength = (value: string): number => + textEncoder.encode (value).byteLength + const isValidSkipReason = (value: unknown): value is PostImportSkipReason => value === 'existing' || value === 'manual' @@ -202,18 +244,25 @@ const compressBytes = async (value: Uint8Array): Promise => { const decompressBytes = async (value: Uint8Array): Promise => { const stream = new Blob ([value]).stream ().pipeThrough (new DecompressionStream ('gzip')) - return new Uint8Array (await new Response (stream).arrayBuffer ()) + const bytes = new Uint8Array (await new Response (stream).arrayBuffer ()) + if (bytes.byteLength > MAX_DECOMPRESSED_STATE_BYTES) + throw new Error ('decompressed state too large') + return bytes } const decodeCompressedState = async (value: string): Promise => { if (!(value.startsWith (COMPRESSED_STATE_PREFIX))) return null + if (value.length > MAX_COMPRESSED_STATE_PARAM_LENGTH) + return null const encoded = value.slice (COMPRESSED_STATE_PREFIX.length) const compressed = decodeBase64UrlBytes (encoded) if (compressed == null) return null + if (compressed.byteLength > MAX_COMPRESSED_STATE_BYTES) + return null try { @@ -318,6 +367,10 @@ const parseManual = (value: string | null): string[] => .filter (entry => entry !== '') +const hasOnlyAllowedManualFields = (value: string[]): boolean => + value.every (field => ALLOWED_MANUAL_FIELDS.includes (field as never)) + + const isStringArray = (value: unknown): value is string[] => Array.isArray (value) && value.every (entry => typeof entry === 'string') @@ -326,10 +379,12 @@ const parseEncodedRow = ( value: unknown, requireUrl: boolean, ): FullRowState | null => { - if (typeof value !== 'object' || value == null || Array.isArray (value)) + if (!(isPlainObject (value))) return null const row = value as Record + if (!(hasOnlyKeys (row, ENCODED_ROW_KEYS))) + return null const url = row['u'] const manual = row['m'] const skip = decodeSkip (row['s']) @@ -341,8 +396,12 @@ const parseEncodedRow = ( if (requireUrl && typeof url !== 'string') return null + if (typeof url === 'string' && byteLength (url) > MAX_QUERY_STRING_BYTES) + return null if (manual != null && !(isStringArray (manual))) return null + if ((manual ?? []).some (field => !(ALLOWED_MANUAL_FIELDS.includes (field as never)))) + return null if (row['s'] != null && skip == null) return null if (row['i'] != null && importStatus == null) @@ -382,6 +441,8 @@ const parseEncodedRow = ( const fieldValue = row[key] if (fieldValue != null && typeof fieldValue !== 'string') throw new Error ('invalid row') + if (typeof fieldValue === 'string' && byteLength (fieldValue) > MAX_QUERY_STRING_BYTES) + throw new Error ('invalid row') return [target, fieldValue ?? ''] }), ) as Record<(typeof stringFields)[number][1], string> @@ -418,6 +479,8 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { const url = params.get ('url') if (url == null || url === '') return null + if (byteLength (url) > MAX_QUERY_STRING_BYTES) + return null const hasOnlyUrl = Array.from (params.keys ()).every ( key => key === 'url' || key === 'session_id') @@ -460,6 +523,8 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { return null if (importStatus != null && !(isValidImportStatus (importStatus))) return null + if (!(hasOnlyAllowedManualFields (parseManual (params.get ('manual'))))) + return null const row = buildRow ({ source_row: 1, @@ -509,7 +574,12 @@ const parseMetaRows = ( return null const urls = urlsValue.split (' ').filter (url => url !== '') - if (urls.length < 2 || urls.length !== parsed.rows.length) + if (urls.length < 2 + || urls.length > MAX_STATE_ROWS + || urls.length !== parsed.rows.length + || parsed.rows.length > MAX_STATE_ROWS) + return null + if (urls.some (url => byteLength (url) > MAX_QUERY_STRING_BYTES)) return null const fullRows = parsed.rows.map ((row, index) => { @@ -520,6 +590,8 @@ const parseMetaRows = ( }) if (fullRows.some (row => row == null)) return null + if ((new Set (fullRows.map (row => (row as FullRowState).source_row))).size !== fullRows.length) + return null const rows = fullRows.map (row => buildRow (row as FullRowState)) return { @@ -545,12 +617,17 @@ const parseFullState = async (stateValue: string): Promise MAX_STATE_ROWS) return null const fullRows = parsed.rows.map (row => parseEncodedRow (row, true)) if (fullRows.some (row => row == null)) return null + if ((new Set (fullRows.map (row => (row as FullRowState).source_row))).size !== fullRows.length) + return null const rows = fullRows.map (row => buildRow (row as FullRowState)) return { diff --git a/frontend/src/pages/posts/PostImportReviewPage.tsx b/frontend/src/pages/posts/PostImportReviewPage.tsx index 3aa2e6f..48be61c 100644 --- a/frontend/src/pages/posts/PostImportReviewPage.tsx +++ b/frontend/src/pages/posts/PostImportReviewPage.tsx @@ -68,6 +68,7 @@ type PreviewResponse = { thumbnailBase?: string originalCreatedFrom?: string originalCreatedBefore?: string + parentPostIds?: string duration?: string videoMs?: number tags?: string @@ -137,8 +138,7 @@ const mergeDryRunRow = ( currentRow: PostImportRow, result: PreviewResponse, ): PostImportRow => - applyThumbnailWarning ( - initialisePreviewRows ([{ + applyThumbnailWarning ({ ...currentRow, url: result.url, attributes: { @@ -150,7 +150,7 @@ const mergeDryRunRow = ( duration: result.duration ?? '', videoMs: result.videoMs ?? '', tags: result.tags ?? '', - parentPostIds: String (currentRow.attributes.parentPostIds ?? '') }, + parentPostIds: String (result.parentPostIds ?? currentRow.attributes.parentPostIds ?? '') }, fieldWarnings: result.fieldWarnings ?? { }, baseWarnings: result.baseWarnings ?? [], validationErrors: { }, @@ -167,7 +167,38 @@ const mergeDryRunRow = ( currentRow.importStatus === 'created' ? 'created' : 'pending', - recoverable: undefined }])[0]) + recoverable: undefined }) + + +const buildPreviewResetSnapshot = ( + preview: PreviewResponse, +): PostImportRow['resetSnapshot'] => ({ + url: preview.url, + attributes: { + title: preview.title ?? '', + thumbnailBase: preview.thumbnailBase ?? '', + originalCreatedFrom: preview.originalCreatedFrom ?? '', + originalCreatedBefore: preview.originalCreatedBefore ?? '', + duration: preview.duration ?? '', + videoMs: preview.videoMs ?? '', + tags: preview.tags ?? '', + parentPostIds: String (preview.parentPostIds ?? '') }, + provenance: { + url: 'manual', + title: 'automatic', + thumbnailBase: 'automatic', + originalCreatedFrom: 'automatic', + originalCreatedBefore: 'automatic', + duration: 'automatic', + videoMs: 'automatic', + tags: 'automatic', + parentPostIds: 'automatic' }, + tagSources: { + automatic: preview.tags ?? '', + manual: '' }, + fieldWarnings: preview.fieldWarnings ?? { }, + baseWarnings: preview.baseWarnings ?? [], + metadataUrl: preview.url }) const indexedBulkResults = ( @@ -216,6 +247,9 @@ const mergePreviewRow = ( currentRow: PostImportRow, preview: PreviewResponse, ): PostImportRow => { + const fieldWarnings = preview.fieldWarnings ?? { } + const baseWarnings = preview.baseWarnings ?? [] + const metadataChanged = currentRow.metadataUrl !== preview.url const nextRow: PostImportRow = { ...currentRow, attributes: { ...currentRow.attributes }, @@ -224,8 +258,8 @@ const mergePreviewRow = ( automatic: currentRow.tagSources?.automatic ?? '', manual: currentRow.tagSources?.manual ?? '' }, url: preview.url, - fieldWarnings: preview.fieldWarnings ?? { }, - baseWarnings: preview.baseWarnings ?? [], + fieldWarnings, + baseWarnings, existingPostId: preview.existingPost?.id, existingPost: preview.existingPost ?? undefined, skipReason: @@ -234,9 +268,14 @@ const mergePreviewRow = ( : currentRow.skipReason === 'manual' ? 'manual' : undefined, + metadataUrl: preview.url, + resetSnapshot: + metadataChanged + ? buildPreviewResetSnapshot (preview) + : currentRow.resetSnapshot, status: - Object.keys (preview.fieldWarnings ?? { }).length > 0 - || (preview.baseWarnings?.length ?? 0) > 0 + Object.keys (fieldWarnings).length > 0 + || baseWarnings.length > 0 ? 'warning' : 'ready' } @@ -251,8 +290,10 @@ const mergePreviewRow = ( if (currentRow.provenance.tags !== 'manual') { nextRow.attributes.tags = preview.tags ?? '' - nextRow.tagSources.automatic = preview.tags ?? '' } + nextRow.tagSources.automatic = preview.tags ?? '' + if (currentRow.provenance.parentPostIds !== 'manual') + nextRow.attributes.parentPostIds = String (preview.parentPostIds ?? '') if (currentRow.provenance.videoMs !== 'manual') nextRow.attributes.videoMs = preview.videoMs ?? '' if (currentRow.provenance.duration !== 'manual') @@ -398,6 +439,7 @@ const PostImportReviewPage: FC = ({ user }) => { const refreshRows = async (baseSession: PostImportSession) => { let nextIndex = 0 + const previews = new Map () const worker = async () => { while (active) { @@ -421,15 +463,7 @@ const PostImportReviewPage: FC = ({ user }) => { signal: controller.signal }) if (!(active) || previewSequenceRef.current !== previewSequence) return - - const latestSession = sessionRef.current ?? baseSession - const currentRow = latestSession.rows.find ( - row => row.sourceRow === baseRow.sourceRow) - if (currentRow == null) - continue - const mergedRow = mergePreviewRow (currentRow, preview) - await persistSession (buildSession ( - replaceImportRow (latestSession.rows, mergedRow))) + previews.set (baseRow.sourceRow, preview) } catch (requestError) { @@ -437,11 +471,6 @@ const PostImportReviewPage: FC = ({ user }) => { return if (!(active) || previewSequenceRef.current !== previewSequence) return - const latestSession = sessionRef.current ?? baseSession - const currentRow = latestSession.rows.find ( - row => row.sourceRow === baseRow.sourceRow) - if (currentRow == null) - continue if (!(isApiError (requestError))) continue } @@ -454,6 +483,21 @@ const PostImportReviewPage: FC = ({ user }) => { await Promise.all ( Array.from ({ length: Math.min (4, baseSession.rows.length) }, () => worker ())) + if (!(active) || previewSequenceRef.current !== previewSequence) + return + + const latestSession = sessionRef.current ?? baseSession + const nextRows = latestSession.rows.map (row => { + const preview = previews.get (row.sourceRow) + if (preview == null + || row.skipReason === 'manual' + || row.importStatus === 'created' + || row.importStatus === 'skipped' + || isNonRecoverableFailedRow (row)) + return row + return mergePreviewRow (row, preview) + }) + await persistSession (buildSession (nextRows)) } const hydrate = async () => { @@ -494,7 +538,7 @@ const PostImportReviewPage: FC = ({ user }) => { duration: preview.duration ?? '', videoMs: preview.videoMs ?? '', tags: preview.tags ?? '', - parentPostIds: '' }, + parentPostIds: String (preview.parentPostIds ?? '') }, fieldWarnings: preview.fieldWarnings ?? { }, baseWarnings: preview.baseWarnings ?? [], validationErrors: { }, @@ -529,7 +573,7 @@ const PostImportReviewPage: FC = ({ user }) => { duration: preview.duration ?? '', videoMs: preview.videoMs ?? '', tags: preview.tags ?? '', - parentPostIds: '' }, + parentPostIds: String (preview.parentPostIds ?? '') }, provenance: { url: 'manual', title: 'automatic', @@ -544,7 +588,8 @@ const PostImportReviewPage: FC = ({ user }) => { automatic: preview.tags ?? '', manual: '' }, fieldWarnings: preview.fieldWarnings ?? { }, - baseWarnings: preview.baseWarnings ?? [] } }])[0])] + baseWarnings: preview.baseWarnings ?? [], + metadataUrl: preview.url } }])[0])] const persisted = await persistSession (buildSession (rows)) if (persisted == null) return @@ -557,10 +602,6 @@ const PostImportReviewPage: FC = ({ user }) => { return } - if (loadedSession == null) - { - await persistSession (loaded) - } void refreshRows (loaded) } @@ -612,8 +653,7 @@ const PostImportReviewPage: FC = ({ user }) => { if (row.sourceRow !== sourceRow) return row if (isExistingSkipRow (row) - || row.importStatus === 'created' - || isNonRecoverableFailedRow (row)) + || row.importStatus === 'created') return row return { ...row, @@ -779,23 +819,8 @@ const PostImportReviewPage: FC = ({ user }) => { const indexedResults = indexedBulkResults ([target], result.results) const latestAfterImport = sessionRef.current ?? pendingSession - const mergedRows = mergeImportResults (latestAfterImport.rows, indexedResults) - const recoverableRows = indexedResults.filter (row => - row.status === 'failed' - && row.recoverable - && Object.keys (row.errors ?? { }).length > 0) - const nextRows = mergedRows.map ((row): PostImportRow => { - const recoverable = recoverableRows.find ( - failedRow => failedRow.sourceRow === row.sourceRow) - if (recoverable == null) - return applyThumbnailWarning (row) - return applyThumbnailWarning ({ - ...row, - importStatus: 'pending', - recoverable: true, - validationErrors: recoverable.errors ?? { }, - importErrors: undefined }) - }) + const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults) + .map (row => applyThumbnailWarning (row)) const persisted = await persistSession (buildSession (nextRows)) if (persisted != null && persisted.rows.every (row => isCompletedReviewRow (row))) @@ -842,23 +867,8 @@ const PostImportReviewPage: FC = ({ user }) => { const indexedResults = indexedBulkResults (processingRows, result.results) const latestAfterImport = sessionRef.current ?? buildSession (currentSession.rows) - const mergedResults = mergeImportResults (latestAfterImport.rows, indexedResults) - const recoverableRows = indexedResults.filter (row => - row.status === 'failed' - && row.recoverable - && Object.keys (row.errors ?? { }).length > 0) - const nextRows = mergedResults.map ((row): PostImportRow => { - const recoverable = recoverableRows.find ( - failedRow => failedRow.sourceRow === row.sourceRow) - if (recoverable == null) - return applyThumbnailWarning (row) - return applyThumbnailWarning ({ - ...row, - importStatus: 'pending', - recoverable: true, - validationErrors: recoverable.errors ?? { }, - importErrors: undefined }) - }) + const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults) + .map (row => applyThumbnailWarning (row)) const persisted = await persistSession (buildSession (nextRows)) if (persisted != null && persisted.rows.every (row => isCompletedReviewRow (row))) @@ -940,8 +950,7 @@ const PostImportReviewPage: FC = ({ user }) => { skipDisabled={ busy || isExistingSkipRow (row) - || row.importStatus === 'created' - || isNonRecoverableFailedRow (row)} + || row.importStatus === 'created'} onEdit={() => editRow (row)} onToggleSkip={checked => toggleManualSkip (row.sourceRow, checked)} onRetry={canRetryResultRow (row) ? () => retry (row.sourceRow) : undefined}