広場投稿追加画面の刷新 (#399) #413

マージ済み
みてるぞ が 103 個のコミットを feature/399 から main へマージ 2026-07-19 00:03:11 +09:00
11個のファイルの変更449行の追加173行の削除
コミット ff970f8171 の変更だけを表示してゐます - すべてのコミットを表示
+21 -4
ファイルの表示
@@ -1,6 +1,5 @@
class PostsController < ApplicationController
Event = Struct.new(:post, :tag, :user, :change_type, :timestamp, keyword_init: true)
MAX_BULK_THUMBNAIL_BYTES = 20 * 1024 * 1024
MAX_BULK_REQUEST_BYTES = 40 * 1024 * 1024
class VideoMsParseError < ArgumentError
@@ -178,7 +177,7 @@ class PostsController < ApplicationController
preflight = PostCreatePreflight.new(
attributes: post_create_attributes,
thumbnail: params[:thumbnail]).run
return render json: preflight if bool?(:dry)
return render json: dry_run_json(preflight) if bool?(:dry)
post = PostCreator.new(actor: current_user,
attributes: post_create_attributes.merge(
@@ -191,7 +190,10 @@ class PostsController < ApplicationController
:original_created_from,
:original_created_before,
:duration,
:video_ms).symbolize_keys).merge(
:video_ms,
:normalised_tags,
:tag_sections,
:normalised_parent_post_ids).symbolize_keys).merge(
thumbnail: params[:thumbnail])).create!
post.reload
@@ -566,7 +568,6 @@ class PostsController < ApplicationController
unless value.is_a?(ActionDispatch::Http::UploadedFile)
raise ArgumentError, 'thumbnail upload が不正です.'
end
raise ArgumentError, 'thumbnail file size が大きすぎます.' if value.size > MAX_BULK_THUMBNAIL_BYTES
thumbnails[index] = value
end
@@ -581,6 +582,22 @@ class PostsController < ApplicationController
PostCompactRepr.base(post)
end
def dry_run_json preflight
preflight.slice(
:url,
:title,
:thumbnail_base,
:tags,
:parent_post_ids,
:original_created_from,
:original_created_before,
:duration,
:video_ms,
:field_warnings,
:base_warnings,
:existing_post)
end
def sync_parent_posts! post, parent_post_ids
if parent_post_ids.include?(post.id)
post.errors.add :parent_post_ids, '自分自身を親投稿にはできません.'
+43 -7
ファイルの表示
@@ -1,21 +1,54 @@
class PostBulkCreator
def initialize actor:, posts:, thumbnails:
@actor = actor
@actor_id = actor.id
@posts = posts
@thumbnails = thumbnails
end
def run
results = @posts.each_with_index.map { |attributes, index|
create_row(attributes, index)
}
results = Array.new(@posts.length)
mutex = Mutex.new
next_index = 0
workers = Array.new(2) do
Thread.new do
ActiveRecord::Base.connection_pool.with_connection do
actor = User.find(@actor_id)
loop do
index = nil
begin
index = mutex.synchronize do
current = next_index
next_index += 1
current
end
break if index >= @posts.length
attributes = @posts[index]
results[index] = create_row(actor, attributes, index)
rescue StandardError => e
Rails.logger.error(
"post_bulk_creator_worker_failure #{ { error: e.class.name,
message: e.message,
index: }.to_json }")
results[index] = {
status: 'failed',
recoverable: false,
errors: { base: ['登録中にエラーが発生しました.'] },
base_errors: [] }
end
end
end
end
end
workers.each(&:join)
{ results: }
end
private
def create_row attributes, index
def create_row actor, attributes, index
preflight =
PostCreatePreflight.new(
attributes: attributes,
@@ -27,7 +60,7 @@ class PostBulkCreator
end
post = PostCreator.new(
actor: @actor,
actor: actor,
attributes: normalised_attributes(attributes, preflight, index)).create!
result = {
status: 'created',
@@ -122,7 +155,10 @@ class PostBulkCreator
original_created_from: preflight[:original_created_from],
original_created_before: preflight[:original_created_before],
duration: preflight[:duration],
video_ms: preflight[:video_ms] }
video_ms: preflight[:video_ms],
normalised_tags: preflight[:normalised_tags],
tag_sections: preflight[:tag_sections],
normalised_parent_post_ids: preflight[:normalised_parent_post_ids] }
end
def thumbnail_for index, attributes
+95
ファイルの表示
@@ -0,0 +1,95 @@
class PostCreatePlan
def initialize attributes:
@attributes = attributes.symbolize_keys
end
def build!
Tag.normalise_tags!(tag_names, with_tagme: false, deny_deprecated: true,
with_sections: true) =>
{ tags:, sections: }
tags = Tag.expand_parent_tags(tags).reject(&:deprecated?)
video_ms = normalise_video_ms(tags)
validate_video_sections!(video_ms, sections)
parent_post_ids = normalise_parent_post_ids
validate_parent_post_ids!(parent_post_ids)
{
url: @attributes[:url],
title: @attributes[:title].to_s,
thumbnail_base: @attributes[:thumbnail_base].presence,
original_created_from: @attributes[:original_created_from].presence,
original_created_before: @attributes[:original_created_before].presence,
tags: serialised_tags(tags, sections),
duration: @attributes[:duration].to_s,
video_ms: video_ms,
parent_post_ids: parent_post_ids.join(' '),
normalised_tags: tags,
tag_sections: sections,
normalised_parent_post_ids: parent_post_ids }
end
private
def tag_names = @attributes[:tags].to_s.split
def normalise_parent_post_ids
Array(@attributes[:parent_post_ids]).flat_map { _1.to_s.split }.map { |token|
id = Integer(token, exception: false)
raise ArgumentError, "親投稿 Id. が不正です: #{ token }" if id.nil? || id <= 0
id
}.uniq.sort
end
def validate_parent_post_ids! ids
missing = ids - Post.where(id: ids).pluck(:id)
raise ArgumentError, "存在しない親投稿 Id. があります: #{ missing.join(' ') }" if missing.present?
end
def serialised_tags tags, sections
tags.uniq(&:id).map { |tag|
"#{ tag.name }#{ sections[tag.id].to_a.map { Post.section_literal(_1) }.join }"
}.sort.join(' ')
end
def normalise_video_ms tags
return nil unless tags.any? { _1.id == Tag.video.id }
video_ms = @attributes[:video_ms]
if video_ms.present?
value = Integer(video_ms, exception: false)
raise PostCreator::VideoMsParseError unless value&.positive?
return value
end
duration = @attributes[:duration]
return nil if duration.blank?
value = Tag.time_to_ms!(duration.to_s, tag_name: '動画時間')
raise PostCreator::VideoMsParseError unless value.positive?
value
rescue Tag::SectionLiteralParseError
raise PostCreator::VideoMsParseError
end
def validate_video_sections! video_ms, sections
return unless video_ms
sections.each_value do |ranges|
ranges.each do |begin_ms, end_ms|
post = Post.new
if begin_ms >= video_ms
post.errors.add :video_ms, 'タグ区間の開始が動画時間以上です.'
raise ActiveRecord::RecordInvalid, post
end
if end_ms && end_ms > video_ms
post.errors.add :video_ms, 'タグ区間の終端が動画時間を超えてゐます.'
raise ActiveRecord::RecordInvalid, post
end
end
end
end
end
+56 -16
ファイルの表示
@@ -18,24 +18,54 @@ class PostCreatePreflight
preview = PostImportPreviewer.new.preview_rows(
rows: [preview_row],
fetch_metadata: false).first
validate_thumbnail_upload!
if preview[:existing_post_id].present?
return {
url: preview[:url],
title: preview[:attributes]['title'],
thumbnail_base: preview[:attributes]['thumbnail_base'],
tags: preview[:attributes]['tags'],
parent_post_ids: preview[:attributes]['parent_post_ids'],
original_created_from: preview[:attributes]['original_created_from'],
original_created_before: preview[:attributes]['original_created_before'],
duration: preview[:attributes]['duration'],
video_ms: preview[:attributes]['video_ms'],
field_warnings: final_field_warnings(preview[:field_warnings] || { }),
base_warnings: preview[:base_warnings],
existing_post: existing_post_compact(preview[:existing_post_id]) }
end
if preview[:validation_errors].present?
raise ValidationFailed.new(fields: preview[:validation_errors])
end
validate_thumbnail_upload!
plan = PostCreatePlan.new(
attributes: {
url: preview[:url],
title: preview[:attributes]['title'],
thumbnail_base: preview[:attributes]['thumbnail_base'],
tags: preview[:attributes]['tags'],
parent_post_ids: preview[:attributes]['parent_post_ids'],
original_created_from: preview[:attributes]['original_created_from'],
original_created_before: preview[:attributes]['original_created_before'],
duration: preview[:attributes]['duration'],
video_ms: preview[:attributes]['video_ms'] }).build!
{
url: preview[:url],
title: preview[:attributes]['title'],
thumbnail_base: preview[:attributes]['thumbnail_base'],
tags: preview[:attributes]['tags'],
parent_post_ids: preview[:attributes]['parent_post_ids'],
original_created_from: preview[:attributes]['original_created_from'],
original_created_before: preview[:attributes]['original_created_before'],
duration: preview[:attributes]['duration'],
video_ms: preview[:attributes]['video_ms'],
field_warnings: preview[:field_warnings],
url: plan[:url],
title: plan[:title],
thumbnail_base: plan[:thumbnail_base],
tags: plan[:tags],
parent_post_ids: plan[:parent_post_ids],
original_created_from: plan[:original_created_from],
original_created_before: plan[:original_created_before],
duration: plan[:duration],
video_ms: plan[:video_ms],
normalised_tags: plan[:normalised_tags],
tag_sections: plan[:tag_sections],
normalised_parent_post_ids: plan[:normalised_parent_post_ids],
field_warnings: final_field_warnings(preview[:field_warnings] || { }),
base_warnings: preview[:base_warnings],
existing_post: existing_post_compact(preview[:existing_post_id]) }
end
@@ -78,10 +108,9 @@ class PostCreatePreflight
return if @attributes[:thumbnail_base].present?
return if @thumbnail.blank?
attachment = Post.resized_thumbnail_attachment(@thumbnail)
attachment[:io].close if attachment[:io].respond_to?(:close)
rescue MiniMagick::Error
raise ValidationFailed.new(fields: { thumbnail: ['サムネイル画像の変換に失敗しました.'] })
PostThumbnailUploadValidator.validate!(@thumbnail)
rescue PostThumbnailUploadValidator::InvalidUpload => e
raise ValidationFailed.new(fields: { thumbnail: [e.message] })
end
def existing_post_compact post_id
@@ -90,4 +119,15 @@ class PostCreatePreflight
post = Post.with_attached_thumbnail.find_by(id: post_id)
PostCompactRepr.base(post)
end
def final_field_warnings field_warnings
thumbnail_warnings = (field_warnings['thumbnail_base'] || []).reject { _1 == 'サムネールなし' }
if @attributes[:thumbnail_base].blank? && @thumbnail.blank?
thumbnail_warnings = (thumbnail_warnings + ['サムネールなし']).uniq
end
{
**field_warnings,
'thumbnail_base' => thumbnail_warnings }
end
end
+20 -48
ファイルの表示
@@ -21,15 +21,13 @@ class PostCreator
ApplicationRecord.transaction do
post.save!
post.thumbnail.attach(thumbnail_attachment) if thumbnail_attachment.present?
Tag.normalise_tags!(tag_names, deny_deprecated: true, with_sections: true) =>
{ tags:, sections: }
tags = planned_tags
sections = planned_sections
TagVersioning.record_tag_snapshots!(tags, created_by_user: @actor)
tags = Tag.expand_parent_tags(tags).reject(&:deprecated?)
post.video_ms = normalise_video_ms(tags)
validate_video_sections!(post.video_ms, sections)
post.video_ms = planned_video_ms
post.save!
sync_post_tags!(post, tags, sections)
sync_parent_posts!(post, parent_post_ids)
sync_parent_posts!(post, planned_parent_post_ids)
PostVersionRecorder.record!(post:, event_type: :create, created_by_user: @actor)
end
post
@@ -46,52 +44,26 @@ class PostCreator
thumbnail_base: @attributes[:thumbnail_base].presence)
end
def tag_names = @attributes[:tags].to_s.split
def planned_tags = planned_create_attributes[:normalised_tags]
def parent_post_ids
Array(@attributes[:parent_post_ids]).flat_map { _1.to_s.split }.map { |token|
id = Integer(token, exception: false)
raise ArgumentError, "親投稿 Id. が不正です: #{ token }" if id.nil? || id <= 0
def planned_sections = planned_create_attributes[:tag_sections]
id
}.uniq
def planned_parent_post_ids = planned_create_attributes[:normalised_parent_post_ids]
def planned_video_ms
planned_create_attributes[:video_ms]
end
def normalise_video_ms tags
return nil unless tags.any? { _1.id == Tag.video.id }
video_ms = @attributes[:video_ms]
if video_ms.present?
value = Integer(video_ms, exception: false)
raise VideoMsParseError unless value&.positive?
return value
end
duration = @attributes[:duration]
return nil if duration.blank?
value = Tag.time_to_ms!(duration.to_s, tag_name: '動画時間')
raise VideoMsParseError unless value.positive?
value
rescue Tag::SectionLiteralParseError
raise VideoMsParseError
end
def validate_video_sections! video_ms, sections
return unless video_ms
sections.each_value do |ranges|
ranges.each do |begin_ms, end_ms|
post = Post.new
if begin_ms >= video_ms
post.errors.add :video_ms, 'タグ区間の開始が動画時間以上です.'
raise ActiveRecord::RecordInvalid, post
end
if end_ms && end_ms > video_ms
post.errors.add :video_ms, 'タグ区間の終端が動画時間を超えてゐます.'
raise ActiveRecord::RecordInvalid, post
end
def planned_create_attributes
@planned_create_attributes ||= begin
if @attributes.key?(:normalised_tags)
{
normalised_tags: @attributes[:normalised_tags],
tag_sections: @attributes[:tag_sections] || { },
normalised_parent_post_ids: @attributes[:normalised_parent_post_ids] || [],
video_ms: @attributes[:video_ms] }
else
PostCreatePlan.new(attributes: @attributes).build!
end
end
end
+2
ファイルの表示
@@ -487,6 +487,8 @@ class PostImportPreviewer
end
def parse_video_ms attributes, errors
return nil unless attributes['tags'].to_s.split.include?('動画')
video_ms = attributes['video_ms']
if video_ms.present?
value = Integer(video_ms, exception: false)
-25
ファイルの表示
@@ -1,25 +0,0 @@
class PostImportUrlListParser
MAX_ROWS = 100
MAX_BYTES = 1.megabyte
MAX_URL_BYTES = 20.kilobytes
def self.parse source
raw = source.to_s
raise ArgumentError, '入力が大きすぎます.' if raw.bytesize > MAX_BYTES
rows = raw.split(/\r\n|\n|\r/).each_with_index.filter_map { |line, index|
url = line.strip
next if url.blank?
if url.bytesize > MAX_URL_BYTES
raise ArgumentError, "#{ index + 1 } 行目: URL が長すぎます."
end
{ source_row: index + 1, url: }
}
raise ArgumentError, 'URL を入力してください.' if rows.empty?
raise ArgumentError, "取込件数は #{ MAX_ROWS } 件までです." if rows.length > MAX_ROWS
rows
end
end
+18
ファイルの表示
@@ -0,0 +1,18 @@
class PostThumbnailUploadValidator
MAX_THUMBNAIL_BYTES = 20 * 1024 * 1024
class InvalidUpload < StandardError; end
def self.validate! thumbnail
return if thumbnail.blank?
raise InvalidUpload, 'thumbnail upload が不正です.' unless thumbnail.is_a?(ActionDispatch::Http::UploadedFile)
raise InvalidUpload, 'thumbnail file size が大きすぎます.' if thumbnail.size > MAX_THUMBNAIL_BYTES
attachment = Post.resized_thumbnail_attachment(thumbnail)
attachment[:io].close if attachment[:io].respond_to?(:close)
rescue MiniMagick::Error
raise InvalidUpload, 'サムネイル画像の変換に失敗しました.'
ensure
thumbnail&.rewind if thumbnail.respond_to?(:rewind)
end
end
+36 -1
ファイルの表示
@@ -11,6 +11,9 @@ const SESSION_VERSION = 3
const SESSION_PREFIX = 'post-import-session:'
const SOURCE_DRAFT_KEY = 'post-import-source-draft'
const SESSION_MAX_AGE_MS = 24 * 60 * 60 * 1000
const MAX_SESSION_JSON_BYTES = 262_144
const MAX_SESSION_ROWS = 100
const MAX_SOURCE_BYTES = 262_144
const SESSION_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
const ATTRIBUTE_KEYS = [
@@ -50,11 +53,15 @@ const SESSION_KEYS = [
'source',
'rows',
'repairMode'] as const
const textEncoder = new TextEncoder ()
const isPlainObject = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && value != null && !(Array.isArray (value))
const byteLength = (value: string): number =>
textEncoder.encode (value).byteLength
export const validatePostImportSessionId = (value: unknown): string | null => {
if (typeof value !== 'string')
@@ -184,6 +191,14 @@ const hasOnlyKeys = (
const sanitiseResetSnapshot = (value: unknown): PostImportResetSnapshot | null => {
if (!(isPlainObject (value)))
return null
if (!(hasOnlyKeys (value, ['url',
'attributes',
'provenance',
'tagSources',
'fieldWarnings',
'baseWarnings',
'metadataUrl'])))
return null
if (typeof value.url !== 'string')
return null
if (!(isPlainObject (value.attributes)))
@@ -353,6 +368,8 @@ const sanitiseRow = (value: unknown): PostImportRow | null => {
return null
if (typeof value.url !== 'string')
return null
if (byteLength (value.url) > 20_480)
return null
if (!(isPlainObject (value.attributes)))
return null
if (!(isPlainObject (value.provenance)))
@@ -450,17 +467,23 @@ const sanitiseSession = (value: unknown): PostImportSession | null => {
return null
if (value.version !== SESSION_VERSION || !(Array.isArray (value.rows)))
return null
if (value.rows.length === 0 || value.rows.length > MAX_SESSION_ROWS)
return null
if (typeof value.expiresAt !== 'string' || isExpiredSession (value.expiresAt))
return null
if (typeof value.source !== 'string' || byteLength (value.source) > MAX_SOURCE_BYTES)
return null
const rows = value.rows.map (sanitiseRow)
if (rows.some (row => row == null))
return null
if ((new Set (rows.map (row => row?.sourceRow))).size !== rows.length)
return null
return {
version: SESSION_VERSION,
expiresAt: value.expiresAt,
source: typeof value.source === 'string' ? value.source : '',
source: value.source,
rows: (rows as PostImportRow[]).map (row => recalculateThumbnailWarning (row)),
repairMode: value.repairMode === 'failed' ? 'failed' : 'all' }
}
@@ -497,6 +520,12 @@ export const cleanupExpiredPostImportSessions = (
const raw = sessionStorage.getItem (key)
if (raw == null)
continue
if (byteLength (raw) > MAX_SESSION_JSON_BYTES)
{
sessionStorage.removeItem (key)
--i
continue
}
try
{
@@ -585,6 +614,11 @@ export const loadPostImportSession = (
const raw = readStorage (sessionKey (validatedId), onError)
if (raw == null)
return null
if (byteLength (raw) > MAX_SESSION_JSON_BYTES)
{
removeStorage (sessionKey (validatedId), onError)
return null
}
try
{
@@ -598,6 +632,7 @@ export const loadPostImportSession = (
}
catch
{
removeStorage (sessionKey (validatedId), onError)
return null
}
}
+81 -4
ファイルの表示
@@ -84,13 +84,55 @@ const STATE_KEYS = [
'created_post_id',
'recoverable'] as const
const SINGLE_ROW_KEYS = [...BASIC_KEYS, ...STATE_KEYS, 'session_id'] as const
const ENCODED_ROW_KEYS = [
'n',
'u',
't',
'h',
'f',
'b',
'x',
'd',
'g',
'p',
'm',
's',
'e',
'i',
'c',
'r'] as const
const ALLOWED_MANUAL_FIELDS = [
'title',
'thumbnailBase',
'originalCreatedFrom',
'originalCreatedBefore',
'duration',
'tags',
'parentPostIds'] as const
const MAX_REGULAR_URL_LENGTH = 768
const MAX_COMPRESSED_STATE_LENGTH = 767
const COMPRESSED_STATE_PREFIX = 'z1.'
const MAX_COMPRESSED_STATE_PARAM_LENGTH = 8_192
const MAX_COMPRESSED_STATE_BYTES = 16_384
const MAX_DECOMPRESSED_STATE_BYTES = 262_144
const MAX_STATE_ROWS = 100
const MAX_QUERY_STRING_BYTES = 20_480
const textEncoder = new TextEncoder ()
const textDecoder = new TextDecoder ()
const isPlainObject = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && value != null && !(Array.isArray (value))
const hasOnlyKeys = (
value: Record<string, unknown>,
allowedKeys: readonly string[],
): boolean =>
Object.keys (value).every (key => allowedKeys.includes (key))
const byteLength = (value: string): number =>
textEncoder.encode (value).byteLength
const isValidSkipReason = (value: unknown): value is PostImportSkipReason =>
value === 'existing' || value === 'manual'
@@ -202,18 +244,25 @@ const compressBytes = async (value: Uint8Array): Promise<Uint8Array> => {
const decompressBytes = async (value: Uint8Array): Promise<Uint8Array> => {
const stream =
new Blob ([value]).stream ().pipeThrough (new DecompressionStream ('gzip'))
return new Uint8Array (await new Response (stream).arrayBuffer ())
const bytes = new Uint8Array (await new Response (stream).arrayBuffer ())
if (bytes.byteLength > MAX_DECOMPRESSED_STATE_BYTES)
throw new Error ('decompressed state too large')
return bytes
}
const decodeCompressedState = async (value: string): Promise<string | null> => {
if (!(value.startsWith (COMPRESSED_STATE_PREFIX)))
return null
if (value.length > MAX_COMPRESSED_STATE_PARAM_LENGTH)
return null
const encoded = value.slice (COMPRESSED_STATE_PREFIX.length)
const compressed = decodeBase64UrlBytes (encoded)
if (compressed == null)
return null
if (compressed.byteLength > MAX_COMPRESSED_STATE_BYTES)
return null
try
{
@@ -318,6 +367,10 @@ const parseManual = (value: string | null): string[] =>
.filter (entry => entry !== '')
const hasOnlyAllowedManualFields = (value: string[]): boolean =>
value.every (field => ALLOWED_MANUAL_FIELDS.includes (field as never))
const isStringArray = (value: unknown): value is string[] =>
Array.isArray (value) && value.every (entry => typeof entry === 'string')
@@ -326,10 +379,12 @@ const parseEncodedRow = (
value: unknown,
requireUrl: boolean,
): FullRowState | null => {
if (typeof value !== 'object' || value == null || Array.isArray (value))
if (!(isPlainObject (value)))
return null
const row = value as Record<string, unknown>
if (!(hasOnlyKeys (row, ENCODED_ROW_KEYS)))
return null
const url = row['u']
const manual = row['m']
const skip = decodeSkip (row['s'])
@@ -341,8 +396,12 @@ const parseEncodedRow = (
if (requireUrl && typeof url !== 'string')
return null
if (typeof url === 'string' && byteLength (url) > MAX_QUERY_STRING_BYTES)
return null
if (manual != null && !(isStringArray (manual)))
return null
if ((manual ?? []).some (field => !(ALLOWED_MANUAL_FIELDS.includes (field as never))))
return null
if (row['s'] != null && skip == null)
return null
if (row['i'] != null && importStatus == null)
@@ -382,6 +441,8 @@ const parseEncodedRow = (
const fieldValue = row[key]
if (fieldValue != null && typeof fieldValue !== 'string')
throw new Error ('invalid row')
if (typeof fieldValue === 'string' && byteLength (fieldValue) > MAX_QUERY_STRING_BYTES)
throw new Error ('invalid row')
return [target, fieldValue ?? '']
}),
) as Record<(typeof stringFields)[number][1], string>
@@ -418,6 +479,8 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => {
const url = params.get ('url')
if (url == null || url === '')
return null
if (byteLength (url) > MAX_QUERY_STRING_BYTES)
return null
const hasOnlyUrl = Array.from (params.keys ()).every (
key => key === 'url' || key === 'session_id')
@@ -460,6 +523,8 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => {
return null
if (importStatus != null && !(isValidImportStatus (importStatus)))
return null
if (!(hasOnlyAllowedManualFields (parseManual (params.get ('manual')))))
return null
const row = buildRow ({
source_row: 1,
@@ -509,7 +574,12 @@ const parseMetaRows = (
return null
const urls = urlsValue.split (' ').filter (url => url !== '')
if (urls.length < 2 || urls.length !== parsed.rows.length)
if (urls.length < 2
|| urls.length > MAX_STATE_ROWS
|| urls.length !== parsed.rows.length
|| parsed.rows.length > MAX_STATE_ROWS)
return null
if (urls.some (url => byteLength (url) > MAX_QUERY_STRING_BYTES))
return null
const fullRows = parsed.rows.map ((row, index) => {
@@ -520,6 +590,8 @@ const parseMetaRows = (
})
if (fullRows.some (row => row == null))
return null
if ((new Set (fullRows.map (row => (row as FullRowState).source_row))).size !== fullRows.length)
return null
const rows = fullRows.map (row => buildRow (row as FullRowState))
return {
@@ -545,12 +617,17 @@ const parseFullState = async (stateValue: string): Promise<ParsedPostNewState |
return null
}
if (parsed.v !== 1 || !(Array.isArray (parsed.rows)) || parsed.rows.length === 0)
if (parsed.v !== 1
|| !(Array.isArray (parsed.rows))
|| parsed.rows.length === 0
|| parsed.rows.length > MAX_STATE_ROWS)
return null
const fullRows = parsed.rows.map (row => parseEncodedRow (row, true))
if (fullRows.some (row => row == null))
return null
if ((new Set (fullRows.map (row => (row as FullRowState).source_row))).size !== fullRows.length)
return null
const rows = fullRows.map (row => buildRow (row as FullRowState))
return {
+77 -68
ファイルの表示
@@ -68,6 +68,7 @@ type PreviewResponse = {
thumbnailBase?: string
originalCreatedFrom?: string
originalCreatedBefore?: string
parentPostIds?: string
duration?: string
videoMs?: number
tags?: string
@@ -137,8 +138,7 @@ const mergeDryRunRow = (
currentRow: PostImportRow,
result: PreviewResponse,
): PostImportRow =>
applyThumbnailWarning (
initialisePreviewRows ([{
applyThumbnailWarning ({
...currentRow,
url: result.url,
attributes: {
@@ -150,7 +150,7 @@ const mergeDryRunRow = (
duration: result.duration ?? '',
videoMs: result.videoMs ?? '',
tags: result.tags ?? '',
parentPostIds: String (currentRow.attributes.parentPostIds ?? '') },
parentPostIds: String (result.parentPostIds ?? currentRow.attributes.parentPostIds ?? '') },
fieldWarnings: result.fieldWarnings ?? { },
baseWarnings: result.baseWarnings ?? [],
validationErrors: { },
@@ -167,7 +167,38 @@ const mergeDryRunRow = (
currentRow.importStatus === 'created'
? 'created'
: 'pending',
recoverable: undefined }])[0])
recoverable: undefined })
const buildPreviewResetSnapshot = (
preview: PreviewResponse,
): PostImportRow['resetSnapshot'] => ({
url: preview.url,
attributes: {
title: preview.title ?? '',
thumbnailBase: preview.thumbnailBase ?? '',
originalCreatedFrom: preview.originalCreatedFrom ?? '',
originalCreatedBefore: preview.originalCreatedBefore ?? '',
duration: preview.duration ?? '',
videoMs: preview.videoMs ?? '',
tags: preview.tags ?? '',
parentPostIds: String (preview.parentPostIds ?? '') },
provenance: {
url: 'manual',
title: 'automatic',
thumbnailBase: 'automatic',
originalCreatedFrom: 'automatic',
originalCreatedBefore: 'automatic',
duration: 'automatic',
videoMs: 'automatic',
tags: 'automatic',
parentPostIds: 'automatic' },
tagSources: {
automatic: preview.tags ?? '',
manual: '' },
fieldWarnings: preview.fieldWarnings ?? { },
baseWarnings: preview.baseWarnings ?? [],
metadataUrl: preview.url })
const indexedBulkResults = (
@@ -216,6 +247,9 @@ const mergePreviewRow = (
currentRow: PostImportRow,
preview: PreviewResponse,
): PostImportRow => {
const fieldWarnings = preview.fieldWarnings ?? { }
const baseWarnings = preview.baseWarnings ?? []
const metadataChanged = currentRow.metadataUrl !== preview.url
const nextRow: PostImportRow = {
...currentRow,
attributes: { ...currentRow.attributes },
@@ -224,8 +258,8 @@ const mergePreviewRow = (
automatic: currentRow.tagSources?.automatic ?? '',
manual: currentRow.tagSources?.manual ?? '' },
url: preview.url,
fieldWarnings: preview.fieldWarnings ?? { },
baseWarnings: preview.baseWarnings ?? [],
fieldWarnings,
baseWarnings,
existingPostId: preview.existingPost?.id,
existingPost: preview.existingPost ?? undefined,
skipReason:
@@ -234,9 +268,14 @@ const mergePreviewRow = (
: currentRow.skipReason === 'manual'
? 'manual'
: undefined,
metadataUrl: preview.url,
resetSnapshot:
metadataChanged
? buildPreviewResetSnapshot (preview)
: currentRow.resetSnapshot,
status:
Object.keys (preview.fieldWarnings ?? { }).length > 0
|| (preview.baseWarnings?.length ?? 0) > 0
Object.keys (fieldWarnings).length > 0
|| baseWarnings.length > 0
? 'warning'
: 'ready' }
@@ -251,8 +290,10 @@ const mergePreviewRow = (
if (currentRow.provenance.tags !== 'manual')
{
nextRow.attributes.tags = preview.tags ?? ''
nextRow.tagSources.automatic = preview.tags ?? ''
}
nextRow.tagSources.automatic = preview.tags ?? ''
if (currentRow.provenance.parentPostIds !== 'manual')
nextRow.attributes.parentPostIds = String (preview.parentPostIds ?? '')
if (currentRow.provenance.videoMs !== 'manual')
nextRow.attributes.videoMs = preview.videoMs ?? ''
if (currentRow.provenance.duration !== 'manual')
@@ -398,6 +439,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const refreshRows = async (baseSession: PostImportSession) => {
let nextIndex = 0
const previews = new Map<number, PreviewResponse> ()
const worker = async () => {
while (active)
{
@@ -421,15 +463,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
signal: controller.signal })
if (!(active) || previewSequenceRef.current !== previewSequence)
return
const latestSession = sessionRef.current ?? baseSession
const currentRow = latestSession.rows.find (
row => row.sourceRow === baseRow.sourceRow)
if (currentRow == null)
continue
const mergedRow = mergePreviewRow (currentRow, preview)
await persistSession (buildSession (
replaceImportRow (latestSession.rows, mergedRow)))
previews.set (baseRow.sourceRow, preview)
}
catch (requestError)
{
@@ -437,11 +471,6 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
return
if (!(active) || previewSequenceRef.current !== previewSequence)
return
const latestSession = sessionRef.current ?? baseSession
const currentRow = latestSession.rows.find (
row => row.sourceRow === baseRow.sourceRow)
if (currentRow == null)
continue
if (!(isApiError (requestError)))
continue
}
@@ -454,6 +483,21 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
await Promise.all (
Array.from ({ length: Math.min (4, baseSession.rows.length) }, () => worker ()))
if (!(active) || previewSequenceRef.current !== previewSequence)
return
const latestSession = sessionRef.current ?? baseSession
const nextRows = latestSession.rows.map (row => {
const preview = previews.get (row.sourceRow)
if (preview == null
|| row.skipReason === 'manual'
|| row.importStatus === 'created'
|| row.importStatus === 'skipped'
|| isNonRecoverableFailedRow (row))
return row
return mergePreviewRow (row, preview)
})
await persistSession (buildSession (nextRows))
}
const hydrate = async () => {
@@ -494,7 +538,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
duration: preview.duration ?? '',
videoMs: preview.videoMs ?? '',
tags: preview.tags ?? '',
parentPostIds: '' },
parentPostIds: String (preview.parentPostIds ?? '') },
fieldWarnings: preview.fieldWarnings ?? { },
baseWarnings: preview.baseWarnings ?? [],
validationErrors: { },
@@ -529,7 +573,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
duration: preview.duration ?? '',
videoMs: preview.videoMs ?? '',
tags: preview.tags ?? '',
parentPostIds: '' },
parentPostIds: String (preview.parentPostIds ?? '') },
provenance: {
url: 'manual',
title: 'automatic',
@@ -544,7 +588,8 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
automatic: preview.tags ?? '',
manual: '' },
fieldWarnings: preview.fieldWarnings ?? { },
baseWarnings: preview.baseWarnings ?? [] } }])[0])]
baseWarnings: preview.baseWarnings ?? [],
metadataUrl: preview.url } }])[0])]
const persisted = await persistSession (buildSession (rows))
if (persisted == null)
return
@@ -557,10 +602,6 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
return
}
if (loadedSession == null)
{
await persistSession (loaded)
}
void refreshRows (loaded)
}
@@ -612,8 +653,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
if (row.sourceRow !== sourceRow)
return row
if (isExistingSkipRow (row)
|| row.importStatus === 'created'
|| isNonRecoverableFailedRow (row))
|| row.importStatus === 'created')
return row
return {
...row,
@@ -779,23 +819,8 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const indexedResults = indexedBulkResults ([target], result.results)
const latestAfterImport = sessionRef.current ?? pendingSession
const mergedRows = mergeImportResults (latestAfterImport.rows, indexedResults)
const recoverableRows = indexedResults.filter (row =>
row.status === 'failed'
&& row.recoverable
&& Object.keys (row.errors ?? { }).length > 0)
const nextRows = mergedRows.map ((row): PostImportRow => {
const recoverable = recoverableRows.find (
failedRow => failedRow.sourceRow === row.sourceRow)
if (recoverable == null)
return applyThumbnailWarning (row)
return applyThumbnailWarning ({
...row,
importStatus: 'pending',
recoverable: true,
validationErrors: recoverable.errors ?? { },
importErrors: undefined })
})
const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults)
.map (row => applyThumbnailWarning (row))
const persisted = await persistSession (buildSession (nextRows))
if (persisted != null
&& persisted.rows.every (row => isCompletedReviewRow (row)))
@@ -842,23 +867,8 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const indexedResults = indexedBulkResults (processingRows, result.results)
const latestAfterImport = sessionRef.current ?? buildSession (currentSession.rows)
const mergedResults = mergeImportResults (latestAfterImport.rows, indexedResults)
const recoverableRows = indexedResults.filter (row =>
row.status === 'failed'
&& row.recoverable
&& Object.keys (row.errors ?? { }).length > 0)
const nextRows = mergedResults.map ((row): PostImportRow => {
const recoverable = recoverableRows.find (
failedRow => failedRow.sourceRow === row.sourceRow)
if (recoverable == null)
return applyThumbnailWarning (row)
return applyThumbnailWarning ({
...row,
importStatus: 'pending',
recoverable: true,
validationErrors: recoverable.errors ?? { },
importErrors: undefined })
})
const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults)
.map (row => applyThumbnailWarning (row))
const persisted = await persistSession (buildSession (nextRows))
if (persisted != null
&& persisted.rows.every (row => isCompletedReviewRow (row)))
@@ -940,8 +950,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
skipDisabled={
busy
|| isExistingSkipRow (row)
|| row.importStatus === 'created'
|| isNonRecoverableFailedRow (row)}
|| row.importStatus === 'created'}
onEdit={() => editRow (row)}
onToggleSkip={checked => toggleManualSkip (row.sourceRow, checked)}
onRetry={canRetryResultRow (row) ? () => retry (row.sourceRow) : undefined}