miteruzo/btrc-hub
1
0
フォーク 0
コード 課題 61 Pull リクエスト 1 リリース Wiki アクティビティ

#399

ソースを参照
このコミットが含まれているのは:
みてるぞ
2026-07-18 00:01:12 +09:00
親 0ae41b6266
コミット ff970f8171
11個のファイルの変更，449行の追加，173行の削除
Patchファイルをダウンロード Diffファイルをダウンロード すべてのファイルを展開 すべてのファイルを折り畳む
frontend/src/lib/postImportStorage.ts
+36 -1
ファイルの表示
@@ -11,6 +11,9 @@ const SESSION_VERSION = 3
const SESSION_PREFIX = 'post-import-session:'
const SOURCE_DRAFT_KEY = 'post-import-source-draft'
const SESSION_MAX_AGE_MS = 24 * 60 * 60 * 1000
const MAX_SESSION_JSON_BYTES = 262_144
const MAX_SESSION_ROWS = 100
const MAX_SOURCE_BYTES = 262_144
const SESSION_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
const ATTRIBUTE_KEYS = [
@@ -50,11 +53,15 @@ const SESSION_KEYS = [
'source',
'rows',
'repairMode'] as const
const textEncoder = new TextEncoder ()
const isPlainObject = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && value != null && !(Array.isArray (value))
const byteLength = (value: string): number =>
textEncoder.encode (value).byteLength
export const validatePostImportSessionId = (value: unknown): string | null => {
if (typeof value !== 'string')
@@ -184,6 +191,14 @@ const hasOnlyKeys = (
const sanitiseResetSnapshot = (value: unknown): PostImportResetSnapshot | null => {
if (!(isPlainObject (value)))
return null
if (!(hasOnlyKeys (value, ['url',
'attributes',
'provenance',
'tagSources',
'fieldWarnings',
'baseWarnings',
'metadataUrl'])))
return null
if (typeof value.url !== 'string')
return null
if (!(isPlainObject (value.attributes)))
@@ -353,6 +368,8 @@ const sanitiseRow = (value: unknown): PostImportRow | null => {
return null
if (typeof value.url !== 'string')
return null
if (byteLength (value.url) > 20_480)
return null
if (!(isPlainObject (value.attributes)))
return null
if (!(isPlainObject (value.provenance)))
@@ -450,17 +467,23 @@ const sanitiseSession = (value: unknown): PostImportSession | null => {
return null
if (value.version !== SESSION_VERSION || !(Array.isArray (value.rows)))
return null
if (value.rows.length === 0 || value.rows.length > MAX_SESSION_ROWS)
return null
if (typeof value.expiresAt !== 'string' || isExpiredSession (value.expiresAt))
return null
if (typeof value.source !== 'string' || byteLength (value.source) > MAX_SOURCE_BYTES)
return null
const rows = value.rows.map (sanitiseRow)
if (rows.some (row => row == null))
return null
if ((new Set (rows.map (row => row?.sourceRow))).size !== rows.length)
return null
return {
version: SESSION_VERSION,
expiresAt: value.expiresAt,
source: typeof value.source === 'string' ? value.source : '',
source: value.source,
rows: (rows as PostImportRow[]).map (row => recalculateThumbnailWarning (row)),
repairMode: value.repairMode === 'failed' ? 'failed' : 'all' }
}
@@ -497,6 +520,12 @@ export const cleanupExpiredPostImportSessions = (
const raw = sessionStorage.getItem (key)
if (raw == null)
continue
if (byteLength (raw) > MAX_SESSION_JSON_BYTES)
{
sessionStorage.removeItem (key)
--i
continue
}
try
{
@@ -585,6 +614,11 @@ export const loadPostImportSession = (
const raw = readStorage (sessionKey (validatedId), onError)
if (raw == null)
return null
if (byteLength (raw) > MAX_SESSION_JSON_BYTES)
{
removeStorage (sessionKey (validatedId), onError)
return null
}
try
{
@@ -598,6 +632,7 @@ export const loadPostImportSession = (
}
catch
{
removeStorage (sessionKey (validatedId), onError)
return null
}
}
frontend/src/lib/postNewQueryState.ts
+81 -4
ファイルの表示
@@ -84,13 +84,55 @@ const STATE_KEYS = [
'created_post_id',
'recoverable'] as const
const SINGLE_ROW_KEYS = [...BASIC_KEYS, ...STATE_KEYS, 'session_id'] as const
const ENCODED_ROW_KEYS = [
'n',
'u',
't',
'h',
'f',
'b',
'x',
'd',
'g',
'p',
'm',
's',
'e',
'i',
'c',
'r'] as const
const ALLOWED_MANUAL_FIELDS = [
'title',
'thumbnailBase',
'originalCreatedFrom',
'originalCreatedBefore',
'duration',
'tags',
'parentPostIds'] as const
const MAX_REGULAR_URL_LENGTH = 768
const MAX_COMPRESSED_STATE_LENGTH = 767
const COMPRESSED_STATE_PREFIX = 'z1.'
const MAX_COMPRESSED_STATE_PARAM_LENGTH = 8_192
const MAX_COMPRESSED_STATE_BYTES = 16_384
const MAX_DECOMPRESSED_STATE_BYTES = 262_144
const MAX_STATE_ROWS = 100
const MAX_QUERY_STRING_BYTES = 20_480
const textEncoder = new TextEncoder ()
const textDecoder = new TextDecoder ()
const isPlainObject = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && value != null && !(Array.isArray (value))
const hasOnlyKeys = (
value: Record<string, unknown>,
allowedKeys: readonly string[],
): boolean =>
Object.keys (value).every (key => allowedKeys.includes (key))
const byteLength = (value: string): number =>
textEncoder.encode (value).byteLength
const isValidSkipReason = (value: unknown): value is PostImportSkipReason =>
value === 'existing' || value === 'manual'
@@ -202,18 +244,25 @@ const compressBytes = async (value: Uint8Array): Promise<Uint8Array> => {
const decompressBytes = async (value: Uint8Array): Promise<Uint8Array> => {
const stream =
new Blob ([value]).stream ().pipeThrough (new DecompressionStream ('gzip'))
return new Uint8Array (await new Response (stream).arrayBuffer ())
const bytes = new Uint8Array (await new Response (stream).arrayBuffer ())
if (bytes.byteLength > MAX_DECOMPRESSED_STATE_BYTES)
throw new Error ('decompressed state too large')
return bytes
}
const decodeCompressedState = async (value: string): Promise<string | null> => {
if (!(value.startsWith (COMPRESSED_STATE_PREFIX)))
return null
if (value.length > MAX_COMPRESSED_STATE_PARAM_LENGTH)
return null
const encoded = value.slice (COMPRESSED_STATE_PREFIX.length)
const compressed = decodeBase64UrlBytes (encoded)
if (compressed == null)
return null
if (compressed.byteLength > MAX_COMPRESSED_STATE_BYTES)
return null
try
{
@@ -318,6 +367,10 @@ const parseManual = (value: string | null): string[] =>
.filter (entry => entry !== '')
const hasOnlyAllowedManualFields = (value: string[]): boolean =>
value.every (field => ALLOWED_MANUAL_FIELDS.includes (field as never))
const isStringArray = (value: unknown): value is string[] =>
Array.isArray (value) && value.every (entry => typeof entry === 'string')
@@ -326,10 +379,12 @@ const parseEncodedRow = (
value: unknown,
requireUrl: boolean,
): FullRowState | null => {
if (typeof value !== 'object' || value == null || Array.isArray (value))
if (!(isPlainObject (value)))
return null
const row = value as Record<string, unknown>
if (!(hasOnlyKeys (row, ENCODED_ROW_KEYS)))
return null
const url = row['u']
const manual = row['m']
const skip = decodeSkip (row['s'])
@@ -341,8 +396,12 @@ const parseEncodedRow = (
if (requireUrl && typeof url !== 'string')
return null
if (typeof url === 'string' && byteLength (url) > MAX_QUERY_STRING_BYTES)
return null
if (manual != null && !(isStringArray (manual)))
return null
if ((manual ?? []).some (field => !(ALLOWED_MANUAL_FIELDS.includes (field as never))))
return null
if (row['s'] != null && skip == null)
return null
if (row['i'] != null && importStatus == null)
@@ -382,6 +441,8 @@ const parseEncodedRow = (
const fieldValue = row[key]
if (fieldValue != null && typeof fieldValue !== 'string')
throw new Error ('invalid row')
if (typeof fieldValue === 'string' && byteLength (fieldValue) > MAX_QUERY_STRING_BYTES)
throw new Error ('invalid row')
return [target, fieldValue ?? '']
}),
) as Record<(typeof stringFields)[number][1], string>
@@ -418,6 +479,8 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => {
const url = params.get ('url')
if (url == null || url === '')
return null
if (byteLength (url) > MAX_QUERY_STRING_BYTES)
return null
const hasOnlyUrl = Array.from (params.keys ()).every (
key => key === 'url' || key === 'session_id')
@@ -460,6 +523,8 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => {
return null
if (importStatus != null && !(isValidImportStatus (importStatus)))
return null
if (!(hasOnlyAllowedManualFields (parseManual (params.get ('manual')))))
return null
const row = buildRow ({
source_row: 1,
@@ -509,7 +574,12 @@ const parseMetaRows = (
return null
const urls = urlsValue.split (' ').filter (url => url !== '')
if (urls.length < 2 || urls.length !== parsed.rows.length)
if (urls.length < 2
|| urls.length > MAX_STATE_ROWS
|| urls.length !== parsed.rows.length
|| parsed.rows.length > MAX_STATE_ROWS)
return null
if (urls.some (url => byteLength (url) > MAX_QUERY_STRING_BYTES))
return null
const fullRows = parsed.rows.map ((row, index) => {
@@ -520,6 +590,8 @@ const parseMetaRows = (
})
if (fullRows.some (row => row == null))
return null
if ((new Set (fullRows.map (row => (row as FullRowState).source_row))).size !== fullRows.length)
return null
const rows = fullRows.map (row => buildRow (row as FullRowState))
return {
@@ -545,12 +617,17 @@ const parseFullState = async (stateValue: string): Promise<ParsedPostNewState |
return null
}
if (parsed.v !== 1 || !(Array.isArray (parsed.rows)) || parsed.rows.length === 0)
if (parsed.v !== 1
|| !(Array.isArray (parsed.rows))
|| parsed.rows.length === 0
|| parsed.rows.length > MAX_STATE_ROWS)
return null
const fullRows = parsed.rows.map (row => parseEncodedRow (row, true))
if (fullRows.some (row => row == null))
return null
if ((new Set (fullRows.map (row => (row as FullRowState).source_row))).size !== fullRows.length)
return null
const rows = fullRows.map (row => buildRow (row as FullRowState))
return {
frontend/src/pages/posts/PostImportReviewPage.tsx
+77 -68
ファイルの表示
@@ -68,6 +68,7 @@ type PreviewResponse = {
thumbnailBase?: string
originalCreatedFrom?: string
originalCreatedBefore?: string
parentPostIds?: string
duration?: string
videoMs?: number
tags?: string
@@ -137,8 +138,7 @@ const mergeDryRunRow = (
currentRow: PostImportRow,
result: PreviewResponse,
): PostImportRow =>
applyThumbnailWarning (
initialisePreviewRows ([{
applyThumbnailWarning ({
...currentRow,
url: result.url,
attributes: {
@@ -150,7 +150,7 @@ const mergeDryRunRow = (
duration: result.duration ?? '',
videoMs: result.videoMs ?? '',
tags: result.tags ?? '',
parentPostIds: String (currentRow.attributes.parentPostIds ?? '') },
parentPostIds: String (result.parentPostIds ?? currentRow.attributes.parentPostIds ?? '') },
fieldWarnings: result.fieldWarnings ?? { },
baseWarnings: result.baseWarnings ?? [],
validationErrors: { },
@@ -167,7 +167,38 @@ const mergeDryRunRow = (
currentRow.importStatus === 'created'
? 'created'
: 'pending',
recoverable: undefined }])[0])
recoverable: undefined })
const buildPreviewResetSnapshot = (
preview: PreviewResponse,
): PostImportRow['resetSnapshot'] => ({
url: preview.url,
attributes: {
title: preview.title ?? '',
thumbnailBase: preview.thumbnailBase ?? '',
originalCreatedFrom: preview.originalCreatedFrom ?? '',
originalCreatedBefore: preview.originalCreatedBefore ?? '',
duration: preview.duration ?? '',
videoMs: preview.videoMs ?? '',
tags: preview.tags ?? '',
parentPostIds: String (preview.parentPostIds ?? '') },
provenance: {
url: 'manual',
title: 'automatic',
thumbnailBase: 'automatic',
originalCreatedFrom: 'automatic',
originalCreatedBefore: 'automatic',
duration: 'automatic',
videoMs: 'automatic',
tags: 'automatic',
parentPostIds: 'automatic' },
tagSources: {
automatic: preview.tags ?? '',
manual: '' },
fieldWarnings: preview.fieldWarnings ?? { },
baseWarnings: preview.baseWarnings ?? [],
metadataUrl: preview.url })
const indexedBulkResults = (
@@ -216,6 +247,9 @@ const mergePreviewRow = (
currentRow: PostImportRow,
preview: PreviewResponse,
): PostImportRow => {
const fieldWarnings = preview.fieldWarnings ?? { }
const baseWarnings = preview.baseWarnings ?? []
const metadataChanged = currentRow.metadataUrl !== preview.url
const nextRow: PostImportRow = {
...currentRow,
attributes: { ...currentRow.attributes },
@@ -224,8 +258,8 @@ const mergePreviewRow = (
automatic: currentRow.tagSources?.automatic ?? '',
manual: currentRow.tagSources?.manual ?? '' },
url: preview.url,
fieldWarnings: preview.fieldWarnings ?? { },
baseWarnings: preview.baseWarnings ?? [],
fieldWarnings,
baseWarnings,
existingPostId: preview.existingPost?.id,
existingPost: preview.existingPost ?? undefined,
skipReason:
@@ -234,9 +268,14 @@ const mergePreviewRow = (
: currentRow.skipReason === 'manual'
? 'manual'
: undefined,
metadataUrl: preview.url,
resetSnapshot:
metadataChanged
? buildPreviewResetSnapshot (preview)
: currentRow.resetSnapshot,
status:
Object.keys (preview.fieldWarnings ?? { }).length > 0
|| (preview.baseWarnings?.length ?? 0) > 0
Object.keys (fieldWarnings).length > 0
|| baseWarnings.length > 0
? 'warning'
: 'ready' }
@@ -251,8 +290,10 @@ const mergePreviewRow = (
if (currentRow.provenance.tags !== 'manual')
{
nextRow.attributes.tags = preview.tags ?? ''
nextRow.tagSources.automatic = preview.tags ?? ''
}
nextRow.tagSources.automatic = preview.tags ?? ''
if (currentRow.provenance.parentPostIds !== 'manual')
nextRow.attributes.parentPostIds = String (preview.parentPostIds ?? '')
if (currentRow.provenance.videoMs !== 'manual')
nextRow.attributes.videoMs = preview.videoMs ?? ''
if (currentRow.provenance.duration !== 'manual')
@@ -398,6 +439,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const refreshRows = async (baseSession: PostImportSession) => {
let nextIndex = 0
const previews = new Map<number, PreviewResponse> ()
const worker = async () => {
while (active)
{
@@ -421,15 +463,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
signal: controller.signal })
if (!(active) || previewSequenceRef.current !== previewSequence)
return
const latestSession = sessionRef.current ?? baseSession
const currentRow = latestSession.rows.find (
row => row.sourceRow === baseRow.sourceRow)
if (currentRow == null)
continue
const mergedRow = mergePreviewRow (currentRow, preview)
await persistSession (buildSession (
replaceImportRow (latestSession.rows, mergedRow)))
previews.set (baseRow.sourceRow, preview)
}
catch (requestError)
{
@@ -437,11 +471,6 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
return
if (!(active) || previewSequenceRef.current !== previewSequence)
return
const latestSession = sessionRef.current ?? baseSession
const currentRow = latestSession.rows.find (
row => row.sourceRow === baseRow.sourceRow)
if (currentRow == null)
continue
if (!(isApiError (requestError)))
continue
}
@@ -454,6 +483,21 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
await Promise.all (
Array.from ({ length: Math.min (4, baseSession.rows.length) }, () => worker ()))
if (!(active) || previewSequenceRef.current !== previewSequence)
return
const latestSession = sessionRef.current ?? baseSession
const nextRows = latestSession.rows.map (row => {
const preview = previews.get (row.sourceRow)
if (preview == null
|| row.skipReason === 'manual'
|| row.importStatus === 'created'
|| row.importStatus === 'skipped'
|| isNonRecoverableFailedRow (row))
return row
return mergePreviewRow (row, preview)
})
await persistSession (buildSession (nextRows))
}
const hydrate = async () => {
@@ -494,7 +538,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
duration: preview.duration ?? '',
videoMs: preview.videoMs ?? '',
tags: preview.tags ?? '',
parentPostIds: '' },
parentPostIds: String (preview.parentPostIds ?? '') },
fieldWarnings: preview.fieldWarnings ?? { },
baseWarnings: preview.baseWarnings ?? [],
validationErrors: { },
@@ -529,7 +573,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
duration: preview.duration ?? '',
videoMs: preview.videoMs ?? '',
tags: preview.tags ?? '',
parentPostIds: '' },
parentPostIds: String (preview.parentPostIds ?? '') },
provenance: {
url: 'manual',
title: 'automatic',
@@ -544,7 +588,8 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
automatic: preview.tags ?? '',
manual: '' },
fieldWarnings: preview.fieldWarnings ?? { },
baseWarnings: preview.baseWarnings ?? [] } }])[0])]
baseWarnings: preview.baseWarnings ?? [],
metadataUrl: preview.url } }])[0])]
const persisted = await persistSession (buildSession (rows))
if (persisted == null)
return
@@ -557,10 +602,6 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
return
}
if (loadedSession == null)
{
await persistSession (loaded)
}
void refreshRows (loaded)
}
@@ -612,8 +653,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
if (row.sourceRow !== sourceRow)
return row
if (isExistingSkipRow (row)
|| row.importStatus === 'created'
|| isNonRecoverableFailedRow (row))
|| row.importStatus === 'created')
return row
return {
...row,
@@ -779,23 +819,8 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const indexedResults = indexedBulkResults ([target], result.results)
const latestAfterImport = sessionRef.current ?? pendingSession
const mergedRows = mergeImportResults (latestAfterImport.rows, indexedResults)
const recoverableRows = indexedResults.filter (row =>
row.status === 'failed'
&& row.recoverable
&& Object.keys (row.errors ?? { }).length > 0)
const nextRows = mergedRows.map ((row): PostImportRow => {
const recoverable = recoverableRows.find (
failedRow => failedRow.sourceRow === row.sourceRow)
if (recoverable == null)
return applyThumbnailWarning (row)
return applyThumbnailWarning ({
...row,
importStatus: 'pending',
recoverable: true,
validationErrors: recoverable.errors ?? { },
importErrors: undefined })
})
const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults)
.map (row => applyThumbnailWarning (row))
const persisted = await persistSession (buildSession (nextRows))
if (persisted != null
&& persisted.rows.every (row => isCompletedReviewRow (row)))
@@ -842,23 +867,8 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
const indexedResults = indexedBulkResults (processingRows, result.results)
const latestAfterImport = sessionRef.current ?? buildSession (currentSession.rows)
const mergedResults = mergeImportResults (latestAfterImport.rows, indexedResults)
const recoverableRows = indexedResults.filter (row =>
row.status === 'failed'
&& row.recoverable
&& Object.keys (row.errors ?? { }).length > 0)
const nextRows = mergedResults.map ((row): PostImportRow => {
const recoverable = recoverableRows.find (
failedRow => failedRow.sourceRow === row.sourceRow)
if (recoverable == null)
return applyThumbnailWarning (row)
return applyThumbnailWarning ({
...row,
importStatus: 'pending',
recoverable: true,
validationErrors: recoverable.errors ?? { },
importErrors: undefined })
})
const nextRows = mergeImportResults (latestAfterImport.rows, indexedResults)
.map (row => applyThumbnailWarning (row))
const persisted = await persistSession (buildSession (nextRows))
if (persisted != null
&& persisted.rows.every (row => isCompletedReviewRow (row)))
@@ -940,8 +950,7 @@ const PostImportReviewPage: FC<Props> = ({ user }) => {
skipDisabled={
busy
|| isExistingSkipRow (row)
|| row.importStatus === 'created'
|| isNonRecoverableFailedRow (row)}
|| row.importStatus === 'created'}
onEdit={() => editRow (row)}
onToggleSkip={checked => toggleManualSkip (row.sourceRow, checked)}
onRetry={canRetryResultRow (row) ? () => retry (row.sourceRow) : undefined}