import { resultRepairMode } from '@/lib/postImportRows' import { validatePostImportSessionId } from '@/lib/postImportStorage' import type { PostImportOrigin, PostImportRepairMode, PostImportRow, PostImportSkipReason, } from '@/lib/postImportTypes' type QueryRowState = { source_row: number title: string thumbnail_base: string original_created_from: string original_created_before: string video_ms: string duration: string tags: string parent_post_ids: string manual: string[] skip: PostImportSkipReason | null existing_post_id: number | null import_status: PostImportRow['importStatus'] | null created_post_id: number | null recoverable: boolean | null } type FullRowState = QueryRowState & { url: string } type MultiRowMetaState = { v: 1 rows: EncodedRowState[] } type EncodedSkip = 'e' | 'm' type EncodedImportStatus = 'p' | 'c' | 's' | 'f' type EncodedRowState = { n?: number u?: string t?: string h?: string f?: string b?: string x?: string d?: string g?: string p?: string m?: string[] s?: EncodedSkip e?: number i?: EncodedImportStatus c?: number r?: boolean } type EncodedState = { v: 1 rows: EncodedRowState[] } export type ParsedPostNewState = { rows: PostImportRow[] source: string repairMode: PostImportRepairMode shortcut: boolean } export type SerialisedPostNewState = { path: string complete: boolean } const BASIC_KEYS = [ 'url', 'title', 'thumbnail_base', 'original_created_from', 'original_created_before', 'video_ms', 'duration', 'tags', 'parent_post_ids'] as const const STATE_KEYS = [ 'manual', 'skip', 'existing_post_id', 'import_status', 'created_post_id', 'recoverable'] as const const SINGLE_ROW_KEYS = [...BASIC_KEYS, ...STATE_KEYS, 'session_id'] as const const ENCODED_ROW_KEYS = [ 'n', 'u', 't', 'h', 'f', 'b', 'x', 'd', 'g', 'p', 'm', 's', 'e', 'i', 'c', 'r'] as const const ALLOWED_MANUAL_FIELDS = [ 'title', 'thumbnailBase', 'originalCreatedFrom', 'originalCreatedBefore', 'duration', 'tags', 'parentPostIds'] as const const MAX_REGULAR_URL_LENGTH = 768 const MAX_COMPRESSED_STATE_LENGTH = 767 const COMPRESSED_STATE_PREFIX = 'z1.' const MAX_COMPRESSED_STATE_PARAM_LENGTH = 8_192 const MAX_COMPRESSED_STATE_BYTES = 16_384 const MAX_DECOMPRESSED_STATE_BYTES = 262_144 const MAX_STATE_ROWS = 100 const MAX_QUERY_STRING_BYTES = 20_480 const textEncoder = new TextEncoder () const textDecoder = new TextDecoder () const isPlainObject = (value: unknown): value is Record => typeof value === 'object' && value != null && !(Array.isArray (value)) const hasOnlyKeys = ( value: Record, allowedKeys: readonly string[], ): boolean => Object.keys (value).every (key => allowedKeys.includes (key)) const byteLength = (value: string): number => textEncoder.encode (value).byteLength const isValidSkipReason = (value: unknown): value is PostImportSkipReason => value === 'existing' || value === 'manual' const isValidImportStatus = ( value: unknown, ): value is PostImportRow['importStatus'] => value === 'pending' || value === 'created' || value === 'skipped' || value === 'failed' const encodeSkip = ( value: PostImportSkipReason | null, ): EncodedSkip | undefined => value === 'existing' ? 'e' : value === 'manual' ? 'm' : undefined const decodeSkip = ( value: unknown, ): PostImportSkipReason | null => { if (value == null) return null if (value === 'e') return 'existing' if (value === 'm') return 'manual' return null } const encodeImportStatus = ( value: PostImportRow['importStatus'] | null, ): EncodedImportStatus | undefined => value === 'pending' ? 'p' : value === 'created' ? 'c' : value === 'skipped' ? 's' : value === 'failed' ? 'f' : undefined const decodeImportStatus = ( value: unknown, ): PostImportRow['importStatus'] | null => { if (value == null) return null if (value === 'p') return 'pending' if (value === 'c') return 'created' if (value === 's') return 'skipped' if (value === 'f') return 'failed' return null } const parsePositiveInt = (value: string | null): number | null => { if (value == null || value === '') return null const parsed = Number.parseInt (value, 10) return Number.isInteger (parsed) && parsed > 0 ? parsed : null } const decodeBase64UrlBytes = (value: string): Uint8Array | null => { try { const padded = value.replaceAll ('-', '+').replaceAll ('_', '/') const remainder = padded.length % 4 const base64 = remainder === 0 ? padded : `${ padded }${ '='.repeat (4 - remainder) }` const binary = window.atob (base64) return Uint8Array.from (binary, char => char.charCodeAt (0)) } catch { return null } } const encodeBase64UrlBytes = (value: Uint8Array): string => { const binary = Array.from (value, byte => String.fromCharCode (byte)).join ('') return window.btoa (binary).replaceAll ('+', '-').replaceAll ('/', '_').replaceAll ('=', '') } const compressBytes = async (value: Uint8Array): Promise => { const stream = new Blob ([value]).stream ().pipeThrough (new CompressionStream ('gzip')) return new Uint8Array (await new Response (stream).arrayBuffer ()) } const decompressBytes = async (value: Uint8Array): Promise => { const stream = new Blob ([value]).stream ().pipeThrough (new DecompressionStream ('gzip')) const bytes = new Uint8Array (await new Response (stream).arrayBuffer ()) if (bytes.byteLength > MAX_DECOMPRESSED_STATE_BYTES) throw new Error ('decompressed state too large') return bytes } const decodeCompressedState = async (value: string): Promise => { if (!(value.startsWith (COMPRESSED_STATE_PREFIX))) return null if (value.length > MAX_COMPRESSED_STATE_PARAM_LENGTH) return null const encoded = value.slice (COMPRESSED_STATE_PREFIX.length) const compressed = decodeBase64UrlBytes (encoded) if (compressed == null) return null if (compressed.byteLength > MAX_COMPRESSED_STATE_BYTES) return null try { return textDecoder.decode (await decompressBytes (compressed)) } catch { return null } } const encodeCompressedState = async (value: string): Promise => { const compressed = await compressBytes (textEncoder.encode (value)) return `${ COMPRESSED_STATE_PREFIX }${ encodeBase64UrlBytes (compressed) }` } const manualFields = (row: PostImportRow): string[] => ['title', 'thumbnailBase', 'originalCreatedFrom', 'originalCreatedBefore', 'duration', 'tags', 'parentPostIds'] .filter (field => row.provenance[field] === 'manual') const baseRowState = (row: PostImportRow): QueryRowState => ({ source_row: row.sourceRow, title: String (row.attributes.title ?? ''), thumbnail_base: String (row.attributes.thumbnailBase ?? ''), original_created_from: String (row.attributes.originalCreatedFrom ?? ''), original_created_before: String (row.attributes.originalCreatedBefore ?? ''), video_ms: String (row.attributes.videoMs ?? ''), duration: String (row.attributes.duration ?? ''), tags: String (row.attributes.tags ?? ''), parent_post_ids: String (row.attributes.parentPostIds ?? ''), manual: manualFields (row), skip: row.skipReason ?? null, existing_post_id: row.existingPostId ?? null, import_status: row.importStatus ?? null, created_post_id: row.createdPostId ?? null, recoverable: row.recoverable ?? null }) const buildRow = (state: FullRowState): PostImportRow => { const manual = new Set (state.manual) const provenance: Record = { url: 'manual', title: manual.has ('title') ? 'manual' : 'automatic', thumbnailBase: manual.has ('thumbnailBase') ? 'manual' : 'automatic', originalCreatedFrom: manual.has ('originalCreatedFrom') ? 'manual' : 'automatic', originalCreatedBefore: manual.has ('originalCreatedBefore') ? 'manual' : 'automatic', videoMs: manual.has ('duration') ? 'manual' : 'automatic', duration: manual.has ('duration') ? 'manual' : 'automatic', tags: manual.has ('tags') ? 'manual' : 'automatic', parentPostIds: manual.has ('parentPostIds') ? 'manual' : 'automatic' } const tagSources = provenance.tags === 'manual' ? { automatic: '', manual: state.tags } : { automatic: state.tags, manual: '' } const attributes = { title: state.title, thumbnailBase: state.thumbnail_base, originalCreatedFrom: state.original_created_from, originalCreatedBefore: state.original_created_before, videoMs: manual.has ('duration') ? '' : state.video_ms, duration: state.duration, tags: state.tags, parentPostIds: state.parent_post_ids } return { sourceRow: state.source_row, url: state.url, attributes, fieldWarnings: { }, baseWarnings: [], validationErrors: { }, provenance, tagSources, status: 'pending', skipReason: state.skip ?? undefined, existingPostId: state.existing_post_id ?? undefined, resetSnapshot: { url: state.url, attributes: { ...attributes }, provenance: { ...provenance }, tagSources: { ...tagSources }, fieldWarnings: { }, baseWarnings: [] }, createdPostId: state.created_post_id ?? undefined, importStatus: state.import_status ?? undefined, recoverable: state.recoverable ?? undefined } } const parseManual = (value: string | null): string[] => (value ?? '') .split (',') .map (entry => entry.trim ()) .filter (entry => entry !== '') const hasOnlyAllowedManualFields = (value: string[]): boolean => value.every (field => ALLOWED_MANUAL_FIELDS.includes (field as never)) const isStringArray = (value: unknown): value is string[] => Array.isArray (value) && value.every (entry => typeof entry === 'string') const parseEncodedRow = ( value: unknown, requireUrl: boolean, ): FullRowState | null => { if (!(isPlainObject (value))) return null const row = value as Record if (!(hasOnlyKeys (row, ENCODED_ROW_KEYS))) return null const url = row['u'] const manual = row['m'] const skip = decodeSkip (row['s']) const importStatus = decodeImportStatus (row['i']) const existingPostId = row['e'] const createdPostId = row['c'] const recoverable = row['r'] const sourceRow = row['n'] if (requireUrl && typeof url !== 'string') return null if (typeof url === 'string' && byteLength (url) > MAX_QUERY_STRING_BYTES) return null if (manual != null && !(isStringArray (manual))) return null if ((manual ?? []).some (field => !(ALLOWED_MANUAL_FIELDS.includes (field as never)))) return null if (row['s'] != null && skip == null) return null if (row['i'] != null && importStatus == null) return null if (existingPostId != null && !(typeof existingPostId === 'number' && Number.isInteger (existingPostId) && existingPostId > 0)) return null if (createdPostId != null && !(typeof createdPostId === 'number' && Number.isInteger (createdPostId) && createdPostId > 0)) return null if (recoverable != null && typeof recoverable !== 'boolean') return null if (sourceRow != null && !(typeof sourceRow === 'number' && Number.isInteger (sourceRow) && sourceRow > 0)) return null const stringFields = [ ['t', 'title'], ['h', 'thumbnail_base'], ['f', 'original_created_from'], ['b', 'original_created_before'], ['x', 'video_ms'], ['d', 'duration'], ['g', 'tags'], ['p', 'parent_post_ids']] as const let parsedStrings: Record<(typeof stringFields)[number][1], string> try { parsedStrings = Object.fromEntries ( stringFields.map (([key, target]) => { const fieldValue = row[key] if (fieldValue != null && typeof fieldValue !== 'string') throw new Error ('invalid row') if (typeof fieldValue === 'string' && byteLength (fieldValue) > MAX_QUERY_STRING_BYTES) throw new Error ('invalid row') return [target, fieldValue ?? ''] }), ) as Record<(typeof stringFields)[number][1], string> } catch { return null } return { source_row: typeof sourceRow === 'number' ? sourceRow : 1, url: typeof url === 'string' ? url : '', title: parsedStrings.title, thumbnail_base: parsedStrings.thumbnail_base, original_created_from: parsedStrings.original_created_from, original_created_before: parsedStrings.original_created_before, video_ms: parsedStrings.video_ms, duration: parsedStrings.duration, tags: parsedStrings.tags, parent_post_ids: parsedStrings.parent_post_ids, manual: manual ?? [], skip, existing_post_id: existingPostId ?? null, import_status: importStatus, created_post_id: createdPostId ?? null, recoverable: recoverable ?? null } } const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { const url = params.get ('url') if (url == null || url === '') return null if (byteLength (url) > MAX_QUERY_STRING_BYTES) return null const hasOnlyUrl = Array.from (params.keys ()).every ( key => key === 'url' || key === 'session_id') if (hasOnlyUrl) { const row = buildRow ({ source_row: 1, url, title: '', thumbnail_base: '', original_created_from: '', original_created_before: '', video_ms: '', duration: '', tags: '', parent_post_ids: '', manual: [], skip: null, existing_post_id: null, import_status: null, created_post_id: null, recoverable: null }) return { rows: [row], source: url, repairMode: 'all', shortcut: true } } if (!(Array.from (params.keys ()).every (key => SINGLE_ROW_KEYS.includes (key as never)))) return null const skipValue = params.get ('skip') const importStatus = params.get ('import_status') const recoverable = params.get ('recoverable') == null ? null : params.get ('recoverable') === 'true' if (skipValue != null && !(isValidSkipReason (skipValue))) return null if (importStatus != null && !(isValidImportStatus (importStatus))) return null if (!(hasOnlyAllowedManualFields (parseManual (params.get ('manual'))))) return null const row = buildRow ({ source_row: 1, url, title: params.get ('title') ?? '', thumbnail_base: params.get ('thumbnail_base') ?? '', original_created_from: params.get ('original_created_from') ?? '', original_created_before: params.get ('original_created_before') ?? '', video_ms: params.get ('video_ms') ?? '', duration: params.get ('duration') ?? '', tags: params.get ('tags') ?? '', parent_post_ids: params.get ('parent_post_ids') ?? '', manual: parseManual (params.get ('manual')), skip: skipValue, existing_post_id: parsePositiveInt (params.get ('existing_post_id')), import_status: importStatus, created_post_id: parsePositiveInt (params.get ('created_post_id')), recoverable }) return { rows: [row], source: url, repairMode: resultRepairMode ([row]), shortcut: false } } const parseMetaRows = ( urlsValue: string, metaValue: string, ): ParsedPostNewState | null => { const decodedBytes = decodeBase64UrlBytes (metaValue) if (decodedBytes == null) return null let parsed: MultiRowMetaState try { parsed = JSON.parse (textDecoder.decode (decodedBytes)) as MultiRowMetaState } catch { return null } if (parsed.v !== 1 || !(Array.isArray (parsed.rows))) return null const urls = urlsValue.split (' ').filter (url => url !== '') if (urls.length < 2 || urls.length > MAX_STATE_ROWS || urls.length !== parsed.rows.length || parsed.rows.length > MAX_STATE_ROWS) return null if (urls.some (url => byteLength (url) > MAX_QUERY_STRING_BYTES)) return null const fullRows = parsed.rows.map ((row, index) => { const parsedRow = parseEncodedRow (row, false) return parsedRow == null ? null : { ...parsedRow, url: urls[index] ?? '' } }) if (fullRows.some (row => row == null)) return null if ((new Set (fullRows.map (row => (row as FullRowState).source_row))).size !== fullRows.length) return null const rows = fullRows.map (row => buildRow (row as FullRowState)) return { rows, source: urls.join ('\n'), repairMode: resultRepairMode (rows), shortcut: false } } const parseFullState = async (stateValue: string): Promise => { const decoded = await decodeCompressedState (stateValue) if (decoded == null) return null let parsed: EncodedState try { parsed = JSON.parse (decoded) as EncodedState } catch { return null } if (parsed.v !== 1 || !(Array.isArray (parsed.rows)) || parsed.rows.length === 0 || parsed.rows.length > MAX_STATE_ROWS) return null const fullRows = parsed.rows.map (row => parseEncodedRow (row, true)) if (fullRows.some (row => row == null)) return null if ((new Set (fullRows.map (row => (row as FullRowState).source_row))).size !== fullRows.length) return null const rows = fullRows.map (row => buildRow (row as FullRowState)) return { rows, source: rows.map (row => row.url).join ('\n'), repairMode: resultRepairMode (rows), shortcut: false } } const regularQuery = (rows: PostImportRow[]): URLSearchParams => { if (rows.length === 1) { const row = rows[0] const params = new URLSearchParams () params.set ('url', row?.url ?? '') params.set ('title', String (row?.attributes.title ?? '')) params.set ('thumbnail_base', String (row?.attributes.thumbnailBase ?? '')) params.set ('original_created_from', String (row?.attributes.originalCreatedFrom ?? '')) params.set ('original_created_before', String (row?.attributes.originalCreatedBefore ?? '')) params.set ('video_ms', String (row?.attributes.videoMs ?? '')) params.set ('duration', String (row?.attributes.duration ?? '')) params.set ('tags', String (row?.attributes.tags ?? '')) params.set ('parent_post_ids', String (row?.attributes.parentPostIds ?? '')) const manual = manualFields (row).join (',') if (manual !== '') params.set ('manual', manual) if (row.skipReason != null) params.set ('skip', row.skipReason) if (row.existingPostId != null) params.set ('existing_post_id', String (row.existingPostId)) if (row.importStatus != null) params.set ('import_status', row.importStatus) if (row.createdPostId != null) params.set ('created_post_id', String (row.createdPostId)) if (row.recoverable != null) params.set ('recoverable', row.recoverable ? 'true' : 'false') return params } const params = new URLSearchParams () params.set ('urls', rows.map (row => row.url).join (' ')) params.set ('meta', encodeBase64UrlBytes (textEncoder.encode (JSON.stringify ({ v: 1, rows: rows.map (row => encodeRowState (row, false)) })))) return params } const encodeRowState = ( row: PostImportRow, includeUrl: boolean, ): EncodedRowState => { const base = baseRowState (row) const encoded: EncodedRowState = includeUrl ? { u: row.url } : { } encoded.n = base.source_row if (base.title !== '') encoded.t = base.title if (base.thumbnail_base !== '') encoded.h = base.thumbnail_base if (base.original_created_from !== '') encoded.f = base.original_created_from if (base.original_created_before !== '') encoded.b = base.original_created_before if (base.video_ms !== '') encoded.x = base.video_ms if (base.duration !== '') encoded.d = base.duration if (base.tags !== '') encoded.g = base.tags if (base.parent_post_ids !== '') encoded.p = base.parent_post_ids if (base.manual.length > 0) encoded.m = [...base.manual] if (base.skip != null) encoded.s = encodeSkip (base.skip) if (base.existing_post_id != null) encoded.e = base.existing_post_id if (base.import_status != null) encoded.i = encodeImportStatus (base.import_status) if (base.created_post_id != null) encoded.c = base.created_post_id if (base.recoverable != null) encoded.r = base.recoverable return encoded } const encodeStateRows = (rows: PostImportRow[]): EncodedState => ({ v: 1, rows: rows.map (row => encodeRowState (row, true)) }) export const hasPostNewReviewState = (search: string): boolean => { const params = new URLSearchParams (search) return params.has ('state') || params.has ('urls') || params.has ('meta') || params.has ('url') || parsePostNewSessionId (search) != null } export const parsePostNewSessionId = (search: string): string | null => validatePostImportSessionId ( new URLSearchParams (search).get ('session_id')) export const appendPostNewSessionId = ( path: string, sessionId: string, ): string => { const validatedId = validatePostImportSessionId (sessionId) if (validatedId == null) return path const separator = path.includes ('?') ? '&' : '?' return `${ path }${ separator }session_id=${ validatedId }` } export const parsePostNewState = async ( search: string, ): Promise => { const params = new URLSearchParams (search) const stateValue = params.get ('state') if (stateValue != null) return await parseFullState (stateValue) const urlsValue = params.get ('urls') const metaValue = params.get ('meta') if (urlsValue != null || metaValue != null) { if (urlsValue == null || metaValue == null) return null return parseMetaRows (urlsValue, metaValue) } return parseSingleRow (params) } const serialiseCompressedRows = async ( rows: PostImportRow[], ): Promise => { try { for (let count = rows.length; count > 0; --count) { const nextRows = rows.slice (0, count) const encoded = encodeStateRows (nextRows) const state = await encodeCompressedState (JSON.stringify (encoded)) const path = `/posts/new?state=${ state }` if (path.length <= MAX_COMPRESSED_STATE_LENGTH) return { path, complete: count === rows.length } } } catch { } return null } export const serialisePostNewState = async ( rows: PostImportRow[], ): Promise => { const regular = regularQuery (rows).toString () const regularPath = `/posts/new?${ regular }` if (regularPath.length < MAX_REGULAR_URL_LENGTH) return { path: regularPath, complete: true } return await serialiseCompressedRows (rows) }