diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index 4bb4b3f..5658ebb 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -131,6 +131,7 @@ class PostsController < ApplicationController title: nil, thumbnail_base: nil, tags: nil, + display_tags: [], original_created_from: nil, original_created_before: nil, duration: nil, @@ -153,6 +154,7 @@ class PostsController < ApplicationController title: metadata[:title], thumbnail_base: metadata[:thumbnail_base], tags: metadata[:tags], + display_tags: metadata[:display_tags], original_created_from: metadata[:original_created_from], original_created_before: metadata[:original_created_before], duration: metadata[:duration], @@ -173,6 +175,7 @@ class PostsController < ApplicationController title: nil, thumbnail_base: nil, tags: nil, + display_tags: [], original_created_from: nil, original_created_before: nil, duration: nil, @@ -633,6 +636,7 @@ class PostsController < ApplicationController :title, :thumbnail_base, :tags, + :display_tags, :parent_post_ids, :original_created_from, :original_created_before, diff --git a/backend/app/representations/post_compact_repr.rb b/backend/app/representations/post_compact_repr.rb index f1a0797..eca81aa 100644 --- a/backend/app/representations/post_compact_repr.rb +++ b/backend/app/representations/post_compact_repr.rb @@ -7,20 +7,13 @@ module PostCompactRepr def base post return nil if post.nil? - { - id: post.id, - title: post.title, - url: post.url, - thumbnail_url: thumbnail_url(post) } - end - - def thumbnail_url post - return nil unless post.thumbnail.attached? - - Rails.application.routes.url_helpers.rails_storage_proxy_url( - post.thumbnail, - only_path: false) - rescue ActionController::UrlGenerationError, ArgumentError, URI::InvalidURIError - nil + PostRepr + .common(post) + .slice( + 'id', + 'title', + 'url', + 'thumbnail', + 'thumbnail_base') end end diff --git a/backend/app/services/post_create_plan.rb b/backend/app/services/post_create_plan.rb index c660e21..ea4712b 100644 --- a/backend/app/services/post_create_plan.rb +++ b/backend/app/services/post_create_plan.rb @@ -27,6 +27,7 @@ class PostCreatePlan original_created_from: @attributes[:original_created_from].presence, original_created_before: @attributes[:original_created_before].presence, tags: serialised_tags(direct_tag_specs, tag_sections), + display_tags: display_tags(direct_tag_specs, tag_sections), duration: @attributes[:duration].to_s, video_ms: video_ms, parent_post_ids: parent_post_ids.join(' '), @@ -204,6 +205,15 @@ class PostCreatePlan }.sort.join(' ') end + def display_tags direct_tag_specs, tag_sections + direct_tag_specs.map { |spec| + { + name: spec[:name], + category: spec[:category].to_s, + section_literals: tag_sections[spec[:name]].to_a.map { Post.section_literal(_1) } } + }.sort_by { _1[:name] } + end + def normalise_video_ms snapshot_tag_specs return nil unless snapshot_tag_specs.any? { _1[:name] == VIDEO_TAG_NAME } diff --git a/backend/app/services/post_create_preflight.rb b/backend/app/services/post_create_preflight.rb index 234483f..c633b47 100644 --- a/backend/app/services/post_create_preflight.rb +++ b/backend/app/services/post_create_preflight.rb @@ -29,6 +29,7 @@ class PostCreatePreflight original_created_before: preview[:attributes]['original_created_before'], duration: preview[:attributes]['duration'], video_ms: preview[:attributes]['video_ms'], + display_tags: preview[:display_tags] || [], field_warnings: final_field_warnings(preview[:field_warnings] || { }), base_warnings: preview[:base_warnings], existing_post_id: preview[:existing_post_id], @@ -63,6 +64,7 @@ class PostCreatePreflight original_created_before: plan[:original_created_before], duration: plan[:duration], video_ms: plan[:video_ms], + display_tags: plan[:display_tags], direct_tag_specs: plan[:direct_tag_specs], default_tag_specs: plan[:default_tag_specs], snapshot_tag_specs: plan[:snapshot_tag_specs], diff --git a/backend/app/services/post_metadata_fetcher.rb b/backend/app/services/post_metadata_fetcher.rb index 5dbff94..33fc3d9 100644 --- a/backend/app/services/post_metadata_fetcher.rb +++ b/backend/app/services/post_metadata_fetcher.rb @@ -32,6 +32,7 @@ class PostMetadataFetcher original_created_from: serialise_time(created_range&.first), original_created_before: serialise_time(created_range&.last), duration: serialise_duration(duration), + display_tags: display_tags(platform_tags), tags: platform_tags.join(' ') } end @@ -42,6 +43,24 @@ class PostMetadataFetcher [] end + def self.display_tags names + return [] if names.blank? + + existing_tags = + Tag + .joins(:tag_name) + .where(tag_names: { name: names }) + .index_by(&:name) + + names.map { |name| + tag = existing_tags[name] + { + name: name, + category: (tag&.category || 'meta'), + section_literals: [] } + } + end + def self.original_created_range value return nil if value.blank? @@ -168,6 +187,7 @@ class PostMetadataFetcher end private_class_method :platform_tags, + :display_tags, :original_created_range, :parse_timestamp_range, :parse_nanoseconds, diff --git a/frontend/src/components/TagLink.tsx b/frontend/src/components/TagLink.tsx index 615f7d1..cfe088e 100644 --- a/frontend/src/components/TagLink.tsx +++ b/frontend/src/components/TagLink.tsx @@ -4,9 +4,13 @@ import { cn } from '@/lib/utils' import type { ComponentProps, CSSProperties, FC, HTMLAttributes } from 'react' -import type { Tag } from '@/types' +import type { Category, Tag } from '@/types' -type CommonProps = { +type LightweightTag = { + name: string + category: Category } + +type FullCommonProps = { tag: Tag nestLevel?: number truncateOnMobile?: boolean @@ -14,18 +18,43 @@ type CommonProps = { withCount?: boolean } type PropsWithLink = - & CommonProps + & FullCommonProps & { linkFlg?: true } & Partial> type PropsWithoutLink = - & CommonProps + & FullCommonProps & { linkFlg: false } & Partial> +type LightweightPropsWithLink = + & { + tag: LightweightTag + nestLevel?: number + truncateOnMobile?: boolean + withWiki: false + withCount: false + linkFlg?: true } + & Partial> + +type LightweightPropsWithoutLink = + & { + tag: LightweightTag + nestLevel?: number + truncateOnMobile?: boolean + withWiki: false + withCount: false + linkFlg: false } + & Partial> + type Props = | PropsWithLink | PropsWithoutLink + | LightweightPropsWithLink + | LightweightPropsWithoutLink + +const isFullTag = (tag: Tag | LightweightTag): tag is Tag => + 'id' in tag const TagLink: FC = ({ tag, @@ -50,12 +79,13 @@ const TagLink: FC = ({ tag, 'inline-flex min-w-0 max-w-full flex-nowrap items-stretch align-baseline gap-x-1 md:items-baseline' const markerWrapClass = 'shrink-0 self-start md:self-auto' const countClass = 'shrink-0 self-end md:self-auto' + const matchedAlias = isFullTag (tag) ? tag.matchedAlias : null const textTitle = title - ?? (tag.matchedAlias == null ? tag.name : `${ tag.matchedAlias } → ${ tag.name }`) + ?? (matchedAlias == null ? tag.name : `${ matchedAlias } → ${ tag.name }`) return ( - {(linkFlg && withWiki) && ( + {(linkFlg && withWiki && isFullTag (tag)) && ( {(tag.materialId != null || tag.hasWiki || tag.hasDeerjikists) ? ( @@ -118,7 +148,7 @@ const TagLink: FC = ({ tag, style={{ paddingLeft: `${ (nestLevel - 1) }rem` }}> ↳ )} - {tag.matchedAlias != null && ( + {matchedAlias != null && ( <> = ({ tag, style={colourStyle} {...props}> @@ -156,7 +186,7 @@ const TagLink: FC = ({ tag, title={textTitle} truncateOnMobile={truncateOnMobile}/> )} - {withCount && ( + {(withCount && isFullTag (tag)) && ( {tag.postCount})} ) } diff --git a/frontend/src/components/posts/import/PostImportRowSummary.tsx b/frontend/src/components/posts/import/PostImportRowSummary.tsx index 9fd5fa2..d2fd4b3 100644 --- a/frontend/src/components/posts/import/PostImportRowSummary.tsx +++ b/frontend/src/components/posts/import/PostImportRowSummary.tsx @@ -1,4 +1,5 @@ import FieldError from '@/components/common/FieldError' +import PostImportTagLinks from '@/components/posts/import/PostImportTagLinks' import { Button } from '@/components/ui/button' import PostImportThumbnailPreview from '@/components/posts/import/PostImportThumbnailPreview' import PostImportStatusBadge from '@/components/posts/import/PostImportStatusBadge' @@ -90,10 +91,7 @@ const PostImportRowSummary: FC = (
{row.url}
- {String (row.attributes.tags ?? '') && ( -
- {String (row.attributes.tags ?? '')} -
)} +
{summaryDate (row)}
@@ -153,10 +151,7 @@ const PostImportRowSummary: FC = (
{displayStatus != null && }
- {String (row.attributes.tags ?? '') && ( -
- {String (row.attributes.tags ?? '')} -
)} +
{summaryDate (row)}
diff --git a/frontend/src/components/posts/import/PostImportTagLinks.tsx b/frontend/src/components/posts/import/PostImportTagLinks.tsx new file mode 100644 index 0000000..7303f72 --- /dev/null +++ b/frontend/src/components/posts/import/PostImportTagLinks.tsx @@ -0,0 +1,35 @@ +import TagLink from '@/components/TagLink' + +import type { FC } from 'react' + +import type { PostImportDisplayTag } from '@/lib/postImportTypes' + +type Props = { + tags: PostImportDisplayTag[] | undefined } + +const PostImportTagLinks: FC = ({ tags }) => { + if (tags == null || tags.length === 0) + return null + + return ( +
+ {tags.map (tag => { + const key = `${ tag.category }:${ tag.name }:${ tag.sectionLiterals?.join ('|') ?? '' }` + return ( + + + {tag.sectionLiterals?.map (literal => ( + + {literal} + ))} + ) + })} +
) +} + +export default PostImportTagLinks diff --git a/frontend/src/lib/postImportRows.ts b/frontend/src/lib/postImportRows.ts index 6de6236..601f7cd 100644 --- a/frontend/src/lib/postImportRows.ts +++ b/frontend/src/lib/postImportRows.ts @@ -68,6 +68,10 @@ export const validatableImportRows = (rows: PostImportRow[]): PostImportRow[] => const buildResetSnapshot = (row: PostImportRow) => ({ url: row.url, attributes: { ...row.attributes }, + displayTags: row.displayTags?.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: tag.sectionLiterals == null ? undefined : [...tag.sectionLiterals] })) ?? [], provenance: { ...row.provenance }, tagSources: { automatic: row.tagSources?.automatic ?? '', @@ -243,6 +247,11 @@ export const buildNextEditedRow = ( ...editingRow, url: draft.url, attributes: nextAttributes, + displayTags: + editingRow.displayTags?.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: tag.sectionLiterals == null ? undefined : [...tag.sectionLiterals] })), thumbnailFile: draft.thumbnailFile, provenance: { ...nextProvenance, @@ -316,6 +325,12 @@ export const mergeValidatedImportRows = ( ...previous, url: row.url, attributes: row.attributes, + displayTags: + row.displayTags?.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null ? undefined : [...tag.sectionLiterals] })), provenance: row.provenance, tagSources: row.tagSources, skipReason: row.skipReason, diff --git a/frontend/src/lib/postImportStorage.ts b/frontend/src/lib/postImportStorage.ts index c75ec84..b339980 100644 --- a/frontend/src/lib/postImportStorage.ts +++ b/frontend/src/lib/postImportStorage.ts @@ -1,4 +1,5 @@ import type { PostImportOrigin, + PostImportDisplayTag, PostImportExistingPost, PostImportResetSnapshot, PostImportRow, @@ -43,6 +44,7 @@ const ROW_KEYS = [ 'existingPostId', 'existingPost', 'metadataUrl', + 'displayTags', 'resetSnapshot', 'createdPostId', 'importStatus', @@ -216,11 +218,53 @@ const hasOnlyKeys = ( ): boolean => Object.keys (value).every (key => allowedKeys.includes (key)) +const sanitiseDisplayTag = (value: unknown): PostImportDisplayTag | null => { + if (!(isPlainObject (value))) + return null + if (!(hasOnlyKeys (value, ['name', 'category', 'sectionLiterals']))) + return null + if (typeof value.name !== 'string' || value.name === '') + return null + if (typeof value.category !== 'string') + return null + if (!(['deerjikist', + 'meme', + 'character', + 'general', + 'material', + 'meta', + 'nico'] as const).includes (value.category as never)) + return null + if (value.sectionLiterals != null + && (!(Array.isArray (value.sectionLiterals)) + || !(value.sectionLiterals.every (entry => typeof entry === 'string')))) + return null + + return { + name: value.name, + category: value.category, + sectionLiterals: + value.sectionLiterals == null + ? undefined + : [...value.sectionLiterals] } +} + +const sanitiseDisplayTags = (value: unknown): PostImportDisplayTag[] | null => { + if (value == null) + return [] + if (!(Array.isArray (value))) + return null + + const tags = value.map (sanitiseDisplayTag) + return tags.some (tag => tag == null) ? null : (tags as PostImportDisplayTag[]) +} + const sanitiseResetSnapshot = (value: unknown): PostImportResetSnapshot | null => { if (!(isPlainObject (value))) return null if (!(hasOnlyKeys (value, ['url', 'attributes', + 'displayTags', 'provenance', 'tagSources', 'fieldWarnings', @@ -258,10 +302,14 @@ const sanitiseResetSnapshot = (value: unknown): PostImportResetSnapshot | null = return null if (value.metadataUrl != null && typeof value.metadataUrl !== 'string') return null + const displayTags = sanitiseDisplayTags (value.displayTags) + if (displayTags == null) + return null return { url: value.url, attributes: value.attributes as Record, + displayTags, provenance: value.provenance as Record, tagSources: value.tagSources as Record, fieldWarnings, @@ -278,14 +326,39 @@ const sanitiseExistingPost = (value: unknown): PostImportExistingPost | undefine return undefined if (typeof value.title !== 'string' || typeof value.url !== 'string') return undefined - if (value.thumbnailUrl != null && typeof value.thumbnailUrl !== 'string') + if (value.thumbnail !== undefined + && value.thumbnail !== null + && typeof value.thumbnail !== 'string') + return undefined + if (value.thumbnailBase !== undefined + && value.thumbnailBase !== null + && typeof value.thumbnailBase !== 'string') + return undefined + if (value.thumbnailUrl !== undefined + && value.thumbnailUrl !== null + && typeof value.thumbnailUrl !== 'string') return undefined return { id: Number (value.id), title: value.title, url: value.url, - thumbnailUrl: value.thumbnailUrl as string | undefined } + thumbnail: + value.thumbnail === null + ? null + : typeof value.thumbnail === 'string' + ? value.thumbnail + : value.thumbnailUrl === null + ? null + : typeof value.thumbnailUrl === 'string' + ? value.thumbnailUrl + : undefined, + thumbnailBase: + value.thumbnailBase === null + ? null + : typeof value.thumbnailBase === 'string' + ? value.thumbnailBase + : undefined } } const canonicalisePersistedRow = ( @@ -319,12 +392,18 @@ const serialiseExistingPost = (value: PostImportExistingPost | undefined) => id: value.id, title: value.title, url: value.url, - thumbnailUrl: value.thumbnailUrl } + thumbnail: value.thumbnail, + thumbnailBase: value.thumbnailBase } const serialiseResetSnapshot = (value: PostImportResetSnapshot) => ({ url: value.url, attributes: Object.fromEntries ( Object.entries (value.attributes).map (([key, entry]) => [key, entry])), + displayTags: value.displayTags.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null ? undefined : [...tag.sectionLiterals] })), provenance: Object.fromEntries ( Object.entries (value.provenance).map (([key, entry]) => [key, entry])), tagSources: { @@ -366,6 +445,12 @@ export const serialisePostImportRow = (row: PostImportRow) => { Object.entries (row.importErrors).map (([key, entry]) => [key, [...entry]])), provenance: Object.fromEntries ( Object.entries (row.provenance).map (([key, entry]) => [key, entry])), + displayTags: + row.displayTags?.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null ? undefined : [...tag.sectionLiterals] })) ?? [], tagSources: row.tagSources == null ? undefined @@ -475,6 +560,9 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { if (!(Object.values (value.tagSources).every (entry => typeof entry === 'string'))) return null } + const displayTags = sanitiseDisplayTags (value.displayTags) + if (displayTags == null) + return null const existingPost = sanitiseExistingPost (value.existingPost) const existingPostId = @@ -496,6 +584,7 @@ const sanitiseRow = (value: unknown): PostImportRow | null => { ? importErrors : undefined, provenance: value.provenance as Record, + displayTags, tagSources: value.tagSources as Record | undefined, status: value.status, skipReason: value.skipReason ?? undefined, diff --git a/frontend/src/lib/postImportTypes.ts b/frontend/src/lib/postImportTypes.ts index 45de47d..9c75eb2 100644 --- a/frontend/src/lib/postImportTypes.ts +++ b/frontend/src/lib/postImportTypes.ts @@ -1,3 +1,5 @@ +import type { Category } from '@/types' + export type PostImportOrigin = 'automatic' | 'manual' export type PostImportRepairMode = 'all' | 'failed' export type PostImportStatus = @@ -9,9 +11,15 @@ export type PostImportSkipReason = 'existing' | 'manual' export type PostImportResultStatus = 'created' | 'skipped' | 'failed' export type PostImportAttributeValue = string | number +export type PostImportDisplayTag = { + name: string + category: Category + sectionLiterals?: string[] } + export type PostImportResetSnapshot = { url: string attributes: Record + displayTags: PostImportDisplayTag[] provenance: Record tagSources: Record fieldWarnings: Record @@ -22,7 +30,8 @@ export type PostImportExistingPost = { id: number title: string url: string - thumbnailUrl?: string } + thumbnail?: string | null + thumbnailBase?: string | null } export type PostImportRow = { sourceRow: number @@ -39,6 +48,7 @@ export type PostImportRow = { existingPostId?: number existingPost?: PostImportExistingPost metadataUrl?: string + displayTags?: PostImportDisplayTag[] resetSnapshot: PostImportResetSnapshot createdPostId?: number importStatus?: PostImportStatus diff --git a/frontend/src/lib/postNewQueryState.ts b/frontend/src/lib/postNewQueryState.ts index d6e6de9..90090a7 100644 --- a/frontend/src/lib/postNewQueryState.ts +++ b/frontend/src/lib/postNewQueryState.ts @@ -1,7 +1,9 @@ +import { CATEGORIES } from '@/consts' import { resultRepairMode } from '@/lib/postImportRows' import { validatePostImportSessionId } from '@/lib/postImportStorage' import type { + PostImportDisplayTag, PostImportOrigin, PostImportRepairMode, PostImportRow, @@ -18,9 +20,11 @@ type QueryRowState = { duration: string tags: string parent_post_ids: string + display_tags: PostImportDisplayTag[] manual: string[] skip: PostImportSkipReason | null existing_post_id: number | null + existing_post: PostImportRow['existingPost'] | null import_status: PostImportRow['importStatus'] | null created_post_id: number | null recoverable: boolean | null } @@ -45,9 +49,11 @@ type EncodedRowState = { d?: string g?: string p?: string + k?: EncodedDisplayTag[] m?: string[] s?: EncodedSkip e?: number + o?: EncodedExistingPost i?: EncodedImportStatus c?: number r?: boolean } @@ -56,6 +62,18 @@ type EncodedState = { v: 1 rows: EncodedRowState[] } +type EncodedDisplayTag = { + n: string + c: PostImportDisplayTag['category'] + s?: string[] } + +type EncodedExistingPost = { + i: number + t: string + u: string + h?: string | null + b?: string | null } + export type ParsedPostNewState = { rows: PostImportRow[] source: string @@ -77,9 +95,11 @@ const BASIC_KEYS = [ 'tags', 'parent_post_ids'] as const const STATE_KEYS = [ + 'display_tags', 'manual', 'skip', 'existing_post_id', + 'existing_post', 'import_status', 'created_post_id', 'recoverable'] as const @@ -95,9 +115,11 @@ const ENCODED_ROW_KEYS = [ 'd', 'g', 'p', + 'k', 'm', 's', 'e', + 'o', 'i', 'c', 'r'] as const @@ -200,6 +222,153 @@ const decodeImportStatus = ( return null } +const sanitiseDisplayTag = (value: unknown): PostImportDisplayTag | null => { + if (!(isPlainObject (value))) + return null + if (!(hasOnlyKeys (value, ['n', 'c', 's']))) + return null + if (typeof value.n !== 'string' || value.n === '') + return null + if (!(CATEGORIES.includes (value.c as never))) + return null + if (value.s != null && !(Array.isArray (value.s)) || !(value.s ?? []).every (entry => typeof entry === 'string')) + return null + + return { + name: value.n, + category: value.c, + sectionLiterals: value.s == null ? undefined : [...value.s] } +} + +const sanitiseDisplayTags = ( + value: unknown, +): PostImportDisplayTag[] | null => { + if (value == null) + return [] + if (!(Array.isArray (value))) + return null + + const tags = value.map (sanitiseDisplayTag) + return tags.some (tag => tag == null) ? null : (tags as PostImportDisplayTag[]) +} + +const encodeDisplayTags = ( + tags: PostImportDisplayTag[] | undefined, +): EncodedDisplayTag[] | undefined => + tags == null || tags.length === 0 + ? undefined + : tags.map (tag => ({ + n: tag.name, + c: tag.category, + s: + tag.sectionLiterals == null || tag.sectionLiterals.length === 0 + ? undefined + : [...tag.sectionLiterals] })) + +const encodeDisplayTagsParam = ( + tags: PostImportDisplayTag[] | undefined, +): string | null => { + const encodedTags = encodeDisplayTags (tags) + if (encodedTags == null) + return null + + return encodeBase64UrlBytes (textEncoder.encode (JSON.stringify (encodedTags))) +} + +const decodeDisplayTagsParam = ( + value: string | null, +): PostImportDisplayTag[] | null => { + if (value == null || value === '') + return [] + + const bytes = decodeBase64UrlBytes (value) + if (bytes == null || bytes.byteLength > MAX_DECOMPRESSED_STATE_BYTES) + return null + + try + { + return sanitiseDisplayTags (JSON.parse (textDecoder.decode (bytes))) + } + catch + { + return null + } +} + + +const sanitiseExistingPost = ( + value: unknown, +): PostImportRow['existingPost'] | null => { + if (value == null) + return null + if (!(isPlainObject (value))) + return null + if (!(hasOnlyKeys (value, ['i', 't', 'u', 'h', 'b']))) + return null + if (!(typeof value.i === 'number' + && Number.isSafeInteger (value.i) + && value.i > 0)) + return null + if (typeof value.t !== 'string' || typeof value.u !== 'string') + return null + if (value.h != null && typeof value.h !== 'string') + return null + if (value.b != null && typeof value.b !== 'string') + return null + + return { + id: value.i, + title: value.t, + url: value.u, + thumbnail: value.h, + thumbnailBase: value.b } +} + + +const encodeExistingPost = ( + post: PostImportRow['existingPost'] | undefined, +): EncodedExistingPost | undefined => + post == null + ? undefined + : { + i: post.id, + t: post.title, + u: post.url, + h: post.thumbnail, + b: post.thumbnailBase } + + +const encodeExistingPostParam = ( + post: PostImportRow['existingPost'] | undefined, +): string | null => { + const encodedPost = encodeExistingPost (post) + if (encodedPost == null) + return null + + return encodeBase64UrlBytes (textEncoder.encode (JSON.stringify (encodedPost))) +} + + +const decodeExistingPostParam = ( + value: string | null, +): PostImportRow['existingPost'] | null => { + if (value == null || value === '') + return null + + const bytes = decodeBase64UrlBytes (value) + if (bytes == null || bytes.byteLength > MAX_DECOMPRESSED_STATE_BYTES) + return null + + try + { + return sanitiseExistingPost (JSON.parse (textDecoder.decode (bytes))) + } + catch + { + return null + } +} + const isStrictPositiveIntegerString = (value: string): boolean => /^[1-9][0-9]*$/.test (value) @@ -338,9 +507,11 @@ const baseRowState = (row: PostImportRow): QueryRowState => ({ duration: String (row.attributes.duration ?? ''), tags: String (row.attributes.tags ?? ''), parent_post_ids: String (row.attributes.parentPostIds ?? ''), + display_tags: row.displayTags ?? [], manual: manualFields (row), skip: row.skipReason ?? null, existing_post_id: row.existingPostId ?? null, + existing_post: row.existingPost ?? null, import_status: row.importStatus ?? null, created_post_id: row.createdPostId ?? null, recoverable: row.recoverable ?? null }) @@ -379,13 +550,36 @@ const buildRow = (state: FullRowState): PostImportRow => { baseWarnings: [], validationErrors: { }, provenance, + displayTags: state.display_tags.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null + ? undefined + : [...tag.sectionLiterals] })), tagSources, status: 'pending', skipReason: state.skip ?? undefined, existingPostId: state.existing_post_id ?? undefined, + existingPost: + state.existing_post == null + ? undefined + : { + id: state.existing_post.id, + title: state.existing_post.title, + url: state.existing_post.url, + thumbnail: state.existing_post.thumbnail, + thumbnailBase: state.existing_post.thumbnailBase }, resetSnapshot: { url: state.url, attributes: { ...attributes }, + displayTags: state.display_tags.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null + ? undefined + : [...tag.sectionLiterals] })), provenance: { ...provenance }, tagSources: { ...tagSources }, fieldWarnings: { }, @@ -429,6 +623,8 @@ const parseEncodedRow = ( const createdPostId = row['c'] const recoverable = row['r'] const sourceRow = row['n'] + const displayTags = sanitiseDisplayTags (row['k']) + const existingPost = sanitiseExistingPost (row['o']) if (requireUrl && typeof url !== 'string') return null @@ -454,6 +650,10 @@ const parseEncodedRow = ( return null if (recoverable != null && typeof recoverable !== 'boolean') return null + if (displayTags == null) + return null + if (row['o'] != null && existingPost == null) + return null if (sourceRow != null && !(typeof sourceRow === 'number' && Number.isSafeInteger (sourceRow) @@ -461,6 +661,8 @@ const parseEncodedRow = ( return null if (skip === 'existing' && existingPostId == null) return null + if (existingPost != null && existingPost.id !== existingPostId) + return null if (importStatus === 'created' && createdPostId == null) return null if (importStatus !== 'created' && createdPostId != null) @@ -512,9 +714,11 @@ const parseEncodedRow = ( duration: parsedStrings.duration, tags: parsedStrings.tags, parent_post_ids: parsedStrings.parent_post_ids, + display_tags: displayTags, manual: manual ?? [], skip, existing_post_id: existingPostId ?? null, + existing_post: existingPost, import_status: importStatus, created_post_id: createdPostId ?? null, recoverable: recoverable ?? null } @@ -543,9 +747,11 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { duration: '', tags: '', parent_post_ids: '', + display_tags: [], manual: [], skip: null, existing_post_id: null, + existing_post: null, import_status: null, created_post_id: null, recoverable: null }) @@ -580,14 +786,22 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { return null const existingPostIdValue = params.get ('existing_post_id') const createdPostIdValue = params.get ('created_post_id') + const displayTags = decodeDisplayTagsParam (params.get ('display_tags')) + const existingPost = decodeExistingPostParam (params.get ('existing_post')) const existingPostId = parsePositiveInt (existingPostIdValue) const createdPostId = parsePositiveInt (createdPostIdValue) + if (displayTags == null) + return null + if (params.get ('existing_post') != null && existingPost == null) + return null if (existingPostIdValue != null && existingPostId == null) return null if (createdPostIdValue != null && createdPostId == null) return null if (skipValue === 'existing' && existingPostId == null) return null + if (existingPost != null && existingPost.id !== existingPostId) + return null if (importStatus === 'created' && createdPostId == null) return null if (importStatus !== 'created' && createdPostIdValue != null) @@ -608,9 +822,11 @@ const parseSingleRow = (params: URLSearchParams): ParsedPostNewState | null => { duration: params.get ('duration') ?? '', tags: params.get ('tags') ?? '', parent_post_ids: params.get ('parent_post_ids') ?? '', + display_tags: displayTags, manual: parseManual (params.get ('manual')), skip: skipValue, existing_post_id: existingPostId, + existing_post: existingPost, import_status: importStatus, created_post_id: createdPostId, recoverable }) @@ -727,6 +943,12 @@ const regularQuery = (rows: PostImportRow[]): URLSearchParams => { params.set ('duration', String (row?.attributes.duration ?? '')) params.set ('tags', String (row?.attributes.tags ?? '')) params.set ('parent_post_ids', String (row?.attributes.parentPostIds ?? '')) + const displayTags = encodeDisplayTagsParam (row?.displayTags) + if (displayTags != null) + params.set ('display_tags', displayTags) + const existingPost = encodeExistingPostParam (row?.existingPost) + if (existingPost != null) + params.set ('existing_post', existingPost) const manual = manualFields (row).join (',') if (manual !== '') params.set ('manual', manual) @@ -776,12 +998,16 @@ const encodeRowState = ( encoded.g = base.tags if (base.parent_post_ids !== '') encoded.p = base.parent_post_ids + if (base.display_tags.length > 0) + encoded.k = encodeDisplayTags (base.display_tags) if (base.manual.length > 0) encoded.m = [...base.manual] if (base.skip != null) encoded.s = encodeSkip (base.skip) if (base.existing_post_id != null) encoded.e = base.existing_post_id + if (base.existing_post != null) + encoded.o = encodeExistingPost (base.existing_post) if (base.import_status != null) encoded.i = encodeImportStatus (base.import_status) if (base.created_post_id != null) diff --git a/frontend/src/pages/posts/PostImportReviewPage.tsx b/frontend/src/pages/posts/PostImportReviewPage.tsx index eea5bd1..50042c5 100644 --- a/frontend/src/pages/posts/PostImportReviewPage.tsx +++ b/frontend/src/pages/posts/PostImportReviewPage.tsx @@ -74,6 +74,7 @@ type PostMetadataResponse = { duration?: string videoMs?: number tags?: string + displayTags?: PostImportRow['displayTags'] fieldWarnings?: Record baseWarnings?: string[] validationErrors?: Record @@ -82,7 +83,8 @@ type PostMetadataResponse = { id: number title: string url: string - thumbnailUrl?: string } | null } + thumbnail?: string | null + thumbnailBase?: string | null } | null } type BulkApiRow = { status: 'created' | 'skipped' | 'failed' @@ -92,7 +94,8 @@ type BulkApiRow = { id: number title: string url: string - thumbnailUrl?: string } | null + thumbnail?: string | null + thumbnailBase?: string | null } | null fieldWarnings?: Record baseWarnings?: string[] errors?: Record @@ -238,6 +241,14 @@ const mergeDryRunRow = ( videoMs: result.videoMs ?? '', tags: result.tags ?? '', parentPostIds: String (result.parentPostIds ?? currentRow.attributes.parentPostIds ?? '') }, + displayTags: + result.displayTags?.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null + ? undefined + : [...tag.sectionLiterals] })) ?? [], fieldWarnings, baseWarnings: result.baseWarnings ?? [], validationErrors: { }, @@ -273,6 +284,14 @@ const buildPreviewResetSnapshot = ( videoMs: preview.videoMs ?? '', tags: preview.tags ?? '', parentPostIds: String (preview.parentPostIds ?? '') }, + displayTags: + preview.displayTags?.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null + ? undefined + : [...tag.sectionLiterals] })) ?? [], provenance: { url: 'manual', title: 'automatic', @@ -368,6 +387,14 @@ const mergePreviewRow = ( ...currentRow, attributes: { ...currentRow.attributes }, provenance: { ...currentRow.provenance }, + displayTags: + currentRow.displayTags?.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null + ? undefined + : [...tag.sectionLiterals] })) ?? [], tagSources: { automatic: currentRow.tagSources?.automatic ?? '', manual: currentRow.tagSources?.manual ?? '' }, @@ -406,6 +433,14 @@ const mergePreviewRow = ( if (currentRow.provenance.tags !== 'manual') { nextRow.attributes.tags = preview.tags ?? '' + nextRow.displayTags = + preview.displayTags?.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null + ? undefined + : [...tag.sectionLiterals] })) ?? [] } nextRow.tagSources.automatic = preview.tags ?? '' if (currentRow.provenance.parentPostIds !== 'manual') @@ -459,7 +494,7 @@ const ExistingSkippedRows: FC = ({ id, rows }) => { rel="noopener noreferrer" className="flex items-center gap-3 p-3">
@@ -809,6 +844,14 @@ const PostImportReviewPage: FC = ({ user }) => { ? { ...currentRow, url: currentRow.resetSnapshot.url, attributes: { ...currentRow.resetSnapshot.attributes }, + displayTags: + currentRow.resetSnapshot.displayTags.map (tag => ({ + name: tag.name, + category: tag.category, + sectionLiterals: + tag.sectionLiterals == null + ? undefined + : [...tag.sectionLiterals] })), provenance: { ...currentRow.resetSnapshot.provenance }, tagSources: { ...currentRow.resetSnapshot.tagSources }, fieldWarnings: Object.fromEntries ( diff --git a/frontend/src/pages/posts/PostImportSourcePage.tsx b/frontend/src/pages/posts/PostImportSourcePage.tsx index 9465e25..53a2b47 100644 --- a/frontend/src/pages/posts/PostImportSourcePage.tsx +++ b/frontend/src/pages/posts/PostImportSourcePage.tsx @@ -76,6 +76,7 @@ const buildInitialRows = (source: string): PostImportRow[] => automatic: '', manual: '' }, status: 'pending' as const, + displayTags: [], resetSnapshot: { url: row.url, attributes: { @@ -100,6 +101,7 @@ const buildInitialRows = (source: string): PostImportRow[] => tagSources: { automatic: '', manual: '' }, + displayTags: [], fieldWarnings: { }, baseWarnings: [] } }))