import { countsFieldLabel } from '@/lib/countsDefinitions' import { apiDownload, apiGet } from '@/lib/api' import { readCountState, encodeCountState } from '@/lib/queryState' import type { CountsAxis, CountsDefinition } from '@/lib/countsDefinitions' import type { LocaleCode } from '@/lib/locales' import type { TagNameSnapshot } from '@/types' export type CountsResource = 'posts' | 'tags' | 'nico_tags' | 'tag_descriptions' | 'tag_groups' | 'materials' export type CountsName = TagNameSnapshot & { languageCode: string; languageName?: string | null; scriptName?: string | null } export type CountsTag = { id: number kind: 'internal' | 'external' tagId: number | null externalTagId: number | null versionNo: number | null name: string | null category: string | null sections: { beginMs: number; endMs: number | null }[] names: CountsName[] } export type CountsVersion = { [key: string]: unknown eventType?: string id: number resourceId: number versionNo: number isLatest: boolean createdAt: string createdByUser: { id: number; name: string | null } | null title?: string | null url?: string thumbnailBase?: string | null tags?: CountsTag[] parents: { id: number }[] videoMs?: number | null originalCreatedFrom?: string | null originalCreatedBefore?: string | null name?: string | null names?: CountsName[] category?: string aliases?: { languageCode: string; name: string; scriptCode: string }[] deprecated?: string | null } export type CountsStateRow = { kind: 'internal' | 'external' tagId: number name: string | null deprecated?: string | null period?: string at: string count: number | null } export type CountsStateTotals = { posts: number | null tags: number | null assignments: number | null deprecatedTags: number | null externalAssignments: number | null } export type CountsPivotHeader = { values: (string | number | null)[] kind: 'data' | 'subtotal' | 'grand_total' } export type CountsResult = { populationCount?: number excludedCount?: number outputCount?: number | null visibleOutputCount?: number outputCountScope?: 'before_authorisation' | 'unavailable_private_history' populationCountScope?: string rowHeaders?: CountsPivotHeader[] columnHeaders?: CountsPivotHeader[] pivotTotals?: { row: (string | number | null)[]; column: (string | number | null)[] count: number }[] executionAt?: string columnValues?: (string | number | null)[][] cells?: { values: (string | number | null)[]; count: number }[] axisLabels?: Record> rows?: { field: string; grain?: string }[] cols?: { field: string; grain?: string }[] count: number page: number limit: number groupCount?: number groups?: { key: string | number | null; name: string | null; count: number }[] versions?: CountsVersion[] at?: string from?: string before?: string grain?: string totals?: CountsStateTotals stateRows?: CountsStateRow[] coverage?: { basis: string; exact: boolean postHistoryFrom: string | null; tagHistoryFrom: string | null } } export type CountSort = { key: string; direction: 'asc' | 'desc' } export type CountField = { key: string; label: string; children?: CountField[] } export type CountRecord = Record export const countColumns: Record = { event_type: 'counts:eventType', thumbnail: 'counts:columns.thumbnailDisplay', resource_id: 'counts:columns.resourceId', version_no: 'counts:columns.version', is_latest: 'counts:columns.latest', title: 'counts:columns.postTitle', url: 'counts:columns.postURL', thumbnail_base: 'counts:columns.thumbnailURL', tags: 'counts:columns.tags', parents: 'shared:labels.parentPosts', video_ms: 'shared:labels.duration', original_created_from: 'counts:columns.originalFrom', original_created_before: 'counts:columns.originalBefore', name: 'counts:columns.name', names: 'counts:columns.names', category: 'common:categoryLabel', aliases: 'tags:aliases', deprecated: 'counts:columns.deprecated', user: 'history:actor', created_at: 'counts:columns.changedAt' } export type CountColumn = string const aliasFields: CountField[] = [ { key: 'name', label: 'history:name' }, { key: 'scriptCode', label: 'history:nameChanges.script' }] const nameFields: CountField[] = [ { key: 'languageCode', label: 'counts:columns.language' }, { key: 'scriptCode', label: 'history:nameChanges.script' }, { key: 'name', label: 'history:name' }, { key: 'aliases', label: 'tags:aliases', children: aliasFields }, { key: 'autoGenerated', label: 'history:nameChanges.automatic' }] const placementFields = (depth: number): CountField[] => [ { key: 'sectionName', label: 'counts:attributes.section_name' }, { key: 'description', label: 'counts:attributes.description' }, { key: 'tagId', label: 'counts:attributes.tag_id' }, { key: 'tagGroupId', label: 'counts:resources.tag_groups' }, ...(depth > 0 ? [{ key: 'children', label: 'counts:attributes.children', children: placementFields (depth - 1) }] : [])] const nested: Partial> = { placements: placementFields (4), export_paths_json: [ { key: 'profile', label: 'counts:attributes.profile' }, { key: 'path', label: 'counts:attributes.path' }], members: [ { key: 'tagId', label: 'counts:columns.resourceId' }, { key: 'tagGroupId', label: 'counts:resources.tag_groups' }, { key: 'versionNo', label: 'history:version' }], tags: [ { key: 'id', label: 'counts:columns.resourceId' }, { key: 'versionNo', label: 'history:version' }, { key: 'kind', label: 'counts:columns.kind' }, { key: 'name', label: 'history:name' }, { key: 'category', label: 'common:categoryLabel' }, { key: 'sections', label: 'counts:columns.sections' }, { key: 'names', label: 'counts:columns.names', children: nameFields }], names: nameFields, aliases: [{ key: 'languageCode', label: 'counts:columns.language' }, ...aliasFields], parents: [{ key: 'id', label: 'counts:columns.resourceId' }] } export const columnsForCounts = ( resource: CountsResource, definition?: CountsDefinition, ): CountColumn[] => definition?.outputFields?.map (field => field.key) ?? [ ...(resource === 'posts' ? ['thumbnail'] : []), 'resource_id', 'version_no', 'is_latest', ...(resource === 'posts' ? ['title', 'url', 'thumbnail_base', 'tags', 'parents', 'video_ms', 'original_created_from', 'original_created_before'] : (resource === 'tags' ? ['name', 'names', 'category', 'aliases', 'parents', 'deprecated'] : (definition?.fields.filter (field => ![ 'resource_id', 'version_no', 'event_type', 'user', 'created_at'] .includes (field.key)).map (field => field.key) ?? []))), 'event_type', 'user', 'created_at'] export const fieldsForCounts = ( resource: CountsResource, definition?: CountsDefinition, ): CountField[] => definition?.outputFields ?? columnsForCounts (resource, definition).map (key => ({ key, label: (key === 'parents' && resource === 'tags') ? 'tags:parents' : (countColumns[key] ?? countsFieldLabel (key)), children: key === 'parents' && resource === 'tags' ? [...(nested.parents ?? []), { key: 'name', label: 'counts:columns.name' }] : nested[key] })) export const orderedCountFields = ( fields: CountField[], params: URLSearchParams, parent = '', ): CountField[] => { const prefix = parent ? `${parent}.` : '' const saved = (params.get ('order') ?? '').split (' ') const keys = [...new Set ([...saved, ...fields.map (field => `${prefix}${field.key}`)])] const ordered = keys.flatMap (key => { const field = fields.find (field => `${prefix}${field.key}` === key) return field == null ? [] : [field] }) return parent ? ordered : [ ...ordered.filter (field => field.key === 'thumbnail'), ...ordered.filter (field => field.key !== 'thumbnail')] } export const reorderCountFields = ( fields: CountField[], params: URLSearchParams, parent: string, from: string, to: string, ): string => { const ordered = orderedCountFields (fields, params, parent) const first = ordered.findIndex (field => field.key === from) const last = ordered.findIndex (field => field.key === to) if (first < 0 || last < 0) return params.get ('order') ?? '' const prefix = parent ? `${parent}.` : '' const current = ordered.map (field => `${prefix}${field.key}`) const next = [...current] next.splice (last, 0, ...next.splice (first, 1)) return [...(params.get ('order') ?? '').split (' ').filter ( key => key && !current.includes (key)), ...next].join (' ') } export const selectedCountColumns = ( resource: CountsResource, params: URLSearchParams, definition?: CountsDefinition, ): CountColumn[] => { const resolved = normaliseCountsParams (resource, params, definition) const common = ['resource_id', 'version_no', 'event_type', 'user', 'created_at'] const defaults: Record = { posts: ['title', 'url', 'tags', 'parents', 'video_ms'], tags: ['name', 'category', 'aliases', 'parents'], nico_tags: ['name', 'linked_tags'], tag_descriptions: ['language_code', 'body', 'source_language_code'], tag_groups: ['name', 'description', 'members'], materials: ['url', 'source_kind', 'tag_id'] } const selected = resolved.has ('columns') ? resolved.get ('columns')!.split (' ') : [...common, ...defaults[resource]] return orderedCountFields (fieldsForCounts (resource, definition), resolved) .map (field => field.key as CountColumn).filter (key => selected.includes (key)) } export const countChildPaths = (fields: CountField[], prefix = ''): string[] => fields.flatMap (field => { const path = prefix ? `${prefix}.${field.key}` : field.key return [path, ...countChildPaths (field.children ?? [], path)] }) export const selectedCountChildren = (params: URLSearchParams): Set => { const selected = new Set (params.has ('children') ? params.get ('children')!.split (' ') : ['tags.name', 'parents.id', 'aliases.name', 'members.tagId']) const clean = (fields: CountField[], prefix: string) => { for (const field of fields) { const path = `${prefix}.${field.key}` if (field.children == null) continue clean (field.children, path) if (!field.children.some (child => selected.has (`${path}.${child.key}`))) selected.delete (path) } } for (const [path, fields] of Object.entries (nested)) clean (fields ?? [], path) return selected } export const readCountSort = (value: string | null): CountSort[] => (value ?? '').split (' ').flatMap (entry => { const [key, direction] = entry.split (':') return (key && (direction === 'asc' || direction === 'desc')) ? [{ key, direction }] : [] }) export const writeCountSort = (sort: CountSort[]): string => sort.map (item => `${item.key}:${item.direction}`).join (' ') export const changeCountSort = ( sort: CountSort[], key: string, additive: boolean, ): CountSort[] => { const previous = sort.find (item => item.key === key) if (!additive) return [{ key, direction: previous?.direction === 'asc' ? 'desc' : 'asc' }] if (previous == null) return [...sort, { key, direction: 'asc' }] return previous.direction === 'desc' ? sort.filter (item => item.key !== key) : sort.map (item => item.key === key ? { key, direction: 'desc' } : item) } export const selectedCountFields = ( fields: CountField[], params: URLSearchParams, selectionKey: string, parent = '', ): CountField[] => { const selected = params.has (selectionKey) ? new Set (params.get (selectionKey)!.split (' ')) : null return orderedCountFields (fields, params, parent) .filter (field => selected == null || selected.has (field.key)) } export const defaultCountsMode = (definition: string | null): 'summary' | 'details' => definition === 'history' ? 'details' : 'summary' export const countsDefinitionParams = ( definition: 'latest' | 'history', previous: URLSearchParams, ): URLSearchParams => { const next = new URLSearchParams ({ definition, mode: defaultCountsMode (definition) }) for (const key of ['timezone', 'row_numbers']) if (previous.has (key)) next.set (key, previous.get (key)!) return next } export const normaliseCountsParams = ( resource: CountsResource, source: URLSearchParams, definition?: CountsDefinition, ): URLSearchParams => { const params = new URLSearchParams (source) if (!params.has ('definition')) params.set ('definition', params.get ('basis') === 'history' ? 'history' : 'latest') if (params.get ('definition') === 'latest' && !params.has ('current')) params.set ('current', params.get ('at') ? '0' : '1') const legacy = params.get ('mode') if (['state', 'trend', 'snapshot'].includes (legacy ?? '')) params.set ('mode', legacy === 'snapshot' ? 'details' : 'summary') if (!params.has ('mode')) params.set ('mode', defaultCountsMode (params.get ('definition'))) if (!params.has ('rows') && params.has ('axis')) { const field = params.get ('axis') === 'month' ? 'created_at' : params.get ('axis')! params.set ('rows', encodeCountState ('rows', [{ field, ...(field === 'created_at' ? { grain: 'month' } : { }) }])) } if (params.has ('before') && !params.has ('through')) params.set ('through', params.get ('before')!) for (const key of ['basis', 'axis', 'latest', 'grain', 'before', 'summary_columns']) params.delete (key) if (params.has ('columns') && ['posts', 'tags'].includes (resource)) { const selected = new Set (params.get ('columns')!.split (' ')) params.set ('columns', columnsForCounts (resource).filter (key => selected.has (key)) .join (' ')) } if (params.has ('sort')) params.set ('sort', params.get ('sort')!.split (' ') .filter (entry => entry.split (':')[0] !== 'original_created_at').join (' ')) if (params.has ('order')) params.set ('order', params.get ('order')!.split (' ') .filter (key => key !== 'original_created_at').join (' ')) const defaults = definition?.defaults?.find (entry => entry.definition === params.get ('definition') && entry.mode === params.get ('mode')) for (const { key, value } of defaults?.params ?? []) { if (params.has (key)) continue if (key === 'sort') { const expected = defaults?.params.find (entry => entry.key === 'rows')?.value const axes = readCountState (params, 'rows', []) const first = expected == null ? null : readCountState (new URLSearchParams ({ rows: expected }), 'rows', [])[0] if (!Array.isArray (axes) || axes[0]?.field !== first?.field || (axes[0]?.grain ?? 'month') !== (first?.grain ?? 'month')) continue } params.set (key, value) } const children = selectedCountChildren (params) if (params.has ('children')) params.set ('children', [...children].filter (Boolean).join (' ')) if (params.has ('columns')) { const fields = fieldsForCounts (resource, definition) params.set ('columns', params.get ('columns')!.split (' ').filter (key => { const field = fields.find (field => field.key === key) return field != null && (field.children == null || field.children.some (child => children.has (`${key}.${child.key}`))) }).join (' ')) } return params } export const isCountsState = (params: URLSearchParams): boolean => params.get ('definition') !== 'history' export const countsUseLatest = (params: URLSearchParams): boolean => params.get ('current') !== '0' export const countsRequestParams = (params: URLSearchParams): URLSearchParams => { const result = new URLSearchParams (params) for (const key of ['columns', 'summary_columns', 'children', 'child_sort', 'nested_json', 'order', 'basis', 'axis', 'latest', 'grain', 'current', 'row_numbers']) result.delete (key) if (isCountsState (params) && countsUseLatest (params)) result.delete ('at') result.sort () return result } export const fetchCounts = ( resource: CountsResource, params: URLSearchParams, locale: LocaleCode, signal?: AbortSignal, ): Promise => apiGet ('/counts/query', { locale, signal, params: { ...Object.fromEntries (countsRequestParams (params)), resource } }) export const exportCounts = ( resource: CountsResource, params: URLSearchParams, locale: LocaleCode, format: 'json' | 'csv', columns: string[], headers: string[], labels: Record, onError?: () => void, ): Promise => apiDownload ('/counts/export', countsExportPayload ( resource, params, format, columns, headers, labels), locale, onError) export const countsExportPayload = ( resource: CountsResource, params: URLSearchParams, format: 'json' | 'csv', columns: string[], headers: string[], labels: Record, definition?: CountsDefinition, ): Record => { const payload = new URLSearchParams (params) if (!payload.has ('children')) { const selected = selectedCountChildren (params) payload.set ('children', countChildPaths (fieldsForCounts (resource, definition)) .filter (path => selected.has (path)).join (' ')) } payload.delete ('current') payload.delete ('row_numbers') if (isCountsState (params) && countsUseLatest (params)) payload.delete ('at') return { ...Object.fromEntries (payload), resource, format, columns: columns.join (' '), headers: JSON.stringify (headers), labels: JSON.stringify (labels) } } const stableValue = (value: unknown): unknown => { if (Array.isArray (value)) return value.map (stableValue) if (value != null && typeof value === 'object') { return Object.fromEntries (Object.entries (value) .sort (([a], [b]) => a < b ? -1 : (a > b ? 1 : 0)) .map (([key, entry]) => [key, stableValue (entry)])) } return value } const compareValue = (first: unknown, second: unknown): number => { if (Array.isArray (first) && Array.isArray (second) && first.length !== second.length) return first.length - second.length if (typeof first === 'number' && typeof second === 'number') return first - second if (typeof first === 'boolean' && typeof second === 'boolean') return Number (first) - Number (second) const left = typeof first === 'object' ? JSON.stringify (stableValue (first)) : String (first) const right = typeof second === 'object' ? JSON.stringify (stableValue (second)) : String (second) return left < right ? -1 : (left > right ? 1 : 0) } export const sortedCountChildren = ( rows: CountRecord[], path: string, params: URLSearchParams, valueFor: (row: CountRecord, key: string) => unknown = (row, key) => row[key], ): CountRecord[] => { const sort = readCountSort (params.get ('child_sort')) .filter (item => item.key.startsWith (`${path}/`)) return rows.map ((row, index) => ({ row, index })).sort ((first, second) => { for (const item of sort) { const key = item.key.slice (path.length + 1) const left = valueFor (first.row, key) const right = valueFor (second.row, key) if (left == null && right != null) return 1 if (left != null && right == null) return -1 if (left == null && right == null) continue const compared = compareValue (left, right) if (compared !== 0) return item.direction === 'asc' ? compared : -compared } return first.index - second.index }).map (item => item.row) } export const countsCSV = (rows: string[][]): string => '\uFEFF' + rows.map (row => row.map (value => { const safe = /^[=+@\-\t\r]/.test (value) ? `'${value}` : value return `"${safe.replaceAll ('"', '""')}"` }).join (',')).join ('\r\n')