diff --git a/backend/app/controllers/counts_states_controller.rb b/backend/app/controllers/counts_states_controller.rb
new file mode 100644
index 0000000..1091bf1
--- /dev/null
+++ b/backend/app/controllers/counts_states_controller.rb
@@ -0,0 +1,10 @@
+class CountsStatesController < ApplicationController
+ def index
+ locale = resolve_locale!
+ result = CountsStateQuery.new(request.query_parameters,
+ language_code: locale.language_code).call
+ render json: result
+ rescue ArgumentError, TypeError
+ head :bad_request
+ end
+end
diff --git a/backend/app/services/counts_query.rb b/backend/app/services/counts_query.rb
index 07e1adf..48e9533 100644
--- a/backend/app/services/counts_query.rb
+++ b/backend/app/services/counts_query.rb
@@ -14,11 +14,12 @@ class CountsQuery
attr_reader :page, :limit, :mode, :axis
- def initialize(scope, resource, params)
+ def initialize(scope, resource, params, latest_expression: nil)
@table = scope.klass.table_name
@resource = resource
@id_column = resource == 'posts' ? 'post_id' : 'tag_id'
@params = params
+ @latest_expression = latest_expression
@scope = scope.except(:select, :order).joins(
"LEFT JOIN users counts_actor ON counts_actor.id = #{ @table }.created_by_user_id")
raise ArgumentError unless (params.keys - KEYS).empty?
@@ -43,6 +44,8 @@ class CountsQuery
def count = @scope.count
def latest_expression
+ return @latest_expression if @latest_expression
+
<<~SQL.squish
NOT EXISTS (SELECT 1 FROM #{ @table } newer
WHERE newer.#{ @id_column } = #{ @table }.#{ @id_column }
diff --git a/backend/app/services/counts_state_query.rb b/backend/app/services/counts_state_query.rb
new file mode 100644
index 0000000..1b263bb
--- /dev/null
+++ b/backend/app/services/counts_state_query.rb
@@ -0,0 +1,293 @@
+# State at an exclusive UTC boundary, reconstructed only from recorded snapshots.
+class CountsStateQuery
+ KEYS = ['locale', 'resource', 'mode', 'at', 'from', 'before', 'grain',
+ 'kind', 'tag', 'page', 'limit', 'sort'].freeze
+ MAX_PERIODS = 366
+
+ def initialize(params, language_code:)
+ raise ArgumentError unless (params.keys - KEYS).empty?
+ raise ArgumentError unless params.values.all? { _1.is_a?(String) }
+
+ @params = params
+ @connection = ApplicationRecord.connection
+ @language = language_code
+ @resource = params.fetch('resource', 'posts')
+ @mode = params.fetch('mode', 'state')
+ @kind = params.fetch('kind', 'internal')
+ @grain = params.fetch('grain', 'month')
+ raise ArgumentError unless ['posts', 'tags'].include?(@resource)
+ raise ArgumentError unless ['state', 'trend', 'snapshot'].include?(@mode)
+ raise ArgumentError unless ['internal', 'external', 'all'].include?(@kind)
+ raise ArgumentError unless ['day', 'month', 'year'].include?(@grain)
+
+ @page = integer('page', 1)
+ @limit = integer('limit', 20)
+ @tag = integer('tag') if params['tag'].present?
+ raise ArgumentError if @limit > 100 || @page > 1_000_000
+ raise ArgumentError if @tag && @kind == 'all'
+ raise ArgumentError if @mode == 'trend' && !@tag
+
+ @now = Time.current.utc
+ @at = timestamp('at') || @now.change(usec: 0)
+ raise ArgumentError if @at > @now
+ end
+
+ def call
+ # Keep totals, rows, and detail pages on one repeatable-read database snapshot.
+ ApplicationRecord.transaction do
+ @post_start = PostVersion.minimum(:created_at)
+ @tag_start = TagVersion.minimum(:created_at)
+ result = { page: @page, limit: @limit, at: @at.iso8601(6),
+ coverage: { basis: 'recorded_snapshots', exact: false,
+ post_history_from: @post_start&.iso8601(6),
+ tag_history_from: @tag_start&.iso8601(6) } }
+ case @mode
+ when 'trend'
+ result.merge(trend)
+ when 'snapshot'
+ result.merge(totals: totals).merge(details)
+ else
+ result.merge(totals: totals).merge(listing)
+ end
+ end
+ end
+
+ private
+
+ def quote(value) = @connection.quote(value)
+
+ def integer(key, default = nil)
+ value = @params[key]
+ return default if value.blank? && default
+ raise ArgumentError unless value&.match?(/\A[1-9][0-9]{0,9}\z/)
+
+ value.to_i
+ end
+
+ def timestamp(key)
+ value = @params[key].presence
+ return nil unless value
+ pattern = /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d{1,6})?)?Z\z/
+ raise ArgumentError unless value.match?(pattern)
+
+ Date.iso8601(value.first(10))
+ Time.iso8601(value.length == 17 ? value.sub(/Z\z/, ':00Z') : value).utc
+ end
+
+ # Version numbers order equal-timestamp writes. Do not filter tags before this selection.
+ def latest_condition(table, row, boundary)
+ key = table == 'post_versions' ? 'post_id' : 'tag_id'
+ <<~SQL.squish
+ #{ row }.created_at < #{ boundary } AND NOT EXISTS (
+ SELECT 1 FROM #{ table } newer
+ WHERE newer.#{ key } = #{ row }.#{ key }
+ AND newer.created_at < #{ boundary }
+ AND newer.version_no > #{ row }.version_no)
+ SQL
+ end
+
+ def state_sql(table, boundary, live: false)
+ sql = "SELECT v.* FROM #{ table } v WHERE #{ latest_condition(table, 'v', boundary) }"
+ live ? "#{ sql } AND v.event_type <> 'discard'" : sql
+ end
+
+ def membership(row)
+ key = @kind == 'external' ? 'external_tag_id' : 'tag_id'
+ "JSON_CONTAINS(#{ row }.tags_json, JSON_OBJECT('#{ key }', #{ @tag }))"
+ end
+
+ # Deprecation keeps a tag in existence. Only an explicit discard removes it.
+ def tag_visible(boundary)
+ return '1 = 1' if @kind == 'external'
+
+ <<~SQL.squish
+ NOT EXISTS (SELECT 1 FROM tag_versions t
+ WHERE t.tag_id = #{ @tag } AND t.event_type = 'discard'
+ AND #{ latest_condition('tag_versions', 't', boundary) })
+ SQL
+ end
+
+ def ctes
+ boundary = quote(@at)
+ <<~SQL
+ WITH post_at AS (#{ state_sql('post_versions', boundary, live: true) }),
+ tag_at AS (#{ state_sql('tag_versions', boundary) }),
+ nico_at AS (#{ state_sql('nico_tag_versions', boundary) }),
+ pairs AS (
+ SELECT DISTINCT p.post_id,
+ CASE WHEN entry.tag_id IS NULL THEN 'external' ELSE 'internal' END AS kind,
+ COALESCE(entry.tag_id, entry.external_tag_id) AS tag_id
+ FROM post_at p
+ JOIN JSON_TABLE(CAST(p.tags_json AS JSON), '$[*]' COLUMNS (
+ tag_id BIGINT PATH '$.tag_id' NULL ON EMPTY ERROR ON ERROR,
+ external_tag_id BIGINT PATH '$.external_tag_id' NULL ON EMPTY ERROR ON ERROR
+ )) entry
+ LEFT JOIN tag_at t ON t.tag_id = entry.tag_id
+ WHERE COALESCE(entry.tag_id, entry.external_tag_id) IS NOT NULL
+ AND (t.event_type IS NULL OR t.event_type <> 'discard')
+ )
+ SQL
+ end
+
+ def totals
+ row = @connection.select_one(<<~SQL)
+ #{ ctes }
+ SELECT (SELECT COUNT(*) FROM post_at) AS posts,
+ (SELECT COUNT(*) FROM tag_at WHERE event_type <> 'discard') AS tags,
+ (SELECT COUNT(*) FROM tag_at WHERE event_type <> 'discard'
+ AND deprecated_at IS NOT NULL) AS deprecated_tags,
+ (SELECT COUNT(*) FROM pairs) AS assignments,
+ (SELECT COUNT(*) FROM pairs WHERE kind = 'external') AS external_assignments
+ SQL
+ row.transform_values!(&:to_i)
+ row.merge('posts' => available?(@post_start, @at) ? row['posts'] : nil,
+ 'assignments' => available?(@post_start, @at) ? row['assignments'] : nil,
+ 'external_assignments' => available?(@post_start, @at) ?
+ row['external_assignments'] : nil,
+ 'tags' => available?(@tag_start, @at) ? row['tags'] : nil,
+ 'deprecated_tags' => available?(@tag_start, @at) ? row['deprecated_tags'] : nil)
+ end
+
+ def available?(first, boundary) = first && first < boundary
+
+ def listing
+ path = quote("$.#{ JSON.generate(@language) }.name")
+ catalog = <<~SQL
+ , tag_counts AS (
+ SELECT membership.kind, membership.tag_id, COUNT(*) AS post_count
+ FROM pairs membership GROUP BY membership.kind, membership.tag_id
+ ), catalog AS (
+ SELECT 'internal' AS kind, tag_id,
+ COALESCE(JSON_UNQUOTE(JSON_EXTRACT(names, #{ path })), name) AS name,
+ deprecated_at FROM tag_at WHERE event_type <> 'discard'
+ UNION ALL
+ SELECT tc.kind, tc.tag_id, external_version.name, NULL AS deprecated_at
+ FROM tag_counts tc
+ LEFT JOIN tag_at internal_version
+ ON tc.kind = 'internal' AND internal_version.tag_id = tc.tag_id
+ LEFT JOIN nico_at external_version
+ ON tc.kind = 'external' AND external_version.tag_id = tc.tag_id
+ WHERE tc.kind = 'external' OR internal_version.tag_id IS NULL
+ )
+ SQL
+ filters = []
+ filters << "c.kind = #{ quote(@kind) }" unless @kind == 'all'
+ filters << "c.tag_id = #{ @tag }" if @tag
+ where = filters.any? ? "WHERE #{ filters.join(' AND ') }" : ''
+ source = 'FROM catalog c LEFT JOIN tag_counts n ON n.kind = c.kind AND n.tag_id = c.tag_id'
+ prefix = "#{ ctes } #{ catalog }"
+ total = @connection.select_value("#{ prefix } SELECT COUNT(*) #{ source } #{ where }").to_i
+ order = order_sql({ 'name' => 'c.name', 'count' => 'COALESCE(n.post_count, 0)',
+ 'tag_id' => 'c.tag_id', 'kind' => 'c.kind',
+ 'deprecated' => 'c.deprecated_at' }, 'count:desc',
+ ['c.kind ASC', 'c.tag_id ASC'])
+ rows = @connection.select_all(<<~SQL)
+ #{ prefix }
+ SELECT c.kind, c.tag_id, c.name, c.deprecated_at,
+ COALESCE(n.post_count, 0) AS post_count #{ source } #{ where }
+ ORDER BY #{ order } LIMIT #{ @limit } OFFSET #{ (@page - 1) * @limit }
+ SQL
+ { count: total, group_count: total, state_rows: rows.map do |row|
+ { kind: row['kind'], tag_id: row['tag_id'], name: row['name'],
+ deprecated: row['deprecated_at']&.iso8601,
+ count: available?(@post_start, @at) ? row['post_count'].to_i : nil,
+ at: @at.iso8601(6) }
+ end }
+ end
+
+ def details
+ table = @resource == 'posts' ? 'post_versions' : 'tag_versions'
+ model = @resource == 'posts' ? PostVersion : TagVersion
+ scope = model.joins("INNER JOIN (#{ state_sql(table, quote(@at), live: true) }) " +
+ "counts_at ON counts_at.id = #{ table }.id")
+ if @tag
+ scope = if @resource == 'posts'
+ scope.where("#{ membership(table) } AND #{ tag_visible(quote(@at)) }")
+ elsif @kind == 'internal'
+ scope.where(tag_id: @tag)
+ else
+ scope.none
+ end
+ end
+ params = @params.slice('locale', 'sort', 'page', 'limit')
+ .merge('mode' => 'details', 'latest' => '0')
+ query = CountsQuery.new(scope, @resource, params, latest_expression: '1')
+ { count: query.count,
+ versions: CountsVersionRepr.many(query.details, resource: @resource,
+ language_code: @language) }
+ end
+
+ def next_boundary(time)
+ case @grain
+ when 'day' then time.beginning_of_day + 1.day
+ when 'month' then time.beginning_of_month.next_month
+ else time.beginning_of_year.next_year
+ end
+ end
+
+ def periods
+ @before = timestamp('before') || @now.beginning_of_month
+ @from = timestamp('from') || @before.prev_year
+ raise ArgumentError if @from >= @before || @before > @now
+
+ rows = []
+ boundary = next_boundary(@from)
+ while boundary <= @before
+ raise ArgumentError if rows.length >= MAX_PERIODS
+
+ format = { 'day' => '%Y-%m-%d', 'month' => '%Y-%m', 'year' => '%Y' }.fetch(@grain)
+ rows << [boundary, (boundary - 1.second).strftime(format)]
+ boundary = next_boundary(boundary)
+ end
+ rows
+ end
+
+ def trend
+ points = periods
+ metadata = { count: points.length, group_count: points.length, state_rows: [],
+ from: @from.iso8601, before: @before.iso8601, grain: @grain }
+ return metadata if points.empty?
+
+ values = points.map do |boundary, period|
+ "SELECT CAST(#{ quote(boundary) } AS DATETIME(6)) AS boundary, " +
+ "#{ quote(period) } AS period"
+ end.join(' UNION ALL ')
+ order = order_sql({ 'period' => 'boundary', 'count' => 'post_count' },
+ 'period:asc', ['boundary ASC'])
+ known = @post_start ? "periods.boundary > #{ quote(@post_start) }" : 'FALSE'
+ rows = @connection.select_all(<<~SQL)
+ WITH periods AS (#{ values }), series AS (
+ SELECT periods.boundary, periods.period,
+ CASE WHEN #{ known } THEN COUNT(DISTINCT p.post_id) ELSE NULL END AS post_count
+ FROM periods
+ LEFT JOIN post_versions p ON p.event_type <> 'discard'
+ AND #{ membership('p') }
+ AND #{ latest_condition('post_versions', 'p', 'periods.boundary') }
+ AND #{ tag_visible('periods.boundary') }
+ GROUP BY periods.boundary, periods.period
+ )
+ SELECT * FROM series ORDER BY #{ order }
+ LIMIT #{ @limit } OFFSET #{ (@page - 1) * @limit }
+ SQL
+ metadata.merge(state_rows: rows.map do |row|
+ { kind: @kind, tag_id: @tag, name: nil, period: row['period'],
+ at: row['boundary'].utc.iso8601(6), count: row['post_count']&.to_i }
+ end)
+ end
+
+ def order_sql(expressions, default, final)
+ raw = @params.fetch('sort', default)
+ sorts = raw.empty? ? [] : raw.split(',', -1).map { _1.split(':', -1) }
+ raise ArgumentError if sorts.map(&:first).uniq.length != sorts.length
+ raise ArgumentError if sorts.length > expressions.length
+
+ order = sorts.flat_map do |key, direction, *extra|
+ if extra.any? || !expressions.key?(key) || !['asc', 'desc'].include?(direction)
+ raise ArgumentError
+ end
+ expression = expressions.fetch(key)
+ ["(#{ expression }) IS NULL ASC", "#{ expression } #{ direction.upcase }"]
+ end
+ (order + final).join(', ')
+ end
+end
diff --git a/backend/config/routes.rb b/backend/config/routes.rb
index 030a874..a350568 100644
--- a/backend/config/routes.rb
+++ b/backend/config/routes.rb
@@ -1,4 +1,5 @@
Rails.application.routes.draw do
+ get 'counts/state', to: 'counts_states#index'
get 'counts/posts', to: 'counts_posts#index'
get 'counts/tags', to: 'counts_tags#index'
diff --git a/frontend/src/components/counts/CountsControls.tsx b/frontend/src/components/counts/CountsControls.tsx
index 1139ebc..6a17e09 100644
--- a/frontend/src/components/counts/CountsControls.tsx
+++ b/frontend/src/components/counts/CountsControls.tsx
@@ -1,10 +1,11 @@
import { useEffect, useId, useState } from 'react'
+import CountsStateFilters from '@/components/counts/CountsStateFilters'
import CountsFieldTree from '@/components/counts/CountsFieldTree'
import DateTimeField from '@/components/common/DateTimeField'
import FormField from '@/components/common/FormField'
import { Button } from '@/components/ui/button'
-import { normaliseCountsParams } from '@/lib/counts'
+import { isCountsState, normaliseCountsParams } from '@/lib/counts'
import { fetchTagByName } from '@/lib/tags'
import { inputClass } from '@/lib/utils'
import { useLocale } from '@/lib/useLocale'
@@ -42,7 +43,8 @@ const CountsControls = (
}, [order, nestedJSON])
const [error, setError] = useState (false)
const [pending, setPending] = useState (false)
- const mode = draft.get ('mode') ?? 'summary'
+ const mode = draft.get ('mode') ?? 'state'
+ const state = isCountsState (draft)
const update = (key: string, value: string) => {
setDraft (previous => {
const next = new URLSearchParams (previous)
@@ -58,11 +60,12 @@ const CountsControls = (
try
{
const next = new URLSearchParams (draft)
- for (const key of resource === 'posts' ? ['tag'] : ['resource_id'])
+ for (const key of (state || resource === 'posts') ? ['tag'] : ['resource_id'])
{
const value = next.get (key)?.trim () ?? ''
if (value && !/^[1-9][0-9]*$/.test (value))
{
+ if (state && next.get ('kind') === 'external') throw new Error ()
const tag = await fetchTagByName (value, locale)
if (tag == null)
throw new Error ()
@@ -96,10 +99,10 @@ const CountsControls = (
{error &&
{label ('shared:messages.checkInputModern')}
}
@@ -115,7 +118,9 @@ const CountsControls = (
}}/>
{label ('nested.asJSON')}
- {mode === 'summary' ? (
+ {(mode === 'state' || mode === 'trend') ? (
+ {label (
+ mode === 'trend' ? 'state.trend' : 'state.list')}
) : (mode === 'summary' ? (