#434

backend/app/controllers/counts_tables_controller.rb

@@ -3,6 +3,8 @@ class CountsTablesController < ApplicationController
     resolve_locale!
     definition = CountsResourceDefinition.new(params[:resource])
     render json: { fields: definition.metadata,
+                   output_fields: definition.output_fields,
+                   filter_fields: definition.filter_metadata,
                    axes: { latest: definition.axes(false), history: definition.axes(true) },
                    grains: CountsAxes::GRAINS, defaults: CountsDefaults.metadata(params[:resource]) }
   rescue ArgumentError

backend/app/models/history_access.rb

@@ -148,4 +148,13 @@ class HistoryAccess < ActiveSupport::CurrentAttributes
 
   # This number deliberately ignores time, identity and content filters to prevent count probes.
   def self.population_count(model) = model.unscoped.count
+
+  # Only the resource-wide denied count is public, never a condition-dependent denied count.
+  def self.population_counts(model)
+    total, excluded = model.unscoped.optimizer_hints('MAX_EXECUTION_TIME(10000)').pick(
+        Arel.sql('COUNT(*)'),
+        Arel.sql("COALESCE(SUM(CASE WHEN #{ sql(model) } THEN 0 ELSE 1 END), 0)"))
+    { population_count: total.to_i, excluded_count: excluded.to_i,
+      excluded_count_scope: 'all_recorded_versions' }
+  end
 end

backend/app/services/counts_condition.rb

@@ -0,0 +1,330 @@
+# Typed predicates over the same recorded fields used by Counts output.
+# Multi-valued comparisons use EXISTS; inequality means no matching element.
+class CountsCondition
+  CHILD_TYPES = { 'id' => :integer, 'versionNo' => :integer, 'tagId' => :integer,
+                  'tagGroupId' => :integer, 'autoGenerated' => :boolean,
+                  'category' => :enum, 'kind' => :enum, 'sections' => :interval }.freeze
+
+  def self.fields(definition)
+    result = definition.metadata.map { _1.merge(labels: [definition.field_label(_1[:key])]) }
+    visit = lambda do |fields, prefix, labels|
+      fields.each do |field|
+        key = [prefix, field[:key]].compact.join('.')
+        path_labels = labels + [field[:label]]
+        type = if prefix
+                 CHILD_TYPES.fetch(field[:key], :string)
+               elsif key == 'is_latest'
+                 :boolean
+               elsif key == 'parents'
+                 :integer
+               elsif key == 'deprecated'
+                 :datetime
+               elsif key == 'thumbnail'
+                 :string
+               else
+                 definition.fields.fetch(key, :string)
+               end
+        type = :string if field[:children] || type == :json
+        type = :integer if ['parents', 'members'].include?(key)
+        physical = result.find { _1[:key] == key }
+        nullable = prefix.present? || (physical ? physical[:nullable] : key != 'is_latest')
+        nullable = false if field[:children]
+        entry = { key:, type:, labels: path_labels, nullable:,
+                  multiple: prefix.present? || field[:children].present?,
+                  collection: field[:children].present? || type == :interval,
+                  operators: CountsResourceDefinition::OPERATORS.fetch(type) }
+        result.reject! { _1[:key] == key }
+        result << entry
+        visit.call(field[:children] || [], key, path_labels)
+      end
+    end
+    visit.call(definition.output_fields, nil, [])
+    # The raw JSON alias has the same membership semantics as the public tag field.
+    if definition.resource == 'posts'
+      result.find { _1[:key] == 'tags_json' }.merge!(type: :string, multiple: true, collection: true,
+          operators: CountsResourceDefinition::OPERATORS.fetch(:string))
+    end
+    result + [{ key: 'user.id', type: :integer, nullable: true,
+                labels: ['history:actor', 'counts:columns.resourceId'],
+                operators: CountsResourceDefinition::OPERATORS.fetch(:integer) },
+              { key: 'user.name', type: :string, nullable: true,
+                labels: ['history:actor', 'history:name'],
+                operators: CountsResourceDefinition::OPERATORS.fetch(:string) }]
+  end
+
+  def initialize(population)
+    @population = population
+    @definition = population.definition
+    @connection = @definition.model.connection
+    @language = Locale.find_by!(code: population.params.fetch('locale', 'ja')).language_code
+  end
+
+  def apply(scope, entry)
+    raise ArgumentError unless entry.is_a?(Hash)
+    raise ArgumentError if (entry.keys - ['field', 'operator', 'value', 'end', 'blank']).any?
+
+    field = @definition.filter_metadata.find { _1[:key] == entry.fetch('field') }
+    raise ArgumentError unless field && field[:operators].include?(entry.fetch('operator'))
+    raise ArgumentError unless [true, false, nil].include?(entry['blank'])
+    values = [entry['value'], entry['end']]
+    raise ArgumentError unless values.all? { _1.nil? || (_1.is_a?(String) && _1.bytesize <= 8192) }
+    if entry['operator'] != 'range' && !values[1].nil? && values[1] != ''
+      raise ArgumentError
+    end
+
+    source, expression = source_for(field[:key])
+    type = field[:type]
+    blank = blank_sql(expression, type)
+    if entry['blank']
+      sql = if source && field[:collection]
+              "NOT EXISTS (SELECT 1 FROM #{ source })"
+            elsif source
+              "NOT EXISTS (SELECT 1 FROM #{ source } WHERE NOT (#{ blank }))"
+            else blank
+            end
+      return scope.where(Arel.sql(sql))
+    end
+    return scope if values.all? { _1.nil? || _1 == '' }
+
+    parsed = values.map { parse(_1, type) }
+    validate_enum!(field[:key], parsed.first) if type == :enum
+    operator = entry['operator']
+    if type == :interval
+      return interval(scope, source, expression, parsed)
+    end
+    clauses = []
+    binds = []
+    if operator == 'range'
+      ['>=', '<='].each_with_index do |comparison, index|
+        next if parsed[index].nil?
+
+        clauses << "#{ expression } #{ comparison } ?"
+        binds << parsed[index]
+      end
+      raise ArgumentError if parsed.all? && parsed.first > parsed.last
+    elsif !parsed.first.nil?
+      comparison = { 'eq' => '=', 'ne' => '=', 'gt' => '>', 'lt' => '<',
+                     'ge' => '>=', 'le' => '<=', 'like' => 'LIKE' }.fetch(operator)
+      clauses << "#{ expression } #{ comparison } ?"
+      binds << parsed.first
+    end
+    return scope if clauses.empty?
+
+    predicate = clauses.join(' AND ')
+    if source
+      predicate = "EXISTS (SELECT 1 FROM #{ source } WHERE #{ predicate })"
+      predicate = "NOT (#{ predicate })" if operator == 'ne'
+    elsif operator == 'ne'
+      predicate = "(NOT (#{ predicate }) OR #{ blank })"
+    end
+    scope.where(predicate, *binds)
+  end
+
+  private
+
+  def parse(value, type)
+    return nil if value.nil? || value == ''
+
+    case type
+    when :integer, :interval
+      raise ArgumentError unless value.match?(/\A-?[0-9]{1,19}\z/)
+
+      number = Integer(value, 10)
+      raise ArgumentError unless number.between?(-9_223_372_036_854_775_808, 9_223_372_036_854_775_807)
+
+      number
+    when :boolean
+      raise ArgumentError unless ['true', 'false'].include?(value)
+
+      value == 'true' ? 1 : 0
+    when :datetime then @population.timestamp(value)
+    else value
+    end
+  end
+
+  def validate_enum!(key, value)
+    allowed = case key.split('.').last
+              when 'event_type' then VersionRecord::DEFAULT_EVENT_TYPE_MAP.values
+              when 'category', 'tag_category' then TagVersion.categories.keys
+              when 'kind' then ['internal', 'external']
+              when 'source_kind' then Material::SOURCE_KINDS
+              end
+    raise ArgumentError if value && allowed && !allowed.include?(value)
+  end
+
+  def blank_sql(expression, type)
+    if [:string, :enum].include?(type)
+      "(#{ expression } IS NULL OR #{ expression } = '')"
+    else
+      "#{ expression } IS NULL"
+    end
+  end
+
+  def scalar(document, key, type = :string)
+    path = @connection.quote("$.#{ key }")
+    value = "JSON_EXTRACT(#{ document }, #{ path })"
+    text = "JSON_UNQUOTE(NULLIF(#{ value }, CAST('null' AS JSON)))"
+    return "CAST(#{ text } AS SIGNED)" if type == :integer
+    return "CASE #{ text } WHEN 'true' THEN 1 WHEN 'false' THEN 0 END" if type == :boolean
+
+    text
+  end
+
+  def json_table(document, path, name)
+    "JSON_TABLE(COALESCE(#{ document }, JSON_ARRAY()), #{ @connection.quote(path) } " +
+        "COLUMNS (item JSON PATH '$')) #{ name }"
+  end
+
+  def names_source(document, alias_name)
+    source = "JSON_TABLE(COALESCE(JSON_KEYS(#{ document }), JSON_ARRAY()), '$[*]' " +
+        "COLUMNS (language_code TEXT PATH '$')) #{ alias_name }"
+    path = "CONCAT('$.', JSON_QUOTE(#{ alias_name }.language_code))"
+    [source, "JSON_EXTRACT(#{ document }, #{ path })", "#{ alias_name }.language_code"]
+  end
+
+  def point
+    @point ||= @population.history ? @population.at :
+        @population.timestamp(@population.params['at']) || @population.at
+  end
+
+  def name_expression(row)
+    language_path = @connection.quote("$.#{ @language.to_json }.name")
+    name = "JSON_UNQUOTE(JSON_EXTRACT(#{ row }.names, #{ language_path }))"
+    "COALESCE(#{ name }, #{ row }.name)"
+  end
+
+  def parent_source
+    column = @definition.resource == 'posts' ? 'parent_post_ids' : 'parent_tag_ids'
+    ids = "TRIM(REGEXP_REPLACE(COALESCE(v.#{ column }, ''), '[[:space:]]+', ' '))"
+    document = "CONCAT('[', REPLACE(#{ ids }, ' ', ','), ']')"
+    source = json_table(document, '$[*]', 'parent_entry')
+    id = 'CAST(parent_entry.item AS UNSIGNED)'
+    return [source, id] if @definition.resource == 'posts'
+
+    bound = @connection.quote(point)
+    source += ' LEFT JOIN tag_versions parent_version ON parent_version.tag_id = ' + id +
+        " AND parent_version.created_at <= #{ bound } AND " +
+        HistoryAccess.sql(TagVersion, 'parent_version') +
+        ' AND NOT EXISTS (SELECT 1 FROM tag_versions newer_parent ' +
+        'WHERE newer_parent.tag_id = parent_version.tag_id ' +
+        "AND newer_parent.created_at <= #{ bound } " +
+        'AND newer_parent.version_no > parent_version.version_no)'
+    [source, id]
+  end
+
+  def post_tag_source
+    source = json_table('v.tags_json', '$[*]', 'tag_snapshot')
+    source += ' LEFT JOIN tag_versions referenced_tag ON referenced_tag.tag_id = ' +
+        scalar('tag_snapshot.item', 'tag_id', :integer) +
+        ' AND referenced_tag.version_no = ' + scalar('tag_snapshot.item', 'version_no', :integer) +
+        " AND referenced_tag.created_at <= #{ @connection.quote(point) } AND " +
+        HistoryAccess.sql(TagVersion, 'referenced_tag')
+    source
+  end
+
+  def language_field(source, document, parts, inherited_language = nil)
+    return [source, scalar(document, 'name')] if parts.empty?
+    return [source, inherited_language] if parts == ['languageCode'] && inherited_language
+
+    if parts.first == 'aliases'
+      aliases = "JSON_EXTRACT(#{ document }, '$.aliases')"
+      source += ' JOIN ' + json_table(aliases, '$[*]', 'alias_entry')
+      child = parts[1] || 'name'
+      return [source, inherited_language] if child == 'languageCode' && inherited_language
+      columns = { 'scriptCode' => 'script_code', 'name' => 'name' }
+      key = columns.fetch(child) { raise ArgumentError }
+      return [source, scalar('alias_entry.item', key)]
+    end
+    key = { 'name' => 'name', 'scriptCode' => 'script_code',
+            'autoGenerated' => 'auto_generated' }.fetch(parts.first) { raise ArgumentError }
+    [source, scalar(document, key, CHILD_TYPES.fetch(parts.first, :string))]
+  end
+
+  def source_for(key)
+    root, *parts = key.split('.')
+    return [nil, "(#{ @population.latest_sql })"] if root == 'is_latest'
+    if root == 'user' && parts.any?
+      expression = parts.first == 'id' ? 'v.created_by_user_id' :
+          '(SELECT actor.name FROM users actor WHERE actor.id = v.created_by_user_id)'
+      return [nil, expression]
+    end
+    return [nil, 'v.thumbnail_base'] if root == 'thumbnail'
+    return [nil, 'v.deprecated_at'] if root == 'deprecated'
+    if root == 'parents'
+      source, id = parent_source
+      return [source, parts.first == 'name' ? name_expression('parent_version') : id]
+    end
+    if root == 'tags' || root == 'tags_json'
+      return post_field(parts)
+    end
+    if root == 'names' || root == 'aliases'
+      source, document, language = names_source('v.names', 'name_language')
+      parts = ['aliases', *parts] if root == 'aliases'
+      return language_field(source, document, parts, language)
+    end
+    if root == 'members' || root == 'placements'
+      path = '$[*]' + parts.take_while { _1 == 'children' }.map { '.children[*]' }.join
+      source = json_table("v.#{ root }", path, 'structure_entry')
+      if root == 'members' && parts.empty?
+        value = "COALESCE(#{ scalar('structure_entry.item', 'tag_id', :integer) }, " +
+            "#{ scalar('structure_entry.item', 'tag_group_id', :integer) })"
+        return [source, value]
+      end
+      leaf = parts.last || 'sectionName'
+      leaf = 'sectionName' if leaf == 'children'
+      column = leaf.gsub(/[A-Z]/) { "_#{ _1.downcase }" }
+      return [source, scalar('structure_entry.item', column, CHILD_TYPES.fetch(leaf, :string))]
+    end
+    if root == 'export_paths_json'
+      source, value, profile = names_source('v.export_paths_json', 'export_profile')
+      return [source, parts.first == 'profile' ? profile : "JSON_UNQUOTE(#{ value })"]
+    end
+    return [nil, name_expression('v')] if root == 'name' && @definition.resource == 'tags'
+
+    [nil, @definition.expression(root)]
+  end
+
+  def post_field(parts)
+    source = post_tag_source
+    document = 'tag_snapshot.item'
+    case parts.first
+    when 'names'
+      names, entry, language = names_source('referenced_tag.names', 'tag_language')
+      language_field("#{ source } JOIN #{ names }", entry, parts.drop(1), language)
+    when 'sections'
+      sections = "JSON_EXTRACT(#{ document }, '$.sections')"
+      source += ' JOIN ' + json_table(sections, '$[*]', 'section_entry')
+      [source, 'section_entry.item']
+    when 'id'
+      [source, "COALESCE(#{ scalar(document, 'tag_id', :integer) }, " +
+          "#{ scalar(document, 'external_tag_id', :integer) })"]
+    when 'versionNo' then [source, scalar(document, 'version_no', :integer)]
+    when 'kind'
+      [source, "CASE WHEN #{ scalar(document, 'tag_id') } IS NULL " +
+          "THEN 'external' ELSE 'internal' END"]
+    when 'category'
+      [source, "COALESCE(#{ scalar(document, 'category') }, referenced_tag.category)"]
+    else
+      [source, "COALESCE(#{ scalar(document, 'name') }, referenced_tag.name)"]
+    end
+  end
+
+  def interval(scope, source, document, bounds)
+    raise ArgumentError if bounds.compact.any?(&:negative?)
+    raise ArgumentError if bounds.all? && bounds.first > bounds.last
+
+    first = scalar(document, 'begin_ms', :integer)
+    last = scalar(document, 'end_ms', :integer)
+    clauses = []
+    values = []
+    if bounds.first
+      clauses << "(#{ last } IS NULL OR #{ last } > ?)"
+      values << bounds.first
+    end
+    if bounds.last
+      clauses << "#{ first } < ?"
+      values << bounds.last
+    end
+    scope.where("EXISTS (SELECT 1 FROM #{ source } WHERE #{ clauses.join(' AND ') })", *values)
+  end
+end

backend/app/services/counts_defaults.rb

@@ -11,22 +11,32 @@ class CountsDefaults
         'sort' => 'a0:desc', 'conditions' => '' } }]
     result << { definition: 'latest', mode: 'summary', params: {
         'rows' => '', 'cols' => '', 'conditions' => '', 'tags' => '' } }
-    if resource == 'posts'
-      result += [{ definition: 'latest', mode: 'details', params: {
-              'columns' => POST_COLUMNS.join(' '), 'order' => POST_COLUMNS.join(' '),
-              'children' => 'tags.name parents.id', 'nested_json' => '0',
-              'conditions' => '', 'tags' => '' } }]
-    end
-    if resource == 'tags'
-      columns = ['resource_id', 'version_no', 'is_latest', 'name', 'category',
-                 'aliases', 'parents', 'deprecated', 'event_type', 'user', 'created_at']
-      result << { definition: 'latest', mode: 'details', params: {
-          'columns' => columns.join(' '), 'order' => columns.join(' '),
-          'children' => 'aliases.name parents.name', 'nested_json' => '0' } }
+    details = detail_params(resource)
+    if details
+      ['latest', 'history'].each do |definition|
+        result << { definition:, mode: 'details', params: details }
+      end
     end
     result
   end
 
+  def self.detail_params(resource)
+    if resource == 'posts'
+      return { 'columns' => POST_COLUMNS.join(' '), 'order' => POST_COLUMNS.join(' '),
+               'children' => 'tags.name parents.id', 'nested_json' => '0',
+               'conditions' => '', 'tags' => '' }
+    end
+    unless resource == 'tags'
+      return { 'columns' => CountsExportProjection.default_columns(resource).join(' '),
+               'children' => CountsExportProjection.default_children(resource).join(' ') }
+    end
+
+    columns = ['resource_id', 'version_no', 'is_latest', 'name', 'category',
+               'aliases', 'parents', 'deprecated', 'event_type', 'user', 'created_at']
+    { 'columns' => columns.join(' '), 'order' => columns.join(' '),
+      'children' => 'aliases.name parents.name', 'nested_json' => '0' }
+  end
+
   def self.for(params)
     settings = entries(params.fetch('resource', 'posts')).find do |entry|
       entry[:definition] == params.fetch('definition', 'latest') &&

backend/app/services/counts_population.rb

@@ -125,50 +125,8 @@ class CountsPopulation
   end
 
   def apply_condition(entry)
-    raise ArgumentError unless entry.is_a?(Hash)
-    raise ArgumentError if (entry.keys - ['field', 'operator', 'value', 'end', 'blank']).any?
-
-    key = entry.fetch('field')
-    type = definition.fields.fetch(key) { raise ArgumentError }
-    operator = entry.fetch('operator')
-    raise ArgumentError unless CountsResourceDefinition::OPERATORS.fetch(type, []).include?(operator)
-    raise ArgumentError unless [true, false, nil].include?(entry['blank'])
-
-    expression = definition.expression(key)
-    blank = type == :string || type == :enum ?
-        "(#{ expression } IS NULL OR #{ expression } = '')" : "#{ expression } IS NULL"
-    if entry['blank']
-      @scope = @scope.where(Arel.sql(blank))
-      return
-    end
-    values = [entry['value'], entry['end']]
-    raise ArgumentError unless values.all? { _1.nil? || _1.is_a?(String) }
-    return if values.all? { _1.nil? || _1 == '' }
-
-    parsed = values.map do |value|
-      next nil if value.nil? || value == ''
-
-      case type
-      when :integer
-        raise ArgumentError unless value.match?(/\A-?\d+\z/)
-
-        value.to_i
-      when :datetime then timestamp(value)
-      else value
-      end
-    end
-    if operator == 'range'
-      @scope = @scope.where("#{ expression } >= ?", parsed[0]) if parsed[0]
-      @scope = @scope.where("#{ expression } <= ?", parsed[1]) if parsed[1]
-      return
-    end
-    return unless parsed[0]
-
-    sql_operator = { 'eq' => '=', 'ne' => '<>', 'gt' => '>', 'lt' => '<',
-                     'ge' => '>=', 'le' => '<=', 'like' => 'LIKE' }.fetch(operator)
-    sql = "#{ expression } #{ sql_operator } ?"
-    sql = "(#{ sql } OR #{ blank })" if operator == 'ne'
-    @scope = @scope.where(sql, parsed[0])
+    @condition ||= CountsCondition.new(self)
+    @scope = @condition.apply(@scope, entry)
   end
 
   def snapshot_membership(value)

backend/app/services/counts_resource_definition.rb

@@ -31,7 +31,35 @@ class CountsResourceDefinition
   OPERATORS = {
       string: ['eq', 'ne', 'like'], enum: ['eq', 'ne'], boolean: ['eq', 'ne'],
       integer: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'],
-      datetime: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'] }.freeze
+      datetime: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'],
+      interval: ['range'] }.freeze
+  LABELS = {
+      'resource_id' => 'counts:columns.resourceId', 'version_no' => 'counts:columns.version',
+      'is_latest' => 'counts:columns.latest', 'event_type' => 'counts:eventType',
+      'user' => 'history:actor', 'created_at' => 'counts:columns.changedAt',
+      'thumbnail' => 'counts:columns.thumbnailDisplay',
+      'thumbnail_base' => 'counts:columns.thumbnailURL',
+      'title' => 'counts:columns.postTitle', 'url' => 'counts:columns.postURL',
+      'tags' => 'counts:columns.tags', 'tags_json' => 'counts:columns.tags',
+      'video_ms' => 'shared:labels.duration', 'name' => 'counts:columns.name',
+      'names' => 'counts:columns.names', 'category' => 'common:categoryLabel',
+      'aliases' => 'tags:aliases', 'deprecated' => 'counts:columns.deprecated',
+      'original_created_from' => 'counts:columns.originalFrom',
+      'original_created_before' => 'counts:columns.originalBefore',
+      'language_code' => 'counts:columns.language' }.freeze
+  CHILD_LABELS = {
+      'id' => 'counts:columns.resourceId', 'versionNo' => 'history:version',
+      'kind' => 'counts:columns.kind', 'name' => 'history:name',
+      'category' => 'common:categoryLabel', 'sections' => 'counts:columns.sections',
+      'names' => 'counts:columns.names', 'aliases' => 'tags:aliases',
+      'languageCode' => 'counts:columns.language',
+      'scriptCode' => 'history:nameChanges.script',
+      'autoGenerated' => 'history:nameChanges.automatic',
+      'tagId' => 'counts:attributes.tag_id', 'tagGroupId' => 'counts:resources.tag_groups',
+      'sectionName' => 'counts:attributes.section_name',
+      'description' => 'counts:attributes.description',
+      'children' => 'counts:attributes.children', 'profile' => 'counts:attributes.profile',
+      'path' => 'counts:attributes.path' }.freeze
 
   attr_reader :model, :keys, :fields, :resource
 
@@ -62,4 +90,57 @@ class CountsResourceDefinition
         operators: OPERATORS.fetch(type, []) }
     end
   end
+
+  def field_label(key) = LABELS.fetch(key, "counts:attributes.#{ key }")
+
+  def output_fields
+    content = case resource
+              when 'posts'
+                ['thumbnail', 'title', 'url', 'thumbnail_base', 'tags', 'parents', 'video_ms',
+                 'original_created_from', 'original_created_before']
+              when 'tags' then ['name', 'names', 'category', 'aliases', 'parents', 'deprecated']
+              else fields.keys - COMMON.keys
+              end
+    keys = ['resource_id', 'version_no', 'is_latest'] + (content - ['thumbnail']) +
+        ['event_type', 'user', 'created_at']
+    keys.unshift('thumbnail') if content.include?('thumbnail')
+    keys.map do |key|
+      label = if key == 'parents'
+                resource == 'tags' ? 'tags:parents' : 'shared:labels.parentPosts'
+              else field_label(key)
+              end
+      field = { key:, label: }
+      children = child_fields(key)
+      field[:children] = children if children.any?
+      field
+    end
+  end
+
+  def child_fields(key, depth = 4)
+    keys = case key
+           when 'tags' then ['id', 'versionNo', 'kind', 'name', 'category', 'sections', 'names']
+           when 'names' then ['languageCode', 'scriptCode', 'name', 'aliases', 'autoGenerated']
+           when 'aliases'
+             resource == 'tags' ? ['languageCode', 'name', 'scriptCode'] : ['name', 'scriptCode']
+           when 'parents' then resource == 'tags' ? ['id', 'name'] : ['id']
+           when 'members' then ['tagId', 'tagGroupId', 'versionNo']
+           when 'export_paths_json' then ['profile', 'path']
+           when 'placements', 'children'
+             scalar = ['sectionName', 'description', 'tagId', 'tagGroupId']
+             scalar + (depth > 0 ? ['children'] : [])
+           else []
+           end
+    keys.map do |child|
+      label = key == 'members' && child == 'tagId' ?
+          'counts:columns.resourceId' : CHILD_LABELS.fetch(child)
+      field = { key: child, label: }
+      children = child_fields(child, child == 'children' ? depth - 1 : depth)
+      # Language is attached to top-level aliases, not to aliases inside a language entry.
+      children = children.reject { _1[:key] == 'languageCode' } if child == 'aliases'
+      field[:children] = children if children.any?
+      field
+    end
+  end
+
+  def filter_metadata = @filter_metadata ||= CountsCondition.fields(self)
 end

backend/app/services/counts_table_export.rb

@@ -67,7 +67,9 @@ class CountsTableExport
           unless started
             metadata = page.except(:versions, :cells, :axis_labels, :column_values,
                 :row_headers, :column_headers, :pivot_totals).merge(
-                resource:, filters: @params.select { |key, _value| CountsTableQuery::KEYS.include?(key) ||
+                resource:, record_unit: @summary && format == 'json' ? 'pivot_cell' : 'row',
+                body_row_totals: 'once_per_logical_group', page_grand_total_repeated: false,
+                filters: @params.select { |key, _value| CountsTableQuery::KEYS.include?(key) ||
         key == 'measures' || QueryState.valid_key?(key) },
                 fields: @query.population.definition.metadata,
                 columns: @columns, column_labels: @headers,

backend/app/services/counts_table_query.rb

@@ -41,17 +41,29 @@ class CountsTableQuery
 
   def call
     ApplicationRecord.transaction do
-      metadata.merge(mode == 'summary' ? summary : details)
+      result = metadata.merge(mode == 'summary' ? summary : details)
+      visible = mode == 'summary' ? result[:visible_output_count] : result[:count]
+      # A scalar result has one row regardless of private attributes or predicates.
+      public_count = result[:excluded_count].zero? || (mode == 'summary' && axes.rows.empty?)
+      result.merge(visible_output_count: visible,
+          output_count: public_count ? visible : nil,
+          output_count_scope: public_count ?
+              'before_authorisation' : 'unavailable_private_history',
+          output_count_unit: 'table_body_rows',
+          page_grand_total_repeated: mode == 'summary' && axes.rows.any? &&
+              result[:group_count] > limit)
     end
   end
 
   def metadata
-    { population_count_scope: 'all_recorded_versions', population_count: HistoryAccess.population_count(@definition.model),
-      page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence,
-      definition: population.history ? 'history' : 'latest',
-      timezone: population.timezone.identifier, rows: axes.rows, cols: axes.columns,
-      fields: @definition.metadata,
-      count: population.scope.except(:select).optimizer_hints('MAX_EXECUTION_TIME(10000)').count }
+    @population_counts ||= HistoryAccess.population_counts(@definition.model)
+    @population_counts.merge(
+        population_count_scope: 'all_recorded_versions',
+        page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence,
+        definition: population.history ? 'history' : 'latest',
+        timezone: population.timezone.identifier, rows: axes.rows, cols: axes.columns,
+        fields: @definition.metadata,
+        count: population.scope.except(:select).optimizer_hints('MAX_EXECUTION_TIME(10000)').count)
   end
 
   def summary
@@ -143,7 +155,9 @@ class CountsTableQuery
       column_headers = [{ values: [], kind: 'grand_total' }]
     end
     totals = CountsPivotTotals.new(population, axes, row_headers, column_headers).call
-    { group_count: total, column_values:, row_headers:, column_headers:, pivot_totals: totals,
+    { group_count: total,
+      visible_output_count: output_row_count(prefix, row_keys, row_depths, total),
+      column_values:, row_headers:, column_headers:, pivot_totals: totals,
       cells: records.map do |row|
         { values: axes.all.each_index.map { row["a#{ _1 }"] }, count: row['count'].to_i }
       end, axis_labels: axis_labels(records, column_values) }
@@ -192,6 +206,21 @@ class CountsTableQuery
 
   private
 
+  def output_row_count(prefix, row_keys, depths, total)
+    # With no row axes the table has exactly one body row, including the zero-count case.
+    return 1 if row_keys.empty?
+
+    subtotals = depths.map do |depth|
+      keys = row_keys.first(depth).join(', ')
+      "(SELECT COUNT(*) FROM (SELECT #{ keys } FROM cells GROUP BY #{ keys }) subtotal_rows)"
+    end
+    expressions = [total.to_s, '1'] + subtotals
+    @output_row_count ||= @connection.select_value(<<~SQL).to_i
+      #{ prefix }
+      SELECT /*+ MAX_EXECUTION_TIME(10000) */ #{ expressions.join(' + ') } AS output_rows
+    SQL
+  end
+
   def locale = @locale ||= Locale.find_by!(code: @params.fetch('locale', 'ja'))
 
   def limited_columns(prefix)