#434

backend/app/models/history_access.rb

@@ -149,7 +149,7 @@ class HistoryAccess < ActiveSupport::CurrentAttributes
   # This number deliberately ignores time, identity and content filters to prevent count probes.
   def self.population_count(model) = model.unscoped.count
 
-  # Only the resource-wide denied count is public, never a condition-dependent denied count.
+  # Resource-wide totals provide a fast path; Counts computes filtered totals separately.
   def self.population_counts(model)
     total, excluded = model.unscoped.optimizer_hints('MAX_EXECUTION_TIME(10000)').pick(
         Arel.sql('COUNT(*)'),

backend/app/representations/counts_version_repr.rb

@@ -23,7 +23,7 @@ class CountsVersionRepr
     @rows.map do |row|
       resource_id = row.public_send(@resource == 'posts' ? :post_id : :tag_id)
       common = { id: row.id, resource_id:,
-                 version_no: row.version_no, is_latest: row.attributes['counts_latest'].to_i == 1,
+                 version_no: row.version_no, latest: row.attributes['counts_latest'].to_i == 1,
                  created_at: row.created_at.iso8601,
                  created_by_user: row.created_by_user_id &&
                      { id: row.created_by_user_id, name: @users[row.created_by_user_id] } }

backend/app/services/counts_condition.rb

@@ -13,7 +13,7 @@ class CountsCondition
         path_labels = labels + [field[:label]]
         type = if prefix
                  CHILD_TYPES.fetch(field[:key], :string)
-               elsif key == 'is_latest'
+               elsif key == 'latest'
                  :boolean
                elsif key == 'parents'
                  :integer
@@ -27,7 +27,7 @@ class CountsCondition
         type = :string if field[:children] || type == :json
         type = :integer if ['parents', 'members'].include?(key)
         physical = result.find { _1[:key] == key }
-        nullable = prefix.present? || (physical ? physical[:nullable] : key != 'is_latest')
+        nullable = prefix.present? || (physical ? physical[:nullable] : key != 'latest')
         nullable = false if field[:children]
         entry = { key:, type:, labels: path_labels, nullable:,
                   multiple: prefix.present? || field[:children].present?,
@@ -204,7 +204,7 @@ class CountsCondition
     bound = @connection.quote(point)
     source += ' LEFT JOIN tag_versions parent_version ON parent_version.tag_id = ' + id +
         " AND parent_version.created_at <= #{ bound } AND " +
-        HistoryAccess.sql(TagVersion, 'parent_version') +
+        @population.access_sql(TagVersion, 'parent_version') +
         ' AND NOT EXISTS (SELECT 1 FROM tag_versions newer_parent ' +
         'WHERE newer_parent.tag_id = parent_version.tag_id ' +
         "AND newer_parent.created_at <= #{ bound } " +
@@ -218,7 +218,7 @@ class CountsCondition
         scalar('tag_snapshot.item', 'tag_id', :integer) +
         ' AND referenced_tag.version_no = ' + scalar('tag_snapshot.item', 'version_no', :integer) +
         " AND referenced_tag.created_at <= #{ @connection.quote(point) } AND " +
-        HistoryAccess.sql(TagVersion, 'referenced_tag')
+        @population.access_sql(TagVersion, 'referenced_tag')
     source
   end
 
@@ -242,7 +242,7 @@ class CountsCondition
 
   def source_for(key)
     root, *parts = key.split('.')
-    return [nil, "(#{ @population.latest_sql })"] if root == 'is_latest'
+    return [nil, "(#{ @population.latest_sql })"] if root == 'latest'
     if root == 'user' && parts.any?
       expression = parts.first == 'id' ? 'v.created_by_user_id' :
           '(SELECT actor.name FROM users actor WHERE actor.id = v.created_by_user_id)'

backend/app/services/counts_defaults.rb

@@ -1,6 +1,6 @@
 # Only confirmed combinations belong here; all others retain the existing behaviour.
 class CountsDefaults
-  POST_COLUMNS = ['resource_id', 'version_no', 'is_latest', 'title', 'url',
+  POST_COLUMNS = ['resource_id', 'version_no', 'latest', 'title', 'url',
                   'thumbnail_base', 'tags', 'parents', 'original_created_from',
                   'original_created_before', 'event_type', 'user', 'created_at'].freeze
 
@@ -31,7 +31,7 @@ class CountsDefaults
                'children' => CountsExportProjection.default_children(resource).join(' ') }
     end
 
-    columns = ['resource_id', 'version_no', 'is_latest', 'name', 'category',
+    columns = ['resource_id', 'version_no', 'latest', 'name', 'category',
                'aliases', 'parents', 'deprecated', 'event_type', 'user', 'created_at']
     { 'columns' => columns.join(' '), 'order' => columns.join(' '),
       'children' => 'aliases.name parents.name', 'nested_json' => '0' }

backend/app/services/counts_export_projection.rb

@@ -3,7 +3,7 @@ require 'set'
 # Export identifiers are stable; UI translations are supplied separately as column labels.
 class CountsExportProjection
   ROOT_KEYS = { 'event_type' => 'event_type', 'resource_id' => 'resource_id', 'version_no' => 'version_no',
-                'is_latest' => 'is_latest', 'thumbnail' => 'thumbnail_url',
+                'latest' => 'latest', 'thumbnail' => 'thumbnail_url',
                 'thumbnail_base' => 'thumbnail_base', 'title' => 'title', 'url' => 'url',
                 'tags' => 'tags', 'parents' => 'parents', 'video_ms' => 'video_ms',
                 'original_created_from' => 'original_created_from',
@@ -42,6 +42,7 @@ class CountsExportProjection
   attr_reader :columns, :headers
 
   def initialize(params, resource:, summary:, tags:)
+    params = CountsResourceDefinition.normalise_params(params)
     if params.key?('definition')
       params = CountsDefaults.for(params.merge('resource' => resource)).merge(params)
     end
@@ -144,7 +145,7 @@ class CountsExportProjection
   def csv_display(value, key)
     return @labels.fetch('bot', '') if key == 'created_by_user' && value.nil?
     return @labels.fetch("event_type:#{ value }", value) if key == 'event_type'
-    return value ? @labels.fetch('latest', 'true') : '' if key == 'is_latest'
+    return value ? @labels.fetch('latest', 'true') : '' if key == 'latest'
     return @labels.fetch("kind:#{ value }", value) if key == 'kind'
     return @labels.fetch("category:#{ value }", value) if key == 'category'
     if ['language', 'script'].include?(key) && value.is_a?(Hash)

backend/app/services/counts_exporter.rb

@@ -7,6 +7,7 @@ class CountsExporter
   attr_reader :format, :resource
 
   def initialize(params, language_code:)
+    params = CountsResourceDefinition.normalise_params(params)
     if params.key?('definition')
       @table_export = CountsTableExport.new(params)
       @format = @table_export.format
@@ -66,7 +67,7 @@ class CountsExporter
                      child_sort: sort_metadata(@params.fetch('child_sort', '')),
                      columns: @projection.metadata,
                      children: @params['children']&.split(' '),
-                     field_order: @QueryState.list(params.fetch('order', '')), timezone: 'UTC',
+                     field_order: QueryState.list(@params.fetch('order', '')), timezone: 'UTC',
                      count: @mode == 'summary' ? page[:group_count] : page[:count],
                      totals: page[:totals], coverage: page[:coverage] }
             output << (format == 'json' ? "{\"metadata\":#{ meta.to_json },\"data\":[" : "\uFEFF")

backend/app/services/counts_population.rb

@@ -2,7 +2,8 @@
 class CountsPopulation
   attr_reader :definition, :params, :at, :history, :scope, :timezone
 
-  def initialize(params)
+  def initialize(params, authorisation_flg: true)
+    @authorisation_flg = authorisation_flg
     @params = params
     @definition = CountsResourceDefinition.new(params.fetch('resource', 'posts'))
     @history = params.fetch('definition', 'latest') == 'history'
@@ -14,7 +15,7 @@ class CountsPopulation
 
     @scope = definition.model.all
     make_population
-    @scope = HistoryAccess.visible(@scope, 'v')
+    @scope = HistoryAccess.visible(@scope, 'v') if @authorisation_flg
     apply_filters
   rescue TZInfo::InvalidTimezoneIdentifier
     raise ArgumentError
@@ -56,6 +57,11 @@ class CountsPopulation
     "(previous.id IS NOT NULL AND NOT (v.#{ column } <=> previous.#{ column }))"
   end
 
+  # Unrestricted predicates are used only for numerical totals, never record output.
+  def access_sql(model, row)
+    @authorisation_flg ? HistoryAccess.sql(model, row) : 'TRUE'
+  end
+
   private
 
   def make_population
@@ -64,7 +70,7 @@ class CountsPopulation
     keys = definition.keys.map { "previous.#{ _1 } = v.#{ _1 }" }.join(' AND ')
     @scope = @scope.joins("LEFT JOIN #{ table } previous ON #{ keys } " +
         'AND previous.version_no = v.version_no - 1 AND ' +
-        HistoryAccess.sql(definition.model, 'previous'))
+        access_sql(definition.model, 'previous'))
     @scope = @scope.where("v.#{ definition.time_column } <= ?", at)
     unless history
       point = timestamp(params['at']) || at

backend/app/services/counts_query.rb

@@ -1,7 +1,7 @@
 # Read-only history queries. SQL identifiers and expressions are server-owned.
 class CountsQuery
   AXES = ['month', 'user'].freeze
-  COMMON_COLUMNS = ['resource_id', 'version_no', 'is_latest', 'event_type', 'user', 'created_at'].freeze
+  COMMON_COLUMNS = ['resource_id', 'version_no', 'latest', 'event_type', 'user', 'created_at'].freeze
   COLUMNS = {
       'posts' => COMMON_COLUMNS + [
           'thumbnail', 'thumbnail_base', 'title', 'url', 'tags', 'parents', 'video_ms',
@@ -15,6 +15,7 @@ class CountsQuery
   attr_reader :page, :limit, :mode, :axis
 
   def initialize(scope, resource, params, latest_expression: nil)
+    params = CountsResourceDefinition.normalise_params(params)
     @table = scope.klass.table_name
     @resource = resource
     @id_column = resource == 'posts' ? 'post_id' : 'tag_id'
@@ -117,7 +118,7 @@ class CountsQuery
   def detail_expressions
     common = { 'resource_id' => ["#{ @table }.#{ @id_column }"],
                'version_no' => ["#{ @table }.version_no"],
-               'is_latest' => ["(#{ latest_expression })"],
+               'latest' => ["(#{ latest_expression })"],
                'user' => ['counts_actor.name', "#{ @table }.created_by_user_id"],
                'created_at' => ["#{ @table }.created_at"] }
     if @resource == 'posts'

backend/app/services/counts_resource_definition.rb

@@ -1,5 +1,22 @@
 # Server-owned version identities, attributes and field types. No client SQL is accepted.
 class CountsResourceDefinition
+  # Compatibility belongs at the request boundary, not in field definitions.
+  def self.normalise_params(params)
+    params.each_with_object({}) do |(key, value), result|
+      result[key] = value
+      next unless value.is_a?(String)
+
+      if ['columns', 'order', 'sort', 'rows', 'cols', 'axis'].include?(key)
+        result[key] = value.split(/ /, -1).map { |token|
+          token.sub(/\Ais_latest(?=:|\z)/, 'latest')
+        }.join(' ')
+      elsif (key.match?(/\A(?:conditions|drill)\.[0-9]+\.field\z/) &&
+          value == 'is_latest')
+        result[key] = 'latest'
+      end
+    end
+  end
+
   COMMON = { 'resource_id' => :integer, 'version_no' => :integer,
              'event_type' => :enum, 'user' => :integer, 'created_at' => :datetime }.freeze
   DEFINITIONS = {
@@ -35,7 +52,7 @@ class CountsResourceDefinition
       interval: ['range'] }.freeze
   LABELS = {
       'resource_id' => 'counts:columns.resourceId', 'version_no' => 'counts:columns.version',
-      'is_latest' => 'counts:columns.latest', 'event_type' => 'counts:eventType',
+      'latest' => 'counts:columns.latest', 'event_type' => 'counts:eventType',
       'user' => 'history:actor', 'created_at' => 'counts:columns.changedAt',
       'thumbnail' => 'counts:columns.thumbnailDisplay',
       'thumbnail_base' => 'counts:columns.thumbnailURL',
@@ -101,7 +118,7 @@ class CountsResourceDefinition
               when 'tags' then ['name', 'names', 'category', 'aliases', 'parents', 'deprecated']
               else fields.keys - COMMON.keys
               end
-    keys = ['resource_id', 'version_no', 'is_latest'] + (content - ['thumbnail']) +
+    keys = ['resource_id', 'version_no', 'latest'] + (content - ['thumbnail']) +
         ['event_type', 'user', 'created_at']
     keys.unshift('thumbnail') if content.include?('thumbnail')
     keys.map do |key|

backend/app/services/counts_table_export.rb

@@ -7,6 +7,7 @@ class CountsTableExport
   attr_reader :format, :resource
 
   def initialize(params)
+    params = CountsResourceDefinition.normalise_params(params)
     QueryState.validate!(params, CountsTableQuery::KEYS + EXTRA + ['measures'])
     raise ArgumentError unless params.values.all? { _1.is_a?(String) }
 
@@ -47,7 +48,7 @@ class CountsTableExport
       if ['posts', 'tags'].include?(resource)
         @projection = CountsExportProjection.new(params, resource:, summary: false, tags: false)
       else
-        allowed = @query.population.definition.fields.keys + ['is_latest']
+        allowed = @query.population.definition.fields.keys + ['latest']
         raise ArgumentError if (@columns - allowed).any?
         raise ArgumentError unless @columns.length == @headers.length && @columns.uniq == @columns
       end
@@ -305,7 +306,7 @@ class CountsTableExport
 
   def csv_logical(key, value)
     return @labels.fetch('bot', '') if key == 'created_by_user' && value.nil?
-    return value ? @labels.fetch('latest', '') : '' if key == 'is_latest'
+    return value ? @labels.fetch('latest', '') : '' if key == 'latest'
     if value.is_a?(Time) || value.is_a?(ActiveSupport::TimeWithZone)
       return @query.population.timezone.utc_to_local(value.utc).iso8601
     end

backend/app/services/counts_table_query.rb

@@ -9,6 +9,7 @@ class CountsTableQuery
   attr_reader :population, :axes, :page, :limit, :mode
 
   def initialize(params)
+    params = CountsResourceDefinition.normalise_params(params)
     QueryState.validate!(params, KEYS + ['measures'])
     raise ArgumentError unless params.values.all? { _1.is_a?(String) }
 
@@ -26,7 +27,7 @@ class CountsTableQuery
 
     @sort = QueryState.list(params.fetch('sort', '')).map { _1.split(':', -1) }
     allowed = mode == 'summary' ? axes.all.each_index.map { "a#{ _1 }" } + ['count'] :
-        @definition.fields.keys + ['is_latest'] +
+        @definition.fields.keys + ['latest'] +
         (@definition.resource == 'posts' ? ['tags', 'parents'] : []) +
         (@definition.resource == 'tags' ? ['parents', 'deprecated'] : [])
     raise ArgumentError if @sort.map(&:first).uniq.length != @sort.length
@@ -43,9 +44,10 @@ class CountsTableQuery
     ApplicationRecord.transaction do
       result = metadata.merge(mode == 'summary' ? summary : details)
       visible = mode == 'summary' ? result[:visible_output_count] : result[:count]
+      totals = output_totals(visible)
       result.merge(visible_output_count: visible,
-          output_count: visible,
-          output_count_scope: 'authorised_results',
+          **totals,
+          output_count_scope: 'before_authorisation',
           output_count_unit: 'table_body_rows',
           page_grand_total_repeated: mode == 'summary' && axes.rows.any? &&
               result[:group_count] > limit)
@@ -54,12 +56,8 @@ class CountsTableQuery
 
   def metadata
     @population_counts ||= HistoryAccess.population_counts(@definition.model)
-    # Global exclusions are not exclusions for this filtered query. Do not expose
-    # private predicate matches through a condition-dependent count.
-    exclusions_known = @population_counts[:excluded_count].zero?
     @population_counts.merge(
-        excluded_count: exclusions_known ? 0 : nil,
-        excluded_count_scope: exclusions_known ? 'current_query' : 'unavailable_private_history',
+        excluded_count: nil, excluded_count_scope: 'current_query',
         population_count_scope: 'all_recorded_versions',
         page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence,
         definition: population.history ? 'history' : 'latest',
@@ -186,7 +184,7 @@ class CountsTableQuery
           [key, row.public_send(@definition.column(key))]
         end
         attributes.merge('id' => row.id, 'resource_id' => row[@definition.keys.first],
-            'is_latest' => row.attributes['counts_latest'].to_i == 1,
+            'latest' => row.attributes['counts_latest'].to_i == 1,
             'created_by_user' => row.created_by_user_id &&
                 { id: row.created_by_user_id, name: users[row.created_by_user_id] })
       end
@@ -217,12 +215,55 @@ class CountsTableQuery
       "(SELECT COUNT(*) FROM (SELECT #{ keys } FROM cells GROUP BY #{ keys }) subtotal_rows)"
     end
     expressions = [total.to_s, '1'] + subtotals
-    @output_row_count ||= @connection.select_value(<<~SQL).to_i
+    @connection.select_value(<<~SQL).to_i
       #{ prefix }
       SELECT /*+ MAX_EXECUTION_TIME(10000) */ #{ expressions.join(' + ') } AS output_rows
     SQL
   end
 
+  def output_totals(visible_rows)
+    return @output_totals if @output_totals
+
+    if HistoryAccess.admin? || @population_counts[:excluded_count].zero?
+      return @output_totals = { output_count: visible_rows, excluded_count: 0 }
+    end
+    # Only numerical aggregates leave this population. Labels, snapshots and
+    # detail pagination continue to use the authorised population exclusively.
+    source = CountsPopulation.new(
+        @params.merge('execution_at' => population.at.iso8601(6)), authorisation_flg: false)
+    records = source.scope.except(:select).optimizer_hints('MAX_EXECUTION_TIME(10000)').count
+    denied = source.scope.except(:select)
+        .where(Arel.sql("NOT (#{ HistoryAccess.sql(@definition.model, 'v') })"))
+        .optimizer_hints('MAX_EXECUTION_TIME(10000)').count
+    rows = records
+    if mode == 'summary'
+      unrestricted_axes = CountsAxes.new(source)
+      relation = source.scope
+      joins = unrestricted_axes.joins
+      relation = relation.joins(joins) unless joins.empty?
+      expressions = unrestricted_axes.expressions
+      keys = axes.rows.each_index.map { "a#{ _1 }" }
+      if keys.empty?
+        rows = 1
+      else
+        selected = expressions.each_with_index.map { |expression, index|
+          "#{ expression } AS a#{ index }"
+        }
+        sql = relation.except(:select).select(Arel.sql(selected.join(', ')))
+            .distinct.to_sql
+        prefix = "WITH cells AS (#{ sql })"
+        groups = @connection.select_value(<<~SQL).to_i
+          #{ prefix }
+          SELECT /*+ MAX_EXECUTION_TIME(10000) */ COUNT(*) FROM (
+            SELECT #{ keys.join(', ') } FROM cells GROUP BY #{ keys.join(', ') }
+          ) output_groups
+        SQL
+        rows = output_row_count(prefix, keys, CountsPivotLayout.depths(keys, @sort), groups)
+      end
+    end
+    @output_totals = { output_count: rows, excluded_count: denied }
+  end
+
   def locale = @locale ||= Locale.find_by!(code: @params.fetch('locale', 'ja'))
 
   def limited_columns(prefix)
@@ -284,7 +325,7 @@ class CountsTableQuery
   end
 
   def detail_expression(key)
-    return "(#{ population.latest_sql })" if key == 'is_latest'
+    return "(#{ population.latest_sql })" if key == 'latest'
     aliases = { 'tags' => 'tags_json', 'parents' => @definition.resource == 'posts' ?
         'parent_post_ids' : 'parent_tag_ids', 'deprecated' => 'deprecated_at' }
     field = aliases.fetch(key, key)