#434

backend/app/controllers/counts_exports_controller.rb

@@ -12,6 +12,15 @@ class CountsExportsController < ApplicationController
     render json: { error: 'export_worker_unavailable' }, status: :service_unavailable
   rescue CountsExportDownload::Busy
     head :too_many_requests
+  rescue CountsAxes::TooManyAxes
+    render json: { error: 'too_many_axes', max_axes: CountsAxes::MAX_AXES }, status: :bad_request
+  rescue CountsAxes::PrivateCondition
+    render json: { error: 'private_condition' }, status: :forbidden
+  rescue CountsTableQuery::TooManyColumns
+    render json: { error: 'too_many_columns', max_columns: CountsTableQuery::MAX_COLUMNS },
+        status: :unprocessable_entity
+  rescue ActiveRecord::StatementTimeout
+    render json: { error: 'counts_query_timeout' }, status: :service_unavailable
   rescue ArgumentError, TypeError, KeyError, JSON::ParserError
     head :bad_request
   end
@@ -48,6 +57,15 @@ class CountsExportsController < ApplicationController
     head :gateway_timeout
   rescue CountsExportDownload::StorageFull, Errno::ENOSPC
     head :insufficient_storage
+  rescue CountsAxes::TooManyAxes
+    render json: { error: 'too_many_axes', max_axes: CountsAxes::MAX_AXES }, status: :bad_request
+  rescue CountsAxes::PrivateCondition
+    render json: { error: 'private_condition' }, status: :forbidden
+  rescue CountsTableQuery::TooManyColumns
+    render json: { error: 'too_many_columns', max_columns: CountsTableQuery::MAX_COLUMNS },
+        status: :unprocessable_entity
+  rescue ActiveRecord::StatementTimeout
+    render json: { error: 'counts_query_timeout' }, status: :service_unavailable
   rescue ArgumentError, TypeError, KeyError, JSON::ParserError
     head :bad_request
   end
@@ -79,36 +97,43 @@ class CountsExportsController < ApplicationController
       CountsExportJobs.download_attempt(params[:job], owner, **arguments, status: 'unavailable')
       return head :not_found
     end
-    # Open before committing the response, but never buffer the entire export.
-    stream = File.open(file.fetch('path'), 'rb')
+    job = params[:job]
+    authority = HistoryAccess.user
+    first_check = true
+    stream = CountsExportDownload::ResponseBody.new(file.fetch('path')) do |status|
+      if status == 'active'
+        if first_check
+          first_check = false
+          fresh = authority && User.find_by(id: authority.id)
+          next false if authority && (!fresh || fresh.banned?)
+
+          authority = fresh
+          HistoryAccess.set(user: authority) do
+            CountsExportDownload.find(arguments[:file_token], owner) &&
+                CountsExportJobs.stream_active?(job, owner, **arguments)
+          end
+        else
+          CountsExportJobs.stream_active?(job, owner, **arguments)
+        end
+      else
+        HistoryAccess.set(user: authority) do
+          CountsExportJobs.download_attempt(job, owner, **arguments, status:)
+        end
+      end
+    end
+    unless CountsExportJobs.download_attempt(job, owner, **arguments, status: 'streaming')
+      stream.close
+      return head :conflict
+    end
     response.headers['Cache-Control'] = 'private, no-store'
     send_file_headers! filename: "counts-#{ file.fetch('resource') }.#{ file.fetch('format') }",
         type: file['format'] == 'json' ? 'application/json' : 'text/csv',
         disposition: 'attachment'
-    job = params[:job]
-    authority = HistoryAccess.user
-    self.response_body = Enumerator.new do |output|
-      begin
-        first = true
-        while (chunk = stream.read(64 * 1024))
-          output << chunk
-          if first
-            HistoryAccess.set(user: authority) do
-              CountsExportJobs.download_attempt(job, owner, **arguments, status: 'started')
-            end
-            first = false
-          end
-        end
-      rescue IOError, SystemCallError => error
-        Rails.logger.warn("counts_download_stream_failure: #{ error.class }")
-        HistoryAccess.set(user: authority) do
-          CountsExportJobs.download_attempt(job, owner, **arguments, status: 'failed')
-        end
-      ensure
-        stream.close
-      end
-    end
+    response.headers['Last-Modified'] = File.mtime(file.fetch('path')).httpdate
+    response.headers['X-Accel-Buffering'] = 'no'
+    self.response_body = stream
   rescue IOError, SystemCallError
+    stream&.close
     CountsExportJobs.download_attempt(params[:job], owner, **arguments, status: 'failed')
     head :service_unavailable
   end
@@ -138,6 +163,15 @@ class CountsExportsController < ApplicationController
         filename: "counts-#{ exporter.resource }.#{ exporter.format }",
         type: exporter.format == 'json' ? 'application/json' : 'text/csv',
         disposition: 'attachment'
+  rescue CountsAxes::TooManyAxes
+    render json: { error: 'too_many_axes', max_axes: CountsAxes::MAX_AXES }, status: :bad_request
+  rescue CountsAxes::PrivateCondition
+    render json: { error: 'private_condition' }, status: :forbidden
+  rescue CountsTableQuery::TooManyColumns
+    render json: { error: 'too_many_columns', max_columns: CountsTableQuery::MAX_COLUMNS },
+        status: :unprocessable_entity
+  rescue ActiveRecord::StatementTimeout
+    render json: { error: 'counts_query_timeout' }, status: :service_unavailable
   rescue ArgumentError, TypeError, KeyError, JSON::ParserError
     head :bad_request
   end

backend/app/models/history_access.rb

@@ -4,6 +4,7 @@ require 'digest'
 class HistoryAccess < ActiveSupport::CurrentAttributes
   attribute :user
   class Forbidden < StandardError; end
+  ARTIFACT_POLICY = 'counts-masked-summary-1'.freeze
 
   KEYS = { 'post_versions' => ['post_id'], 'tag_versions' => ['tag_id'],
            'nico_tag_versions' => ['tag_id'],
@@ -18,10 +19,12 @@ class HistoryAccess < ActiveSupport::CurrentAttributes
   end
 
   def self.authority
-    { 'policy_revision' => revision, 'admin' => admin?, 'actor_id' => user&.id }
+    { 'artifact_policy' => ARTIFACT_POLICY, 'policy_revision' => revision,
+      'admin' => admin?, 'actor_id' => user&.id }
   end
 
   def self.authorised_artifact?(metadata)
+    return false unless metadata['artifact_policy'] == ARTIFACT_POLICY
     return false unless metadata['policy_revision'].is_a?(String)
     if metadata['admin']
       admin? && user.id == metadata['actor_id']
@@ -72,13 +75,47 @@ class HistoryAccess < ActiveSupport::CurrentAttributes
         "ON reference_entry.reference_id = CAST(blocked.#{ key } AS CHAR))"
   end
 
+  def self.snapshot_reference(model, row, external: false)
+    key = external ? 'external_tag_id' : 'tag_id'
+    table = model.table_name
+    source = "JSON_TABLE(COALESCE(#{ row }.tags_json, JSON_ARRAY()), '$[*]' COLUMNS (" +
+        "tag_id BIGINT PATH '$.#{ key }', version_no INT PATH '$.version_no')) tag_reference"
+    match = 'reference_version.tag_id = tag_reference.tag_id'
+    if external
+      match += " AND reference_version.created_at <= #{ row }.created_at"
+      latest = "AND NOT EXISTS (SELECT 1 FROM #{ table } later_reference " +
+          'WHERE later_reference.tag_id = reference_version.tag_id ' +
+          "AND later_reference.created_at <= #{ row }.created_at " +
+          'AND later_reference.version_no > reference_version.version_no)'
+    else
+      match += ' AND reference_version.version_no = tag_reference.version_no'
+      latest = ''
+    end
+    "NOT EXISTS (SELECT 1 FROM #{ source } JOIN #{ table } reference_version ON #{ match } " +
+        "WHERE NOT (#{ sql(model, 'reference_version') }) #{ latest })"
+  end
+
   def self.reference_sql(table, row)
     case table
     when 'post_versions'
       [['tag_versions', 'tag_id', 'tag_id'],
        ['nico_tag_versions', 'tag_id', 'external_tag_id']].map do |target, key, field|
         json_reference(target, key, "JSON_EXTRACT(#{ row }.tags_json, '$[*].#{ field }')")
-      end + [list_reference('post_versions', 'post_id', "#{ row }.parent_post_ids")]
+      end + [list_reference('post_versions', 'post_id', "#{ row }.parent_post_ids"),
+             snapshot_reference(TagVersion, row),
+             snapshot_reference(NicoTagVersion, row, external: true)]
+    when 'nico_tag_versions'
+      # Legacy linked_tags snapshots contain names, not IDs. Match recorded names
+      # conservatively; never expose a private linked name through an external tag.
+      names = "JSON_MERGE_PRESERVE(JSON_ARRAY(private_tag.name), " +
+          "COALESCE(JSON_EXTRACT(private_tag.names, '$**.name'), JSON_ARRAY()))"
+      ["NOT EXISTS (SELECT 1 FROM tag_versions private_tag " +
+          "JOIN (#{ discarded_keys('tag_versions', 'tag_id') }) blocked " +
+          "ON blocked.tag_id = private_tag.tag_id " +
+          "JOIN JSON_TABLE(#{ names }, '$[*]' COLUMNS (name TEXT PATH '$')) private_name " +
+          "WHERE private_name.name <> '' AND " +
+          "LOCATE(CONCAT(' ', private_name.name, ' '), " +
+          "CONCAT(' ', COALESCE(#{ row }.linked_tags, ''), ' ')) > 0)"]
     when 'tag_versions'
       [list_reference('tag_versions', 'tag_id', "#{ row }.parent_tag_ids")]
     when 'tag_description_versions'

backend/app/services/counts_axes.rb

@@ -61,7 +61,11 @@ class CountsAxes
             expression, axis.fetch('grain', 'month'), @population.timezone) : expression
       end
       if !HistoryAccess.admin? && !PUBLIC_FIELDS.include?(field)
-        expression = "CASE WHEN #{ HistoryAccess.sql(@definition.model, 'v') } " +
+        visible = HistoryAccess.sql(@definition.model, 'v')
+        if field == 'changed_field'
+          visible = "(#{ visible }) AND (#{ HistoryAccess.sql(@definition.model, 'previous') })"
+        end
+        expression = "CASE WHEN #{ visible } " +
             "THEN CAST((#{ expression }) AS CHAR) ELSE '#{ MASKED_VALUE }' END"
       end
       expression
@@ -72,8 +76,10 @@ class CountsAxes
     return 'COUNT(DISTINCT v.id)' unless rows.any? { _1['field'] == 'tags' }
 
     # Keep tag occurrences even when several private tags share one masked label.
-    "COUNT(DISTINCT v.id, COALESCE(CONCAT('internal:', tag_entry.tag_id), " +
-        "CONCAT('external:', tag_entry.external_tag_id)))"
+    values = ['v.id', "COALESCE(CONCAT('internal:', tag_entry.tag_id), " +
+        "CONCAT('external:', tag_entry.external_tag_id))"]
+    values << 'changed.field_key' if rows.any? { _1['field'] == 'changed_field' }
+    "COUNT(DISTINCT #{ values.join(', ') })"
   end
 
   def joins(fields = all.map { _1['field'] })

backend/app/services/counts_export_download.rb

@@ -13,6 +13,46 @@ class CountsExportDownload
   class TimedOut < StandardError; end
   class StorageFull < StandardError; end
 
+  # Rack body deliberately has no to_path: sendfile middleware must not bypass ACK.
+  class ResponseBody
+    def initialize(path, &state)
+      File.open(path, 'rb') { |file| file.stat }
+      @path = path
+      @stream = nil
+      @closed = false
+      @state = state
+    end
+
+    def each
+      return if @closed
+
+      @stream = File.open(@path, 'rb')
+      first = true
+      while (chunk = @stream.read(64 * 1024))
+        raise IOError, 'Download no longer authorised' unless @state.call('active')
+
+        yield chunk
+        if first
+          # This confirms yielding to Rack, never browser receipt or saving.
+          @state.call('started')
+          first = false
+        end
+      end
+      @state.call('failed') if first
+    rescue StandardError => error
+      Rails.logger.warn("counts_download_stream_failure: #{ error.class }")
+      @state.call('failed')
+      raise
+    ensure
+      close
+    end
+
+    def close
+      @closed = true
+      @stream.close if @stream && !@stream.closed?
+    end
+  end
+
   def self.directory
     Rails.root.join(ENV.fetch('COUNTS_EXPORT_DIRECTORY', 'tmp/counts_exports'))
   end

backend/app/services/counts_export_jobs.rb

@@ -45,6 +45,20 @@ class CountsExportJobs
     write(state)
   end
 
+  def self.failure_code(error)
+    case error
+    when Cancelled then 'cancelled'
+    when CountsAxes::TooManyAxes then 'too_many_axes'
+    when CountsAxes::PrivateCondition then 'private_condition'
+    when CountsTableQuery::TooManyColumns then 'too_many_columns'
+    when ActiveRecord::StatementTimeout then 'counts_query_timeout'
+    when CountsExportDownload::TimedOut then 'generation_timeout'
+    when CountsExportDownload::TooLarge then 'export_too_large'
+    when CountsExportDownload::StorageFull, Errno::ENOSPC then 'export_storage_full'
+    else 'generation_failed'
+    end
+  end
+
   def self.directory = CountsExportDownload.directory.join('jobs')
   def self.verifier = Rails.application.message_verifier('counts_export_job')
 
@@ -176,6 +190,12 @@ class CountsExportJobs
       return unless state && state['owner'] == owner && state['expires_at'] > Time.current.to_f
 
       expire_stalled(state)
+      if (state['status'] == 'ready' &&
+          ['pending', 'streaming'].include?(state['download_status']) &&
+          state.fetch('download_expires_at', 0) <= Time.current.to_f)
+        state['download_status'] = 'timeout'
+        write(state)
+      end
 
       if cancel && ['queued', 'running', 'ready'].include?(state['status'])
         state['status'] = state['status'] == 'running' ? 'cancelling' : 'cancelled'
@@ -183,8 +203,11 @@ class CountsExportJobs
         state.delete('token')
         write(state)
       end
-      state.slice('status', 'phase', 'processed', 'total', 'format', 'token', 'error',
+      result = state.slice('status', 'phase', 'processed', 'total', 'format', 'token', 'error',
           'updated_at', 'heartbeat_at', 'download_attempt', 'download_status')
+      result['max_axes'] = CountsAxes::MAX_AXES if state['error'] == 'too_many_axes'
+      result['max_columns'] = CountsTableQuery::MAX_COLUMNS if state['error'] == 'too_many_columns'
+      result
     end
   end
 
@@ -198,18 +221,53 @@ class CountsExportJobs
           state['expires_at'] > Time.current.to_f && HistoryAccess.authorised_artifact?(state))
       if attempt
         return unless state['download_attempt'] == attempt && state['token'] == file_token
-        return unless ['pending', 'started'].include?(state['download_status'])
+        return unless state['download_status'] == 'started' ||
+            state.fetch('download_expires_at', 0) > Time.current.to_f
 
-        state['download_status'] = status if status
+        transitions = { 'streaming' => ['pending'], 'started' => ['streaming'],
+                        'failed' => ['pending', 'streaming', 'started'],
+                        'unavailable' => ['pending', 'streaming'] }
+        if status == 'active'
+          return state.slice('download_attempt', 'download_status', 'token') if
+              ['streaming', 'started'].include?(state['download_status'])
+          return
+        end
+        if status
+          return unless transitions.fetch(status, []).include?(state['download_status'])
+
+          state['download_status'] = status
+        else
+          return unless state['download_status'] == 'pending'
+        end
       else
+        # Repeated acceptance POSTs do not invalidate an in-flight GET or issue duplicates.
+        if (state['download_status'] == 'started' ||
+            (['pending', 'streaming'].include?(state['download_status']) &&
+             state.fetch('download_expires_at', 0) > Time.current.to_f))
+          return state.slice('download_attempt', 'download_status', 'token')
+        end
         state['download_attempt'] = SecureRandom.hex(24)
         state['download_status'] = 'pending'
+        state['download_expires_at'] = Time.current.to_f + 45
       end
       write(state)
       state.slice('download_attempt', 'download_status', 'token')
     end
   end
 
+  # GET already verified authority. During streaming, observe cancellation without
+  # repeating the six-table policy revision aggregate for every 64KiB chunk.
+  def self.stream_active?(token, owner, file_token:, attempt:)
+    id = authorised_id(token, owner)
+    state = id && read(id)
+    state && state['owner'] == owner && state['status'] == 'ready' &&
+        state['expires_at'] > Time.current.to_f && state['token'] == file_token &&
+        state['download_attempt'] == attempt &&
+        (state['download_status'] == 'started' ||
+         (state['download_status'] == 'streaming' &&
+          state.fetch('download_expires_at', 0) > Time.current.to_f))
+  end
+
   def self.checkpoint(id, processed = nil, total = nil, phase: nil)
     synchronise do
       state = read(id)
@@ -398,7 +456,7 @@ class CountsExportJobs
         if current
           unless current['status'] == 'failed'
             current['status'] = error.is_a?(Cancelled) ? 'cancelled' : 'failed'
-            current['error'] = error.class.name
+            current['error'] = failure_code(error)
           end
           current['updated_at'] = Time.current.to_f
           write(current)

backend/app/services/counts_pivot_totals.rb

@@ -1,4 +1,4 @@
-# Re-aggregate the population, never add up already grouped cells (multi-value axes overlap).
+# Re-aggregate distinct membership; tag row tuples alone use additive contributions.
 class CountsPivotTotals
   def initialize(population, axes, row_headers, column_headers)
     @population = population
@@ -8,8 +8,12 @@ class CountsPivotTotals
   end
 
   def call
-    # MySQL's unlimited LIMIT prevents CTE merging: authorise once for all branches.
-    basis = @population.scope.except(:select, :order).select('v.*')
+    # Materialise full-cell membership once, including omitted multi-value axes.
+    membership = @population.scope.except(:select, :order)
+    joins = @axes.joins
+    membership = membership.joins(joins) unless joins.empty?
+    # Retain exactly the records contributing a full cell, including changed_field.
+    basis = membership.select('v.*').distinct
         .limit(18_446_744_073_709_551_615).to_sql
     queries = @rows.flat_map do |row_depth, headers|
       @columns.filter_map do |column_depth, columns|
@@ -18,7 +22,7 @@ class CountsPivotTotals
         indexes = (0...row_depth).to_a +
             (0...column_depth).map { @axes.rows.length + _1 }
         expressions = indexes.map { @axes.expressions[_1] }
-        scope = @population.scope.except(:select, :order, :where, :from).from('authorised v')
+        scope = @population.scope.except(:select, :order, :where, :from).from('population_members v')
         additive = @axes.rows.any? { _1['field'] == 'tags' }
         retained = additive ? (0...@axes.rows.length).to_a | indexes : indexes
         joins = @axes.joins(retained.map { @axes.all[_1].fetch('field') })
@@ -47,9 +51,12 @@ class CountsPivotTotals
         if additive
           # Sum full row tuples, not page cells; preserve DISTINCT inside each tuple.
           leaf = retained.map { @axes.expressions[_1] }
-          selected = leaf.each_with_index.map { |expression, index| "#{ expression } AS k#{ index }" }
+          selected = leaf.each_with_index.map do |expression, index|
+            "#{ expression } AS k#{ index }"
+          end
+          selected << "#{ @axes.count_expression } AS count"
           inner = scope.group(*leaf.map { Arel.sql(_1) })
-              .select(Arel.sql((selected + ["#{ @axes.count_expression } AS count"]).join(', '))).to_sql
+              .select(Arel.sql(selected.join(', '))).to_sql
           coordinates = indexes.map { "k#{ retained.index(_1) }" }
           projected = coordinates.each_with_index.map { |key, index| "#{ key } AS v#{ index }" }
           output = ["#{ row_depth } AS row_depth", "#{ column_depth } AS column_depth"] +
@@ -67,7 +74,7 @@ class CountsPivotTotals
 
     sql = queries.map { "(#{ _1 })" }.join(' UNION ALL ')
     @population.definition.model.connection.select_all(
-        "WITH authorised AS (#{ basis }) " +
+        "WITH population_members AS (#{ basis }) " +
         "SELECT /*+ MAX_EXECUTION_TIME(10000) */ * FROM (#{ sql }) totals").map do |record|
       row_depth = record['row_depth'].to_i
       column_depth = record['column_depth'].to_i

backend/app/services/counts_population.rb

@@ -57,7 +57,8 @@ class CountsPopulation
     "(previous.id IS NOT NULL AND NOT (v.#{ column } <=> previous.#{ column }))"
   end
 
-  # Unrestricted predicates are used only for numerical totals, never record output.
+  # Aggregate sources are unrestricted; CountsAxes masks their output coordinates.
+  # Never serialise raw records from an unrestricted population.
   def access_sql(model, row)
     @authorisation_flg ? HistoryAccess.sql(model, row) : 'TRUE'
   end
@@ -67,6 +68,16 @@ class CountsPopulation
 
     key = entry.fetch('field')
     raise ArgumentError unless definition.axes(history).include?(key)
+    value = entry['value']
+    unless value.nil?
+      valid = case definition.fields[key]
+              when :integer
+                value.is_a?(Integer) || (value.is_a?(String) && value.match?(/\A-?[0-9]+\z/))
+              when :boolean then [true, false].include?(value)
+              else value.is_a?(String)
+              end
+      raise ArgumentError unless valid
+    end
     if entry.key?('grain')
       unless definition.fields[key] == :datetime && CountsAxes::GRAINS.include?(entry['grain'])
         raise ArgumentError

backend/app/services/counts_table_export.rb

@@ -253,6 +253,8 @@ class CountsTableExport
   end
 
   def summary_value(value, index, page)
+    return '********' if value == CountsAxes::MASKED_VALUE
+
     field = @query.axes.all.fetch(index).fetch('field')
     return @labels.fetch('bot', '') if field == 'user' && value.nil?
     if ['language_code', 'source_language_code'].include?(field)

backend/app/services/counts_table_query.rb

@@ -49,12 +49,18 @@ class CountsTableQuery
       result = metadata.merge(mode == 'summary' ? summary : details)
       visible = mode == 'summary' ? result[:visible_output_count] : result[:count]
       totals = output_totals(visible)
-      result.merge(visible_output_count: visible,
+      response = result.merge(visible_output_count: visible,
           **totals,
           output_count_scope: 'before_authorisation',
           output_count_unit: 'regular_result_rows',
           page_grand_total_repeated: mode == 'summary' && axes.rows.any? &&
               result[:group_count] > limit)
+      if mode == 'summary'
+        response.except!(:excluded_count, :excluded_count_scope)
+        response[:column_values_editable] = HistoryAccess.admin? ||
+            !result.fetch(:column_values, []).flatten.include?(CountsAxes::MASKED_VALUE)
+      end
+      response
     end
   end
 
@@ -228,17 +234,17 @@ class CountsTableQuery
 
     fields = QueryState.records(@params, 'conditions') + QueryState.records(@params, 'drill')
     private_fields = fields.any? { !CountsAxes::PUBLIC_FIELDS.include?(_1['field']) }
-    private_parameters = ['tags', 'tag_refs', 'tag', 'kind', 'resource_id', 'user_id',
-                          'name_scope'].any? do |key|
+    keys = ['tags', 'tag_refs', 'tag', 'kind', 'resource_id', 'user_id']
+    private_parameters = keys.any? do |key|
       @params.any? { |name, value| (name == key || name.start_with?("#{ key }.")) && value != '' }
     end
     raise CountsAxes::PrivateCondition if private_fields || private_parameters
 
     ['column_order', 'measures'].each do |key|
-      next if QueryState.records(@params, key).empty?
-      columns = QueryState.axes(@params.fetch('cols', ''))
-      raise CountsAxes::PrivateCondition if columns.any? do |axis|
-        !CountsAxes::PUBLIC_FIELDS.include?(axis['field'])
+      # Coordinates are compared only to already masked expressions, never raw
+      # values. Public coordinates remain sortable; a guessed private ID cannot match.
+      raise CountsAxes::PrivateCondition if QueryState.records(@params, key).any? do |record|
+        record['value'] == CountsAxes::MASKED_VALUE
       end
     end
   end
@@ -350,6 +356,11 @@ class CountsTableQuery
   def measure_relation values
     row_expressions = axes.expressions.first(axes.rows.length)
     source = summary_population.scope.except(:select, :order)
+    # Omitted multi-value column axes still determine full-cell membership.
+    membership = summary_population.scope.except(:select, :order)
+    all_joins = axes.joins
+    membership = membership.joins(all_joins) unless all_joins.empty?
+    source = source.where("v.id IN (#{ membership.select('v.id').distinct.to_sql })")
     fields = axes.rows + axes.columns.first(values.length)
     joins = axes.joins(fields.map { _1.fetch('field') })
     source = source.joins(joins) unless joins.empty?
@@ -367,7 +378,15 @@ class CountsTableQuery
   def axis_order_sql key, direction = 'asc', qualifier: nil
     column = qualifier ? "#{ qualifier }.#{ key }" : key
     expression = column
-    if axes.all.fetch(key.delete_prefix('a').to_i).fetch('field') == 'category'
+    field = axes.all.fetch(key.delete_prefix('a').to_i).fetch('field')
+    if !HistoryAccess.admin? && [:integer, :boolean].include?(@definition.fields[field])
+      masked = @connection.quote(CountsAxes::MASKED_VALUE)
+      expression = "CASE WHEN #{ column } <=> #{ masked } THEN NULL " +
+          "ELSE CAST(#{ column } AS SIGNED) END"
+      return "#{ column } IS NULL ASC, (#{ column } <=> #{ masked }) ASC, " +
+          "#{ expression } #{ direction.upcase }"
+    end
+    if field == 'category'
       # Tag's enum preserves the same category order as TagsController.
       cases = Tag.categories.keys.each_with_index.map { |category, index|
         "WHEN #{ @connection.quote(category) } THEN #{ index }"
@@ -418,7 +437,10 @@ class CountsTableQuery
       labels = case field
                when 'tags' then tag_labels(values)
                when 'user'
-                 User.where(id: values).pluck(:id, :name).to_h.transform_keys(&:to_s)
+                 actors = HistoryAccess.visible(population.scope, 'v').except(:select, :order)
+                     .select('v.created_by_user_id')
+                 User.where(id: values).where(id: actors)
+                     .pluck(:id, :name).to_h.transform_keys(&:to_s)
                else {}
                end
       [field, labels]

backend/spec/models/history_access_spec.rb

@@ -67,3 +67,64 @@ RSpec.describe HistoryAccess do
     end
   end
 end
+
+RSpec.describe 'Counts artifact policy and external references' do
+  it 'rejects files created before the masked-summary policy' do
+    HistoryAccess.set(user: nil) do
+      metadata = HistoryAccess.authority.except('artifact_policy')
+      expect(HistoryAccess.authorised_artifact?(metadata)).to be(false)
+    end
+  end
+
+  it 'does not expose a private linked tag name through an external history snapshot' do
+    tag = create(:tag, primary_name: 'private-linked-name')
+    TagVersion.create!(tag:, version_no: 1, event_type: 'discard', name: 'private-linked-name',
+        category: 'general', aliases: '', parent_tag_ids: '', created_at: Time.utc(2024),
+        names: { ja: { name: 'private-linked-name', script_code: 'Jpan',
+                       auto_generated: false, aliases: [] } })
+    external = create(:external_tag, name: 'public-external')
+    row = NicoTagVersion.create!(external_tag: external, version_no: 1, event_type: 'create',
+        name: 'nico:public-external', linked_tags: 'private-linked-name',
+        created_at: Time.utc(2024))
+    post = create(:post)
+    snapshot = PostVersion.create!(post:, version_no: 1, event_type: 'create',
+        title: 'external reference', url: post.url, tags: '',
+        tags_json: [{ external_tag_id: external.id }], parent_post_ids: '',
+        created_at: Time.utc(2024, 2))
+    HistoryAccess.set(user: nil) do
+      expect(HistoryAccess.visible(NicoTagVersion.where(id: row.id))).to be_empty
+      expect(HistoryAccess.visible(PostVersion.where(id: snapshot.id))).to be_empty
+    end
+    HistoryAccess.set(user: create(:user, :admin)) do
+      expect(HistoryAccess.visible(NicoTagVersion.where(id: row.id))).to exist
+    end
+    public_row = NicoTagVersion.create!(external_tag: external, version_no: 2, event_type: 'update',
+        name: 'nico:public-external', linked_tags: 'private-linked-name-extra',
+        created_at: Time.utc(2024, 7))
+    HistoryAccess.set(user: nil) do
+      expect(HistoryAccess.visible(NicoTagVersion.where(id: public_row.id))).to exist
+      expect(HistoryAccess.visible(PostVersion.where(id: snapshot.id))).to be_empty
+    end
+  end
+
+  it 'propagates the recorded tag version policy to its post snapshot' do
+    parent = create(:tag, primary_name: 'private parent')
+    child = create(:tag, primary_name: 'referencing child')
+    [[parent, 'discard', '', 'private parent'],
+     [child, 'create', parent.id.to_s, 'referencing child']].each do |tag, event, parents, name|
+      TagVersion.create!(tag:, version_no: 1, event_type: event, name:,
+          category: 'general', aliases: '', parent_tag_ids: parents, created_at: Time.utc(2023),
+          names: { ja: { name:, script_code: 'Jpan',
+                         auto_generated: false, aliases: [] } })
+    end
+    post = create(:post)
+    row = PostVersion.create!(post:, version_no: 1, event_type: 'create', title: 'snapshot',
+        url: post.url, tags: '', parent_post_ids: '',
+        tags_json: [{ tag_id: child.id, version_no: 1, name: 'referencing child', sections: [] }],
+        created_at: Time.utc(2024))
+    HistoryAccess.set(user: nil) do
+      expect(HistoryAccess.visible(PostVersion.where(id: row.id))).to be_empty
+    end
+  end
+
+end

backend/spec/requests/counts_download_spec.rb

@@ -0,0 +1,98 @@
+require 'rails_helper'
+require 'tmpdir'
+
+RSpec.describe 'Counts native GET acknowledgement', type: :request do
+  include ActiveSupport::Testing::TimeHelpers
+  around do |example|
+    Dir.mktmpdir('counts-download-request') do |path|
+      allow(CountsExportDownload).to receive(:directory).and_return(Pathname.new(path))
+      example.run
+    end
+  end
+
+  def ready_job(format)
+    CountsExportJobs.pulse
+    post '/counts/export/jobs', params: { resource: 'posts', definition: 'history',
+        mode: 'details', columns: 'resource_id', locale: 'ja', format: }
+    expect(response).to have_http_status(:accepted)
+    job = response.parsed_body.fetch('job')
+    signature = CountsExportJobs.verifier.verified(job, purpose: 'counts_export_job')
+    owner = signature.fetch('owner')
+    exporter = instance_double(CountsExporter, resource: 'posts', format:, body: ['payload'])
+    file = HistoryAccess.set(user: nil) { CountsExportDownload.prepare(exporter, owner) }
+    state = CountsExportJobs.read(signature.fetch('id'))
+    state.merge!('status' => 'ready', 'token' => file)
+    CountsExportJobs.write(state)
+    post "/counts/export/jobs/#{ job }/download"
+    expect(response).to have_http_status(:ok)
+    [job, file, response.parsed_body.fetch('download_attempt')]
+  end
+
+  ['csv', 'json'].each do |format|
+    it "acknowledges only an authorised #{ format } GET body, not HEAD or acceptance" do
+      job, file, attempt = ready_job(format)
+      get "/counts/export/jobs/#{ job }"
+      expect(response.parsed_body['download_status']).to eq('pending')
+      head "/counts/export/files/#{ file }", params: { job:, attempt: }
+      expect(response).to have_http_status(:method_not_allowed)
+      get "/counts/export/jobs/#{ job }"
+      expect(response.parsed_body['download_status']).to eq('pending')
+      get "/counts/export/files/#{ file }", params: { job:, attempt: }
+      expect(response).to have_http_status(:ok)
+      expect(response.headers['Content-Disposition']).to include("counts-posts.#{ format }")
+      expect(response.body).to eq('payload')
+      get "/counts/export/jobs/#{ job }"
+      expect(response.parsed_body['download_status']).to eq('started')
+      get "/counts/export/files/#{ file }", params: { job:, attempt: }
+      expect(response).to have_http_status(:not_found)
+    end
+  end
+
+  it 'rejects wrong attempts, cancellation and a newly discarded resource' do
+    job, file, attempt = ready_job('json')
+    get "/counts/export/files/#{ file }", params: { job:, attempt: 'wrong' }
+    expect(response).to have_http_status(:not_found)
+    delete "/counts/export/jobs/#{ job }"
+    expect(response).to have_http_status(:ok)
+    get "/counts/export/files/#{ file }", params: { job:, attempt: }
+    expect(response).to have_http_status(:not_found)
+    job, file, attempt = ready_job('json')
+    record = create(:post)
+    PostVersion.create!(post: record, version_no: 1, event_type: 'discard', title: 'private',
+        url: record.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024))
+    get "/counts/export/files/#{ file }", params: { job:, attempt: }
+    expect(response).to have_http_status(:not_found)
+    expect(response.body).not_to include('payload', 'private')
+  end
+  it 'does not let a second browser session inspect the job or download its file' do
+    job, file, attempt = ready_job('json')
+    other = ActionDispatch::Integration::Session.new(Rails.application)
+    other.get("/counts/export/jobs/#{ job }")
+    expect(other.response).to have_http_status(:not_found)
+    other.get("/counts/export/files/#{ file }", params: { job:, attempt: })
+    expect(other.response).to have_http_status(:not_found)
+    expect(other.response.body).not_to include('payload')
+  end
+
+  it 'does not acknowledge an expired attempt' do
+    job, file, attempt = ready_job('json')
+    travel_to(Time.current + 46.seconds) do
+      get "/counts/export/files/#{ file }", params: { job:, attempt: }
+      expect(response).to have_http_status(:not_found)
+      get "/counts/export/jobs/#{ job }"
+      expect(response.parsed_body['download_status']).to eq('timeout')
+    end
+  end
+
+  it 'records a missing file without accepting the GET as started' do
+    job, file, attempt = ready_job('json')
+    owner = CountsExportJobs.verifier.verified(job, purpose: 'counts_export_job').fetch('owner')
+    metadata = HistoryAccess.set(user: nil) { CountsExportDownload.find(file, owner) }
+    File.unlink(metadata.fetch('path'))
+    get "/counts/export/files/#{ file }", params: { job:, attempt: }
+    expect(response).to have_http_status(:not_found)
+    get "/counts/export/jobs/#{ job }"
+    expect(response.parsed_body['download_status']).to eq('unavailable')
+  end
+
+end

backend/spec/requests/counts_table_spec.rb

@@ -266,4 +266,12 @@ RSpec.describe 'Counts table API', type: :request do
     expect(json).to eq('error' => 'private_condition')
   end
 
+  it 'reports the combined axis limit without replacing it with a generic 400' do
+    get '/counts/query', params: base.merge(mode: 'summary', sort: '',
+        rows: 'resource_id version_no event_type user created_at:month',
+        cols: 'title url thumbnail_base video_ms')
+    expect(response).to have_http_status(:bad_request)
+    expect(response.parsed_body).to eq('error' => 'too_many_axes', 'max_axes' => 8)
+  end
+
 end

backend/spec/services/counts_export_spec.rb

@@ -193,11 +193,49 @@ RSpec.describe 'Counts download acknowledgement' do
     arguments = { file_token: 'file-token', attempt: receipt.fetch('download_attempt') }
     expect(CountsExportJobs.download_attempt(token, 'other', **arguments)).to be_nil
     expect(CountsExportJobs.download_attempt(token, 'owner', **arguments.merge(file_token: 'wrong'))).to be_nil
+    CountsExportJobs.download_attempt(token, 'owner', **arguments, status: 'streaming')
     CountsExportJobs.download_attempt(token, 'owner', **arguments, status: 'started')
     expect(CountsExportJobs.status(token, 'owner')['download_status']).to eq('started')
-    CountsExportJobs.download_attempt(token, 'owner')
+    expect(CountsExportJobs.download_attempt(token, 'owner')['download_attempt'])
+        .to eq(receipt['download_attempt'])
+    CountsExportJobs.download_attempt(token, 'owner', **arguments, status: 'failed')
+    fresh = CountsExportJobs.download_attempt(token, 'owner')
+    expect(fresh['download_attempt']).not_to eq(receipt['download_attempt'])
     expect(CountsExportJobs.download_attempt(token, 'owner', **arguments)).to be_nil
     CountsExportJobs.status(token, 'owner', cancel: true)
     expect(CountsExportJobs.download_attempt(token, 'owner')).to be_nil
   end
 end
+
+RSpec.describe CountsExportDownload::ResponseBody do
+  it 'acknowledges only after the first Rack yield and closes its bounded file body' do
+    Dir.mktmpdir('counts-body-spec') do |directory|
+      path = File.join(directory, 'file')
+      File.binwrite(path, 'x' * (64 * 1024 + 3))
+      states = []
+      body = described_class.new(path) { |state| states << state; true }
+      chunks = []
+      body.each do |chunk|
+        expect(states).not_to include('started') if chunks.empty?
+        chunks << chunk
+      end
+      expect(chunks.map(&:bytesize)).to eq([64 * 1024, 3])
+      expect(states.count('started')).to eq(1)
+      expect(body).not_to respond_to(:to_path)
+      expect { body.close }.not_to raise_error
+    end
+  end
+
+  it 'records a failed stream without converting an exception into a second HTTP response' do
+    Dir.mktmpdir('counts-body-spec') do |directory|
+      path = File.join(directory, 'file')
+      File.binwrite(path, 'data')
+      states = []
+      body = described_class.new(path) { |state| states << state; true }
+      expect { body.each { raise IOError, 'connection closed' } }.to raise_error(IOError)
+      expect(states).to include('failed')
+      expect(states).not_to include('started')
+      expect { body.close }.not_to raise_error
+    end
+  end
+end

backend/spec/services/counts_pivot_security_spec.rb

@@ -0,0 +1,169 @@
+require 'rails_helper'
+require 'csv'
+
+RSpec.describe 'Counts masked pivot contracts' do
+  let(:admin) { create(:user, :admin) }
+  let(:member) { create(:user, :member) }
+  let(:guest) { create(:user, :guest) }
+  let(:params) do
+    { 'resource' => 'posts', 'definition' => 'history', 'mode' => 'summary',
+      'locale' => 'ja', 'execution_at' => '2025-01-01T00:00:00Z',
+      'timezone' => 'UTC', 'rows' => 'tags', 'cols' => 'event_type', 'sort' => '' }
+  end
+  let!(:tags) do
+    ['A', 'B', 'C'].map do |name|
+      tag = create(:tag, primary_name: name)
+      TagVersion.create!(tag:, version_no: 1, event_type: 'create', name:,
+          category: 'general', aliases: '', parent_tag_ids: '',
+          names: { ja: { name:, script_code: 'Jpan', auto_generated: false, aliases: [] } },
+          created_at: Time.utc(2023))
+      tag
+    end
+  end
+
+  before do
+    [[0, 1], [0], [1, 2]].each_with_index do |indexes, index|
+      post = create(:post)
+      snapshot = indexes.map { { tag_id: tags[_1].id, version_no: 1, sections: [] } }
+      PostVersion.create!(post:, version_no: 1, event_type: 'create',
+          title: "hidden-title-#{ index }", url: post.url, tags: '', tags_json: snapshot,
+          parent_post_ids: '', created_by_user: member, created_at: Time.utc(2024, 1))
+      next if index == 2
+
+      PostVersion.create!(post:, version_no: 2, event_type: 'discard',
+          title: "private-discard-#{ index }", url: post.url, tags: '', tags_json: [],
+          parent_post_ids: '', created_by_user: member, created_at: Time.utc(2024, 2))
+    end
+  end
+
+  def result(user, extra = {})
+    HistoryAccess.set(user:) { CountsTableQuery.new(params.merge(extra)).call }
+  end
+
+  def grand(result)
+    result.fetch(:pivot_totals).find { _1[:row].empty? && _1[:column].empty? }.fetch(:count)
+  end
+
+  it 'preserves five tag occurrences after merging private tags and across pages' do
+    [nil, guest, member, admin].each do |user|
+      page = result(user)
+      expect(grand(page)).to eq(5)
+      expect(page[:cells].sum { _1[:count] }).to eq(5)
+      expect(page).not_to have_key(:excluded_count)
+      paged = result(user, 'limit' => '1')
+      expect(grand(paged)).to eq(5)
+      expect(result(user, 'limit' => '1', 'page' => '2')[:group_count])
+          .to eq(page[:group_count])
+    end
+    public_page = result(nil)
+    hidden = public_page[:cells].select { _1[:values].first == CountsAxes::MASKED_VALUE }
+    expect(hidden.sum { _1[:count] }).to eq(3)
+    expect(public_page[:axis_labels]['a0']).not_to have_key("internal:#{ tags.first.id }")
+    expect(public_page.to_json).not_to include('hidden-title', 'private-discard')
+  end
+
+  it 'retains distinct membership when tags are not a row axis' do
+    [nil, admin].each do |user|
+      expect(grand(result(user, 'rows' => 'event_type', 'cols' => 'tags'))).to eq(3)
+      expect(result(user, 'rows' => '', 'cols' => '')[:cells].sole[:count]).to eq(5)
+      details = result(user, 'mode' => 'details', 'rows' => '', 'cols' => '')
+      expect(details[:count]).to eq(user == admin ? 5 : 1)
+    end
+  end
+
+  it 'uses changed-record membership for totals and hides changes to private predecessors' do
+    post = PostVersion.order(:id).last.post
+    PostVersion.create!(post:, version_no: 2, event_type: 'update', title: 'changed-public-title',
+        url: post.url, tags: '',
+        tags_json: [{ tag_id: tags.last.id, version_no: 1, sections: [] }],
+        parent_post_ids: '', created_at: Time.utc(2024, 3))
+    restored = PostVersion.order(:id).first.post
+    PostVersion.create!(post: restored, version_no: 3, event_type: 'restore',
+        title: 'restored-public-title', url: restored.url, tags: '',
+        tags_json: [{ tag_id: tags.first.id, version_no: 1, sections: [] }], parent_post_ids: '',
+        created_at: Time.utc(2024, 4))
+    [nil, admin].each do |user|
+      page = result(user, 'rows' => 'tags changed_field', 'cols' => 'event_type')
+      expect(grand(page)).to eq(page[:cells].sum { _1[:count] })
+      expect(page[:cells]).not_to be_empty
+      if user.nil?
+        expect(page[:cells].any? { _1[:values][1] == CountsAxes::MASKED_VALUE }).to be(true)
+      end
+      expect(page.to_json).not_to include('hidden-title', 'private-discard')
+    end
+  end
+
+  it 'counts distinct changed records when tags are not additive row tuples' do
+    [nil, admin].each do |user|
+      page = result(user, 'rows' => 'changed_field', 'cols' => 'event_type')
+      expect(grand(page)).to eq(2)
+      if user.nil?
+        expect(page[:cells].map { _1[:values].first }.uniq).to eq([CountsAxes::MASKED_VALUE])
+      end
+      # Changed discard snapshots have no tags: no full cell contributes here.
+      empty = result(user, 'rows' => 'changed_field', 'cols' => 'tags')
+      expect(empty[:cells]).to be_empty
+      expect(grand(empty)).to eq(0)
+    end
+  end
+
+  it 'keeps NULL as a public data value rather than treating it as a rollup' do
+    post = create(:post)
+    PostVersion.create!(post:, version_no: 1, event_type: 'create', title: 'public bot',
+        url: post.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024, 5))
+    page = result(nil, 'rows' => 'user', 'cols' => 'event_type')
+    expect(page[:cells].select { _1[:values].first.nil? }.sum { _1[:count] }).to eq(1)
+    expect(grand(page)).to eq(6)
+    expect(page[:row_headers]).to include(hash_including(values: [nil], kind: 'data'))
+  end
+
+  it 'sorts public numeric values numerically alongside a masked group' do
+    post = PostVersion.order(:id).last.post
+    [2, 10].each do |number|
+      PostVersion.create!(post:, version_no: number, event_type: 'update',
+          title: 'public numeric sort', url: post.url, tags: '', tags_json: [],
+          parent_post_ids: '', created_at: Time.utc(2024, 5))
+    end
+    page = result(nil, 'rows' => 'version_no', 'cols' => '')
+    values = page[:row_headers].select { _1[:kind] == 'data' }.map { _1[:values].first }
+    expect(values.reject { _1 == CountsAxes::MASKED_VALUE }.map(&:to_i)).to eq([1, 2, 10])
+    expect(values.last).to eq(CountsAxes::MASKED_VALUE)
+  end
+
+  it 'rejects private predicates and supplied measure coordinates before SQL labels escape' do
+    probes = [{ 'resource_id' => PostVersion.first.post_id.to_s },
+        { 'tags' => 'A' }, { 'tag_refs.0.name' => 'A', 'tag_refs.0.tag_id' => tags.first.id.to_s },
+        { 'conditions.0.field' => 'title', 'conditions.0.operator' => 'eq',
+          'conditions.0.value' => 'hidden-title-0' },
+        { 'drill.0.field' => 'tags', 'drill.0.value' => "internal:#{ tags.first.id }" },
+        { 'cols' => 'tags', 'column_order.0.value' => CountsAxes::MASKED_VALUE }]
+    HistoryAccess.set(user: nil) do
+      probes.each do |probe|
+        expect { CountsTableQuery.new(params.merge(probe)) }
+            .to raise_error(CountsAxes::PrivateCondition)
+      end
+    end
+  end
+
+  it 'keeps public column sorting usable while guessed private coordinates match nothing' do
+    first = result(nil, 'rows' => 'event_type', 'cols' => 'tags')
+    probe = result(nil, 'rows' => 'event_type', 'cols' => 'tags',
+        'column_order.0.value' => "internal:#{ tags.first.id }")
+    expect(probe[:column_values]).to eq(first[:column_values])
+    expect(probe[:cells]).to eq(first[:cells])
+    expect(probe[:column_values_editable]).to be(false)
+    expect(probe.to_json).not_to include("internal:#{ tags.first.id }")
+  end
+
+  it 'exports masked coordinates and safe CSV labels from the same pivot' do
+    HistoryAccess.set(user: nil) do
+      payload = params.merge('format' => 'json')
+      json = JSON.parse(CountsTableExport.new(payload).body.to_a.join)
+      expect(json['data'].sum { _1['kind'] == 'data' ? _1['count'] : 0 }).to eq(5)
+      expect(json.to_json).not_to include('hidden-title', 'private-discard')
+      csv = CountsTableExport.new(payload.merge('format' => 'csv')).body.to_a.join
+      expect(csv).to include('********')
+      expect(csv).not_to include(CountsAxes::MASKED_VALUE, 'hidden-title', 'private-discard')
+    end
+  end
+end