#434

backend/app/services/counts_axes.rb

@@ -61,7 +61,11 @@ class CountsAxes
             expression, axis.fetch('grain', 'month'), @population.timezone) : expression
       end
       if !HistoryAccess.admin? && !PUBLIC_FIELDS.include?(field)
-        expression = "CASE WHEN #{ HistoryAccess.sql(@definition.model, 'v') } " +
+        visible = HistoryAccess.sql(@definition.model, 'v')
+        if field == 'changed_field'
+          visible = "(#{ visible }) AND (#{ HistoryAccess.sql(@definition.model, 'previous') })"
+        end
+        expression = "CASE WHEN #{ visible } " +
             "THEN CAST((#{ expression }) AS CHAR) ELSE '#{ MASKED_VALUE }' END"
       end
       expression
@@ -72,8 +76,10 @@ class CountsAxes
     return 'COUNT(DISTINCT v.id)' unless rows.any? { _1['field'] == 'tags' }
 
     # Keep tag occurrences even when several private tags share one masked label.
-    "COUNT(DISTINCT v.id, COALESCE(CONCAT('internal:', tag_entry.tag_id), " +
-        "CONCAT('external:', tag_entry.external_tag_id)))"
+    values = ['v.id', "COALESCE(CONCAT('internal:', tag_entry.tag_id), " +
+        "CONCAT('external:', tag_entry.external_tag_id))"]
+    values << 'changed.field_key' if rows.any? { _1['field'] == 'changed_field' }
+    "COUNT(DISTINCT #{ values.join(', ') })"
   end
 
   def joins(fields = all.map { _1['field'] })

backend/app/services/counts_export_download.rb

@@ -13,6 +13,46 @@ class CountsExportDownload
   class TimedOut < StandardError; end
   class StorageFull < StandardError; end
 
+  # Rack body deliberately has no to_path: sendfile middleware must not bypass ACK.
+  class ResponseBody
+    def initialize(path, &state)
+      File.open(path, 'rb') { |file| file.stat }
+      @path = path
+      @stream = nil
+      @closed = false
+      @state = state
+    end
+
+    def each
+      return if @closed
+
+      @stream = File.open(@path, 'rb')
+      first = true
+      while (chunk = @stream.read(64 * 1024))
+        raise IOError, 'Download no longer authorised' unless @state.call('active')
+
+        yield chunk
+        if first
+          # This confirms yielding to Rack, never browser receipt or saving.
+          @state.call('started')
+          first = false
+        end
+      end
+      @state.call('failed') if first
+    rescue StandardError => error
+      Rails.logger.warn("counts_download_stream_failure: #{ error.class }")
+      @state.call('failed')
+      raise
+    ensure
+      close
+    end
+
+    def close
+      @closed = true
+      @stream.close if @stream && !@stream.closed?
+    end
+  end
+
   def self.directory
     Rails.root.join(ENV.fetch('COUNTS_EXPORT_DIRECTORY', 'tmp/counts_exports'))
   end

backend/app/services/counts_export_jobs.rb

@@ -45,6 +45,20 @@ class CountsExportJobs
     write(state)
   end
 
+  def self.failure_code(error)
+    case error
+    when Cancelled then 'cancelled'
+    when CountsAxes::TooManyAxes then 'too_many_axes'
+    when CountsAxes::PrivateCondition then 'private_condition'
+    when CountsTableQuery::TooManyColumns then 'too_many_columns'
+    when ActiveRecord::StatementTimeout then 'counts_query_timeout'
+    when CountsExportDownload::TimedOut then 'generation_timeout'
+    when CountsExportDownload::TooLarge then 'export_too_large'
+    when CountsExportDownload::StorageFull, Errno::ENOSPC then 'export_storage_full'
+    else 'generation_failed'
+    end
+  end
+
   def self.directory = CountsExportDownload.directory.join('jobs')
   def self.verifier = Rails.application.message_verifier('counts_export_job')
 
@@ -176,6 +190,12 @@ class CountsExportJobs
       return unless state && state['owner'] == owner && state['expires_at'] > Time.current.to_f
 
       expire_stalled(state)
+      if (state['status'] == 'ready' &&
+          ['pending', 'streaming'].include?(state['download_status']) &&
+          state.fetch('download_expires_at', 0) <= Time.current.to_f)
+        state['download_status'] = 'timeout'
+        write(state)
+      end
 
       if cancel && ['queued', 'running', 'ready'].include?(state['status'])
         state['status'] = state['status'] == 'running' ? 'cancelling' : 'cancelled'
@@ -183,8 +203,11 @@ class CountsExportJobs
         state.delete('token')
         write(state)
       end
-      state.slice('status', 'phase', 'processed', 'total', 'format', 'token', 'error',
+      result = state.slice('status', 'phase', 'processed', 'total', 'format', 'token', 'error',
           'updated_at', 'heartbeat_at', 'download_attempt', 'download_status')
+      result['max_axes'] = CountsAxes::MAX_AXES if state['error'] == 'too_many_axes'
+      result['max_columns'] = CountsTableQuery::MAX_COLUMNS if state['error'] == 'too_many_columns'
+      result
     end
   end
 
@@ -198,18 +221,53 @@ class CountsExportJobs
           state['expires_at'] > Time.current.to_f && HistoryAccess.authorised_artifact?(state))
       if attempt
         return unless state['download_attempt'] == attempt && state['token'] == file_token
-        return unless ['pending', 'started'].include?(state['download_status'])
+        return unless state['download_status'] == 'started' ||
+            state.fetch('download_expires_at', 0) > Time.current.to_f
 
-        state['download_status'] = status if status
+        transitions = { 'streaming' => ['pending'], 'started' => ['streaming'],
+                        'failed' => ['pending', 'streaming', 'started'],
+                        'unavailable' => ['pending', 'streaming'] }
+        if status == 'active'
+          return state.slice('download_attempt', 'download_status', 'token') if
+              ['streaming', 'started'].include?(state['download_status'])
+          return
+        end
+        if status
+          return unless transitions.fetch(status, []).include?(state['download_status'])
+
+          state['download_status'] = status
+        else
+          return unless state['download_status'] == 'pending'
+        end
       else
+        # Repeated acceptance POSTs do not invalidate an in-flight GET or issue duplicates.
+        if (state['download_status'] == 'started' ||
+            (['pending', 'streaming'].include?(state['download_status']) &&
+             state.fetch('download_expires_at', 0) > Time.current.to_f))
+          return state.slice('download_attempt', 'download_status', 'token')
+        end
         state['download_attempt'] = SecureRandom.hex(24)
         state['download_status'] = 'pending'
+        state['download_expires_at'] = Time.current.to_f + 45
       end
       write(state)
       state.slice('download_attempt', 'download_status', 'token')
     end
   end
 
+  # GET already verified authority. During streaming, observe cancellation without
+  # repeating the six-table policy revision aggregate for every 64KiB chunk.
+  def self.stream_active?(token, owner, file_token:, attempt:)
+    id = authorised_id(token, owner)
+    state = id && read(id)
+    state && state['owner'] == owner && state['status'] == 'ready' &&
+        state['expires_at'] > Time.current.to_f && state['token'] == file_token &&
+        state['download_attempt'] == attempt &&
+        (state['download_status'] == 'started' ||
+         (state['download_status'] == 'streaming' &&
+          state.fetch('download_expires_at', 0) > Time.current.to_f))
+  end
+
   def self.checkpoint(id, processed = nil, total = nil, phase: nil)
     synchronise do
       state = read(id)
@@ -398,7 +456,7 @@ class CountsExportJobs
         if current
           unless current['status'] == 'failed'
             current['status'] = error.is_a?(Cancelled) ? 'cancelled' : 'failed'
-            current['error'] = error.class.name
+            current['error'] = failure_code(error)
           end
           current['updated_at'] = Time.current.to_f
           write(current)

backend/app/services/counts_pivot_totals.rb

@@ -1,4 +1,4 @@
-# Re-aggregate the population, never add up already grouped cells (multi-value axes overlap).
+# Re-aggregate distinct membership; tag row tuples alone use additive contributions.
 class CountsPivotTotals
   def initialize(population, axes, row_headers, column_headers)
     @population = population
@@ -8,8 +8,12 @@ class CountsPivotTotals
   end
 
   def call
-    # MySQL's unlimited LIMIT prevents CTE merging: authorise once for all branches.
-    basis = @population.scope.except(:select, :order).select('v.*')
+    # Materialise full-cell membership once, including omitted multi-value axes.
+    membership = @population.scope.except(:select, :order)
+    joins = @axes.joins
+    membership = membership.joins(joins) unless joins.empty?
+    # Retain exactly the records contributing a full cell, including changed_field.
+    basis = membership.select('v.*').distinct
         .limit(18_446_744_073_709_551_615).to_sql
     queries = @rows.flat_map do |row_depth, headers|
       @columns.filter_map do |column_depth, columns|
@@ -18,7 +22,7 @@ class CountsPivotTotals
         indexes = (0...row_depth).to_a +
             (0...column_depth).map { @axes.rows.length + _1 }
         expressions = indexes.map { @axes.expressions[_1] }
-        scope = @population.scope.except(:select, :order, :where, :from).from('authorised v')
+        scope = @population.scope.except(:select, :order, :where, :from).from('population_members v')
         additive = @axes.rows.any? { _1['field'] == 'tags' }
         retained = additive ? (0...@axes.rows.length).to_a | indexes : indexes
         joins = @axes.joins(retained.map { @axes.all[_1].fetch('field') })
@@ -47,9 +51,12 @@ class CountsPivotTotals
         if additive
           # Sum full row tuples, not page cells; preserve DISTINCT inside each tuple.
           leaf = retained.map { @axes.expressions[_1] }
-          selected = leaf.each_with_index.map { |expression, index| "#{ expression } AS k#{ index }" }
+          selected = leaf.each_with_index.map do |expression, index|
+            "#{ expression } AS k#{ index }"
+          end
+          selected << "#{ @axes.count_expression } AS count"
           inner = scope.group(*leaf.map { Arel.sql(_1) })
-              .select(Arel.sql((selected + ["#{ @axes.count_expression } AS count"]).join(', '))).to_sql
+              .select(Arel.sql(selected.join(', '))).to_sql
           coordinates = indexes.map { "k#{ retained.index(_1) }" }
           projected = coordinates.each_with_index.map { |key, index| "#{ key } AS v#{ index }" }
           output = ["#{ row_depth } AS row_depth", "#{ column_depth } AS column_depth"] +
@@ -67,7 +74,7 @@ class CountsPivotTotals
 
     sql = queries.map { "(#{ _1 })" }.join(' UNION ALL ')
     @population.definition.model.connection.select_all(
-        "WITH authorised AS (#{ basis }) " +
+        "WITH population_members AS (#{ basis }) " +
         "SELECT /*+ MAX_EXECUTION_TIME(10000) */ * FROM (#{ sql }) totals").map do |record|
       row_depth = record['row_depth'].to_i
       column_depth = record['column_depth'].to_i

backend/app/services/counts_population.rb

@@ -57,7 +57,8 @@ class CountsPopulation
     "(previous.id IS NOT NULL AND NOT (v.#{ column } <=> previous.#{ column }))"
   end
 
-  # Unrestricted predicates are used only for numerical totals, never record output.
+  # Aggregate sources are unrestricted; CountsAxes masks their output coordinates.
+  # Never serialise raw records from an unrestricted population.
   def access_sql(model, row)
     @authorisation_flg ? HistoryAccess.sql(model, row) : 'TRUE'
   end
@@ -67,6 +68,16 @@ class CountsPopulation
 
     key = entry.fetch('field')
     raise ArgumentError unless definition.axes(history).include?(key)
+    value = entry['value']
+    unless value.nil?
+      valid = case definition.fields[key]
+              when :integer
+                value.is_a?(Integer) || (value.is_a?(String) && value.match?(/\A-?[0-9]+\z/))
+              when :boolean then [true, false].include?(value)
+              else value.is_a?(String)
+              end
+      raise ArgumentError unless valid
+    end
     if entry.key?('grain')
       unless definition.fields[key] == :datetime && CountsAxes::GRAINS.include?(entry['grain'])
         raise ArgumentError

backend/app/services/counts_table_export.rb

@@ -253,6 +253,8 @@ class CountsTableExport
   end
 
   def summary_value(value, index, page)
+    return '********' if value == CountsAxes::MASKED_VALUE
+
     field = @query.axes.all.fetch(index).fetch('field')
     return @labels.fetch('bot', '') if field == 'user' && value.nil?
     if ['language_code', 'source_language_code'].include?(field)

backend/app/services/counts_table_query.rb

@@ -49,12 +49,18 @@ class CountsTableQuery
       result = metadata.merge(mode == 'summary' ? summary : details)
       visible = mode == 'summary' ? result[:visible_output_count] : result[:count]
       totals = output_totals(visible)
-      result.merge(visible_output_count: visible,
+      response = result.merge(visible_output_count: visible,
           **totals,
           output_count_scope: 'before_authorisation',
           output_count_unit: 'regular_result_rows',
           page_grand_total_repeated: mode == 'summary' && axes.rows.any? &&
               result[:group_count] > limit)
+      if mode == 'summary'
+        response.except!(:excluded_count, :excluded_count_scope)
+        response[:column_values_editable] = HistoryAccess.admin? ||
+            !result.fetch(:column_values, []).flatten.include?(CountsAxes::MASKED_VALUE)
+      end
+      response
     end
   end
 
@@ -228,17 +234,17 @@ class CountsTableQuery
 
     fields = QueryState.records(@params, 'conditions') + QueryState.records(@params, 'drill')
     private_fields = fields.any? { !CountsAxes::PUBLIC_FIELDS.include?(_1['field']) }
-    private_parameters = ['tags', 'tag_refs', 'tag', 'kind', 'resource_id', 'user_id',
-                          'name_scope'].any? do |key|
+    keys = ['tags', 'tag_refs', 'tag', 'kind', 'resource_id', 'user_id']
+    private_parameters = keys.any? do |key|
       @params.any? { |name, value| (name == key || name.start_with?("#{ key }.")) && value != '' }
     end
     raise CountsAxes::PrivateCondition if private_fields || private_parameters
 
     ['column_order', 'measures'].each do |key|
-      next if QueryState.records(@params, key).empty?
-      columns = QueryState.axes(@params.fetch('cols', ''))
-      raise CountsAxes::PrivateCondition if columns.any? do |axis|
-        !CountsAxes::PUBLIC_FIELDS.include?(axis['field'])
+      # Coordinates are compared only to already masked expressions, never raw
+      # values. Public coordinates remain sortable; a guessed private ID cannot match.
+      raise CountsAxes::PrivateCondition if QueryState.records(@params, key).any? do |record|
+        record['value'] == CountsAxes::MASKED_VALUE
       end
     end
   end
@@ -350,6 +356,11 @@ class CountsTableQuery
   def measure_relation values
     row_expressions = axes.expressions.first(axes.rows.length)
     source = summary_population.scope.except(:select, :order)
+    # Omitted multi-value column axes still determine full-cell membership.
+    membership = summary_population.scope.except(:select, :order)
+    all_joins = axes.joins
+    membership = membership.joins(all_joins) unless all_joins.empty?
+    source = source.where("v.id IN (#{ membership.select('v.id').distinct.to_sql })")
     fields = axes.rows + axes.columns.first(values.length)
     joins = axes.joins(fields.map { _1.fetch('field') })
     source = source.joins(joins) unless joins.empty?
@@ -367,7 +378,15 @@ class CountsTableQuery
   def axis_order_sql key, direction = 'asc', qualifier: nil
     column = qualifier ? "#{ qualifier }.#{ key }" : key
     expression = column
-    if axes.all.fetch(key.delete_prefix('a').to_i).fetch('field') == 'category'
+    field = axes.all.fetch(key.delete_prefix('a').to_i).fetch('field')
+    if !HistoryAccess.admin? && [:integer, :boolean].include?(@definition.fields[field])
+      masked = @connection.quote(CountsAxes::MASKED_VALUE)
+      expression = "CASE WHEN #{ column } <=> #{ masked } THEN NULL " +
+          "ELSE CAST(#{ column } AS SIGNED) END"
+      return "#{ column } IS NULL ASC, (#{ column } <=> #{ masked }) ASC, " +
+          "#{ expression } #{ direction.upcase }"
+    end
+    if field == 'category'
       # Tag's enum preserves the same category order as TagsController.
       cases = Tag.categories.keys.each_with_index.map { |category, index|
         "WHEN #{ @connection.quote(category) } THEN #{ index }"
@@ -418,7 +437,10 @@ class CountsTableQuery
       labels = case field
                when 'tags' then tag_labels(values)
                when 'user'
-                 User.where(id: values).pluck(:id, :name).to_h.transform_keys(&:to_s)
+                 actors = HistoryAccess.visible(population.scope, 'v').except(:select, :order)
+                     .select('v.created_by_user_id')
+                 User.where(id: values).where(id: actors)
+                     .pluck(:id, :name).to_h.transform_keys(&:to_s)
                else {}
                end
       [field, labels]