#434

backend/app/controllers/counts_exports_controller.rb

@@ -99,6 +99,11 @@ class CountsExportsController < ApplicationController
             first = false
           end
         end
+      rescue IOError, SystemCallError => error
+        Rails.logger.warn("counts_download_stream_failure: #{ error.class }")
+        HistoryAccess.set(user: authority) do
+          CountsExportJobs.download_attempt(job, owner, **arguments, status: 'failed')
+        end
       ensure
         stream.close
       end

backend/app/controllers/counts_tables_controller.rb

@@ -18,6 +18,10 @@ class CountsTablesController < ApplicationController
   def index
     resolve_locale!
     render json: CountsTableQuery.new(request.query_parameters).call
+  rescue CountsAxes::TooManyAxes
+    render json: { error: 'too_many_axes', max_axes: CountsAxes::MAX_AXES }, status: :bad_request
+  rescue CountsAxes::PrivateCondition
+    render json: { error: 'private_condition' }, status: :forbidden
   rescue CountsTableQuery::TooManyColumns
     render json: { error: 'too_many_columns', max_columns: CountsTableQuery::MAX_COLUMNS },
            status: :unprocessable_entity

backend/app/services/counts_axes.rb

@@ -68,6 +68,14 @@ class CountsAxes
     end
   end
 
+  def count_expression
+    return 'COUNT(DISTINCT v.id)' unless rows.any? { _1['field'] == 'tags' }
+
+    # Keep tag occurrences even when several private tags share one masked label.
+    "COUNT(DISTINCT v.id, COALESCE(CONCAT('internal:', tag_entry.tag_id), " +
+        "CONCAT('external:', tag_entry.external_tag_id)))"
+  end
+
   def joins(fields = all.map { _1['field'] })
     result = ''
     if fields.include?('tags')
@@ -85,9 +93,6 @@ class CountsAxes
       condition = @definition.changes.map do |field|
         "(changed.field_key = '#{ field }' AND #{ @population.changed_sql(field) })"
       end.join(' OR ')
-      unless HistoryAccess.admin?
-        condition = "(#{ condition }) OR NOT (#{ HistoryAccess.sql(@definition.model, 'v') })"
-      end
       result += " JOIN (#{ fields }) changed ON (#{ condition })"
     end
     result

backend/app/services/counts_export_jobs.rb

@@ -198,7 +198,7 @@ class CountsExportJobs
           state['expires_at'] > Time.current.to_f && HistoryAccess.authorised_artifact?(state))
       if attempt
         return unless state['download_attempt'] == attempt && state['token'] == file_token
-        return unless state['download_status'] == 'pending'
+        return unless ['pending', 'started'].include?(state['download_status'])
 
         state['download_status'] = status if status
       else

backend/app/services/counts_pivot_totals.rb

@@ -49,7 +49,7 @@ class CountsPivotTotals
           leaf = retained.map { @axes.expressions[_1] }
           selected = leaf.each_with_index.map { |expression, index| "#{ expression } AS k#{ index }" }
           inner = scope.group(*leaf.map { Arel.sql(_1) })
-              .select(Arel.sql((selected + ['COUNT(DISTINCT v.id) AS count']).join(', '))).to_sql
+              .select(Arel.sql((selected + ["#{ @axes.count_expression } AS count"]).join(', '))).to_sql
           coordinates = indexes.map { "k#{ retained.index(_1) }" }
           projected = coordinates.each_with_index.map { |key, index| "#{ key } AS v#{ index }" }
           output = ["#{ row_depth } AS row_depth", "#{ column_depth } AS column_depth"] +

backend/app/services/counts_population.rb

@@ -63,6 +63,8 @@ class CountsPopulation
   end
 
   def axis_condition(scope, entry)
+    raise CountsAxes::PrivateCondition if entry['value'] == CountsAxes::MASKED_VALUE
+
     key = entry.fetch('field')
     raise ArgumentError unless definition.axes(history).include?(key)
     if entry.key?('grain')

backend/app/services/counts_table_query.rb

@@ -88,7 +88,7 @@ class CountsTableQuery
             source.except(:select).select('COUNT(DISTINCT v.id) AS count').to_sql
           else
             source.except(:select).group(*expressions.map { Arel.sql(_1) })
-                .select(Arel.sql((selected + ['COUNT(DISTINCT v.id) AS count']).join(', ')))
+                .select(Arel.sql((selected + ["#{ axes.count_expression } AS count"]).join(', ')))
                 .to_sql
           end
     prefix = "WITH cells AS (#{ sql })"
@@ -359,7 +359,7 @@ class CountsTableQuery
     end
     selected = row_expressions.each_with_index.map { |expression, index|
       "#{ expression } AS a#{ index }"
-    } + ['COUNT(DISTINCT v.id) AS count']
+    } + ["#{ axes.count_expression } AS count"]
     source = source.group(*row_expressions.map { Arel.sql(_1) }) if row_expressions.any?
     source.select(Arel.sql(selected.join(', '))).to_sql
   end

backend/spec/requests/counts_table_spec.rb

@@ -232,7 +232,7 @@ RSpec.describe 'Counts table API', type: :request do
       it 'does not resolve private condition names even when the result set is empty' do
         TagVersion.create!(tag:, version_no: 2, event_type: 'discard', name: 'secret tag',
             category: 'general', aliases: '', parent_tag_ids: '',
-            names: { ja: { name: 'secret tag', script_code: 'Jpan', aliases: [] } },
+            names: { ja: { name: 'secret tag', script_code: 'Jpan', auto_generated: false, aliases: [] } },
             created_at: Time.utc(2024, 2))
         result = query(axis('tags', "internal:#{ tag.id }").merge(definition: 'latest'))
         expect(result['versions']).to be_empty
@@ -245,4 +245,25 @@ RSpec.describe 'Counts table API', type: :request do
       end
     end
   end
+  it 'includes private history in the public timeline without an exclusion count' do
+    PostVersion.create!(post: record, version_no: 4, event_type: 'discard', url: record.url,
+        title: 'hidden title', tags: '', tags_json: [], parent_post_ids: '',
+        created_at: Time.utc(2024, 3))
+    result = query(mode: 'summary', rows: 'created_at:month', cols: 'event_type')
+    expect(result['excluded_count']).to be_nil
+    expect(result['count']).to eq(4)
+    expect(result['cells'].sum { _1['count'] }).to eq(4)
+    expect(response.body).not_to include('hidden title')
+    masked = query(mode: 'summary', rows: 'resource_id', cols: '')
+    expect(masked['cells'].map { _1['values'] }).to eq([[CountsAxes::MASKED_VALUE]])
+    expect(masked['cells'].first['count']).to eq(4)
+    expect(query['versions']).to be_empty
+  end
+
+  it 'rejects private summary predicates instead of exposing a count oracle' do
+    get '/counts/query', params: base.merge(mode: 'summary', resource_id: record.id.to_s)
+    expect(response).to have_http_status(:forbidden)
+    expect(json).to eq('error' => 'private_condition')
+  end
+
 end

backend/spec/requests/post_locale_filter_identity_spec.rb

@@ -21,29 +21,7 @@ RSpec.describe 'Post filter identity across locales', type: :request do
 
   ['nico:shared', 'union_alias', 'not:nico:shared'].each do |literal|
     it "preserves the complete match set for #{ literal }" do
-      reference_params = { tag_refs: '' }
-
-      references.each_with_index do |reference, index|
-        %w[name tag_id external_tag_id negative missing].each do |field|
-          value = reference[field]
-          next if value.nil?
-          next if field == 'missing' && value == false
-
-          reference_params["tag_refs.#{index}.#{field}"] =
-              case value
-              when true
-                '1'
-              when false
-                '0'
-              else
-                value.to_s
-              end
-        end
-      end
-
-      get '/posts', params: {
-          locale: 'ain-Kana',
-          tags: references.map { _1.fetch('name') }.join(' ') }.merge(reference_params)
+      get '/posts', params: { locale: 'ja', tags: literal }
       expect(response).to have_http_status(:ok)
       original_ids = json.fetch('posts').map { _1.fetch('id') }
       expected = literal.start_with?('not:') ? [other_post.id] :
@@ -56,9 +34,18 @@ RSpec.describe 'Post filter identity across locales', type: :request do
       references = json.fetch('references')
       expect(references.first).to include('tag_id' => tag.id, 'external_tag_id' => external.id)
 
-      get '/posts', params: {
-          locale: 'ain-Kana', tags: references.map { _1.fetch('name') }.join(' '),
-          tag_refs: references.to_json }
+      reference_params = { tag_refs: '' }
+      references.each_with_index do |reference, index|
+        ['name', 'tag_id', 'external_tag_id', 'negative', 'missing'].each do |field|
+          value = reference[field]
+          next if value.nil? || (field == 'missing' && value == false)
+
+          reference_params["tag_refs.#{ index }.#{ field }"] =
+              [true, false].include?(value) ? (value ? '1' : '0') : value.to_s
+        end
+      end
+      get '/posts', params: { locale: 'ain-Kana',
+          tags: references.map { _1.fetch('name') }.join(' ') }.merge(reference_params)
       expect(response).to have_http_status(:ok)
       expect(json.fetch('posts').map { _1.fetch('id') }).to match_array(original_ids)
     end

backend/spec/requests/tag_description_wiki_spec.rb

@@ -63,7 +63,7 @@ RSpec.describe 'Tag description Wiki compatibility', type: :request do
     end
 
     it 'does not fall back when the new current revision is missing' do
-      versions.last.destroy!
+      TagDescriptionVersion.where(id: versions.last.id).delete_all
       without_legacy_reads { get "/wiki/#{ description.id }" }
       expect(response).to have_http_status(:not_found)
     end

backend/spec/services/counts_export_spec.rb

@@ -171,3 +171,33 @@ RSpec.describe 'Counts export contracts' do
     end
   end
 end
+
+RSpec.describe 'Counts download acknowledgement' do
+  around do |example|
+    Dir.mktmpdir('counts-ack-spec') do |path|
+      allow(CountsExportDownload).to receive(:directory).and_return(Pathname.new(path))
+      HistoryAccess.set(user: nil) { example.run }
+    end
+  end
+
+  it 'binds an acknowledgement to the owner, job, file token and fresh attempt' do
+    CountsExportJobs.pulse
+    payload = { 'resource' => 'posts', 'definition' => 'history', 'mode' => 'details',
+                'format' => 'json', 'locale' => 'ja' }
+    token = CountsExportJobs.start(payload, 'owner', 'ja')
+    id = CountsExportJobs.authorised_id(token, 'owner')
+    state = CountsExportJobs.read(id).merge('status' => 'ready', 'token' => 'file-token')
+    CountsExportJobs.write(state)
+    receipt = CountsExportJobs.download_attempt(token, 'owner')
+    expect(receipt['download_status']).to eq('pending')
+    arguments = { file_token: 'file-token', attempt: receipt.fetch('download_attempt') }
+    expect(CountsExportJobs.download_attempt(token, 'other', **arguments)).to be_nil
+    expect(CountsExportJobs.download_attempt(token, 'owner', **arguments.merge(file_token: 'wrong'))).to be_nil
+    CountsExportJobs.download_attempt(token, 'owner', **arguments, status: 'started')
+    expect(CountsExportJobs.status(token, 'owner')['download_status']).to eq('started')
+    CountsExportJobs.download_attempt(token, 'owner')
+    expect(CountsExportJobs.download_attempt(token, 'owner', **arguments)).to be_nil
+    CountsExportJobs.status(token, 'owner', cancel: true)
+    expect(CountsExportJobs.download_attempt(token, 'owner')).to be_nil
+  end
+end