#434

backend/spec/models/history_access_spec.rb

@@ -67,3 +67,64 @@ RSpec.describe HistoryAccess do
     end
   end
 end
+
+RSpec.describe 'Counts artifact policy and external references' do
+  it 'rejects files created before the masked-summary policy' do
+    HistoryAccess.set(user: nil) do
+      metadata = HistoryAccess.authority.except('artifact_policy')
+      expect(HistoryAccess.authorised_artifact?(metadata)).to be(false)
+    end
+  end
+
+  it 'does not expose a private linked tag name through an external history snapshot' do
+    tag = create(:tag, primary_name: 'private-linked-name')
+    TagVersion.create!(tag:, version_no: 1, event_type: 'discard', name: 'private-linked-name',
+        category: 'general', aliases: '', parent_tag_ids: '', created_at: Time.utc(2024),
+        names: { ja: { name: 'private-linked-name', script_code: 'Jpan',
+                       auto_generated: false, aliases: [] } })
+    external = create(:external_tag, name: 'public-external')
+    row = NicoTagVersion.create!(external_tag: external, version_no: 1, event_type: 'create',
+        name: 'nico:public-external', linked_tags: 'private-linked-name',
+        created_at: Time.utc(2024))
+    post = create(:post)
+    snapshot = PostVersion.create!(post:, version_no: 1, event_type: 'create',
+        title: 'external reference', url: post.url, tags: '',
+        tags_json: [{ external_tag_id: external.id }], parent_post_ids: '',
+        created_at: Time.utc(2024, 2))
+    HistoryAccess.set(user: nil) do
+      expect(HistoryAccess.visible(NicoTagVersion.where(id: row.id))).to be_empty
+      expect(HistoryAccess.visible(PostVersion.where(id: snapshot.id))).to be_empty
+    end
+    HistoryAccess.set(user: create(:user, :admin)) do
+      expect(HistoryAccess.visible(NicoTagVersion.where(id: row.id))).to exist
+    end
+    public_row = NicoTagVersion.create!(external_tag: external, version_no: 2, event_type: 'update',
+        name: 'nico:public-external', linked_tags: 'private-linked-name-extra',
+        created_at: Time.utc(2024, 7))
+    HistoryAccess.set(user: nil) do
+      expect(HistoryAccess.visible(NicoTagVersion.where(id: public_row.id))).to exist
+      expect(HistoryAccess.visible(PostVersion.where(id: snapshot.id))).to be_empty
+    end
+  end
+
+  it 'propagates the recorded tag version policy to its post snapshot' do
+    parent = create(:tag, primary_name: 'private parent')
+    child = create(:tag, primary_name: 'referencing child')
+    [[parent, 'discard', '', 'private parent'],
+     [child, 'create', parent.id.to_s, 'referencing child']].each do |tag, event, parents, name|
+      TagVersion.create!(tag:, version_no: 1, event_type: event, name:,
+          category: 'general', aliases: '', parent_tag_ids: parents, created_at: Time.utc(2023),
+          names: { ja: { name:, script_code: 'Jpan',
+                         auto_generated: false, aliases: [] } })
+    end
+    post = create(:post)
+    row = PostVersion.create!(post:, version_no: 1, event_type: 'create', title: 'snapshot',
+        url: post.url, tags: '', parent_post_ids: '',
+        tags_json: [{ tag_id: child.id, version_no: 1, name: 'referencing child', sections: [] }],
+        created_at: Time.utc(2024))
+    HistoryAccess.set(user: nil) do
+      expect(HistoryAccess.visible(PostVersion.where(id: row.id))).to be_empty
+    end
+  end
+
+end

backend/spec/requests/counts_download_spec.rb

@@ -0,0 +1,98 @@
+require 'rails_helper'
+require 'tmpdir'
+
+RSpec.describe 'Counts native GET acknowledgement', type: :request do
+  include ActiveSupport::Testing::TimeHelpers
+  around do |example|
+    Dir.mktmpdir('counts-download-request') do |path|
+      allow(CountsExportDownload).to receive(:directory).and_return(Pathname.new(path))
+      example.run
+    end
+  end
+
+  def ready_job(format)
+    CountsExportJobs.pulse
+    post '/counts/export/jobs', params: { resource: 'posts', definition: 'history',
+        mode: 'details', columns: 'resource_id', locale: 'ja', format: }
+    expect(response).to have_http_status(:accepted)
+    job = response.parsed_body.fetch('job')
+    signature = CountsExportJobs.verifier.verified(job, purpose: 'counts_export_job')
+    owner = signature.fetch('owner')
+    exporter = instance_double(CountsExporter, resource: 'posts', format:, body: ['payload'])
+    file = HistoryAccess.set(user: nil) { CountsExportDownload.prepare(exporter, owner) }
+    state = CountsExportJobs.read(signature.fetch('id'))
+    state.merge!('status' => 'ready', 'token' => file)
+    CountsExportJobs.write(state)
+    post "/counts/export/jobs/#{ job }/download"
+    expect(response).to have_http_status(:ok)
+    [job, file, response.parsed_body.fetch('download_attempt')]
+  end
+
+  ['csv', 'json'].each do |format|
+    it "acknowledges only an authorised #{ format } GET body, not HEAD or acceptance" do
+      job, file, attempt = ready_job(format)
+      get "/counts/export/jobs/#{ job }"
+      expect(response.parsed_body['download_status']).to eq('pending')
+      head "/counts/export/files/#{ file }", params: { job:, attempt: }
+      expect(response).to have_http_status(:method_not_allowed)
+      get "/counts/export/jobs/#{ job }"
+      expect(response.parsed_body['download_status']).to eq('pending')
+      get "/counts/export/files/#{ file }", params: { job:, attempt: }
+      expect(response).to have_http_status(:ok)
+      expect(response.headers['Content-Disposition']).to include("counts-posts.#{ format }")
+      expect(response.body).to eq('payload')
+      get "/counts/export/jobs/#{ job }"
+      expect(response.parsed_body['download_status']).to eq('started')
+      get "/counts/export/files/#{ file }", params: { job:, attempt: }
+      expect(response).to have_http_status(:not_found)
+    end
+  end
+
+  it 'rejects wrong attempts, cancellation and a newly discarded resource' do
+    job, file, attempt = ready_job('json')
+    get "/counts/export/files/#{ file }", params: { job:, attempt: 'wrong' }
+    expect(response).to have_http_status(:not_found)
+    delete "/counts/export/jobs/#{ job }"
+    expect(response).to have_http_status(:ok)
+    get "/counts/export/files/#{ file }", params: { job:, attempt: }
+    expect(response).to have_http_status(:not_found)
+    job, file, attempt = ready_job('json')
+    record = create(:post)
+    PostVersion.create!(post: record, version_no: 1, event_type: 'discard', title: 'private',
+        url: record.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024))
+    get "/counts/export/files/#{ file }", params: { job:, attempt: }
+    expect(response).to have_http_status(:not_found)
+    expect(response.body).not_to include('payload', 'private')
+  end
+  it 'does not let a second browser session inspect the job or download its file' do
+    job, file, attempt = ready_job('json')
+    other = ActionDispatch::Integration::Session.new(Rails.application)
+    other.get("/counts/export/jobs/#{ job }")
+    expect(other.response).to have_http_status(:not_found)
+    other.get("/counts/export/files/#{ file }", params: { job:, attempt: })
+    expect(other.response).to have_http_status(:not_found)
+    expect(other.response.body).not_to include('payload')
+  end
+
+  it 'does not acknowledge an expired attempt' do
+    job, file, attempt = ready_job('json')
+    travel_to(Time.current + 46.seconds) do
+      get "/counts/export/files/#{ file }", params: { job:, attempt: }
+      expect(response).to have_http_status(:not_found)
+      get "/counts/export/jobs/#{ job }"
+      expect(response.parsed_body['download_status']).to eq('timeout')
+    end
+  end
+
+  it 'records a missing file without accepting the GET as started' do
+    job, file, attempt = ready_job('json')
+    owner = CountsExportJobs.verifier.verified(job, purpose: 'counts_export_job').fetch('owner')
+    metadata = HistoryAccess.set(user: nil) { CountsExportDownload.find(file, owner) }
+    File.unlink(metadata.fetch('path'))
+    get "/counts/export/files/#{ file }", params: { job:, attempt: }
+    expect(response).to have_http_status(:not_found)
+    get "/counts/export/jobs/#{ job }"
+    expect(response.parsed_body['download_status']).to eq('unavailable')
+  end
+
+end

backend/spec/requests/counts_table_spec.rb

@@ -266,4 +266,12 @@ RSpec.describe 'Counts table API', type: :request do
     expect(json).to eq('error' => 'private_condition')
   end
 
+  it 'reports the combined axis limit without replacing it with a generic 400' do
+    get '/counts/query', params: base.merge(mode: 'summary', sort: '',
+        rows: 'resource_id version_no event_type user created_at:month',
+        cols: 'title url thumbnail_base video_ms')
+    expect(response).to have_http_status(:bad_request)
+    expect(response.parsed_body).to eq('error' => 'too_many_axes', 'max_axes' => 8)
+  end
+
 end

backend/spec/services/counts_export_spec.rb

@@ -193,11 +193,49 @@ RSpec.describe 'Counts download acknowledgement' do
     arguments = { file_token: 'file-token', attempt: receipt.fetch('download_attempt') }
     expect(CountsExportJobs.download_attempt(token, 'other', **arguments)).to be_nil
     expect(CountsExportJobs.download_attempt(token, 'owner', **arguments.merge(file_token: 'wrong'))).to be_nil
+    CountsExportJobs.download_attempt(token, 'owner', **arguments, status: 'streaming')
     CountsExportJobs.download_attempt(token, 'owner', **arguments, status: 'started')
     expect(CountsExportJobs.status(token, 'owner')['download_status']).to eq('started')
-    CountsExportJobs.download_attempt(token, 'owner')
+    expect(CountsExportJobs.download_attempt(token, 'owner')['download_attempt'])
+        .to eq(receipt['download_attempt'])
+    CountsExportJobs.download_attempt(token, 'owner', **arguments, status: 'failed')
+    fresh = CountsExportJobs.download_attempt(token, 'owner')
+    expect(fresh['download_attempt']).not_to eq(receipt['download_attempt'])
     expect(CountsExportJobs.download_attempt(token, 'owner', **arguments)).to be_nil
     CountsExportJobs.status(token, 'owner', cancel: true)
     expect(CountsExportJobs.download_attempt(token, 'owner')).to be_nil
   end
 end
+
+RSpec.describe CountsExportDownload::ResponseBody do
+  it 'acknowledges only after the first Rack yield and closes its bounded file body' do
+    Dir.mktmpdir('counts-body-spec') do |directory|
+      path = File.join(directory, 'file')
+      File.binwrite(path, 'x' * (64 * 1024 + 3))
+      states = []
+      body = described_class.new(path) { |state| states << state; true }
+      chunks = []
+      body.each do |chunk|
+        expect(states).not_to include('started') if chunks.empty?
+        chunks << chunk
+      end
+      expect(chunks.map(&:bytesize)).to eq([64 * 1024, 3])
+      expect(states.count('started')).to eq(1)
+      expect(body).not_to respond_to(:to_path)
+      expect { body.close }.not_to raise_error
+    end
+  end
+
+  it 'records a failed stream without converting an exception into a second HTTP response' do
+    Dir.mktmpdir('counts-body-spec') do |directory|
+      path = File.join(directory, 'file')
+      File.binwrite(path, 'data')
+      states = []
+      body = described_class.new(path) { |state| states << state; true }
+      expect { body.each { raise IOError, 'connection closed' } }.to raise_error(IOError)
+      expect(states).to include('failed')
+      expect(states).not_to include('started')
+      expect { body.close }.not_to raise_error
+    end
+  end
+end

backend/spec/services/counts_pivot_security_spec.rb

@@ -0,0 +1,169 @@
+require 'rails_helper'
+require 'csv'
+
+RSpec.describe 'Counts masked pivot contracts' do
+  let(:admin) { create(:user, :admin) }
+  let(:member) { create(:user, :member) }
+  let(:guest) { create(:user, :guest) }
+  let(:params) do
+    { 'resource' => 'posts', 'definition' => 'history', 'mode' => 'summary',
+      'locale' => 'ja', 'execution_at' => '2025-01-01T00:00:00Z',
+      'timezone' => 'UTC', 'rows' => 'tags', 'cols' => 'event_type', 'sort' => '' }
+  end
+  let!(:tags) do
+    ['A', 'B', 'C'].map do |name|
+      tag = create(:tag, primary_name: name)
+      TagVersion.create!(tag:, version_no: 1, event_type: 'create', name:,
+          category: 'general', aliases: '', parent_tag_ids: '',
+          names: { ja: { name:, script_code: 'Jpan', auto_generated: false, aliases: [] } },
+          created_at: Time.utc(2023))
+      tag
+    end
+  end
+
+  before do
+    [[0, 1], [0], [1, 2]].each_with_index do |indexes, index|
+      post = create(:post)
+      snapshot = indexes.map { { tag_id: tags[_1].id, version_no: 1, sections: [] } }
+      PostVersion.create!(post:, version_no: 1, event_type: 'create',
+          title: "hidden-title-#{ index }", url: post.url, tags: '', tags_json: snapshot,
+          parent_post_ids: '', created_by_user: member, created_at: Time.utc(2024, 1))
+      next if index == 2
+
+      PostVersion.create!(post:, version_no: 2, event_type: 'discard',
+          title: "private-discard-#{ index }", url: post.url, tags: '', tags_json: [],
+          parent_post_ids: '', created_by_user: member, created_at: Time.utc(2024, 2))
+    end
+  end
+
+  def result(user, extra = {})
+    HistoryAccess.set(user:) { CountsTableQuery.new(params.merge(extra)).call }
+  end
+
+  def grand(result)
+    result.fetch(:pivot_totals).find { _1[:row].empty? && _1[:column].empty? }.fetch(:count)
+  end
+
+  it 'preserves five tag occurrences after merging private tags and across pages' do
+    [nil, guest, member, admin].each do |user|
+      page = result(user)
+      expect(grand(page)).to eq(5)
+      expect(page[:cells].sum { _1[:count] }).to eq(5)
+      expect(page).not_to have_key(:excluded_count)
+      paged = result(user, 'limit' => '1')
+      expect(grand(paged)).to eq(5)
+      expect(result(user, 'limit' => '1', 'page' => '2')[:group_count])
+          .to eq(page[:group_count])
+    end
+    public_page = result(nil)
+    hidden = public_page[:cells].select { _1[:values].first == CountsAxes::MASKED_VALUE }
+    expect(hidden.sum { _1[:count] }).to eq(3)
+    expect(public_page[:axis_labels]['a0']).not_to have_key("internal:#{ tags.first.id }")
+    expect(public_page.to_json).not_to include('hidden-title', 'private-discard')
+  end
+
+  it 'retains distinct membership when tags are not a row axis' do
+    [nil, admin].each do |user|
+      expect(grand(result(user, 'rows' => 'event_type', 'cols' => 'tags'))).to eq(3)
+      expect(result(user, 'rows' => '', 'cols' => '')[:cells].sole[:count]).to eq(5)
+      details = result(user, 'mode' => 'details', 'rows' => '', 'cols' => '')
+      expect(details[:count]).to eq(user == admin ? 5 : 1)
+    end
+  end
+
+  it 'uses changed-record membership for totals and hides changes to private predecessors' do
+    post = PostVersion.order(:id).last.post
+    PostVersion.create!(post:, version_no: 2, event_type: 'update', title: 'changed-public-title',
+        url: post.url, tags: '',
+        tags_json: [{ tag_id: tags.last.id, version_no: 1, sections: [] }],
+        parent_post_ids: '', created_at: Time.utc(2024, 3))
+    restored = PostVersion.order(:id).first.post
+    PostVersion.create!(post: restored, version_no: 3, event_type: 'restore',
+        title: 'restored-public-title', url: restored.url, tags: '',
+        tags_json: [{ tag_id: tags.first.id, version_no: 1, sections: [] }], parent_post_ids: '',
+        created_at: Time.utc(2024, 4))
+    [nil, admin].each do |user|
+      page = result(user, 'rows' => 'tags changed_field', 'cols' => 'event_type')
+      expect(grand(page)).to eq(page[:cells].sum { _1[:count] })
+      expect(page[:cells]).not_to be_empty
+      if user.nil?
+        expect(page[:cells].any? { _1[:values][1] == CountsAxes::MASKED_VALUE }).to be(true)
+      end
+      expect(page.to_json).not_to include('hidden-title', 'private-discard')
+    end
+  end
+
+  it 'counts distinct changed records when tags are not additive row tuples' do
+    [nil, admin].each do |user|
+      page = result(user, 'rows' => 'changed_field', 'cols' => 'event_type')
+      expect(grand(page)).to eq(2)
+      if user.nil?
+        expect(page[:cells].map { _1[:values].first }.uniq).to eq([CountsAxes::MASKED_VALUE])
+      end
+      # Changed discard snapshots have no tags: no full cell contributes here.
+      empty = result(user, 'rows' => 'changed_field', 'cols' => 'tags')
+      expect(empty[:cells]).to be_empty
+      expect(grand(empty)).to eq(0)
+    end
+  end
+
+  it 'keeps NULL as a public data value rather than treating it as a rollup' do
+    post = create(:post)
+    PostVersion.create!(post:, version_no: 1, event_type: 'create', title: 'public bot',
+        url: post.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024, 5))
+    page = result(nil, 'rows' => 'user', 'cols' => 'event_type')
+    expect(page[:cells].select { _1[:values].first.nil? }.sum { _1[:count] }).to eq(1)
+    expect(grand(page)).to eq(6)
+    expect(page[:row_headers]).to include(hash_including(values: [nil], kind: 'data'))
+  end
+
+  it 'sorts public numeric values numerically alongside a masked group' do
+    post = PostVersion.order(:id).last.post
+    [2, 10].each do |number|
+      PostVersion.create!(post:, version_no: number, event_type: 'update',
+          title: 'public numeric sort', url: post.url, tags: '', tags_json: [],
+          parent_post_ids: '', created_at: Time.utc(2024, 5))
+    end
+    page = result(nil, 'rows' => 'version_no', 'cols' => '')
+    values = page[:row_headers].select { _1[:kind] == 'data' }.map { _1[:values].first }
+    expect(values.reject { _1 == CountsAxes::MASKED_VALUE }.map(&:to_i)).to eq([1, 2, 10])
+    expect(values.last).to eq(CountsAxes::MASKED_VALUE)
+  end
+
+  it 'rejects private predicates and supplied measure coordinates before SQL labels escape' do
+    probes = [{ 'resource_id' => PostVersion.first.post_id.to_s },
+        { 'tags' => 'A' }, { 'tag_refs.0.name' => 'A', 'tag_refs.0.tag_id' => tags.first.id.to_s },
+        { 'conditions.0.field' => 'title', 'conditions.0.operator' => 'eq',
+          'conditions.0.value' => 'hidden-title-0' },
+        { 'drill.0.field' => 'tags', 'drill.0.value' => "internal:#{ tags.first.id }" },
+        { 'cols' => 'tags', 'column_order.0.value' => CountsAxes::MASKED_VALUE }]
+    HistoryAccess.set(user: nil) do
+      probes.each do |probe|
+        expect { CountsTableQuery.new(params.merge(probe)) }
+            .to raise_error(CountsAxes::PrivateCondition)
+      end
+    end
+  end
+
+  it 'keeps public column sorting usable while guessed private coordinates match nothing' do
+    first = result(nil, 'rows' => 'event_type', 'cols' => 'tags')
+    probe = result(nil, 'rows' => 'event_type', 'cols' => 'tags',
+        'column_order.0.value' => "internal:#{ tags.first.id }")
+    expect(probe[:column_values]).to eq(first[:column_values])
+    expect(probe[:cells]).to eq(first[:cells])
+    expect(probe[:column_values_editable]).to be(false)
+    expect(probe.to_json).not_to include("internal:#{ tags.first.id }")
+  end
+
+  it 'exports masked coordinates and safe CSV labels from the same pivot' do
+    HistoryAccess.set(user: nil) do
+      payload = params.merge('format' => 'json')
+      json = JSON.parse(CountsTableExport.new(payload).body.to_a.join)
+      expect(json['data'].sum { _1['kind'] == 'data' ? _1['count'] : 0 }).to eq(5)
+      expect(json.to_json).not_to include('hidden-title', 'private-discard')
+      csv = CountsTableExport.new(payload.merge('format' => 'csv')).body.to_a.join
+      expect(csv).to include('********')
+      expect(csv).not_to include(CountsAxes::MASKED_VALUE, 'hidden-title', 'private-discard')
+    end
+  end
+end