#434

backend/spec/requests/counts_history_boundaries_spec.rb

@@ -0,0 +1,40 @@
+require 'rails_helper'
+
+RSpec.describe 'History boundaries shared with Counts', type: :request do
+  let!(:tag) { create(:tag, primary_name: 'description tag') }
+  let!(:description) do
+    TagDescription.create!(tag:, language_code: 'ja', version_no: 2, body: 'private body')
+  end
+
+  before do
+    ['create', 'discard'].each_with_index do |event, index|
+      TagDescriptionVersion.create!(tag_id: tag.id, language_code: 'ja',
+          version_no: index + 1, event_type: event, body: 'private body',
+          created_at: Time.utc(2024))
+    end
+  end
+
+  it 'denies the same discarded description through Counts, the facade and the ordinary API' do
+    get '/counts/query', params: { resource: 'tag_descriptions', definition: 'history',
+        mode: 'details', locale: 'ja', execution_at: '2025-01-01T00:00:00Z' }
+    expect(response).to have_http_status(:ok)
+    expect(response.parsed_body).to include('count' => 0, 'output_count' => 2,
+        'excluded_count' => 2, 'versions' => [])
+    ["/wiki/#{ description.id }", "/tags/#{ tag.id }/description"].each do |path|
+      get path, params: { locale: 'ja' }
+      expect(response).to have_http_status(:not_found)
+      expect(response.body).not_to include('private body')
+    end
+    get '/sitemap_entries/descriptions', params: { locale: 'ja' }
+    expect(response).to have_http_status(:ok)
+    expect(response.parsed_body['ids']).not_to include(description.id)
+  end
+
+  it 'permits the server-authenticated administrator to read the discarded description' do
+    sign_in_as(create(:user, :admin))
+    get "/tags/#{ tag.id }/description", params: { locale: 'ja' }
+    expect(response).to have_http_status(:ok)
+    expect(response.parsed_body['body']).to eq('private body')
+    expect(response.headers['Cache-Control']).to include('no-store')
+  end
+end

backend/spec/requests/counts_spec.rb

@@ -1,6 +1,6 @@
 require 'rails_helper'
 
-RSpec.describe 'Counts API', type: :request do
+RSpec.describe 'Legacy Counts API', type: :request do
   let!(:actor) { create(:user, :member) }
   let!(:post) { create(:post) }
   let!(:tag) { create(:tag, primary_name: 'counts tag', category: :general) }
@@ -28,16 +28,16 @@ RSpec.describe 'Counts API', type: :request do
     context resource do
       let(:path) { "/counts/#{ resource }" }
       let(:id) { resource == 'posts' ? post.id : tag.id }
-      let(:filters) { { resource_id: id, user_id: actor.id,
+      let(:filters) { { resource_id: id, user_id: actor.id, latest: '0',
                        from: '2024-01-01T00:00Z', before: '2024-02-01T00:00Z' } }
 
       it 'uses the same AND filters for SQL totals, groups and details' do
-        get path, params: filters.merge(axis: 'event_type')
+        get path, params: filters.merge(axis: 'month')
         expect(response).to have_http_status(:ok)
         summary = response.parsed_body
         expect(summary['count']).to eq(2)
         expect(summary['groups'].sum { _1['count'] }).to eq(2)
-        expect(summary['groups'].map { _1['key'] }).to eq(['create', 'update'])
+        expect(summary['groups'].map { _1['key'] }).to eq(['2024-01'])
 
         get path, params: filters.merge(mode: 'details')
         expect(response.parsed_body['count']).to eq(2)
@@ -45,16 +45,16 @@ RSpec.describe 'Counts API', type: :request do
       end
 
       it 'paginates details and groups independently from totals' do
-        get path, params: { resource_id: id, mode: 'details', limit: 1, page: 2 }
+        get path, params: { resource_id: id, latest: '0', mode: 'details', limit: 1, page: 2 }
         expect(response.parsed_body['count']).to eq(3)
         expect(response.parsed_body['versions'].sole['version_no']).to eq(2)
-        get path, params: { resource_id: id, axis: 'month', limit: 1, page: 2 }
+        get path, params: { resource_id: id, latest: '0', axis: 'month', limit: 1, page: 2 }
         expect(response.parsed_body['group_count']).to eq(2)
         expect(response.parsed_body['groups'].sole).to include('key' => '2024-02', 'count' => 1)
       end
 
       it 'groups null actors and supports their drill-down filter' do
-        get path, params: { resource_id: id, axis: 'user' }
+        get path, params: { resource_id: id, latest: '0', axis: 'user' }
         expect(response.parsed_body['groups']).to include(
             { 'key' => nil, 'name' => nil, 'count' => 1 })
         get path, params: { resource_id: id, mode: 'details', user_id: 'none' }
@@ -62,7 +62,7 @@ RSpec.describe 'Counts API', type: :request do
       end
 
       it 'uses an inclusive start and exclusive end' do
-        get path, params: { resource_id: id,
+        get path, params: { resource_id: id, latest: '0',
                             from: '2024-01-01T00:00Z', before: '2024-01-02T00:00Z' }
         expect(response.parsed_body['count']).to eq(1)
       end
@@ -88,9 +88,10 @@ RSpec.describe 'Counts API', type: :request do
   end
 
   it 'matches removed tags from the previous snapshot, not current post_tags' do
-    get '/counts/posts', params: { tag: tag.id, mode: 'details' }
+    get '/counts/posts', params: { tag: tag.id, latest: '0', mode: 'details' }
     expect(response.parsed_body['count']).to eq(2)
     expect(response.parsed_body['versions'].map { _1['version_no'] }).to eq([2, 1])
-    expect(response.parsed_body['versions'].first['tags'].sole['type']).to eq('removed')
+    expect(response.parsed_body['versions'].first['tags']).to be_empty
+    expect(response.parsed_body['versions'].last['tags'].sole['id']).to eq(tag.id)
   end
 end

backend/spec/requests/counts_table_spec.rb

@@ -0,0 +1,248 @@
+require 'rails_helper'
+
+RSpec.describe 'Counts table API', type: :request do
+  include_context 'Ainu locale'
+
+  let!(:actor) { create(:user, :member, name: 'Counts actor') }
+  let!(:record) { create(:post) }
+  let!(:tag) { create(:tag, primary_name: 'alpha', category: :general) }
+  let!(:external) { create(:external_tag, name: 'beta') }
+  let(:point) { '2025-01-01T00:00:00Z' }
+  let(:base) do
+    { resource: 'posts', definition: 'history', mode: 'details',
+      execution_at: point, timezone: 'UTC', locale: 'ja' }
+  end
+
+  before do
+    TagVersion.create!(tag:, version_no: 1, event_type: 'create', name: 'alpha',
+        category: 'general', aliases: '', parent_tag_ids: '',
+        names: { ja: { name: 'alpha', script_code: 'Jpan', aliases: [] } },
+        created_at: Time.utc(2023))
+    NicoTagVersion.create!(external_tag: external, version_no: 1,
+        event_type: 'create', name: 'beta', linked_tags: '', created_at: Time.utc(2023))
+    snapshots = [
+        { tag_id: tag.id, version_no: 1, name: 'alpha', category: 'general', sections: [] },
+        { external_tag_id: external.id, name: 'beta', sections: [] }]
+    [1, 2, 3].each do |number|
+      PostVersion.create!(post: record, version_no: number,
+          event_type: number == 1 ? 'create' : 'update', url: record.url,
+          title: "snapshot #{ number }", tags: '', parent_post_ids: '',
+          tags_json: snapshots.first(number == 1 ? 2 : (number == 2 ? 1 : 0)),
+          created_by_user: number == 3 ? nil : actor,
+          created_at: Time.utc(2024, 1, 31, 15) + (number - 1).hours)
+    end
+  end
+
+  def query(extra = {})
+    get '/counts/query', params: base.merge(extra)
+    expect(response).to have_http_status(:ok)
+    response.parsed_body
+  end
+
+  def axis(field, value, type: 'string', grain: nil, root: 'conditions')
+    result = { root => '', "#{ root }.0.field" => field,
+               "#{ root }.0.value_type" => type }
+    result["#{ root }.0.operator"] = 'axis_eq' if root == 'conditions'
+    result["#{ root }.0.value"] = value unless type == 'null'
+    result["#{ root }.0.grain"] = grain if grain
+    result
+  end
+
+  def versions(result) = result.fetch('versions').map { _1.fetch('version_no') }
+
+  it 'separates latest and history and defaults to stable detail ordering and limit 100' do
+    history = query
+    expect(versions(history)).to eq([1, 2, 3])
+    expect(history).to include('limit' => 100, 'count' => 3, 'output_count' => 3,
+        'visible_output_count' => 3, 'excluded_count' => 0)
+    expect(history['versions'].map { _1['latest'] }).to eq([false, false, true])
+    expect(versions(query(definition: 'latest'))).to eq([3])
+    expect(versions(query(limit: 1, page: 2))).to eq([2])
+    expect(versions(query(sort: 'version_no:desc', limit: 1))).to eq([3])
+  end
+
+  it 'counts regular pivot rows separately from totals and from underlying records' do
+    total = query(mode: 'summary', rows: '', cols: '')
+    expect(total).to include('count' => 3, 'output_count' => 1, 'group_count' => 1)
+    expect(total['cells']).to eq([{ 'values' => [], 'count' => 3 }])
+    pivot = query(mode: 'summary', rows: 'event_type', cols: '', sort: '', limit: 1)
+    expect(pivot).to include('group_count' => 2, 'visible_output_count' => 2)
+    expect(pivot['row_headers'].count { _1['kind'] == 'data' }).to eq(1)
+    expect(pivot['row_headers']).to include(hash_including('kind' => 'grand_total'))
+    expect(query(mode: 'summary', rows: '', cols: 'event_type', sort: '')['output_count']).to eq(1)
+  end
+
+  it 'preserves timezone, NULL and legacy drill semantics and permits condition removal' do
+    month = axis('created_at', '2024-02', grain: 'month')
+    expect(versions(query(month.merge(timezone: 'Asia/Tokyo')))).to eq([1, 2, 3])
+    expect(versions(query(month))).to eq([])
+    old = axis('created_at', '2024-02', grain: 'month', root: 'drill')
+    expect(versions(query(old.merge(timezone: 'Asia/Tokyo')))).to eq([1, 2, 3])
+    expect(versions(query(old.merge(month).merge(timezone: 'Asia/Tokyo')))).to eq([1, 2, 3])
+    expect(versions(query(axis('user', nil, type: 'null')))).to eq([3])
+    expect(versions(query(axis('changed_field', 'title')))).to eq([2, 3])
+    expect(versions(query)).to eq([1, 2, 3])
+  end
+
+  it 'evaluates latest before filtering the version set' do
+    predicate = { conditions: '', 'conditions.0.field' => 'latest',
+                  'conditions.0.operator' => 'eq', 'conditions.0.value' => 'false' }
+    expect(versions(query(predicate))).to eq([1, 2])
+    expect(versions(query(predicate.merge('conditions.0.value' => 'true')))).to eq([3])
+  end
+
+  it 'keeps ANY, ALL and NONE distinct without multiplying parent records' do
+    values = { conditions: '', 'conditions.0.field' => 'tags',
+               'conditions.0.values' => '1', condition_values: '',
+               'condition_values.0.condition' => '0',
+               'condition_values.0.value_type' => 'string',
+               'condition_values.0.value' => "internal:#{ tag.id }",
+               'condition_values.1.condition' => '0',
+               'condition_values.1.value_type' => 'string',
+               'condition_values.1.value' => "external:#{ external.id }" }
+    { 'any' => [1, 2], 'all' => [1], 'none' => [3] }.each do |operator, expected|
+      expect(versions(query(values.merge('conditions.0.operator' => operator)))).to eq(expected)
+    end
+    result = query(axis('tags', "internal:#{ tag.id }"))
+    expect(result['versions'].first['tags'].length).to eq(2)
+  end
+
+  it 'distinguishes IN, NOT IN, an empty selection and an unspecified selection' do
+    predicate = { conditions: '', 'conditions.0.field' => 'version_no',
+                  'conditions.0.operator' => 'in', 'conditions.0.values' => '1',
+                  condition_values: '', 'condition_values.0.condition' => '0',
+                  'condition_values.0.value_type' => 'number',
+                  'condition_values.0.value' => '1' }
+    expect(versions(query(predicate))).to eq([1])
+    expect(versions(query(predicate.merge('conditions.0.operator' => 'not_in')))).to eq([2, 3])
+    empty = predicate.reject { |key, _value| key.to_s.start_with?('condition_values.') }
+    expect(versions(query(empty))).to be_empty
+    expect(versions(query(empty.except('conditions.0.values')))).to eq([1, 2, 3])
+  end
+
+  it 'canonicalises unreferenced tags while preserving partial stable references' do
+    reference = { tag_refs: '', 'tag_refs.0.name' => 'alpha',
+                  'tag_refs.0.tag_id' => tag.id.to_s }
+    expect(versions(query(tags: 'alpha'))).to eq([1, 2])
+    expect(versions(query(reference.merge(tags: 'alpha nico:beta')))).to eq([1])
+    expect(versions(query(reference.merge(tags: 'nico:beta alpha')))).to eq([1])
+    expect(versions(query(reference.merge(tags: 'alpha not:nico:beta')))).to eq([2])
+    expect(versions(query(reference.merge(tags: 'alpha nico:beta', match: 'any')))).to eq([1, 2])
+    expect(versions(query(tags: 'not:alpha'))).to eq([3])
+    get '/counts/query', params: base.merge(reference).merge(tags: 'different')
+    expect(response).to have_http_status(:bad_request)
+  end
+
+  it 'rejects malformed structured conditions and axes without accepting arbitrary SQL' do
+    [
+        { rows: '[{"field":"user"}]' }, { rows: 'user user' },
+        { rows: 'user:month' }, { rows: 'password' }, { limit: 101 }, { limit: 0 },
+        { page: '1x' }, { conditions: 'eyJmaWVsZCI6InVzZXIifQ==' },
+        { conditions: '', 'conditions.1.field' => 'user' },
+        axis('password', '1'), axis('changed_field', 'title').merge(definition: 'latest'),
+        axis('user', 'yes', type: 'boolean'),
+        { conditions: '', 'conditions.0.field' => 'title',
+          'conditions.0.operator' => 'eq', 'conditions.0.value' => 'x' * 8193 }].each do |invalid|
+      get '/counts/query', params: base.merge(invalid)
+      expect(response).to have_http_status(:bad_request), invalid.inspect
+    end
+  end
+
+  it 'rejects duplicate URL keys and JSON query payloads at the HTTP boundary' do
+    ['resource=posts&limit=20&limit=50', 'resource=posts&headers=%5B%22Id.%22%5D',
+     'resource=posts&limit%5B%5D=20'].each do |query_string|
+      get "/counts/query?#{ query_string }"
+      expect(response).to have_http_status(:bad_request)
+    end
+  end
+
+  it 'reports column overflow separately from malformed input' do
+    101.times do |index|
+      PostVersion.create!(post: record, version_no: index + 4,
+          event_type: 'update', url: record.url, title: "column #{ index }",
+          tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024, 3))
+    end
+    get '/counts/query', params: base.merge(mode: 'summary', rows: '', cols: 'title', sort: '')
+    expect(response).to have_http_status(:unprocessable_entity)
+    expect(response.parsed_body).to eq('error' => 'too_many_columns', 'max_columns' => 100)
+  end
+
+  it 'defines all six resources with filterable output fields and shared detail defaults' do
+    locale
+    CountsResourceDefinition::DEFINITIONS.each_key do |resource|
+      get "/counts/definitions/#{ resource }", params: { locale: 'ain-Kana' }
+      expect(response).to have_http_status(:ok)
+      body = response.parsed_body
+      filters = body['filter_fields'].map { _1['key'] }
+      walk = lambda do |fields, prefix|
+        fields.each do |field|
+          key = [prefix, field['key']].compact.join('.')
+          expect(filters).to include(key)
+          walk.call(field.fetch('children', []), key)
+        end
+      end
+      walk.call(body['output_fields'], nil)
+      defaults = body['defaults'].select { _1['mode'] == 'details' }
+      expect(defaults.map { _1['definition'] }).to eq(['latest', 'history'])
+      expect(defaults.first['params']).to eq(defaults.last['params'])
+      settings = defaults.first['params'].to_h { [_1['key'], _1['value']] }
+      expect(settings['sort']).to eq('resource_id:asc version_no:asc')
+      if resource == 'posts'
+        expect(settings['children'].split).to eq(['tags.name_ain', 'parents.id'])
+      end
+      keys = body['name_columns'].map { _1['key'] }
+      languages = Language.order(:created_at, :code).pluck(:code)
+      expect(keys).to eq(languages.map { "name_#{ _1 }" } + languages.map { "aliases_#{ _1 }" })
+    end
+  end
+
+  [nil, :guest, :member, :admin].each do |role|
+    context "authority #{ role || 'anonymous' }" do
+      before do
+        sign_in_as(create(:user, role)) if role
+        # Equal timestamps deliberately require version ordering, not a date comparison.
+        ['discard', 'restore', 'discard', 'restore'].each_with_index do |event, index|
+          PostVersion.create!(post: record, version_no: index + 4, event_type: event,
+              url: record.url, title: event == 'restore' ? 'restored' : 'private snapshot',
+              tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024, 3))
+        end
+      end
+
+      it 'keeps every earlier discard boundary private but counts denied records' do
+        result = query
+        expected = role == :admin ? (1..7).to_a : [7]
+        expect(versions(result)).to eq(expected)
+        expect(result).to include('output_count' => 7,
+            'visible_output_count' => expected.length, 'excluded_count' => 7 - expected.length)
+        expect(response.headers['Cache-Control']).to include('no-store')
+        expect(response.body).not_to include('private snapshot') unless role == :admin
+      end
+
+      it 'applies the same version boundary to the ordinary history endpoint and old values' do
+        get '/posts/versions', params: { post: record.id, locale: 'ja' }
+        expect(response).to have_http_status(:ok)
+        rows = response.parsed_body.fetch('versions')
+        expect(rows.map { _1['version_no'] }).to eq(role == :admin ? (1..7).to_a.reverse : [7])
+        unless role == :admin
+          expect(rows.sole.dig('title', 'prev')).to be_nil
+          expect(response.body).not_to include('private snapshot')
+        end
+      end
+
+      it 'does not resolve private condition names even when the result set is empty' do
+        TagVersion.create!(tag:, version_no: 2, event_type: 'discard', name: 'secret tag',
+            category: 'general', aliases: '', parent_tag_ids: '',
+            names: { ja: { name: 'secret tag', script_code: 'Jpan', aliases: [] } },
+            created_at: Time.utc(2024, 2))
+        result = query(axis('tags', "internal:#{ tag.id }").merge(definition: 'latest'))
+        expect(result['versions']).to be_empty
+        expected = role == :admin ? { "internal:#{ tag.id }" => 'secret tag' } : {}
+        expect(result['condition_labels']['tags']).to eq(expected)
+        get '/counts/definitions/posts/candidates', params: {
+            field: 'tags', query: 'secret', execution_at: point, locale: 'ja' }
+        expect(response).to have_http_status(:ok)
+        expect(response.parsed_body['rows']).to be_empty
+      end
+    end
+  end
+end