diff --git a/backend/spec/models/history_access_spec.rb b/backend/spec/models/history_access_spec.rb new file mode 100644 index 0000000..4c3f537 --- /dev/null +++ b/backend/spec/models/history_access_spec.rb @@ -0,0 +1,67 @@ +require 'rails_helper' + +RSpec.describe HistoryAccess do + include_context 'Ainu locale' + + let!(:tag) { create(:tag) } + + it 'uses the complete tag-description identity and version order at equal timestamps' do + locale + rows = ['ja', 'ain'].flat_map do |language| + ['create', 'discard', 'restore'].each_with_index.map do |event, index| + TagDescriptionVersion.create!(tag_id: tag.id, language_code: language, + version_no: index + 1, event_type: language == 'ain' ? 'update' : event, + body: "#{ language } #{ index }", created_at: Time.utc(2024)) + end + end + described_class.set(user: nil) do + expect(described_class.visible(TagDescriptionVersion.where(id: rows.map(&:id))).pluck(:id)) + .to match_array([rows[2], *rows.last(3)].map(&:id)) + expect { described_class.require!(rows.first) }.to raise_error(HistoryAccess::Forbidden) + end + described_class.set(user: create(:user, :admin)) do + expect(described_class.visible(TagDescriptionVersion.where(id: rows.map(&:id))).count).to eq(6) + end + end + + it 'invalidates public artifacts after a new discard and binds admin artifacts to their actor' do + public_metadata = described_class.set(user: nil) { described_class.authority } + admin = create(:user, :admin) + admin_metadata = described_class.set(user: admin) { described_class.authority } + TagVersion.create!(tag:, version_no: 1, event_type: 'discard', name: 'private', + category: 'general', names: { ja: { name: 'private' } }, + aliases: '', parent_tag_ids: '', created_at: Time.utc(2024)) + described_class.set(user: nil) do + expect(described_class.authorised_artifact?(public_metadata)).to be(false) + expect(described_class.authorised_artifact?(admin_metadata)).to be(false) + end + described_class.set(user: create(:user, :admin)) do + expect(described_class.authorised_artifact?(admin_metadata)).to be(false) + end + described_class.set(user: admin) do + expect(described_class.authorised_artifact?(admin_metadata)).to be(true) + admin.update!(role: :guest) + expect(described_class.authorised_artifact?(admin_metadata)).to be(false) + end + end + + it 'denies a blob attached to a discarded post including its derived attachment' do + post = create(:post) + blob = ActiveStorage::Blob.create_before_direct_upload!(filename: 'private.png', + byte_size: 1, checksum: Base64.strict_encode64('checksum'), content_type: 'image/png') + ActiveStorage::Attachment.create!(name: 'thumbnail', record: post, blob:) + PostVersion.create!(post:, version_no: 1, event_type: 'discard', url: post.url, + title: 'private', tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024)) + variant = ActiveStorage::VariantRecord.create!(blob:, variation_digest: 'spec-variation') + derived = ActiveStorage::Blob.create_before_direct_upload!(filename: 'derived.png', + byte_size: 1, checksum: Base64.strict_encode64('checksum'), content_type: 'image/png') + ActiveStorage::Attachment.create!(name: 'image', record: variant, blob: derived) + described_class.set(user: nil) do + expect(described_class.blob_visible?(blob)).to be(false) + expect(described_class.blob_visible?(derived)).to be(false) + end + described_class.set(user: create(:user, :admin)) do + expect(described_class.blob_visible?(derived)).to be(true) + end + end +end diff --git a/backend/spec/requests/counts_history_boundaries_spec.rb b/backend/spec/requests/counts_history_boundaries_spec.rb new file mode 100644 index 0000000..0bdb514 --- /dev/null +++ b/backend/spec/requests/counts_history_boundaries_spec.rb @@ -0,0 +1,40 @@ +require 'rails_helper' + +RSpec.describe 'History boundaries shared with Counts', type: :request do + let!(:tag) { create(:tag, primary_name: 'description tag') } + let!(:description) do + TagDescription.create!(tag:, language_code: 'ja', version_no: 2, body: 'private body') + end + + before do + ['create', 'discard'].each_with_index do |event, index| + TagDescriptionVersion.create!(tag_id: tag.id, language_code: 'ja', + version_no: index + 1, event_type: event, body: 'private body', + created_at: Time.utc(2024)) + end + end + + it 'denies the same discarded description through Counts, the facade and the ordinary API' do + get '/counts/query', params: { resource: 'tag_descriptions', definition: 'history', + mode: 'details', locale: 'ja', execution_at: '2025-01-01T00:00:00Z' } + expect(response).to have_http_status(:ok) + expect(response.parsed_body).to include('count' => 0, 'output_count' => 2, + 'excluded_count' => 2, 'versions' => []) + ["/wiki/#{ description.id }", "/tags/#{ tag.id }/description"].each do |path| + get path, params: { locale: 'ja' } + expect(response).to have_http_status(:not_found) + expect(response.body).not_to include('private body') + end + get '/sitemap_entries/descriptions', params: { locale: 'ja' } + expect(response).to have_http_status(:ok) + expect(response.parsed_body['ids']).not_to include(description.id) + end + + it 'permits the server-authenticated administrator to read the discarded description' do + sign_in_as(create(:user, :admin)) + get "/tags/#{ tag.id }/description", params: { locale: 'ja' } + expect(response).to have_http_status(:ok) + expect(response.parsed_body['body']).to eq('private body') + expect(response.headers['Cache-Control']).to include('no-store') + end +end diff --git a/backend/spec/requests/counts_spec.rb b/backend/spec/requests/counts_spec.rb index bbd3a28..0026d49 100644 --- a/backend/spec/requests/counts_spec.rb +++ b/backend/spec/requests/counts_spec.rb @@ -1,6 +1,6 @@ require 'rails_helper' -RSpec.describe 'Counts API', type: :request do +RSpec.describe 'Legacy Counts API', type: :request do let!(:actor) { create(:user, :member) } let!(:post) { create(:post) } let!(:tag) { create(:tag, primary_name: 'counts tag', category: :general) } @@ -28,16 +28,16 @@ RSpec.describe 'Counts API', type: :request do context resource do let(:path) { "/counts/#{ resource }" } let(:id) { resource == 'posts' ? post.id : tag.id } - let(:filters) { { resource_id: id, user_id: actor.id, + let(:filters) { { resource_id: id, user_id: actor.id, latest: '0', from: '2024-01-01T00:00Z', before: '2024-02-01T00:00Z' } } it 'uses the same AND filters for SQL totals, groups and details' do - get path, params: filters.merge(axis: 'event_type') + get path, params: filters.merge(axis: 'month') expect(response).to have_http_status(:ok) summary = response.parsed_body expect(summary['count']).to eq(2) expect(summary['groups'].sum { _1['count'] }).to eq(2) - expect(summary['groups'].map { _1['key'] }).to eq(['create', 'update']) + expect(summary['groups'].map { _1['key'] }).to eq(['2024-01']) get path, params: filters.merge(mode: 'details') expect(response.parsed_body['count']).to eq(2) @@ -45,16 +45,16 @@ RSpec.describe 'Counts API', type: :request do end it 'paginates details and groups independently from totals' do - get path, params: { resource_id: id, mode: 'details', limit: 1, page: 2 } + get path, params: { resource_id: id, latest: '0', mode: 'details', limit: 1, page: 2 } expect(response.parsed_body['count']).to eq(3) expect(response.parsed_body['versions'].sole['version_no']).to eq(2) - get path, params: { resource_id: id, axis: 'month', limit: 1, page: 2 } + get path, params: { resource_id: id, latest: '0', axis: 'month', limit: 1, page: 2 } expect(response.parsed_body['group_count']).to eq(2) expect(response.parsed_body['groups'].sole).to include('key' => '2024-02', 'count' => 1) end it 'groups null actors and supports their drill-down filter' do - get path, params: { resource_id: id, axis: 'user' } + get path, params: { resource_id: id, latest: '0', axis: 'user' } expect(response.parsed_body['groups']).to include( { 'key' => nil, 'name' => nil, 'count' => 1 }) get path, params: { resource_id: id, mode: 'details', user_id: 'none' } @@ -62,7 +62,7 @@ RSpec.describe 'Counts API', type: :request do end it 'uses an inclusive start and exclusive end' do - get path, params: { resource_id: id, + get path, params: { resource_id: id, latest: '0', from: '2024-01-01T00:00Z', before: '2024-01-02T00:00Z' } expect(response.parsed_body['count']).to eq(1) end @@ -88,9 +88,10 @@ RSpec.describe 'Counts API', type: :request do end it 'matches removed tags from the previous snapshot, not current post_tags' do - get '/counts/posts', params: { tag: tag.id, mode: 'details' } + get '/counts/posts', params: { tag: tag.id, latest: '0', mode: 'details' } expect(response.parsed_body['count']).to eq(2) expect(response.parsed_body['versions'].map { _1['version_no'] }).to eq([2, 1]) - expect(response.parsed_body['versions'].first['tags'].sole['type']).to eq('removed') + expect(response.parsed_body['versions'].first['tags']).to be_empty + expect(response.parsed_body['versions'].last['tags'].sole['id']).to eq(tag.id) end end diff --git a/backend/spec/requests/counts_table_spec.rb b/backend/spec/requests/counts_table_spec.rb new file mode 100644 index 0000000..bda0ff3 --- /dev/null +++ b/backend/spec/requests/counts_table_spec.rb @@ -0,0 +1,248 @@ +require 'rails_helper' + +RSpec.describe 'Counts table API', type: :request do + include_context 'Ainu locale' + + let!(:actor) { create(:user, :member, name: 'Counts actor') } + let!(:record) { create(:post) } + let!(:tag) { create(:tag, primary_name: 'alpha', category: :general) } + let!(:external) { create(:external_tag, name: 'beta') } + let(:point) { '2025-01-01T00:00:00Z' } + let(:base) do + { resource: 'posts', definition: 'history', mode: 'details', + execution_at: point, timezone: 'UTC', locale: 'ja' } + end + + before do + TagVersion.create!(tag:, version_no: 1, event_type: 'create', name: 'alpha', + category: 'general', aliases: '', parent_tag_ids: '', + names: { ja: { name: 'alpha', script_code: 'Jpan', aliases: [] } }, + created_at: Time.utc(2023)) + NicoTagVersion.create!(external_tag: external, version_no: 1, + event_type: 'create', name: 'beta', linked_tags: '', created_at: Time.utc(2023)) + snapshots = [ + { tag_id: tag.id, version_no: 1, name: 'alpha', category: 'general', sections: [] }, + { external_tag_id: external.id, name: 'beta', sections: [] }] + [1, 2, 3].each do |number| + PostVersion.create!(post: record, version_no: number, + event_type: number == 1 ? 'create' : 'update', url: record.url, + title: "snapshot #{ number }", tags: '', parent_post_ids: '', + tags_json: snapshots.first(number == 1 ? 2 : (number == 2 ? 1 : 0)), + created_by_user: number == 3 ? nil : actor, + created_at: Time.utc(2024, 1, 31, 15) + (number - 1).hours) + end + end + + def query(extra = {}) + get '/counts/query', params: base.merge(extra) + expect(response).to have_http_status(:ok) + response.parsed_body + end + + def axis(field, value, type: 'string', grain: nil, root: 'conditions') + result = { root => '', "#{ root }.0.field" => field, + "#{ root }.0.value_type" => type } + result["#{ root }.0.operator"] = 'axis_eq' if root == 'conditions' + result["#{ root }.0.value"] = value unless type == 'null' + result["#{ root }.0.grain"] = grain if grain + result + end + + def versions(result) = result.fetch('versions').map { _1.fetch('version_no') } + + it 'separates latest and history and defaults to stable detail ordering and limit 100' do + history = query + expect(versions(history)).to eq([1, 2, 3]) + expect(history).to include('limit' => 100, 'count' => 3, 'output_count' => 3, + 'visible_output_count' => 3, 'excluded_count' => 0) + expect(history['versions'].map { _1['latest'] }).to eq([false, false, true]) + expect(versions(query(definition: 'latest'))).to eq([3]) + expect(versions(query(limit: 1, page: 2))).to eq([2]) + expect(versions(query(sort: 'version_no:desc', limit: 1))).to eq([3]) + end + + it 'counts regular pivot rows separately from totals and from underlying records' do + total = query(mode: 'summary', rows: '', cols: '') + expect(total).to include('count' => 3, 'output_count' => 1, 'group_count' => 1) + expect(total['cells']).to eq([{ 'values' => [], 'count' => 3 }]) + pivot = query(mode: 'summary', rows: 'event_type', cols: '', sort: '', limit: 1) + expect(pivot).to include('group_count' => 2, 'visible_output_count' => 2) + expect(pivot['row_headers'].count { _1['kind'] == 'data' }).to eq(1) + expect(pivot['row_headers']).to include(hash_including('kind' => 'grand_total')) + expect(query(mode: 'summary', rows: '', cols: 'event_type', sort: '')['output_count']).to eq(1) + end + + it 'preserves timezone, NULL and legacy drill semantics and permits condition removal' do + month = axis('created_at', '2024-02', grain: 'month') + expect(versions(query(month.merge(timezone: 'Asia/Tokyo')))).to eq([1, 2, 3]) + expect(versions(query(month))).to eq([]) + old = axis('created_at', '2024-02', grain: 'month', root: 'drill') + expect(versions(query(old.merge(timezone: 'Asia/Tokyo')))).to eq([1, 2, 3]) + expect(versions(query(old.merge(month).merge(timezone: 'Asia/Tokyo')))).to eq([1, 2, 3]) + expect(versions(query(axis('user', nil, type: 'null')))).to eq([3]) + expect(versions(query(axis('changed_field', 'title')))).to eq([2, 3]) + expect(versions(query)).to eq([1, 2, 3]) + end + + it 'evaluates latest before filtering the version set' do + predicate = { conditions: '', 'conditions.0.field' => 'latest', + 'conditions.0.operator' => 'eq', 'conditions.0.value' => 'false' } + expect(versions(query(predicate))).to eq([1, 2]) + expect(versions(query(predicate.merge('conditions.0.value' => 'true')))).to eq([3]) + end + + it 'keeps ANY, ALL and NONE distinct without multiplying parent records' do + values = { conditions: '', 'conditions.0.field' => 'tags', + 'conditions.0.values' => '1', condition_values: '', + 'condition_values.0.condition' => '0', + 'condition_values.0.value_type' => 'string', + 'condition_values.0.value' => "internal:#{ tag.id }", + 'condition_values.1.condition' => '0', + 'condition_values.1.value_type' => 'string', + 'condition_values.1.value' => "external:#{ external.id }" } + { 'any' => [1, 2], 'all' => [1], 'none' => [3] }.each do |operator, expected| + expect(versions(query(values.merge('conditions.0.operator' => operator)))).to eq(expected) + end + result = query(axis('tags', "internal:#{ tag.id }")) + expect(result['versions'].first['tags'].length).to eq(2) + end + + it 'distinguishes IN, NOT IN, an empty selection and an unspecified selection' do + predicate = { conditions: '', 'conditions.0.field' => 'version_no', + 'conditions.0.operator' => 'in', 'conditions.0.values' => '1', + condition_values: '', 'condition_values.0.condition' => '0', + 'condition_values.0.value_type' => 'number', + 'condition_values.0.value' => '1' } + expect(versions(query(predicate))).to eq([1]) + expect(versions(query(predicate.merge('conditions.0.operator' => 'not_in')))).to eq([2, 3]) + empty = predicate.reject { |key, _value| key.to_s.start_with?('condition_values.') } + expect(versions(query(empty))).to be_empty + expect(versions(query(empty.except('conditions.0.values')))).to eq([1, 2, 3]) + end + + it 'canonicalises unreferenced tags while preserving partial stable references' do + reference = { tag_refs: '', 'tag_refs.0.name' => 'alpha', + 'tag_refs.0.tag_id' => tag.id.to_s } + expect(versions(query(tags: 'alpha'))).to eq([1, 2]) + expect(versions(query(reference.merge(tags: 'alpha nico:beta')))).to eq([1]) + expect(versions(query(reference.merge(tags: 'nico:beta alpha')))).to eq([1]) + expect(versions(query(reference.merge(tags: 'alpha not:nico:beta')))).to eq([2]) + expect(versions(query(reference.merge(tags: 'alpha nico:beta', match: 'any')))).to eq([1, 2]) + expect(versions(query(tags: 'not:alpha'))).to eq([3]) + get '/counts/query', params: base.merge(reference).merge(tags: 'different') + expect(response).to have_http_status(:bad_request) + end + + it 'rejects malformed structured conditions and axes without accepting arbitrary SQL' do + [ + { rows: '[{"field":"user"}]' }, { rows: 'user user' }, + { rows: 'user:month' }, { rows: 'password' }, { limit: 101 }, { limit: 0 }, + { page: '1x' }, { conditions: 'eyJmaWVsZCI6InVzZXIifQ==' }, + { conditions: '', 'conditions.1.field' => 'user' }, + axis('password', '1'), axis('changed_field', 'title').merge(definition: 'latest'), + axis('user', 'yes', type: 'boolean'), + { conditions: '', 'conditions.0.field' => 'title', + 'conditions.0.operator' => 'eq', 'conditions.0.value' => 'x' * 8193 }].each do |invalid| + get '/counts/query', params: base.merge(invalid) + expect(response).to have_http_status(:bad_request), invalid.inspect + end + end + + it 'rejects duplicate URL keys and JSON query payloads at the HTTP boundary' do + ['resource=posts&limit=20&limit=50', 'resource=posts&headers=%5B%22Id.%22%5D', + 'resource=posts&limit%5B%5D=20'].each do |query_string| + get "/counts/query?#{ query_string }" + expect(response).to have_http_status(:bad_request) + end + end + + it 'reports column overflow separately from malformed input' do + 101.times do |index| + PostVersion.create!(post: record, version_no: index + 4, + event_type: 'update', url: record.url, title: "column #{ index }", + tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024, 3)) + end + get '/counts/query', params: base.merge(mode: 'summary', rows: '', cols: 'title', sort: '') + expect(response).to have_http_status(:unprocessable_entity) + expect(response.parsed_body).to eq('error' => 'too_many_columns', 'max_columns' => 100) + end + + it 'defines all six resources with filterable output fields and shared detail defaults' do + locale + CountsResourceDefinition::DEFINITIONS.each_key do |resource| + get "/counts/definitions/#{ resource }", params: { locale: 'ain-Kana' } + expect(response).to have_http_status(:ok) + body = response.parsed_body + filters = body['filter_fields'].map { _1['key'] } + walk = lambda do |fields, prefix| + fields.each do |field| + key = [prefix, field['key']].compact.join('.') + expect(filters).to include(key) + walk.call(field.fetch('children', []), key) + end + end + walk.call(body['output_fields'], nil) + defaults = body['defaults'].select { _1['mode'] == 'details' } + expect(defaults.map { _1['definition'] }).to eq(['latest', 'history']) + expect(defaults.first['params']).to eq(defaults.last['params']) + settings = defaults.first['params'].to_h { [_1['key'], _1['value']] } + expect(settings['sort']).to eq('resource_id:asc version_no:asc') + if resource == 'posts' + expect(settings['children'].split).to eq(['tags.name_ain', 'parents.id']) + end + keys = body['name_columns'].map { _1['key'] } + languages = Language.order(:created_at, :code).pluck(:code) + expect(keys).to eq(languages.map { "name_#{ _1 }" } + languages.map { "aliases_#{ _1 }" }) + end + end + + [nil, :guest, :member, :admin].each do |role| + context "authority #{ role || 'anonymous' }" do + before do + sign_in_as(create(:user, role)) if role + # Equal timestamps deliberately require version ordering, not a date comparison. + ['discard', 'restore', 'discard', 'restore'].each_with_index do |event, index| + PostVersion.create!(post: record, version_no: index + 4, event_type: event, + url: record.url, title: event == 'restore' ? 'restored' : 'private snapshot', + tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024, 3)) + end + end + + it 'keeps every earlier discard boundary private but counts denied records' do + result = query + expected = role == :admin ? (1..7).to_a : [7] + expect(versions(result)).to eq(expected) + expect(result).to include('output_count' => 7, + 'visible_output_count' => expected.length, 'excluded_count' => 7 - expected.length) + expect(response.headers['Cache-Control']).to include('no-store') + expect(response.body).not_to include('private snapshot') unless role == :admin + end + + it 'applies the same version boundary to the ordinary history endpoint and old values' do + get '/posts/versions', params: { post: record.id, locale: 'ja' } + expect(response).to have_http_status(:ok) + rows = response.parsed_body.fetch('versions') + expect(rows.map { _1['version_no'] }).to eq(role == :admin ? (1..7).to_a.reverse : [7]) + unless role == :admin + expect(rows.sole.dig('title', 'prev')).to be_nil + expect(response.body).not_to include('private snapshot') + end + end + + it 'does not resolve private condition names even when the result set is empty' do + TagVersion.create!(tag:, version_no: 2, event_type: 'discard', name: 'secret tag', + category: 'general', aliases: '', parent_tag_ids: '', + names: { ja: { name: 'secret tag', script_code: 'Jpan', aliases: [] } }, + created_at: Time.utc(2024, 2)) + result = query(axis('tags', "internal:#{ tag.id }").merge(definition: 'latest')) + expect(result['versions']).to be_empty + expected = role == :admin ? { "internal:#{ tag.id }" => 'secret tag' } : {} + expect(result['condition_labels']['tags']).to eq(expected) + get '/counts/definitions/posts/candidates', params: { + field: 'tags', query: 'secret', execution_at: point, locale: 'ja' } + expect(response).to have_http_status(:ok) + expect(response.parsed_body['rows']).to be_empty + end + end + end +end diff --git a/backend/spec/services/counts_export_spec.rb b/backend/spec/services/counts_export_spec.rb new file mode 100644 index 0000000..bf2c304 --- /dev/null +++ b/backend/spec/services/counts_export_spec.rb @@ -0,0 +1,168 @@ +require 'rails_helper' +require 'tmpdir' +require 'csv' + +RSpec.describe 'Counts export contracts' do + include ActiveSupport::Testing::TimeHelpers + let(:payload) do + { 'resource' => 'posts', 'definition' => 'history', 'mode' => 'details', + 'locale' => 'ja', 'execution_at' => '2024-03-01T00:00:00Z', + 'columns' => 'title', 'headers' => '["Title"]', 'format' => 'json' } + end + + around do |example| + Dir.mktmpdir('counts-export-spec') do |path| + allow(CountsExportDownload).to receive(:directory).and_return(Pathname.new(path)) + HistoryAccess.set(user: nil) { example.run } + end + end + + context 'serialization after the authorised query' do + let(:rows) { [{ title: "=formula\n\"quoted\"" }, { title: 'second' }] } + let(:page) do + { count: 2, output_count: 2, visible_output_count: 2, page: 1, limit: 100, + versions: rows, execution_at: payload.fetch('execution_at') } + end + + before do + query = CountsTableQuery.new(payload.slice(*CountsTableQuery::KEYS)) + # SQL/authority are exercised in request specs; this unit boundary supplies their page. + allow(query).to receive(:each_page) do |before_page:, &block| + before_page.call + block.call(page) + end + allow(CountsTableQuery).to receive(:new).and_return(query) + allow(ApplicationRecord).to receive(:transaction).with(isolation: :repeatable_read).and_yield + end + + it 'reports aggregation, actual output progress and regular-row metadata' do + progress = [] + phases = [] + body = CountsTableExport.new(payload).body( + progress: ->(done, total) { progress << [done, total] }, + phase: ->(value) { phases << value }).to_a.join + result = JSON.parse(body) + expect(result['data']).to eq(rows.map(&:stringify_keys)) + expect(result['metadata']).to include('logical_count_unit' => 'regular_result_rows', + 'aggregate_records_in_logical_count' => false, 'output_count' => 2) + expect(phases).to eq(['aggregating', 'writing']) + expect(progress).to include([0, 2], [1, 2], [2, 2]) + expect(progress.last).to eq([2, 2]) + end + + it 'preserves CSV multiline text and quotes while neutralising spreadsheet formulas' do + body = CountsTableExport.new(payload.merge('format' => 'csv')).body.to_a.join + expect(CSV.parse(body.delete_prefix("\uFEFF"))) + .to eq([['Title'], ["'=formula\n\"quoted\""], ['second']]) + end + + it 'completes an isolated worker job with its real processed count and cancellable file' do + CountsExportJobs.pulse + token = CountsExportJobs.start(payload, 'owner', 'ja') + id = CountsExportJobs.authorised_id(token, 'owner') + state = CountsExportJobs.read(id).merge('status' => 'running', 'phase' => 'aggregating') + CountsExportJobs.write(state) + # Invoke the worker entrypoint directly; never fork or start an external worker in specs. + CountsExportJobs.perform(state) + ready = CountsExportJobs.status(token, 'owner') + expect(ready).to include('status' => 'ready', 'processed' => 2, 'total' => 2) + file = CountsExportDownload.find(ready.fetch('token'), 'owner') + expect(JSON.parse(File.read(file.fetch('path')))['data'].length).to eq(2) + CountsExportJobs.status(token, 'owner', cancel: true) + expect(CountsExportDownload.find(ready.fetch('token'), 'owner')).to be_nil + end + + it 'marks a generation exception as failed without issuing a download token' do + CountsExportJobs.pulse + token = CountsExportJobs.start(payload, 'owner', 'ja') + id = CountsExportJobs.authorised_id(token, 'owner') + state = CountsExportJobs.read(id).merge('status' => 'running') + CountsExportJobs.write(state) + allow(CountsExportDownload).to receive(:prepare).and_raise(CountsExportDownload::TooLarge) + CountsExportJobs.perform(state) + expect(CountsExportJobs.status(token, 'owner')).to include('status' => 'failed') + expect(CountsExportJobs.status(token, 'owner')).not_to have_key('token') + end + end + + it 'retains multilingual names and all inner rows in JSON with one selected display column' do + names = [{ language_code: 'ja', name: 'alpha', script_code: 'Jpan', aliases: [] }, + { language_code: 'ain', name: 'recorded', script_code: 'Kana', + aliases: [{ name: 'alias', script_code: 'Latn' }] }] + rows = [{ id: 1, name_ja: 'alpha', names: }, { id: 2, name_ja: 'beta', names: }] + projection = CountsExportProjection.new(payload.merge('columns' => 'tags', + 'headers' => '["Tags"]', 'children' => 'tags.name_ja'), + resource: 'posts', summary: false, tags: false) + result = projection.project({ tags: rows }) + expect(result['tags'].length).to eq(2) + expect(result['tags'].map { _1['name_ja'] }).to eq(['alpha', 'beta']) + expect(result['tags'].map { _1['names'] }).to eq([names, names]) + end + + it 'requires a live worker, enforces queue capacity and binds jobs to their owner' do + expect { CountsExportJobs.start(payload, 'owner', 'ja') } + .to raise_error(CountsExportJobs::WorkerUnavailable) + CountsExportJobs.pulse + token = CountsExportJobs.start(payload, 'owner', 'ja') + expect(CountsExportJobs.status(token, 'other')).to be_nil + expect(CountsExportJobs.status(token, 'owner')).to include( + 'status' => 'queued', 'phase' => 'queued', 'processed' => 0, 'total' => nil) + (CountsExportDownload::MAX_CONCURRENT_EXPORTS - 1).times do + CountsExportJobs.start(payload, 'owner', 'ja') + end + expect { CountsExportJobs.start(payload, 'owner', 'ja') } + .to raise_error(CountsExportDownload::Busy) + expect(CountsExportJobs.status(token, 'owner', cancel: true)['status']).to eq('cancelled') + end + + it 'fails stale queued/running jobs and preserves actual progress without a fake total' do + CountsExportJobs.pulse + token = CountsExportJobs.start(payload, 'owner', 'ja') + id = CountsExportJobs.authorised_id(token, 'owner') + state = CountsExportJobs.read(id) + state.merge!('status' => 'running', 'phase' => 'aggregating') + CountsExportJobs.write(state) + CountsExportJobs.checkpoint(id, 5, nil, phase: 'writing') + expect(CountsExportJobs.status(token, 'owner')).to include( + 'phase' => 'writing', 'processed' => 5, 'total' => nil) + state = CountsExportJobs.read(id) + state['heartbeat_at'] = (Time.current - CountsExportJobs::HEARTBEAT_SECONDS - 1).to_f + CountsExportJobs.write(state) + expect(CountsExportJobs.status(token, 'owner')).to include( + 'status' => 'failed', 'error' => 'worker_stalled', 'processed' => 5) + expect { CountsExportJobs.checkpoint(id, 6) }.to raise_error(CountsExportJobs::Cancelled) + end + + it 'revokes an already generated download on a new discard and rejects wrong owners' do + exporter = instance_double(CountsExporter, resource: 'posts', format: 'json', body: ['{}']) + token = CountsExportDownload.prepare(exporter, 'owner') + expect(CountsExportDownload.find(token, 'owner')).not_to be_nil + expect(CountsExportDownload.find(token, 'other')).to be_nil + post = create(:post) + PostVersion.create!(post:, version_no: 1, event_type: 'discard', url: post.url, + title: 'private', tags: '', tags_json: [], parent_post_ids: '', created_at: Time.utc(2024)) + expect(CountsExportDownload.find(token, 'owner')).to be_nil + end + + it 'rejects oversized output and removes its partial file' do + stub_const('CountsExportDownload::MAX_FILE_BYTES', 4) + exporter = instance_double(CountsExporter, resource: 'posts', format: 'json', body: ['12345']) + expect { CountsExportDownload.prepare(exporter, 'owner') } + .to raise_error(CountsExportDownload::TooLarge) + expect(Dir.glob(CountsExportDownload.directory.join('counts-*'))).to be_empty + end + + it 'expires queued work after worker loss and rejects expired download signatures' do + CountsExportJobs.pulse + token = CountsExportJobs.start(payload, 'owner', 'ja') + travel_to(Time.current + CountsExportJobs::HEARTBEAT_SECONDS + 1) do + expect(CountsExportJobs.status(token, 'owner')).to include( + 'status' => 'failed', 'error' => 'worker_unavailable') + end + exporter = instance_double(CountsExporter, resource: 'posts', format: 'json', body: ['{}']) + download = CountsExportDownload.prepare(exporter, 'owner') + travel_to(Time.current + CountsExportDownload::EXPIRY + 1) do + expect(CountsExportDownload.find(download, 'owner')).to be_nil + end + end +end diff --git a/backend/spec/services/query_state_spec.rb b/backend/spec/services/query_state_spec.rb new file mode 100644 index 0000000..81f84bc --- /dev/null +++ b/backend/spec/services/query_state_spec.rb @@ -0,0 +1,33 @@ +require 'rails_helper' + +RSpec.describe QueryState do + it 'decodes typed axis values and keeps absence distinct from an empty set' do + { 'boolean' => ['false', false], 'number' => ['7', 7], + 'string' => ['a b & c', 'a b & c'], 'null' => ['', nil] }.each do |type, (raw, expected)| + params = { 'conditions' => '', 'conditions.0.field' => 'user', + 'conditions.0.operator' => 'axis_eq', + 'conditions.0.value_type' => type, 'conditions.0.value' => raw } + expect(described_class.records(params, 'conditions').sole['value']).to eq(expected) + end + expect(described_class.records({}, 'conditions')).to eq([]) + expect(described_class.records({ 'conditions' => '' }, 'conditions')).to eq([]) + expect(described_class.records({ 'conditions' => '', 'conditions.0.values' => '1' }, + 'conditions')).to eq([{ 'values' => true }]) + end + + it 'rejects gaps, unknown keys, JSON, wrong types and oversized lists' do + [ + { 'conditions' => '[]' }, { 'conditions' => '', 'conditions.1.field' => 'user' }, + { 'conditions' => '', 'conditions.0.sql' => 'TRUE' }, + { 'conditions' => '', 'conditions.0.blank' => 'true' }, + { 'conditions' => '', 'conditions.0.value' => ['x'] }, + { 'drill' => '', 'drill.0.field' => 'user', 'drill.0.value_type' => 'object' }, + { 'conditions' => 'x' * 131_073 }].each do |params| + expect { described_class.validate!(params, ['conditions', 'drill']) } + .to raise_error(ArgumentError) + end + ['user user', 'user title', ' user', (1..201).to_a.join(' ')].each do |value| + expect { described_class.list(value) }.to raise_error(ArgumentError) + end + end +end diff --git a/frontend/server.ssr.test.mjs b/frontend/server.ssr.test.mjs index 98a62b6..55bc512 100644 --- a/frontend/server.ssr.test.mjs +++ b/frontend/server.ssr.test.mjs @@ -62,14 +62,37 @@ const responseFor = url => { return { locales: [{ code: 'ja', name: '日本語' }, { code: 'ain-Kana', name: 'アイヌ' }] } if (path === '/users/settings') return { theme: 'system', wiki_editor_mode: 'split' } if (path === '/users/theme_slots') return [] - if (path === '/counts/posts' || path === '/counts/tags') { - const resource = path.endsWith ('posts') ? 'posts' : 'tags' - return { count: 1, page: 1, limit: 20, group_count: 1, - groups: [{ key: url.searchParams.get ('axis') === 'user' ? 1 : '2024-01', - name: null, count: 1 }], - versions: resource === 'posts' - ? [{ post_id: 1, version_no: 1, title: { current: `SSR count ${locale}` } }] - : [{ tag_id: 1, version_no: 1, name: { current: `SSR count ${locale}` } }] } + if (path.startsWith ('/counts/definitions/')) { + const resource = path.split ('/')[3] + const language = locale === 'ain-Kana' ? 'ain' : 'ja' + const names = ['ja', 'ain'].map (code => ({ key: `name_${code}`, + label: 'counts:localeName', name_locale: { code, language_code: code } })) + const columns = resource === 'posts' ? ['resource_id', 'title'] + : ['resource_id', `name_${language}`] + return { fields: [{ key: 'resource_id', type: 'integer', nullable: false, + operators: ['eq', 'ne'] }], + output_fields: [{ key: 'resource_id', label: 'counts:columns.resourceId' }, + ...(resource === 'posts' ? [{ key: 'title', label: 'counts:columns.postTitle' }] : names)], + name_columns: names, filter_fields: [], column_aliases: [], condition_aliases: [], + axes: { latest: ['resource_id'], history: ['resource_id'] }, grains: ['month'], + defaults: [ + { definition: 'latest', mode: 'summary', params: [ + { key: 'rows', value: '' }, { key: 'cols', value: '' }] }, + ...['latest', 'history'].map (definition => ({ definition, mode: 'details', params: [ + { key: 'columns', value: columns.join (' ') }, + { key: 'sort', value: 'resource_id:asc version_no:asc' }] }))] } + } + if (path === '/counts/query') { + const header = { values: [], kind: 'data', depth: 0, axis_count: 0, label_key: null } + return { count: 1, page: 1, limit: 100, group_count: 1, + output_count: 1, visible_output_count: 1, excluded_count: 0, + excluded_count_scope: 'current_query', execution_at: date, + rows: [], cols: [], row_headers: [header], column_headers: [header], + column_values: [[]], cells: [{ values: [], count: 1 }], pivot_totals: [], + condition_labels: {}, axis_labels: {}, versions: [{ id: 1, resource_id: 1, + version_no: 1, latest: true, created_at: date, created_by_user: null, + title: 'SSR count', name_ja: 'SSR count', name_ain: 'SSR count', + names: [], aliases: [], parents: [], tags: [] }] } } if (path === '/posts') return { posts: [post (locale)], count: 1 } if (path === '/gekanator/posts') return { posts: [post (locale)] } @@ -309,21 +332,24 @@ after (async () => { if (errors.length) throw new AggregateError (errors, 'SSR teardown failed') }) -test ('counts SSR renders history data and retains URL query state', async () => { +test ('counts SSR uses query/definitions, noindex and the common site title', async () => { for (const locale of ['ja', 'ain-Kana']) { for (const resource of ['posts', 'tags']) { - const column = resource === 'posts' ? 'title' : 'name' - const query = `mode=details&columns=${column}&user_id=4` + const column = resource === 'posts' ? 'title' : (locale === 'ja' ? 'name_ja' : 'name_ain') + const query = `definition=history&mode=details&columns=resource_id+${column}&user_id=4` const result = await renderPage (`/${locale}/counts/${resource}?${query}`, origin, apiURL) assert.equal (result.status, 200) - assert.match (result.html, new RegExp (`SSR count ${locale}`)) + assert.match (result.html, /SSR count/) assert.match (result.html, new RegExp (`/${locale}/${resource}/1`)) assert.doesNotMatch (result.html, /counts:|columns\.changedAt|fields\.actor/) - assert.match (result.html, /aria-expanded="false"/) - if (locale === 'ain-Kana') - assert.match (result.html, /lang="ja"[^>]*>集計実行/) - assert.ok (requests.some (request => request.url.includes (`/counts/${resource}`) - && request.url.includes ('user_id=4') && request.url.includes (`locale=${locale}`))) + assert.match (result.head, /]*name="robots"[^>]*content="noindex"/) + assert.match (result.head, /
+ className="max-h-72 w-full overflow-y-auto whitespace-pre-wrap
+ [overflow-wrap:anywhere] font-sans text-xs">
{JSON.stringify (countsNestedExport (rows, fields, path, params, locale))}
) : (columns.length > 0 && rows.length > 0 && (