From 6a0f1ec81990307c5e226cb352a59de8ff101733 Mon Sep 17 00:00:00 2001 From: miteruzo Date: Sat, 31 Oct 2026 04:08:05 +0900 Subject: [PATCH] #434 --- .../controllers/counts_exports_controller.rb | 5 +++ .../controllers/counts_tables_controller.rb | 4 ++ backend/app/services/counts_axes.rb | 11 ++++-- backend/app/services/counts_export_jobs.rb | 2 +- backend/app/services/counts_pivot_totals.rb | 2 +- backend/app/services/counts_population.rb | 2 + backend/app/services/counts_table_query.rb | 4 +- backend/spec/requests/counts_table_spec.rb | 23 ++++++++++- .../post_locale_filter_identity_spec.rb | 39 +++++++------------ .../requests/tag_description_wiki_spec.rb | 2 +- backend/spec/services/counts_export_spec.rb | 30 ++++++++++++++ docs/counts-exports.md | 20 ++++++++++ frontend/src/components/counts/CountsAxes.tsx | 2 +- .../src/components/counts/CountsControls.tsx | 6 +-- .../counts/CountsExportProgress.tsx | 28 +++++++++---- .../components/counts/CountsSortHeader.tsx | 6 ++- .../counts/CountsSummaryResults.tsx | 11 ++++-- frontend/src/lib/countsDisplay.ts | 1 + frontend/src/locales/ain-Kana/counts.json | 18 ++++++++- frontend/src/locales/ja/counts.json | 11 +++++- .../src/pages/CountsResourcePage.test.tsx | 1 + frontend/src/pages/CountsResourcePage.tsx | 16 ++++++-- .../src/pages/help/HelpChangesPage.test.tsx | 1 + 23 files changed, 188 insertions(+), 57 deletions(-) diff --git a/backend/app/controllers/counts_exports_controller.rb b/backend/app/controllers/counts_exports_controller.rb index 1a76464..c966bd7 100644 --- a/backend/app/controllers/counts_exports_controller.rb +++ b/backend/app/controllers/counts_exports_controller.rb @@ -99,6 +99,11 @@ class CountsExportsController < ApplicationController first = false end end + rescue IOError, SystemCallError => error + Rails.logger.warn("counts_download_stream_failure: #{ error.class }") + HistoryAccess.set(user: authority) do + CountsExportJobs.download_attempt(job, owner, **arguments, status: 'failed') + end ensure stream.close end diff --git a/backend/app/controllers/counts_tables_controller.rb b/backend/app/controllers/counts_tables_controller.rb index 9f88753..15ea3f1 100644 --- a/backend/app/controllers/counts_tables_controller.rb +++ b/backend/app/controllers/counts_tables_controller.rb @@ -18,6 +18,10 @@ class CountsTablesController < ApplicationController def index resolve_locale! render json: CountsTableQuery.new(request.query_parameters).call + rescue CountsAxes::TooManyAxes + render json: { error: 'too_many_axes', max_axes: CountsAxes::MAX_AXES }, status: :bad_request + rescue CountsAxes::PrivateCondition + render json: { error: 'private_condition' }, status: :forbidden rescue CountsTableQuery::TooManyColumns render json: { error: 'too_many_columns', max_columns: CountsTableQuery::MAX_COLUMNS }, status: :unprocessable_entity diff --git a/backend/app/services/counts_axes.rb b/backend/app/services/counts_axes.rb index 5ec084b..c7c6af8 100644 --- a/backend/app/services/counts_axes.rb +++ b/backend/app/services/counts_axes.rb @@ -68,6 +68,14 @@ class CountsAxes end end + def count_expression + return 'COUNT(DISTINCT v.id)' unless rows.any? { _1['field'] == 'tags' } + + # Keep tag occurrences even when several private tags share one masked label. + "COUNT(DISTINCT v.id, COALESCE(CONCAT('internal:', tag_entry.tag_id), " + + "CONCAT('external:', tag_entry.external_tag_id)))" + end + def joins(fields = all.map { _1['field'] }) result = '' if fields.include?('tags') @@ -85,9 +93,6 @@ class CountsAxes condition = @definition.changes.map do |field| "(changed.field_key = '#{ field }' AND #{ @population.changed_sql(field) })" end.join(' OR ') - unless HistoryAccess.admin? - condition = "(#{ condition }) OR NOT (#{ HistoryAccess.sql(@definition.model, 'v') })" - end result += " JOIN (#{ fields }) changed ON (#{ condition })" end result diff --git a/backend/app/services/counts_export_jobs.rb b/backend/app/services/counts_export_jobs.rb index 34b1ff8..7c575ef 100644 --- a/backend/app/services/counts_export_jobs.rb +++ b/backend/app/services/counts_export_jobs.rb @@ -198,7 +198,7 @@ class CountsExportJobs state['expires_at'] > Time.current.to_f && HistoryAccess.authorised_artifact?(state)) if attempt return unless state['download_attempt'] == attempt && state['token'] == file_token - return unless state['download_status'] == 'pending' + return unless ['pending', 'started'].include?(state['download_status']) state['download_status'] = status if status else diff --git a/backend/app/services/counts_pivot_totals.rb b/backend/app/services/counts_pivot_totals.rb index f25a0ab..29c81b7 100644 --- a/backend/app/services/counts_pivot_totals.rb +++ b/backend/app/services/counts_pivot_totals.rb @@ -49,7 +49,7 @@ class CountsPivotTotals leaf = retained.map { @axes.expressions[_1] } selected = leaf.each_with_index.map { |expression, index| "#{ expression } AS k#{ index }" } inner = scope.group(*leaf.map { Arel.sql(_1) }) - .select(Arel.sql((selected + ['COUNT(DISTINCT v.id) AS count']).join(', '))).to_sql + .select(Arel.sql((selected + ["#{ @axes.count_expression } AS count"]).join(', '))).to_sql coordinates = indexes.map { "k#{ retained.index(_1) }" } projected = coordinates.each_with_index.map { |key, index| "#{ key } AS v#{ index }" } output = ["#{ row_depth } AS row_depth", "#{ column_depth } AS column_depth"] + diff --git a/backend/app/services/counts_population.rb b/backend/app/services/counts_population.rb index 986895d..9dd0c4e 100644 --- a/backend/app/services/counts_population.rb +++ b/backend/app/services/counts_population.rb @@ -63,6 +63,8 @@ class CountsPopulation end def axis_condition(scope, entry) + raise CountsAxes::PrivateCondition if entry['value'] == CountsAxes::MASKED_VALUE + key = entry.fetch('field') raise ArgumentError unless definition.axes(history).include?(key) if entry.key?('grain') diff --git a/backend/app/services/counts_table_query.rb b/backend/app/services/counts_table_query.rb index bf4ee40..62f291d 100644 --- a/backend/app/services/counts_table_query.rb +++ b/backend/app/services/counts_table_query.rb @@ -88,7 +88,7 @@ class CountsTableQuery source.except(:select).select('COUNT(DISTINCT v.id) AS count').to_sql else source.except(:select).group(*expressions.map { Arel.sql(_1) }) - .select(Arel.sql((selected + ['COUNT(DISTINCT v.id) AS count']).join(', '))) + .select(Arel.sql((selected + ["#{ axes.count_expression } AS count"]).join(', '))) .to_sql end prefix = "WITH cells AS (#{ sql })" @@ -359,7 +359,7 @@ class CountsTableQuery end selected = row_expressions.each_with_index.map { |expression, index| "#{ expression } AS a#{ index }" - } + ['COUNT(DISTINCT v.id) AS count'] + } + ["#{ axes.count_expression } AS count"] source = source.group(*row_expressions.map { Arel.sql(_1) }) if row_expressions.any? source.select(Arel.sql(selected.join(', '))).to_sql end diff --git a/backend/spec/requests/counts_table_spec.rb b/backend/spec/requests/counts_table_spec.rb index 90c7e1b..4c96c40 100644 --- a/backend/spec/requests/counts_table_spec.rb +++ b/backend/spec/requests/counts_table_spec.rb @@ -232,7 +232,7 @@ RSpec.describe 'Counts table API', type: :request do it 'does not resolve private condition names even when the result set is empty' do TagVersion.create!(tag:, version_no: 2, event_type: 'discard', name: 'secret tag', category: 'general', aliases: '', parent_tag_ids: '', - names: { ja: { name: 'secret tag', script_code: 'Jpan', aliases: [] } }, + names: { ja: { name: 'secret tag', script_code: 'Jpan', auto_generated: false, aliases: [] } }, created_at: Time.utc(2024, 2)) result = query(axis('tags', "internal:#{ tag.id }").merge(definition: 'latest')) expect(result['versions']).to be_empty @@ -245,4 +245,25 @@ RSpec.describe 'Counts table API', type: :request do end end end + it 'includes private history in the public timeline without an exclusion count' do + PostVersion.create!(post: record, version_no: 4, event_type: 'discard', url: record.url, + title: 'hidden title', tags: '', tags_json: [], parent_post_ids: '', + created_at: Time.utc(2024, 3)) + result = query(mode: 'summary', rows: 'created_at:month', cols: 'event_type') + expect(result['excluded_count']).to be_nil + expect(result['count']).to eq(4) + expect(result['cells'].sum { _1['count'] }).to eq(4) + expect(response.body).not_to include('hidden title') + masked = query(mode: 'summary', rows: 'resource_id', cols: '') + expect(masked['cells'].map { _1['values'] }).to eq([[CountsAxes::MASKED_VALUE]]) + expect(masked['cells'].first['count']).to eq(4) + expect(query['versions']).to be_empty + end + + it 'rejects private summary predicates instead of exposing a count oracle' do + get '/counts/query', params: base.merge(mode: 'summary', resource_id: record.id.to_s) + expect(response).to have_http_status(:forbidden) + expect(json).to eq('error' => 'private_condition') + end + end diff --git a/backend/spec/requests/post_locale_filter_identity_spec.rb b/backend/spec/requests/post_locale_filter_identity_spec.rb index 3f165b1..50dc16a 100644 --- a/backend/spec/requests/post_locale_filter_identity_spec.rb +++ b/backend/spec/requests/post_locale_filter_identity_spec.rb @@ -21,29 +21,7 @@ RSpec.describe 'Post filter identity across locales', type: :request do ['nico:shared', 'union_alias', 'not:nico:shared'].each do |literal| it "preserves the complete match set for #{ literal }" do - reference_params = { tag_refs: '' } - - references.each_with_index do |reference, index| - %w[name tag_id external_tag_id negative missing].each do |field| - value = reference[field] - next if value.nil? - next if field == 'missing' && value == false - - reference_params["tag_refs.#{index}.#{field}"] = - case value - when true - '1' - when false - '0' - else - value.to_s - end - end - end - - get '/posts', params: { - locale: 'ain-Kana', - tags: references.map { _1.fetch('name') }.join(' ') }.merge(reference_params) + get '/posts', params: { locale: 'ja', tags: literal } expect(response).to have_http_status(:ok) original_ids = json.fetch('posts').map { _1.fetch('id') } expected = literal.start_with?('not:') ? [other_post.id] : @@ -56,9 +34,18 @@ RSpec.describe 'Post filter identity across locales', type: :request do references = json.fetch('references') expect(references.first).to include('tag_id' => tag.id, 'external_tag_id' => external.id) - get '/posts', params: { - locale: 'ain-Kana', tags: references.map { _1.fetch('name') }.join(' '), - tag_refs: references.to_json } + reference_params = { tag_refs: '' } + references.each_with_index do |reference, index| + ['name', 'tag_id', 'external_tag_id', 'negative', 'missing'].each do |field| + value = reference[field] + next if value.nil? || (field == 'missing' && value == false) + + reference_params["tag_refs.#{ index }.#{ field }"] = + [true, false].include?(value) ? (value ? '1' : '0') : value.to_s + end + end + get '/posts', params: { locale: 'ain-Kana', + tags: references.map { _1.fetch('name') }.join(' ') }.merge(reference_params) expect(response).to have_http_status(:ok) expect(json.fetch('posts').map { _1.fetch('id') }).to match_array(original_ids) end diff --git a/backend/spec/requests/tag_description_wiki_spec.rb b/backend/spec/requests/tag_description_wiki_spec.rb index 374b7dd..19b0e19 100644 --- a/backend/spec/requests/tag_description_wiki_spec.rb +++ b/backend/spec/requests/tag_description_wiki_spec.rb @@ -63,7 +63,7 @@ RSpec.describe 'Tag description Wiki compatibility', type: :request do end it 'does not fall back when the new current revision is missing' do - versions.last.destroy! + TagDescriptionVersion.where(id: versions.last.id).delete_all without_legacy_reads { get "/wiki/#{ description.id }" } expect(response).to have_http_status(:not_found) end diff --git a/backend/spec/services/counts_export_spec.rb b/backend/spec/services/counts_export_spec.rb index f7b92e3..15e3b73 100644 --- a/backend/spec/services/counts_export_spec.rb +++ b/backend/spec/services/counts_export_spec.rb @@ -171,3 +171,33 @@ RSpec.describe 'Counts export contracts' do end end end + +RSpec.describe 'Counts download acknowledgement' do + around do |example| + Dir.mktmpdir('counts-ack-spec') do |path| + allow(CountsExportDownload).to receive(:directory).and_return(Pathname.new(path)) + HistoryAccess.set(user: nil) { example.run } + end + end + + it 'binds an acknowledgement to the owner, job, file token and fresh attempt' do + CountsExportJobs.pulse + payload = { 'resource' => 'posts', 'definition' => 'history', 'mode' => 'details', + 'format' => 'json', 'locale' => 'ja' } + token = CountsExportJobs.start(payload, 'owner', 'ja') + id = CountsExportJobs.authorised_id(token, 'owner') + state = CountsExportJobs.read(id).merge('status' => 'ready', 'token' => 'file-token') + CountsExportJobs.write(state) + receipt = CountsExportJobs.download_attempt(token, 'owner') + expect(receipt['download_status']).to eq('pending') + arguments = { file_token: 'file-token', attempt: receipt.fetch('download_attempt') } + expect(CountsExportJobs.download_attempt(token, 'other', **arguments)).to be_nil + expect(CountsExportJobs.download_attempt(token, 'owner', **arguments.merge(file_token: 'wrong'))).to be_nil + CountsExportJobs.download_attempt(token, 'owner', **arguments, status: 'started') + expect(CountsExportJobs.status(token, 'owner')['download_status']).to eq('started') + CountsExportJobs.download_attempt(token, 'owner') + expect(CountsExportJobs.download_attempt(token, 'owner', **arguments)).to be_nil + CountsExportJobs.status(token, 'owner', cancel: true) + expect(CountsExportJobs.download_attempt(token, 'owner')).to be_nil + end +end diff --git a/docs/counts-exports.md b/docs/counts-exports.md index c3ae6ea..ec8d202 100644 --- a/docs/counts-exports.md +++ b/docs/counts-exports.md @@ -105,3 +105,23 @@ CSV の物理行と JSON の pivot_cell には集計レコードを含むため 追加は許可後の新規 add_index migration として行い、既存 migration は変更しない。 実DBでのインデックス容量・作成時ロック・実行計画・時間は未確認。 索引追加なしでの最大データ量に対する応答時間は保証しない。 + + +## GET 応答開始の確認（静的実装・未実測） + +進捗画面は ready 後に owner/job に紐づく download_attempt を登録し、同じ attempt を +native iframe の実 GET に渡す。サーバーは署名・所有者・権限 revision・期限・ファイルを +再検証し、64KiB の最初のチャンクを Rack 出力側へ渡した後に started を記録する。 +HEAD や受付 POST では started にしない。確認には30秒の独立した期限がある。 +サーバーの出力開始はブラウザー受信・保存完了を保証しない。途中の切断もあり得る。 +開始確認前の失敗はダイアログに残る。再実行はダイアログを閉じてエクスポートをやり直す。 + +Web と worker は同じ COUNTS_EXPORT_DIRECTORY、Rails secret、DB、実行ユーザーで +共有 POSIX filesystem を使うこと。別ホストではローカル tmp を共有できない。 +systemd の EnvironmentFile と Web の環境を照合し、directory の owner と0700相当の +アクセス制限、容量、期限切れ削除、worker heartbeat と restart を公開前に確認する。 +この作業では systemctl・worker 起動・本番環境への適用を行っていない。 + +リリース保留：非公開グループと複数値・変更項目軸の交差集計、生成ファイルへの秘匿値の +非混入、実 GET の middleware/proxy 経由の開始通知は、回帰テストと実環境の確認が必要。 +テスト・型チェック・build は未実行。 diff --git a/frontend/src/components/counts/CountsAxes.tsx b/frontend/src/components/counts/CountsAxes.tsx index 175546d..89b0816 100644 --- a/frontend/src/components/counts/CountsAxes.tsx +++ b/frontend/src/components/counts/CountsAxes.tsx @@ -51,7 +51,7 @@ const CountsAxes = ({ params, update }: { return
{label (id === 'available' ? 'axisItems' : id)} -
{content}
} renderItem={(id, group) => { const axis = axes.find (axis => axis.field === id) diff --git a/frontend/src/components/counts/CountsControls.tsx b/frontend/src/components/counts/CountsControls.tsx index 5a9a3c1..02f375f 100644 --- a/frontend/src/components/counts/CountsControls.tsx +++ b/frontend/src/components/counts/CountsControls.tsx @@ -185,16 +185,16 @@ const CountsControls = ( className={inputClass (false, 'w-20 py-1')} onChange={event => update ('limit', event.target.value)}> {[20, 50, 100].map (count => )} + {message ('shared:labels.resultsCount', { count }).text})} - } {details && resource === 'posts' &&