From 2563b3fef30c4e9dc00ad9744d8d7885350ce957 Mon Sep 17 00:00:00 2001 From: miteruzo Date: Mon, 26 Oct 2026 21:24:32 +0900 Subject: [PATCH] #434 --- .../controllers/counts_exports_controller.rb | 30 +++ .../app/services/counts_export_download.rb | 18 +- backend/app/services/counts_export_jobs.rb | 236 ++++++++++++++++++ .../app/services/counts_export_projection.rb | 22 +- backend/app/services/counts_exporter.rb | 10 +- backend/app/services/counts_table_export.rb | 19 +- backend/config/routes.rb | 3 + backend/lib/tasks/counts_exports.rake | 11 + docs/counts-exports.md | 53 ++++ .../counts/CountsAdditionalConditions.tsx | 27 +- .../src/components/counts/CountsControls.tsx | 9 +- .../counts/CountsExportProgress.tsx | 130 ++++++++++ .../components/counts/CountsNestedTable.tsx | 12 +- .../components/dialogues/DialogueProvider.tsx | 20 +- frontend/src/components/ui/dialog.tsx | 8 +- frontend/src/lib/api.ts | 9 + frontend/src/lib/counts.ts | 44 +++- frontend/src/lib/countsDisplay.ts | 60 ++++- frontend/src/lib/dialogues/useDialogue.ts | 1 + frontend/src/locales/ja/counts.json | 13 + frontend/src/pages/CountsResourcePage.tsx | 22 +- 21 files changed, 703 insertions(+), 54 deletions(-) create mode 100644 backend/app/services/counts_export_jobs.rb create mode 100644 backend/lib/tasks/counts_exports.rake create mode 100644 docs/counts-exports.md create mode 100644 frontend/src/components/counts/CountsExportProgress.tsx diff --git a/backend/app/controllers/counts_exports_controller.rb b/backend/app/controllers/counts_exports_controller.rb index 537d512..7503679 100644 --- a/backend/app/controllers/counts_exports_controller.rb +++ b/backend/app/controllers/counts_exports_controller.rb @@ -1,6 +1,27 @@ class CountsExportsController < ApplicationController wrap_parameters false + def create_job + locale = resolve_locale! + payload = request.request_parameters.merge(request.query_parameters) + session[:counts_export_owner] ||= SecureRandom.hex(32) + job = CountsExportJobs.start(payload, session[:counts_export_owner], locale.language_code) + response.headers['Cache-Control'] = 'private, no-store' + render json: { job: }, status: :accepted + rescue CountsExportDownload::Busy + head :too_many_requests + rescue ArgumentError, TypeError, KeyError, JSON::ParserError + head :bad_request + end + + def job_status + job_state(cancel: false) + end + + def cancel_job + job_state(cancel: true) + end + def prepare locale = resolve_locale! payload = request.request_parameters.merge(request.query_parameters) @@ -32,6 +53,15 @@ class CountsExportsController < ApplicationController disposition: 'attachment' end + def job_state(cancel:) + result = CountsExportJobs.status(params[:token], session[:counts_export_owner], cancel:) + return head :not_found unless result + + response.headers['Cache-Control'] = 'private, no-store' + render json: result + end + private :job_state + def index locale = resolve_locale! payload = request.request_parameters.except('authenticity_token') diff --git a/backend/app/services/counts_export_download.rb b/backend/app/services/counts_export_download.rb index b773c5c..a7c5568 100644 --- a/backend/app/services/counts_export_download.rb +++ b/backend/app/services/counts_export_download.rb @@ -13,15 +13,17 @@ class CountsExportDownload class TimedOut < StandardError; end class StorageFull < StandardError; end - def self.directory = Rails.root.join('tmp', 'counts_exports') + def self.directory + Rails.root.join(ENV.fetch('COUNTS_EXPORT_DIRECTORY', 'tmp/counts_exports')) + end def self.verifier = Rails.application.message_verifier('counts_export_download') - def self.prepare(exporter, owner) + def self.prepare(exporter, owner, progress: nil, check: nil, identifier: nil) FileUtils.mkdir_p(directory, mode: 0o700) slot = acquire_slot begin - generate(exporter, owner) + generate(exporter, owner, progress:, check:, identifier:) ensure slot.close end @@ -38,7 +40,7 @@ class CountsExportDownload raise Busy end - def self.generate(exporter, owner) + def self.generate(exporter, owner, progress: nil, check: nil, identifier: nil) FileUtils.mkdir_p(directory, mode: 0o700) Dir.glob(directory.join('counts-*.export')).each do |path| File.unlink(path) if File.mtime(path) < Time.current - EXPIRY @@ -53,14 +55,16 @@ class CountsExportDownload reserved = MAX_CONCURRENT_EXPORTS * MAX_FILE_BYTES raise StorageFull if stored + reserved > MAX_STORED_BYTES - file = Tempfile.create(['counts-', '.part'], directory) + prefix = identifier ? "counts-#{ identifier }-" : 'counts-' + file = Tempfile.create([prefix, '.part'], directory) complete = false bytes = 0 deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + MAX_EXPORT_SECONDS path = file.path begin file.binmode - exporter.body.each do |chunk| + exporter.body(progress:).each do |chunk| + check&.call raise TimedOut if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline bytes += chunk.bytesize @@ -68,6 +72,8 @@ class CountsExportDownload file.write(chunk) end + file.flush + file.fsync file.close finished_path = path.sub(/\.part\z/, '.export') File.rename(path, finished_path) diff --git a/backend/app/services/counts_export_jobs.rb b/backend/app/services/counts_export_jobs.rb new file mode 100644 index 0000000..41f4afc --- /dev/null +++ b/backend/app/services/counts_export_jobs.rb @@ -0,0 +1,236 @@ +require 'json' +require 'fileutils' + +# A durable, bounded export spool. Web and supervised workers share the same POSIX volume. +class CountsExportJobs + class Cancelled < StandardError; end + ACTIVE = ['queued', 'running', 'cancelling'].freeze + + def self.directory = CountsExportDownload.directory.join('jobs') + def self.verifier = Rails.application.message_verifier('counts_export_job') + + def self.synchronise + FileUtils.mkdir_p(directory, mode: 0o700) + File.open(directory.join('.registry.lock'), File::RDWR | File::CREAT, 0o600) do |lock| + lock.flock(File::LOCK_EX) + yield + end + end + + def self.write(state) + path = directory.join("#{ state.fetch('id') }.json") + temporary = "#{ path }.#{ SecureRandom.hex(8) }.tmp" + File.open(temporary, 'w', 0o600) do |file| + file.write(JSON.generate(state)) + file.flush + file.fsync + end + File.rename(temporary, path) + ensure + File.unlink(temporary) if temporary && File.exist?(temporary) + end + + def self.read(id) + JSON.parse(File.read(directory.join("#{ id }.json"))) + rescue Errno::ENOENT + nil + end + + def self.remove_download(state) + name = state['download_file'] + return unless name&.match?(/\Acounts-[a-zA-Z0-9_-]+\.export\z/) + + File.unlink(CountsExportDownload.directory.join(name)) + rescue Errno::ENOENT + nil + end + + def self.remove_partial(id) + Dir.glob(CountsExportDownload.directory.join("counts-#{ id }-*")).each do |path| + File.unlink(path) + rescue Errno::ENOENT + nil + end + end + + def self.clean + Dir.glob(directory.join('*.tmp')).each do |path| + File.unlink(path) if File.mtime(path) < Time.current - CountsExportDownload::EXPIRY + end + Dir.glob(directory.join('*.run')).each do |path| + id = File.basename(path, '.run') + next if directory.join("#{ id }.json").exist? + + File.open(path, File::RDWR) do |lock| + File.unlink(path) if lock.flock(File::LOCK_EX | File::LOCK_NB) + end + end + Dir.glob(directory.join('*.json')).each do |path| + state = JSON.parse(File.read(path)) + next if state.fetch('expires_at') > Time.current.to_f + + run = directory.join("#{ state.fetch('id') }.run") + File.open(run, File::RDWR | File::CREAT, 0o600) do |lock| + unless lock.flock(File::LOCK_EX | File::LOCK_NB) + state['status'] = 'cancelling' + write(state) + next + end + remove_download(state) + remove_partial(state.fetch('id')) + File.unlink(path) + File.unlink(run) + end + end + end + + def self.start(payload, owner, language) + payload = payload.dup + time_key = payload.key?('definition') ? 'execution_at' : 'at' + if payload[time_key].nil? || payload[time_key] == '' + payload[time_key] = Time.current.utc.iso8601(6) + end + exporter = CountsExporter.new(payload, language_code: language) + synchronise do + clean + active = Dir.glob(directory.join('*.json')).count do |path| + ACTIVE.include?(JSON.parse(File.read(path))['status']) + end + raise CountsExportDownload::Busy if active >= CountsExportDownload::MAX_CONCURRENT_EXPORTS + + id = SecureRandom.hex(24) + state = { 'id' => id, 'owner' => owner, 'payload' => payload, + 'language' => language, 'resource' => exporter.resource, + 'format' => exporter.format, 'status' => 'queued', + 'processed' => 0, 'total' => nil, + 'expires_at' => (Time.current + CountsExportDownload::EXPIRY).to_f } + write(state) + verifier.generate({ 'id' => id, 'owner' => owner }, + expires_in: CountsExportDownload::EXPIRY, purpose: 'counts_export_job') + end + end + + def self.authorised_id(token, owner) + value = verifier.verified(token, purpose: 'counts_export_job') + return unless value.is_a?(Hash) && value['owner'] == owner + return unless value['id']&.match?(/\A[a-f0-9]{48}\z/) + + value['id'] + end + + def self.status(token, owner, cancel: false) + id = authorised_id(token, owner) + return unless id + + synchronise do + state = read(id) + return unless state && state['owner'] == owner && state['expires_at'] > Time.current.to_f + + if cancel && ['queued', 'running', 'ready'].include?(state['status']) + state['status'] = state['status'] == 'running' ? 'cancelling' : 'cancelled' + remove_download(state) + state.delete('token') + write(state) + end + state.slice('status', 'processed', 'total', 'format', 'token', 'error') + end + end + + def self.checkpoint(id, processed = nil, total = nil) + synchronise do + state = read(id) + unless state && state['status'] == 'running' && state['expires_at'] > Time.current.to_f + raise Cancelled + end + if processed + state.merge!('processed' => processed, 'total' => total) + write(state) + end + end + end + + def self.work_one + FileUtils.mkdir_p(directory, mode: 0o700) + Dir.glob(directory.join('*.json')).sort.each do |path| + id = File.basename(path, '.json') + File.open(directory.join("#{ id }.run"), File::RDWR | File::CREAT, 0o600) do |lock| + next unless lock.flock(File::LOCK_EX | File::LOCK_NB) + + state = synchronise do + candidate = read(id) + next unless candidate && ACTIVE.include?(candidate['status']) + + if candidate['status'] != 'queued' + candidate['status'] = candidate['status'] == 'cancelling' ? 'cancelled' : 'failed' + remove_partial(id) + candidate['error'] = 'interrupted' + write(candidate) + next + end + candidate['status'] = 'running' + write(candidate) + candidate + end + next unless state + + perform(state) + return true + end + end + synchronise { clean } + false + end + + def self.perform(state) + id = state.fetch('id') + token = nil + begin + checkpoint(id) + exporter = CountsExporter.new(state.fetch('payload'), language_code: state.fetch('language')) + last = Process.clock_gettime(Process::CLOCK_MONOTONIC) + progress = lambda do |processed, total| + now = Process.clock_gettime(Process::CLOCK_MONOTONIC) + if processed.zero? || now - last >= 0.25 || processed == total + checkpoint(id, processed, total) + last = now + end + end + check = lambda do + now = Process.clock_gettime(Process::CLOCK_MONOTONIC) + if now - last >= 0.25 + checkpoint(id) + last = now + end + end + token = CountsExportDownload.prepare(exporter, state.fetch('owner'), + progress:, check:, identifier: id) + synchronise do + current = read(id) + if (!current || current['status'] != 'running' || + current['expires_at'] <= Time.current.to_f) + file = CountsExportDownload.find(token, state.fetch('owner')) + File.unlink(file.fetch('path')) if file + raise Cancelled + end + file = CountsExportDownload.find(token, state.fetch('owner')) + current.merge!('status' => 'ready', 'token' => token, + 'download_file' => File.basename(file.fetch('path'))) + current['total'] = current['processed'] + write(current) + end + rescue StandardError => error + remove_partial(id) + synchronise do + current = read(id) + if current + current['status'] = error.is_a?(Cancelled) ? 'cancelled' : 'failed' + current['error'] = error.class.name + write(current) + end + end + unless error.is_a?(Cancelled) + Rails.logger.error("counts_export_job #{ id }: #{ error.class }") + end + end + end +end diff --git a/backend/app/services/counts_export_projection.rb b/backend/app/services/counts_export_projection.rb index 0b15f1e..3285d3b 100644 --- a/backend/app/services/counts_export_projection.rb +++ b/backend/app/services/counts_export_projection.rb @@ -24,6 +24,22 @@ class CountsExportProjection 'aliases' => ['languageCode', 'name', 'scriptCode'], 'parents' => ['id'] }.freeze + def self.default_columns(resource) + fields = { 'posts' => ['title', 'url', 'tags', 'parents', 'video_ms'], + 'tags' => ['name', 'category', 'aliases', 'parents'], + 'nico_tags' => ['name', 'linked_tags'], + 'tag_descriptions' => ['language_code', 'body', 'source_language_code'], + 'tag_groups' => ['name', 'description', 'members'], + 'materials' => ['url', 'tag_id', 'source_kind'] }.fetch(resource) + ['resource_id', 'version_no'] + fields + ['event_type', 'user', 'created_at'] + end + + def self.default_children(resource) + { 'posts' => ['tags.name', 'parents.id'], + 'tags' => ['aliases.name', 'parents.id'], + 'tag_groups' => ['members.tagId'] }.fetch(resource, []) + end + attr_reader :columns, :headers def initialize(params, resource:, summary:, tags:) @@ -37,7 +53,8 @@ class CountsExportProjection else CountsQuery::COLUMNS.fetch(resource) end - @columns = params.fetch('columns', allowed.join(',')).split(',') + defaults = summary ? allowed : self.class.default_columns(resource) + @columns = params.fetch('columns', defaults.join(',')).split(',') raise ArgumentError if (@columns - allowed).any? || @columns.uniq != @columns @headers = JSON.parse(params.fetch('headers', JSON.generate(@columns))) @@ -55,7 +72,8 @@ class CountsExportProjection [path, direction] end - @selected = params.key?('children') ? params['children'].split(',').to_set : nil + @selected = params.key?('children') ? params['children'].split(',').to_set : + self.class.default_children(resource).to_set valid = CHILDREN.flat_map do |key, fields| paths(fields, key) end diff --git a/backend/app/services/counts_exporter.rb b/backend/app/services/counts_exporter.rb index d6ad94c..baf1a25 100644 --- a/backend/app/services/counts_exporter.rb +++ b/backend/app/services/counts_exporter.rb @@ -47,14 +47,17 @@ class CountsExporter end end - def body - return @table_export.body if @table_export + def body(progress: nil) + return @table_export.body(progress:) if @table_export Enumerator.new do |output| ApplicationRecord.transaction(isolation: :repeatable_read) do + processed = 0 first_row = true started = false pages do |page| + total = @mode == 'details' ? page[:count] : nil + progress&.call(processed, total) unless started meta = { resource:, mode: @mode, basis: @history ? 'history' : 'state', at: @query_params['at'], boundary: 'inclusive', @@ -83,8 +86,11 @@ class CountsExporter output << CSV.generate_line(values, row_sep: "\r\n") end first_row = false + processed += 1 + progress&.call(processed, total) end end + progress&.call(processed, processed) output << ']}' if format == 'json' end end diff --git a/backend/app/services/counts_table_export.rb b/backend/app/services/counts_table_export.rb index 3f9fbf6..6f89b7e 100644 --- a/backend/app/services/counts_table_export.rb +++ b/backend/app/services/counts_table_export.rb @@ -10,6 +10,10 @@ class CountsTableExport raise ArgumentError if (params.keys - CountsTableQuery::KEYS - EXTRA).any? raise ArgumentError unless params.values.all? { _1.is_a?(String) } + unless params.key?('children') + params = params.merge('children' => CountsExportProjection.default_children( + params.fetch('resource', 'posts')).join(',')) + end @params = params @labels = JSON.parse(params.fetch('labels', '{}')) raise ArgumentError unless @labels.is_a?(Hash) @@ -27,8 +31,11 @@ class CountsTableExport @query = CountsTableQuery.new(params.slice(*CountsTableQuery::KEYS) .merge('page' => '1', 'limit' => '100')) @summary = @query.mode == 'summary' - @columns = params.fetch('columns', '').split(',') - @headers = JSON.parse(params.fetch('headers', '[]')) + default_columns = @summary ? @query.axes.all.map { _1.fetch('field') } + ['count'] : + CountsExportProjection.default_columns(resource) + @columns = params.fetch('columns', default_columns.join(',')).split(',') + default_headers = @columns.map { @labels.fetch(_1, _1) } + @headers = JSON.parse(params.fetch('headers', default_headers.to_json)) raise ArgumentError unless @headers.is_a?(Array) && @headers.all? { _1.is_a?(String) } if @summary && @headers.any? && @headers.length != @query.axes.all.length + 1 raise ArgumentError @@ -45,13 +52,16 @@ class CountsTableExport end end - def body + def body(progress: nil) Enumerator.new do |output| ApplicationRecord.transaction(isolation: :repeatable_read) do @language_names = Language.pluck(:code, :name).to_h + processed = 0 started = false first = true @query.each_page do |page| + total = @summary ? nil : page[:count] + progress&.call(processed, total) unless started metadata = page.except(:versions, :cells, :axis_labels, :column_values).merge( resource:, filters: @params.slice(*CountsTableQuery::KEYS), @@ -84,8 +94,11 @@ class CountsTableExport output << CSV.generate_line(values.map { safe(_1) }, row_sep: "\r\n") end first = false + processed += 1 + progress&.call(processed, total) end end + progress&.call(processed, processed) output << ']}' if format == 'json' end end diff --git a/backend/config/routes.rb b/backend/config/routes.rb index 0d5e948..faecb5c 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -1,6 +1,9 @@ Rails.application.routes.draw do get 'counts/definitions/:resource', to: 'counts_tables#definition' get 'counts/query', to: 'counts_tables#index' + post 'counts/export/jobs', to: 'counts_exports#create_job' + get 'counts/export/jobs/:token', to: 'counts_exports#job_status' + delete 'counts/export/jobs/:token', to: 'counts_exports#cancel_job' post 'counts/export/prepare', to: 'counts_exports#prepare' get 'counts/export/files/:token', to: 'counts_exports#download' get 'counts/export', to: 'counts_exports#index' diff --git a/backend/lib/tasks/counts_exports.rake b/backend/lib/tasks/counts_exports.rake new file mode 100644 index 0000000..4c4668f --- /dev/null +++ b/backend/lib/tasks/counts_exports.rake @@ -0,0 +1,11 @@ +namespace :counts do + desc 'Process the persistent Counts export queue (run under a process supervisor)' + task export_worker: :environment do + stopping = false + ['TERM', 'INT'].each { |signal| Signal.trap(signal) { stopping = true } } + until stopping + worked = Rails.application.executor.wrap { CountsExportJobs.work_one } + sleep(1) unless worked + end + end +end diff --git a/docs/counts-exports.md b/docs/counts-exports.md new file mode 100644 index 0000000..c878fb0 --- /dev/null +++ b/docs/counts-exports.md @@ -0,0 +1,53 @@ +# Counts エクスポートワーカー + +Counts の画面からのエクスポートは永続ファイルキューへ登録する。 +Active Job の既定のプロセス内 async アダプターは使用しない。 +追加 gem、外部キュー、DB マイグレーションは不要。 + +## 起動・配置 + +backend で `bundle exec rake counts:export_worker` を、systemd 等の監視対象として +別プロセスで起動する。Web プロセス内では起動しない。 +ワーカー未起動の間は待機状態となる。1 プロセスは同時に1ジョブを処理し、 +全体の実行枠は既存の2枠を共有する。ワーカーの起動・配備は実施していない。 + +Web とワーカーは同一リリース、Rails secret、アプリケーション環境、DB 接続先を +使用する。DB コネクションはワーカープロセスごとに必要。 +`COUNTS_EXPORT_DIRECTORY` を同じ永続 POSIX 共有ディレクトリに設定する。 +省略時は `backend/tmp/counts_exports`。 +全プロセスで同一の所有者とアクセス権を使用する。 +複数ホストでは、flock・同一ファイルシステム内の atomic rename・fsync が正しく +機能する共有ボリュームが必須。オブジェクトストレージや各コンテナ独立の tmp は不可。 +ジョブ JSON と完成ファイルが格納されるため、このディレクトリを静的公開しない。 + +## API と進捗 + +- POST `/counts/export/jobs`：要求検証・実行時点の固定・署名付きジョブトークンの発行。 +- GET `/counts/export/jobs/:token`：状態、実際の処理済みレコード数、判明している総数。 +- DELETE `/counts/export/jobs/:token`：取消を永続記録。 +- GET `/counts/export/files/:token`：生成完了後の署名付きファイルトークンによる取得。 + +ジョブ・ファイルのトークンは作成元のセッションに紐付く。 +既存の BAN・CSRF 制御を使用する。フロントは約800ms間隔で状態を取得する。 +明細の総数は同じ読み取りトランザクション内の SQL 件数を使用する。 +集計結果の出力セル総数は生成終了まで確定させず、不確定進捗として扱う。 +完成状態はファイルの生成完了であり、ブラウザの保存完了を意味しない。 + +## 取消・障害・制限 + +通常は約250ms間隔の生成チェックポイントで取消を検知する。 +DB 問い合わせ・ファイル I/O がブロックしている間は即時に中断できない。 +取消済みへの遷移は生成処理の終了と一時ファイル削除後に行う。 +既に始まった HTTP 転送やブラウザ側の保存は取り消せない。 + +ワーカーが異常終了したジョブは、次のワーカーが実行ロックを取得した際に失敗状態にし、 +そのジョブの残存生成ファイルを削除する。別の読み取り状態で自動再実行しない。 +原子的に書き換える状態ファイルとプロセス間ロックにより、二重処理・取消との競合を防ぐ。 + +既存の同時実行2、1ファイル128MiB、保存領域512MiB、生成120秒、期限10分を維持する。 +生成時間制限もチェックポイント方式であり、実行中 SQL の強制停止ではない。 +期限切れのジョブ・ファイル・残存状態書込ファイルは登録時またはワーカーの巡回時に削除する。 +ワーカーと Web の両方を停止した間は巡回削除も停止する。 + +これらの処理は静的確認のみで、実環境のキュー処理・取消・転送・共有ボリュームでの +排他動作は未確認。 diff --git a/frontend/src/components/counts/CountsAdditionalConditions.tsx b/frontend/src/components/counts/CountsAdditionalConditions.tsx index 9f58347..ce768d8 100644 --- a/frontend/src/components/counts/CountsAdditionalConditions.tsx +++ b/frontend/src/components/counts/CountsAdditionalConditions.tsx @@ -21,21 +21,36 @@ const ConditionEditor = ( ) => { const { label, message } = useUITranslation ('counts') const fields = countsFilterFields (definition) - const [field, setField] = useState (initial?.field ?? 'resource_id') - const [operator, setOperator] = useState (initial?.operator ?? 'eq') + const [condition, setCondition] = useState (() => { + const next = { ...initial, field: initial?.field ?? 'resource_id', + operator: initial?.operator ?? 'eq' } + if (next.operator !== 'range') delete next.end + return next + }) + const { field, operator } = condition const operators = definition.fields.find (item => item.key === field)?.operators ?? [] useEffect (() => { controls.setActions ([{ label: label (initial == null ? 'add' : 'shared:actions.edit'), - onSelect: () => save ({ ...initial, field, operator }) }]) - }, [controls, field, operator, initial, label, save]) + onSelect: () => save (condition) }]) + }, [controls, condition, initial, label, save]) return
diff --git a/frontend/src/components/counts/CountsControls.tsx b/frontend/src/components/counts/CountsControls.tsx index 29fefb5..79da94c 100644 --- a/frontend/src/components/counts/CountsControls.tsx +++ b/frontend/src/components/counts/CountsControls.tsx @@ -11,6 +11,7 @@ import TagInput from '@/components/common/TagInput' import { Button } from '@/components/ui/button' import { countsUseLatest, fieldsForCounts, normaliseCountsParams, selectedCountChildren, selectedCountColumns } from '@/lib/counts' +import { countsNestedIsScalar } from '@/lib/countsDisplay' import { useCountsDefinition } from '@/lib/countsDefinitions' import { useClientMotion } from '@/lib/useClientMotion' import { inputClass } from '@/lib/utils' @@ -43,11 +44,13 @@ const CountsControls = ( const thumbnail = selectedCountColumns (resource, applied, definition).includes ('thumbnail') const mode = draft.get ('mode') ?? 'summary' const columns = selectedCountColumns (resource, draft, definition) - const children = selectedCountChildren (draft) const details = mode === 'details' const hasNested = details && fieldsForCounts (resource, definition).some (field => - columns.includes (field.key) && field.children?.some (child => - children.has (`${field.key}.${child.key}`))) + columns.includes (field.key) && field.children != null + && !countsNestedIsScalar (field.children, field.key, draft) + && field.children.some (child => selectedCountChildren (draft) + .has (`${field.key}.${child.key}`))) + // Immediate presentation changes merge into the pending query, never replace it. useEffect (() => { setDraft (previous => { const next = new URLSearchParams (previous) diff --git a/frontend/src/components/counts/CountsExportProgress.tsx b/frontend/src/components/counts/CountsExportProgress.tsx new file mode 100644 index 0000000..ae8dd63 --- /dev/null +++ b/frontend/src/components/counts/CountsExportProgress.tsx @@ -0,0 +1,130 @@ +import { useEffect, useRef, useState } from 'react' + +import LocalisedText from '@/components/LocalisedText' +import { apiDelete, apiGet, apiPost, isApiError, startAPIDownload } from '@/lib/api' +import { useUITranslation } from '@/lib/useUITranslation' + +import type { DialogueFormControls } from '@/lib/dialogues/useDialogue' +import type { LocaleCode } from '@/lib/locales' + +export type CountsExportHandle = { cancel: () => Promise } +type ExportStatus = { + status: 'starting' | 'queued' | 'running' | 'cancelling' | 'cancelled' | 'ready' | 'failed' + processed: number; total: number | null; token?: string } + + +const CountsExportProgress = ( + { payload, locale, handle, controls }: { + payload: Record; locale: LocaleCode + handle: CountsExportHandle; controls: DialogueFormControls }, +) => { + const { label, message } = useUITranslation ('counts') + const [state, setState] = useState ({ + status: 'starting', processed: 0, total: null }) + const [cancelError, setCancelError] = useState (false) + const terminal = useRef (false) + const cancelled = useRef (false) + const job = useRef (null) + const creation = useRef | null> (null) + const downloaded = useRef (false) + useEffect (() => { + handle.cancel = async () => { + if (terminal.current) return true + cancelled.current = true + if (job.current != null) + { + try + { + setCancelError (false) + const result = await apiDelete ( + `/counts/export/jobs/${encodeURIComponent (job.current)}`, { locale }) + setState (result) + } + catch + { + setCancelError (true) + } + } + return false + } + }, [handle, locale]) + useEffect (() => { + let active = true + let timer: ReturnType | undefined + const poll = async (token: string) => { + try + { + const response = await apiGet ( + `/counts/export/jobs/${encodeURIComponent (token)}`, { locale }) + if (!active) return + const result = (cancelled.current && response.status === 'ready') + ? await apiDelete ( + `/counts/export/jobs/${encodeURIComponent (token)}`, { locale }) : response + if (!active) return + setState (result) + terminal.current = ['ready', 'failed', 'cancelled'].includes (result.status) + if (result.status === 'ready' && result.token != null && !downloaded.current) + { + downloaded.current = true + startAPIDownload ('/counts/export', result.token, () => { + if (active) setState (previous => ({ ...previous, status: 'failed' })) + }) + } + if (!terminal.current) timer = setTimeout (() => poll (token), 800) + } + catch (error) + { + if (active) + { + if (isApiError (error) && error.response?.status === 404) + { + terminal.current = true + setState (previous => ({ ...previous, status: 'failed' })) + return + } + setCancelError (true) + timer = setTimeout (() => poll (token), 2000) + } + } + } + creation.current ??= apiPost<{ job: string }> ('/counts/export/jobs', payload, { locale }) + creation.current.then (async result => { + if (!active) return + job.current = result.job + if (cancelled.current) await handle.cancel () + await poll (result.job) + }).catch (() => { + if (active) + { + terminal.current = true + setState (previous => ({ ...previous, status: 'failed' })) + } + }) + return () => { + active = false + if (timer != null) clearTimeout (timer) + } + }, [handle, locale, payload]) + useEffect (() => { + controls.setActions ([]) + }, [controls]) + const percent = state.total == null ? null : (state.total === 0 ? 100 + : Math.min (100, Math.floor (state.processed / state.total * 100))) + const phase = message (`exportProgress.${state.status}`) + return
+

{label ('exportProgress.format', { + format: message (`exportFormats.${payload.format}`).text })}

+

+ {phase.text} +

+ +

{percent == null + ? label ('exportProgress.processed', { count: state.processed }) + : label ('exportProgress.measured', { + processed: state.processed, total: state.total ?? 0, percent })}

+ {cancelError &&

{label ('common:fetchError')}

} +
+} + +export default CountsExportProgress diff --git a/frontend/src/components/counts/CountsNestedTable.tsx b/frontend/src/components/counts/CountsNestedTable.tsx index 8093050..5349448 100644 --- a/frontend/src/components/counts/CountsNestedTable.tsx +++ b/frontend/src/components/counts/CountsNestedTable.tsx @@ -5,7 +5,8 @@ import CountsSelectableTable from '@/components/counts/CountsSelectableTable' import { changeCountSort, readCountSort, selectedCountChildren, writeCountSort, orderedCountFields } from '@/lib/counts' import { countsLogicalName, countsNestedClipboardTable, countsNestedExport, - countsNestedOrder, countsNestedText } from '@/lib/countsDisplay' + countsNestedOrder, countsNestedText, countsNestedIsScalar, countsNestedScalarText } + from '@/lib/countsDisplay' import { useLocale } from '@/lib/useLocale' import { useUITranslation } from '@/lib/useUITranslation' @@ -54,14 +55,15 @@ const CountsNestedTable = ( return {String (value)} return display.text } + if (countsNestedIsScalar (fields, path, params)) + return {countsNestedScalarText (rows, fields, path, params, locale)} return (
{params.get ('nested_json') === '1' ? ( 
-	      {JSON.stringify (countsNestedExport (rows, fields, path, params, locale), null, 2)}
+		className="max-h-72 w-full overflow-auto whitespace-pre text-xs">
+	      {JSON.stringify (countsNestedExport (rows, fields, path, params, locale))}
) : (columns.length > 0 && rows.length > 0 && (
@@ -87,7 +89,7 @@ const CountsNestedTable = ( {columns.map (field => ( = ({ children }) => { }, [closeRequest, pendingIds]) + const dismiss = async (request: DialogueRequest) => { + if (request.kind === 'form' && request.options.onCancel != null) + { + setPendingIds (current => [...current, request.id]) + try + { + if (await request.options.onCancel () === false) return + } + finally + { + setPendingIds (current => current.filter (id => id !== request.id)) + } + } + closeRequest (request.id, request.kind === 'confirm' ? false : null) + } + const active = queue[0] const startActions = active?.kind === 'form' @@ -179,7 +195,7 @@ const DialogueProvider: FC = ({ children }) => { nestedConfirm?.parentId === active.id || pendingIds.includes (active.id) if (!(open) && !(blocked)) - closeRequest (active.id, active.kind !== 'confirm' && null) + dismiss (active) }}> = ({ children }) => {