このコミットが含まれているのは:
2026-10-26 21:24:32 +09:00
コミット 2563b3fef3
21個のファイルの変更、703行の追加、54行の削除
+30
ファイルの表示
@@ -1,6 +1,27 @@
class CountsExportsController < ApplicationController
wrap_parameters false
def create_job
locale = resolve_locale!
payload = request.request_parameters.merge(request.query_parameters)
session[:counts_export_owner] ||= SecureRandom.hex(32)
job = CountsExportJobs.start(payload, session[:counts_export_owner], locale.language_code)
response.headers['Cache-Control'] = 'private, no-store'
render json: { job: }, status: :accepted
rescue CountsExportDownload::Busy
head :too_many_requests
rescue ArgumentError, TypeError, KeyError, JSON::ParserError
head :bad_request
end
def job_status
job_state(cancel: false)
end
def cancel_job
job_state(cancel: true)
end
def prepare
locale = resolve_locale!
payload = request.request_parameters.merge(request.query_parameters)
@@ -32,6 +53,15 @@ class CountsExportsController < ApplicationController
disposition: 'attachment'
end
def job_state(cancel:)
result = CountsExportJobs.status(params[:token], session[:counts_export_owner], cancel:)
return head :not_found unless result
response.headers['Cache-Control'] = 'private, no-store'
render json: result
end
private :job_state
def index
locale = resolve_locale!
payload = request.request_parameters.except('authenticity_token')
+12 -6
ファイルの表示
@@ -13,15 +13,17 @@ class CountsExportDownload
class TimedOut < StandardError; end
class StorageFull < StandardError; end
def self.directory = Rails.root.join('tmp', 'counts_exports')
def self.directory
Rails.root.join(ENV.fetch('COUNTS_EXPORT_DIRECTORY', 'tmp/counts_exports'))
end
def self.verifier = Rails.application.message_verifier('counts_export_download')
def self.prepare(exporter, owner)
def self.prepare(exporter, owner, progress: nil, check: nil, identifier: nil)
FileUtils.mkdir_p(directory, mode: 0o700)
slot = acquire_slot
begin
generate(exporter, owner)
generate(exporter, owner, progress:, check:, identifier:)
ensure
slot.close
end
@@ -38,7 +40,7 @@ class CountsExportDownload
raise Busy
end
def self.generate(exporter, owner)
def self.generate(exporter, owner, progress: nil, check: nil, identifier: nil)
FileUtils.mkdir_p(directory, mode: 0o700)
Dir.glob(directory.join('counts-*.export')).each do |path|
File.unlink(path) if File.mtime(path) < Time.current - EXPIRY
@@ -53,14 +55,16 @@ class CountsExportDownload
reserved = MAX_CONCURRENT_EXPORTS * MAX_FILE_BYTES
raise StorageFull if stored + reserved > MAX_STORED_BYTES
file = Tempfile.create(['counts-', '.part'], directory)
prefix = identifier ? "counts-#{ identifier }-" : 'counts-'
file = Tempfile.create([prefix, '.part'], directory)
complete = false
bytes = 0
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + MAX_EXPORT_SECONDS
path = file.path
begin
file.binmode
exporter.body.each do |chunk|
exporter.body(progress:).each do |chunk|
check&.call
raise TimedOut if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
bytes += chunk.bytesize
@@ -68,6 +72,8 @@ class CountsExportDownload
file.write(chunk)
end
file.flush
file.fsync
file.close
finished_path = path.sub(/\.part\z/, '.export')
File.rename(path, finished_path)
+236
ファイルの表示
@@ -0,0 +1,236 @@
require 'json'
require 'fileutils'
# A durable, bounded export spool. Web and supervised workers share the same POSIX volume.
class CountsExportJobs
class Cancelled < StandardError; end
ACTIVE = ['queued', 'running', 'cancelling'].freeze
def self.directory = CountsExportDownload.directory.join('jobs')
def self.verifier = Rails.application.message_verifier('counts_export_job')
def self.synchronise
FileUtils.mkdir_p(directory, mode: 0o700)
File.open(directory.join('.registry.lock'), File::RDWR | File::CREAT, 0o600) do |lock|
lock.flock(File::LOCK_EX)
yield
end
end
def self.write(state)
path = directory.join("#{ state.fetch('id') }.json")
temporary = "#{ path }.#{ SecureRandom.hex(8) }.tmp"
File.open(temporary, 'w', 0o600) do |file|
file.write(JSON.generate(state))
file.flush
file.fsync
end
File.rename(temporary, path)
ensure
File.unlink(temporary) if temporary && File.exist?(temporary)
end
def self.read(id)
JSON.parse(File.read(directory.join("#{ id }.json")))
rescue Errno::ENOENT
nil
end
def self.remove_download(state)
name = state['download_file']
return unless name&.match?(/\Acounts-[a-zA-Z0-9_-]+\.export\z/)
File.unlink(CountsExportDownload.directory.join(name))
rescue Errno::ENOENT
nil
end
def self.remove_partial(id)
Dir.glob(CountsExportDownload.directory.join("counts-#{ id }-*")).each do |path|
File.unlink(path)
rescue Errno::ENOENT
nil
end
end
def self.clean
Dir.glob(directory.join('*.tmp')).each do |path|
File.unlink(path) if File.mtime(path) < Time.current - CountsExportDownload::EXPIRY
end
Dir.glob(directory.join('*.run')).each do |path|
id = File.basename(path, '.run')
next if directory.join("#{ id }.json").exist?
File.open(path, File::RDWR) do |lock|
File.unlink(path) if lock.flock(File::LOCK_EX | File::LOCK_NB)
end
end
Dir.glob(directory.join('*.json')).each do |path|
state = JSON.parse(File.read(path))
next if state.fetch('expires_at') > Time.current.to_f
run = directory.join("#{ state.fetch('id') }.run")
File.open(run, File::RDWR | File::CREAT, 0o600) do |lock|
unless lock.flock(File::LOCK_EX | File::LOCK_NB)
state['status'] = 'cancelling'
write(state)
next
end
remove_download(state)
remove_partial(state.fetch('id'))
File.unlink(path)
File.unlink(run)
end
end
end
def self.start(payload, owner, language)
payload = payload.dup
time_key = payload.key?('definition') ? 'execution_at' : 'at'
if payload[time_key].nil? || payload[time_key] == ''
payload[time_key] = Time.current.utc.iso8601(6)
end
exporter = CountsExporter.new(payload, language_code: language)
synchronise do
clean
active = Dir.glob(directory.join('*.json')).count do |path|
ACTIVE.include?(JSON.parse(File.read(path))['status'])
end
raise CountsExportDownload::Busy if active >= CountsExportDownload::MAX_CONCURRENT_EXPORTS
id = SecureRandom.hex(24)
state = { 'id' => id, 'owner' => owner, 'payload' => payload,
'language' => language, 'resource' => exporter.resource,
'format' => exporter.format, 'status' => 'queued',
'processed' => 0, 'total' => nil,
'expires_at' => (Time.current + CountsExportDownload::EXPIRY).to_f }
write(state)
verifier.generate({ 'id' => id, 'owner' => owner },
expires_in: CountsExportDownload::EXPIRY, purpose: 'counts_export_job')
end
end
def self.authorised_id(token, owner)
value = verifier.verified(token, purpose: 'counts_export_job')
return unless value.is_a?(Hash) && value['owner'] == owner
return unless value['id']&.match?(/\A[a-f0-9]{48}\z/)
value['id']
end
def self.status(token, owner, cancel: false)
id = authorised_id(token, owner)
return unless id
synchronise do
state = read(id)
return unless state && state['owner'] == owner && state['expires_at'] > Time.current.to_f
if cancel && ['queued', 'running', 'ready'].include?(state['status'])
state['status'] = state['status'] == 'running' ? 'cancelling' : 'cancelled'
remove_download(state)
state.delete('token')
write(state)
end
state.slice('status', 'processed', 'total', 'format', 'token', 'error')
end
end
def self.checkpoint(id, processed = nil, total = nil)
synchronise do
state = read(id)
unless state && state['status'] == 'running' && state['expires_at'] > Time.current.to_f
raise Cancelled
end
if processed
state.merge!('processed' => processed, 'total' => total)
write(state)
end
end
end
def self.work_one
FileUtils.mkdir_p(directory, mode: 0o700)
Dir.glob(directory.join('*.json')).sort.each do |path|
id = File.basename(path, '.json')
File.open(directory.join("#{ id }.run"), File::RDWR | File::CREAT, 0o600) do |lock|
next unless lock.flock(File::LOCK_EX | File::LOCK_NB)
state = synchronise do
candidate = read(id)
next unless candidate && ACTIVE.include?(candidate['status'])
if candidate['status'] != 'queued'
candidate['status'] = candidate['status'] == 'cancelling' ? 'cancelled' : 'failed'
remove_partial(id)
candidate['error'] = 'interrupted'
write(candidate)
next
end
candidate['status'] = 'running'
write(candidate)
candidate
end
next unless state
perform(state)
return true
end
end
synchronise { clean }
false
end
def self.perform(state)
id = state.fetch('id')
token = nil
begin
checkpoint(id)
exporter = CountsExporter.new(state.fetch('payload'), language_code: state.fetch('language'))
last = Process.clock_gettime(Process::CLOCK_MONOTONIC)
progress = lambda do |processed, total|
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
if processed.zero? || now - last >= 0.25 || processed == total
checkpoint(id, processed, total)
last = now
end
end
check = lambda do
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
if now - last >= 0.25
checkpoint(id)
last = now
end
end
token = CountsExportDownload.prepare(exporter, state.fetch('owner'),
progress:, check:, identifier: id)
synchronise do
current = read(id)
if (!current || current['status'] != 'running' ||
current['expires_at'] <= Time.current.to_f)
file = CountsExportDownload.find(token, state.fetch('owner'))
File.unlink(file.fetch('path')) if file
raise Cancelled
end
file = CountsExportDownload.find(token, state.fetch('owner'))
current.merge!('status' => 'ready', 'token' => token,
'download_file' => File.basename(file.fetch('path')))
current['total'] = current['processed']
write(current)
end
rescue StandardError => error
remove_partial(id)
synchronise do
current = read(id)
if current
current['status'] = error.is_a?(Cancelled) ? 'cancelled' : 'failed'
current['error'] = error.class.name
write(current)
end
end
unless error.is_a?(Cancelled)
Rails.logger.error("counts_export_job #{ id }: #{ error.class }")
end
end
end
end
+20 -2
ファイルの表示
@@ -24,6 +24,22 @@ class CountsExportProjection
'aliases' => ['languageCode', 'name', 'scriptCode'],
'parents' => ['id'] }.freeze
def self.default_columns(resource)
fields = { 'posts' => ['title', 'url', 'tags', 'parents', 'video_ms'],
'tags' => ['name', 'category', 'aliases', 'parents'],
'nico_tags' => ['name', 'linked_tags'],
'tag_descriptions' => ['language_code', 'body', 'source_language_code'],
'tag_groups' => ['name', 'description', 'members'],
'materials' => ['url', 'tag_id', 'source_kind'] }.fetch(resource)
['resource_id', 'version_no'] + fields + ['event_type', 'user', 'created_at']
end
def self.default_children(resource)
{ 'posts' => ['tags.name', 'parents.id'],
'tags' => ['aliases.name', 'parents.id'],
'tag_groups' => ['members.tagId'] }.fetch(resource, [])
end
attr_reader :columns, :headers
def initialize(params, resource:, summary:, tags:)
@@ -37,7 +53,8 @@ class CountsExportProjection
else
CountsQuery::COLUMNS.fetch(resource)
end
@columns = params.fetch('columns', allowed.join(',')).split(',')
defaults = summary ? allowed : self.class.default_columns(resource)
@columns = params.fetch('columns', defaults.join(',')).split(',')
raise ArgumentError if (@columns - allowed).any? || @columns.uniq != @columns
@headers = JSON.parse(params.fetch('headers', JSON.generate(@columns)))
@@ -55,7 +72,8 @@ class CountsExportProjection
[path, direction]
end
@selected = params.key?('children') ? params['children'].split(',').to_set : nil
@selected = params.key?('children') ? params['children'].split(',').to_set :
self.class.default_children(resource).to_set
valid = CHILDREN.flat_map do |key, fields|
paths(fields, key)
end
+8 -2
ファイルの表示
@@ -47,14 +47,17 @@ class CountsExporter
end
end
def body
return @table_export.body if @table_export
def body(progress: nil)
return @table_export.body(progress:) if @table_export
Enumerator.new do |output|
ApplicationRecord.transaction(isolation: :repeatable_read) do
processed = 0
first_row = true
started = false
pages do |page|
total = @mode == 'details' ? page[:count] : nil
progress&.call(processed, total)
unless started
meta = { resource:, mode: @mode, basis: @history ? 'history' : 'state',
at: @query_params['at'], boundary: 'inclusive',
@@ -83,8 +86,11 @@ class CountsExporter
output << CSV.generate_line(values, row_sep: "\r\n")
end
first_row = false
processed += 1
progress&.call(processed, total)
end
end
progress&.call(processed, processed)
output << ']}' if format == 'json'
end
end
+16 -3
ファイルの表示
@@ -10,6 +10,10 @@ class CountsTableExport
raise ArgumentError if (params.keys - CountsTableQuery::KEYS - EXTRA).any?
raise ArgumentError unless params.values.all? { _1.is_a?(String) }
unless params.key?('children')
params = params.merge('children' => CountsExportProjection.default_children(
params.fetch('resource', 'posts')).join(','))
end
@params = params
@labels = JSON.parse(params.fetch('labels', '{}'))
raise ArgumentError unless @labels.is_a?(Hash)
@@ -27,8 +31,11 @@ class CountsTableExport
@query = CountsTableQuery.new(params.slice(*CountsTableQuery::KEYS)
.merge('page' => '1', 'limit' => '100'))
@summary = @query.mode == 'summary'
@columns = params.fetch('columns', '').split(',')
@headers = JSON.parse(params.fetch('headers', '[]'))
default_columns = @summary ? @query.axes.all.map { _1.fetch('field') } + ['count'] :
CountsExportProjection.default_columns(resource)
@columns = params.fetch('columns', default_columns.join(',')).split(',')
default_headers = @columns.map { @labels.fetch(_1, _1) }
@headers = JSON.parse(params.fetch('headers', default_headers.to_json))
raise ArgumentError unless @headers.is_a?(Array) && @headers.all? { _1.is_a?(String) }
if @summary && @headers.any? && @headers.length != @query.axes.all.length + 1
raise ArgumentError
@@ -45,13 +52,16 @@ class CountsTableExport
end
end
def body
def body(progress: nil)
Enumerator.new do |output|
ApplicationRecord.transaction(isolation: :repeatable_read) do
@language_names = Language.pluck(:code, :name).to_h
processed = 0
started = false
first = true
@query.each_page do |page|
total = @summary ? nil : page[:count]
progress&.call(processed, total)
unless started
metadata = page.except(:versions, :cells, :axis_labels, :column_values).merge(
resource:, filters: @params.slice(*CountsTableQuery::KEYS),
@@ -84,8 +94,11 @@ class CountsTableExport
output << CSV.generate_line(values.map { safe(_1) }, row_sep: "\r\n")
end
first = false
processed += 1
progress&.call(processed, total)
end
end
progress&.call(processed, processed)
output << ']}' if format == 'json'
end
end
+3
ファイルの表示
@@ -1,6 +1,9 @@
Rails.application.routes.draw do
get 'counts/definitions/:resource', to: 'counts_tables#definition'
get 'counts/query', to: 'counts_tables#index'
post 'counts/export/jobs', to: 'counts_exports#create_job'
get 'counts/export/jobs/:token', to: 'counts_exports#job_status'
delete 'counts/export/jobs/:token', to: 'counts_exports#cancel_job'
post 'counts/export/prepare', to: 'counts_exports#prepare'
get 'counts/export/files/:token', to: 'counts_exports#download'
get 'counts/export', to: 'counts_exports#index'
+11
ファイルの表示
@@ -0,0 +1,11 @@
namespace :counts do
desc 'Process the persistent Counts export queue (run under a process supervisor)'
task export_worker: :environment do
stopping = false
['TERM', 'INT'].each { |signal| Signal.trap(signal) { stopping = true } }
until stopping
worked = Rails.application.executor.wrap { CountsExportJobs.work_one }
sleep(1) unless worked
end
end
end
+53
ファイルの表示
@@ -0,0 +1,53 @@
# Counts エクスポートワーカー
Counts の画面からのエクスポートは永続ファイルキューへ登録する。
Active Job の既定のプロセス内 async アダプターは使用しない。
追加 gem、外部キュー、DB マイグレーションは不要。
## 起動・配置
backend で `bundle exec rake counts:export_worker` を、systemd 等の監視対象として
別プロセスで起動する。Web プロセス内では起動しない。
ワーカー未起動の間は待機状態となる。1 プロセスは同時に1ジョブを処理し、
全体の実行枠は既存の2枠を共有する。ワーカーの起動・配備は実施していない。
Web とワーカーは同一リリース、Rails secret、アプリケーション環境、DB 接続先を
使用する。DB コネクションはワーカープロセスごとに必要。
`COUNTS_EXPORT_DIRECTORY` を同じ永続 POSIX 共有ディレクトリに設定する。
省略時は `backend/tmp/counts_exports`。
全プロセスで同一の所有者とアクセス権を使用する。
複数ホストでは、flock・同一ファイルシステム内の atomic rename・fsync が正しく
機能する共有ボリュームが必須。オブジェクトストレージや各コンテナ独立の tmp は不可。
ジョブ JSON と完成ファイルが格納されるため、このディレクトリを静的公開しない。
## API と進捗
- POST `/counts/export/jobs`:要求検証・実行時点の固定・署名付きジョブトークンの発行。
- GET `/counts/export/jobs/:token`:状態、実際の処理済みレコード数、判明している総数。
- DELETE `/counts/export/jobs/:token`:取消を永続記録。
- GET `/counts/export/files/:token`:生成完了後の署名付きファイルトークンによる取得。
ジョブ・ファイルのトークンは作成元のセッションに紐付く。
既存の BAN・CSRF 制御を使用する。フロントは約800ms間隔で状態を取得する。
明細の総数は同じ読み取りトランザクション内の SQL 件数を使用する。
集計結果の出力セル総数は生成終了まで確定させず、不確定進捗として扱う。
完成状態はファイルの生成完了であり、ブラウザの保存完了を意味しない。
## 取消・障害・制限
通常は約250ms間隔の生成チェックポイントで取消を検知する。
DB 問い合わせ・ファイル I/O がブロックしている間は即時に中断できない。
取消済みへの遷移は生成処理の終了と一時ファイル削除後に行う。
既に始まった HTTP 転送やブラウザ側の保存は取り消せない。
ワーカーが異常終了したジョブは、次のワーカーが実行ロックを取得した際に失敗状態にし、
そのジョブの残存生成ファイルを削除する。別の読み取り状態で自動再実行しない。
原子的に書き換える状態ファイルとプロセス間ロックにより、二重処理・取消との競合を防ぐ。
既存の同時実行2、1ファイル128MiB、保存領域512MiB、生成120秒、期限10分を維持する。
生成時間制限もチェックポイント方式であり、実行中 SQL の強制停止ではない。
期限切れのジョブ・ファイル・残存状態書込ファイルは登録時またはワーカーの巡回時に削除する。
ワーカーと Web の両方を停止した間は巡回削除も停止する。
これらの処理は静的確認のみで、実環境のキュー処理・取消・転送・共有ボリュームでの
排他動作は未確認。
+21 -6
ファイルの表示
@@ -21,21 +21,36 @@ const ConditionEditor = (
) => {
const { label, message } = useUITranslation ('counts')
const fields = countsFilterFields (definition)
const [field, setField] = useState (initial?.field ?? 'resource_id')
const [operator, setOperator] = useState (initial?.operator ?? 'eq')
const [condition, setCondition] = useState<CountsCondition> (() => {
const next = { ...initial, field: initial?.field ?? 'resource_id',
operator: initial?.operator ?? 'eq' }
if (next.operator !== 'range') delete next.end
return next
})
const { field, operator } = condition
const operators = definition.fields.find (item => item.key === field)?.operators ?? []
useEffect (() => {
controls.setActions ([{ label: label (initial == null ? 'add' : 'shared:actions.edit'),
onSelect: () => save ({ ...initial, field, operator }) }])
}, [controls, field, operator, initial, label, save])
onSelect: () => save (condition) }])
}, [controls, condition, initial, label, save])
return <div className="space-y-2">
<select value={field} aria-label={message ('field').text} className={inputClass ()}
onChange={event => { setField (event.target.value); setOperator ('eq') }}>
onChange={event => {
const field = event.target.value
const operator = definition.fields.find (item => item.key === field)?.operators[0]
if (operator != null) setCondition ({ field, operator })
}}>
{Object.keys (fields).filter (key => fields[key] !== 'json').map (key => (
<option key={key} value={key}>{message (countsFieldLabel (key)).text}</option>))}
</select>
<select value={operator} aria-label={message ('operator').text} className={inputClass ()}
onChange={event => setOperator (event.target.value)}>
onChange={event => setCondition (previous => {
const operator = event.target.value
const next = { ...previous, operator }
if (operator !== 'range') delete next.end
if ((operator === 'like') !== (previous.operator === 'like')) delete next.value
return next
})}>
{operators.map (key => <option key={key} value={key}>
{message (`operators.${key}`).text}</option>)}
</select>
+6 -3
ファイルの表示
@@ -11,6 +11,7 @@ import TagInput from '@/components/common/TagInput'
import { Button } from '@/components/ui/button'
import { countsUseLatest, fieldsForCounts, normaliseCountsParams,
selectedCountChildren, selectedCountColumns } from '@/lib/counts'
import { countsNestedIsScalar } from '@/lib/countsDisplay'
import { useCountsDefinition } from '@/lib/countsDefinitions'
import { useClientMotion } from '@/lib/useClientMotion'
import { inputClass } from '@/lib/utils'
@@ -43,11 +44,13 @@ const CountsControls = (
const thumbnail = selectedCountColumns (resource, applied, definition).includes ('thumbnail')
const mode = draft.get ('mode') ?? 'summary'
const columns = selectedCountColumns (resource, draft, definition)
const children = selectedCountChildren (draft)
const details = mode === 'details'
const hasNested = details && fieldsForCounts (resource, definition).some (field =>
columns.includes (field.key) && field.children?.some (child =>
children.has (`${field.key}.${child.key}`)))
columns.includes (field.key) && field.children != null
&& !countsNestedIsScalar (field.children, field.key, draft)
&& field.children.some (child => selectedCountChildren (draft)
.has (`${field.key}.${child.key}`)))
// Immediate presentation changes merge into the pending query, never replace it.
useEffect (() => {
setDraft (previous => {
const next = new URLSearchParams (previous)
+130
ファイルの表示
@@ -0,0 +1,130 @@
import { useEffect, useRef, useState } from 'react'
import LocalisedText from '@/components/LocalisedText'
import { apiDelete, apiGet, apiPost, isApiError, startAPIDownload } from '@/lib/api'
import { useUITranslation } from '@/lib/useUITranslation'
import type { DialogueFormControls } from '@/lib/dialogues/useDialogue'
import type { LocaleCode } from '@/lib/locales'
export type CountsExportHandle = { cancel: () => Promise<boolean> }
type ExportStatus = {
status: 'starting' | 'queued' | 'running' | 'cancelling' | 'cancelled' | 'ready' | 'failed'
processed: number; total: number | null; token?: string }
const CountsExportProgress = (
{ payload, locale, handle, controls }: {
payload: Record<string, string>; locale: LocaleCode
handle: CountsExportHandle; controls: DialogueFormControls },
) => {
const { label, message } = useUITranslation ('counts')
const [state, setState] = useState<ExportStatus> ({
status: 'starting', processed: 0, total: null })
const [cancelError, setCancelError] = useState (false)
const terminal = useRef (false)
const cancelled = useRef (false)
const job = useRef<string | null> (null)
const creation = useRef<Promise<{ job: string }> | null> (null)
const downloaded = useRef (false)
useEffect (() => {
handle.cancel = async () => {
if (terminal.current) return true
cancelled.current = true
if (job.current != null)
{
try
{
setCancelError (false)
const result = await apiDelete<ExportStatus> (
`/counts/export/jobs/${encodeURIComponent (job.current)}`, { locale })
setState (result)
}
catch
{
setCancelError (true)
}
}
return false
}
}, [handle, locale])
useEffect (() => {
let active = true
let timer: ReturnType<typeof setTimeout> | undefined
const poll = async (token: string) => {
try
{
const response = await apiGet<ExportStatus> (
`/counts/export/jobs/${encodeURIComponent (token)}`, { locale })
if (!active) return
const result = (cancelled.current && response.status === 'ready')
? await apiDelete<ExportStatus> (
`/counts/export/jobs/${encodeURIComponent (token)}`, { locale }) : response
if (!active) return
setState (result)
terminal.current = ['ready', 'failed', 'cancelled'].includes (result.status)
if (result.status === 'ready' && result.token != null && !downloaded.current)
{
downloaded.current = true
startAPIDownload ('/counts/export', result.token, () => {
if (active) setState (previous => ({ ...previous, status: 'failed' }))
})
}
if (!terminal.current) timer = setTimeout (() => poll (token), 800)
}
catch (error)
{
if (active)
{
if (isApiError (error) && error.response?.status === 404)
{
terminal.current = true
setState (previous => ({ ...previous, status: 'failed' }))
return
}
setCancelError (true)
timer = setTimeout (() => poll (token), 2000)
}
}
}
creation.current ??= apiPost<{ job: string }> ('/counts/export/jobs', payload, { locale })
creation.current.then (async result => {
if (!active) return
job.current = result.job
if (cancelled.current) await handle.cancel ()
await poll (result.job)
}).catch (() => {
if (active)
{
terminal.current = true
setState (previous => ({ ...previous, status: 'failed' }))
}
})
return () => {
active = false
if (timer != null) clearTimeout (timer)
}
}, [handle, locale, payload])
useEffect (() => {
controls.setActions ([])
}, [controls])
const percent = state.total == null ? null : (state.total === 0 ? 100
: Math.min (100, Math.floor (state.processed / state.total * 100)))
const phase = message (`exportProgress.${state.status}`)
return <div className="space-y-3 py-2">
<p>{label ('exportProgress.format', {
format: message (`exportFormats.${payload.format}`).text })}</p>
<p role="status" aria-live="polite">
<LocalisedText language={phase.language}>{phase.text}</LocalisedText>
</p>
<progress className="h-3 w-full" max={100} value={percent ?? undefined}
aria-label={phase.text} lang={phase.language}/>
<p aria-live="polite">{percent == null
? label ('exportProgress.processed', { count: state.processed })
: label ('exportProgress.measured', {
processed: state.processed, total: state.total ?? 0, percent })}</p>
{cancelError && <p role="alert">{label ('common:fetchError')}</p>}
</div>
}
export default CountsExportProgress
+7 -5
ファイルの表示
@@ -5,7 +5,8 @@ import CountsSelectableTable from '@/components/counts/CountsSelectableTable'
import { changeCountSort, readCountSort, selectedCountChildren,
writeCountSort, orderedCountFields } from '@/lib/counts'
import { countsLogicalName, countsNestedClipboardTable, countsNestedExport,
countsNestedOrder, countsNestedText } from '@/lib/countsDisplay'
countsNestedOrder, countsNestedText, countsNestedIsScalar, countsNestedScalarText }
from '@/lib/countsDisplay'
import { useLocale } from '@/lib/useLocale'
import { useUITranslation } from '@/lib/useUITranslation'
@@ -54,14 +55,15 @@ const CountsNestedTable = (
return <LocalisedText language={row.languageCode}>{String (value)}</LocalisedText>
return display.text
}
if (countsNestedIsScalar (fields, path, params))
return <span>{countsNestedScalarText (rows, fields, path, params, locale)}</span>
return (
<div className="counts-nested w-full min-w-0 max-w-full">
{params.get ('nested_json') === '1' ? (
<pre data-counts-scroll
aria-label={title.text} lang={title.language}
className="max-h-72 w-full overflow-auto whitespace-pre-wrap
break-words text-xs">
{JSON.stringify (countsNestedExport (rows, fields, path, params, locale), null, 2)}
className="max-h-72 w-full overflow-auto whitespace-pre text-xs">
{JSON.stringify (countsNestedExport (rows, fields, path, params, locale))}
</pre>) : (columns.length > 0 && rows.length > 0 && (
<div data-counts-scroll
className="relative isolate max-h-72 w-full min-w-0 overflow-auto">
@@ -87,7 +89,7 @@ const CountsNestedTable = (
<tr key={index} className="border-t dark:border-slate-700">
{columns.map (field => (
<td key={field.key}
className="overflow-hidden break-words border-t px-2 py-1
className="overflow-hidden break-words border-t px-3 py-2
align-top dark:border-slate-700"
data-counts-copy={
field.children
+18 -2
ファイルの表示
@@ -156,6 +156,22 @@ const DialogueProvider: FC<Props> = ({ children }) => {
},
[closeRequest, pendingIds])
const dismiss = async (request: DialogueRequest) => {
if (request.kind === 'form' && request.options.onCancel != null)
{
setPendingIds (current => [...current, request.id])
try
{
if (await request.options.onCancel () === false) return
}
finally
{
setPendingIds (current => current.filter (id => id !== request.id))
}
}
closeRequest (request.id, request.kind === 'confirm' ? false : null)
}
const active = queue[0]
const startActions =
active?.kind === 'form'
@@ -179,7 +195,7 @@ const DialogueProvider: FC<Props> = ({ children }) => {
nestedConfirm?.parentId === active.id
|| pendingIds.includes (active.id)
if (!(open) && !(blocked))
closeRequest (active.id, active.kind !== 'confirm' && null)
dismiss (active)
}}>
<DialogContent
closeLabel={(active.kind === 'confirm' || active.kind === 'choice')
@@ -244,7 +260,7 @@ const DialogueProvider: FC<Props> = ({ children }) => {
<Button
className="w-full md:w-auto"
variant="outline"
onClick={() => closeRequest (active.id)}
onClick={() => dismiss (active)}
disabled={pendingIds.includes (active.id)
|| nestedConfirm?.parentId === active.id}>
{active.options.cancelText ?? label ('actions.cancel')}
+6 -2
ファイルの表示
@@ -7,6 +7,7 @@ import { useTranslation } from 'react-i18next'
import LocalisedText from '@/components/LocalisedText'
import { translateUIMessage } from '@/lib/i18n'
import { useClientMotion } from '@/lib/useClientMotion'
import { cn } from '@/lib/utils'
const Dialog = DialogPrimitive.Root
@@ -37,14 +38,17 @@ const DialogContent = React.forwardRef<
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Content>
& { closeLabel?: React.ReactNode }
>(
({ className, children, closeLabel, ...props }, ref) => {
({ className, children, closeLabel, style, ...props }, ref) => {
const { i18n } = useTranslation ('shared')
const motion = useClientMotion ()
const animationDuration = `${motion.duration ('popover')}ms`
const close = translateUIMessage ('dialogue.close', 'shared', i18n.language)
return (
<DialogPortal>
<DialogOverlay/>
<DialogOverlay style={{ animationDuration }}/>
<DialogPrimitive.Content
ref={ref}
style={{ animationDuration, ...style }}
className={cn (
'fixed left-[50%] top-[50%] z-50 grid w-[calc(100%-2rem)] max-w-lg',
'translate-x-[-50%] translate-y-[-50%]',
+9
ファイルの表示
@@ -211,6 +211,15 @@ export const apiDownload = async (
const { token } = await apiPost<{ token: string }> (`${path}/prepare`, body, { locale })
if (typeof token !== 'string' || token.length === 0)
throw new Error ('Missing download token')
startAPIDownload (path, token, onError)
}
export const startAPIDownload = (
path: string,
token: string,
onError?: () => void,
): void => {
const frame = document.createElement ('iframe')
frame.hidden = true
frame.addEventListener ('load', () => {
+33 -11
ファイルの表示
@@ -206,10 +206,16 @@ export const selectedCountColumns = (
params: URLSearchParams,
definition?: CountsDefinition,
): CountColumn[] => {
const selected = params.has ('columns') ? params.get ('columns')!.split (',') :
columnsForCounts (resource, definition).filter (key => ![
'thumbnail', 'original_created_from', 'original_created_before']
.includes (key))
const common = ['resource_id', 'version_no', 'event_type', 'user', 'created_at']
const defaults: Record<CountsResource, string[]> = {
posts: ['title', 'url', 'tags', 'parents', 'video_ms'],
tags: ['name', 'category', 'aliases', 'parents'],
nico_tags: ['name', 'linked_tags'],
tag_descriptions: ['language_code', 'body', 'source_language_code'],
tag_groups: ['name', 'description', 'members'],
materials: ['url', 'source_kind', 'tag_id'] }
const selected = params.has ('columns') ? params.get ('columns')!.split (',')
: [...common, ...defaults[resource]]
return orderedCountFields (fieldsForCounts (resource, definition), params)
.map (field => field.key as CountColumn).filter (key => selected.includes (key))
}
@@ -223,9 +229,8 @@ export const countChildPaths = (fields: CountField[], prefix = ''): string[] =>
export const selectedCountChildren = (params: URLSearchParams): Set<string> =>
new Set (params.has ('children') ? params.get ('children')!.split (',') :
Object.entries (nested).flatMap (([key, fields]) =>
countChildPaths (fields ?? [], key)))
new Set (params.has ('children') ? params.get ('children')!.split (',')
: ['tags.name', 'parents.id', 'aliases.name', 'members.tagId'])
export const readCountSort = (value: string | null): CountSort[] =>
@@ -342,15 +347,32 @@ export const exportCounts = (
headers: string[],
labels: Record<string, string>,
onError?: () => void,
): Promise<void> => {
): Promise<void> => apiDownload ('/counts/export', countsExportPayload (
resource, params, format, columns, headers, labels), locale, onError)
export const countsExportPayload = (
resource: CountsResource,
params: URLSearchParams,
format: 'json' | 'csv',
columns: string[],
headers: string[],
labels: Record<string, string>,
definition?: CountsDefinition,
): Record<string, string> => {
const payload = new URLSearchParams (params)
if (!payload.has ('children'))
{
const selected = selectedCountChildren (params)
payload.set ('children', countChildPaths (fieldsForCounts (resource, definition))
.filter (path => selected.has (path)).join (','))
}
payload.delete ('current')
payload.delete ('row_numbers')
if (isCountsState (params) && countsUseLatest (params)) payload.delete ('at')
return apiDownload ('/counts/export', {
...Object.fromEntries (payload), resource, format,
return { ...Object.fromEntries (payload), resource, format,
columns: columns.join (','), headers: JSON.stringify (headers),
labels: JSON.stringify (labels) }, locale, onError)
labels: JSON.stringify (labels) }
}
+53 -7
ファイルの表示
@@ -194,6 +194,38 @@ export const countsNestedExport = (
}
// A chain with one selected scalar leaf is displayed and copied as one cell.
export const countsNestedIsScalar = (
fields: CountField[],
path: string,
params: URLSearchParams,
): boolean => {
const selected = selectedCountChildren (params)
const columns = fields.filter (field => selected.has (`${path}.${field.key}`))
if (columns.length !== 1) return false
const field = columns[0]
return field.children == null
|| countsNestedIsScalar (field.children, `${path}.${field.key}`, params)
}
export const countsNestedScalarText = (
rows: CountRecord[],
fields: CountField[],
path: string,
params: URLSearchParams,
locale: LocaleCode,
): string => {
const selected = selectedCountChildren (params)
const field = fields.find (field => selected.has (`${path}.${field.key}`))
if (field == null) return ''
return countsNestedOrder (rows, path, params, locale).map (row => field.children == null
? countsLogicalName (row, field.key, locale).text
: countsNestedScalarText (row[field.key] as CountRecord[] ?? [], field.children,
`${path}.${field.key}`, params, locale)).join (' ')
}
export const countsNestedText = (
rows: CountRecord[],
fields: CountField[],
@@ -201,6 +233,8 @@ export const countsNestedText = (
params: URLSearchParams,
locale: LocaleCode,
): string => {
if (countsNestedIsScalar (fields, path, params))
return countsNestedScalarText (rows, fields, path, params, locale)
const selected = selectedCountChildren (params)
const columns = orderedCountFields (fields, params, path)
.filter (field => selected.has (`${path}.${field.key}`))
@@ -224,9 +258,11 @@ export const countsCellCopy = (
): string => {
if (field.children == null) return countsCellText (row, field.key as CountColumn, locale, params)
const rows = countsNestedRows (row, field.key)
if (countsNestedIsScalar (field.children, field.key, params))
return countsNestedScalarText (rows, field.children, field.key, params, locale)
return params.get ('nested_json') === '1'
? JSON.stringify (
countsNestedExport (rows, field.children, field.key, params, locale), null, 2)
countsNestedExport (rows, field.children, field.key, params, locale))
: countsNestedText (rows, field.children, field.key, params, locale)
}
@@ -297,7 +333,9 @@ const clipboardColumns = (
return orderedCountFields (fields, params, path)
.filter (field => selected.has (`${path}.${field.key}`))
.map (field => ({ label: translateUIMessage (field.label, 'counts', locale).text,
children: field.children == null ? undefined : clipboardColumns (
children: (field.children == null
|| countsNestedIsScalar (field.children, `${path}.${field.key}`, params))
? undefined : clipboardColumns (
field.children, `${path}.${field.key}`, params, locale) }))
}
@@ -316,8 +354,11 @@ export const countsNestedClipboardTable = (
columns: clipboardColumns (fields, path, params, locale),
rows: countsNestedOrder (rows, path, params, locale).map (row => columns.map (field =>
field.children == null ? countsLogicalName (row, field.key, locale).text
: countsNestedClipboardTable (row[field.key] as CountRecord[] ?? [],
field.children, `${path}.${field.key}`, params, locale))) }
: (countsNestedIsScalar (field.children, `${path}.${field.key}`, params)
? countsNestedScalarText (row[field.key] as CountRecord[] ?? [],
field.children, `${path}.${field.key}`, params, locale)
: countsNestedClipboardTable (row[field.key] as CountRecord[] ?? [],
field.children, `${path}.${field.key}`, params, locale)))) }
}
@@ -330,7 +371,9 @@ export const countsDetailClipboardTable = (
): CountsClipboardTable => {
const columns = fields.map (field => ({
label: translateUIMessage (field.label, 'counts', locale).text,
children: field.children == null ? undefined : clipboardColumns (
children: (field.children == null
|| countsNestedIsScalar (field.children, field.key, params))
? undefined : clipboardColumns (
field.children, field.key, params, locale) }))
if (rowStart != null)
columns.unshift ({ label: translateUIMessage ('rowNumbers', 'counts', locale).text,
@@ -338,8 +381,11 @@ export const countsDetailClipboardTable = (
return { columns, rows: rows.map ((row, index) => {
const cells: CountsClipboardCell[] = fields.map (field => field.children == null
? countsCellText (row, field.key, locale, params)
: countsNestedClipboardTable (countsNestedRows (row, field.key), field.children,
field.key, params, locale))
: (countsNestedIsScalar (field.children, field.key, params)
? countsNestedScalarText (countsNestedRows (row, field.key), field.children,
field.key, params, locale)
: countsNestedClipboardTable (countsNestedRows (row, field.key), field.children,
field.key, params, locale)))
return rowStart == null ? cells : [String (rowStart + index), ...cells]
}) }
}
+1
ファイルの表示
@@ -41,6 +41,7 @@ type DialogueFormOptions = { title: ReactNode
description?: ReactNode
body: (controls: DialogueFormControls) => ReactNode
cancelText?: ReactNode
onCancel?: () => Promise<boolean | void> | boolean | void
footerLayout?: 'inline'
size?: 'default' | 'large' | 'viewport' }
+13
ファイルの表示
@@ -13,6 +13,19 @@
"output": "出力項目",
"filters": "抽出条件",
"mode": "表示",
"exportProgress": {
"title": "エクスポート",
"format": "形式:{{format}}",
"starting": "開始中",
"queued": "待機中",
"running": "生成中",
"cancelling": "取消処理中",
"cancelled": "取消済み",
"ready": "ファイル生成完了・ダウンロード開始(保存完了は未確認)",
"failed": "エラー",
"processed": "{{count}}件処理済み",
"measured": "{{processed}} / {{total}}件({{percent}}%)"
},
"displaySettings": "表示設定",
"summary": "集計",
"details": "明細",
+17 -5
ファイルの表示
@@ -1,8 +1,9 @@
import { useQuery } from '@tanstack/react-query'
import { useEffect, useState } from 'react'
import { useEffect, useRef, useState } from 'react'
import { useSearchParams } from 'react-router-dom'
import PrefetchLink from '@/components/PrefetchLink'
import CountsExportProgress from '@/components/counts/CountsExportProgress'
import CountsControls from '@/components/counts/CountsControls'
import CountsSummaryResults from '@/components/counts/CountsSummaryResults'
import CountsResults from '@/components/counts/CountsResults'
@@ -14,7 +15,7 @@ import Pagination from '@/components/common/Pagination'
import MainArea from '@/components/layout/MainArea'
import SidebarComponent from '@/components/layout/SidebarComponent'
import { Button } from '@/components/ui/button'
import { countsRequestParams, defaultCountsMode, exportCounts, fieldsForCounts, fetchCounts,
import { countsRequestParams, defaultCountsMode, countsExportPayload, fieldsForCounts, fetchCounts,
isCountsState, normaliseCountsParams, selectedCountColumns } from '@/lib/counts'
import { countsExportLabels } from '@/lib/countsDisplay'
import { CountsDefinitionContext, countsAxisFields, countsAxisLabel,
@@ -24,6 +25,7 @@ import { CountsDefinitionContext, countsAxisFields, countsAxisLabel,
import { countsKeys } from '@/lib/queryKeys'
import { cn } from '@/lib/utils'
import { useClientMotion } from '@/lib/useClientMotion'
import useDialogue from '@/lib/dialogues/useDialogue'
import { useLocale } from '@/lib/useLocale'
import { useUITranslation } from '@/lib/useUITranslation'
@@ -42,7 +44,7 @@ const CountsResourceContent = ({ resource }: { resource: CountsResource }) => {
controlParams.delete ('order')
controlParams.delete ('row_numbers')
controlParams.set ('columns', selectedCountColumns (resource, params, definition)
.filter (column => column !== 'thumbnail').join (','))
.filter (column => column !== 'thumbnail').sort ().join (','))
const request = countsRequestParams (params)
const [open, setOpen] = useState (false)
const [sidebarOpen, setSidebarOpen] = useState (false)
@@ -52,12 +54,14 @@ const CountsResourceContent = ({ resource }: { resource: CountsResource }) => {
setExpanded (previous => new Set ([...previous, resource]))
}, [resource])
const locale = useLocale ()
const dialogue = useDialogue ()
const { label, message } = useUITranslation ('counts')
const mode = params.get ('mode') ?? 'summary'
const state = isCountsState (params)
const cacheParams = new URLSearchParams (request)
cacheParams.set ('definition', state ? 'latest' : 'history')
const [exporting, setExporting] = useState (false)
const exportInFlight = useRef (false)
const [exportError, setExportError] = useState (false)
const { data, isPending, isError, isPlaceholderData, refetch } = useQuery ({
enabled: params.has ('timezone'),
@@ -103,6 +107,8 @@ const CountsResourceContent = ({ resource }: { resource: CountsResource }) => {
setParams (next)
}
const download = async (format: 'csv' | 'json') => {
if (exportInFlight.current) return
exportInFlight.current = true
setExporting (true)
setExportError (false)
try
@@ -119,12 +125,17 @@ const CountsResourceContent = ({ resource }: { resource: CountsResource }) => {
? [...axes.map (axis => countsAxisLabel (axis, definition, locale).text),
message ('count').text]
: fields.map (field => message (field.label).text)
await exportCounts (resource, params, locale, format,
const payload = countsExportPayload (resource, params, format,
fields.map (field => field.key), headers,
{ ...countsExportLabels (resource, locale, definition),
...Object.fromEntries (countsAxisFields (definition, !state).map (field => [
field.key, message (field.label).text])), count: message ('count').text },
() => setExportError (true))
definition)
const handle = { cancel: async () => false }
await dialogue.form ({ title: label ('exportProgress.title'),
footerLayout: 'inline', onCancel: () => handle.cancel (),
body: controls => <CountsExportProgress payload={payload} locale={locale}
handle={handle} controls={controls}/> })
}
catch
@@ -133,6 +144,7 @@ const CountsResourceContent = ({ resource }: { resource: CountsResource }) => {
}
finally
{
exportInFlight.current = false
setExporting (false)
}
}