コミットを比較
3
コミット
| 作成者 | SHA1 | 日付 | |
|---|---|---|---|
|
|
3232c57cc4 | ||
|
|
084bc95102 | ||
|
|
225f55a3b9 |
@@ -0,0 +1,35 @@
|
||||
## 背景
|
||||
|
||||
なぜ必要か。
|
||||
|
||||
## 対象範囲
|
||||
|
||||
- backend:
|
||||
- frontend:
|
||||
- docs:
|
||||
- migration:
|
||||
|
||||
## やること
|
||||
|
||||
- [ ]
|
||||
|
||||
## 受け入れ条件
|
||||
|
||||
- [ ]
|
||||
|
||||
## 実行すべき確認
|
||||
|
||||
- [ ] `cd backend && bundle exec rspec`
|
||||
- [ ] `cd frontend && npm run build`
|
||||
- [ ] `cd frontend && npm run lint`
|
||||
|
||||
## 禁止事項
|
||||
|
||||
- unrelated refactor はしない
|
||||
- 既存 API response shape を壊さない
|
||||
- 認証・認可・BAN を弱めない
|
||||
|
||||
## Codex への指示
|
||||
|
||||
この issue を読んで実装してください。
|
||||
不明点があれば、実装前に調査結果と選択肢を提示してください。
|
||||
@@ -100,9 +100,18 @@ class PostVersionsController < ApplicationController
|
||||
|
||||
def external_tag_names_for snapshots
|
||||
ids = snapshots.flatten.filter_map { _1['external_tag_id'] }.uniq
|
||||
ExternalTag.where(id: ids).pluck(:id, :platform, :name).to_h { |id, platform, name|
|
||||
names = ExternalTag.where(id: ids).pluck(:id, :platform, :name).to_h { |id, platform, name|
|
||||
[id, "#{ platform }:#{ name }"]
|
||||
}
|
||||
missing_ids = ids - names.keys
|
||||
|
||||
HistoryAccess.visible(NicoTagVersion.all)
|
||||
.where(tag_id: missing_ids)
|
||||
.order(:tag_id, version_no: :desc)
|
||||
.pluck(:tag_id, :name)
|
||||
.each { |id, name| names[id] ||= name }
|
||||
|
||||
names
|
||||
end
|
||||
|
||||
def snapshot_tag_literals snapshots, tag_names, external_tag_names
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
class StaticMirrorsController < ApplicationController
|
||||
before_action :development_only
|
||||
wrap_parameters false
|
||||
|
||||
def show
|
||||
render json: { resource_limit: StaticMirror::RESOURCE_LIMIT,
|
||||
name_limit: StaticMirror::NAME_LIMIT, max_bytes: StaticMirror::MAX_BYTES,
|
||||
max_seconds: StaticMirrorJob::MAX_SECONDS, concurrency: 1,
|
||||
cooldown_seconds: StaticMirrorJob::COOLDOWN,
|
||||
expiry_seconds: StaticMirrorJob::EXPIRY.to_i, exclusions: StaticMirror::EXCLUSIONS }
|
||||
end
|
||||
|
||||
def create
|
||||
# No date input is accepted: this endpoint cannot promise historical restoration.
|
||||
payload = request.request_parameters.except('authenticity_token')
|
||||
.merge(request.query_parameters)
|
||||
return head :bad_request unless (payload.keys - ['locale']).empty?
|
||||
|
||||
locale = resolve_locale!
|
||||
session[:static_mirror_owner] ||= SecureRandom.hex(32)
|
||||
job = StaticMirrorJob.start(owner: session[:static_mirror_owner], locale: locale.code,
|
||||
origin: request.base_url)
|
||||
render json: { job: }, status: :accepted
|
||||
rescue StaticMirrorJob::Busy
|
||||
response.headers['Retry-After'] = StaticMirrorJob::COOLDOWN.to_s
|
||||
head :too_many_requests
|
||||
rescue ActiveJob::EnqueueError
|
||||
render json: { error: 'generation_failed' }, status: :service_unavailable
|
||||
end
|
||||
|
||||
def status
|
||||
value = StaticMirrorJob.status(params[:id], session[:static_mirror_owner])
|
||||
return head :not_found unless value
|
||||
|
||||
render json: value
|
||||
rescue StaticMirrorJob::Busy
|
||||
head :service_unavailable
|
||||
end
|
||||
|
||||
def download
|
||||
bytes = StaticMirrorJob.download(params[:id], session[:static_mirror_owner])
|
||||
return head :not_found unless bytes
|
||||
|
||||
send_data bytes, filename: 'static-mirror.zip', type: 'application/zip',
|
||||
disposition: 'attachment'
|
||||
rescue StaticMirrorJob::Busy
|
||||
head :service_unavailable
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def development_only
|
||||
head :not_found unless Rails.env.development?
|
||||
end
|
||||
end
|
||||
@@ -1,155 +0,0 @@
|
||||
require 'fileutils'
|
||||
require 'timeout'
|
||||
|
||||
# Development PoC: one filesystem slot and Rails' in-process async executor.
|
||||
# A process restart loses queued work; stale state fails rather than reporting success.
|
||||
class StaticMirrorJob < ApplicationJob
|
||||
self.queue_adapter = :async
|
||||
EXPIRY = 10.minutes
|
||||
COOLDOWN = 60
|
||||
MAX_SECONDS = 60
|
||||
class Busy < StandardError; end
|
||||
|
||||
def self.directory = Rails.root.join('tmp/static_mirror')
|
||||
|
||||
def self.synchronise(blocking: false)
|
||||
FileUtils.mkdir_p(directory, mode: 0o700)
|
||||
File.open(directory.join('.registry.lock'), File::RDWR | File::CREAT, 0o600) do |lock|
|
||||
mode = blocking ? File::LOCK_EX : File::LOCK_EX | File::LOCK_NB
|
||||
raise Busy unless lock.flock(mode)
|
||||
|
||||
yield
|
||||
end
|
||||
end
|
||||
|
||||
def self.state
|
||||
JSON.parse(directory.join('state.json').read)
|
||||
rescue Errno::ENOENT
|
||||
nil
|
||||
end
|
||||
|
||||
def self.write(value)
|
||||
path = directory.join('state.tmp')
|
||||
File.open(path, 'w', 0o600) { _1.write(JSON.generate(value)) }
|
||||
File.rename(path, directory.join('state.json'))
|
||||
ensure
|
||||
File.unlink(path) if path&.exist?
|
||||
end
|
||||
|
||||
def self.start(owner:, locale:, origin:)
|
||||
id = synchronise do
|
||||
previous = state
|
||||
if previous && Time.current.to_f - previous.fetch('created_at') < COOLDOWN
|
||||
raise Busy
|
||||
end
|
||||
File.open(directory.join('.generation.lock'), File::RDWR | File::CREAT, 0o600) do |lock|
|
||||
raise Busy unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
||||
|
||||
FileUtils.rm_f(directory.join('archive.zip'))
|
||||
FileUtils.rm_f(directory.join('archive.part'))
|
||||
identifier = SecureRandom.hex(24)
|
||||
write({ 'id' => identifier, 'owner' => owner, 'locale' => locale, 'origin' => origin,
|
||||
'created_at' => Time.current.to_f, 'expiry_at' => (Time.current + EXPIRY).to_f,
|
||||
'status' => 'queued', 'processed' => 0, 'total' => nil, 'data_at' => nil })
|
||||
identifier
|
||||
end
|
||||
end
|
||||
perform_later(id)
|
||||
id
|
||||
rescue ActiveJob::EnqueueError
|
||||
update(id) { _1.merge!('status' => 'failed', 'error' => 'generation_failed') } if id
|
||||
raise
|
||||
end
|
||||
|
||||
def self.authorised_state(id, owner)
|
||||
value = state
|
||||
return unless owner.present? && value && value['id'] == id && value['owner'] == owner
|
||||
return unless value['expiry_at'] > Time.current.to_f
|
||||
|
||||
if (['queued', 'running'].include?(value['status']) &&
|
||||
Time.current.to_f - value['created_at'] > MAX_SECONDS + 15)
|
||||
value.merge!('status' => 'failed', 'error' => 'worker_stopped')
|
||||
write(value)
|
||||
end
|
||||
if value['status'] == 'ready'
|
||||
permitted = HistoryAccess.set(user: nil) { HistoryAccess.authorised_artifact?(value) }
|
||||
return unless permitted && directory.join('archive.zip').file?
|
||||
end
|
||||
value
|
||||
end
|
||||
|
||||
def self.status(id, owner)
|
||||
synchronise do
|
||||
authorised_state(id, owner)&.slice('status', 'processed', 'total', 'data_at', 'error')
|
||||
end
|
||||
end
|
||||
|
||||
def self.download(id, owner)
|
||||
synchronise do
|
||||
value = authorised_state(id, owner)
|
||||
return unless value && value['status'] == 'ready'
|
||||
|
||||
bytes = directory.join('archive.zip').binread
|
||||
# Revalidate after reading; send_data avoids delayed sendfile reuse of a stale path.
|
||||
permitted = HistoryAccess.set(user: nil) { HistoryAccess.authorised_artifact?(value) }
|
||||
permitted ? bytes : nil
|
||||
end
|
||||
end
|
||||
|
||||
def self.update(id)
|
||||
# Worker updates may wait for a short status/download read; request intake never waits.
|
||||
synchronise(blocking: true) do
|
||||
value = state
|
||||
return unless value && value['id'] == id
|
||||
|
||||
yield value
|
||||
write(value)
|
||||
end
|
||||
end
|
||||
|
||||
def perform(id)
|
||||
File.open(self.class.directory.join('.generation.lock'), File::RDWR) do |lock|
|
||||
return unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
||||
|
||||
value = self.class.state
|
||||
return unless value && value['id'] == id && value['status'] == 'queued'
|
||||
|
||||
generate(id, value)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def generate(id, value)
|
||||
self.class.update(id) { _1['status'] = 'running' }
|
||||
remaining = MAX_SECONDS - (Time.current.to_f - value.fetch('created_at'))
|
||||
raise Timeout::Error unless remaining.positive?
|
||||
|
||||
Timeout.timeout(remaining) do
|
||||
locale = Locale.find_by!(code: value.fetch('locale'), deprecated_at: nil)
|
||||
mirror = StaticMirror.new(locale:, origin: value.fetch('origin'))
|
||||
bytes = mirror.export do |processed, total|
|
||||
self.class.update(id) do |metadata|
|
||||
metadata.merge!('processed' => processed, 'total' => total, 'data_at' => mirror.data_at)
|
||||
end
|
||||
end
|
||||
part = self.class.directory.join('archive.part')
|
||||
File.open(part, 'wb', 0o600) { _1.write(bytes) }
|
||||
File.rename(part, self.class.directory.join('archive.zip'))
|
||||
self.class.update(id) do |metadata|
|
||||
metadata.merge!(mirror.authority).merge!('status' => 'ready', 'data_at' => mirror.data_at)
|
||||
end
|
||||
end
|
||||
rescue StandardError => error
|
||||
FileUtils.rm_f(self.class.directory.join('archive.part'))
|
||||
FileUtils.rm_f(self.class.directory.join('archive.zip'))
|
||||
code = case error
|
||||
when StaticMirror::TooLarge then 'too_large'
|
||||
when Timeout::Error then 'generation_timeout'
|
||||
when HistoryAccess::Forbidden then 'publication_changed'
|
||||
else 'generation_failed'
|
||||
end
|
||||
self.class.update(id) { _1.merge!('status' => 'failed', 'error' => code) }
|
||||
Rails.logger.warn("static_mirror_failure: #{ error.class }")
|
||||
end
|
||||
end
|
||||
@@ -1,11 +1,8 @@
|
||||
class ExternalTag < ApplicationRecord
|
||||
before_create { SharedResourceIdAllocator.assign!(self) }
|
||||
# Must run before dependent association callbacks.
|
||||
before_destroy :prevent_destroy
|
||||
|
||||
enum :platform, nico: 'nico'
|
||||
validates :platform, presence: true, inclusion: { in: ExternalTag.platforms.keys }
|
||||
validate :immutable_identity, on: :update
|
||||
|
||||
has_many :post_external_tags, dependent: :delete_all
|
||||
has_many :posts, through: :post_external_tags
|
||||
@@ -19,21 +16,11 @@ class ExternalTag < ApplicationRecord
|
||||
|
||||
has_many :linked_tags, through: :nico_tag_relations, source: :tag
|
||||
|
||||
def snapshot_linked_tag_ids = linked_tags.distinct.order(:id).pluck(:id)
|
||||
|
||||
def delete
|
||||
raise ActiveRecord::ReadOnlyRecord, '外部タグは削除できません.'
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def immutable_identity
|
||||
errors.add(:platform, 'は変更できません.') if will_save_change_to_platform?
|
||||
errors.add(:name, 'は変更できません.') if will_save_change_to_name?
|
||||
end
|
||||
|
||||
def prevent_destroy
|
||||
errors.add(:base, '外部タグは削除できません.')
|
||||
throw(:abort)
|
||||
def snapshot_linked_tag_names language_code = 'ja'
|
||||
linked_tags
|
||||
.joins(:tag_names)
|
||||
.where(tag_names: { language_code:, primary_flg: true })
|
||||
.order('tag_names.name')
|
||||
.pluck('tag_names.name')
|
||||
end
|
||||
end
|
||||
|
||||
@@ -106,7 +106,21 @@ class HistoryAccess < ActiveSupport::CurrentAttributes
|
||||
snapshot_reference(TagVersion, row),
|
||||
snapshot_reference(NicoTagVersion, row, external: true)]
|
||||
when 'nico_tag_versions'
|
||||
[list_reference('tag_versions', 'tag_id', "#{ row }.linked_tag_ids")]
|
||||
# Legacy linked_tags snapshots contain names, not IDs. Match recorded names
|
||||
# conservatively; never expose a private linked name through an external tag.
|
||||
names = "JSON_MERGE_PRESERVE(JSON_ARRAY(private_tag.name), " +
|
||||
"COALESCE(JSON_EXTRACT(private_tag.names, '$**.name'), JSON_ARRAY()))"
|
||||
# DISTINCT prevents merging this independent name set into the correlated scan.
|
||||
# Keep byte-distinct names even when the column collation considers them equal.
|
||||
private_names = 'SELECT DISTINCT private_name.name, ' +
|
||||
'CAST(private_name.name AS BINARY) AS name_bytes FROM tag_versions private_tag ' +
|
||||
"JOIN (#{ discarded_keys('tag_versions', 'tag_id') }) blocked " +
|
||||
'ON blocked.tag_id = private_tag.tag_id ' +
|
||||
"JOIN JSON_TABLE(#{ names }, '$[*]' COLUMNS (name TEXT PATH '$')) private_name " +
|
||||
"WHERE private_name.name <> ''"
|
||||
["NOT EXISTS (SELECT 1 FROM (#{ private_names }) private_name WHERE " +
|
||||
"LOCATE(CONCAT(' ', private_name.name, ' '), " +
|
||||
"CONCAT(' ', COALESCE(#{ row }.linked_tags, ''), ' ')) > 0)"]
|
||||
when 'tag_versions'
|
||||
[list_reference('tag_versions', 'tag_id', "#{ row }.parent_tag_ids")]
|
||||
when 'tag_description_versions'
|
||||
|
||||
@@ -3,10 +3,5 @@ class NicoTagVersion < ApplicationRecord
|
||||
|
||||
belongs_to :external_tag, foreign_key: :tag_id, inverse_of: :nico_tag_versions
|
||||
|
||||
validates :event_type, inclusion: { in: ['create', 'update'] }
|
||||
|
||||
def name
|
||||
tag = external_tag
|
||||
"#{ tag.platform }:#{ tag.name }" if tag
|
||||
end
|
||||
validates :name, presence: true
|
||||
end
|
||||
|
||||
@@ -139,8 +139,6 @@ class CountsVersionRepr
|
||||
result = {}
|
||||
table = model.table_name
|
||||
comparison = @inclusive ? '<=' : '<'
|
||||
name = model == NicoTagVersion ?
|
||||
CountsResourceDefinition.new('nico_tags').expression('name', table) : 'name'
|
||||
requests.each_slice(100) do |batch|
|
||||
queries = batch.map do |id, point|
|
||||
scope = HistoryAccess.visible(model.where(tag_id: id))
|
||||
@@ -150,8 +148,7 @@ class CountsVersionRepr
|
||||
"AND newer.version_no > #{ table }.version_no)", point)
|
||||
quoted = model.connection.quote(point.iso8601(6))
|
||||
names = model == TagVersion ? 'names' : 'JSON_OBJECT() AS names'
|
||||
scope.select("tag_id, #{ name } AS name, #{ names }, " +
|
||||
"#{ quoted } AS reference_point").to_sql
|
||||
scope.select("tag_id, name, #{ names }, #{ quoted } AS reference_point").to_sql
|
||||
end
|
||||
next if queries.empty?
|
||||
|
||||
|
||||
@@ -6,8 +6,14 @@ class CountsCandidates
|
||||
|
||||
def initialize(params)
|
||||
QueryState.validate!(params, KEYS)
|
||||
raise ArgumentError unless params.fetch('resource') == 'posts'
|
||||
raise ArgumentError unless ['tags', 'tags_json'].include?(params.fetch('field'))
|
||||
definition = CountsResourceDefinition.new(params.fetch('resource'))
|
||||
fields = ['resource_id', 'user']
|
||||
fields += ['tags', 'tags_json', 'tags.id'] if definition.resource == 'posts'
|
||||
fields << 'members' if definition.resource == 'tag_groups'
|
||||
if definition.resource == 'materials'
|
||||
fields += ['updated_by_user_id', 'file_blob_id', 'source_file_id', 'drive_file_id']
|
||||
end
|
||||
raise ArgumentError unless fields.include?(params.fetch('field'))
|
||||
|
||||
@params = params
|
||||
@page = Integer(params.fetch('page', '1'), 10)
|
||||
@@ -18,28 +24,28 @@ class CountsCandidates
|
||||
end
|
||||
|
||||
def call
|
||||
references = ['tags', 'tags_json', 'tags.id', 'members']
|
||||
return resource_candidates unless references.include?(@params['field'])
|
||||
|
||||
options = @params.slice('locale', 'at', 'execution_at', 'timezone')
|
||||
.merge('definition' => 'latest', 'resource' => 'tags')
|
||||
internal = CountsPopulation.new(options)
|
||||
external = CountsPopulation.new(options.merge('resource' => 'nico_tags',
|
||||
groups_flg = @params['field'] == 'members'
|
||||
external = CountsPopulation.new(options.merge(
|
||||
'resource' => groups_flg ? 'tag_groups' : 'nico_tags',
|
||||
'execution_at' => internal.at.iso8601(6)))
|
||||
connection = TagVersion.connection
|
||||
sources = [[internal, 'internal', 'v.category', 'v.names'],
|
||||
[external, 'external', 'NULL', 'JSON_OBJECT()']]
|
||||
[external, groups_flg ? 'group' : 'external', 'NULL', 'JSON_OBJECT()']]
|
||||
sources = sources.map do |population, kind, category, names|
|
||||
scope = population.scope.except(:select)
|
||||
name = population.definition.expression('name')
|
||||
if @params['query'].present?
|
||||
pattern = "%#{ ApplicationRecord.sanitize_sql_like(@params['query']) }%"
|
||||
scope = scope.where("(#{ name } LIKE ? OR CAST(v.tag_id AS CHAR) = ? OR EXISTS (" +
|
||||
"SELECT 1 FROM JSON_TABLE(COALESCE(JSON_EXTRACT(#{ names }, '$**.name'), " +
|
||||
"JSON_ARRAY()), '$[*]' COLUMNS (name TEXT PATH '$')) candidate_name " +
|
||||
'WHERE candidate_name.name LIKE ?))', pattern, @params['query'], pattern)
|
||||
end
|
||||
identity = "v.#{ population.definition.keys.first }"
|
||||
scope = search(scope, identity, 'v.name', names)
|
||||
value = ['tags.id', 'members'].include?(@params['field']) ?
|
||||
"CAST(#{ identity } AS CHAR)" : "CONCAT('#{ kind }:', #{ identity })"
|
||||
scope.select(Arel.sql(
|
||||
"CONCAT('#{ kind }:', v.tag_id) AS value, v.tag_id AS id, " +
|
||||
"#{ name } AS name, #{ names } AS names, " +
|
||||
"#{ category } AS category, '#{ kind }' AS kind"))
|
||||
"#{ value } AS value, #{ identity } AS id, " +
|
||||
"v.name AS name, #{ names } AS names, #{ category } AS category, '#{ kind }' AS kind"))
|
||||
.to_sql
|
||||
end
|
||||
clauses = []
|
||||
@@ -57,4 +63,63 @@ class CountsCandidates
|
||||
{ rows: rows.first(LIMIT), page: @page, more: rows.length > LIMIT,
|
||||
execution_at: internal.at.iso8601(6) }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def search(scope, identity, name, names)
|
||||
return scope if @params['query'].blank?
|
||||
|
||||
pattern = "%#{ ApplicationRecord.sanitize_sql_like(@params['query']) }%"
|
||||
scope.where("(#{ name } LIKE ? OR CAST(#{ identity } AS CHAR) = ? OR EXISTS (" +
|
||||
"SELECT 1 FROM JSON_TABLE(COALESCE(JSON_EXTRACT(#{ names }, '$**.name'), " +
|
||||
"JSON_ARRAY()), '$[*]' COLUMNS (name TEXT PATH '$')) candidate_name " +
|
||||
'WHERE candidate_name.name LIKE ?))', pattern, @params['query'], pattern)
|
||||
end
|
||||
|
||||
def resource_candidates
|
||||
options = @params.slice('at', 'execution_at', 'timezone')
|
||||
.merge('resource' => @params.fetch('resource'), 'definition' => 'latest')
|
||||
actor_flg = ['user', 'updated_by_user_id'].include?(@params['field'])
|
||||
options['definition'] = 'history' if actor_flg
|
||||
population = CountsPopulation.new(options)
|
||||
scope = population.scope.except(:select)
|
||||
if actor_flg
|
||||
point = population.timestamp(@params['at']) || population.at
|
||||
raise ArgumentError if point > population.at
|
||||
|
||||
column = @params['field'] == 'user' ? 'created_by_user_id' : 'updated_by_user_id'
|
||||
identities = scope.where('v.created_at <= ?', point).select("v.#{ column }")
|
||||
scope = User.where(id: identities)
|
||||
identity = 'users.id'
|
||||
name = 'users.name'
|
||||
names = 'JSON_OBJECT()'
|
||||
else
|
||||
definition = population.definition
|
||||
column = @params['field'] == 'resource_id' ? definition.keys.first : @params['field']
|
||||
if column == 'drive_file_id'
|
||||
column = 'source_file_id'
|
||||
scope = scope.where('v.source_kind' => 'google_drive_file')
|
||||
end
|
||||
identity = "v.#{ column }"
|
||||
label = ['title', 'name', 'url'].find { definition.fields.key?(_1) }
|
||||
name = label ? "v.#{ label }" : 'NULL'
|
||||
names = definition.resource == 'tags' ? 'v.names' : 'JSON_OBJECT()'
|
||||
if ['file_blob_id', 'source_file_id', 'drive_file_id'].include?(@params['field'])
|
||||
scope = scope.where.not("v.#{ column }" => nil).group(Arel.sql(identity))
|
||||
name = 'v.file_filename'
|
||||
end
|
||||
end
|
||||
scope = search(scope, identity, name, names)
|
||||
title = ['file_blob_id', 'source_file_id', 'drive_file_id'].include?(@params['field']) ?
|
||||
"MIN(#{ name })" : name
|
||||
query = scope.optimizer_hints('MAX_EXECUTION_TIME(10000)').distinct
|
||||
.select(Arel.sql("#{ identity } AS id, #{ title } AS name, #{ names } AS names"))
|
||||
.order(Arel.sql(identity)).limit(LIMIT + 1).offset((@page - 1) * LIMIT)
|
||||
rows = scope.klass.connection.select_all(query.to_sql).to_a.map do |values|
|
||||
values['names'] = JSON.parse(values['names']) if values['names'].is_a?(String)
|
||||
values.merge('value' => values.fetch('id').to_s)
|
||||
end
|
||||
{ rows: rows.first(LIMIT), page: @page, more: rows.length > LIMIT,
|
||||
execution_at: population.at.iso8601(6) }
|
||||
end
|
||||
end
|
||||
|
||||
@@ -113,7 +113,7 @@ class CountsPopulation
|
||||
else
|
||||
raise ArgumentError unless definition.axes(history).include?(key)
|
||||
|
||||
scope = scope.where("#{ definition.expression(key) } <=> ?", entry['value'])
|
||||
scope = scope.where(definition.expression(key) => entry['value'])
|
||||
end
|
||||
scope
|
||||
end
|
||||
|
||||
@@ -105,25 +105,12 @@ class CountsResourceDefinition
|
||||
def time_column = 'created_at'
|
||||
def column(key)
|
||||
raise ArgumentError unless fields.key?(key)
|
||||
# Keep the V1 Counts field name at the API boundary only.
|
||||
return 'linked_tag_ids' if resource == 'nico_tags' && key == 'linked_tags'
|
||||
|
||||
{ 'resource_id' => keys.first, 'user' => 'created_by_user_id' }.fetch(key, key)
|
||||
end
|
||||
|
||||
def expression(key, row = 'v')
|
||||
if resource == 'nico_tags' && key == 'name'
|
||||
return "(SELECT CONCAT(external_tags.platform, ':', external_tags.name) " +
|
||||
"FROM external_tags WHERE external_tags.id = #{ row }.tag_id)"
|
||||
end
|
||||
|
||||
"#{ row }.#{ column(key) }"
|
||||
end
|
||||
|
||||
def changes
|
||||
columns = fields.keys - COMMON.keys - ['updated_at', 'updated_by_user_id']
|
||||
resource == 'nico_tags' ? columns - ['name'] : columns
|
||||
end
|
||||
def expression(key, row = 'v') = "#{ row }.#{ column(key) }"
|
||||
def changes = fields.keys - COMMON.keys - ['updated_at', 'updated_by_user_id']
|
||||
def axes(history)
|
||||
fields.reject { |_key, type| type == :json }.keys +
|
||||
(resource == 'posts' ? ['tags'] : []) + (history ? ['changed_field'] : [])
|
||||
@@ -131,9 +118,7 @@ class CountsResourceDefinition
|
||||
|
||||
def metadata
|
||||
fields.map do |key, type|
|
||||
nullable = (resource == 'nico_tags' && key == 'name') ||
|
||||
model.columns_hash.fetch(column(key)).null
|
||||
{ key:, type:, nullable:,
|
||||
{ key:, type:, nullable: model.columns_hash.fetch(column(key)).null,
|
||||
operators: OPERATORS.fetch(type, []) }
|
||||
end
|
||||
end
|
||||
|
||||
@@ -201,8 +201,6 @@ class CountsStateQuery
|
||||
end
|
||||
|
||||
def listing
|
||||
external_name = CountsResourceDefinition.new('nico_tags')
|
||||
.expression('name', 'external_version')
|
||||
catalog = <<~SQL
|
||||
, tag_counts AS (
|
||||
SELECT membership.kind, membership.tag_id, COUNT(*) AS post_count
|
||||
@@ -211,7 +209,7 @@ class CountsStateQuery
|
||||
SELECT 'internal' AS kind, tag_id, name, names,
|
||||
deprecated_at FROM tag_at WHERE event_type <> 'discard'
|
||||
UNION ALL
|
||||
SELECT tc.kind, tc.tag_id, #{ external_name } AS name, JSON_OBJECT() AS names,
|
||||
SELECT tc.kind, tc.tag_id, external_version.name, JSON_OBJECT() AS names,
|
||||
NULL AS deprecated_at
|
||||
FROM tag_counts tc
|
||||
LEFT JOIN tag_at internal_version
|
||||
|
||||
@@ -182,7 +182,6 @@ class CountsTableQuery
|
||||
order = sort_sql { detail_expression(_1) }
|
||||
query = query.order(Arel.sql((order + ['v.id ASC']).join(', ')))
|
||||
.limit(limit).offset((page - 1) * limit)
|
||||
query = query.preload(:external_tag) if @definition.resource == 'nico_tags'
|
||||
rows = query.to_a
|
||||
if ['posts', 'tags'].include?(@definition.resource)
|
||||
point = population.history ? population.at :
|
||||
@@ -487,7 +486,6 @@ class CountsTableQuery
|
||||
|
||||
table = model.table_name
|
||||
scope = HistoryAccess.visible(model.where(tag_id: ids)).where('created_at <= ?', point)
|
||||
scope = scope.includes(:external_tag) if model == NicoTagVersion
|
||||
bound = @connection.quote(point)
|
||||
if population.history && @definition.resource == 'posts'
|
||||
key = kind == 'internal' ? 'tag_id' : 'external_tag_id'
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
class NicoTagVersionRecorder < VersionRecorder
|
||||
EVENT_TYPES = ['create', 'update'].freeze
|
||||
def self.record! external_tag:, event_type:, created_by_user:
|
||||
new(external_tag:, event_type:, created_by_user:).record!
|
||||
end
|
||||
@@ -15,7 +14,8 @@ class NicoTagVersionRecorder < VersionRecorder
|
||||
def record_key = :external_tag
|
||||
|
||||
def snapshot_attributes
|
||||
{ linked_tag_ids: @record.snapshot_linked_tag_ids.join(' ') }
|
||||
{ name: "#{ @record.platform }:#{ @record.name }",
|
||||
linked_tags: @record.snapshot_linked_tag_names.join(' ') }
|
||||
end
|
||||
|
||||
def tracks_version_no_on_record? = false
|
||||
|
||||
@@ -1,201 +0,0 @@
|
||||
require 'cgi'
|
||||
require 'digest'
|
||||
require 'erb'
|
||||
require 'pathname'
|
||||
require 'uri'
|
||||
|
||||
# A deliberately small current-state projection, always under public authority.
|
||||
class StaticMirror
|
||||
RESOURCE_LIMIT = 100
|
||||
NAME_LIMIT = 2_000
|
||||
MAX_BYTES = 8.megabytes
|
||||
EXCLUSIONS = [
|
||||
'historical_state', 'unversioned_resources', 'tag_descriptions', 'help', 'materials',
|
||||
'external_tags', 'parent_relations', 'tag_sections', 'users', 'audit_data',
|
||||
'authentication', 'editing', 'images', 'videos', 'resources_beyond_limits'].freeze
|
||||
class TooLarge < StandardError; end
|
||||
class UnsafeUrl < StandardError; end
|
||||
|
||||
attr_reader :data_at, :authority, :counts
|
||||
|
||||
def initialize(locale:, origin:)
|
||||
@locale = locale
|
||||
@origin = validated_url(origin).sub(%r{/+\z}, '')
|
||||
end
|
||||
|
||||
def export(&progress)
|
||||
@files = { }
|
||||
@bytes = 0
|
||||
HistoryAccess.set(user: nil) do
|
||||
capture
|
||||
progress&.call(0, @posts.length + @tags.length)
|
||||
generate_pages(&progress)
|
||||
raise HistoryAccess::Forbidden unless HistoryAccess.authorised_artifact?(authority)
|
||||
|
||||
generated_at = Time.current.utc
|
||||
manifest = { format_version: 1, generated_at: generated_at.iso8601(6),
|
||||
data_at:, locale: @locale.code, resources: counts, exclusions: EXCLUSIONS,
|
||||
fields: { posts: ['id', 'title', 'url', 'tags'], tags: ['id', 'category', 'names'] },
|
||||
limits: { posts: RESOURCE_LIMIT, tags: RESOURCE_LIMIT, names: NAME_LIMIT,
|
||||
bytes: MAX_BYTES },
|
||||
external_assets: 'not_included', checksum_scope: 'all_files_except_manifest.json',
|
||||
files: @files.map { |path, data| { path:, sha256: Digest::SHA256.hexdigest(data) } } }
|
||||
add_file('manifest.json', JSON.pretty_generate(manifest))
|
||||
entries = @files.map do |path, data|
|
||||
MaterialZipExporter::Entry.new(path: "archive/#{ path }", data:, mtime: generated_at)
|
||||
end
|
||||
bytes = MaterialZipExporter::ZipWriter.write(entries)
|
||||
raise TooLarge if bytes.bytesize > MAX_BYTES
|
||||
|
||||
bytes
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def capture
|
||||
CountsExporter.with_read_snapshot do
|
||||
@data_at = Time.current.utc.iso8601(6)
|
||||
@authority = HistoryAccess.authority
|
||||
# Requiring a matching immutable revision closes the legacy unversioned gap.
|
||||
@posts = confirmed(Post, PostVersion, :post_versions).order(:id)
|
||||
.limit(RESOURCE_LIMIT).select(:id, :title, :url).to_a
|
||||
@tags = confirmed(Tag, TagVersion, :tag_versions).where(deprecated_at: nil)
|
||||
.order(:id).limit(RESOURCE_LIMIT).select(:id, :category).to_a
|
||||
@names = TagName.where(tag_id: @tags.map(&:id)).order(:tag_id, :language_code, :id)
|
||||
.limit(NAME_LIMIT + 1).pluck(:tag_id, :language_code, :script_code, :name, :primary_flg)
|
||||
raise TooLarge if @names.length > NAME_LIMIT
|
||||
|
||||
@names = @names.group_by(&:first)
|
||||
@relations = PostTag.where(post_id: @posts.map(&:id), tag_id: @tags.map(&:id))
|
||||
.order(:post_id, :tag_id).pluck(:post_id, :tag_id)
|
||||
@counts = { posts: @posts.length, tags: @tags.length }
|
||||
end
|
||||
end
|
||||
|
||||
def confirmed(model, version_model, association)
|
||||
HistoryAccess.current(model.all, version_model).joins(association)
|
||||
.where("#{ version_model.table_name }.version_no = #{ model.table_name }.version_no")
|
||||
end
|
||||
|
||||
def generate_pages
|
||||
assets = Rails.root.join('app/views/static_mirror')
|
||||
['archive.css', 'archive.js'].each do |name|
|
||||
add_file("assets/#{ name }", assets.join(name).read)
|
||||
end
|
||||
add_file('robots.txt', "User-agent: *\nDisallow: /\n")
|
||||
add_file('archive-config.js', 'window.archiveConfig = { origin: null };')
|
||||
page('index.html', 'Static Mirror 生成機能', '', search_form + limitations)
|
||||
page("#{ @locale.code }/posts/index.html", 'Post', '/posts', post_list(@posts))
|
||||
page("#{ @locale.code }/tags/index.html", 'Tag', '/tags', tag_list(@tags))
|
||||
processed = 0
|
||||
@posts.each do |post|
|
||||
tags = @tags.select { |tag| @relations.include?([post.id, tag.id]) }
|
||||
content = "<p>#{ external_link(post.url) }</p>#{ tag_list(tags) }"
|
||||
page(resource_path('posts', post.id), post.title.presence || post.id.to_s,
|
||||
"/posts/#{ post.id }", content)
|
||||
processed += 1
|
||||
yield(processed, @posts.length + @tags.length) if block_given?
|
||||
end
|
||||
@tags.each do |tag|
|
||||
posts = @posts.select { |post| @relations.include?([post.id, tag.id]) }
|
||||
names = @names.fetch(tag.id, []).map do |_, language, script, name, _primary|
|
||||
"<li><span lang=\"#{ h(language) }\">#{ h(name) }</span> (#{ h(script) })</li>"
|
||||
end.join
|
||||
content = "<p>#{ h(tag.category) }</p><ul>#{ names }</ul>#{ post_list(posts) }"
|
||||
page(resource_path('tags', tag.id), tag_name(tag), "/tags/#{ tag.id }", content)
|
||||
processed += 1
|
||||
yield(processed, @posts.length + @tags.length) if block_given?
|
||||
end
|
||||
index = @posts.map { |post|
|
||||
{ path: resource_path('posts', post.id), title: post.title.presence || post.id.to_s }
|
||||
} + @tags.map { |tag|
|
||||
{ path: resource_path('tags', tag.id), title: tag_name(tag),
|
||||
names: @names.fetch(tag.id, []).map { _1[3] } }
|
||||
}
|
||||
add_file('search-index.json', JSON.generate(index))
|
||||
end
|
||||
|
||||
def page(path, title, original_path, content)
|
||||
@path = path
|
||||
@title = title
|
||||
@original_path = "/#{ @locale.code }#{ original_path }"
|
||||
@content = content
|
||||
template = Rails.root.join('app/views/static_mirror/page.html.erb').read
|
||||
add_file(path, ERB.new(template).result(binding))
|
||||
end
|
||||
|
||||
def add_file(path, data)
|
||||
@bytes += data.bytesize
|
||||
raise TooLarge if @bytes > MAX_BYTES
|
||||
|
||||
@files[path] = data
|
||||
end
|
||||
|
||||
def resource_path(resource, id) = "#{ @locale.code }/#{ resource }/#{ id }/index.html"
|
||||
|
||||
def relative(path)
|
||||
Pathname.new(path).relative_path_from(Pathname.new(@path).dirname).to_s
|
||||
end
|
||||
|
||||
def h(value) = CGI.escapeHTML(value.to_s)
|
||||
|
||||
def tag_name(tag)
|
||||
@names.fetch(tag.id, []).find { _1[1] == @locale.language_code && _1[4] }&.fetch(3) ||
|
||||
tag.id.to_s
|
||||
end
|
||||
|
||||
# Links are resolved after the page path is known; only generated numeric IDs enter paths.
|
||||
def post_list(posts)
|
||||
'<h2>Post</h2><ul>' + posts.map { |post|
|
||||
title = post.title.presence || post.id.to_s
|
||||
"<li><a data-path=\"#{ resource_path('posts', post.id) }\">#{ h(title) }</a></li>"
|
||||
}.join + '</ul>'
|
||||
end
|
||||
|
||||
def tag_list(tags)
|
||||
'<h2>Tag</h2><ul>' + tags.map { |tag|
|
||||
"<li><a data-path=\"#{ resource_path('tags', tag.id) }\">#{ h(tag_name(tag)) }</a></li>"
|
||||
}.join + '</ul>'
|
||||
end
|
||||
|
||||
def linked_content
|
||||
@content.gsub(/data-path="([^"]+)"/) { "href=\"#{ h(relative(Regexp.last_match(1))) }\"" }
|
||||
end
|
||||
|
||||
def validated_url(value)
|
||||
uri = URI.parse(value.to_s)
|
||||
unless uri.is_a?(URI::HTTP) && uri.host.present? && uri.userinfo.nil?
|
||||
raise UnsafeUrl
|
||||
end
|
||||
|
||||
uri.to_s
|
||||
rescue URI::InvalidURIError
|
||||
raise UnsafeUrl
|
||||
end
|
||||
|
||||
def external_link(value)
|
||||
url = validated_url(value)
|
||||
"<a href=\"#{ h(url) }\" target=\"_blank\" rel=\"noopener noreferrer\">#{ h(url) }</a>"
|
||||
rescue UnsafeUrl
|
||||
''
|
||||
end
|
||||
|
||||
def search_form
|
||||
'<form id="archive-search"><label>検索 <input name="q" type="search"></label>' +
|
||||
'<button type="submit">検索</button></form><ul id="search-results"></ul>'
|
||||
end
|
||||
|
||||
def limitations
|
||||
<<~HTML
|
||||
<h2>未対応機能</h2>
|
||||
<p>Tag Description, Help, Material, 外部タグ, 親子関係, 区間, 履歴, ログイン, 作成, 編輯</p>
|
||||
<h2>出力件数の制限</h2><p>Post: #{ RESOURCE_LIMIT }, Tag: #{ RESOURCE_LIMIT }</p>
|
||||
<p>TagName: #{ NAME_LIMIT }, #{ MAX_BYTES } B</p>
|
||||
<h2>外部画像や動画への依存</h2><p>ローカルへ取り込めていない資産: 外部画像や動画</p>
|
||||
<h2>データ保護上の制約</h2>
|
||||
<p>公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。</p>
|
||||
<p>今回は現在状態のみの対応</p>
|
||||
HTML
|
||||
end
|
||||
end
|
||||
@@ -1,7 +0,0 @@
|
||||
:root { color-scheme: light dark; }
|
||||
body { max-width: 65rem; margin: auto; padding: 1rem; font: 1rem/1.6 sans-serif; }
|
||||
header { border-bottom: 1px solid; padding-bottom: 1rem; }
|
||||
nav { display: flex; flex-wrap: wrap; gap: 1rem; }
|
||||
a, p, li { overflow-wrap: anywhere; }
|
||||
li { margin-block: .4rem; }
|
||||
input, button { font: inherit; }
|
||||
@@ -1,61 +0,0 @@
|
||||
(() => {
|
||||
const root = new URL ('../', document.currentScript.src)
|
||||
const origin = window.archiveConfig?.origin
|
||||
if (typeof origin === 'string')
|
||||
{
|
||||
try
|
||||
{
|
||||
const url = new URL (origin)
|
||||
if (['http:', 'https:'].includes (url.protocol) && !url.username && !url.password)
|
||||
{
|
||||
url.search = ''
|
||||
url.hash = ''
|
||||
document.querySelectorAll ('[data-original]').forEach (link => {
|
||||
link.href = url.href.replace (/\/+$/, '') + link.dataset.original
|
||||
})
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Each HTML retains its original link when configuration is absent or invalid.
|
||||
}
|
||||
}
|
||||
const form = document.getElementById ('archive-search')
|
||||
const results = document.getElementById ('search-results')
|
||||
if (!form || !results) return
|
||||
form.addEventListener ('submit', async event => {
|
||||
event.preventDefault ()
|
||||
results.replaceChildren ()
|
||||
const query = (new FormData (form).get ('q') ?? '').toString ().toLocaleLowerCase ()
|
||||
if (!query.trim ()) return
|
||||
try
|
||||
{
|
||||
const response = await fetch (new URL ('search-index.json', root))
|
||||
if (!response.ok) throw new Error ('Search index unavailable')
|
||||
const entries = await response.json ()
|
||||
for (const entry of entries)
|
||||
{
|
||||
const text = [entry.title, ...(entry.names ?? [])].join (' ').toLocaleLowerCase ()
|
||||
if (!text.includes (query)) continue
|
||||
const url = new URL (entry.path, root)
|
||||
if (url.origin !== root.origin || !url.pathname.startsWith (root.pathname)) continue
|
||||
const item = document.createElement ('li')
|
||||
const link = document.createElement ('a')
|
||||
link.setAttribute ('href', entry.path)
|
||||
link.textContent = entry.title
|
||||
item.append (link)
|
||||
results.append (item)
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Lists remain available if the index cannot be loaded.
|
||||
for (const link of document.querySelectorAll ('nav a:not([data-original])'))
|
||||
{
|
||||
const item = document.createElement ('li')
|
||||
item.append (link.cloneNode (true))
|
||||
results.append (item)
|
||||
}
|
||||
}
|
||||
})
|
||||
})()
|
||||
@@ -1,28 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="ja">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="robots" content="noindex, nofollow">
|
||||
<meta name="referrer" content="no-referrer">
|
||||
<meta http-equiv="Content-Security-Policy"
|
||||
content="default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; form-action 'none'; base-uri 'none'">
|
||||
<title><%= h(@title) %></title>
|
||||
<link rel="stylesheet" href="<%= h(relative('assets/archive.css')) %>">
|
||||
<script defer src="<%= h(relative('archive-config.js')) %>"></script>
|
||||
<script defer src="<%= h(relative('assets/archive.js')) %>"></script>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<p>静的保存版 — データ基準時点: <time><%= h(data_at) %></time></p>
|
||||
<nav>
|
||||
<a href="<%= h(relative('index.html')) %>">Static Mirror 生成機能</a>
|
||||
<a href="<%= h(relative("#{ @locale.code }/posts/index.html")) %>">Post</a>
|
||||
<a href="<%= h(relative("#{ @locale.code }/tags/index.html")) %>">Tag</a>
|
||||
<a href="<%= h(@origin + @original_path) %>" data-original="<%= h(@original_path) %>"
|
||||
target="_blank" rel="noopener noreferrer">本家へのリンク</a>
|
||||
</nav>
|
||||
</header>
|
||||
<main><h1><%= h(@title) %></h1><%= linked_content %></main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,9 +1,4 @@
|
||||
Rails.application.routes.draw do
|
||||
get 'static_mirror', to: 'static_mirrors#show'
|
||||
post 'static_mirror/jobs', to: 'static_mirrors#create'
|
||||
get 'static_mirror/jobs/:id', to: 'static_mirrors#status'
|
||||
get 'static_mirror/files/:id', to: 'static_mirrors#download'
|
||||
|
||||
get 'counts/definitions/:resource', to: 'counts_tables#definition'
|
||||
get 'counts/definitions/:resource/candidates', to: 'counts_tables#candidates'
|
||||
get 'counts/query', to: 'counts_tables#index'
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
class AddLinkedTagIdsToNicoTagVersions < ActiveRecord::Migration[8.0]
|
||||
def change
|
||||
add_column :nico_tag_versions, :linked_tag_ids, :text
|
||||
end
|
||||
end
|
||||
@@ -1,42 +0,0 @@
|
||||
class MigrateLinkedTagsInNicoTagVersions < ActiveRecord::Migration[8.0]
|
||||
class MigrationTagVersion < ActiveRecord::Base
|
||||
self.table_name = 'tag_versions'
|
||||
end
|
||||
|
||||
class MigrationNicoTagVersion < ActiveRecord::Base
|
||||
self.table_name = 'nico_tag_versions'
|
||||
end
|
||||
|
||||
def up
|
||||
# 除票外部タグの恢復はないので除票履歴があれば全消しでよいはず.
|
||||
execute <<~SQL
|
||||
DELETE
|
||||
t1
|
||||
FROM
|
||||
nico_tag_versions t1
|
||||
INNER JOIN
|
||||
nico_tag_versions t2
|
||||
ON
|
||||
t1.tag_id = t2.tag_id
|
||||
AND t2.event_type = 'discard'
|
||||
SQL
|
||||
|
||||
MigrationNicoTagVersion.find_each do |ntv|
|
||||
linked_tag_ids = ntv.linked_tags.split.map { |name|
|
||||
# 厳密には一意ぢゃないけどめんだぅだからこれでいいや.
|
||||
MigrationTagVersion
|
||||
.order(created_at: :desc, id: :desc)
|
||||
.find_by("names ->> '$.ja.name' = ?", name)
|
||||
&.tag_id
|
||||
}.compact.uniq.sort.join(' ')
|
||||
|
||||
ntv.update_columns(linked_tag_ids:)
|
||||
end
|
||||
|
||||
change_column_null :nico_tag_versions, :linked_tag_ids, false
|
||||
end
|
||||
|
||||
def down
|
||||
raise ActiveRecord::IrreversibleMigration, '戻せません.'
|
||||
end
|
||||
end
|
||||
@@ -1,6 +0,0 @@
|
||||
class RemoveLinkedTagsFromNicoTagVersions < ActiveRecord::Migration[8.0]
|
||||
def change
|
||||
remove_column :nico_tag_versions, :linked_tags
|
||||
remove_column :nico_tag_versions, :name
|
||||
end
|
||||
end
|
||||
生成ファイル
+3
-2
@@ -10,7 +10,7 @@
|
||||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema[8.0].define(version: 2026_11_08_020000) do
|
||||
ActiveRecord::Schema[8.0].define(version: 2026_10_15_010000) do
|
||||
create_table "active_storage_attachments", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
|
||||
t.string "name", null: false
|
||||
t.string "record_type", null: false
|
||||
@@ -301,9 +301,10 @@ ActiveRecord::Schema[8.0].define(version: 2026_11_08_020000) do
|
||||
t.bigint "tag_id", null: false
|
||||
t.integer "version_no", null: false
|
||||
t.string "event_type", null: false
|
||||
t.string "name", null: false
|
||||
t.text "linked_tags", null: false
|
||||
t.datetime "created_at", null: false
|
||||
t.bigint "created_by_user_id"
|
||||
t.text "linked_tag_ids", null: false
|
||||
t.index ["created_at"], name: "index_nico_tag_versions_on_created_at"
|
||||
t.index ["created_by_user_id", "created_at"], name: "index_nico_tag_versions_on_created_by_user_id_and_created_at", order: { created_at: :desc }
|
||||
t.index ["tag_id", "created_at"], name: "index_nico_tag_versions_on_tag_id_and_created_at", order: { created_at: :desc }
|
||||
|
||||
@@ -11,63 +11,28 @@ RSpec.describe ExternalTag, type: :model do
|
||||
expect(external.reload.name).to eq('raw tag[]')
|
||||
end
|
||||
|
||||
it 'does not allow a persisted external tag to change its platform or name' do
|
||||
external = create(:external_tag, name: 'immutable')
|
||||
|
||||
expect { external.update!(name: 'renamed') }
|
||||
.to raise_error(ActiveRecord::RecordInvalid)
|
||||
expect(external.errors[:name]).to include('は変更できません.')
|
||||
|
||||
external.reload
|
||||
external.platform = nil
|
||||
expect { external.save! }.to raise_error(ActiveRecord::RecordInvalid)
|
||||
expect(external.errors[:platform]).to include('は変更できません.')
|
||||
|
||||
expect(external.reload).to have_attributes(platform: 'nico', name: 'immutable')
|
||||
end
|
||||
|
||||
it 'rejects delete, which bypasses destroy callbacks' do
|
||||
external = create(:external_tag)
|
||||
|
||||
expect { external.delete }.to raise_error(ActiveRecord::ReadOnlyRecord)
|
||||
expect(external.reload).to be_persisted
|
||||
end
|
||||
|
||||
it 'increments the counter through post associations without changing identity' do
|
||||
external = create(:external_tag, name: 'counter')
|
||||
post = create(:post)
|
||||
|
||||
expect {
|
||||
PostExternalTag.create!(post:, external_tag: external)
|
||||
}.to change { external.reload.post_count }.by(1)
|
||||
|
||||
expect(external.reload).to have_attributes(platform: 'nico', name: 'counter')
|
||||
end
|
||||
|
||||
it 'does not destroy external tags or their post associations and history' do
|
||||
it 'deletes post associations without deleting posts or version history' do
|
||||
external = create(:external_tag)
|
||||
post = create(:post)
|
||||
PostExternalTag.create!(post:, external_tag: external)
|
||||
version = NicoTagVersionRecorder.record!(
|
||||
external_tag: external, event_type: :create, created_by_user: nil)
|
||||
|
||||
expect { external.destroy! }.to raise_error(ActiveRecord::RecordNotDestroyed)
|
||||
external.destroy!
|
||||
|
||||
expect(external.reload).to be_persisted
|
||||
expect(PostExternalTag.exists?(post:, external_tag: external)).to be(true)
|
||||
expect(post.reload.external_tags).to contain_exactly(external)
|
||||
expect(PostExternalTag.where(external_tag_id: external.id)).to be_empty
|
||||
expect(post.reload.external_tags).to be_empty
|
||||
expect(version.reload.tag_id).to eq(external.id)
|
||||
end
|
||||
|
||||
it 'does not destroy an external tag or its links to internal tags' do
|
||||
it 'deletes external links without deleting linked internal tags' do
|
||||
external = create(:external_tag)
|
||||
tag = create(:tag)
|
||||
relation = NicoTagRelation.create!(nico_tag: external, tag:)
|
||||
NicoTagRelation.create!(nico_tag: external, tag:)
|
||||
|
||||
expect { external.destroy! }.to raise_error(ActiveRecord::RecordNotDestroyed)
|
||||
external.destroy!
|
||||
|
||||
expect(external.reload).to be_persisted
|
||||
expect(relation.reload).to be_persisted
|
||||
expect(NicoTagRelation.where(nico_tag_id: external.id)).to be_empty
|
||||
expect(Tag.exists?(tag.id)).to be(true)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -76,7 +76,7 @@ RSpec.describe 'Counts artifact policy and external references' do
|
||||
end
|
||||
end
|
||||
|
||||
it 'does not expose a private linked tag ID through an external history snapshot' do
|
||||
it 'does not expose a private linked tag name through an external history snapshot' do
|
||||
tag = create(:tag, primary_name: 'private-linked-name')
|
||||
TagVersion.create!(tag:, version_no: 1, event_type: 'discard', name: 'private-linked-name',
|
||||
category: 'general', aliases: '', parent_tag_ids: '', created_at: Time.utc(2024),
|
||||
@@ -84,7 +84,7 @@ RSpec.describe 'Counts artifact policy and external references' do
|
||||
auto_generated: false, aliases: [] } })
|
||||
external = create(:external_tag, name: 'public-external')
|
||||
row = NicoTagVersion.create!(external_tag: external, version_no: 1, event_type: 'create',
|
||||
linked_tag_ids: tag.id.to_s,
|
||||
name: 'nico:public-external', linked_tags: 'private-linked-name',
|
||||
created_at: Time.utc(2024))
|
||||
post = create(:post)
|
||||
snapshot = PostVersion.create!(post:, version_no: 1, event_type: 'create',
|
||||
@@ -99,7 +99,7 @@ RSpec.describe 'Counts artifact policy and external references' do
|
||||
expect(HistoryAccess.visible(NicoTagVersion.where(id: row.id))).to exist
|
||||
end
|
||||
public_row = NicoTagVersion.create!(external_tag: external, version_no: 2, event_type: 'update',
|
||||
linked_tag_ids: '',
|
||||
name: 'nico:public-external', linked_tags: 'private-linked-name-extra',
|
||||
created_at: Time.utc(2024, 7))
|
||||
HistoryAccess.set(user: nil) do
|
||||
expect(HistoryAccess.visible(NicoTagVersion.where(id: public_row.id))).to exist
|
||||
|
||||
@@ -10,22 +10,23 @@ RSpec.describe 'Shared Tag and ExternalTag IDs', type: :model do
|
||||
expect(next_tag.id).to be > external.id
|
||||
end
|
||||
|
||||
it 'does not reuse IDs after internal deletion while external tags remain' do
|
||||
it 'does not reuse physically deleted IDs in either table' do
|
||||
tag = create(:tag)
|
||||
external = create(:external_tag)
|
||||
TagName.where(tag_id: tag.id).delete_all
|
||||
tag.destroy!
|
||||
expect(ExternalTag.exists?(external.id)).to be(true)
|
||||
external.destroy!
|
||||
|
||||
expect(create(:tag).id).to be > external.id
|
||||
expect(create(:external_tag).id).to be > external.id
|
||||
end
|
||||
|
||||
it 'retains overlapping IDs after a tag update and a new allocation' do
|
||||
it 'retains existing overlapping IDs without renumbering on update or insert' do
|
||||
tag = create(:tag)
|
||||
external = create(:external_tag)
|
||||
external.update_columns(id: tag.id)
|
||||
tag.update!(category: :meta)
|
||||
external.update!(name: 'updated_external')
|
||||
before_tag = tag.reload.attributes
|
||||
before_external = external.reload.attributes
|
||||
|
||||
|
||||
@@ -47,34 +47,24 @@ RSpec.describe VersionRecord, type: :model do
|
||||
external_tag: nico_tag,
|
||||
version_no: 1,
|
||||
event_type: 'create',
|
||||
linked_tag_ids: '',
|
||||
name: "nico:#{ nico_tag.name }",
|
||||
linked_tags: '',
|
||||
created_at: Time.current,
|
||||
created_by_user: nil
|
||||
)
|
||||
|
||||
expect {
|
||||
version.update!(linked_tag_ids: '123')
|
||||
version.update!(name: 'nico:changed')
|
||||
}.to raise_error(ActiveRecord::ReadOnlyRecord)
|
||||
end
|
||||
|
||||
|
||||
it 'rejects discard and restore versions for external tags' do
|
||||
[:discard, :restore].each do |event_type|
|
||||
version = NicoTagVersion.new(
|
||||
external_tag: nico_tag, version_no: 1, event_type:,
|
||||
linked_tag_ids: '', created_at: Time.current)
|
||||
|
||||
expect(version).not_to be_valid
|
||||
expect(version.errors[:event_type]).not_to be_empty
|
||||
end
|
||||
end
|
||||
|
||||
it 'prevents NicoTagVersion destroy' do
|
||||
version = NicoTagVersion.create!(
|
||||
external_tag: nico_tag,
|
||||
version_no: 1,
|
||||
event_type: 'create',
|
||||
linked_tag_ids: '',
|
||||
name: "nico:#{ nico_tag.name }",
|
||||
linked_tags: '',
|
||||
created_at: Time.current,
|
||||
created_by_user: nil
|
||||
)
|
||||
|
||||
@@ -19,7 +19,7 @@ RSpec.describe 'Counts table API', type: :request do
|
||||
names: { ja: { name: 'alpha', script_code: 'Jpan', auto_generated: false, aliases: [] } },
|
||||
created_at: Time.utc(2023))
|
||||
NicoTagVersion.create!(external_tag: external, version_no: 1,
|
||||
event_type: 'create', linked_tag_ids: '', created_at: Time.utc(2023))
|
||||
event_type: 'create', name: 'beta', linked_tags: '', created_at: Time.utc(2023))
|
||||
snapshots = [
|
||||
{ tag_id: tag.id, version_no: 1, name: 'alpha', category: 'general', sections: [] },
|
||||
{ external_tag_id: external.id }]
|
||||
@@ -50,115 +50,6 @@ RSpec.describe 'Counts table API', type: :request do
|
||||
|
||||
def versions(result) = result.fetch('versions').map { _1.fetch('version_no') }
|
||||
|
||||
context 'nico link ID snapshots through the V1 field name' do
|
||||
let(:base) do
|
||||
super().merge(resource: 'nico_tags')
|
||||
end
|
||||
|
||||
before do
|
||||
NicoTagVersion.create!(external_tag: external, version_no: 2,
|
||||
event_type: 'update', linked_tag_ids: tag.id.to_s,
|
||||
created_at: Time.utc(2024, 1, 1))
|
||||
NicoTagVersion.create!(external_tag: external, version_no: 3,
|
||||
event_type: 'update', linked_tag_ids: '',
|
||||
created_at: Time.utc(2024, 2, 1))
|
||||
end
|
||||
|
||||
it 'returns ID values under linked_tags without exposing the physical column key' do
|
||||
result = query
|
||||
expect(versions(result)).to eq([1, 2, 3])
|
||||
expect(result['versions'].map { _1.fetch('linked_tags') }).to eq(['', tag.id.to_s, ''])
|
||||
result['versions'].each do |row|
|
||||
expect(row).not_to have_key('linked_tag_ids')
|
||||
expect(row.fetch('name')).to eq('nico:beta')
|
||||
end
|
||||
end
|
||||
|
||||
it 'preloads external names separately for aliased link-change details' do
|
||||
statements = []
|
||||
listener = lambda do |_name, _start, _finish, _id, payload|
|
||||
statements << payload[:sql]
|
||||
end
|
||||
ActiveSupport::Notifications.subscribed(listener, 'sql.active_record') do
|
||||
result = query(axis('changed_field', 'linked_tags'))
|
||||
expect(versions(result)).to eq([2, 3])
|
||||
expect(result.fetch('versions').map { _1.fetch('name') })
|
||||
.to eq(['nico:beta', 'nico:beta'])
|
||||
end
|
||||
|
||||
details = statements.find { _1.include?('AS counts_latest') }
|
||||
expect(details).not_to be_nil
|
||||
expect(details).not_to include('JOIN `external_tags`')
|
||||
names = statements.select { _1.match?(/SELECT .* FROM `external_tags`/) }
|
||||
expect(names.length).to eq(1)
|
||||
end
|
||||
|
||||
it 'advertises the legacy field, label and default columns only' do
|
||||
get '/counts/definitions/nico_tags', params: { locale: 'ja' }
|
||||
expect(response).to have_http_status(:ok)
|
||||
body = response.parsed_body
|
||||
['fields', 'filter_fields', 'output_fields'].each do |collection|
|
||||
keys = body.fetch(collection).map { _1.fetch('key') }
|
||||
expect(keys).to include('linked_tags')
|
||||
expect(keys).not_to include('linked_tag_ids')
|
||||
end
|
||||
field = body.fetch('output_fields').find { _1['key'] == 'linked_tags' }
|
||||
expect(field.fetch('label')).to eq('counts:attributes.linked_tags')
|
||||
body.fetch('axes').each_value do |keys|
|
||||
expect(keys).to include('linked_tags')
|
||||
expect(keys).not_to include('linked_tag_ids')
|
||||
end
|
||||
body.fetch('defaults').select { _1['mode'] == 'details' }.each do |entry|
|
||||
settings = entry.fetch('params').to_h { [_1.fetch('key'), _1.fetch('value')] }
|
||||
expect(settings.fetch('columns').split).to include('name', 'linked_tags')
|
||||
expect(settings.fetch('columns').split).not_to include('linked_tag_ids')
|
||||
end
|
||||
end
|
||||
|
||||
it 'filters IDs and legacy drills and compares link changes through linked_tags' do
|
||||
predicate = { conditions: '', 'conditions.0.field' => 'linked_tags',
|
||||
'conditions.0.operator' => 'eq', 'conditions.0.value' => tag.id.to_s }
|
||||
expect(versions(query(predicate))).to eq([2])
|
||||
expect(versions(query(axis('linked_tags', tag.id.to_s, root: 'drill')))).to eq([2])
|
||||
expect(versions(query(axis('changed_field', 'linked_tags')))).to eq([2, 3])
|
||||
expect(versions(query(predicate.merge('conditions.0.value' => 'alpha')))).to be_empty
|
||||
end
|
||||
|
||||
it 'sorts ID snapshots through the public field in both directions' do
|
||||
expect(versions(query(sort: 'linked_tags:asc version_no:asc'))).to eq([1, 3, 2])
|
||||
expect(versions(query(sort: 'linked_tags:desc version_no:asc'))).to eq([2, 1, 3])
|
||||
end
|
||||
|
||||
it 'restores historical link IDs before applying a latest snapshot filter' do
|
||||
predicate = { definition: 'latest', at: '2024-01-15T00:00:00Z',
|
||||
conditions: '', 'conditions.0.field' => 'linked_tags',
|
||||
'conditions.0.operator' => 'eq', 'conditions.0.value' => tag.id.to_s }
|
||||
expect(versions(query(predicate))).to eq([2])
|
||||
expect(versions(query(predicate.except(:at)))).to be_empty
|
||||
end
|
||||
|
||||
it 'groups by linked_tags ID values and accepts the resulting drill condition' do
|
||||
sign_in_as(create(:user, :admin))
|
||||
result = query(mode: 'summary', rows: 'linked_tags', cols: '', sort: '')
|
||||
expect(result.fetch('cells')).to match_array([
|
||||
{ 'values' => [''], 'count' => 2 },
|
||||
{ 'values' => [tag.id.to_s], 'count' => 1 }])
|
||||
expect(versions(query(axis('linked_tags', tag.id.to_s)))).to eq([2])
|
||||
end
|
||||
|
||||
it 'rejects the physical column as a public filter, axis, sort or change field' do
|
||||
sign_in_as(create(:user, :admin))
|
||||
[
|
||||
axis('linked_tag_ids', tag.id.to_s),
|
||||
axis('changed_field', 'linked_tag_ids'),
|
||||
{ mode: 'summary', rows: 'linked_tag_ids', cols: '', sort: '' },
|
||||
{ sort: 'linked_tag_ids:asc' }].each do |invalid|
|
||||
get '/counts/query', params: base.merge(invalid)
|
||||
expect(response).to have_http_status(:bad_request), invalid.inspect
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
it 'identifies a totals timeout without exposing SQL in the error response' do
|
||||
allow_any_instance_of(CountsPivotTotals).to receive(:call)
|
||||
.and_raise(CountsPivotTotals::TimedOut, 'private SQL')
|
||||
@@ -332,10 +223,11 @@ RSpec.describe 'Counts table API', type: :request do
|
||||
expect(versions(query(tags: 'renamed-alpha'))).to eq([1, 2, 4])
|
||||
end
|
||||
|
||||
it 'resolves immutable external names across historical snapshots' do
|
||||
expect(versions(query(tags: 'nico:beta'))).to eq([1])
|
||||
expect(versions(query(tags: 'nico:renamed-beta'))).to eq([])
|
||||
expect(versions(query(tags: 'not:nico:beta'))).to eq([2, 3, 4])
|
||||
it 'resolves external names from current registrations too' do
|
||||
external.update!(name: 'renamed-beta')
|
||||
expect(versions(query(tags: 'nico:renamed-beta'))).to eq([1])
|
||||
expect(versions(query(tags: 'nico:beta'))).to eq([])
|
||||
expect(versions(query(tags: 'not:nico:renamed-beta'))).to eq([2, 3, 4])
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -38,9 +38,12 @@ RSpec.describe 'NicoTags', type: :request do
|
||||
end
|
||||
|
||||
it 'filters by nico tag name, linked tag name, and link status' do
|
||||
linked = create(:external_tag, name: 'search_linked')
|
||||
unlinked = create(:external_tag, name: 'search_unlinked')
|
||||
other = create(:external_tag, name: 'other')
|
||||
linked = create(:external_tag)
|
||||
linked.update!(name: 'search_linked')
|
||||
unlinked = create(:external_tag)
|
||||
unlinked.update!(name: 'search_unlinked')
|
||||
other = create(:external_tag)
|
||||
other.update!(name: 'other')
|
||||
destination = create(:tag, category: :general)
|
||||
primary_tag_name_for(destination, 'ja').update!(name: 'destination_search')
|
||||
NicoTagRelation.create!(nico_tag: linked, tag: destination)
|
||||
@@ -77,9 +80,11 @@ RSpec.describe 'NicoTags', type: :request do
|
||||
end
|
||||
|
||||
it 'sorts by name and timestamps' do
|
||||
older = create(:external_tag, name: 'ordered_a')
|
||||
older = create(:external_tag)
|
||||
older.update!(name: 'ordered_a')
|
||||
older.update_columns(created_at: 2.days.ago)
|
||||
newer = create(:external_tag, name: 'ordered_b')
|
||||
newer = create(:external_tag)
|
||||
newer.update!(name: 'ordered_b')
|
||||
newer.update_columns(created_at: 1.day.ago)
|
||||
older_post_tag =
|
||||
PostExternalTag.create!(post: create(:post), external_tag: older)
|
||||
@@ -184,8 +189,9 @@ RSpec.describe 'NicoTags', type: :request do
|
||||
|
||||
versions = nico_tag.reload.nico_tag_versions.order(:version_no)
|
||||
expect(versions.map(&:event_type)).to eq(['create', 'update'])
|
||||
expect(versions.last.linked_tag_ids.split.map(&:to_i))
|
||||
.to match_array([linked_a.id, linked_b.id])
|
||||
expect(versions.last.linked_tags.split).to match_array([
|
||||
'nico_linked_a',
|
||||
'nico_linked_b'])
|
||||
expect(versions.last.created_by_user_id).to eq(admin.id)
|
||||
end
|
||||
|
||||
@@ -204,7 +210,7 @@ RSpec.describe 'NicoTags', type: :request do
|
||||
expect(json).to eq([])
|
||||
expect(nico_tag.reload.linked_tags).to be_empty
|
||||
expect(nico_tag.nico_tag_versions.order(:version_no).last)
|
||||
.to have_attributes(linked_tag_ids: '', created_by_user: member)
|
||||
.to have_attributes(linked_tags: '', created_by_user: member)
|
||||
end
|
||||
|
||||
it 'returns the tags field error when a nico tag is specified directly' do
|
||||
|
||||
@@ -20,7 +20,7 @@ RSpec.describe 'Post history privacy', type: :request do
|
||||
created_at: origin + index.hours)
|
||||
end
|
||||
NicoTagVersion.create!(external_tag: external, version_no: 1, event_type: 'create',
|
||||
linked_tag_ids: tag.id.to_s,
|
||||
name: 'nico:external-history-tag', linked_tags: 'private-history-tag',
|
||||
created_at: origin)
|
||||
['create', 'discard', 'restore', 'update'].each_with_index do |event, index|
|
||||
snapshots =
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
require 'rails_helper'
|
||||
require 'tmpdir'
|
||||
|
||||
RSpec.describe 'Static Mirror API', type: :request do
|
||||
include ActiveSupport::Testing::TimeHelpers
|
||||
|
||||
around do |example|
|
||||
Dir.mktmpdir('static-mirror-spec') do |path|
|
||||
allow(StaticMirrorJob).to receive(:directory).and_return(Pathname.new(path))
|
||||
example.run
|
||||
end
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Rails.env).to receive(:development?).and_return(true)
|
||||
allow(StaticMirrorJob).to receive(:perform_later)
|
||||
end
|
||||
|
||||
def create_job
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:accepted)
|
||||
json.fetch('job')
|
||||
end
|
||||
|
||||
it 'accepts anonymous jobs asynchronously and serves the completed ZIP' do
|
||||
record = create(:post, title: 'mirror-public-api-title')
|
||||
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
||||
title: record.title, url: record.url, tags: '', tags_json: [], parent_post_ids: '',
|
||||
created_at: Time.current)
|
||||
get '/static_mirror'
|
||||
expect(json).to include('resource_limit' => 100, 'concurrency' => 1)
|
||||
id = create_job
|
||||
expect(StaticMirrorJob).to have_received(:perform_later).with(id)
|
||||
expect(StaticMirrorJob.directory.join('archive.zip')).not_to exist
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'queued', 'data_at' => nil)
|
||||
StaticMirrorJob.perform_now(id)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'ready', 'processed' => 1, 'total' => 1)
|
||||
expect(json.fetch('data_at')).to be_present
|
||||
expect(json.keys).not_to include('owner', 'origin', 'actor_id', 'policy_revision')
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:ok)
|
||||
expect(response.media_type).to eq('application/zip')
|
||||
expect(response.body.byteslice(0, 4)).to eq("PK\x03\x04".b)
|
||||
expect(response.body).to include(record.title)
|
||||
expect(response.headers['Cache-Control']).to include('no-store')
|
||||
end
|
||||
|
||||
it 'rejects historical and unknown input instead of pretending to restore it' do
|
||||
['at', 'data_at', 'admin', 'limit'].each do |key|
|
||||
post '/static_mirror/jobs', params: { locale: 'ja', key => '2020-01-01' }, as: :json
|
||||
expect(response).to have_http_status(:bad_request)
|
||||
end
|
||||
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
||||
end
|
||||
|
||||
it 'limits requests globally even across fresh anonymous sessions' do
|
||||
create_job
|
||||
other = ActionDispatch::Integration::Session.new(Rails.application)
|
||||
other.post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(other.response).to have_http_status(:too_many_requests)
|
||||
expect(other.response.headers['Retry-After']).to eq('60')
|
||||
expect(StaticMirrorJob).to have_received(:perform_later).once
|
||||
end
|
||||
|
||||
it 'binds job status and downloads to the creating session' do
|
||||
id = create_job
|
||||
StaticMirrorJob.perform_now(id)
|
||||
other = ActionDispatch::Integration::Session.new(Rails.application)
|
||||
["/static_mirror/jobs/#{ id }", "/static_mirror/files/#{ id }"].each do |path|
|
||||
other.get path
|
||||
expect(other.response).to have_http_status(:not_found)
|
||||
end
|
||||
end
|
||||
|
||||
it 'refuses ready artifacts after a new discard even for admins' do
|
||||
id = create_job
|
||||
StaticMirrorJob.perform_now(id)
|
||||
record = create(:post)
|
||||
PostVersion.create!(post: record, version_no: 1, event_type: 'discard',
|
||||
url: record.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.current)
|
||||
sign_in_as(create(:user, :admin))
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
|
||||
it 'fails lost jobs and rejects expired downloads' do
|
||||
id = create_job
|
||||
travel 80.seconds do
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'failed', 'error' => 'worker_stopped')
|
||||
end
|
||||
travel 11.minutes do
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
end
|
||||
|
||||
it 'rejects a previously ready ZIP after its expiry' do
|
||||
id = create_job
|
||||
StaticMirrorJob.perform_now(id)
|
||||
travel 11.minutes do
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
end
|
||||
|
||||
it 'keeps simultaneous generation bounded after the cooldown' do
|
||||
create_job
|
||||
lock = File.open(StaticMirrorJob.directory.join('.generation.lock'), File::RDWR)
|
||||
lock.flock(File::LOCK_EX)
|
||||
travel 61.seconds do
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:too_many_requests)
|
||||
end
|
||||
ensure
|
||||
lock&.close
|
||||
end
|
||||
|
||||
it 'reports bounded-generation failures without returning internal exception messages' do
|
||||
id = create_job
|
||||
allow(StaticMirror).to receive(:new).and_raise(StaticMirror::TooLarge, 'private details')
|
||||
StaticMirrorJob.perform_now(id)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'failed', 'error' => 'too_large')
|
||||
expect(response.body).not_to include('private details')
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
|
||||
it 'is unavailable outside development' do
|
||||
allow(Rails.env).to receive(:development?).and_return(false)
|
||||
get '/static_mirror'
|
||||
expect(response).to have_http_status(:not_found)
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get '/static_mirror/jobs/unknown'
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get '/static_mirror/files/unknown'
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
|
||||
it 'preserves user and IP BAN checks' do
|
||||
user = create(:user, banned_at: Time.current)
|
||||
sign_in_as(user)
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:forbidden)
|
||||
sign_out
|
||||
IpAddress.create!(ip_address: IPAddr.new('127.0.0.1').hton, banned_at: Time.current)
|
||||
get '/static_mirror', headers: { 'REMOTE_ADDR' => '127.0.0.1' }
|
||||
expect(response).to have_http_status(:forbidden)
|
||||
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
||||
end
|
||||
|
||||
it 'requires the normal CSRF token for anonymous generation', csrf: true do
|
||||
get '/users/csrf'
|
||||
token = json.fetch('token')
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:forbidden)
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json,
|
||||
headers: { 'X-CSRF-Token' => token }
|
||||
expect(response).to have_http_status(:accepted)
|
||||
end
|
||||
end
|
||||
@@ -22,55 +22,6 @@ RSpec.describe 'Counts export contracts' do
|
||||
.and_return(@counts_export_dir)
|
||||
end
|
||||
|
||||
context 'nico link IDs with the V1 export field name' do
|
||||
let(:tag) { create(:tag, primary_name: 'linked export tag') }
|
||||
let(:external) { create(:external_tag, name: 'external export tag') }
|
||||
let(:payload) do
|
||||
super().merge('resource' => 'nico_tags', 'columns' => 'linked_tags',
|
||||
'headers' => '["linked_tags"]')
|
||||
end
|
||||
|
||||
before do
|
||||
NicoTagVersion.create!(external_tag: external, version_no: 1,
|
||||
event_type: 'create', linked_tag_ids: '', created_at: Time.utc(2023))
|
||||
NicoTagVersion.create!(external_tag: external, version_no: 2,
|
||||
event_type: 'update', linked_tag_ids: tag.id.to_s, created_at: Time.utc(2024))
|
||||
end
|
||||
|
||||
it 'exports recorded IDs with the legacy JSON key and metadata' do
|
||||
result = JSON.parse(CountsTableExport.new(payload).body.to_a.join)
|
||||
expect(result.fetch('data')).to eq([
|
||||
{ 'linked_tags' => '' }, { 'linked_tags' => tag.id.to_s }])
|
||||
expect(result.fetch('metadata').fetch('columns')).to eq(['linked_tags'])
|
||||
keys = result.fetch('metadata').fetch('fields').map { _1.fetch('key') }
|
||||
expect(keys).to include('linked_tags')
|
||||
expect(keys).not_to include('linked_tag_ids')
|
||||
end
|
||||
|
||||
it 'exports the selected legacy CSV header and ID values' do
|
||||
body = CountsTableExport.new(payload.merge('format' => 'csv')).body.to_a.join
|
||||
expect(CSV.parse(body.delete_prefix("\uFEFF")))
|
||||
.to eq([['linked_tags'], [''], [tag.id.to_s]])
|
||||
end
|
||||
|
||||
it 'keeps the legacy link column in default JSON exports' do
|
||||
options = payload.except('columns', 'headers')
|
||||
result = JSON.parse(CountsTableExport.new(options).body.to_a.join)
|
||||
expect(result.fetch('metadata').fetch('columns')).to include('linked_tags')
|
||||
result.fetch('data').each do |row|
|
||||
expect(row).to have_key('linked_tags')
|
||||
expect(row).not_to have_key('linked_tag_ids')
|
||||
end
|
||||
end
|
||||
|
||||
it 'rejects the physical column as a selected CSV or JSON field' do
|
||||
['csv', 'json'].each do |format|
|
||||
options = payload.merge('format' => format, 'columns' => 'linked_tag_ids')
|
||||
expect { CountsTableExport.new(options) }.to raise_error(ArgumentError)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context 'serialization after the authorised query' do
|
||||
let(:rows) { [{ title: "=formula\n\"quoted\"" }, { title: 'second' }] }
|
||||
let(:page) do
|
||||
|
||||
@@ -8,23 +8,21 @@ RSpec.describe NicoTagVersionRecorder do
|
||||
described_class.record!(external_tag:, event_type:, created_by_user: member)
|
||||
end
|
||||
|
||||
it 'records the external association and derives the platform-qualified name' do
|
||||
it 'records the external association and platform-qualified name' do
|
||||
version = record(:create)
|
||||
|
||||
expect(version).to have_attributes(
|
||||
external_tag:, version_no: 1, event_type: 'create',
|
||||
name: 'nico:raw tag[]', linked_tag_ids: '', created_by_user: member)
|
||||
name: 'nico:raw tag[]', linked_tags: '', created_by_user: member)
|
||||
expect(external_tag.reload.nico_tag_versions).to contain_exactly(version)
|
||||
end
|
||||
|
||||
it 'uses the latest history number when the record has no version_no column' do
|
||||
first = record(:create)
|
||||
tag = create(:tag)
|
||||
NicoTagRelation.create!(nico_tag: external_tag, tag:)
|
||||
external_tag.update!(name: 'changed')
|
||||
second = record(:update)
|
||||
|
||||
expect(second).to have_attributes(version_no: 2, name: 'nico:raw tag[]',
|
||||
linked_tag_ids: tag.id.to_s)
|
||||
expect(second).to have_attributes(version_no: 2, name: 'nico:changed')
|
||||
expect(first.reload.name).to eq('nico:raw tag[]')
|
||||
expect(external_tag.reload.has_attribute?(:version_no)).to be(false)
|
||||
end
|
||||
@@ -42,7 +40,7 @@ RSpec.describe NicoTagVersionRecorder do
|
||||
|
||||
expect(version).to have_attributes(
|
||||
external_tag: external, name: 'registered_external:foo', version_no: 1,
|
||||
event_type: 'create', linked_tag_ids: '', created_by_user: member)
|
||||
event_type: 'create', linked_tags: '', created_by_user: member)
|
||||
end
|
||||
|
||||
it 'returns the latest version without appending an unchanged snapshot' do
|
||||
@@ -66,21 +64,13 @@ RSpec.describe NicoTagVersionRecorder do
|
||||
expect(external_tag.nico_tag_versions).to contain_exactly(first)
|
||||
end
|
||||
|
||||
|
||||
it 'rejects discard and restore events for external tags' do
|
||||
[:discard, :restore].each do |event_type|
|
||||
expect { record(event_type) }
|
||||
.to raise_error(ArgumentError, "Invalid event_type: #{ event_type }")
|
||||
end
|
||||
end
|
||||
|
||||
it 'records sorted linked internal IDs and later link removal' do
|
||||
it 'records sorted linked internal names and later link removal' do
|
||||
tags = ['z_link', 'a_link'].map { |name| create(:tag, primary_name: name, category: :general) }
|
||||
tags.each { |tag| NicoTagRelation.create!(nico_tag: external_tag, tag:) }
|
||||
|
||||
expect(record(:create).linked_tag_ids).to eq(tags.map(&:id).sort.join(' '))
|
||||
expect(record(:create).linked_tags).to eq('a_link z_link')
|
||||
external_tag.linked_tags = [tags.first]
|
||||
|
||||
expect(record(:update)).to have_attributes(version_no: 2, linked_tag_ids: tags.first.id.to_s)
|
||||
expect(record(:update)).to have_attributes(version_no: 2, linked_tags: 'z_link')
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe StaticMirror do
|
||||
include_context 'Ainu locale'
|
||||
|
||||
let(:tag) { create(:tag, primary_name: 'mirror-public-tag') }
|
||||
let(:post) { create(:post, title: '<img src=x onerror=alert(1)>') }
|
||||
let(:mirror) { described_class.new(locale: Locale.nipponese, origin: 'https://source.test') }
|
||||
|
||||
def snapshot_tag(record, event: 'create', number: 1)
|
||||
TagVersion.create!(tag: record, version_no: number, event_type: event,
|
||||
name: record.name('ja'), category: record.category, aliases: '', parent_tag_ids: '',
|
||||
names: { ja: { name: record.name('ja'), script_code: 'Jpan',
|
||||
auto_generated: false, aliases: [] } }, created_at: Time.current)
|
||||
end
|
||||
|
||||
def snapshot_post(record, tags: [])
|
||||
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
||||
title: record.title, url: record.url, tags: '', tags_json: tags,
|
||||
parent_post_ids: '', created_at: Time.current)
|
||||
end
|
||||
|
||||
# Read STORE entries independently of the production writer and verify CRCs.
|
||||
def archive_files(bytes)
|
||||
offset = 0
|
||||
files = { }
|
||||
while bytes.byteslice(offset, 4).unpack1('V') == 0x04034b50
|
||||
header = bytes.byteslice(offset, 30).unpack('VvvvvvVVVvv')
|
||||
size, name_size, extra_size = header[7], header[9], header[10]
|
||||
path = bytes.byteslice(offset + 30, name_size).force_encoding('UTF-8')
|
||||
data = bytes.byteslice(offset + 30 + name_size + extra_size, size)
|
||||
expect(Zlib.crc32(data)).to eq(header[6])
|
||||
files[path.delete_prefix('archive/')] = data.force_encoding('UTF-8')
|
||||
offset += 30 + name_size + extra_size + size
|
||||
end
|
||||
expect(bytes.byteslice(-22, 4).unpack1('V')).to eq(0x06054b50)
|
||||
files
|
||||
end
|
||||
|
||||
before do
|
||||
snapshot_tag(tag)
|
||||
PostTag.create!(post:, tag:)
|
||||
snapshot_post(post, tags: [{ tag_id: tag.id, version_no: 1, sections: [] }])
|
||||
end
|
||||
|
||||
it 'generates a navigable ZIP, escaped HTML, search data and verified hashes' do
|
||||
progress = []
|
||||
files = archive_files(mirror.export { |done, total| progress << [done, total] })
|
||||
expect(progress.last).to eq([2, 2])
|
||||
expect(files.keys).to include('index.html', 'robots.txt', 'assets/archive.css',
|
||||
'assets/archive.js', 'archive-config.js', 'search-index.json', 'manifest.json',
|
||||
'ja/posts/index.html', "ja/posts/#{ post.id }/index.html",
|
||||
'ja/tags/index.html', "ja/tags/#{ tag.id }/index.html")
|
||||
files.select { |path, _| path.end_with?('.html') }.each do |path, html|
|
||||
document = Nokogiri::HTML(html)
|
||||
expect(document.at_css('meta[name="robots"]')['content']).to include('noindex')
|
||||
expect(document.text).to include('静的保存版', mirror.data_at)
|
||||
expect(document.css('img')).to be_empty
|
||||
expect(document.at_css('[data-original]')['href']).to start_with('https://source.test/ja')
|
||||
document.css('a[href], link[href], script[src]').each do |node|
|
||||
target = node['href'] || node['src']
|
||||
next if target.start_with?('https://')
|
||||
|
||||
expect(target).not_to start_with('/')
|
||||
resolved = Pathname.new(path).dirname.join(target).cleanpath.to_s
|
||||
expect(files).to have_key(resolved)
|
||||
end
|
||||
end
|
||||
detail = Nokogiri::HTML(files.fetch("ja/posts/#{ post.id }/index.html"))
|
||||
expect(detail.at_css('h1').text).to eq(post.title)
|
||||
expect(detail.css('main a').map { _1['href'] })
|
||||
.to include("../../tags/#{ tag.id }/index.html")
|
||||
manifest = JSON.parse(files.fetch('manifest.json'))
|
||||
expect(manifest).to include('format_version' => 1, 'locale' => 'ja',
|
||||
'data_at' => mirror.data_at, 'resources' => { 'posts' => 1, 'tags' => 1 })
|
||||
expect(Time.iso8601(manifest.fetch('generated_at'))).to be >= Time.iso8601(mirror.data_at)
|
||||
expect(manifest.fetch('files').map { _1.fetch('path') })
|
||||
.to match_array(files.keys - ['manifest.json'])
|
||||
manifest.fetch('files').each do |entry|
|
||||
digest = Digest::SHA256.hexdigest(files.fetch(entry.fetch('path')))
|
||||
expect(entry.fetch('sha256')).to eq(digest)
|
||||
end
|
||||
expect(JSON.parse(files.fetch('search-index.json')).map { _1.fetch('path') })
|
||||
.to include("ja/posts/#{ post.id }/index.html", "ja/tags/#{ tag.id }/index.html")
|
||||
end
|
||||
|
||||
it 'keeps private and unversioned records out even for an admin caller' do
|
||||
private_tag = create(:tag, primary_name: 'mirror-secret-tag')
|
||||
snapshot_tag(private_tag)
|
||||
snapshot_tag(private_tag, event: 'discard', number: 2)
|
||||
private_post = create(:post, title: 'mirror-secret-post')
|
||||
snapshot_post(private_post)
|
||||
PostVersion.create!(post: private_post, version_no: 2, event_type: 'discard',
|
||||
url: private_post.url, tags: '', tags_json: [], parent_post_ids: '',
|
||||
created_at: Time.current)
|
||||
legacy = create(:post, title: 'mirror-unversioned-secret')
|
||||
referencing = create(:post, title: 'mirror-secret-reference')
|
||||
snapshot_post(referencing,
|
||||
tags: [{ tag_id: private_tag.id, version_no: 1, sections: [] }])
|
||||
files = HistoryAccess.set(user: create(:user, :admin)) { archive_files(mirror.export) }
|
||||
expect(files.values.join).not_to include('mirror-secret', legacy.title, private_post.url)
|
||||
expect(files.keys).not_to include("ja/tags/#{ private_tag.id }/index.html",
|
||||
"ja/posts/#{ private_post.id }/index.html", "ja/posts/#{ referencing.id }/index.html")
|
||||
expect(mirror.authority.fetch('admin')).to be(false)
|
||||
end
|
||||
|
||||
it 'applies later discard boundaries after a restoration' do
|
||||
snapshot_tag(tag, event: 'discard', number: 2)
|
||||
snapshot_tag(tag, event: 'restore', number: 3)
|
||||
tag.update_columns(version_no: 3)
|
||||
expect(archive_files(mirror.export)).to have_key("ja/tags/#{ tag.id }/index.html")
|
||||
snapshot_tag(tag, event: 'discard', number: 4)
|
||||
second = described_class.new(locale: Locale.nipponese, origin: 'https://source.test')
|
||||
files = archive_files(second.export)
|
||||
expect(files.keys).not_to include("ja/tags/#{ tag.id }/index.html")
|
||||
expect(files.values.join).not_to include('mirror-public-tag')
|
||||
end
|
||||
|
||||
it 'does not include credentials or executable URL schemes' do
|
||||
post.update_columns(url: 'https://secret-user:secret-password@source.test/path')
|
||||
content = archive_files(mirror.export).values.join
|
||||
expect(content).not_to include('secret-password', 'secret-user')
|
||||
post.update_columns(url: 'javascript:alert(1)')
|
||||
expect(archive_files(mirror.export).values.join).not_to include('javascript:')
|
||||
end
|
||||
|
||||
it 'fails if publication changes during generation' do
|
||||
expect do
|
||||
mirror.export do |done, _|
|
||||
snapshot_tag(tag, event: 'discard', number: 2) if done.zero?
|
||||
end
|
||||
end.to raise_error(HistoryAccess::Forbidden)
|
||||
end
|
||||
|
||||
it 'bounds output and never reports hidden population counts' do
|
||||
stub_const('StaticMirror::RESOURCE_LIMIT', 1)
|
||||
second = create(:post, title: 'mirror-beyond-limit')
|
||||
snapshot_post(second)
|
||||
files = archive_files(mirror.export)
|
||||
expect(files.values.join).not_to include(second.title, 'population_count', 'excluded_count')
|
||||
stub_const('StaticMirror::MAX_BYTES', 16)
|
||||
expect { mirror.export }.to raise_error(StaticMirror::TooLarge)
|
||||
end
|
||||
|
||||
it 'keeps all registered language names and aliases without locale filtering' do
|
||||
locale
|
||||
create(:tag_name, :alias, tag:, name: 'mirror-alias', language_code: 'ja', script_code: 'Jpan')
|
||||
files = archive_files(described_class.new(locale:, origin: 'https://source.test').export)
|
||||
detail = files.fetch("ain-Kana/tags/#{ tag.id }/index.html")
|
||||
expect(detail).to include('mirror-public-tag', 'mirror-alias', 'lang="ain"', 'lang="ja"')
|
||||
end
|
||||
|
||||
it 'fails rather than truncating registered aliases at the name limit' do
|
||||
create(:tag_name, :alias, tag:, name: 'mirror-extra-alias')
|
||||
stub_const('StaticMirror::NAME_LIMIT', 1)
|
||||
expect { mirror.export }.to raise_error(StaticMirror::TooLarge)
|
||||
end
|
||||
end
|
||||
-3366
ファイル差分が大きすぎるため省略します
差分を読み込み
+4
-12
@@ -158,15 +158,7 @@ private ID の正誤に応じて許可可否を変えないため、条件の一
|
||||
非公開列を含む列順変更は API の column_values_editable でも不可と通知する。
|
||||
公開列の値を使うソートは引き続き利用できる。
|
||||
明細には元の認可を適用し、抽出条件自体や取得済みの全言語値を表示設定で改変しない。
|
||||
外部タグ履歴の linked_tag_ids は、連携する内部タグの ID を半角スペース区切りで
|
||||
記録する。除票履歴のある内部タグの ID を含むスナップショットを非管理者へ公開しない。
|
||||
名前や UI ロカールを参照 ID の判定に使わない。
|
||||
外部タグ名は履歴に保存せず、external_tags の platform と name を参照する。
|
||||
|
||||
V1 Counts の公開フィールド名は linked_tags を維持し、DB の linked_tag_ids へ対応付ける。
|
||||
明細・フィルタ・集計軸・変更項目・ソート・CSV / JSON 出力で同じ対応付けを使う。
|
||||
既定列と既存の表示ラベルも linked_tags のままとし、値は ID のスペース区切りとする。
|
||||
linked_tag_ids は物理カラム名であり、Counts の公開フィールドとしては受け付けない。
|
||||
旧 URL や保存済み設定のフィールド指定は維持できるが、値にタグ名を指定した検索条件や
|
||||
ドリル条件は ID 値への見直しが必要となる。旧履歴の名称は移行後の ID 列から復元できず、
|
||||
現在の名称からの推定変換も行わない。
|
||||
外部タグ履歴の linked_tags は名前しか記録していないため、非公開タグの記録名を含む
|
||||
スナップショットを保守的に非公開とする。名前全体を空白区切りの境界で照合する。
|
||||
区切りが曖昧な旧データでは過剰な秘匿の可能性はあるが、参照 ID がない旧形式から
|
||||
名前の公開可否を楽観的に推定しない。
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
# Static Mirror PoC (#483)
|
||||
|
||||
現在の公開状態の一部を、Rails で HTML にして ZIP 出力する開発環境専用の PoC。
|
||||
Post は ID・タイトル・URL・出力済み Tag との関連、Tag は ID・カテゴリ・全言語の
|
||||
名前と別名・出力済み Post との関連を出力する。それ以外の Post / Tag 属性は対象外。
|
||||
DB backup / restore とは別機能で、日時指定の復元機能はない。
|
||||
本番環境では全 API が 404 になる。依存パッケージ・DB schema の変更はない。
|
||||
|
||||
## 操作
|
||||
|
||||
既存のローカル開発用 MySQL 設定と locale 登録を用意した環境で、
|
||||
`backend/` の `bin/rails server -p 3002` と、`frontend/` の `npm run dev` を使う。
|
||||
Vite の既存 `/api` proxy は3002番ポートを参照する。これらの起動コマンドは
|
||||
今回実行していない。DB の準備・migration・依存導入も実行していない。
|
||||
|
||||
1. `http://localhost:5173/ja/static-mirror` を開く。開発時の「その他」にもリンクがある。
|
||||
有効な locale では `/:locale/static-mirror` から利用できる。
|
||||
2. 「スナップショット生成」を押す。ログインは不要。通常の CSRF、IP BAN、user BAN
|
||||
の境界を通り、セッションに紐づくジョブを登録する。
|
||||
3. 状態・処理件数・データ基準時点を確認し、完了後「ZIP ダウンロード」を押す。
|
||||
4. ZIP を展開し、`archive/` を通常の静的 HTTP サーバーへ配置する。
|
||||
`/保存版/archive/index.html` のようなサブディレクトリ配置に対応する。
|
||||
DB、Node.js、原サーバー、外部 API は閲覧先では不要。
|
||||
|
||||
一覧・詳細・相互リンクは JavaScript 無効でも利用できる。検索は HTTP 配信した
|
||||
`search-index.json` を JavaScript が読む。`file://` からの検索は保証しない。
|
||||
外部動画・画像・サムネール・素材ファイルは取得も同梱もしない。Post の外部 URL は
|
||||
通常のリンクとして残すため、そのリンク先の閲覧は外部サイトの存続に依存する。
|
||||
|
||||
各 HTML に静的保存版の表示、UTC のデータ基準時点、`noindex, nofollow`、
|
||||
本家への絶対リンクを焼き込む。内部リンク・CSS・JS は相対参照。
|
||||
`archive-config.js` の `window.archiveConfig.origin` を HTTP(S) の本家基点 URL
|
||||
(例: `https://example.org`)に変更すれば本家リンクをまとめて変更できる。
|
||||
設定 JS が削除・欠落・不正な場合や JS 無効時にも HTML の初期リンクが残る。
|
||||
設定は配置先ミラーの URL ではなく、本家の基点を指す。
|
||||
|
||||
## API と処理上限
|
||||
|
||||
| API | 役割 |
|
||||
| --- | --- |
|
||||
| `GET /static_mirror` | 件数・容量・実行制限と対象外機能 |
|
||||
| `POST /static_mirror/jobs` | 現在状態のジョブ登録。入力は `locale` のみ |
|
||||
| `GET /static_mirror/jobs/:id` | 所有セッションの状態・処理進捗・基準時点・失敗コード |
|
||||
| `GET /static_mirror/files/:id` | 所有セッションの完成 ZIP を再認可して取得 |
|
||||
|
||||
通常どおりフロントエンドの API transport と Query key を使用する。
|
||||
URL に JSON を格納しない。`at` 等の任意時点入力・未知の生成オプションは400で拒否する。
|
||||
|
||||
- Post / Tag は各100件。公開認可後に ID 昇順で選ぶ。非公開件数や全母集団件数は返さない。
|
||||
- 対象 Tag の全言語名・別名は最大2,000行。超過時は失敗させ、黙って切り捨てない。
|
||||
言語・文字体系は保持し、表示対象言語に登録がない場合は ID を使う。
|
||||
- Post と Tag の関連は両者が出力集合にあるものだけ。出力外への内部リンクは生成しない。
|
||||
- 展開後のファイル合計及び ZIP は各8 MiB 以下。超過時は配布しない。
|
||||
- 同時実行1件、全セッションを通じて新規登録は60秒に1件、登録から処理期限60秒。
|
||||
競合・受付制限は429と `Retry-After`。状態取得・取得時の短いロック競合は503。
|
||||
- ZIP とジョブの認可期限は10分。次のジョブ登録で前の ZIP / ジョブは置き換える。
|
||||
期限後も最後のファイルがディスクに残り得るが、API では取得できない。
|
||||
容量は単一 ZIP と作業ファイルに限定され、無制限に蓄積しない。
|
||||
|
||||
Rails の既存 Active Job と `:async` adapter を使い、生成は HTTP request の外で行う。
|
||||
既存 `CountsExportJobs` は Counts の入力・投影・配信確認に結合しているため変更せず、
|
||||
本機能の状態・単一成果物の管理だけを feature 固有の Job に置く。
|
||||
受付・状態・ZIP は `backend/tmp/static_mirror/` の単一 spool を使う。
|
||||
プロセス間の filesystem lock で実行枠と更新を制御する。各ファイルは0600、directory は
|
||||
0700で作成する。Web プロセス再起動で非永続キューが失われる点が PoC の限界。
|
||||
進まない queued / running は75秒後の状態取得で failed になる。再開や再実行はしない。
|
||||
|
||||
失敗コードは `too_large`、`generation_timeout`、`publication_changed`、
|
||||
`generation_failed`、`worker_stopped`。内部の例外本文・パス・資源属性は応答に含めない。
|
||||
画面では失敗コード又は既存の通信・認可エラー文言を表示する。
|
||||
|
||||
## 公開制御と整合性
|
||||
|
||||
生成者の role によらず `HistoryAccess.set(user: nil)` の公開権限を使う。
|
||||
既存の `HistoryAccess.current` と確定済みの同一 `version_no` の存在確認を組み合わせる。
|
||||
独自の role 判定や簡易な除票判定は作らない。対応する immutable revision がない
|
||||
legacy record は安全側で対象外とする。公開不可の参照を含む確定版も既存認可で除外する。
|
||||
廃止 Tag は対象外。HTML / 検索 JSON / manifest の件数には認可済みの出力集合だけが入る。
|
||||
user、actor、履歴版番号、監査、認証、blob 情報は投影しない。
|
||||
|
||||
`CountsExporter.with_read_snapshot` の既存 repeatable-read 境界を再利用し、Post、Tag、
|
||||
全言語 TagName、PostTag、publication revision を同じ読み取り view から取得する。
|
||||
これには既存 HistoryAccess の SQL が動作する MySQL 環境が必要。未対応の adapter や
|
||||
未整備の DB では生成失敗とし、認可を迂回した出力へフォールバックしない。
|
||||
`data_at` はこの現在状態読み取りの開始時刻であり、履歴の指定日時復元を表さない。
|
||||
`generated_at` はファイル生成後の別の時刻。
|
||||
|
||||
生成の最後と状態・ZIP 取得時に既存 artifact policy / discard revision を再検証する。
|
||||
生成中の除票は失敗にし、生成後の除票では古い ZIP の再取得を404にする。
|
||||
ZIP は再認可後のバイト列として返し、遅延 sendfile に古いパスを渡さない。
|
||||
HTTP 応答は既存境界の `private, no-store`。ダウンロード済み ZIP の回収や、
|
||||
最終認可と送出の間に発生する公開制御変更の完全な排他は保証できない。
|
||||
新しい公開制御を導入する場合は共通 policy と artifact revision の連動が必要。
|
||||
|
||||
本文・名前・URL は escape する。外部リンクは HTTP(S)・host・userinfo 不在を検証し、
|
||||
不正な URL はリンクとして出さない。外部 URL の取得は行わないため SSRF 用 DNS 検査を
|
||||
流用しない。HTML の CSP は外部スクリプト・画像・通信を許可しない。
|
||||
|
||||
`manifest.json` に形式版、生成日時、基準時点、locale、出力件数、対象外項目、上限、
|
||||
各ファイルの相対パスと SHA-256 を記録する。自己ハッシュは成立しないため manifest
|
||||
自身だけは一覧の対象外とし、`checksum_scope` にその範囲を記録する。
|
||||
|
||||
## 過去時点復元の調査
|
||||
|
||||
- `post_versions` / `tag_versions` は親 ID と正の `version_no` に一意制約があり、
|
||||
readonly の `VersionRecord` を使う。`created_at <= 時点` の中で最大の確定版を
|
||||
選ぶことは可能。同一時刻の順序は時刻だけで決めず版番号を併用する必要がある。
|
||||
- Tag は `names` に言語別の名前・script・別名を保存する。古い行の移行では
|
||||
日本語として backfill し、旧 aliases 文字列を分割している。失われた区切りや
|
||||
当時未記録の他言語名を現在の TagName で補完して正確と称することはできない。
|
||||
- Post は `tags_json`、区間、Tag 版参照、`parent_post_ids` を、Tag は
|
||||
`parent_tag_ids` を記録する。ただし legacy `tags_json` は名前・時刻からの補完を
|
||||
含む。全時代・全書き込み経路の関係変更の完全性を検証する必要がある。
|
||||
外部 Tag 名の現行参照など、immutable history だけでは揃わない値もある。
|
||||
- Tag Description は `(tag_id, language_code, version_no)` の版に本文と翻訳元を記録する。
|
||||
Material にはファイル情報等の版があるが、blob 本体の永久保存や除票・公開抑止との
|
||||
整合は別途必要。Help はフロントエンドの文書配信に依存し、DB の任意時点だけでは
|
||||
当時の公開版を復元できない。
|
||||
- 当時の publication policy そのものや将来追加される公開抑止の完全な履歴は、
|
||||
これらの内容版だけでは保証できない。指定時点の再現と現在の公開許可を別々に評価し、
|
||||
現在の全後続除票境界を版番号と複合キーで適用する必要がある。恢復しても過去の
|
||||
秘匿履歴を再公開しない。Counts の件数例外はこの機能へ拡張しない。
|
||||
- 正確な時点復元には、全 resource の履歴の起点・欠落・backfill の識別、関係・名前・
|
||||
policy・locale・asset の時点整合性、immutable な Help の版、削除済み asset の保存方針、
|
||||
一つの DB read view と現在の認可の再検証、同時刻更新の順序・境界の設計が必要。
|
||||
|
||||
このため日時入力は実装していない。履歴評価はソース調査であり、実 DB の網羅監査ではない。
|
||||
|
||||
## 検証と残課題
|
||||
|
||||
追加した回帰テスト:
|
||||
|
||||
- `backend/spec/services/static_mirror_spec.rb`: ZIP の CRC / 構造、全ファイルの hash、
|
||||
内部参照、escape、基準時点、公開権限、後続除票、言語名・別名、容量・件数上限。
|
||||
- `backend/spec/requests/static_mirrors_spec.rb`: 一般受付、非同期境界、所有権、
|
||||
公開制御変更、失敗・期限・競合・全体 rate limit、開発環境制限、CSRF と BAN。
|
||||
- `frontend/src/pages/StaticMirrorPage.test.tsx`: 開始、進捗、取得、失敗、
|
||||
受付エラー、認可喪失、日本語 fallback の言語属性。
|
||||
- `frontend/src/lib/staticMirrorAssets.test.ts`: 静的検索の相対参照・escape と
|
||||
サブディレクトリ配備、本家設定の欠落・不正・変更。
|
||||
|
||||
実施したのは既存コードの調査と差分の目視確認のみ。テスト、lint、build、型検査、
|
||||
migration、実ブラウザーでの表示、ZIP 展開・HTTP 配置は実行していない。
|
||||
管理者指定の suite による結果確認と、サブディレクトリ配置・JS 無効・原サーバー停止・
|
||||
設定 JS 欠落・静的検索の実画面確認が残る。
|
||||
|
||||
本番公開には durable な worker / queue、復旧・取消・削除と運用監視、適切な利用者別
|
||||
rate limit、容量・時間・負荷の実測、公開制御の監査と変更検知、長時間配信の再認可方針、
|
||||
asset の権利・保管方針、追加 resource の安全な投影、UI 文言・翻訳の確定が必要。
|
||||
本 PoC の開発環境制限を単に外して公開することは想定しない。
|
||||
|
||||
## 変更ファイルの責務
|
||||
|
||||
| ファイル | 責務 |
|
||||
| --- | --- |
|
||||
| `backend/config/routes.rb` | metadata・job・ZIP の4経路 |
|
||||
| `backend/app/controllers/static_mirrors_controller.rb` | 通常認証境界、開発制限、入力・HTTP 応答 |
|
||||
| `backend/app/jobs/static_mirror_job.rb` | 単一生成枠、受付制限、状態・成果物・期限・取得認可 |
|
||||
| `backend/app/services/static_mirror.rb` | 公開データの一貫した投影、ページ・検索・manifest・ZIP |
|
||||
| `backend/app/views/static_mirror/page.html.erb` | escape 済み HTML、ナビ、基準時点・本家リンク・CSP |
|
||||
| `backend/app/views/static_mirror/archive.css` | 静的サイトの読み取り表示 |
|
||||
| `backend/app/views/static_mirror/archive.js` | ローカル検索と本家リンクの設定変更 |
|
||||
| `frontend/src/App.tsx` | locale 付き操作画面経路 |
|
||||
| `frontend/src/lib/queryKeys.ts` | 既存 Query key entrypoint への追加 |
|
||||
| `frontend/src/pages/StaticMirrorPage.tsx` | 開始・制限・進捗・失敗・ZIP 操作 |
|
||||
| `frontend/src/pages/MorePage.tsx` | 開発時の画面への導線 |
|
||||
| `frontend/src/locales/{ja,ain-Kana}/archive.json` | Issue 指定の文言と未翻訳 fallback |
|
||||
| 上記4テストファイル | 回帰テスト |
|
||||
| `docs/static-mirror.md` | 手順・制約・調査・未検証事項 |
|
||||
@@ -83,7 +83,6 @@ const MaterialSyncSuppressionsPage = lazyPage (
|
||||
() => import ('@/pages/materials/MaterialSyncSuppressionsPage'))
|
||||
const CountsResourcePage = lazyPage (() => import ('@/pages/counts/CountsResourcePage'))
|
||||
const MorePage = lazyPage (() => import ('@/pages/MorePage'))
|
||||
const StaticMirrorPage = lazyPage (() => import ('@/pages/StaticMirrorPage'))
|
||||
const GekanatorUnavailablePage = lazyPage (() => import ('@/pages/GekanatorUnavailablePage'))
|
||||
const NicoTagListPage = lazyPage (() => import ('@/pages/tags/NicoTagListPage'))
|
||||
const TOSPage = lazyPage (() => import ('@/pages/TOSPage'))
|
||||
@@ -127,8 +126,6 @@ export const pageRoutes = (
|
||||
user: User | null,
|
||||
setUser: Dispatch<SetStateAction<User | null>>,
|
||||
): PageRoute[] => [
|
||||
{ path: 'static-mirror', element: <StaticMirrorPage/>,
|
||||
handle: { preload: StaticMirrorPage.preload, robots: 'noindex' } },
|
||||
{ path: '', element: <LocaleRedirect to="/posts"/>,
|
||||
handle: { redirectTo: '/posts' } },
|
||||
{ path: 'posts', element: <PostListPage/>,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
|
||||
import ResourcePickerButton from '@/components/common/ResourcePickerButton'
|
||||
import { useLocale } from '@/lib/useLocale'
|
||||
import PostOriginalCreatedTimeField from '@/components/PostOriginalCreatedTimeField'
|
||||
import FieldError from '@/components/common/FieldError'
|
||||
@@ -184,7 +185,12 @@ const PostEditForm: FC<Props> = ({ post, onSave }) => {
|
||||
value={parentPostIds}
|
||||
disabled={disabled}
|
||||
errors={fieldErrors.parentPostIds}
|
||||
onChange={setParentPostIds}/>
|
||||
onChange={setParentPostIds}
|
||||
after={<ResourcePickerButton
|
||||
source={{ resource: 'posts', field: 'resource_id' }}
|
||||
values={parentPostIds.split (/\s+/).filter (Boolean)}
|
||||
maximum={Number.MAX_SAFE_INTEGER} disabled={disabled}
|
||||
onSelect={values => setParentPostIds (values.join (' '))}/>}/>
|
||||
|
||||
<PostTagsField
|
||||
disabled={disabled}
|
||||
|
||||
+46
-31
@@ -10,16 +10,19 @@ import { useUITranslation } from '@/lib/useUITranslation'
|
||||
import { inputClass } from '@/lib/utils'
|
||||
|
||||
import type { DialogueFormControls } from '@/lib/dialogues/useDialogue'
|
||||
import type { CountsCandidate, CountsDefinition, CountsFieldMetadata }
|
||||
from '@/lib/countsDefinitions'
|
||||
import type { CountsResource } from '@/lib/counts'
|
||||
import type { CountsCandidate } from '@/lib/countsDefinitions'
|
||||
|
||||
|
||||
const CountsCandidatePicker = (
|
||||
{ field, definition, params, initial, maximum, controls, save }: {
|
||||
field: CountsFieldMetadata; params: URLSearchParams
|
||||
definition: CountsDefinition
|
||||
type CandidateSource = {
|
||||
resource: CountsResource; field: string; params?: URLSearchParams }
|
||||
|
||||
const CandidatePicker = (
|
||||
{ source, choices = [], categories = [], initial, maximum, controls, onSelect }: {
|
||||
source?: CandidateSource; choices?: CountsCandidate[]
|
||||
categories?: { value: string; label: string }[]
|
||||
initial: (string | number | boolean)[]; maximum: number
|
||||
controls: DialogueFormControls; save: (values: string[]) => void },
|
||||
controls: DialogueFormControls; onSelect: (values: string[]) => void },
|
||||
) => {
|
||||
const { label, message } = useUITranslation ('counts')
|
||||
const locale = useLocale ()
|
||||
@@ -29,49 +32,58 @@ const CountsCandidatePicker = (
|
||||
const [search, setSearch] = useState ('')
|
||||
const [category, setCategory] = useState ('')
|
||||
const [page, setPage] = useState (1)
|
||||
const params = source?.params ?? new URLSearchParams ()
|
||||
const point = useRef (params.get ('execution_at'))
|
||||
const request = new URLSearchParams ({ field: field.key, query: search, page: String (page) })
|
||||
const resource = source?.resource ?? 'posts'
|
||||
const request = new URLSearchParams ({
|
||||
field: source?.field ?? '', query: search, page: String (page) })
|
||||
for (const key of ['timezone', 'at', 'execution_at'])
|
||||
if (params.has (key)) request.set (key, params.get (key)!)
|
||||
if (countsUseLatest (params)) request.delete ('at')
|
||||
if (category) request.set ('category', category)
|
||||
const dynamic = field.candidateSource === 'tags'
|
||||
const dynamic = source != null
|
||||
const { data, isFetching: fetching, isError: errorFlg } = useQuery ({
|
||||
queryKey: countsKeys.candidates (request.toString (), locale), enabled: dynamic,
|
||||
queryKey: countsKeys.candidates (resource, request.toString (), locale), enabled: dynamic,
|
||||
queryFn: ({ signal }) => {
|
||||
const pinned = new URLSearchParams (request)
|
||||
if (point.current != null) pinned.set ('execution_at', point.current)
|
||||
return fetchCountsCandidates (pinned, locale, signal)
|
||||
}, retry: false })
|
||||
return fetchCountsCandidates (resource, pinned, locale, signal)
|
||||
}, retry: false, staleTime: 0, gcTime: 0 })
|
||||
useEffect (() => {
|
||||
if (data != null) point.current = data.executionAt
|
||||
}, [data])
|
||||
useEffect (() => {
|
||||
controls.setActions ([{ label: label ('counts:pattern.selection'),
|
||||
onSelect: () => save ([...selected]) }])
|
||||
}, [controls, label, save, selected])
|
||||
const fixed: CountsCandidate[] = (field.choices ?? []).map (choice => ({
|
||||
value: choice.value, name: message (choice.label).text }))
|
||||
const rows = dynamic ? data?.rows ?? [] : fixed.filter (row =>
|
||||
onSelect: () => onSelect ([...selected]) }])
|
||||
}, [controls, label, onSelect, selected])
|
||||
const rows = dynamic ? data?.rows ?? [] : choices.filter (row =>
|
||||
row.name?.includes (search))
|
||||
const headings: Record<string, string> = {
|
||||
id: 'columns.resourceId', name: 'columns.name', category: 'common:categoryLabel',
|
||||
kind: 'columns.kind' }
|
||||
const available = dynamic ? ['id', 'name', 'category', 'kind'] : ['name']
|
||||
const referenceFlg = ['tags', 'tags_json', 'tags.id', 'members'].includes (source?.field ?? '')
|
||||
const available =
|
||||
dynamic
|
||||
? (referenceFlg ? ['id', 'name', 'category', 'kind'] : ['id', 'name']) : ['name']
|
||||
const visible = available.filter (column => columns.includes (column))
|
||||
const categories = definition.filterFields?.find (field =>
|
||||
field.candidateSource === 'categories')?.choices ?? []
|
||||
const cell = (row: CountsCandidate, column: string) => {
|
||||
if (column === 'category') return row.category == null ? ''
|
||||
: message (`common:categories.${row.category}`).text
|
||||
if (column === 'kind') return row.kind == null ? '' : message (`kinds.${row.kind}`).text
|
||||
if (column === 'category')
|
||||
return row.category == null ? '' : message (`common:categories.${row.category}`).text
|
||||
if (column === 'kind')
|
||||
{
|
||||
const key = row.kind === 'group' ? 'resources.tag_groups' : `kinds.${row.kind}`
|
||||
return row.kind == null ? '' : message (key).text
|
||||
}
|
||||
return column === 'id' ? row.id : row.name
|
||||
}
|
||||
return <div className="min-w-0 space-y-2">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<input className={inputClass (false, 'h-9 min-w-0 flex-1')}
|
||||
aria-label={message ('common:search').text} value={draft}
|
||||
onChange={event => setDraft (event.target.value)}
|
||||
aria-label={message ('common:search').text} value={draft} maxLength={1000}
|
||||
onChange={event => {
|
||||
const value = event.target.value
|
||||
if (new TextEncoder ().encode (value).length <= 1000) setDraft (value)
|
||||
}}
|
||||
onKeyDown={event => {
|
||||
if (event.key !== 'Enter') return
|
||||
event.preventDefault ()
|
||||
@@ -81,7 +93,7 @@ const CountsCandidatePicker = (
|
||||
<Button type="button" onClick={() => { setSearch (draft); setPage (1) }}>
|
||||
{label ('common:search')}</Button>
|
||||
</div>
|
||||
{dynamic && <select className={inputClass (false, 'h-9 w-full')}
|
||||
{categories.length > 0 && <select className={inputClass (false, 'h-9 w-full')}
|
||||
aria-label={message ('common:categoryLabel').text} value={category}
|
||||
onChange={event => { setCategory (event.target.value); setPage (1) }}>
|
||||
<option value=""/>
|
||||
@@ -108,12 +120,13 @@ const CountsCandidatePicker = (
|
||||
{visible.map (column => <th key={column} className="p-2">
|
||||
{label (headings[column])}</th>)}
|
||||
</tr></thead>
|
||||
<tbody>{rows.map (row => <tr key={row.value} className="border-t dark:border-slate-700">
|
||||
<tbody>{rows.map (row => <tr key={`${row.kind ?? ''}:${row.value}`}
|
||||
className="border-t dark:border-slate-700">
|
||||
<td className="p-2"><input type="checkbox" checked={selected.has (row.value)}
|
||||
aria-label={row.name ?? String (row.id ?? '')}
|
||||
disabled={!selected.has (row.value) && selected.size >= maximum}
|
||||
disabled={maximum !== 1 && !selected.has (row.value) && selected.size >= maximum}
|
||||
onChange={event => setSelected (previous => {
|
||||
const next = new Set (previous)
|
||||
const next = new Set (maximum === 1 ? [] : previous)
|
||||
if (event.target.checked) next.add (row.value)
|
||||
else next.delete (row.value)
|
||||
return next
|
||||
@@ -128,7 +141,8 @@ const CountsCandidatePicker = (
|
||||
onClick={() => setPage (previous => previous - 1)}>
|
||||
{label ('common:pagination.previous')}</Button>
|
||||
<span>{page}</span>
|
||||
<Button type="button" variant="outline" disabled={!data?.more || fetching}
|
||||
<Button type="button" variant="outline"
|
||||
disabled={!data?.more || page >= 100_000 || fetching}
|
||||
onClick={() => setPage (previous => previous + 1)}>
|
||||
{label ('common:pagination.next')}</Button>
|
||||
</div>}
|
||||
@@ -136,4 +150,5 @@ const CountsCandidatePicker = (
|
||||
}
|
||||
|
||||
|
||||
export default CountsCandidatePicker
|
||||
export default CandidatePicker
|
||||
export type { CandidateSource }
|
||||
@@ -0,0 +1,27 @@
|
||||
import CandidatePicker from '@/components/common/CandidatePicker'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import useDialogue from '@/lib/dialogues/useDialogue'
|
||||
import { useUITranslation } from '@/lib/useUITranslation'
|
||||
|
||||
import type { CandidateSource } from '@/components/common/CandidatePicker'
|
||||
|
||||
|
||||
const ResourcePickerButton = (
|
||||
{ source, values, maximum = 1, disabled, onSelect }: {
|
||||
source: CandidateSource; values: (string | number | boolean)[]
|
||||
maximum?: number; disabled?: boolean
|
||||
onSelect: (values: string[]) => void },
|
||||
) => {
|
||||
const dialogue = useDialogue ()
|
||||
const { label } = useUITranslation ('counts')
|
||||
return <Button type="button" variant="outline" disabled={disabled || maximum === 0}
|
||||
onClick={() => void dialogue.form ({
|
||||
title: label ('pattern.link'), size: 'large', footerLayout: 'inline',
|
||||
body: controls => <CandidatePicker source={source} initial={values}
|
||||
maximum={maximum} controls={controls} onSelect={onSelect}/> })}>
|
||||
{label ('pattern.selection')}
|
||||
</Button>
|
||||
}
|
||||
|
||||
|
||||
export default ResourcePickerButton
|
||||
@@ -2,9 +2,10 @@ import { useEffect, useState } from 'react'
|
||||
|
||||
import CountsConditionValues from '@/components/counts/CountsConditionValues'
|
||||
import DateTimeField from '@/components/common/DateTimeField'
|
||||
import ResourcePickerButton from '@/components/common/ResourcePickerButton'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { countsConditionInputs, countsEventTypes, countsFieldLabel, countsFilterFields,
|
||||
countsSetOperators, useCountsDefinition }
|
||||
import { countsCandidateSource, countsConditionInputs, countsEventTypes, countsFieldLabel,
|
||||
countsFilterFields, countsSetOperators, useCountsDefinition }
|
||||
from '@/lib/countsDefinitions'
|
||||
import useDialogue from '@/lib/dialogues/useDialogue'
|
||||
import { countsConditionValue } from '@/lib/countsDisplay'
|
||||
@@ -15,7 +16,7 @@ import { useUITranslation } from '@/lib/useUITranslation'
|
||||
|
||||
import { readCountState, encodeCountState } from '@/lib/queryState'
|
||||
|
||||
import type { CountsResult } from '@/lib/counts'
|
||||
import type { CountsResource, CountsResult } from '@/lib/counts'
|
||||
import type { DialogueFormControls } from '@/lib/dialogues/useDialogue'
|
||||
import type { CountsCondition, CountsDefinition } from '@/lib/countsDefinitions'
|
||||
|
||||
@@ -94,10 +95,12 @@ const ConditionEditor = (
|
||||
}
|
||||
|
||||
|
||||
const CountsAdditionalConditions = ({ params, update, conditionLabels }: {
|
||||
conditionLabels?: CountsResult['conditionLabels']
|
||||
params: URLSearchParams
|
||||
update: (key: string, value: string) => void },
|
||||
const CountsAdditionalConditions = (
|
||||
{ resource, params, update, conditionLabels }: {
|
||||
resource: CountsResource
|
||||
conditionLabels?: CountsResult['conditionLabels']
|
||||
params: URLSearchParams
|
||||
update: (key: string, value: string) => void },
|
||||
) => {
|
||||
const { label, message } = useUITranslation ('counts')
|
||||
const definition = useCountsDefinition ()
|
||||
@@ -149,11 +152,19 @@ const CountsAdditionalConditions = ({ params, update, conditionLabels }: {
|
||||
value={condition[key] == null ? '' : String (condition[key])} disabled={condition.blank}
|
||||
aria-label={aria.text} lang={aria.language}
|
||||
onChange={value => patch (index, { [key]: value ?? '' })}/>
|
||||
return <input value={condition[key] == null ? '' : String (condition[key])} disabled={condition.blank}
|
||||
type={(type === 'integer' || type === 'interval') ? 'number' : 'text'}
|
||||
className={inputClass (false, 'h-10 w-36 min-w-0 max-w-full py-1')}
|
||||
aria-label={aria.text} lang={aria.language}
|
||||
onChange={event => patch (index, { [key]: event.target.value })}/>
|
||||
const source = countsCandidateSource (resource, condition.field, params)
|
||||
const value = condition[key]
|
||||
return <>
|
||||
<input value={condition[key] == null ? '' : String (condition[key])}
|
||||
disabled={condition.blank}
|
||||
type={(type === 'integer' || type === 'interval') ? 'number' : 'text'}
|
||||
className={inputClass (false, 'h-10 w-36 min-w-0 max-w-full py-1')}
|
||||
aria-label={aria.text} lang={aria.language}
|
||||
onChange={event => patch (index, { [key]: event.target.value })}/>
|
||||
{source != null && <ResourcePickerButton source={source} disabled={condition.blank}
|
||||
values={value == null ? [] : [value]}
|
||||
onSelect={values => patch (index, { [key]: values[0] ?? '' })}/>}
|
||||
</>
|
||||
}
|
||||
return <fieldset className="min-w-0 space-y-1">
|
||||
<legend className="flex items-center gap-2">{label ('additionalConditions')}
|
||||
@@ -182,7 +193,7 @@ const CountsAdditionalConditions = ({ params, update, conditionLabels }: {
|
||||
</Button>
|
||||
{countsSetOperators.includes (condition.operator) ? (
|
||||
<CountsConditionValues key={`${condition.field}:${condition.operator}`}
|
||||
condition={condition}
|
||||
condition={condition} resource={resource}
|
||||
field={metadata.find (field => field.key === condition.field)!}
|
||||
definition={definition} params={params}
|
||||
maximum={Math.max (0, 100 - conditions.reduce ((sum, item, position) =>
|
||||
|
||||
@@ -1,18 +1,22 @@
|
||||
import { useState } from 'react'
|
||||
|
||||
import CountsCandidatePicker from '@/components/counts/CountsCandidatePicker'
|
||||
import CandidatePicker from '@/components/common/CandidatePicker'
|
||||
import DateTimeField from '@/components/common/DateTimeField'
|
||||
import ResourcePickerButton from '@/components/common/ResourcePickerButton'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { countsCandidateSource } from '@/lib/countsDefinitions'
|
||||
import useDialogue from '@/lib/dialogues/useDialogue'
|
||||
import { inputClass } from '@/lib/utils'
|
||||
import { useUITranslation } from '@/lib/useUITranslation'
|
||||
|
||||
import type { CountsResource } from '@/lib/counts'
|
||||
import type { CountsCondition, CountsDefinition, CountsFieldMetadata }
|
||||
from '@/lib/countsDefinitions'
|
||||
|
||||
|
||||
const CountsConditionValues = (
|
||||
{ condition, field, definition, params, maximum, change }: {
|
||||
{ resource, condition, field, definition, params, maximum, change }: {
|
||||
resource: CountsResource
|
||||
condition: CountsCondition; field: CountsFieldMetadata; definition: CountsDefinition
|
||||
params: URLSearchParams; maximum: number
|
||||
change: (values: (string | number | boolean)[]) => void },
|
||||
@@ -21,11 +25,21 @@ const CountsConditionValues = (
|
||||
const dialogue = useDialogue ()
|
||||
const [draft, setDraft] = useState ('')
|
||||
const values = condition.values ?? []
|
||||
const source = countsCandidateSource (resource, field.key, params)
|
||||
const pick = () => {
|
||||
void dialogue.form ({ title: label (field.candidateSource === 'tags'
|
||||
? 'counts:pattern.link' : 'counts:pattern.pattern'), size: 'large', footerLayout: 'inline',
|
||||
body: controls => <CountsCandidatePicker field={field} definition={definition}
|
||||
params={params} initial={values} maximum={maximum} controls={controls} save={change}/> })
|
||||
body: controls => <CandidatePicker
|
||||
source={
|
||||
field.candidateSource === 'tags'
|
||||
? { resource: 'posts', field: field.key, params } : undefined}
|
||||
choices={(field.choices ?? []).map (choice => ({
|
||||
value: choice.value, name: message (choice.label).text }))}
|
||||
categories={
|
||||
field.candidateSource === 'tags'
|
||||
? definition.filterFields?.find (field =>
|
||||
field.candidateSource === 'categories')?.choices : undefined}
|
||||
initial={values} maximum={maximum} controls={controls} onSelect={change}/> })
|
||||
}
|
||||
if (condition.input === 'list' || field.candidateSource === 'tags')
|
||||
return <Button type="button" variant="outline" disabled={condition.blank} onClick={pick}>
|
||||
@@ -73,6 +87,8 @@ const CountsConditionValues = (
|
||||
onClick={() => change (values.filter ((_, position) => position !== index))}>×</Button>
|
||||
</span>)}
|
||||
{input ()}
|
||||
{source != null && <ResourcePickerButton source={source} values={values}
|
||||
maximum={maximum} disabled={condition.blank} onSelect={change}/>}
|
||||
<Button type="button" variant="outline" disabled={condition.blank || draft === ''
|
||||
|| values.length >= maximum} onClick={append}>{label ('add')}</Button>
|
||||
<Button type="button" variant="ghost" disabled={condition.blank}
|
||||
|
||||
@@ -7,6 +7,7 @@ import AnimatedCollapse from '@/components/common/AnimatedCollapse'
|
||||
import DateTimeField from '@/components/common/DateTimeField'
|
||||
import FormField from '@/components/common/FormField'
|
||||
import HelpPopover from '@/components/common/HelpPopover'
|
||||
import ResourcePickerButton from '@/components/common/ResourcePickerButton'
|
||||
import TagInput from '@/components/common/TagInput'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { countsRequestParams, countsUseLatest, fieldsForCounts, normaliseCountsParams, readCountSort,
|
||||
@@ -223,11 +224,17 @@ const CountsControls = (
|
||||
{['resource_id', 'user_id'].map (key => <FormField key={key}
|
||||
label={label (key === 'resource_id' ? 'columns.resourceId'
|
||||
: 'fields.actor')}>
|
||||
{() => <input value={draft.get (key) ?? ''}
|
||||
aria-label={message (key === 'resource_id' ? 'columns.resourceId'
|
||||
: 'fields.actor').text}
|
||||
className={inputClass (false, 'w-28 py-1')}
|
||||
onChange={event => update (key, event.target.value)}/>}
|
||||
{() => <div className="flex flex-wrap items-center gap-1">
|
||||
<input value={draft.get (key) ?? ''}
|
||||
aria-label={message (key === 'resource_id' ? 'columns.resourceId'
|
||||
: 'fields.actor').text}
|
||||
className={inputClass (false, 'w-28 py-1')}
|
||||
onChange={event => update (key, event.target.value)}/>
|
||||
<ResourcePickerButton source={{ resource,
|
||||
field: key === 'resource_id' ? key : 'user', params: draft }}
|
||||
values={draft.get (key) ? [draft.get (key)!] : []}
|
||||
onSelect={values => update (key, values[0] ?? '')}/>
|
||||
</div>}
|
||||
</FormField>)}
|
||||
</div>
|
||||
{resource === 'posts' && <FormField label={label ('columns.tags')}>
|
||||
@@ -263,7 +270,7 @@ const CountsControls = (
|
||||
</>}
|
||||
</div>}
|
||||
</FormField>}
|
||||
<CountsAdditionalConditions params={draft} update={update}
|
||||
<CountsAdditionalConditions resource={resource} params={draft} update={update}
|
||||
conditionLabels={(labelParams != null
|
||||
&& draft.get ('current') === labelParams.get ('current')
|
||||
&& draft.get ('at') === labelParams.get ('at')
|
||||
|
||||
@@ -227,11 +227,13 @@ const DialogueProvider: FC<Props> = ({ children }) => {
|
||||
</DialogHeader>
|
||||
|
||||
<div className="min-h-0 flex-1 overflow-y-auto overscroll-contain">
|
||||
{active.options.body (
|
||||
formControls.current[active.id] ??= {
|
||||
<DialogueProvider key={active.id}>
|
||||
{active.options.body (
|
||||
formControls.current[active.id] ??= {
|
||||
close: () => closeRequest (active.id),
|
||||
setActions: actions => setRequestActions (active.id, actions),
|
||||
confirm: options => openNestedConfirm (active.id, options) })}
|
||||
</DialogueProvider>
|
||||
</div>
|
||||
|
||||
<DialogFooter
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import ResourcePickerButton from '@/components/common/ResourcePickerButton'
|
||||
import PostOriginalCreatedTimeField from '@/components/PostOriginalCreatedTimeField'
|
||||
import PostTagsField from '@/components/posts/PostTagsField'
|
||||
import PostTextField from '@/components/posts/PostTextField'
|
||||
@@ -88,7 +89,12 @@ const PostCoreDataFields: FC<PostCoreDataFieldsProps> = (
|
||||
disabled={parentPostIds.disabled}
|
||||
warnings={parentPostIds.warnings}
|
||||
errors={parentPostIds.errors}
|
||||
onChange={parentPostIds.onChange}/>
|
||||
onChange={parentPostIds.onChange}
|
||||
after={<ResourcePickerButton
|
||||
source={{ resource: 'posts', field: 'resource_id' }}
|
||||
values={parentPostIds.value.split (/\s+/).filter (Boolean)}
|
||||
maximum={Number.MAX_SAFE_INTEGER} disabled={parentPostIds.disabled}
|
||||
onSelect={values => parentPostIds.onChange (values.join (' '))}/>}/>
|
||||
</>)
|
||||
|
||||
export default PostCoreDataFields
|
||||
|
||||
@@ -34,18 +34,22 @@ const PostTextField: FC<Props> = (
|
||||
<FormField label={label} messages={errors}>
|
||||
{({ describedBy, invalid }) => (
|
||||
<>
|
||||
<input
|
||||
type={type}
|
||||
value={value}
|
||||
disabled={disabled}
|
||||
placeholder={placeholder}
|
||||
onBlur={onBlur}
|
||||
onChange={ev => onChange (ev.target.value)}
|
||||
aria-describedby={describedBy}
|
||||
aria-invalid={invalid}
|
||||
className={inputClass (invalid, className)}/>
|
||||
<FieldWarning messages={warnings}/>
|
||||
{after}
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="min-w-0 flex-1">
|
||||
<input
|
||||
type={type}
|
||||
value={value}
|
||||
disabled={disabled}
|
||||
placeholder={placeholder}
|
||||
onBlur={onBlur}
|
||||
onChange={ev => onChange (ev.target.value)}
|
||||
aria-describedby={describedBy}
|
||||
aria-invalid={invalid}
|
||||
className={inputClass (invalid, className)}/>
|
||||
</div>
|
||||
{after}
|
||||
</div>
|
||||
<FieldWarning messages={warnings}/>
|
||||
</>)}
|
||||
</FormField>)
|
||||
|
||||
|
||||
@@ -89,18 +89,19 @@ export const fetchCountsDefinition = (
|
||||
.then (definition => localiseCountsDefinition (definition, locale))
|
||||
|
||||
export type CountsCandidate = {
|
||||
value: string; id?: number; name: string | null; category?: string | null
|
||||
value: string; id?: number | string; name: string | null; category?: string | null
|
||||
names?: Record<string, unknown>
|
||||
kind?: 'internal' | 'external' }
|
||||
kind?: 'internal' | 'external' | 'group' }
|
||||
export type CountsCandidatesResult = {
|
||||
rows: CountsCandidate[]; page: number; more: boolean; executionAt: string }
|
||||
|
||||
export const fetchCountsCandidates = (
|
||||
resource: CountsResource,
|
||||
params: URLSearchParams,
|
||||
locale: LocaleCode,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CountsCandidatesResult> => apiGet (
|
||||
'/counts/definitions/posts/candidates', {
|
||||
`/counts/definitions/${encodeURIComponent (resource)}/candidates`, {
|
||||
params: Object.fromEntries (params), locale, signal })
|
||||
|
||||
export const countsSetOperators = ['in', 'not_in', 'any', 'all', 'none']
|
||||
@@ -155,3 +156,25 @@ export const countsAxisFields = (
|
||||
history: boolean,
|
||||
): CountField[] => definition.axes[history ? 'history' : 'latest']
|
||||
.map (key => ({ key, label: countsFieldLabel (key) }))
|
||||
|
||||
|
||||
export const countsCandidateSource = (
|
||||
resource: CountsResource,
|
||||
field: string,
|
||||
params: URLSearchParams,
|
||||
): { resource: CountsResource; field: string; params: URLSearchParams } | undefined => {
|
||||
if (['resource_id', 'user', 'updated_by_user_id', 'file_blob_id', 'source_file_id']
|
||||
.includes (field)) return { resource, field, params }
|
||||
if (field === 'user.id') return { resource, field: 'user', params }
|
||||
if ((field === 'tags.id' && resource === 'posts')
|
||||
|| (field === 'members' && resource === 'tag_groups'))
|
||||
return { resource, field, params }
|
||||
const leaf = field.split ('.').at (-1) ?? ''
|
||||
const targets: Record<string, CountsResource> = {
|
||||
tagGroupId: 'tag_groups', tagId: 'tags', tag_id: 'tags', parent_id: 'materials',
|
||||
parents: resource === 'posts' ? 'posts' : 'tags',
|
||||
'parents.id': resource === 'posts' ? 'posts' : 'tags',
|
||||
parent_post_ids: 'posts', parent_tag_ids: 'tags' }
|
||||
const target = targets[field] ?? targets[leaf]
|
||||
return target == null ? undefined : { resource: target, field: 'resource_id', params }
|
||||
}
|
||||
|
||||
@@ -7,10 +7,6 @@ import type {
|
||||
MaterialFilter,
|
||||
WikiTagReference } from '@/types'
|
||||
|
||||
export const staticMirrorKeys = {
|
||||
definition: ['static_mirror', 'definition'] as const,
|
||||
job: (id: string | null) => ['static_mirror', 'jobs', id] as const }
|
||||
|
||||
export const localesKeys = {
|
||||
root: ['locales'] as const,
|
||||
index: ['locales', 'index'] as const }
|
||||
@@ -178,6 +174,7 @@ export const theatresKeys = {
|
||||
export const countsKeys = {
|
||||
definition: (resource: string, locale: LocaleCode = 'ja') =>
|
||||
['counts', 'definition', resource, locale] as const,
|
||||
candidates: (params: string, locale: LocaleCode) => ['counts', 'candidates', locale, params] as const,
|
||||
candidates: (resource: string, params: string, locale: LocaleCode) =>
|
||||
['counts', 'candidates', resource, locale, params] as const,
|
||||
index: (resource: string, params: string, locale: LocaleCode) =>
|
||||
['counts', resource, locale, params] as const }
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { URL as NodeURL } from 'node:url'
|
||||
import { fireEvent, waitFor } from '@testing-library/react'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const script = readFileSync (new NodeURL (
|
||||
'../../../backend/app/views/static_mirror/archive.js', import.meta.url), 'utf8')
|
||||
const archiveWindow = window as typeof window & { archiveConfig?: { origin: string | null } }
|
||||
|
||||
beforeEach (() => {
|
||||
const element = document.createElement ('script')
|
||||
element.src = 'https://mirror.test/subdirectory/archive/assets/archive.js'
|
||||
vi.spyOn (document, 'currentScript', 'get').mockReturnValue (element)
|
||||
delete archiveWindow.archiveConfig
|
||||
document.body.innerHTML = `
|
||||
<nav><a href="ja/posts/index.html">Post</a>
|
||||
<a data-original="/ja/posts/1" href="https://source.test/ja/posts/1">Source</a></nav>
|
||||
<form id="archive-search"><input name="q" value="alias"></form>
|
||||
<ul id="search-results"></ul>`
|
||||
})
|
||||
afterEach (() => {
|
||||
delete archiveWindow.archiveConfig
|
||||
document.body.replaceChildren ()
|
||||
vi.unstubAllGlobals ()
|
||||
})
|
||||
|
||||
describe ('Portable static mirror assets', () => {
|
||||
it ('searches local JSON by aliases using safe text and relative links', async () => {
|
||||
const fetcher = vi.fn ().mockResolvedValue ({ ok: true, json: async () => [
|
||||
{ path: 'ja/tags/1/index.html', title: '<img src=x onerror=alert(1)>', names: ['alias'] },
|
||||
{ path: 'https://outside.test/', title: 'alias' },
|
||||
{ path: '../outside.html', title: 'alias' }] })
|
||||
vi.stubGlobal ('fetch', fetcher)
|
||||
new Function (script) ()
|
||||
fireEvent.submit (document.querySelector ('form')!)
|
||||
await waitFor (() => expect (document.querySelectorAll ('#search-results a')).toHaveLength (1))
|
||||
const link = document.querySelector ('#search-results a')!
|
||||
expect (link).toHaveAttribute ('href', 'ja/tags/1/index.html')
|
||||
expect (link.textContent).toBe ('<img src=x onerror=alert(1)>')
|
||||
expect (document.querySelector ('img')).toBeNull ()
|
||||
expect (fetcher).toHaveBeenCalledTimes (1)
|
||||
expect (fetcher.mock.calls[0][0].href)
|
||||
.toBe ('https://mirror.test/subdirectory/archive/search-index.json')
|
||||
})
|
||||
|
||||
it ('retains baked links without config and changes only source links with valid config', () => {
|
||||
new Function (script) ()
|
||||
const source = document.querySelector ('[data-original]')!
|
||||
expect (source).toHaveAttribute ('href', 'https://source.test/ja/posts/1')
|
||||
archiveWindow.archiveConfig = { origin: 'javascript:alert(1)' }
|
||||
new Function (script) ()
|
||||
expect (source).toHaveAttribute ('href', 'https://source.test/ja/posts/1')
|
||||
archiveWindow.archiveConfig = { origin: 'https://replacement.test/base/' }
|
||||
new Function (script) ()
|
||||
expect (source).toHaveAttribute ('href', 'https://replacement.test/base/ja/posts/1')
|
||||
expect (document.querySelector ('a:not([data-original])'))
|
||||
.toHaveAttribute ('href', 'ja/posts/index.html')
|
||||
})
|
||||
})
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"_untranslated": ["*"],
|
||||
"title": "Static Mirror 生成機能",
|
||||
"generation": "スナップショット生成",
|
||||
"dataAt": "データ基準時点",
|
||||
"currentOnly": "今回は現在状態のみの対応",
|
||||
"developmentOnly": "開発環境限定",
|
||||
"limits": "出力件数の制限",
|
||||
"concurrency": "同時実行数",
|
||||
"exclusions": "未対応機能",
|
||||
"protection": "データ保護上の制約",
|
||||
"protectionDetails": "公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。",
|
||||
"externalAssets": "外部画像や動画への依存",
|
||||
"externalMedia": "外部画像や動画",
|
||||
"externalOmissions": "ローカルへ取り込めていない資産",
|
||||
"download": "ZIP ダウンロード",
|
||||
"zip": "ZIP",
|
||||
"failure": "失敗時の理由"
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"title": "Static Mirror 生成機能",
|
||||
"generation": "スナップショット生成",
|
||||
"dataAt": "データ基準時点",
|
||||
"currentOnly": "今回は現在状態のみの対応",
|
||||
"developmentOnly": "開発環境限定",
|
||||
"limits": "出力件数の制限",
|
||||
"concurrency": "同時実行数",
|
||||
"exclusions": "未対応機能",
|
||||
"protection": "データ保護上の制約",
|
||||
"protectionDetails": "公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。",
|
||||
"externalAssets": "外部画像や動画への依存",
|
||||
"externalMedia": "外部画像や動画",
|
||||
"externalOmissions": "ローカルへ取り込めていない資産",
|
||||
"download": "ZIP ダウンロード",
|
||||
"zip": "ZIP",
|
||||
"failure": "失敗時の理由"
|
||||
}
|
||||
@@ -16,17 +16,12 @@ import type { User } from '@/types'
|
||||
const MorePage: FC = () => {
|
||||
const location = useLocation ()
|
||||
const { message } = useUITranslation ('shared')
|
||||
const { label: archiveLabel } = useUITranslation ('archive')
|
||||
const menu = menuOutline (
|
||||
{ tag: null, wikiId: null, user: { } as User, pathName: location.pathname })
|
||||
|
||||
return (
|
||||
<MainArea className="md:flex">
|
||||
<PageHead title={message ('labels.menu')}/>
|
||||
{import.meta.env.DEV && (
|
||||
<PrefetchLink to="/static-mirror">
|
||||
{archiveLabel ('title')}
|
||||
</PrefetchLink>)}
|
||||
|
||||
{[...Array (Math.ceil (menu.length / 4)).keys ()].map (i => (
|
||||
<div key={i} className="flex-1 mx-16">
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
import { act, fireEvent, screen, waitFor } from '@testing-library/react'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import StaticMirrorPage from '@/pages/StaticMirrorPage'
|
||||
import { staticMirrorKeys } from '@/lib/queryKeys'
|
||||
import { renderWithProviders } from '@/test/render'
|
||||
|
||||
const api = vi.hoisted (() => ({
|
||||
apiGet: vi.fn (), apiPost: vi.fn (), startAPIDownload: vi.fn () }))
|
||||
vi.mock ('@/lib/api', async original => ({
|
||||
...await original<typeof import ('@/lib/api')> (), ...api }))
|
||||
|
||||
const definition = {
|
||||
resourceLimit: 100, nameLimit: 2000, maxBytes: 8388608, maxSeconds: 60,
|
||||
concurrency: 1, cooldownSeconds: 60, expirySeconds: 600,
|
||||
exclusions: ['historical_state', 'unversioned_resources', 'images', 'videos'] }
|
||||
const ready = {
|
||||
status: 'ready', dataAt: '2026-11-16T00:00:00.000000Z', processed: 2, total: 2 }
|
||||
|
||||
beforeEach (() => {
|
||||
vi.clearAllMocks ()
|
||||
api.apiGet.mockImplementation (async (path: string) =>
|
||||
path === '/static_mirror' ? definition : ready)
|
||||
api.apiPost.mockResolvedValue ({ job: 'opaque-job' })
|
||||
api.startAPIDownload.mockReturnValue (vi.fn ())
|
||||
})
|
||||
|
||||
const startGeneration = async () => {
|
||||
const button = screen.getByRole ('button', { name: 'スナップショット生成' })
|
||||
await waitFor (() => expect (button).toBeEnabled ())
|
||||
fireEvent.click (button)
|
||||
}
|
||||
|
||||
describe ('Static Mirror generation', () => {
|
||||
it ('starts a current snapshot and downloads through the common API', async () => {
|
||||
const view = renderWithProviders (<StaticMirrorPage/>, { route: '/ja/static-mirror' })
|
||||
await startGeneration ()
|
||||
expect (api.apiPost).toHaveBeenCalledWith ('/static_mirror/jobs', { }, { locale: 'ja' })
|
||||
fireEvent.click (await screen.findByRole ('button', { name: 'ZIP ダウンロード' }))
|
||||
expect (screen.getByText (ready.dataAt)).toBeInTheDocument ()
|
||||
expect (screen.getByText ('言語別名称').parentElement).toHaveTextContent ('2000')
|
||||
expect (api.startAPIDownload).toHaveBeenCalledWith (
|
||||
'/static_mirror', 'opaque-job', expect.any (Function))
|
||||
const release = api.startAPIDownload.mock.results[0].value
|
||||
view.unmount ()
|
||||
expect (release).toHaveBeenCalled ()
|
||||
})
|
||||
|
||||
it ('shows generation progress and offers the ZIP only after readiness', async () => {
|
||||
let state = { ...ready, status: 'running', processed: 1 }
|
||||
api.apiGet.mockImplementation (async (path: string) =>
|
||||
path === '/static_mirror' ? definition : state)
|
||||
const view = renderWithProviders (<StaticMirrorPage/>)
|
||||
await startGeneration ()
|
||||
expect (await screen.findByText ('生成中')).toBeInTheDocument ()
|
||||
expect (screen.queryByRole ('button', { name: 'ZIP ダウンロード' })).toBeNull ()
|
||||
expect (screen.getByRole ('button', { name: 'スナップショット生成' })).toBeDisabled ()
|
||||
state = ready
|
||||
await act (async () => {
|
||||
await view.queryClient.invalidateQueries ({ queryKey: staticMirrorKeys.job ('opaque-job') })
|
||||
})
|
||||
expect (await screen.findByRole ('button', { name: 'ZIP ダウンロード' })).toBeEnabled ()
|
||||
})
|
||||
|
||||
it ('keeps failure reasons visible without offering a ZIP', async () => {
|
||||
api.apiGet.mockImplementation (async (path: string) =>
|
||||
path === '/static_mirror' ? definition : { ...ready, status: 'failed', error: 'too_large' })
|
||||
renderWithProviders (<StaticMirrorPage/>)
|
||||
await startGeneration ()
|
||||
expect (await screen.findByRole ('alert')).toHaveTextContent ('too_large')
|
||||
expect (screen.queryByRole ('button', { name: 'ZIP ダウンロード' })).toBeNull ()
|
||||
})
|
||||
|
||||
it ('shows intake limits and revoked downloads using existing error copy', async () => {
|
||||
api.apiPost.mockRejectedValueOnce ({ isAxiosError: true, response: { status: 429 } })
|
||||
renderWithProviders (<StaticMirrorPage/>)
|
||||
await startGeneration ()
|
||||
expect (await screen.findByRole ('alert')).toHaveTextContent ('HTTP 429')
|
||||
api.apiGet.mockImplementation (async (path: string) => {
|
||||
if (path === '/static_mirror') return definition
|
||||
throw { isAxiosError: true, response: { status: 404 } }
|
||||
})
|
||||
await startGeneration ()
|
||||
await waitFor (() => expect (screen.getByRole ('alert')).toHaveTextContent ('有効期限'))
|
||||
expect (screen.getByRole ('button', { name: 'スナップショット生成' })).toBeEnabled ()
|
||||
})
|
||||
|
||||
it ('preserves Japanese fallback language metadata for untranslated UI', async () => {
|
||||
renderWithProviders (<StaticMirrorPage/>, { route: '/ain-Kana/static-mirror' })
|
||||
await startGeneration ()
|
||||
expect (api.apiPost).toHaveBeenCalledWith (
|
||||
'/static_mirror/jobs', { }, { locale: 'ain-Kana' })
|
||||
expect (screen.getByRole ('heading', { name: 'Static Mirror 生成機能' })
|
||||
.querySelector ('[lang="ja"]')).toBeInTheDocument ()
|
||||
})
|
||||
})
|
||||
@@ -1,140 +0,0 @@
|
||||
import { useMutation, useQuery } from '@tanstack/react-query'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
|
||||
import PageHead from '@/components/common/PageHead'
|
||||
import PageTitle from '@/components/common/PageTitle'
|
||||
import MainArea from '@/components/layout/MainArea'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { apiGet, apiPost, isApiError, startAPIDownload } from '@/lib/api'
|
||||
import { staticMirrorKeys } from '@/lib/queryKeys'
|
||||
import { useLocale } from '@/lib/useLocale'
|
||||
import { useUITranslation } from '@/lib/useUITranslation'
|
||||
|
||||
type Definition = {
|
||||
resourceLimit: number
|
||||
nameLimit: number
|
||||
maxBytes: number
|
||||
maxSeconds: number
|
||||
concurrency: number
|
||||
cooldownSeconds: number
|
||||
expirySeconds: number
|
||||
exclusions: string[] }
|
||||
|
||||
type Job = {
|
||||
status: 'queued' | 'running' | 'ready' | 'failed'
|
||||
dataAt: string | null
|
||||
processed: number
|
||||
total: number | null
|
||||
error?: string }
|
||||
|
||||
const StaticMirrorPage = () => {
|
||||
const locale = useLocale ()
|
||||
const { label, message } = useUITranslation ('archive')
|
||||
const [job, setJob] = useState<string | null> (null)
|
||||
const [downloadFailed, setDownloadFailed] = useState (false)
|
||||
const downloadRef = useRef<(() => void) | null> (null)
|
||||
useEffect (() => () => downloadRef.current?.(), [])
|
||||
|
||||
const definition = useQuery ({
|
||||
queryKey: staticMirrorKeys.definition,
|
||||
queryFn: () => apiGet<Definition> ('/static_mirror'), retry: false })
|
||||
const { mutate: start, isPending: pending, error: creationError } = useMutation ({
|
||||
mutationFn: () => apiPost<{ job: string }> ('/static_mirror/jobs', { }, { locale }),
|
||||
onSuccess: result => {
|
||||
downloadRef.current?.()
|
||||
setDownloadFailed (false)
|
||||
setJob (result.job)
|
||||
} })
|
||||
const status = useQuery ({
|
||||
queryKey: staticMirrorKeys.job (job),
|
||||
queryFn: () => apiGet<Job> (`/static_mirror/jobs/${encodeURIComponent (job!)}`),
|
||||
enabled: job != null,
|
||||
retry: false,
|
||||
refetchInterval: query =>
|
||||
(query.state.error != null
|
||||
|| ['ready', 'failed'].includes (query.state.data?.status ?? ''))
|
||||
? false : 1000 })
|
||||
const unavailable = isApiError (status.error) && status.error.response?.status === 404
|
||||
const active = job != null && !unavailable
|
||||
&& !['ready', 'failed'].includes (status.data?.status ?? '')
|
||||
const error = creationError ?? status.error ?? definition.error
|
||||
const errorStatus = isApiError (error) ? error.response?.status : null
|
||||
const metadata = definition.data
|
||||
const percentage =
|
||||
(status.data?.total == null || status.data.total === 0)
|
||||
? 100
|
||||
: Math.round (100 * status.data.processed / status.data.total)
|
||||
|
||||
const download = () => {
|
||||
if (job == null) return
|
||||
downloadRef.current?.()
|
||||
setDownloadFailed (false)
|
||||
downloadRef.current = startAPIDownload (
|
||||
'/static_mirror', job, () => setDownloadFailed (true))
|
||||
}
|
||||
|
||||
return (
|
||||
<MainArea className="space-y-4">
|
||||
<PageHead title={message ('title')}/>
|
||||
<PageTitle>{label ('title')}</PageTitle>
|
||||
<p>{label ('developmentOnly')} / {label ('currentOnly')}</p>
|
||||
{metadata != null && (
|
||||
<dl>
|
||||
<dt>{label ('limits')}</dt>
|
||||
<dd>
|
||||
{label ('counts:resources.posts')} / {label ('counts:resources.tags')}:
|
||||
{' '}{metadata.resourceLimit}
|
||||
</dd>
|
||||
<dd>{label ('counts:columns.names')}: {metadata.nameLimit}</dd>
|
||||
<dd>{metadata.maxBytes} B / {metadata.maxSeconds} s</dd>
|
||||
<dt>{label ('concurrency')}</dt>
|
||||
<dd>{metadata.concurrency} / {metadata.cooldownSeconds} s</dd>
|
||||
<dd>{label ('zip')}: {metadata.expirySeconds} s</dd>
|
||||
<dt>{label ('exclusions')}</dt>
|
||||
<dd>
|
||||
{['counts:resources.tag_descriptions', 'navigation:help',
|
||||
'counts:resources.materials', 'counts:resources.nico_tags',
|
||||
'counts:attributes.parent_post_ids', 'counts:attributes.parent_tag_ids',
|
||||
'counts:columns.sections', 'navigation:history', 'navigation:edit']
|
||||
.map (key => <span key={key}>{label (key)}{' / '}</span>)}
|
||||
</dd>
|
||||
<dt>{label ('externalAssets')}</dt>
|
||||
<dd>{label ('externalOmissions')}: {label ('externalMedia')}</dd>
|
||||
<dt>{label ('protection')}</dt>
|
||||
<dd>{label ('protectionDetails')}</dd>
|
||||
</dl>)}
|
||||
<Button onClick={() => start ()}
|
||||
disabled={metadata == null || pending || active}>
|
||||
{label ('generation')}
|
||||
</Button>
|
||||
{status.data != null && (
|
||||
<div role="status">
|
||||
<p>{label ('dataAt')}: <time>{status.data.dataAt ?? '—'}</time></p>
|
||||
<p>
|
||||
{status.data.status === 'ready'
|
||||
? label ('download')
|
||||
: label (`counts:exportProgress.${status.data.status}`)}
|
||||
</p>
|
||||
{status.data.total != null && (
|
||||
<p>{label ('counts:exportProgress.measured', {
|
||||
processed: status.data.processed, total: status.data.total,
|
||||
percent: percentage })}</p>)}
|
||||
{status.data.status === 'ready' && (
|
||||
<Button onClick={download}>{label ('download')}</Button>)}
|
||||
{status.data.error != null && (
|
||||
<p role="alert">{label ('failure')}: <code>{status.data.error}</code></p>)}
|
||||
</div>)}
|
||||
{error != null && (
|
||||
<p role="alert">
|
||||
{errorStatus === 404
|
||||
? label ('counts:downloadErrors.unavailable')
|
||||
: (errorStatus != null
|
||||
? label ('counts:errors.http', { status: errorStatus })
|
||||
: label ('counts:errors.network'))}
|
||||
</p>)}
|
||||
{status.error != null && (
|
||||
<Button onClick={() => status.refetch ()}>{label ('shared:actions.retry')}</Button>)}
|
||||
{downloadFailed && <p role="alert">{label ('counts:downloadErrors.failed')}</p>}
|
||||
</MainArea>)
|
||||
}
|
||||
export default StaticMirrorPage
|
||||
@@ -6,6 +6,7 @@ import { useLocale, useLocaleNavigate } from '@/lib/useLocale'
|
||||
import PrefetchLink from '@/components/PrefetchLink'
|
||||
import FormField from '@/components/common/FormField'
|
||||
import PageTitle from '@/components/common/PageTitle'
|
||||
import ResourcePickerButton from '@/components/common/ResourcePickerButton'
|
||||
import Pagination from '@/components/common/Pagination'
|
||||
import MainArea from '@/components/layout/MainArea'
|
||||
import PageHead from '@/components/common/PageHead'
|
||||
@@ -96,13 +97,19 @@ const MaterialHistoryPage: FC = () => {
|
||||
<div className="space-y-3">
|
||||
<FormField label={label ('materialId')}>
|
||||
{({ invalid }) => (
|
||||
<input
|
||||
<div className="flex items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={materialIdInput}
|
||||
aria-label={message ('materialId').text}
|
||||
lang={message ('materialId').language}
|
||||
onChange={e => setMaterialIdInput (e.target.value)}
|
||||
className={inputClass (invalid)}/>)}
|
||||
className={inputClass (invalid, 'min-w-0 flex-1')}/>
|
||||
<ResourcePickerButton
|
||||
source={{ resource: 'materials', field: 'resource_id' }}
|
||||
values={materialIdInput === '' ? [] : [materialIdInput]}
|
||||
onSelect={values => setMaterialIdInput (values[0] ?? '')}/>
|
||||
</div>)}
|
||||
</FormField>
|
||||
|
||||
<FormField label={label ('tagName')}>
|
||||
|
||||
@@ -4,6 +4,7 @@ import { useState } from 'react'
|
||||
import { useLocale } from '@/lib/useLocale'
|
||||
import FormField from '@/components/common/FormField'
|
||||
import PageTitle from '@/components/common/PageTitle'
|
||||
import ResourcePickerButton from '@/components/common/ResourcePickerButton'
|
||||
import MainArea from '@/components/layout/MainArea'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { toast } from '@/components/ui/use-toast'
|
||||
@@ -126,11 +127,17 @@ const MaterialSyncSuppressionsPage: FC = () => {
|
||||
|
||||
<FormField label={label ('suppressions.driveFileId')}>
|
||||
{({ invalid }) => (
|
||||
<input
|
||||
<div className="flex items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={driveFileId}
|
||||
onChange={e => setDriveFileId (e.target.value)}
|
||||
className={inputClass (invalid)}/>)}
|
||||
className={inputClass (invalid, 'min-w-0 flex-1')}/>
|
||||
<ResourcePickerButton
|
||||
source={{ resource: 'materials', field: 'drive_file_id' }}
|
||||
values={driveFileId === '' ? [] : [driveFileId]}
|
||||
onSelect={values => setDriveFileId (values[0] ?? '')}/>
|
||||
</div>)}
|
||||
</FormField>
|
||||
|
||||
<FormField label={label ('suppressions.reason')}>
|
||||
|
||||
新しいイシューから参照
ユーザーをブロックする