コミットを比較
2
コミット
main
..
feature/483
| 作成者 | SHA1 | 日付 | |
|---|---|---|---|
|
|
9c71a9fc02 | ||
|
|
cb1f947a36 |
@@ -0,0 +1,54 @@
|
|||||||
|
class StaticMirrorsController < ApplicationController
|
||||||
|
before_action :development_only
|
||||||
|
wrap_parameters false
|
||||||
|
|
||||||
|
def show
|
||||||
|
render json: { resource_limit: StaticMirror::RESOURCE_LIMIT,
|
||||||
|
name_limit: StaticMirror::NAME_LIMIT, max_bytes: StaticMirror::MAX_BYTES,
|
||||||
|
max_seconds: StaticMirrorJob::MAX_SECONDS, concurrency: 1,
|
||||||
|
cooldown_seconds: StaticMirrorJob::COOLDOWN,
|
||||||
|
expiry_seconds: StaticMirrorJob::EXPIRY.to_i, exclusions: StaticMirror::EXCLUSIONS }
|
||||||
|
end
|
||||||
|
|
||||||
|
def create
|
||||||
|
# No date input is accepted: this endpoint cannot promise historical restoration.
|
||||||
|
payload = request.request_parameters.except('authenticity_token')
|
||||||
|
.merge(request.query_parameters)
|
||||||
|
return head :bad_request unless (payload.keys - ['locale']).empty?
|
||||||
|
|
||||||
|
locale = resolve_locale!
|
||||||
|
session[:static_mirror_owner] ||= SecureRandom.hex(32)
|
||||||
|
job = StaticMirrorJob.start(owner: session[:static_mirror_owner], locale: locale.code)
|
||||||
|
render json: { job: }, status: :accepted
|
||||||
|
rescue StaticMirrorJob::Busy
|
||||||
|
response.headers['Retry-After'] = StaticMirrorJob::COOLDOWN.to_s
|
||||||
|
head :too_many_requests
|
||||||
|
rescue ActiveJob::EnqueueError
|
||||||
|
render json: { error: 'generation_failed' }, status: :service_unavailable
|
||||||
|
end
|
||||||
|
|
||||||
|
def status
|
||||||
|
value = StaticMirrorJob.status(params[:id], session[:static_mirror_owner])
|
||||||
|
return head :not_found unless value
|
||||||
|
|
||||||
|
render json: value
|
||||||
|
rescue StaticMirrorJob::Busy
|
||||||
|
head :service_unavailable
|
||||||
|
end
|
||||||
|
|
||||||
|
def download
|
||||||
|
bytes = StaticMirrorJob.download(params[:id], session[:static_mirror_owner])
|
||||||
|
return head :not_found unless bytes
|
||||||
|
|
||||||
|
send_data bytes, filename: 'static-mirror.zip', type: 'application/zip',
|
||||||
|
disposition: 'attachment'
|
||||||
|
rescue StaticMirrorJob::Busy
|
||||||
|
head :service_unavailable
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def development_only
|
||||||
|
head :not_found unless Rails.env.development?
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
require 'fileutils'
|
||||||
|
require 'timeout'
|
||||||
|
|
||||||
|
# Development PoC: one filesystem slot and Rails' in-process async executor.
|
||||||
|
# A process restart loses queued work; stale state fails rather than reporting success.
|
||||||
|
class StaticMirrorJob < ApplicationJob
|
||||||
|
self.queue_adapter = :async
|
||||||
|
EXPIRY = 10.minutes
|
||||||
|
COOLDOWN = 60
|
||||||
|
MAX_SECONDS = 60
|
||||||
|
ACTIVE = ['queued', 'running'].freeze
|
||||||
|
class Busy < StandardError; end
|
||||||
|
|
||||||
|
def self.directory = Rails.root.join('tmp/static_mirror')
|
||||||
|
|
||||||
|
def self.synchronise(blocking: false)
|
||||||
|
FileUtils.mkdir_p(directory, mode: 0o700)
|
||||||
|
File.open(directory.join('.registry.lock'), File::RDWR | File::CREAT, 0o600) do |lock|
|
||||||
|
mode = blocking ? File::LOCK_EX : File::LOCK_EX | File::LOCK_NB
|
||||||
|
raise Busy unless lock.flock(mode)
|
||||||
|
|
||||||
|
yield
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.state
|
||||||
|
JSON.parse(directory.join('state.json').read)
|
||||||
|
rescue Errno::ENOENT
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.write(value)
|
||||||
|
path = directory.join('state.tmp')
|
||||||
|
File.open(path, 'w', 0o600) { _1.write(JSON.generate(value)) }
|
||||||
|
File.rename(path, directory.join('state.json'))
|
||||||
|
ensure
|
||||||
|
File.unlink(path) if path&.exist?
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.expire_stalled(value)
|
||||||
|
return unless value && ACTIVE.include?(value['status'])
|
||||||
|
return unless Time.current.to_f - value.fetch('created_at') > MAX_SECONDS + 15
|
||||||
|
|
||||||
|
value.merge!('status' => 'failed', 'error' => 'worker_stopped')
|
||||||
|
write(value)
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.start(owner:, locale:)
|
||||||
|
id = synchronise do
|
||||||
|
previous = state
|
||||||
|
expire_stalled(previous)
|
||||||
|
raise Busy if previous && ACTIVE.include?(previous['status'])
|
||||||
|
if previous && Time.current.to_f - previous.fetch('created_at') < COOLDOWN
|
||||||
|
raise Busy
|
||||||
|
end
|
||||||
|
File.open(directory.join('.generation.lock'), File::RDWR | File::CREAT, 0o600) do |lock|
|
||||||
|
raise Busy unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
||||||
|
|
||||||
|
FileUtils.rm_f(directory.join('archive.zip'))
|
||||||
|
FileUtils.rm_f(directory.join('archive.part'))
|
||||||
|
identifier = SecureRandom.hex(24)
|
||||||
|
write({ 'id' => identifier, 'owner' => owner, 'locale' => locale,
|
||||||
|
'created_at' => Time.current.to_f, 'expiry_at' => (Time.current + EXPIRY).to_f,
|
||||||
|
'status' => 'queued', 'processed' => 0, 'total' => nil, 'data_at' => nil })
|
||||||
|
identifier
|
||||||
|
end
|
||||||
|
end
|
||||||
|
perform_later(id)
|
||||||
|
id
|
||||||
|
rescue ActiveJob::EnqueueError
|
||||||
|
update(id) { _1.merge!('status' => 'failed', 'error' => 'generation_failed') } if id
|
||||||
|
raise
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.authorised_state(id, owner)
|
||||||
|
value = state
|
||||||
|
return unless owner.present? && value && value['id'] == id && value['owner'] == owner
|
||||||
|
return unless value['expiry_at'] > Time.current.to_f
|
||||||
|
|
||||||
|
expire_stalled(value)
|
||||||
|
if value['status'] == 'ready'
|
||||||
|
permitted = StaticMirror.authorised_artifact?(value)
|
||||||
|
return unless permitted && directory.join('archive.zip').file?
|
||||||
|
end
|
||||||
|
value
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.status(id, owner)
|
||||||
|
synchronise do
|
||||||
|
authorised_state(id, owner)&.slice('status', 'processed', 'total', 'data_at', 'error')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.download(id, owner)
|
||||||
|
synchronise do
|
||||||
|
value = authorised_state(id, owner)
|
||||||
|
return unless value && value['status'] == 'ready'
|
||||||
|
|
||||||
|
bytes = directory.join('archive.zip').binread
|
||||||
|
# Revalidate after reading; send_data avoids delayed sendfile reuse of a stale path.
|
||||||
|
permitted = StaticMirror.authorised_artifact?(value)
|
||||||
|
permitted ? bytes : nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.update(id)
|
||||||
|
# Worker updates may wait for a short status/download read; request intake never waits.
|
||||||
|
synchronise(blocking: true) do
|
||||||
|
value = state
|
||||||
|
return unless value && value['id'] == id
|
||||||
|
|
||||||
|
yield value
|
||||||
|
write(value)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def perform(id)
|
||||||
|
File.open(self.class.directory.join('.generation.lock'), File::RDWR) do |lock|
|
||||||
|
# Intake briefly owns this lock. A queued worker must wait, never disappear.
|
||||||
|
lock.flock(File::LOCK_EX)
|
||||||
|
|
||||||
|
value = self.class.state
|
||||||
|
return unless value && value['id'] == id && value['status'] == 'queued'
|
||||||
|
|
||||||
|
generate(id, value)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def generate(id, value)
|
||||||
|
self.class.update(id) { _1['status'] = 'running' }
|
||||||
|
remaining = MAX_SECONDS - (Time.current.to_f - value.fetch('created_at'))
|
||||||
|
raise Timeout::Error unless remaining.positive?
|
||||||
|
|
||||||
|
Timeout.timeout(remaining) do
|
||||||
|
locale = Locale.find_by!(code: value.fetch('locale'), deprecated_at: nil)
|
||||||
|
mirror = StaticMirror.new(locale:)
|
||||||
|
bytes = mirror.export do |processed, total|
|
||||||
|
self.class.update(id) do |metadata|
|
||||||
|
metadata.merge!('processed' => processed, 'total' => total, 'data_at' => mirror.data_at)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
part = self.class.directory.join('archive.part')
|
||||||
|
File.open(part, 'wb', 0o600) { _1.write(bytes) }
|
||||||
|
File.rename(part, self.class.directory.join('archive.zip'))
|
||||||
|
self.class.update(id) do |metadata|
|
||||||
|
metadata.merge!(mirror.authority).merge!('status' => 'ready', 'data_at' => mirror.data_at)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
rescue StandardError => error
|
||||||
|
FileUtils.rm_f(self.class.directory.join('archive.part'))
|
||||||
|
FileUtils.rm_f(self.class.directory.join('archive.zip'))
|
||||||
|
code = case error
|
||||||
|
when StaticMirror::TooLarge then 'too_large'
|
||||||
|
when Timeout::Error then 'generation_timeout'
|
||||||
|
when HistoryAccess::Forbidden then 'publication_changed'
|
||||||
|
else 'generation_failed'
|
||||||
|
end
|
||||||
|
self.class.update(id) { _1.merge!('status' => 'failed', 'error' => code) }
|
||||||
|
Rails.logger.warn("static_mirror_failure: #{ error.class }")
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
require 'cgi'
|
||||||
|
require 'digest'
|
||||||
|
require 'erb'
|
||||||
|
require 'pathname'
|
||||||
|
require 'uri'
|
||||||
|
|
||||||
|
# A deliberately small current-state projection, always under public authority.
|
||||||
|
class StaticMirror
|
||||||
|
ORIGIN = 'https://hub.nizika.monster'.freeze
|
||||||
|
PUBLICATION_POLICY = 'static-mirror-public-projection-1'.freeze
|
||||||
|
DATA_AT_BASIS = 'capture_started_at'.freeze
|
||||||
|
RESOURCE_LIMIT = 100
|
||||||
|
NAME_LIMIT = 2_000
|
||||||
|
MAX_BYTES = 8.megabytes
|
||||||
|
EXCLUSIONS = [
|
||||||
|
'historical_state', 'unversioned_resources', 'tag_descriptions', 'help', 'materials',
|
||||||
|
'external_tags', 'parent_relations', 'tag_sections', 'users', 'audit_data',
|
||||||
|
'authentication', 'editing', 'images', 'videos', 'resources_beyond_limits'].freeze
|
||||||
|
class TooLarge < StandardError; end
|
||||||
|
class UnsafeUrl < StandardError; end
|
||||||
|
|
||||||
|
attr_reader :data_at, :authority, :counts
|
||||||
|
|
||||||
|
def initialize(locale:, origin: ORIGIN)
|
||||||
|
@locale = locale
|
||||||
|
@origin = validated_url(origin).sub(%r{/+\z}, '')
|
||||||
|
end
|
||||||
|
|
||||||
|
def export(&progress)
|
||||||
|
@files = { }
|
||||||
|
@bytes = 0
|
||||||
|
HistoryAccess.set(user: nil) do
|
||||||
|
capture
|
||||||
|
progress&.call(0, @posts.length + @tags.length)
|
||||||
|
generate_pages(&progress)
|
||||||
|
raise HistoryAccess::Forbidden unless self.class.authorised_artifact?(authority)
|
||||||
|
|
||||||
|
generated_at = Time.current.utc
|
||||||
|
manifest = { format_version: 1, generated_at: generated_at.iso8601(6),
|
||||||
|
data_at:, data_at_basis: DATA_AT_BASIS,
|
||||||
|
locale: @locale.code, resources: counts, exclusions: EXCLUSIONS,
|
||||||
|
fields: { posts: ['id', 'title', 'url', 'tags'], tags: ['id', 'category', 'names'] },
|
||||||
|
limits: { posts: RESOURCE_LIMIT, tags: RESOURCE_LIMIT, names: NAME_LIMIT,
|
||||||
|
bytes: MAX_BYTES },
|
||||||
|
external_assets: 'not_included', checksum_scope: 'all_files_except_manifest.json',
|
||||||
|
files: @files.map { |path, data| { path:, sha256: Digest::SHA256.hexdigest(data) } } }
|
||||||
|
add_file('manifest.json', JSON.pretty_generate(manifest))
|
||||||
|
entries = @files.map do |path, data|
|
||||||
|
MaterialZipExporter::Entry.new(path: "archive/#{ path }", data:, mtime: generated_at)
|
||||||
|
end
|
||||||
|
bytes = MaterialZipExporter::ZipWriter.write(entries)
|
||||||
|
raise TooLarge if bytes.bytesize > MAX_BYTES
|
||||||
|
|
||||||
|
bytes
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
# Static distribution has no admin mode or Counts aggregate exception.
|
||||||
|
# Recheck the exact public projection, including edits without a new discard.
|
||||||
|
def self.authorised_artifact?(metadata)
|
||||||
|
return false unless metadata['publication_policy'] == PUBLICATION_POLICY
|
||||||
|
ids = metadata.values_at('post_ids', 'tag_ids')
|
||||||
|
unless ids.all? { |values|
|
||||||
|
values.is_a?(Array) && values.length <= RESOURCE_LIMIT &&
|
||||||
|
values.all? { _1.is_a?(Integer) && _1.positive? }
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
end
|
||||||
|
|
||||||
|
HistoryAccess.set(user: nil) do
|
||||||
|
ApplicationRecord.uncached do
|
||||||
|
CountsExporter.with_read_snapshot do
|
||||||
|
Locale.where(code: metadata['locale'], deprecated_at: nil).exists? &&
|
||||||
|
metadata['history_revision'] == HistoryAccess.revision &&
|
||||||
|
metadata['content_digest'] == projection_digest(
|
||||||
|
read_projection(post_ids: ids[0], tag_ids: ids[1]))
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
rescue TooLarge
|
||||||
|
false
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.read_projection(post_ids: nil, tag_ids: nil)
|
||||||
|
posts = confirmed(Post, PostVersion, :post_versions)
|
||||||
|
tags = confirmed(Tag, TagVersion, :tag_versions).where(deprecated_at: nil)
|
||||||
|
posts = posts.where(id: post_ids) unless post_ids.nil?
|
||||||
|
tags = tags.where(id: tag_ids) unless tag_ids.nil?
|
||||||
|
posts = posts.order(:id).limit(RESOURCE_LIMIT).select(:id, :version_no, :title, :url).to_a
|
||||||
|
tags = tags.order(:id).limit(RESOURCE_LIMIT).select(:id, :version_no, :category).to_a
|
||||||
|
names = TagName.where(tag_id: tags.map(&:id)).order(:tag_id, :language_code, :id)
|
||||||
|
.limit(NAME_LIMIT + 1).pluck(:tag_id, :language_code, :script_code, :name, :primary_flg)
|
||||||
|
raise TooLarge if names.length > NAME_LIMIT
|
||||||
|
|
||||||
|
relations = PostTag.where(post_id: posts.map(&:id), tag_id: tags.map(&:id))
|
||||||
|
.order(:post_id, :tag_id).pluck(:post_id, :tag_id)
|
||||||
|
[posts, tags, names, relations]
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.confirmed(model, version_model, association)
|
||||||
|
# Requiring a matching immutable revision closes the legacy unversioned gap.
|
||||||
|
HistoryAccess.current(model.all, version_model).joins(association)
|
||||||
|
.where("#{ version_model.table_name }.version_no = #{ model.table_name }.version_no")
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.projection_digest(projection)
|
||||||
|
posts, tags, names, relations = projection
|
||||||
|
Digest::SHA256.hexdigest(JSON.generate(
|
||||||
|
[posts.map(&:attributes), tags.map(&:attributes), names, relations]))
|
||||||
|
end
|
||||||
|
|
||||||
|
private_class_method :confirmed
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def capture
|
||||||
|
# Server clock at capture entry; MySQL establishes the read view later.
|
||||||
|
@data_at = Time.current.utc.iso8601(6)
|
||||||
|
CountsExporter.with_read_snapshot do
|
||||||
|
projection = self.class.read_projection
|
||||||
|
@posts, @tags, names, @relations = projection
|
||||||
|
@authority = { 'publication_policy' => PUBLICATION_POLICY,
|
||||||
|
'locale' => @locale.code,
|
||||||
|
'history_revision' => HistoryAccess.revision,
|
||||||
|
'post_ids' => @posts.map(&:id), 'tag_ids' => @tags.map(&:id),
|
||||||
|
'content_digest' => self.class.projection_digest(projection) }
|
||||||
|
@names = names.group_by(&:first)
|
||||||
|
@counts = { posts: @posts.length, tags: @tags.length }
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def generate_pages
|
||||||
|
assets = Rails.root.join('app/views/static_mirror')
|
||||||
|
['archive.css', 'archive.js'].each do |name|
|
||||||
|
add_file("assets/#{ name }", assets.join(name).read)
|
||||||
|
end
|
||||||
|
add_file('robots.txt', "User-agent: *\nDisallow: /\n")
|
||||||
|
add_file('archive-config.js', 'window.archiveConfig = { origin: null };')
|
||||||
|
page('index.html', 'Static Mirror 生成機能', '', search_form + limitations)
|
||||||
|
page("#{ @locale.code }/posts/index.html", 'Post', '/posts', post_list(@posts))
|
||||||
|
page("#{ @locale.code }/tags/index.html", 'Tag', '/tags', tag_list(@tags))
|
||||||
|
processed = 0
|
||||||
|
@posts.each do |post|
|
||||||
|
tags = @tags.select { |tag| @relations.include?([post.id, tag.id]) }
|
||||||
|
content = "<p>#{ external_link(post.url) }</p>#{ tag_list(tags) }"
|
||||||
|
page(resource_path('posts', post.id), post.title.presence || post.id.to_s,
|
||||||
|
"/posts/#{ post.id }", content)
|
||||||
|
processed += 1
|
||||||
|
yield(processed, @posts.length + @tags.length) if block_given?
|
||||||
|
end
|
||||||
|
@tags.each do |tag|
|
||||||
|
posts = @posts.select { |post| @relations.include?([post.id, tag.id]) }
|
||||||
|
names = @names.fetch(tag.id, []).map do |_, language, script, name, _primary|
|
||||||
|
"<li><span lang=\"#{ h(language) }\">#{ h(name) }</span> (#{ h(script) })</li>"
|
||||||
|
end.join
|
||||||
|
content = "<p>#{ h(tag.category) }</p><ul>#{ names }</ul>#{ post_list(posts) }"
|
||||||
|
page(resource_path('tags', tag.id), tag_name(tag), "/tags/#{ tag.id }", content)
|
||||||
|
processed += 1
|
||||||
|
yield(processed, @posts.length + @tags.length) if block_given?
|
||||||
|
end
|
||||||
|
index = @posts.map { |post|
|
||||||
|
tag_ids = @relations.filter_map { |post_id, tag_id| tag_id if post_id == post.id }
|
||||||
|
names = tag_ids.flat_map { |id| @names.fetch(id, []).map { _1[3] } }.uniq
|
||||||
|
{ path: resource_path('posts', post.id), title: post.title.presence || post.id.to_s,
|
||||||
|
names: }
|
||||||
|
} + @tags.map { |tag|
|
||||||
|
{ path: resource_path('tags', tag.id), title: tag_name(tag),
|
||||||
|
names: @names.fetch(tag.id, []).map { _1[3] } }
|
||||||
|
}
|
||||||
|
add_file('search-index.json', JSON.generate(index))
|
||||||
|
end
|
||||||
|
|
||||||
|
def page(path, title, original_path, content)
|
||||||
|
@path = path
|
||||||
|
@title = title
|
||||||
|
@original_path = "/#{ @locale.code }#{ original_path }"
|
||||||
|
@content = content
|
||||||
|
template = Rails.root.join('app/views/static_mirror/page.html.erb').read
|
||||||
|
add_file(path, ERB.new(template).result(binding))
|
||||||
|
end
|
||||||
|
|
||||||
|
def add_file(path, data)
|
||||||
|
@bytes += data.bytesize
|
||||||
|
raise TooLarge if @bytes > MAX_BYTES
|
||||||
|
|
||||||
|
@files[path] = data
|
||||||
|
end
|
||||||
|
|
||||||
|
def resource_path(resource, id) = "#{ @locale.code }/#{ resource }/#{ id }/index.html"
|
||||||
|
|
||||||
|
def relative(path)
|
||||||
|
Pathname.new(path).relative_path_from(Pathname.new(@path).dirname).to_s
|
||||||
|
end
|
||||||
|
|
||||||
|
def h(value) = CGI.escapeHTML(value.to_s)
|
||||||
|
|
||||||
|
def tag_name(tag)
|
||||||
|
@names.fetch(tag.id, []).find { _1[1] == @locale.language_code && _1[4] }&.fetch(3) ||
|
||||||
|
tag.id.to_s
|
||||||
|
end
|
||||||
|
|
||||||
|
# Links are resolved after the page path is known; only generated numeric IDs enter paths.
|
||||||
|
def post_list(posts)
|
||||||
|
'<h2>Post</h2><ul>' + posts.map { |post|
|
||||||
|
title = post.title.presence || post.id.to_s
|
||||||
|
"<li><a data-path=\"#{ resource_path('posts', post.id) }\">#{ h(title) }</a></li>"
|
||||||
|
}.join + '</ul>'
|
||||||
|
end
|
||||||
|
|
||||||
|
def tag_list(tags)
|
||||||
|
'<h2>Tag</h2><ul>' + tags.map { |tag|
|
||||||
|
"<li><a data-path=\"#{ resource_path('tags', tag.id) }\">#{ h(tag_name(tag)) }</a></li>"
|
||||||
|
}.join + '</ul>'
|
||||||
|
end
|
||||||
|
|
||||||
|
def linked_content
|
||||||
|
@content.gsub(/data-path="([^"]+)"/) { "href=\"#{ h(relative(Regexp.last_match(1))) }\"" }
|
||||||
|
end
|
||||||
|
|
||||||
|
def validated_url(value)
|
||||||
|
uri = URI.parse(value.to_s)
|
||||||
|
unless uri.is_a?(URI::HTTP) && uri.host.present? && uri.userinfo.nil?
|
||||||
|
raise UnsafeUrl
|
||||||
|
end
|
||||||
|
|
||||||
|
uri.to_s
|
||||||
|
rescue URI::InvalidURIError
|
||||||
|
raise UnsafeUrl
|
||||||
|
end
|
||||||
|
|
||||||
|
def external_link(value)
|
||||||
|
url = validated_url(value)
|
||||||
|
"<a href=\"#{ h(url) }\" target=\"_blank\" rel=\"noopener noreferrer\">#{ h(url) }</a>"
|
||||||
|
rescue UnsafeUrl
|
||||||
|
''
|
||||||
|
end
|
||||||
|
|
||||||
|
def search_form
|
||||||
|
'<form id="archive-search"><label>検索 <input name="q" type="search"></label>' +
|
||||||
|
'<button type="submit">検索</button></form><ul id="search-results"></ul>'
|
||||||
|
end
|
||||||
|
|
||||||
|
def limitations
|
||||||
|
<<~HTML
|
||||||
|
<h2>未対応機能</h2>
|
||||||
|
<p>Tag Description, Help, Material, 外部タグ, 親子関係, 区間, 履歴, ログイン, 作成, 編輯</p>
|
||||||
|
<h2>出力件数の制限</h2><p>Post: #{ RESOURCE_LIMIT }, Tag: #{ RESOURCE_LIMIT }</p>
|
||||||
|
<p>TagName: #{ NAME_LIMIT }, #{ MAX_BYTES } B</p>
|
||||||
|
<h2>外部画像や動画への依存</h2><p>ローカルへ取り込めていない資産: 外部画像や動画</p>
|
||||||
|
<h2>データ保護上の制約</h2>
|
||||||
|
<p>公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。</p>
|
||||||
|
<p>今回は現在状態のみの対応</p>
|
||||||
|
HTML
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
:root { color-scheme: light dark; }
|
||||||
|
body { max-width: 65rem; margin: auto; padding: 1rem; font: 1rem/1.6 sans-serif; }
|
||||||
|
header { border-bottom: 1px solid; padding-bottom: 1rem; }
|
||||||
|
nav { display: flex; flex-wrap: wrap; gap: 1rem; }
|
||||||
|
a, p, li { overflow-wrap: anywhere; }
|
||||||
|
li { margin-block: .4rem; }
|
||||||
|
input, button { font: inherit; }
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
(() => {
|
||||||
|
const root = new URL ('../', document.currentScript.src)
|
||||||
|
const origin = window.archiveConfig?.origin
|
||||||
|
if (typeof origin === 'string')
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
const url = new URL (origin)
|
||||||
|
if (['http:', 'https:'].includes (url.protocol) && !url.username && !url.password)
|
||||||
|
{
|
||||||
|
url.search = ''
|
||||||
|
url.hash = ''
|
||||||
|
document.querySelectorAll ('[data-original]').forEach (link => {
|
||||||
|
link.href = url.href.replace (/\/+$/, '') + link.dataset.original
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Each HTML retains its original link when configuration is absent or invalid.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const form = document.getElementById ('archive-search')
|
||||||
|
const results = document.getElementById ('search-results')
|
||||||
|
if (!form || !results) return
|
||||||
|
form.addEventListener ('submit', async event => {
|
||||||
|
event.preventDefault ()
|
||||||
|
results.replaceChildren ()
|
||||||
|
const query = (new FormData (form).get ('q') ?? '').toString ().toLocaleLowerCase ()
|
||||||
|
if (!query.trim ()) return
|
||||||
|
try
|
||||||
|
{
|
||||||
|
const response = await fetch (new URL ('search-index.json', root))
|
||||||
|
if (!response.ok) throw new Error ('Search index unavailable')
|
||||||
|
const entries = await response.json ()
|
||||||
|
for (const entry of entries)
|
||||||
|
{
|
||||||
|
const text = [entry.title, ...(entry.names ?? [])].join (' ').toLocaleLowerCase ()
|
||||||
|
if (!text.includes (query)) continue
|
||||||
|
const url = new URL (entry.path, root)
|
||||||
|
if (url.origin !== root.origin || !url.pathname.startsWith (root.pathname)) continue
|
||||||
|
const item = document.createElement ('li')
|
||||||
|
const link = document.createElement ('a')
|
||||||
|
link.setAttribute ('href', entry.path)
|
||||||
|
link.textContent = entry.title
|
||||||
|
item.append (link)
|
||||||
|
results.append (item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Lists remain available if the index cannot be loaded.
|
||||||
|
for (const link of document.querySelectorAll ('nav a:not([data-original])'))
|
||||||
|
{
|
||||||
|
const item = document.createElement ('li')
|
||||||
|
item.append (link.cloneNode (true))
|
||||||
|
results.append (item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})()
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="ja">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<meta name="robots" content="noindex, nofollow">
|
||||||
|
<meta name="referrer" content="no-referrer">
|
||||||
|
<meta http-equiv="Content-Security-Policy"
|
||||||
|
content="default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; form-action 'none'; base-uri 'none'">
|
||||||
|
<title><%= h(@title) %></title>
|
||||||
|
<link rel="stylesheet" href="<%= h(relative('assets/archive.css')) %>">
|
||||||
|
<script defer src="<%= h(relative('archive-config.js')) %>"></script>
|
||||||
|
<script defer src="<%= h(relative('assets/archive.js')) %>"></script>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header>
|
||||||
|
<p>静的保存版 — データ基準時点: <time><%= h(data_at) %></time></p>
|
||||||
|
<nav>
|
||||||
|
<a href="<%= h(relative('index.html')) %>">Static Mirror 生成機能</a>
|
||||||
|
<a href="<%= h(relative("#{ @locale.code }/posts/index.html")) %>">Post</a>
|
||||||
|
<a href="<%= h(relative("#{ @locale.code }/tags/index.html")) %>">Tag</a>
|
||||||
|
<a href="<%= h(@origin + @original_path) %>" data-original="<%= h(@original_path) %>"
|
||||||
|
target="_blank" rel="noopener noreferrer">本家へのリンク</a>
|
||||||
|
</nav>
|
||||||
|
</header>
|
||||||
|
<main><h1><%= h(@title) %></h1><%= linked_content %></main>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -1,4 +1,9 @@
|
|||||||
Rails.application.routes.draw do
|
Rails.application.routes.draw do
|
||||||
|
get 'static_mirror', to: 'static_mirrors#show'
|
||||||
|
post 'static_mirror/jobs', to: 'static_mirrors#create'
|
||||||
|
get 'static_mirror/jobs/:id', to: 'static_mirrors#status'
|
||||||
|
get 'static_mirror/files/:id', to: 'static_mirrors#download'
|
||||||
|
|
||||||
get 'counts/definitions/:resource', to: 'counts_tables#definition'
|
get 'counts/definitions/:resource', to: 'counts_tables#definition'
|
||||||
get 'counts/definitions/:resource/candidates', to: 'counts_tables#candidates'
|
get 'counts/definitions/:resource/candidates', to: 'counts_tables#candidates'
|
||||||
get 'counts/query', to: 'counts_tables#index'
|
get 'counts/query', to: 'counts_tables#index'
|
||||||
|
|||||||
@@ -0,0 +1,257 @@
|
|||||||
|
require 'rails_helper'
|
||||||
|
require 'tmpdir'
|
||||||
|
|
||||||
|
RSpec.describe 'Static Mirror API', type: :request do
|
||||||
|
include ActiveSupport::Testing::TimeHelpers
|
||||||
|
|
||||||
|
around do |example|
|
||||||
|
Dir.mktmpdir('static-mirror-spec') do |path|
|
||||||
|
allow(StaticMirrorJob).to receive(:directory).and_return(Pathname.new(path))
|
||||||
|
example.run
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
before do
|
||||||
|
allow(Rails.env).to receive(:development?).and_return(true)
|
||||||
|
allow(StaticMirrorJob).to receive(:perform_later)
|
||||||
|
end
|
||||||
|
|
||||||
|
def create_job
|
||||||
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||||
|
expect(response).to have_http_status(:accepted)
|
||||||
|
json.fetch('job')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'accepts anonymous jobs asynchronously and serves the completed ZIP' do
|
||||||
|
record = create(:post, title: 'mirror-public-api-title')
|
||||||
|
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
||||||
|
title: record.title, url: record.url, tags: '', tags_json: [], parent_post_ids: '',
|
||||||
|
created_at: Time.current)
|
||||||
|
get '/static_mirror'
|
||||||
|
expect(json).to include('resource_limit' => 100, 'concurrency' => 1)
|
||||||
|
id = create_job
|
||||||
|
expect(StaticMirrorJob).to have_received(:perform_later).with(id)
|
||||||
|
expect(StaticMirrorJob.directory.join('archive.zip')).not_to exist
|
||||||
|
get "/static_mirror/jobs/#{ id }"
|
||||||
|
expect(json).to include('status' => 'queued', 'data_at' => nil)
|
||||||
|
StaticMirrorJob.perform_now(id)
|
||||||
|
get "/static_mirror/jobs/#{ id }"
|
||||||
|
expect(json).to include('status' => 'ready', 'processed' => 1, 'total' => 1)
|
||||||
|
expect(json.fetch('data_at')).to be_present
|
||||||
|
expect(json.keys).not_to include('owner', 'origin', 'actor_id', 'policy_revision',
|
||||||
|
'publication_policy', 'history_revision', 'post_ids', 'tag_ids', 'content_digest')
|
||||||
|
get "/static_mirror/files/#{ id }"
|
||||||
|
expect(response).to have_http_status(:ok)
|
||||||
|
expect(response.media_type).to eq('application/zip')
|
||||||
|
expect(response.body.byteslice(0, 4)).to eq("PK\x03\x04".b)
|
||||||
|
expect(response.body).to include(record.title)
|
||||||
|
expect(response.headers['Cache-Control']).to include('no-store')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'bakes the canonical source origin regardless of request and forwarded hosts' do
|
||||||
|
host! 'mirror-intake.test'
|
||||||
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json,
|
||||||
|
headers: { 'X-Forwarded-Host' => 'untrusted-source.test',
|
||||||
|
'X-Forwarded-Proto' => 'https' }
|
||||||
|
expect(response).to have_http_status(:accepted)
|
||||||
|
id = json.fetch('job')
|
||||||
|
expect(StaticMirrorJob.state).not_to have_key('origin')
|
||||||
|
StaticMirrorJob.perform_now(id)
|
||||||
|
get "/static_mirror/files/#{ id }"
|
||||||
|
expect(response).to have_http_status(:ok)
|
||||||
|
expect(response.body).to include('https://hub.nizika.monster/ja')
|
||||||
|
expect(response.body).not_to include('mirror-intake.test', 'untrusted-source.test')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rejects historical and unknown input instead of pretending to restore it' do
|
||||||
|
['at', 'data_at', 'admin', 'limit'].each do |key|
|
||||||
|
post '/static_mirror/jobs', params: { locale: 'ja', key => '2020-01-01' }, as: :json
|
||||||
|
expect(response).to have_http_status(:bad_request)
|
||||||
|
end
|
||||||
|
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'limits requests globally even across fresh anonymous sessions' do
|
||||||
|
create_job
|
||||||
|
other = ActionDispatch::Integration::Session.new(Rails.application)
|
||||||
|
other.post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||||
|
expect(other.response).to have_http_status(:too_many_requests)
|
||||||
|
expect(other.response.headers['Retry-After']).to eq('60')
|
||||||
|
expect(StaticMirrorJob).to have_received(:perform_later).once
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'binds job status and downloads to the creating session' do
|
||||||
|
id = create_job
|
||||||
|
StaticMirrorJob.perform_now(id)
|
||||||
|
other = ActionDispatch::Integration::Session.new(Rails.application)
|
||||||
|
["/static_mirror/jobs/#{ id }", "/static_mirror/files/#{ id }"].each do |path|
|
||||||
|
other.get path
|
||||||
|
expect(other.response).to have_http_status(:not_found)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'refuses ready artifacts after a new discard even for admins' do
|
||||||
|
id = create_job
|
||||||
|
StaticMirrorJob.perform_now(id)
|
||||||
|
record = create(:post)
|
||||||
|
PostVersion.create!(post: record, version_no: 1, event_type: 'discard',
|
||||||
|
url: record.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.current)
|
||||||
|
sign_in_as(create(:user, :admin))
|
||||||
|
get "/static_mirror/files/#{ id }"
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
get "/static_mirror/jobs/#{ id }"
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rechecks ready ZIPs after a non-discard content change' do
|
||||||
|
record = create(:post, title: 'mirror-withdrawn-title')
|
||||||
|
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
||||||
|
title: record.title, url: record.url, tags: '', tags_json: [], parent_post_ids: '',
|
||||||
|
created_at: Time.current)
|
||||||
|
id = create_job
|
||||||
|
StaticMirrorJob.perform_now(id)
|
||||||
|
record.update!(title: 'mirror-corrected-title')
|
||||||
|
get "/static_mirror/jobs/#{ id }"
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
get "/static_mirror/files/#{ id }"
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
expect(response.body).not_to include('mirror-withdrawn-title')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'fails lost jobs and rejects expired downloads' do
|
||||||
|
id = create_job
|
||||||
|
travel 80.seconds do
|
||||||
|
get "/static_mirror/jobs/#{ id }"
|
||||||
|
expect(json).to include('status' => 'failed', 'error' => 'worker_stopped')
|
||||||
|
end
|
||||||
|
travel 11.minutes do
|
||||||
|
get "/static_mirror/files/#{ id }"
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rejects a previously ready ZIP after its expiry' do
|
||||||
|
id = create_job
|
||||||
|
StaticMirrorJob.perform_now(id)
|
||||||
|
travel 11.minutes do
|
||||||
|
get "/static_mirror/files/#{ id }"
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
get "/static_mirror/jobs/#{ id }"
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'preserves the queued reservation after the cooldown' do
|
||||||
|
id = create_job
|
||||||
|
travel 61.seconds do
|
||||||
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||||
|
expect(response).to have_http_status(:too_many_requests)
|
||||||
|
expect(StaticMirrorJob.state).to include('id' => id, 'status' => 'queued')
|
||||||
|
expect(StaticMirrorJob).to have_received(:perform_later).once
|
||||||
|
StaticMirrorJob.perform_now(id)
|
||||||
|
get "/static_mirror/jobs/#{ id }"
|
||||||
|
expect(json).to include('status' => 'failed', 'error' => 'generation_timeout')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'reclaims a stalled queued reservation without waiting for a status request' do
|
||||||
|
id = create_job
|
||||||
|
travel 80.seconds do
|
||||||
|
second_id = create_job
|
||||||
|
expect(second_id).not_to eq(id)
|
||||||
|
StaticMirrorJob.perform_now(id)
|
||||||
|
expect(StaticMirrorJob.state).to include('id' => second_id, 'status' => 'queued')
|
||||||
|
StaticMirrorJob.perform_now(second_id)
|
||||||
|
get "/static_mirror/jobs/#{ second_id }"
|
||||||
|
expect(json).to include('status' => 'ready')
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'waits for brief intake lock contention instead of losing the queued worker' do
|
||||||
|
id = create_job
|
||||||
|
path = StaticMirrorJob.directory.join('.generation.lock')
|
||||||
|
lock = File.open(path, File::RDWR)
|
||||||
|
lock.flock(File::LOCK_EX)
|
||||||
|
attempts = Queue.new
|
||||||
|
worker = StaticMirrorJob.new
|
||||||
|
allow(worker).to receive(:generate)
|
||||||
|
allow(File).to receive(:open).and_call_original
|
||||||
|
allow(File).to receive(:open).with(path, File::RDWR).and_wrap_original do |original, *args, &block|
|
||||||
|
original.call(*args) do |file|
|
||||||
|
allow(file).to receive(:flock).and_wrap_original do |flock, mode|
|
||||||
|
attempts << mode
|
||||||
|
flock.call(mode)
|
||||||
|
end
|
||||||
|
block.call(file)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
thread = Thread.new { worker.perform(id) }
|
||||||
|
Timeout.timeout(5) do
|
||||||
|
expect(attempts.pop).to eq(File::LOCK_EX)
|
||||||
|
lock.close
|
||||||
|
thread.join
|
||||||
|
end
|
||||||
|
expect(worker).to have_received(:generate).with(id, hash_including('status' => 'queued'))
|
||||||
|
ensure
|
||||||
|
lock&.close unless lock&.closed?
|
||||||
|
thread&.kill
|
||||||
|
thread&.join
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'keeps the generation lock reserved even when stale metadata expires' do
|
||||||
|
create_job
|
||||||
|
lock = File.open(StaticMirrorJob.directory.join('.generation.lock'), File::RDWR)
|
||||||
|
lock.flock(File::LOCK_EX)
|
||||||
|
travel 80.seconds do
|
||||||
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||||
|
expect(response).to have_http_status(:too_many_requests)
|
||||||
|
end
|
||||||
|
ensure
|
||||||
|
lock&.close
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'reports bounded-generation failures without returning internal exception messages' do
|
||||||
|
id = create_job
|
||||||
|
allow(StaticMirror).to receive(:new).and_raise(StaticMirror::TooLarge, 'private details')
|
||||||
|
StaticMirrorJob.perform_now(id)
|
||||||
|
get "/static_mirror/jobs/#{ id }"
|
||||||
|
expect(json).to include('status' => 'failed', 'error' => 'too_large')
|
||||||
|
expect(response.body).not_to include('private details')
|
||||||
|
get "/static_mirror/files/#{ id }"
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'is unavailable outside development' do
|
||||||
|
allow(Rails.env).to receive(:development?).and_return(false)
|
||||||
|
get '/static_mirror'
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
get '/static_mirror/jobs/unknown'
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
get '/static_mirror/files/unknown'
|
||||||
|
expect(response).to have_http_status(:not_found)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'preserves user and IP BAN checks' do
|
||||||
|
user = create(:user, banned_at: Time.current)
|
||||||
|
sign_in_as(user)
|
||||||
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||||
|
expect(response).to have_http_status(:forbidden)
|
||||||
|
sign_out
|
||||||
|
IpAddress.create!(ip_address: IPAddr.new('127.0.0.1').hton, banned_at: Time.current)
|
||||||
|
get '/static_mirror', headers: { 'REMOTE_ADDR' => '127.0.0.1' }
|
||||||
|
expect(response).to have_http_status(:forbidden)
|
||||||
|
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'requires the normal CSRF token for anonymous generation', csrf: true do
|
||||||
|
get '/users/csrf'
|
||||||
|
token = json.fetch('token')
|
||||||
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||||
|
expect(response).to have_http_status(:forbidden)
|
||||||
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json,
|
||||||
|
headers: { 'X-CSRF-Token' => token }
|
||||||
|
expect(response).to have_http_status(:accepted)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,249 @@
|
|||||||
|
require 'rails_helper'
|
||||||
|
|
||||||
|
RSpec.describe StaticMirror do
|
||||||
|
include_context 'Ainu locale'
|
||||||
|
include ActiveSupport::Testing::TimeHelpers
|
||||||
|
|
||||||
|
let(:tag) { create(:tag, primary_name: 'mirror-public-tag') }
|
||||||
|
let(:post) { create(:post, title: '<img src=x onerror=alert(1)>') }
|
||||||
|
let(:mirror) { described_class.new(locale: Locale.nipponese, origin: 'https://source.test') }
|
||||||
|
|
||||||
|
def snapshot_tag(record, event: 'create', number: 1)
|
||||||
|
TagVersion.create!(tag: record, version_no: number, event_type: event,
|
||||||
|
name: record.name('ja'), category: record.category, aliases: '', parent_tag_ids: '',
|
||||||
|
names: { ja: { name: record.name('ja'), script_code: 'Jpan',
|
||||||
|
auto_generated: false, aliases: [] } }, created_at: Time.current)
|
||||||
|
end
|
||||||
|
|
||||||
|
def snapshot_post(record, tags: [])
|
||||||
|
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
||||||
|
title: record.title, url: record.url, tags: '', tags_json: tags,
|
||||||
|
parent_post_ids: '', created_at: Time.current)
|
||||||
|
end
|
||||||
|
|
||||||
|
# Read STORE entries independently of the production writer and verify CRCs.
|
||||||
|
def archive_files(bytes)
|
||||||
|
offset = 0
|
||||||
|
files = { }
|
||||||
|
while bytes.byteslice(offset, 4).unpack1('V') == 0x04034b50
|
||||||
|
header = bytes.byteslice(offset, 30).unpack('VvvvvvVVVvv')
|
||||||
|
size, name_size, extra_size = header[7], header[9], header[10]
|
||||||
|
path = bytes.byteslice(offset + 30, name_size).force_encoding('UTF-8')
|
||||||
|
data = bytes.byteslice(offset + 30 + name_size + extra_size, size)
|
||||||
|
expect(Zlib.crc32(data)).to eq(header[6])
|
||||||
|
files[path.delete_prefix('archive/')] = data.force_encoding('UTF-8')
|
||||||
|
offset += 30 + name_size + extra_size + size
|
||||||
|
end
|
||||||
|
expect(bytes.byteslice(-22, 4).unpack1('V')).to eq(0x06054b50)
|
||||||
|
files
|
||||||
|
end
|
||||||
|
|
||||||
|
before do
|
||||||
|
snapshot_tag(tag)
|
||||||
|
PostTag.create!(post:, tag:)
|
||||||
|
snapshot_post(post, tags: [{ tag_id: tag.id, version_no: 1, sections: [] }])
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'generates a navigable ZIP, escaped HTML, search data and verified hashes' do
|
||||||
|
progress = []
|
||||||
|
files = archive_files(mirror.export { |done, total| progress << [done, total] })
|
||||||
|
expect(progress.last).to eq([2, 2])
|
||||||
|
expect(files.keys).to include('index.html', 'robots.txt', 'assets/archive.css',
|
||||||
|
'assets/archive.js', 'archive-config.js', 'search-index.json', 'manifest.json',
|
||||||
|
'ja/posts/index.html', "ja/posts/#{ post.id }/index.html",
|
||||||
|
'ja/tags/index.html', "ja/tags/#{ tag.id }/index.html")
|
||||||
|
files.select { |path, _| path.end_with?('.html') }.each do |path, html|
|
||||||
|
document = Nokogiri::HTML(html)
|
||||||
|
expect(document.at_css('meta[name="robots"]')['content']).to include('noindex')
|
||||||
|
expect(document.text).to include('静的保存版', mirror.data_at)
|
||||||
|
expect(document.css('img')).to be_empty
|
||||||
|
expect(document.at_css('[data-original]')['href']).to start_with('https://source.test/ja')
|
||||||
|
document.css('a[href], link[href], script[src]').each do |node|
|
||||||
|
target = node['href'] || node['src']
|
||||||
|
next if target.start_with?('https://')
|
||||||
|
|
||||||
|
expect(target).not_to start_with('/')
|
||||||
|
resolved = Pathname.new(path).dirname.join(target).cleanpath.to_s
|
||||||
|
expect(files).to have_key(resolved)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
detail = Nokogiri::HTML(files.fetch("ja/posts/#{ post.id }/index.html"))
|
||||||
|
expect(detail.at_css('h1').text).to eq(post.title)
|
||||||
|
expect(detail.css('main a').map { _1['href'] })
|
||||||
|
.to include("../../tags/#{ tag.id }/index.html")
|
||||||
|
manifest = JSON.parse(files.fetch('manifest.json'))
|
||||||
|
expect(manifest).to include('format_version' => 1, 'locale' => 'ja',
|
||||||
|
'data_at' => mirror.data_at, 'data_at_basis' => 'capture_started_at',
|
||||||
|
'resources' => { 'posts' => 1, 'tags' => 1 })
|
||||||
|
expect(Time.iso8601(manifest.fetch('generated_at'))).to be >= Time.iso8601(mirror.data_at)
|
||||||
|
expect(manifest.fetch('files').map { _1.fetch('path') })
|
||||||
|
.to match_array(files.keys - ['manifest.json'])
|
||||||
|
manifest.fetch('files').each do |entry|
|
||||||
|
digest = Digest::SHA256.hexdigest(files.fetch(entry.fetch('path')))
|
||||||
|
expect(entry.fetch('sha256')).to eq(digest)
|
||||||
|
end
|
||||||
|
expect(JSON.parse(files.fetch('search-index.json')).map { _1.fetch('path') })
|
||||||
|
.to include("ja/posts/#{ post.id }/index.html", "ja/tags/#{ tag.id }/index.html")
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'keeps private and unversioned records out even for an admin caller' do
|
||||||
|
private_tag = create(:tag, primary_name: 'mirror-secret-tag')
|
||||||
|
snapshot_tag(private_tag)
|
||||||
|
snapshot_tag(private_tag, event: 'discard', number: 2)
|
||||||
|
private_post = create(:post, title: 'mirror-secret-post')
|
||||||
|
snapshot_post(private_post)
|
||||||
|
PostVersion.create!(post: private_post, version_no: 2, event_type: 'discard',
|
||||||
|
url: private_post.url, tags: '', tags_json: [], parent_post_ids: '',
|
||||||
|
created_at: Time.current)
|
||||||
|
legacy = create(:post, title: 'mirror-unversioned-secret')
|
||||||
|
referencing = create(:post, title: 'mirror-secret-reference')
|
||||||
|
snapshot_post(referencing,
|
||||||
|
tags: [{ tag_id: private_tag.id, version_no: 1, sections: [] }])
|
||||||
|
files = HistoryAccess.set(user: create(:user, :admin)) { archive_files(mirror.export) }
|
||||||
|
expect(files.values.join).not_to include('mirror-secret', legacy.title, private_post.url)
|
||||||
|
expect(files.keys).not_to include("ja/tags/#{ private_tag.id }/index.html",
|
||||||
|
"ja/posts/#{ private_post.id }/index.html", "ja/posts/#{ referencing.id }/index.html")
|
||||||
|
expect(mirror.authority.fetch('publication_policy'))
|
||||||
|
.to eq('static-mirror-public-projection-1')
|
||||||
|
expect(mirror.authority.keys).not_to include('admin', 'actor_id', 'artifact_policy')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'applies later discard boundaries after a restoration' do
|
||||||
|
snapshot_tag(tag, event: 'discard', number: 2)
|
||||||
|
snapshot_tag(tag, event: 'restore', number: 3)
|
||||||
|
tag.update_columns(version_no: 3)
|
||||||
|
expect(archive_files(mirror.export)).to have_key("ja/tags/#{ tag.id }/index.html")
|
||||||
|
snapshot_tag(tag, event: 'discard', number: 4)
|
||||||
|
second = described_class.new(locale: Locale.nipponese, origin: 'https://source.test')
|
||||||
|
files = archive_files(second.export)
|
||||||
|
expect(files.keys).not_to include("ja/tags/#{ tag.id }/index.html")
|
||||||
|
expect(files.values.join).not_to include('mirror-public-tag')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'does not include credentials or executable URL schemes' do
|
||||||
|
post.update_columns(url: 'https://secret-user:secret-password@source.test/path')
|
||||||
|
content = archive_files(mirror.export).values.join
|
||||||
|
expect(content).not_to include('secret-password', 'secret-user')
|
||||||
|
post.update_columns(url: 'javascript:alert(1)')
|
||||||
|
expect(archive_files(mirror.export).values.join).not_to include('javascript:')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'fails if publication changes during generation' do
|
||||||
|
expect do
|
||||||
|
mirror.export do |done, _|
|
||||||
|
snapshot_tag(tag, event: 'discard', number: 2) if done.zero?
|
||||||
|
end
|
||||||
|
end.to raise_error(HistoryAccess::Forbidden)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'owns the static publication contract independently of Counts' do
|
||||||
|
allow(HistoryAccess).to receive(:authority).and_raise('Counts metadata is not applicable')
|
||||||
|
allow(HistoryAccess).to receive(:authorised_artifact?).and_return(false)
|
||||||
|
archive_files(mirror.export)
|
||||||
|
expect(described_class.authorised_artifact?(mirror.authority)).to be(true)
|
||||||
|
metadata = mirror.authority.except('publication_policy')
|
||||||
|
.merge('artifact_policy' => HistoryAccess::ARTIFACT_POLICY)
|
||||||
|
expect(described_class.authorised_artifact?(metadata)).to be(false)
|
||||||
|
stub_const('HistoryAccess::ARTIFACT_POLICY', 'another-counts-policy')
|
||||||
|
expect(described_class.authorised_artifact?(mirror.authority)).to be(true)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rejects stale projections after edits without discard, including for admins' do
|
||||||
|
create(:tag_name, :alias, tag:, name: 'mirror-old-alias')
|
||||||
|
archive_files(mirror.export)
|
||||||
|
HistoryAccess.set(user: create(:user, :admin)) do
|
||||||
|
expect(described_class.authorised_artifact?(mirror.authority)).to be(true)
|
||||||
|
TagName.find_by!(tag:, name: 'mirror-old-alias').update!(name: 'mirror-new-alias')
|
||||||
|
expect(described_class.authorised_artifact?(mirror.authority)).to be(false)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rejects a deprecated tag and its names without requiring a discard' do
|
||||||
|
archive_files(mirror.export)
|
||||||
|
tag.update!(deprecated_at: Time.current)
|
||||||
|
expect(described_class.authorised_artifact?(mirror.authority)).to be(false)
|
||||||
|
files = archive_files(mirror.export)
|
||||||
|
expect(files.keys).not_to include("ja/tags/#{ tag.id }/index.html")
|
||||||
|
expect(files.values.join).not_to include('mirror-public-tag')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rejects stale post content and relations without a discard' do
|
||||||
|
archive_files(mirror.export)
|
||||||
|
post.update!(title: 'mirror-corrected-title')
|
||||||
|
expect(described_class.authorised_artifact?(mirror.authority)).to be(false)
|
||||||
|
archive_files(mirror.export)
|
||||||
|
PostTag.where(post:, tag:).delete_all
|
||||||
|
expect(described_class.authorised_artifact?(mirror.authority)).to be(false)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'rejects an artifact after its locale is deprecated' do
|
||||||
|
archive_files(mirror.export)
|
||||||
|
Locale.nipponese.update!(deprecated_at: Time.current)
|
||||||
|
expect(described_class.authorised_artifact?(mirror.authority)).to be(false)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'fails if a tag is deprecated during generation' do
|
||||||
|
expect do
|
||||||
|
mirror.export do |processed, _total|
|
||||||
|
tag.update!(deprecated_at: Time.current) if processed.zero?
|
||||||
|
end
|
||||||
|
end.to raise_error(HistoryAccess::Forbidden)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'records capture entry rather than claiming the exact DB read-view time' do
|
||||||
|
freeze_time do
|
||||||
|
capture_at = Time.current.utc.iso8601(6)
|
||||||
|
allow(CountsExporter).to receive(:with_read_snapshot).and_wrap_original do |original, &block|
|
||||||
|
travel 5.seconds
|
||||||
|
original.call(&block)
|
||||||
|
end
|
||||||
|
files = archive_files(mirror.export)
|
||||||
|
manifest = JSON.parse(files.fetch('manifest.json'))
|
||||||
|
expect(manifest).to include('data_at' => capture_at, 'data_at_basis' => 'capture_started_at')
|
||||||
|
expect(Time.iso8601(manifest.fetch('generated_at'))).to be > Time.iso8601(capture_at)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'bounds output and never reports hidden population counts' do
|
||||||
|
stub_const('StaticMirror::RESOURCE_LIMIT', 1)
|
||||||
|
second = create(:post, title: 'mirror-beyond-limit')
|
||||||
|
snapshot_post(second)
|
||||||
|
files = archive_files(mirror.export)
|
||||||
|
expect(files.values.join).not_to include(second.title, 'population_count', 'excluded_count')
|
||||||
|
stub_const('StaticMirror::MAX_BYTES', 16)
|
||||||
|
expect { mirror.export }.to raise_error(StaticMirror::TooLarge)
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'keeps all registered language names and aliases without locale filtering' do
|
||||||
|
locale
|
||||||
|
create(:tag_name, :alias, tag:, name: 'mirror-alias', language_code: 'ja', script_code: 'Jpan')
|
||||||
|
files = archive_files(described_class.new(locale:, origin: 'https://source.test').export)
|
||||||
|
detail = files.fetch("ain-Kana/tags/#{ tag.id }/index.html")
|
||||||
|
expect(detail).to include('mirror-public-tag', 'mirror-alias', 'lang="ain"', 'lang="ja"')
|
||||||
|
index = JSON.parse(files.fetch('search-index.json'))
|
||||||
|
entry = index.find { _1.fetch('path') == "ain-Kana/posts/#{ post.id }/index.html" }
|
||||||
|
expect(entry.fetch('names')).to include('mirror-public-tag', 'mirror-alias')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'indexes only names of public linked tags within the output limit' do
|
||||||
|
other = create(:tag, primary_name: 'mirror-unlinked-tag')
|
||||||
|
snapshot_tag(other)
|
||||||
|
private_tag = create(:tag, primary_name: 'mirror-private-tag')
|
||||||
|
snapshot_tag(private_tag, event: 'discard')
|
||||||
|
PostTag.create!(post:, tag: private_tag)
|
||||||
|
files = archive_files(mirror.export)
|
||||||
|
index = JSON.parse(files.fetch('search-index.json'))
|
||||||
|
entry = index.find { _1.fetch('path') == "ja/posts/#{ post.id }/index.html" }
|
||||||
|
expect(entry.fetch('names')).to include('mirror-public-tag')
|
||||||
|
expect(entry.fetch('names')).not_to include('mirror-private-tag', 'mirror-unlinked-tag')
|
||||||
|
stub_const('StaticMirror::RESOURCE_LIMIT', 1)
|
||||||
|
expect(archive_files(mirror.export).fetch('search-index.json'))
|
||||||
|
.not_to include('mirror-unlinked-tag', 'mirror-private-tag')
|
||||||
|
end
|
||||||
|
|
||||||
|
it 'fails rather than truncating registered aliases at the name limit' do
|
||||||
|
create(:tag_name, :alias, tag:, name: 'mirror-extra-alias')
|
||||||
|
stub_const('StaticMirror::NAME_LIMIT', 1)
|
||||||
|
expect { mirror.export }.to raise_error(StaticMirror::TooLarge)
|
||||||
|
end
|
||||||
|
end
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
# Static Mirror PoC (#483)
|
||||||
|
|
||||||
|
現在の公開状態の一部を、Rails で HTML にして ZIP 出力する開発環境専用の PoC。
|
||||||
|
Post は ID・タイトル・URL・出力済み Tag との関連、Tag は ID・カテゴリ・全言語の
|
||||||
|
名前と別名・出力済み Post との関連を出力する。それ以外の Post / Tag 属性は対象外。
|
||||||
|
DB backup / restore とは別機能で、日時指定の復元機能はない。
|
||||||
|
本番環境では全 API が 404 になる。依存パッケージ・DB schema の変更はない。
|
||||||
|
|
||||||
|
## 操作
|
||||||
|
|
||||||
|
既存のローカル開発用 MySQL 設定と locale 登録を用意した環境で、
|
||||||
|
`backend/` の `bin/rails server -p 3002` と、`frontend/` の `npm run dev` を使う。
|
||||||
|
Vite の既存 `/api` proxy は3002番ポートを参照する。これらの起動コマンドは
|
||||||
|
今回実行していない。DB の準備・migration・依存導入も実行していない。
|
||||||
|
|
||||||
|
1. `http://localhost:5173/ja/static-mirror` を開く。開発時の「その他」にもリンクがある。
|
||||||
|
有効な locale では `/:locale/static-mirror` から利用できる。
|
||||||
|
2. 「スナップショット生成」を押す。ログインは不要。通常の CSRF、IP BAN、user BAN
|
||||||
|
の境界を通り、セッションに紐づくジョブを登録する。
|
||||||
|
3. 状態・処理件数・データ基準時点を確認し、完了後「ZIP ダウンロード」を押す。
|
||||||
|
4. ZIP を展開し、`archive/` を通常の静的 HTTP サーバーへ配置する。
|
||||||
|
`/保存版/archive/index.html` のようなサブディレクトリ配置に対応する。
|
||||||
|
DB、Node.js、原サーバー、外部 API は閲覧先では不要。
|
||||||
|
|
||||||
|
一覧・詳細・相互リンクは JavaScript 無効でも利用できる。検索は HTTP 配信した
|
||||||
|
`search-index.json` を JavaScript が読む。`file://` からの検索は保証しない。
|
||||||
|
Post はタイトルに加えて、出力対象の関連 Tag の全言語名・別名から検索できる。
|
||||||
|
非公開・廃止・件数上限外の Tag や、関連しない Tag の名前は Post の検索項目に含めない。
|
||||||
|
外部動画・画像・サムネール・素材ファイルは取得も同梱もしない。Post の外部 URL は
|
||||||
|
通常のリンクとして残すため、そのリンク先の閲覧は外部サイトの存続に依存する。
|
||||||
|
|
||||||
|
各 HTML に静的保存版の表示、UTC のデータ基準時点、`noindex, nofollow`、
|
||||||
|
本家への絶対リンクを焼き込む。内部リンク・CSS・JS は相対参照。
|
||||||
|
本家リンクの初期オリジンは `https://hub.nizika.monster`。request の Host、
|
||||||
|
forwarded header、API のオリジンは採用しない。
|
||||||
|
`archive-config.js` の `window.archiveConfig.origin` を HTTP(S) の本家基点 URL
|
||||||
|
(例: `https://example.org`)に変更すれば本家リンクをまとめて変更できる。
|
||||||
|
設定 JS が削除・欠落・不正な場合や JS 無効時にも HTML の初期リンクが残る。
|
||||||
|
設定は配置先ミラーの URL ではなく、本家の基点を指す。
|
||||||
|
|
||||||
|
## API と処理上限
|
||||||
|
|
||||||
|
| API | 役割 |
|
||||||
|
| --- | --- |
|
||||||
|
| `GET /static_mirror` | 件数・容量・実行制限と対象外機能 |
|
||||||
|
| `POST /static_mirror/jobs` | 現在状態のジョブ登録。入力は `locale` のみ |
|
||||||
|
| `GET /static_mirror/jobs/:id` | 所有セッションの状態・処理進捗・基準時点・失敗コード |
|
||||||
|
| `GET /static_mirror/files/:id` | 所有セッションの完成 ZIP を再認可して取得 |
|
||||||
|
|
||||||
|
通常どおりフロントエンドの API transport と Query key を使用する。
|
||||||
|
URL に JSON を格納しない。`at` 等の任意時点入力・未知の生成オプションは400で拒否する。
|
||||||
|
|
||||||
|
- Post / Tag は各100件。公開認可後に ID 昇順で選ぶ。非公開件数や全母集団件数は返さない。
|
||||||
|
- 対象 Tag の全言語名・別名は最大2,000行。超過時は失敗させ、黙って切り捨てない。
|
||||||
|
言語・文字体系は保持し、表示対象言語に登録がない場合は ID を使う。
|
||||||
|
- Post と Tag の関連は両者が出力集合にあるものだけ。出力外への内部リンクは生成しない。
|
||||||
|
- 展開後のファイル合計及び ZIP は各8 MiB 以下。超過時は配布しない。
|
||||||
|
- 同時実行1件、全セッションを通じて新規登録は60秒に1件、登録から処理期限60秒。
|
||||||
|
競合・受付制限は429と `Retry-After`。状態取得・取得時の短いロック競合は503。
|
||||||
|
- ZIP とジョブの認可期限は10分。次のジョブ登録で前の ZIP / ジョブは置き換える。
|
||||||
|
queued / running の予約は cooldown が過ぎても置き換えない。75秒を超えた予約は
|
||||||
|
状態取得又は次の受付で failed にし、実行ロックが空いている場合だけ次を受け付ける。
|
||||||
|
期限後も最後のファイルがディスクに残り得るが、API では取得できない。
|
||||||
|
容量は単一 ZIP と作業ファイルに限定され、無制限に蓄積しない。
|
||||||
|
|
||||||
|
Rails の既存 Active Job と `:async` adapter を使い、生成は HTTP request の外で行う。
|
||||||
|
既存 `CountsExportJobs` は Counts の入力・投影・配信確認に結合しているため変更せず、
|
||||||
|
本機能の状態・単一成果物の管理だけを feature 固有の Job に置く。
|
||||||
|
受付・状態・ZIP は `backend/tmp/static_mirror/` の単一 spool を使う。
|
||||||
|
プロセス間の filesystem lock で実行枠と更新を制御する。各ファイルは0600、directory は
|
||||||
|
0700で作成する。Web プロセス再起動で非永続キューが失われる点が PoC の限界。
|
||||||
|
進まない queued / running は75秒経過後の状態取得又は次の受付で failed になる。
|
||||||
|
worker は受付中の短い実行ロック競合を待ち、競合だけを理由に queued を放棄しない。
|
||||||
|
再開や再実行はしない。実行ロック待ちも登録からの60秒の処理期限に含まれる。
|
||||||
|
|
||||||
|
失敗コードは `too_large`、`generation_timeout`、`publication_changed`、
|
||||||
|
`generation_failed`、`worker_stopped`。内部の例外本文・パス・資源属性は応答に含めない。
|
||||||
|
画面では失敗コード又は既存の通信・認可エラー文言を表示する。
|
||||||
|
|
||||||
|
## 公開制御と整合性
|
||||||
|
|
||||||
|
生成者の role によらず `HistoryAccess.set(user: nil)` の公開権限を使う。
|
||||||
|
既存の `HistoryAccess.current` と確定済みの同一 `version_no` の存在確認を組み合わせる。
|
||||||
|
独自の role 判定や簡易な除票判定は作らない。対応する immutable revision がない
|
||||||
|
legacy record は安全側で対象外とする。公開不可の参照を含む確定版も既存認可で除外する。
|
||||||
|
廃止 Tag は対象外。HTML / 検索 JSON / manifest の件数には認可済みの出力集合だけが入る。
|
||||||
|
user、actor、履歴版番号、監査、認証、blob 情報は投影しない。
|
||||||
|
|
||||||
|
公開条件の調査範囲は V1 の Post / Tag / TagName / Locale の model、schema、
|
||||||
|
閲覧 controller・representation、HistoryAccess と Material の抑止経路。
|
||||||
|
Post / Tag / TagName には別の private / visibility カラムや、汎用の公開抑止 scope は
|
||||||
|
見つからなかった。Tag の `deprecated_at` は除票とは別の廃止状態で、通常の Post
|
||||||
|
表示からも除外されるため、ミラーから引き続き除外する。Locale の廃止後も配布しない。
|
||||||
|
`MaterialImportBlock`、`MaterialSyncSuppression` は著作権・個人情報等を理由にした
|
||||||
|
素材取込・同期の抑止であり、Post / Tag の汎用認可として流用しない。Material と
|
||||||
|
関連ファイル・監査用の理由・同期元情報は全て出力対象外のままにする。
|
||||||
|
既存の通常 Post / Tag 閲覧経路が共通認可を一律適用しているとは仮定せず、
|
||||||
|
ミラー自身は必ず HistoryAccess を経由する。新たな抑止条件を導入する場合には
|
||||||
|
適用範囲を確認して共通認可と静的配布契約を更新する必要がある。
|
||||||
|
|
||||||
|
`CountsExporter.with_read_snapshot` の既存 repeatable-read 境界を再利用し、Post、Tag、
|
||||||
|
全言語 TagName、PostTag、publication revision を同じ読み取り view から取得する。
|
||||||
|
これには既存 HistoryAccess の SQL が動作する MySQL 環境が必要。未対応の adapter や
|
||||||
|
未整備の DB では生成失敗とし、認可を迂回した出力へフォールバックしない。
|
||||||
|
`data_at` は DB 読み取り transaction を要求する直前の、取得処理開始時の
|
||||||
|
サーバー時計(UTC)。manifest の `data_at_basis: capture_started_at` に意味を記録する。
|
||||||
|
MySQL の read view は後の最初の consistent read で成立するため、その厳密な成立時刻や
|
||||||
|
`data_at` 以前のコミットだけを含むことは保証しない。履歴の指定日時復元も表さない。
|
||||||
|
独立した worker では repeatable-read を開始するが、呼出元の transaction がある場合は
|
||||||
|
既存 helper の契約どおり、その transaction の view と分離レベルを引き継ぐ。
|
||||||
|
`generated_at` はファイル生成後の別の時刻。
|
||||||
|
|
||||||
|
静的配布契約は `StaticMirror::PUBLICATION_POLICY` の
|
||||||
|
`static-mirror-public-projection-1`。Counts 用 `HistoryAccess::ARTIFACT_POLICY`、
|
||||||
|
`authority`、`authorised_artifact?` は使わない。共通 HistoryAccess の現在版認可と
|
||||||
|
全後続除票の revision 検知は再利用し、集計件数の例外や admin の配布モードは設けない。
|
||||||
|
旧 Counts 契約の metadata しかない ZIP は再生成を必要とする。
|
||||||
|
|
||||||
|
生成の最後と状態・ZIP 取得時に、別の読み取り transaction で共通 discard revision と
|
||||||
|
出力対象 ID の公開投影を再取得する。SQL query cache を使わず、対象の存在、確定版、
|
||||||
|
Tag / Locale の廃止、タイトル・URL・category、全言語名・別名・文字体系、関連の変更を
|
||||||
|
検知する。除票を伴わない訂正・削除も投影 digest の不一致として配布を止める。
|
||||||
|
対象 ID と digest はサーバー側ジョブ metadata にだけ置き、manifest や API 状態応答へ
|
||||||
|
出さない。対象は各100件・名前2,000行以内の一括取得で、行ごとの追加 query は行わない。
|
||||||
|
生成中の除票は失敗にし、生成後の除票では古い ZIP の再取得を404にする。
|
||||||
|
ZIP は再認可後のバイト列として返し、遅延 sendfile に古いパスを渡さない。
|
||||||
|
HTTP 応答は既存境界の `private, no-store`。ダウンロード済み ZIP の回収や、
|
||||||
|
最終認可と送出の間に発生する公開制御変更の完全な排他は保証できない。
|
||||||
|
新しい公開制御を導入する場合は共通 policy と静的配布契約の連動が必要。
|
||||||
|
|
||||||
|
本文・名前・URL は escape する。外部リンクは HTTP(S)・host・userinfo 不在を検証し、
|
||||||
|
不正な URL はリンクとして出さない。外部 URL の取得は行わないため SSRF 用 DNS 検査を
|
||||||
|
流用しない。HTML の CSP は外部スクリプト・画像・通信を許可しない。
|
||||||
|
|
||||||
|
`manifest.json` に形式版、生成日時、基準時点、locale、出力件数、対象外項目、上限、
|
||||||
|
各ファイルの相対パスと SHA-256 を記録する。自己ハッシュは成立しないため manifest
|
||||||
|
自身だけは一覧の対象外とし、`checksum_scope` にその範囲を記録する。
|
||||||
|
|
||||||
|
## 過去時点復元の調査
|
||||||
|
|
||||||
|
- `post_versions` / `tag_versions` は親 ID と正の `version_no` に一意制約があり、
|
||||||
|
readonly の `VersionRecord` を使う。`created_at <= 時点` の中で最大の確定版を
|
||||||
|
選ぶことは可能。同一時刻の順序は時刻だけで決めず版番号を併用する必要がある。
|
||||||
|
- Tag は `names` に言語別の名前・script・別名を保存する。古い行の移行では
|
||||||
|
日本語として backfill し、旧 aliases 文字列を分割している。失われた区切りや
|
||||||
|
当時未記録の他言語名を現在の TagName で補完して正確と称することはできない。
|
||||||
|
- Post は `tags_json`、区間、Tag 版参照、`parent_post_ids` を、Tag は
|
||||||
|
`parent_tag_ids` を記録する。ただし legacy `tags_json` は名前・時刻からの補完を
|
||||||
|
含む。全時代・全書き込み経路の関係変更の完全性を検証する必要がある。
|
||||||
|
外部 Tag 名の現行参照など、immutable history だけでは揃わない値もある。
|
||||||
|
- Tag Description は `(tag_id, language_code, version_no)` の版に本文と翻訳元を記録する。
|
||||||
|
Material にはファイル情報等の版があるが、blob 本体の永久保存や除票・公開抑止との
|
||||||
|
整合は別途必要。Help はフロントエンドの文書配信に依存し、DB の任意時点だけでは
|
||||||
|
当時の公開版を復元できない。
|
||||||
|
- 当時の publication policy そのものや将来追加される公開抑止の完全な履歴は、
|
||||||
|
これらの内容版だけでは保証できない。指定時点の再現と現在の公開許可を別々に評価し、
|
||||||
|
現在の全後続除票境界を版番号と複合キーで適用する必要がある。恢復しても過去の
|
||||||
|
秘匿履歴を再公開しない。Counts の件数例外はこの機能へ拡張しない。
|
||||||
|
- 正確な時点復元には、全 resource の履歴の起点・欠落・backfill の識別、関係・名前・
|
||||||
|
policy・locale・asset の時点整合性、immutable な Help の版、削除済み asset の保存方針、
|
||||||
|
一つの DB read view と現在の認可の再検証、同時刻更新の順序・境界の設計が必要。
|
||||||
|
|
||||||
|
このため日時入力は実装していない。履歴評価はソース調査であり、実 DB の網羅監査ではない。
|
||||||
|
|
||||||
|
## 検証と残課題
|
||||||
|
|
||||||
|
追加した回帰テスト:
|
||||||
|
|
||||||
|
- `backend/spec/services/static_mirror_spec.rb`: ZIP の CRC / 構造、全ファイルの hash、
|
||||||
|
内部参照、escape、基準時点、公開権限、後続除票、言語名・別名、容量・件数上限。
|
||||||
|
今回は Counts 契約との独立性、除票なしの訂正・廃止・関連変更、取得開始時刻の意味、
|
||||||
|
Post 検索への関連名の投影を追加。
|
||||||
|
- `backend/spec/requests/static_mirrors_spec.rb`: 一般受付、非同期境界、所有権、
|
||||||
|
公開制御変更、失敗・期限・競合・全体 rate limit、開発環境制限、CSRF と BAN。
|
||||||
|
今回は Host に依存しない本家リンク、訂正後の取得拒否、queued 予約の保持・期限切れ、
|
||||||
|
実 filesystem lock を使う受付競合待ちを追加。
|
||||||
|
- `frontend/src/pages/StaticMirrorPage.test.tsx`: 開始、進捗、取得、失敗、
|
||||||
|
受付エラー、認可喪失、日本語 fallback の言語属性。
|
||||||
|
- `frontend/src/lib/staticMirrorAssets.test.ts`: 静的検索の相対参照・escape と
|
||||||
|
サブディレクトリ配備、本家設定の欠落・不正・変更。
|
||||||
|
今回は Tag 名・別名から関連 Post が見つかるケースを追加。検索 JS の変更は不要。
|
||||||
|
|
||||||
|
実施したのは既存コードの調査、差分の目視確認と `git diff --check`(成功)。
|
||||||
|
テスト、lint、build、型検査、
|
||||||
|
migration、実ブラウザーでの表示、ZIP 展開・HTTP 配置は実行していない。
|
||||||
|
管理者指定の suite による結果確認と、サブディレクトリ配置・JS 無効・原サーバー停止・
|
||||||
|
設定 JS 欠落・静的検索の実画面確認が残る。
|
||||||
|
|
||||||
|
本番公開には durable な worker / queue、復旧・取消・削除と運用監視、適切な利用者別
|
||||||
|
rate limit、容量・時間・負荷の実測、公開制御の監査と変更検知、長時間配信の再認可方針、
|
||||||
|
asset の権利・保管方針、追加 resource の安全な投影、UI 文言・翻訳の確定が必要。
|
||||||
|
本 PoC の開発環境制限を単に外して公開することは想定しない。
|
||||||
|
|
||||||
|
## 今回のレビュー対応の変更ファイル
|
||||||
|
|
||||||
|
| ファイル | 修正と理由 |
|
||||||
|
| --- | --- |
|
||||||
|
| `backend/app/controllers/static_mirrors_controller.rb` | request オリジンを生成処理へ渡さず、本家リンクへの Host 混入を防止 |
|
||||||
|
| `backend/app/jobs/static_mirror_job.rb` | 静的配布専用の再認可、queued 予約と期限切れの共通処理、worker のロック待ちでジョブ消失を防止 |
|
||||||
|
| `backend/app/services/static_mirror.rb` | Counts と独立した公開投影の再検証、関連 Tag 名を含む Post 検索、正規の本家リンク初期値、`data_at_basis` |
|
||||||
|
| `backend/spec/services/static_mirror_spec.rb` | 専用契約・訂正・廃止・関連変更・検索投影・時刻意味の回帰テスト |
|
||||||
|
| `backend/spec/requests/static_mirrors_spec.rb` | 本家オリジン・ZIP 再認可・queued 予約・実行ロック競合の回帰テスト |
|
||||||
|
| `frontend/src/lib/staticMirrorAssets.test.ts` | 名前・別名を使う Post 検索の回帰テスト |
|
||||||
|
| `docs/static-mirror.md` | 公開条件の調査結果、配布契約、ロックと時刻の保証範囲・未解決事項を更新 |
|
||||||
|
|
||||||
|
V2、DB schema / migration、既存の Counts 契約・共通認可、UI 文言は変更していない。
|
||||||
|
|
||||||
|
## 変更ファイルの責務
|
||||||
|
|
||||||
|
| ファイル | 責務 |
|
||||||
|
| --- | --- |
|
||||||
|
| `backend/config/routes.rb` | metadata・job・ZIP の4経路 |
|
||||||
|
| `backend/app/controllers/static_mirrors_controller.rb` | 通常認証境界、開発制限、入力・HTTP 応答 |
|
||||||
|
| `backend/app/jobs/static_mirror_job.rb` | 単一生成枠、受付制限、状態・成果物・期限・取得認可 |
|
||||||
|
| `backend/app/services/static_mirror.rb` | 公開データの一貫した投影、ページ・検索・manifest・ZIP |
|
||||||
|
| `backend/app/views/static_mirror/page.html.erb` | escape 済み HTML、ナビ、基準時点・本家リンク・CSP |
|
||||||
|
| `backend/app/views/static_mirror/archive.css` | 静的サイトの読み取り表示 |
|
||||||
|
| `backend/app/views/static_mirror/archive.js` | ローカル検索と本家リンクの設定変更 |
|
||||||
|
| `frontend/src/App.tsx` | locale 付き操作画面経路 |
|
||||||
|
| `frontend/src/lib/queryKeys.ts` | 既存 Query key entrypoint への追加 |
|
||||||
|
| `frontend/src/pages/StaticMirrorPage.tsx` | 開始・制限・進捗・失敗・ZIP 操作 |
|
||||||
|
| `frontend/src/pages/MorePage.tsx` | 開発時の画面への導線 |
|
||||||
|
| `frontend/src/locales/{ja,ain-Kana}/archive.json` | Issue 指定の文言と未翻訳 fallback |
|
||||||
|
| 上記4テストファイル | 回帰テスト |
|
||||||
|
| `docs/static-mirror.md` | 手順・制約・調査・未検証事項 |
|
||||||
@@ -83,6 +83,7 @@ const MaterialSyncSuppressionsPage = lazyPage (
|
|||||||
() => import ('@/pages/materials/MaterialSyncSuppressionsPage'))
|
() => import ('@/pages/materials/MaterialSyncSuppressionsPage'))
|
||||||
const CountsResourcePage = lazyPage (() => import ('@/pages/counts/CountsResourcePage'))
|
const CountsResourcePage = lazyPage (() => import ('@/pages/counts/CountsResourcePage'))
|
||||||
const MorePage = lazyPage (() => import ('@/pages/MorePage'))
|
const MorePage = lazyPage (() => import ('@/pages/MorePage'))
|
||||||
|
const StaticMirrorPage = lazyPage (() => import ('@/pages/StaticMirrorPage'))
|
||||||
const GekanatorUnavailablePage = lazyPage (() => import ('@/pages/GekanatorUnavailablePage'))
|
const GekanatorUnavailablePage = lazyPage (() => import ('@/pages/GekanatorUnavailablePage'))
|
||||||
const NicoTagListPage = lazyPage (() => import ('@/pages/tags/NicoTagListPage'))
|
const NicoTagListPage = lazyPage (() => import ('@/pages/tags/NicoTagListPage'))
|
||||||
const TOSPage = lazyPage (() => import ('@/pages/TOSPage'))
|
const TOSPage = lazyPage (() => import ('@/pages/TOSPage'))
|
||||||
@@ -126,6 +127,8 @@ export const pageRoutes = (
|
|||||||
user: User | null,
|
user: User | null,
|
||||||
setUser: Dispatch<SetStateAction<User | null>>,
|
setUser: Dispatch<SetStateAction<User | null>>,
|
||||||
): PageRoute[] => [
|
): PageRoute[] => [
|
||||||
|
{ path: 'static-mirror', element: <StaticMirrorPage/>,
|
||||||
|
handle: { preload: StaticMirrorPage.preload, robots: 'noindex' } },
|
||||||
{ path: '', element: <LocaleRedirect to="/posts"/>,
|
{ path: '', element: <LocaleRedirect to="/posts"/>,
|
||||||
handle: { redirectTo: '/posts' } },
|
handle: { redirectTo: '/posts' } },
|
||||||
{ path: 'posts', element: <PostListPage/>,
|
{ path: 'posts', element: <PostListPage/>,
|
||||||
|
|||||||
@@ -7,6 +7,10 @@ import type {
|
|||||||
MaterialFilter,
|
MaterialFilter,
|
||||||
WikiTagReference } from '@/types'
|
WikiTagReference } from '@/types'
|
||||||
|
|
||||||
|
export const staticMirrorKeys = {
|
||||||
|
definition: ['static_mirror', 'definition'] as const,
|
||||||
|
job: (id: string | null) => ['static_mirror', 'jobs', id] as const }
|
||||||
|
|
||||||
export const localesKeys = {
|
export const localesKeys = {
|
||||||
root: ['locales'] as const,
|
root: ['locales'] as const,
|
||||||
index: ['locales', 'index'] as const }
|
index: ['locales', 'index'] as const }
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { readFileSync } from 'node:fs'
|
||||||
|
import { URL as NodeURL } from 'node:url'
|
||||||
|
import { fireEvent, waitFor } from '@testing-library/react'
|
||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
|
||||||
|
const script = readFileSync (new NodeURL (
|
||||||
|
'../../../backend/app/views/static_mirror/archive.js', import.meta.url), 'utf8')
|
||||||
|
const archiveWindow = window as typeof window & { archiveConfig?: { origin: string | null } }
|
||||||
|
|
||||||
|
beforeEach (() => {
|
||||||
|
const element = document.createElement ('script')
|
||||||
|
element.src = 'https://mirror.test/subdirectory/archive/assets/archive.js'
|
||||||
|
vi.spyOn (document, 'currentScript', 'get').mockReturnValue (element)
|
||||||
|
delete archiveWindow.archiveConfig
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<nav><a href="ja/posts/index.html">Post</a>
|
||||||
|
<a data-original="/ja/posts/1" href="https://source.test/ja/posts/1">Source</a></nav>
|
||||||
|
<form id="archive-search"><input name="q" value="alias"></form>
|
||||||
|
<ul id="search-results"></ul>`
|
||||||
|
})
|
||||||
|
afterEach (() => {
|
||||||
|
delete archiveWindow.archiveConfig
|
||||||
|
document.body.replaceChildren ()
|
||||||
|
vi.unstubAllGlobals ()
|
||||||
|
})
|
||||||
|
|
||||||
|
describe ('Portable static mirror assets', () => {
|
||||||
|
it ('searches local JSON by aliases using safe text and relative links', async () => {
|
||||||
|
const fetcher = vi.fn ().mockResolvedValue ({ ok: true, json: async () => [
|
||||||
|
{ path: 'ja/tags/1/index.html', title: '<img src=x onerror=alert(1)>', names: ['alias'] },
|
||||||
|
{ path: 'https://outside.test/', title: 'alias' },
|
||||||
|
{ path: '../outside.html', title: 'alias' }] })
|
||||||
|
vi.stubGlobal ('fetch', fetcher)
|
||||||
|
new Function (script) ()
|
||||||
|
fireEvent.submit (document.querySelector ('form')!)
|
||||||
|
await waitFor (() => expect (document.querySelectorAll ('#search-results a')).toHaveLength (1))
|
||||||
|
const link = document.querySelector ('#search-results a')!
|
||||||
|
expect (link).toHaveAttribute ('href', 'ja/tags/1/index.html')
|
||||||
|
expect (link.textContent).toBe ('<img src=x onerror=alert(1)>')
|
||||||
|
expect (document.querySelector ('img')).toBeNull ()
|
||||||
|
expect (fetcher).toHaveBeenCalledTimes (1)
|
||||||
|
expect (fetcher.mock.calls[0][0].href)
|
||||||
|
.toBe ('https://mirror.test/subdirectory/archive/search-index.json')
|
||||||
|
})
|
||||||
|
|
||||||
|
it ('retains baked links without config and changes only source links with valid config', () => {
|
||||||
|
new Function (script) ()
|
||||||
|
const source = document.querySelector ('[data-original]')!
|
||||||
|
expect (source).toHaveAttribute ('href', 'https://source.test/ja/posts/1')
|
||||||
|
archiveWindow.archiveConfig = { origin: 'javascript:alert(1)' }
|
||||||
|
new Function (script) ()
|
||||||
|
expect (source).toHaveAttribute ('href', 'https://source.test/ja/posts/1')
|
||||||
|
archiveWindow.archiveConfig = { origin: 'https://replacement.test/base/' }
|
||||||
|
new Function (script) ()
|
||||||
|
expect (source).toHaveAttribute ('href', 'https://replacement.test/base/ja/posts/1')
|
||||||
|
expect (document.querySelector ('a:not([data-original])'))
|
||||||
|
.toHaveAttribute ('href', 'ja/posts/index.html')
|
||||||
|
})
|
||||||
|
|
||||||
|
it.each (['tag name', 'alias']) ('finds linked posts by %s', async query => {
|
||||||
|
const input = document.querySelector ('input')!
|
||||||
|
input.value = query
|
||||||
|
vi.stubGlobal ('fetch', vi.fn ().mockResolvedValue ({ ok: true, json: async () => [
|
||||||
|
{ path: 'ja/posts/1/index.html', title: 'Post title', names: ['Tag Name', 'Alias'] },
|
||||||
|
{ path: 'ja/posts/2/index.html', title: 'Unrelated post', names: ['Other tag'] },
|
||||||
|
{ path: 'ja/tags/1/index.html', title: 'Tag Name', names: ['Tag Name', 'Alias'] }] }))
|
||||||
|
new Function (script) ()
|
||||||
|
fireEvent.submit (document.querySelector ('form')!)
|
||||||
|
await waitFor (() => expect (document.querySelectorAll ('#search-results a')).toHaveLength (2))
|
||||||
|
const links = [...document.querySelectorAll ('#search-results a')]
|
||||||
|
expect (links.map (link => link.getAttribute ('href')))
|
||||||
|
.toEqual (['ja/posts/1/index.html', 'ja/tags/1/index.html'])
|
||||||
|
expect (links[0].textContent).toBe ('Post title')
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"_untranslated": ["*"],
|
||||||
|
"title": "Static Mirror 生成機能",
|
||||||
|
"generation": "スナップショット生成",
|
||||||
|
"dataAt": "データ基準時点",
|
||||||
|
"currentOnly": "今回は現在状態のみの対応",
|
||||||
|
"developmentOnly": "開発環境限定",
|
||||||
|
"limits": "出力件数の制限",
|
||||||
|
"concurrency": "同時実行数",
|
||||||
|
"exclusions": "未対応機能",
|
||||||
|
"protection": "データ保護上の制約",
|
||||||
|
"protectionDetails": "公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。",
|
||||||
|
"externalAssets": "外部画像や動画への依存",
|
||||||
|
"externalMedia": "外部画像や動画",
|
||||||
|
"externalOmissions": "ローカルへ取り込めていない資産",
|
||||||
|
"download": "ZIP ダウンロード",
|
||||||
|
"zip": "ZIP",
|
||||||
|
"failure": "失敗時の理由"
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"title": "Static Mirror 生成機能",
|
||||||
|
"generation": "スナップショット生成",
|
||||||
|
"dataAt": "データ基準時点",
|
||||||
|
"currentOnly": "今回は現在状態のみの対応",
|
||||||
|
"developmentOnly": "開発環境限定",
|
||||||
|
"limits": "出力件数の制限",
|
||||||
|
"concurrency": "同時実行数",
|
||||||
|
"exclusions": "未対応機能",
|
||||||
|
"protection": "データ保護上の制約",
|
||||||
|
"protectionDetails": "公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。",
|
||||||
|
"externalAssets": "外部画像や動画への依存",
|
||||||
|
"externalMedia": "外部画像や動画",
|
||||||
|
"externalOmissions": "ローカルへ取り込めていない資産",
|
||||||
|
"download": "ZIP ダウンロード",
|
||||||
|
"zip": "ZIP",
|
||||||
|
"failure": "失敗時の理由"
|
||||||
|
}
|
||||||
@@ -16,12 +16,17 @@ import type { User } from '@/types'
|
|||||||
const MorePage: FC = () => {
|
const MorePage: FC = () => {
|
||||||
const location = useLocation ()
|
const location = useLocation ()
|
||||||
const { message } = useUITranslation ('shared')
|
const { message } = useUITranslation ('shared')
|
||||||
|
const { label: archiveLabel } = useUITranslation ('archive')
|
||||||
const menu = menuOutline (
|
const menu = menuOutline (
|
||||||
{ tag: null, wikiId: null, user: { } as User, pathName: location.pathname })
|
{ tag: null, wikiId: null, user: { } as User, pathName: location.pathname })
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<MainArea className="md:flex">
|
<MainArea className="md:flex">
|
||||||
<PageHead title={message ('labels.menu')}/>
|
<PageHead title={message ('labels.menu')}/>
|
||||||
|
{import.meta.env.DEV && (
|
||||||
|
<PrefetchLink to="/static-mirror">
|
||||||
|
{archiveLabel ('title')}
|
||||||
|
</PrefetchLink>)}
|
||||||
|
|
||||||
{[...Array (Math.ceil (menu.length / 4)).keys ()].map (i => (
|
{[...Array (Math.ceil (menu.length / 4)).keys ()].map (i => (
|
||||||
<div key={i} className="flex-1 mx-16">
|
<div key={i} className="flex-1 mx-16">
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
import { act, fireEvent, screen, waitFor } from '@testing-library/react'
|
||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||||
|
|
||||||
|
import StaticMirrorPage from '@/pages/StaticMirrorPage'
|
||||||
|
import { staticMirrorKeys } from '@/lib/queryKeys'
|
||||||
|
import { renderWithProviders } from '@/test/render'
|
||||||
|
|
||||||
|
const api = vi.hoisted (() => ({
|
||||||
|
apiGet: vi.fn (), apiPost: vi.fn (), startAPIDownload: vi.fn () }))
|
||||||
|
vi.mock ('@/lib/api', async original => ({
|
||||||
|
...await original<typeof import ('@/lib/api')> (), ...api }))
|
||||||
|
|
||||||
|
const definition = {
|
||||||
|
resourceLimit: 100, nameLimit: 2000, maxBytes: 8388608, maxSeconds: 60,
|
||||||
|
concurrency: 1, cooldownSeconds: 60, expirySeconds: 600,
|
||||||
|
exclusions: ['historical_state', 'unversioned_resources', 'images', 'videos'] }
|
||||||
|
const ready = {
|
||||||
|
status: 'ready', dataAt: '2026-11-16T00:00:00.000000Z', processed: 2, total: 2 }
|
||||||
|
|
||||||
|
beforeEach (() => {
|
||||||
|
vi.clearAllMocks ()
|
||||||
|
api.apiGet.mockImplementation (async (path: string) =>
|
||||||
|
path === '/static_mirror' ? definition : ready)
|
||||||
|
api.apiPost.mockResolvedValue ({ job: 'opaque-job' })
|
||||||
|
api.startAPIDownload.mockReturnValue (vi.fn ())
|
||||||
|
})
|
||||||
|
|
||||||
|
const startGeneration = async () => {
|
||||||
|
const button = screen.getByRole ('button', { name: 'スナップショット生成' })
|
||||||
|
await waitFor (() => expect (button).toBeEnabled ())
|
||||||
|
fireEvent.click (button)
|
||||||
|
}
|
||||||
|
|
||||||
|
describe ('Static Mirror generation', () => {
|
||||||
|
it ('starts a current snapshot and downloads through the common API', async () => {
|
||||||
|
const view = renderWithProviders (<StaticMirrorPage/>, { route: '/ja/static-mirror' })
|
||||||
|
await startGeneration ()
|
||||||
|
expect (api.apiPost).toHaveBeenCalledWith ('/static_mirror/jobs', { }, { locale: 'ja' })
|
||||||
|
fireEvent.click (await screen.findByRole ('button', { name: 'ZIP ダウンロード' }))
|
||||||
|
expect (screen.getByText (ready.dataAt)).toBeInTheDocument ()
|
||||||
|
expect (screen.getByText ('言語別名称').parentElement).toHaveTextContent ('2000')
|
||||||
|
expect (api.startAPIDownload).toHaveBeenCalledWith (
|
||||||
|
'/static_mirror', 'opaque-job', expect.any (Function))
|
||||||
|
const release = api.startAPIDownload.mock.results[0].value
|
||||||
|
view.unmount ()
|
||||||
|
expect (release).toHaveBeenCalled ()
|
||||||
|
})
|
||||||
|
|
||||||
|
it ('shows generation progress and offers the ZIP only after readiness', async () => {
|
||||||
|
let state = { ...ready, status: 'running', processed: 1 }
|
||||||
|
api.apiGet.mockImplementation (async (path: string) =>
|
||||||
|
path === '/static_mirror' ? definition : state)
|
||||||
|
const view = renderWithProviders (<StaticMirrorPage/>)
|
||||||
|
await startGeneration ()
|
||||||
|
expect (await screen.findByText ('生成中')).toBeInTheDocument ()
|
||||||
|
expect (screen.queryByRole ('button', { name: 'ZIP ダウンロード' })).toBeNull ()
|
||||||
|
expect (screen.getByRole ('button', { name: 'スナップショット生成' })).toBeDisabled ()
|
||||||
|
state = ready
|
||||||
|
await act (async () => {
|
||||||
|
await view.queryClient.invalidateQueries ({ queryKey: staticMirrorKeys.job ('opaque-job') })
|
||||||
|
})
|
||||||
|
expect (await screen.findByRole ('button', { name: 'ZIP ダウンロード' })).toBeEnabled ()
|
||||||
|
})
|
||||||
|
|
||||||
|
it ('keeps failure reasons visible without offering a ZIP', async () => {
|
||||||
|
api.apiGet.mockImplementation (async (path: string) =>
|
||||||
|
path === '/static_mirror' ? definition : { ...ready, status: 'failed', error: 'too_large' })
|
||||||
|
renderWithProviders (<StaticMirrorPage/>)
|
||||||
|
await startGeneration ()
|
||||||
|
expect (await screen.findByRole ('alert')).toHaveTextContent ('too_large')
|
||||||
|
expect (screen.queryByRole ('button', { name: 'ZIP ダウンロード' })).toBeNull ()
|
||||||
|
})
|
||||||
|
|
||||||
|
it ('shows intake limits and revoked downloads using existing error copy', async () => {
|
||||||
|
api.apiPost.mockRejectedValueOnce ({ isAxiosError: true, response: { status: 429 } })
|
||||||
|
renderWithProviders (<StaticMirrorPage/>)
|
||||||
|
await startGeneration ()
|
||||||
|
expect (await screen.findByRole ('alert')).toHaveTextContent ('HTTP 429')
|
||||||
|
api.apiGet.mockImplementation (async (path: string) => {
|
||||||
|
if (path === '/static_mirror') return definition
|
||||||
|
throw { isAxiosError: true, response: { status: 404 } }
|
||||||
|
})
|
||||||
|
await startGeneration ()
|
||||||
|
await waitFor (() => expect (screen.getByRole ('alert')).toHaveTextContent ('有効期限'))
|
||||||
|
expect (screen.getByRole ('button', { name: 'スナップショット生成' })).toBeEnabled ()
|
||||||
|
})
|
||||||
|
|
||||||
|
it ('preserves Japanese fallback language metadata for untranslated UI', async () => {
|
||||||
|
renderWithProviders (<StaticMirrorPage/>, { route: '/ain-Kana/static-mirror' })
|
||||||
|
await startGeneration ()
|
||||||
|
expect (api.apiPost).toHaveBeenCalledWith (
|
||||||
|
'/static_mirror/jobs', { }, { locale: 'ain-Kana' })
|
||||||
|
expect (screen.getByRole ('heading', { name: 'Static Mirror 生成機能' })
|
||||||
|
.querySelector ('[lang="ja"]')).toBeInTheDocument ()
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
import { useMutation, useQuery } from '@tanstack/react-query'
|
||||||
|
import { useEffect, useRef, useState } from 'react'
|
||||||
|
|
||||||
|
import PageHead from '@/components/common/PageHead'
|
||||||
|
import PageTitle from '@/components/common/PageTitle'
|
||||||
|
import MainArea from '@/components/layout/MainArea'
|
||||||
|
import { Button } from '@/components/ui/button'
|
||||||
|
import { apiGet, apiPost, isApiError, startAPIDownload } from '@/lib/api'
|
||||||
|
import { staticMirrorKeys } from '@/lib/queryKeys'
|
||||||
|
import { useLocale } from '@/lib/useLocale'
|
||||||
|
import { useUITranslation } from '@/lib/useUITranslation'
|
||||||
|
|
||||||
|
type Definition = {
|
||||||
|
resourceLimit: number
|
||||||
|
nameLimit: number
|
||||||
|
maxBytes: number
|
||||||
|
maxSeconds: number
|
||||||
|
concurrency: number
|
||||||
|
cooldownSeconds: number
|
||||||
|
expirySeconds: number
|
||||||
|
exclusions: string[] }
|
||||||
|
|
||||||
|
type Job = {
|
||||||
|
status: 'queued' | 'running' | 'ready' | 'failed'
|
||||||
|
dataAt: string | null
|
||||||
|
processed: number
|
||||||
|
total: number | null
|
||||||
|
error?: string }
|
||||||
|
|
||||||
|
const StaticMirrorPage = () => {
|
||||||
|
const locale = useLocale ()
|
||||||
|
const { label, message } = useUITranslation ('archive')
|
||||||
|
const [job, setJob] = useState<string | null> (null)
|
||||||
|
const [downloadFailed, setDownloadFailed] = useState (false)
|
||||||
|
const downloadRef = useRef<(() => void) | null> (null)
|
||||||
|
useEffect (() => () => downloadRef.current?.(), [])
|
||||||
|
|
||||||
|
const definition = useQuery ({
|
||||||
|
queryKey: staticMirrorKeys.definition,
|
||||||
|
queryFn: () => apiGet<Definition> ('/static_mirror'), retry: false })
|
||||||
|
const { mutate: start, isPending: pending, error: creationError } = useMutation ({
|
||||||
|
mutationFn: () => apiPost<{ job: string }> ('/static_mirror/jobs', { }, { locale }),
|
||||||
|
onSuccess: result => {
|
||||||
|
downloadRef.current?.()
|
||||||
|
setDownloadFailed (false)
|
||||||
|
setJob (result.job)
|
||||||
|
} })
|
||||||
|
const status = useQuery ({
|
||||||
|
queryKey: staticMirrorKeys.job (job),
|
||||||
|
queryFn: () => apiGet<Job> (`/static_mirror/jobs/${encodeURIComponent (job!)}`),
|
||||||
|
enabled: job != null,
|
||||||
|
retry: false,
|
||||||
|
refetchInterval: query =>
|
||||||
|
(query.state.error != null
|
||||||
|
|| ['ready', 'failed'].includes (query.state.data?.status ?? ''))
|
||||||
|
? false : 1000 })
|
||||||
|
const unavailable = isApiError (status.error) && status.error.response?.status === 404
|
||||||
|
const active = job != null && !unavailable
|
||||||
|
&& !['ready', 'failed'].includes (status.data?.status ?? '')
|
||||||
|
const error = creationError ?? status.error ?? definition.error
|
||||||
|
const errorStatus = isApiError (error) ? error.response?.status : null
|
||||||
|
const metadata = definition.data
|
||||||
|
const percentage =
|
||||||
|
(status.data?.total == null || status.data.total === 0)
|
||||||
|
? 100
|
||||||
|
: Math.round (100 * status.data.processed / status.data.total)
|
||||||
|
|
||||||
|
const download = () => {
|
||||||
|
if (job == null) return
|
||||||
|
downloadRef.current?.()
|
||||||
|
setDownloadFailed (false)
|
||||||
|
downloadRef.current = startAPIDownload (
|
||||||
|
'/static_mirror', job, () => setDownloadFailed (true))
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<MainArea className="space-y-4">
|
||||||
|
<PageHead title={message ('title')}/>
|
||||||
|
<PageTitle>{label ('title')}</PageTitle>
|
||||||
|
<p>{label ('developmentOnly')} / {label ('currentOnly')}</p>
|
||||||
|
{metadata != null && (
|
||||||
|
<dl>
|
||||||
|
<dt>{label ('limits')}</dt>
|
||||||
|
<dd>
|
||||||
|
{label ('counts:resources.posts')} / {label ('counts:resources.tags')}:
|
||||||
|
{' '}{metadata.resourceLimit}
|
||||||
|
</dd>
|
||||||
|
<dd>{label ('counts:columns.names')}: {metadata.nameLimit}</dd>
|
||||||
|
<dd>{metadata.maxBytes} B / {metadata.maxSeconds} s</dd>
|
||||||
|
<dt>{label ('concurrency')}</dt>
|
||||||
|
<dd>{metadata.concurrency} / {metadata.cooldownSeconds} s</dd>
|
||||||
|
<dd>{label ('zip')}: {metadata.expirySeconds} s</dd>
|
||||||
|
<dt>{label ('exclusions')}</dt>
|
||||||
|
<dd>
|
||||||
|
{['counts:resources.tag_descriptions', 'navigation:help',
|
||||||
|
'counts:resources.materials', 'counts:resources.nico_tags',
|
||||||
|
'counts:attributes.parent_post_ids', 'counts:attributes.parent_tag_ids',
|
||||||
|
'counts:columns.sections', 'navigation:history', 'navigation:edit']
|
||||||
|
.map (key => <span key={key}>{label (key)}{' / '}</span>)}
|
||||||
|
</dd>
|
||||||
|
<dt>{label ('externalAssets')}</dt>
|
||||||
|
<dd>{label ('externalOmissions')}: {label ('externalMedia')}</dd>
|
||||||
|
<dt>{label ('protection')}</dt>
|
||||||
|
<dd>{label ('protectionDetails')}</dd>
|
||||||
|
</dl>)}
|
||||||
|
<Button onClick={() => start ()}
|
||||||
|
disabled={metadata == null || pending || active}>
|
||||||
|
{label ('generation')}
|
||||||
|
</Button>
|
||||||
|
{status.data != null && (
|
||||||
|
<div role="status">
|
||||||
|
<p>{label ('dataAt')}: <time>{status.data.dataAt ?? '—'}</time></p>
|
||||||
|
<p>
|
||||||
|
{status.data.status === 'ready'
|
||||||
|
? label ('download')
|
||||||
|
: label (`counts:exportProgress.${status.data.status}`)}
|
||||||
|
</p>
|
||||||
|
{status.data.total != null && (
|
||||||
|
<p>{label ('counts:exportProgress.measured', {
|
||||||
|
processed: status.data.processed, total: status.data.total,
|
||||||
|
percent: percentage })}</p>)}
|
||||||
|
{status.data.status === 'ready' && (
|
||||||
|
<Button onClick={download}>{label ('download')}</Button>)}
|
||||||
|
{status.data.error != null && (
|
||||||
|
<p role="alert">{label ('failure')}: <code>{status.data.error}</code></p>)}
|
||||||
|
</div>)}
|
||||||
|
{error != null && (
|
||||||
|
<p role="alert">
|
||||||
|
{errorStatus === 404
|
||||||
|
? label ('counts:downloadErrors.unavailable')
|
||||||
|
: (errorStatus != null
|
||||||
|
? label ('counts:errors.http', { status: errorStatus })
|
||||||
|
: label ('counts:errors.network'))}
|
||||||
|
</p>)}
|
||||||
|
{status.error != null && (
|
||||||
|
<Button onClick={() => status.refetch ()}>{label ('shared:actions.retry')}</Button>)}
|
||||||
|
{downloadFailed && <p role="alert">{label ('counts:downloadErrors.failed')}</p>}
|
||||||
|
</MainArea>)
|
||||||
|
}
|
||||||
|
export default StaticMirrorPage
|
||||||
新しいイシューから参照
ユーザーをブロックする