diff --git a/backend/app/controllers/static_mirrors_controller.rb b/backend/app/controllers/static_mirrors_controller.rb new file mode 100644 index 0000000..b79fe23 --- /dev/null +++ b/backend/app/controllers/static_mirrors_controller.rb @@ -0,0 +1,55 @@ +class StaticMirrorsController < ApplicationController + before_action :development_only + wrap_parameters false + + def show + render json: { resource_limit: StaticMirror::RESOURCE_LIMIT, + name_limit: StaticMirror::NAME_LIMIT, max_bytes: StaticMirror::MAX_BYTES, + max_seconds: StaticMirrorJob::MAX_SECONDS, concurrency: 1, + cooldown_seconds: StaticMirrorJob::COOLDOWN, + expiry_seconds: StaticMirrorJob::EXPIRY.to_i, exclusions: StaticMirror::EXCLUSIONS } + end + + def create + # No date input is accepted: this endpoint cannot promise historical restoration. + payload = request.request_parameters.except('authenticity_token') + .merge(request.query_parameters) + return head :bad_request unless (payload.keys - ['locale']).empty? + + locale = resolve_locale! + session[:static_mirror_owner] ||= SecureRandom.hex(32) + job = StaticMirrorJob.start(owner: session[:static_mirror_owner], locale: locale.code, + origin: request.base_url) + render json: { job: }, status: :accepted + rescue StaticMirrorJob::Busy + response.headers['Retry-After'] = StaticMirrorJob::COOLDOWN.to_s + head :too_many_requests + rescue ActiveJob::EnqueueError + render json: { error: 'generation_failed' }, status: :service_unavailable + end + + def status + value = StaticMirrorJob.status(params[:id], session[:static_mirror_owner]) + return head :not_found unless value + + render json: value + rescue StaticMirrorJob::Busy + head :service_unavailable + end + + def download + bytes = StaticMirrorJob.download(params[:id], session[:static_mirror_owner]) + return head :not_found unless bytes + + send_data bytes, filename: 'static-mirror.zip', type: 'application/zip', + disposition: 'attachment' + rescue StaticMirrorJob::Busy + head :service_unavailable + end + + private + + def development_only + head :not_found unless Rails.env.development? + end +end diff --git a/backend/app/jobs/static_mirror_job.rb b/backend/app/jobs/static_mirror_job.rb new file mode 100644 index 0000000..5d26bde --- /dev/null +++ b/backend/app/jobs/static_mirror_job.rb @@ -0,0 +1,155 @@ +require 'fileutils' +require 'timeout' + +# Development PoC: one filesystem slot and Rails' in-process async executor. +# A process restart loses queued work; stale state fails rather than reporting success. +class StaticMirrorJob < ApplicationJob + self.queue_adapter = :async + EXPIRY = 10.minutes + COOLDOWN = 60 + MAX_SECONDS = 60 + class Busy < StandardError; end + + def self.directory = Rails.root.join('tmp/static_mirror') + + def self.synchronise(blocking: false) + FileUtils.mkdir_p(directory, mode: 0o700) + File.open(directory.join('.registry.lock'), File::RDWR | File::CREAT, 0o600) do |lock| + mode = blocking ? File::LOCK_EX : File::LOCK_EX | File::LOCK_NB + raise Busy unless lock.flock(mode) + + yield + end + end + + def self.state + JSON.parse(directory.join('state.json').read) + rescue Errno::ENOENT + nil + end + + def self.write(value) + path = directory.join('state.tmp') + File.open(path, 'w', 0o600) { _1.write(JSON.generate(value)) } + File.rename(path, directory.join('state.json')) + ensure + File.unlink(path) if path&.exist? + end + + def self.start(owner:, locale:, origin:) + id = synchronise do + previous = state + if previous && Time.current.to_f - previous.fetch('created_at') < COOLDOWN + raise Busy + end + File.open(directory.join('.generation.lock'), File::RDWR | File::CREAT, 0o600) do |lock| + raise Busy unless lock.flock(File::LOCK_EX | File::LOCK_NB) + + FileUtils.rm_f(directory.join('archive.zip')) + FileUtils.rm_f(directory.join('archive.part')) + identifier = SecureRandom.hex(24) + write({ 'id' => identifier, 'owner' => owner, 'locale' => locale, 'origin' => origin, + 'created_at' => Time.current.to_f, 'expiry_at' => (Time.current + EXPIRY).to_f, + 'status' => 'queued', 'processed' => 0, 'total' => nil, 'data_at' => nil }) + identifier + end + end + perform_later(id) + id + rescue ActiveJob::EnqueueError + update(id) { _1.merge!('status' => 'failed', 'error' => 'generation_failed') } if id + raise + end + + def self.authorised_state(id, owner) + value = state + return unless owner.present? && value && value['id'] == id && value['owner'] == owner + return unless value['expiry_at'] > Time.current.to_f + + if (['queued', 'running'].include?(value['status']) && + Time.current.to_f - value['created_at'] > MAX_SECONDS + 15) + value.merge!('status' => 'failed', 'error' => 'worker_stopped') + write(value) + end + if value['status'] == 'ready' + permitted = HistoryAccess.set(user: nil) { HistoryAccess.authorised_artifact?(value) } + return unless permitted && directory.join('archive.zip').file? + end + value + end + + def self.status(id, owner) + synchronise do + authorised_state(id, owner)&.slice('status', 'processed', 'total', 'data_at', 'error') + end + end + + def self.download(id, owner) + synchronise do + value = authorised_state(id, owner) + return unless value && value['status'] == 'ready' + + bytes = directory.join('archive.zip').binread + # Revalidate after reading; send_data avoids delayed sendfile reuse of a stale path. + permitted = HistoryAccess.set(user: nil) { HistoryAccess.authorised_artifact?(value) } + permitted ? bytes : nil + end + end + + def self.update(id) + # Worker updates may wait for a short status/download read; request intake never waits. + synchronise(blocking: true) do + value = state + return unless value && value['id'] == id + + yield value + write(value) + end + end + + def perform(id) + File.open(self.class.directory.join('.generation.lock'), File::RDWR) do |lock| + return unless lock.flock(File::LOCK_EX | File::LOCK_NB) + + value = self.class.state + return unless value && value['id'] == id && value['status'] == 'queued' + + generate(id, value) + end + end + + private + + def generate(id, value) + self.class.update(id) { _1['status'] = 'running' } + remaining = MAX_SECONDS - (Time.current.to_f - value.fetch('created_at')) + raise Timeout::Error unless remaining.positive? + + Timeout.timeout(remaining) do + locale = Locale.find_by!(code: value.fetch('locale'), deprecated_at: nil) + mirror = StaticMirror.new(locale:, origin: value.fetch('origin')) + bytes = mirror.export do |processed, total| + self.class.update(id) do |metadata| + metadata.merge!('processed' => processed, 'total' => total, 'data_at' => mirror.data_at) + end + end + part = self.class.directory.join('archive.part') + File.open(part, 'wb', 0o600) { _1.write(bytes) } + File.rename(part, self.class.directory.join('archive.zip')) + self.class.update(id) do |metadata| + metadata.merge!(mirror.authority).merge!('status' => 'ready', 'data_at' => mirror.data_at) + end + end + rescue StandardError => error + FileUtils.rm_f(self.class.directory.join('archive.part')) + FileUtils.rm_f(self.class.directory.join('archive.zip')) + code = case error + when StaticMirror::TooLarge then 'too_large' + when Timeout::Error then 'generation_timeout' + when HistoryAccess::Forbidden then 'publication_changed' + else 'generation_failed' + end + self.class.update(id) { _1.merge!('status' => 'failed', 'error' => code) } + Rails.logger.warn("static_mirror_failure: #{ error.class }") + end +end diff --git a/backend/app/services/static_mirror.rb b/backend/app/services/static_mirror.rb new file mode 100644 index 0000000..a6382e3 --- /dev/null +++ b/backend/app/services/static_mirror.rb @@ -0,0 +1,201 @@ +require 'cgi' +require 'digest' +require 'erb' +require 'pathname' +require 'uri' + +# A deliberately small current-state projection, always under public authority. +class StaticMirror + RESOURCE_LIMIT = 100 + NAME_LIMIT = 2_000 + MAX_BYTES = 8.megabytes + EXCLUSIONS = [ + 'historical_state', 'unversioned_resources', 'tag_descriptions', 'help', 'materials', + 'external_tags', 'parent_relations', 'tag_sections', 'users', 'audit_data', + 'authentication', 'editing', 'images', 'videos', 'resources_beyond_limits'].freeze + class TooLarge < StandardError; end + class UnsafeUrl < StandardError; end + + attr_reader :data_at, :authority, :counts + + def initialize(locale:, origin:) + @locale = locale + @origin = validated_url(origin).sub(%r{/+\z}, '') + end + + def export(&progress) + @files = { } + @bytes = 0 + HistoryAccess.set(user: nil) do + capture + progress&.call(0, @posts.length + @tags.length) + generate_pages(&progress) + raise HistoryAccess::Forbidden unless HistoryAccess.authorised_artifact?(authority) + + generated_at = Time.current.utc + manifest = { format_version: 1, generated_at: generated_at.iso8601(6), + data_at:, locale: @locale.code, resources: counts, exclusions: EXCLUSIONS, + fields: { posts: ['id', 'title', 'url', 'tags'], tags: ['id', 'category', 'names'] }, + limits: { posts: RESOURCE_LIMIT, tags: RESOURCE_LIMIT, names: NAME_LIMIT, + bytes: MAX_BYTES }, + external_assets: 'not_included', checksum_scope: 'all_files_except_manifest.json', + files: @files.map { |path, data| { path:, sha256: Digest::SHA256.hexdigest(data) } } } + add_file('manifest.json', JSON.pretty_generate(manifest)) + entries = @files.map do |path, data| + MaterialZipExporter::Entry.new(path: "archive/#{ path }", data:, mtime: generated_at) + end + bytes = MaterialZipExporter::ZipWriter.write(entries) + raise TooLarge if bytes.bytesize > MAX_BYTES + + bytes + end + end + + private + + def capture + CountsExporter.with_read_snapshot do + @data_at = Time.current.utc.iso8601(6) + @authority = HistoryAccess.authority + # Requiring a matching immutable revision closes the legacy unversioned gap. + @posts = confirmed(Post, PostVersion, :post_versions).order(:id) + .limit(RESOURCE_LIMIT).select(:id, :title, :url).to_a + @tags = confirmed(Tag, TagVersion, :tag_versions).where(deprecated_at: nil) + .order(:id).limit(RESOURCE_LIMIT).select(:id, :category).to_a + @names = TagName.where(tag_id: @tags.map(&:id)).order(:tag_id, :language_code, :id) + .limit(NAME_LIMIT + 1).pluck(:tag_id, :language_code, :script_code, :name, :primary_flg) + raise TooLarge if @names.length > NAME_LIMIT + + @names = @names.group_by(&:first) + @relations = PostTag.where(post_id: @posts.map(&:id), tag_id: @tags.map(&:id)) + .order(:post_id, :tag_id).pluck(:post_id, :tag_id) + @counts = { posts: @posts.length, tags: @tags.length } + end + end + + def confirmed(model, version_model, association) + HistoryAccess.current(model.all, version_model).joins(association) + .where("#{ version_model.table_name }.version_no = #{ model.table_name }.version_no") + end + + def generate_pages + assets = Rails.root.join('app/views/static_mirror') + ['archive.css', 'archive.js'].each do |name| + add_file("assets/#{ name }", assets.join(name).read) + end + add_file('robots.txt', "User-agent: *\nDisallow: /\n") + add_file('archive-config.js', 'window.archiveConfig = { origin: null };') + page('index.html', 'Static Mirror 生成機能', '', search_form + limitations) + page("#{ @locale.code }/posts/index.html", 'Post', '/posts', post_list(@posts)) + page("#{ @locale.code }/tags/index.html", 'Tag', '/tags', tag_list(@tags)) + processed = 0 + @posts.each do |post| + tags = @tags.select { |tag| @relations.include?([post.id, tag.id]) } + content = "

#{ external_link(post.url) }

#{ tag_list(tags) }" + page(resource_path('posts', post.id), post.title.presence || post.id.to_s, + "/posts/#{ post.id }", content) + processed += 1 + yield(processed, @posts.length + @tags.length) if block_given? + end + @tags.each do |tag| + posts = @posts.select { |post| @relations.include?([post.id, tag.id]) } + names = @names.fetch(tag.id, []).map do |_, language, script, name, _primary| + "
  • #{ h(name) } (#{ h(script) })
  • " + end.join + content = "

    #{ h(tag.category) }

    #{ post_list(posts) }" + page(resource_path('tags', tag.id), tag_name(tag), "/tags/#{ tag.id }", content) + processed += 1 + yield(processed, @posts.length + @tags.length) if block_given? + end + index = @posts.map { |post| + { path: resource_path('posts', post.id), title: post.title.presence || post.id.to_s } + } + @tags.map { |tag| + { path: resource_path('tags', tag.id), title: tag_name(tag), + names: @names.fetch(tag.id, []).map { _1[3] } } + } + add_file('search-index.json', JSON.generate(index)) + end + + def page(path, title, original_path, content) + @path = path + @title = title + @original_path = "/#{ @locale.code }#{ original_path }" + @content = content + template = Rails.root.join('app/views/static_mirror/page.html.erb').read + add_file(path, ERB.new(template).result(binding)) + end + + def add_file(path, data) + @bytes += data.bytesize + raise TooLarge if @bytes > MAX_BYTES + + @files[path] = data + end + + def resource_path(resource, id) = "#{ @locale.code }/#{ resource }/#{ id }/index.html" + + def relative(path) + Pathname.new(path).relative_path_from(Pathname.new(@path).dirname).to_s + end + + def h(value) = CGI.escapeHTML(value.to_s) + + def tag_name(tag) + @names.fetch(tag.id, []).find { _1[1] == @locale.language_code && _1[4] }&.fetch(3) || + tag.id.to_s + end + + # Links are resolved after the page path is known; only generated numeric IDs enter paths. + def post_list(posts) + '

    Post

    ' + end + + def tag_list(tags) + '

    Tag

    ' + end + + def linked_content + @content.gsub(/data-path="([^"]+)"/) { "href=\"#{ h(relative(Regexp.last_match(1))) }\"" } + end + + def validated_url(value) + uri = URI.parse(value.to_s) + unless uri.is_a?(URI::HTTP) && uri.host.present? && uri.userinfo.nil? + raise UnsafeUrl + end + + uri.to_s + rescue URI::InvalidURIError + raise UnsafeUrl + end + + def external_link(value) + url = validated_url(value) + "#{ h(url) }" + rescue UnsafeUrl + '' + end + + def search_form + '' + end + + def limitations + <<~HTML +

    未対応機能

    +

    Tag Description, Help, Material, 外部タグ, 親子関係, 区間, 履歴, ログイン, 作成, 編輯

    +

    出力件数の制限

    Post: #{ RESOURCE_LIMIT }, Tag: #{ RESOURCE_LIMIT }

    +

    TagName: #{ NAME_LIMIT }, #{ MAX_BYTES } B

    +

    外部画像や動画への依存

    ローカルへ取り込めていない資産: 外部画像や動画

    +

    データ保護上の制約

    +

    公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。

    +

    今回は現在状態のみの対応

    + HTML + end +end diff --git a/backend/app/views/static_mirror/archive.css b/backend/app/views/static_mirror/archive.css new file mode 100644 index 0000000..5bd54f2 --- /dev/null +++ b/backend/app/views/static_mirror/archive.css @@ -0,0 +1,7 @@ +:root { color-scheme: light dark; } +body { max-width: 65rem; margin: auto; padding: 1rem; font: 1rem/1.6 sans-serif; } +header { border-bottom: 1px solid; padding-bottom: 1rem; } +nav { display: flex; flex-wrap: wrap; gap: 1rem; } +a, p, li { overflow-wrap: anywhere; } +li { margin-block: .4rem; } +input, button { font: inherit; } diff --git a/backend/app/views/static_mirror/archive.js b/backend/app/views/static_mirror/archive.js new file mode 100644 index 0000000..e5a3db9 --- /dev/null +++ b/backend/app/views/static_mirror/archive.js @@ -0,0 +1,61 @@ +(() => { + const root = new URL ('../', document.currentScript.src) + const origin = window.archiveConfig?.origin + if (typeof origin === 'string') + { + try + { + const url = new URL (origin) + if (['http:', 'https:'].includes (url.protocol) && !url.username && !url.password) + { + url.search = '' + url.hash = '' + document.querySelectorAll ('[data-original]').forEach (link => { + link.href = url.href.replace (/\/+$/, '') + link.dataset.original + }) + } + } + catch + { + // Each HTML retains its original link when configuration is absent or invalid. + } + } + const form = document.getElementById ('archive-search') + const results = document.getElementById ('search-results') + if (!form || !results) return + form.addEventListener ('submit', async event => { + event.preventDefault () + results.replaceChildren () + const query = (new FormData (form).get ('q') ?? '').toString ().toLocaleLowerCase () + if (!query.trim ()) return + try + { + const response = await fetch (new URL ('search-index.json', root)) + if (!response.ok) throw new Error ('Search index unavailable') + const entries = await response.json () + for (const entry of entries) + { + const text = [entry.title, ...(entry.names ?? [])].join (' ').toLocaleLowerCase () + if (!text.includes (query)) continue + const url = new URL (entry.path, root) + if (url.origin !== root.origin || !url.pathname.startsWith (root.pathname)) continue + const item = document.createElement ('li') + const link = document.createElement ('a') + link.setAttribute ('href', entry.path) + link.textContent = entry.title + item.append (link) + results.append (item) + } + } + catch + { + // Lists remain available if the index cannot be loaded. + for (const link of document.querySelectorAll ('nav a:not([data-original])')) + { + const item = document.createElement ('li') + item.append (link.cloneNode (true)) + results.append (item) + } + } + }) +})() diff --git a/backend/app/views/static_mirror/page.html.erb b/backend/app/views/static_mirror/page.html.erb new file mode 100644 index 0000000..a9bbfea --- /dev/null +++ b/backend/app/views/static_mirror/page.html.erb @@ -0,0 +1,28 @@ + + + + + + + + + <%= h(@title) %> + + + + + +
    +

    静的保存版 — データ基準時点:

    + +
    +

    <%= h(@title) %>

    <%= linked_content %>
    + + diff --git a/backend/config/routes.rb b/backend/config/routes.rb index 62f7f3f..1d1a6d4 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -1,4 +1,9 @@ Rails.application.routes.draw do + get 'static_mirror', to: 'static_mirrors#show' + post 'static_mirror/jobs', to: 'static_mirrors#create' + get 'static_mirror/jobs/:id', to: 'static_mirrors#status' + get 'static_mirror/files/:id', to: 'static_mirrors#download' + get 'counts/definitions/:resource', to: 'counts_tables#definition' get 'counts/definitions/:resource/candidates', to: 'counts_tables#candidates' get 'counts/query', to: 'counts_tables#index' diff --git a/backend/spec/requests/static_mirrors_spec.rb b/backend/spec/requests/static_mirrors_spec.rb new file mode 100644 index 0000000..a6ae780 --- /dev/null +++ b/backend/spec/requests/static_mirrors_spec.rb @@ -0,0 +1,169 @@ +require 'rails_helper' +require 'tmpdir' + +RSpec.describe 'Static Mirror API', type: :request do + include ActiveSupport::Testing::TimeHelpers + + around do |example| + Dir.mktmpdir('static-mirror-spec') do |path| + allow(StaticMirrorJob).to receive(:directory).and_return(Pathname.new(path)) + example.run + end + end + + before do + allow(Rails.env).to receive(:development?).and_return(true) + allow(StaticMirrorJob).to receive(:perform_later) + end + + def create_job + post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json + expect(response).to have_http_status(:accepted) + json.fetch('job') + end + + it 'accepts anonymous jobs asynchronously and serves the completed ZIP' do + record = create(:post, title: 'mirror-public-api-title') + PostVersion.create!(post: record, version_no: 1, event_type: 'create', + title: record.title, url: record.url, tags: '', tags_json: [], parent_post_ids: '', + created_at: Time.current) + get '/static_mirror' + expect(json).to include('resource_limit' => 100, 'concurrency' => 1) + id = create_job + expect(StaticMirrorJob).to have_received(:perform_later).with(id) + expect(StaticMirrorJob.directory.join('archive.zip')).not_to exist + get "/static_mirror/jobs/#{ id }" + expect(json).to include('status' => 'queued', 'data_at' => nil) + StaticMirrorJob.perform_now(id) + get "/static_mirror/jobs/#{ id }" + expect(json).to include('status' => 'ready', 'processed' => 1, 'total' => 1) + expect(json.fetch('data_at')).to be_present + expect(json.keys).not_to include('owner', 'origin', 'actor_id', 'policy_revision') + get "/static_mirror/files/#{ id }" + expect(response).to have_http_status(:ok) + expect(response.media_type).to eq('application/zip') + expect(response.body.byteslice(0, 4)).to eq("PK\x03\x04".b) + expect(response.body).to include(record.title) + expect(response.headers['Cache-Control']).to include('no-store') + end + + it 'rejects historical and unknown input instead of pretending to restore it' do + ['at', 'data_at', 'admin', 'limit'].each do |key| + post '/static_mirror/jobs', params: { locale: 'ja', key => '2020-01-01' }, as: :json + expect(response).to have_http_status(:bad_request) + end + expect(StaticMirrorJob).not_to have_received(:perform_later) + end + + it 'limits requests globally even across fresh anonymous sessions' do + create_job + other = ActionDispatch::Integration::Session.new(Rails.application) + other.post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json + expect(other.response).to have_http_status(:too_many_requests) + expect(other.response.headers['Retry-After']).to eq('60') + expect(StaticMirrorJob).to have_received(:perform_later).once + end + + it 'binds job status and downloads to the creating session' do + id = create_job + StaticMirrorJob.perform_now(id) + other = ActionDispatch::Integration::Session.new(Rails.application) + ["/static_mirror/jobs/#{ id }", "/static_mirror/files/#{ id }"].each do |path| + other.get path + expect(other.response).to have_http_status(:not_found) + end + end + + it 'refuses ready artifacts after a new discard even for admins' do + id = create_job + StaticMirrorJob.perform_now(id) + record = create(:post) + PostVersion.create!(post: record, version_no: 1, event_type: 'discard', + url: record.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.current) + sign_in_as(create(:user, :admin)) + get "/static_mirror/files/#{ id }" + expect(response).to have_http_status(:not_found) + get "/static_mirror/jobs/#{ id }" + expect(response).to have_http_status(:not_found) + end + + it 'fails lost jobs and rejects expired downloads' do + id = create_job + travel 80.seconds do + get "/static_mirror/jobs/#{ id }" + expect(json).to include('status' => 'failed', 'error' => 'worker_stopped') + end + travel 11.minutes do + get "/static_mirror/files/#{ id }" + expect(response).to have_http_status(:not_found) + end + end + + it 'rejects a previously ready ZIP after its expiry' do + id = create_job + StaticMirrorJob.perform_now(id) + travel 11.minutes do + get "/static_mirror/files/#{ id }" + expect(response).to have_http_status(:not_found) + get "/static_mirror/jobs/#{ id }" + expect(response).to have_http_status(:not_found) + end + end + + it 'keeps simultaneous generation bounded after the cooldown' do + create_job + lock = File.open(StaticMirrorJob.directory.join('.generation.lock'), File::RDWR) + lock.flock(File::LOCK_EX) + travel 61.seconds do + post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json + expect(response).to have_http_status(:too_many_requests) + end + ensure + lock&.close + end + + it 'reports bounded-generation failures without returning internal exception messages' do + id = create_job + allow(StaticMirror).to receive(:new).and_raise(StaticMirror::TooLarge, 'private details') + StaticMirrorJob.perform_now(id) + get "/static_mirror/jobs/#{ id }" + expect(json).to include('status' => 'failed', 'error' => 'too_large') + expect(response.body).not_to include('private details') + get "/static_mirror/files/#{ id }" + expect(response).to have_http_status(:not_found) + end + + it 'is unavailable outside development' do + allow(Rails.env).to receive(:development?).and_return(false) + get '/static_mirror' + expect(response).to have_http_status(:not_found) + post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json + expect(response).to have_http_status(:not_found) + get '/static_mirror/jobs/unknown' + expect(response).to have_http_status(:not_found) + get '/static_mirror/files/unknown' + expect(response).to have_http_status(:not_found) + end + + it 'preserves user and IP BAN checks' do + user = create(:user, banned_at: Time.current) + sign_in_as(user) + post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json + expect(response).to have_http_status(:forbidden) + sign_out + IpAddress.create!(ip_address: IPAddr.new('127.0.0.1').hton, banned_at: Time.current) + get '/static_mirror', headers: { 'REMOTE_ADDR' => '127.0.0.1' } + expect(response).to have_http_status(:forbidden) + expect(StaticMirrorJob).not_to have_received(:perform_later) + end + + it 'requires the normal CSRF token for anonymous generation', csrf: true do + get '/users/csrf' + token = json.fetch('token') + post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json + expect(response).to have_http_status(:forbidden) + post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json, + headers: { 'X-CSRF-Token' => token } + expect(response).to have_http_status(:accepted) + end +end diff --git a/backend/spec/services/static_mirror_spec.rb b/backend/spec/services/static_mirror_spec.rb new file mode 100644 index 0000000..4f5870d --- /dev/null +++ b/backend/spec/services/static_mirror_spec.rb @@ -0,0 +1,158 @@ +require 'rails_helper' + +RSpec.describe StaticMirror do + include_context 'Ainu locale' + + let(:tag) { create(:tag, primary_name: 'mirror-public-tag') } + let(:post) { create(:post, title: '') } + let(:mirror) { described_class.new(locale: Locale.nipponese, origin: 'https://source.test') } + + def snapshot_tag(record, event: 'create', number: 1) + TagVersion.create!(tag: record, version_no: number, event_type: event, + name: record.name('ja'), category: record.category, aliases: '', parent_tag_ids: '', + names: { ja: { name: record.name('ja'), script_code: 'Jpan', + auto_generated: false, aliases: [] } }, created_at: Time.current) + end + + def snapshot_post(record, tags: []) + PostVersion.create!(post: record, version_no: 1, event_type: 'create', + title: record.title, url: record.url, tags: '', tags_json: tags, + parent_post_ids: '', created_at: Time.current) + end + + # Read STORE entries independently of the production writer and verify CRCs. + def archive_files(bytes) + offset = 0 + files = { } + while bytes.byteslice(offset, 4).unpack1('V') == 0x04034b50 + header = bytes.byteslice(offset, 30).unpack('VvvvvvVVVvv') + size, name_size, extra_size = header[7], header[9], header[10] + path = bytes.byteslice(offset + 30, name_size).force_encoding('UTF-8') + data = bytes.byteslice(offset + 30 + name_size + extra_size, size) + expect(Zlib.crc32(data)).to eq(header[6]) + files[path.delete_prefix('archive/')] = data.force_encoding('UTF-8') + offset += 30 + name_size + extra_size + size + end + expect(bytes.byteslice(-22, 4).unpack1('V')).to eq(0x06054b50) + files + end + + before do + snapshot_tag(tag) + PostTag.create!(post:, tag:) + snapshot_post(post, tags: [{ tag_id: tag.id, version_no: 1, sections: [] }]) + end + + it 'generates a navigable ZIP, escaped HTML, search data and verified hashes' do + progress = [] + files = archive_files(mirror.export { |done, total| progress << [done, total] }) + expect(progress.last).to eq([2, 2]) + expect(files.keys).to include('index.html', 'robots.txt', 'assets/archive.css', + 'assets/archive.js', 'archive-config.js', 'search-index.json', 'manifest.json', + 'ja/posts/index.html', "ja/posts/#{ post.id }/index.html", + 'ja/tags/index.html', "ja/tags/#{ tag.id }/index.html") + files.select { |path, _| path.end_with?('.html') }.each do |path, html| + document = Nokogiri::HTML(html) + expect(document.at_css('meta[name="robots"]')['content']).to include('noindex') + expect(document.text).to include('静的保存版', mirror.data_at) + expect(document.css('img')).to be_empty + expect(document.at_css('[data-original]')['href']).to start_with('https://source.test/ja') + document.css('a[href], link[href], script[src]').each do |node| + target = node['href'] || node['src'] + next if target.start_with?('https://') + + expect(target).not_to start_with('/') + resolved = Pathname.new(path).dirname.join(target).cleanpath.to_s + expect(files).to have_key(resolved) + end + end + detail = Nokogiri::HTML(files.fetch("ja/posts/#{ post.id }/index.html")) + expect(detail.at_css('h1').text).to eq(post.title) + expect(detail.css('main a').map { _1['href'] }) + .to include("../../tags/#{ tag.id }/index.html") + manifest = JSON.parse(files.fetch('manifest.json')) + expect(manifest).to include('format_version' => 1, 'locale' => 'ja', + 'data_at' => mirror.data_at, 'resources' => { 'posts' => 1, 'tags' => 1 }) + expect(Time.iso8601(manifest.fetch('generated_at'))).to be >= Time.iso8601(mirror.data_at) + expect(manifest.fetch('files').map { _1.fetch('path') }) + .to match_array(files.keys - ['manifest.json']) + manifest.fetch('files').each do |entry| + digest = Digest::SHA256.hexdigest(files.fetch(entry.fetch('path'))) + expect(entry.fetch('sha256')).to eq(digest) + end + expect(JSON.parse(files.fetch('search-index.json')).map { _1.fetch('path') }) + .to include("ja/posts/#{ post.id }/index.html", "ja/tags/#{ tag.id }/index.html") + end + + it 'keeps private and unversioned records out even for an admin caller' do + private_tag = create(:tag, primary_name: 'mirror-secret-tag') + snapshot_tag(private_tag) + snapshot_tag(private_tag, event: 'discard', number: 2) + private_post = create(:post, title: 'mirror-secret-post') + snapshot_post(private_post) + PostVersion.create!(post: private_post, version_no: 2, event_type: 'discard', + url: private_post.url, tags: '', tags_json: [], parent_post_ids: '', + created_at: Time.current) + legacy = create(:post, title: 'mirror-unversioned-secret') + referencing = create(:post, title: 'mirror-secret-reference') + snapshot_post(referencing, + tags: [{ tag_id: private_tag.id, version_no: 1, sections: [] }]) + files = HistoryAccess.set(user: create(:user, :admin)) { archive_files(mirror.export) } + expect(files.values.join).not_to include('mirror-secret', legacy.title, private_post.url) + expect(files.keys).not_to include("ja/tags/#{ private_tag.id }/index.html", + "ja/posts/#{ private_post.id }/index.html", "ja/posts/#{ referencing.id }/index.html") + expect(mirror.authority.fetch('admin')).to be(false) + end + + it 'applies later discard boundaries after a restoration' do + snapshot_tag(tag, event: 'discard', number: 2) + snapshot_tag(tag, event: 'restore', number: 3) + tag.update_columns(version_no: 3) + expect(archive_files(mirror.export)).to have_key("ja/tags/#{ tag.id }/index.html") + snapshot_tag(tag, event: 'discard', number: 4) + second = described_class.new(locale: Locale.nipponese, origin: 'https://source.test') + files = archive_files(second.export) + expect(files.keys).not_to include("ja/tags/#{ tag.id }/index.html") + expect(files.values.join).not_to include('mirror-public-tag') + end + + it 'does not include credentials or executable URL schemes' do + post.update_columns(url: 'https://secret-user:secret-password@source.test/path') + content = archive_files(mirror.export).values.join + expect(content).not_to include('secret-password', 'secret-user') + post.update_columns(url: 'javascript:alert(1)') + expect(archive_files(mirror.export).values.join).not_to include('javascript:') + end + + it 'fails if publication changes during generation' do + expect do + mirror.export do |done, _| + snapshot_tag(tag, event: 'discard', number: 2) if done.zero? + end + end.to raise_error(HistoryAccess::Forbidden) + end + + it 'bounds output and never reports hidden population counts' do + stub_const('StaticMirror::RESOURCE_LIMIT', 1) + second = create(:post, title: 'mirror-beyond-limit') + snapshot_post(second) + files = archive_files(mirror.export) + expect(files.values.join).not_to include(second.title, 'population_count', 'excluded_count') + stub_const('StaticMirror::MAX_BYTES', 16) + expect { mirror.export }.to raise_error(StaticMirror::TooLarge) + end + + it 'keeps all registered language names and aliases without locale filtering' do + locale + create(:tag_name, :alias, tag:, name: 'mirror-alias', language_code: 'ja', script_code: 'Jpan') + files = archive_files(described_class.new(locale:, origin: 'https://source.test').export) + detail = files.fetch("ain-Kana/tags/#{ tag.id }/index.html") + expect(detail).to include('mirror-public-tag', 'mirror-alias', 'lang="ain"', 'lang="ja"') + end + + it 'fails rather than truncating registered aliases at the name limit' do + create(:tag_name, :alias, tag:, name: 'mirror-extra-alias') + stub_const('StaticMirror::NAME_LIMIT', 1) + expect { mirror.export }.to raise_error(StaticMirror::TooLarge) + end +end diff --git a/docs/static-mirror.md b/docs/static-mirror.md new file mode 100644 index 0000000..80462b2 --- /dev/null +++ b/docs/static-mirror.md @@ -0,0 +1,169 @@ +# Static Mirror PoC (#483) + +現在の公開状態の一部を、Rails で HTML にして ZIP 出力する開発環境専用の PoC。 +Post は ID・タイトル・URL・出力済み Tag との関連、Tag は ID・カテゴリ・全言語の +名前と別名・出力済み Post との関連を出力する。それ以外の Post / Tag 属性は対象外。 +DB backup / restore とは別機能で、日時指定の復元機能はない。 +本番環境では全 API が 404 になる。依存パッケージ・DB schema の変更はない。 + +## 操作 + +既存のローカル開発用 MySQL 設定と locale 登録を用意した環境で、 +`backend/` の `bin/rails server -p 3002` と、`frontend/` の `npm run dev` を使う。 +Vite の既存 `/api` proxy は3002番ポートを参照する。これらの起動コマンドは +今回実行していない。DB の準備・migration・依存導入も実行していない。 + +1. `http://localhost:5173/ja/static-mirror` を開く。開発時の「その他」にもリンクがある。 + 有効な locale では `/:locale/static-mirror` から利用できる。 +2. 「スナップショット生成」を押す。ログインは不要。通常の CSRF、IP BAN、user BAN + の境界を通り、セッションに紐づくジョブを登録する。 +3. 状態・処理件数・データ基準時点を確認し、完了後「ZIP ダウンロード」を押す。 +4. ZIP を展開し、`archive/` を通常の静的 HTTP サーバーへ配置する。 + `/保存版/archive/index.html` のようなサブディレクトリ配置に対応する。 + DB、Node.js、原サーバー、外部 API は閲覧先では不要。 + +一覧・詳細・相互リンクは JavaScript 無効でも利用できる。検索は HTTP 配信した +`search-index.json` を JavaScript が読む。`file://` からの検索は保証しない。 +外部動画・画像・サムネール・素材ファイルは取得も同梱もしない。Post の外部 URL は +通常のリンクとして残すため、そのリンク先の閲覧は外部サイトの存続に依存する。 + +各 HTML に静的保存版の表示、UTC のデータ基準時点、`noindex, nofollow`、 +本家への絶対リンクを焼き込む。内部リンク・CSS・JS は相対参照。 +`archive-config.js` の `window.archiveConfig.origin` を HTTP(S) の本家基点 URL +(例: `https://example.org`)に変更すれば本家リンクをまとめて変更できる。 +設定 JS が削除・欠落・不正な場合や JS 無効時にも HTML の初期リンクが残る。 +設定は配置先ミラーの URL ではなく、本家の基点を指す。 + +## API と処理上限 + +| API | 役割 | +| --- | --- | +| `GET /static_mirror` | 件数・容量・実行制限と対象外機能 | +| `POST /static_mirror/jobs` | 現在状態のジョブ登録。入力は `locale` のみ | +| `GET /static_mirror/jobs/:id` | 所有セッションの状態・処理進捗・基準時点・失敗コード | +| `GET /static_mirror/files/:id` | 所有セッションの完成 ZIP を再認可して取得 | + +通常どおりフロントエンドの API transport と Query key を使用する。 +URL に JSON を格納しない。`at` 等の任意時点入力・未知の生成オプションは400で拒否する。 + +- Post / Tag は各100件。公開認可後に ID 昇順で選ぶ。非公開件数や全母集団件数は返さない。 +- 対象 Tag の全言語名・別名は最大2,000行。超過時は失敗させ、黙って切り捨てない。 + 言語・文字体系は保持し、表示対象言語に登録がない場合は ID を使う。 +- Post と Tag の関連は両者が出力集合にあるものだけ。出力外への内部リンクは生成しない。 +- 展開後のファイル合計及び ZIP は各8 MiB 以下。超過時は配布しない。 +- 同時実行1件、全セッションを通じて新規登録は60秒に1件、登録から処理期限60秒。 + 競合・受付制限は429と `Retry-After`。状態取得・取得時の短いロック競合は503。 +- ZIP とジョブの認可期限は10分。次のジョブ登録で前の ZIP / ジョブは置き換える。 + 期限後も最後のファイルがディスクに残り得るが、API では取得できない。 + 容量は単一 ZIP と作業ファイルに限定され、無制限に蓄積しない。 + +Rails の既存 Active Job と `:async` adapter を使い、生成は HTTP request の外で行う。 +既存 `CountsExportJobs` は Counts の入力・投影・配信確認に結合しているため変更せず、 +本機能の状態・単一成果物の管理だけを feature 固有の Job に置く。 +受付・状態・ZIP は `backend/tmp/static_mirror/` の単一 spool を使う。 +プロセス間の filesystem lock で実行枠と更新を制御する。各ファイルは0600、directory は +0700で作成する。Web プロセス再起動で非永続キューが失われる点が PoC の限界。 +進まない queued / running は75秒後の状態取得で failed になる。再開や再実行はしない。 + +失敗コードは `too_large`、`generation_timeout`、`publication_changed`、 +`generation_failed`、`worker_stopped`。内部の例外本文・パス・資源属性は応答に含めない。 +画面では失敗コード又は既存の通信・認可エラー文言を表示する。 + +## 公開制御と整合性 + +生成者の role によらず `HistoryAccess.set(user: nil)` の公開権限を使う。 +既存の `HistoryAccess.current` と確定済みの同一 `version_no` の存在確認を組み合わせる。 +独自の role 判定や簡易な除票判定は作らない。対応する immutable revision がない +legacy record は安全側で対象外とする。公開不可の参照を含む確定版も既存認可で除外する。 +廃止 Tag は対象外。HTML / 検索 JSON / manifest の件数には認可済みの出力集合だけが入る。 +user、actor、履歴版番号、監査、認証、blob 情報は投影しない。 + +`CountsExporter.with_read_snapshot` の既存 repeatable-read 境界を再利用し、Post、Tag、 +全言語 TagName、PostTag、publication revision を同じ読み取り view から取得する。 +これには既存 HistoryAccess の SQL が動作する MySQL 環境が必要。未対応の adapter や +未整備の DB では生成失敗とし、認可を迂回した出力へフォールバックしない。 +`data_at` はこの現在状態読み取りの開始時刻であり、履歴の指定日時復元を表さない。 +`generated_at` はファイル生成後の別の時刻。 + +生成の最後と状態・ZIP 取得時に既存 artifact policy / discard revision を再検証する。 +生成中の除票は失敗にし、生成後の除票では古い ZIP の再取得を404にする。 +ZIP は再認可後のバイト列として返し、遅延 sendfile に古いパスを渡さない。 +HTTP 応答は既存境界の `private, no-store`。ダウンロード済み ZIP の回収や、 +最終認可と送出の間に発生する公開制御変更の完全な排他は保証できない。 +新しい公開制御を導入する場合は共通 policy と artifact revision の連動が必要。 + +本文・名前・URL は escape する。外部リンクは HTTP(S)・host・userinfo 不在を検証し、 +不正な URL はリンクとして出さない。外部 URL の取得は行わないため SSRF 用 DNS 検査を +流用しない。HTML の CSP は外部スクリプト・画像・通信を許可しない。 + +`manifest.json` に形式版、生成日時、基準時点、locale、出力件数、対象外項目、上限、 +各ファイルの相対パスと SHA-256 を記録する。自己ハッシュは成立しないため manifest +自身だけは一覧の対象外とし、`checksum_scope` にその範囲を記録する。 + +## 過去時点復元の調査 + +- `post_versions` / `tag_versions` は親 ID と正の `version_no` に一意制約があり、 + readonly の `VersionRecord` を使う。`created_at <= 時点` の中で最大の確定版を + 選ぶことは可能。同一時刻の順序は時刻だけで決めず版番号を併用する必要がある。 +- Tag は `names` に言語別の名前・script・別名を保存する。古い行の移行では + 日本語として backfill し、旧 aliases 文字列を分割している。失われた区切りや + 当時未記録の他言語名を現在の TagName で補完して正確と称することはできない。 +- Post は `tags_json`、区間、Tag 版参照、`parent_post_ids` を、Tag は + `parent_tag_ids` を記録する。ただし legacy `tags_json` は名前・時刻からの補完を + 含む。全時代・全書き込み経路の関係変更の完全性を検証する必要がある。 + 外部 Tag 名の現行参照など、immutable history だけでは揃わない値もある。 +- Tag Description は `(tag_id, language_code, version_no)` の版に本文と翻訳元を記録する。 + Material にはファイル情報等の版があるが、blob 本体の永久保存や除票・公開抑止との + 整合は別途必要。Help はフロントエンドの文書配信に依存し、DB の任意時点だけでは + 当時の公開版を復元できない。 +- 当時の publication policy そのものや将来追加される公開抑止の完全な履歴は、 + これらの内容版だけでは保証できない。指定時点の再現と現在の公開許可を別々に評価し、 + 現在の全後続除票境界を版番号と複合キーで適用する必要がある。恢復しても過去の + 秘匿履歴を再公開しない。Counts の件数例外はこの機能へ拡張しない。 +- 正確な時点復元には、全 resource の履歴の起点・欠落・backfill の識別、関係・名前・ + policy・locale・asset の時点整合性、immutable な Help の版、削除済み asset の保存方針、 + 一つの DB read view と現在の認可の再検証、同時刻更新の順序・境界の設計が必要。 + +このため日時入力は実装していない。履歴評価はソース調査であり、実 DB の網羅監査ではない。 + +## 検証と残課題 + +追加した回帰テスト: + +- `backend/spec/services/static_mirror_spec.rb`: ZIP の CRC / 構造、全ファイルの hash、 + 内部参照、escape、基準時点、公開権限、後続除票、言語名・別名、容量・件数上限。 +- `backend/spec/requests/static_mirrors_spec.rb`: 一般受付、非同期境界、所有権、 + 公開制御変更、失敗・期限・競合・全体 rate limit、開発環境制限、CSRF と BAN。 +- `frontend/src/pages/StaticMirrorPage.test.tsx`: 開始、進捗、取得、失敗、 + 受付エラー、認可喪失、日本語 fallback の言語属性。 +- `frontend/src/lib/staticMirrorAssets.test.ts`: 静的検索の相対参照・escape と + サブディレクトリ配備、本家設定の欠落・不正・変更。 + +実施したのは既存コードの調査と差分の目視確認のみ。テスト、lint、build、型検査、 +migration、実ブラウザーでの表示、ZIP 展開・HTTP 配置は実行していない。 +管理者指定の suite による結果確認と、サブディレクトリ配置・JS 無効・原サーバー停止・ +設定 JS 欠落・静的検索の実画面確認が残る。 + +本番公開には durable な worker / queue、復旧・取消・削除と運用監視、適切な利用者別 +rate limit、容量・時間・負荷の実測、公開制御の監査と変更検知、長時間配信の再認可方針、 +asset の権利・保管方針、追加 resource の安全な投影、UI 文言・翻訳の確定が必要。 +本 PoC の開発環境制限を単に外して公開することは想定しない。 + +## 変更ファイルの責務 + +| ファイル | 責務 | +| --- | --- | +| `backend/config/routes.rb` | metadata・job・ZIP の4経路 | +| `backend/app/controllers/static_mirrors_controller.rb` | 通常認証境界、開発制限、入力・HTTP 応答 | +| `backend/app/jobs/static_mirror_job.rb` | 単一生成枠、受付制限、状態・成果物・期限・取得認可 | +| `backend/app/services/static_mirror.rb` | 公開データの一貫した投影、ページ・検索・manifest・ZIP | +| `backend/app/views/static_mirror/page.html.erb` | escape 済み HTML、ナビ、基準時点・本家リンク・CSP | +| `backend/app/views/static_mirror/archive.css` | 静的サイトの読み取り表示 | +| `backend/app/views/static_mirror/archive.js` | ローカル検索と本家リンクの設定変更 | +| `frontend/src/App.tsx` | locale 付き操作画面経路 | +| `frontend/src/lib/queryKeys.ts` | 既存 Query key entrypoint への追加 | +| `frontend/src/pages/StaticMirrorPage.tsx` | 開始・制限・進捗・失敗・ZIP 操作 | +| `frontend/src/pages/MorePage.tsx` | 開発時の画面への導線 | +| `frontend/src/locales/{ja,ain-Kana}/archive.json` | Issue 指定の文言と未翻訳 fallback | +| 上記4テストファイル | 回帰テスト | +| `docs/static-mirror.md` | 手順・制約・調査・未検証事項 | diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index a6799ad..96452e2 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -83,6 +83,7 @@ const MaterialSyncSuppressionsPage = lazyPage ( () => import ('@/pages/materials/MaterialSyncSuppressionsPage')) const CountsResourcePage = lazyPage (() => import ('@/pages/counts/CountsResourcePage')) const MorePage = lazyPage (() => import ('@/pages/MorePage')) +const StaticMirrorPage = lazyPage (() => import ('@/pages/StaticMirrorPage')) const GekanatorUnavailablePage = lazyPage (() => import ('@/pages/GekanatorUnavailablePage')) const NicoTagListPage = lazyPage (() => import ('@/pages/tags/NicoTagListPage')) const TOSPage = lazyPage (() => import ('@/pages/TOSPage')) @@ -126,6 +127,8 @@ export const pageRoutes = ( user: User | null, setUser: Dispatch>, ): PageRoute[] => [ + { path: 'static-mirror', element: , + handle: { preload: StaticMirrorPage.preload, robots: 'noindex' } }, { path: '', element: , handle: { redirectTo: '/posts' } }, { path: 'posts', element: , diff --git a/frontend/src/lib/queryKeys.ts b/frontend/src/lib/queryKeys.ts index 7e1d09c..bf3cf35 100644 --- a/frontend/src/lib/queryKeys.ts +++ b/frontend/src/lib/queryKeys.ts @@ -7,6 +7,10 @@ import type { MaterialFilter, WikiTagReference } from '@/types' +export const staticMirrorKeys = { + definition: ['static_mirror', 'definition'] as const, + job: (id: string | null) => ['static_mirror', 'jobs', id] as const } + export const localesKeys = { root: ['locales'] as const, index: ['locales', 'index'] as const } diff --git a/frontend/src/lib/staticMirrorAssets.test.ts b/frontend/src/lib/staticMirrorAssets.test.ts new file mode 100644 index 0000000..5c2156a --- /dev/null +++ b/frontend/src/lib/staticMirrorAssets.test.ts @@ -0,0 +1,59 @@ +import { readFileSync } from 'node:fs' +import { URL as NodeURL } from 'node:url' +import { fireEvent, waitFor } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const script = readFileSync (new NodeURL ( + '../../../backend/app/views/static_mirror/archive.js', import.meta.url), 'utf8') +const archiveWindow = window as typeof window & { archiveConfig?: { origin: string | null } } + +beforeEach (() => { + const element = document.createElement ('script') + element.src = 'https://mirror.test/subdirectory/archive/assets/archive.js' + vi.spyOn (document, 'currentScript', 'get').mockReturnValue (element) + delete archiveWindow.archiveConfig + document.body.innerHTML = ` + + +
      ` +}) +afterEach (() => { + delete archiveWindow.archiveConfig + document.body.replaceChildren () + vi.unstubAllGlobals () +}) + +describe ('Portable static mirror assets', () => { + it ('searches local JSON by aliases using safe text and relative links', async () => { + const fetcher = vi.fn ().mockResolvedValue ({ ok: true, json: async () => [ + { path: 'ja/tags/1/index.html', title: '', names: ['alias'] }, + { path: 'https://outside.test/', title: 'alias' }, + { path: '../outside.html', title: 'alias' }] }) + vi.stubGlobal ('fetch', fetcher) + new Function (script) () + fireEvent.submit (document.querySelector ('form')!) + await waitFor (() => expect (document.querySelectorAll ('#search-results a')).toHaveLength (1)) + const link = document.querySelector ('#search-results a')! + expect (link).toHaveAttribute ('href', 'ja/tags/1/index.html') + expect (link.textContent).toBe ('') + expect (document.querySelector ('img')).toBeNull () + expect (fetcher).toHaveBeenCalledTimes (1) + expect (fetcher.mock.calls[0][0].href) + .toBe ('https://mirror.test/subdirectory/archive/search-index.json') + }) + + it ('retains baked links without config and changes only source links with valid config', () => { + new Function (script) () + const source = document.querySelector ('[data-original]')! + expect (source).toHaveAttribute ('href', 'https://source.test/ja/posts/1') + archiveWindow.archiveConfig = { origin: 'javascript:alert(1)' } + new Function (script) () + expect (source).toHaveAttribute ('href', 'https://source.test/ja/posts/1') + archiveWindow.archiveConfig = { origin: 'https://replacement.test/base/' } + new Function (script) () + expect (source).toHaveAttribute ('href', 'https://replacement.test/base/ja/posts/1') + expect (document.querySelector ('a:not([data-original])')) + .toHaveAttribute ('href', 'ja/posts/index.html') + }) +}) diff --git a/frontend/src/locales/ain-Kana/archive.json b/frontend/src/locales/ain-Kana/archive.json new file mode 100644 index 0000000..a8175ac --- /dev/null +++ b/frontend/src/locales/ain-Kana/archive.json @@ -0,0 +1,19 @@ +{ + "_untranslated": ["*"], + "title": "Static Mirror 生成機能", + "generation": "スナップショット生成", + "dataAt": "データ基準時点", + "currentOnly": "今回は現在状態のみの対応", + "developmentOnly": "開発環境限定", + "limits": "出力件数の制限", + "concurrency": "同時実行数", + "exclusions": "未対応機能", + "protection": "データ保護上の制約", + "protectionDetails": "公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。", + "externalAssets": "外部画像や動画への依存", + "externalMedia": "外部画像や動画", + "externalOmissions": "ローカルへ取り込めていない資産", + "download": "ZIP ダウンロード", + "zip": "ZIP", + "failure": "失敗時の理由" +} diff --git a/frontend/src/locales/ja/archive.json b/frontend/src/locales/ja/archive.json new file mode 100644 index 0000000..866b0f5 --- /dev/null +++ b/frontend/src/locales/ja/archive.json @@ -0,0 +1,18 @@ +{ + "title": "Static Mirror 生成機能", + "generation": "スナップショット生成", + "dataAt": "データ基準時点", + "currentOnly": "今回は現在状態のみの対応", + "developmentOnly": "開発環境限定", + "limits": "出力件数の制限", + "concurrency": "同時実行数", + "exclusions": "未対応機能", + "protection": "データ保護上の制約", + "protectionDetails": "公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。", + "externalAssets": "外部画像や動画への依存", + "externalMedia": "外部画像や動画", + "externalOmissions": "ローカルへ取り込めていない資産", + "download": "ZIP ダウンロード", + "zip": "ZIP", + "failure": "失敗時の理由" +} diff --git a/frontend/src/pages/MorePage.tsx b/frontend/src/pages/MorePage.tsx index f6aba5c..4aeff61 100644 --- a/frontend/src/pages/MorePage.tsx +++ b/frontend/src/pages/MorePage.tsx @@ -16,12 +16,17 @@ import type { User } from '@/types' const MorePage: FC = () => { const location = useLocation () const { message } = useUITranslation ('shared') + const { label: archiveLabel } = useUITranslation ('archive') const menu = menuOutline ( { tag: null, wikiId: null, user: { } as User, pathName: location.pathname }) return ( + {import.meta.env.DEV && ( + + {archiveLabel ('title')} + )} {[...Array (Math.ceil (menu.length / 4)).keys ()].map (i => (
      diff --git a/frontend/src/pages/StaticMirrorPage.test.tsx b/frontend/src/pages/StaticMirrorPage.test.tsx new file mode 100644 index 0000000..246465d --- /dev/null +++ b/frontend/src/pages/StaticMirrorPage.test.tsx @@ -0,0 +1,96 @@ +import { act, fireEvent, screen, waitFor } from '@testing-library/react' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import StaticMirrorPage from '@/pages/StaticMirrorPage' +import { staticMirrorKeys } from '@/lib/queryKeys' +import { renderWithProviders } from '@/test/render' + +const api = vi.hoisted (() => ({ + apiGet: vi.fn (), apiPost: vi.fn (), startAPIDownload: vi.fn () })) +vi.mock ('@/lib/api', async original => ({ + ...await original (), ...api })) + +const definition = { + resourceLimit: 100, nameLimit: 2000, maxBytes: 8388608, maxSeconds: 60, + concurrency: 1, cooldownSeconds: 60, expirySeconds: 600, + exclusions: ['historical_state', 'unversioned_resources', 'images', 'videos'] } +const ready = { + status: 'ready', dataAt: '2026-11-16T00:00:00.000000Z', processed: 2, total: 2 } + +beforeEach (() => { + vi.clearAllMocks () + api.apiGet.mockImplementation (async (path: string) => + path === '/static_mirror' ? definition : ready) + api.apiPost.mockResolvedValue ({ job: 'opaque-job' }) + api.startAPIDownload.mockReturnValue (vi.fn ()) +}) + +const startGeneration = async () => { + const button = screen.getByRole ('button', { name: 'スナップショット生成' }) + await waitFor (() => expect (button).toBeEnabled ()) + fireEvent.click (button) +} + +describe ('Static Mirror generation', () => { + it ('starts a current snapshot and downloads through the common API', async () => { + const view = renderWithProviders (, { route: '/ja/static-mirror' }) + await startGeneration () + expect (api.apiPost).toHaveBeenCalledWith ('/static_mirror/jobs', { }, { locale: 'ja' }) + fireEvent.click (await screen.findByRole ('button', { name: 'ZIP ダウンロード' })) + expect (screen.getByText (ready.dataAt)).toBeInTheDocument () + expect (screen.getByText ('言語別名称').parentElement).toHaveTextContent ('2000') + expect (api.startAPIDownload).toHaveBeenCalledWith ( + '/static_mirror', 'opaque-job', expect.any (Function)) + const release = api.startAPIDownload.mock.results[0].value + view.unmount () + expect (release).toHaveBeenCalled () + }) + + it ('shows generation progress and offers the ZIP only after readiness', async () => { + let state = { ...ready, status: 'running', processed: 1 } + api.apiGet.mockImplementation (async (path: string) => + path === '/static_mirror' ? definition : state) + const view = renderWithProviders () + await startGeneration () + expect (await screen.findByText ('生成中')).toBeInTheDocument () + expect (screen.queryByRole ('button', { name: 'ZIP ダウンロード' })).toBeNull () + expect (screen.getByRole ('button', { name: 'スナップショット生成' })).toBeDisabled () + state = ready + await act (async () => { + await view.queryClient.invalidateQueries ({ queryKey: staticMirrorKeys.job ('opaque-job') }) + }) + expect (await screen.findByRole ('button', { name: 'ZIP ダウンロード' })).toBeEnabled () + }) + + it ('keeps failure reasons visible without offering a ZIP', async () => { + api.apiGet.mockImplementation (async (path: string) => + path === '/static_mirror' ? definition : { ...ready, status: 'failed', error: 'too_large' }) + renderWithProviders () + await startGeneration () + expect (await screen.findByRole ('alert')).toHaveTextContent ('too_large') + expect (screen.queryByRole ('button', { name: 'ZIP ダウンロード' })).toBeNull () + }) + + it ('shows intake limits and revoked downloads using existing error copy', async () => { + api.apiPost.mockRejectedValueOnce ({ isAxiosError: true, response: { status: 429 } }) + renderWithProviders () + await startGeneration () + expect (await screen.findByRole ('alert')).toHaveTextContent ('HTTP 429') + api.apiGet.mockImplementation (async (path: string) => { + if (path === '/static_mirror') return definition + throw { isAxiosError: true, response: { status: 404 } } + }) + await startGeneration () + await waitFor (() => expect (screen.getByRole ('alert')).toHaveTextContent ('有効期限')) + expect (screen.getByRole ('button', { name: 'スナップショット生成' })).toBeEnabled () + }) + + it ('preserves Japanese fallback language metadata for untranslated UI', async () => { + renderWithProviders (, { route: '/ain-Kana/static-mirror' }) + await startGeneration () + expect (api.apiPost).toHaveBeenCalledWith ( + '/static_mirror/jobs', { }, { locale: 'ain-Kana' }) + expect (screen.getByRole ('heading', { name: 'Static Mirror 生成機能' }) + .querySelector ('[lang="ja"]')).toBeInTheDocument () + }) +}) diff --git a/frontend/src/pages/StaticMirrorPage.tsx b/frontend/src/pages/StaticMirrorPage.tsx new file mode 100644 index 0000000..0593f30 --- /dev/null +++ b/frontend/src/pages/StaticMirrorPage.tsx @@ -0,0 +1,140 @@ +import { useMutation, useQuery } from '@tanstack/react-query' +import { useEffect, useRef, useState } from 'react' + +import PageHead from '@/components/common/PageHead' +import PageTitle from '@/components/common/PageTitle' +import MainArea from '@/components/layout/MainArea' +import { Button } from '@/components/ui/button' +import { apiGet, apiPost, isApiError, startAPIDownload } from '@/lib/api' +import { staticMirrorKeys } from '@/lib/queryKeys' +import { useLocale } from '@/lib/useLocale' +import { useUITranslation } from '@/lib/useUITranslation' + +type Definition = { + resourceLimit: number + nameLimit: number + maxBytes: number + maxSeconds: number + concurrency: number + cooldownSeconds: number + expirySeconds: number + exclusions: string[] } + +type Job = { + status: 'queued' | 'running' | 'ready' | 'failed' + dataAt: string | null + processed: number + total: number | null + error?: string } + +const StaticMirrorPage = () => { + const locale = useLocale () + const { label, message } = useUITranslation ('archive') + const [job, setJob] = useState (null) + const [downloadFailed, setDownloadFailed] = useState (false) + const downloadRef = useRef<(() => void) | null> (null) + useEffect (() => () => downloadRef.current?.(), []) + + const definition = useQuery ({ + queryKey: staticMirrorKeys.definition, + queryFn: () => apiGet ('/static_mirror'), retry: false }) + const { mutate: start, isPending: pending, error: creationError } = useMutation ({ + mutationFn: () => apiPost<{ job: string }> ('/static_mirror/jobs', { }, { locale }), + onSuccess: result => { + downloadRef.current?.() + setDownloadFailed (false) + setJob (result.job) + } }) + const status = useQuery ({ + queryKey: staticMirrorKeys.job (job), + queryFn: () => apiGet (`/static_mirror/jobs/${encodeURIComponent (job!)}`), + enabled: job != null, + retry: false, + refetchInterval: query => + (query.state.error != null + || ['ready', 'failed'].includes (query.state.data?.status ?? '')) + ? false : 1000 }) + const unavailable = isApiError (status.error) && status.error.response?.status === 404 + const active = job != null && !unavailable + && !['ready', 'failed'].includes (status.data?.status ?? '') + const error = creationError ?? status.error ?? definition.error + const errorStatus = isApiError (error) ? error.response?.status : null + const metadata = definition.data + const percentage = + (status.data?.total == null || status.data.total === 0) + ? 100 + : Math.round (100 * status.data.processed / status.data.total) + + const download = () => { + if (job == null) return + downloadRef.current?.() + setDownloadFailed (false) + downloadRef.current = startAPIDownload ( + '/static_mirror', job, () => setDownloadFailed (true)) + } + + return ( + + + {label ('title')} +

      {label ('developmentOnly')} / {label ('currentOnly')}

      + {metadata != null && ( +
      +
      {label ('limits')}
      +
      + {label ('counts:resources.posts')} / {label ('counts:resources.tags')}: + {' '}{metadata.resourceLimit} +
      +
      {label ('counts:columns.names')}: {metadata.nameLimit}
      +
      {metadata.maxBytes} B / {metadata.maxSeconds} s
      +
      {label ('concurrency')}
      +
      {metadata.concurrency} / {metadata.cooldownSeconds} s
      +
      {label ('zip')}: {metadata.expirySeconds} s
      +
      {label ('exclusions')}
      +
      + {['counts:resources.tag_descriptions', 'navigation:help', + 'counts:resources.materials', 'counts:resources.nico_tags', + 'counts:attributes.parent_post_ids', 'counts:attributes.parent_tag_ids', + 'counts:columns.sections', 'navigation:history', 'navigation:edit'] + .map (key => {label (key)}{' / '})} +
      +
      {label ('externalAssets')}
      +
      {label ('externalOmissions')}: {label ('externalMedia')}
      +
      {label ('protection')}
      +
      {label ('protectionDetails')}
      +
      )} + + {status.data != null && ( +
      +

      {label ('dataAt')}:

      +

      + {status.data.status === 'ready' + ? label ('download') + : label (`counts:exportProgress.${status.data.status}`)} +

      + {status.data.total != null && ( +

      {label ('counts:exportProgress.measured', { + processed: status.data.processed, total: status.data.total, + percent: percentage })}

      )} + {status.data.status === 'ready' && ( + )} + {status.data.error != null && ( +

      {label ('failure')}: {status.data.error}

      )} +
      )} + {error != null && ( +

      + {errorStatus === 404 + ? label ('counts:downloadErrors.unavailable') + : (errorStatus != null + ? label ('counts:errors.http', { status: errorStatus }) + : label ('counts:errors.network'))} +

      )} + {status.error != null && ( + )} + {downloadFailed &&

      {label ('counts:downloadErrors.failed')}

      } +
      ) +} +export default StaticMirrorPage