AI: address issue #483
このコミットが含まれているのは:
@@ -0,0 +1,55 @@
|
||||
class StaticMirrorsController < ApplicationController
|
||||
before_action :development_only
|
||||
wrap_parameters false
|
||||
|
||||
def show
|
||||
render json: { resource_limit: StaticMirror::RESOURCE_LIMIT,
|
||||
name_limit: StaticMirror::NAME_LIMIT, max_bytes: StaticMirror::MAX_BYTES,
|
||||
max_seconds: StaticMirrorJob::MAX_SECONDS, concurrency: 1,
|
||||
cooldown_seconds: StaticMirrorJob::COOLDOWN,
|
||||
expiry_seconds: StaticMirrorJob::EXPIRY.to_i, exclusions: StaticMirror::EXCLUSIONS }
|
||||
end
|
||||
|
||||
def create
|
||||
# No date input is accepted: this endpoint cannot promise historical restoration.
|
||||
payload = request.request_parameters.except('authenticity_token')
|
||||
.merge(request.query_parameters)
|
||||
return head :bad_request unless (payload.keys - ['locale']).empty?
|
||||
|
||||
locale = resolve_locale!
|
||||
session[:static_mirror_owner] ||= SecureRandom.hex(32)
|
||||
job = StaticMirrorJob.start(owner: session[:static_mirror_owner], locale: locale.code,
|
||||
origin: request.base_url)
|
||||
render json: { job: }, status: :accepted
|
||||
rescue StaticMirrorJob::Busy
|
||||
response.headers['Retry-After'] = StaticMirrorJob::COOLDOWN.to_s
|
||||
head :too_many_requests
|
||||
rescue ActiveJob::EnqueueError
|
||||
render json: { error: 'generation_failed' }, status: :service_unavailable
|
||||
end
|
||||
|
||||
def status
|
||||
value = StaticMirrorJob.status(params[:id], session[:static_mirror_owner])
|
||||
return head :not_found unless value
|
||||
|
||||
render json: value
|
||||
rescue StaticMirrorJob::Busy
|
||||
head :service_unavailable
|
||||
end
|
||||
|
||||
def download
|
||||
bytes = StaticMirrorJob.download(params[:id], session[:static_mirror_owner])
|
||||
return head :not_found unless bytes
|
||||
|
||||
send_data bytes, filename: 'static-mirror.zip', type: 'application/zip',
|
||||
disposition: 'attachment'
|
||||
rescue StaticMirrorJob::Busy
|
||||
head :service_unavailable
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def development_only
|
||||
head :not_found unless Rails.env.development?
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,155 @@
|
||||
require 'fileutils'
|
||||
require 'timeout'
|
||||
|
||||
# Development PoC: one filesystem slot and Rails' in-process async executor.
|
||||
# A process restart loses queued work; stale state fails rather than reporting success.
|
||||
class StaticMirrorJob < ApplicationJob
|
||||
self.queue_adapter = :async
|
||||
EXPIRY = 10.minutes
|
||||
COOLDOWN = 60
|
||||
MAX_SECONDS = 60
|
||||
class Busy < StandardError; end
|
||||
|
||||
def self.directory = Rails.root.join('tmp/static_mirror')
|
||||
|
||||
def self.synchronise(blocking: false)
|
||||
FileUtils.mkdir_p(directory, mode: 0o700)
|
||||
File.open(directory.join('.registry.lock'), File::RDWR | File::CREAT, 0o600) do |lock|
|
||||
mode = blocking ? File::LOCK_EX : File::LOCK_EX | File::LOCK_NB
|
||||
raise Busy unless lock.flock(mode)
|
||||
|
||||
yield
|
||||
end
|
||||
end
|
||||
|
||||
def self.state
|
||||
JSON.parse(directory.join('state.json').read)
|
||||
rescue Errno::ENOENT
|
||||
nil
|
||||
end
|
||||
|
||||
def self.write(value)
|
||||
path = directory.join('state.tmp')
|
||||
File.open(path, 'w', 0o600) { _1.write(JSON.generate(value)) }
|
||||
File.rename(path, directory.join('state.json'))
|
||||
ensure
|
||||
File.unlink(path) if path&.exist?
|
||||
end
|
||||
|
||||
def self.start(owner:, locale:, origin:)
|
||||
id = synchronise do
|
||||
previous = state
|
||||
if previous && Time.current.to_f - previous.fetch('created_at') < COOLDOWN
|
||||
raise Busy
|
||||
end
|
||||
File.open(directory.join('.generation.lock'), File::RDWR | File::CREAT, 0o600) do |lock|
|
||||
raise Busy unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
||||
|
||||
FileUtils.rm_f(directory.join('archive.zip'))
|
||||
FileUtils.rm_f(directory.join('archive.part'))
|
||||
identifier = SecureRandom.hex(24)
|
||||
write({ 'id' => identifier, 'owner' => owner, 'locale' => locale, 'origin' => origin,
|
||||
'created_at' => Time.current.to_f, 'expiry_at' => (Time.current + EXPIRY).to_f,
|
||||
'status' => 'queued', 'processed' => 0, 'total' => nil, 'data_at' => nil })
|
||||
identifier
|
||||
end
|
||||
end
|
||||
perform_later(id)
|
||||
id
|
||||
rescue ActiveJob::EnqueueError
|
||||
update(id) { _1.merge!('status' => 'failed', 'error' => 'generation_failed') } if id
|
||||
raise
|
||||
end
|
||||
|
||||
def self.authorised_state(id, owner)
|
||||
value = state
|
||||
return unless owner.present? && value && value['id'] == id && value['owner'] == owner
|
||||
return unless value['expiry_at'] > Time.current.to_f
|
||||
|
||||
if (['queued', 'running'].include?(value['status']) &&
|
||||
Time.current.to_f - value['created_at'] > MAX_SECONDS + 15)
|
||||
value.merge!('status' => 'failed', 'error' => 'worker_stopped')
|
||||
write(value)
|
||||
end
|
||||
if value['status'] == 'ready'
|
||||
permitted = HistoryAccess.set(user: nil) { HistoryAccess.authorised_artifact?(value) }
|
||||
return unless permitted && directory.join('archive.zip').file?
|
||||
end
|
||||
value
|
||||
end
|
||||
|
||||
def self.status(id, owner)
|
||||
synchronise do
|
||||
authorised_state(id, owner)&.slice('status', 'processed', 'total', 'data_at', 'error')
|
||||
end
|
||||
end
|
||||
|
||||
def self.download(id, owner)
|
||||
synchronise do
|
||||
value = authorised_state(id, owner)
|
||||
return unless value && value['status'] == 'ready'
|
||||
|
||||
bytes = directory.join('archive.zip').binread
|
||||
# Revalidate after reading; send_data avoids delayed sendfile reuse of a stale path.
|
||||
permitted = HistoryAccess.set(user: nil) { HistoryAccess.authorised_artifact?(value) }
|
||||
permitted ? bytes : nil
|
||||
end
|
||||
end
|
||||
|
||||
def self.update(id)
|
||||
# Worker updates may wait for a short status/download read; request intake never waits.
|
||||
synchronise(blocking: true) do
|
||||
value = state
|
||||
return unless value && value['id'] == id
|
||||
|
||||
yield value
|
||||
write(value)
|
||||
end
|
||||
end
|
||||
|
||||
def perform(id)
|
||||
File.open(self.class.directory.join('.generation.lock'), File::RDWR) do |lock|
|
||||
return unless lock.flock(File::LOCK_EX | File::LOCK_NB)
|
||||
|
||||
value = self.class.state
|
||||
return unless value && value['id'] == id && value['status'] == 'queued'
|
||||
|
||||
generate(id, value)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def generate(id, value)
|
||||
self.class.update(id) { _1['status'] = 'running' }
|
||||
remaining = MAX_SECONDS - (Time.current.to_f - value.fetch('created_at'))
|
||||
raise Timeout::Error unless remaining.positive?
|
||||
|
||||
Timeout.timeout(remaining) do
|
||||
locale = Locale.find_by!(code: value.fetch('locale'), deprecated_at: nil)
|
||||
mirror = StaticMirror.new(locale:, origin: value.fetch('origin'))
|
||||
bytes = mirror.export do |processed, total|
|
||||
self.class.update(id) do |metadata|
|
||||
metadata.merge!('processed' => processed, 'total' => total, 'data_at' => mirror.data_at)
|
||||
end
|
||||
end
|
||||
part = self.class.directory.join('archive.part')
|
||||
File.open(part, 'wb', 0o600) { _1.write(bytes) }
|
||||
File.rename(part, self.class.directory.join('archive.zip'))
|
||||
self.class.update(id) do |metadata|
|
||||
metadata.merge!(mirror.authority).merge!('status' => 'ready', 'data_at' => mirror.data_at)
|
||||
end
|
||||
end
|
||||
rescue StandardError => error
|
||||
FileUtils.rm_f(self.class.directory.join('archive.part'))
|
||||
FileUtils.rm_f(self.class.directory.join('archive.zip'))
|
||||
code = case error
|
||||
when StaticMirror::TooLarge then 'too_large'
|
||||
when Timeout::Error then 'generation_timeout'
|
||||
when HistoryAccess::Forbidden then 'publication_changed'
|
||||
else 'generation_failed'
|
||||
end
|
||||
self.class.update(id) { _1.merge!('status' => 'failed', 'error' => code) }
|
||||
Rails.logger.warn("static_mirror_failure: #{ error.class }")
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,201 @@
|
||||
require 'cgi'
|
||||
require 'digest'
|
||||
require 'erb'
|
||||
require 'pathname'
|
||||
require 'uri'
|
||||
|
||||
# A deliberately small current-state projection, always under public authority.
|
||||
class StaticMirror
|
||||
RESOURCE_LIMIT = 100
|
||||
NAME_LIMIT = 2_000
|
||||
MAX_BYTES = 8.megabytes
|
||||
EXCLUSIONS = [
|
||||
'historical_state', 'unversioned_resources', 'tag_descriptions', 'help', 'materials',
|
||||
'external_tags', 'parent_relations', 'tag_sections', 'users', 'audit_data',
|
||||
'authentication', 'editing', 'images', 'videos', 'resources_beyond_limits'].freeze
|
||||
class TooLarge < StandardError; end
|
||||
class UnsafeUrl < StandardError; end
|
||||
|
||||
attr_reader :data_at, :authority, :counts
|
||||
|
||||
def initialize(locale:, origin:)
|
||||
@locale = locale
|
||||
@origin = validated_url(origin).sub(%r{/+\z}, '')
|
||||
end
|
||||
|
||||
def export(&progress)
|
||||
@files = { }
|
||||
@bytes = 0
|
||||
HistoryAccess.set(user: nil) do
|
||||
capture
|
||||
progress&.call(0, @posts.length + @tags.length)
|
||||
generate_pages(&progress)
|
||||
raise HistoryAccess::Forbidden unless HistoryAccess.authorised_artifact?(authority)
|
||||
|
||||
generated_at = Time.current.utc
|
||||
manifest = { format_version: 1, generated_at: generated_at.iso8601(6),
|
||||
data_at:, locale: @locale.code, resources: counts, exclusions: EXCLUSIONS,
|
||||
fields: { posts: ['id', 'title', 'url', 'tags'], tags: ['id', 'category', 'names'] },
|
||||
limits: { posts: RESOURCE_LIMIT, tags: RESOURCE_LIMIT, names: NAME_LIMIT,
|
||||
bytes: MAX_BYTES },
|
||||
external_assets: 'not_included', checksum_scope: 'all_files_except_manifest.json',
|
||||
files: @files.map { |path, data| { path:, sha256: Digest::SHA256.hexdigest(data) } } }
|
||||
add_file('manifest.json', JSON.pretty_generate(manifest))
|
||||
entries = @files.map do |path, data|
|
||||
MaterialZipExporter::Entry.new(path: "archive/#{ path }", data:, mtime: generated_at)
|
||||
end
|
||||
bytes = MaterialZipExporter::ZipWriter.write(entries)
|
||||
raise TooLarge if bytes.bytesize > MAX_BYTES
|
||||
|
||||
bytes
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def capture
|
||||
CountsExporter.with_read_snapshot do
|
||||
@data_at = Time.current.utc.iso8601(6)
|
||||
@authority = HistoryAccess.authority
|
||||
# Requiring a matching immutable revision closes the legacy unversioned gap.
|
||||
@posts = confirmed(Post, PostVersion, :post_versions).order(:id)
|
||||
.limit(RESOURCE_LIMIT).select(:id, :title, :url).to_a
|
||||
@tags = confirmed(Tag, TagVersion, :tag_versions).where(deprecated_at: nil)
|
||||
.order(:id).limit(RESOURCE_LIMIT).select(:id, :category).to_a
|
||||
@names = TagName.where(tag_id: @tags.map(&:id)).order(:tag_id, :language_code, :id)
|
||||
.limit(NAME_LIMIT + 1).pluck(:tag_id, :language_code, :script_code, :name, :primary_flg)
|
||||
raise TooLarge if @names.length > NAME_LIMIT
|
||||
|
||||
@names = @names.group_by(&:first)
|
||||
@relations = PostTag.where(post_id: @posts.map(&:id), tag_id: @tags.map(&:id))
|
||||
.order(:post_id, :tag_id).pluck(:post_id, :tag_id)
|
||||
@counts = { posts: @posts.length, tags: @tags.length }
|
||||
end
|
||||
end
|
||||
|
||||
def confirmed(model, version_model, association)
|
||||
HistoryAccess.current(model.all, version_model).joins(association)
|
||||
.where("#{ version_model.table_name }.version_no = #{ model.table_name }.version_no")
|
||||
end
|
||||
|
||||
def generate_pages
|
||||
assets = Rails.root.join('app/views/static_mirror')
|
||||
['archive.css', 'archive.js'].each do |name|
|
||||
add_file("assets/#{ name }", assets.join(name).read)
|
||||
end
|
||||
add_file('robots.txt', "User-agent: *\nDisallow: /\n")
|
||||
add_file('archive-config.js', 'window.archiveConfig = { origin: null };')
|
||||
page('index.html', 'Static Mirror 生成機能', '', search_form + limitations)
|
||||
page("#{ @locale.code }/posts/index.html", 'Post', '/posts', post_list(@posts))
|
||||
page("#{ @locale.code }/tags/index.html", 'Tag', '/tags', tag_list(@tags))
|
||||
processed = 0
|
||||
@posts.each do |post|
|
||||
tags = @tags.select { |tag| @relations.include?([post.id, tag.id]) }
|
||||
content = "<p>#{ external_link(post.url) }</p>#{ tag_list(tags) }"
|
||||
page(resource_path('posts', post.id), post.title.presence || post.id.to_s,
|
||||
"/posts/#{ post.id }", content)
|
||||
processed += 1
|
||||
yield(processed, @posts.length + @tags.length) if block_given?
|
||||
end
|
||||
@tags.each do |tag|
|
||||
posts = @posts.select { |post| @relations.include?([post.id, tag.id]) }
|
||||
names = @names.fetch(tag.id, []).map do |_, language, script, name, _primary|
|
||||
"<li><span lang=\"#{ h(language) }\">#{ h(name) }</span> (#{ h(script) })</li>"
|
||||
end.join
|
||||
content = "<p>#{ h(tag.category) }</p><ul>#{ names }</ul>#{ post_list(posts) }"
|
||||
page(resource_path('tags', tag.id), tag_name(tag), "/tags/#{ tag.id }", content)
|
||||
processed += 1
|
||||
yield(processed, @posts.length + @tags.length) if block_given?
|
||||
end
|
||||
index = @posts.map { |post|
|
||||
{ path: resource_path('posts', post.id), title: post.title.presence || post.id.to_s }
|
||||
} + @tags.map { |tag|
|
||||
{ path: resource_path('tags', tag.id), title: tag_name(tag),
|
||||
names: @names.fetch(tag.id, []).map { _1[3] } }
|
||||
}
|
||||
add_file('search-index.json', JSON.generate(index))
|
||||
end
|
||||
|
||||
def page(path, title, original_path, content)
|
||||
@path = path
|
||||
@title = title
|
||||
@original_path = "/#{ @locale.code }#{ original_path }"
|
||||
@content = content
|
||||
template = Rails.root.join('app/views/static_mirror/page.html.erb').read
|
||||
add_file(path, ERB.new(template).result(binding))
|
||||
end
|
||||
|
||||
def add_file(path, data)
|
||||
@bytes += data.bytesize
|
||||
raise TooLarge if @bytes > MAX_BYTES
|
||||
|
||||
@files[path] = data
|
||||
end
|
||||
|
||||
def resource_path(resource, id) = "#{ @locale.code }/#{ resource }/#{ id }/index.html"
|
||||
|
||||
def relative(path)
|
||||
Pathname.new(path).relative_path_from(Pathname.new(@path).dirname).to_s
|
||||
end
|
||||
|
||||
def h(value) = CGI.escapeHTML(value.to_s)
|
||||
|
||||
def tag_name(tag)
|
||||
@names.fetch(tag.id, []).find { _1[1] == @locale.language_code && _1[4] }&.fetch(3) ||
|
||||
tag.id.to_s
|
||||
end
|
||||
|
||||
# Links are resolved after the page path is known; only generated numeric IDs enter paths.
|
||||
def post_list(posts)
|
||||
'<h2>Post</h2><ul>' + posts.map { |post|
|
||||
title = post.title.presence || post.id.to_s
|
||||
"<li><a data-path=\"#{ resource_path('posts', post.id) }\">#{ h(title) }</a></li>"
|
||||
}.join + '</ul>'
|
||||
end
|
||||
|
||||
def tag_list(tags)
|
||||
'<h2>Tag</h2><ul>' + tags.map { |tag|
|
||||
"<li><a data-path=\"#{ resource_path('tags', tag.id) }\">#{ h(tag_name(tag)) }</a></li>"
|
||||
}.join + '</ul>'
|
||||
end
|
||||
|
||||
def linked_content
|
||||
@content.gsub(/data-path="([^"]+)"/) { "href=\"#{ h(relative(Regexp.last_match(1))) }\"" }
|
||||
end
|
||||
|
||||
def validated_url(value)
|
||||
uri = URI.parse(value.to_s)
|
||||
unless uri.is_a?(URI::HTTP) && uri.host.present? && uri.userinfo.nil?
|
||||
raise UnsafeUrl
|
||||
end
|
||||
|
||||
uri.to_s
|
||||
rescue URI::InvalidURIError
|
||||
raise UnsafeUrl
|
||||
end
|
||||
|
||||
def external_link(value)
|
||||
url = validated_url(value)
|
||||
"<a href=\"#{ h(url) }\" target=\"_blank\" rel=\"noopener noreferrer\">#{ h(url) }</a>"
|
||||
rescue UnsafeUrl
|
||||
''
|
||||
end
|
||||
|
||||
def search_form
|
||||
'<form id="archive-search"><label>検索 <input name="q" type="search"></label>' +
|
||||
'<button type="submit">検索</button></form><ul id="search-results"></ul>'
|
||||
end
|
||||
|
||||
def limitations
|
||||
<<~HTML
|
||||
<h2>未対応機能</h2>
|
||||
<p>Tag Description, Help, Material, 外部タグ, 親子関係, 区間, 履歴, ログイン, 作成, 編輯</p>
|
||||
<h2>出力件数の制限</h2><p>Post: #{ RESOURCE_LIMIT }, Tag: #{ RESOURCE_LIMIT }</p>
|
||||
<p>TagName: #{ NAME_LIMIT }, #{ MAX_BYTES } B</p>
|
||||
<h2>外部画像や動画への依存</h2><p>ローカルへ取り込めていない資産: 外部画像や動画</p>
|
||||
<h2>データ保護上の制約</h2>
|
||||
<p>公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。</p>
|
||||
<p>今回は現在状態のみの対応</p>
|
||||
HTML
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,7 @@
|
||||
:root { color-scheme: light dark; }
|
||||
body { max-width: 65rem; margin: auto; padding: 1rem; font: 1rem/1.6 sans-serif; }
|
||||
header { border-bottom: 1px solid; padding-bottom: 1rem; }
|
||||
nav { display: flex; flex-wrap: wrap; gap: 1rem; }
|
||||
a, p, li { overflow-wrap: anywhere; }
|
||||
li { margin-block: .4rem; }
|
||||
input, button { font: inherit; }
|
||||
@@ -0,0 +1,61 @@
|
||||
(() => {
|
||||
const root = new URL ('../', document.currentScript.src)
|
||||
const origin = window.archiveConfig?.origin
|
||||
if (typeof origin === 'string')
|
||||
{
|
||||
try
|
||||
{
|
||||
const url = new URL (origin)
|
||||
if (['http:', 'https:'].includes (url.protocol) && !url.username && !url.password)
|
||||
{
|
||||
url.search = ''
|
||||
url.hash = ''
|
||||
document.querySelectorAll ('[data-original]').forEach (link => {
|
||||
link.href = url.href.replace (/\/+$/, '') + link.dataset.original
|
||||
})
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Each HTML retains its original link when configuration is absent or invalid.
|
||||
}
|
||||
}
|
||||
const form = document.getElementById ('archive-search')
|
||||
const results = document.getElementById ('search-results')
|
||||
if (!form || !results) return
|
||||
form.addEventListener ('submit', async event => {
|
||||
event.preventDefault ()
|
||||
results.replaceChildren ()
|
||||
const query = (new FormData (form).get ('q') ?? '').toString ().toLocaleLowerCase ()
|
||||
if (!query.trim ()) return
|
||||
try
|
||||
{
|
||||
const response = await fetch (new URL ('search-index.json', root))
|
||||
if (!response.ok) throw new Error ('Search index unavailable')
|
||||
const entries = await response.json ()
|
||||
for (const entry of entries)
|
||||
{
|
||||
const text = [entry.title, ...(entry.names ?? [])].join (' ').toLocaleLowerCase ()
|
||||
if (!text.includes (query)) continue
|
||||
const url = new URL (entry.path, root)
|
||||
if (url.origin !== root.origin || !url.pathname.startsWith (root.pathname)) continue
|
||||
const item = document.createElement ('li')
|
||||
const link = document.createElement ('a')
|
||||
link.setAttribute ('href', entry.path)
|
||||
link.textContent = entry.title
|
||||
item.append (link)
|
||||
results.append (item)
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Lists remain available if the index cannot be loaded.
|
||||
for (const link of document.querySelectorAll ('nav a:not([data-original])'))
|
||||
{
|
||||
const item = document.createElement ('li')
|
||||
item.append (link.cloneNode (true))
|
||||
results.append (item)
|
||||
}
|
||||
}
|
||||
})
|
||||
})()
|
||||
@@ -0,0 +1,28 @@
|
||||
<!doctype html>
|
||||
<html lang="ja">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="robots" content="noindex, nofollow">
|
||||
<meta name="referrer" content="no-referrer">
|
||||
<meta http-equiv="Content-Security-Policy"
|
||||
content="default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; form-action 'none'; base-uri 'none'">
|
||||
<title><%= h(@title) %></title>
|
||||
<link rel="stylesheet" href="<%= h(relative('assets/archive.css')) %>">
|
||||
<script defer src="<%= h(relative('archive-config.js')) %>"></script>
|
||||
<script defer src="<%= h(relative('assets/archive.js')) %>"></script>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<p>静的保存版 — データ基準時点: <time><%= h(data_at) %></time></p>
|
||||
<nav>
|
||||
<a href="<%= h(relative('index.html')) %>">Static Mirror 生成機能</a>
|
||||
<a href="<%= h(relative("#{ @locale.code }/posts/index.html")) %>">Post</a>
|
||||
<a href="<%= h(relative("#{ @locale.code }/tags/index.html")) %>">Tag</a>
|
||||
<a href="<%= h(@origin + @original_path) %>" data-original="<%= h(@original_path) %>"
|
||||
target="_blank" rel="noopener noreferrer">本家へのリンク</a>
|
||||
</nav>
|
||||
</header>
|
||||
<main><h1><%= h(@title) %></h1><%= linked_content %></main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,4 +1,9 @@
|
||||
Rails.application.routes.draw do
|
||||
get 'static_mirror', to: 'static_mirrors#show'
|
||||
post 'static_mirror/jobs', to: 'static_mirrors#create'
|
||||
get 'static_mirror/jobs/:id', to: 'static_mirrors#status'
|
||||
get 'static_mirror/files/:id', to: 'static_mirrors#download'
|
||||
|
||||
get 'counts/definitions/:resource', to: 'counts_tables#definition'
|
||||
get 'counts/definitions/:resource/candidates', to: 'counts_tables#candidates'
|
||||
get 'counts/query', to: 'counts_tables#index'
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
require 'rails_helper'
|
||||
require 'tmpdir'
|
||||
|
||||
RSpec.describe 'Static Mirror API', type: :request do
|
||||
include ActiveSupport::Testing::TimeHelpers
|
||||
|
||||
around do |example|
|
||||
Dir.mktmpdir('static-mirror-spec') do |path|
|
||||
allow(StaticMirrorJob).to receive(:directory).and_return(Pathname.new(path))
|
||||
example.run
|
||||
end
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Rails.env).to receive(:development?).and_return(true)
|
||||
allow(StaticMirrorJob).to receive(:perform_later)
|
||||
end
|
||||
|
||||
def create_job
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:accepted)
|
||||
json.fetch('job')
|
||||
end
|
||||
|
||||
it 'accepts anonymous jobs asynchronously and serves the completed ZIP' do
|
||||
record = create(:post, title: 'mirror-public-api-title')
|
||||
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
||||
title: record.title, url: record.url, tags: '', tags_json: [], parent_post_ids: '',
|
||||
created_at: Time.current)
|
||||
get '/static_mirror'
|
||||
expect(json).to include('resource_limit' => 100, 'concurrency' => 1)
|
||||
id = create_job
|
||||
expect(StaticMirrorJob).to have_received(:perform_later).with(id)
|
||||
expect(StaticMirrorJob.directory.join('archive.zip')).not_to exist
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'queued', 'data_at' => nil)
|
||||
StaticMirrorJob.perform_now(id)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'ready', 'processed' => 1, 'total' => 1)
|
||||
expect(json.fetch('data_at')).to be_present
|
||||
expect(json.keys).not_to include('owner', 'origin', 'actor_id', 'policy_revision')
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:ok)
|
||||
expect(response.media_type).to eq('application/zip')
|
||||
expect(response.body.byteslice(0, 4)).to eq("PK\x03\x04".b)
|
||||
expect(response.body).to include(record.title)
|
||||
expect(response.headers['Cache-Control']).to include('no-store')
|
||||
end
|
||||
|
||||
it 'rejects historical and unknown input instead of pretending to restore it' do
|
||||
['at', 'data_at', 'admin', 'limit'].each do |key|
|
||||
post '/static_mirror/jobs', params: { locale: 'ja', key => '2020-01-01' }, as: :json
|
||||
expect(response).to have_http_status(:bad_request)
|
||||
end
|
||||
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
||||
end
|
||||
|
||||
it 'limits requests globally even across fresh anonymous sessions' do
|
||||
create_job
|
||||
other = ActionDispatch::Integration::Session.new(Rails.application)
|
||||
other.post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(other.response).to have_http_status(:too_many_requests)
|
||||
expect(other.response.headers['Retry-After']).to eq('60')
|
||||
expect(StaticMirrorJob).to have_received(:perform_later).once
|
||||
end
|
||||
|
||||
it 'binds job status and downloads to the creating session' do
|
||||
id = create_job
|
||||
StaticMirrorJob.perform_now(id)
|
||||
other = ActionDispatch::Integration::Session.new(Rails.application)
|
||||
["/static_mirror/jobs/#{ id }", "/static_mirror/files/#{ id }"].each do |path|
|
||||
other.get path
|
||||
expect(other.response).to have_http_status(:not_found)
|
||||
end
|
||||
end
|
||||
|
||||
it 'refuses ready artifacts after a new discard even for admins' do
|
||||
id = create_job
|
||||
StaticMirrorJob.perform_now(id)
|
||||
record = create(:post)
|
||||
PostVersion.create!(post: record, version_no: 1, event_type: 'discard',
|
||||
url: record.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.current)
|
||||
sign_in_as(create(:user, :admin))
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
|
||||
it 'fails lost jobs and rejects expired downloads' do
|
||||
id = create_job
|
||||
travel 80.seconds do
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'failed', 'error' => 'worker_stopped')
|
||||
end
|
||||
travel 11.minutes do
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
end
|
||||
|
||||
it 'rejects a previously ready ZIP after its expiry' do
|
||||
id = create_job
|
||||
StaticMirrorJob.perform_now(id)
|
||||
travel 11.minutes do
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
end
|
||||
|
||||
it 'keeps simultaneous generation bounded after the cooldown' do
|
||||
create_job
|
||||
lock = File.open(StaticMirrorJob.directory.join('.generation.lock'), File::RDWR)
|
||||
lock.flock(File::LOCK_EX)
|
||||
travel 61.seconds do
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:too_many_requests)
|
||||
end
|
||||
ensure
|
||||
lock&.close
|
||||
end
|
||||
|
||||
it 'reports bounded-generation failures without returning internal exception messages' do
|
||||
id = create_job
|
||||
allow(StaticMirror).to receive(:new).and_raise(StaticMirror::TooLarge, 'private details')
|
||||
StaticMirrorJob.perform_now(id)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'failed', 'error' => 'too_large')
|
||||
expect(response.body).not_to include('private details')
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
|
||||
it 'is unavailable outside development' do
|
||||
allow(Rails.env).to receive(:development?).and_return(false)
|
||||
get '/static_mirror'
|
||||
expect(response).to have_http_status(:not_found)
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get '/static_mirror/jobs/unknown'
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get '/static_mirror/files/unknown'
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
|
||||
it 'preserves user and IP BAN checks' do
|
||||
user = create(:user, banned_at: Time.current)
|
||||
sign_in_as(user)
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:forbidden)
|
||||
sign_out
|
||||
IpAddress.create!(ip_address: IPAddr.new('127.0.0.1').hton, banned_at: Time.current)
|
||||
get '/static_mirror', headers: { 'REMOTE_ADDR' => '127.0.0.1' }
|
||||
expect(response).to have_http_status(:forbidden)
|
||||
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
||||
end
|
||||
|
||||
it 'requires the normal CSRF token for anonymous generation', csrf: true do
|
||||
get '/users/csrf'
|
||||
token = json.fetch('token')
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:forbidden)
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json,
|
||||
headers: { 'X-CSRF-Token' => token }
|
||||
expect(response).to have_http_status(:accepted)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,158 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe StaticMirror do
|
||||
include_context 'Ainu locale'
|
||||
|
||||
let(:tag) { create(:tag, primary_name: 'mirror-public-tag') }
|
||||
let(:post) { create(:post, title: '<img src=x onerror=alert(1)>') }
|
||||
let(:mirror) { described_class.new(locale: Locale.nipponese, origin: 'https://source.test') }
|
||||
|
||||
def snapshot_tag(record, event: 'create', number: 1)
|
||||
TagVersion.create!(tag: record, version_no: number, event_type: event,
|
||||
name: record.name('ja'), category: record.category, aliases: '', parent_tag_ids: '',
|
||||
names: { ja: { name: record.name('ja'), script_code: 'Jpan',
|
||||
auto_generated: false, aliases: [] } }, created_at: Time.current)
|
||||
end
|
||||
|
||||
def snapshot_post(record, tags: [])
|
||||
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
||||
title: record.title, url: record.url, tags: '', tags_json: tags,
|
||||
parent_post_ids: '', created_at: Time.current)
|
||||
end
|
||||
|
||||
# Read STORE entries independently of the production writer and verify CRCs.
|
||||
def archive_files(bytes)
|
||||
offset = 0
|
||||
files = { }
|
||||
while bytes.byteslice(offset, 4).unpack1('V') == 0x04034b50
|
||||
header = bytes.byteslice(offset, 30).unpack('VvvvvvVVVvv')
|
||||
size, name_size, extra_size = header[7], header[9], header[10]
|
||||
path = bytes.byteslice(offset + 30, name_size).force_encoding('UTF-8')
|
||||
data = bytes.byteslice(offset + 30 + name_size + extra_size, size)
|
||||
expect(Zlib.crc32(data)).to eq(header[6])
|
||||
files[path.delete_prefix('archive/')] = data.force_encoding('UTF-8')
|
||||
offset += 30 + name_size + extra_size + size
|
||||
end
|
||||
expect(bytes.byteslice(-22, 4).unpack1('V')).to eq(0x06054b50)
|
||||
files
|
||||
end
|
||||
|
||||
before do
|
||||
snapshot_tag(tag)
|
||||
PostTag.create!(post:, tag:)
|
||||
snapshot_post(post, tags: [{ tag_id: tag.id, version_no: 1, sections: [] }])
|
||||
end
|
||||
|
||||
it 'generates a navigable ZIP, escaped HTML, search data and verified hashes' do
|
||||
progress = []
|
||||
files = archive_files(mirror.export { |done, total| progress << [done, total] })
|
||||
expect(progress.last).to eq([2, 2])
|
||||
expect(files.keys).to include('index.html', 'robots.txt', 'assets/archive.css',
|
||||
'assets/archive.js', 'archive-config.js', 'search-index.json', 'manifest.json',
|
||||
'ja/posts/index.html', "ja/posts/#{ post.id }/index.html",
|
||||
'ja/tags/index.html', "ja/tags/#{ tag.id }/index.html")
|
||||
files.select { |path, _| path.end_with?('.html') }.each do |path, html|
|
||||
document = Nokogiri::HTML(html)
|
||||
expect(document.at_css('meta[name="robots"]')['content']).to include('noindex')
|
||||
expect(document.text).to include('静的保存版', mirror.data_at)
|
||||
expect(document.css('img')).to be_empty
|
||||
expect(document.at_css('[data-original]')['href']).to start_with('https://source.test/ja')
|
||||
document.css('a[href], link[href], script[src]').each do |node|
|
||||
target = node['href'] || node['src']
|
||||
next if target.start_with?('https://')
|
||||
|
||||
expect(target).not_to start_with('/')
|
||||
resolved = Pathname.new(path).dirname.join(target).cleanpath.to_s
|
||||
expect(files).to have_key(resolved)
|
||||
end
|
||||
end
|
||||
detail = Nokogiri::HTML(files.fetch("ja/posts/#{ post.id }/index.html"))
|
||||
expect(detail.at_css('h1').text).to eq(post.title)
|
||||
expect(detail.css('main a').map { _1['href'] })
|
||||
.to include("../../tags/#{ tag.id }/index.html")
|
||||
manifest = JSON.parse(files.fetch('manifest.json'))
|
||||
expect(manifest).to include('format_version' => 1, 'locale' => 'ja',
|
||||
'data_at' => mirror.data_at, 'resources' => { 'posts' => 1, 'tags' => 1 })
|
||||
expect(Time.iso8601(manifest.fetch('generated_at'))).to be >= Time.iso8601(mirror.data_at)
|
||||
expect(manifest.fetch('files').map { _1.fetch('path') })
|
||||
.to match_array(files.keys - ['manifest.json'])
|
||||
manifest.fetch('files').each do |entry|
|
||||
digest = Digest::SHA256.hexdigest(files.fetch(entry.fetch('path')))
|
||||
expect(entry.fetch('sha256')).to eq(digest)
|
||||
end
|
||||
expect(JSON.parse(files.fetch('search-index.json')).map { _1.fetch('path') })
|
||||
.to include("ja/posts/#{ post.id }/index.html", "ja/tags/#{ tag.id }/index.html")
|
||||
end
|
||||
|
||||
it 'keeps private and unversioned records out even for an admin caller' do
|
||||
private_tag = create(:tag, primary_name: 'mirror-secret-tag')
|
||||
snapshot_tag(private_tag)
|
||||
snapshot_tag(private_tag, event: 'discard', number: 2)
|
||||
private_post = create(:post, title: 'mirror-secret-post')
|
||||
snapshot_post(private_post)
|
||||
PostVersion.create!(post: private_post, version_no: 2, event_type: 'discard',
|
||||
url: private_post.url, tags: '', tags_json: [], parent_post_ids: '',
|
||||
created_at: Time.current)
|
||||
legacy = create(:post, title: 'mirror-unversioned-secret')
|
||||
referencing = create(:post, title: 'mirror-secret-reference')
|
||||
snapshot_post(referencing,
|
||||
tags: [{ tag_id: private_tag.id, version_no: 1, sections: [] }])
|
||||
files = HistoryAccess.set(user: create(:user, :admin)) { archive_files(mirror.export) }
|
||||
expect(files.values.join).not_to include('mirror-secret', legacy.title, private_post.url)
|
||||
expect(files.keys).not_to include("ja/tags/#{ private_tag.id }/index.html",
|
||||
"ja/posts/#{ private_post.id }/index.html", "ja/posts/#{ referencing.id }/index.html")
|
||||
expect(mirror.authority.fetch('admin')).to be(false)
|
||||
end
|
||||
|
||||
it 'applies later discard boundaries after a restoration' do
|
||||
snapshot_tag(tag, event: 'discard', number: 2)
|
||||
snapshot_tag(tag, event: 'restore', number: 3)
|
||||
tag.update_columns(version_no: 3)
|
||||
expect(archive_files(mirror.export)).to have_key("ja/tags/#{ tag.id }/index.html")
|
||||
snapshot_tag(tag, event: 'discard', number: 4)
|
||||
second = described_class.new(locale: Locale.nipponese, origin: 'https://source.test')
|
||||
files = archive_files(second.export)
|
||||
expect(files.keys).not_to include("ja/tags/#{ tag.id }/index.html")
|
||||
expect(files.values.join).not_to include('mirror-public-tag')
|
||||
end
|
||||
|
||||
it 'does not include credentials or executable URL schemes' do
|
||||
post.update_columns(url: 'https://secret-user:secret-password@source.test/path')
|
||||
content = archive_files(mirror.export).values.join
|
||||
expect(content).not_to include('secret-password', 'secret-user')
|
||||
post.update_columns(url: 'javascript:alert(1)')
|
||||
expect(archive_files(mirror.export).values.join).not_to include('javascript:')
|
||||
end
|
||||
|
||||
it 'fails if publication changes during generation' do
|
||||
expect do
|
||||
mirror.export do |done, _|
|
||||
snapshot_tag(tag, event: 'discard', number: 2) if done.zero?
|
||||
end
|
||||
end.to raise_error(HistoryAccess::Forbidden)
|
||||
end
|
||||
|
||||
it 'bounds output and never reports hidden population counts' do
|
||||
stub_const('StaticMirror::RESOURCE_LIMIT', 1)
|
||||
second = create(:post, title: 'mirror-beyond-limit')
|
||||
snapshot_post(second)
|
||||
files = archive_files(mirror.export)
|
||||
expect(files.values.join).not_to include(second.title, 'population_count', 'excluded_count')
|
||||
stub_const('StaticMirror::MAX_BYTES', 16)
|
||||
expect { mirror.export }.to raise_error(StaticMirror::TooLarge)
|
||||
end
|
||||
|
||||
it 'keeps all registered language names and aliases without locale filtering' do
|
||||
locale
|
||||
create(:tag_name, :alias, tag:, name: 'mirror-alias', language_code: 'ja', script_code: 'Jpan')
|
||||
files = archive_files(described_class.new(locale:, origin: 'https://source.test').export)
|
||||
detail = files.fetch("ain-Kana/tags/#{ tag.id }/index.html")
|
||||
expect(detail).to include('mirror-public-tag', 'mirror-alias', 'lang="ain"', 'lang="ja"')
|
||||
end
|
||||
|
||||
it 'fails rather than truncating registered aliases at the name limit' do
|
||||
create(:tag_name, :alias, tag:, name: 'mirror-extra-alias')
|
||||
stub_const('StaticMirror::NAME_LIMIT', 1)
|
||||
expect { mirror.export }.to raise_error(StaticMirror::TooLarge)
|
||||
end
|
||||
end
|
||||
新しいイシューから参照
ユーザーをブロックする