AI: address issue #483
このコミットが含まれているのは:
@@ -0,0 +1,169 @@
|
||||
require 'rails_helper'
|
||||
require 'tmpdir'
|
||||
|
||||
RSpec.describe 'Static Mirror API', type: :request do
|
||||
include ActiveSupport::Testing::TimeHelpers
|
||||
|
||||
around do |example|
|
||||
Dir.mktmpdir('static-mirror-spec') do |path|
|
||||
allow(StaticMirrorJob).to receive(:directory).and_return(Pathname.new(path))
|
||||
example.run
|
||||
end
|
||||
end
|
||||
|
||||
before do
|
||||
allow(Rails.env).to receive(:development?).and_return(true)
|
||||
allow(StaticMirrorJob).to receive(:perform_later)
|
||||
end
|
||||
|
||||
def create_job
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:accepted)
|
||||
json.fetch('job')
|
||||
end
|
||||
|
||||
it 'accepts anonymous jobs asynchronously and serves the completed ZIP' do
|
||||
record = create(:post, title: 'mirror-public-api-title')
|
||||
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
||||
title: record.title, url: record.url, tags: '', tags_json: [], parent_post_ids: '',
|
||||
created_at: Time.current)
|
||||
get '/static_mirror'
|
||||
expect(json).to include('resource_limit' => 100, 'concurrency' => 1)
|
||||
id = create_job
|
||||
expect(StaticMirrorJob).to have_received(:perform_later).with(id)
|
||||
expect(StaticMirrorJob.directory.join('archive.zip')).not_to exist
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'queued', 'data_at' => nil)
|
||||
StaticMirrorJob.perform_now(id)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'ready', 'processed' => 1, 'total' => 1)
|
||||
expect(json.fetch('data_at')).to be_present
|
||||
expect(json.keys).not_to include('owner', 'origin', 'actor_id', 'policy_revision')
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:ok)
|
||||
expect(response.media_type).to eq('application/zip')
|
||||
expect(response.body.byteslice(0, 4)).to eq("PK\x03\x04".b)
|
||||
expect(response.body).to include(record.title)
|
||||
expect(response.headers['Cache-Control']).to include('no-store')
|
||||
end
|
||||
|
||||
it 'rejects historical and unknown input instead of pretending to restore it' do
|
||||
['at', 'data_at', 'admin', 'limit'].each do |key|
|
||||
post '/static_mirror/jobs', params: { locale: 'ja', key => '2020-01-01' }, as: :json
|
||||
expect(response).to have_http_status(:bad_request)
|
||||
end
|
||||
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
||||
end
|
||||
|
||||
it 'limits requests globally even across fresh anonymous sessions' do
|
||||
create_job
|
||||
other = ActionDispatch::Integration::Session.new(Rails.application)
|
||||
other.post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(other.response).to have_http_status(:too_many_requests)
|
||||
expect(other.response.headers['Retry-After']).to eq('60')
|
||||
expect(StaticMirrorJob).to have_received(:perform_later).once
|
||||
end
|
||||
|
||||
it 'binds job status and downloads to the creating session' do
|
||||
id = create_job
|
||||
StaticMirrorJob.perform_now(id)
|
||||
other = ActionDispatch::Integration::Session.new(Rails.application)
|
||||
["/static_mirror/jobs/#{ id }", "/static_mirror/files/#{ id }"].each do |path|
|
||||
other.get path
|
||||
expect(other.response).to have_http_status(:not_found)
|
||||
end
|
||||
end
|
||||
|
||||
it 'refuses ready artifacts after a new discard even for admins' do
|
||||
id = create_job
|
||||
StaticMirrorJob.perform_now(id)
|
||||
record = create(:post)
|
||||
PostVersion.create!(post: record, version_no: 1, event_type: 'discard',
|
||||
url: record.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.current)
|
||||
sign_in_as(create(:user, :admin))
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
|
||||
it 'fails lost jobs and rejects expired downloads' do
|
||||
id = create_job
|
||||
travel 80.seconds do
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'failed', 'error' => 'worker_stopped')
|
||||
end
|
||||
travel 11.minutes do
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
end
|
||||
|
||||
it 'rejects a previously ready ZIP after its expiry' do
|
||||
id = create_job
|
||||
StaticMirrorJob.perform_now(id)
|
||||
travel 11.minutes do
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
end
|
||||
|
||||
it 'keeps simultaneous generation bounded after the cooldown' do
|
||||
create_job
|
||||
lock = File.open(StaticMirrorJob.directory.join('.generation.lock'), File::RDWR)
|
||||
lock.flock(File::LOCK_EX)
|
||||
travel 61.seconds do
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:too_many_requests)
|
||||
end
|
||||
ensure
|
||||
lock&.close
|
||||
end
|
||||
|
||||
it 'reports bounded-generation failures without returning internal exception messages' do
|
||||
id = create_job
|
||||
allow(StaticMirror).to receive(:new).and_raise(StaticMirror::TooLarge, 'private details')
|
||||
StaticMirrorJob.perform_now(id)
|
||||
get "/static_mirror/jobs/#{ id }"
|
||||
expect(json).to include('status' => 'failed', 'error' => 'too_large')
|
||||
expect(response.body).not_to include('private details')
|
||||
get "/static_mirror/files/#{ id }"
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
|
||||
it 'is unavailable outside development' do
|
||||
allow(Rails.env).to receive(:development?).and_return(false)
|
||||
get '/static_mirror'
|
||||
expect(response).to have_http_status(:not_found)
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get '/static_mirror/jobs/unknown'
|
||||
expect(response).to have_http_status(:not_found)
|
||||
get '/static_mirror/files/unknown'
|
||||
expect(response).to have_http_status(:not_found)
|
||||
end
|
||||
|
||||
it 'preserves user and IP BAN checks' do
|
||||
user = create(:user, banned_at: Time.current)
|
||||
sign_in_as(user)
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:forbidden)
|
||||
sign_out
|
||||
IpAddress.create!(ip_address: IPAddr.new('127.0.0.1').hton, banned_at: Time.current)
|
||||
get '/static_mirror', headers: { 'REMOTE_ADDR' => '127.0.0.1' }
|
||||
expect(response).to have_http_status(:forbidden)
|
||||
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
||||
end
|
||||
|
||||
it 'requires the normal CSRF token for anonymous generation', csrf: true do
|
||||
get '/users/csrf'
|
||||
token = json.fetch('token')
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
||||
expect(response).to have_http_status(:forbidden)
|
||||
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json,
|
||||
headers: { 'X-CSRF-Token' => token }
|
||||
expect(response).to have_http_status(:accepted)
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,158 @@
|
||||
require 'rails_helper'
|
||||
|
||||
RSpec.describe StaticMirror do
|
||||
include_context 'Ainu locale'
|
||||
|
||||
let(:tag) { create(:tag, primary_name: 'mirror-public-tag') }
|
||||
let(:post) { create(:post, title: '<img src=x onerror=alert(1)>') }
|
||||
let(:mirror) { described_class.new(locale: Locale.nipponese, origin: 'https://source.test') }
|
||||
|
||||
def snapshot_tag(record, event: 'create', number: 1)
|
||||
TagVersion.create!(tag: record, version_no: number, event_type: event,
|
||||
name: record.name('ja'), category: record.category, aliases: '', parent_tag_ids: '',
|
||||
names: { ja: { name: record.name('ja'), script_code: 'Jpan',
|
||||
auto_generated: false, aliases: [] } }, created_at: Time.current)
|
||||
end
|
||||
|
||||
def snapshot_post(record, tags: [])
|
||||
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
||||
title: record.title, url: record.url, tags: '', tags_json: tags,
|
||||
parent_post_ids: '', created_at: Time.current)
|
||||
end
|
||||
|
||||
# Read STORE entries independently of the production writer and verify CRCs.
|
||||
def archive_files(bytes)
|
||||
offset = 0
|
||||
files = { }
|
||||
while bytes.byteslice(offset, 4).unpack1('V') == 0x04034b50
|
||||
header = bytes.byteslice(offset, 30).unpack('VvvvvvVVVvv')
|
||||
size, name_size, extra_size = header[7], header[9], header[10]
|
||||
path = bytes.byteslice(offset + 30, name_size).force_encoding('UTF-8')
|
||||
data = bytes.byteslice(offset + 30 + name_size + extra_size, size)
|
||||
expect(Zlib.crc32(data)).to eq(header[6])
|
||||
files[path.delete_prefix('archive/')] = data.force_encoding('UTF-8')
|
||||
offset += 30 + name_size + extra_size + size
|
||||
end
|
||||
expect(bytes.byteslice(-22, 4).unpack1('V')).to eq(0x06054b50)
|
||||
files
|
||||
end
|
||||
|
||||
before do
|
||||
snapshot_tag(tag)
|
||||
PostTag.create!(post:, tag:)
|
||||
snapshot_post(post, tags: [{ tag_id: tag.id, version_no: 1, sections: [] }])
|
||||
end
|
||||
|
||||
it 'generates a navigable ZIP, escaped HTML, search data and verified hashes' do
|
||||
progress = []
|
||||
files = archive_files(mirror.export { |done, total| progress << [done, total] })
|
||||
expect(progress.last).to eq([2, 2])
|
||||
expect(files.keys).to include('index.html', 'robots.txt', 'assets/archive.css',
|
||||
'assets/archive.js', 'archive-config.js', 'search-index.json', 'manifest.json',
|
||||
'ja/posts/index.html', "ja/posts/#{ post.id }/index.html",
|
||||
'ja/tags/index.html', "ja/tags/#{ tag.id }/index.html")
|
||||
files.select { |path, _| path.end_with?('.html') }.each do |path, html|
|
||||
document = Nokogiri::HTML(html)
|
||||
expect(document.at_css('meta[name="robots"]')['content']).to include('noindex')
|
||||
expect(document.text).to include('静的保存版', mirror.data_at)
|
||||
expect(document.css('img')).to be_empty
|
||||
expect(document.at_css('[data-original]')['href']).to start_with('https://source.test/ja')
|
||||
document.css('a[href], link[href], script[src]').each do |node|
|
||||
target = node['href'] || node['src']
|
||||
next if target.start_with?('https://')
|
||||
|
||||
expect(target).not_to start_with('/')
|
||||
resolved = Pathname.new(path).dirname.join(target).cleanpath.to_s
|
||||
expect(files).to have_key(resolved)
|
||||
end
|
||||
end
|
||||
detail = Nokogiri::HTML(files.fetch("ja/posts/#{ post.id }/index.html"))
|
||||
expect(detail.at_css('h1').text).to eq(post.title)
|
||||
expect(detail.css('main a').map { _1['href'] })
|
||||
.to include("../../tags/#{ tag.id }/index.html")
|
||||
manifest = JSON.parse(files.fetch('manifest.json'))
|
||||
expect(manifest).to include('format_version' => 1, 'locale' => 'ja',
|
||||
'data_at' => mirror.data_at, 'resources' => { 'posts' => 1, 'tags' => 1 })
|
||||
expect(Time.iso8601(manifest.fetch('generated_at'))).to be >= Time.iso8601(mirror.data_at)
|
||||
expect(manifest.fetch('files').map { _1.fetch('path') })
|
||||
.to match_array(files.keys - ['manifest.json'])
|
||||
manifest.fetch('files').each do |entry|
|
||||
digest = Digest::SHA256.hexdigest(files.fetch(entry.fetch('path')))
|
||||
expect(entry.fetch('sha256')).to eq(digest)
|
||||
end
|
||||
expect(JSON.parse(files.fetch('search-index.json')).map { _1.fetch('path') })
|
||||
.to include("ja/posts/#{ post.id }/index.html", "ja/tags/#{ tag.id }/index.html")
|
||||
end
|
||||
|
||||
it 'keeps private and unversioned records out even for an admin caller' do
|
||||
private_tag = create(:tag, primary_name: 'mirror-secret-tag')
|
||||
snapshot_tag(private_tag)
|
||||
snapshot_tag(private_tag, event: 'discard', number: 2)
|
||||
private_post = create(:post, title: 'mirror-secret-post')
|
||||
snapshot_post(private_post)
|
||||
PostVersion.create!(post: private_post, version_no: 2, event_type: 'discard',
|
||||
url: private_post.url, tags: '', tags_json: [], parent_post_ids: '',
|
||||
created_at: Time.current)
|
||||
legacy = create(:post, title: 'mirror-unversioned-secret')
|
||||
referencing = create(:post, title: 'mirror-secret-reference')
|
||||
snapshot_post(referencing,
|
||||
tags: [{ tag_id: private_tag.id, version_no: 1, sections: [] }])
|
||||
files = HistoryAccess.set(user: create(:user, :admin)) { archive_files(mirror.export) }
|
||||
expect(files.values.join).not_to include('mirror-secret', legacy.title, private_post.url)
|
||||
expect(files.keys).not_to include("ja/tags/#{ private_tag.id }/index.html",
|
||||
"ja/posts/#{ private_post.id }/index.html", "ja/posts/#{ referencing.id }/index.html")
|
||||
expect(mirror.authority.fetch('admin')).to be(false)
|
||||
end
|
||||
|
||||
it 'applies later discard boundaries after a restoration' do
|
||||
snapshot_tag(tag, event: 'discard', number: 2)
|
||||
snapshot_tag(tag, event: 'restore', number: 3)
|
||||
tag.update_columns(version_no: 3)
|
||||
expect(archive_files(mirror.export)).to have_key("ja/tags/#{ tag.id }/index.html")
|
||||
snapshot_tag(tag, event: 'discard', number: 4)
|
||||
second = described_class.new(locale: Locale.nipponese, origin: 'https://source.test')
|
||||
files = archive_files(second.export)
|
||||
expect(files.keys).not_to include("ja/tags/#{ tag.id }/index.html")
|
||||
expect(files.values.join).not_to include('mirror-public-tag')
|
||||
end
|
||||
|
||||
it 'does not include credentials or executable URL schemes' do
|
||||
post.update_columns(url: 'https://secret-user:secret-password@source.test/path')
|
||||
content = archive_files(mirror.export).values.join
|
||||
expect(content).not_to include('secret-password', 'secret-user')
|
||||
post.update_columns(url: 'javascript:alert(1)')
|
||||
expect(archive_files(mirror.export).values.join).not_to include('javascript:')
|
||||
end
|
||||
|
||||
it 'fails if publication changes during generation' do
|
||||
expect do
|
||||
mirror.export do |done, _|
|
||||
snapshot_tag(tag, event: 'discard', number: 2) if done.zero?
|
||||
end
|
||||
end.to raise_error(HistoryAccess::Forbidden)
|
||||
end
|
||||
|
||||
it 'bounds output and never reports hidden population counts' do
|
||||
stub_const('StaticMirror::RESOURCE_LIMIT', 1)
|
||||
second = create(:post, title: 'mirror-beyond-limit')
|
||||
snapshot_post(second)
|
||||
files = archive_files(mirror.export)
|
||||
expect(files.values.join).not_to include(second.title, 'population_count', 'excluded_count')
|
||||
stub_const('StaticMirror::MAX_BYTES', 16)
|
||||
expect { mirror.export }.to raise_error(StaticMirror::TooLarge)
|
||||
end
|
||||
|
||||
it 'keeps all registered language names and aliases without locale filtering' do
|
||||
locale
|
||||
create(:tag_name, :alias, tag:, name: 'mirror-alias', language_code: 'ja', script_code: 'Jpan')
|
||||
files = archive_files(described_class.new(locale:, origin: 'https://source.test').export)
|
||||
detail = files.fetch("ain-Kana/tags/#{ tag.id }/index.html")
|
||||
expect(detail).to include('mirror-public-tag', 'mirror-alias', 'lang="ain"', 'lang="ja"')
|
||||
end
|
||||
|
||||
it 'fails rather than truncating registered aliases at the name limit' do
|
||||
create(:tag_name, :alias, tag:, name: 'mirror-extra-alias')
|
||||
stub_const('StaticMirror::NAME_LIMIT', 1)
|
||||
expect { mirror.export }.to raise_error(StaticMirror::TooLarge)
|
||||
end
|
||||
end
|
||||
新しいイシューから参照
ユーザーをブロックする