258 行
10 KiB
Ruby
258 行
10 KiB
Ruby
require 'rails_helper'
|
|
require 'tmpdir'
|
|
|
|
RSpec.describe 'Static Mirror API', type: :request do
|
|
include ActiveSupport::Testing::TimeHelpers
|
|
|
|
around do |example|
|
|
Dir.mktmpdir('static-mirror-spec') do |path|
|
|
allow(StaticMirrorJob).to receive(:directory).and_return(Pathname.new(path))
|
|
example.run
|
|
end
|
|
end
|
|
|
|
before do
|
|
allow(Rails.env).to receive(:development?).and_return(true)
|
|
allow(StaticMirrorJob).to receive(:perform_later)
|
|
end
|
|
|
|
def create_job
|
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
|
expect(response).to have_http_status(:accepted)
|
|
json.fetch('job')
|
|
end
|
|
|
|
it 'accepts anonymous jobs asynchronously and serves the completed ZIP' do
|
|
record = create(:post, title: 'mirror-public-api-title')
|
|
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
|
title: record.title, url: record.url, tags: '', tags_json: [], parent_post_ids: '',
|
|
created_at: Time.current)
|
|
get '/static_mirror'
|
|
expect(json).to include('resource_limit' => 100, 'concurrency' => 1)
|
|
id = create_job
|
|
expect(StaticMirrorJob).to have_received(:perform_later).with(id)
|
|
expect(StaticMirrorJob.directory.join('archive.zip')).not_to exist
|
|
get "/static_mirror/jobs/#{ id }"
|
|
expect(json).to include('status' => 'queued', 'data_at' => nil)
|
|
StaticMirrorJob.perform_now(id)
|
|
get "/static_mirror/jobs/#{ id }"
|
|
expect(json).to include('status' => 'ready', 'processed' => 1, 'total' => 1)
|
|
expect(json.fetch('data_at')).to be_present
|
|
expect(json.keys).not_to include('owner', 'origin', 'actor_id', 'policy_revision',
|
|
'publication_policy', 'history_revision', 'post_ids', 'tag_ids', 'content_digest')
|
|
get "/static_mirror/files/#{ id }"
|
|
expect(response).to have_http_status(:ok)
|
|
expect(response.media_type).to eq('application/zip')
|
|
expect(response.body.byteslice(0, 4)).to eq("PK\x03\x04".b)
|
|
expect(response.body).to include(record.title)
|
|
expect(response.headers['Cache-Control']).to include('no-store')
|
|
end
|
|
|
|
it 'bakes the canonical source origin regardless of request and forwarded hosts' do
|
|
host! 'mirror-intake.test'
|
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json,
|
|
headers: { 'X-Forwarded-Host' => 'untrusted-source.test',
|
|
'X-Forwarded-Proto' => 'https' }
|
|
expect(response).to have_http_status(:accepted)
|
|
id = json.fetch('job')
|
|
expect(StaticMirrorJob.state).not_to have_key('origin')
|
|
StaticMirrorJob.perform_now(id)
|
|
get "/static_mirror/files/#{ id }"
|
|
expect(response).to have_http_status(:ok)
|
|
expect(response.body).to include('https://hub.nizika.monster/ja')
|
|
expect(response.body).not_to include('mirror-intake.test', 'untrusted-source.test')
|
|
end
|
|
|
|
it 'rejects historical and unknown input instead of pretending to restore it' do
|
|
['at', 'data_at', 'admin', 'limit'].each do |key|
|
|
post '/static_mirror/jobs', params: { locale: 'ja', key => '2020-01-01' }, as: :json
|
|
expect(response).to have_http_status(:bad_request)
|
|
end
|
|
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
|
end
|
|
|
|
it 'limits requests globally even across fresh anonymous sessions' do
|
|
create_job
|
|
other = ActionDispatch::Integration::Session.new(Rails.application)
|
|
other.post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
|
expect(other.response).to have_http_status(:too_many_requests)
|
|
expect(other.response.headers['Retry-After']).to eq('60')
|
|
expect(StaticMirrorJob).to have_received(:perform_later).once
|
|
end
|
|
|
|
it 'binds job status and downloads to the creating session' do
|
|
id = create_job
|
|
StaticMirrorJob.perform_now(id)
|
|
other = ActionDispatch::Integration::Session.new(Rails.application)
|
|
["/static_mirror/jobs/#{ id }", "/static_mirror/files/#{ id }"].each do |path|
|
|
other.get path
|
|
expect(other.response).to have_http_status(:not_found)
|
|
end
|
|
end
|
|
|
|
it 'refuses ready artifacts after a new discard even for admins' do
|
|
id = create_job
|
|
StaticMirrorJob.perform_now(id)
|
|
record = create(:post)
|
|
PostVersion.create!(post: record, version_no: 1, event_type: 'discard',
|
|
url: record.url, tags: '', tags_json: [], parent_post_ids: '', created_at: Time.current)
|
|
sign_in_as(create(:user, :admin))
|
|
get "/static_mirror/files/#{ id }"
|
|
expect(response).to have_http_status(:not_found)
|
|
get "/static_mirror/jobs/#{ id }"
|
|
expect(response).to have_http_status(:not_found)
|
|
end
|
|
|
|
it 'rechecks ready ZIPs after a non-discard content change' do
|
|
record = create(:post, title: 'mirror-withdrawn-title')
|
|
PostVersion.create!(post: record, version_no: 1, event_type: 'create',
|
|
title: record.title, url: record.url, tags: '', tags_json: [], parent_post_ids: '',
|
|
created_at: Time.current)
|
|
id = create_job
|
|
StaticMirrorJob.perform_now(id)
|
|
record.update!(title: 'mirror-corrected-title')
|
|
get "/static_mirror/jobs/#{ id }"
|
|
expect(response).to have_http_status(:not_found)
|
|
get "/static_mirror/files/#{ id }"
|
|
expect(response).to have_http_status(:not_found)
|
|
expect(response.body).not_to include('mirror-withdrawn-title')
|
|
end
|
|
|
|
it 'fails lost jobs and rejects expired downloads' do
|
|
id = create_job
|
|
travel 80.seconds do
|
|
get "/static_mirror/jobs/#{ id }"
|
|
expect(json).to include('status' => 'failed', 'error' => 'worker_stopped')
|
|
end
|
|
travel 11.minutes do
|
|
get "/static_mirror/files/#{ id }"
|
|
expect(response).to have_http_status(:not_found)
|
|
end
|
|
end
|
|
|
|
it 'rejects a previously ready ZIP after its expiry' do
|
|
id = create_job
|
|
StaticMirrorJob.perform_now(id)
|
|
travel 11.minutes do
|
|
get "/static_mirror/files/#{ id }"
|
|
expect(response).to have_http_status(:not_found)
|
|
get "/static_mirror/jobs/#{ id }"
|
|
expect(response).to have_http_status(:not_found)
|
|
end
|
|
end
|
|
|
|
it 'preserves the queued reservation after the cooldown' do
|
|
id = create_job
|
|
travel 61.seconds do
|
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
|
expect(response).to have_http_status(:too_many_requests)
|
|
expect(StaticMirrorJob.state).to include('id' => id, 'status' => 'queued')
|
|
expect(StaticMirrorJob).to have_received(:perform_later).once
|
|
StaticMirrorJob.perform_now(id)
|
|
get "/static_mirror/jobs/#{ id }"
|
|
expect(json).to include('status' => 'failed', 'error' => 'generation_timeout')
|
|
end
|
|
end
|
|
|
|
it 'reclaims a stalled queued reservation without waiting for a status request' do
|
|
id = create_job
|
|
travel 80.seconds do
|
|
second_id = create_job
|
|
expect(second_id).not_to eq(id)
|
|
StaticMirrorJob.perform_now(id)
|
|
expect(StaticMirrorJob.state).to include('id' => second_id, 'status' => 'queued')
|
|
StaticMirrorJob.perform_now(second_id)
|
|
get "/static_mirror/jobs/#{ second_id }"
|
|
expect(json).to include('status' => 'ready')
|
|
end
|
|
end
|
|
|
|
it 'waits for brief intake lock contention instead of losing the queued worker' do
|
|
id = create_job
|
|
path = StaticMirrorJob.directory.join('.generation.lock')
|
|
lock = File.open(path, File::RDWR)
|
|
lock.flock(File::LOCK_EX)
|
|
attempts = Queue.new
|
|
worker = StaticMirrorJob.new
|
|
allow(worker).to receive(:generate)
|
|
allow(File).to receive(:open).and_call_original
|
|
allow(File).to receive(:open).with(path, File::RDWR).and_wrap_original do |original, *args, &block|
|
|
original.call(*args) do |file|
|
|
allow(file).to receive(:flock).and_wrap_original do |flock, mode|
|
|
attempts << mode
|
|
flock.call(mode)
|
|
end
|
|
block.call(file)
|
|
end
|
|
end
|
|
thread = Thread.new { worker.perform(id) }
|
|
Timeout.timeout(5) do
|
|
expect(attempts.pop).to eq(File::LOCK_EX)
|
|
lock.close
|
|
thread.join
|
|
end
|
|
expect(worker).to have_received(:generate).with(id, hash_including('status' => 'queued'))
|
|
ensure
|
|
lock&.close unless lock&.closed?
|
|
thread&.kill
|
|
thread&.join
|
|
end
|
|
|
|
it 'keeps the generation lock reserved even when stale metadata expires' do
|
|
create_job
|
|
lock = File.open(StaticMirrorJob.directory.join('.generation.lock'), File::RDWR)
|
|
lock.flock(File::LOCK_EX)
|
|
travel 80.seconds do
|
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
|
expect(response).to have_http_status(:too_many_requests)
|
|
end
|
|
ensure
|
|
lock&.close
|
|
end
|
|
|
|
it 'reports bounded-generation failures without returning internal exception messages' do
|
|
id = create_job
|
|
allow(StaticMirror).to receive(:new).and_raise(StaticMirror::TooLarge, 'private details')
|
|
StaticMirrorJob.perform_now(id)
|
|
get "/static_mirror/jobs/#{ id }"
|
|
expect(json).to include('status' => 'failed', 'error' => 'too_large')
|
|
expect(response.body).not_to include('private details')
|
|
get "/static_mirror/files/#{ id }"
|
|
expect(response).to have_http_status(:not_found)
|
|
end
|
|
|
|
it 'is unavailable outside development' do
|
|
allow(Rails.env).to receive(:development?).and_return(false)
|
|
get '/static_mirror'
|
|
expect(response).to have_http_status(:not_found)
|
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
|
expect(response).to have_http_status(:not_found)
|
|
get '/static_mirror/jobs/unknown'
|
|
expect(response).to have_http_status(:not_found)
|
|
get '/static_mirror/files/unknown'
|
|
expect(response).to have_http_status(:not_found)
|
|
end
|
|
|
|
it 'preserves user and IP BAN checks' do
|
|
user = create(:user, banned_at: Time.current)
|
|
sign_in_as(user)
|
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
|
expect(response).to have_http_status(:forbidden)
|
|
sign_out
|
|
IpAddress.create!(ip_address: IPAddr.new('127.0.0.1').hton, banned_at: Time.current)
|
|
get '/static_mirror', headers: { 'REMOTE_ADDR' => '127.0.0.1' }
|
|
expect(response).to have_http_status(:forbidden)
|
|
expect(StaticMirrorJob).not_to have_received(:perform_later)
|
|
end
|
|
|
|
it 'requires the normal CSRF token for anonymous generation', csrf: true do
|
|
get '/users/csrf'
|
|
token = json.fetch('token')
|
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json
|
|
expect(response).to have_http_status(:forbidden)
|
|
post '/static_mirror/jobs', params: { locale: 'ja' }, as: :json,
|
|
headers: { 'X-CSRF-Token' => token }
|
|
expect(response).to have_http_status(:accepted)
|
|
end
|
|
end
|