AI: address issue #483
このコミットが含まれているのは:
@@ -0,0 +1,201 @@
|
||||
require 'cgi'
|
||||
require 'digest'
|
||||
require 'erb'
|
||||
require 'pathname'
|
||||
require 'uri'
|
||||
|
||||
# A deliberately small current-state projection, always under public authority.
|
||||
class StaticMirror
|
||||
RESOURCE_LIMIT = 100
|
||||
NAME_LIMIT = 2_000
|
||||
MAX_BYTES = 8.megabytes
|
||||
EXCLUSIONS = [
|
||||
'historical_state', 'unversioned_resources', 'tag_descriptions', 'help', 'materials',
|
||||
'external_tags', 'parent_relations', 'tag_sections', 'users', 'audit_data',
|
||||
'authentication', 'editing', 'images', 'videos', 'resources_beyond_limits'].freeze
|
||||
class TooLarge < StandardError; end
|
||||
class UnsafeUrl < StandardError; end
|
||||
|
||||
attr_reader :data_at, :authority, :counts
|
||||
|
||||
def initialize(locale:, origin:)
|
||||
@locale = locale
|
||||
@origin = validated_url(origin).sub(%r{/+\z}, '')
|
||||
end
|
||||
|
||||
def export(&progress)
|
||||
@files = { }
|
||||
@bytes = 0
|
||||
HistoryAccess.set(user: nil) do
|
||||
capture
|
||||
progress&.call(0, @posts.length + @tags.length)
|
||||
generate_pages(&progress)
|
||||
raise HistoryAccess::Forbidden unless HistoryAccess.authorised_artifact?(authority)
|
||||
|
||||
generated_at = Time.current.utc
|
||||
manifest = { format_version: 1, generated_at: generated_at.iso8601(6),
|
||||
data_at:, locale: @locale.code, resources: counts, exclusions: EXCLUSIONS,
|
||||
fields: { posts: ['id', 'title', 'url', 'tags'], tags: ['id', 'category', 'names'] },
|
||||
limits: { posts: RESOURCE_LIMIT, tags: RESOURCE_LIMIT, names: NAME_LIMIT,
|
||||
bytes: MAX_BYTES },
|
||||
external_assets: 'not_included', checksum_scope: 'all_files_except_manifest.json',
|
||||
files: @files.map { |path, data| { path:, sha256: Digest::SHA256.hexdigest(data) } } }
|
||||
add_file('manifest.json', JSON.pretty_generate(manifest))
|
||||
entries = @files.map do |path, data|
|
||||
MaterialZipExporter::Entry.new(path: "archive/#{ path }", data:, mtime: generated_at)
|
||||
end
|
||||
bytes = MaterialZipExporter::ZipWriter.write(entries)
|
||||
raise TooLarge if bytes.bytesize > MAX_BYTES
|
||||
|
||||
bytes
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def capture
|
||||
CountsExporter.with_read_snapshot do
|
||||
@data_at = Time.current.utc.iso8601(6)
|
||||
@authority = HistoryAccess.authority
|
||||
# Requiring a matching immutable revision closes the legacy unversioned gap.
|
||||
@posts = confirmed(Post, PostVersion, :post_versions).order(:id)
|
||||
.limit(RESOURCE_LIMIT).select(:id, :title, :url).to_a
|
||||
@tags = confirmed(Tag, TagVersion, :tag_versions).where(deprecated_at: nil)
|
||||
.order(:id).limit(RESOURCE_LIMIT).select(:id, :category).to_a
|
||||
@names = TagName.where(tag_id: @tags.map(&:id)).order(:tag_id, :language_code, :id)
|
||||
.limit(NAME_LIMIT + 1).pluck(:tag_id, :language_code, :script_code, :name, :primary_flg)
|
||||
raise TooLarge if @names.length > NAME_LIMIT
|
||||
|
||||
@names = @names.group_by(&:first)
|
||||
@relations = PostTag.where(post_id: @posts.map(&:id), tag_id: @tags.map(&:id))
|
||||
.order(:post_id, :tag_id).pluck(:post_id, :tag_id)
|
||||
@counts = { posts: @posts.length, tags: @tags.length }
|
||||
end
|
||||
end
|
||||
|
||||
def confirmed(model, version_model, association)
|
||||
HistoryAccess.current(model.all, version_model).joins(association)
|
||||
.where("#{ version_model.table_name }.version_no = #{ model.table_name }.version_no")
|
||||
end
|
||||
|
||||
def generate_pages
|
||||
assets = Rails.root.join('app/views/static_mirror')
|
||||
['archive.css', 'archive.js'].each do |name|
|
||||
add_file("assets/#{ name }", assets.join(name).read)
|
||||
end
|
||||
add_file('robots.txt', "User-agent: *\nDisallow: /\n")
|
||||
add_file('archive-config.js', 'window.archiveConfig = { origin: null };')
|
||||
page('index.html', 'Static Mirror 生成機能', '', search_form + limitations)
|
||||
page("#{ @locale.code }/posts/index.html", 'Post', '/posts', post_list(@posts))
|
||||
page("#{ @locale.code }/tags/index.html", 'Tag', '/tags', tag_list(@tags))
|
||||
processed = 0
|
||||
@posts.each do |post|
|
||||
tags = @tags.select { |tag| @relations.include?([post.id, tag.id]) }
|
||||
content = "<p>#{ external_link(post.url) }</p>#{ tag_list(tags) }"
|
||||
page(resource_path('posts', post.id), post.title.presence || post.id.to_s,
|
||||
"/posts/#{ post.id }", content)
|
||||
processed += 1
|
||||
yield(processed, @posts.length + @tags.length) if block_given?
|
||||
end
|
||||
@tags.each do |tag|
|
||||
posts = @posts.select { |post| @relations.include?([post.id, tag.id]) }
|
||||
names = @names.fetch(tag.id, []).map do |_, language, script, name, _primary|
|
||||
"<li><span lang=\"#{ h(language) }\">#{ h(name) }</span> (#{ h(script) })</li>"
|
||||
end.join
|
||||
content = "<p>#{ h(tag.category) }</p><ul>#{ names }</ul>#{ post_list(posts) }"
|
||||
page(resource_path('tags', tag.id), tag_name(tag), "/tags/#{ tag.id }", content)
|
||||
processed += 1
|
||||
yield(processed, @posts.length + @tags.length) if block_given?
|
||||
end
|
||||
index = @posts.map { |post|
|
||||
{ path: resource_path('posts', post.id), title: post.title.presence || post.id.to_s }
|
||||
} + @tags.map { |tag|
|
||||
{ path: resource_path('tags', tag.id), title: tag_name(tag),
|
||||
names: @names.fetch(tag.id, []).map { _1[3] } }
|
||||
}
|
||||
add_file('search-index.json', JSON.generate(index))
|
||||
end
|
||||
|
||||
def page(path, title, original_path, content)
|
||||
@path = path
|
||||
@title = title
|
||||
@original_path = "/#{ @locale.code }#{ original_path }"
|
||||
@content = content
|
||||
template = Rails.root.join('app/views/static_mirror/page.html.erb').read
|
||||
add_file(path, ERB.new(template).result(binding))
|
||||
end
|
||||
|
||||
def add_file(path, data)
|
||||
@bytes += data.bytesize
|
||||
raise TooLarge if @bytes > MAX_BYTES
|
||||
|
||||
@files[path] = data
|
||||
end
|
||||
|
||||
def resource_path(resource, id) = "#{ @locale.code }/#{ resource }/#{ id }/index.html"
|
||||
|
||||
def relative(path)
|
||||
Pathname.new(path).relative_path_from(Pathname.new(@path).dirname).to_s
|
||||
end
|
||||
|
||||
def h(value) = CGI.escapeHTML(value.to_s)
|
||||
|
||||
def tag_name(tag)
|
||||
@names.fetch(tag.id, []).find { _1[1] == @locale.language_code && _1[4] }&.fetch(3) ||
|
||||
tag.id.to_s
|
||||
end
|
||||
|
||||
# Links are resolved after the page path is known; only generated numeric IDs enter paths.
|
||||
def post_list(posts)
|
||||
'<h2>Post</h2><ul>' + posts.map { |post|
|
||||
title = post.title.presence || post.id.to_s
|
||||
"<li><a data-path=\"#{ resource_path('posts', post.id) }\">#{ h(title) }</a></li>"
|
||||
}.join + '</ul>'
|
||||
end
|
||||
|
||||
def tag_list(tags)
|
||||
'<h2>Tag</h2><ul>' + tags.map { |tag|
|
||||
"<li><a data-path=\"#{ resource_path('tags', tag.id) }\">#{ h(tag_name(tag)) }</a></li>"
|
||||
}.join + '</ul>'
|
||||
end
|
||||
|
||||
def linked_content
|
||||
@content.gsub(/data-path="([^"]+)"/) { "href=\"#{ h(relative(Regexp.last_match(1))) }\"" }
|
||||
end
|
||||
|
||||
def validated_url(value)
|
||||
uri = URI.parse(value.to_s)
|
||||
unless uri.is_a?(URI::HTTP) && uri.host.present? && uri.userinfo.nil?
|
||||
raise UnsafeUrl
|
||||
end
|
||||
|
||||
uri.to_s
|
||||
rescue URI::InvalidURIError
|
||||
raise UnsafeUrl
|
||||
end
|
||||
|
||||
def external_link(value)
|
||||
url = validated_url(value)
|
||||
"<a href=\"#{ h(url) }\" target=\"_blank\" rel=\"noopener noreferrer\">#{ h(url) }</a>"
|
||||
rescue UnsafeUrl
|
||||
''
|
||||
end
|
||||
|
||||
def search_form
|
||||
'<form id="archive-search"><label>検索 <input name="q" type="search"></label>' +
|
||||
'<button type="submit">検索</button></form><ul id="search-results"></ul>'
|
||||
end
|
||||
|
||||
def limitations
|
||||
<<~HTML
|
||||
<h2>未対応機能</h2>
|
||||
<p>Tag Description, Help, Material, 外部タグ, 親子関係, 区間, 履歴, ログイン, 作成, 編輯</p>
|
||||
<h2>出力件数の制限</h2><p>Post: #{ RESOURCE_LIMIT }, Tag: #{ RESOURCE_LIMIT }</p>
|
||||
<p>TagName: #{ NAME_LIMIT }, #{ MAX_BYTES } B</p>
|
||||
<h2>外部画像や動画への依存</h2><p>ローカルへ取り込めていない資産: 外部画像や動画</p>
|
||||
<h2>データ保護上の制約</h2>
|
||||
<p>公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。</p>
|
||||
<p>今回は現在状態のみの対応</p>
|
||||
HTML
|
||||
end
|
||||
end
|
||||
新しいイシューから参照
ユーザーをブロックする