require 'cgi' require 'digest' require 'erb' require 'pathname' require 'uri' # A deliberately small current-state projection, always under public authority. class StaticMirror RESOURCE_LIMIT = 100 NAME_LIMIT = 2_000 MAX_BYTES = 8.megabytes EXCLUSIONS = [ 'historical_state', 'unversioned_resources', 'tag_descriptions', 'help', 'materials', 'external_tags', 'parent_relations', 'tag_sections', 'users', 'audit_data', 'authentication', 'editing', 'images', 'videos', 'resources_beyond_limits'].freeze class TooLarge < StandardError; end class UnsafeUrl < StandardError; end attr_reader :data_at, :authority, :counts def initialize(locale:, origin:) @locale = locale @origin = validated_url(origin).sub(%r{/+\z}, '') end def export(&progress) @files = { } @bytes = 0 HistoryAccess.set(user: nil) do capture progress&.call(0, @posts.length + @tags.length) generate_pages(&progress) raise HistoryAccess::Forbidden unless HistoryAccess.authorised_artifact?(authority) generated_at = Time.current.utc manifest = { format_version: 1, generated_at: generated_at.iso8601(6), data_at:, locale: @locale.code, resources: counts, exclusions: EXCLUSIONS, fields: { posts: ['id', 'title', 'url', 'tags'], tags: ['id', 'category', 'names'] }, limits: { posts: RESOURCE_LIMIT, tags: RESOURCE_LIMIT, names: NAME_LIMIT, bytes: MAX_BYTES }, external_assets: 'not_included', checksum_scope: 'all_files_except_manifest.json', files: @files.map { |path, data| { path:, sha256: Digest::SHA256.hexdigest(data) } } } add_file('manifest.json', JSON.pretty_generate(manifest)) entries = @files.map do |path, data| MaterialZipExporter::Entry.new(path: "archive/#{ path }", data:, mtime: generated_at) end bytes = MaterialZipExporter::ZipWriter.write(entries) raise TooLarge if bytes.bytesize > MAX_BYTES bytes end end private def capture CountsExporter.with_read_snapshot do @data_at = Time.current.utc.iso8601(6) @authority = HistoryAccess.authority # Requiring a matching immutable revision closes the legacy unversioned gap. @posts = confirmed(Post, PostVersion, :post_versions).order(:id) .limit(RESOURCE_LIMIT).select(:id, :title, :url).to_a @tags = confirmed(Tag, TagVersion, :tag_versions).where(deprecated_at: nil) .order(:id).limit(RESOURCE_LIMIT).select(:id, :category).to_a @names = TagName.where(tag_id: @tags.map(&:id)).order(:tag_id, :language_code, :id) .limit(NAME_LIMIT + 1).pluck(:tag_id, :language_code, :script_code, :name, :primary_flg) raise TooLarge if @names.length > NAME_LIMIT @names = @names.group_by(&:first) @relations = PostTag.where(post_id: @posts.map(&:id), tag_id: @tags.map(&:id)) .order(:post_id, :tag_id).pluck(:post_id, :tag_id) @counts = { posts: @posts.length, tags: @tags.length } end end def confirmed(model, version_model, association) HistoryAccess.current(model.all, version_model).joins(association) .where("#{ version_model.table_name }.version_no = #{ model.table_name }.version_no") end def generate_pages assets = Rails.root.join('app/views/static_mirror') ['archive.css', 'archive.js'].each do |name| add_file("assets/#{ name }", assets.join(name).read) end add_file('robots.txt', "User-agent: *\nDisallow: /\n") add_file('archive-config.js', 'window.archiveConfig = { origin: null };') page('index.html', 'Static Mirror 生成機能', '', search_form + limitations) page("#{ @locale.code }/posts/index.html", 'Post', '/posts', post_list(@posts)) page("#{ @locale.code }/tags/index.html", 'Tag', '/tags', tag_list(@tags)) processed = 0 @posts.each do |post| tags = @tags.select { |tag| @relations.include?([post.id, tag.id]) } content = "

#{ external_link(post.url) }

#{ tag_list(tags) }" page(resource_path('posts', post.id), post.title.presence || post.id.to_s, "/posts/#{ post.id }", content) processed += 1 yield(processed, @posts.length + @tags.length) if block_given? end @tags.each do |tag| posts = @posts.select { |post| @relations.include?([post.id, tag.id]) } names = @names.fetch(tag.id, []).map do |_, language, script, name, _primary| "
  • #{ h(name) } (#{ h(script) })
  • " end.join content = "

    #{ h(tag.category) }

    #{ post_list(posts) }" page(resource_path('tags', tag.id), tag_name(tag), "/tags/#{ tag.id }", content) processed += 1 yield(processed, @posts.length + @tags.length) if block_given? end index = @posts.map { |post| { path: resource_path('posts', post.id), title: post.title.presence || post.id.to_s } } + @tags.map { |tag| { path: resource_path('tags', tag.id), title: tag_name(tag), names: @names.fetch(tag.id, []).map { _1[3] } } } add_file('search-index.json', JSON.generate(index)) end def page(path, title, original_path, content) @path = path @title = title @original_path = "/#{ @locale.code }#{ original_path }" @content = content template = Rails.root.join('app/views/static_mirror/page.html.erb').read add_file(path, ERB.new(template).result(binding)) end def add_file(path, data) @bytes += data.bytesize raise TooLarge if @bytes > MAX_BYTES @files[path] = data end def resource_path(resource, id) = "#{ @locale.code }/#{ resource }/#{ id }/index.html" def relative(path) Pathname.new(path).relative_path_from(Pathname.new(@path).dirname).to_s end def h(value) = CGI.escapeHTML(value.to_s) def tag_name(tag) @names.fetch(tag.id, []).find { _1[1] == @locale.language_code && _1[4] }&.fetch(3) || tag.id.to_s end # Links are resolved after the page path is known; only generated numeric IDs enter paths. def post_list(posts) '

    Post

    ' end def tag_list(tags) '

    Tag

    ' end def linked_content @content.gsub(/data-path="([^"]+)"/) { "href=\"#{ h(relative(Regexp.last_match(1))) }\"" } end def validated_url(value) uri = URI.parse(value.to_s) unless uri.is_a?(URI::HTTP) && uri.host.present? && uri.userinfo.nil? raise UnsafeUrl end uri.to_s rescue URI::InvalidURIError raise UnsafeUrl end def external_link(value) url = validated_url(value) "#{ h(url) }" rescue UnsafeUrl '' end def search_form '' end def limitations <<~HTML

    未対応機能

    Tag Description, Help, Material, 外部タグ, 親子関係, 区間, 履歴, ログイン, 作成, 編輯

    出力件数の制限

    Post: #{ RESOURCE_LIMIT }, Tag: #{ RESOURCE_LIMIT }

    TagName: #{ NAME_LIMIT }, #{ MAX_BYTES } B

    外部画像や動画への依存

    ローカルへ取り込めていない資産: 外部画像や動画

    データ保護上の制約

    公開抑止、除票、非公開情報、認証情報、監査専用情報を ZIP に含めない。

    今回は現在状態のみの対応

    HTML end end