#434

backend/AGENTS.md

@@ -1,5 +1,20 @@
 # backend/AGENTS.md
 
+## Mandatory security and URL contracts
+
+- URL クエリへの JSON 格納を禁止する。Base64・16進数等による迂回も禁止。
+  単純な配列は半角スペース区切り、構造化条件は番号付きの個別キーで表す。
+  未指定と明示的な空選択を区別し、自由入力を区切り文字で分割しない。
+  Frontend / Backend で同じ文法・上限・検証を使い、新しい URL 状態の追加前に確認する。
+- 一度除票されたリソースの除票以前（除票版を含む）の履歴は admin 以外へ公開しない。
+  恢復しても解除しない。すべての後続除票を確定版番号と正しい複合キーで判定する。
+  本文・属性・ID・版番号・関連参照・旧値を API / SSR / キャッシュ / 出力から漏らさない。
+  サーバーが確定した current_user の role のみを認可に使用する。
+  秘匿版も安全な全体件数には含めるが、公開明細・内訳・検索件数とは分離する。
+  秘匿属性を条件・軸・ソート・差分件数から推測できる実装を禁止する。
+- 履歴読み取りは HistoryAccess の共通認可を必須とする。書込用 relation を公開しない。
+  export は生成時と取得時に認可し、権限変更・除票後に古いファイルを再利用しない。
+
 ## Scope
 
 These rules apply to work under `backend/`.

backend/app/controllers/application_controller.rb

@@ -1,4 +1,5 @@
 class ApplicationController < ActionController::API
+  include RequestAccess
   include ActionController::Cookies
   include ActionController::RequestForgeryProtection
 
@@ -8,34 +9,46 @@ class ApplicationController < ActionController::API
 
   rescue_from ActiveRecord::RecordInvalid, with: :render_record_invalid
   rescue_from ActiveRecord::RecordNotUnique, with: :render_record_not_unique
+  rescue_from(ActiveRecord::RecordNotFound) { head :not_found }
   rescue_from(UnsupportedLocale) { head :not_found }
 
   before_action :reject_banned_ip_address!
   before_action :authenticate_user
   before_action :reject_banned_user!
   before_action :validate_explicit_locale!
+  before_action :enforce_query_contract!
+  around_action :with_history_authority
+  rescue_from(HistoryAccess::Forbidden) { head :not_found }
   protect_from_forgery with: :exception
   self.forgery_protection_origin_check = true
   rescue_from ActionController::InvalidAuthenticityToken, with: :reject_invalid_csrf
   after_action :prevent_authenticated_caching
 
-  def current_user = @current_user
-
   private
 
+  def enforce_query_contract!
+    pairs = URI.decode_www_form(request.query_string)
+    return head :bad_request if pairs.length > 1000 || pairs.map(&:first).uniq.length != pairs.length
+    return head :bad_request if request.query_parameters.values.any? { !_1.is_a?(String) }
+    if (controller_name.start_with?('counts_') &&
+        (request.query_parameters.keys & ['headers', 'labels']).any?)
+      return head :bad_request
+    end
+  rescue ArgumentError
+    head :bad_request
+  end
+
+  def with_history_authority(&action)
+    response.headers['Cache-Control'] = 'private, no-store'
+    HistoryAccess.set(user: current_user, &action)
+  end
+
   def reject_invalid_csrf
     response.headers['X-CSRF-Invalid'] = '1'
     response.headers['Cache-Control'] = 'private, no-store'
     head :forbidden
   end
 
-  def authenticate_user
-    code = session[:inheritance_code]
-    return if code.blank?
-
-    @current_user = User.find_by(inheritance_code: code)
-  end
-
   def prevent_authenticated_caching
     response.headers['Cache-Control'] = 'private, no-store' if current_user
   end
@@ -79,19 +92,6 @@ class ApplicationController < ActionController::API
     render_validation_error base: ['すでに存在してゐます．']
   end
 
-  def reject_banned_ip_address!
-    ip_address = IpAddress.find_by(ip_address: IPAddr.new(request.remote_ip).hton)
-    return unless ip_address&.banned?
-
-    head :forbidden
-  end
-
-  def reject_banned_user!
-    return unless current_user&.banned?
-
-    head :forbidden
-  end
-
   def render_validation_error record = nil, fields: { }, base: [], status: :unprocessable_entity
     errors = { }
 

backend/app/controllers/concerns/counts_listing.rb

@@ -6,7 +6,7 @@ module CountsListing
     resource = is_a?(CountsPostsController) ? 'posts' : 'tags'
     query = CountsQuery.new(version_scope, resource, request.query_parameters.except('locale')
         .merge('locale' => locale.code))
-    result = { count: query.count, page: query.page, limit: query.limit,
+    result = { population_count_scope: 'all_recorded_versions', population_count: HistoryAccess.population_count(query.relation.klass), count: query.count, page: query.page, limit: query.limit,
                at: request.query_parameters['at'].presence || Time.current.utc.iso8601(6) }
     if query.mode == 'summary'
       result.merge!(query.summary)

backend/app/controllers/concerns/history_storage_access.rb

@@ -0,0 +1,52 @@
+# Permanent storage URLs must not bypass a resource's later discard boundary.
+module HistoryStorageAccess
+  extend ActiveSupport::Concern
+
+  included do
+    include RequestAccess
+    prepend_before_action :authorise_history_storage!
+  end
+
+  private
+
+  def authorise_history_storage!
+    response.headers['Cache-Control'] = 'private, no-store'
+    return unless request.get? || request.head?
+
+    reject_banned_ip_address!
+    return if performed?
+
+    authenticate_user
+    reject_banned_user!
+    return if performed?
+
+    signed = params[:signed_blob_id] || params[:signed_id]
+    blob = if signed
+             ActiveStorage::Blob.find_signed(signed)
+           elsif params[:encoded_key]
+             payload = ActiveStorage.verifier.verified(params[:encoded_key], purpose: :blob_key)
+             key = payload && (payload[:key] || payload['key'])
+             if key
+               original = key.start_with?('variants/') ? key.split('/')[1] : key
+               ActiveStorage::Blob.find_by(key: original)
+             end
+           end
+    return head :not_found unless blob
+
+    allowed = HistoryAccess.set(user: current_user) { HistoryAccess.blob_visible?(blob) }
+    head :not_found unless allowed
+  rescue ActiveSupport::MessageVerifier::InvalidSignature
+    head :not_found
+  end
+
+  # Override the engine's public-forever caching before a streaming response commits.
+  def http_cache_forever(**_options)
+    response.headers['Cache-Control'] = 'private, no-store'
+    yield
+  end
+
+  def expires_in(*arguments, **options)
+    super
+    response.headers['Cache-Control'] = 'private, no-store'
+  end
+end

backend/app/controllers/concerns/request_access.rb

@@ -0,0 +1,28 @@
+# Canonical read-side session and BAN boundary, shared by API and storage controllers.
+module RequestAccess
+  extend ActiveSupport::Concern
+
+  def current_user = @current_user
+
+  private
+
+  def authenticate_user
+    code = session[:inheritance_code]
+    return if code.blank?
+
+    @current_user = User.find_by(inheritance_code: code)
+  end
+
+  def reject_banned_ip_address!
+    ip_address = IpAddress.find_by(ip_address: IPAddr.new(request.remote_ip).hton)
+    return unless ip_address&.banned?
+
+    head :forbidden
+  end
+
+  def reject_banned_user!
+    return unless current_user&.banned?
+
+    head :forbidden
+  end
+end

backend/app/controllers/counts_exports_controller.rb

@@ -68,15 +68,15 @@ class CountsExportsController < ApplicationController
         .merge(request.query_parameters)
     exporter = CountsExporter.new(payload,
                                   language_code: locale.language_code)
-    response.headers['Content-Type'] = exporter.format == 'json' ?
-        'application/json; charset=utf-8' : 'text/csv; charset=utf-8'
-    response.headers['Content-Disposition'] =
-        "attachment; filename=\"counts-#{ exporter.resource }.#{ exporter.format }\""
-    response.headers['Cache-Control'] = 'private, no-store'
-    response.headers['X-Accel-Buffering'] = 'no'
-    # Rack::ETag otherwise materialises the response to calculate a digest.
-    response.headers['Last-Modified'] = Time.current.httpdate
-    self.response_body = exporter.body
+    session[:counts_export_owner] ||= SecureRandom.hex(32)
+    token = CountsExportDownload.prepare(exporter, session[:counts_export_owner])
+    file = CountsExportDownload.find(token, session[:counts_export_owner])
+    return head :not_found unless file
+
+    send_file file.fetch('path'),
+        filename: "counts-#{ exporter.resource }.#{ exporter.format }",
+        type: exporter.format == 'json' ? 'application/json' : 'text/csv',
+        disposition: 'attachment'
   rescue ArgumentError, TypeError, KeyError, JSON::ParserError
     head :bad_request
   end

backend/app/controllers/material_versions_controller.rb

@@ -8,7 +8,7 @@ class MaterialVersionsController < ApplicationController
 
     offset = (page - 1) * limit
 
-    q = MaterialVersion.includes(:created_by_user)
+    q = HistoryAccess.visible(MaterialVersion.includes(:created_by_user))
     q = q.where(material_id: params[:material_id]) if params[:material_id].present?
     q = q.where(tag_name: params[:tag].to_s.strip) if params[:tag].present?
     q = q.where(event_type: params[:event_type]) if params[:event_type].present?
@@ -20,7 +20,7 @@ class MaterialVersionsController < ApplicationController
         .limit(limit)
         .offset(offset)
 
-    render json: { versions: serialise_versions(versions), count: }
+    render json: { versions: serialise_versions(versions), count:, population_count: HistoryAccess.population_count(MaterialVersion) }
   end
 
   private

backend/app/controllers/post_versions_controller.rb

@@ -14,7 +14,7 @@ class PostVersionsController < ApplicationController
 
     offset = (page - 1) * limit
 
-    q = version_scope
+    q = HistoryAccess.visible(version_scope)
     q = q.where('post_versions.post_id = ?', post_id) if post_id
     if external_tag_id || (tag_id && !(Tag.exists?(id: tag_id)))
       q = q.where('JSON_CONTAINS(post_versions.tags_json,' +
@@ -32,7 +32,7 @@ class PostVersionsController < ApplicationController
                 .limit(limit)
                 .offset(offset)
 
-    render json: { versions: serialise_versions(versions, locale.language_code), count: }
+    render json: { versions: serialise_versions(versions, locale.language_code), count:, population_count: HistoryAccess.population_count(PostVersion) }
   end
 
   private
@@ -101,7 +101,7 @@ class PostVersionsController < ApplicationController
     }
     missing_ids = ids - names.keys
 
-    NicoTagVersion
+    HistoryAccess.visible(NicoTagVersion.all)
         .where(tag_id: missing_ids)
         .order(:tag_id, version_no: :desc)
         .pluck(:tag_id, :name)

backend/app/controllers/posts_controller.rb

@@ -38,7 +38,7 @@ class PostsController < ApplicationController
     offset = (page - 1) * limit
 
     pt_max_sql =
-        PostVersion
+        HistoryAccess.visible(PostVersion.all)
         .select('post_id, MAX(created_at) AS max_updated_at')
         .group('post_id')
         .to_sql
@@ -344,6 +344,7 @@ class PostsController < ApplicationController
       unless force
         base_version = post.post_versions.find_by!(version_no: base_version_no)
 
+        HistoryAccess.require!(base_version)
         base_snapshot = post_snapshot_from_version(base_version)
         current_snapshot = post_snapshot_from_record(post, locale)
       end
@@ -449,7 +450,7 @@ class PostsController < ApplicationController
   private
 
   def tag_filter_references literals
-    PostTagSearch.parse_references(params[:tag_refs], literals)
+    PostTagSearch.parse_references(params.to_unsafe_h, literals)
   rescue ArgumentError, JSON::ParserError, TypeError
     raise ActionController::BadRequest
   end

backend/app/controllers/sitemap_entries_controller.rb

@@ -42,7 +42,7 @@ class SitemapEntriesController < ApplicationController
     when 'descriptions'
       TagDescription.where(language_code: locale.language_code, tag_id: named_tags)
           .where.not(body: [nil, ''])
-          .where(TagDescriptionVersion
+          .where(HistoryAccess.visible(TagDescriptionVersion.all)
               .where('tag_description_versions.tag_id = tag_descriptions.tag_id')
               .where('tag_description_versions.language_code = tag_descriptions.language_code')
               .where('tag_description_versions.version_no = tag_descriptions.version_no')
@@ -50,6 +50,7 @@ class SitemapEntriesController < ApplicationController
     when 'tag_groups'
       TagGroup.joins(:versions)
           .where('tag_group_versions.version_no = tag_groups.version_no')
+          .where(Arel.sql(HistoryAccess.sql(TagGroupVersion)))
     when 'theatres'
       Theatre.kept
     end

backend/app/controllers/tag_descriptions_controller.rb

@@ -49,6 +49,7 @@ class TagDescriptionsController < ApplicationController
   private
 
   def representation description
+    HistoryAccess.require_current!(description, TagDescriptionVersion)
     description.as_json(only: [
         :tag_id,
         :language_code,

backend/app/controllers/tag_groups_controller.rb

@@ -1,7 +1,7 @@
 class TagGroupsController < ApplicationController
   def index
     locale = resolve_locale!
-    groups = TagGroup.includes(:created_by_user, :updated_by_user).order(:id)
+    groups = HistoryAccess.current(TagGroup.all, TagGroupVersion).includes(:created_by_user, :updated_by_user).order(:id)
     render json: TagGroupRepr.many(locale, groups.to_a)
   end
 

backend/app/controllers/tag_versions_controller.rb

@@ -10,7 +10,7 @@ class TagVersionsController < ApplicationController
 
     offset = (page - 1) * limit
 
-    q = version_scope
+    q = HistoryAccess.visible(version_scope)
     q = q.where('tag_versions.tag_id = ?', tag_id) if tag_id
 
     count = q.except(:select, :order, :limit, :offset).count
@@ -19,7 +19,7 @@ class TagVersionsController < ApplicationController
                 .limit(limit)
                 .offset(offset)
 
-    render json: { versions: serialise_versions(versions, locale.language_code), count: }
+    render json: { versions: serialise_versions(versions, locale.language_code), count:, population_count: HistoryAccess.population_count(TagVersion) }
   end
 
   private
@@ -31,6 +31,7 @@ class TagVersionsController < ApplicationController
             ON
               prev.tag_id = tag_versions.tag_id
               AND prev.version_no = tag_versions.version_no - 1
+              AND #{ HistoryAccess.sql(TagVersion, 'prev') }
         SQL
       .select('tag_versions.*', 'prev.name AS prev_name', 'prev.category AS prev_category',
               'prev.deprecated_at AS prev_deprecated_at',

backend/app/controllers/wiki_pages_controller.rb

@@ -8,12 +8,12 @@ class WikiPagesController < ApplicationController
 
     title = params[:title].to_s.strip
     descriptions =
-        TagDescription
+        HistoryAccess.current(TagDescription.all, TagDescriptionVersion)
         .where(language_code: locale.language_code)
         .includes(tag: :tag_names)
 
     descriptions = descriptions.order(updated_at: :desc, id: :asc)
-    tag_groups = TagGroup.order(updated_at: :desc, id: :asc)
+    tag_groups = HistoryAccess.current(TagGroup.all, TagGroupVersion).order(updated_at: :desc, id: :asc)
     if title.present?
       pattern = "%#{ ApplicationRecord.sanitize_sql_like(title) }%"
       descriptions =
@@ -41,13 +41,13 @@ class WikiPagesController < ApplicationController
     return head :not_found unless language.is_a?(String)
     return head :not_found unless Language.exists?(code: language)
 
-    names = TagDescription.joins(tag: :tag_names)
+    names = HistoryAccess.current(TagDescription.all, TagDescriptionVersion).joins(tag: :tag_names)
         .where(language_code: locale.language_code, tag_names: { language_code: language })
         .pluck('tag_descriptions.tag_id', 'tag_descriptions.id', 'tag_names.name')
     candidates = names.map do |tag_id, description_id, name|
       WikiPageRepr.autolink_tag(locale, tag_id:, description_id:, name:, language:)
     end
-    candidates.concat(TagGroup.order(:id).pluck(:id, :name).map do |id, name|
+    candidates.concat(HistoryAccess.current(TagGroup.all, TagGroupVersion).order(:id).pluck(:id, :name).map do |id, name|
       WikiPageRepr.autolink_group(locale, id:, name:, language:)
     end)
     render json: candidates
@@ -286,12 +286,12 @@ class WikiPagesController < ApplicationController
 
     tag_group = id && TagGroup.find_by(id:)
     if tag_group
-      return render json: tag_group_changes(TagGroupVersion.where(tag_group_id: tag_group.id))
+      return render json: tag_group_changes(HistoryAccess.visible(TagGroupVersion.all).where(tag_group_id: tag_group.id))
     end
 
     json = []
     unless id
-      json.concat(tag_group_changes(TagGroupVersion.all))
+      json.concat(tag_group_changes(HistoryAccess.visible(TagGroupVersion.all)))
       json.concat(description_changes(locale,
                                       TagDescription.where(language_code: locale.language_code)))
       json = json.each_with_index.sort_by do |row, index|
@@ -327,7 +327,7 @@ class WikiPagesController < ApplicationController
 
   def render_group_representation representation, version, body: nil, status: :ok
     body ||= TagGroupMarkdownRenderer.render(representation)
-    versions = TagGroupVersion.where(tag_group_id: version.tag_group_id)
+    versions = HistoryAccess.visible(TagGroupVersion.all).where(tag_group_id: version.tag_group_id)
     pred = versions.find_by(version_no: version.version_no - 1)&.id
     succ = versions.find_by(version_no: version.version_no + 1)&.id
     updated_at = version.created_at
@@ -346,11 +346,11 @@ class WikiPagesController < ApplicationController
   end
 
   def current_group_version group
-    TagGroupVersion.find_by(tag_group_id: group.id, version_no: group.version_no)
+    HistoryAccess.visible(TagGroupVersion.all).find_by(tag_group_id: group.id, version_no: group.version_no)
   end
 
   def group_version group, version_id
-    TagGroupVersion.find_by(tag_group_id: group.id, id: version_id)
+    HistoryAccess.visible(TagGroupVersion.all).find_by(tag_group_id: group.id, id: version_id)
   end
 
   def wiki_diff from_body, to_body
@@ -373,7 +373,7 @@ class WikiPagesController < ApplicationController
   def tag_group_changes versions
     rows = versions.includes(:created_by_user).order(id: :desc).limit(200).to_a
     groups = TagGroup.where(id: rows.map(&:tag_group_id).uniq).index_by(&:id)
-    predecessors = TagGroupVersion.where(tag_group_id: groups.keys,
+    predecessors = HistoryAccess.visible(TagGroupVersion.all).where(tag_group_id: groups.keys,
                                          version_no: rows.map { _1.version_no - 1 })
         .pluck(:tag_group_id, :version_no, :id)
         .to_h { |group_id, number, version_id| [[group_id, number], version_id] }
@@ -406,18 +406,18 @@ class WikiPagesController < ApplicationController
   end
 
   def description_versions description
-    TagDescriptionVersion.where(tag_id: description.tag_id,
+    HistoryAccess.visible(TagDescriptionVersion.all).where(tag_id: description.tag_id,
                                 language_code: description.language_code)
   end
 
   def tag_description_by_title locale, title
-    TagDescription.joins(tag: :tag_names)
+    HistoryAccess.current(TagDescription.all, TagDescriptionVersion).joins(tag: :tag_names)
         .find_by(language_code: locale.language_code,
                  tag_names: { language_code: locale.language_code, name: title })
   end
 
   def description_changes locale, descriptions
-    versions = TagDescriptionVersion.where(language_code: locale.language_code,
+    versions = HistoryAccess.visible(TagDescriptionVersion.all).where(language_code: locale.language_code,
                                            tag_id: descriptions.select(:tag_id))
         .order(id: :desc).limit(200).to_a
     return [] if versions.empty?
@@ -425,7 +425,7 @@ class WikiPagesController < ApplicationController
     by_tag = descriptions.where(tag_id: versions.map(&:tag_id))
         .includes(tag: :tag_names).index_by(&:tag_id)
     users = User.where(id: versions.map(&:created_by_user_id)).index_by(&:id)
-    predecessors = TagDescriptionVersion.where(
+    predecessors = HistoryAccess.visible(TagDescriptionVersion.all).where(
         language_code: locale.language_code, tag_id: by_tag.keys,
         version_no: versions.map { |version| version.version_no - 1 })
         .pluck(:tag_id, :version_no, :id)

backend/app/models/history_access.rb

@@ -0,0 +1,151 @@
+require 'digest'
+
+# Request/worker authority comes only from a server-resolved user. Default is public.
+class HistoryAccess < ActiveSupport::CurrentAttributes
+  attribute :user
+  class Forbidden < StandardError; end
+
+  KEYS = { 'post_versions' => ['post_id'], 'tag_versions' => ['tag_id'],
+           'nico_tag_versions' => ['tag_id'],
+           'tag_description_versions' => ['tag_id', 'language_code'],
+           'tag_group_versions' => ['tag_group_id'], 'material_versions' => ['material_id'] }.freeze
+
+  def self.revision
+    values = KEYS.keys.map do |table|
+      table.classify.constantize.unscoped.where(event_type: 'discard').maximum(:id)
+    end
+    Digest::SHA256.hexdigest(values.to_json)
+  end
+
+  def self.authority
+    { 'policy_revision' => revision, 'admin' => admin?, 'actor_id' => user&.id }
+  end
+
+  def self.authorised_artifact?(metadata)
+    return false unless metadata['policy_revision'].is_a?(String)
+    if metadata['admin']
+      admin? && user.id == metadata['actor_id']
+    else
+      metadata['policy_revision'] == revision
+    end
+  end
+
+  def self.admin? = user&.admin? == true
+
+  def self.sql(model, row = model.table_name, references: true)
+    return 'TRUE' if admin?
+
+    table = model.table_name
+    keys = KEYS.fetch(table)
+    matches = keys.map { "boundary.#{ _1 } = #{ row }.#{ _1 }" }.join(' AND ')
+    clauses = ["NOT EXISTS (SELECT 1 FROM #{ table } boundary WHERE #{ matches } " +
+        "AND boundary.event_type = 'discard' AND boundary.version_no >= #{ row }.version_no)"]
+    clauses += reference_sql(table, row) if references
+    clauses.map { "(#{ _1 })" }.join(' AND ')
+  end
+
+  def self.discard_reference(table, key, expression)
+    "NOT EXISTS (SELECT 1 FROM #{ table } blocked WHERE blocked.event_type = 'discard' " +
+        "AND #{ expression.call("blocked.#{ key }") })"
+  end
+
+  def self.reference_sql(table, row)
+    case table
+    when 'post_versions'
+      [['tag_versions', 'tag_id', 'tag_id'],
+       ['nico_tag_versions', 'tag_id', 'external_tag_id']].map do |target, key, field|
+        discard_reference(target, key, ->(id) {
+          "JSON_CONTAINS(#{ row }.tags_json, JSON_OBJECT('#{ field }', #{ id }))"
+        })
+      end + [discard_reference('post_versions', 'post_id', ->(id) {
+        "FIND_IN_SET(#{ id }, REPLACE(#{ row }.parent_post_ids, ' ', ',')) > 0"
+      })]
+    when 'tag_versions'
+      [discard_reference('tag_versions', 'tag_id', ->(id) {
+        "FIND_IN_SET(#{ id }, REPLACE(#{ row }.parent_tag_ids, ' ', ',')) > 0"
+      })]
+    when 'tag_description_versions'
+      [discard_reference('tag_versions', 'tag_id', ->(id) { "#{ row }.tag_id = #{ id }" })]
+    when 'material_versions'
+      [discard_reference('tag_versions', 'tag_id', ->(id) { "#{ row }.tag_id = #{ id }" }),
+       discard_reference('material_versions', 'material_id', ->(id) {
+         "#{ row }.parent_id = #{ id }"
+       })]
+    when 'tag_group_versions'
+      [['tag_versions', 'tag_id'], ['tag_group_versions', 'tag_group_id']].map do |target, field|
+        discard_reference(target, field, ->(id) {
+          ['members', 'placements'].map do |column|
+            "JSON_CONTAINS(COALESCE(JSON_EXTRACT(#{ row }.#{ column }, " +
+                "'$**.#{ field }'), JSON_ARRAY()), CAST(#{ id } AS CHAR))"
+          end.join(' OR ')
+        })
+      end
+    else []
+    end
+  end
+
+  def self.visible(scope, row = scope.klass.table_name)
+    scope.where(Arel.sql(sql(scope.klass, row)))
+  end
+
+  def self.current(scope, version_model)
+    return scope if admin?
+
+    table = scope.klass.table_name
+    keys = KEYS.fetch(version_model.table_name)
+    matches = keys.map do |key|
+      column = scope.klass.column_names.include?(key) ? key : 'id'
+      "current_snapshot.#{ key } = #{ table }.#{ column }"
+    end.join(' AND ')
+    boundary = "#{ matches } AND current_snapshot.version_no >= #{ table }.version_no " +
+        "AND current_snapshot.event_type = 'discard'"
+    scope = scope.where(Arel.sql("NOT EXISTS (SELECT 1 FROM #{ version_model.table_name } " +
+        "current_snapshot WHERE #{ boundary })"))
+    condition = "#{ matches } AND current_snapshot.version_no = #{ table }.version_no"
+    scope.where(Arel.sql("NOT EXISTS (SELECT 1 FROM #{ version_model.table_name } " +
+        "current_snapshot WHERE #{ condition } AND NOT (#{ sql(version_model, 'current_snapshot') }))"))
+  end
+
+  def self.require_current!(record, version_model)
+    return if admin?
+    raise Forbidden unless current(record.class.where(id: record.id), version_model).exists?
+  end
+
+  def self.blob_visible?(blob, seen = [])
+    return true if admin?
+    return false if seen.include?(blob.id)
+    seen = seen + [blob.id]
+    private_versions = MaterialVersion.where(file_blob_id: blob.id)
+        .where(Arel.sql("NOT (#{ sql(MaterialVersion) })"))
+    return false if private_versions.exists?
+
+    blob.attachments.each do |attachment|
+      model, key = { 'Post' => [PostVersion, 'post_id'],
+                     'Material' => [MaterialVersion, 'material_id'] }[attachment.record_type]
+      if model && model.where(key => attachment.record_id, event_type: 'discard').exists?
+        return false
+      end
+      if attachment.record_type == 'ActiveStorage::VariantRecord'
+        original = ActiveStorage::VariantRecord.find_by(id: attachment.record_id)&.blob
+        return false unless original && blob_visible?(original, seen)
+      end
+    end
+    true
+  end
+
+  def self.require_many!(records)
+    return if admin? || records.empty?
+
+    ids = records.map(&:id).uniq
+    raise Forbidden unless visible(records.first.class.where(id: ids)).count == ids.length
+  end
+
+  def self.require!(record)
+    return record if admin? || visible(record.class.where(id: record.id)).exists?
+
+    raise Forbidden
+  end
+
+  # This number deliberately ignores time, identity and content filters to prevent count probes.
+  def self.population_count(model) = model.unscoped.count
+end

backend/app/models/post_version.rb

@@ -3,7 +3,8 @@ class PostVersion < ApplicationRecord
 
   scope :with_previous_version, -> {
     joins('LEFT JOIN post_versions prev ON prev.post_id = post_versions.post_id ' +
-          'AND prev.version_no = post_versions.version_no - 1')
+          'AND prev.version_no = post_versions.version_no - 1 AND ' +
+          HistoryAccess.sql(PostVersion, 'prev'))
   }
 
   belongs_to :post

backend/app/models/tag_description_version.rb

@@ -1,3 +1,5 @@
 class TagDescriptionVersion < ApplicationRecord
+  include VersionRecord
+
   before_create { SharedResourceIdAllocator.assign!(self) }
 end

backend/app/models/version_record.rb

@@ -6,6 +6,8 @@ module VersionRecord
                              discard: 'discard',
                              restore: 'restore' }.freeze
 
+  def require_history_access! = HistoryAccess.require!(self)
+
   def readonly? = persisted?
 
   included do

backend/app/representations/counts_version_repr.rb

@@ -6,6 +6,7 @@ class CountsVersionRepr
 
   def initialize(rows, resource:, language_code:, at:, inclusive:)
     @rows = rows
+    HistoryAccess.require_many!(rows)
     @at = at
     @inclusive = inclusive
     @resource = resource
@@ -15,6 +16,7 @@ class CountsVersionRepr
     @scripts = master_names(Script)
     @tag_versions = {}
     load_tag_versions if resource == 'posts'
+    load_parent_versions if resource == 'tags'
   end
 
   def serialise
@@ -47,7 +49,7 @@ class CountsVersionRepr
         .uniq
     pairs.each_slice(500) do |batch|
       placeholders = batch.map { '(?, ?)' }.join(', ')
-      scope = TagVersion.all
+      scope = HistoryAccess.visible(TagVersion.all)
       if @at
         scope = scope.where("created_at #{ @inclusive ? '<=' : '<' } ?", @at)
       end
@@ -86,6 +88,31 @@ class CountsVersionRepr
       original_created_before: row.original_created_before&.iso8601 }
   end
 
+  def parent_point(row) = @at || row.created_at
+
+  def load_parent_versions
+    @parent_versions = {}
+    requests = @rows.flat_map do |row|
+      row.parent_tag_ids.to_s.split.map { [_1.to_i, parent_point(row)] }
+    end.uniq
+    requests.each_slice(100) do |batch|
+      queries = batch.map do |id, point|
+        scope = HistoryAccess.visible(TagVersion.where(tag_id: id))
+            .where('created_at <= ?', point)
+            .where('NOT EXISTS (SELECT 1 FROM tag_versions newer ' +
+                'WHERE newer.tag_id = tag_versions.tag_id AND newer.created_at <= ? ' +
+                'AND newer.version_no > tag_versions.version_no)', point)
+        quoted = TagVersion.connection.quote(point.iso8601(6))
+        scope.select("tag_id, name, names, #{ quoted } AS reference_point").to_sql
+      end
+      next if queries.empty?
+
+      TagVersion.connection.select_all(queries.map { "(#{ _1 })" }.join(' UNION ALL ')).each do |value|
+        @parent_versions[[value['tag_id'], value['reference_point']]] = value
+      end
+    end
+  end
+
   def tag(row)
     recorded_names = names(row.names)
     selected = recorded_names.find { _1[:language_code] == @language }
@@ -97,7 +124,12 @@ class CountsVersionRepr
     end
     { name: selected&.fetch(:name) || row.name, names: recorded_names,
       category: row.category, aliases:,
-      parents: row.parent_tag_ids.to_s.split.map { { id: _1.to_i } },
+      parents: row.parent_tag_ids.to_s.split.map do |id|
+        value = @parent_versions[[id.to_i, parent_point(row).iso8601(6)]]
+        snapshots = value && json(value['names'])
+        name = snapshots&.dig(@language, 'name') || value&.fetch('name')
+        { id: id.to_i, name: }
+      end,
       deprecated: row.deprecated_at&.iso8601 }
   end
 end

backend/app/representations/tag_group_repr.rb

@@ -44,6 +44,7 @@ module TagGroupRepr
   end
 
   def one locale, group
+    HistoryAccess.require_current!(group, TagGroupVersion)
     many(locale, [group]).first
   end
 
@@ -56,10 +57,11 @@ module TagGroupRepr
   end
 
   def current_version group
-    TagGroupVersion.find_by(tag_group_id: group.id, version_no: group.version_no)
+    HistoryAccess.visible(TagGroupVersion.all).find_by(tag_group_id: group.id, version_no: group.version_no)
   end
 
   def historical locale, version
+    HistoryAccess.require!(version)
     members = Array(version.members)
     placements = Array(version.placements)
     tag_references = members.filter_map { |member| member['tag_id'] } +
@@ -153,13 +155,13 @@ module TagGroupRepr
   end
 
   def historical_tag_versions ids
-    TagVersion.where(tag_id: ids.uniq).to_a.index_by do |version|
+    HistoryAccess.visible(TagVersion.all).where(tag_id: ids.uniq).to_a.index_by do |version|
       [version.tag_id, version.version_no]
     end
   end
 
   def historical_group_versions ids
-    TagGroupVersion.where(tag_group_id: ids.uniq).to_a.index_by do |version|
+    HistoryAccess.visible(TagGroupVersion.all).where(tag_group_id: ids.uniq).to_a.index_by do |version|
       [version.tag_group_id, version.version_no]
     end
   end

backend/app/representations/wiki_page_repr.rb

@@ -5,6 +5,7 @@ module WikiPageRepr
   module_function
 
   def tag_description locale, description
+    HistoryAccess.require_current!(description, TagDescriptionVersion)
     tag = description.tag
 
     { 'id' => description.id,

backend/app/services/counts_axes.rb

@@ -33,8 +33,8 @@ class CountsAxes
   def initialize(population)
     @population = population
     @definition = population.definition
-    @rows = parse(population.params.fetch('rows', '[]'))
-    @columns = parse(population.params.fetch('cols', '[]'))
+    @rows = parse(population.params.fetch('rows', ''))
+    @columns = parse(population.params.fetch('cols', ''))
     @all = rows + columns
     raise ArgumentError if all.length > 8 || all.map { _1['field'] }.uniq.length != all.length
   end
@@ -81,7 +81,7 @@ class CountsAxes
   private
 
   def parse(value)
-    axes = JSON.parse(value)
+    axes = QueryState.axes(value)
     raise ArgumentError unless axes.is_a?(Array)
 
     axes.each do |axis|

backend/app/services/counts_defaults.rb

@@ -6,16 +6,23 @@ class CountsDefaults
 
   def self.entries(resource)
     result = [{ definition: 'history', mode: 'summary', params: {
-        'rows' => [{ field: 'created_at', grain: 'month' }].to_json,
-        'cols' => [{ field: 'event_type' }].to_json,
-        'sort' => 'a0:desc', 'conditions' => '[]' } }]
+        'rows' => 'created_at:month',
+        'cols' => 'event_type',
+        'sort' => 'a0:desc', 'conditions' => '' } }]
+    result << { definition: 'latest', mode: 'summary', params: {
+        'rows' => '', 'cols' => '', 'conditions' => '', 'tags' => '' } }
     if resource == 'posts'
-      result += [{ definition: 'latest', mode: 'summary', params: {
-          'rows' => '[]', 'cols' => '[]', 'conditions' => '[]', 'tags' => '' } },
-          { definition: 'latest', mode: 'details', params: {
-              'columns' => POST_COLUMNS.join(','), 'order' => POST_COLUMNS.join(','),
-              'children' => 'tags.name,parents.id', 'nested_json' => '0',
-              'conditions' => '[]', 'tags' => '' } }]
+      result += [{ definition: 'latest', mode: 'details', params: {
+              'columns' => POST_COLUMNS.join(' '), 'order' => POST_COLUMNS.join(' '),
+              'children' => 'tags.name parents.id', 'nested_json' => '0',
+              'conditions' => '', 'tags' => '' } }]
+    end
+    if resource == 'tags'
+      columns = ['resource_id', 'version_no', 'is_latest', 'name', 'category',
+                 'aliases', 'parents', 'deprecated', 'event_type', 'user', 'created_at']
+      result << { definition: 'latest', mode: 'details', params: {
+          'columns' => columns.join(' '), 'order' => columns.join(' '),
+          'children' => 'aliases.name parents.name', 'nested_json' => '0' } }
     end
     result
   end
@@ -26,8 +33,8 @@ class CountsDefaults
           entry[:mode] == params.fetch('mode', 'summary')
     end&.fetch(:params) || {}
     if settings.key?('sort') && params.key?('rows')
-      actual = JSON.parse(params['rows'])
-      expected = JSON.parse(settings.fetch('rows'))
+      actual = QueryState.axes(params['rows'])
+      expected = QueryState.axes(settings.fetch('rows'))
       compatible = actual.is_a?(Array) && actual.first.is_a?(Hash) &&
           actual.first['field'] == expected.first['field'] &&
           actual.first.fetch('grain', 'month') == expected.first['grain']

backend/app/services/counts_export_download.rb

@@ -20,10 +20,11 @@ class CountsExportDownload
 
   def self.prepare(exporter, owner, progress: nil, check: nil, identifier: nil)
     FileUtils.mkdir_p(directory, mode: 0o700)
+    authority = HistoryAccess.authority
     slot = acquire_slot
 
     begin
-      generate(exporter, owner, progress:, check:, identifier:)
+      generate(exporter, owner, progress:, check:, identifier:, authority:)
     ensure
       slot.close
     end
@@ -40,7 +41,7 @@ class CountsExportDownload
     raise Busy
   end
 
-  def self.generate(exporter, owner, progress: nil, check: nil, identifier: nil)
+  def self.generate(exporter, owner, progress: nil, check: nil, identifier: nil, authority: HistoryAccess.authority)
     FileUtils.mkdir_p(directory, mode: 0o700)
     Dir.glob(directory.join('counts-*.export')).each do |path|
       File.unlink(path) if File.mtime(path) < Time.current - EXPIRY
@@ -80,7 +81,7 @@ class CountsExportDownload
       path = finished_path
       token = verifier.generate({ 'file' => File.basename(path),
                           'resource' => exporter.resource, 'format' => exporter.format,
-                          'owner' => owner }, expires_in: EXPIRY, purpose: 'counts_export')
+                          'owner' => owner, **authority }, expires_in: EXPIRY, purpose: 'counts_export')
       complete = true
       token
     ensure
@@ -92,6 +93,7 @@ class CountsExportDownload
   def self.find(token, owner)
     metadata = verifier.verified(token, purpose: 'counts_export')
     return unless metadata.is_a?(Hash) && metadata['owner'] == owner
+    return unless HistoryAccess.authorised_artifact?(metadata)
     return unless metadata['file']&.match?(/\Acounts-[a-zA-Z0-9_-]+\.export\z/)
     return unless CountsResourceDefinition::DEFINITIONS.key?(metadata['resource'])
     return unless ['json', 'csv'].include?(metadata['format'])

backend/app/services/counts_export_jobs.rb

@@ -103,6 +103,7 @@ class CountsExportJobs
           'language' => language, 'resource' => exporter.resource,
           'format' => exporter.format, 'status' => 'queued',
           'processed' => 0, 'total' => nil,
+          **HistoryAccess.authority,
           'expires_at' => (Time.current + CountsExportDownload::EXPIRY).to_f }
       write(state)
       verifier.generate({ 'id' => id, 'owner' => owner },
@@ -126,6 +127,8 @@ class CountsExportJobs
       state = read(id)
       return unless state && state['owner'] == owner && state['expires_at'] > Time.current.to_f
 
+      return unless HistoryAccess.authorised_artifact?(state)
+
       if cancel && ['queued', 'running', 'ready'].include?(state['status'])
         state['status'] = state['status'] == 'running' ? 'cancelling' : 'cancelled'
         remove_download(state)
@@ -182,6 +185,22 @@ class CountsExportJobs
   end
 
   def self.perform(state)
+    actor = state['actor_id'] && User.find_by(id: state['actor_id'])
+    actor = nil unless state['admin'] && actor&.admin? && !actor.banned?
+    HistoryAccess.set(user: actor) do
+      unless HistoryAccess.authorised_artifact?(state)
+        synchronise do
+          current = read(state.fetch('id'))
+          current['status'] = 'failed'
+          write(current)
+        end
+        return
+      end
+      perform_authorised(state)
+    end
+  end
+
+  def self.perform_authorised(state)
     id = state.fetch('id')
     token = nil
     begin

backend/app/services/counts_export_projection.rb

@@ -21,7 +21,7 @@ class CountsExportProjection
       'tags' => ['id', 'versionNo', 'kind', 'name', 'category', 'sections', 'names'],
       'names' => NAME_FIELDS,
       'aliases' => ['languageCode', 'name', 'scriptCode'],
-      'parents' => ['id'] }.freeze
+      'parents' => ['id', 'name'] }.freeze
 
   def self.default_columns(resource)
     fields = { 'posts' => ['title', 'url', 'tags', 'parents', 'video_ms'],
@@ -56,7 +56,7 @@ class CountsExportProjection
                 CountsQuery::COLUMNS.fetch(resource)
               end
     defaults = summary ? allowed : self.class.default_columns(resource)
-    @columns = params.fetch('columns', defaults.join(',')).split(',')
+    @columns = QueryState.list(params.fetch('columns', defaults.join(' ')))
     raise ArgumentError if (@columns - allowed).any? || @columns.uniq != @columns
 
     @headers = JSON.parse(params.fetch('headers', JSON.generate(@columns)))
@@ -64,8 +64,8 @@ class CountsExportProjection
         @headers.all? { _1.is_a?(String) && _1.length <= 200 })
       raise ArgumentError
     end
-    @order = params.fetch('order', '').split(',')
-    @sorts = params.fetch('child_sort', '').split(',').map do |entry|
+    @order = QueryState.list(params.fetch('order', ''))
+    @sorts = QueryState.list(params.fetch('child_sort', '')).map do |entry|
       parts = entry.split(':', -1)
       raise ArgumentError unless parts.length == 2
 
@@ -74,12 +74,28 @@ class CountsExportProjection
 
       [path, direction]
     end
-    @selected = params.key?('children') ? params['children'].split(',').to_set :
+    @selected = params.key?('children') ? QueryState.list(params['children']).to_set :
         self.class.default_children(resource).to_set
     valid = CHILDREN.flat_map do |key, fields|
       paths(fields, key)
     end
     raise ArgumentError if @selected && (@selected.to_a - valid).any?
+    prune = lambda do |fields, prefix|
+      fields.each do |key|
+        path = "#{ prefix }.#{ key }"
+        children = fields_for(key)
+        next if children.empty?
+
+        prune.call(children, path)
+        @selected.delete(path) unless children.any? { @selected.include?("#{ path }.#{ _1 }") }
+      end
+    end
+    CHILDREN.each { |key, fields| prune.call(fields, key) }
+    pairs = @columns.zip(@headers).select do |key, _header|
+      !CHILDREN.key?(key) || CHILDREN[key].any? { @selected.include?("#{ key }.#{ _1 }") }
+    end
+    @columns = pairs.map(&:first)
+    @headers = pairs.map(&:last)
     allowed_sorts = valid.map do |path|
       parts = path.split('.')
       field = parts.pop

backend/app/services/counts_exporter.rb

@@ -14,7 +14,7 @@ class CountsExporter
       return
     end
     allowed = (CountsQuery::KEYS + CountsStateQuery::KEYS + EXTRA_KEYS).uniq
-    raise ArgumentError if (params.keys - allowed).any?
+    QueryState.validate!(params, allowed)
     raise ArgumentError unless params.values.all? { _1.is_a?(String) }
 
     @params = params
@@ -65,8 +65,8 @@ class CountsExporter
                      sort: export_sort,
                      child_sort: sort_metadata(@params.fetch('child_sort', '')),
                      columns: @projection.metadata,
-                     children: @params['children']&.split(','),
-                     field_order: @params.fetch('order', '').split(','), timezone: 'UTC',
+                     children: @params['children']&.split(' '),
+                     field_order: @QueryState.list(params.fetch('order', '')), timezone: 'UTC',
                      count: @mode == 'summary' ? page[:group_count] : page[:count],
                      totals: page[:totals], coverage: page[:coverage] }
             output << (format == 'json' ? "{\"metadata\":#{ meta.to_json },\"data\":[" : "\uFEFF")
@@ -103,7 +103,7 @@ class CountsExporter
   end
 
   def sort_metadata(value)
-    value.split(',').map do |entry|
+    value.split(' ').map do |entry|
       key, direction = entry.split(':', 2)
       { key:, direction: }
     end

backend/app/services/counts_pivot_layout.rb

@@ -2,10 +2,10 @@
 class CountsPivotLayout
   def self.depths(keys, sorts, measures: true)
     return [] if keys.length < 2 || sorts.first&.first == 'count' ||
-        sorts.first&.first&.start_with?('measure_')
+        sorts.first&.first&.match?(/\Am[0-9]+\z/)
 
     relevant = sorts.map(&:first).select { keys.include?(_1) || (measures && (_1 == 'count' ||
-        _1.start_with?('measure_'))) }
+        _1.match?(/\Am[0-9]+\z/))) }
     order = (relevant + keys).uniq
     (1...keys.length).select { order.first(_1).sort == keys.first(_1).sort }
   end

backend/app/services/counts_population.rb

@@ -14,6 +14,7 @@ class CountsPopulation
 
     @scope = definition.model.all
     make_population
+    @scope = HistoryAccess.visible(@scope, 'v')
     apply_filters
   rescue TZInfo::InvalidTimezoneIdentifier
     raise ArgumentError
@@ -62,7 +63,8 @@ class CountsPopulation
     @scope = @scope.from("#{ table } v").select('v.*')
     keys = definition.keys.map { "previous.#{ _1 } = v.#{ _1 }" }.join(' AND ')
     @scope = @scope.joins("LEFT JOIN #{ table } previous ON #{ keys } " +
-        'AND previous.version_no = v.version_no - 1')
+        'AND previous.version_no = v.version_no - 1 AND ' +
+        HistoryAccess.sql(definition.model, 'previous'))
     @scope = @scope.where("v.#{ definition.time_column } <= ?", at)
     unless history
       point = timestamp(params['at']) || at
@@ -92,11 +94,11 @@ class CountsPopulation
       @scope = @scope.where("#{ definition.expression(key) } = ?", value.to_i)
     end
     apply_tags if params['tags'].present? || params['tag'].present?
-    conditions = JSON.parse(params.fetch('conditions', '[]'))
+    conditions = QueryState.records(params, 'conditions')
     raise ArgumentError unless conditions.is_a?(Array) && conditions.length <= 100
 
     conditions.each { apply_condition(_1) }
-    drills = JSON.parse(params.fetch('drill', '[]'))
+    drills = QueryState.records(params, 'drill')
     raise ArgumentError unless drills.is_a?(Array) && drills.length <= 100
 
     drills.each do |entry|
@@ -190,7 +192,7 @@ class CountsPopulation
     return unless params['tags'].present?
 
     locale = Locale.find_by!(code: params.fetch('locale', 'ja'))
-    refs = PostTagSearch.parse_references(params['tag_refs'], params['tags'].split)
+    refs = PostTagSearch.parse_references(params, params['tags'].split)
     raw = params['tags']
     literals = PostTagSearch.literals(locale, raw, references: refs)
     predicates = literals.map do |literal|

backend/app/services/counts_query.rb

@@ -20,6 +20,7 @@ class CountsQuery
     @id_column = resource == 'posts' ? 'post_id' : 'tag_id'
     @params = params
     @latest_expression = latest_expression
+    scope = HistoryAccess.visible(scope)
     @scope = scope.except(:select, :order).joins(
         "LEFT JOIN users counts_actor ON counts_actor.id = #{ @table }.created_by_user_id")
     raise ArgumentError unless (params.keys - KEYS).empty?
@@ -33,7 +34,7 @@ class CountsQuery
     @limit = integer('limit', 20)
     raise ArgumentError if limit > 100 || page > 1_000_000
 
-    columns = params.fetch('columns', COLUMNS.fetch(resource).join(',')).split(',')
+    columns = QueryState.list(params.fetch('columns', COLUMNS.fetch(resource).join(' ')))
     if columns.empty? || columns.uniq != columns || (columns - COLUMNS.fetch(resource)).any?
       raise ArgumentError
     end
@@ -92,7 +93,7 @@ class CountsQuery
     value = @params.fetch('sort', mode == 'details' ? 'created_at:desc' : 'group:asc')
     return [] if value.empty?
 
-    values = value.split(',', -1).map { _1.split(':', -1) }
+    values = QueryState.list(value).map { _1.split(':', -1) }
     allowed = mode == 'details' ? COLUMNS.fetch(@resource) - ['thumbnail'] : ['group', 'count']
     invalid = values.any? do |key, direction, *extra|
       extra.any? || !allowed.include?(key) || !['asc', 'desc'].include?(direction)

backend/app/services/counts_state_query.rb

@@ -46,11 +46,12 @@ class CountsStateQuery
     # Keep totals, rows, and detail pages on one repeatable-read database snapshot.
     ApplicationRecord.transaction do
       unless @history_loaded
-        @post_start = PostVersion.minimum(:created_at)
-        @tag_start = TagVersion.minimum(:created_at)
+        @post_start = HistoryAccess.visible(PostVersion.all).minimum(:created_at)
+        @tag_start = HistoryAccess.visible(TagVersion.all).minimum(:created_at)
         @history_loaded = true
       end
-      result = { page: @page, limit: @limit, at: @at.iso8601(6),
+      result = { population_count: HistoryAccess.population_count(
+          @resource == 'posts' ? PostVersion : TagVersion), page: @page, limit: @limit, at: @at.iso8601(6),
                  coverage: { basis: 'recorded_snapshots', exact: false,
                              post_history_from: @post_start&.iso8601(6),
                              tag_history_from: @tag_start&.iso8601(6) } }
@@ -131,7 +132,10 @@ class CountsStateQuery
   end
 
   def state_sql(table, boundary, live: false)
-    sql = "SELECT v.* FROM #{ table } v WHERE #{ latest_condition(table, 'v', boundary) }"
+    model = { 'post_versions' => PostVersion, 'tag_versions' => TagVersion,
+              'nico_tag_versions' => NicoTagVersion }.fetch(table)
+    sql = "SELECT v.* FROM #{ table } v WHERE #{ latest_condition(table, 'v', boundary) } " +
+        "AND (#{ HistoryAccess.sql(model, 'v') })"
     live ? "#{ sql } AND v.event_type <> 'discard'" : sql
   end
 
@@ -350,7 +354,7 @@ class CountsStateQuery
 
   def order_sql(expressions, default, final)
     raw = @params.fetch('sort', default)
-    sorts = raw.empty? ? [] : raw.split(',', -1).map { _1.split(':', -1) }
+    sorts = raw.empty? ? [] : QueryState.list(raw).map { _1.split(':', -1) }
     raise ArgumentError if sorts.map(&:first).uniq.length != sorts.length
     raise ArgumentError if sorts.length > expressions.length
 

backend/app/services/counts_table_export.rb

@@ -7,20 +7,20 @@ class CountsTableExport
   attr_reader :format, :resource
 
   def initialize(params)
-    raise ArgumentError if (params.keys - CountsTableQuery::KEYS - EXTRA).any?
+    QueryState.validate!(params, CountsTableQuery::KEYS + EXTRA + ['measures'])
     raise ArgumentError unless params.values.all? { _1.is_a?(String) }
 
     params = CountsDefaults.for(params).merge(params)
     unless params.key?('children')
       params = params.merge('children' => CountsExportProjection.default_children(
-          params.fetch('resource', 'posts')).join(','))
+          params.fetch('resource', 'posts')).join(' '))
     end
     @params = params
     @labels = JSON.parse(params.fetch('labels', '{}'))
     raise ArgumentError unless @labels.is_a?(Hash)
-    @selected = params.key?('children') ? params['children'].split(',') : nil
-    @order = params.fetch('order', '').split(',')
-    @child_sort = params.fetch('child_sort', '').split(',').map { _1.split(':', -1) }
+    @selected = params.key?('children') ? QueryState.list(params['children']) : nil
+    @order = QueryState.list(params.fetch('order', ''))
+    @child_sort = QueryState.list(params.fetch('child_sort', '')).map { _1.split(':', -1) }
     @child_sort.each do |path, direction, *extra|
       raise ArgumentError if extra.any? || !['asc', 'desc'].include?(direction)
       raise ArgumentError unless path&.match?(/\A[a-zA-Z_]+(?:\.[a-zA-Z]+)*\/[a-zA-Z]+\z/)
@@ -29,12 +29,13 @@ class CountsTableExport
     @resource = params.fetch('resource', 'posts')
     raise ArgumentError unless ['json', 'csv'].include?(format)
 
-    @query = CountsTableQuery.new(params.slice(*CountsTableQuery::KEYS)
+    @query = CountsTableQuery.new(params.select { |key, _value| CountsTableQuery::KEYS.include?(key) ||
+        key == 'measures' || QueryState.valid_key?(key) }
         .merge('page' => '1', 'limit' => '100'))
     @summary = @query.mode == 'summary'
     default_columns = @summary ? @query.axes.all.map { _1.fetch('field') } + ['count'] :
         CountsExportProjection.default_columns(resource)
-    @columns = params.fetch('columns', default_columns.join(',')).split(',')
+    @columns = QueryState.list(params.fetch('columns', default_columns.join(' ')))
     default_headers = @columns.map { @labels.fetch(_1, _1) }
     @headers = JSON.parse(params.fetch('headers', default_headers.to_json))
     raise ArgumentError unless @headers.is_a?(Array) && @headers.all? { _1.is_a?(String) }
@@ -66,7 +67,8 @@ class CountsTableExport
           unless started
             metadata = page.except(:versions, :cells, :axis_labels, :column_values,
                 :row_headers, :column_headers, :pivot_totals).merge(
-                resource:, filters: @params.slice(*CountsTableQuery::KEYS),
+                resource:, filters: @params.select { |key, _value| CountsTableQuery::KEYS.include?(key) ||
+        key == 'measures' || QueryState.valid_key?(key) },
                 fields: @query.population.definition.metadata,
                 columns: @columns, column_labels: @headers,
                 order: @params['order'], children: @params['children'],
@@ -138,13 +140,23 @@ class CountsTableExport
   end
 
   def total_label(header)
-    @labels.fetch(header[:kind] == 'grand_total' ? 'grandTotal' : 'subtotal', '')
+    key = header[:kind] == 'grand_total' ?
+        (@query.axes.all.length == 1 ? 'singleTotal' : 'grandTotal') : 'subtotal'
+    @labels.fetch(key, '')
   end
 
   def pivot_row_labels(header, page)
     values = header[:values].each_with_index.map { |value, index|
       summary_value(value, index, page)
     }
+    if header[:kind] == 'data' && @previous_pivot_row&.fetch(:kind) == 'data'
+      values.each_index do |index|
+        if header[:values].first(index + 1) == @previous_pivot_row[:values].first(index + 1)
+          values[index] = ''
+        end
+      end
+    end
+    @previous_pivot_row = header
     values << total_label(header) unless header[:kind] == 'data'
     leading = [@query.axes.rows.length, @query.axes.columns.any? ? 1 : 0].max
     Array.new(leading) { values[_1] || '' }

backend/app/services/counts_table_query.rb

@@ -5,11 +5,11 @@ class CountsTableQuery
 
   KEYS = ['resource', 'definition', 'mode', 'at', 'execution_at', 'from', 'through',
           'timezone', 'rows', 'cols', 'conditions', 'drill', 'tags', 'tag_refs', 'match',
-          'tag', 'kind', 'resource_id', 'user_id', 'page', 'limit', 'sort', 'locale'].freeze
+          'tag', 'kind', 'resource_id', 'user_id', 'page', 'limit', 'sort', 'locale', 'measures', 'column_order'].freeze
   attr_reader :population, :axes, :page, :limit, :mode
 
   def initialize(params)
-    raise ArgumentError if (params.keys - KEYS).any?
+    QueryState.validate!(params, KEYS + ['measures'])
     raise ArgumentError unless params.values.all? { _1.is_a?(String) }
 
     params = CountsDefaults.for(params).slice(*KEYS).merge(params)
@@ -24,14 +24,14 @@ class CountsTableQuery
     raise ArgumentError unless page.between?(1, 1_000_000) && limit.between?(1, 100)
     raise ArgumentError unless ['summary', 'details'].include?(mode)
 
-    @sort = params.fetch('sort', '').split(',').map { _1.split(':', -1) }
+    @sort = QueryState.list(params.fetch('sort', '')).map { _1.split(':', -1) }
     allowed = mode == 'summary' ? axes.all.each_index.map { "a#{ _1 }" } + ['count'] :
         @definition.fields.keys + ['is_latest'] +
         (@definition.resource == 'posts' ? ['tags', 'parents'] : []) +
         (@definition.resource == 'tags' ? ['parents', 'deprecated'] : [])
     raise ArgumentError if @sort.map(&:first).uniq.length != @sort.length
     @sort.each do |key, direction, *extra|
-      measure = mode == 'summary' && key&.match?(/\Ameasure_[0-9a-f]+\z/)
+      measure = mode == 'summary' && key&.match?(/\Am[0-9]{1,2}\z/)
       measure_values(key) if measure
       if extra.any? || (!allowed.include?(key) && !measure) || !['asc', 'desc'].include?(direction)
         raise ArgumentError
@@ -46,7 +46,8 @@ class CountsTableQuery
   end
 
   def metadata
-    { page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence,
+    { population_count_scope: 'all_recorded_versions', population_count: HistoryAccess.population_count(@definition.model),
+      page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence,
       definition: population.history ? 'history' : 'latest',
       timezone: population.timezone.identifier, rows: axes.rows, cols: axes.columns,
       fields: @definition.metadata,
@@ -76,7 +77,7 @@ class CountsTableQuery
     measure_joins = []
     extras = []
     row_sort = @sort.each_with_index.filter_map do |(key, direction), index|
-      if key == 'count' || key.start_with?('measure_')
+      if key == 'count' || key.match?(/\Am[0-9]+\z/)
         values = key == 'count' ? [] : measure_values(key)
         relation = measure_relation(values)
         matches = row_keys.map { "grouped.#{ _1 } <=> sort#{ index }.#{ _1 }" }
@@ -125,7 +126,12 @@ class CountsTableQuery
     row_values = records.map { |record| row_keys.map { record[_1] } }.uniq
     row_depths = CountsPivotLayout.depths(row_keys, @sort)
     column_keys = axes.columns.each_index.map { "a#{ axes.rows.length + _1 }" }
-    column_depths = CountsPivotLayout.depths(column_keys, @sort, measures: false)
+    column_sort = @sort
+    if @params.key?('column_order') && column_keys.any? &&
+        !(@sort.first&.first == 'count' || @sort.first&.first&.match?(/\Am[0-9]+\z/))
+      column_sort = [[column_keys.first, 'asc']] + @sort
+    end
+    column_depths = CountsPivotLayout.depths(column_keys, column_sort, measures: false)
     row_headers = CountsPivotLayout.headers(row_values, row_depths,
         next_values: next_row && row_keys.map { next_row[_1] }, grand: axes.rows.any?)
     if axes.rows.empty? && axes.columns.any?
@@ -197,6 +203,15 @@ class CountsTableQuery
     order = directions.map { |key, direction|
       "#{ key } IS NULL ASC, #{ key } #{ direction.upcase }"
     }.join(', ')
+    custom = QueryState.records(@params, 'column_order')
+    if custom.any?
+      raise ArgumentError unless custom.map { _1['value'] }.uniq.length == custom.length
+
+      cases = custom.each_with_index.map do |record, index|
+        "WHEN #{ keys.first } <=> #{ @connection.quote(record['value']) } THEN #{ index }"
+      end.join(' ')
+      order = "CASE #{ cases } ELSE #{ custom.length } END, #{ order }"
+    end
     values = @connection.select_rows(<<~SQL)
       #{ prefix }
       SELECT /*+ MAX_EXECUTION_TIME(10000) */ DISTINCT #{ keys.join(', ') } FROM cells
@@ -225,7 +240,11 @@ class CountsTableQuery
   end
 
   def measure_values(key)
-    value = JSON.parse([key.delete_prefix('measure_')].pack('H*'))
+    records = QueryState.records(@params, 'measures').select { _1['key'] == key }
+    raise ArgumentError if records.empty?
+    raise ArgumentError unless records.map { _1['axis'] }.sort == (0...records.length).to_a
+
+    value = records.sort_by { _1['axis'] }.map { _1['value'] }
     unless (value.is_a?(Array) && value.length <= axes.columns.length &&
         value.all? { _1.nil? || _1.is_a?(String) || _1.is_a?(Numeric) })
       raise ArgumentError

backend/app/services/post_tag_search.rb

@@ -1,9 +1,9 @@
 # Shared literal parsing and reference resolution for current and snapshot post search.
 class PostTagSearch
   def self.parse_references(raw, literals)
-    return nil if raw.blank?
+    return nil unless raw.key?('tag_refs')
 
-    references = JSON.parse(raw)
+    references = QueryState.records(raw, 'tag_refs')
     valid = references.is_a?(Array) && references.all? do |reference|
       next false unless reference.is_a?(Hash) && reference['name'].is_a?(String)
 

backend/app/services/query_state.rb

@@ -0,0 +1,88 @@
+# Public query grammar shared with frontend/src/lib/queryState.ts.
+class QueryState
+  SCHEMAS = {
+      'conditions' => ['field', 'operator', 'value', 'end', 'blank'],
+      'drill' => ['field', 'grain', 'value', 'value_type'],
+      'tag_refs' => ['name', 'tag_id', 'external_tag_id', 'negative', 'missing'],
+      'column_order' => ['value', 'value_type'],
+      'measures' => ['key', 'axis', 'value', 'value_type'] }.freeze
+
+  def self.list(value)
+    return [] if value.nil? || value == ''
+    raise ArgumentError unless value.is_a?(String) && value.bytesize <= 32_768
+
+    values = value.split(/ /, -1)
+    raise ArgumentError if values.length > 200 || values.any?(&:empty?) || values.uniq != values
+
+    values
+  end
+
+  def self.valid_key?(key)
+    root, index, field, extra = key.split('.', -1)
+    !extra && SCHEMAS.key?(root) && index&.match?(/\A(?:0|[1-9][0-9]?)\z/) &&
+        SCHEMAS[root].include?(field)
+  end
+
+  def self.validate!(params, allowed)
+    raise ArgumentError if params.length > 1000
+    params.each do |key, value|
+      raise ArgumentError unless allowed.include?(key) || valid_key?(key)
+      raise ArgumentError unless value.is_a?(String) && value.bytesize <= 131_072
+    end
+    SCHEMAS.each_key { records(params, _1) }
+  end
+
+  def self.records(params, key)
+    records = {}
+    params.each do |name, value|
+      next unless name.start_with?("#{ key }.")
+      raise ArgumentError unless params.key?(key) && valid_key?(name)
+
+      _root, index, field = name.split('.')
+      record = records[index.to_i] ||= {}
+      if ['blank', 'negative', 'missing'].include?(field)
+        raise ArgumentError unless ['0', '1'].include?(value)
+
+        value = value == '1'
+      elsif ['tag_id', 'external_tag_id', 'axis'].include?(field)
+        raise ArgumentError unless value.match?(/\A[0-9]{1,18}\z/)
+
+        value = value.to_i
+      end
+      record[field] = value
+    end
+    raise ArgumentError if params.key?(key) && params[key] != ''
+    raise ArgumentError unless records.keys.sort == (0...records.length).to_a
+
+    records.sort.map do |_index, record|
+      type = record.delete('value_type')
+      if ['drill', 'measures', 'column_order'].include?(key)
+        raise ArgumentError unless ['null', 'string', 'number', 'boolean'].include?(type)
+
+        raise ArgumentError unless type == 'null' || record.key?('value')
+
+        value = record['value']
+        record['value'] = case type
+                          when 'null' then nil
+                          when 'number'
+                            raise ArgumentError unless value&.match?(/\A-?[0-9]{1,18}\z/)
+                            Integer(value, 10)
+                          when 'boolean'
+                            raise ArgumentError unless ['true', 'false'].include?(value)
+                            value == 'true'
+                          else value
+                          end
+      end
+      record
+    end
+  end
+
+  def self.axes(value)
+    list(value).map do |token|
+      field, grain, extra = token.split(':', -1)
+      raise ArgumentError if extra || !field.match?(/\A[a-z_]+\z/) || grain == ''
+
+      grain ? { 'field' => field, 'grain' => grain } : { 'field' => field }
+    end
+  end
+end

backend/config/initializers/history_storage_access.rb

@@ -0,0 +1,7 @@
+Rails.application.config.to_prepare do
+  [ActiveStorage::Blobs::ProxyController, ActiveStorage::Blobs::RedirectController,
+   ActiveStorage::Representations::ProxyController,
+   ActiveStorage::Representations::RedirectController, ActiveStorage::DiskController].each do |controller|
+    controller.include(HistoryStorageAccess) unless controller < HistoryStorageAccess
+  end
+end