#434

backend/app/controllers/concerns/counts_listing.rb

@@ -10,7 +10,8 @@ module CountsListing
     if query.mode == 'summary'
       result.merge!(query.summary)
     else
-      result[:versions] = serialise_versions(query.details(version_scope), locale.language_code)
+      result[:versions] = CountsVersionRepr.many(
+          query.details, resource:, language_code: locale.language_code)
     end
     render json: result
   rescue ArgumentError, TypeError

backend/app/representations/counts_version_repr.rb

@@ -0,0 +1,97 @@
+# Counts uses recorded values only; history/diff endpoints retain their own contract.
+class CountsVersionRepr
+  def self.many(rows, resource:, language_code:)
+    new(rows.to_a, resource:, language_code:).serialise
+  end
+
+  def initialize(rows, resource:, language_code:)
+    @rows = rows
+    @resource = resource
+    @language = language_code
+    @users = User.where(id: rows.map(&:created_by_user_id).compact).pluck(:id, :name).to_h
+    @languages = master_names(Language)
+    @scripts = master_names(Script)
+    @tag_versions = {}
+    load_tag_versions if resource == 'posts'
+  end
+
+  def serialise
+    @rows.map do |row|
+      resource_id = row.public_send(@resource == 'posts' ? :post_id : :tag_id)
+      common = { id: row.id, resource_id:,
+                 version_no: row.version_no, is_latest: row.attributes['counts_latest'].to_i == 1,
+                 created_at: row.created_at.iso8601,
+                 created_by_user: row.created_by_user_id &&
+                     { id: row.created_by_user_id, name: @users[row.created_by_user_id] } }
+      common.merge(@resource == 'posts' ? post(row) : tag(row))
+    end
+  end
+
+  private
+
+  def master_names(model)
+    model.select('code AS counts_code', :name).to_h do |record|
+      [record.attributes['counts_code'], record.name]
+    end
+  end
+
+  def json(value)
+    value.is_a?(String) ? JSON.parse(value) : value
+  end
+
+  def load_tag_versions
+    pairs = @rows.flat_map { Array(json(_1.tags_json)) }
+        .filter_map { |entry| [entry['tag_id'], entry['version_no']] if entry['tag_id'] }
+        .uniq
+    pairs.each_slice(500) do |batch|
+      placeholders = batch.map { '(?, ?)' }.join(', ')
+      TagVersion.where(["(tag_id, version_no) IN (#{ placeholders })", *batch.flatten])
+          .select(:tag_id, :version_no, :name, :category, :names).each do |version|
+        @tag_versions[[version.tag_id, version.version_no]] = version
+      end
+    end
+  end
+
+  def names(value)
+    (json(value) || {}).sort.map do |language, entry|
+      aliases = entry.fetch('aliases', []).map do |alias_entry|
+        alias_entry.merge('script_name' => @scripts[alias_entry['script_code']])
+      end
+      { language_code: language, language_name: @languages[language],
+        script_code: entry['script_code'], script_name: @scripts[entry['script_code']],
+        name: entry['name'], auto_generated: entry['auto_generated'], aliases: }
+    end
+  end
+
+  def post(row)
+    snapshots = Array(json(row.tags_json)).map do |entry|
+      version = @tag_versions[[entry['tag_id'], entry['version_no']]]
+      { id: entry['tag_id'] || entry['external_tag_id'],
+        kind: entry['tag_id'] ? 'internal' : 'external',
+        tag_id: entry['tag_id'], external_tag_id: entry['external_tag_id'],
+        version_no: entry['version_no'], name: entry['name'] || version&.name,
+        category: entry['category'] || version&.category,
+        sections: entry.fetch('sections', []), names: version ? names(version.names) : [] }
+    end
+    { title: row.title, url: row.url, thumbnail_base: row.thumbnail_base,
+      tags: snapshots, parents: row.parent_post_ids.to_s.split.map { { id: _1.to_i } },
+      video_ms: row.video_ms,
+      original_created_from: row.original_created_from&.iso8601,
+      original_created_before: row.original_created_before&.iso8601 }
+  end
+
+  def tag(row)
+    recorded_names = names(row.names)
+    selected = recorded_names.find { _1[:language_code] == @language }
+    aliases = recorded_names.flat_map do |entry|
+      entry[:aliases].map {
+        _1.merge('language_code' => entry[:language_code],
+                 'language_name' => entry[:language_name])
+      }
+    end
+    { name: selected&.fetch(:name) || row.name, names: recorded_names,
+      category: row.category, aliases:,
+      parents: row.parent_tag_ids.to_s.split.map { { id: _1.to_i } },
+      deprecated: row.deprecated_at&.iso8601 }
+  end
+end

backend/app/services/counts_query.rb

@@ -1,59 +1,71 @@
-# Read-only history queries. All identifiers and SQL expressions are server-owned.
+# Read-only history queries. SQL identifiers and expressions are server-owned.
 class CountsQuery
-  EVENTS = ['create', 'update', 'discard', 'restore'].freeze
-  AXES = ['event_type', 'month', 'user'].freeze
-  COMMON_COLUMNS = ['created_at', 'resource_id', 'version_no', 'event_type', 'user'].freeze
+  AXES = ['month', 'user'].freeze
+  COMMON_COLUMNS = ['resource_id', 'version_no', 'is_latest', 'user', 'created_at'].freeze
   COLUMNS = {
-      'posts' => COMMON_COLUMNS + ['title', 'url', 'tags', 'added_tags', 'removed_tags'],
-      'tags' => COMMON_COLUMNS + ['name', 'category', 'aliases', 'parents', 'deprecated'] }.freeze
+      'posts' => COMMON_COLUMNS + [
+          'thumbnail', 'thumbnail_base', 'title', 'url', 'tags', 'parents', 'video_ms',
+          'original_created_at', 'original_created_from', 'original_created_before'],
+      'tags' => COMMON_COLUMNS + [
+          'name', 'names', 'category', 'aliases', 'parents', 'deprecated'] }.freeze
   KEYS = [
-      'locale', 'mode', 'axis', 'page', 'limit', 'from', 'before',
-      'user_id', 'event_type', 'resource_id', 'tag', 'columns'].freeze
+      'locale', 'mode', 'axis', 'page', 'limit', 'from', 'before', 'latest',
+      'user_id', 'resource_id', 'tag', 'columns', 'sort'].freeze
 
-  attr_reader :page, :limit, :mode, :axis, :columns
+  attr_reader :page, :limit, :mode, :axis
 
   def initialize(scope, resource, params)
-    @scope = scope.except(:select, :order)
-    @resource = resource
     @table = scope.klass.table_name
+    @resource = resource
+    @id_column = resource == 'posts' ? 'post_id' : 'tag_id'
     @params = params
+    @scope = scope.except(:select, :order).joins(
+        "LEFT JOIN users counts_actor ON counts_actor.id = #{ @table }.created_by_user_id")
     raise ArgumentError unless (params.keys - KEYS).empty?
     raise ArgumentError unless params.values.all? { _1.is_a?(String) }
 
     @mode = params.fetch('mode', 'summary')
-    @axis = params.fetch('axis', 'event_type')
+    @axis = params.fetch('axis', 'month')
     raise ArgumentError unless ['summary', 'details'].include?(mode) && AXES.include?(axis)
 
     @page = integer('page', 1)
     @limit = integer('limit', 20)
     raise ArgumentError if limit > 100 || page > 1_000_000
 
-    @columns = params.fetch('columns', COLUMNS.fetch(resource).join(',')).split(',')
+    columns = params.fetch('columns', COLUMNS.fetch(resource).join(',')).split(',')
     if columns.empty? || columns.uniq != columns || (columns - COLUMNS.fetch(resource)).any?
       raise ArgumentError
     end
+    @sort = parse_sort
     apply_filters
   end
 
   def count = @scope.count
 
-  def details(scope)
-    scope.where(id: @scope.select("#{ @table }.id"))
-        .order(Arel.sql("#{ @table }.created_at DESC, #{ @table }.id DESC"))
+  def latest_expression
+    <<~SQL.squish
+      NOT EXISTS (SELECT 1 FROM #{ @table } newer
+        WHERE newer.#{ @id_column } = #{ @table }.#{ @id_column }
+          AND newer.version_no > #{ @table }.version_no)
+    SQL
+  end
+
+  def details
+    @scope.select("#{ @table }.*", Arel.sql("(#{ latest_expression }) AS counts_latest"))
+        .order(Arel.sql(order_sql(detail_expressions, ["#{ @table }.id DESC"])))
         .limit(limit).offset((page - 1) * limit)
   end
 
   def summary
-    expression = case axis
-                 when 'event_type' then "#{ @table }.event_type"
-                 when 'user' then "#{ @table }.created_by_user_id"
-                 when 'month' then "DATE_FORMAT(#{ @table }.created_at, '%Y-%m')"
-                 end
+    expression = axis == 'month' ? "DATE_FORMAT(#{ @table }.created_at, '%Y-%m')" :
+                                  "#{ @table }.created_by_user_id"
     grouped = @scope.group(Arel.sql(expression))
-    # A derived table counts groups without materialising every group in Ruby.
     sql = grouped.select(Arel.sql("#{ expression } AS group_key")).to_sql
     groups = @scope.klass.connection.select_value("SELECT COUNT(*) FROM (#{ sql }) counts_groups")
-    rows = grouped.order(Arel.sql("#{ expression } ASC"))
+    group_order = axis == 'user' ? 'MAX(counts_actor.name)' : expression
+    expressions = { 'group' => [group_order], 'count' => ['COUNT(*)'] }
+    final = ["(#{ expression }) IS NULL ASC", "#{ expression } ASC"]
+    rows = grouped.order(Arel.sql(order_sql(expressions, final)))
         .limit(limit).offset((page - 1) * limit).count
     names = axis == 'user' ? User.where(id: rows.keys.compact).pluck(:id, :name).to_h : {}
     { group_count: groups.to_i,
@@ -62,6 +74,79 @@ class CountsQuery
 
   private
 
+  def parse_sort
+    value = @params.fetch('sort', mode == 'details' ? 'created_at:desc' : 'group:asc')
+    return [] if value.empty?
+
+    values = value.split(',', -1).map { _1.split(':', -1) }
+    allowed = mode == 'details' ? COLUMNS.fetch(@resource) - ['thumbnail'] : ['group', 'count']
+    invalid = values.any? do |key, direction, *extra|
+      extra.any? || !allowed.include?(key) || !['asc', 'desc'].include?(direction)
+    end
+    if (values.empty? || values.length > allowed.length || invalid ||
+        values.map(&:first).uniq.length != values.length)
+      raise ArgumentError
+    end
+    values
+  end
+
+  # NULL always sorts last, including descending keys. The final key is unique.
+  def order_sql(expressions, final)
+    (@sort.flat_map do |key, direction|
+      expressions.fetch(key).flat_map do |expression|
+        ["(#{ expression }) IS NULL ASC", "#{ expression } #{ direction.upcase }"]
+      end
+    end + final).join(', ')
+  end
+
+  def detail_expressions
+    common = { 'resource_id' => ["#{ @table }.#{ @id_column }"],
+               'version_no' => ["#{ @table }.version_no"],
+               'is_latest' => ["(#{ latest_expression })"],
+               'user' => ['counts_actor.name', "#{ @table }.created_by_user_id"],
+               'created_at' => ["#{ @table }.created_at"] }
+    if @resource == 'posts'
+      fields = [
+          'title', 'url', 'thumbnail_base', 'video_ms',
+          'original_created_from', 'original_created_before']
+      common.merge(fields.to_h { [_1, ["#{ @table }.#{ _1 }"]] }).merge(
+          'original_created_at' => [
+              'post_versions.original_created_from', 'post_versions.original_created_before'],
+          'tags' => json_order('post_versions.tags_json'),
+          'parents' => list_order('post_versions.parent_post_ids'))
+    else
+      name = "JSON_UNQUOTE(JSON_EXTRACT(tag_versions.names, #{ locale_name_path }))"
+      common.merge(
+          'name' => ["COALESCE(#{ name }, tag_versions.name)"],
+          'names' => json_order('tag_versions.names'),
+          'category' => ['tag_versions.category'],
+          'aliases' => json_order(
+              "JSON_EXTRACT(tag_versions.names, '$.*.aliases[*]')"),
+          'parents' => list_order('tag_versions.parent_tag_ids'),
+          'deprecated' => ['tag_versions.deprecated_at'])
+    end
+  end
+
+  def locale_name_path
+    language = Locale.find_by(code: @params.fetch('locale', 'ja'))&.language_code || 'ja'
+    path = "$.#{ JSON.generate(language) }.name"
+    @scope.klass.connection.quote(path)
+  end
+
+  # Multi-valued parent cells sort by cardinality, then binary snapshot text.
+  def json_order(column)
+    [
+        "COALESCE(JSON_LENGTH(#{ column }), 0)",
+        "COALESCE(CAST(#{ column } AS CHAR), '[]') COLLATE utf8mb4_bin"]
+  end
+
+  def list_order(column)
+    [
+        "CASE WHEN #{ column } = '' THEN 0 ELSE " +
+            "1 + LENGTH(#{ column }) - LENGTH(REPLACE(#{ column }, ' ', '')) END",
+        "#{ column } COLLATE utf8mb4_bin"]
+  end
+
   def integer(key, default = nil)
     value = @params[key].presence
     return default if value.nil? && !default.nil?
@@ -83,22 +168,18 @@ class CountsQuery
     first = timestamp('from')
     last = timestamp('before')
     raise ArgumentError if first && last && first >= last
+    raise ArgumentError unless ['0', '1'].include?(@params.fetch('latest', '0'))
 
+    @scope = @scope.where(Arel.sql(latest_expression)) if @params['latest'] == '1'
     @scope = @scope.where("#{ @table }.created_at >= ?", first) if first
     @scope = @scope.where("#{ @table }.created_at < ?", last) if last
     if @params['resource_id'].present?
-      id_column = @resource == 'posts' ? 'post_id' : 'tag_id'
-      @scope = @scope.where(@table => { id_column => integer('resource_id') })
+      @scope = @scope.where(@table => { @id_column => integer('resource_id') })
     end
     if @params['user_id'].present?
       user = @params['user_id'] == 'none' ? nil : integer('user_id')
       @scope = @scope.where(@table => { created_by_user_id: user })
     end
-    if @params['event_type'].present?
-      raise ArgumentError unless EVENTS.include?(@params['event_type'])
-
-      @scope = @scope.where(@table => { event_type: @params['event_type'] })
-    end
     return unless @params['tag'].present?
     raise ArgumentError unless @resource == 'posts'