#434

backend/app/services/counts_table_query.rb

@@ -29,6 +29,7 @@ class CountsTableQuery
     @sort = QueryState.list(params.fetch('sort', '')).map { _1.split(':', -1) }
     allowed = mode == 'summary' ? axes.all.each_index.map { "a#{ _1 }" } + ['count'] :
         @definition.fields.keys + ['latest'] +
+        (@definition.resource == 'tags' ? @definition.name_columns.map { _1[:key] } : []) +
         (@definition.resource == 'posts' ? ['tags', 'parents'] : []) +
         (@definition.resource == 'tags' ? ['parents', 'deprecated'] : [])
     raise ArgumentError if @sort.map(&:first).uniq.length != @sort.length
@@ -49,25 +50,31 @@ class CountsTableQuery
       result.merge(visible_output_count: visible,
           **totals,
           output_count_scope: 'before_authorisation',
-          output_count_unit: 'table_body_rows',
+          output_count_unit: 'regular_result_rows',
           page_grand_total_repeated: mode == 'summary' && axes.rows.any? &&
               result[:group_count] > limit)
     end
   end
 
   def metadata
-    @population_counts ||= HistoryAccess.population_counts(@definition.model)
-    @population_counts.merge(
+    { population_count: nil, population_count_computed: false,
         excluded_count: nil, excluded_count_scope: 'current_query',
         population_count_scope: 'all_recorded_versions',
         page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence,
         definition: population.history ? 'history' : 'latest',
         timezone: population.timezone.identifier, rows: axes.rows, cols: axes.columns,
         fields: @definition.metadata,
-        count: population.scope.except(:select).optimizer_hints('MAX_EXECUTION_TIME(10000)').count)
+        count: record_counts[:visible] }
   end
 
   def summary
+    if axes.all.empty?
+      headers = [{ values: [], kind: 'data', depth: 0, axis_count: 0, label_key: nil }]
+      return { group_count: 1, visible_output_count: 1, column_values: [[]],
+          row_headers: page == 1 ? headers : [], column_headers: headers, pivot_totals: [],
+          cells: page == 1 ? [{ values: [], count: record_counts[:visible] }] : [],
+          axis_labels: {} }
+    end
     expressions = axes.expressions
     selected = expressions.each_with_index.map do |expression, index|
       "#{ expression } AS a#{ index }"
@@ -107,10 +114,6 @@ class CountsTableQuery
     grouped = "SELECT #{ row_selection } FROM cells #{ row_group }"
     selection = (['grouped.*'] + extras).join(', ')
     group_sql = "SELECT #{ selection } FROM (#{ grouped }) grouped #{ measure_joins.join(' ') }"
-    total = @connection.select_value(<<~SQL).to_i
-      #{ prefix }
-      SELECT /*+ MAX_EXECUTION_TIME(10000) */ COUNT(*) FROM (#{ group_sql }) row_groups
-    SQL
     join = row_keys.map { "cells.#{ _1 } <=> page_rows.#{ _1 }" }.join(' AND ')
     join = 'TRUE' if row_keys.empty?
     order_cols = @sort.filter_map do |key, direction|
@@ -120,22 +123,24 @@ class CountsTableQuery
     end
     order = ['page_rows.position'] + order_cols +
         axes.all.each_index.map { axis_order_sql("a#{ _1 }", qualifier: 'cells') }
-    records = @connection.select_all(<<~SQL)
+    result = @connection.select_all(<<~SQL).to_a
       #{ prefix }, row_groups AS (#{ group_sql }),
       numbered AS (SELECT row_groups.*,
         ROW_NUMBER() OVER (ORDER BY #{ order_rows.join(', ') }) AS position FROM row_groups),
-      page_rows AS (SELECT * FROM numbered
-        WHERE position > #{ (page - 1) * limit } AND position <= #{ page * limit })
-      SELECT /*+ MAX_EXECUTION_TIME(10000) */ cells.* FROM cells JOIN page_rows ON #{ join }
+      group_totals AS (SELECT COUNT(*) AS group_count FROM row_groups)
+      SELECT /*+ MAX_EXECUTION_TIME(10000) */ cells.*,
+        page_rows.position AS counts_position, group_totals.group_count AS counts_group_count
+      FROM group_totals LEFT JOIN numbered page_rows
+        ON page_rows.position > #{ (page - 1) * limit }
+        AND page_rows.position <= #{ page * limit + 1 }
+      LEFT JOIN cells ON #{ join }
       ORDER BY #{ order.join(', ') }
     SQL
-    next_row = @connection.select_one(<<~SQL)
-      #{ prefix }, row_groups AS (#{ group_sql }),
-      numbered AS (SELECT row_groups.*,
-        ROW_NUMBER() OVER (ORDER BY #{ order_rows.join(', ') }) AS position FROM row_groups)
-      SELECT /*+ MAX_EXECUTION_TIME(10000) */ * FROM numbered
-      WHERE position = #{ page * limit + 1 }
-    SQL
+    total = result.first.fetch('counts_group_count').to_i
+    next_row = result.find { _1['counts_position'].to_i == page * limit + 1 }
+    records = result.select do |record|
+      record['counts_position'] && record['counts_position'].to_i <= page * limit
+    end
     row_values = records.map { |record| row_keys.map { record[_1] } }.uniq
     row_depths = CountsPivotLayout.depths(row_keys, @sort)
     column_keys = axes.columns.each_index.map { "a#{ axes.rows.length + _1 }" }
@@ -162,11 +167,11 @@ class CountsTableQuery
     end
     totals = CountsPivotTotals.new(population, axes, row_headers, column_headers).call
     { group_count: total,
-      visible_output_count: output_row_count(prefix, row_keys, row_depths, total),
+      visible_output_count: axes.rows.empty? ? 1 : total,
       column_values:, row_headers:, column_headers:, pivot_totals: totals,
       cells: records.map do |row|
         { values: axes.all.each_index.map { row["a#{ _1 }"] }, count: row['count'].to_i }
-      end, axis_labels: axis_labels(records, column_values) }
+      end, axis_labels: axis_labels(records, column_values), axis_names: @axis_names }
   end
 
   def details
@@ -181,7 +186,7 @@ class CountsTableQuery
           population.timestamp(@params['at']) || population.at
       values = CountsVersionRepr.many(rows, resource: @definition.resource,
           language_code: locale.language_code,
-          at: point)
+          at: point, history_flg: population.history)
       values.each_with_index { |value, index| value[:event_type] = rows[index].event_type }
     else
       users = User.where(id: rows.map(&:created_by_user_id)).pluck(:id, :name).to_h
@@ -198,9 +203,10 @@ class CountsTableQuery
     { versions: values }
   end
 
-  def each_page
+  def each_page(before_page: nil)
     total = nil
     loop do
+      before_page&.call
       result = call
       total ||= mode == 'summary' ? result[:group_count] : result[:count]
       yield result
@@ -212,40 +218,50 @@ class CountsTableQuery
 
   private
 
-  def output_row_count(prefix, row_keys, depths, total)
-    # With no row axes the table has exactly one body row, including the zero-count case.
-    return 1 if row_keys.empty?
+  def unrestricted_population
+    @unrestricted_population ||= CountsPopulation.new(
+        @params.merge('execution_at' => population.at.iso8601(6)), authorisation_flg: false)
+  end
 
-    subtotals = depths.map do |depth|
-      keys = row_keys.first(depth).join(', ')
-      "(SELECT COUNT(*) FROM (SELECT #{ keys } FROM cells GROUP BY #{ keys }) subtotal_rows)"
+  def record_counts
+    return @record_counts if @record_counts
+
+    if HistoryAccess.admin?
+      total = population.scope.except(:select)
+          .optimizer_hints('MAX_EXECUTION_TIME(10000)').count
+      return @record_counts = { total:, visible: total, denied: 0 }
     end
-    expressions = [total.to_s, '1'] + subtotals
-    @connection.select_value(<<~SQL).to_i
-      #{ prefix }
-      SELECT /*+ MAX_EXECUTION_TIME(10000) */ #{ expressions.join(' + ') } AS output_rows
-    SQL
+    total, denied = unrestricted_population.scope.except(:select)
+        .optimizer_hints('MAX_EXECUTION_TIME(10000)').pick(Arel.sql('COUNT(*)'),
+            Arel.sql("COALESCE(SUM(CASE WHEN #{ HistoryAccess.sql(@definition.model, 'v') } " +
+                'THEN 0 ELSE 1 END), 0)'))
+    # Related/previous-version predicates may differ under public authority.
+    shared = QueryState.records(@params, 'conditions').empty? &&
+        QueryState.records(@params, 'drill').none? { _1['field'] == 'changed_field' }
+    visible = if shared
+                total.to_i - denied.to_i
+              else
+                population.scope.except(:select)
+                    .optimizer_hints('MAX_EXECUTION_TIME(10000)').count
+              end
+    @record_counts = { total: total.to_i, denied: denied.to_i, visible: }
   end
 
   def output_totals(visible_rows)
     return @output_totals if @output_totals
 
-    if HistoryAccess.admin? || @population_counts[:excluded_count].zero?
+    if HistoryAccess.admin?
       return @output_totals = { output_count: visible_rows, excluded_count: 0 }
     end
     # Only numerical aggregates leave this population. Labels, snapshots and
     # detail pagination continue to use the authorised population exclusively.
-    source = CountsPopulation.new(
-        @params.merge('execution_at' => population.at.iso8601(6)), authorisation_flg: false)
-    denied = source.scope.except(:select)
-        .where(Arel.sql("NOT (#{ HistoryAccess.sql(@definition.model, 'v') })"))
-        .optimizer_hints('MAX_EXECUTION_TIME(10000)').count
+    source = unrestricted_population
     rows = if mode == 'details'
-             source.scope.except(:select).optimizer_hints('MAX_EXECUTION_TIME(10000)').count
+             record_counts[:total]
            else
              summary_output_count(source)
            end
-    @output_totals = { output_count: rows, excluded_count: denied }
+    @output_totals = { output_count: rows, excluded_count: record_counts[:denied] }
   end
 
   def summary_output_count(source)
@@ -263,13 +279,12 @@ class CountsTableQuery
     sql = relation.except(:select).select(Arel.sql(selected.join(', ')))
         .distinct.to_sql
     prefix = "WITH cells AS (#{ sql })"
-    groups = @connection.select_value(<<~SQL).to_i
+    @connection.select_value(<<~SQL).to_i
       #{ prefix }
       SELECT /*+ MAX_EXECUTION_TIME(10000) */ COUNT(*) FROM (
         SELECT #{ keys.join(', ') } FROM cells GROUP BY #{ keys.join(', ') }
       ) output_groups
     SQL
-    output_row_count(prefix, keys, CountsPivotLayout.depths(keys, @sort), groups)
   end
 
   def locale = @locale ||= Locale.find_by!(code: @params.fetch('locale', 'ja'))
@@ -345,6 +360,8 @@ class CountsTableQuery
 
   def detail_expression(key)
     return "(#{ population.latest_sql })" if key == 'latest'
+    return @definition.name_expression(key) if @definition.resource == 'tags' &&
+        @definition.name_column(key)
     aliases = { 'tags' => 'tags_json', 'parents' => @definition.resource == 'posts' ?
         'parent_post_ids' : 'parent_tag_ids', 'deprecated' => 'deprecated_at' }
     field = aliases.fetch(key, key)
@@ -363,6 +380,7 @@ class CountsTableQuery
 
   def axis_labels(records, column_values)
     result = {}
+    @axis_names = {}
     axes.all.each_with_index do |axis, index|
       values = records.map { _1["a#{ index }"] }
       if index >= axes.rows.length
@@ -375,7 +393,9 @@ class CountsTableQuery
                when 'language_code', 'source_language_code'
                  Language.where(code: values).pluck(:code, :name).to_h
                when 'tags'
-                 tag_labels(values)
+                 labels = tag_labels(values)
+                 @axis_names["a#{ index }"] = @tag_names
+                 labels
                else {}
                end
       result["a#{ index }"] = labels
@@ -386,20 +406,43 @@ class CountsTableQuery
   def tag_labels(values)
     internal = values.grep(/\Ainternal:/).map { _1.split(':').last.to_i }
     external = values.grep(/\Aexternal:/).map { _1.split(':').last.to_i }
-    point = population.timestamp(@params['at']) || population.at
+    point = population.history ? population.at :
+        population.timestamp(@params['at']) || population.at
     labels = {}
+    @tag_names = {}
     sources = [[TagVersion, internal, 'internal'], [NicoTagVersion, external, 'external']]
     sources.each do |model, ids, kind|
       next if ids.empty?
 
       table = model.table_name
-      model.where(tag_id: ids).where('created_at <= ?', point)
-          .where("NOT EXISTS (SELECT 1 FROM #{ table } newer " +
-              "WHERE newer.tag_id = #{ table }.tag_id AND newer.created_at <= ? " +
-              "AND newer.version_no > #{ table }.version_no)", point).each do |row|
-        names = row.respond_to?(:names) ? row.names : nil
-        name = names&.dig(locale.language_code, 'name') || row.name
-        labels["#{ kind }:#{ row.tag_id }"] = name
+      scope = HistoryAccess.visible(model.where(tag_id: ids)).where('created_at <= ?', point)
+      bound = @connection.quote(point)
+      if population.history && @definition.resource == 'posts'
+        key = kind == 'internal' ? 'tag_id' : 'external_tag_id'
+        pairs = population.scope.except(:select, :order).joins(
+            "JOIN JSON_TABLE(v.tags_json, '$[*]' COLUMNS (tag_id BIGINT PATH '$.#{ key }', " +
+            "version_no INT PATH '$.version_no' NULL ON EMPTY)) recorded_tag")
+            .where('recorded_tag.tag_id IN (?)', ids).group('recorded_tag.tag_id')
+        value = kind == 'internal' ? 'MAX(recorded_tag.version_no)' : 'MAX(v.created_at)'
+        pairs = pairs.select(Arel.sql("recorded_tag.tag_id, #{ value } AS reference_point"))
+        scope = scope.joins("JOIN (#{ pairs.to_sql }) recorded " +
+            "ON recorded.tag_id = #{ table }.tag_id")
+        if kind == 'internal'
+          scope = scope.where("#{ table }.version_no = recorded.reference_point")
+        else
+          scope = scope.where("#{ table }.created_at <= recorded.reference_point")
+          bound = 'recorded.reference_point'
+        end
+      end
+      unless population.history && kind == 'internal' && @definition.resource == 'posts'
+        scope = scope.where("NOT EXISTS (SELECT 1 FROM #{ table } newer " +
+            "WHERE newer.tag_id = #{ table }.tag_id AND newer.created_at <= #{ bound } " +
+            "AND newer.version_no > #{ table }.version_no)")
+      end
+      scope.each do |row|
+        id = "#{ kind }:#{ row.tag_id }"
+        labels[id] = row.name
+        @tag_names[id] = row.respond_to?(:names) ? row.names : {}
       end
     end
     labels