#434

backend/app/controllers/counts_tables_controller.rb

@@ -4,7 +4,7 @@ class CountsTablesController < ApplicationController
     definition = CountsResourceDefinition.new(params[:resource])
     render json: { fields: definition.metadata,
                    axes: { latest: definition.axes(false), history: definition.axes(true) },
-                   grains: CountsAxes::GRAINS }
+                   grains: CountsAxes::GRAINS, defaults: CountsDefaults.metadata(params[:resource]) }
   rescue ArgumentError
     head :bad_request
   end

backend/app/services/counts_axes.rb

@@ -40,7 +40,7 @@ class CountsAxes
   end
 
   def expressions
-    all.map do |axis|
+    @expressions ||= all.map do |axis|
       field = axis.fetch('field')
       case field
       when 'tags'
@@ -56,9 +56,9 @@ class CountsAxes
     end
   end
 
-  def joins
+  def joins(fields = all.map { _1['field'] })
     result = ''
-    if all.any? { _1['field'] == 'tags' }
+    if fields.include?('tags')
       result += <<~SQL.squish
          JOIN JSON_TABLE(CAST(v.tags_json AS JSON), '$[*]' COLUMNS (
           tag_id BIGINT PATH '$.tag_id' NULL ON EMPTY,
@@ -66,7 +66,7 @@ class CountsAxes
 
       SQL
     end
-    if all.any? { _1['field'] == 'changed_field' }
+    if fields.include?('changed_field')
       fields = @definition.changes.map do |field|
         "SELECT #{ ApplicationRecord.connection.quote(field) } AS field_key"
       end.join(' UNION ALL ')
@@ -89,6 +89,8 @@ class CountsAxes
       raise ArgumentError unless @definition.axes(@population.history).include?(axis['field'])
       if @definition.fields[axis['field']] == :datetime
         raise ArgumentError unless GRAINS.include?(axis.fetch('grain', 'month'))
+
+        axis['grain'] ||= 'month'
       elsif axis.key?('grain')
         raise ArgumentError
       end

backend/app/services/counts_defaults.rb

@@ -0,0 +1,46 @@
+# Only confirmed combinations belong here; all others retain the existing behaviour.
+class CountsDefaults
+  POST_COLUMNS = ['resource_id', 'version_no', 'is_latest', 'title', 'url',
+                  'thumbnail_base', 'tags', 'parents', 'original_created_from',
+                  'original_created_before', 'event_type', 'user', 'created_at'].freeze
+
+  def self.entries(resource)
+    result = [{ definition: 'history', mode: 'summary', params: {
+        'rows' => [{ field: 'created_at', grain: 'month' }].to_json,
+        'cols' => [{ field: 'event_type' }].to_json,
+        'sort' => 'a0:desc', 'conditions' => '[]' } }]
+    if resource == 'posts'
+      result += [{ definition: 'latest', mode: 'summary', params: {
+          'rows' => '[]', 'cols' => '[]', 'conditions' => '[]', 'tags' => '' } },
+          { definition: 'latest', mode: 'details', params: {
+              'columns' => POST_COLUMNS.join(','), 'order' => POST_COLUMNS.join(','),
+              'children' => 'tags.name,parents.id', 'nested_json' => '0',
+              'conditions' => '[]', 'tags' => '' } }]
+    end
+    result
+  end
+
+  def self.for(params)
+    settings = entries(params.fetch('resource', 'posts')).find do |entry|
+      entry[:definition] == params.fetch('definition', 'latest') &&
+          entry[:mode] == params.fetch('mode', 'summary')
+    end&.fetch(:params) || {}
+    if settings.key?('sort') && params.key?('rows')
+      actual = JSON.parse(params['rows'])
+      expected = JSON.parse(settings.fetch('rows'))
+      compatible = actual.is_a?(Array) && actual.first.is_a?(Hash) &&
+          actual.first['field'] == expected.first['field'] &&
+          actual.first.fetch('grain', 'month') == expected.first['grain']
+      unless compatible
+        settings = settings.except('sort')
+      end
+    end
+    settings
+  end
+
+  def self.metadata(resource)
+    entries(resource).map do |entry|
+      entry.merge(params: entry[:params].map { |key, value| { key:, value: } })
+    end
+  end
+end

backend/app/services/counts_export_projection.rb

@@ -6,7 +6,6 @@ class CountsExportProjection
                 'is_latest' => 'is_latest', 'thumbnail' => 'thumbnail_url',
                 'thumbnail_base' => 'thumbnail_base', 'title' => 'title', 'url' => 'url',
                 'tags' => 'tags', 'parents' => 'parents', 'video_ms' => 'video_ms',
-                'original_created_at' => 'original_created_at',
                 'original_created_from' => 'original_created_from',
                 'original_created_before' => 'original_created_before',
                 'user' => 'created_by_user', 'created_at' => 'created_at',
@@ -43,6 +42,9 @@ class CountsExportProjection
   attr_reader :columns, :headers
 
   def initialize(params, resource:, summary:, tags:)
+    if params.key?('definition')
+      params = CountsDefaults.for(params.merge('resource' => resource)).merge(params)
+    end
     @params = params
     @labels = JSON.parse(params.fetch('labels', '{}'))
     raise ArgumentError unless @labels.is_a?(Hash) && @labels.length <= 100
@@ -103,8 +105,6 @@ class CountsExportProjection
       else
         value = case key
                 when 'thumbnail' then row[:thumbnail_base]
-                when 'original_created_at'
-                  { from: row[:original_created_from], before: row[:original_created_before] }
                 else row[ROOT_KEYS.fetch(key).to_sym]
                 end
         value = nested(value || [], CHILDREN.fetch(key), key) if CHILDREN.key?(key)

backend/app/services/counts_pivot_layout.rb

@@ -0,0 +1,28 @@
+# Coordinates carry their aggregation depth; SQL NULL remains an ordinary axis value.
+class CountsPivotLayout
+  def self.depths(keys, sorts, measures: true)
+    return [] if keys.length < 2 || sorts.first&.first == 'count' ||
+        sorts.first&.first&.start_with?('measure_')
+
+    relevant = sorts.map(&:first).select { keys.include?(_1) || (measures && (_1 == 'count' ||
+        _1.start_with?('measure_'))) }
+    order = (relevant + keys).uniq
+    (1...keys.length).select { order.first(_1).sort == keys.first(_1).sort }
+  end
+
+  def self.headers(values, depths, next_values: nil, grand: false)
+    result = []
+    values.each_with_index do |tuple, index|
+      result << { values: tuple, kind: 'data' }
+      following = values[index + 1] || next_values
+      depths.reverse_each do |depth|
+        prefix = tuple.first(depth)
+        next if following && following.first(depth) == prefix
+
+        result << { values: prefix, kind: 'subtotal' }
+      end
+    end
+    result << { values: [], kind: 'grand_total' } if grand
+    result
+  end
+end

backend/app/services/counts_pivot_totals.rb

@@ -0,0 +1,59 @@
+# Re-aggregate the population, never add up already grouped cells (multi-value axes overlap).
+class CountsPivotTotals
+  def initialize(population, axes, row_headers, column_headers)
+    @population = population
+    @axes = axes
+    @rows = row_headers.group_by { _1[:values].length }
+    @columns = column_headers.group_by { _1[:values].length }
+  end
+
+  def call
+    queries = @rows.flat_map do |row_depth, headers|
+      @columns.filter_map do |column_depth, columns|
+        next if row_depth == @axes.rows.length && column_depth == @axes.columns.length
+
+        indexes = (0...row_depth).to_a +
+            (0...column_depth).map { @axes.rows.length + _1 }
+        expressions = indexes.map { @axes.expressions[_1] }
+        scope = @population.scope.except(:select, :order)
+        joins = @axes.joins(indexes.map { @axes.all[_1].fetch('field') })
+        scope = scope.joins(joins) unless joins.empty?
+        if row_depth.positive?
+          tuples = headers.map { _1[:values] }.uniq
+          predicate = tuples.map { Array.new(row_depth) { |i|
+            "(#{ expressions[i] }) <=> ?"
+          }.join(' AND ') }.map { "(#{ _1 })" }.join(' OR ')
+          scope = scope.where(predicate, *tuples.flatten)
+        end
+        if column_depth.positive?
+          tuples = columns.map { _1[:values] }.uniq
+          predicate = tuples.map { Array.new(column_depth) { |i|
+            "(#{ expressions[row_depth + i] }) <=> ?"
+          }.join(' AND ') }.map { "(#{ _1 })" }.join(' OR ')
+          scope = scope.where(predicate, *tuples.flatten)
+        end
+        grouped = expressions.each_with_index.map { |expression, index|
+          "#{ expression } AS v#{ index }"
+        }
+        # Fixed-width result coordinates, independent of NULL and of translated labels.
+        padding = (expressions.length...@axes.all.length).map { "NULL AS v#{ _1 }" }
+        fields = ["#{ row_depth } AS row_depth", "#{ column_depth } AS column_depth"] +
+            grouped + padding + ['COUNT(DISTINCT v.id) AS count']
+        scope = scope.group(*expressions.map { Arel.sql(_1) }) if expressions.any?
+        scope.optimizer_hints('MAX_EXECUTION_TIME(10000)')
+            .select(Arel.sql(fields.join(', '))).to_sql
+      end
+    end
+    return [] if queries.empty?
+
+    sql = queries.map { "(#{ _1 })" }.join(' UNION ALL ')
+    @population.definition.model.connection.select_all(
+        "SELECT /*+ MAX_EXECUTION_TIME(10000) */ * FROM (#{ sql }) totals").map do |record|
+      row_depth = record['row_depth'].to_i
+      column_depth = record['column_depth'].to_i
+      { row: (0...row_depth).map { record["v#{ _1 }"] },
+        column: (0...column_depth).map { record["v#{ row_depth + _1 }"] },
+        count: record['count'].to_i }
+    end
+  end
+end

backend/app/services/counts_query.rb

@@ -5,7 +5,7 @@ class CountsQuery
   COLUMNS = {
       'posts' => COMMON_COLUMNS + [
           'thumbnail', 'thumbnail_base', 'title', 'url', 'tags', 'parents', 'video_ms',
-          'original_created_at', 'original_created_from', 'original_created_before'],
+          'original_created_from', 'original_created_before'],
       'tags' => COMMON_COLUMNS + [
           'name', 'names', 'category', 'aliases', 'parents', 'deprecated'] }.freeze
   KEYS = [
@@ -124,8 +124,6 @@ class CountsQuery
           'title', 'url', 'thumbnail_base', 'video_ms',
           'original_created_from', 'original_created_before']
       common.merge(fields.to_h { [_1, ["#{ @table }.#{ _1 }"]] }).merge(
-          'original_created_at' => [
-              'post_versions.original_created_from', 'post_versions.original_created_before'],
           'tags' => json_order('post_versions.tags_json'),
           'parents' => list_order('post_versions.parent_post_ids'))
     else

backend/app/services/counts_table_export.rb

@@ -10,6 +10,7 @@ class CountsTableExport
     raise ArgumentError if (params.keys - CountsTableQuery::KEYS - EXTRA).any?
     raise ArgumentError unless params.values.all? { _1.is_a?(String) }
 
+    params = CountsDefaults.for(params).merge(params)
     unless params.key?('children')
       params = params.merge('children' => CountsExportProjection.default_children(
           params.fetch('resource', 'posts')).join(','))
@@ -63,7 +64,8 @@ class CountsTableExport
           total = @summary ? nil : page[:count]
           progress&.call(processed, total)
           unless started
-            metadata = page.except(:versions, :cells, :axis_labels, :column_values).merge(
+            metadata = page.except(:versions, :cells, :axis_labels, :column_values,
+                :row_headers, :column_headers, :pivot_totals).merge(
                 resource:, filters: @params.slice(*CountsTableQuery::KEYS),
                 fields: @query.population.definition.metadata,
                 columns: @columns, column_labels: @headers,
@@ -71,12 +73,35 @@ class CountsTableExport
                 sort: @params.fetch('sort', ''), child_sort: @params['child_sort'])
             output << (format == 'json' ? "{\"metadata\":#{ metadata.to_json },\"data\":[" : "\uFEFF")
             if format == 'csv'
-              headers = @summary ? summary_headers : @headers
-              output << CSV.generate_line(headers.map { safe(_1) }, row_sep: "\r\n")
+              headings = @summary ? pivot_headers(page) : [@headers]
+              headings.each do |headers|
+                output << CSV.generate_line(headers.map { safe(_1) }, row_sep: "\r\n")
+              end
             end
             started = true
           end
-          (page[:cells] || page[:versions]).each do |row|
+          if @summary
+            pivot_rows(page).each do |header|
+              records = page.fetch(:column_headers).map { pivot_cell(header, _1, page) }
+              if format == 'csv'
+                labels = pivot_row_labels(header, page)
+                values = labels + records.map { _1[:count] }
+                output << CSV.generate_line(values.map { safe(_1) }, row_sep: "\r\n")
+                processed += records.length
+                progress&.call(processed, nil)
+              else
+                records.each do |record|
+                  output << ',' unless first
+                  output << record.to_json
+                  first = false
+                  processed += 1
+                  progress&.call(processed, nil)
+                end
+              end
+            end
+            next
+          end
+          page[:versions].each do |row|
             record = project(row)
             if format == 'json'
               output << ',' unless first
@@ -106,12 +131,76 @@ class CountsTableExport
 
   private
 
-  def summary_headers
-    return @headers if @headers.any?
+  def pivot_rows(page)
+    page.fetch(:row_headers).reject do |header|
+      header[:kind] == 'grand_total' && @query.page * @query.limit < page[:group_count]
+    end
+  end
 
-    labels = @labels
-    @query.axes.all.map { labels.fetch(_1['field'], _1['field']) } +
-        [labels.fetch('count', 'count')]
+  def total_label(header)
+    @labels.fetch(header[:kind] == 'grand_total' ? 'grandTotal' : 'subtotal', '')
+  end
+
+  def pivot_row_labels(header, page)
+    values = header[:values].each_with_index.map { |value, index|
+      summary_value(value, index, page)
+    }
+    values << total_label(header) unless header[:kind] == 'data'
+    leading = [@query.axes.rows.length, @query.axes.columns.any? ? 1 : 0].max
+    Array.new(leading) { values[_1] || '' }
+  end
+
+  def pivot_headers(page)
+    row_count = @query.axes.rows.length
+    cols = @query.axes.columns
+    leading = [row_count, cols.any? ? 1 : 0].max
+    columns = page.fetch(:column_headers)
+    Array.new([cols.length, 1].max) do |level|
+      prefix = Array.new(leading) { level.zero? ? @headers[_1].to_s : '' }
+      prefix = [''] if row_count.zero? && cols.any?
+      prefix + columns.each_with_index.map do |header, index|
+        values = header[:values]
+        if cols.empty?
+          header[:kind] == 'data' ? @labels.fetch('count', '') : total_label(header)
+        elsif level == values.length && header[:kind] != 'data'
+          total_label(header)
+        elsif level < values.length
+          previous = index.positive? ? columns[index - 1][:values] : nil
+          if level < cols.length - 1 && previous && previous.first(level + 1) == values.first(level + 1)
+            ''
+          else
+            summary_value(values[level], row_count + level, page)
+          end
+        else
+          ''
+        end
+      end
+    end
+  end
+
+  def pivot_cell(row, column, page)
+    @pivot_lookup ||= {}
+    # Discard the previous page's lookup; memory remains bounded by one page and its rollups.
+    unless @pivot_page.equal?(page)
+      @pivot_page = page
+      @pivot_lookup = page.fetch(:cells).to_h do |cell|
+        values = cell[:values]
+        [[values.first(@query.axes.rows.length), values.drop(@query.axes.rows.length)], cell[:count]]
+      end
+      page.fetch(:pivot_totals).each { @pivot_lookup[[_1[:row], _1[:column]]] = _1[:count] }
+    end
+    kinds = [row[:kind], column[:kind]]
+    kind = if kinds.include?('grand_total')
+             'grand_total'
+           elsif kinds.include?('subtotal')
+             'subtotal'
+           else
+             'data'
+           end
+    { row: row[:values], column: column[:values], kind:,
+      row_kind: row[:kind], column_kind: column[:kind],
+      row_depth: row[:values].length, column_depth: column[:values].length,
+      count: @pivot_lookup.fetch([row[:values], column[:values]], 0) }
   end
 
   def summary_value(value, index, page)

backend/app/services/counts_table_query.rb

@@ -12,6 +12,7 @@ class CountsTableQuery
     raise ArgumentError if (params.keys - KEYS).any?
     raise ArgumentError unless params.values.all? { _1.is_a?(String) }
 
+    params = CountsDefaults.for(params).slice(*KEYS).merge(params)
     @params = params
     @population = CountsPopulation.new(params)
     @definition = population.definition
@@ -26,7 +27,7 @@ class CountsTableQuery
     @sort = params.fetch('sort', '').split(',').map { _1.split(':', -1) }
     allowed = mode == 'summary' ? axes.all.each_index.map { "a#{ _1 }" } + ['count'] :
         @definition.fields.keys + ['is_latest'] +
-        (@definition.resource == 'posts' ? ['tags', 'parents', 'original_created_at'] : []) +
+        (@definition.resource == 'posts' ? ['tags', 'parents'] : []) +
         (@definition.resource == 'tags' ? ['parents', 'deprecated'] : [])
     raise ArgumentError if @sort.map(&:first).uniq.length != @sort.length
     @sort.each do |key, direction, *extra|
@@ -72,25 +73,26 @@ class CountsTableQuery
     # Page whole row tuples so a pivot row is never split across display pages.
     row_selection = row_keys.empty? ? '1 AS singleton' : row_keys.join(', ')
     row_group = row_keys.empty? ? 'GROUP BY singleton' : "GROUP BY #{ row_keys.join(', ') }"
+    measure_joins = []
+    extras = []
     row_sort = @sort.each_with_index.filter_map do |(key, direction), index|
       if key == 'count' || key.start_with?('measure_')
-        condition = if key == 'count'
-                      'TRUE'
-                    else
-                      measure_values(key).each_with_index.map do |value, offset|
-                        "a#{ axes.rows.length + offset } <=> #{ @connection.quote(value) }"
-                      end.join(' AND ').presence || 'TRUE'
-                    end
-        ["MAX(CASE WHEN #{ condition } THEN count END) AS sort#{ index }",
-         "sort#{ index } IS NULL ASC, sort#{ index } #{ direction.upcase }"]
+        values = key == 'count' ? [] : measure_values(key)
+        relation = measure_relation(values)
+        matches = row_keys.map { "grouped.#{ _1 } <=> sort#{ index }.#{ _1 }" }
+        condition = matches.empty? ? 'TRUE' : matches.join(' AND ')
+        measure_joins << "LEFT JOIN (#{ relation }) sort#{ index } ON #{ condition }"
+        extras << "sort#{ index }.count AS sort#{ index }"
+        "sort#{ index } IS NULL ASC, sort#{ index } #{ direction.upcase }"
       elsif row_keys.include?(key)
-        [nil, "#{ key } IS NULL ASC, #{ key } #{ direction.upcase }"]
+        "#{ key } IS NULL ASC, #{ key } #{ direction.upcase }"
       end
     end
-    extras = row_sort.filter_map(&:first)
-    order_rows = row_sort.map(&:last) + row_keys.map { "#{ _1 } IS NULL, #{ _1 }" }
+    order_rows = row_sort + row_keys.map { "#{ _1 } IS NULL, #{ _1 }" }
     order_rows = ['singleton ASC'] if order_rows.empty?
-    group_sql = "SELECT #{ ([row_selection] + extras).join(', ') } FROM cells #{ row_group }"
+    grouped = "SELECT #{ row_selection } FROM cells #{ row_group }"
+    selection = (['grouped.*'] + extras).join(', ')
+    group_sql = "SELECT #{ selection } FROM (#{ grouped }) grouped #{ measure_joins.join(' ') }"
     total = @connection.select_value(<<~SQL).to_i
       #{ prefix }
       SELECT /*+ MAX_EXECUTION_TIME(10000) */ COUNT(*) FROM (#{ group_sql }) row_groups
@@ -113,7 +115,30 @@ class CountsTableQuery
       SELECT /*+ MAX_EXECUTION_TIME(10000) */ cells.* FROM cells JOIN page_rows ON #{ join }
       ORDER BY #{ order.join(', ') }
     SQL
-    { group_count: total, column_values:, cells: records.map do |row|
+    next_row = @connection.select_one(<<~SQL)
+      #{ prefix }, row_groups AS (#{ group_sql }),
+      numbered AS (SELECT row_groups.*,
+        ROW_NUMBER() OVER (ORDER BY #{ order_rows.join(', ') }) AS position FROM row_groups)
+      SELECT /*+ MAX_EXECUTION_TIME(10000) */ * FROM numbered
+      WHERE position = #{ page * limit + 1 }
+    SQL
+    row_values = records.map { |record| row_keys.map { record[_1] } }.uniq
+    row_depths = CountsPivotLayout.depths(row_keys, @sort)
+    column_keys = axes.columns.each_index.map { "a#{ axes.rows.length + _1 }" }
+    column_depths = CountsPivotLayout.depths(column_keys, @sort, measures: false)
+    row_headers = CountsPivotLayout.headers(row_values, row_depths,
+        next_values: next_row && row_keys.map { next_row[_1] }, grand: axes.rows.any?)
+    if axes.rows.empty? && axes.columns.any?
+      row_headers = [{ values: [], kind: 'grand_total' }]
+    end
+    column_headers = CountsPivotLayout.headers(column_values, column_depths,
+        grand: axes.columns.any?)
+    if axes.columns.empty? && axes.rows.any?
+      column_headers = [{ values: [], kind: 'grand_total' }]
+    end
+    totals = CountsPivotTotals.new(population, axes, row_headers, column_headers).call
+    { group_count: total, column_values:, row_headers:, column_headers:, pivot_totals: totals,
+      cells: records.map do |row|
         { values: axes.all.each_index.map { row["a#{ _1 }"] }, count: row['count'].to_i }
       end, axis_labels: axis_labels(records, column_values) }
   end
@@ -167,7 +192,11 @@ class CountsTableQuery
     keys = axes.columns.each_index.map { "a#{ axes.rows.length + _1 }" }
     return [[]] if keys.empty?
 
-    order = keys.map { "#{ _1 } IS NULL, #{ _1 }" }.join(', ')
+    configured = @sort.select { |key, _direction| keys.include?(key) }
+    directions = (configured + keys.map { [_1, 'asc'] }).uniq { _1.first }
+    order = directions.map { |key, direction|
+      "#{ key } IS NULL ASC, #{ key } #{ direction.upcase }"
+    }.join(', ')
     values = @connection.select_rows(<<~SQL)
       #{ prefix }
       SELECT /*+ MAX_EXECUTION_TIME(10000) */ DISTINCT #{ keys.join(', ') } FROM cells
@@ -178,9 +207,26 @@ class CountsTableQuery
     values
   end
 
+  def measure_relation(values)
+    row_expressions = axes.expressions.first(axes.rows.length)
+    source = population.scope.except(:select, :order)
+    fields = axes.rows + axes.columns.first(values.length)
+    joins = axes.joins(fields.map { _1.fetch('field') })
+    source = source.joins(joins) unless joins.empty?
+    values.each_with_index do |value, index|
+      expression = axes.expressions[axes.rows.length + index]
+      source = source.where("(#{ expression }) <=> ?", value)
+    end
+    selected = row_expressions.each_with_index.map { |expression, index|
+      "#{ expression } AS a#{ index }"
+    } + ['COUNT(DISTINCT v.id) AS count']
+    source = source.group(*row_expressions.map { Arel.sql(_1) }) if row_expressions.any?
+    source.select(Arel.sql(selected.join(', '))).to_sql
+  end
+
   def measure_values(key)
     value = JSON.parse([key.delete_prefix('measure_')].pack('H*'))
-    unless (value.is_a?(Array) && value.length == axes.columns.length &&
+    unless (value.is_a?(Array) && value.length <= axes.columns.length &&
         value.all? { _1.nil? || _1.is_a?(String) || _1.is_a?(Numeric) })
       raise ArgumentError
     end
@@ -190,8 +236,7 @@ class CountsTableQuery
   def detail_expression(key)
     return "(#{ population.latest_sql })" if key == 'is_latest'
     aliases = { 'tags' => 'tags_json', 'parents' => @definition.resource == 'posts' ?
-        'parent_post_ids' : 'parent_tag_ids', 'deprecated' => 'deprecated_at',
-                'original_created_at' => 'original_created_from' }
+        'parent_post_ids' : 'parent_tag_ids', 'deprecated' => 'deprecated_at' }
     field = aliases.fetch(key, key)
     expression = @definition.expression(field)
     return expression unless @definition.fields[field] == :json