#434

backend/app/services/counts_state_query.rb

@@ -1,9 +1,11 @@
-# State at an exclusive UTC boundary, reconstructed only from recorded snapshots.
+# Restores one recorded version per resource before applying record filters.
 class CountsStateQuery
   KEYS = ['locale', 'resource', 'mode', 'at', 'from', 'before', 'grain',
-          'kind', 'tag', 'page', 'limit', 'sort'].freeze
+          'kind', 'tag', 'page', 'limit', 'sort', 'axis', 'user_id', 'resource_id'].freeze
   MAX_PERIODS = 366
 
+  attr_reader :at
+
   def initialize(params, language_code:)
     raise ArgumentError unless (params.keys - KEYS).empty?
     raise ArgumentError unless params.values.all? { _1.is_a?(String) }
@@ -12,14 +14,21 @@ class CountsStateQuery
     @connection = ApplicationRecord.connection
     @language = language_code
     @resource = params.fetch('resource', 'posts')
-    @mode = params.fetch('mode', 'state')
+    @mode = params.fetch('mode', 'summary')
+    @inclusive = ['summary', 'details'].include?(@mode)
+    @axis = params.fetch('axis', @resource == 'posts' ? 'tags' : 'month')
     @kind = params.fetch('kind', 'internal')
     @grain = params.fetch('grain', 'month')
     raise ArgumentError unless ['posts', 'tags'].include?(@resource)
-    raise ArgumentError unless ['state', 'trend', 'snapshot'].include?(@mode)
+    raise ArgumentError unless ['summary', 'details', 'state', 'trend', 'snapshot'].include?(@mode)
     raise ArgumentError unless ['internal', 'external', 'all'].include?(@kind)
     raise ArgumentError unless ['day', 'month', 'year'].include?(@grain)
 
+    raise ArgumentError unless ['tags', 'month', 'user'].include?(@axis)
+    if @mode == 'summary' && @axis == 'tags' && @resource != 'posts'
+      raise ArgumentError
+    end
+
     @page = integer('page', 1)
     @limit = integer('limit', 20)
     @tag = integer('tag') if params['tag'].present?
@@ -28,15 +37,19 @@ class CountsStateQuery
     raise ArgumentError if @mode == 'trend' && !@tag
 
     @now = Time.current.utc
-    @at = timestamp('at') || @now.change(usec: 0)
+    @at = timestamp('at') || @now
     raise ArgumentError if @at > @now
   end
 
-  def call
+  def call(metadata: true)
+    @metadata = metadata
     # Keep totals, rows, and detail pages on one repeatable-read database snapshot.
     ApplicationRecord.transaction do
-      @post_start = PostVersion.minimum(:created_at)
-      @tag_start = TagVersion.minimum(:created_at)
+      unless @history_loaded
+        @post_start = PostVersion.minimum(:created_at)
+        @tag_start = TagVersion.minimum(:created_at)
+        @history_loaded = true
+      end
       result = { page: @page, limit: @limit, at: @at.iso8601(6),
                  coverage: { basis: 'recorded_snapshots', exact: false,
                              post_history_from: @post_start&.iso8601(6),
@@ -44,14 +57,45 @@ class CountsStateQuery
       case @mode
       when 'trend'
         result.merge(trend)
-      when 'snapshot'
-        result.merge(totals: totals).merge(details)
+      when 'summary'
+        result.merge(metadata ? { totals: totals } : {}).merge(summary)
+      when 'details', 'snapshot'
+        result.merge(metadata ? { totals: totals } : {}).merge(details)
       else
-        result.merge(totals: totals).merge(listing)
+        result.merge(metadata ? { totals: totals } : {}).merge(listing)
       end
     end
   end
 
+  def each_page
+    return enum_for(:each_page) unless block_given?
+
+    @page = 1
+    first = call
+    yield first
+    total = ['summary', 'state', 'trend'].include?(@mode) ? first[:group_count] : first[:count]
+    while @page * @limit < total
+      @page += 1
+      yield call(metadata: false)
+    end
+  end
+
+  def validate_query!
+    filtered_scope(@resource)
+    case @mode
+    when 'summary'
+      if @axis == 'tags'
+        order_sql({ 'name' => 'name', 'count' => 'post_count', 'tag_id' => 'tag_id',
+                    'kind' => 'kind', 'deprecated' => 'deprecated_at' }, 'count:desc', [])
+      else
+        record_query('summary')
+      end
+    when 'details', 'snapshot'
+      record_query('details')
+    end
+    self
+  end
+
   private
 
   def quote(value) = @connection.quote(value)
@@ -78,10 +122,10 @@ class CountsStateQuery
   def latest_condition(table, row, boundary)
     key = table == 'post_versions' ? 'post_id' : 'tag_id'
     <<~SQL.squish
-      #{ row }.created_at < #{ boundary } AND NOT EXISTS (
+      #{ row }.created_at #{ @inclusive ? '<=' : '<' } #{ boundary } AND NOT EXISTS (
         SELECT 1 FROM #{ table } newer
         WHERE newer.#{ key } = #{ row }.#{ key }
-          AND newer.created_at < #{ boundary }
+          AND newer.created_at #{ @inclusive ? '<=' : '<' } #{ boundary }
           AND newer.version_no > #{ row }.version_no)
     SQL
   end
@@ -110,7 +154,7 @@ class CountsStateQuery
   def ctes
     boundary = quote(@at)
     <<~SQL
-      WITH post_at AS (#{ state_sql('post_versions', boundary, live: true) }),
+      WITH post_at AS (#{ filtered_scope('posts').to_sql }),
       tag_at AS (#{ state_sql('tag_versions', boundary) }),
       nico_at AS (#{ state_sql('nico_tag_versions', boundary) }),
       pairs AS (
@@ -133,7 +177,7 @@ class CountsStateQuery
     row = @connection.select_one(<<~SQL)
       #{ ctes }
       SELECT (SELECT COUNT(*) FROM post_at) AS posts,
-        (SELECT COUNT(*) FROM tag_at WHERE event_type <> 'discard') AS tags,
+        (SELECT COUNT(*) FROM (#{ filtered_scope('tags').to_sql }) counted_tags) AS tags,
         (SELECT COUNT(*) FROM tag_at WHERE event_type <> 'discard'
           AND deprecated_at IS NOT NULL) AS deprecated_tags,
         (SELECT COUNT(*) FROM pairs) AS assignments,
@@ -148,7 +192,9 @@ class CountsStateQuery
               'deprecated_tags' => available?(@tag_start, @at) ? row['deprecated_tags'] : nil)
   end
 
-  def available?(first, boundary) = first && first < boundary
+  def available?(first, boundary)
+    first && (@inclusive ? first <= boundary : first < boundary)
+  end
 
   def listing
     path = quote("$.#{ JSON.generate(@language) }.name")
@@ -176,7 +222,9 @@ class CountsStateQuery
     where = filters.any? ? "WHERE #{ filters.join(' AND ') }" : ''
     source = 'FROM catalog c LEFT JOIN tag_counts n ON n.kind = c.kind AND n.tag_id = c.tag_id'
     prefix = "#{ ctes } #{ catalog }"
-    total = @connection.select_value("#{ prefix } SELECT COUNT(*) #{ source } #{ where }").to_i
+    total = if @metadata
+              @connection.select_value("#{ prefix } SELECT COUNT(*) #{ source } #{ where }").to_i
+            end
     order = order_sql({ 'name' => 'c.name', 'count' => 'COALESCE(n.post_count, 0)',
                         'tag_id' => 'c.tag_id', 'kind' => 'c.kind',
                         'deprecated' => 'c.deprecated_at' }, 'count:desc',
@@ -195,13 +243,14 @@ class CountsStateQuery
       end }
   end
 
-  def details
-    table = @resource == 'posts' ? 'post_versions' : 'tag_versions'
-    model = @resource == 'posts' ? PostVersion : TagVersion
-    scope = model.joins("INNER JOIN (#{ state_sql(table, quote(@at), live: true) }) " +
-                        "counts_at ON counts_at.id = #{ table }.id")
+  # All record filters apply after selecting the latest recorded version at the point.
+  def restored_scope(resource)
+    model = resource == 'posts' ? PostVersion : TagVersion
+    table = model.table_name
+    scope = model.where(Arel.sql(latest_condition(table, table, quote(@at))))
+        .where.not(event_type: 'discard')
     if @tag
-      scope = if @resource == 'posts'
+      scope = if resource == 'posts'
                 scope.where("#{ membership(table) } AND #{ tag_visible(quote(@at)) }")
               elsif @kind == 'internal'
                 scope.where(tag_id: @tag)
@@ -209,12 +258,36 @@ class CountsStateQuery
                 scope.none
               end
     end
+    scope
+  end
+
+  def filtered_scope(resource)
+    params = @params.slice('from', 'before', 'user_id', 'resource_id')
+        .merge('mode' => 'details', 'latest' => '0', 'sort' => '')
+    CountsQuery.new(restored_scope(resource), resource, params,
+                    latest_expression: '1').relation
+  end
+
+  def record_query(mode)
     params = @params.slice('locale', 'sort', 'page', 'limit')
-        .merge('mode' => 'details', 'latest' => '0')
-    query = CountsQuery.new(scope, @resource, params, latest_expression: '1')
-    { count: query.count,
+        .merge('mode' => mode, 'latest' => '0', 'axis' => @axis == 'tags' ? 'month' : @axis,
+               'page' => @page.to_s, 'limit' => @limit.to_s)
+    CountsQuery.new(filtered_scope(@resource), @resource, params, latest_expression: '1')
+  end
+
+  def summary
+    return listing if @axis == 'tags'
+
+    query = record_query('summary')
+    { count: @metadata ? query.count : nil }.merge(query.summary(include_count: @metadata))
+  end
+
+  def details
+    query = record_query('details')
+    { count: @metadata ? query.count : nil,
       versions: CountsVersionRepr.many(query.details, resource: @resource,
-                                      language_code: @language) }
+                                      language_code: @language, at: @at,
+                                      inclusive: @inclusive) }
   end
 
   def next_boundary(time)