#434

backend/app/controllers/concerns/counts_listing.rb

@@ -0,0 +1,19 @@
+module CountsListing
+  extend ActiveSupport::Concern
+
+  def index
+    locale = resolve_locale!
+    resource = is_a?(CountsPostsController) ? 'posts' : 'tags'
+    query = CountsQuery.new(version_scope, resource, request.query_parameters.except('locale')
+        .merge('locale' => locale.code))
+    result = { count: query.count, page: query.page, limit: query.limit }
+    if query.mode == 'summary'
+      result.merge!(query.summary)
+    else
+      result[:versions] = serialise_versions(query.details(version_scope), locale.language_code)
+    end
+    render json: result
+  rescue ArgumentError, TypeError
+    head :bad_request
+  end
+end

backend/app/controllers/counts_posts_controller.rb

@@ -0,0 +1,3 @@
+class CountsPostsController < PostVersionsController
+  include CountsListing
+end

backend/app/controllers/counts_tags_controller.rb

@@ -0,0 +1,3 @@
+class CountsTagsController < TagVersionsController
+  include CountsListing
+end

backend/app/controllers/post_versions_controller.rb

@@ -14,21 +14,7 @@ class PostVersionsController < ApplicationController
 
     offset = (page - 1) * limit
 
-    q = PostVersion.joins(<<~SQL.squish)
-          LEFT JOIN
-            post_versions prev
-            ON
-              prev.post_id = post_versions.post_id
-              AND prev.version_no = post_versions.version_no - 1
-        SQL
-        .select('post_versions.*',
-                'prev.title AS prev_title',
-                'prev.url AS prev_url',
-                'prev.thumbnail_base AS prev_thumbnail_base',
-                'prev.tags_json AS prev_tags_json',
-                'prev.video_ms AS prev_video_ms',
-                'prev.original_created_from AS prev_original_created_from',
-                'prev.original_created_before AS prev_original_created_before')
+    q = version_scope
     q = q.where('post_versions.post_id = ?', post_id) if post_id
     if external_tag_id || (tag_id && !(Tag.exists?(id: tag_id)))
       q = q.where('JSON_CONTAINS(post_versions.tags_json,' +
@@ -51,6 +37,24 @@ class PostVersionsController < ApplicationController
 
   private
 
+  def version_scope
+    PostVersion.joins(<<~SQL.squish)
+          LEFT JOIN
+            post_versions prev
+            ON
+              prev.post_id = post_versions.post_id
+              AND prev.version_no = post_versions.version_no - 1
+        SQL
+        .select('post_versions.*',
+                'prev.title AS prev_title',
+                'prev.url AS prev_url',
+                'prev.thumbnail_base AS prev_thumbnail_base',
+                'prev.tags_json AS prev_tags_json',
+                'prev.video_ms AS prev_video_ms',
+                'prev.original_created_from AS prev_original_created_from',
+                'prev.original_created_before AS prev_original_created_before')
+  end
+
   def serialise_versions rows, language_code
     rows = rows.to_a
     user_ids = rows.map(&:created_by_user_id).compact.uniq

backend/app/controllers/tag_versions_controller.rb

@@ -10,17 +10,7 @@ class TagVersionsController < ApplicationController
 
     offset = (page - 1) * limit
 
-    q = TagVersion.joins(<<~SQL.squish)
-          LEFT JOIN
-            tag_versions prev
-            ON
-              prev.tag_id = tag_versions.tag_id
-              AND prev.version_no = tag_versions.version_no - 1
-        SQL
-      .select('tag_versions.*', 'prev.name AS prev_name', 'prev.category AS prev_category',
-              'prev.deprecated_at AS prev_deprecated_at',
-              'prev.aliases AS prev_aliases', 'prev.names AS prev_names',
-              'prev.parent_tag_ids AS prev_parent_tag_ids')
+    q = version_scope
     q = q.where('tag_versions.tag_id = ?', tag_id) if tag_id
 
     count = q.except(:select, :order, :limit, :offset).count
@@ -34,6 +24,20 @@ class TagVersionsController < ApplicationController
 
   private
 
+  def version_scope
+    TagVersion.joins(<<~SQL.squish)
+          LEFT JOIN
+            tag_versions prev
+            ON
+              prev.tag_id = tag_versions.tag_id
+              AND prev.version_no = tag_versions.version_no - 1
+        SQL
+      .select('tag_versions.*', 'prev.name AS prev_name', 'prev.category AS prev_category',
+              'prev.deprecated_at AS prev_deprecated_at',
+              'prev.aliases AS prev_aliases', 'prev.names AS prev_names',
+              'prev.parent_tag_ids AS prev_parent_tag_ids')
+  end
+
   def serialise_versions rows, language_code
     rows = rows.to_a
     tag_ids = rows.map(&:tag_id).uniq

backend/app/services/counts_query.rb

@@ -0,0 +1,110 @@
+# Read-only history queries. All identifiers and SQL expressions are server-owned.
+class CountsQuery
+  EVENTS = ['create', 'update', 'discard', 'restore'].freeze
+  AXES = ['event_type', 'month', 'user'].freeze
+  COMMON_COLUMNS = ['created_at', 'resource_id', 'version_no', 'event_type', 'user'].freeze
+  COLUMNS = {
+      'posts' => COMMON_COLUMNS + ['title', 'url', 'tags', 'added_tags', 'removed_tags'],
+      'tags' => COMMON_COLUMNS + ['name', 'category', 'aliases', 'parents', 'deprecated'] }.freeze
+  KEYS = [
+      'locale', 'mode', 'axis', 'page', 'limit', 'from', 'before',
+      'user_id', 'event_type', 'resource_id', 'tag', 'columns'].freeze
+
+  attr_reader :page, :limit, :mode, :axis, :columns
+
+  def initialize(scope, resource, params)
+    @scope = scope.except(:select, :order)
+    @resource = resource
+    @table = scope.klass.table_name
+    @params = params
+    raise ArgumentError unless (params.keys - KEYS).empty?
+    raise ArgumentError unless params.values.all? { _1.is_a?(String) }
+
+    @mode = params.fetch('mode', 'summary')
+    @axis = params.fetch('axis', 'event_type')
+    raise ArgumentError unless ['summary', 'details'].include?(mode) && AXES.include?(axis)
+
+    @page = integer('page', 1)
+    @limit = integer('limit', 20)
+    raise ArgumentError if limit > 100 || page > 1_000_000
+
+    @columns = params.fetch('columns', COLUMNS.fetch(resource).join(',')).split(',')
+    if columns.empty? || columns.uniq != columns || (columns - COLUMNS.fetch(resource)).any?
+      raise ArgumentError
+    end
+    apply_filters
+  end
+
+  def count = @scope.count
+
+  def details(scope)
+    scope.where(id: @scope.select("#{ @table }.id"))
+        .order(Arel.sql("#{ @table }.created_at DESC, #{ @table }.id DESC"))
+        .limit(limit).offset((page - 1) * limit)
+  end
+
+  def summary
+    expression = case axis
+                 when 'event_type' then "#{ @table }.event_type"
+                 when 'user' then "#{ @table }.created_by_user_id"
+                 when 'month' then "DATE_FORMAT(#{ @table }.created_at, '%Y-%m')"
+                 end
+    grouped = @scope.group(Arel.sql(expression))
+    # A derived table counts groups without materialising every group in Ruby.
+    sql = grouped.select(Arel.sql("#{ expression } AS group_key")).to_sql
+    groups = @scope.klass.connection.select_value("SELECT COUNT(*) FROM (#{ sql }) counts_groups")
+    rows = grouped.order(Arel.sql("#{ expression } ASC"))
+        .limit(limit).offset((page - 1) * limit).count
+    names = axis == 'user' ? User.where(id: rows.keys.compact).pluck(:id, :name).to_h : {}
+    { group_count: groups.to_i,
+      groups: rows.map { |key, value| { key:, name: names[key], count: value } } }
+  end
+
+  private
+
+  def integer(key, default = nil)
+    value = @params[key].presence
+    return default if value.nil? && !default.nil?
+    raise ArgumentError unless value&.match?(/\A[1-9][0-9]{0,9}\z/)
+
+    value.to_i
+  end
+
+  def timestamp(key)
+    value = @params[key].presence
+    return nil unless value
+    raise ArgumentError unless value.match?(/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2})?Z\z/)
+
+    Date.iso8601(value.first(10))
+    Time.iso8601(value.length == 17 ? value.sub(/Z\z/, ':00Z') : value)
+  end
+
+  def apply_filters
+    first = timestamp('from')
+    last = timestamp('before')
+    raise ArgumentError if first && last && first >= last
+
+    @scope = @scope.where("#{ @table }.created_at >= ?", first) if first
+    @scope = @scope.where("#{ @table }.created_at < ?", last) if last
+    if @params['resource_id'].present?
+      id_column = @resource == 'posts' ? 'post_id' : 'tag_id'
+      @scope = @scope.where(@table => { id_column => integer('resource_id') })
+    end
+    if @params['user_id'].present?
+      user = @params['user_id'] == 'none' ? nil : integer('user_id')
+      @scope = @scope.where(@table => { created_by_user_id: user })
+    end
+    if @params['event_type'].present?
+      raise ArgumentError unless EVENTS.include?(@params['event_type'])
+
+      @scope = @scope.where(@table => { event_type: @params['event_type'] })
+    end
+    return unless @params['tag'].present?
+    raise ArgumentError unless @resource == 'posts'
+
+    id = integer('tag')
+    @scope = @scope.where(
+        "JSON_CONTAINS(post_versions.tags_json, JSON_OBJECT('tag_id', CAST(? AS UNSIGNED))) OR " +
+        "JSON_CONTAINS(prev.tags_json, JSON_OBJECT('tag_id', CAST(? AS UNSIGNED)))", id, id)
+  end
+end

backend/config/routes.rb

@@ -1,4 +1,7 @@
 Rails.application.routes.draw do
+  get 'counts/posts', to: 'counts_posts#index'
+  get 'counts/tags', to: 'counts_tags#index'
+
   resources :locales, only: [:index]
   resources :sitemap_entries, only: [:index, :show], param: :resource
 

backend/spec/requests/counts_spec.rb

@@ -0,0 +1,96 @@
+require 'rails_helper'
+
+RSpec.describe 'Counts API', type: :request do
+  let!(:actor) { create(:user, :member) }
+  let!(:post) { create(:post) }
+  let!(:tag) { create(:tag, primary_name: 'counts tag', category: :general) }
+
+  before do
+    snapshot = { tag_id: tag.id, version_no: 1, name: 'counts tag',
+                 category: 'general', sections: [] }
+    [1, 2, 3].each do |number|
+      time = Time.utc(2024, number == 3 ? 2 : 1, number, 0, 0)
+      PostVersion.create!(post:, version_no: number,
+          event_type: number == 1 ? 'create' : 'update',
+          tags: '', url: post.url, title: "Snapshot #{ number }", parent_post_ids: '',
+          tags_json: number == 1 ? [snapshot] : [],
+          created_by_user: number == 3 ? nil : actor, created_at: time)
+      TagVersion.create!(tag:, version_no: number,
+          event_type: number == 1 ? 'create' : 'update',
+          name: 'counts tag', category: 'general', aliases: '', parent_tag_ids: '',
+          names: { ja: { name: 'counts tag', script_code: 'Jpan',
+                         auto_generated: false, aliases: [] } },
+          created_by_user: number == 3 ? nil : actor, created_at: time)
+    end
+  end
+
+  ['posts', 'tags'].each do |resource|
+    context resource do
+      let(:path) { "/counts/#{ resource }" }
+      let(:id) { resource == 'posts' ? post.id : tag.id }
+      let(:filters) { { resource_id: id, user_id: actor.id,
+                       from: '2024-01-01T00:00Z', before: '2024-02-01T00:00Z' } }
+
+      it 'uses the same AND filters for SQL totals, groups and details' do
+        get path, params: filters.merge(axis: 'event_type')
+        expect(response).to have_http_status(:ok)
+        summary = response.parsed_body
+        expect(summary['count']).to eq(2)
+        expect(summary['groups'].sum { _1['count'] }).to eq(2)
+        expect(summary['groups'].map { _1['key'] }).to eq(['create', 'update'])
+
+        get path, params: filters.merge(mode: 'details')
+        expect(response.parsed_body['count']).to eq(2)
+        expect(response.parsed_body['versions'].map { _1['version_no'] }).to eq([2, 1])
+      end
+
+      it 'paginates details and groups independently from totals' do
+        get path, params: { resource_id: id, mode: 'details', limit: 1, page: 2 }
+        expect(response.parsed_body['count']).to eq(3)
+        expect(response.parsed_body['versions'].sole['version_no']).to eq(2)
+        get path, params: { resource_id: id, axis: 'month', limit: 1, page: 2 }
+        expect(response.parsed_body['group_count']).to eq(2)
+        expect(response.parsed_body['groups'].sole).to include('key' => '2024-02', 'count' => 1)
+      end
+
+      it 'groups null actors and supports their drill-down filter' do
+        get path, params: { resource_id: id, axis: 'user' }
+        expect(response.parsed_body['groups']).to include(
+            { 'key' => nil, 'name' => nil, 'count' => 1 })
+        get path, params: { resource_id: id, mode: 'details', user_id: 'none' }
+        expect(response.parsed_body['count']).to eq(1)
+      end
+
+      it 'uses an inclusive start and exclusive end' do
+        get path, params: { resource_id: id,
+                            from: '2024-01-01T00:00Z', before: '2024-01-02T00:00Z' }
+        expect(response.parsed_body['count']).to eq(1)
+      end
+
+      it 'rejects unsupported or malformed query values' do
+        [
+            { axis: 'SQL' }, { limit: 101 }, { page: '1x' }, { user_id: '-1' },
+            { event_type: 'invalid' }, { columns: 'password' }, { order: 'id' },
+            { from: '2024-02-30T00:00Z' }, { from: '2024-01-01' },
+            { user_id: ['1'] }, { mode: 'invalid' }].each do |invalid|
+          get path, params: invalid
+          expect(response).to have_http_status(:bad_request)
+        end
+      end
+
+      it 'retains the existing banned-user boundary' do
+        actor.update!(banned_at: Time.current)
+        sign_in_as(actor)
+        get path
+        expect(response).to have_http_status(:forbidden)
+      end
+    end
+  end
+
+  it 'matches removed tags from the previous snapshot, not current post_tags' do
+    get '/counts/posts', params: { tag: tag.id, mode: 'details' }
+    expect(response.parsed_body['count']).to eq(2)
+    expect(response.parsed_body['versions'].map { _1['version_no'] }).to eq([2, 1])
+    expect(response.parsed_body['versions'].first['tags'].sole['type']).to eq('removed')
+  end
+end