diff --git a/backend/app/controllers/concerns/counts_listing.rb b/backend/app/controllers/concerns/counts_listing.rb index a1a4769..bbaeb2e 100644 --- a/backend/app/controllers/concerns/counts_listing.rb +++ b/backend/app/controllers/concerns/counts_listing.rb @@ -6,7 +6,8 @@ module CountsListing resource = is_a?(CountsPostsController) ? 'posts' : 'tags' query = CountsQuery.new(version_scope, resource, request.query_parameters.except('locale') .merge('locale' => locale.code)) - result = { count: query.count, page: query.page, limit: query.limit } + result = { count: query.count, page: query.page, limit: query.limit, + at: request.query_parameters['at'].presence || Time.current.utc.iso8601(6) } if query.mode == 'summary' result.merge!(query.summary) else diff --git a/backend/app/controllers/counts_exports_controller.rb b/backend/app/controllers/counts_exports_controller.rb new file mode 100644 index 0000000..cb4d9de --- /dev/null +++ b/backend/app/controllers/counts_exports_controller.rb @@ -0,0 +1,20 @@ +class CountsExportsController < ApplicationController + wrap_parameters false + + def index + locale = resolve_locale! + exporter = CountsExporter.new(request.request_parameters.merge(request.query_parameters), + language_code: locale.language_code) + response.headers['Content-Type'] = exporter.format == 'json' ? + 'application/json; charset=utf-8' : 'text/csv; charset=utf-8' + response.headers['Content-Disposition'] = + "attachment; filename=\"counts-#{ exporter.resource }.#{ exporter.format }\"" + response.headers['Cache-Control'] = 'private, no-store' + response.headers['X-Accel-Buffering'] = 'no' + # Rack::ETag otherwise materialises the response to calculate a digest. + response.headers['Last-Modified'] = Time.current.httpdate + self.response_body = exporter.body + rescue ArgumentError, TypeError, JSON::ParserError + head :bad_request + end +end diff --git a/backend/app/controllers/post_versions_controller.rb b/backend/app/controllers/post_versions_controller.rb index e37bdb5..23cd083 100644 --- a/backend/app/controllers/post_versions_controller.rb +++ b/backend/app/controllers/post_versions_controller.rb @@ -38,13 +38,7 @@ class PostVersionsController < ApplicationController private def version_scope - PostVersion.joins(<<~SQL.squish) - LEFT JOIN - post_versions prev - ON - prev.post_id = post_versions.post_id - AND prev.version_no = post_versions.version_no - 1 - SQL + PostVersion.with_previous_version .select('post_versions.*', 'prev.title AS prev_title', 'prev.url AS prev_url', diff --git a/backend/app/models/post_version.rb b/backend/app/models/post_version.rb index 8fe52bb..745505d 100644 --- a/backend/app/models/post_version.rb +++ b/backend/app/models/post_version.rb @@ -1,6 +1,11 @@ class PostVersion < ApplicationRecord include VersionRecord + scope :with_previous_version, -> { + joins('LEFT JOIN post_versions prev ON prev.post_id = post_versions.post_id ' + + 'AND prev.version_no = post_versions.version_no - 1') + } + belongs_to :post belongs_to :parent, class_name: 'Post', optional: true diff --git a/backend/app/representations/counts_version_repr.rb b/backend/app/representations/counts_version_repr.rb index e3a7c5b..0d8b44d 100644 --- a/backend/app/representations/counts_version_repr.rb +++ b/backend/app/representations/counts_version_repr.rb @@ -1,11 +1,13 @@ # Counts uses recorded values only; history/diff endpoints retain their own contract. class CountsVersionRepr - def self.many(rows, resource:, language_code:) - new(rows.to_a, resource:, language_code:).serialise + def self.many(rows, resource:, language_code:, at: nil, inclusive: true) + new(rows.to_a, resource:, language_code:, at:, inclusive:).serialise end - def initialize(rows, resource:, language_code:) + def initialize(rows, resource:, language_code:, at:, inclusive:) @rows = rows + @at = at + @inclusive = inclusive @resource = resource @language = language_code @users = User.where(id: rows.map(&:created_by_user_id).compact).pluck(:id, :name).to_h @@ -45,7 +47,11 @@ class CountsVersionRepr .uniq pairs.each_slice(500) do |batch| placeholders = batch.map { '(?, ?)' }.join(', ') - TagVersion.where(["(tag_id, version_no) IN (#{ placeholders })", *batch.flatten]) + scope = TagVersion.all + if @at + scope = scope.where("created_at #{ @inclusive ? '<=' : '<' } ?", @at) + end + scope.where(["(tag_id, version_no) IN (#{ placeholders })", *batch.flatten]) .select(:tag_id, :version_no, :name, :category, :names).each do |version| @tag_versions[[version.tag_id, version.version_no]] = version end diff --git a/backend/app/services/counts_export_projection.rb b/backend/app/services/counts_export_projection.rb new file mode 100644 index 0000000..6ab6370 --- /dev/null +++ b/backend/app/services/counts_export_projection.rb @@ -0,0 +1,229 @@ +require 'set' + +# Export identifiers are stable; UI translations are supplied separately as column labels. +class CountsExportProjection + ROOT_KEYS = { 'resource_id' => 'resource_id', 'version_no' => 'version_no', + 'is_latest' => 'is_latest', 'thumbnail' => 'thumbnail_url', + 'thumbnail_base' => 'thumbnail_base', 'title' => 'title', 'url' => 'url', + 'tags' => 'tags', 'parents' => 'parents', 'video_ms' => 'video_ms', + 'original_created_at' => 'original_created_at', + 'original_created_from' => 'original_created_from', + 'original_created_before' => 'original_created_before', + 'user' => 'created_by_user', 'created_at' => 'created_at', + 'name' => 'name', 'names' => 'names', 'category' => 'category', + 'aliases' => 'aliases', 'deprecated' => 'deprecated' }.freeze + CHILD_KEYS = { 'id' => 'id', 'versionNo' => 'version_no', 'kind' => 'kind', + 'name' => 'name', 'category' => 'category', 'sections' => 'sections', + 'names' => 'names', 'aliases' => 'aliases', + 'languageCode' => 'language', 'scriptCode' => 'script', + 'autoGenerated' => 'auto_generated' }.freeze + NAME_FIELDS = ['languageCode', 'scriptCode', 'name', 'aliases', 'autoGenerated'].freeze + CHILDREN = { + 'tags' => ['id', 'versionNo', 'kind', 'name', 'category', 'sections', 'names'], + 'names' => NAME_FIELDS, + 'aliases' => ['languageCode', 'name', 'scriptCode'], + 'parents' => ['id'] }.freeze + + attr_reader :columns, :headers + + def initialize(params, resource:, summary:, tags:) + @params = params + @labels = JSON.parse(params.fetch('labels', '{}')) + raise ArgumentError unless @labels.is_a?(Hash) && @labels.length <= 100 + raise ArgumentError unless @labels.all? { |key, value| valid_label?(key, value) } + @summary = summary + allowed = if summary + tags ? ['kind', 'tag_id', 'name', 'deprecated', 'count'] : ['group', 'count'] + else + CountsQuery::COLUMNS.fetch(resource) + end + @columns = params.fetch('columns', allowed.join(',')).split(',') + raise ArgumentError if (@columns - allowed).any? || @columns.uniq != @columns + + @headers = JSON.parse(params.fetch('headers', JSON.generate(@columns))) + unless (@headers.is_a?(Array) && @headers.length == @columns.length && + @headers.all? { _1.is_a?(String) && _1.length <= 200 }) + raise ArgumentError + end + @order = params.fetch('order', '').split(',') + @sorts = params.fetch('child_sort', '').split(',').map do |entry| + parts = entry.split(':', -1) + raise ArgumentError unless parts.length == 2 + + path, direction = parts + raise ArgumentError unless ['asc', 'desc'].include?(direction) + + [path, direction] + end + @selected = params.key?('children') ? params['children'].split(',').to_set : nil + valid = CHILDREN.flat_map do |key, fields| + paths(fields, key) + end + raise ArgumentError if @selected && (@selected.to_a - valid).any? + allowed_sorts = valid.map do |path| + parts = path.split('.') + field = parts.pop + "#{ parts.join('.') }/#{ field }" + end + if ((@sorts.map(&:first) - allowed_sorts).any? || + @sorts.map(&:first).uniq.length != @sorts.length) + raise ArgumentError + end + end + + def metadata + @columns.each_with_index.map do |key, index| + { key: @summary ? key : ROOT_KEYS.fetch(key), label: @headers[index] } + end + end + + def project(row) + @columns.to_h do |key| + if @summary + value = key == 'group' ? { key: row[:key], name: row[:name] } : row[key.to_sym] + [key, value] + else + value = case key + when 'thumbnail' then row[:thumbnail_base] + when 'original_created_at' + { from: row[:original_created_from], before: row[:original_created_before] } + else row[ROOT_KEYS.fetch(key).to_sym] + end + value = nested(value || [], CHILDREN.fetch(key), key) if CHILDREN.key?(key) + [ROOT_KEYS.fetch(key), value] + end + end + end + + def csv_record(record, row) + record.map do |key, value| + key == 'group' ? row[:name] || row[:key] : csv_display(value, key) + end + end + + private + + def valid_label?(key, value) + key.is_a?(String) && value.is_a?(String) && key.length <= 100 && value.length <= 300 + end + + def csv_display(value, key) + return value ? @labels.fetch('latest', 'true') : '' if key == 'is_latest' + return @labels.fetch("kind:#{ value }", value) if key == 'kind' + return @labels.fetch("category:#{ value }", value) if key == 'category' + if ['language', 'script'].include?(key) && value.is_a?(Hash) + return @labels.fetch("#{ key }:#{ value[:code] }", + value[:name] || @labels.fetch('notRegistered', '')) + end + return @labels.fetch("boolean:#{ value }", value) if value == true || value == false + return value.map { csv_display(_1, key) } if value.is_a?(Array) + if value.is_a?(Hash) + return value.to_h do |field, item| + [@labels.fetch(field.to_s, field), csv_display(item, field.to_s)] + end + end + + value + end + + def fields_for(key) + return NAME_FIELDS if key == 'names' + return ['name', 'scriptCode'] if key == 'aliases' + + [] + end + + def paths(fields, prefix) + fields.flat_map do |key| + path = "#{ prefix }.#{ key }" + [path, *paths(fields_for(key), path)] + end + end + + def fetch(row, key) = row.key?(key) ? row[key] : row[key.to_sym] + + def named_code(row, kind) + code = fetch(row, "#{ kind }_code") + name = fetch(row, "#{ kind }_name") + { code:, name: name.present? && name.to_s.downcase != code.to_s.downcase ? name : nil } + end + + def stable(value) + case value + when Hash + value.stringify_keys.sort.to_h.transform_values { stable(_1) } + when Array + value.map { stable(_1) } + else + value + end + end + + def sort_value(row, key) + if key == 'languageCode' + language = named_code(row, 'language') + return @labels.fetch("language:#{ language[:code] }", language[:name]) + end + return named_code(row, 'script')[:name] if key == 'scriptCode' + + fetch(row, CHILD_KEYS.fetch(key)) + end + + def comparison(left, right) + if left.is_a?(Array) && right.is_a?(Array) && left.length != right.length + return left.length <=> right.length + end + return left <=> right if left.is_a?(Numeric) && right.is_a?(Numeric) + if [true, false].include?(left) && [true, false].include?(right) + return (left ? 1 : 0) <=> (right ? 1 : 0) + end + + a = left.is_a?(Hash) || left.is_a?(Array) ? JSON.generate(stable(left)) : left.to_s + b = right.is_a?(Hash) || right.is_a?(Array) ? JSON.generate(stable(right)) : right.to_s + a <=> b + end + + def sorted(rows, prefix, fields) + sorts = @sorts.filter_map do |path, direction| + parent, key = path.split('/', 2) + next unless parent == prefix + raise ArgumentError unless fields.include?(key) + + [key, direction] + end + rows.each_with_index.sort do |(left, a), (right, b)| + result = 0 + sorts.each do |key, direction| + first = sort_value(left, key) + last = sort_value(right, key) + result = if first.nil? || last.nil? + first.nil? == last.nil? ? 0 : (first.nil? ? 1 : -1) + else + comparison(first, last) * (direction == 'asc' ? 1 : -1) + end + break unless result.zero? + end + result.zero? ? a <=> b : result + end.map(&:first) + end + + def nested(rows, fields, prefix) + keys = fields.select { !@selected || @selected.include?("#{ prefix }.#{ _1 }") } + ordered = (@order + keys.map { "#{ prefix }.#{ _1 }" }).uniq + .filter_map { |path| keys.find { "#{ prefix }.#{ _1 }" == path } } + sorted(rows, prefix, fields).map do |row| + ordered.to_h do |key| + stable = CHILD_KEYS.fetch(key) + value = case key + when 'languageCode' then named_code(row, 'language') + when 'scriptCode' then named_code(row, 'script') + else fetch(row, stable) + end + if fields_for(key).any? + value = nested(value || [], fields_for(key), "#{ prefix }.#{ key }") + end + [stable, value] + end + end + end +end diff --git a/backend/app/services/counts_exporter.rb b/backend/app/services/counts_exporter.rb new file mode 100644 index 0000000..8b797d9 --- /dev/null +++ b/backend/app/services/counts_exporter.rb @@ -0,0 +1,138 @@ +require 'csv' + +# One read view and a fixed point for the entire download; only one page is held in memory. +class CountsExporter + EXTRA_KEYS = ['basis', 'format', 'resource', 'headers', 'children', 'order', + 'summary_columns', 'nested_json', 'child_sort', 'labels'].freeze + attr_reader :format, :resource + + def initialize(params, language_code:) + allowed = (CountsQuery::KEYS + CountsStateQuery::KEYS + EXTRA_KEYS).uniq + raise ArgumentError if (params.keys - allowed).any? + raise ArgumentError unless params.values.all? { _1.is_a?(String) } + + @params = params + @language = language_code + @format = params.fetch('format', 'json') + @resource = params.fetch('resource', 'posts') + @mode = params.fetch('mode', 'summary') + @history = params.fetch('basis', 'state') == 'history' + raise ArgumentError unless ['json', 'csv'].include?(format) + raise ArgumentError unless ['posts', 'tags'].include?(resource) + raise ArgumentError unless ['summary', 'details'].include?(@mode) + raise ArgumentError unless ['state', 'history'].include?(params.fetch('basis', 'state')) + + @query_params = params.except(*EXTRA_KEYS, 'columns', 'page', 'limit') + .merge('mode' => @mode, 'page' => '1', 'limit' => '100') + point = CountsStateQuery.new(@query_params.slice('at').merge('resource' => resource), + language_code: @language).at + @query_params['at'] = point.iso8601(6) + @query_params['axis'] ||= !@history && resource == 'posts' ? 'tags' : 'month' + @projection = CountsExportProjection.new(params, resource:, + summary: @mode == 'summary', + tags: !@history && @query_params['axis'] == 'tags') + if @history + @query_params.delete('kind') + history_query + else + @query_params.delete('latest') + @state_query = CountsStateQuery.new(@query_params.merge('resource' => resource), + language_code: @language).validate_query! + end + end + + def body + Enumerator.new do |output| + ApplicationRecord.transaction(isolation: :repeatable_read) do + first_row = true + started = false + pages do |page| + unless started + meta = { resource:, mode: @mode, basis: @history ? 'history' : 'state', + at: @query_params['at'], boundary: 'inclusive', + filters: @query_params.except('sort', 'page', 'limit', 'mode'), + sort: export_sort, + child_sort: sort_metadata(@params.fetch('child_sort', '')), + columns: @projection.metadata, + children: @params['children']&.split(','), + field_order: @params.fetch('order', '').split(','), timezone: 'UTC', + count: @mode == 'summary' ? page[:group_count] : page[:count], + totals: page[:totals], coverage: page[:coverage] } + output << (format == 'json' ? "{\"metadata\":#{ meta.to_json },\"data\":[" : "\uFEFF") + if format == 'csv' + output << CSV.generate_line( + @projection.headers.map { csv_value(_1) }, row_sep: "\r\n") + end + started = true + end + (page[:versions] || page[:state_rows] || page[:groups] || []).each do |row| + record = @projection.project(row) + if format == 'json' + output << ',' unless first_row + output << record.to_json + else + values = @projection.csv_record(record, row).map { csv_value(_1) } + output << CSV.generate_line(values, row_sep: "\r\n") + end + first_row = false + end + end + output << ']}' if format == 'json' + end + end + end + + private + + def export_sort + sort_metadata(@query_params.fetch('sort', default_sort)) + end + + def sort_metadata(value) + value.split(',').map do |entry| + key, direction = entry.split(':', 2) + { key:, direction: } + end + end + + def default_sort + return 'created_at:desc' if @mode == 'details' + + !@history && @query_params['axis'] == 'tags' ? 'count:desc' : 'group:asc' + end + + def history_query + scope = resource == 'posts' ? PostVersion.with_previous_version : TagVersion.all + CountsQuery.new(scope, resource, @query_params) + end + + def pages + unless @history + @state_query.each_page { yield _1 } + return + end + query = history_query + total = @mode == 'summary' ? nil : query.count + loop do + result = if @mode == 'summary' + query.summary(include_count: @query_params['page'] == '1') + else + { count: total, + versions: CountsVersionRepr.many(query.details, resource:, + language_code: @language, + at: Time.iso8601(@query_params['at'])) } + end + total ||= result[:group_count] + yield result + break if @query_params['page'].to_i * 100 >= total + + @query_params['page'] = (@query_params['page'].to_i + 1).to_s + query = history_query + end + end + + def csv_value(value) + text = value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : value.to_s + text.match?(/\A\s*[=+@-]/) ? "'#{ text }" : text + end +end diff --git a/backend/app/services/counts_query.rb b/backend/app/services/counts_query.rb index 48e9533..832b98a 100644 --- a/backend/app/services/counts_query.rb +++ b/backend/app/services/counts_query.rb @@ -10,7 +10,7 @@ class CountsQuery 'name', 'names', 'category', 'aliases', 'parents', 'deprecated'] }.freeze KEYS = [ 'locale', 'mode', 'axis', 'page', 'limit', 'from', 'before', 'latest', - 'user_id', 'resource_id', 'tag', 'columns', 'sort'].freeze + 'user_id', 'resource_id', 'tag', 'columns', 'sort', 'at'].freeze attr_reader :page, :limit, :mode, :axis @@ -41,15 +41,19 @@ class CountsQuery apply_filters end + def relation = @scope + def count = @scope.count def latest_expression return @latest_expression if @latest_expression + at = timestamp('at') + cutoff = at ? "AND newer.created_at <= #{ @scope.klass.connection.quote(at) }" : '' <<~SQL.squish NOT EXISTS (SELECT 1 FROM #{ @table } newer WHERE newer.#{ @id_column } = #{ @table }.#{ @id_column } - AND newer.version_no > #{ @table }.version_no) + AND newer.version_no > #{ @table }.version_no #{ cutoff }) SQL end @@ -59,7 +63,7 @@ class CountsQuery .limit(limit).offset((page - 1) * limit) end - def summary + def summary(include_count: true) expression = axis == 'month' ? "DATE_FORMAT(#{ @table }.created_at, '%Y-%m')" : "#{ @table }.created_by_user_id" group_order = axis == 'user' ? 'MAX(counts_actor.name)' : expression @@ -68,13 +72,13 @@ class CountsQuery "#{ group_order } AS group_name")).to_sql connection = @scope.klass.connection source = "FROM (#{ sql }) counts_groups" - groups = connection.select_value("SELECT COUNT(*) #{ source }") + groups = include_count ? connection.select_value("SELECT COUNT(*) #{ source }") : nil expressions = { 'group' => ['group_name'], 'count' => ['record_count'] } order = order_sql(expressions, ['group_key IS NULL ASC', 'group_key ASC']) rows = connection.select_all( "SELECT group_key, group_name, record_count #{ source } " + "ORDER BY #{ order } LIMIT #{ limit } OFFSET #{ (page - 1) * limit }") - { group_count: groups.to_i, + { group_count: groups&.to_i, groups: rows.map do |row| { key: row['group_key'], name: axis == 'user' ? row['group_name'] : nil, @@ -168,18 +172,21 @@ class CountsQuery def timestamp(key) value = @params[key].presence return nil unless value - raise ArgumentError unless value.match?(/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2})?Z\z/) + pattern = /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d{1,6})?)?Z\z/ + raise ArgumentError unless value.match?(pattern) Date.iso8601(value.first(10)) Time.iso8601(value.length == 17 ? value.sub(/Z\z/, ':00Z') : value) end def apply_filters + at = timestamp('at') first = timestamp('from') last = timestamp('before') raise ArgumentError if first && last && first >= last raise ArgumentError unless ['0', '1'].include?(@params.fetch('latest', '1')) + @scope = @scope.where("#{ @table }.created_at <= ?", at) if at @scope = @scope.where(Arel.sql(latest_expression)) unless @params['latest'] == '0' @scope = @scope.where("#{ @table }.created_at >= ?", first) if first @scope = @scope.where("#{ @table }.created_at < ?", last) if last diff --git a/backend/app/services/counts_state_query.rb b/backend/app/services/counts_state_query.rb index 1b263bb..48d70c7 100644 --- a/backend/app/services/counts_state_query.rb +++ b/backend/app/services/counts_state_query.rb @@ -1,9 +1,11 @@ -# State at an exclusive UTC boundary, reconstructed only from recorded snapshots. +# Restores one recorded version per resource before applying record filters. class CountsStateQuery KEYS = ['locale', 'resource', 'mode', 'at', 'from', 'before', 'grain', - 'kind', 'tag', 'page', 'limit', 'sort'].freeze + 'kind', 'tag', 'page', 'limit', 'sort', 'axis', 'user_id', 'resource_id'].freeze MAX_PERIODS = 366 + attr_reader :at + def initialize(params, language_code:) raise ArgumentError unless (params.keys - KEYS).empty? raise ArgumentError unless params.values.all? { _1.is_a?(String) } @@ -12,14 +14,21 @@ class CountsStateQuery @connection = ApplicationRecord.connection @language = language_code @resource = params.fetch('resource', 'posts') - @mode = params.fetch('mode', 'state') + @mode = params.fetch('mode', 'summary') + @inclusive = ['summary', 'details'].include?(@mode) + @axis = params.fetch('axis', @resource == 'posts' ? 'tags' : 'month') @kind = params.fetch('kind', 'internal') @grain = params.fetch('grain', 'month') raise ArgumentError unless ['posts', 'tags'].include?(@resource) - raise ArgumentError unless ['state', 'trend', 'snapshot'].include?(@mode) + raise ArgumentError unless ['summary', 'details', 'state', 'trend', 'snapshot'].include?(@mode) raise ArgumentError unless ['internal', 'external', 'all'].include?(@kind) raise ArgumentError unless ['day', 'month', 'year'].include?(@grain) + raise ArgumentError unless ['tags', 'month', 'user'].include?(@axis) + if @mode == 'summary' && @axis == 'tags' && @resource != 'posts' + raise ArgumentError + end + @page = integer('page', 1) @limit = integer('limit', 20) @tag = integer('tag') if params['tag'].present? @@ -28,15 +37,19 @@ class CountsStateQuery raise ArgumentError if @mode == 'trend' && !@tag @now = Time.current.utc - @at = timestamp('at') || @now.change(usec: 0) + @at = timestamp('at') || @now raise ArgumentError if @at > @now end - def call + def call(metadata: true) + @metadata = metadata # Keep totals, rows, and detail pages on one repeatable-read database snapshot. ApplicationRecord.transaction do - @post_start = PostVersion.minimum(:created_at) - @tag_start = TagVersion.minimum(:created_at) + unless @history_loaded + @post_start = PostVersion.minimum(:created_at) + @tag_start = TagVersion.minimum(:created_at) + @history_loaded = true + end result = { page: @page, limit: @limit, at: @at.iso8601(6), coverage: { basis: 'recorded_snapshots', exact: false, post_history_from: @post_start&.iso8601(6), @@ -44,14 +57,45 @@ class CountsStateQuery case @mode when 'trend' result.merge(trend) - when 'snapshot' - result.merge(totals: totals).merge(details) + when 'summary' + result.merge(metadata ? { totals: totals } : {}).merge(summary) + when 'details', 'snapshot' + result.merge(metadata ? { totals: totals } : {}).merge(details) else - result.merge(totals: totals).merge(listing) + result.merge(metadata ? { totals: totals } : {}).merge(listing) end end end + def each_page + return enum_for(:each_page) unless block_given? + + @page = 1 + first = call + yield first + total = ['summary', 'state', 'trend'].include?(@mode) ? first[:group_count] : first[:count] + while @page * @limit < total + @page += 1 + yield call(metadata: false) + end + end + + def validate_query! + filtered_scope(@resource) + case @mode + when 'summary' + if @axis == 'tags' + order_sql({ 'name' => 'name', 'count' => 'post_count', 'tag_id' => 'tag_id', + 'kind' => 'kind', 'deprecated' => 'deprecated_at' }, 'count:desc', []) + else + record_query('summary') + end + when 'details', 'snapshot' + record_query('details') + end + self + end + private def quote(value) = @connection.quote(value) @@ -78,10 +122,10 @@ class CountsStateQuery def latest_condition(table, row, boundary) key = table == 'post_versions' ? 'post_id' : 'tag_id' <<~SQL.squish - #{ row }.created_at < #{ boundary } AND NOT EXISTS ( + #{ row }.created_at #{ @inclusive ? '<=' : '<' } #{ boundary } AND NOT EXISTS ( SELECT 1 FROM #{ table } newer WHERE newer.#{ key } = #{ row }.#{ key } - AND newer.created_at < #{ boundary } + AND newer.created_at #{ @inclusive ? '<=' : '<' } #{ boundary } AND newer.version_no > #{ row }.version_no) SQL end @@ -110,7 +154,7 @@ class CountsStateQuery def ctes boundary = quote(@at) <<~SQL - WITH post_at AS (#{ state_sql('post_versions', boundary, live: true) }), + WITH post_at AS (#{ filtered_scope('posts').to_sql }), tag_at AS (#{ state_sql('tag_versions', boundary) }), nico_at AS (#{ state_sql('nico_tag_versions', boundary) }), pairs AS ( @@ -133,7 +177,7 @@ class CountsStateQuery row = @connection.select_one(<<~SQL) #{ ctes } SELECT (SELECT COUNT(*) FROM post_at) AS posts, - (SELECT COUNT(*) FROM tag_at WHERE event_type <> 'discard') AS tags, + (SELECT COUNT(*) FROM (#{ filtered_scope('tags').to_sql }) counted_tags) AS tags, (SELECT COUNT(*) FROM tag_at WHERE event_type <> 'discard' AND deprecated_at IS NOT NULL) AS deprecated_tags, (SELECT COUNT(*) FROM pairs) AS assignments, @@ -148,7 +192,9 @@ class CountsStateQuery 'deprecated_tags' => available?(@tag_start, @at) ? row['deprecated_tags'] : nil) end - def available?(first, boundary) = first && first < boundary + def available?(first, boundary) + first && (@inclusive ? first <= boundary : first < boundary) + end def listing path = quote("$.#{ JSON.generate(@language) }.name") @@ -176,7 +222,9 @@ class CountsStateQuery where = filters.any? ? "WHERE #{ filters.join(' AND ') }" : '' source = 'FROM catalog c LEFT JOIN tag_counts n ON n.kind = c.kind AND n.tag_id = c.tag_id' prefix = "#{ ctes } #{ catalog }" - total = @connection.select_value("#{ prefix } SELECT COUNT(*) #{ source } #{ where }").to_i + total = if @metadata + @connection.select_value("#{ prefix } SELECT COUNT(*) #{ source } #{ where }").to_i + end order = order_sql({ 'name' => 'c.name', 'count' => 'COALESCE(n.post_count, 0)', 'tag_id' => 'c.tag_id', 'kind' => 'c.kind', 'deprecated' => 'c.deprecated_at' }, 'count:desc', @@ -195,13 +243,14 @@ class CountsStateQuery end } end - def details - table = @resource == 'posts' ? 'post_versions' : 'tag_versions' - model = @resource == 'posts' ? PostVersion : TagVersion - scope = model.joins("INNER JOIN (#{ state_sql(table, quote(@at), live: true) }) " + - "counts_at ON counts_at.id = #{ table }.id") + # All record filters apply after selecting the latest recorded version at the point. + def restored_scope(resource) + model = resource == 'posts' ? PostVersion : TagVersion + table = model.table_name + scope = model.where(Arel.sql(latest_condition(table, table, quote(@at)))) + .where.not(event_type: 'discard') if @tag - scope = if @resource == 'posts' + scope = if resource == 'posts' scope.where("#{ membership(table) } AND #{ tag_visible(quote(@at)) }") elsif @kind == 'internal' scope.where(tag_id: @tag) @@ -209,12 +258,36 @@ class CountsStateQuery scope.none end end + scope + end + + def filtered_scope(resource) + params = @params.slice('from', 'before', 'user_id', 'resource_id') + .merge('mode' => 'details', 'latest' => '0', 'sort' => '') + CountsQuery.new(restored_scope(resource), resource, params, + latest_expression: '1').relation + end + + def record_query(mode) params = @params.slice('locale', 'sort', 'page', 'limit') - .merge('mode' => 'details', 'latest' => '0') - query = CountsQuery.new(scope, @resource, params, latest_expression: '1') - { count: query.count, + .merge('mode' => mode, 'latest' => '0', 'axis' => @axis == 'tags' ? 'month' : @axis, + 'page' => @page.to_s, 'limit' => @limit.to_s) + CountsQuery.new(filtered_scope(@resource), @resource, params, latest_expression: '1') + end + + def summary + return listing if @axis == 'tags' + + query = record_query('summary') + { count: @metadata ? query.count : nil }.merge(query.summary(include_count: @metadata)) + end + + def details + query = record_query('details') + { count: @metadata ? query.count : nil, versions: CountsVersionRepr.many(query.details, resource: @resource, - language_code: @language) } + language_code: @language, at: @at, + inclusive: @inclusive) } end def next_boundary(time) diff --git a/backend/config/routes.rb b/backend/config/routes.rb index a350568..002149b 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -1,4 +1,6 @@ Rails.application.routes.draw do + get 'counts/export', to: 'counts_exports#index' + post 'counts/export', to: 'counts_exports#index' get 'counts/state', to: 'counts_states#index' get 'counts/posts', to: 'counts_posts#index' get 'counts/tags', to: 'counts_tags#index' diff --git a/frontend/src/components/counts/CountsControls.tsx b/frontend/src/components/counts/CountsControls.tsx index 6a17e09..22ab6b1 100644 --- a/frontend/src/components/counts/CountsControls.tsx +++ b/frontend/src/components/counts/CountsControls.tsx @@ -1,28 +1,31 @@ import { useEffect, useId, useState } from 'react' -import CountsStateFilters from '@/components/counts/CountsStateFilters' import CountsFieldTree from '@/components/counts/CountsFieldTree' import DateTimeField from '@/components/common/DateTimeField' import FormField from '@/components/common/FormField' import { Button } from '@/components/ui/button' import { isCountsState, normaliseCountsParams } from '@/lib/counts' +import { countsSummaryFields } from '@/lib/countsState' import { fetchTagByName } from '@/lib/tags' import { inputClass } from '@/lib/utils' import { useLocale } from '@/lib/useLocale' import { useUITranslation } from '@/lib/useUITranslation' +import type { ReactNode } from 'react' + import type { CountsResource } from '@/lib/counts' const CountsControls = ( - { resource, applied, open, setOpen, execute, displayJSON, reorder }: { + { resource, applied, open, setOpen, execute, displayJSON, reorder, actions }: { resource: CountsResource applied: URLSearchParams open: boolean setOpen: (open: boolean) => void execute: (params: URLSearchParams) => void displayJSON: (enabled: boolean) => void - reorder: (value: string) => void }, + reorder: (value: string) => void + actions?: ReactNode }, ) => { const { label, message } = useUITranslation ('counts') const locale = useLocale () @@ -43,13 +46,18 @@ const CountsControls = ( }, [order, nestedJSON]) const [error, setError] = useState (false) const [pending, setPending] = useState (false) - const mode = draft.get ('mode') ?? 'state' + const mode = draft.get ('mode') ?? 'summary' const state = isCountsState (draft) + const axes = (state && resource === 'posts') ? ['tags', 'month', 'user'] : ['month', 'user'] const update = (key: string, value: string) => { setDraft (previous => { const next = new URLSearchParams (previous) - if (key === 'mode') next.delete ('sort') - if (value || key === 'columns' || key === 'children') next.set (key, value) + if (['mode', 'axis', 'basis'].includes (key)) next.delete ('sort') + if (key === 'axis' || key === 'basis') next.delete ('summary_columns') + if (key === 'basis' && value === 'history' && next.get ('axis') === 'tags') + next.set ('axis', 'month') + if (value || ['columns', 'children', 'summary_columns'].includes (key)) + next.set (key, value) else next.delete (key) return next }) @@ -60,7 +68,7 @@ const CountsControls = ( try { const next = new URLSearchParams (draft) - for (const key of (state || resource === 'posts') ? ['tag'] : ['resource_id']) + for (const key of resource === 'posts' ? ['tag'] : ['resource_id']) { const value = next.get (key)?.trim () ?? '' if (value && !/^[1-9][0-9]*$/.test (value)) @@ -99,14 +107,16 @@ const CountsControls = ( + {actions} {error &&

{label ('shared:messages.checkInputModern')}

} -