diff --git a/backend/app/controllers/counts_tables_controller.rb b/backend/app/controllers/counts_tables_controller.rb index a93aaa7..f3620c3 100644 --- a/backend/app/controllers/counts_tables_controller.rb +++ b/backend/app/controllers/counts_tables_controller.rb @@ -3,6 +3,8 @@ class CountsTablesController < ApplicationController resolve_locale! definition = CountsResourceDefinition.new(params[:resource]) render json: { fields: definition.metadata, + output_fields: definition.output_fields, + filter_fields: definition.filter_metadata, axes: { latest: definition.axes(false), history: definition.axes(true) }, grains: CountsAxes::GRAINS, defaults: CountsDefaults.metadata(params[:resource]) } rescue ArgumentError diff --git a/backend/app/models/history_access.rb b/backend/app/models/history_access.rb index b3344af..018e3d0 100644 --- a/backend/app/models/history_access.rb +++ b/backend/app/models/history_access.rb @@ -148,4 +148,13 @@ class HistoryAccess < ActiveSupport::CurrentAttributes # This number deliberately ignores time, identity and content filters to prevent count probes. def self.population_count(model) = model.unscoped.count + + # Only the resource-wide denied count is public, never a condition-dependent denied count. + def self.population_counts(model) + total, excluded = model.unscoped.optimizer_hints('MAX_EXECUTION_TIME(10000)').pick( + Arel.sql('COUNT(*)'), + Arel.sql("COALESCE(SUM(CASE WHEN #{ sql(model) } THEN 0 ELSE 1 END), 0)")) + { population_count: total.to_i, excluded_count: excluded.to_i, + excluded_count_scope: 'all_recorded_versions' } + end end diff --git a/backend/app/services/counts_condition.rb b/backend/app/services/counts_condition.rb new file mode 100644 index 0000000..5e4e494 --- /dev/null +++ b/backend/app/services/counts_condition.rb @@ -0,0 +1,330 @@ +# Typed predicates over the same recorded fields used by Counts output. +# Multi-valued comparisons use EXISTS; inequality means no matching element. +class CountsCondition + CHILD_TYPES = { 'id' => :integer, 'versionNo' => :integer, 'tagId' => :integer, + 'tagGroupId' => :integer, 'autoGenerated' => :boolean, + 'category' => :enum, 'kind' => :enum, 'sections' => :interval }.freeze + + def self.fields(definition) + result = definition.metadata.map { _1.merge(labels: [definition.field_label(_1[:key])]) } + visit = lambda do |fields, prefix, labels| + fields.each do |field| + key = [prefix, field[:key]].compact.join('.') + path_labels = labels + [field[:label]] + type = if prefix + CHILD_TYPES.fetch(field[:key], :string) + elsif key == 'is_latest' + :boolean + elsif key == 'parents' + :integer + elsif key == 'deprecated' + :datetime + elsif key == 'thumbnail' + :string + else + definition.fields.fetch(key, :string) + end + type = :string if field[:children] || type == :json + type = :integer if ['parents', 'members'].include?(key) + physical = result.find { _1[:key] == key } + nullable = prefix.present? || (physical ? physical[:nullable] : key != 'is_latest') + nullable = false if field[:children] + entry = { key:, type:, labels: path_labels, nullable:, + multiple: prefix.present? || field[:children].present?, + collection: field[:children].present? || type == :interval, + operators: CountsResourceDefinition::OPERATORS.fetch(type) } + result.reject! { _1[:key] == key } + result << entry + visit.call(field[:children] || [], key, path_labels) + end + end + visit.call(definition.output_fields, nil, []) + # The raw JSON alias has the same membership semantics as the public tag field. + if definition.resource == 'posts' + result.find { _1[:key] == 'tags_json' }.merge!(type: :string, multiple: true, collection: true, + operators: CountsResourceDefinition::OPERATORS.fetch(:string)) + end + result + [{ key: 'user.id', type: :integer, nullable: true, + labels: ['history:actor', 'counts:columns.resourceId'], + operators: CountsResourceDefinition::OPERATORS.fetch(:integer) }, + { key: 'user.name', type: :string, nullable: true, + labels: ['history:actor', 'history:name'], + operators: CountsResourceDefinition::OPERATORS.fetch(:string) }] + end + + def initialize(population) + @population = population + @definition = population.definition + @connection = @definition.model.connection + @language = Locale.find_by!(code: population.params.fetch('locale', 'ja')).language_code + end + + def apply(scope, entry) + raise ArgumentError unless entry.is_a?(Hash) + raise ArgumentError if (entry.keys - ['field', 'operator', 'value', 'end', 'blank']).any? + + field = @definition.filter_metadata.find { _1[:key] == entry.fetch('field') } + raise ArgumentError unless field && field[:operators].include?(entry.fetch('operator')) + raise ArgumentError unless [true, false, nil].include?(entry['blank']) + values = [entry['value'], entry['end']] + raise ArgumentError unless values.all? { _1.nil? || (_1.is_a?(String) && _1.bytesize <= 8192) } + if entry['operator'] != 'range' && !values[1].nil? && values[1] != '' + raise ArgumentError + end + + source, expression = source_for(field[:key]) + type = field[:type] + blank = blank_sql(expression, type) + if entry['blank'] + sql = if source && field[:collection] + "NOT EXISTS (SELECT 1 FROM #{ source })" + elsif source + "NOT EXISTS (SELECT 1 FROM #{ source } WHERE NOT (#{ blank }))" + else blank + end + return scope.where(Arel.sql(sql)) + end + return scope if values.all? { _1.nil? || _1 == '' } + + parsed = values.map { parse(_1, type) } + validate_enum!(field[:key], parsed.first) if type == :enum + operator = entry['operator'] + if type == :interval + return interval(scope, source, expression, parsed) + end + clauses = [] + binds = [] + if operator == 'range' + ['>=', '<='].each_with_index do |comparison, index| + next if parsed[index].nil? + + clauses << "#{ expression } #{ comparison } ?" + binds << parsed[index] + end + raise ArgumentError if parsed.all? && parsed.first > parsed.last + elsif !parsed.first.nil? + comparison = { 'eq' => '=', 'ne' => '=', 'gt' => '>', 'lt' => '<', + 'ge' => '>=', 'le' => '<=', 'like' => 'LIKE' }.fetch(operator) + clauses << "#{ expression } #{ comparison } ?" + binds << parsed.first + end + return scope if clauses.empty? + + predicate = clauses.join(' AND ') + if source + predicate = "EXISTS (SELECT 1 FROM #{ source } WHERE #{ predicate })" + predicate = "NOT (#{ predicate })" if operator == 'ne' + elsif operator == 'ne' + predicate = "(NOT (#{ predicate }) OR #{ blank })" + end + scope.where(predicate, *binds) + end + + private + + def parse(value, type) + return nil if value.nil? || value == '' + + case type + when :integer, :interval + raise ArgumentError unless value.match?(/\A-?[0-9]{1,19}\z/) + + number = Integer(value, 10) + raise ArgumentError unless number.between?(-9_223_372_036_854_775_808, 9_223_372_036_854_775_807) + + number + when :boolean + raise ArgumentError unless ['true', 'false'].include?(value) + + value == 'true' ? 1 : 0 + when :datetime then @population.timestamp(value) + else value + end + end + + def validate_enum!(key, value) + allowed = case key.split('.').last + when 'event_type' then VersionRecord::DEFAULT_EVENT_TYPE_MAP.values + when 'category', 'tag_category' then TagVersion.categories.keys + when 'kind' then ['internal', 'external'] + when 'source_kind' then Material::SOURCE_KINDS + end + raise ArgumentError if value && allowed && !allowed.include?(value) + end + + def blank_sql(expression, type) + if [:string, :enum].include?(type) + "(#{ expression } IS NULL OR #{ expression } = '')" + else + "#{ expression } IS NULL" + end + end + + def scalar(document, key, type = :string) + path = @connection.quote("$.#{ key }") + value = "JSON_EXTRACT(#{ document }, #{ path })" + text = "JSON_UNQUOTE(NULLIF(#{ value }, CAST('null' AS JSON)))" + return "CAST(#{ text } AS SIGNED)" if type == :integer + return "CASE #{ text } WHEN 'true' THEN 1 WHEN 'false' THEN 0 END" if type == :boolean + + text + end + + def json_table(document, path, name) + "JSON_TABLE(COALESCE(#{ document }, JSON_ARRAY()), #{ @connection.quote(path) } " + + "COLUMNS (item JSON PATH '$')) #{ name }" + end + + def names_source(document, alias_name) + source = "JSON_TABLE(COALESCE(JSON_KEYS(#{ document }), JSON_ARRAY()), '$[*]' " + + "COLUMNS (language_code TEXT PATH '$')) #{ alias_name }" + path = "CONCAT('$.', JSON_QUOTE(#{ alias_name }.language_code))" + [source, "JSON_EXTRACT(#{ document }, #{ path })", "#{ alias_name }.language_code"] + end + + def point + @point ||= @population.history ? @population.at : + @population.timestamp(@population.params['at']) || @population.at + end + + def name_expression(row) + language_path = @connection.quote("$.#{ @language.to_json }.name") + name = "JSON_UNQUOTE(JSON_EXTRACT(#{ row }.names, #{ language_path }))" + "COALESCE(#{ name }, #{ row }.name)" + end + + def parent_source + column = @definition.resource == 'posts' ? 'parent_post_ids' : 'parent_tag_ids' + ids = "TRIM(REGEXP_REPLACE(COALESCE(v.#{ column }, ''), '[[:space:]]+', ' '))" + document = "CONCAT('[', REPLACE(#{ ids }, ' ', ','), ']')" + source = json_table(document, '$[*]', 'parent_entry') + id = 'CAST(parent_entry.item AS UNSIGNED)' + return [source, id] if @definition.resource == 'posts' + + bound = @connection.quote(point) + source += ' LEFT JOIN tag_versions parent_version ON parent_version.tag_id = ' + id + + " AND parent_version.created_at <= #{ bound } AND " + + HistoryAccess.sql(TagVersion, 'parent_version') + + ' AND NOT EXISTS (SELECT 1 FROM tag_versions newer_parent ' + + 'WHERE newer_parent.tag_id = parent_version.tag_id ' + + "AND newer_parent.created_at <= #{ bound } " + + 'AND newer_parent.version_no > parent_version.version_no)' + [source, id] + end + + def post_tag_source + source = json_table('v.tags_json', '$[*]', 'tag_snapshot') + source += ' LEFT JOIN tag_versions referenced_tag ON referenced_tag.tag_id = ' + + scalar('tag_snapshot.item', 'tag_id', :integer) + + ' AND referenced_tag.version_no = ' + scalar('tag_snapshot.item', 'version_no', :integer) + + " AND referenced_tag.created_at <= #{ @connection.quote(point) } AND " + + HistoryAccess.sql(TagVersion, 'referenced_tag') + source + end + + def language_field(source, document, parts, inherited_language = nil) + return [source, scalar(document, 'name')] if parts.empty? + return [source, inherited_language] if parts == ['languageCode'] && inherited_language + + if parts.first == 'aliases' + aliases = "JSON_EXTRACT(#{ document }, '$.aliases')" + source += ' JOIN ' + json_table(aliases, '$[*]', 'alias_entry') + child = parts[1] || 'name' + return [source, inherited_language] if child == 'languageCode' && inherited_language + columns = { 'scriptCode' => 'script_code', 'name' => 'name' } + key = columns.fetch(child) { raise ArgumentError } + return [source, scalar('alias_entry.item', key)] + end + key = { 'name' => 'name', 'scriptCode' => 'script_code', + 'autoGenerated' => 'auto_generated' }.fetch(parts.first) { raise ArgumentError } + [source, scalar(document, key, CHILD_TYPES.fetch(parts.first, :string))] + end + + def source_for(key) + root, *parts = key.split('.') + return [nil, "(#{ @population.latest_sql })"] if root == 'is_latest' + if root == 'user' && parts.any? + expression = parts.first == 'id' ? 'v.created_by_user_id' : + '(SELECT actor.name FROM users actor WHERE actor.id = v.created_by_user_id)' + return [nil, expression] + end + return [nil, 'v.thumbnail_base'] if root == 'thumbnail' + return [nil, 'v.deprecated_at'] if root == 'deprecated' + if root == 'parents' + source, id = parent_source + return [source, parts.first == 'name' ? name_expression('parent_version') : id] + end + if root == 'tags' || root == 'tags_json' + return post_field(parts) + end + if root == 'names' || root == 'aliases' + source, document, language = names_source('v.names', 'name_language') + parts = ['aliases', *parts] if root == 'aliases' + return language_field(source, document, parts, language) + end + if root == 'members' || root == 'placements' + path = '$[*]' + parts.take_while { _1 == 'children' }.map { '.children[*]' }.join + source = json_table("v.#{ root }", path, 'structure_entry') + if root == 'members' && parts.empty? + value = "COALESCE(#{ scalar('structure_entry.item', 'tag_id', :integer) }, " + + "#{ scalar('structure_entry.item', 'tag_group_id', :integer) })" + return [source, value] + end + leaf = parts.last || 'sectionName' + leaf = 'sectionName' if leaf == 'children' + column = leaf.gsub(/[A-Z]/) { "_#{ _1.downcase }" } + return [source, scalar('structure_entry.item', column, CHILD_TYPES.fetch(leaf, :string))] + end + if root == 'export_paths_json' + source, value, profile = names_source('v.export_paths_json', 'export_profile') + return [source, parts.first == 'profile' ? profile : "JSON_UNQUOTE(#{ value })"] + end + return [nil, name_expression('v')] if root == 'name' && @definition.resource == 'tags' + + [nil, @definition.expression(root)] + end + + def post_field(parts) + source = post_tag_source + document = 'tag_snapshot.item' + case parts.first + when 'names' + names, entry, language = names_source('referenced_tag.names', 'tag_language') + language_field("#{ source } JOIN #{ names }", entry, parts.drop(1), language) + when 'sections' + sections = "JSON_EXTRACT(#{ document }, '$.sections')" + source += ' JOIN ' + json_table(sections, '$[*]', 'section_entry') + [source, 'section_entry.item'] + when 'id' + [source, "COALESCE(#{ scalar(document, 'tag_id', :integer) }, " + + "#{ scalar(document, 'external_tag_id', :integer) })"] + when 'versionNo' then [source, scalar(document, 'version_no', :integer)] + when 'kind' + [source, "CASE WHEN #{ scalar(document, 'tag_id') } IS NULL " + + "THEN 'external' ELSE 'internal' END"] + when 'category' + [source, "COALESCE(#{ scalar(document, 'category') }, referenced_tag.category)"] + else + [source, "COALESCE(#{ scalar(document, 'name') }, referenced_tag.name)"] + end + end + + def interval(scope, source, document, bounds) + raise ArgumentError if bounds.compact.any?(&:negative?) + raise ArgumentError if bounds.all? && bounds.first > bounds.last + + first = scalar(document, 'begin_ms', :integer) + last = scalar(document, 'end_ms', :integer) + clauses = [] + values = [] + if bounds.first + clauses << "(#{ last } IS NULL OR #{ last } > ?)" + values << bounds.first + end + if bounds.last + clauses << "#{ first } < ?" + values << bounds.last + end + scope.where("EXISTS (SELECT 1 FROM #{ source } WHERE #{ clauses.join(' AND ') })", *values) + end +end diff --git a/backend/app/services/counts_defaults.rb b/backend/app/services/counts_defaults.rb index b448695..d223cbf 100644 --- a/backend/app/services/counts_defaults.rb +++ b/backend/app/services/counts_defaults.rb @@ -11,22 +11,32 @@ class CountsDefaults 'sort' => 'a0:desc', 'conditions' => '' } }] result << { definition: 'latest', mode: 'summary', params: { 'rows' => '', 'cols' => '', 'conditions' => '', 'tags' => '' } } - if resource == 'posts' - result += [{ definition: 'latest', mode: 'details', params: { - 'columns' => POST_COLUMNS.join(' '), 'order' => POST_COLUMNS.join(' '), - 'children' => 'tags.name parents.id', 'nested_json' => '0', - 'conditions' => '', 'tags' => '' } }] - end - if resource == 'tags' - columns = ['resource_id', 'version_no', 'is_latest', 'name', 'category', - 'aliases', 'parents', 'deprecated', 'event_type', 'user', 'created_at'] - result << { definition: 'latest', mode: 'details', params: { - 'columns' => columns.join(' '), 'order' => columns.join(' '), - 'children' => 'aliases.name parents.name', 'nested_json' => '0' } } + details = detail_params(resource) + if details + ['latest', 'history'].each do |definition| + result << { definition:, mode: 'details', params: details } + end end result end + def self.detail_params(resource) + if resource == 'posts' + return { 'columns' => POST_COLUMNS.join(' '), 'order' => POST_COLUMNS.join(' '), + 'children' => 'tags.name parents.id', 'nested_json' => '0', + 'conditions' => '', 'tags' => '' } + end + unless resource == 'tags' + return { 'columns' => CountsExportProjection.default_columns(resource).join(' '), + 'children' => CountsExportProjection.default_children(resource).join(' ') } + end + + columns = ['resource_id', 'version_no', 'is_latest', 'name', 'category', + 'aliases', 'parents', 'deprecated', 'event_type', 'user', 'created_at'] + { 'columns' => columns.join(' '), 'order' => columns.join(' '), + 'children' => 'aliases.name parents.name', 'nested_json' => '0' } + end + def self.for(params) settings = entries(params.fetch('resource', 'posts')).find do |entry| entry[:definition] == params.fetch('definition', 'latest') && diff --git a/backend/app/services/counts_population.rb b/backend/app/services/counts_population.rb index 19446cb..c38be27 100644 --- a/backend/app/services/counts_population.rb +++ b/backend/app/services/counts_population.rb @@ -125,50 +125,8 @@ class CountsPopulation end def apply_condition(entry) - raise ArgumentError unless entry.is_a?(Hash) - raise ArgumentError if (entry.keys - ['field', 'operator', 'value', 'end', 'blank']).any? - - key = entry.fetch('field') - type = definition.fields.fetch(key) { raise ArgumentError } - operator = entry.fetch('operator') - raise ArgumentError unless CountsResourceDefinition::OPERATORS.fetch(type, []).include?(operator) - raise ArgumentError unless [true, false, nil].include?(entry['blank']) - - expression = definition.expression(key) - blank = type == :string || type == :enum ? - "(#{ expression } IS NULL OR #{ expression } = '')" : "#{ expression } IS NULL" - if entry['blank'] - @scope = @scope.where(Arel.sql(blank)) - return - end - values = [entry['value'], entry['end']] - raise ArgumentError unless values.all? { _1.nil? || _1.is_a?(String) } - return if values.all? { _1.nil? || _1 == '' } - - parsed = values.map do |value| - next nil if value.nil? || value == '' - - case type - when :integer - raise ArgumentError unless value.match?(/\A-?\d+\z/) - - value.to_i - when :datetime then timestamp(value) - else value - end - end - if operator == 'range' - @scope = @scope.where("#{ expression } >= ?", parsed[0]) if parsed[0] - @scope = @scope.where("#{ expression } <= ?", parsed[1]) if parsed[1] - return - end - return unless parsed[0] - - sql_operator = { 'eq' => '=', 'ne' => '<>', 'gt' => '>', 'lt' => '<', - 'ge' => '>=', 'le' => '<=', 'like' => 'LIKE' }.fetch(operator) - sql = "#{ expression } #{ sql_operator } ?" - sql = "(#{ sql } OR #{ blank })" if operator == 'ne' - @scope = @scope.where(sql, parsed[0]) + @condition ||= CountsCondition.new(self) + @scope = @condition.apply(@scope, entry) end def snapshot_membership(value) diff --git a/backend/app/services/counts_resource_definition.rb b/backend/app/services/counts_resource_definition.rb index cfebcea..bb909cd 100644 --- a/backend/app/services/counts_resource_definition.rb +++ b/backend/app/services/counts_resource_definition.rb @@ -31,7 +31,35 @@ class CountsResourceDefinition OPERATORS = { string: ['eq', 'ne', 'like'], enum: ['eq', 'ne'], boolean: ['eq', 'ne'], integer: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'], - datetime: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'] }.freeze + datetime: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'], + interval: ['range'] }.freeze + LABELS = { + 'resource_id' => 'counts:columns.resourceId', 'version_no' => 'counts:columns.version', + 'is_latest' => 'counts:columns.latest', 'event_type' => 'counts:eventType', + 'user' => 'history:actor', 'created_at' => 'counts:columns.changedAt', + 'thumbnail' => 'counts:columns.thumbnailDisplay', + 'thumbnail_base' => 'counts:columns.thumbnailURL', + 'title' => 'counts:columns.postTitle', 'url' => 'counts:columns.postURL', + 'tags' => 'counts:columns.tags', 'tags_json' => 'counts:columns.tags', + 'video_ms' => 'shared:labels.duration', 'name' => 'counts:columns.name', + 'names' => 'counts:columns.names', 'category' => 'common:categoryLabel', + 'aliases' => 'tags:aliases', 'deprecated' => 'counts:columns.deprecated', + 'original_created_from' => 'counts:columns.originalFrom', + 'original_created_before' => 'counts:columns.originalBefore', + 'language_code' => 'counts:columns.language' }.freeze + CHILD_LABELS = { + 'id' => 'counts:columns.resourceId', 'versionNo' => 'history:version', + 'kind' => 'counts:columns.kind', 'name' => 'history:name', + 'category' => 'common:categoryLabel', 'sections' => 'counts:columns.sections', + 'names' => 'counts:columns.names', 'aliases' => 'tags:aliases', + 'languageCode' => 'counts:columns.language', + 'scriptCode' => 'history:nameChanges.script', + 'autoGenerated' => 'history:nameChanges.automatic', + 'tagId' => 'counts:attributes.tag_id', 'tagGroupId' => 'counts:resources.tag_groups', + 'sectionName' => 'counts:attributes.section_name', + 'description' => 'counts:attributes.description', + 'children' => 'counts:attributes.children', 'profile' => 'counts:attributes.profile', + 'path' => 'counts:attributes.path' }.freeze attr_reader :model, :keys, :fields, :resource @@ -62,4 +90,57 @@ class CountsResourceDefinition operators: OPERATORS.fetch(type, []) } end end + + def field_label(key) = LABELS.fetch(key, "counts:attributes.#{ key }") + + def output_fields + content = case resource + when 'posts' + ['thumbnail', 'title', 'url', 'thumbnail_base', 'tags', 'parents', 'video_ms', + 'original_created_from', 'original_created_before'] + when 'tags' then ['name', 'names', 'category', 'aliases', 'parents', 'deprecated'] + else fields.keys - COMMON.keys + end + keys = ['resource_id', 'version_no', 'is_latest'] + (content - ['thumbnail']) + + ['event_type', 'user', 'created_at'] + keys.unshift('thumbnail') if content.include?('thumbnail') + keys.map do |key| + label = if key == 'parents' + resource == 'tags' ? 'tags:parents' : 'shared:labels.parentPosts' + else field_label(key) + end + field = { key:, label: } + children = child_fields(key) + field[:children] = children if children.any? + field + end + end + + def child_fields(key, depth = 4) + keys = case key + when 'tags' then ['id', 'versionNo', 'kind', 'name', 'category', 'sections', 'names'] + when 'names' then ['languageCode', 'scriptCode', 'name', 'aliases', 'autoGenerated'] + when 'aliases' + resource == 'tags' ? ['languageCode', 'name', 'scriptCode'] : ['name', 'scriptCode'] + when 'parents' then resource == 'tags' ? ['id', 'name'] : ['id'] + when 'members' then ['tagId', 'tagGroupId', 'versionNo'] + when 'export_paths_json' then ['profile', 'path'] + when 'placements', 'children' + scalar = ['sectionName', 'description', 'tagId', 'tagGroupId'] + scalar + (depth > 0 ? ['children'] : []) + else [] + end + keys.map do |child| + label = key == 'members' && child == 'tagId' ? + 'counts:columns.resourceId' : CHILD_LABELS.fetch(child) + field = { key: child, label: } + children = child_fields(child, child == 'children' ? depth - 1 : depth) + # Language is attached to top-level aliases, not to aliases inside a language entry. + children = children.reject { _1[:key] == 'languageCode' } if child == 'aliases' + field[:children] = children if children.any? + field + end + end + + def filter_metadata = @filter_metadata ||= CountsCondition.fields(self) end diff --git a/backend/app/services/counts_table_export.rb b/backend/app/services/counts_table_export.rb index e50d98b..87981b2 100644 --- a/backend/app/services/counts_table_export.rb +++ b/backend/app/services/counts_table_export.rb @@ -67,7 +67,9 @@ class CountsTableExport unless started metadata = page.except(:versions, :cells, :axis_labels, :column_values, :row_headers, :column_headers, :pivot_totals).merge( - resource:, filters: @params.select { |key, _value| CountsTableQuery::KEYS.include?(key) || + resource:, record_unit: @summary && format == 'json' ? 'pivot_cell' : 'row', + body_row_totals: 'once_per_logical_group', page_grand_total_repeated: false, + filters: @params.select { |key, _value| CountsTableQuery::KEYS.include?(key) || key == 'measures' || QueryState.valid_key?(key) }, fields: @query.population.definition.metadata, columns: @columns, column_labels: @headers, diff --git a/backend/app/services/counts_table_query.rb b/backend/app/services/counts_table_query.rb index 51093e2..21f7c56 100644 --- a/backend/app/services/counts_table_query.rb +++ b/backend/app/services/counts_table_query.rb @@ -41,17 +41,29 @@ class CountsTableQuery def call ApplicationRecord.transaction do - metadata.merge(mode == 'summary' ? summary : details) + result = metadata.merge(mode == 'summary' ? summary : details) + visible = mode == 'summary' ? result[:visible_output_count] : result[:count] + # A scalar result has one row regardless of private attributes or predicates. + public_count = result[:excluded_count].zero? || (mode == 'summary' && axes.rows.empty?) + result.merge(visible_output_count: visible, + output_count: public_count ? visible : nil, + output_count_scope: public_count ? + 'before_authorisation' : 'unavailable_private_history', + output_count_unit: 'table_body_rows', + page_grand_total_repeated: mode == 'summary' && axes.rows.any? && + result[:group_count] > limit) end end def metadata - { population_count_scope: 'all_recorded_versions', population_count: HistoryAccess.population_count(@definition.model), - page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence, - definition: population.history ? 'history' : 'latest', - timezone: population.timezone.identifier, rows: axes.rows, cols: axes.columns, - fields: @definition.metadata, - count: population.scope.except(:select).optimizer_hints('MAX_EXECUTION_TIME(10000)').count } + @population_counts ||= HistoryAccess.population_counts(@definition.model) + @population_counts.merge( + population_count_scope: 'all_recorded_versions', + page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence, + definition: population.history ? 'history' : 'latest', + timezone: population.timezone.identifier, rows: axes.rows, cols: axes.columns, + fields: @definition.metadata, + count: population.scope.except(:select).optimizer_hints('MAX_EXECUTION_TIME(10000)').count) end def summary @@ -143,7 +155,9 @@ class CountsTableQuery column_headers = [{ values: [], kind: 'grand_total' }] end totals = CountsPivotTotals.new(population, axes, row_headers, column_headers).call - { group_count: total, column_values:, row_headers:, column_headers:, pivot_totals: totals, + { group_count: total, + visible_output_count: output_row_count(prefix, row_keys, row_depths, total), + column_values:, row_headers:, column_headers:, pivot_totals: totals, cells: records.map do |row| { values: axes.all.each_index.map { row["a#{ _1 }"] }, count: row['count'].to_i } end, axis_labels: axis_labels(records, column_values) } @@ -192,6 +206,21 @@ class CountsTableQuery private + def output_row_count(prefix, row_keys, depths, total) + # With no row axes the table has exactly one body row, including the zero-count case. + return 1 if row_keys.empty? + + subtotals = depths.map do |depth| + keys = row_keys.first(depth).join(', ') + "(SELECT COUNT(*) FROM (SELECT #{ keys } FROM cells GROUP BY #{ keys }) subtotal_rows)" + end + expressions = [total.to_s, '1'] + subtotals + @output_row_count ||= @connection.select_value(<<~SQL).to_i + #{ prefix } + SELECT /*+ MAX_EXECUTION_TIME(10000) */ #{ expressions.join(' + ') } AS output_rows + SQL + end + def locale = @locale ||= Locale.find_by!(code: @params.fetch('locale', 'ja')) def limited_columns(prefix) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index dc27053..a46f65f 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -261,6 +261,8 @@ const RoutedApp = ({ locales }: { locales: AvailableLocale[] }) => { const locale = useLocale () const route = resolvePageRoute (location.pathname) const robots = route?.route.handle?.robots + const pagePath = unlocalisedPath (location.pathname) + const countsPage = pagePath === '/counts' || pagePath.startsWith ('/counts/') const switchLocale = async (targetLocale: LocaleCode): Promise => { const resolve = route?.route.handle?.switchLocale return (resolve != null && route != null) @@ -377,7 +379,7 @@ const RoutedApp = ({ locales }: { locales: AvailableLocale[] }) => { }> diff --git a/frontend/src/components/counts/CountsAdditionalConditions.tsx b/frontend/src/components/counts/CountsAdditionalConditions.tsx index 4ce91a1..8a61509 100644 --- a/frontend/src/components/counts/CountsAdditionalConditions.tsx +++ b/frontend/src/components/counts/CountsAdditionalConditions.tsx @@ -22,7 +22,9 @@ const ConditionEditor = ( save: (condition: CountsCondition) => void }, ) => { const { label, message } = useUITranslation ('counts') - const fields = countsFilterFields (definition) + const fields = definition.filterFields ?? definition.fields + const fieldName = (key: string) => (fields.find (item => item.key === key)?.labels + ?? [countsFieldLabel (key)]).map (key => message (key).text).join (' › ') const [condition, setCondition] = useState (() => { const next = { ...initial, field: initial?.field ?? 'resource_id', operator: initial?.operator ?? 'eq' } @@ -30,7 +32,7 @@ const ConditionEditor = ( return next }) const { field, operator } = condition - const operators = definition.fields.find (item => item.key === field)?.operators ?? [] + const operators = fields.find (item => item.key === field)?.operators ?? [] useEffect (() => { controls.setActions ([{ label: label (initial == null ? 'add' : 'shared:actions.edit'), onSelect: () => save (condition) }]) @@ -39,11 +41,11 @@ const ConditionEditor = ( patch (index, { [key]: event.target.value })}> + )} + + } + if (type === 'datetime') + return patch (index, { [key]: value ?? '' })}/> + return patch (index, { [key]: event.target.value })}/> + } return
{label ('additionalConditions')} {conditions.map ((condition, index) => { - const datetime = countsFilterFields (definition)[condition.field] === 'datetime' return
{(condition.operator === 'range' ? ['value', 'end'] as const : ['value'] as const) - .map (key => condition.field === 'event_type' ? ( - ) : (datetime ? patch (index, { [key]: value ?? '' })}/> : ( - patch (index, { [key]: event.target.value })}/>)))} + .map (key => valueInput (condition, index, key))}