451 行
20 KiB
Ruby
451 行
20 KiB
Ruby
# Common entrypoint for six version resources; legacy endpoints keep their contracts.
|
|
class CountsTableQuery
|
|
MAX_COLUMNS = 100
|
|
class TooManyColumns < ArgumentError; end
|
|
|
|
KEYS = ['resource', 'definition', 'mode', 'at', 'execution_at', 'from', 'through',
|
|
'timezone', 'rows', 'cols', 'conditions', 'drill', 'tags', 'tag_refs', 'match',
|
|
'tag', 'kind', 'resource_id', 'user_id', 'page', 'limit', 'sort', 'locale',
|
|
'measures', 'column_order', 'condition_values'].freeze
|
|
attr_reader :population, :axes, :page, :limit, :mode
|
|
|
|
def initialize(params)
|
|
params = CountsResourceDefinition.normalise_params(params)
|
|
QueryState.validate!(params, KEYS + ['measures'])
|
|
raise ArgumentError unless params.values.all? { _1.is_a?(String) }
|
|
|
|
params = CountsDefaults.for(params).slice(*KEYS).merge(params)
|
|
@params = params
|
|
@population = CountsPopulation.new(params)
|
|
@definition = population.definition
|
|
@connection = @definition.model.connection
|
|
@axes = CountsAxes.new(population)
|
|
@page = Integer(params.fetch('page', '1'), 10)
|
|
@limit = Integer(params.fetch('limit', '20'), 10)
|
|
@mode = params.fetch('mode', 'summary')
|
|
raise ArgumentError unless page.between?(1, 1_000_000) && limit.between?(1, 100)
|
|
raise ArgumentError unless ['summary', 'details'].include?(mode)
|
|
|
|
@sort = QueryState.list(params.fetch('sort', '')).map { _1.split(':', -1) }
|
|
allowed = mode == 'summary' ? axes.all.each_index.map { "a#{ _1 }" } + ['count'] :
|
|
@definition.fields.keys + ['latest'] +
|
|
(@definition.resource == 'tags' ? @definition.name_columns.map { _1[:key] } : []) +
|
|
(@definition.resource == 'posts' ? ['tags', 'parents'] : []) +
|
|
(@definition.resource == 'tags' ? ['parents', 'deprecated'] : [])
|
|
raise ArgumentError if @sort.map(&:first).uniq.length != @sort.length
|
|
@sort.each do |key, direction, *extra|
|
|
measure = mode == 'summary' && key&.match?(/\Am[0-9]{1,2}\z/)
|
|
measure_values(key) if measure
|
|
if extra.any? || (!allowed.include?(key) && !measure) || !['asc', 'desc'].include?(direction)
|
|
raise ArgumentError
|
|
end
|
|
end
|
|
end
|
|
|
|
def call
|
|
ApplicationRecord.transaction do
|
|
result = metadata.merge(mode == 'summary' ? summary : details)
|
|
visible = mode == 'summary' ? result[:visible_output_count] : result[:count]
|
|
totals = output_totals(visible)
|
|
result.merge(visible_output_count: visible,
|
|
**totals,
|
|
output_count_scope: 'before_authorisation',
|
|
output_count_unit: 'regular_result_rows',
|
|
page_grand_total_repeated: mode == 'summary' && axes.rows.any? &&
|
|
result[:group_count] > limit)
|
|
end
|
|
end
|
|
|
|
def metadata
|
|
{ population_count: nil, population_count_computed: false,
|
|
excluded_count: nil, excluded_count_scope: 'current_query',
|
|
population_count_scope: 'all_recorded_versions',
|
|
page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence,
|
|
definition: population.history ? 'history' : 'latest',
|
|
timezone: population.timezone.identifier, rows: axes.rows, cols: axes.columns,
|
|
fields: @definition.metadata,
|
|
count: record_counts[:visible] }
|
|
end
|
|
|
|
def summary
|
|
if axes.all.empty?
|
|
headers = [{ values: [], kind: 'data', depth: 0, axis_count: 0, label_key: nil }]
|
|
return { group_count: 1, visible_output_count: 1, column_values: [[]],
|
|
row_headers: page == 1 ? headers : [], column_headers: headers, pivot_totals: [],
|
|
cells: page == 1 ? [{ values: [], count: record_counts[:visible] }] : [],
|
|
axis_labels: {} }
|
|
end
|
|
expressions = axes.expressions
|
|
selected = expressions.each_with_index.map do |expression, index|
|
|
"#{ expression } AS a#{ index }"
|
|
end
|
|
source = population.scope
|
|
source = source.joins(axes.joins) unless axes.joins.empty?
|
|
sql = if expressions.empty?
|
|
source.except(:select).select('COUNT(DISTINCT v.id) AS count').to_sql
|
|
else
|
|
source.except(:select).group(*expressions.map { Arel.sql(_1) })
|
|
.select(Arel.sql((selected + ['COUNT(DISTINCT v.id) AS count']).join(', ')))
|
|
.to_sql
|
|
end
|
|
prefix = "WITH cells AS (#{ sql })"
|
|
column_values = limited_columns(prefix)
|
|
row_keys = axes.rows.each_index.map { "a#{ _1 }" }
|
|
# Page whole row tuples so a pivot row is never split across display pages.
|
|
row_selection = row_keys.empty? ? '1 AS singleton' : row_keys.join(', ')
|
|
row_group = row_keys.empty? ? 'GROUP BY singleton' : "GROUP BY #{ row_keys.join(', ') }"
|
|
measure_joins = []
|
|
extras = []
|
|
row_sort = @sort.each_with_index.filter_map do |(key, direction), index|
|
|
if key == 'count' || key.match?(/\Am[0-9]+\z/)
|
|
values = key == 'count' ? [] : measure_values(key)
|
|
relation = measure_relation(values)
|
|
matches = row_keys.map { "grouped.#{ _1 } <=> sort#{ index }.#{ _1 }" }
|
|
condition = matches.empty? ? 'TRUE' : matches.join(' AND ')
|
|
measure_joins << "LEFT JOIN (#{ relation }) sort#{ index } ON #{ condition }"
|
|
extras << "sort#{ index }.count AS sort#{ index }"
|
|
"sort#{ index } IS NULL ASC, sort#{ index } #{ direction.upcase }"
|
|
elsif row_keys.include?(key)
|
|
axis_order_sql(key, direction)
|
|
end
|
|
end
|
|
order_rows = row_sort + row_keys.map { axis_order_sql(_1) }
|
|
order_rows = ['singleton ASC'] if order_rows.empty?
|
|
grouped = "SELECT #{ row_selection } FROM cells #{ row_group }"
|
|
selection = (['grouped.*'] + extras).join(', ')
|
|
group_sql = "SELECT #{ selection } FROM (#{ grouped }) grouped #{ measure_joins.join(' ') }"
|
|
join = row_keys.map { "cells.#{ _1 } <=> page_rows.#{ _1 }" }.join(' AND ')
|
|
join = 'TRUE' if row_keys.empty?
|
|
order_cols = @sort.filter_map do |key, direction|
|
|
next unless key.match?(/\Aa[0-9]+\z/) && !row_keys.include?(key)
|
|
|
|
axis_order_sql(key, direction, qualifier: 'cells')
|
|
end
|
|
order = ['page_rows.position'] + order_cols +
|
|
axes.all.each_index.map { axis_order_sql("a#{ _1 }", qualifier: 'cells') }
|
|
result = @connection.select_all(<<~SQL).to_a
|
|
#{ prefix }, row_groups AS (#{ group_sql }),
|
|
numbered AS (SELECT row_groups.*,
|
|
ROW_NUMBER() OVER (ORDER BY #{ order_rows.join(', ') }) AS position FROM row_groups),
|
|
group_totals AS (SELECT COUNT(*) AS group_count FROM row_groups)
|
|
SELECT /*+ MAX_EXECUTION_TIME(10000) */ cells.*,
|
|
page_rows.position AS counts_position, group_totals.group_count AS counts_group_count
|
|
FROM group_totals LEFT JOIN numbered page_rows
|
|
ON page_rows.position > #{ (page - 1) * limit }
|
|
AND page_rows.position <= #{ page * limit + 1 }
|
|
LEFT JOIN cells ON #{ join }
|
|
ORDER BY #{ order.join(', ') }
|
|
SQL
|
|
total = result.first.fetch('counts_group_count').to_i
|
|
next_row = result.find { _1['counts_position'].to_i == page * limit + 1 }
|
|
records = result.select do |record|
|
|
record['counts_position'] && record['counts_position'].to_i <= page * limit
|
|
end
|
|
row_values = records.map { |record| row_keys.map { record[_1] } }.uniq
|
|
row_depths = CountsPivotLayout.depths(row_keys, @sort)
|
|
column_keys = axes.columns.each_index.map { "a#{ axes.rows.length + _1 }" }
|
|
column_sort = @sort
|
|
if @params.key?('column_order') && column_keys.any? &&
|
|
!(@sort.first&.first == 'count' || @sort.first&.first&.match?(/\Am[0-9]+\z/))
|
|
column_sort = [[column_keys.first, 'asc']] + @sort
|
|
end
|
|
column_depths = CountsPivotLayout.depths(column_keys, column_sort, measures: false)
|
|
row_headers = CountsPivotLayout.headers(row_values, row_depths,
|
|
next_values: next_row && row_keys.map { next_row[_1] }, grand: axes.rows.any?,
|
|
axis_count: axes.rows.length, other_count: axes.columns.length)
|
|
if axes.rows.empty? && axes.columns.any?
|
|
row_headers = CountsPivotLayout.headers([[]], [], axis_count: 0,
|
|
other_count: axes.columns.length).map { _1.merge(kind: 'grand_total') }
|
|
end
|
|
column_headers = CountsPivotLayout.headers(column_values, column_depths,
|
|
grand: axes.columns.any?, axis_count: axes.columns.length,
|
|
other_count: axes.rows.length, direction: :column)
|
|
if axes.columns.empty? && axes.rows.any?
|
|
column_headers = CountsPivotLayout.headers([[]], [], axis_count: 0,
|
|
other_count: axes.rows.length, direction: :column)
|
|
.map { _1.merge(kind: 'grand_total') }
|
|
end
|
|
totals = CountsPivotTotals.new(population, axes, row_headers, column_headers).call
|
|
{ group_count: total,
|
|
visible_output_count: axes.rows.empty? ? 1 : total,
|
|
column_values:, row_headers:, column_headers:, pivot_totals: totals,
|
|
cells: records.map do |row|
|
|
{ values: axes.all.each_index.map { row["a#{ _1 }"] }, count: row['count'].to_i }
|
|
end, axis_labels: axis_labels(records, column_values), axis_names: @axis_names }
|
|
end
|
|
|
|
def details
|
|
query = population.scope.select(Arel.sql(
|
|
"(#{ population.latest_sql }) AS counts_latest"))
|
|
order = sort_sql { detail_expression(_1) }
|
|
query = query.order(Arel.sql((order + ['v.id DESC']).join(', ')))
|
|
.limit(limit).offset((page - 1) * limit)
|
|
rows = query.to_a
|
|
if ['posts', 'tags'].include?(@definition.resource)
|
|
point = population.history ? population.at :
|
|
population.timestamp(@params['at']) || population.at
|
|
values = CountsVersionRepr.many(rows, resource: @definition.resource,
|
|
language_code: locale.language_code,
|
|
at: point, history_flg: population.history)
|
|
values.each_with_index { |value, index| value[:event_type] = rows[index].event_type }
|
|
else
|
|
users = User.where(id: rows.map(&:created_by_user_id)).pluck(:id, :name).to_h
|
|
values = rows.map do |row|
|
|
attributes = @definition.fields.keys.to_h do |key|
|
|
[key, row.public_send(@definition.column(key))]
|
|
end
|
|
attributes.merge('id' => row.id, 'resource_id' => row[@definition.keys.first],
|
|
'latest' => row.attributes['counts_latest'].to_i == 1,
|
|
'created_by_user' => row.created_by_user_id &&
|
|
{ id: row.created_by_user_id, name: users[row.created_by_user_id] })
|
|
end
|
|
end
|
|
{ versions: values }
|
|
end
|
|
|
|
def each_page(before_page: nil)
|
|
total = nil
|
|
loop do
|
|
before_page&.call
|
|
result = call
|
|
total ||= mode == 'summary' ? result[:group_count] : result[:count]
|
|
yield result
|
|
break if page * limit >= total
|
|
|
|
@page += 1
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def unrestricted_population
|
|
@unrestricted_population ||= CountsPopulation.new(
|
|
@params.merge('execution_at' => population.at.iso8601(6)), authorisation_flg: false)
|
|
end
|
|
|
|
def record_counts
|
|
return @record_counts if @record_counts
|
|
|
|
if HistoryAccess.admin?
|
|
total = population.scope.except(:select)
|
|
.optimizer_hints('MAX_EXECUTION_TIME(10000)').count
|
|
return @record_counts = { total:, visible: total, denied: 0 }
|
|
end
|
|
total, denied = unrestricted_population.scope.except(:select)
|
|
.optimizer_hints('MAX_EXECUTION_TIME(10000)').pick(Arel.sql('COUNT(*)'),
|
|
Arel.sql("COALESCE(SUM(CASE WHEN #{ HistoryAccess.sql(@definition.model, 'v') } " +
|
|
'THEN 0 ELSE 1 END), 0)'))
|
|
# Related/previous-version predicates may differ under public authority.
|
|
shared = QueryState.records(@params, 'conditions').empty? &&
|
|
QueryState.records(@params, 'drill').none? { _1['field'] == 'changed_field' }
|
|
visible = if shared
|
|
total.to_i - denied.to_i
|
|
else
|
|
population.scope.except(:select)
|
|
.optimizer_hints('MAX_EXECUTION_TIME(10000)').count
|
|
end
|
|
@record_counts = { total: total.to_i, denied: denied.to_i, visible: }
|
|
end
|
|
|
|
def output_totals(visible_rows)
|
|
return @output_totals if @output_totals
|
|
|
|
if HistoryAccess.admin?
|
|
return @output_totals = { output_count: visible_rows, excluded_count: 0 }
|
|
end
|
|
# Only numerical aggregates leave this population. Labels, snapshots and
|
|
# detail pagination continue to use the authorised population exclusively.
|
|
source = unrestricted_population
|
|
rows = if mode == 'details'
|
|
record_counts[:total]
|
|
else
|
|
summary_output_count(source)
|
|
end
|
|
@output_totals = { output_count: rows, excluded_count: record_counts[:denied] }
|
|
end
|
|
|
|
def summary_output_count(source)
|
|
keys = axes.rows.each_index.map { "a#{ _1 }" }
|
|
return 1 if keys.empty?
|
|
|
|
unrestricted_axes = CountsAxes.new(source)
|
|
relation = source.scope
|
|
joins = unrestricted_axes.joins
|
|
relation = relation.joins(joins) unless joins.empty?
|
|
expressions = unrestricted_axes.expressions
|
|
selected = expressions.each_with_index.map { |expression, index|
|
|
"#{ expression } AS a#{ index }"
|
|
}
|
|
sql = relation.except(:select).select(Arel.sql(selected.join(', ')))
|
|
.distinct.to_sql
|
|
prefix = "WITH cells AS (#{ sql })"
|
|
@connection.select_value(<<~SQL).to_i
|
|
#{ prefix }
|
|
SELECT /*+ MAX_EXECUTION_TIME(10000) */ COUNT(*) FROM (
|
|
SELECT #{ keys.join(', ') } FROM cells GROUP BY #{ keys.join(', ') }
|
|
) output_groups
|
|
SQL
|
|
end
|
|
|
|
def locale = @locale ||= Locale.find_by!(code: @params.fetch('locale', 'ja'))
|
|
|
|
def limited_columns(prefix)
|
|
keys = axes.columns.each_index.map { "a#{ axes.rows.length + _1 }" }
|
|
return [[]] if keys.empty?
|
|
|
|
configured = @sort.select { |key, _direction| keys.include?(key) }
|
|
directions = (configured + keys.map { [_1, 'asc'] }).uniq { _1.first }
|
|
order = directions.map { |key, direction| axis_order_sql(key, direction) }.join(', ')
|
|
custom = QueryState.records(@params, 'column_order')
|
|
if custom.any?
|
|
raise ArgumentError unless custom.map { _1['value'] }.uniq.length == custom.length
|
|
|
|
cases = custom.each_with_index.map do |record, index|
|
|
"WHEN #{ keys.first } <=> #{ @connection.quote(record['value']) } THEN #{ index }"
|
|
end.join(' ')
|
|
order = "CASE #{ cases } ELSE #{ custom.length } END, #{ order }"
|
|
end
|
|
values = @connection.select_rows(<<~SQL)
|
|
#{ prefix }
|
|
SELECT /*+ MAX_EXECUTION_TIME(10000) */ DISTINCT #{ keys.join(', ') } FROM cells
|
|
ORDER BY #{ order } LIMIT #{ MAX_COLUMNS + 1 }
|
|
SQL
|
|
raise TooManyColumns if values.length > MAX_COLUMNS
|
|
|
|
values
|
|
end
|
|
|
|
def measure_relation(values)
|
|
row_expressions = axes.expressions.first(axes.rows.length)
|
|
source = population.scope.except(:select, :order)
|
|
fields = axes.rows + axes.columns.first(values.length)
|
|
joins = axes.joins(fields.map { _1.fetch('field') })
|
|
source = source.joins(joins) unless joins.empty?
|
|
values.each_with_index do |value, index|
|
|
expression = axes.expressions[axes.rows.length + index]
|
|
source = source.where("(#{ expression }) <=> ?", value)
|
|
end
|
|
selected = row_expressions.each_with_index.map { |expression, index|
|
|
"#{ expression } AS a#{ index }"
|
|
} + ['COUNT(DISTINCT v.id) AS count']
|
|
source = source.group(*row_expressions.map { Arel.sql(_1) }) if row_expressions.any?
|
|
source.select(Arel.sql(selected.join(', '))).to_sql
|
|
end
|
|
|
|
def axis_order_sql(key, direction = 'asc', qualifier: nil)
|
|
column = qualifier ? "#{ qualifier }.#{ key }" : key
|
|
expression = column
|
|
if axes.all.fetch(key.delete_prefix('a').to_i).fetch('field') == 'category'
|
|
# Tag's enum preserves the same category order as TagsController.
|
|
cases = Tag.categories.keys.each_with_index.map { |category, index|
|
|
"WHEN #{ @connection.quote(category) } THEN #{ index }"
|
|
}.join(' ')
|
|
expression = "CASE #{ column } #{ cases } ELSE #{ Tag.categories.length } END"
|
|
end
|
|
"#{ column } IS NULL ASC, #{ expression } #{ direction.upcase }"
|
|
end
|
|
|
|
def measure_values(key)
|
|
records = QueryState.records(@params, 'measures').select { _1['key'] == key }
|
|
raise ArgumentError if records.empty?
|
|
raise ArgumentError unless records.map { _1['axis'] }.sort == (0...records.length).to_a
|
|
|
|
value = records.sort_by { _1['axis'] }.map { _1['value'] }
|
|
unless (value.is_a?(Array) && value.length <= axes.columns.length &&
|
|
value.all? { _1.nil? || _1.is_a?(String) || _1.is_a?(Numeric) })
|
|
raise ArgumentError
|
|
end
|
|
value
|
|
end
|
|
|
|
def detail_expression(key)
|
|
return "(#{ population.latest_sql })" if key == 'latest'
|
|
return @definition.name_expression(key) if @definition.resource == 'tags' &&
|
|
@definition.name_column(key)
|
|
aliases = { 'tags' => 'tags_json', 'parents' => @definition.resource == 'posts' ?
|
|
'parent_post_ids' : 'parent_tag_ids', 'deprecated' => 'deprecated_at' }
|
|
field = aliases.fetch(key, key)
|
|
expression = @definition.expression(field)
|
|
return expression unless @definition.fields[field] == :json
|
|
|
|
"CAST(#{ expression } AS CHAR) COLLATE utf8mb4_bin"
|
|
end
|
|
|
|
def sort_sql
|
|
@sort.flat_map do |key, direction|
|
|
expression = yield key
|
|
["(#{ expression }) IS NULL ASC", "#{ expression } #{ direction.upcase }"]
|
|
end
|
|
end
|
|
|
|
def axis_labels(records, column_values)
|
|
result = {}
|
|
@axis_names = {}
|
|
axes.all.each_with_index do |axis, index|
|
|
values = records.map { _1["a#{ index }"] }
|
|
if index >= axes.rows.length
|
|
values += column_values.map { _1[index - axes.rows.length] }
|
|
end
|
|
values = values.compact.uniq
|
|
labels = case axis['field']
|
|
when 'user'
|
|
User.where(id: values).pluck(:id, :name).to_h.transform_keys(&:to_s)
|
|
when 'language_code', 'source_language_code'
|
|
values.to_h { [_1, _1] }
|
|
when 'tags'
|
|
labels = tag_labels(values)
|
|
@axis_names["a#{ index }"] = @tag_names
|
|
labels
|
|
else {}
|
|
end
|
|
result["a#{ index }"] = labels
|
|
end
|
|
result
|
|
end
|
|
|
|
def tag_labels(values)
|
|
internal = values.grep(/\Ainternal:/).map { _1.split(':').last.to_i }
|
|
external = values.grep(/\Aexternal:/).map { _1.split(':').last.to_i }
|
|
point = population.history ? population.at :
|
|
population.timestamp(@params['at']) || population.at
|
|
labels = {}
|
|
@tag_names = {}
|
|
sources = [[TagVersion, internal, 'internal'], [NicoTagVersion, external, 'external']]
|
|
sources.each do |model, ids, kind|
|
|
next if ids.empty?
|
|
|
|
table = model.table_name
|
|
scope = HistoryAccess.visible(model.where(tag_id: ids)).where('created_at <= ?', point)
|
|
bound = @connection.quote(point)
|
|
if population.history && @definition.resource == 'posts'
|
|
key = kind == 'internal' ? 'tag_id' : 'external_tag_id'
|
|
pairs = population.scope.except(:select, :order).joins(
|
|
"JOIN JSON_TABLE(v.tags_json, '$[*]' COLUMNS (tag_id BIGINT PATH '$.#{ key }', " +
|
|
"version_no INT PATH '$.version_no' NULL ON EMPTY)) recorded_tag")
|
|
.where('recorded_tag.tag_id IN (?)', ids).group('recorded_tag.tag_id')
|
|
value = kind == 'internal' ? 'MAX(recorded_tag.version_no)' : 'MAX(v.created_at)'
|
|
pairs = pairs.select(Arel.sql("recorded_tag.tag_id, #{ value } AS reference_point"))
|
|
scope = scope.joins("JOIN (#{ pairs.to_sql }) recorded " +
|
|
"ON recorded.tag_id = #{ table }.tag_id")
|
|
if kind == 'internal'
|
|
scope = scope.where("#{ table }.version_no = recorded.reference_point")
|
|
else
|
|
scope = scope.where("#{ table }.created_at <= recorded.reference_point")
|
|
bound = 'recorded.reference_point'
|
|
end
|
|
end
|
|
unless population.history && kind == 'internal' && @definition.resource == 'posts'
|
|
scope = scope.where("NOT EXISTS (SELECT 1 FROM #{ table } newer " +
|
|
"WHERE newer.tag_id = #{ table }.tag_id AND newer.created_at <= #{ bound } " +
|
|
"AND newer.version_no > #{ table }.version_no)")
|
|
end
|
|
scope.each do |row|
|
|
id = "#{ kind }:#{ row.tag_id }"
|
|
labels[id] = row.name
|
|
@tag_names[id] = row.respond_to?(:names) ? row.names : {}
|
|
end
|
|
end
|
|
labels
|
|
end
|
|
end
|