#434

backend/app/controllers/counts_tables_controller.rb

@@ -20,4 +20,12 @@ class CountsTablesController < ApplicationController
   rescue ArgumentError, TypeError, KeyError, JSON::ParserError
     head :bad_request
   end
+
+  def candidates
+    resolve_locale!
+    render json: CountsCandidates.new(request.query_parameters.merge(
+        'resource' => request.path_parameters[:resource])).call
+  rescue ArgumentError, TypeError, KeyError
+    head :bad_request
+  end
 end

backend/app/services/counts_candidates.rb

@@ -0,0 +1,55 @@
+# Bounded, authorised candidates at the same recorded point as the Counts query.
+class CountsCandidates
+  KEYS = ['resource', 'field', 'query', 'category', 'page', 'locale', 'at',
+          'execution_at', 'timezone'].freeze
+  LIMIT = 20
+
+  def initialize(params)
+    QueryState.validate!(params, KEYS)
+    raise ArgumentError unless params.fetch('resource') == 'posts'
+    raise ArgumentError unless ['tags', 'tags_json'].include?(params.fetch('field'))
+
+    @params = params
+    @page = Integer(params.fetch('page', '1'), 10)
+    raise ArgumentError unless @page.between?(1, 100_000)
+    raise ArgumentError if params.fetch('query', '').bytesize > 1000
+    category = params['category']
+    raise ArgumentError if category.present? && !Tag.categories.key?(category)
+  end
+
+  def call
+    options = @params.slice('locale', 'at', 'execution_at', 'timezone')
+        .merge('definition' => 'latest', 'resource' => 'tags')
+    internal = CountsPopulation.new(options)
+    external = CountsPopulation.new(options.merge('resource' => 'nico_tags',
+        'execution_at' => internal.at.iso8601(6)))
+    language = Locale.find_by!(code: @params.fetch('locale', 'ja')).language_code
+    connection = TagVersion.connection
+    path = connection.quote("$.#{ language.to_json }.name")
+    name = "COALESCE(JSON_UNQUOTE(JSON_EXTRACT(v.names, #{ path })), v.name)"
+    sources = [[internal, 'internal', name, 'v.category'],
+               [external, 'external', 'v.name', 'NULL']].map do |population, kind, title, category|
+      population.scope.except(:select).select(Arel.sql(
+          "CONCAT('#{ kind }:', v.tag_id) AS value, v.tag_id AS id, " +
+          "#{ title } AS name, #{ category } AS category, '#{ kind }' AS kind")).to_sql
+    end
+    clauses = []
+    binds = []
+    if @params['query'].present?
+      clauses << '(name LIKE ? OR CAST(id AS CHAR) = ?)'
+      binds += ["%#{ ApplicationRecord.sanitize_sql_like(@params['query']) }%", @params['query']]
+    end
+    if @params['category'].present?
+      clauses << 'category = ?'
+      binds << @params['category']
+    end
+    where = clauses.empty? ? '' : "WHERE #{ clauses.join(' AND ') }"
+    sql = <<~SQL
+      SELECT /*+ MAX_EXECUTION_TIME(10000) */ * FROM (#{ sources.join(' UNION ALL ') }) choices
+      #{ where } ORDER BY kind, id LIMIT #{ LIMIT + 1 } OFFSET #{ (@page - 1) * LIMIT }
+    SQL
+    rows = connection.select_all(ApplicationRecord.sanitize_sql_array([sql, *binds])).to_a
+    { rows: rows.first(LIMIT), page: @page, more: rows.length > LIMIT,
+      execution_at: internal.at.iso8601(6) }
+  end
+end

backend/app/services/counts_condition.rb

@@ -44,12 +44,31 @@ class CountsCondition
       result.find { _1[:key] == 'tags_json' }.merge!(type: :string, multiple: true, collection: true,
           operators: CountsResourceDefinition::OPERATORS.fetch(:string))
     end
-    result + [{ key: 'user.id', type: :integer, nullable: true,
+    result += [{ key: 'user.id', type: :integer, nullable: true,
                 labels: ['history:actor', 'counts:columns.resourceId'],
                 operators: CountsResourceDefinition::OPERATORS.fetch(:integer) },
               { key: 'user.name', type: :string, nullable: true,
                 labels: ['history:actor', 'history:name'],
                 operators: CountsResourceDefinition::OPERATORS.fetch(:string) }]
+    result.each do |field|
+      next if field[:type] == :interval
+
+      if field[:multiple]
+        field[:operators] = field[:operators] - ['in', 'not_in'] + ['any', 'all', 'none']
+      end
+      field[:input_types] = ['single', 'multiple']
+      if ['category', 'tag_category'].include?(field[:key].split('.').last)
+        field[:candidate_source] = 'categories'
+        field[:choices] = Tag.categories.keys.map { |value|
+          { value:, label: "common:categories.#{ value }" }
+        }
+      elsif definition.resource == 'posts' && ['tags', 'tags_json'].include?(field[:key])
+        field[:candidate_source] = 'tags'
+        field[:set_type] = 'tag_reference'
+      end
+      field[:input_types] << 'list' if field[:candidate_source]
+    end
+    result
   end
 
   def initialize(population)
@@ -61,11 +80,17 @@ class CountsCondition
 
   def apply(scope, entry)
     raise ArgumentError unless entry.is_a?(Hash)
-    raise ArgumentError if (entry.keys - ['field', 'operator', 'value', 'end', 'blank']).any?
+    allowed = ['field', 'operator', 'value', 'end', 'blank', 'input', 'values']
+    raise ArgumentError if (entry.keys - allowed).any?
 
     field = @definition.filter_metadata.find { _1[:key] == entry.fetch('field') }
     raise ArgumentError unless field && field[:operators].include?(entry.fetch('operator'))
     raise ArgumentError unless [true, false, nil].include?(entry['blank'])
+    input = entry.fetch('input', 'single')
+    raise ArgumentError unless ['single', 'multiple', 'list'].include?(input)
+    if input == 'list' && !field[:candidate_source]
+      raise ArgumentError
+    end
     values = [entry['value'], entry['end']]
     raise ArgumentError unless values.all? { _1.nil? || (_1.is_a?(String) && _1.bytesize <= 8192) }
     if entry['operator'] != 'range' && !values[1].nil? && values[1] != ''
@@ -84,6 +109,10 @@ class CountsCondition
             end
       return scope.where(Arel.sql(sql))
     end
+    if ['in', 'not_in', 'any', 'all', 'none'].include?(entry['operator'])
+      return set_condition(scope, entry, field)
+    end
+    raise ArgumentError if entry.key?('values') || input != 'single'
     return scope if values.all? { _1.nil? || _1 == '' }
 
     parsed = values.map { parse(_1, type) }
@@ -122,6 +151,58 @@ class CountsCondition
 
   private
 
+  def set_condition(scope, entry, field)
+    return scope unless entry.key?('values')
+
+    values = entry['values']
+    raise ArgumentError unless values.is_a?(Array) && values.length <= 100
+    raise ArgumentError if [entry['value'], entry['end']].any? { !_1.nil? && _1 != '' }
+
+    parsed = values.map do |value|
+      unless (value.is_a?(String) || value.is_a?(Numeric) || [true, false].include?(value))
+        raise ArgumentError
+      end
+      raise ArgumentError if value.to_s.bytesize > 8192
+
+      if field[:set_type] == 'tag_reference'
+        raise ArgumentError unless value.is_a?(String) &&
+            value.match?(/\A(?:internal|external):[1-9][0-9]{0,17}\z/)
+        value
+      else
+        raise ArgumentError if value == '' && ![:string, :enum].include?(field[:type])
+        result = value == '' ? '' : parse(value.to_s, field[:type])
+        validate_enum!(field[:key], result) if field[:type] == :enum
+        result
+      end
+    end
+    raise ArgumentError unless parsed.uniq == parsed
+
+    operator = entry['operator']
+    if parsed.empty?
+      return ['in', 'any'].include?(operator) ? scope.none : scope
+    end
+    source, expression = source_for(field[:key])
+    if field[:set_type] == 'tag_reference'
+      source = post_tag_source
+      id = "COALESCE(#{ scalar('tag_snapshot.item', 'tag_id', :integer) }, " +
+          "#{ scalar('tag_snapshot.item', 'external_tag_id', :integer) })"
+      expression = "CONCAT(CASE WHEN #{ scalar('tag_snapshot.item', 'tag_id') } IS NULL " +
+          "THEN 'external:' ELSE 'internal:' END, #{ id })"
+    end
+    predicates = parsed.map do |_value|
+      clause = "#{ expression } = ?"
+      clause = "EXISTS (SELECT 1 FROM #{ source } WHERE #{ clause })" if source
+      clause
+    end
+    conjunction = operator == 'all' ? ' AND ' : ' OR '
+    predicate = "(#{ predicates.join(conjunction) })"
+    if ['not_in', 'none'].include?(operator)
+      predicate = source ? "NOT #{ predicate }" :
+          "(NOT #{ predicate } OR #{ blank_sql(expression, field[:type]) })"
+    end
+    scope.where(predicate, *parsed)
+  end
+
   def parse(value, type)
     return nil if value.nil? || value == ''
 

backend/app/services/counts_pivot_layout.rb

@@ -10,7 +10,21 @@ class CountsPivotLayout
     (1...keys.length).select { order.first(_1).sort == keys.first(_1).sort }
   end
 
-  def self.headers(values, depths, next_values: nil, grand: false)
+  def self.label_key(row_count, column_count, row_depth, column_depth)
+    total = row_count + column_count
+    collapsed = total - row_depth - column_depth
+    return nil if collapsed.zero?
+    return 'grandTotal' if total >= 3 && collapsed == total
+    if (collapsed == 1 &&
+        ((row_count > 1 && row_depth < row_count) ||
+         (column_count > 1 && column_depth < column_count)))
+      return 'subtotal'
+    end
+    'singleTotal'
+  end
+
+  def self.headers(values, depths, next_values: nil, grand: false, axis_count: nil,
+                   other_count: 0, direction: :row)
     result = []
     values.each_with_index do |tuple, index|
       result << { values: tuple, kind: 'data' }
@@ -23,6 +37,15 @@ class CountsPivotLayout
       end
     end
     result << { values: [], kind: 'grand_total' } if grand
+    count = axis_count || values.first&.length || 0
+    result.each do |header|
+      depth = header[:values].length
+      label = if header[:kind] != 'data'
+                direction == :row ? label_key(count, other_count, depth, other_count) :
+                    label_key(other_count, count, other_count, depth)
+              end
+      header.merge!(depth:, axis_count: count, label_key: label)
+    end
     result
   end
 end

backend/app/services/counts_pivot_totals.rb

@@ -53,6 +53,9 @@ class CountsPivotTotals
       column_depth = record['column_depth'].to_i
       { row: (0...row_depth).map { record["v#{ _1 }"] },
         column: (0...column_depth).map { record["v#{ row_depth + _1 }"] },
+        row_depth:, column_depth:,
+        label_key: CountsPivotLayout.label_key(@axes.rows.length, @axes.columns.length,
+            row_depth, column_depth),
         count: record['count'].to_i }
     end
   end

backend/app/services/counts_population.rb

@@ -103,7 +103,21 @@ class CountsPopulation
     conditions = QueryState.records(params, 'conditions')
     raise ArgumentError unless conditions.is_a?(Array) && conditions.length <= 100
 
-    conditions.each { apply_condition(_1) }
+    values = QueryState.records(params, 'condition_values')
+    values.each do |entry|
+      index = entry.fetch('condition')
+      raise ArgumentError unless index.between?(0, conditions.length - 1)
+      raise ArgumentError unless conditions[index]['values'] == true
+    end
+    conditions.each_with_index do |entry, index|
+      if entry.key?('values')
+        raise ArgumentError unless entry['values'] == true
+
+        entry = entry.merge('values' => values.select { _1['condition'] == index }
+            .map { _1.fetch('value') })
+      end
+      apply_condition(entry)
+    end
     drills = QueryState.records(params, 'drill')
     raise ArgumentError unless drills.is_a?(Array) && drills.length <= 100
 

backend/app/services/counts_resource_definition.rb

@@ -46,9 +46,10 @@ class CountsResourceDefinition
           'source_kind' => :enum, 'source_uri' => :string, 'source_path' => :string,
           'source_file_id' => :string, 'normalized_source_key' => :string }] }.freeze
   OPERATORS = {
-      string: ['eq', 'ne', 'like'], enum: ['eq', 'ne'], boolean: ['eq', 'ne'],
-      integer: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'],
-      datetime: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'],
+      string: ['eq', 'ne', 'like', 'in', 'not_in'],
+      enum: ['eq', 'ne', 'in', 'not_in'], boolean: ['eq', 'ne', 'in', 'not_in'],
+      integer: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range', 'in', 'not_in'],
+      datetime: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range', 'in', 'not_in'],
       interval: ['range'] }.freeze
   LABELS = {
       'resource_id' => 'counts:columns.resourceId', 'version_no' => 'counts:columns.version',

backend/app/services/counts_table_export.rb

@@ -143,8 +143,8 @@ class CountsTableExport
   end
 
   def total_label(header)
-    key = header[:kind] == 'grand_total' ?
-        (@query.axes.all.length == 1 ? 'singleTotal' : 'grandTotal') : 'subtotal'
+    key = header[:label_key] || (header[:kind] == 'grand_total' ?
+        (@query.axes.all.length == 1 ? 'singleTotal' : 'grandTotal') : 'subtotal')
     @labels.fetch(key, '')
   end
 
@@ -152,9 +152,10 @@ class CountsTableExport
     values = header[:values].each_with_index.map { |value, index|
       summary_value(value, index, page)
     }
-    if header[:kind] == 'data' && @previous_pivot_row&.fetch(:kind) == 'data'
+    if @previous_pivot_row
       values.each_index do |index|
-        if header[:values].first(index + 1) == @previous_pivot_row[:values].first(index + 1)
+        if (@previous_pivot_row[:values].length > index &&
+            header[:values].first(index + 1) == @previous_pivot_row[:values].first(index + 1))
           values[index] = ''
         end
       end
@@ -234,6 +235,8 @@ class CountsTableExport
     { row: row[:values], column: column[:values], kind:,
       row_kind: row[:kind], column_kind: column[:kind],
       row_depth: row[:values].length, column_depth: column[:values].length,
+      label_key: CountsPivotLayout.label_key(@query.axes.rows.length, @query.axes.columns.length,
+          row[:values].length, column[:values].length),
       count: @pivot_lookup.fetch([row[:values], column[:values]], 0) }
   end
 

backend/app/services/counts_table_query.rb

@@ -5,7 +5,8 @@ class CountsTableQuery
 
   KEYS = ['resource', 'definition', 'mode', 'at', 'execution_at', 'from', 'through',
           'timezone', 'rows', 'cols', 'conditions', 'drill', 'tags', 'tag_refs', 'match',
-          'tag', 'kind', 'resource_id', 'user_id', 'page', 'limit', 'sort', 'locale', 'measures', 'column_order'].freeze
+          'tag', 'kind', 'resource_id', 'user_id', 'page', 'limit', 'sort', 'locale',
+          'measures', 'column_order', 'condition_values'].freeze
   attr_reader :population, :axes, :page, :limit, :mode
 
   def initialize(params)
@@ -145,14 +146,19 @@ class CountsTableQuery
     end
     column_depths = CountsPivotLayout.depths(column_keys, column_sort, measures: false)
     row_headers = CountsPivotLayout.headers(row_values, row_depths,
-        next_values: next_row && row_keys.map { next_row[_1] }, grand: axes.rows.any?)
+        next_values: next_row && row_keys.map { next_row[_1] }, grand: axes.rows.any?,
+        axis_count: axes.rows.length, other_count: axes.columns.length)
     if axes.rows.empty? && axes.columns.any?
-      row_headers = [{ values: [], kind: 'grand_total' }]
+      row_headers = CountsPivotLayout.headers([[]], [], axis_count: 0,
+          other_count: axes.columns.length).map { _1.merge(kind: 'grand_total') }
     end
     column_headers = CountsPivotLayout.headers(column_values, column_depths,
-        grand: axes.columns.any?)
+        grand: axes.columns.any?, axis_count: axes.columns.length,
+        other_count: axes.rows.length, direction: :column)
     if axes.columns.empty? && axes.rows.any?
-      column_headers = [{ values: [], kind: 'grand_total' }]
+      column_headers = CountsPivotLayout.headers([[]], [], axis_count: 0,
+          other_count: axes.rows.length, direction: :column)
+          .map { _1.merge(kind: 'grand_total') }
     end
     totals = CountsPivotTotals.new(population, axes, row_headers, column_headers).call
     { group_count: total,

backend/app/services/query_state.rb

@@ -1,7 +1,8 @@
 # Public query grammar shared with frontend/src/lib/queryState.ts.
 class QueryState
   SCHEMAS = {
-      'conditions' => ['field', 'operator', 'value', 'end', 'blank'],
+      'conditions' => ['field', 'operator', 'value', 'end', 'blank', 'input', 'values'],
+      'condition_values' => ['condition', 'value', 'value_type'],
       'drill' => ['field', 'grain', 'value', 'value_type'],
       'tag_refs' => ['name', 'tag_id', 'external_tag_id', 'negative', 'missing'],
       'column_order' => ['value', 'value_type'],
@@ -40,11 +41,11 @@ class QueryState
 
       _root, index, field = name.split('.')
       record = records[index.to_i] ||= {}
-      if ['blank', 'negative', 'missing'].include?(field)
+      if ['blank', 'negative', 'missing', 'values'].include?(field)
         raise ArgumentError unless ['0', '1'].include?(value)
 
         value = value == '1'
-      elsif ['tag_id', 'external_tag_id', 'axis'].include?(field)
+      elsif ['tag_id', 'external_tag_id', 'axis', 'condition'].include?(field)
         raise ArgumentError unless value.match?(/\A[0-9]{1,18}\z/)
 
         value = value.to_i
@@ -56,7 +57,7 @@ class QueryState
 
     records.sort.map do |_index, record|
       type = record.delete('value_type')
-      if ['drill', 'measures', 'column_order'].include?(key)
+      if ['drill', 'measures', 'column_order', 'condition_values'].include?(key)
         raise ArgumentError unless ['null', 'string', 'number', 'boolean'].include?(type)
 
         raise ArgumentError unless type == 'null' || record.key?('value')

backend/config/routes.rb

@@ -1,5 +1,6 @@
 Rails.application.routes.draw do
   get 'counts/definitions/:resource', to: 'counts_tables#definition'
+  get 'counts/definitions/:resource/candidates', to: 'counts_tables#candidates'
   get 'counts/query', to: 'counts_tables#index'
   post 'counts/export/jobs', to: 'counts_exports#create_job'
   get 'counts/export/jobs/:token', to: 'counts_exports#job_status'