diff --git a/backend/app/controllers/counts_exports_controller.rb b/backend/app/controllers/counts_exports_controller.rb index cb4d9de..de48b16 100644 --- a/backend/app/controllers/counts_exports_controller.rb +++ b/backend/app/controllers/counts_exports_controller.rb @@ -3,7 +3,9 @@ class CountsExportsController < ApplicationController def index locale = resolve_locale! - exporter = CountsExporter.new(request.request_parameters.merge(request.query_parameters), + payload = request.request_parameters.except('authenticity_token') + .merge(request.query_parameters) + exporter = CountsExporter.new(payload, language_code: locale.language_code) response.headers['Content-Type'] = exporter.format == 'json' ? 'application/json; charset=utf-8' : 'text/csv; charset=utf-8' @@ -14,7 +16,7 @@ class CountsExportsController < ApplicationController # Rack::ETag otherwise materialises the response to calculate a digest. response.headers['Last-Modified'] = Time.current.httpdate self.response_body = exporter.body - rescue ArgumentError, TypeError, JSON::ParserError + rescue ArgumentError, TypeError, KeyError, JSON::ParserError head :bad_request end end diff --git a/backend/app/controllers/counts_tables_controller.rb b/backend/app/controllers/counts_tables_controller.rb new file mode 100644 index 0000000..ccffdab --- /dev/null +++ b/backend/app/controllers/counts_tables_controller.rb @@ -0,0 +1,8 @@ +class CountsTablesController < ApplicationController + def index + resolve_locale! + render json: CountsTableQuery.new(request.query_parameters).call + rescue ArgumentError, TypeError, KeyError, JSON::ParserError + head :bad_request + end +end diff --git a/backend/app/controllers/posts_controller.rb b/backend/app/controllers/posts_controller.rb index 9513e97..dc052cc 100644 --- a/backend/app/controllers/posts_controller.rb +++ b/backend/app/controllers/posts_controller.rb @@ -449,24 +449,8 @@ class PostsController < ApplicationController private def tag_filter_references literals - raw = params[:tag_refs] - return nil if raw.blank? - - references = JSON.parse(raw) - valid = references.is_a?(Array) && references.all? do |reference| - next false unless reference.is_a?(Hash) && reference['name'].is_a?(String) - - ids = [reference['tag_id'], reference['external_tag_id']].compact - ((ids.any? || reference['missing'] == true) && - ids.all? { _1.is_a?(Integer) && _1.positive? } && - (!reference.key?('missing') || (reference['missing'] == true && ids.empty?)) && - (!reference.key?('negative') || [true, false].include?(reference['negative']))) - end - raise ActionController::BadRequest unless valid - return nil unless references.map { _1['name'] } == literals - - references - rescue JSON::ParserError, TypeError + PostTagSearch.parse_references(params[:tag_refs], literals) + rescue ArgumentError, JSON::ParserError, TypeError raise ActionController::BadRequest end @@ -482,17 +466,7 @@ class PostsController < ApplicationController def filter_posts_by_tags locale, tag_names, match_type references = tag_filter_references(tag_names) - literals = tag_names.each_with_index.map do |raw_name, index| - name = raw_name.sub(/\Anot:/i, '') - name = TagName.canonicalise(locale, name).first unless references - reference = references&.[](index) - negative = if reference&.key?('negative') - reference['negative'] - else - raw_name.downcase.start_with?('not:') - end - { name:, negative:, reference: } - end + literals = PostTagSearch.literals(locale, tag_names.join(' '), references:) return Post.all if literals.empty? @@ -523,31 +497,10 @@ class PostsController < ApplicationController end def tagged_post_ids_for locale, name, reference: nil - if reference - matches = Post.none - if reference['tag_id'] - matches = matches.or(Post.where(id: PostTag.where( - tag_id: reference['tag_id']).select(:post_id))) - end - if reference['external_tag_id'] - matches = matches.or(Post.where(id: PostExternalTag.where( - external_tag_id: reference['external_tag_id']).select(:post_id))) - end - return matches.select(:id) - end - - posts_by_internal_tags = - Post - .joins(tags: :tag_names) - .where(tag_names: { language_code: locale.language_code, name: }) - .select(:id) - - posts_by_external_tags = - Post - .joins(:external_tags) - .where("CONCAT(external_tags.platform, ':', external_tags.name) = ?", name) - - (posts_by_internal_tags + posts_by_external_tags).uniq(&:id) + internal, external = PostTagSearch.references(locale, { name:, reference: }) + Post.where(id: PostTag.where(tag_id: Array(internal)).select(:post_id)) + .or(Post.where(id: PostExternalTag.where(external_tag_id: Array(external)) + .select(:post_id))).select(:id) end def sync_post_tags! post, desired_tags, sections diff --git a/backend/app/services/counts_axes.rb b/backend/app/services/counts_axes.rb new file mode 100644 index 0000000..0697966 --- /dev/null +++ b/backend/app/services/counts_axes.rb @@ -0,0 +1,98 @@ +class CountsAxes + GRAINS = ['year', 'quarter', 'month', 'week', 'day', 'hour'].freeze + + def self.time_expression(expression, grain, timezone) + raise ArgumentError unless GRAINS.include?(grain) + + connection = ApplicationRecord.connection + # Use the installed TZInfo database, including DST, rather than MySQL timezone tables. + first = Time.utc(1900) + last = Time.utc(2101) + offset = timezone.period_for_utc(first).utc_total_offset + clauses = timezone.transitions_up_to(last, first).filter_map do |transition| + next if transition.offset.utc_total_offset == offset + + previous = offset + offset = transition.offset.utc_total_offset + "WHEN #{ expression } < #{ connection.quote(transition.at.to_time) } THEN #{ previous }" + end + seconds = clauses.empty? ? offset.to_s : "CASE #{ clauses.join(' ') } ELSE #{ offset } END" + local = "DATE_ADD(#{ expression }, INTERVAL (#{ seconds }) SECOND)" + case grain + when 'year' then "DATE_FORMAT(#{ local }, '%Y')" + when 'quarter' then "CONCAT(YEAR(#{ local }), '-Q', QUARTER(#{ local }))" + when 'month' then "DATE_FORMAT(#{ local }, '%Y-%m')" + when 'week' then "DATE_FORMAT(#{ local }, '%x-W%v')" + when 'day' then "DATE_FORMAT(#{ local }, '%Y-%m-%d')" + else "DATE_FORMAT(#{ local }, '%Y-%m-%d %H:00')" + end + end + + attr_reader :rows, :columns, :all + + def initialize(population) + @population = population + @definition = population.definition + @rows = parse(population.params.fetch('rows', '[]')) + @columns = parse(population.params.fetch('cols', '[]')) + @all = rows + columns + raise ArgumentError if all.length > 8 || all.map { _1['field'] }.uniq.length != all.length + end + + def expressions + all.map do |axis| + field = axis.fetch('field') + case field + when 'tags' + "CASE WHEN tag_entry.tag_id IS NOT NULL " + + "THEN CONCAT('internal:', tag_entry.tag_id) " + + "ELSE CONCAT('external:', tag_entry.external_tag_id) END" + when 'changed_field' then 'changed.field_key' + else + expression = @definition.expression(field) + @definition.fields[field] == :datetime ? self.class.time_expression( + expression, axis.fetch('grain', 'month'), @population.timezone) : expression + end + end + end + + def joins + result = '' + if all.any? { _1['field'] == 'tags' } + result += <<~SQL.squish + JOIN JSON_TABLE(CAST(v.tags_json AS JSON), '$[*]' COLUMNS ( + tag_id BIGINT PATH '$.tag_id' NULL ON EMPTY, + external_tag_id BIGINT PATH '$.external_tag_id' NULL ON EMPTY)) tag_entry + + SQL + end + if all.any? { _1['field'] == 'changed_field' } + fields = @definition.changes.map do |field| + "SELECT #{ ApplicationRecord.connection.quote(field) } AS field_key" + end.join(' UNION ALL ') + condition = @definition.changes.map do |field| + "(changed.field_key = '#{ field }' AND #{ @population.changed_sql(field) })" + end.join(' OR ') + result += " JOIN (#{ fields }) changed ON (#{ condition })" + end + result + end + + private + + def parse(value) + axes = JSON.parse(value) + raise ArgumentError unless axes.is_a?(Array) + + axes.each do |axis| + raise ArgumentError unless axis.is_a?(Hash) && (axis.keys - ['field', 'grain']).empty? + raise ArgumentError unless @definition.axes(@population.history).include?(axis['field']) + if @definition.fields[axis['field']] == :datetime + raise ArgumentError unless GRAINS.include?(axis.fetch('grain', 'month')) + elsif axis.key?('grain') + raise ArgumentError + end + end + axes + end +end diff --git a/backend/app/services/counts_export_projection.rb b/backend/app/services/counts_export_projection.rb index 6ab6370..0e42632 100644 --- a/backend/app/services/counts_export_projection.rb +++ b/backend/app/services/counts_export_projection.rb @@ -2,7 +2,7 @@ require 'set' # Export identifiers are stable; UI translations are supplied separately as column labels. class CountsExportProjection - ROOT_KEYS = { 'resource_id' => 'resource_id', 'version_no' => 'version_no', + ROOT_KEYS = { 'event_type' => 'event_type', 'resource_id' => 'resource_id', 'version_no' => 'version_no', 'is_latest' => 'is_latest', 'thumbnail' => 'thumbnail_url', 'thumbnail_base' => 'thumbnail_base', 'title' => 'title', 'url' => 'url', 'tags' => 'tags', 'parents' => 'parents', 'video_ms' => 'video_ms', diff --git a/backend/app/services/counts_exporter.rb b/backend/app/services/counts_exporter.rb index 8b797d9..d6ad94c 100644 --- a/backend/app/services/counts_exporter.rb +++ b/backend/app/services/counts_exporter.rb @@ -7,6 +7,12 @@ class CountsExporter attr_reader :format, :resource def initialize(params, language_code:) + if params.key?('definition') + @table_export = CountsTableExport.new(params) + @format = @table_export.format + @resource = @table_export.resource + return + end allowed = (CountsQuery::KEYS + CountsStateQuery::KEYS + EXTRA_KEYS).uniq raise ArgumentError if (params.keys - allowed).any? raise ArgumentError unless params.values.all? { _1.is_a?(String) } @@ -42,6 +48,8 @@ class CountsExporter end def body + return @table_export.body if @table_export + Enumerator.new do |output| ApplicationRecord.transaction(isolation: :repeatable_read) do first_row = true diff --git a/backend/app/services/counts_population.rb b/backend/app/services/counts_population.rb new file mode 100644 index 0000000..a4d3975 --- /dev/null +++ b/backend/app/services/counts_population.rb @@ -0,0 +1,208 @@ +# Population first, then filters. Previous versions are joined before period filtering. +class CountsPopulation + attr_reader :definition, :params, :at, :history, :scope, :timezone + + def initialize(params) + @params = params + @definition = CountsResourceDefinition.new(params.fetch('resource', 'posts')) + @history = params.fetch('definition', 'latest') == 'history' + raise ArgumentError unless ['latest', 'history'].include?(params.fetch('definition', 'latest')) + + @timezone = TZInfo::Timezone.get(params.fetch('timezone', 'UTC')) + @at = timestamp(params['execution_at']) || Time.current.utc + raise ArgumentError if at > Time.current + 1.second + + @scope = definition.model.all + make_population + apply_filters + rescue TZInfo::InvalidTimezoneIdentifier + raise ArgumentError + end + + def timestamp(value, end_date: false) + return nil if value.nil? || value == '' + raise ArgumentError unless value.is_a?(String) + + if value.match?(/\A\d{4}-\d{2}-\d{2}\z/) + date = Date.iso8601(value) + date += 1 if end_date + timezone.local_to_utc(Time.utc(date.year, date.month, date.day)) + else + raise ArgumentError unless value.match?(/(?:Z|[+-]\d{2}:\d{2})\z/) + + time = Time.iso8601(value).utc + raise ArgumentError unless time.year.between?(1900, 2100) + + time + end + rescue TZInfo::AmbiguousTime, TZInfo::PeriodNotFound, Date::Error + raise ArgumentError + end + + def latest_sql(row = 'v', cutoff = nil) + cutoff ||= history ? at : timestamp(params['at']) || at + keys = definition.keys.map { "newer.#{ _1 } = #{ row }.#{ _1 }" }.join(' AND ') + bound = definition.model.connection.quote(cutoff) + <<~SQL.squish + NOT EXISTS (SELECT 1 FROM #{ definition.table } newer + WHERE #{ keys } AND newer.version_no > #{ row }.version_no + AND newer.#{ definition.time_column } <= #{ bound }) + SQL + end + + def changed_sql(key) + column = definition.column(key) + "(previous.id IS NOT NULL AND NOT (v.#{ column } <=> previous.#{ column }))" + end + + private + + def make_population + table = definition.table + @scope = @scope.from("#{ table } v").select('v.*') + keys = definition.keys.map { "previous.#{ _1 } = v.#{ _1 }" }.join(' AND ') + @scope = @scope.joins("LEFT JOIN #{ table } previous ON #{ keys } " + + 'AND previous.version_no = v.version_no - 1') + @scope = @scope.where("v.#{ definition.time_column } <= ?", at) + unless history + point = timestamp(params['at']) || at + raise ArgumentError if point > at + + @scope = @scope.where("v.#{ definition.time_column } <= ?", point) + .where(Arel.sql(latest_sql('v', point))).where.not('v.event_type' => 'discard') + return + end + first = timestamp(params['from']) + last = timestamp(params['through'], end_date: true) + raise ArgumentError if first && last && first > last + + @scope = @scope.where('v.created_at >= ?', first) if first + if last + operator = params['through'].length == 10 ? '<' : '<=' + @scope = @scope.where("v.created_at #{ operator } ?", last) + end + end + + def apply_filters + ['resource_id', 'user'].each do |key| + value = params[key == 'user' ? 'user_id' : key] + next if value.nil? || value == '' + raise ArgumentError unless value.match?(/\A[0-9]+\z/) + + @scope = @scope.where("#{ definition.expression(key) } = ?", value.to_i) + end + apply_tags if params['tags'].present? || params['tag'].present? + conditions = JSON.parse(params.fetch('conditions', '[]')) + raise ArgumentError unless conditions.is_a?(Array) && conditions.length <= 100 + + conditions.each { apply_condition(_1) } + drills = JSON.parse(params.fetch('drill', '[]')) + raise ArgumentError unless drills.is_a?(Array) && drills.length <= 100 + + drills.each do |entry| + key = entry.fetch('field') + if key == 'tags' + @scope = @scope.where(snapshot_membership(entry.fetch('value'))) + elsif key == 'changed_field' + raise ArgumentError unless history && definition.changes.include?(entry['value']) + + @scope = @scope.where(Arel.sql(changed_sql(entry['value']))) + elsif definition.fields[key] == :datetime && entry['grain'] + @scope = @scope.where(definition.expression(key) => nil) if entry['value'].nil? + if entry['value'] + expression = CountsAxes.time_expression(definition.expression(key), + entry['grain'], timezone) + @scope = @scope.where("#{ expression } = ?", entry['value']) + end + else + raise ArgumentError unless definition.axes(history).include?(key) + + @scope = @scope.where(definition.expression(key) => entry['value']) + end + end + end + + def apply_condition(entry) + raise ArgumentError unless entry.is_a?(Hash) + raise ArgumentError if (entry.keys - ['field', 'operator', 'value', 'end', 'blank']).any? + + key = entry.fetch('field') + type = definition.fields.fetch(key) { raise ArgumentError } + operator = entry.fetch('operator') + raise ArgumentError unless CountsResourceDefinition::OPERATORS.fetch(type, []).include?(operator) + raise ArgumentError unless [true, false, nil].include?(entry['blank']) + + expression = definition.expression(key) + blank = type == :string || type == :enum ? + "(#{ expression } IS NULL OR #{ expression } = '')" : "#{ expression } IS NULL" + if entry['blank'] + @scope = @scope.where(Arel.sql(blank)) + return + end + values = [entry['value'], entry['end']] + raise ArgumentError unless values.all? { _1.nil? || _1.is_a?(String) } + return if values.all? { _1.nil? || _1 == '' } + + parsed = values.map do |value| + next nil if value.nil? || value == '' + + case type + when :integer + raise ArgumentError unless value.match?(/\A-?\d+\z/) + + value.to_i + when :datetime then timestamp(value) + else value + end + end + if operator == 'range' + @scope = @scope.where("#{ expression } >= ?", parsed[0]) if parsed[0] + @scope = @scope.where("#{ expression } <= ?", parsed[1]) if parsed[1] + return + end + return unless parsed[0] + + sql_operator = { 'eq' => '=', 'ne' => '<>', 'gt' => '>', 'lt' => '<', + 'ge' => '>=', 'le' => '<=', 'like' => 'LIKE' }.fetch(operator) + sql = "#{ expression } #{ sql_operator } ?" + sql = "(#{ sql } OR #{ blank })" if operator == 'ne' + @scope = @scope.where(sql, parsed[0]) + end + + def snapshot_membership(value) + kind, identifier = value.to_s.split(':', 2) + raise ArgumentError unless ['internal', 'external'].include?(kind) + raise ArgumentError unless identifier&.match?(/\A[1-9]\d*\z/) + + key = kind == 'internal' ? 'tag_id' : 'external_tag_id' + definition.model.sanitize_sql_array([ + "JSON_CONTAINS(v.tags_json, JSON_OBJECT('#{ key }', CAST(? AS UNSIGNED)))", + identifier.to_i]) + end + + def apply_tags + raise ArgumentError unless definition.resource == 'posts' + + if params['tag'].present? + @scope = @scope.where(snapshot_membership( + "#{ params.fetch('kind', 'internal') }:#{ params['tag'] }")) + end + return unless params['tags'].present? + + locale = Locale.find_by!(code: params.fetch('locale', 'ja')) + refs = PostTagSearch.parse_references(params['tag_refs'], params['tags'].split) + raw = params['tags'] + literals = PostTagSearch.literals(locale, raw, references: refs) + predicates = literals.map do |literal| + internal, external = PostTagSearch.references(locale, literal) + parts = Array(internal).map { snapshot_membership("internal:#{ _1 }") } + + Array(external).map { snapshot_membership("external:#{ _1 }") } + expression = parts.empty? ? 'FALSE' : "(#{ parts.join(' OR ') })" + literal[:negative] ? "NOT #{ expression }" : expression + end + return if predicates.empty? + + any = params['match'] == 'any' || raw.split.any? { _1.downcase == 'type:or' } + @scope = @scope.where(Arel.sql("(#{ predicates.join(any ? ' OR ' : ' AND ') })")) + end +end diff --git a/backend/app/services/counts_query.rb b/backend/app/services/counts_query.rb index 832b98a..fb47736 100644 --- a/backend/app/services/counts_query.rb +++ b/backend/app/services/counts_query.rb @@ -1,7 +1,7 @@ # Read-only history queries. SQL identifiers and expressions are server-owned. class CountsQuery AXES = ['month', 'user'].freeze - COMMON_COLUMNS = ['resource_id', 'version_no', 'is_latest', 'user', 'created_at'].freeze + COMMON_COLUMNS = ['resource_id', 'version_no', 'is_latest', 'event_type', 'user', 'created_at'].freeze COLUMNS = { 'posts' => COMMON_COLUMNS + [ 'thumbnail', 'thumbnail_base', 'title', 'url', 'tags', 'parents', 'video_ms', diff --git a/backend/app/services/counts_resource_definition.rb b/backend/app/services/counts_resource_definition.rb new file mode 100644 index 0000000..cfebcea --- /dev/null +++ b/backend/app/services/counts_resource_definition.rb @@ -0,0 +1,65 @@ +# Server-owned version identities, attributes and field types. No client SQL is accepted. +class CountsResourceDefinition + COMMON = { 'resource_id' => :integer, 'version_no' => :integer, + 'event_type' => :enum, 'user' => :integer, 'created_at' => :datetime }.freeze + DEFINITIONS = { + 'posts' => [PostVersion, ['post_id'], { + 'title' => :string, 'url' => :string, 'thumbnail_base' => :string, + 'tags_json' => :json, 'parent_post_ids' => :string, 'video_ms' => :integer, + 'original_created_from' => :datetime, 'original_created_before' => :datetime }], + 'tags' => [TagVersion, ['tag_id'], { + 'name' => :string, 'names' => :json, 'category' => :enum, 'aliases' => :string, + 'parent_tag_ids' => :string, 'deprecated_at' => :datetime }], + 'nico_tags' => [NicoTagVersion, ['tag_id'], { + 'name' => :string, 'linked_tags' => :string }], + 'tag_descriptions' => [TagDescriptionVersion, ['tag_id', 'language_code'], { + 'language_code' => :enum, 'body' => :string, + 'source_language_code' => :enum, 'source_version_no' => :integer }], + 'tag_groups' => [TagGroupVersion, ['tag_group_id'], { + 'name' => :string, 'description' => :string, + 'members' => :json, 'placements' => :json }], + 'materials' => [MaterialVersion, ['material_id'], { + 'url' => :string, 'parent_id' => :integer, 'tag_id' => :integer, + 'tag_name' => :string, 'tag_category' => :enum, + 'discarded_at' => :datetime, 'updated_at' => :datetime, + 'updated_by_user_id' => :integer, 'export_paths_json' => :json, + 'file_blob_id' => :integer, 'file_filename' => :string, + 'file_content_type' => :string, 'file_byte_size' => :integer, + 'file_checksum' => :string, 'file_sha256' => :string, + 'source_kind' => :enum, 'source_uri' => :string, 'source_path' => :string, + 'source_file_id' => :string, 'normalized_source_key' => :string }] }.freeze + OPERATORS = { + string: ['eq', 'ne', 'like'], enum: ['eq', 'ne'], boolean: ['eq', 'ne'], + integer: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'], + datetime: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'] }.freeze + + attr_reader :model, :keys, :fields, :resource + + def initialize(resource) + @resource = resource + @model, @keys, attributes = DEFINITIONS.fetch(resource) { raise ArgumentError } + @fields = COMMON.merge(attributes) + end + + def table = model.table_name + def time_column = 'created_at' + def column(key) + raise ArgumentError unless fields.key?(key) + + { 'resource_id' => keys.first, 'user' => 'created_by_user_id' }.fetch(key, key) + end + + def expression(key, row = 'v') = "#{ row }.#{ column(key) }" + def changes = fields.keys - COMMON.keys - ['updated_at', 'updated_by_user_id'] + def axes(history) + fields.reject { |_key, type| type == :json }.keys + + (resource == 'posts' ? ['tags'] : []) + (history ? ['changed_field'] : []) + end + + def metadata + fields.map do |key, type| + { key:, type:, nullable: model.columns_hash.fetch(column(key)).null, + operators: OPERATORS.fetch(type, []) } + end + end +end diff --git a/backend/app/services/counts_table_export.rb b/backend/app/services/counts_table_export.rb new file mode 100644 index 0000000..aeb0caf --- /dev/null +++ b/backend/app/services/counts_table_export.rb @@ -0,0 +1,172 @@ +require 'csv' + +# Export uses the exact same population and grouped cells as the interactive table. +class CountsTableExport + EXTRA = ['format', 'columns', 'headers', 'labels', 'children', 'child_sort', + 'order', 'nested_json', 'summary_columns', 'basis', 'axis', 'latest'].freeze + attr_reader :format, :resource + + def initialize(params) + raise ArgumentError if (params.keys - CountsTableQuery::KEYS - EXTRA).any? + raise ArgumentError unless params.values.all? { _1.is_a?(String) } + + @params = params + @labels = JSON.parse(params.fetch('labels', '{}')) + raise ArgumentError unless @labels.is_a?(Hash) + @selected = params.key?('children') ? params['children'].split(',') : nil + @order = params.fetch('order', '').split(',') + @child_sort = params.fetch('child_sort', '').split(',').map { _1.split(':', -1) } + @child_sort.each do |path, direction, *extra| + raise ArgumentError if extra.any? || !['asc', 'desc'].include?(direction) + raise ArgumentError unless path&.match?(/\A[a-zA-Z_]+(?:\.[a-zA-Z]+)*\/[a-zA-Z]+\z/) + end + @format = params.fetch('format', 'json') + @resource = params.fetch('resource', 'posts') + raise ArgumentError unless ['json', 'csv'].include?(format) + + @query = CountsTableQuery.new(params.slice(*CountsTableQuery::KEYS) + .merge('page' => '1', 'limit' => '100')) + @summary = @query.mode == 'summary' + @columns = params.fetch('columns', '').split(',') + @headers = JSON.parse(params.fetch('headers', '[]')) + raise ArgumentError unless @headers.is_a?(Array) && @headers.all? { _1.is_a?(String) } + + unless @summary + if ['posts', 'tags'].include?(resource) + @projection = CountsExportProjection.new(params, resource:, summary: false, tags: false) + else + allowed = @query.population.definition.fields.keys + ['is_latest'] + raise ArgumentError if (@columns - allowed).any? + raise ArgumentError unless @columns.length == @headers.length && @columns.uniq == @columns + end + end + end + + def body + Enumerator.new do |output| + ApplicationRecord.transaction(isolation: :repeatable_read) do + @language_names = Language.pluck(:code, :name).to_h + started = false + first = true + @query.each_page do |page| + unless started + metadata = page.except(:versions, :cells, :axis_labels, :column_values).merge( + resource:, filters: @params.slice(*CountsTableQuery::KEYS), + fields: @query.population.definition.metadata, + columns: @columns, column_labels: @headers, + order: @params['order'], children: @params['children'], + sort: @params.fetch('sort', ''), child_sort: @params['child_sort']) + output << (format == 'json' ? "{\"metadata\":#{ metadata.to_json },\"data\":[" : "\uFEFF") + if format == 'csv' + headers = @summary ? summary_headers : @headers + output << CSV.generate_line(headers.map { safe(_1) }, row_sep: "\r\n") + end + started = true + end + (page[:cells] || page[:versions]).each do |row| + record = project(row) + if format == 'json' + output << ',' unless first + output << record.to_json + else + values = if @summary + row[:values].each_with_index.map do |value, index| + page[:axis_labels].fetch("a#{ index }", {}).fetch(value.to_s, value) + end + [row[:count]] + elsif @projection + @projection.csv_record(record, row) + else + record.map { |key, value| csv_logical(key, value) } + end + output << CSV.generate_line(values.map { safe(_1) }, row_sep: "\r\n") + end + first = false + end + end + output << ']}' if format == 'json' + end + end + end + + private + + def summary_headers + labels = @labels + @query.axes.all.map { labels.fetch(_1['field'], _1['field']) } + + [labels.fetch('count', 'count')] + end + + def project(row) + return { row: row[:values].first(@query.axes.rows.length), + column: row[:values].drop(@query.axes.rows.length), count: row[:count] } if @summary + return @projection.project(row) if @projection + + @columns.to_h do |column| + key = column == 'user' ? 'created_by_user' : column + value = row[key] + value = project_nested(value, key) if ['members', 'placements', 'export_paths_json'].include?(key) + [key, value] + end + end + + def camel(key) + key.gsub(/_([a-z])/) { Regexp.last_match(1).upcase } + end + + def project_nested(value, path) + rows = path == 'export_paths_json' ? (value || {}).map { |profile, target| + { 'profile' => profile, 'path' => target } + } : Array(value) + sorts = @child_sort.filter_map do |key, direction| + parent, field = key.split('/', 2) + [field, direction] if parent == path + end + rows = rows.each_with_index.sort do |(first, a), (last, b)| + compared = 0 + sorts.each do |field, direction| + left = first.find { |key, _| camel(key) == field }&.last + right = last.find { |key, _| camel(key) == field }&.last + compared = if left.nil? || right.nil? + left.nil? == right.nil? ? 0 : (left.nil? ? 1 : -1) + else + (left.to_s <=> right.to_s) * (direction == 'asc' ? 1 : -1) + end + break unless compared.zero? + end + compared.zero? ? a <=> b : compared + end.map(&:first) + rows.map do |row| + fields = row.keys.select { !@selected || @selected.include?("#{ path }.#{ camel(_1) }") } + ordered = (@order + fields.map { "#{ path }.#{ camel(_1) }" }).uniq + .filter_map { |item| fields.find { "#{ path }.#{ camel(_1) }" == item } } + ordered.to_h do |key| + item = row[key] + item = project_nested(item, "#{ path }.#{ camel(key) }") if item.is_a?(Array) + [key, item] + end + end + end + + def csv_logical(key, value) + return value ? @labels.fetch('latest', '') : '' if key == 'is_latest' + if value.is_a?(Time) || value.is_a?(ActiveSupport::TimeWithZone) + return @query.population.timezone.utc_to_local(value.utc).iso8601 + end + if key == 'language_code' || key == 'source_language_code' + return @language_names[value] + end + if value.is_a?(Hash) + return value.to_h { |field, item| [@labels.fetch(camel(field.to_s), ''), csv_logical(field, item)] } + end + return value.map { csv_logical(key, _1) } if value.is_a?(Array) + + @labels.fetch("#{ key }:#{ value }", value) + end + + def safe(value) + return value if value.is_a?(Numeric) + + text = value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : value.to_s + text.match?(/\A\s*[=+@-]/) ? "'#{ text }" : text + end +end diff --git a/backend/app/services/counts_table_query.rb b/backend/app/services/counts_table_query.rb new file mode 100644 index 0000000..a47c6dd --- /dev/null +++ b/backend/app/services/counts_table_query.rb @@ -0,0 +1,226 @@ +# Common entrypoint for six version resources; legacy endpoints keep their contracts. +class CountsTableQuery + KEYS = ['resource', 'definition', 'mode', 'at', 'execution_at', 'from', 'through', + 'timezone', 'rows', 'cols', 'conditions', 'drill', 'tags', 'tag_refs', 'match', + 'tag', 'kind', 'resource_id', 'user_id', 'page', 'limit', 'sort', 'locale'].freeze + attr_reader :population, :axes, :page, :limit, :mode + + def initialize(params) + raise ArgumentError if (params.keys - KEYS).any? + raise ArgumentError unless params.values.all? { _1.is_a?(String) } + + @params = params + @population = CountsPopulation.new(params) + @definition = population.definition + @connection = @definition.model.connection + @axes = CountsAxes.new(population) + @page = Integer(params.fetch('page', '1'), 10) + @limit = Integer(params.fetch('limit', '20'), 10) + @mode = params.fetch('mode', 'summary') + raise ArgumentError unless page.between?(1, 1_000_000) && limit.between?(1, 100) + raise ArgumentError unless ['summary', 'details'].include?(mode) + + @sort = params.fetch('sort', '').split(',').map { _1.split(':', -1) } + allowed = mode == 'summary' ? axes.all.each_index.map { "a#{ _1 }" } + ['count'] : + @definition.fields.keys + ['is_latest'] + + (@definition.resource == 'posts' ? ['tags', 'parents', 'original_created_at'] : []) + + (@definition.resource == 'tags' ? ['parents', 'deprecated'] : []) + raise ArgumentError if @sort.map(&:first).uniq.length != @sort.length + @sort.each do |key, direction, *extra| + measure = mode == 'summary' && key&.match?(/\Ameasure_[0-9a-f]+\z/) + measure_values(key) if measure + if extra.any? || (!allowed.include?(key) && !measure) || !['asc', 'desc'].include?(direction) + raise ArgumentError + end + end + end + + def call + ApplicationRecord.transaction do + metadata.merge(mode == 'summary' ? summary : details) + end + end + + def metadata + { page:, limit:, execution_at: population.at.iso8601(6), at: @params['at'].presence, + definition: population.history ? 'history' : 'latest', + timezone: population.timezone.identifier, rows: axes.rows, cols: axes.columns, + fields: @definition.metadata, count: population.scope.except(:select).count } + end + + def summary + expressions = axes.expressions + selected = expressions.each_with_index.map { |expression, index| "#{ expression } AS a#{ index }" } + source = population.scope + source = source.joins(axes.joins) unless axes.joins.empty? + sql = if expressions.empty? + source.except(:select).select('COUNT(DISTINCT v.id) AS count').to_sql + else + source.except(:select).group(*expressions.map { Arel.sql(_1) }) + .select(Arel.sql((selected + ['COUNT(DISTINCT v.id) AS count']).join(', '))) + .to_sql + end + prefix = "WITH cells AS (#{ sql })" + row_keys = axes.rows.each_index.map { "a#{ _1 }" } + # Page whole row tuples so a pivot row is never split across display pages. + row_selection = row_keys.empty? ? '1 AS singleton' : row_keys.join(', ') + row_group = row_keys.empty? ? 'GROUP BY singleton' : "GROUP BY #{ row_keys.join(', ') }" + row_sort = @sort.each_with_index.filter_map do |(key, direction), index| + if key == 'count' || key.start_with?('measure_') + condition = if key == 'count' + 'TRUE' + else + measure_values(key).each_with_index.map do |value, offset| + "a#{ axes.rows.length + offset } <=> #{ @connection.quote(value) }" + end.join(' AND ').presence || 'TRUE' + end + ["MAX(CASE WHEN #{ condition } THEN count END) AS sort#{ index }", + "sort#{ index } IS NULL ASC, sort#{ index } #{ direction.upcase }"] + elsif row_keys.include?(key) + [nil, "#{ key } IS NULL ASC, #{ key } #{ direction.upcase }"] + end + end + extras = row_sort.filter_map(&:first) + order_rows = row_sort.map(&:last) + row_keys.map { "#{ _1 } IS NULL, #{ _1 }" } + order_rows = ['singleton ASC'] if order_rows.empty? + group_sql = "SELECT #{ ([row_selection] + extras).join(', ') } FROM cells #{ row_group }" + total = @connection.select_value( + "#{ prefix } SELECT COUNT(*) FROM (#{ group_sql }) row_groups").to_i + join = row_keys.empty? ? 'TRUE' : row_keys.map { "cells.#{ _1 } <=> page_rows.#{ _1 }" }.join(' AND ') + order_cols = @sort.filter_map do |key, direction| + "cells.#{ key } #{ direction.upcase }" if key.match?(/\Aa[0-9]+\z/) && !row_keys.include?(key) + end + records = @connection.select_all(<<~SQL) + #{ prefix }, row_groups AS (#{ group_sql }), + numbered AS (SELECT row_groups.*, + ROW_NUMBER() OVER (ORDER BY #{ order_rows.join(', ') }) AS position FROM row_groups), + page_rows AS (SELECT * FROM numbered + WHERE position > #{ (page - 1) * limit } AND position <= #{ page * limit }) + SELECT cells.* FROM cells JOIN page_rows ON #{ join } + ORDER BY #{ (['page_rows.position'] + order_cols + axes.all.each_index.map { "cells.a#{ _1 }" }).join(', ') } + SQL + column_keys = axes.columns.each_index.map { "a#{ axes.rows.length + _1 }" } + column_values = if column_keys.empty? + [[]] + else + @connection.select_rows( + "#{ prefix } SELECT DISTINCT #{ column_keys.join(', ') } FROM cells " + + "ORDER BY #{ column_keys.map { "#{ _1 } IS NULL, #{ _1 }" }.join(', ') }") + end + { group_count: total, column_values:, cells: records.map do |row| + { values: axes.all.each_index.map { row["a#{ _1 }"] }, count: row['count'].to_i } + end, axis_labels: axis_labels(records, column_values) } + end + + def details + query = population.scope.select(Arel.sql( + "(#{ population.latest_sql }) AS counts_latest")) + order = sort_sql { detail_expression(_1) } + query = query.order(Arel.sql((order + ['v.id DESC']).join(', '))) + .limit(limit).offset((page - 1) * limit) + rows = query.to_a + if ['posts', 'tags'].include?(@definition.resource) + values = CountsVersionRepr.many(rows, resource: @definition.resource, + language_code: locale.language_code, + at: population.history ? population.at : population.timestamp(@params['at']) || population.at) + values.each_with_index { |value, index| value[:event_type] = rows[index].event_type } + else + users = User.where(id: rows.map(&:created_by_user_id)).pluck(:id, :name).to_h + values = rows.map do |row| + attributes = @definition.fields.keys.to_h do |key| + [key, row.public_send(@definition.column(key))] + end + attributes.merge('id' => row.id, 'resource_id' => row[@definition.keys.first], + 'is_latest' => row.attributes['counts_latest'].to_i == 1, + 'created_by_user' => row.created_by_user_id && + { id: row.created_by_user_id, name: users[row.created_by_user_id] }) + end + end + { versions: values } + end + + def each_page + total = nil + loop do + result = call + total ||= mode == 'summary' ? result[:group_count] : result[:count] + yield result + break if page * limit >= total + + @page += 1 + end + end + + private + + def locale = @locale ||= Locale.find_by!(code: @params.fetch('locale', 'ja')) + + def measure_values(key) + value = JSON.parse([key.delete_prefix('measure_')].pack('H*')) + unless (value.is_a?(Array) && value.length == axes.columns.length && + value.all? { _1.nil? || _1.is_a?(String) || _1.is_a?(Numeric) }) + raise ArgumentError + end + value + end + + def detail_expression(key) + return "(#{ population.latest_sql })" if key == 'is_latest' + aliases = { 'tags' => 'tags_json', 'parents' => @definition.resource == 'posts' ? + 'parent_post_ids' : 'parent_tag_ids', 'deprecated' => 'deprecated_at', + 'original_created_at' => 'original_created_from' } + field = aliases.fetch(key, key) + expression = @definition.expression(field) + @definition.fields[field] == :json ? "CAST(#{ expression } AS CHAR) COLLATE utf8mb4_bin" : expression + end + + def sort_sql + @sort.flat_map do |key, direction| + expression = yield key + ["(#{ expression }) IS NULL ASC", "#{ expression } #{ direction.upcase }"] + end + end + + def axis_labels(records, column_values) + result = {} + axes.all.each_with_index do |axis, index| + values = records.map { _1["a#{ index }"] } + if index >= axes.rows.length + values += column_values.map { _1[index - axes.rows.length] } + end + values = values.compact.uniq + labels = case axis['field'] + when 'user' + User.where(id: values).pluck(:id, :name).to_h.transform_keys(&:to_s) + when 'language_code', 'source_language_code' + Language.where(code: values).pluck(:code, :name).to_h + when 'tags' + tag_labels(values) + else {} + end + result["a#{ index }"] = labels + end + result + end + + def tag_labels(values) + internal = values.grep(/\Ainternal:/).map { _1.split(':').last.to_i } + external = values.grep(/\Aexternal:/).map { _1.split(':').last.to_i } + point = population.timestamp(@params['at']) || population.at + labels = {} + [[TagVersion, internal, 'internal'], [NicoTagVersion, external, 'external']].each do |model, ids, kind| + next if ids.empty? + + table = model.table_name + model.where(tag_id: ids).where('created_at <= ?', point) + .where("NOT EXISTS (SELECT 1 FROM #{ table } newer " + + "WHERE newer.tag_id = #{ table }.tag_id AND newer.created_at <= ? " + + "AND newer.version_no > #{ table }.version_no)", point).each do |row| + names = row.respond_to?(:names) ? row.names : nil + name = names&.dig(locale.language_code, 'name') || row.name + labels["#{ kind }:#{ row.tag_id }"] = name + end + end + labels + end +end diff --git a/backend/app/services/post_tag_search.rb b/backend/app/services/post_tag_search.rb new file mode 100644 index 0000000..c59f1a9 --- /dev/null +++ b/backend/app/services/post_tag_search.rb @@ -0,0 +1,42 @@ +# Shared literal parsing and reference resolution for current and snapshot post search. +class PostTagSearch + def self.parse_references(raw, literals) + return nil if raw.blank? + + references = JSON.parse(raw) + valid = references.is_a?(Array) && references.all? do |reference| + next false unless reference.is_a?(Hash) && reference['name'].is_a?(String) + + ids = [reference['tag_id'], reference['external_tag_id']].compact + ((ids.any? || reference['missing'] == true) && + ids.all? { _1.is_a?(Integer) && _1.positive? } && + (!reference.key?('missing') || (reference['missing'] == true && ids.empty?)) && + (!reference.key?('negative') || [true, false].include?(reference['negative']))) + end + raise ArgumentError unless valid + + references.map { _1['name'] } == literals ? references : nil + end + + def self.literals(locale, raw, references: nil) + raw.split.each_with_index.filter_map do |token, index| + next if token.downcase == 'type:or' + + reference = references&.[](index) + name = token.sub(/\Anot:/i, '') + name = TagName.canonicalise(locale, name).first unless reference + negative = reference&.key?('negative') ? reference['negative'] : + token.downcase.start_with?('not:') + { name:, negative:, reference: } + end + end + + def self.references(locale, literal) + reference = literal[:reference] + return [reference['tag_id'], reference['external_tag_id']] if reference + + [TagName.where(language_code: locale.language_code, name: literal[:name]) + .distinct.pluck(:tag_id), + ExternalTag.where("CONCAT(platform, ':', name) = ?", literal[:name]).pluck(:id)] + end +end diff --git a/backend/config/routes.rb b/backend/config/routes.rb index 002149b..a8f5df2 100644 --- a/backend/config/routes.rb +++ b/backend/config/routes.rb @@ -1,4 +1,5 @@ Rails.application.routes.draw do + get 'counts/query', to: 'counts_tables#index' get 'counts/export', to: 'counts_exports#index' post 'counts/export', to: 'counts_exports#index' get 'counts/state', to: 'counts_states#index' diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6e53548..1bc73e8 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -202,6 +202,14 @@ export const pageRoutes = ( handle: { preload: CountsResourcePage.preload } }, { path: 'counts/tags', element: , handle: { preload: CountsResourcePage.preload } }, + { path: 'counts/nico_tags', element: , + handle: { preload: CountsResourcePage.preload } }, + { path: 'counts/tag_descriptions', element: , + handle: { preload: CountsResourcePage.preload } }, + { path: 'counts/tag_groups', element: , + handle: { preload: CountsResourcePage.preload } }, + { path: 'counts/materials', element: , + handle: { preload: CountsResourcePage.preload } }, { path: 'more', element: , handle: { preload: MorePage.preload, sitemap: ['static'] } }, { path: '*', element: , handle: { notFound: true } }] diff --git a/frontend/src/components/common/DateTimeField.tsx b/frontend/src/components/common/DateTimeField.tsx index 74fbec8..2ea074e 100644 --- a/frontend/src/components/common/DateTimeField.tsx +++ b/frontend/src/components/common/DateTimeField.tsx @@ -8,13 +8,13 @@ import type { ComponentPropsWithoutRef, FC, FocusEvent } from 'react' const pad = (n: number): string => n.toString ().padStart (2, '0') -const toDateTimeLocalValue = (value: Date) => { +const toDateTimeLocalValue = (value: Date, seconds = false) => { const y = value.getFullYear () const m = pad (value.getMonth () + 1) const day = pad (value.getDate ()) const h = pad (value.getHours ()) const min = pad (value.getMinutes ()) - return `${ y }-${ m }-${ day }T${ h }:${ min }` + return `${ y }-${ m }-${ day }T${ h }:${ min }${seconds ? `:${pad (value.getSeconds ())}` : ''}` } @@ -37,15 +37,17 @@ type Props = Omit, 'onChange'> & { onChange?: (isoUTC: string | null) => void className?: string onBlur?: (ev: FocusEvent) => void - invalid?: boolean } + invalid?: boolean + precision?: 'minute' | 'second' } -const DateTimeField: FC = ({ value, onChange, className, onBlur, invalid, ...rest }) => { +const DateTimeField: FC = props => { + const { value, onChange, className, onBlur, invalid, precision = 'minute', ...rest } = props const [local, setLocal] = useState ('') useEffect (() => { - setLocal (value ? toDateTimeLocalValue (new Date (value)) : '') - }, [value]) + setLocal (value ? toDateTimeLocalValue (new Date (value), precision === 'second') : '') + }, [value, precision]) return ( = ({ value, onChange, className, onBlur, invalid, 'focus:ring-2 focus:ring-blue-200']), className)} type="datetime-local" - step={60} + step={precision === 'second' ? 1 : 60} value={local} aria-invalid={invalid} onChange={ev => { const v = ev.target.value setLocal (v) - onChange?.(v ? toMinutePrecisionIsoUtc (v) : null) + const iso = v ? (precision === 'second' + ? new Date (v).toISOString () : toMinutePrecisionIsoUtc (v)) : null + onChange?.(iso) }} onBlur={onBlur}/>) } diff --git a/frontend/src/components/common/HelpPopover.tsx b/frontend/src/components/common/HelpPopover.tsx new file mode 100644 index 0000000..3ae8bbd --- /dev/null +++ b/frontend/src/components/common/HelpPopover.tsx @@ -0,0 +1,35 @@ +import { useId, useRef } from 'react' +import { CircleHelp } from 'lucide-react' + +import type { ReactNode } from 'react' + + +const HelpPopover = ({ label, children }: { + label: { text: string; language: string }; children: ReactNode }, +) => { + const id = useId () + const trigger = useRef (null) + const panel = useRef (null) + return <> + + + +} + + +export default HelpPopover diff --git a/frontend/src/components/common/TagInput.tsx b/frontend/src/components/common/TagInput.tsx index be12274..d2967ff 100644 --- a/frontend/src/components/common/TagInput.tsx +++ b/frontend/src/components/common/TagInput.tsx @@ -12,6 +12,7 @@ import type { Tag } from '@/types' type Props = { includeNico?: boolean + includeUnused?: boolean describedBy?: string invalid?: boolean value: string @@ -20,6 +21,7 @@ type Props = { const TagInput: FC = ({ includeNico = false, + includeUnused = false, describedBy, invalid, value, @@ -57,11 +59,11 @@ const TagInput: FC = ({ { const tags = await apiGet ('/tags/autocomplete', { locale, signal: controller.signal, - params: { q: request.query, nico: includeNico ? '1' : '0' } }) + params: { q: request.query.replace (/^not:/i, ''), nico: includeNico ? '1' : '0' } }) if (active) { setResult ({ locale, includeNico, request, - tags: tags.filter (tag => tag.postCount > 0) }) + tags: includeUnused ? tags : tags.filter (tag => tag.postCount > 0) }) } } catch @@ -74,7 +76,7 @@ const TagInput: FC = ({ active = false controller.abort () } - }, [request, locale, includeNico]) + }, [request, locale, includeNico, includeUnused]) useEffect (() => { setActiveIndex (-1) @@ -85,7 +87,8 @@ const TagInput: FC = ({ // TODO: TagSearch からのコピペのため，共通化を考へる． const handleTagSelect = (tag: Tag) => { const parts = value?.split (' ') - parts[parts.length - 1] = tag.name + const negative = /^not:/i.test (parts[parts.length - 1]) + parts[parts.length - 1] = `${negative ? 'not:' : ''}${tag.name}` setValue (parts.join (' ') + ' ') setRequest ({ query: '' }) setActiveIndex (-1) diff --git a/frontend/src/components/counts/CountsAdditionalConditions.tsx b/frontend/src/components/counts/CountsAdditionalConditions.tsx new file mode 100644 index 0000000..adaae03 --- /dev/null +++ b/frontend/src/components/counts/CountsAdditionalConditions.tsx @@ -0,0 +1,90 @@ +import { useState } from 'react' + +import DateTimeField from '@/components/common/DateTimeField' +import { Button } from '@/components/ui/button' +import { countsFieldLabel, countsFilterFields, countsOperators, readCountsJSON } + from '@/lib/countsDefinitions' +import useDialogue from '@/lib/dialogues/useDialogue' +import { inputClass } from '@/lib/utils' +import { useUITranslation } from '@/lib/useUITranslation' + +import type { CountsResource } from '@/lib/counts' +import type { CountsCondition } from '@/lib/countsDefinitions' + + +const ConditionEditor = ({ resource, initial, save }: { + resource: CountsResource; initial?: CountsCondition + save: (condition: CountsCondition) => void }, +) => { + const { label, message } = useUITranslation ('counts') + const fields = countsFilterFields (resource) + const [field, setField] = useState (initial?.field ?? 'resource_id') + const [operator, setOperator] = useState (initial?.operator ?? 'eq') + return
+ + + +
+} + + +const CountsAdditionalConditions = ({ resource, params, update }: { + resource: CountsResource; params: URLSearchParams + update: (key: string, value: string) => void }, +) => { + const { label, message } = useUITranslation ('counts') + const dialogue = useDialogue () + const conditions = readCountsJSON (params, 'conditions', []) + const write = (next: CountsCondition[]) => update ('conditions', JSON.stringify (next)) + const edit = (index?: number) => { + void dialogue.form ({ title: label ('additionalConditions'), body: controls => ( + { + write (index == null ? [...conditions, condition] + : conditions.map ((item, position) => position === index ? condition : item)) + controls.close () + }}/>) }) + } + const patch = (index: number, change: Partial) => write ( + conditions.map ((item, position) => position === index ? { ...item, ...change } : item)) + return
+ {label ('additionalConditions')} + + {conditions.map ((condition, index) => { + const datetime = countsFilterFields (resource)[condition.field] === 'datetime' + return
+ + {(condition.operator === 'range' ? ['value', 'end'] as const : ['value'] as const) + .map (key => datetime ? patch (index, { [key]: value ?? '' })}/> : ( + patch (index, { [key]: event.target.value })}/>))} + + +
+ })} +
+} + + +export default CountsAdditionalConditions diff --git a/frontend/src/components/counts/CountsAxes.tsx b/frontend/src/components/counts/CountsAxes.tsx new file mode 100644 index 0000000..1cfffe4 --- /dev/null +++ b/frontend/src/components/counts/CountsAxes.tsx @@ -0,0 +1,125 @@ +import { DndContext, MouseSensor, TouchSensor, closestCenter, + useDraggable, useDroppable, useSensor, useSensors } from '@dnd-kit/core' + +import { countsAxisFields, countsFilterFields, countsGrains, readCountsJSON } + from '@/lib/countsDefinitions' +import { cn, inputClass } from '@/lib/utils' +import { useUITranslation } from '@/lib/useUITranslation' + +import type { ReactNode } from 'react' + +import type { CountsResource } from '@/lib/counts' +import type { CountsAxis } from '@/lib/countsDefinitions' + + +const AxisItem = ({ id, name, children }: { id: string; name: string; children: ReactNode }) => { + const drag = useDraggable ({ id }) + const drop = useDroppable ({ id }) + return
+ {children} +
+} + + +const AxisZone = ({ id, children }: { id: string; children: ReactNode }) => { + const drop = useDroppable ({ id }) + return
+ {children} +
+} + + +const CountsAxes = ({ resource, params, update }: { + resource: CountsResource; params: URLSearchParams + update: (key: string, value: string) => void }, +) => { + const { label, message } = useUITranslation ('counts') + const history = params.get ('definition') === 'history' + const available = countsAxisFields (resource, history) + const rows = readCountsJSON (params, 'rows', []) + const cols = readCountsJSON (params, 'cols', []) + const groups = { rows, cols } + const sensors = useSensors ( + useSensor (MouseSensor, { activationConstraint: { distance: 4 } }), + useSensor (TouchSensor, { activationConstraint: { delay: 350, tolerance: 8 } })) + const place = (field: string, target: string, position?: string) => { + for (const key of ['rows', 'cols'] as const) + { + const existing = [...rows, ...cols].find (axis => axis.field === field) + const next = groups[key].filter (axis => axis.field !== field) + if (target === key) + { + const index = position == null ? next.length : next.findIndex ( + axis => axis.field === position) + next.splice (index < 0 ? next.length : index, 0, existing ?? { + field, ...(countsFilterFields (resource)[field] === 'datetime' + ? { grain: 'month' } : { }) }) + } + update (key, JSON.stringify (next)) + } + } + return undefined, onDragOver: () => undefined, + onDragMove: () => undefined, onDragEnd: () => undefined, + onDragCancel: () => undefined }, + screenReaderInstructions: { draggable: message ('axis').text } }} + onDragEnd={({ active, over }) => { + if (over == null) return + const [, field] = String (active.id).split ('/') + const [zone, position] = String (over.id).split ('/') + place (field, zone, position) + }}> +
+
{label ('axisItems')} + {available.filter (field => ![...rows, ...cols].some ( + axis => axis.field === field.key)).map (field => ( + + {label (field.label)} + {(['rows', 'cols'] as const).map (target => )} + ))} +
+ {(['rows', 'cols'] as const).map (zone =>
+ {label (zone)} + {groups[zone].map ((axis, index) => { + const field = available.find (field => field.key === axis.field) + return + {field == null ? null : label (field.label)} + {countsFilterFields (resource)[axis.field] === 'datetime' && } + + + + + })} + +
)} +
+ +} + + +export default CountsAxes diff --git a/frontend/src/components/counts/CountsControls.tsx b/frontend/src/components/counts/CountsControls.tsx index 22ab6b1..aa572db 100644 --- a/frontend/src/components/counts/CountsControls.tsx +++ b/frontend/src/components/counts/CountsControls.tsx @@ -1,14 +1,15 @@ import { useEffect, useId, useState } from 'react' +import CountsAdditionalConditions from '@/components/counts/CountsAdditionalConditions' +import CountsAxes from '@/components/counts/CountsAxes' import CountsFieldTree from '@/components/counts/CountsFieldTree' import DateTimeField from '@/components/common/DateTimeField' import FormField from '@/components/common/FormField' +import HelpPopover from '@/components/common/HelpPopover' +import TagInput from '@/components/common/TagInput' import { Button } from '@/components/ui/button' -import { isCountsState, normaliseCountsParams } from '@/lib/counts' -import { countsSummaryFields } from '@/lib/countsState' -import { fetchTagByName } from '@/lib/tags' +import { normaliseCountsParams } from '@/lib/counts' import { inputClass } from '@/lib/utils' -import { useLocale } from '@/lib/useLocale' import { useUITranslation } from '@/lib/useUITranslation' import type { ReactNode } from 'react' @@ -18,17 +19,13 @@ import type { CountsResource } from '@/lib/counts' const CountsControls = ( { resource, applied, open, setOpen, execute, displayJSON, reorder, actions }: { - resource: CountsResource - applied: URLSearchParams - open: boolean - setOpen: (open: boolean) => void - execute: (params: URLSearchParams) => void - displayJSON: (enabled: boolean) => void - reorder: (value: string) => void - actions?: ReactNode }, + resource: CountsResource; applied: URLSearchParams; open: boolean + setOpen: (open: boolean) => void + execute: (params: URLSearchParams) => void + displayJSON: (enabled: boolean) => void + reorder: (value: string) => void; actions?: ReactNode }, ) => { const { label, message } = useUITranslation ('counts') - const locale = useLocale () const id = useId () const [draft, setDraft] = useState (() => new URLSearchParams (applied)) const order = applied.get ('order') @@ -38,175 +35,96 @@ const CountsControls = ( const next = new URLSearchParams (previous) for (const [key, value] of Object.entries ({ order, nested_json: nestedJSON })) { - if (value == null) next.delete (key) - else next.set (key, value) + if (value == null) next.delete (key) + else next.set (key, value) } return next }) }, [order, nestedJSON]) - const [error, setError] = useState (false) - const [pending, setPending] = useState (false) const mode = draft.get ('mode') ?? 'summary' - const state = isCountsState (draft) - const axes = (state && resource === 'posts') ? ['tags', 'month', 'user'] : ['month', 'user'] - const update = (key: string, value: string) => { - setDraft (previous => { - const next = new URLSearchParams (previous) - if (['mode', 'axis', 'basis'].includes (key)) next.delete ('sort') - if (key === 'axis' || key === 'basis') next.delete ('summary_columns') - if (key === 'basis' && value === 'history' && next.get ('axis') === 'tags') - next.set ('axis', 'month') - if (value || ['columns', 'children', 'summary_columns'].includes (key)) - next.set (key, value) - else next.delete (key) - return next - }) - } - const submit = async () => { - setError (false) - setPending (true) - try - { - const next = new URLSearchParams (draft) - for (const key of resource === 'posts' ? ['tag'] : ['resource_id']) - { - const value = next.get (key)?.trim () ?? '' - if (value && !/^[1-9][0-9]*$/.test (value)) - { - if (state && next.get ('kind') === 'external') throw new Error () - const tag = await fetchTagByName (value, locale) - if (tag == null) - throw new Error () - next.set (key, String (tag.id)) - } - } - next.delete ('page') - execute (normaliseCountsParams (resource, next)) - } - catch - { - setError (true) - } - finally - { - setPending (false) - } - } - - return ( -
{ - event.preventDefault () - void submit () - }}> -
- - - - {actions} -
- {error &&

{label ('shared:messages.checkInputModern')}

} - -
) + const history = draft.get ('definition') === 'history' + const update = (key: string, value: string) => setDraft (previous => { + const next = new URLSearchParams (previous) + if (['mode', 'rows', 'cols'].includes (key)) next.delete ('sort') + next.set (key, value) + return next + }) + return
{ + event.preventDefault () + const next = new URLSearchParams (draft) + next.delete ('page') + next.delete ('execution_at') + execute (normaliseCountsParams (resource, next)) + }}> +
+ {history ? <> + {['from', 'through'].map (key => )} + : } +
+
+ + + {actions} +
+ +
} diff --git a/frontend/src/components/counts/CountsFieldTree.tsx b/frontend/src/components/counts/CountsFieldTree.tsx index b8d9307..24fea83 100644 --- a/frontend/src/components/counts/CountsFieldTree.tsx +++ b/frontend/src/components/counts/CountsFieldTree.tsx @@ -124,7 +124,6 @@ const CountsFieldTree = ( params: URLSearchParams update: (key: string, value: string) => void }, ) => { - const { label } = useUITranslation ('counts') const fields = suppliedFields ?? fieldsForCounts (resource) const roots = suppliedFields == null ? selectedCountColumns (resource, params) : selectedCountFields (fields, params, selectionKey, parent).map (field => field.key) @@ -151,7 +150,6 @@ const CountsFieldTree = ( <> - {suppliedFields == null &&

{label ('nested.selection')}

} ) } diff --git a/frontend/src/components/counts/CountsNestedTable.tsx b/frontend/src/components/counts/CountsNestedTable.tsx index c0d3ace..3093b3c 100644 --- a/frontend/src/components/counts/CountsNestedTable.tsx +++ b/frontend/src/components/counts/CountsNestedTable.tsx @@ -34,6 +34,10 @@ const CountsNestedTable = ( const scalar = (row: CountRecord, field: string) => { const value = row[field] if (value == null) return null + if (field === 'tagId' || field === 'tagGroupId') + return + {String (value)} if (field === 'id') { if (row.kind === 'external') return String (value) diff --git a/frontend/src/components/counts/CountsResults.tsx b/frontend/src/components/counts/CountsResults.tsx index e47fb40..abdd31f 100644 --- a/frontend/src/components/counts/CountsResults.tsx +++ b/frontend/src/components/counts/CountsResults.tsx @@ -7,12 +7,12 @@ import CountsSelectableTable from '@/components/counts/CountsSelectableTable' import PostThumbnailPreview from '@/components/posts/PostThumbnailPreview' import { changeCountSort, countColumns, countsRequestParams, fieldsForCounts, readCountSort, selectedCountColumns, writeCountSort } from '@/lib/counts' -import { countsCellCopy, countsCellText, countsImageURL } from '@/lib/countsDisplay' +import { countsCellCopy, countsCellText, countsImageURL, countsNestedRows } from '@/lib/countsDisplay' import useDialogue from '@/lib/dialogues/useDialogue' import { useLocale } from '@/lib/useLocale' import { useUITranslation } from '@/lib/useUITranslation' -import type { CountColumn, CountRecord, CountsResource, +import type { CountColumn, CountsResource, CountsResult, CountsVersion } from '@/lib/counts' @@ -34,17 +34,23 @@ const CountsResults = ( update ('sort', writeCountSort (changeCountSort (sorts, key, additive))) const cell = (row: CountsVersion, column: CountColumn) => { const field = fields.find (field => field.key === column)! - const value = countsCellText (row, column, locale) + const value = countsCellText (row, column, locale, params) if (field.children != null) { - return } switch (column) { case 'resource_id': - return + if (resource === 'nico_tags') return + {row.resourceId} + if (resource === 'tag_groups') return + {row.resourceId} + return {row.resourceId} case 'thumbnail': { diff --git a/frontend/src/components/counts/CountsSummaryResults.tsx b/frontend/src/components/counts/CountsSummaryResults.tsx index adb434e..30e8b1c 100644 --- a/frontend/src/components/counts/CountsSummaryResults.tsx +++ b/frontend/src/components/counts/CountsSummaryResults.tsx @@ -1,62 +1,98 @@ -import LocalisedText from '@/components/LocalisedText' import PrefetchLink from '@/components/PrefetchLink' import CountsSelectableTable from '@/components/counts/CountsSelectableTable' import CountsSortHeader from '@/components/counts/CountsSortHeader' import { changeCountSort, countsRequestParams, readCountSort, writeCountSort } from '@/lib/counts' -import { countsSummaryCell, countsSummaryDrill, countsSummaryRows, - selectedCountsSummaryFields } from '@/lib/countsState' -import { useLocale } from '@/lib/useLocale' +import { countsFieldLabel, readCountsJSON } from '@/lib/countsDefinitions' import { useUITranslation } from '@/lib/useUITranslation' import type { CountsResource, CountsResult } from '@/lib/counts' +import type { CountsAxis } from '@/lib/countsDefinitions' const CountsSummaryResults = ( { resource, data, params, pending, update }: { - resource: CountsResource - data: CountsResult - params: URLSearchParams - pending: boolean - update: (key: string, value: string) => void }, + resource: CountsResource; data: CountsResult; params: URLSearchParams + pending: boolean; update: (key: string, value: string) => void }, ) => { - const locale = useLocale () const { label, message } = useUITranslation ('counts') - const fields = selectedCountsSummaryFields (resource, params) - const rows = countsSummaryRows (data) - const sorts = readCountSort (params.get ('sort') ?? ( - params.get ('axis') === 'tags' ? 'count:desc' : 'group:asc')) - return ( -
- {fields.length === 0 ?

{label ('nested.noColumns')}

: ( - field.key)} - identity={`${countsRequestParams (params)}:${fields.map (field => field.key)}`} - className="text-left text-sm"> - - {fields.map (field => ( - update ( - 'sort', writeCountSort (changeCountSort (sorts, key, additive)))}/>))} - - {rows.map (row => ( - - {fields.map (field => { - const value = countsSummaryCell (row, field.key, locale) - const content = value.language == null ? value.text : ( - {value.text}) - return - {(['count', 'group'].includes (field.key) && row.count != null) - ? - {content} - : content} - - })} - ))} - )} - {rows.length === 0 &&

{label ('empty')}

} -
) + const rows = data.rows ?? [] + const cols = data.cols ?? [] + const axes = [...rows, ...cols] + const cells = data.cells ?? [] + const rowKeys = [...new Set (cells.map (cell => JSON.stringify (cell.values.slice (0, rows.length))))] + const columnKeys = data.columnValues?.map (values => JSON.stringify (values)) + ?? [...new Set (cells.map (cell => JSON.stringify (cell.values.slice (rows.length))))] + if (cols.length === 0 && columnKeys.length === 0) columnKeys.push ('[]') + const sorts = readCountSort (params.get ('sort')) + const display = (value: string | number | null, axis: CountsAxis, index: number): string => { + if (value == null) return '' + const logical = data.axisLabels?.[`a${index}`]?.[String (value)] + if (logical != null) return logical + if (['tags', 'user', 'language_code', 'source_language_code'].includes (axis.field)) + return message ('notRegistered').text + if (axis.field === 'changed_field') return message (countsFieldLabel (String (value))).text + if (axis.field === 'event_type') return message (`events.${value}`).text + if (axis.field === 'category' || axis.field === 'tag_category') + return message (`common:categories.${value}`).text + return String (value) + } + const drill = (values: (string | number | null)[]) => { + const next = new URLSearchParams (params) + const existing = readCountsJSON (params, 'drill', []) + next.set ('drill', JSON.stringify ([...existing, + ...axes.map ((axis, index) => ({ ...axis, value: values[index] }))])) + next.set ('mode', 'details') + next.delete ('sort') + next.delete ('page') + return `/counts/${resource}?${next}` + } + const keys = [...rows.map ((_, index) => `a${index}`), + ...columnKeys.map ((_, index) => `cell${index}`)] + return
+ + + {rows.map ((axis, index) => update ('sort', writeCountSort ( + changeCountSort (sorts, key, additive)))}/>)} + {columnKeys.map ((key, index) => { + const values = JSON.parse (key) as (string | number | null)[] + const name = values.map ((value, position) => + `${message (countsFieldLabel (cols[position].field)).text}: ${ + display (value, cols[position], rows.length + position)}`).join (' / ') + const sortKey = cols.length === 0 ? 'count' : `measure_${[ + ...new TextEncoder ().encode (key)].map (byte => + byte.toString (16).padStart (2, '0')).join ('')}` + return sort.key === sortKey + ? { ...sort, key: `cell${index}` } : sort)} + change={(_, additive) => update ('sort', writeCountSort ( + changeCountSort (sorts, sortKey, additive)))}/> + })} + + {rowKeys.map (rowKey => { + const rowValues = JSON.parse (rowKey) as (string | number | null)[] + return + {rowValues.map ((value, index) => + {display (value, rows[index], index)})} + {columnKeys.map (columnKey => { + const columnValues = JSON.parse (columnKey) as (string | number | null)[] + const values = [...rowValues, ...columnValues] + const cell = cells.find (cell => JSON.stringify (cell.values) === JSON.stringify (values)) + return + {cell?.count ?? 0} + + })} + + })} + + {cells.length === 0 &&

{label ('empty')}

} +
} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 817bbfc..663f623 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -199,3 +199,28 @@ export const apiDelete = async ( export const isApiError = (err: unknown): err is AxiosError => axios.isAxiosError (err) + + +// Native downloads let the browser stream to disk without buffering a Blob in app memory. +export const apiDownload = async ( + path: string, + body: Record, + locale: LocaleCode, +): Promise => { + const token = await csrfToken () + const form = document.createElement ('form') + form.method = 'post' + form.action = `${API_BASE_URL}${path}?${new URLSearchParams ({ locale })}` + form.hidden = true + for (const [name, value] of Object.entries ({ ...body, authenticity_token: token })) + { + const input = document.createElement ('input') + input.type = 'hidden' + input.name = name + input.value = value + form.append (input) + } + document.body.append (form) + form.submit () + form.remove () +} diff --git a/frontend/src/lib/counts.ts b/frontend/src/lib/counts.ts index d64ccf8..5cf7324 100644 --- a/frontend/src/lib/counts.ts +++ b/frontend/src/lib/counts.ts @@ -1,9 +1,11 @@ -import { apiGet, apiPost } from '@/lib/api' +import { countsAttributes, countsFieldLabel } from '@/lib/countsDefinitions' +import { apiDownload, apiGet } from '@/lib/api' import type { LocaleCode } from '@/lib/locales' import type { TagNameSnapshot } from '@/types' -export type CountsResource = 'posts' | 'tags' +export type CountsResource = 'posts' | 'tags' | 'nico_tags' + | 'tag_descriptions' | 'tag_groups' | 'materials' export type CountsName = TagNameSnapshot & { languageCode: string; languageName?: string | null; scriptName?: string | null } export type CountsTag = { @@ -17,6 +19,8 @@ export type CountsTag = { sections: { beginMs: number; endMs: number | null }[] names: CountsName[] } export type CountsVersion = { + [key: string]: unknown + eventType?: string id: number resourceId: number versionNo: number @@ -51,6 +55,12 @@ export type CountsStateTotals = { deprecatedTags: number | null externalAssignments: number | null } export type CountsResult = { + executionAt?: string + columnValues?: (string | number | null)[][] + cells?: { values: (string | number | null)[]; count: number }[] + axisLabels?: Record> + rows?: { field: string; grain?: string }[] + cols?: { field: string; grain?: string }[] count: number page: number limit: number @@ -69,7 +79,8 @@ export type CountSort = { key: string; direction: 'asc' | 'desc' } export type CountField = { key: string; label: string; children?: CountField[] } export type CountRecord = Record -export const countColumns = { +export const countColumns: Record = { + event_type: 'counts:eventType', thumbnail: 'counts:columns.thumbnailDisplay', resource_id: 'counts:columns.resourceId', version_no: 'history:version', is_latest: 'counts:columns.latest', title: 'counts:columns.postTitle', url: 'counts:columns.postURL', @@ -82,7 +93,7 @@ export const countColumns = { category: 'common:categoryLabel', aliases: 'tags:aliases', deprecated: 'counts:columns.deprecated', user: 'history:actor', created_at: 'counts:columns.changedAt' } -export type CountColumn = keyof typeof countColumns +export type CountColumn = string const aliasFields: CountField[] = [ { key: 'name', label: 'history:name' }, @@ -93,7 +104,24 @@ const nameFields: CountField[] = [ { key: 'name', label: 'history:name' }, { key: 'aliases', label: 'tags:aliases', children: aliasFields }, { key: 'autoGenerated', label: 'history:nameChanges.automatic' }] +const placementFields = (depth: number): CountField[] => [ + { key: 'sectionName', label: 'counts:attributes.section_name' }, + { key: 'description', label: 'counts:attributes.description' }, + { key: 'tagId', label: 'counts:attributes.tag_id' }, + { key: 'tagGroupId', label: 'counts:resources.tag_groups' }, + ...(depth > 0 ? [{ key: 'children', label: 'counts:attributes.children', + children: placementFields (depth - 1) }] : [])] + + const nested: Partial> = { + placements: placementFields (4), + export_paths_json: [ + { key: 'profile', label: 'counts:attributes.profile' }, + { key: 'path', label: 'counts:attributes.path' }], + members: [ + { key: 'tagId', label: 'counts:columns.resourceId' }, + { key: 'tagGroupId', label: 'counts:resources.tag_groups' }, + { key: 'versionNo', label: 'history:version' }], tags: [ { key: 'id', label: 'counts:columns.resourceId' }, { key: 'versionNo', label: 'history:version' }, @@ -108,19 +136,22 @@ const nested: Partial> = { export const columnsForCounts = (resource: CountsResource): CountColumn[] => [ - ...(resource === 'posts' ? ['thumbnail'] as const : []), + ...(resource === 'posts' ? ['thumbnail'] : []), 'resource_id', 'version_no', 'is_latest', ...(resource === 'posts' - ? ['title', 'url', 'thumbnail_base', 'tags', 'parents', 'video_ms', - 'original_created_at', 'original_created_from', 'original_created_before'] as const - : ['name', 'names', 'category', 'aliases', 'parents', 'deprecated'] as const), - 'user', 'created_at'] + ? ['title', 'url', 'thumbnail_base', 'tags', 'parents', 'video_ms', + 'original_created_at', 'original_created_from', 'original_created_before'] + : (resource === 'tags' + ? ['name', 'names', 'category', 'aliases', 'parents', 'deprecated'] + : Object.keys (countsAttributes[resource]))), + 'event_type', 'user', 'created_at'] export const fieldsForCounts = (resource: CountsResource): CountField[] => columnsForCounts (resource).map (key => ({ key, - label: (key === 'parents' && resource === 'tags') ? 'tags:parents' : countColumns[key], + label: (key === 'parents' && resource === 'tags') ? 'tags:parents' + : (countColumns[key] ?? countsFieldLabel (key)), children: nested[key] })) @@ -185,7 +216,7 @@ export const countChildPaths = (fields: CountField[], prefix = ''): string[] => export const selectedCountChildren = (params: URLSearchParams): Set => new Set (params.has ('children') ? params.get ('children')!.split (',') : Object.entries (nested).flatMap (([key, fields]) => - countChildPaths (fields, key))) + countChildPaths (fields ?? [], key))) export const readCountSort = (value: string | null): CountSort[] => @@ -234,24 +265,22 @@ export const normaliseCountsParams = ( source: URLSearchParams, ): URLSearchParams => { const params = new URLSearchParams (source) - params.delete ('event_type') + if (!params.has ('definition')) + params.set ('definition', params.get ('basis') === 'history' ? 'history' : 'latest') const legacy = params.get ('mode') if (['state', 'trend', 'snapshot'].includes (legacy ?? '')) - { params.set ('mode', legacy === 'snapshot' ? 'details' : 'summary') - if (legacy !== 'snapshot') params.set ('axis', resource === 'posts' ? 'tags' : 'month') - if (legacy === 'trend') - { - if (params.get ('before')) params.set ('at', params.get ('before')!) - params.delete ('from') - params.delete ('before') - } - } if (!params.has ('mode')) params.set ('mode', 'summary') - const axes = (resource === 'posts' && params.get ('basis') !== 'history') - ? ['tags', 'month', 'user'] : ['month', 'user'] - if (!axes.includes (params.get ('axis') ?? '')) params.set ('axis', axes[0]) - params.delete ('grain') + if (!params.has ('rows') && params.has ('axis')) + { + const field = params.get ('axis') === 'month' ? 'created_at' : params.get ('axis')! + params.set ('rows', JSON.stringify ([{ + field, ...(field === 'created_at' ? { grain: 'month' } : { }) }])) + } + if (params.has ('before') && !params.has ('through')) + params.set ('through', params.get ('before')!) + for (const key of ['basis', 'axis', 'latest', 'grain', 'before', 'summary_columns']) + params.delete (key) if (params.has ('columns')) { const selected = new Set (params.get ('columns')!.split (',')) @@ -263,14 +292,13 @@ export const normaliseCountsParams = ( export const isCountsState = (params: URLSearchParams): boolean => - params.get ('basis') !== 'history' + params.get ('definition') !== 'history' export const countsRequestParams = (params: URLSearchParams): URLSearchParams => { const result = new URLSearchParams (params) - const hidden = isCountsState (params) ? ['latest'] : ['kind'] for (const key of ['columns', 'summary_columns', 'children', 'child_sort', - 'nested_json', 'order', 'basis', 'grain', ...hidden]) + 'nested_json', 'order', 'basis', 'axis', 'latest', 'grain']) result.delete (key) return result } @@ -280,10 +308,9 @@ export const fetchCounts = ( resource: CountsResource, params: URLSearchParams, locale: LocaleCode, -): Promise => apiGet ( - isCountsState (params) ? '/counts/state' : `/counts/${resource}`, { - locale, params: { ...Object.fromEntries (countsRequestParams (params)), - ...(isCountsState (params) ? { resource } : { }) } }) +): Promise => apiGet ('/counts/query', { + locale, params: { ...Object.fromEntries (countsRequestParams (params)), resource } }) + export const exportCounts = ( @@ -294,10 +321,10 @@ export const exportCounts = ( columns: string[], headers: string[], labels: Record, -): Promise => apiPost ('/counts/export', { +): Promise => apiDownload ('/counts/export', { ...Object.fromEntries (params), resource, format, columns: columns.join (','), headers: JSON.stringify (headers), - labels: JSON.stringify (labels) }, { locale, responseType: 'blob' }) + labels: JSON.stringify (labels) }, locale) diff --git a/frontend/src/lib/countsDefinitions.ts b/frontend/src/lib/countsDefinitions.ts new file mode 100644 index 0000000..696d946 --- /dev/null +++ b/frontend/src/lib/countsDefinitions.ts @@ -0,0 +1,72 @@ +import type { CountField, CountsResource } from '@/lib/counts' + +export type CountsAxis = { field: string; grain?: string } +export type CountsCondition = { + field: string; operator: string; value?: string; end?: string; blank?: boolean } +export type CountsFieldType = 'string' | 'integer' | 'datetime' | 'enum' | 'json' + +export const countsResources: CountsResource[] = [ + 'posts', 'tags', 'nico_tags', 'tag_descriptions', 'tag_groups', 'materials'] +export const countsGrains = ['year', 'quarter', 'month', 'week', 'day', 'hour'] +export const countsOperators: Record = { + string: ['eq', 'ne', 'like'], enum: ['eq', 'ne'], json: [], + integer: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'], + datetime: ['eq', 'ne', 'gt', 'lt', 'ge', 'le', 'range'] } + +export const countsAttributes: Record> = { + posts: { title: 'string', url: 'string', thumbnail_base: 'string', tags_json: 'json', + parent_post_ids: 'string', video_ms: 'integer', original_created_from: 'datetime', + original_created_before: 'datetime' }, + tags: { name: 'string', names: 'json', category: 'enum', aliases: 'string', + parent_tag_ids: 'string', deprecated_at: 'datetime' }, + nico_tags: { name: 'string', linked_tags: 'string' }, + tag_descriptions: { language_code: 'enum', body: 'string', source_language_code: 'enum', + source_version_no: 'integer' }, + tag_groups: { name: 'string', description: 'string', members: 'json', placements: 'json' }, + materials: { url: 'string', parent_id: 'integer', tag_id: 'integer', tag_name: 'string', + tag_category: 'enum', discarded_at: 'datetime', updated_at: 'datetime', + updated_by_user_id: 'integer', export_paths_json: 'json', file_blob_id: 'integer', + file_filename: 'string', file_content_type: 'string', file_byte_size: 'integer', + file_checksum: 'string', file_sha256: 'string', source_kind: 'enum', source_uri: 'string', + source_path: 'string', source_file_id: 'string', normalized_source_key: 'string' } } + + +export const countsFilterFields = ( + resource: CountsResource, +): Record => ({ + resource_id: 'integer' as const, version_no: 'integer' as const, + event_type: 'enum' as const, user: 'integer' as const, created_at: 'datetime' as const, + ...countsAttributes[resource] }) + + +export const countsFieldLabel = (key: string): string => { + const labels: Record = { + resource_id: 'columns.resourceId', version_no: 'history:version', + user: 'history:actor', created_at: 'columns.changedAt', name: 'history:name', + title: 'columns.postTitle', url: 'columns.postURL', thumbnail_base: 'columns.thumbnailURL', + tags: 'columns.tags', tags_json: 'columns.tags', names: 'columns.names', + aliases: 'tags:aliases', category: 'common:categoryLabel', + language_code: 'columns.language', source_language_code: 'attributes.sourceLanguage', + changed_field: 'changedField' } + return labels[key] ?? `attributes.${key}` +} + + +export const countsAxisFields = (resource: CountsResource, history: boolean): CountField[] => [ + ...Object.entries (countsFilterFields (resource)) + .filter (([, type]) => type !== 'json') + .map (([key]) => ({ key, label: countsFieldLabel (key) })), + ...(resource === 'posts' ? [{ key: 'tags', label: 'columns.tags' }] : []), + ...(history ? [{ key: 'changed_field', label: 'changedField' }] : [])] + + +export const readCountsJSON = (params: URLSearchParams, key: string, fallback: T): T => { + try + { + return JSON.parse (params.get (key) ?? JSON.stringify (fallback)) as T + } + catch + { + return fallback + } +} diff --git a/frontend/src/lib/countsDisplay.ts b/frontend/src/lib/countsDisplay.ts index 37c1523..50ed06f 100644 --- a/frontend/src/lib/countsDisplay.ts +++ b/frontend/src/lib/countsDisplay.ts @@ -12,7 +12,10 @@ export const countsCellText = ( row: CountsVersion, column: CountColumn, locale: LocaleCode, + params = new URLSearchParams (), ): string => { + const timezone = params.get ('timezone') ?? 'UTC' + const date = (value: string) => localisedDateString (value, locale, null, timezone) switch (column) { case 'resource_id': return String (row.resourceId) @@ -28,6 +31,10 @@ export const countsCellText = ( return translateUIMessage (`common:categories.${row.category}`, 'counts', locale).text case 'video_ms': return row.videoMs == null ? '' : msToTime (row.videoMs) case 'original_created_at': + if (timezone !== 'Asia/Tokyo') + return translateUIMessage ('sections.range', 'counts', locale, { + begin: row.originalCreatedFrom == null ? '' : date (row.originalCreatedFrom), + end: row.originalCreatedBefore == null ? '' : date (row.originalCreatedBefore) }).text return originalCreatedAtString ( row.originalCreatedFrom ?? null, row.originalCreatedBefore ?? null, locale) case 'original_created_from': @@ -35,10 +42,10 @@ export const countsCellText = ( { const value = column === 'original_created_from' ? row.originalCreatedFrom : row.originalCreatedBefore - return value == null ? '' : localisedDateString (value, locale) + return value == null ? '' : date (value) } case 'deprecated': - return row.deprecated == null ? '' : localisedDateString (row.deprecated, locale) + return row.deprecated == null ? '' : date (row.deprecated) case 'user': if (row.createdByUser == null) return translateUIMessage ('history:bot', 'counts', locale).text @@ -46,12 +53,33 @@ export const countsCellText = ( ? `${row.createdByUser.name} (#${row.createdByUser.id})` : translateUIMessage ('history:anonymous', 'counts', locale, { id: row.createdByUser.id }).text - case 'created_at': return localisedDateString (row.createdAt, locale) - default: return '' + case 'created_at': return date (row.createdAt) + default: + { + const key = column.replace (/_([a-z])/g, (_, letter: string) => letter.toUpperCase ()) + const value = row[key] + if (['discarded_at', 'updated_at'].includes (column) && typeof value === 'string') + return date (value) + if (column === 'event_type') + return translateUIMessage (`events.${value}`, 'counts', locale).text + if (column === 'language_code' || column === 'source_language_code') + return countsLogicalName ({ languageCode: value }, 'languageCode', locale).text + return value == null ? '' : (typeof value === 'object' + ? JSON.stringify (value, null, 2) : String (value)) + } } } +export const countsNestedRows = (row: CountsVersion, column: string): CountRecord[] => { + const key = column.replace (/_([a-z])/g, (_, letter: string) => letter.toUpperCase ()) + const value = row[key] + if (column === 'export_paths_json' && value != null && typeof value === 'object') + return Object.entries (value).map (([profile, path]) => ({ profile, path })) + return Array.isArray (value) ? value as CountRecord[] : [] +} + + export const countsCSVCell = ( row: CountsVersion, column: CountColumn, @@ -62,10 +90,10 @@ export const countsCSVCell = ( const field = fieldsForCounts (resource).find (field => field.key === column)! if (field.children != null) { - const rows = row[column as keyof CountsVersion] as CountRecord[] ?? [] + const rows = countsNestedRows (row, column) return JSON.stringify (countsNestedExport (rows, field.children, column, params, locale)) } - return countsCellText (row, column, locale) + return countsCellText (row, column, locale, params) } @@ -189,8 +217,8 @@ export const countsCellCopy = ( params: URLSearchParams, locale: LocaleCode, ): string => { - if (field.children == null) return countsCellText (row, field.key as CountColumn, locale) - const rows = row[field.key as keyof CountsVersion] as CountRecord[] ?? [] + if (field.children == null) return countsCellText (row, field.key as CountColumn, locale, params) + const rows = countsNestedRows (row, field.key) return params.get ('nested_json') === '1' ? JSON.stringify ( countsNestedExport (rows, field.children, field.key, params, locale), null, 2) @@ -240,6 +268,10 @@ export const countsExportLabels = ( for (const key of ['ja', 'ain']) labels[`language:${key}`] = translateUIMessage ( `history:languageNames.${key}`, 'counts', locale).text + labels.id = translateUIMessage ('columns.resourceId', 'counts', locale).text + labels.name = translateUIMessage ('history:name', 'counts', locale).text + for (const event of ['create', 'update', 'discard', 'restore']) + labels[`event_type:${event}`] = translateUIMessage (`events.${event}`, 'counts', locale).text labels.latest = translateUIMessage ('latestValue', 'counts', locale).text labels.notRegistered = translateUIMessage ('notRegistered', 'counts', locale).text return labels diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index a4c54e1..94665bb 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -17,24 +17,26 @@ export const cn = (...inputs: ClassValue[]) => twMerge (clsx (...inputs)) type DatePrecision = 'month' | 'day' | 'hour' | 'minute' | 'second' | null -const dateFormatters = new Map () -let ainuDateFormatter: Intl.DateTimeFormat | null = null +const dateFormatters = new Map () +const ainuDateFormatters = new Map () const formatDateString = ( date: Date, unknown: DatePrecision, locale: LocaleCode, + timeZone = 'Asia/Tokyo', ): string => { + const key = `${timeZone}/${unknown}` if (Number.isNaN (date.getTime ())) return date.toString () if (locale === 'ja') { - if (!dateFormatters.has (unknown)) + if (!dateFormatters.has (key)) { - dateFormatters.set (unknown, new Intl.DateTimeFormat ('ja-JP-u-ca-japanese', { + dateFormatters.set (key, new Intl.DateTimeFormat ('ja-JP-u-ca-japanese', { era: 'long', year: 'numeric', - timeZone: 'Asia/Tokyo', + timeZone, month: unknown === 'month' ? undefined : 'long', day: unknown != null && ['month', 'day'].includes (unknown) ? undefined : 'numeric', weekday: unknown != null && ['month', 'day'].includes (unknown) ? undefined : 'short', @@ -42,14 +44,15 @@ const formatDateString = ( minute: unknown == null || unknown === 'second' ? 'numeric' : undefined, second: unknown == null ? 'numeric' : undefined })) } - return dateFormatters.get (unknown)!.format (date) + return dateFormatters.get (key)!.format (date) } - ainuDateFormatter ??= new Intl.DateTimeFormat ('en-CA', { + if (!ainuDateFormatters.has (timeZone)) + ainuDateFormatters.set (timeZone, new Intl.DateTimeFormat ('en-CA', { year: 'numeric', month: '2-digit', day: '2-digit', - timeZone: 'Asia/Tokyo', + timeZone, hour: '2-digit', minute: '2-digit', second: '2-digit', - hourCycle: 'h23' }) - const parts = ainuDateFormatter.formatToParts (date) + hourCycle: 'h23' })) + const parts = ainuDateFormatters.get (timeZone)!.formatToParts (date) const part = (type: string) => parts.find (entry => entry.type === type)?.value ?? '' const omitDay = unknown != null && ['month', 'day'].includes (unknown) const dateParts = [part ('year'), @@ -76,7 +79,8 @@ export const localisedDateString = ( value: string | Date, locale: LocaleCode, unknown: DatePrecision = null, -): string => dateString (value, unknown, locale) + timeZone?: string, +): string => formatDateString (toDate (value), unknown, locale, timeZone) export const originalCreatedAtString = ( diff --git a/frontend/src/locales/ja/counts.json b/frontend/src/locales/ja/counts.json index 558fdb9..a835f6c 100644 --- a/frontend/src/locales/ja/counts.json +++ b/frontend/src/locales/ja/counts.json @@ -1,9 +1,13 @@ { "titles": { - "posts": "投稿の集計・明細", - "tags": "タグの集計・明細" + "posts": "広場の集計・明細", + "tags": "タグの集計・明細", + "nico_tags": "ニコニコタグの集計・明細", + "tag_descriptions": "タグ説明の集計・明細", + "tag_groups": "タググループの集計・明細", + "materials": "素材の集計・明細" }, - "target": "集計対象", + "target": "集計定義", "run": "集計実行", "conditions": "集計条件", "output": "出力項目", @@ -119,5 +123,99 @@ "exportFormats": { "json": "JSON", "csv": "CSV" + }, + "resources": { + "posts": "広場", + "tags": "タグ", + "nico_tags": "ニコニコタグ", + "tag_descriptions": "タグ説明", + "tag_groups": "タググループ", + "materials": "素材" + }, + "latest": "最新", + "history": "履歴", + "referenceAt": "基準日時", + "referenceHelp": "指定した日時における各データの最新版を集計・表示します。空欄の場合は現在の最新版を使用します。", + "from": "開始（以上）", + "through": "終了（以下）", + "axisItems": "項目一覧", + "rows": "行", + "cols": "列", + "axisPlace": "{{field}}を{{target}}に配置", + "field": "項目", + "operator": "演算子", + "add": "追加", + "remove": "削除", + "value": "値", + "blank": "空欄", + "additionalConditions": "追加条件", + "changedField": "変更項目", + "eventType": "異動区分", + "count": "件数", + "recordTotal": "総件数：{{count}}", + "grain": "粒度", + "grains": { + "year": "年", + "quarter": "四半期", + "month": "月", + "week": "週", + "day": "日", + "hour": "時" + }, + "operators": { + "eq": "等しい", + "ne": "等しくない", + "gt": "より大きい", + "lt": "未満", + "ge": "以上", + "le": "以下", + "like": "パターン", + "range": "範囲" + }, + "events": { + "create": "作成", + "update": "更新", + "discard": "破棄", + "restore": "復元" + }, + "attributes": { + "sourceLanguage": "翻訳元の言語", + "source_language_code": "翻訳元の言語", + "source_version_no": "翻訳元の版番号", + "event_type": "異動区分", + "body": "本文", + "linked_tags": "対応タグ", + "description": "説明", + "members": "構成要素", + "placements": "配置", + "parent_id": "親素材", + "tag_id": "タグ", + "tag_name": "タグ名称", + "tag_category": "タグカテゴリ", + "discarded_at": "破棄日時", + "updated_at": "履歴行更新日時", + "updated_by_user_id": "履歴行更新者", + "export_paths_json": "出力先", + "file_blob_id": "ファイル識別子", + "file_filename": "ファイル名", + "file_content_type": "ファイル形式", + "file_byte_size": "ファイル容量", + "file_checksum": "チェックサム", + "file_sha256": "SHA-256", + "source_kind": "取得元の種類", + "source_uri": "取得元 URI", + "source_path": "取得元のパス", + "source_file_id": "取得元ファイル識別子", + "normalized_source_key": "取得元識別キー", + "parent_post_ids": "親投稿", + "parent_tag_ids": "上位タグ", + "video_ms": "動画時間", + "original_created_from": "オリジナルの投稿日時（開始境界）", + "original_created_before": "オリジナルの投稿日時（終了境界）", + "deprecated_at": "廃止日時", + "section_name": "節の名称", + "children": "子項目", + "profile": "出力設定", + "path": "出力パス" } } diff --git a/frontend/src/pages/CountsResourcePage.tsx b/frontend/src/pages/CountsResourcePage.tsx index 3f0da37..e610b6b 100644 --- a/frontend/src/pages/CountsResourcePage.tsx +++ b/frontend/src/pages/CountsResourcePage.tsx @@ -16,12 +16,15 @@ import { Button } from '@/components/ui/button' import { countsRequestParams, exportCounts, fieldsForCounts, fetchCounts, isCountsState, normaliseCountsParams, selectedCountColumns } from '@/lib/counts' import { countsExportLabels } from '@/lib/countsDisplay' -import { selectedCountsSummaryFields } from '@/lib/countsState' +import { countsAxisFields, countsResources, countsFieldLabel, readCountsJSON } + from '@/lib/countsDefinitions' import { countsKeys } from '@/lib/queryKeys' import { cn, inputClass } from '@/lib/utils' import { useLocale } from '@/lib/useLocale' import { useUITranslation } from '@/lib/useUITranslation' +import type { CountsAxis } from '@/lib/countsDefinitions' + import type { CountsResource } from '@/lib/counts' @@ -40,19 +43,21 @@ const CountsResourcePage = ({ resource }: { resource: CountsResource }) => { const mode = params.get ('mode') ?? 'summary' const state = isCountsState (params) const cacheParams = new URLSearchParams (request) - cacheParams.set ('basis', state ? 'state' : 'history') + cacheParams.set ('definition', state ? 'latest' : 'history') const [format, setFormat] = useState<'json' | 'csv'> ('json') const [exporting, setExporting] = useState (false) const [exportError, setExportError] = useState (false) const { data, isPending, isError, isPlaceholderData, refetch } = useQuery ({ + enabled: params.has ('timezone'), queryKey: countsKeys.index (resource, cacheParams.toString (), locale), queryFn: () => fetchCounts (resource, params, locale), retry: false, placeholderData: (previous, query) => { const previousParams = new URLSearchParams (String (query?.queryKey[3] ?? '')) return (query?.queryKey[1] === resource && query?.queryKey[2] === locale && previousParams.get ('mode') === mode - && previousParams.get ('axis') === params.get ('axis') - && previousParams.get ('basis') === cacheParams.get ('basis')) ? previous : undefined + && previousParams.get ('rows') === params.get ('rows') + && previousParams.get ('cols') === params.get ('cols') + && previousParams.get ('definition') === cacheParams.get ('definition')) ? previous : undefined } }) useEffect (() => { @@ -61,12 +66,21 @@ const CountsResourcePage = ({ resource }: { resource: CountsResource }) => { }, [canonical, source, setParams]) useEffect (() => { - if (data?.at == null || isPlaceholderData) return - const next = new URLSearchParams (canonical) - if (!next.get ('at')) next.set ('at', data.at) + const timezone = Intl.DateTimeFormat ().resolvedOptions ().timeZone + if (params.get ('timezone') !== timezone) + { + const next = new URLSearchParams (canonical) + next.set ('timezone', timezone) + setParams (next, { replace: true }) + } + else if (data?.executionAt != null && !isPlaceholderData && !params.has ('execution_at')) + { + const next = new URLSearchParams (canonical) + next.set ('execution_at', data.executionAt) + setParams (next, { replace: true }) + } + }, [canonical, data, isPlaceholderData, params, setParams]) - if (next.toString () !== canonical) setParams (next, { replace: true }) - }, [canonical, data, isPlaceholderData, setParams]) const update = (key: string, value: string) => { const next = new URLSearchParams (params) @@ -80,18 +94,19 @@ const CountsResourcePage = ({ resource }: { resource: CountsResource }) => { setExportError (false) try { - const fields = mode === 'summary' ? selectedCountsSummaryFields (resource, params) + const fields = mode === 'summary' ? [ + ...[...readCountsJSON (params, 'rows', []), + ...readCountsJSON (params, 'cols', [])].map (axis => ({ + key: axis.field, label: countsFieldLabel (axis.field) })), + { key: 'count', label: 'count' }] : selectedCountColumns (resource, params).map (column => fieldsForCounts (resource).find (field => field.key === column)!) - const file = await exportCounts (resource, params, locale, format, + await exportCounts (resource, params, locale, format, fields.map (field => field.key), fields.map (field => message (field.label).text), - countsExportLabels (resource, locale)) - const url = URL.createObjectURL (file) - const link = document.createElement ('a') - link.href = url - link.download = `counts-${resource}.${format}` - link.click () - setTimeout (() => URL.revokeObjectURL (url), 1000) + { ...countsExportLabels (resource, locale), + ...Object.fromEntries (countsAxisFields (resource, !state).map (field => [ + field.key, message (field.label).text])), count: message ('count').text }) + } catch { @@ -117,17 +132,23 @@ const CountsResourcePage = ({ resource }: { resource: CountsResource }) => { lang={message ('target').language}>

{label ('target')}

    - {(['posts', 'tags'] as const).map (target => { - const next = new URLSearchParams (params) - for (const key of ['resource_id', 'tag', 'columns', 'summary_columns', - 'sort', 'page', 'children', 'child_sort']) next.delete (key) - return
  • - {label (target === 'posts' ? 'history:post' : 'navigation:tags')} -
    • - })} + {countsResources.map (target =>
  • + {label (`resources.${target}`)} +
      + {['latest', 'history'].map (definition => { + const next = new URLSearchParams ({ definition, mode }) + for (const key of ['timezone', 'at', 'execution_at']) + if (params.get (key)) next.set (key, params.get (key)!) + const active = target === resource + && params.get ('definition') === definition + return
    • + {label (definition)}
      • + })} +
    +
    • )}
@@ -153,7 +174,7 @@ const CountsResourcePage = ({ resource }: { resource: CountsResource }) => { }/> {exportError &&

{label ('common:fetchError')}

} @@ -161,14 +182,7 @@ const CountsResourcePage = ({ resource }: { resource: CountsResource }) => { {isPending &&

{label ('shared:messages.loadingEllipsis')}

} {isError &&

{label ('common:fetchError')}

} {data != null && <> - {data.at &&
- {label ('state.boundary')}：{data.at.replace ('T', ' ')} - {data.totals && (['posts', 'tags', 'assignments'] as const).map (key => ( - {label (`state.totals.${key}`)}： - {data.totals![key] ?? '—'} - ))} -
} - {!state &&

{label ('total', { count: data.count })}

} +

{label ('recordTotal', { count: data.count })}

{mode === 'summary' ? : (