require 'csv' # Export uses the exact same population and grouped cells as the interactive table. class CountsTableExport EXTRA = ['format', 'columns', 'headers', 'labels', 'children', 'child_sort', 'order', 'nested_json', 'summary_columns', 'basis', 'axis', 'latest'].freeze attr_reader :format, :resource def initialize(params) QueryState.validate!(params, CountsTableQuery::KEYS + EXTRA + ['measures']) raise ArgumentError unless params.values.all? { _1.is_a?(String) } params = CountsDefaults.for(params).merge(params) unless params.key?('children') params = params.merge('children' => CountsExportProjection.default_children( params.fetch('resource', 'posts')).join(' ')) end @params = params @labels = JSON.parse(params.fetch('labels', '{}')) raise ArgumentError unless @labels.is_a?(Hash) @selected = params.key?('children') ? QueryState.list(params['children']) : nil @order = QueryState.list(params.fetch('order', '')) @child_sort = QueryState.list(params.fetch('child_sort', '')).map { _1.split(':', -1) } @child_sort.each do |path, direction, *extra| raise ArgumentError if extra.any? || !['asc', 'desc'].include?(direction) raise ArgumentError unless path&.match?(/\A[a-zA-Z_]+(?:\.[a-zA-Z]+)*\/[a-zA-Z]+\z/) end @format = params.fetch('format', 'json') @resource = params.fetch('resource', 'posts') raise ArgumentError unless ['json', 'csv'].include?(format) @query = CountsTableQuery.new(params.select { |key, _value| CountsTableQuery::KEYS.include?(key) || key == 'measures' || QueryState.valid_key?(key) } .merge('page' => '1', 'limit' => '100')) @summary = @query.mode == 'summary' default_columns = @summary ? @query.axes.all.map { _1.fetch('field') } + ['count'] : CountsExportProjection.default_columns(resource) @columns = QueryState.list(params.fetch('columns', default_columns.join(' '))) default_headers = @columns.map { @labels.fetch(_1, _1) } @headers = JSON.parse(params.fetch('headers', default_headers.to_json)) raise ArgumentError unless @headers.is_a?(Array) && @headers.all? { _1.is_a?(String) } if @summary && @headers.any? && @headers.length != @query.axes.all.length + 1 raise ArgumentError end unless @summary if ['posts', 'tags'].include?(resource) @projection = CountsExportProjection.new(params, resource:, summary: false, tags: false) else allowed = @query.population.definition.fields.keys + ['is_latest'] raise ArgumentError if (@columns - allowed).any? raise ArgumentError unless @columns.length == @headers.length && @columns.uniq == @columns end end end def body(progress: nil) Enumerator.new do |output| ApplicationRecord.transaction(isolation: :repeatable_read) do @language_names = Language.pluck(:code, :name).to_h processed = 0 started = false first = true @query.each_page do |page| total = @summary ? nil : page[:count] progress&.call(processed, total) unless started metadata = page.except(:versions, :cells, :axis_labels, :column_values, :row_headers, :column_headers, :pivot_totals).merge( resource:, record_unit: @summary && format == 'json' ? 'pivot_cell' : 'row', body_row_totals: 'once_per_logical_group', page_grand_total_repeated: false, filters: @params.select { |key, _value| CountsTableQuery::KEYS.include?(key) || key == 'measures' || QueryState.valid_key?(key) }, fields: @query.population.definition.metadata, columns: @columns, column_labels: @headers, order: @params['order'], children: @params['children'], sort: @params.fetch('sort', ''), child_sort: @params['child_sort']) output << (format == 'json' ? "{\"metadata\":#{ metadata.to_json },\"data\":[" : "\uFEFF") if format == 'csv' headings = @summary ? pivot_headers(page) : [@headers] headings.each do |headers| output << CSV.generate_line(headers.map { safe(_1) }, row_sep: "\r\n") end end started = true end if @summary pivot_rows(page).each do |header| records = page.fetch(:column_headers).map { pivot_cell(header, _1, page) } if format == 'csv' labels = pivot_row_labels(header, page) values = labels + records.map { _1[:count] } output << CSV.generate_line(values.map { safe(_1) }, row_sep: "\r\n") processed += records.length progress&.call(processed, nil) else records.each do |record| output << ',' unless first output << record.to_json first = false processed += 1 progress&.call(processed, nil) end end end next end page[:versions].each do |row| record = project(row) if format == 'json' output << ',' unless first output << record.to_json else values = if @summary row[:values].each_with_index.map do |value, index| summary_value(value, index, page) end + [row[:count]] elsif @projection @projection.csv_record(record, row) else record.map { |key, value| csv_logical(key, value) } end output << CSV.generate_line(values.map { safe(_1) }, row_sep: "\r\n") end first = false processed += 1 progress&.call(processed, total) end end progress&.call(processed, processed) output << ']}' if format == 'json' end end end private def pivot_rows(page) page.fetch(:row_headers).reject do |header| header[:kind] == 'grand_total' && @query.page * @query.limit < page[:group_count] end end def total_label(header) key = header[:kind] == 'grand_total' ? (@query.axes.all.length == 1 ? 'singleTotal' : 'grandTotal') : 'subtotal' @labels.fetch(key, '') end def pivot_row_labels(header, page) values = header[:values].each_with_index.map { |value, index| summary_value(value, index, page) } if header[:kind] == 'data' && @previous_pivot_row&.fetch(:kind) == 'data' values.each_index do |index| if header[:values].first(index + 1) == @previous_pivot_row[:values].first(index + 1) values[index] = '' end end end @previous_pivot_row = header values << total_label(header) unless header[:kind] == 'data' leading = [@query.axes.rows.length, @query.axes.columns.any? ? 1 : 0].max Array.new(leading) { values[_1] || '' } end def pivot_headers(page) row_count = @query.axes.rows.length cols = @query.axes.columns leading = [row_count, cols.any? ? 1 : 0].max columns = page.fetch(:column_headers) headings = Array.new([cols.length, 1].max) do |level| prefix = Array.new(leading) { level.zero? ? @headers[_1].to_s : '' } prefix = [''] if row_count.zero? && cols.any? prefix + columns.each_with_index.map do |header, index| values = header[:values] if cols.empty? @labels.fetch('count', '') elsif level == values.length && header[:kind] != 'data' total_label(header) elsif level < values.length previous = index.positive? ? columns[index - 1][:values] : nil if level < cols.length - 1 && previous && previous.first(level + 1) == values.first(level + 1) '' else summary_value(values[level], row_count + level, page) end else '' end end end return headings if cols.empty? cols.each_index.flat_map do |level| prefix = Array.new(leading, '') prefix = headings.first.first(leading) if level.zero? names = columns.each_with_index.map do |header, index| next '' if header[:values].length < level previous = index.positive? ? columns[index - 1][:values] : nil if (previous && previous.length >= level && previous.first(level) == header[:values].first(level)) '' else @headers[row_count + level].to_s end end headings[level] = Array.new(leading, '') + headings[level].drop(leading) if level.zero? [prefix + names, headings[level]] end end def pivot_cell(row, column, page) @pivot_lookup ||= {} # Discard the previous page's lookup; memory remains bounded by one page and its rollups. unless @pivot_page.equal?(page) @pivot_page = page @pivot_lookup = page.fetch(:cells).to_h do |cell| values = cell[:values] [[values.first(@query.axes.rows.length), values.drop(@query.axes.rows.length)], cell[:count]] end page.fetch(:pivot_totals).each { @pivot_lookup[[_1[:row], _1[:column]]] = _1[:count] } end kinds = [row[:kind], column[:kind]] kind = if kinds.include?('grand_total') 'grand_total' elsif kinds.include?('subtotal') 'subtotal' else 'data' end { row: row[:values], column: column[:values], kind:, row_kind: row[:kind], column_kind: column[:kind], row_depth: row[:values].length, column_depth: column[:values].length, count: @pivot_lookup.fetch([row[:values], column[:values]], 0) } end def summary_value(value, index, page) field = @query.axes.all.fetch(index).fetch('field') return @labels.fetch('bot', '') if field == 'user' && value.nil? names = page[:axis_labels].fetch("a#{ index }", {}) name = field == 'user' ? names[value.to_s] : names.fetch(value.to_s, value) if field == 'user' name = @labels.fetch('notRegistered', '') if name.nil? || name == '' pattern = @labels.fetch('actorFormat', '{{name}} (#{{id}})') return pattern.gsub(/\{\{(name|id)\}\}/) do Regexp.last_match(1) == 'name' ? name.to_s : value.to_s end end @labels.fetch("#{ field }:#{ value }", name) end def project(row) return { row: row[:values].first(@query.axes.rows.length), column: row[:values].drop(@query.axes.rows.length), count: row[:count] } if @summary return @projection.project(row) if @projection @columns.to_h do |column| key = column == 'user' ? 'created_by_user' : column value = row[key] value = project_nested(value, key) if ['members', 'placements', 'export_paths_json'].include?(key) [key, value] end end def camel(key) key.gsub(/_([a-z])/) { Regexp.last_match(1).upcase } end def project_nested(value, path) rows = path == 'export_paths_json' ? (value || {}).map { |profile, target| { 'profile' => profile, 'path' => target } } : Array(value) sorts = @child_sort.filter_map do |key, direction| parent, field = key.split('/', 2) [field, direction] if parent == path end rows = rows.each_with_index.sort do |(first, a), (last, b)| compared = 0 sorts.each do |field, direction| left = first.find { |key, _| camel(key) == field }&.last right = last.find { |key, _| camel(key) == field }&.last compared = if left.nil? || right.nil? left.nil? == right.nil? ? 0 : (left.nil? ? 1 : -1) else (left.to_s <=> right.to_s) * (direction == 'asc' ? 1 : -1) end break unless compared.zero? end compared.zero? ? a <=> b : compared end.map(&:first) rows.map do |row| fields = row.keys.select { !@selected || @selected.include?("#{ path }.#{ camel(_1) }") } ordered = (@order + fields.map { "#{ path }.#{ camel(_1) }" }).uniq .filter_map { |item| fields.find { "#{ path }.#{ camel(_1) }" == item } } ordered.to_h do |key| item = row[key] item = project_nested(item, "#{ path }.#{ camel(key) }") if item.is_a?(Array) [key, item] end end end def csv_logical(key, value) return @labels.fetch('bot', '') if key == 'created_by_user' && value.nil? return value ? @labels.fetch('latest', '') : '' if key == 'is_latest' if value.is_a?(Time) || value.is_a?(ActiveSupport::TimeWithZone) return @query.population.timezone.utc_to_local(value.utc).iso8601 end if key == 'language_code' || key == 'source_language_code' return @language_names[value] end if value.is_a?(Hash) return value.to_h { |field, item| [@labels.fetch(camel(field.to_s), ''), csv_logical(field, item)] } end return value.map { csv_logical(key, _1) } if value.is_a?(Array) @labels.fetch("#{ key }:#{ value }", value) end def safe(value) return value if value.is_a?(Numeric) text = value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : value.to_s text.match?(/\A\s*[=+@-]/) ? "'#{ text }" : text end end