330 行
13 KiB
Ruby
330 行
13 KiB
Ruby
require 'csv'
|
|
|
|
# Export uses the exact same population and grouped cells as the interactive table.
|
|
class CountsTableExport
|
|
EXTRA = ['format', 'columns', 'headers', 'labels', 'children', 'child_sort',
|
|
'order', 'nested_json', 'summary_columns', 'basis', 'axis', 'latest'].freeze
|
|
attr_reader :format, :resource
|
|
|
|
def initialize(params)
|
|
QueryState.validate!(params, CountsTableQuery::KEYS + EXTRA + ['measures'])
|
|
raise ArgumentError unless params.values.all? { _1.is_a?(String) }
|
|
|
|
params = CountsDefaults.for(params).merge(params)
|
|
unless params.key?('children')
|
|
params = params.merge('children' => CountsExportProjection.default_children(
|
|
params.fetch('resource', 'posts')).join(' '))
|
|
end
|
|
@params = params
|
|
@labels = JSON.parse(params.fetch('labels', '{}'))
|
|
raise ArgumentError unless @labels.is_a?(Hash)
|
|
@selected = params.key?('children') ? QueryState.list(params['children']) : nil
|
|
@order = QueryState.list(params.fetch('order', ''))
|
|
@child_sort = QueryState.list(params.fetch('child_sort', '')).map { _1.split(':', -1) }
|
|
@child_sort.each do |path, direction, *extra|
|
|
raise ArgumentError if extra.any? || !['asc', 'desc'].include?(direction)
|
|
raise ArgumentError unless path&.match?(/\A[a-zA-Z_]+(?:\.[a-zA-Z]+)*\/[a-zA-Z]+\z/)
|
|
end
|
|
@format = params.fetch('format', 'json')
|
|
@resource = params.fetch('resource', 'posts')
|
|
raise ArgumentError unless ['json', 'csv'].include?(format)
|
|
|
|
@query = CountsTableQuery.new(params.select { |key, _value| CountsTableQuery::KEYS.include?(key) ||
|
|
key == 'measures' || QueryState.valid_key?(key) }
|
|
.merge('page' => '1', 'limit' => '100'))
|
|
@summary = @query.mode == 'summary'
|
|
default_columns = @summary ? @query.axes.all.map { _1.fetch('field') } + ['count'] :
|
|
CountsExportProjection.default_columns(resource)
|
|
@columns = QueryState.list(params.fetch('columns', default_columns.join(' ')))
|
|
default_headers = @columns.map { @labels.fetch(_1, _1) }
|
|
@headers = JSON.parse(params.fetch('headers', default_headers.to_json))
|
|
raise ArgumentError unless @headers.is_a?(Array) && @headers.all? { _1.is_a?(String) }
|
|
if @summary && @headers.any? && @headers.length != @query.axes.all.length + 1
|
|
raise ArgumentError
|
|
end
|
|
|
|
unless @summary
|
|
if ['posts', 'tags'].include?(resource)
|
|
@projection = CountsExportProjection.new(params, resource:, summary: false, tags: false)
|
|
else
|
|
allowed = @query.population.definition.fields.keys + ['is_latest']
|
|
raise ArgumentError if (@columns - allowed).any?
|
|
raise ArgumentError unless @columns.length == @headers.length && @columns.uniq == @columns
|
|
end
|
|
end
|
|
end
|
|
|
|
def body(progress: nil)
|
|
Enumerator.new do |output|
|
|
ApplicationRecord.transaction(isolation: :repeatable_read) do
|
|
@language_names = Language.pluck(:code, :name).to_h
|
|
processed = 0
|
|
started = false
|
|
first = true
|
|
@query.each_page do |page|
|
|
total = @summary ? nil : page[:count]
|
|
progress&.call(processed, total)
|
|
unless started
|
|
metadata = page.except(:versions, :cells, :axis_labels, :column_values,
|
|
:row_headers, :column_headers, :pivot_totals).merge(
|
|
resource:, record_unit: @summary && format == 'json' ? 'pivot_cell' : 'row',
|
|
body_row_totals: 'once_per_logical_group', page_grand_total_repeated: false,
|
|
filters: @params.select { |key, _value| CountsTableQuery::KEYS.include?(key) ||
|
|
key == 'measures' || QueryState.valid_key?(key) },
|
|
fields: @query.population.definition.metadata,
|
|
columns: @columns, column_labels: @headers,
|
|
order: @params['order'], children: @params['children'],
|
|
sort: @params.fetch('sort', ''), child_sort: @params['child_sort'])
|
|
output << (format == 'json' ? "{\"metadata\":#{ metadata.to_json },\"data\":[" : "\uFEFF")
|
|
if format == 'csv'
|
|
headings = @summary ? pivot_headers(page) : [@headers]
|
|
headings.each do |headers|
|
|
output << CSV.generate_line(headers.map { safe(_1) }, row_sep: "\r\n")
|
|
end
|
|
end
|
|
started = true
|
|
end
|
|
if @summary
|
|
pivot_rows(page).each do |header|
|
|
records = page.fetch(:column_headers).map { pivot_cell(header, _1, page) }
|
|
if format == 'csv'
|
|
labels = pivot_row_labels(header, page)
|
|
values = labels + records.map { _1[:count] }
|
|
output << CSV.generate_line(values.map { safe(_1) }, row_sep: "\r\n")
|
|
processed += records.length
|
|
progress&.call(processed, nil)
|
|
else
|
|
records.each do |record|
|
|
output << ',' unless first
|
|
output << record.to_json
|
|
first = false
|
|
processed += 1
|
|
progress&.call(processed, nil)
|
|
end
|
|
end
|
|
end
|
|
next
|
|
end
|
|
page[:versions].each do |row|
|
|
record = project(row)
|
|
if format == 'json'
|
|
output << ',' unless first
|
|
output << record.to_json
|
|
else
|
|
values = if @summary
|
|
row[:values].each_with_index.map do |value, index|
|
|
summary_value(value, index, page)
|
|
end + [row[:count]]
|
|
elsif @projection
|
|
@projection.csv_record(record, row)
|
|
else
|
|
record.map { |key, value| csv_logical(key, value) }
|
|
end
|
|
output << CSV.generate_line(values.map { safe(_1) }, row_sep: "\r\n")
|
|
end
|
|
first = false
|
|
processed += 1
|
|
progress&.call(processed, total)
|
|
end
|
|
end
|
|
progress&.call(processed, processed)
|
|
output << ']}' if format == 'json'
|
|
end
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def pivot_rows(page)
|
|
page.fetch(:row_headers).reject do |header|
|
|
header[:kind] == 'grand_total' && @query.page * @query.limit < page[:group_count]
|
|
end
|
|
end
|
|
|
|
def total_label(header)
|
|
key = header[:kind] == 'grand_total' ?
|
|
(@query.axes.all.length == 1 ? 'singleTotal' : 'grandTotal') : 'subtotal'
|
|
@labels.fetch(key, '')
|
|
end
|
|
|
|
def pivot_row_labels(header, page)
|
|
values = header[:values].each_with_index.map { |value, index|
|
|
summary_value(value, index, page)
|
|
}
|
|
if header[:kind] == 'data' && @previous_pivot_row&.fetch(:kind) == 'data'
|
|
values.each_index do |index|
|
|
if header[:values].first(index + 1) == @previous_pivot_row[:values].first(index + 1)
|
|
values[index] = ''
|
|
end
|
|
end
|
|
end
|
|
@previous_pivot_row = header
|
|
values << total_label(header) unless header[:kind] == 'data'
|
|
leading = [@query.axes.rows.length, @query.axes.columns.any? ? 1 : 0].max
|
|
Array.new(leading) { values[_1] || '' }
|
|
end
|
|
|
|
def pivot_headers(page)
|
|
row_count = @query.axes.rows.length
|
|
cols = @query.axes.columns
|
|
leading = [row_count, cols.any? ? 1 : 0].max
|
|
columns = page.fetch(:column_headers)
|
|
headings = Array.new([cols.length, 1].max) do |level|
|
|
prefix = Array.new(leading) { level.zero? ? @headers[_1].to_s : '' }
|
|
prefix = [''] if row_count.zero? && cols.any?
|
|
prefix + columns.each_with_index.map do |header, index|
|
|
values = header[:values]
|
|
if cols.empty?
|
|
@labels.fetch('count', '')
|
|
elsif level == values.length && header[:kind] != 'data'
|
|
total_label(header)
|
|
elsif level < values.length
|
|
previous = index.positive? ? columns[index - 1][:values] : nil
|
|
if level < cols.length - 1 && previous && previous.first(level + 1) == values.first(level + 1)
|
|
''
|
|
else
|
|
summary_value(values[level], row_count + level, page)
|
|
end
|
|
else
|
|
''
|
|
end
|
|
end
|
|
end
|
|
return headings if cols.empty?
|
|
|
|
cols.each_index.flat_map do |level|
|
|
prefix = Array.new(leading, '')
|
|
prefix = headings.first.first(leading) if level.zero?
|
|
names = columns.each_with_index.map do |header, index|
|
|
next '' if header[:values].length < level
|
|
|
|
previous = index.positive? ? columns[index - 1][:values] : nil
|
|
if (previous && previous.length >= level &&
|
|
previous.first(level) == header[:values].first(level))
|
|
''
|
|
else
|
|
@headers[row_count + level].to_s
|
|
end
|
|
end
|
|
headings[level] = Array.new(leading, '') + headings[level].drop(leading) if level.zero?
|
|
[prefix + names, headings[level]]
|
|
end
|
|
end
|
|
|
|
def pivot_cell(row, column, page)
|
|
@pivot_lookup ||= {}
|
|
# Discard the previous page's lookup; memory remains bounded by one page and its rollups.
|
|
unless @pivot_page.equal?(page)
|
|
@pivot_page = page
|
|
@pivot_lookup = page.fetch(:cells).to_h do |cell|
|
|
values = cell[:values]
|
|
[[values.first(@query.axes.rows.length), values.drop(@query.axes.rows.length)], cell[:count]]
|
|
end
|
|
page.fetch(:pivot_totals).each { @pivot_lookup[[_1[:row], _1[:column]]] = _1[:count] }
|
|
end
|
|
kinds = [row[:kind], column[:kind]]
|
|
kind = if kinds.include?('grand_total')
|
|
'grand_total'
|
|
elsif kinds.include?('subtotal')
|
|
'subtotal'
|
|
else
|
|
'data'
|
|
end
|
|
{ row: row[:values], column: column[:values], kind:,
|
|
row_kind: row[:kind], column_kind: column[:kind],
|
|
row_depth: row[:values].length, column_depth: column[:values].length,
|
|
count: @pivot_lookup.fetch([row[:values], column[:values]], 0) }
|
|
end
|
|
|
|
def summary_value(value, index, page)
|
|
field = @query.axes.all.fetch(index).fetch('field')
|
|
return @labels.fetch('bot', '') if field == 'user' && value.nil?
|
|
|
|
names = page[:axis_labels].fetch("a#{ index }", {})
|
|
name = field == 'user' ? names[value.to_s] : names.fetch(value.to_s, value)
|
|
if field == 'user'
|
|
name = @labels.fetch('notRegistered', '') if name.nil? || name == ''
|
|
pattern = @labels.fetch('actorFormat', '{{name}} (#{{id}})')
|
|
return pattern.gsub(/\{\{(name|id)\}\}/) do
|
|
Regexp.last_match(1) == 'name' ? name.to_s : value.to_s
|
|
end
|
|
end
|
|
@labels.fetch("#{ field }:#{ value }", name)
|
|
end
|
|
|
|
def project(row)
|
|
return { row: row[:values].first(@query.axes.rows.length),
|
|
column: row[:values].drop(@query.axes.rows.length), count: row[:count] } if @summary
|
|
return @projection.project(row) if @projection
|
|
|
|
@columns.to_h do |column|
|
|
key = column == 'user' ? 'created_by_user' : column
|
|
value = row[key]
|
|
value = project_nested(value, key) if ['members', 'placements', 'export_paths_json'].include?(key)
|
|
[key, value]
|
|
end
|
|
end
|
|
|
|
def camel(key)
|
|
key.gsub(/_([a-z])/) { Regexp.last_match(1).upcase }
|
|
end
|
|
|
|
def project_nested(value, path)
|
|
rows = path == 'export_paths_json' ? (value || {}).map { |profile, target|
|
|
{ 'profile' => profile, 'path' => target }
|
|
} : Array(value)
|
|
sorts = @child_sort.filter_map do |key, direction|
|
|
parent, field = key.split('/', 2)
|
|
[field, direction] if parent == path
|
|
end
|
|
rows = rows.each_with_index.sort do |(first, a), (last, b)|
|
|
compared = 0
|
|
sorts.each do |field, direction|
|
|
left = first.find { |key, _| camel(key) == field }&.last
|
|
right = last.find { |key, _| camel(key) == field }&.last
|
|
compared = if left.nil? || right.nil?
|
|
left.nil? == right.nil? ? 0 : (left.nil? ? 1 : -1)
|
|
else
|
|
(left.to_s <=> right.to_s) * (direction == 'asc' ? 1 : -1)
|
|
end
|
|
break unless compared.zero?
|
|
end
|
|
compared.zero? ? a <=> b : compared
|
|
end.map(&:first)
|
|
rows.map do |row|
|
|
fields = row.keys.select { !@selected || @selected.include?("#{ path }.#{ camel(_1) }") }
|
|
ordered = (@order + fields.map { "#{ path }.#{ camel(_1) }" }).uniq
|
|
.filter_map { |item| fields.find { "#{ path }.#{ camel(_1) }" == item } }
|
|
ordered.to_h do |key|
|
|
item = row[key]
|
|
item = project_nested(item, "#{ path }.#{ camel(key) }") if item.is_a?(Array)
|
|
[key, item]
|
|
end
|
|
end
|
|
end
|
|
|
|
def csv_logical(key, value)
|
|
return @labels.fetch('bot', '') if key == 'created_by_user' && value.nil?
|
|
return value ? @labels.fetch('latest', '') : '' if key == 'is_latest'
|
|
if value.is_a?(Time) || value.is_a?(ActiveSupport::TimeWithZone)
|
|
return @query.population.timezone.utc_to_local(value.utc).iso8601
|
|
end
|
|
if key == 'language_code' || key == 'source_language_code'
|
|
return @language_names[value]
|
|
end
|
|
if value.is_a?(Hash)
|
|
return value.to_h { |field, item| [@labels.fetch(camel(field.to_s), ''), csv_logical(field, item)] }
|
|
end
|
|
return value.map { csv_logical(key, _1) } if value.is_a?(Array)
|
|
|
|
@labels.fetch("#{ key }:#{ value }", value)
|
|
end
|
|
|
|
def safe(value)
|
|
return value if value.is_a?(Numeric)
|
|
|
|
text = value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : value.to_s
|
|
text.match?(/\A\s*[=+@-]/) ? "'#{ text }" : text
|
|
end
|
|
end
|