require 'csv' # One read view and a fixed point for the entire download; only one page is held in memory. class CountsExporter EXTRA_KEYS = ['basis', 'format', 'resource', 'headers', 'children', 'order', 'summary_columns', 'nested_json', 'child_sort', 'labels'].freeze attr_reader :format, :resource def initialize(params, language_code:) if params.key?('definition') @table_export = CountsTableExport.new(params) @format = @table_export.format @resource = @table_export.resource return end allowed = (CountsQuery::KEYS + CountsStateQuery::KEYS + EXTRA_KEYS).uniq raise ArgumentError if (params.keys - allowed).any? raise ArgumentError unless params.values.all? { _1.is_a?(String) } @params = params @language = language_code @format = params.fetch('format', 'json') @resource = params.fetch('resource', 'posts') @mode = params.fetch('mode', 'summary') @history = params.fetch('basis', 'state') == 'history' raise ArgumentError unless ['json', 'csv'].include?(format) raise ArgumentError unless ['posts', 'tags'].include?(resource) raise ArgumentError unless ['summary', 'details'].include?(@mode) raise ArgumentError unless ['state', 'history'].include?(params.fetch('basis', 'state')) @query_params = params.except(*EXTRA_KEYS, 'columns', 'page', 'limit') .merge('mode' => @mode, 'page' => '1', 'limit' => '100') point = CountsStateQuery.new(@query_params.slice('at').merge('resource' => resource), language_code: @language).at @query_params['at'] = point.iso8601(6) @query_params['axis'] ||= !@history && resource == 'posts' ? 'tags' : 'month' @projection = CountsExportProjection.new(params, resource:, summary: @mode == 'summary', tags: !@history && @query_params['axis'] == 'tags') if @history @query_params.delete('kind') history_query else @query_params.delete('latest') @state_query = CountsStateQuery.new(@query_params.merge('resource' => resource), language_code: @language).validate_query! end end def body(progress: nil) return @table_export.body(progress:) if @table_export Enumerator.new do |output| ApplicationRecord.transaction(isolation: :repeatable_read) do processed = 0 first_row = true started = false pages do |page| total = @mode == 'details' ? page[:count] : nil progress&.call(processed, total) unless started meta = { resource:, mode: @mode, basis: @history ? 'history' : 'state', at: @query_params['at'], boundary: 'inclusive', filters: @query_params.except('sort', 'page', 'limit', 'mode'), sort: export_sort, child_sort: sort_metadata(@params.fetch('child_sort', '')), columns: @projection.metadata, children: @params['children']&.split(','), field_order: @params.fetch('order', '').split(','), timezone: 'UTC', count: @mode == 'summary' ? page[:group_count] : page[:count], totals: page[:totals], coverage: page[:coverage] } output << (format == 'json' ? "{\"metadata\":#{ meta.to_json },\"data\":[" : "\uFEFF") if format == 'csv' output << CSV.generate_line( @projection.headers.map { csv_value(_1) }, row_sep: "\r\n") end started = true end (page[:versions] || page[:state_rows] || page[:groups] || []).each do |row| record = @projection.project(row) if format == 'json' output << ',' unless first_row output << record.to_json else values = @projection.csv_record(record, row).map { csv_value(_1) } output << CSV.generate_line(values, row_sep: "\r\n") end first_row = false processed += 1 progress&.call(processed, total) end end progress&.call(processed, processed) output << ']}' if format == 'json' end end end private def export_sort sort_metadata(@query_params.fetch('sort', default_sort)) end def sort_metadata(value) value.split(',').map do |entry| key, direction = entry.split(':', 2) { key:, direction: } end end def default_sort return 'created_at:desc' if @mode == 'details' !@history && @query_params['axis'] == 'tags' ? 'count:desc' : 'group:asc' end def history_query scope = resource == 'posts' ? PostVersion.with_previous_version : TagVersion.all CountsQuery.new(scope, resource, @query_params) end def pages unless @history @state_query.each_page { yield _1 } return end query = history_query total = @mode == 'summary' ? nil : query.count loop do result = if @mode == 'summary' query.summary(include_count: @query_params['page'] == '1') else { count: total, versions: CountsVersionRepr.many(query.details, resource:, language_code: @language, at: Time.iso8601(@query_params['at'])) } end total ||= result[:group_count] yield result break if @query_params['page'].to_i * 100 >= total @query_params['page'] = (@query_params['page'].to_i + 1).to_s query = history_query end end def csv_value(value) text = value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : value.to_s text.match?(/\A\s*[=+@-]/) ? "'#{ text }" : text end end