class RelationalExporter::Runner

Attributes

logger[RW]
schema[RW]

Public Class Methods

new(options={}) click to toggle source
# File lib/relational_exporter.rb, line 14
def initialize(options={})
  @logger = options[:logger] ? options[:logger] : Logger.new(STDERR)

  @connection_config = options[:connection_config]
  begin
    ActiveRecord::Base.establish_connection @connection_config
    ActiveRecord::Base.connection.active?
  rescue Exception => e
    raise "Database connection failed: #{e.message}"
  end

  @schema = Hashie::Mash.new(options[:schema] || YAML.load_file(options[:schema_file]))

  load_models
end

Public Instance Methods

export(options, &block) click to toggle source
# File lib/relational_exporter.rb, line 30
def export(options, &block)
  ActiveRecord::Base.logger = @logger
  Celluloid.logger = @logger

  options = Hashie::Mash.new options

  main_klass = options.output.model.to_s.classify.constantize

  main_klass.set_scope_from_hash options.output.scope.as_json

  total_records = main_klass.find_all_by_scope(options.output.scope.as_json).count
  remaining_records = total_records

  csv_builder = RelationalExporter::CsvBuilder.new options.file_path
  Celluloid::Actor[:csv_builder] = csv_builder
  result = csv_builder.future.start
  pool_size = options.workers || 10
  pool = RelationalExporter::RecordWorker.pool(size: pool_size)
  get_headers = true

  record_sequence = -1
  batch_count = 0

  batch_options = Hashie::Mash.new({batch_size: 100}.merge(options.batch_options || {}))
  limit = options.limit.nil? ? nil : options.limit.to_i
  max_records = limit.nil? ? total_records : [limit, total_records].min

  @logger.info "CSV export will process #{max_records} of #{total_records} total records."

  all_bm = Benchmark.measure do
    catch(:hit_limit) do
      main_klass.find_all_by_scope(options.output.scope.as_json).find_in_batches(batch_options.to_h.symbolize_keys) do |records|
        batch_count+=1
        batch_bm = Benchmark.measure do
          records.each do |record|
            record_sequence += 1
            remaining_records -= 1

            args = [record_sequence, record, options.output.associations, get_headers]
            if get_headers
              pool.get_csv_row(*args)
              get_headers = false
            else
              pool.async.get_csv_row(*args)
            end

            throw :hit_limit if !limit.nil? && (record_sequence == max_records)
          end
        end

        @logger.debug "Batch of #{records.size} queued. #{remaining_records} remaining. Benchmark: #{batch_bm}"
      end
    end

    csv_builder.end_index = record_sequence

    @logger.info "CSV export complete <#{options.file_path}>" if result.value === true
  end

  @logger.debug "#{batch_count} batches processed. Benchmark: #{all_bm}"

  pool.terminate
  csv_builder.terminate
end

Private Instance Methods

load_models() click to toggle source
# File lib/relational_exporter.rb, line 106
def load_models
  @schema.each do |model, options|
    klass = Object.const_set model.to_s.classify, Class.new(ActiveRecord::Base)
    # klass.extend ActiveRecordExtension
    options.each do |method, calls|
      method = "#{method}=".to_sym if klass.respond_to?("#{method}=")
      if calls.respond_to? :each_pair
        calls.each do |association, association_options|
          association_options = symbolize_options association_options
          klass.send method, association.to_sym, *association_options
        end
      else
        klass.send method, *calls
      end
    end
  end
end
symbolize_options(options) click to toggle source
# File lib/relational_exporter.rb, line 97
def symbolize_options(options)
  options = options.as_json
  if options.is_a? Hash
    options.deep_symbolize_keys!
  elsif options.is_a? Array
    options.map { |val| symbolize_options val }
  end
end