class S3DataPacker::JSONBatch

Attributes

batch[R]
delimitter[R]
item_count[R]

Public Class Methods

new(opts = {}) click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 5
def initialize opts = {}
  @delimitter = "\r\n"
  @workdir = opts[:workdir]
  @filename_generator = opts[:filename_generator]
  @pre_processor = opts[:pre_processor] # Should be a proc
  @size = opts[:size]
  @item_count = 0
  init_workdir!
end

Public Instance Methods

append_data!(data) click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 41
def append_data! data
  digested = pre_proccess_data(data)
  batch << "#{digested}#{delimitter}"
  @item_count += 1
end
close!() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 51
def close!
  batch.close
end
delete!() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 55
def delete!
  close! if !@batch.closed?
  File.delete(path) if File.exist?(path)
  reset!
end
filename_generator() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 23
def filename_generator
  @filename_generator ||= S3DataPacker::FilenameGenerator.new
end
finalize!() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 61
def finalize!
  close! if !batch.closed?
  final_path = batch.path
  final_path = compress! if S3DataPacker.config.compress_batch?
  reset!
  final_path
end
full?() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 27
def full?
  item_count >= size
end
generate_filename() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 31
def generate_filename
  name = filename_generator.generate!
  "#{workdir}/#{name}.json"
end
new_file!() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 36
def new_file!
  close! if @batch
  @batch = File.open(generate_filename, 'w')
end
path() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 47
def path
  batch.path
end
size() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 15
def size
  @size ||= S3DataPacker.config.batch_size
end
workdir() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 19
def workdir
  @workdir ||= S3DataPacker.config.workdir
end

Private Instance Methods

compress!() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 81
def compress!
  new_path = "#{batch.path}.gz"
  `gzip #{batch.path}`
  new_path
end
init_workdir!() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 77
def init_workdir!
  Dir.mkdir(workdir) unless Dir.exist?(workdir)
end
pre_proccess_data(raw_data) click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 71
def pre_proccess_data(raw_data)
  # Transformations here, return string for this one
  return @pre_processor.call(raw_data) if @pre_processor
  raw_data
end
reset!() click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 87
def reset!
  @batch = nil
  @item_count = 0
end