class S3DataPacker::JSONBatch
Attributes
batch[R]
delimitter[R]
item_count[R]
Public Class Methods
new(opts = {})
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 5 def initialize opts = {} @delimitter = "\r\n" @workdir = opts[:workdir] @filename_generator = opts[:filename_generator] @pre_processor = opts[:pre_processor] # Should be a proc @size = opts[:size] @item_count = 0 init_workdir! end
Public Instance Methods
append_data!(data)
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 41 def append_data! data digested = pre_proccess_data(data) batch << "#{digested}#{delimitter}" @item_count += 1 end
close!()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 51 def close! batch.close end
delete!()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 55 def delete! close! if !@batch.closed? File.delete(path) if File.exist?(path) reset! end
filename_generator()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 23 def filename_generator @filename_generator ||= S3DataPacker::FilenameGenerator.new end
finalize!()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 61 def finalize! close! if !batch.closed? final_path = batch.path final_path = compress! if S3DataPacker.config.compress_batch? reset! final_path end
full?()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 27 def full? item_count >= size end
generate_filename()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 31 def generate_filename name = filename_generator.generate! "#{workdir}/#{name}.json" end
new_file!()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 36 def new_file! close! if @batch @batch = File.open(generate_filename, 'w') end
path()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 47 def path batch.path end
size()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 15 def size @size ||= S3DataPacker.config.batch_size end
workdir()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 19 def workdir @workdir ||= S3DataPacker.config.workdir end
Private Instance Methods
compress!()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 81 def compress! new_path = "#{batch.path}.gz" `gzip #{batch.path}` new_path end
init_workdir!()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 77 def init_workdir! Dir.mkdir(workdir) unless Dir.exist?(workdir) end
pre_proccess_data(raw_data)
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 71 def pre_proccess_data(raw_data) # Transformations here, return string for this one return @pre_processor.call(raw_data) if @pre_processor raw_data end
reset!()
click to toggle source
# File lib/s3_data_packer/json_batch.rb, line 87 def reset! @batch = nil @item_count = 0 end