class S3DataPacker::Configuration

Attributes

batch_size[RW]

Number of items of the final batch size

cleanup_batch[RW]

Whether to keep or delete the finalized batch file. Set to false if you want to keep the output files in the workdir.

compress_batch[RW]

Whether to compress the final batch file or not. If set to true, the output file will be compressed with GZip, and the uncompressed file will be removed.

logger[RW]

Standard logger to output information

max_queue_size[RW]

Maximum number of items to maintain in queue to not overflow while workers process items.

max_queue_wait[RW]

Time in seconds to wait when the queue reached max_queue_size to keep adding new items.

output_filename_pattern[RW]

Desired pattern to construct output filenames

output_filename_prefix[RW]

String prefix to include in output filenames for the batches

output_filename_splitter[RW]

Splitter character to concat different values to generate a filename

output_filename_suffix[RW]

String suffix to include in output filenames for the batches

s3_api_key[RW]
s3_api_secret[RW]
s3_region[RW]
thread_count[RW]

How many threads to run for reading and processing items. This needs to be balanced out with the speed at which item keys are gathered to prevent emptying the queue too early.

thread_lock_wait_time[RW]

Time in seconds for thread to wait when locked.

thread_sleep_time[RW]

Time in seconds to let a thread sleep when there’s no pending items in queue.

workdir[RW]

Directory to keep working files. Make sure you have permissions on the path set. If the path does not exist, the packer will try to create it before using it.

Public Class Methods

new() click to toggle source
# File lib/s3_data_packer/configuration.rb, line 57
def initialize
  @thread_count = 2
  @thread_sleep_time = 1
  @thread_lock_wait_time = 1
  @max_queue_size = 10000
  @max_queue_wait = 5
  @batch_size = 100000
  @workdir = 'tmp/s3_data_packer'
  @cleanup_batch = true
  @compress_batch = true
  @output_filename_prefix = nil
  @output_filename_suffix = 'batch'
  @output_filename_pattern = %i[timecode_int suffix]
  @output_filename_splitter = '_'
end

Public Instance Methods

cleanup_batch?() click to toggle source
# File lib/s3_data_packer/configuration.rb, line 77
def cleanup_batch?
  cleanup_batch == true
end
compress_batch?() click to toggle source
# File lib/s3_data_packer/configuration.rb, line 73
def compress_batch?
  compress_batch == true
end
default_s3_credentials() click to toggle source
# File lib/s3_data_packer/configuration.rb, line 85
def default_s3_credentials
  return nil unless s3_credentials?

  Aws::Credentials.new(s3_api_key, s3_api_secret)
end
filename_generator_defaults() click to toggle source
# File lib/s3_data_packer/configuration.rb, line 91
def filename_generator_defaults
  { prefix: output_filename_prefix,
    suffix: output_filename_suffix,
    pattern: output_filename_pattern,
    splitter: output_filename_splitter }
end
s3_credentials?() click to toggle source
# File lib/s3_data_packer/configuration.rb, line 81
def s3_credentials?
  s3_api_key && s3_api_secret
end