class BigShift::UnloadManifest
Attributes
bucket_name[R]
manifest_key[R]
prefix[R]
Public Class Methods
new(s3_resource, cs_service, bucket_name, prefix)
click to toggle source
# File lib/bigshift/unload_manifest.rb, line 7 def initialize(s3_resource, cs_service, bucket_name, prefix) @s3_resource = s3_resource @cs_service = cs_service @bucket_name = bucket_name @prefix = prefix @manifest_key = "#{@prefix}manifest" end
Public Instance Methods
count()
click to toggle source
# File lib/bigshift/unload_manifest.rb, line 24 def count keys.size end
keys()
click to toggle source
# File lib/bigshift/unload_manifest.rb, line 15 def keys @keys ||= begin bucket = @s3_resource.bucket(@bucket_name) object = bucket.object(@manifest_key) manifest = JSON.load(object.get.body) manifest['entries'].map { |entry| entry['url'].sub(%r{\As3://[^/]+/}, '') } end end
total_file_size()
click to toggle source
# File lib/bigshift/unload_manifest.rb, line 28 def total_file_size @total_file_size ||= file_sizes.values.reduce(:+) end
validate_transfer(cs_bucket_name)
click to toggle source
# File lib/bigshift/unload_manifest.rb, line 32 def validate_transfer(cs_bucket_name) objects = @cs_service.list_objects(cs_bucket_name, prefix: @prefix) cs_file_sizes = objects.items.each_with_object({}) do |item, acc| acc[item.name] = item.size.to_i end missing_files = (file_sizes.keys - cs_file_sizes.keys) extra_files = cs_file_sizes.keys - file_sizes.keys common_files = (cs_file_sizes.keys & file_sizes.keys) size_mismatches = common_files.select { |name| file_sizes[name] != cs_file_sizes[name] } errors = [] unless missing_files.empty? errors << "missing files: #{missing_files.join(', ')}" end unless extra_files.empty? errors << "extra files: #{extra_files.join(', ')}" end unless size_mismatches.empty? messages = size_mismatches.map { |name| sprintf('%s (%d != %d)', name, cs_file_sizes[name], file_sizes[name]) } errors << "size mismatches: #{messages.join(', ')}" end unless errors.empty? raise TransferValidationError, "Transferred files don't match unload manifest: #{errors.join('; ')}" end end
Private Instance Methods
file_sizes()
click to toggle source
# File lib/bigshift/unload_manifest.rb, line 59 def file_sizes @file_sizes ||= begin bucket = @s3_resource.bucket(@bucket_name) objects = bucket.objects(prefix: @prefix) objects.each_with_object({}) do |object, acc| if keys.include?(object.key) acc[object.key] = object.size end end end end