module Linecook::S3Manager
Constants
- EXPIRY
- PREFIX
Public Instance Methods
clean(retention, group)
click to toggle source
# File lib/linecook-gem/image/s3.rb, line 21 def clean(retention, group) full_set = list(group: group) keep = full_set.reverse.take(retention) destroy = full_set - keep destroy.each_slice(1000).each do |garbage| to_destroy = garbage.map { |x| { key: x } } client.delete_objects(bucket: Linecook.config[:aws][:s3][:bucket], delete: { objects: to_destroy} ) end return destroy end
latest(group)
click to toggle source
# File lib/linecook-gem/image/s3.rb, line 32 def latest(group) objects = list_objects(group: group).sort! { |a,b| a.last_modified <=> b.last_modified } key = objects.last ? objects.last.key : nil end
list(group: nil)
click to toggle source
# File lib/linecook-gem/image/s3.rb, line 17 def list(group: nil) list_objects(group: group).map{ |x| x.key if x.key =~ /\.tar\.xz/ }.compact end
upload(path, group: nil)
click to toggle source
# File lib/linecook-gem/image/s3.rb, line 37 def upload(path, group: nil) File.open(path, 'rb') do |file| fid = File.basename(path) pbar = ProgressBar.create(title: fid, total: file.size) common_opts = { bucket: Linecook.config[:aws][:s3][:bucket], key: File.join([PREFIX, group, fid].compact) } resp = client.create_multipart_upload(storage_class: 'STANDARD', server_side_encryption: 'AES256', **common_opts) id = resp.upload_id part = 0 total = 0 parts = [] while content = file.read(1048576 * 20) part += 1 resp = client.upload_part(body: content, content_length: content.length, part_number: part, upload_id: id, **common_opts) parts << { etag: resp.etag, part_number: part } total += content.length pbar.progress = total pbar.title = "#{fid} - (#{((total.to_f/file.size.to_f)*100.0).round(2)}%)" end client.complete_multipart_upload(upload_id: id, multipart_upload: { parts: parts }, **common_opts) end end
url(id, group: nil)
click to toggle source
# File lib/linecook-gem/image/s3.rb, line 10 def url(id, group: nil) client s3 = Aws::S3::Resource.new obj = s3.bucket(Linecook.config[:aws][:s3][:bucket]).object(File.join([PREFIX, group, "#{id}.tar.xz"].compact)) obj.presigned_url(:get, expires_in: EXPIRY * 60) end
Private Instance Methods
client()
click to toggle source
# File lib/linecook-gem/image/s3.rb, line 73 def client @client ||= begin Aws.config[:credentials] = Aws::Credentials.new(Linecook.config[:aws][:access_key], Linecook.config[:aws][:secret_key]) Aws.config[:region] = 'us-east-1' Aws::S3::Client.new end end
list_objects(group: nil)
click to toggle source
# File lib/linecook-gem/image/s3.rb, line 61 def list_objects(group: nil) contents = [] marker = nil loop do resp = client.list_objects(bucket: Linecook.config[:aws][:s3][:bucket], prefix: File.join([PREFIX, group].compact), marker: marker) break unless resp.contents.last marker = resp.contents.last.key contents += resp.contents end contents.sort { |a,b| a.last_modified <=> b.last_modified } end