class Bricolage::S3DataSource
Attributes
bucket_name[R]
encryption[R]
endpoint[R]
iam_role[R]
Redshift attached IAM role ARN
prefix[R]
region[R]
Public Class Methods
new( endpoint: 's3-ap-northeast-1.amazonaws.com', region: 'ap-northeast-1', bucket: nil, prefix: nil, access_key_id: nil, secret_access_key: nil, iam_role: nil, master_symmetric_key: nil, encryption: nil, s3cfg: nil )
click to toggle source
# File lib/bricolage/s3datasource.rb, line 12 def initialize( endpoint: 's3-ap-northeast-1.amazonaws.com', region: 'ap-northeast-1', bucket: nil, prefix: nil, access_key_id: nil, secret_access_key: nil, iam_role: nil, master_symmetric_key: nil, encryption: nil, s3cfg: nil ) @endpoint = (/\Ahttps?:/ =~ endpoint) ? endpoint : "https://#{endpoint}" @region = region @bucket_name = bucket @prefix = (prefix && prefix.empty?) ? nil : prefix @access_key_id = access_key_id @secret_access_key = secret_access_key @iam_role = iam_role @master_symmetric_key = master_symmetric_key @encryption = encryption @s3cfg_path = s3cfg @s3cfg = @s3cfg_path ? load_s3cfg(@s3cfg_path) : nil end
Public Instance Methods
access_key()
click to toggle source
AWS access key ID. This property may be nil, we can use EC2 instance or ECS task attached IAM role in that case.
# File lib/bricolage/s3datasource.rb, line 63 def access_key @access_key_id || get_s3cfg('access_key') end
bucket()
click to toggle source
# File lib/bricolage/s3datasource.rb, line 109 def bucket @resource ||= Aws::S3::Resource.new(client: client) @bucket ||= @resource.bucket(@bucket_name) end
client()
click to toggle source
Ruby Interface
# File lib/bricolage/s3datasource.rb, line 105 def client @client ||= Aws::S3::Client.new(region: @region, endpoint: @endpoint, access_key_id: access_key, secret_access_key: secret_key) end
credential_string()
click to toggle source
For Redshift COPY/UNLOAD
# File lib/bricolage/s3datasource.rb, line 47 def credential_string if @iam_role "aws_iam_role=#{@iam_role}" elsif access_key [ "aws_access_key_id=#{access_key}", "aws_secret_access_key=#{secret_key}", (@master_symmetric_key && "master_symmetric_key=#{@master_symmetric_key}") ].compact.join(';') else raise ParameterError, "[s3:#{@bucket_name}] credential string requested but no credentials exist" end end
encrypted?()
click to toggle source
# File lib/bricolage/s3datasource.rb, line 97 def encrypted? !!(@master_symmetric_key or @encryption) end
new_task()
click to toggle source
# File lib/bricolage/s3datasource.rb, line 42 def new_task S3Task.new(self) end
object(rel, no_prefix: false)
click to toggle source
# File lib/bricolage/s3datasource.rb, line 114 def object(rel, no_prefix: false) bucket.object(path(rel, no_prefix: no_prefix)) end
path(rel, no_prefix: false)
click to toggle source
# File lib/bricolage/s3datasource.rb, line 122 def path(rel, no_prefix: false) path = (no_prefix || !@prefix) ? rel.to_s : "#{@prefix}/#{rel}" path.sub(%r<\A/>, '').gsub(%r<//>, '/') end
redshift_loader_source?()
click to toggle source
# File lib/bricolage/psqldatasource.rb, line 157 def redshift_loader_source? true end
secret_key()
click to toggle source
AWS secret access key.
# File lib/bricolage/s3datasource.rb, line 68 def secret_key @secret_access_key || get_s3cfg('secret_key') end
traverse(rel, no_prefix: false)
click to toggle source
# File lib/bricolage/s3datasource.rb, line 127 def traverse(rel, no_prefix: false) retries = client.config.retry_limit begin bucket.objects(prefix: path(rel, no_prefix: no_prefix)).to_a rescue Aws::Xml::Parser::ParsingError => e retries -= 1 if retries >= 0 logger.warn "Retry Bucket#objects() for XML parsing error: #{e.message}" sleep 1 retry end raise end end
url(rel, no_prefix: false)
click to toggle source
# File lib/bricolage/s3datasource.rb, line 118 def url(rel, no_prefix: false) "s3://#{@bucket_name}/#{path(rel, no_prefix: no_prefix)}" end
Private Instance Methods
get_s3cfg(key)
click to toggle source
# File lib/bricolage/s3datasource.rb, line 75 def get_s3cfg(key) return nil unless @s3cfg @s3cfg[key] or raise ParameterError, "[s3:#{@bucket_name}] missing s3cfg entry: #{key}" end
load_s3cfg(path)
click to toggle source
# File lib/bricolage/s3datasource.rb, line 81 def load_s3cfg(path) h = {} File.foreach(path) do |line| case line when /\A\s*\w+\s*=\s*/ key, value = line.split('=', 2) val = value.strip h[key.strip] = val.empty? ? nil : val end end h end