class Bricolage::S3DataSource

Attributes

bucket_name[R]
encryption[R]
endpoint[R]
iam_role[R]

Redshift attached IAM role ARN

prefix[R]
region[R]

Public Class Methods

new( endpoint: 's3-ap-northeast-1.amazonaws.com', region: 'ap-northeast-1', bucket: nil, prefix: nil, access_key_id: nil, secret_access_key: nil, iam_role: nil, master_symmetric_key: nil, encryption: nil, s3cfg: nil ) click to toggle source
# File lib/bricolage/s3datasource.rb, line 12
def initialize(
    endpoint: 's3-ap-northeast-1.amazonaws.com',
    region: 'ap-northeast-1',
    bucket: nil,
    prefix: nil,
    access_key_id: nil,
    secret_access_key: nil,
    iam_role: nil,
    master_symmetric_key: nil,
    encryption: nil,
    s3cfg: nil
)
  @endpoint = (/\Ahttps?:/ =~ endpoint) ? endpoint : "https://#{endpoint}"
  @region = region
  @bucket_name = bucket
  @prefix = (prefix && prefix.empty?) ? nil : prefix
  @access_key_id = access_key_id
  @secret_access_key = secret_access_key
  @iam_role = iam_role
  @master_symmetric_key = master_symmetric_key
  @encryption = encryption
  @s3cfg_path = s3cfg
  @s3cfg = @s3cfg_path ? load_s3cfg(@s3cfg_path) : nil
end

Public Instance Methods

access_key() click to toggle source

AWS access key ID. This property may be nil, we can use EC2 instance or ECS task attached IAM role in that case.

# File lib/bricolage/s3datasource.rb, line 63
def access_key
  @access_key_id || get_s3cfg('access_key')
end
bucket() click to toggle source
# File lib/bricolage/s3datasource.rb, line 109
def bucket
  @resource ||= Aws::S3::Resource.new(client: client)
  @bucket ||= @resource.bucket(@bucket_name)
end
client() click to toggle source

Ruby Interface

# File lib/bricolage/s3datasource.rb, line 105
def client
  @client ||= Aws::S3::Client.new(region: @region, endpoint: @endpoint, access_key_id: access_key, secret_access_key: secret_key)
end
credential_string() click to toggle source

For Redshift COPY/UNLOAD

# File lib/bricolage/s3datasource.rb, line 47
def credential_string
  if @iam_role
    "aws_iam_role=#{@iam_role}"
  elsif access_key
    [
      "aws_access_key_id=#{access_key}",
      "aws_secret_access_key=#{secret_key}",
      (@master_symmetric_key && "master_symmetric_key=#{@master_symmetric_key}")
    ].compact.join(';')
  else
    raise ParameterError, "[s3:#{@bucket_name}] credential string requested but no credentials exist"
  end
end
encrypted?() click to toggle source
# File lib/bricolage/s3datasource.rb, line 97
def encrypted?
  !!(@master_symmetric_key or @encryption)
end
new_task() click to toggle source
# File lib/bricolage/s3datasource.rb, line 42
def new_task
  S3Task.new(self)
end
object(rel, no_prefix: false) click to toggle source
# File lib/bricolage/s3datasource.rb, line 114
def object(rel, no_prefix: false)
  bucket.object(path(rel, no_prefix: no_prefix))
end
path(rel, no_prefix: false) click to toggle source
# File lib/bricolage/s3datasource.rb, line 122
def path(rel, no_prefix: false)
  path = (no_prefix || !@prefix) ? rel.to_s : "#{@prefix}/#{rel}"
  path.sub(%r<\A/>, '').gsub(%r<//>, '/')
end
redshift_loader_source?() click to toggle source
# File lib/bricolage/psqldatasource.rb, line 157
def redshift_loader_source?
  true
end
secret_key() click to toggle source

AWS secret access key.

# File lib/bricolage/s3datasource.rb, line 68
def secret_key
  @secret_access_key || get_s3cfg('secret_key')
end
traverse(rel, no_prefix: false) click to toggle source
# File lib/bricolage/s3datasource.rb, line 127
def traverse(rel, no_prefix: false)
  retries = client.config.retry_limit
  begin
    bucket.objects(prefix: path(rel, no_prefix: no_prefix)).to_a
  rescue Aws::Xml::Parser::ParsingError => e
    retries -= 1
    if retries >= 0
      logger.warn "Retry Bucket#objects() for XML parsing error: #{e.message}"
      sleep 1
      retry
    end
    raise
  end
end
url(rel, no_prefix: false) click to toggle source
# File lib/bricolage/s3datasource.rb, line 118
def url(rel, no_prefix: false)
  "s3://#{@bucket_name}/#{path(rel, no_prefix: no_prefix)}"
end

Private Instance Methods

get_s3cfg(key) click to toggle source
# File lib/bricolage/s3datasource.rb, line 75
def get_s3cfg(key)
  return nil unless @s3cfg
  @s3cfg[key] or raise ParameterError, "[s3:#{@bucket_name}] missing s3cfg entry: #{key}"
end
load_s3cfg(path) click to toggle source
# File lib/bricolage/s3datasource.rb, line 81
def load_s3cfg(path)
  h = {}
  File.foreach(path) do |line|
    case line
    when /\A\s*\w+\s*=\s*/
      key, value = line.split('=', 2)
      val = value.strip
      h[key.strip] = val.empty? ? nil : val
    end
  end
  h
end