module Miasma::Contrib::AwsApiCore::ApiCommon

Common API setup

Public Class Methods

included(klass) click to toggle source
# File lib/miasma/contrib/aws.rb, line 360
def self.included(klass)
  klass.class_eval do
    include Bogo::Logger::Helpers
    attribute :aws_profile_name, [FalseClass, String], :default => ENV.fetch("AWS_PROFILE", "default")
    attribute :aws_sts_token, String
    attribute :aws_sts_role_arn, String
    attribute :aws_sts_external_id, String
    attribute :aws_sts_role_session_name, String
    attribute :aws_sts_region, String
    attribute :aws_sts_host, String
    attribute :aws_sts_session_token, String
    attribute :aws_sts_session_token_code, [String, Proc, Method]
    attribute :aws_sts_mfa_serial_number, [String]
    attribute :aws_credentials_file, String,
      :required => true,
      :default => ENV.fetch("AWS_SHARED_CREDENTIALS_FILE", File.join(Dir.home, ".aws/credentials"))
    attribute :aws_config_file, String,
      :required => true,
      :default => ENV.fetch("AWS_CONFIG_FILE", File.join(Dir.home, ".aws/config"))
    attribute :aws_access_key_id, String, :required => true, :default => ENV["AWS_ACCESS_KEY_ID"]
    attribute :aws_secret_access_key, String, :required => true, :default => ENV["AWS_SECRET_ACCESS_KEY"]
    attribute :aws_iam_instance_profile, [TrueClass, FalseClass], :default => false
    attribute :aws_ecs_task_profile, [TrueClass, FalseClass], :default => false
    attribute :aws_region, String, :required => true, :default => ENV["AWS_DEFAULT_REGION"]
    attribute :aws_host, String
    attribute :aws_bucket_region, String
    attribute :api_endpoint, String, :required => true, :default => "amazonaws.com"
    attribute :euca_compat, Symbol, :allowed_values => [:path, :dns],
                                    :coerce => lambda { |v| v.is_a?(String) ? v.to_sym : v }
    attribute :euca_dns_map, Smash, :coerce => lambda { |v| v.to_smash },
                                    :default => Smash.new
    attribute :ssl_enabled, [TrueClass, FalseClass], :default => true
  end

  # AWS config file key remapping
  klass.const_set(:CONFIG_FILE_REMAP,
                  Smash.new(
    "region" => "aws_region",
    "role_arn" => "aws_sts_role_arn",
    "aws_security_token" => "aws_sts_token",
    "aws_session_token" => "aws_sts_session_token",
  ).to_smash.freeze)
  klass.const_set(:INSTANCE_PROFILE_HOST, "http://169.254.169.254".freeze)
  klass.const_set(
    :INSTANCE_PROFILE_PATH,
    "latest/meta-data/iam/security-credentials".freeze
  )
  klass.const_set(
    :INSTANCE_PROFILE_AZ_PATH,
    "latest/meta-data/placement/availability-zone".freeze
  )
  klass.const_set(:ECS_TASK_PROFILE_HOST, "http://169.254.170.2".freeze)
  klass.const_set(
    :ECS_TASK_PROFILE_PATH, ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
  )
  # Reload sts tokens if expiry is within the next 10 minutes
  klass.const_set(:STS_TOKEN_EXPIRY_BUFFER, 600)
end

Public Instance Methods

after_setup(creds) click to toggle source

Persist any underlying stored credential data that is not a defined attribute (things like STS information)

@param creds [Hash] @return [TrueClass]

# File lib/miasma/contrib/aws.rb, line 498
def after_setup(creds)
  logger.debug("running after setup configuration updates")
  skip = self.class.attributes.keys.map(&:to_s)
  creds.each do |k, v|
    k = k.to_s
    if k.start_with?("aws_") && !skip.include?(k)
      data[k] = v
    end
  end
end
api_for(type) click to toggle source

Build new API for specified type using current provider / creds

@param type [Symbol] api type @return [Api]

# File lib/miasma/contrib/aws.rb, line 423
def api_for(type)
  memoize(type) do
    logger.debug("building API for type `#{type}`")
    creds = attributes.dup
    creds.delete(:aws_host)
    Miasma.api(
      Smash.new(
        :type => type,
        :provider => provider,
        :credentials => creds,
      )
    )
  end
end
connect() click to toggle source

Setup for API connections

# File lib/miasma/contrib/aws.rb, line 710
def connect
  unless aws_host
    if euca_compat
      service_name = (self.class.const_defined?(:EUCA_API_SERVICE) ?
        self.class::EUCA_API_SERVICE :
        self.class::API_SERVICE)
    else
      service_name = self.class::API_SERVICE.downcase
    end
    if euca_compat == :path
      self.aws_host = [
        api_endpoint,
        "services",
        service_name,
      ].join("/")
    elsif euca_compat == :dns && euca_dns_map[service_name]
      self.aws_host = [
        euca_dns_map[service_name],
        api_endpoint,
      ].join(".")
    else
      self.aws_host = [
        service_name,
        aws_region,
        api_endpoint,
      ].join(".")
    end
  end
end
connection() click to toggle source

@return [HTTP] connection for requests (forces headers)

Calls superclass method
# File lib/miasma/contrib/aws.rb, line 771
def connection
  super.headers(
    "Host" => aws_host,
    "X-Amz-Date" => Contrib::AwsApiCore.time_iso8601,
  )
end
custom_setup(creds) click to toggle source

Provide custom setup functionality to support alternative credential loading.

@param creds [Hash] @return [TrueClass]

# File lib/miasma/contrib/aws.rb, line 443
def custom_setup(creds)
  logger.debug("running custom setup configuration updates")
  cred_file = load_aws_file(creds.fetch(
    :aws_credentials_file, aws_credentials_file
  ))
  config_file = load_aws_file(creds.fetch(
    :aws_config_file, aws_config_file
  ))
  # Load any configuration available from the config file
  profile = creds.fetch(:aws_profile_name, aws_profile_name)
  profile_list = [profile].compact
  new_config_creds = Smash.new
  while profile
    logger.debug("loading aws configuration profile: #{profile}")
    new_config_creds = config_file.fetch(profile, Smash.new).merge(
      new_config_creds
    )
    profile = new_config_creds.delete(:source_profile)
    profile_list << profile
  end
  new_config_creds = config_file.fetch(:default, Smash.new).merge(
    new_config_creds
  )
  # Load any configuration available from the creds file
  new_creds = Smash.new
  profile_list.each do |profile|
    logger.debug("loading aws credentials profile: #{profile}")
    new_creds = cred_file.fetch(profile, Smash.new).merge(
      new_creds
    )
    profile = new_creds.delete(:source_profile)
  end
  new_creds = cred_file.fetch(:default, Smash.new).merge(
    new_creds
  )
  new_creds = new_creds.merge(new_config_creds)
  # Provided credentials override any config file or creds
  # file configuration so set them into new creds if available
  new_creds.merge!(creds)
  # Replace creds hash with updated hash so it is loaded with
  # updated values
  creds.replace(new_creds)
  if creds[:aws_iam_instance_profile]
    self.class.const_get(:ECS_TASK_PROFILE_PATH).nil? ?
      load_instance_credentials!(creds) :
      load_ecs_credentials!(creds)
  end
  true
end
endpoint() click to toggle source

@return [String] endpoint for request

# File lib/miasma/contrib/aws.rb, line 779
def endpoint
  "http#{"s" if ssl_enabled}://#{aws_host}"
end
extract_creds(data) click to toggle source

Return hash with needed information to assume role

@param data [Hash] @return [Hash]

# File lib/miasma/contrib/aws.rb, line 579
def extract_creds(data)
  c = Smash.new
  c[:aws_access_key_id] = data["AccessKeyId"]
  c[:aws_secret_access_key] = data["SecretAccessKey"]
  c[:aws_sts_token] = data["Token"]
  c[:aws_sts_token_expires] = Time.xmlschema(data["Expiration"])
  c[:aws_sts_role_arn] = data["RoleArn"] # used in ECS Role but not instance role
  c
end
get_credential(key, data_hash = nil) click to toggle source

Return correct credential value based on STS context

@param key [String, Symbol] credential suffix @return [Object]

# File lib/miasma/contrib/aws.rb, line 754
def get_credential(key, data_hash = nil)
  data_hash = attributes if data_hash.nil?
  if data_hash[:aws_sts_token]
    data_hash.fetch("aws_sts_#{key}", data_hash["aws_#{key}"])
  elsif data_hash[:aws_sts_session_token]
    data_hash.fetch("aws_sts_session_#{key}", data_hash["aws_#{key}"])
  else
    data_hash["aws_#{key}"]
  end
end
get_region() click to toggle source

Return region from meta-data service

@return [String]

# File lib/miasma/contrib/aws.rb, line 592
def get_region
  logger.debug("fetching region from meta-data service")
  az = HTTP.get(
    [
      self.class.const_get(:INSTANCE_PROFILE_HOST),
      self.class.const_get(:INSTANCE_PROFILE_AZ_PATH),
    ].join("/")
  ).body.to_s.strip
  az.sub!(/[a-zA-Z]+$/, "")
  logger.debug("region from meta-data service: #{az}")
  az
end
load_aws_file(file_path) click to toggle source

Load configuration from the AWS configuration file

@param file_path [String] path to configuration file @return [Smash]

# File lib/miasma/contrib/aws.rb, line 661
def load_aws_file(file_path)
  if File.exist?(file_path)
    logger.debug("loading aws file @ #{file_path}")
    Smash.new.tap do |creds|
      key = :default
      File.readlines(file_path).each_with_index do |line, idx|
        line.strip!
        next if line.empty? || line.start_with?("#")
        if line.start_with?("[")
          unless line.end_with?("]")
            raise ArgumentError,
              "Failed to parse aws file! (#{file_path} line #{idx + 1})"
          end
          key = line.tr("[]", "").strip.sub(/^profile /, "")
          creds[key] = Smash.new
        else
          unless key
            raise ArgumentError,
              "Failed to parse aws file! (#{file_path} line #{idx + 1}) " \
              "- No section defined!"
          end
          line_args = line.split("=", 2).map(&:strip)
          line_args.first.replace(
            self.class.const_get(:CONFIG_FILE_REMAP).fetch(
              line_args.first, line_args.first
            )
          )
          if line_args.last.start_with?('"')
            unless line_args.last.end_with?('"')
              raise ArgumentError,
                "Failed to parse aws file! (#{file_path} line #{idx + 1})"
            end
            line_args.last.replace(line_args.last[1..-2]) # NOTE: strip quoted values
          end
          begin
            creds[key].merge!(Smash[*line_args])
          rescue => e
            raise ArgumentError,
              "Failed to parse aws file! (#{file_path} line #{idx + 1})"
          end
        end
      end
    end
  else
    Smash.new
  end
end
load_ecs_credentials!(creds) click to toggle source

Attempt to load credentials from instance metadata

@param creds [Hash] @return [TrueClass]

# File lib/miasma/contrib/aws.rb, line 548
def load_ecs_credentials!(creds)
  logger.debug("loading ECS credentials")
  # As per docs ECS_TASK_PROFILE_PATH is defined as
  # /credential_provider_version/credentials?id=task_UUID
  # where AWS fills in the version and UUID.
  # @see https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
  data = HTTP.get(
    [
      self.class.const_get(:ECS_TASK_PROFILE_HOST),
      self.class.const_get(:ECS_TASK_PROFILE_PATH),
    ].join
  ).body
  unless data.is_a?(Hash)
    begin
      data = MultiJson.load(data.to_s)
    rescue MultiJson::ParseError => err
      logger.debug("failed to parse ECS credentials - #{err}")
      data = {}
    end
  end
  creds.merge!(extract_creds(data))
  unless creds[:aws_region]
    creds[:aws_region] = get_region
  end
  true
end
load_instance_credentials!(creds) click to toggle source

Attempt to load credentials from instance metadata

@param creds [Hash] @return [TrueClass]

# File lib/miasma/contrib/aws.rb, line 513
def load_instance_credentials!(creds)
  logger.debug("loading instance credentials")
  role = HTTP.get(
    [
      self.class.const_get(:INSTANCE_PROFILE_HOST),
      self.class.const_get(:INSTANCE_PROFILE_PATH),
      "",
    ].join("/")
  ).body.to_s.strip
  data = HTTP.get(
    [
      self.class.const_get(:INSTANCE_PROFILE_HOST),
      self.class.const_get(:INSTANCE_PROFILE_PATH),
      role,
    ].join("/")
  ).body
  unless data.is_a?(Hash)
    begin
      data = MultiJson.load(data.to_s)
    rescue MultiJson::ParseError => err
      logger.debug("failed to parse instance credentials - #{err}")
      data = {}
    end
  end
  creds.merge!(extract_creds(data))
  unless creds[:aws_region]
    creds[:aws_region] = get_region
  end
  true
end
make_request(connection, http_method, request_args) click to toggle source

Override to inject signature

@param connection [HTTP] @param http_method [Symbol] @param request_args [Array] @return [HTTP::Response]

# File lib/miasma/contrib/aws.rb, line 789
def make_request(connection, http_method, request_args)
  logger.debug("making #{http_method.to_s.upcase} request - #{request_args.inspect}")
  dest, options = request_args
  path = URI.parse(dest).path
  options = options ? options.to_smash : Smash.new
  options[:headers] = Smash[connection.default_options.headers.to_a].
    merge(options.fetch(:headers, Smash.new))
  if self.class::API_VERSION
    if options[:form]
      options.set(:form, "Version", self.class::API_VERSION)
    else
      options[:params] = options.fetch(
        :params, Smash.new
      ).to_smash.deep_merge(
        Smash.new(
          "Version" => self.class::API_VERSION,
        )
      )
    end
  end
  if aws_sts_session_token || aws_sts_session_token_code
    if sts_mfa_session_update_required?
      sts_mfa_session!(data)
    end
    options.set(:headers, "X-Amz-Security-Token", aws_sts_session_token)
  end
  if aws_sts_token || aws_sts_role_arn
    if sts_assume_role_update_required?
      sts_assume_role!(data)
    end
    options.set(:headers, "X-Amz-Security-Token", aws_sts_token)
  end
  signature = signer.generate(http_method, path, options)
  update_request(connection, options)
  options = Hash[options.map { |k, v| [k.to_sym, v] }]
  connection.auth(signature).send(http_method, dest, options)
end
perform_request_retry(exception) click to toggle source

Determine if a retry is allowed based on exception

@param exception [Exception] @return [TrueClass, FalseClass]

# File lib/miasma/contrib/aws.rb, line 869
def perform_request_retry(exception)
  if exception.is_a?(Miasma::Error::ApiError)
    if [400, 500, 503].include?(exception.response.code)
      if exception.response.code == 400
        exception.response.body.to_s.downcase.include?("throttl")
      else
        true
      end
    else
      false
    end
  end
end
retryable_allowed?(*_) click to toggle source

Always allow retry

@return [TrueClass]

# File lib/miasma/contrib/aws.rb, line 886
def retryable_allowed?(*_)
  true
end
signer() click to toggle source

@return [Contrib::AwsApiCore::SignatureV4]

# File lib/miasma/contrib/aws.rb, line 741
def signer
  Contrib::AwsApiCore::SignatureV4.new(
    get_credential(:access_key_id),
    get_credential(:secret_access_key),
    aws_region,
    self.class::API_SERVICE
  )
end
sts_assume_role!(creds) click to toggle source

Assume requested role and replace key id and secret

@param creds [Hash] @return [TrueClass]

# File lib/miasma/contrib/aws.rb, line 633
def sts_assume_role!(creds)
  if sts_assume_role_update_required?(creds)
    logger.debug("loading STS assume role")
    sts = Miasma::Contrib::Aws::Api::Sts.new(
      :aws_access_key_id => get_credential(:access_key_id, creds),
      :aws_secret_access_key => get_credential(:secret_access_key, creds),
      :aws_region => creds.fetch(:aws_sts_region, "us-east-1"),
      :aws_credentials_file => creds.fetch(
        :aws_credentials_file, aws_credentials_file
      ),
      :aws_config_file => creds.fetch(:aws_config_file, aws_config_file),
      :aws_host => creds[:aws_sts_host],
      :aws_sts_token => creds[:aws_sts_session_token],
    )
    role_info = sts.assume_role(
      creds[:aws_sts_role_arn],
      :session_name => creds[:aws_sts_role_session_name],
      :external_id => creds[:aws_sts_external_id],
    )
    creds.merge!(role_info)
  end
  true
end
sts_assume_role_update_required?(args = {}) click to toggle source

@return [TrueClass, FalseClass] @note update check only applied if assuming role

# File lib/miasma/contrib/aws.rb, line 829
def sts_assume_role_update_required?(args = {})
  sts_attribute_update_required?(:aws_sts_role_arn,
                                 :aws_sts_token_expires, args)
end
sts_attribute_update_required?(key, expiry_key, args = {}) click to toggle source

Check if STS attribute requires update

@param key [String, Symbol] token key @param expiry_key [String, Symbol] expiry of token (Time instance) @param args [Hash] overrides to check instead of instance values @return [TrueClass, FalseClass]

# File lib/miasma/contrib/aws.rb, line 847
def sts_attribute_update_required?(key, expiry_key, args = {})
  if args.to_smash.fetch(key, attributes[key])
    expiry = args.to_smash.fetch(expiry_key, attributes[expiry_key])
    expiry.nil? || expiry - self.class.const_get(:STS_TOKEN_EXPIRY_BUFFER) <= Time.now
  else
    false
  end
end
sts_mfa_session!(creds) click to toggle source
# File lib/miasma/contrib/aws.rb, line 605
def sts_mfa_session!(creds)
  if sts_mfa_session_update_required?(creds)
    logger.debug("loading STS MFA session")
    sts = Miasma::Contrib::Aws::Api::Sts.new(
      :aws_access_key_id => creds[:aws_access_key_id],
      :aws_secret_access_key => creds[:aws_secret_access_key],
      :aws_region => creds.fetch(:aws_sts_region, "us-east-1"),
      :aws_credentials_file => creds.fetch(
        :aws_credentials_file, aws_credentials_file
      ),
      :aws_config_file => creds.fetch(:aws_config_file, aws_config_file),
      :aws_profile_name => creds[:aws_profile_name],
      :aws_host => creds[:aws_sts_host],
    )
    creds.merge!(
      sts.mfa_session(
        creds[:aws_sts_session_token_code],
        :mfa_serial => creds[:aws_sts_mfa_serial_number],
      )
    )
  end
  true
end
sts_mfa_session_update_required?(args = {}) click to toggle source

@return [TrueClass, FalseClass] @note update check only applied if assuming role

# File lib/miasma/contrib/aws.rb, line 836
def sts_mfa_session_update_required?(args = {})
  sts_attribute_update_required?(:aws_sts_session_token_code,
                                 :aws_sts_session_token_expires, args)
end
update_request(con, opts) click to toggle source

Simple callback to allow request option adjustments prior to signature calculation

@param opts [Smash] request options @return [TrueClass]

# File lib/miasma/contrib/aws.rb, line 861
def update_request(con, opts)
  true
end
uri_escape(string) click to toggle source

@return [String] custom escape for aws compat

# File lib/miasma/contrib/aws.rb, line 766
def uri_escape(string)
  signer.safe_escape(string)
end