module Miasma::Contrib::AwsApiCore::ApiCommon
Common API setup
Public Class Methods
# File lib/miasma/contrib/aws.rb, line 360 def self.included(klass) klass.class_eval do include Bogo::Logger::Helpers attribute :aws_profile_name, [FalseClass, String], :default => ENV.fetch("AWS_PROFILE", "default") attribute :aws_sts_token, String attribute :aws_sts_role_arn, String attribute :aws_sts_external_id, String attribute :aws_sts_role_session_name, String attribute :aws_sts_region, String attribute :aws_sts_host, String attribute :aws_sts_session_token, String attribute :aws_sts_session_token_code, [String, Proc, Method] attribute :aws_sts_mfa_serial_number, [String] attribute :aws_credentials_file, String, :required => true, :default => ENV.fetch("AWS_SHARED_CREDENTIALS_FILE", File.join(Dir.home, ".aws/credentials")) attribute :aws_config_file, String, :required => true, :default => ENV.fetch("AWS_CONFIG_FILE", File.join(Dir.home, ".aws/config")) attribute :aws_access_key_id, String, :required => true, :default => ENV["AWS_ACCESS_KEY_ID"] attribute :aws_secret_access_key, String, :required => true, :default => ENV["AWS_SECRET_ACCESS_KEY"] attribute :aws_iam_instance_profile, [TrueClass, FalseClass], :default => false attribute :aws_ecs_task_profile, [TrueClass, FalseClass], :default => false attribute :aws_region, String, :required => true, :default => ENV["AWS_DEFAULT_REGION"] attribute :aws_host, String attribute :aws_bucket_region, String attribute :api_endpoint, String, :required => true, :default => "amazonaws.com" attribute :euca_compat, Symbol, :allowed_values => [:path, :dns], :coerce => lambda { |v| v.is_a?(String) ? v.to_sym : v } attribute :euca_dns_map, Smash, :coerce => lambda { |v| v.to_smash }, :default => Smash.new attribute :ssl_enabled, [TrueClass, FalseClass], :default => true end # AWS config file key remapping klass.const_set(:CONFIG_FILE_REMAP, Smash.new( "region" => "aws_region", "role_arn" => "aws_sts_role_arn", "aws_security_token" => "aws_sts_token", "aws_session_token" => "aws_sts_session_token", ).to_smash.freeze) klass.const_set(:INSTANCE_PROFILE_HOST, "http://169.254.169.254".freeze) klass.const_set( :INSTANCE_PROFILE_PATH, "latest/meta-data/iam/security-credentials".freeze ) klass.const_set( :INSTANCE_PROFILE_AZ_PATH, "latest/meta-data/placement/availability-zone".freeze ) klass.const_set(:ECS_TASK_PROFILE_HOST, "http://169.254.170.2".freeze) klass.const_set( :ECS_TASK_PROFILE_PATH, ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] ) # Reload sts tokens if expiry is within the next 10 minutes klass.const_set(:STS_TOKEN_EXPIRY_BUFFER, 600) end
Public Instance Methods
Persist any underlying stored credential data that is not a defined attribute (things like STS information)
@param creds [Hash] @return [TrueClass]
# File lib/miasma/contrib/aws.rb, line 498 def after_setup(creds) logger.debug("running after setup configuration updates") skip = self.class.attributes.keys.map(&:to_s) creds.each do |k, v| k = k.to_s if k.start_with?("aws_") && !skip.include?(k) data[k] = v end end end
Build new API for specified type using current provider / creds
@param type [Symbol] api type @return [Api]
# File lib/miasma/contrib/aws.rb, line 423 def api_for(type) memoize(type) do logger.debug("building API for type `#{type}`") creds = attributes.dup creds.delete(:aws_host) Miasma.api( Smash.new( :type => type, :provider => provider, :credentials => creds, ) ) end end
Setup for API connections
# File lib/miasma/contrib/aws.rb, line 710 def connect unless aws_host if euca_compat service_name = (self.class.const_defined?(:EUCA_API_SERVICE) ? self.class::EUCA_API_SERVICE : self.class::API_SERVICE) else service_name = self.class::API_SERVICE.downcase end if euca_compat == :path self.aws_host = [ api_endpoint, "services", service_name, ].join("/") elsif euca_compat == :dns && euca_dns_map[service_name] self.aws_host = [ euca_dns_map[service_name], api_endpoint, ].join(".") else self.aws_host = [ service_name, aws_region, api_endpoint, ].join(".") end end end
@return [HTTP] connection for requests (forces headers)
# File lib/miasma/contrib/aws.rb, line 771 def connection super.headers( "Host" => aws_host, "X-Amz-Date" => Contrib::AwsApiCore.time_iso8601, ) end
Provide custom setup functionality to support alternative credential loading.
@param creds [Hash] @return [TrueClass]
# File lib/miasma/contrib/aws.rb, line 443 def custom_setup(creds) logger.debug("running custom setup configuration updates") cred_file = load_aws_file(creds.fetch( :aws_credentials_file, aws_credentials_file )) config_file = load_aws_file(creds.fetch( :aws_config_file, aws_config_file )) # Load any configuration available from the config file profile = creds.fetch(:aws_profile_name, aws_profile_name) profile_list = [profile].compact new_config_creds = Smash.new while profile logger.debug("loading aws configuration profile: #{profile}") new_config_creds = config_file.fetch(profile, Smash.new).merge( new_config_creds ) profile = new_config_creds.delete(:source_profile) profile_list << profile end new_config_creds = config_file.fetch(:default, Smash.new).merge( new_config_creds ) # Load any configuration available from the creds file new_creds = Smash.new profile_list.each do |profile| logger.debug("loading aws credentials profile: #{profile}") new_creds = cred_file.fetch(profile, Smash.new).merge( new_creds ) profile = new_creds.delete(:source_profile) end new_creds = cred_file.fetch(:default, Smash.new).merge( new_creds ) new_creds = new_creds.merge(new_config_creds) # Provided credentials override any config file or creds # file configuration so set them into new creds if available new_creds.merge!(creds) # Replace creds hash with updated hash so it is loaded with # updated values creds.replace(new_creds) if creds[:aws_iam_instance_profile] self.class.const_get(:ECS_TASK_PROFILE_PATH).nil? ? load_instance_credentials!(creds) : load_ecs_credentials!(creds) end true end
@return [String] endpoint for request
# File lib/miasma/contrib/aws.rb, line 779 def endpoint "http#{"s" if ssl_enabled}://#{aws_host}" end
Return hash with needed information to assume role
@param data [Hash] @return [Hash]
# File lib/miasma/contrib/aws.rb, line 579 def extract_creds(data) c = Smash.new c[:aws_access_key_id] = data["AccessKeyId"] c[:aws_secret_access_key] = data["SecretAccessKey"] c[:aws_sts_token] = data["Token"] c[:aws_sts_token_expires] = Time.xmlschema(data["Expiration"]) c[:aws_sts_role_arn] = data["RoleArn"] # used in ECS Role but not instance role c end
Return correct credential value based on STS context
@param key [String, Symbol] credential suffix @return [Object]
# File lib/miasma/contrib/aws.rb, line 754 def get_credential(key, data_hash = nil) data_hash = attributes if data_hash.nil? if data_hash[:aws_sts_token] data_hash.fetch("aws_sts_#{key}", data_hash["aws_#{key}"]) elsif data_hash[:aws_sts_session_token] data_hash.fetch("aws_sts_session_#{key}", data_hash["aws_#{key}"]) else data_hash["aws_#{key}"] end end
Return region from meta-data service
@return [String]
# File lib/miasma/contrib/aws.rb, line 592 def get_region logger.debug("fetching region from meta-data service") az = HTTP.get( [ self.class.const_get(:INSTANCE_PROFILE_HOST), self.class.const_get(:INSTANCE_PROFILE_AZ_PATH), ].join("/") ).body.to_s.strip az.sub!(/[a-zA-Z]+$/, "") logger.debug("region from meta-data service: #{az}") az end
Load configuration from the AWS configuration file
@param file_path [String] path to configuration file @return [Smash]
# File lib/miasma/contrib/aws.rb, line 661 def load_aws_file(file_path) if File.exist?(file_path) logger.debug("loading aws file @ #{file_path}") Smash.new.tap do |creds| key = :default File.readlines(file_path).each_with_index do |line, idx| line.strip! next if line.empty? || line.start_with?("#") if line.start_with?("[") unless line.end_with?("]") raise ArgumentError, "Failed to parse aws file! (#{file_path} line #{idx + 1})" end key = line.tr("[]", "").strip.sub(/^profile /, "") creds[key] = Smash.new else unless key raise ArgumentError, "Failed to parse aws file! (#{file_path} line #{idx + 1}) " \ "- No section defined!" end line_args = line.split("=", 2).map(&:strip) line_args.first.replace( self.class.const_get(:CONFIG_FILE_REMAP).fetch( line_args.first, line_args.first ) ) if line_args.last.start_with?('"') unless line_args.last.end_with?('"') raise ArgumentError, "Failed to parse aws file! (#{file_path} line #{idx + 1})" end line_args.last.replace(line_args.last[1..-2]) # NOTE: strip quoted values end begin creds[key].merge!(Smash[*line_args]) rescue => e raise ArgumentError, "Failed to parse aws file! (#{file_path} line #{idx + 1})" end end end end else Smash.new end end
Attempt to load credentials from instance metadata
@param creds [Hash] @return [TrueClass]
# File lib/miasma/contrib/aws.rb, line 548 def load_ecs_credentials!(creds) logger.debug("loading ECS credentials") # As per docs ECS_TASK_PROFILE_PATH is defined as # /credential_provider_version/credentials?id=task_UUID # where AWS fills in the version and UUID. # @see https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html data = HTTP.get( [ self.class.const_get(:ECS_TASK_PROFILE_HOST), self.class.const_get(:ECS_TASK_PROFILE_PATH), ].join ).body unless data.is_a?(Hash) begin data = MultiJson.load(data.to_s) rescue MultiJson::ParseError => err logger.debug("failed to parse ECS credentials - #{err}") data = {} end end creds.merge!(extract_creds(data)) unless creds[:aws_region] creds[:aws_region] = get_region end true end
Attempt to load credentials from instance metadata
@param creds [Hash] @return [TrueClass]
# File lib/miasma/contrib/aws.rb, line 513 def load_instance_credentials!(creds) logger.debug("loading instance credentials") role = HTTP.get( [ self.class.const_get(:INSTANCE_PROFILE_HOST), self.class.const_get(:INSTANCE_PROFILE_PATH), "", ].join("/") ).body.to_s.strip data = HTTP.get( [ self.class.const_get(:INSTANCE_PROFILE_HOST), self.class.const_get(:INSTANCE_PROFILE_PATH), role, ].join("/") ).body unless data.is_a?(Hash) begin data = MultiJson.load(data.to_s) rescue MultiJson::ParseError => err logger.debug("failed to parse instance credentials - #{err}") data = {} end end creds.merge!(extract_creds(data)) unless creds[:aws_region] creds[:aws_region] = get_region end true end
Override to inject signature
@param connection [HTTP] @param http_method [Symbol] @param request_args [Array] @return [HTTP::Response]
# File lib/miasma/contrib/aws.rb, line 789 def make_request(connection, http_method, request_args) logger.debug("making #{http_method.to_s.upcase} request - #{request_args.inspect}") dest, options = request_args path = URI.parse(dest).path options = options ? options.to_smash : Smash.new options[:headers] = Smash[connection.default_options.headers.to_a]. merge(options.fetch(:headers, Smash.new)) if self.class::API_VERSION if options[:form] options.set(:form, "Version", self.class::API_VERSION) else options[:params] = options.fetch( :params, Smash.new ).to_smash.deep_merge( Smash.new( "Version" => self.class::API_VERSION, ) ) end end if aws_sts_session_token || aws_sts_session_token_code if sts_mfa_session_update_required? sts_mfa_session!(data) end options.set(:headers, "X-Amz-Security-Token", aws_sts_session_token) end if aws_sts_token || aws_sts_role_arn if sts_assume_role_update_required? sts_assume_role!(data) end options.set(:headers, "X-Amz-Security-Token", aws_sts_token) end signature = signer.generate(http_method, path, options) update_request(connection, options) options = Hash[options.map { |k, v| [k.to_sym, v] }] connection.auth(signature).send(http_method, dest, options) end
Determine if a retry is allowed based on exception
@param exception [Exception] @return [TrueClass, FalseClass]
# File lib/miasma/contrib/aws.rb, line 869 def perform_request_retry(exception) if exception.is_a?(Miasma::Error::ApiError) if [400, 500, 503].include?(exception.response.code) if exception.response.code == 400 exception.response.body.to_s.downcase.include?("throttl") else true end else false end end end
Always allow retry
@return [TrueClass]
# File lib/miasma/contrib/aws.rb, line 886 def retryable_allowed?(*_) true end
@return [Contrib::AwsApiCore::SignatureV4]
# File lib/miasma/contrib/aws.rb, line 741 def signer Contrib::AwsApiCore::SignatureV4.new( get_credential(:access_key_id), get_credential(:secret_access_key), aws_region, self.class::API_SERVICE ) end
Assume requested role and replace key id and secret
@param creds [Hash] @return [TrueClass]
# File lib/miasma/contrib/aws.rb, line 633 def sts_assume_role!(creds) if sts_assume_role_update_required?(creds) logger.debug("loading STS assume role") sts = Miasma::Contrib::Aws::Api::Sts.new( :aws_access_key_id => get_credential(:access_key_id, creds), :aws_secret_access_key => get_credential(:secret_access_key, creds), :aws_region => creds.fetch(:aws_sts_region, "us-east-1"), :aws_credentials_file => creds.fetch( :aws_credentials_file, aws_credentials_file ), :aws_config_file => creds.fetch(:aws_config_file, aws_config_file), :aws_host => creds[:aws_sts_host], :aws_sts_token => creds[:aws_sts_session_token], ) role_info = sts.assume_role( creds[:aws_sts_role_arn], :session_name => creds[:aws_sts_role_session_name], :external_id => creds[:aws_sts_external_id], ) creds.merge!(role_info) end true end
@return [TrueClass, FalseClass] @note update check only applied if assuming role
# File lib/miasma/contrib/aws.rb, line 829 def sts_assume_role_update_required?(args = {}) sts_attribute_update_required?(:aws_sts_role_arn, :aws_sts_token_expires, args) end
Check if STS attribute requires update
@param key [String, Symbol] token key @param expiry_key [String, Symbol] expiry of token (Time instance) @param args [Hash] overrides to check instead of instance values @return [TrueClass, FalseClass]
# File lib/miasma/contrib/aws.rb, line 847 def sts_attribute_update_required?(key, expiry_key, args = {}) if args.to_smash.fetch(key, attributes[key]) expiry = args.to_smash.fetch(expiry_key, attributes[expiry_key]) expiry.nil? || expiry - self.class.const_get(:STS_TOKEN_EXPIRY_BUFFER) <= Time.now else false end end
# File lib/miasma/contrib/aws.rb, line 605 def sts_mfa_session!(creds) if sts_mfa_session_update_required?(creds) logger.debug("loading STS MFA session") sts = Miasma::Contrib::Aws::Api::Sts.new( :aws_access_key_id => creds[:aws_access_key_id], :aws_secret_access_key => creds[:aws_secret_access_key], :aws_region => creds.fetch(:aws_sts_region, "us-east-1"), :aws_credentials_file => creds.fetch( :aws_credentials_file, aws_credentials_file ), :aws_config_file => creds.fetch(:aws_config_file, aws_config_file), :aws_profile_name => creds[:aws_profile_name], :aws_host => creds[:aws_sts_host], ) creds.merge!( sts.mfa_session( creds[:aws_sts_session_token_code], :mfa_serial => creds[:aws_sts_mfa_serial_number], ) ) end true end
@return [TrueClass, FalseClass] @note update check only applied if assuming role
# File lib/miasma/contrib/aws.rb, line 836 def sts_mfa_session_update_required?(args = {}) sts_attribute_update_required?(:aws_sts_session_token_code, :aws_sts_session_token_expires, args) end
Simple callback to allow request option adjustments prior to signature calculation
@param opts [Smash] request options @return [TrueClass]
# File lib/miasma/contrib/aws.rb, line 861 def update_request(con, opts) true end
@return [String] custom escape for aws compat
# File lib/miasma/contrib/aws.rb, line 766 def uri_escape(string) signer.safe_escape(string) end