class Common::Utils

Utilities for managing the resource used when writing to the Google API.

Public Class Methods

new(log) click to toggle source
# File lib/fluent/plugin/common.rb, line 103
def initialize(log)
  @log = log
end

Public Instance Methods

check_required_metadata_variables(platform, project_id, zone, vm_id) click to toggle source

Check required variables like @project_id, @vm_id, @vm_name and @zone.

# File lib/fluent/plugin/common.rb, line 161
def check_required_metadata_variables(platform, project_id, zone, vm_id)
  missing = []
  missing << 'project_id' unless project_id
  if platform != Platform::OTHER
    missing << 'zone' unless zone
    missing << 'vm_id' unless vm_id
  end
  return if missing.empty?

  raise Fluent::ConfigError,
        "Unable to obtain metadata parameters: #{missing.join(' ')}"
end
create_monitored_resource(type, labels) click to toggle source

Create a monitored resource from type and labels.

# File lib/fluent/plugin/common.rb, line 227
def create_monitored_resource(type, labels)
  Google::Apis::LoggingV2::MonitoredResource.new(
    type: type, labels: labels.to_h
  )
end
detect_platform(use_metadata_service) click to toggle source

Determine what platform we are running on by consulting the metadata service (unless the user has explicitly disabled using that).

# File lib/fluent/plugin/common.rb, line 109
def detect_platform(use_metadata_service)
  unless use_metadata_service
    @log.info 'use_metadata_service is false; not detecting platform'
    return Platform::OTHER
  end

  begin
    URI.open("http://#{METADATA_SERVICE_ADDR}", proxy: false) do |f|
      if f.meta['metadata-flavor'] == 'Google'
        @log.info 'Detected GCE platform'
        return Platform::GCE
      end
      if f.meta['server'] == 'EC2ws'
        @log.info 'Detected EC2 platform'
        return Platform::EC2
      end
    end
  rescue StandardError => e
    @log.error 'Failed to access metadata service: ', error: e
  end

  @log.info 'Unable to determine platform'
  Platform::OTHER
end
determine_agent_level_monitored_resource_labels( platform, type, vm_id, zone ) click to toggle source

Determine agent level monitored resource labels based on the resource type. Each resource type has its own labels that need to be filled in.

# File lib/fluent/plugin/common.rb, line 288
def determine_agent_level_monitored_resource_labels(
  platform, type, vm_id, zone
)
  case type
  # GAE app.
  when APPENGINE_CONSTANTS[:resource_type]
    return {
      'module_id' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/gae_backend_name'),
      'version_id' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/gae_backend_version')
    }

  # GCE.
  when COMPUTE_CONSTANTS[:resource_type]
    raise "Cannot construct a #{type} resource without vm_id and zone" \
      unless vm_id && zone

    return {
      'instance_id' => vm_id,
      'zone' => zone
    }

  # GKE container.
  when GKE_CONSTANTS[:resource_type]
    raise "Cannot construct a #{type} resource without vm_id and zone" \
      unless vm_id && zone

    return {
      'instance_id' => vm_id,
      'zone' => zone,
      'cluster_name' =>
        fetch_gce_metadata(platform, 'instance/attributes/cluster-name')
    }

  # Cloud Dataproc.
  when DATAPROC_CONSTANTS[:resource_type]
    return {
      'cluster_uuid' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/dataproc-cluster-uuid'),
      'cluster_name' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/dataproc-cluster-name'),
      'region' =>
        fetch_gce_metadata(platform,
                           'instance/attributes/dataproc-region')
    }

  # EC2.
  when EC2_CONSTANTS[:resource_type]
    raise "Cannot construct a #{type} resource without vm_id and zone" \
      unless vm_id && zone

    labels = {
      'instance_id' => vm_id,
      'region' => zone
    }
    labels['aws_account'] = ec2_metadata(platform)['accountId'] if
      ec2_metadata(platform).key?('accountId')
    return labels
  end

  {}
rescue StandardError => e
  if [Platform::GCE, Platform::EC2].include?(platform)
    @log.error "Failed to set monitored resource labels for #{type}: ",
               error: e
  end
  {}
end
determine_agent_level_monitored_resource_type( platform, subservice_name, detect_subservice ) click to toggle source

Determine agent level monitored resource type.

# File lib/fluent/plugin/common.rb, line 253
def determine_agent_level_monitored_resource_type(
  platform, subservice_name, detect_subservice
)
  case platform
  when Platform::OTHER
    # Unknown platform will be defaulted to GCE instance.
    COMPUTE_CONSTANTS[:resource_type]

  when Platform::EC2
    EC2_CONSTANTS[:resource_type]

  when Platform::GCE
    # Resource types determined by subservice_name config.
    return SUBSERVICE_MAP[subservice_name] if subservice_name

    # Resource types determined by detect_subservice config.
    if detect_subservice
      begin
        attributes = fetch_gce_metadata(platform,
                                        'instance/attributes/').split.to_set
        SUBSERVICE_METADATA_ATTRIBUTES.each do |resource_type, expected|
          return resource_type if attributes.superset?(expected)
        end
      rescue StandardError => e
        @log.error 'Failed to detect subservice: ', error: e
      end
    end

    # GCE instance.
    COMPUTE_CONSTANTS[:resource_type]
  end
end
determine_agent_level_monitored_resource_via_legacy( platform, subservice_name, detect_subservice, vm_id, zone ) click to toggle source

Retrieve monitored resource via the legacy way.

Note: This is just a failover plan if we fail to get metadata from Metadata Agent. Thus it should be equivalent to what Metadata Agent returns.

# File lib/fluent/plugin/common.rb, line 238
def determine_agent_level_monitored_resource_via_legacy(
  platform, subservice_name, detect_subservice, vm_id, zone
)
  resource_type = determine_agent_level_monitored_resource_type(
    platform, subservice_name, detect_subservice
  )
  create_monitored_resource(
    resource_type,
    determine_agent_level_monitored_resource_labels(
      platform, resource_type, vm_id, zone
    )
  )
end
ec2_metadata(platform) click to toggle source

EC2 Metadata server returns everything in one call. Store it after the first fetch to avoid making multiple calls.

# File lib/fluent/plugin/common.rb, line 145
def ec2_metadata(platform)
  raise "Called ec2_metadata with platform=#{platform}" unless
    platform == Platform::EC2

  unless @ec2_metadata
    # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
    URI.open("http://#{METADATA_SERVICE_ADDR}/latest/dynamic/instance-identity/document", proxy: false) do |f|
      contents = f.read
      @ec2_metadata = JSON.parse(contents)
    end
  end

  @ec2_metadata
end
fetch_gce_metadata(platform, metadata_path) click to toggle source
# File lib/fluent/plugin/common.rb, line 134
def fetch_gce_metadata(platform, metadata_path)
  raise "Called fetch_gce_metadata with platform=#{platform}" unless
    platform == Platform::GCE

  # See https://cloud.google.com/compute/docs/metadata
  URI.open("http://#{METADATA_SERVICE_ADDR}/computeMetadata/v1/#{metadata_path}",
           'Metadata-Flavor' => 'Google', :proxy => false, &:read)
end
get_location(platform, zone, use_aws_availability_zone) click to toggle source
  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it locally.

# File lib/fluent/plugin/common.rb, line 207
def get_location(platform, zone, use_aws_availability_zone)
  # Response format: "projects/<number>/zones/<zone>"
  if platform == Platform::GCE
    zone ||= fetch_gce_metadata(platform,
                                'instance/zone').rpartition('/')[2]
  end
  aws_location_key = if use_aws_availability_zone
                       'availabilityZone'
                     else
                       'region'
                     end
  zone ||= "aws:#{ec2_metadata(platform)[aws_location_key]}" if
    platform == Platform::EC2 &&
    ec2_metadata(platform).key?(aws_location_key)
  zone
rescue StandardError => e
  @log.error 'Failed to obtain location: ', error: e
end
get_project_id(platform, project_id) click to toggle source
  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it by calling metadata server directly.

  3. If still not set, try to obtain it from the credentials.

# File lib/fluent/plugin/common.rb, line 177
def get_project_id(platform, project_id)
  project_id ||= CredentialsInfo.project_id
  project_id ||= fetch_gce_metadata(platform, 'project/project-id') if
    platform == Platform::GCE
  project_id
end
get_vm_id(platform, vm_id) click to toggle source
  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it by calling metadata servers directly.

# File lib/fluent/plugin/common.rb, line 186
def get_vm_id(platform, vm_id)
  vm_id ||= fetch_gce_metadata(platform, 'instance/id') if
    platform == Platform::GCE
  vm_id ||= ec2_metadata(platform)['instanceId'] if
    platform == Platform::EC2
  vm_id
rescue StandardError => e
  @log.error 'Failed to obtain vm_id: ', error: e
end
get_vm_name(vm_name) click to toggle source
  1. Return the value if it is explicitly set in the config already.

  2. If not, try to retrieve it locally.

# File lib/fluent/plugin/common.rb, line 198
def get_vm_name(vm_name)
  vm_name ||= Socket.gethostname
  vm_name
rescue StandardError => e
  @log.error 'Failed to obtain vm name: ', error: e
end