class LogStash::Filters::Metrics

The metrics filter is useful for aggregating metrics.

For example, if you have a field 'response' that is a http response code, and you want to count each kind of response, you can do this:

filter {
  metrics {
    meter => [ "http.%{response}" ]
    add_tag => "metric"
  }
}

Metrics are flushed every 5 seconds by default or according to 'flush_interval'. Metrics appear as new events in the event stream and go through any filters that occur after as well as outputs.

In general, you will want to add a tag to your metrics and have an output explicitly look for that tag.

The event that is flushed will include every 'meter' and 'timer' metric in the following way:

#### 'meter' values

For a `meter => “something”` you will receive the following fields:

#### 'timer' values

For a `timer => [ “thing”, “%{duration}” ]` you will receive the following fields:

#### Example: computing event rate

For a simple example, let's track how many events per second are running through logstash:

input {
  generator {
    type => "generated"
  }
}

filter {
  metrics {
    type => "generated"
    meter => "events"
    add_tag => "metric"
  }
}

output {
  stdout {
    # only emit events with the 'metric' tag
    tags => "metric"
    message => "rate: %{events.rate_1m}"
  }
}

Running the above:

% java -jar logstash.jar agent -f example.conf
rate: 23721.983566819246
rate: 24811.395722536377
rate: 25875.892745934525
rate: 26836.42375967113

We see the output includes our 'events' 1-minute rate.

In the real world, you would emit this to graphite or another metrics store, like so:

output {
  graphite {
    metrics => [ "events.rate_1m", "%{events.rate_1m}" ]
  }
}

Public Instance Methods

filter(event) click to toggle source
# File lib/logstash/filters/metrics.rb, line 151
def filter(event)
  return unless filter?(event)

  # TODO(piavlo): This should probably be moved to base filter class.
  if @ignore_older_than > 0 && Time.now - event["@timestamp"] > @ignore_older_than
    @logger.debug("Skipping metriks for old event", :event => event)
    return
  end

  @meter.each do |m|
    @metric_meters[event.sprintf(m)].mark
  end

  @timer.each do |name, value|
    @metric_timers[event.sprintf(name)].update(event.sprintf(value).to_f)
  end
end
flush() click to toggle source
# File lib/logstash/filters/metrics.rb, line 169
def flush
  # Add 5 seconds to @last_flush and @last_clear counters
  # since this method is called every 5 seconds.
  @last_flush += 5
  @last_clear += 5

  # Do nothing if there's nothing to do ;)
  return unless should_flush?

  event = LogStash::Event.new
  event["message"] = Socket.gethostname
  @metric_meters.each do |name, metric|
    flush_rates event, name, metric
    metric.clear if should_clear?
  end

  @metric_timers.each do |name, metric|
    flush_rates event, name, metric
    # These 4 values are not sliding, so they probably are not useful.
    event["#{name}.min"] = metric.min
    event["#{name}.max"] = metric.max
    # timer's stddev currently returns variance, fix it.
    event["#{name}.stddev"] = metric.stddev ** 0.5
    event["#{name}.mean"] = metric.mean

    @percentiles.each do |percentile|
      event["#{name}.p#{percentile}"] = metric.snapshot.value(percentile / 100)
    end
    metric.clear if should_clear?
  end

  # Reset counter since metrics were flushed
  @last_flush = 0

  if should_clear?
    #Reset counter since metrics were cleared
    @last_clear = 0
    initialize_metrics
  end

  filter_matched(event)
  return [event]
end
register() click to toggle source
# File lib/logstash/filters/metrics.rb, line 139
def register
  require "metriks"
  require "socket"
  @last_flush = 0 # how many seconds ago the metrics where flushed.
  @last_clear = 0 # how many seconds ago the metrics where cleared.
  @random_key_preffix = SecureRandom.hex
  unless (@rates - [1, 5, 15]).empty?
    raise LogStash::ConfigurationError, "Invalid rates configuration. possible rates are 1, 5, 15. Rates: #{rates}."
  end
  initialize_metrics
end

Private Instance Methods

flush_rates(event, name, metric) click to toggle source
# File lib/logstash/filters/metrics.rb, line 214
def flush_rates(event, name, metric)
    event["#{name}.count"] = metric.count
    event["#{name}.rate_1m"] = metric.one_minute_rate if @rates.include? 1
    event["#{name}.rate_5m"] = metric.five_minute_rate if @rates.include? 5
    event["#{name}.rate_15m"] = metric.fifteen_minute_rate if @rates.include? 15
end
initialize_metrics() click to toggle source
# File lib/logstash/filters/metrics.rb, line 221
def initialize_metrics
  @metric_meters = Hash.new { |h,k| h[k] = Metriks.meter metric_key(k) }
  @metric_timers = Hash.new { |h,k| h[k] = Metriks.timer metric_key(k) }
end
metric_key(key) click to toggle source
# File lib/logstash/filters/metrics.rb, line 226
def metric_key(key)
  "#{@random_key_preffix}_#{key}"
end
should_clear?() click to toggle source
# File lib/logstash/filters/metrics.rb, line 234
def should_clear?
  @clear_interval > 0 && @last_clear >= @clear_interval
end
should_flush?() click to toggle source
# File lib/logstash/filters/metrics.rb, line 230
def should_flush?
  @last_flush >= @flush_interval && (@metric_meters.any? || @metric_timers.any?)
end