class LogStash::Filters::KV

This filter helps automatically parse messages which are of the 'foo=bar' variety.

For example, if you have a log message which contains 'ip=1.2.3.4 error=REFUSED', you can parse those automatically by doing:

filter {
  kv { }
}

The above will result in a message of “ip=1.2.3.4 error=REFUSED” having the fields:

This is great for postfix, iptables, and other types of logs that tend towards 'key=value' syntax.

Further, this can often be used to parse query parameters like 'foo=bar&baz=fizz' by setting the field_split to “&”

Public Instance Methods

filter(event) click to toggle source
# File lib/logstash/filters/kv.rb, line 166
def filter(event)
  return unless filter?(event)

  kv = Hash.new

  value = event[@source]

  case value
    when nil; # Nothing to do
    when String; kv = parse(value, event, kv)
    when Array; value.each { |v| kv = parse(v, event, kv) }
    else
      @logger.warn("kv filter has no support for this type of data",
                   :type => value.class, :value => value)
  end # case value

  # Add default key-values for missing keys
  kv = @default_keys.merge(kv)

  # If we have any keys, create/append the hash
  if kv.length > 0
    if @target.nil?
      # Default is to write to the root of the event.
      dest = event.to_hash
    else
      if !event[@target].is_a?(Hash)
        @logger.debug("Overwriting existing target field", :target => @target)
        dest = event[@target] = {}
      else
        dest = event[@target]
      end
    end

    dest.merge!(kv)
    filter_matched(event)
  end
end
register() click to toggle source
# File lib/logstash/filters/kv.rb, line 160
def register
  @trim_re = Regexp.new("[#{@trim}]") if !@trim.nil?
  @trimkey_re = Regexp.new("[#{@trimkey}]") if !@trimkey.nil?
  @scan_re = Regexp.new("((?:\\\\ |[^"+@field_split+@value_split+"])+)["+@value_split+"](?:\"([^\"]+)\"|'([^']+)'|((?:\\\\ |[^"+@field_split+"])+))")
end

Private Instance Methods

parse(text, event, kv_keys) click to toggle source
# File lib/logstash/filters/kv.rb, line 205
def parse(text, event, kv_keys)
  if !event =~ /[@field_split]/
    return kv_keys
  end
  text.scan(@scan_re) do |key, v1, v2, v3|
    value = v1 || v2 || v3
    key = @trimkey.nil? ? key : key.gsub(@trimkey_re, "")

    # Bail out as per the values of @include_keys and @exclude_keys
    next if not @include_keys.empty? and not @include_keys.include?(key)
    next if @exclude_keys.include?(key)

    key = event.sprintf(@prefix) + key

    value = @trim.nil? ? value : value.gsub(@trim_re, "")
    if kv_keys.has_key?(key)
      if kv_keys[key].is_a? Array
        kv_keys[key].push(value)
      else
        kv_keys[key] = [kv_keys[key], value]
      end
    else
      kv_keys[key] = value
    end
  end
  return kv_keys
end