class LogStash::Filters::KV
This filter helps automatically parse messages which are of the 'foo=bar' variety.
For example, if you have a log message which contains 'ip=1.2.3.4 error=REFUSED', you can parse those automatically by doing:
filter { kv { } }
The above will result in a message of “ip=1.2.3.4 error=REFUSED” having the fields:
-
ip: 1.2.3.4
-
error: REFUSED
This is great for postfix, iptables, and other types of logs that tend towards 'key=value' syntax.
Further, this can often be used to parse query parameters like 'foo=bar&baz=fizz' by setting the field_split to “&”
Public Instance Methods
filter(event)
click to toggle source
# File lib/logstash/filters/kv.rb, line 166 def filter(event) return unless filter?(event) kv = Hash.new value = event[@source] case value when nil; # Nothing to do when String; kv = parse(value, event, kv) when Array; value.each { |v| kv = parse(v, event, kv) } else @logger.warn("kv filter has no support for this type of data", :type => value.class, :value => value) end # case value # Add default key-values for missing keys kv = @default_keys.merge(kv) # If we have any keys, create/append the hash if kv.length > 0 if @target.nil? # Default is to write to the root of the event. dest = event.to_hash else if !event[@target].is_a?(Hash) @logger.debug("Overwriting existing target field", :target => @target) dest = event[@target] = {} else dest = event[@target] end end dest.merge!(kv) filter_matched(event) end end
register()
click to toggle source
# File lib/logstash/filters/kv.rb, line 160 def register @trim_re = Regexp.new("[#{@trim}]") if !@trim.nil? @trimkey_re = Regexp.new("[#{@trimkey}]") if !@trimkey.nil? @scan_re = Regexp.new("((?:\\\\ |[^"+@field_split+@value_split+"])+)["+@value_split+"](?:\"([^\"]+)\"|'([^']+)'|((?:\\\\ |[^"+@field_split+"])+))") end
Private Instance Methods
parse(text, event, kv_keys)
click to toggle source
# File lib/logstash/filters/kv.rb, line 205 def parse(text, event, kv_keys) if !event =~ /[@field_split]/ return kv_keys end text.scan(@scan_re) do |key, v1, v2, v3| value = v1 || v2 || v3 key = @trimkey.nil? ? key : key.gsub(@trimkey_re, "") # Bail out as per the values of @include_keys and @exclude_keys next if not @include_keys.empty? and not @include_keys.include?(key) next if @exclude_keys.include?(key) key = event.sprintf(@prefix) + key value = @trim.nil? ? value : value.gsub(@trim_re, "") if kv_keys.has_key?(key) if kv_keys[key].is_a? Array kv_keys[key].push(value) else kv_keys[key] = [kv_keys[key], value] end else kv_keys[key] = value end end return kv_keys end