class LogStash::Filters::Date
The date filter is used for parsing dates from fields and using that date or timestamp as the timestamp for the event.
For example, syslog events usually have timestamps like this:
"Apr 17 09:32:01"
You would use the date format “MMM dd HH:mm:ss” to parse this.
The date filter is especially important for sorting events and for backfilling old data. If you don't get the date correct in your event, then searching for them later will likely sort out of order.
In the absence of this filter, logstash will choose a timestamp based on the first time it sees the event (at input time), if the timestamp is not already set in the event. For example, with file input, the timestamp is set to the time of each read.
Constants
- DATEPATTERNS
LOGSTASH-34
- JavaException
- UTC
Public Class Methods
The 'date' filter will take a value from your event and use it as the event timestamp. This is useful for parsing logs generated on remote servers or for importing old logs.
The config looks like this:
filter { date { type => "typename" filename => fieldformat # Example: timestamp => "mmm DD HH:mm:ss" } }
The format is whatever is supported by Joda; generally: download.oracle.com/javase/1.4.2/docs/api/java/text/SimpleDateFormat.html
TODO(sissel): Support 'seconds since epoch' parsing (nagios uses this)
LogStash::Filters::Base::new
# File lib/logstash/filters/date.rb, line 113 def initialize(config = {}) super @parsers = Hash.new { |h,k| h[k] = [] } end
Public Instance Methods
def register
# File lib/logstash/filters/date.rb, line 196 def filter(event) @logger.debug? && @logger.debug("Date filter: received event", :type => event["type"]) return unless filter?(event) @parsers.each do |field, fieldparsers| @logger.debug? && @logger.debug("Date filter looking for field", :type => event["type"], :field => field) next unless event.include?(field) fieldvalues = event[field] fieldvalues = [fieldvalues] if !fieldvalues.is_a?(Array) fieldvalues.each do |value| next if value.nil? begin epochmillis = nil success = false last_exception = RuntimeError.new "Unknown" fieldparsers.each do |parserconfig| parser = parserconfig[:parser] begin epochmillis = parser.call(value) success = true break # success rescue StandardError, JavaException => e last_exception = e end end # fieldparsers.each raise last_exception unless success # Convert joda DateTime to a ruby Time event[@target] = Time.at(epochmillis / 1000, (epochmillis % 1000) * 1000) #event[@target] = Time.at(epochmillis / 1000.0).utc @logger.debug? && @logger.debug("Date parsing done", :value => value, :timestamp => event[@target]) rescue StandardError, JavaException => e @logger.warn("Failed parsing date from field", :field => field, :value => value, :exception => e) # Raising here will bubble all the way up and cause an exit. # TODO(sissel): Maybe we shouldn't raise? # TODO(sissel): What do we do on a failure? Tag it like grok does? #raise e end # begin end # fieldvalue.each end # @parsers.each filter_matched(event) if !event.cancelled? return event end
# File lib/logstash/filters/date.rb, line 130 def register require "java" if @match.length < 2 raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register", :plugin => "filter", :type => "date", :error => "The match setting should contains first a field name and at least one date format, current value is #{@match}") end # TODO(sissel): Need a way of capturing regexp configs better. locale = parseLocale(@config["locale"][0]) if @config["locale"] != nil and @config["locale"][0] != nil setupMatcher(@config["match"].shift, locale, @config["match"] ) end
# File lib/logstash/filters/date.rb, line 142 def setupMatcher(field, locale, value) value.each do |format| case format when "ISO8601" joda_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser if @timezone joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone)) else joda_parser = joda_parser.withOffsetParsed end parser = lambda { |date| joda_parser.parseMillis(date) } when "UNIX" # unix epoch joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance) #parser = lambda { |date| joda_instant.call((date.to_f * 1000).to_i).to_java.toDateTime } parser = lambda { |date| (date.to_f * 1000).to_i } when "UNIX_MS" # unix epoch in ms joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance) parser = lambda do |date| #return joda_instant.call(date.to_i).to_java.toDateTime return date.to_i end when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance) parser = lambda do |date| # Skip leading "@" if it is present (common in tai64n times) date = date[1..-1] if date[0, 1] == "@" #return joda_instant.call((date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)).to_java.toDateTime return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000) end else joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format).withDefaultYear(Time.new.year) if @timezone joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone)) else joda_parser = joda_parser.withOffsetParsed end if (locale != nil) joda_parser = joda_parser.withLocale(locale) end parser = lambda { |date| joda_parser.parseMillis(date) } end @logger.debug("Adding type with date config", :type => @type, :field => field, :format => format) @parsers[field] << { :parser => parser, :format => format } end end
Private Instance Methods
# File lib/logstash/filters/date.rb, line 120 def parseLocale(localeString) return nil if localeString == nil matches = localeString.match(/(?<lang>.+?)(?:_(?<country>.+?))?(?:_(?<variant>.+))?/) lang = matches['lang'] == nil ? "" : matches['lang'].strip() country = matches['country'] == nil ? "" : matches['country'].strip() variant = matches['variant'] == nil ? "" : matches['variant'].strip() return lang.length > 0 ? java.util.Locale.new(lang, country, variant) : nil end