class LogStash::Filters::Date

The date filter is used for parsing dates from fields and using that date or timestamp as the timestamp for the event.

For example, syslog events usually have timestamps like this:

"Apr 17 09:32:01"

You would use the date format “MMM dd HH:mm:ss” to parse this.

The date filter is especially important for sorting events and for backfilling old data. If you don't get the date correct in your event, then searching for them later will likely sort out of order.

In the absence of this filter, logstash will choose a timestamp based on the first time it sees the event (at input time), if the timestamp is not already set in the event. For example, with file input, the timestamp is set to the time of each read.

Constants

DATEPATTERNS

LOGSTASH-34

JavaException
UTC

Public Class Methods

new(config = {}) click to toggle source

The 'date' filter will take a value from your event and use it as the event timestamp. This is useful for parsing logs generated on remote servers or for importing old logs.

The config looks like this:

filter {
  date {
    type => "typename"
    filename => fieldformat
    # Example:
    timestamp => "mmm DD HH:mm:ss"
  }
}

The format is whatever is supported by Joda; generally: download.oracle.com/javase/1.4.2/docs/api/java/text/SimpleDateFormat.html

TODO(sissel): Support 'seconds since epoch' parsing (nagios uses this)

Calls superclass method LogStash::Filters::Base::new
# File lib/logstash/filters/date.rb, line 113
def initialize(config = {})
  super

  @parsers = Hash.new { |h,k| h[k] = [] }
end

Public Instance Methods

filter(event) click to toggle source

def register

# File lib/logstash/filters/date.rb, line 196
def filter(event)
  @logger.debug? && @logger.debug("Date filter: received event", :type => event["type"])
  return unless filter?(event)
  @parsers.each do |field, fieldparsers|
    @logger.debug? && @logger.debug("Date filter looking for field",
                                    :type => event["type"], :field => field)
    next unless event.include?(field)

    fieldvalues = event[field]
    fieldvalues = [fieldvalues] if !fieldvalues.is_a?(Array)
    fieldvalues.each do |value|
      next if value.nil?
      begin
        epochmillis = nil
        success = false
        last_exception = RuntimeError.new "Unknown"
        fieldparsers.each do |parserconfig|
          parser = parserconfig[:parser]
          begin
            epochmillis = parser.call(value)
            success = true
            break # success
          rescue StandardError, JavaException => e
            last_exception = e
          end
        end # fieldparsers.each

        raise last_exception unless success

        # Convert joda DateTime to a ruby Time
        event[@target] = Time.at(epochmillis / 1000, (epochmillis % 1000) * 1000)
        #event[@target] = Time.at(epochmillis / 1000.0).utc

        @logger.debug? && @logger.debug("Date parsing done", :value => value, :timestamp => event[@target])
      rescue StandardError, JavaException => e
        @logger.warn("Failed parsing date from field", :field => field,
                     :value => value, :exception => e)
        # Raising here will bubble all the way up and cause an exit.
        # TODO(sissel): Maybe we shouldn't raise?
        # TODO(sissel): What do we do on a failure? Tag it like grok does?
        #raise e
      end # begin
    end # fieldvalue.each
  end # @parsers.each

  filter_matched(event) if !event.cancelled?
  return event
end
register() click to toggle source
# File lib/logstash/filters/date.rb, line 130
def register
  require "java"
  if @match.length < 2
    raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register", 
      :plugin => "filter", :type => "date",
      :error => "The match setting should contains first a field name and at least one date format, current value is #{@match}")
  end
  # TODO(sissel): Need a way of capturing regexp configs better.
  locale = parseLocale(@config["locale"][0]) if @config["locale"] != nil and @config["locale"][0] != nil
  setupMatcher(@config["match"].shift, locale, @config["match"] )
end
setupMatcher(field, locale, value) click to toggle source
# File lib/logstash/filters/date.rb, line 142
def setupMatcher(field, locale, value)
  value.each do |format|
    case format
      when "ISO8601"
        joda_parser = org.joda.time.format.ISODateTimeFormat.dateTimeParser
        if @timezone
          joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
        else
          joda_parser = joda_parser.withOffsetParsed
        end
        parser = lambda { |date| joda_parser.parseMillis(date) }
      when "UNIX" # unix epoch
        joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance)
        #parser = lambda { |date| joda_instant.call((date.to_f * 1000).to_i).to_java.toDateTime }
        parser = lambda { |date| (date.to_f * 1000).to_i }
      when "UNIX_MS" # unix epoch in ms
        joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance)
        parser = lambda do |date| 
          #return joda_instant.call(date.to_i).to_java.toDateTime
          return date.to_i
        end
      when "TAI64N" # TAI64 with nanoseconds, -10000 accounts for leap seconds
        joda_instant = org.joda.time.Instant.java_class.constructor(Java::long).method(:new_instance)
        parser = lambda do |date| 
          # Skip leading "@" if it is present (common in tai64n times)
          date = date[1..-1] if date[0, 1] == "@"
          #return joda_instant.call((date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)).to_java.toDateTime
          return (date[1..15].hex * 1000 - 10000)+(date[16..23].hex/1000000)
        end
      else
        joda_parser = org.joda.time.format.DateTimeFormat.forPattern(format).withDefaultYear(Time.new.year)
        if @timezone
          joda_parser = joda_parser.withZone(org.joda.time.DateTimeZone.forID(@timezone))
        else
          joda_parser = joda_parser.withOffsetParsed
        end
        if (locale != nil)
          joda_parser = joda_parser.withLocale(locale)
        end
        parser = lambda { |date| joda_parser.parseMillis(date) }
    end

    @logger.debug("Adding type with date config", :type => @type,
                  :field => field, :format => format)
    @parsers[field] << {
      :parser => parser,
      :format => format
    }
  end
end

Private Instance Methods

parseLocale(localeString) click to toggle source
# File lib/logstash/filters/date.rb, line 120
def parseLocale(localeString)
  return nil if localeString == nil
  matches = localeString.match(/(?<lang>.+?)(?:_(?<country>.+?))?(?:_(?<variant>.+))?/)
  lang = matches['lang'] == nil ? "" : matches['lang'].strip()
  country = matches['country'] == nil ? "" : matches['country'].strip()
  variant = matches['variant'] == nil ? "" : matches['variant'].strip()
  return lang.length > 0 ? java.util.Locale.new(lang, country, variant) : nil
end