Merge pull request #12 from ecwws/custom_fields

allow custom fields to be set

Author: ecwws

README.md

@@ -19,10 +19,15 @@ elasticsearch.
   such value is detected, it will be converted to iso8601 format for easier
   consumption of elasticsearch when dynamic mapping is used.**
 
-* If a field named `timestamp` or `time` or `syslog_timestamp` exists, it will
-  parse that field and conver it to format '%Y-%m-%dT%H:%M:%S.%L%z' then store it
-  in `@timestamp` field. In addition, a field `fluent_converted_timestamp`
-  is added to the object with the same value.
+* By default, it will check whether fields named `timestamp`, `time`, or
+  `syslog_timestamp` exists, if so it will parse that field and conver it to
+  format '%Y-%m-%dT%H:%M:%S.%L%z' then store it in `@timestamp` field. In
+  addition, a field `fluent_converted_timestamp` is added to the object with
+  the same value.
+
+* (>=0.3.0) the list of fields can be overriden by setting the
+  `timestamp_fields` parameter. It accepts a list of strings, the default is set
+  to: `['@timestamp', 'timestamp', 'time', 'syslog_timestamp']`
 
 * If none of the above field exists, it will insert current event time in
   '%Y-%m-%dT%H:%M:%S.%L%z' format as the `@timestamp` field. A field

fluent-plugin-elasticsearch-timestamp-check.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elasticsearch-timestamp-check"
-  spec.version       = "0.2.8"
+  spec.version       = "0.3.0"
   spec.authors       = ["Richard Li"]
   spec.email         = ["[email protected]"]
   spec.description   = %q{fluent filter plugin to ensure @timestamp is in proper format}

lib/fluent/plugin/filter_elasticsearch_timestamp_check.rb

@@ -7,6 +7,7 @@ class ElasticsearchTimestampCheckFilter < Filter
     Fluent::Plugin.register_filter('elasticsearch_timestamp_check', self)
 
     config_param :subsecond_precision, :integer, default: 3
+    config_param :timestamp_fields, :array, default: ['@timestamp', 'timestamp', 'time', 'syslog_timestamp'], value_type: :string
 
     def configure(conf)
       super
@@ -33,7 +34,7 @@ def shutdown
     end
 
     def filter(tag, time, record)
-      %w{@timestamp timestamp time syslog_timestamp}.map do |field|
+      @timestamp_fields.map do |field|
         record[field]
       end.compact.each do |timestamp|
         begin
@@ -58,6 +59,7 @@ def filter(tag, time, record)
           $log.debug("Timestamp parsed: #{record['@timestamp']}")
           break
         rescue ArgumentError
+          $log.debug("#{field} (#{timestamp}) failed to parse, trying next")
         end
       end