Merge pull request #74 from fluent/support-timezone

Support timezone

Author: okkez

README.md

@@ -104,6 +104,17 @@ You can use this parser without `multiline_start_regexp` when you know your data
 * **grok_name_key** (string) (optional): The key name to store grok section's name
 * **multi_line_start_regexp** (string) (optional): The regexp to match beginning of multiline. This is only for "multiline_grok".
 
+### \<grok\> section (optional) (multiple)
+
+* **name** (string) (optional): The name of this grok section
+* **pattern** (string) (required): The pattern of grok
+* **keep_time_key** (bool) (optional): If true, keep time field in the record.
+* **time_key** (string) (optional): Specify time field for event time. If the event doesn't have this field, current time is used.
+  * Default value: `time`.
+* **time_format** (string) (optional): Process value using specified format. This is available only when time_type is string
+* **timezone** (string) (optional): Use specified timezone. one can parse/format the time value in the specified timezone.
+
+
 ## Examples
 
 ### Using grok\_failure\_key
@@ -183,6 +194,28 @@ This will add keys like following:
 Add `grokfailure` key to the record if the record does not match any grok pattern.
 See also test code for more details.
 
+## How to parse time value using specific timezone
+
+```aconf
+<source>
+  @type tail
+  path /path/to/log
+  tag grokked_log
+  <parse>
+    @type grok
+    <grok>
+      name mylog-without-timezone
+      pattern %{DATESTAMP:time} %{GREEDYDATE:message}
+      timezone Asia/Tokyo
+    </grok>
+  </parse>
+</source>
+```
+
+This will parse the `time` value as "Asia/Tokyo" timezone.
+
+See [Config: Parse Section - Fluentd](https://docs.fluentd.org/configuration/parse-section) for more details about timezone.
+
 ## How to write Grok patterns
 
 Grok patterns look like `%{PATTERN_NAME:name}` where ":name" is optional. If "name" is provided, then it

lib/fluent/plugin/grok.rb

@@ -27,6 +27,8 @@ def initialize(plugin, conf)
       @multiline_mode = false
       @conf = conf
       @plugin = plugin
+      @time_format = nil
+      @timezone = nil
       if @plugin.respond_to?(:firstline?)
         @multiline_mode = true
       end
@@ -39,6 +41,9 @@ def initialize(plugin, conf)
       if @plugin.respond_to?(:time_format)
         @time_format = @plugin.time_format
       end
+      if @plugin.respond_to?(:timezone)
+        @timezone = @plugin.timezone
+      end
     end
 
     def add_patterns_from_file(path)
@@ -105,6 +110,9 @@ def expand_pattern_expression_grok_section(conf)
       if conf["time_format"] || @time_format
         _conf["time_format"] = conf["time_format"] || @time_format
       end
+      if conf["timezone"] || @timezone
+        _conf["timezone"] = conf["timezone"] || @timezone
+      end
       _conf["expression"] = regexp
       config = Fluent::Config::Element.new("parse", "", _conf, [])
       parser = Fluent::Plugin::RegexpParser.new

lib/fluent/plugin/parser_grok.rb

@@ -22,9 +22,14 @@ class GrokParser < Parser
         config_param :name, :string, default: nil
         desc "The pattern of grok"
         config_param :pattern, :string
+        desc "If true, keep time field in the record."
         config_param :keep_time_key, :bool, default: false
+        desc "Specify time field for event time. If the event doesn't have this field, current time is used."
         config_param :time_key, :string, default: "time"
+        desc "Process value using specified format. This is available only when time_type is string"
         config_param :time_format, :string, default: nil
+        desc "Use specified timezone. one can parse/format the time value in the specified timezone."
+        config_param :timezone, :string, default: nil
       end
 
       def initialize

test/test_grok_parser.rb

@@ -374,6 +374,46 @@ class GrokParserTest < ::Test::Unit::TestCase
         assert_equal(expected_record, record)
       end
     end
+
+    test "timezone" do
+      d = create_driver(%[
+        <grok>
+          pattern %{TIMESTAMP_ISO8601:time} %{GREEDYDATA:message}
+          time_key time
+          time_format %Y-%m-%d %H:%M:%S
+          timezone Europe/Berlin
+        </grok>
+      ])
+      d.instance.parse("2019-02-01 12:34:56 This is test") do |time, record|
+        assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
+        assert_equal({ "message" => "This is test" }, record)
+      end
+    end
+
+    test "multiple timezone" do
+      d = create_driver(%[
+        <grok>
+          pattern %{TIMESTAMP_ISO8601:time} 1 %{GREEDYDATA:message}
+          time_key time
+          time_format %Y-%m-%d %H:%M:%S
+          timezone Europe/Berlin
+        </grok>
+        <grok>
+          pattern %{TIMESTAMP_ISO8601:time} 2 %{GREEDYDATA:message}
+          time_key time
+          time_format %Y-%m-%d %H:%M:%S
+          timezone Asia/Aden
+        </grok>
+      ])
+      d.instance.parse("2019-02-01 12:34:56 1 This is test") do |time, record|
+        assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
+        assert_equal({ "message" => "This is test" }, record)
+      end
+      d.instance.parse("2019-02-01 12:34:56 2 This is test") do |time, record|
+        assert_equal(event_time("2019-02-01 12:34:56 +0300"), time)
+        assert_equal({ "message" => "This is test" }, record)
+      end
+    end
   end
 
   private