Merge pull request #59 from fluent/support-time-format-parameter-in-grok-section

Support time_format parameter in grok section

Author: okkez

lib/fluent/plugin/grok.rb

@@ -36,6 +36,9 @@ def initialize(plugin, conf)
       if @plugin.respond_to?(:keep_time_key)
         @keep_time_key = @plugin.keep_time_key
       end
+      if @plugin.respond_to?(:time_format)
+        @time_format = @plugin.time_format
+      end
     end
 
     def add_patterns_from_file(path)
@@ -48,10 +51,10 @@ def add_patterns_from_file(path)
 
     def setup
       if @plugin.grok_pattern
-        @parsers[:grok_pattern] = expand_pattern_expression(@plugin.grok_pattern, @conf)
+        @parsers[:grok_pattern] = expand_pattern_expression_grok_pattern(@plugin.grok_pattern, @conf)
       else
         @plugin.grok_confs.each.with_index do |grok_conf, index|
-          @parsers[grok_conf.name || index] = expand_pattern_expression(grok_conf.pattern, grok_conf)
+          @parsers[grok_conf.name || index] = expand_pattern_expression_grok_section(grok_conf)
         end
       end
       @parsers.reject! do |key, parser|
@@ -64,7 +67,7 @@ def setup
 
     private
 
-    def expand_pattern_expression(grok_pattern, conf)
+    def expand_pattern_expression_grok_pattern(grok_pattern, conf)
       regexp, types = expand_pattern(grok_pattern)
       $log.info "Expanded the pattern #{grok_pattern} into #{regexp}"
       _conf = conf.to_h
@@ -83,6 +86,37 @@ def expand_pattern_expression(grok_pattern, conf)
       nil
     end
 
+    def expand_pattern_expression_grok_section(conf)
+      regexp, types = expand_pattern(conf.pattern)
+      $log.info "Expanded the pattern #{conf.pattern} into #{regexp}"
+      _conf = conf.to_h
+      unless types.empty?
+        _conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",")
+      end
+      if conf["multiline"] ||  @multiline_mode
+        _conf["multiline"] = conf["multiline"] ||  @multiline_mode
+      end
+      if conf["keep_time_key"] || @keep_time_key
+        _conf["keep_time_key"] = conf["keep_time_key"] || @keep_time_key
+      end
+      if conf["time_key"]
+        _conf["time_key"] = conf["time_key"]
+      end
+      if conf["time_format"] || @time_format
+        _conf["time_format"] = conf["time_format"] || @time_format
+      end
+      _conf["expression"] = regexp
+      config = Fluent::Config::Element.new("parse", "", _conf, [])
+      parser = Fluent::Plugin::RegexpParser.new
+      parser.configure(config)
+      parser
+    rescue GrokPatternNotFoundError => e
+      raise e
+    rescue => e
+      $log.error(error: e)
+      nil
+    end
+
     def expand_pattern(pattern)
       # It's okay to modify in place. no need to expand it more than once.
       type_map = {}

lib/fluent/plugin/parser_grok.rb

@@ -22,6 +22,9 @@ class GrokParser < Parser
         config_param :name, :string, default: nil
         desc "The pattern of grok"
         config_param :pattern, :string
+        config_param :keep_time_key, :bool, default: false
+        config_param :time_key, :string, default: "time"
+        config_param :time_format, :string, default: nil
       end
 
       def initialize

test/test_grok_parser.rb

@@ -321,6 +321,34 @@ class GrokParserTest < ::Test::Unit::TestCase
     end
   end
 
+  sub_test_case "grok section" do
+    test "complex pattern" do
+      d = create_driver(%[
+        <grok>
+          pattern %{COMBINEDAPACHELOG}
+          time_key timestamp
+          time_format %d/%b/%Y:%H:%M:%S %z
+        </grok>
+      ])
+      expected_record = {
+        "clientip"    => "127.0.0.1",
+        "ident"       => "192.168.0.1",
+        "auth"        => "-",
+        "verb"        => "GET",
+        "request"     => "/",
+        "httpversion" => "1.1",
+        "response"    => "200",
+        "bytes"       => "777",
+        "referrer"    => "\"-\"",
+        "agent"       => "\"Opera/12.0\""
+      }
+      d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record|
+        assert_equal(expected_record, record)
+        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time)
+      end
+    end
+  end
+
   private
 
   def create_driver(conf)