Skip to content

Commit f2d9f1f

Browse files
okkezokkez
authored
Merge pull request #53 from fluent/grok_name_key
Add grok_name_key
2 parents 7a48854 + ffec54c commit f2d9f1f

File tree

6 files changed

+170
-33
lines changed

6 files changed

+170
-33
lines changed

‎.travis.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ sudo: false
22
language: ruby
33

44
rvm:
5-
- 2.1
6-
- 2.2
7-
- 2.3.3
8-
- 2.4.0
5+
- 2.2.10
6+
- 2.3.7
7+
- 2.4.4
8+
- 2.5.1
99

‎README.md‎

Lines changed: 41 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -95,21 +95,19 @@ You can use this parser without `multiline_start_regexp` when you know your data
9595

9696
## Configurations
9797

98-
**time_format**
98+
* See also: [TimeParameters Plugin Overview](https://docs.fluentd.org/v1.0/articles/timeparameters-plugin-overview)
99+
* See also: [Parser Plugin Overview](https://docs.fluentd.org/v1.0/articles/parser-plugin-overview)
99100

100-
The format of the time field.
101+
* **time_format** (string) (optional): The format of the time field.
102+
* **grok_pattern** (string) (optional): The pattern of grok. You cannot specify multiple grok pattern with this.
103+
* **custom_pattern_path** (string) (optional): Path to the file that includes custom grok patterns
104+
* **grok_failure_key** (string) (optional): The key has grok failure reason.
105+
* **grok_name_key** (string) (optional): The key name to store grok section's name
106+
* **multi_line_start_regexp** (string) (optional): The regexp to match beginning of multiline. This is only for "multiline_grok".
101107

102-
**grok_pattern**
108+
## Examples
103109

104-
The pattern of grok. You cannot specify multiple grok pattern with this.
105-
106-
**custom_pattern_path**
107-
108-
Path to the file that includes custom grok patterns
109-
110-
**grok_failure_key**
111-
112-
The key has grok failure reason. Default is `nil`.
110+
### Using grok\_failure\_key
113111

114112
```aconf
115113
<source>
@@ -149,21 +147,42 @@ This generates following events:
149147
2016-11-28 13:07:09.010400923 +0900 dummy.log: {"message1":"/","prog":"bar","path":"/"}
150148
```
151149

152-
153-
**grok/pattern**
154-
155-
Section for grok patterns. You can use multiple grok patterns with
156-
multiple `<grok>` sections.
150+
### Using grok\_name\_key
157151

158152
```aconf
159-
<grok>
160-
pattern %{IP:ipaddress}
161-
</grok>
153+
<source>
154+
@type tail
155+
path /path/to/log
156+
tag grokked_log
157+
grok_name_key grok_name
158+
grok_failure_key grokfailure
159+
<parse>
160+
@type grok
161+
<grok>
162+
name apache_log
163+
pattern %{COMBINEDAPACHELOG}
164+
time_format "%d/%b/%Y:%H:%M:%S %z"
165+
</grok>
166+
<grok>
167+
name ip_address
168+
pattern %{IP:ip_address}
169+
</grok>
170+
<grok>
171+
name rest_message
172+
pattern %{GREEDYDATA:message}
173+
</grok>
174+
</parse>
175+
</source>
162176
```
163177

164-
**multiline_start_regexp**
178+
This will add keys like following:
179+
180+
* Add `grok_name: "apache_log"` if the record matches `COMBINEDAPACHELOG`
181+
* Add `grok_name: "ip_address"` if the record matches `IP`
182+
* Add `grok_name: "rest_message"` if the record matches `GREEDYDATA`
165183

166-
The regexp to match beginning of multiline. This is only for "multiline_grok".
184+
Add `grokfailure` key to the record if the record does not match any grok pattern.
185+
See also test code for more details.
167186

168187
## How to write Grok patterns
169188

‎lib/fluent/plugin/grok.rb‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class GrokPatternNotFoundError < StandardError
2323

2424
def initialize(plugin, conf)
2525
@pattern_map = {}
26-
@parsers = []
26+
@parsers = {}
2727
@multiline_mode = false
2828
@conf = conf
2929
@plugin = plugin
@@ -45,13 +45,15 @@ def add_patterns_from_file(path)
4545

4646
def setup
4747
if @plugin.grok_pattern
48-
@parsers << expand_pattern_expression(@plugin.grok_pattern, @conf)
48+
@parsers[:grok_pattern] = expand_pattern_expression(@plugin.grok_pattern, @conf)
4949
else
50-
@plugin.grok_confs.each do |grok_conf|
51-
@parsers << expand_pattern_expression(grok_conf.pattern, grok_conf)
50+
@plugin.grok_confs.each.with_index do |grok_conf, index|
51+
@parsers[grok_conf.name || index] = expand_pattern_expression(grok_conf.pattern, grok_conf)
5252
end
5353
end
54-
@parsers.compact!
54+
@parsers.reject! do |key, parser|
55+
parser.nil?
56+
end
5557
if @parsers.empty?
5658
raise Fluent::ConfigError, 'no grok patterns. Check configuration, e.g. typo, configuration syntax, etc'
5759
end

‎lib/fluent/plugin/parser_grok.rb‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,12 @@ class GrokParser < Parser
1414
config_param :custom_pattern_path, :string, default: nil
1515
desc "The key has grok failure reason"
1616
config_param :grok_failure_key, :string, default: nil
17+
desc "The key name to store grok section's name"
18+
config_param :grok_name_key, :string, default: nil
1719

1820
config_section :grok, param_name: "grok_confs", multi: true do
21+
desc "The name of this grok section"
22+
config_param :name, :string, default: nil
1923
desc "The pattern of grok"
2024
config_param :pattern, :string
2125
end
@@ -49,9 +53,10 @@ def configure(conf={})
4953
end
5054

5155
def parse(text)
52-
@grok.parsers.each do |parser|
56+
@grok.parsers.each do |name_or_index, parser|
5357
parser.parse(text) do |time, record|
5458
if time and record
59+
record[@grok_name_key] = name_or_index if @grok_name_key
5560
yield time, record
5661
return
5762
end

‎lib/fluent/plugin/parser_multiline_grok.rb‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@ def firstline?(text)
1717
end
1818

1919
def parse(text)
20-
@grok.parsers.each do |parser|
20+
@grok.parsers.each do |name_or_index, parser|
2121
parser.parse(text) do |time, record|
2222
if time and record
23+
record[@grok_name_key] = name_or_index if @grok_name_key
2324
yield time, record
2425
return
2526
end

‎test/test_grok_parser.rb‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,116 @@ class GrokParserTest < ::Test::Unit::TestCase
191191
assert_equal("unknown value conversion for key:'path', type:'foo'", error_message)
192192
end
193193

194+
sub_test_case "grok_name_key" do
195+
test "one grok section with name" do
196+
d = create_driver(%[
197+
grok_name_key grok_name
198+
<grok>
199+
name path
200+
pattern %{PATH:path}
201+
</grok>
202+
])
203+
expected = {
204+
"path" => "/",
205+
"grok_name" => "path"
206+
}
207+
d.instance.parse("/") do |time, record|
208+
assert_equal(expected, record)
209+
end
210+
end
211+
212+
test "one grok section without name" do
213+
d = create_driver(%[
214+
grok_name_key grok_name
215+
<grok>
216+
pattern %{PATH:path}
217+
</grok>
218+
])
219+
expected = {
220+
"path" => "/",
221+
"grok_name" => 0
222+
}
223+
d.instance.parse("/") do |time, record|
224+
assert_equal(expected, record)
225+
end
226+
end
227+
228+
test "multiple grok sections with name" do
229+
d = create_driver(%[
230+
grok_name_key grok_name
231+
<grok>
232+
name path
233+
pattern %{PATH:path}
234+
</grok>
235+
<grok>
236+
name ip
237+
pattern %{IP:ip_address}
238+
</grok>
239+
])
240+
expected = [
241+
{ "path" => "/", "grok_name" => "path" },
242+
{ "ip_address" => "127.0.0.1", "grok_name" => "ip" },
243+
]
244+
records = []
245+
d.instance.parse("/") do |time, record|
246+
records << record
247+
end
248+
d.instance.parse("127.0.0.1") do |time, record|
249+
records << record
250+
end
251+
assert_equal(expected, records)
252+
end
253+
254+
test "multiple grok sections without name" do
255+
d = create_driver(%[
256+
grok_name_key grok_name
257+
<grok>
258+
pattern %{PATH:path}
259+
</grok>
260+
<grok>
261+
pattern %{IP:ip_address}
262+
</grok>
263+
])
264+
expected = [
265+
{ "path" => "/", "grok_name" => 0 },
266+
{ "ip_address" => "127.0.0.1", "grok_name" => 1 },
267+
]
268+
records = []
269+
d.instance.parse("/") do |time, record|
270+
records << record
271+
end
272+
d.instance.parse("127.0.0.1") do |time, record|
273+
records << record
274+
end
275+
assert_equal(expected, records)
276+
end
277+
278+
test "multiple grok sections with both name and index" do
279+
d = create_driver(%[
280+
grok_name_key grok_name
281+
<grok>
282+
name path
283+
pattern %{PATH:path}
284+
</grok>
285+
<grok>
286+
pattern %{IP:ip_address}
287+
</grok>
288+
])
289+
expected = [
290+
{ "path" => "/", "grok_name" => "path" },
291+
{ "ip_address" => "127.0.0.1", "grok_name" => 1 },
292+
]
293+
records = []
294+
d.instance.parse("/") do |time, record|
295+
records << record
296+
end
297+
d.instance.parse("127.0.0.1") do |time, record|
298+
records << record
299+
end
300+
assert_equal(expected, records)
301+
end
302+
end
303+
194304
private
195305

196306
def create_driver(conf)

0 commit comments

Comments
 (0)