Skip to content

Commit a088cee

Browse files
mchromemchrome
authored
feat(config): add 'wildcard-min-distance' config option (#296)
1 parent c608a1c commit a088cee

File tree

8 files changed

+317
-2
lines changed

8 files changed

+317
-2
lines changed

‎config/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,7 @@ type ClickHouse struct {
221221
TagsAdaptiveQueries int `toml:"tags-adaptive-queries" json:"tags-adaptive-queries" comment:"Tags adaptive queries (based on load average) for increase/decrease concurrent queries"`
222222
TagsLimiter limiter.ServerLimiter `toml:"-" json:"-"`
223223

224+
WildcardMinDistance int `toml:"wildcard-min-distance" json:"wildcard-min-distance" comment:"If a wildcard appears both at the start and the end of a plain query at a distance (in terms of nodes) less than wildcard-min-distance, then it will be discarded. This parameter can be used to discard expensive queries."`
224225
TagsMinInQuery int `toml:"tags-min-in-query" json:"tags-min-in-query" comment:"Minimum tags in seriesByTag query"`
225226
TagsMinInAutocomplete int `toml:"tags-min-in-autocomplete" json:"tags-min-in-autocomplete" comment:"Minimum tags in autocomplete query"`
226227

‎doc/config.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,8 @@ Only one tag used as filter for index field Tag1, see graphite_tagged table [str
311311
tags-concurrent-queries = 0
312312
# Tags adaptive queries (based on load average) for increase/decrease concurrent queries
313313
tags-adaptive-queries = 0
314+
# If a wildcard appears both at the start and the end of a plain query at a distance (in terms of nodes) less than wildcard-min-distance, then it will be discarded. This parameter can be used to discard expensive queries.
315+
wildcard-min-distance = 0
314316
# Minimum tags in seriesByTag query
315317
tags-min-in-query = 0
316318
# Minimum tags in autocomplete query

‎finder/index.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,15 +155,22 @@ func (idx *IndexFinder) whereFilter(query string, from int64, until int64) *wher
155155
return w
156156
}
157157

158-
func (idx *IndexFinder) validatePlainQuery(query string) error {
158+
func (idx *IndexFinder) validatePlainQuery(query string, wildcardMinDistance int) error {
159159
if where.HasUnmatchedBrackets(query) {
160160
return errs.NewErrorWithCode("query has unmatched brackets", http.StatusBadRequest)
161161
}
162+
163+
var maxDist = where.MaxWildcardDistance(query)
164+
165+
if maxDist != -1 && maxDist < wildcardMinDistance {
166+
return errs.NewErrorWithCode("query has wildcards way too early at the start and at the end of it", http.StatusBadRequest)
167+
}
168+
162169
return nil
163170
}
164171

165172
func (idx *IndexFinder) Execute(ctx context.Context, config *config.Config, query string, from int64, until int64, stat *FinderStat) (err error) {
166-
err = idx.validatePlainQuery(query)
173+
err = idx.validatePlainQuery(query, config.ClickHouse.WildcardMinDistance)
167174
if err != nil {
168175
return err
169176
}

‎pkg/where/where.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,19 @@ func IndexWildcard(target string) int {
7777
return strings.IndexAny(target, "[]{}*?")
7878
}
7979

80+
func MaxWildcardDistance(query string) int {
81+
if !HasWildcard(query) {
82+
return -1
83+
}
84+
85+
w := IndexWildcard(query)
86+
firstWildcardNode := strings.Count(query[:w], ".")
87+
w = IndexLastWildcard(query)
88+
lastWildcardNode := strings.Count(query[w:], ".")
89+
90+
return max(firstWildcardNode, lastWildcardNode)
91+
}
92+
8093
func NonRegexpPrefix(expr string) string {
8194
s := regexp.QuoteMeta(expr)
8295
for i := 0; i < len(expr); i++ {

‎pkg/where/where_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,3 +72,23 @@ func TestNonRegexpPrefix(t *testing.T) {
7272
assert.Equal(t, test.prefix, prefix, testName)
7373
}
7474
}
75+
76+
func TestMaxWildcardDistance(t *testing.T) {
77+
table := []struct {
78+
glob string
79+
dist int
80+
}{
81+
{`a.b.c.d.e`, -1},
82+
{`test.*.foo.bar`, 2},
83+
{`test.foo.*.*.bar.count`, 2},
84+
{`test.foo.bar.*.bar.foo.test`, 3},
85+
{`test.foo.bar.foobar.*.middle.*.foobar.bar.foo.test`, 4},
86+
{`*.test.foo.bar.*`, 0},
87+
}
88+
89+
for _, test := range table {
90+
testName := fmt.Sprintf("glob: %#v", test.glob)
91+
dist := MaxWildcardDistance(test.glob)
92+
assert.Equal(t, test.dist, dist, testName)
93+
}
94+
}

‎tests/wildcard_min_distance/carbon-clickhouse.conf.tpl‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
[common]
2+
3+
[data]
4+
path = "/etc/carbon-clickhouse/data"
5+
chunk-interval = "1s"
6+
chunk-auto-interval = ""
7+
8+
[upload.graphite_index]
9+
type = "index"
10+
table = "graphite_index"
11+
url = "{{ .CLICKHOUSE_URL }}/"
12+
timeout = "2m30s"
13+
cache-ttl = "1h"
14+
15+
[upload.graphite_tags]
16+
type = "tagged"
17+
table = "graphite_tags"
18+
threads = 3
19+
url = "{{ .CLICKHOUSE_URL }}/"
20+
timeout = "2m30s"
21+
cache-ttl = "1h"
22+
23+
[upload.graphite_reverse]
24+
type = "points-reverse"
25+
table = "graphite_reverse"
26+
url = "{{ .CLICKHOUSE_URL }}/"
27+
timeout = "2m30s"
28+
zero-timestamp = false
29+
30+
[upload.graphite]
31+
type = "points"
32+
table = "graphite"
33+
url = "{{ .CLICKHOUSE_URL }}/"
34+
timeout = "2m30s"
35+
zero-timestamp = false
36+
37+
[tcp]
38+
listen = ":2003"
39+
enabled = true
40+
drop-future = "0s"
41+
drop-past = "0s"
42+
43+
[logging]
44+
file = "/etc/carbon-clickhouse/carbon-clickhouse.log"
45+
level = "debug"

‎tests/wildcard_min_distance/graphite-clickhouse.conf.tpl‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
[common]
2+
listen = "{{ .GCH_ADDR }}"
3+
max-cpu = 0
4+
max-metrics-in-render-answer = 10000
5+
max-metrics-per-target = 10000
6+
headers-to-log = [ "X-Ctx-Carbonapi-Uuid" ]
7+
8+
[clickhouse]
9+
url = "{{ .CLICKHOUSE_URL }}/?max_rows_to_read=500000000&max_result_bytes=1073741824&readonly=2&log_queries=1"
10+
data-timeout = "30s"
11+
12+
wildcard-min-distance = 1
13+
14+
index-table = "graphite_index"
15+
index-use-daily = true
16+
index-timeout = "1m"
17+
internal-aggregation = true
18+
19+
tagged-table = "graphite_tags"
20+
tagged-autocomplete-days = 1
21+
22+
[[data-table]]
23+
# # clickhouse table name
24+
table = "graphite"
25+
# # points in table are stored with reverse path
26+
reverse = false
27+
rollup-conf = "auto"
28+
29+
[[logging]]
30+
logger = ""
31+
file = "{{ .GCH_DIR }}/graphite-clickhouse.log"
32+
level = "info"
33+
encoding = "json"
34+
encoding-time = "iso8601"
35+
encoding-duration = "seconds"

‎tests/wildcard_min_distance/test.toml‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
[test]
2+
precision = "10s"
3+
4+
[[test.clickhouse]]
5+
version = "21.3"
6+
dir = "tests/clickhouse/rollup"
7+
8+
[[test.clickhouse]]
9+
version = "22.8"
10+
dir = "tests/clickhouse/rollup"
11+
12+
[[test.clickhouse]]
13+
version = "24.2"
14+
dir = "tests/clickhouse/rollup"
15+
16+
[test.carbon_clickhouse]
17+
template = "carbon-clickhouse.conf.tpl"
18+
19+
[[test.graphite_clickhouse]]
20+
template = "graphite-clickhouse.conf.tpl"
21+
22+
[[test.input]]
23+
name = "team_one.prod.test.metric_one"
24+
points = [{value = 1.0, time = "rnow-10"}]
25+
26+
[[test.input]]
27+
name = "team_two.stage.test.metric_one"
28+
points = [{value = 1.0, time = "rnow-10"}]
29+
30+
[[test.input]]
31+
name = "team_one.dev.test.metric_two"
32+
points = [{value = 1.0, time = "rnow-10"}]
33+
34+
[[test.input]]
35+
name = "team_one.dev.nontest.metric_one"
36+
points = [{value = 1.0, time = "rnow-10"}]
37+
38+
[[test.render_checks]]
39+
from = "rnow-10"
40+
until = "rnow+1"
41+
timeout = "1h"
42+
targets = [
43+
"team_one.prod.test.metric_one",
44+
]
45+
46+
[[test.render_checks.result]]
47+
name = "team_one.prod.test.metric_one"
48+
path = "team_one.prod.test.metric_one"
49+
consolidation = "avg"
50+
start = "rnow-10"
51+
stop = "rnow+10"
52+
step = 10
53+
req_start = "rnow-10"
54+
req_stop = "rnow+10"
55+
values = [1.0, nan]
56+
57+
58+
[[test.render_checks]]
59+
from = "rnow-10"
60+
until = "rnow+1"
61+
timeout = "1h"
62+
targets = [
63+
"*.dev.test.metric_two",
64+
]
65+
66+
[[test.render_checks.result]]
67+
name = "team_one.dev.test.metric_two"
68+
path = "*.dev.test.metric_two"
69+
consolidation = "avg"
70+
start = "rnow-10"
71+
stop = "rnow+10"
72+
step = 10
73+
req_start = "rnow-10"
74+
req_stop = "rnow+10"
75+
values = [1.0, nan]
76+
77+
[[test.render_checks]]
78+
from = "rnow-10"
79+
until = "rnow+1"
80+
timeout = "1h"
81+
targets = [
82+
"*.*.test.metric_one",
83+
]
84+
85+
[[test.render_checks.result]]
86+
name = "team_one.prod.test.metric_one"
87+
path = "*.*.test.metric_one"
88+
consolidation = "avg"
89+
start = "rnow-10"
90+
stop = "rnow+10"
91+
step = 10
92+
req_start = "rnow-10"
93+
req_stop = "rnow+10"
94+
values = [1.0, nan]
95+
96+
[[test.render_checks.result]]
97+
name = "team_two.stage.test.metric_one"
98+
path = "*.*.test.metric_one"
99+
consolidation = "avg"
100+
start = "rnow-10"
101+
stop = "rnow+10"
102+
step = 10
103+
req_start = "rnow-10"
104+
req_stop = "rnow+10"
105+
values = [1.0, nan]
106+
107+
108+
[[test.render_checks]]
109+
from = "rnow-10"
110+
until = "rnow+1"
111+
timeout = "1h"
112+
targets = [
113+
"team_two.stage.test.*",
114+
]
115+
116+
[[test.render_checks.result]]
117+
name = "team_two.stage.test.metric_one"
118+
path = "team_two.stage.test.*"
119+
consolidation = "avg"
120+
start = "rnow-10"
121+
stop = "rnow+10"
122+
step = 10
123+
req_start = "rnow-10"
124+
req_stop = "rnow+10"
125+
values = [1.0, nan]
126+
127+
[[test.render_checks]]
128+
from = "rnow-10"
129+
until = "rnow+1"
130+
timeout = "1h"
131+
targets = [
132+
"team_one.*.test.*",
133+
]
134+
135+
[[test.render_checks.result]]
136+
name = "team_one.prod.test.metric_one"
137+
path = "team_one.*.test.*"
138+
consolidation = "avg"
139+
start = "rnow-10"
140+
stop = "rnow+10"
141+
step = 10
142+
req_start = "rnow-10"
143+
req_stop = "rnow+10"
144+
values = [1.0, nan]
145+
146+
[[test.render_checks.result]]
147+
name = "team_one.dev.test.metric_two"
148+
path = "team_one.*.test.*"
149+
consolidation = "avg"
150+
start = "rnow-10"
151+
stop = "rnow+10"
152+
step = 10
153+
req_start = "rnow-10"
154+
req_stop = "rnow+10"
155+
values = [1.0, nan]
156+
157+
[[test.render_checks]]
158+
from = "rnow-10"
159+
until = "rnow+1"
160+
timeout = "1h"
161+
targets = [
162+
"*.prod.test.*",
163+
]
164+
error_regexp = "^400: query has wildcards way too early at the start and at the end of it"
165+
166+
[[test.render_checks]]
167+
from = "rnow-10"
168+
until = "rnow+1"
169+
timeout = "1h"
170+
targets = [
171+
"*.*.test.*",
172+
]
173+
error_regexp = "^400: query has wildcards way too early at the start and at the end of it"
174+
175+
[[test.render_checks]]
176+
from = "rnow-10"
177+
until = "rnow+1"
178+
timeout = "1h"
179+
targets = [
180+
"*.*.*.*",
181+
]
182+
error_regexp = "^400: query has wildcards way too early at the start and at the end of it"
183+
184+
185+
[[test.render_checks]]
186+
from = "rnow-10"
187+
until = "rnow+1"
188+
timeout = "1h"
189+
targets = [
190+
"*.*",
191+
]
192+
error_regexp = "^400: query has wildcards way too early at the start and at the end of it"

0 commit comments

Comments
 (0)