Skip to content

Commit e45cbd8

Browse files
alanprotyeya24
alanprot
and
yeya24
authored
Release 1.15.1 (#5291)
* Handle grpc code resource exhausted for store gateway (#5286) * handle grpc code resource exhausted for store gateway Signed-off-by: Ben Ye <benye@amazon.com> * fix lint Signed-off-by: Ben Ye <benye@amazon.com> * update changelog Signed-off-by: Ben Ye <benye@amazon.com> * try fixing test Signed-off-by: Ben Ye <benye@amazon.com> * try to fix E2E test Signed-off-by: Ben Ye <benye@amazon.com> * lint Signed-off-by: Ben Ye <benye@amazon.com> * try again Signed-off-by: Ben Ye <benye@amazon.com> * fix message Signed-off-by: Ben Ye <benye@amazon.com> * remove labels API Signed-off-by: Ben Ye <benye@amazon.com> * remove logic to check string contains Signed-off-by: Ben Ye <benye@amazon.com> * make limiter vars private Signed-off-by: Ben Ye <benye@amazon.com> --------- Signed-off-by: Ben Ye <benye@amazon.com> * Validating new fields on the PagerDuty AM config (#5290) * Not Allowing to set the new service_key_file and routing_key_file on AM config Signed-off-by: Alan Protasio <alanprot@gmail.com> * changelog Signed-off-by: Alan Protasio <alanprot@gmail.com> * fix multiples types Signed-off-by: Alan Protasio <alanprot@gmail.com> --------- Signed-off-by: Alan Protasio <alanprot@gmail.com> --------- Signed-off-by: Ben Ye <benye@amazon.com> Signed-off-by: Alan Protasio <alanprot@gmail.com> Co-authored-by: Ben Ye <benye@amazon.com>
1 parent 92fcee2 commit e45cbd8

File tree

9 files changed

+252
-24
lines changed

9 files changed

+252
-24
lines changed

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22

33
## master / unreleased
44

5+
## 1.15.1 2023-04-26
6+
7+
* [CHANGE] Alertmanager: Validating new fields on the PagerDuty AM config. #5290
8+
* [BUGFIX] Querier: Convert gRPC `ResourceExhausted` status code from store gateway to 422 limit error. #5286
9+
510
## 1.15.0 2023-04-19
611

712
* [CHANGE] Storage: Make Max exemplars config per tenant instead of global configuration. #5080 #5122

‎integration/e2ecortex/client.go‎

Lines changed: 83 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,88 @@ func (c *Client) QueryRangeRaw(query string, start, end time.Time, step time.Dur
150150
}
151151

152152
// QueryRaw runs a query directly against the querier API.
153-
func (c *Client) QueryRaw(query string) (*http.Response, []byte, error) {
154-
addr := fmt.Sprintf("http://%s/api/prom/api/v1/query?query=%s", c.querierAddress, url.QueryEscape(query))
153+
func (c *Client) QueryRaw(query string, ts time.Time) (*http.Response, []byte, error) {
154+
u := &url.URL{
155+
Scheme: "http",
156+
Path: fmt.Sprintf("%s/api/prom/api/v1/query", c.querierAddress),
157+
}
158+
q := u.Query()
159+
q.Set("query", query)
155160

156-
return c.query(addr)
161+
if !ts.IsZero() {
162+
q.Set("time", FormatTime(ts))
163+
}
164+
u.RawQuery = q.Encode()
165+
return c.query(u.String())
166+
}
167+
168+
// SeriesRaw runs a series request directly against the querier API.
169+
func (c *Client) SeriesRaw(matches []string, startTime, endTime time.Time) (*http.Response, []byte, error) {
170+
u := &url.URL{
171+
Scheme: "http",
172+
Path: fmt.Sprintf("%s/api/prom/api/v1/series", c.querierAddress),
173+
}
174+
q := u.Query()
175+
176+
for _, m := range matches {
177+
q.Add("match[]", m)
178+
}
179+
180+
if !startTime.IsZero() {
181+
q.Set("start", FormatTime(startTime))
182+
}
183+
if !endTime.IsZero() {
184+
q.Set("end", FormatTime(endTime))
185+
}
186+
187+
u.RawQuery = q.Encode()
188+
return c.query(u.String())
189+
}
190+
191+
// LabelNamesRaw runs a label names request directly against the querier API.
192+
func (c *Client) LabelNamesRaw(matches []string, startTime, endTime time.Time) (*http.Response, []byte, error) {
193+
u := &url.URL{
194+
Scheme: "http",
195+
Path: fmt.Sprintf("%s/api/prom/api/v1/labels", c.querierAddress),
196+
}
197+
q := u.Query()
198+
199+
for _, m := range matches {
200+
q.Add("match[]", m)
201+
}
202+
203+
if !startTime.IsZero() {
204+
q.Set("start", FormatTime(startTime))
205+
}
206+
if !endTime.IsZero() {
207+
q.Set("end", FormatTime(endTime))
208+
}
209+
210+
u.RawQuery = q.Encode()
211+
return c.query(u.String())
212+
}
213+
214+
// LabelValuesRaw runs a label values request directly against the querier API.
215+
func (c *Client) LabelValuesRaw(label string, matches []string, startTime, endTime time.Time) (*http.Response, []byte, error) {
216+
u := &url.URL{
217+
Scheme: "http",
218+
Path: fmt.Sprintf("%s/api/prom/api/v1/label/%s/values", c.querierAddress, label),
219+
}
220+
q := u.Query()
221+
222+
for _, m := range matches {
223+
q.Add("match[]", m)
224+
}
225+
226+
if !startTime.IsZero() {
227+
q.Set("start", FormatTime(startTime))
228+
}
229+
if !endTime.IsZero() {
230+
q.Set("end", FormatTime(endTime))
231+
}
232+
233+
u.RawQuery = q.Encode()
234+
return c.query(u.String())
157235
}
158236

159237
// RemoteRead runs a remote read query.
@@ -259,8 +337,8 @@ func (c *Client) LabelValues(label string, start, end time.Time, matches []strin
259337
}
260338

261339
// LabelNames gets label names
262-
func (c *Client) LabelNames(start, end time.Time) ([]string, error) {
263-
result, _, err := c.querierClient.LabelNames(context.Background(), nil, start, end)
340+
func (c *Client) LabelNames(start, end time.Time, matchers ...string) ([]string, error) {
341+
result, _, err := c.querierClient.LabelNames(context.Background(), matchers, start, end)
264342
return result, err
265343
}
266344

‎integration/querier_test.go‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package integration
55

66
import (
77
"fmt"
8+
"net/http"
89
"strconv"
910
"strings"
1011
"testing"
@@ -818,6 +819,90 @@ func TestQuerierWithBlocksStorageOnMissingBlocksFromStorage(t *testing.T) {
818819
assert.Contains(t, err.Error(), "500")
819820
}
820821

822+
func TestQuerierWithBlocksStorageLimits(t *testing.T) {
823+
const blockRangePeriod = 5 * time.Second
824+
825+
s, err := e2e.NewScenario(networkName)
826+
require.NoError(t, err)
827+
defer s.Close()
828+
829+
// Configure the blocks storage to frequently compact TSDB head
830+
// and ship blocks to the storage.
831+
flags := mergeFlags(BlocksStorageFlags(), map[string]string{
832+
"-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(),
833+
"-blocks-storage.tsdb.ship-interval": "1s",
834+
"-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(),
835+
})
836+
837+
// Start dependencies.
838+
consul := e2edb.NewConsul()
839+
minio := e2edb.NewMinio(9000, flags["-blocks-storage.s3.bucket-name"])
840+
require.NoError(t, s.StartAndWaitReady(consul, minio))
841+
842+
// Start Cortex components for the write path.
843+
distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "")
844+
ingester := e2ecortex.NewIngester("ingester", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "")
845+
require.NoError(t, s.StartAndWaitReady(distributor, ingester))
846+
847+
// Wait until the distributor has updated the ring.
848+
require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total"))
849+
850+
// Push some series to Cortex.
851+
c, err := e2ecortex.NewClient(distributor.HTTPEndpoint(), "", "", "", "user-1")
852+
require.NoError(t, err)
853+
854+
seriesTimestamp := time.Now()
855+
series2Timestamp := seriesTimestamp.Add(blockRangePeriod * 2)
856+
series1, _ := generateSeries("series_1", seriesTimestamp, prompb.Label{Name: "job", Value: "test"})
857+
series2, _ := generateSeries("series_2", series2Timestamp, prompb.Label{Name: "job", Value: "test"})
858+
859+
res, err := c.Push(series1)
860+
require.NoError(t, err)
861+
require.Equal(t, 200, res.StatusCode)
862+
863+
res, err = c.Push(series2)
864+
require.NoError(t, err)
865+
require.Equal(t, 200, res.StatusCode)
866+
867+
// Wait until the TSDB head is compacted and shipped to the storage.
868+
// The shipped block contains the 1st series, while the 2ns series in in the head.
869+
require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(1), "cortex_ingester_shipper_uploads_total"))
870+
require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(2), "cortex_ingester_memory_series_created_total"))
871+
require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(1), "cortex_ingester_memory_series_removed_total"))
872+
require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(1), "cortex_ingester_memory_series"))
873+
874+
// Start the querier and store-gateway, and configure them to frequently sync blocks fast enough to trigger consistency check.
875+
storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(flags, map[string]string{
876+
"-blocks-storage.bucket-store.sync-interval": "5s",
877+
"-querier.max-fetched-series-per-query": "1",
878+
}), "")
879+
querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(flags, map[string]string{
880+
"-blocks-storage.bucket-store.sync-interval": "5s",
881+
"-querier.max-fetched-series-per-query": "1",
882+
}), "")
883+
require.NoError(t, s.StartAndWaitReady(querier, storeGateway))
884+
885+
// Wait until the querier and store-gateway have updated the ring, and wait until the blocks are old enough for consistency check
886+
require.NoError(t, querier.WaitSumMetrics(e2e.Equals(512*2), "cortex_ring_tokens_total"))
887+
require.NoError(t, storeGateway.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total"))
888+
require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.GreaterOrEqual(4), []string{"cortex_querier_blocks_scan_duration_seconds"}, e2e.WithMetricCount))
889+
890+
// Query back the series.
891+
c, err = e2ecortex.NewClient("", querier.HTTPEndpoint(), "", "", "user-1")
892+
require.NoError(t, err)
893+
894+
// We expect all queries hitting 422 exceeded series limit
895+
resp, body, err := c.QueryRaw(`{job="test"}`, series2Timestamp)
896+
require.NoError(t, err)
897+
require.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode)
898+
require.Contains(t, string(body), "max number of series limit")
899+
900+
resp, body, err = c.SeriesRaw([]string{`{job="test"}`}, series2Timestamp.Add(-time.Hour), series2Timestamp)
901+
require.NoError(t, err)
902+
require.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode)
903+
require.Contains(t, string(body), "max number of series limit")
904+
}
905+
821906
func TestQueryLimitsWithBlocksStorageRunningInMicroServices(t *testing.T) {
822907
const blockRangePeriod = 5 * time.Second
823908

‎integration/query_frontend_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
293293

294294
// No need to repeat the test on missing metric name for each user.
295295
if userID == 0 && cfg.testMissingMetricName {
296-
res, body, err := c.QueryRaw("{instance=~\"hello.*\"}")
296+
res, body, err := c.QueryRaw("{instance=~\"hello.*\"}", time.Now())
297297
require.NoError(t, err)
298298
require.Equal(t, 422, res.StatusCode)
299299
require.Contains(t, string(body), "query must contain metric name")
@@ -317,7 +317,7 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
317317

318318
// No need to repeat the test on Server-Timing header for each user.
319319
if userID == 0 && cfg.queryStatsEnabled {
320-
res, _, err := c.QueryRaw("{instance=~\"hello.*\"}")
320+
res, _, err := c.QueryRaw("{instance=~\"hello.*\"}", time.Now())
321321
require.NoError(t, err)
322322
require.Regexp(t, "querier_wall_time;dur=[0-9.]*, response_time;dur=[0-9.]*$", res.Header.Values("Server-Timing")[0])
323323
}

‎integration/zone_aware_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func TestZoneAwareReplication(t *testing.T) {
135135
require.NoError(t, ingester3.Kill())
136136

137137
// Query back any series => fail (either because of a timeout or 500)
138-
result, _, err := client.QueryRaw("series_1")
138+
result, _, err := client.QueryRaw("series_1", time.Now())
139139
if !errors.Is(err, context.DeadlineExceeded) {
140140
require.NoError(t, err)
141141
require.Equal(t, 500, result.StatusCode)

‎pkg/alertmanager/api.go‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,14 @@ const (
4040
)
4141

4242
var (
43-
errPasswordFileNotAllowed = errors.New("setting password_file, bearer_token_file and credentials_file is not allowed")
44-
errOAuth2SecretFileNotAllowed = errors.New("setting OAuth2 client_secret_file is not allowed")
45-
errTLSFileNotAllowed = errors.New("setting TLS ca_file, cert_file and key_file is not allowed")
46-
errSlackAPIURLFileNotAllowed = errors.New("setting Slack api_url_file and global slack_api_url_file is not allowed")
47-
errVictorOpsAPIKeyFileNotAllowed = errors.New("setting VictorOps api_key_file is not allowed")
48-
errOpsGenieAPIKeyFileNotAllowed = errors.New("setting OpsGenie api_key_file is not allowed")
43+
errPasswordFileNotAllowed = errors.New("setting password_file, bearer_token_file and credentials_file is not allowed")
44+
errOAuth2SecretFileNotAllowed = errors.New("setting OAuth2 client_secret_file is not allowed")
45+
errTLSFileNotAllowed = errors.New("setting TLS ca_file, cert_file and key_file is not allowed")
46+
errSlackAPIURLFileNotAllowed = errors.New("setting Slack api_url_file and global slack_api_url_file is not allowed")
47+
errVictorOpsAPIKeyFileNotAllowed = errors.New("setting VictorOps api_key_file is not allowed")
48+
errOpsGenieAPIKeyFileNotAllowed = errors.New("setting OpsGenie api_key_file is not allowed")
49+
errPagerDutyRoutingKeyFileNotAllowed = errors.New("setting PagerDuty routing_key_file is not allowed")
50+
errPagerDutyServiceKeyFileNotAllowed = errors.New("setting PagerDuty service_key_file is not allowed")
4951
)
5052

5153
// UserConfig is used to communicate a users alertmanager configs
@@ -356,6 +358,11 @@ func validateAlertmanagerConfig(cfg interface{}) error {
356358
if err := validateVictorOpsConfig(v.Interface().(config.VictorOpsConfig)); err != nil {
357359
return err
358360
}
361+
362+
case reflect.TypeOf(config.PagerdutyConfig{}):
363+
if err := validatePagerdutyConfig(v.Interface().(config.PagerdutyConfig)); err != nil {
364+
return err
365+
}
359366
}
360367

361368
// If the input config is a struct, recursively iterate on all fields.
@@ -430,7 +437,7 @@ func validateReceiverTLSConfig(cfg commoncfg.TLSConfig) error {
430437
}
431438

432439
// validateGlobalConfig validates the Global config and returns an error if it contains
433-
// settings now allowed by Cortex.
440+
// settings not allowed by Cortex.
434441
func validateGlobalConfig(cfg config.GlobalConfig) error {
435442
if cfg.OpsGenieAPIKeyFile != "" {
436443
return errOpsGenieAPIKeyFileNotAllowed
@@ -442,7 +449,7 @@ func validateGlobalConfig(cfg config.GlobalConfig) error {
442449
}
443450

444451
// validateOpsGenieConfig validates the OpsGenie config and returns an error if it contains
445-
// settings now allowed by Cortex.
452+
// settings not allowed by Cortex.
446453
func validateOpsGenieConfig(cfg config.OpsGenieConfig) error {
447454
if cfg.APIKeyFile != "" {
448455
return errOpsGenieAPIKeyFileNotAllowed
@@ -451,7 +458,7 @@ func validateOpsGenieConfig(cfg config.OpsGenieConfig) error {
451458
}
452459

453460
// validateSlackConfig validates the Slack config and returns an error if it contains
454-
// settings now allowed by Cortex.
461+
// settings not allowed by Cortex.
455462
func validateSlackConfig(cfg config.SlackConfig) error {
456463
if cfg.APIURLFile != "" {
457464
return errSlackAPIURLFileNotAllowed
@@ -460,10 +467,24 @@ func validateSlackConfig(cfg config.SlackConfig) error {
460467
}
461468

462469
// validateVictorOpsConfig validates the VictorOps config and returns an error if it contains
463-
// settings now allowed by Cortex.
470+
// settings not allowed by Cortex.
464471
func validateVictorOpsConfig(cfg config.VictorOpsConfig) error {
465472
if cfg.APIKeyFile != "" {
466473
return errVictorOpsAPIKeyFileNotAllowed
467474
}
468475
return nil
469476
}
477+
478+
// validatePagerdutyConfig validates the pager duty config and returns an error if it contains
479+
// settings not allowed by Cortex.
480+
func validatePagerdutyConfig(cfg config.PagerdutyConfig) error {
481+
if cfg.RoutingKeyFile != "" {
482+
return errPagerDutyRoutingKeyFileNotAllowed
483+
}
484+
485+
if cfg.ServiceKeyFile != "" {
486+
return errPagerDutyServiceKeyFileNotAllowed
487+
}
488+
489+
return nil
490+
}

‎pkg/alertmanager/api_test.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -563,6 +563,34 @@ template_files:
563563
maxTemplateSize: 20,
564564
err: nil,
565565
},
566+
{
567+
name: "Should return error if PagerDuty routing_key_file is set",
568+
cfg: `
569+
alertmanager_config: |
570+
receivers:
571+
- name: default-receiver
572+
pagerduty_configs:
573+
- routing_key_file: /secrets
574+
575+
route:
576+
receiver: 'default-receiver'
577+
`,
578+
err: errors.Wrap(errPagerDutyRoutingKeyFileNotAllowed, "error validating Alertmanager config"),
579+
},
580+
{
581+
name: "Should return error if PagerDuty service_key_file is set",
582+
cfg: `
583+
alertmanager_config: |
584+
receivers:
585+
- name: default-receiver
586+
pagerduty_configs:
587+
- service_key_file: /secrets
588+
589+
route:
590+
receiver: 'default-receiver'
591+
`,
592+
err: errors.Wrap(errPagerDutyServiceKeyFileNotAllowed, "error validating Alertmanager config"),
593+
},
566594
}
567595

568596
limits := &mockAlertManagerLimits{}

0 commit comments

Comments
 (0)