Resource based throttling bug fixes (#7001)

* remove limiting query rejection to only adhoc queries for ingester. This shouldn't have been included as discussed in PR #6947

Signed-off-by: Erlan Zholdubai uulu <erlanz@amazon.com>

* bug fixes for query rejection. fix metric and remove unused enabled flag.

Signed-off-by: Erlan Zholdubai uulu <erlanz@amazon.com>

---------

Signed-off-by: Erlan Zholdubai uulu <erlanz@amazon.com>

Author: erlan-z

docs/blocks-storage/store-gateway.md

@@ -357,12 +357,6 @@ store_gateway:
 
   query_protection:
     rejection:
-      # EXPERIMENTAL: Enable query rejection feature, where the component return
-      # 503 to all incoming query requests when the configured thresholds are
-      # breached.
-      # CLI flag: -store-gateway.query-protection.rejection.enabled
-      [enabled: <boolean> | default = false]
-
       threshold:
         # EXPERIMENTAL: Max CPU utilization that this ingester can reach before
         # rejecting new query request (across all tenants) in percentage,

docs/configuration/config-file-reference.md

@@ -3731,12 +3731,6 @@ instance_limits:
 
 query_protection:
   rejection:
-    # EXPERIMENTAL: Enable query rejection feature, where the component return
-    # 503 to all incoming query requests when the configured thresholds are
-    # breached.
-    # CLI flag: -ingester.query-protection.rejection.enabled
-    [enabled: <boolean> | default = false]
-
     threshold:
       # EXPERIMENTAL: Max CPU utilization that this ingester can reach before
       # rejecting new query request (across all tenants) in percentage, between
@@ -6472,12 +6466,6 @@ sharding_ring:
 
 query_protection:
   rejection:
-    # EXPERIMENTAL: Enable query rejection feature, where the component return
-    # 503 to all incoming query requests when the configured thresholds are
-    # breached.
-    # CLI flag: -store-gateway.query-protection.rejection.enabled
-    [enabled: <boolean> | default = false]
-
     threshold:
       # EXPERIMENTAL: Max CPU utilization that this ingester can reach before
       # rejecting new query request (across all tenants) in percentage, between

pkg/configs/query_protection.go

@@ -14,7 +14,6 @@ type QueryProtection struct {
 }
 
 type rejection struct {
-	Enabled   bool      `yaml:"enabled"`
 	Threshold threshold `yaml:"threshold"`
 }
 
@@ -24,7 +23,6 @@ type threshold struct {
 }
 
 func (cfg *QueryProtection) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
-	f.BoolVar(&cfg.Rejection.Enabled, prefix+"query-protection.rejection.enabled", false, "EXPERIMENTAL: Enable query rejection feature, where the component return 503 to all incoming query requests when the configured thresholds are breached.")
 	f.Float64Var(&cfg.Rejection.Threshold.CPUUtilization, prefix+"query-protection.rejection.threshold.cpu-utilization", 0, "EXPERIMENTAL: Max CPU utilization that this ingester can reach before rejecting new query request (across all tenants) in percentage, between 0 and 1. monitored_resources config must include the resource type. 0 to disable.")
 	f.Float64Var(&cfg.Rejection.Threshold.HeapUtilization, prefix+"query-protection.rejection.threshold.heap-utilization", 0, "EXPERIMENTAL: Max heap utilization that this ingester can reach before rejecting new query request (across all tenants) in percentage, between 0 and 1. monitored_resources config must include the resource type. 0 to disable.")
 }

pkg/frontend/transport/handler.go

@@ -584,7 +584,10 @@ func (f *Handler) reportQueryStats(r *http.Request, source, userID string, query
 			reason = reasonChunksLimitStoreGateway
 		} else if strings.Contains(errMsg, limitBytesStoreGateway) {
 			reason = reasonBytesLimitStoreGateway
-		} else if strings.Contains(errMsg, limiter.ErrResourceLimitReachedStr) {
+		}
+	} else if statusCode == http.StatusServiceUnavailable && error != nil {
+		errMsg := error.Error()
+		if strings.Contains(errMsg, limiter.ErrResourceLimitReachedStr) {
 			reason = reasonResourceExhausted
 		}
 	}

pkg/frontend/transport/handler_test.go

@@ -390,15 +390,15 @@ func TestHandler_ServeHTTP(t *testing.T) {
 			roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
 				resourceLimitReachedErr := &limiter.ResourceLimitReachedError{}
 				return &http.Response{
-					StatusCode: http.StatusUnprocessableEntity,
+					StatusCode: http.StatusServiceUnavailable,
 					Body:       io.NopCloser(strings.NewReader(resourceLimitReachedErr.Error())),
 				}, nil
 			}),
 			additionalMetricsCheckFunc: func(h *Handler) {
 				v := promtest.ToFloat64(h.rejectedQueries.WithLabelValues(reasonResourceExhausted, requestmeta.SourceAPI, userID))
 				assert.Equal(t, float64(1), v)
 			},
-			expectedStatusCode: http.StatusUnprocessableEntity,
+			expectedStatusCode: http.StatusServiceUnavailable,
 		},
 		{
 			name:            "test cortex_slow_queries_total",