Add Overrides API component and rename old overrides to overrides-configs

docs/api/_index.md

@@ -67,6 +67,9 @@ For the sake of clarity, in this document we have grouped API endpoints by servi
 | [Delete Alertmanager configuration](#delete-alertmanager-configuration) | Alertmanager || `DELETE /api/v1/alerts` |
 | [Tenant delete request](#tenant-delete-request) | Purger || `POST /purger/delete_tenant` |
 | [Tenant delete status](#tenant-delete-status) | Purger || `GET /purger/delete_tenant_status` |
+| [Get user overrides](#get-user-overrides) | Overrides || `GET /api/v1/user-overrides` |
+| [Set user overrides](#set-user-overrides) | Overrides || `POST /api/v1/user-overrides` |
+| [Delete user overrides](#delete-user-overrides) | Overrides || `DELETE /api/v1/user-overrides` |
 | [Store-gateway ring status](#store-gateway-ring-status) | Store-gateway || `GET /store-gateway/ring` |
 | [Compactor ring status](#compactor-ring-status) | Compactor || `GET /compactor/ring` |
 | [Get rule files](#get-rule-files) | Configs API (deprecated) || `GET /api/prom/configs/rules` |
@@ -888,6 +891,64 @@ Returns status of tenant deletion. Output format to be defined. Experimental.
 
 _Requires [authentication](#authentication)._
 
+## Overrides
+
+The Overrides service provides an API for managing user overrides.
+
+### Get user overrides
+
+```
+GET /api/v1/user-overrides
+```
+
+Get the current overrides for the authenticated tenant. Returns the overrides in JSON format.
+
+_Requires [authentication](#authentication)._
+
+### Set user overrides
+
+```
+POST /api/v1/user-overrides
+```
+
+Set or update overrides for the authenticated tenant. The request body should contain a JSON object with the override values.
+
+_Requires [authentication](#authentication)._
+
+### Delete user overrides
+
+```
+DELETE /api/v1/user-overrides
+```
+
+Delete all overrides for the authenticated tenant. This will revert the tenant to using default values.
+
+_Requires [authentication](#authentication)._
+
+#### Example request body for PUT
+
+```json
+{
+  "ingestion_rate": 50000,
+  "max_global_series_per_user": 1000000,
+  "ruler_max_rules_per_rule_group": 100
+}
+```
+
+#### Supported limits
+
+The following limits can be modified via the API:
+- `max_global_series_per_user`
+- `max_global_series_per_metric`
+- `ingestion_rate`
+- `ingestion_burst_size`
+- `ruler_max_rules_per_rule_group`
+- `ruler_max_rule_groups_per_tenant`
+
+#### Hard limits
+
+Overrides are validated against hard limits defined in the runtime configuration file. If a requested override exceeds the hard limit for the tenant, the request will be rejected with a 400 status code.
+
 ## Store-gateway
 
 ### Store-gateway ring status

docs/configuration/v1-guarantees.md

@@ -35,6 +35,8 @@ Cortex is an actively developed project and we want to encourage the introductio
 
 Currently experimental features are:
 
+- Overrides API
+  - Runtime configuration API for managing tenant limits
 - Ruler
   - Evaluate rules to query frontend instead of ingesters (enabled via `-ruler.frontend-address`).
   - When `-ruler.frontend-address` is specified, the response format can be specified (via `-ruler.query-response-format`).

docs/proposals/user-overrides-api.md

@@ -40,7 +40,7 @@ Response format:
 }
 ```
 
-#### 2. PUT /api/v1/user-overrides
+#### 2. POST /api/v1/user-overrides
 Updates overrides for a specific tenant. The request body should contain only the overrides that need to be updated.
 
 Request body:

integration/overrides_test.go

@@ -0,0 +1,276 @@
+//go:build integration
+// +build integration
+
+package integration
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/thanos-io/objstore/providers/s3"
+	"gopkg.in/yaml.v3"
+
+	"github.com/cortexproject/cortex/integration/e2e"
+	e2edb "github.com/cortexproject/cortex/integration/e2e/db"
+	"github.com/cortexproject/cortex/integration/e2ecortex"
+)
+
+func TestOverridesAPIWithRunningCortex(t *testing.T) {
+	s, err := e2e.NewScenario(networkName)
+	require.NoError(t, err)
+	defer s.Close()
+
+	minio := e2edb.NewMinio(9000, "cortex")
+	require.NoError(t, s.StartAndWaitReady(minio))
+
+	runtimeConfig := map[string]interface{}{
+		"overrides": map[string]interface{}{
+			"user1": map[string]interface{}{
+				"ingestion_rate": 5000,
+			},
+		},
+		"api_allowed_limits": []string{
+			"ingestion_rate",
+			"max_global_series_per_user",
+			"max_global_series_per_metric",
+			"ingestion_burst_size",
+			"ruler_max_rules_per_rule_group",
+			"ruler_max_rule_groups_per_tenant",
+		},
+	}
+	runtimeConfigData, err := yaml.Marshal(runtimeConfig)
+	require.NoError(t, err)
+
+	s3Client, err := s3.NewBucketWithConfig(nil, s3.Config{
+		Endpoint:  minio.HTTPEndpoint(),
+		Insecure:  true,
+		Bucket:    "cortex",
+		AccessKey: e2edb.MinioAccessKey,
+		SecretKey: e2edb.MinioSecretKey,
+	}, "overrides-test", nil)
+	require.NoError(t, err)
+
+	require.NoError(t, s3Client.Upload(context.Background(), "runtime.yaml", bytes.NewReader(runtimeConfigData)))
+
+	flags := map[string]string{
+		"-target": "overrides",
+
+		"-runtime-config.file":                 "runtime.yaml",
+		"-runtime-config.backend":              "s3",
+		"-runtime-config.s3.access-key-id":     e2edb.MinioAccessKey,
+		"-runtime-config.s3.secret-access-key": e2edb.MinioSecretKey,
+		"-runtime-config.s3.bucket-name":       "cortex",
+		"-runtime-config.s3.endpoint":          minio.NetworkHTTPEndpoint(),
+		"-runtime-config.s3.insecure":          "true",
+	}
+
+	cortexSvc := e2ecortex.NewSingleBinary("cortex-overrides", flags, "")
+	require.NoError(t, s.StartAndWaitReady(cortexSvc))
+
+	t.Run("GET overrides for existing user", func(t *testing.T) {
+		req, err := http.NewRequest("GET", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", nil)
+		require.NoError(t, err)
+		req.Header.Set("X-Scope-OrgID", "user1")
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		var overrides map[string]interface{}
+		err = json.NewDecoder(resp.Body).Decode(&overrides)
+		require.NoError(t, err)
+
+		assert.Equal(t, float64(5000), overrides["ingestion_rate"])
+	})
+
+	t.Run("GET overrides for non-existing user", func(t *testing.T) {
+		req, err := http.NewRequest("GET", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", nil)
+		require.NoError(t, err)
+		req.Header.Set("X-Scope-OrgID", "user2")
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		var overrides map[string]interface{}
+		err = json.NewDecoder(resp.Body).Decode(&overrides)
+		require.NoError(t, err)
+
+		assert.Empty(t, overrides)
+	})
+
+	t.Run("POST overrides for new user", func(t *testing.T) {
+		newOverrides := map[string]interface{}{
+			"ingestion_rate":       6000,
+			"ingestion_burst_size": 7000,
+		}
+		requestBody, err := json.Marshal(newOverrides)
+		require.NoError(t, err)
+
+		req, err := http.NewRequest("POST", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", bytes.NewReader(requestBody))
+		require.NoError(t, err)
+		req.Header.Set("X-Scope-OrgID", "user3")
+		req.Header.Set("Content-Type", "application/json")
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		req, err = http.NewRequest("GET", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", nil)
+		require.NoError(t, err)
+		req.Header.Set("X-Scope-OrgID", "user3")
+
+		resp, err = http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		var savedOverrides map[string]interface{}
+		err = json.NewDecoder(resp.Body).Decode(&savedOverrides)
+		require.NoError(t, err)
+
+		assert.Equal(t, float64(6000), savedOverrides["ingestion_rate"])
+		assert.Equal(t, float64(7000), savedOverrides["ingestion_burst_size"])
+	})
+
+	t.Run("POST overrides with invalid limit", func(t *testing.T) {
+		invalidOverrides := map[string]interface{}{
+			"invalid_limit": 5000,
+		}
+		requestBody, err := json.Marshal(invalidOverrides)
+		require.NoError(t, err)
+
+		req, err := http.NewRequest("POST", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", bytes.NewReader(requestBody))
+		require.NoError(t, err)
+		req.Header.Set("X-Scope-OrgID", "user4")
+		req.Header.Set("Content-Type", "application/json")
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	})
+
+	t.Run("POST overrides with invalid JSON", func(t *testing.T) {
+		req, err := http.NewRequest("POST", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", bytes.NewReader([]byte("invalid json")))
+		require.NoError(t, err)
+		req.Header.Set("X-Scope-OrgID", "user5")
+		req.Header.Set("Content-Type", "application/json")
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	})
+
+	t.Run("DELETE overrides", func(t *testing.T) {
+		req, err := http.NewRequest("DELETE", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", nil)
+		require.NoError(t, err)
+		req.Header.Set("X-Scope-OrgID", "user1")
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		req, err = http.NewRequest("GET", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", nil)
+		require.NoError(t, err)
+		req.Header.Set("X-Scope-OrgID", "user1")
+
+		resp, err = http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		var overrides map[string]interface{}
+		err = json.NewDecoder(resp.Body).Decode(&overrides)
+		require.NoError(t, err)
+
+		assert.Empty(t, overrides)
+	})
+
+	require.NoError(t, s.Stop(cortexSvc))
+}
+
+func TestOverridesAPITenantExtraction(t *testing.T) {
+	s, err := e2e.NewScenario(networkName)
+	require.NoError(t, err)
+	defer s.Close()
+
+	minio := e2edb.NewMinio(9010, "cortex")
+	require.NoError(t, s.StartAndWaitReady(minio))
+
+	// Upload an empty runtime config file to S3
+	runtimeConfig := map[string]interface{}{
+		"overrides": map[string]interface{}{},
+	}
+	runtimeConfigData, err := yaml.Marshal(runtimeConfig)
+	require.NoError(t, err)
+
+	s3Client, err := s3.NewBucketWithConfig(nil, s3.Config{
+		Endpoint:  minio.HTTPEndpoint(),
+		Insecure:  true,
+		Bucket:    "cortex",
+		AccessKey: e2edb.MinioAccessKey,
+		SecretKey: e2edb.MinioSecretKey,
+	}, "overrides-test-tenant", nil)
+	require.NoError(t, err)
+
+	require.NoError(t, s3Client.Upload(context.Background(), "runtime.yaml", bytes.NewReader(runtimeConfigData)))
+
+	flags := map[string]string{
+		"-target": "overrides",
+
+		"-runtime-config.file":                 "runtime.yaml",
+		"-runtime-config.backend":              "s3",
+		"-runtime-config.s3.access-key-id":     e2edb.MinioAccessKey,
+		"-runtime-config.s3.secret-access-key": e2edb.MinioSecretKey,
+		"-runtime-config.s3.bucket-name":       "cortex",
+		"-runtime-config.s3.endpoint":          minio.NetworkHTTPEndpoint(),
+		"-runtime-config.s3.insecure":          "true",
+	}
+
+	cortexSvc := e2ecortex.NewSingleBinary("cortex-overrides-tenant", flags, "")
+	require.NoError(t, s.StartAndWaitReady(cortexSvc))
+
+	t.Run("no tenant header", func(t *testing.T) {
+		req, err := http.NewRequest("GET", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", nil)
+		require.NoError(t, err)
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+	})
+
+	t.Run("empty tenant header", func(t *testing.T) {
+		req, err := http.NewRequest("GET", "http://"+cortexSvc.HTTPEndpoint()+"/api/v1/user-overrides", nil)
+		require.NoError(t, err)
+		req.Header.Set("X-Scope-OrgID", "")
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)
+	})
+
+	require.NoError(t, s.Stop(cortexSvc))
+}

pkg/api/api.go

@@ -30,6 +30,7 @@ import (
 	frontendv2 "github.com/cortexproject/cortex/pkg/frontend/v2"
 	"github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb"
 	"github.com/cortexproject/cortex/pkg/ingester/client"
+	"github.com/cortexproject/cortex/pkg/overrides"
 	"github.com/cortexproject/cortex/pkg/purger"
 	"github.com/cortexproject/cortex/pkg/querier"
 	"github.com/cortexproject/cortex/pkg/ring"
@@ -385,6 +386,17 @@ func (a *API) RegisterRulerAPI(r *ruler.API) {
 	a.RegisterRoute(path.Join(a.cfg.LegacyHTTPPrefix, "/rules/{namespace}"), http.HandlerFunc(r.DeleteNamespace), true, "DELETE")
 }
 
+// RegisterOverrides registers routes associated with the Overrides API
+func (a *API) RegisterOverrides(o *overrides.API) {
+	// Register individual overrides API routes with the main API
+	a.RegisterRoute("/api/v1/user-overrides", http.HandlerFunc(o.GetOverrides), true, "GET")
+	a.RegisterRoute("/api/v1/user-overrides", http.HandlerFunc(o.SetOverrides), true, "POST")
+	a.RegisterRoute("/api/v1/user-overrides", http.HandlerFunc(o.DeleteOverrides), true, "DELETE")
+
+	// Add link to the index page
+	a.indexPage.AddLink(SectionAdminEndpoints, "/api/v1/user-overrides", "User Overrides API")
+}
+
 // RegisterRing registers the ring UI page associated with the distributor for writes.
 func (a *API) RegisterRing(r *ring.Ring) {
 	a.indexPage.AddLink(SectionAdminEndpoints, "/ingester/ring", "Ingester Ring Status")