Unify chunks and metadata cache backend to bucket cache (#6829)

* unify chunks and metadata cache to bucket cache

Signed-off-by: yeya24 <benye@amazon.com>

* update docs

Signed-off-by: Ben Ye <benye@amazon.com>

* update changelog

Signed-off-by: Ben Ye <benye@amazon.com>

---------

Signed-off-by: yeya24 <benye@amazon.com>
Signed-off-by: Ben Ye <benye@amazon.com>

Author: yeya24

CHANGELOG.md

@@ -46,6 +46,7 @@
 * [ENHANCEMENT] Compactor: Optimize cleaner run time. #6815
 * [ENHANCEMENT] Parquet Storage: Allow percentage based dynamic shard size for Parquet Converter. #6817
 * [ENHANCEMENT] Query Frontend: Enhance the performance of the JSON codec. #6816
+* [ENHANCEMENT] Metadata Cache: Support inmemory and multi level cache backend. #6829
 * [ENHANCEMENT] Store Gateway: Allow to ignore syncing blocks older than certain time using `ignore_blocks_before`. #6830
 * [BUGFIX] Ingester: Avoid error or early throttling when READONLY ingesters are present in the ring #6517
 * [BUGFIX] Ingester: Fix labelset data race condition. #6573

docs/blocks-storage/querier.md

@@ -854,8 +854,8 @@ blocks_storage:
       [backend: <string> | default = ""]
 
       inmemory:
-        # Maximum size in bytes of in-memory chunk cache used to speed up chunk
-        # lookups (shared between all tenants).
+        # Maximum size in bytes of in-memory chunks cache used (shared between
+        # all tenants).
         # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes
         [max_size_bytes: <int> | default = 1073741824]
 
@@ -1095,11 +1095,19 @@ blocks_storage:
       [subrange_ttl: <duration> | default = 24h]
 
     metadata_cache:
-      # Backend for metadata cache, if not empty. Supported values: memcached,
-      # redis, and '' (disable).
+      # The metadata cache backend type. Single or Multiple cache backend can be
+      # provided. Supported values in single cache: memcached, redis, inmemory,
+      # and '' (disable). Supported values in multi level cache: a
+      # comma-separated list of (inmemory, memcached, redis)
       # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend
       [backend: <string> | default = ""]
 
+      inmemory:
+        # Maximum size in bytes of in-memory metadata cache used (shared between
+        # all tenants).
+        # CLI flag: -blocks-storage.bucket-store.metadata-cache.inmemory.max-size-bytes
+        [max_size_bytes: <int> | default = 1073741824]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV
@@ -1301,6 +1309,21 @@ blocks_storage:
           # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent
           [failure_percent: <float> | default = 0.05]
 
+      multilevel:
+        # The maximum number of concurrent asynchronous operations can occur
+        # when backfilling cache items.
+        # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-concurrency
+        [max_async_concurrency: <int> | default = 3]
+
+        # The maximum number of enqueued asynchronous operations allowed when
+        # backfilling cache items.
+        # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-buffer-size
+        [max_async_buffer_size: <int> | default = 10000]
+
+        # The maximum number of items to backfill per asynchronous operation.
+        # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-backfill-items
+        [max_backfill_items: <int> | default = 10000]
+
       # How long to cache list of tenants in the bucket.
       # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl
       [tenants_list_ttl: <duration> | default = 15m]

docs/blocks-storage/store-gateway.md

@@ -975,8 +975,8 @@ blocks_storage:
       [backend: <string> | default = ""]
 
       inmemory:
-        # Maximum size in bytes of in-memory chunk cache used to speed up chunk
-        # lookups (shared between all tenants).
+        # Maximum size in bytes of in-memory chunks cache used (shared between
+        # all tenants).
         # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes
         [max_size_bytes: <int> | default = 1073741824]
 
@@ -1216,11 +1216,19 @@ blocks_storage:
       [subrange_ttl: <duration> | default = 24h]
 
     metadata_cache:
-      # Backend for metadata cache, if not empty. Supported values: memcached,
-      # redis, and '' (disable).
+      # The metadata cache backend type. Single or Multiple cache backend can be
+      # provided. Supported values in single cache: memcached, redis, inmemory,
+      # and '' (disable). Supported values in multi level cache: a
+      # comma-separated list of (inmemory, memcached, redis)
       # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend
       [backend: <string> | default = ""]
 
+      inmemory:
+        # Maximum size in bytes of in-memory metadata cache used (shared between
+        # all tenants).
+        # CLI flag: -blocks-storage.bucket-store.metadata-cache.inmemory.max-size-bytes
+        [max_size_bytes: <int> | default = 1073741824]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV
@@ -1422,6 +1430,21 @@ blocks_storage:
           # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent
           [failure_percent: <float> | default = 0.05]
 
+      multilevel:
+        # The maximum number of concurrent asynchronous operations can occur
+        # when backfilling cache items.
+        # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-concurrency
+        [max_async_concurrency: <int> | default = 3]
+
+        # The maximum number of enqueued asynchronous operations allowed when
+        # backfilling cache items.
+        # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-buffer-size
+        [max_async_buffer_size: <int> | default = 10000]
+
+        # The maximum number of items to backfill per asynchronous operation.
+        # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-backfill-items
+        [max_backfill_items: <int> | default = 10000]
+
       # How long to cache list of tenants in the bucket.
       # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl
       [tenants_list_ttl: <duration> | default = 15m]

docs/configuration/config-file-reference.md

@@ -1450,8 +1450,8 @@ bucket_store:
     [backend: <string> | default = ""]
 
     inmemory:
-      # Maximum size in bytes of in-memory chunk cache used to speed up chunk
-      # lookups (shared between all tenants).
+      # Maximum size in bytes of in-memory chunks cache used (shared between all
+      # tenants).
       # CLI flag: -blocks-storage.bucket-store.chunks-cache.inmemory.max-size-bytes
       [max_size_bytes: <int> | default = 1073741824]
 
@@ -1688,11 +1688,19 @@ bucket_store:
     [subrange_ttl: <duration> | default = 24h]
 
   metadata_cache:
-    # Backend for metadata cache, if not empty. Supported values: memcached,
-    # redis, and '' (disable).
+    # The metadata cache backend type. Single or Multiple cache backend can be
+    # provided. Supported values in single cache: memcached, redis, inmemory,
+    # and '' (disable). Supported values in multi level cache: a comma-separated
+    # list of (inmemory, memcached, redis)
     # CLI flag: -blocks-storage.bucket-store.metadata-cache.backend
     [backend: <string> | default = ""]
 
+    inmemory:
+      # Maximum size in bytes of in-memory metadata cache used (shared between
+      # all tenants).
+      # CLI flag: -blocks-storage.bucket-store.metadata-cache.inmemory.max-size-bytes
+      [max_size_bytes: <int> | default = 1073741824]
+
     memcached:
       # Comma separated list of memcached addresses. Supported prefixes are:
       # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
@@ -1892,6 +1900,21 @@ bucket_store:
         # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-async.circuit-breaker.failure-percent
         [failure_percent: <float> | default = 0.05]
 
+    multilevel:
+      # The maximum number of concurrent asynchronous operations can occur when
+      # backfilling cache items.
+      # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-concurrency
+      [max_async_concurrency: <int> | default = 3]
+
+      # The maximum number of enqueued asynchronous operations allowed when
+      # backfilling cache items.
+      # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-async-buffer-size
+      [max_async_buffer_size: <int> | default = 10000]
+
+      # The maximum number of items to backfill per asynchronous operation.
+      # CLI flag: -blocks-storage.bucket-store.metadata-cache.multilevel.max-backfill-items
+      [max_backfill_items: <int> | default = 10000]
+
     # How long to cache list of tenants in the bucket.
     # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl
     [tenants_list_ttl: <duration> | default = 15m]