store-gateway: enable lazy loading concurrency by default (grafana#6004)

* store-gateway: enable lazy loading concurrency by default

Lazy loading concurrency has shows good results with slower disks internally at GL, so this PR is enabling it by default.

Signed-off-by: Dimitar Dimitrov <[email protected]>

* Add CHANGELOG.md entry

Signed-off-by: Dimitar Dimitrov <[email protected]>

* Update tests

Signed-off-by: Dimitar Dimitrov <[email protected]>

* Update tests

Signed-off-by: Dimitar Dimitrov <[email protected]>

---------

Signed-off-by: Dimitar Dimitrov <[email protected]>

Author: dimitarvdimitrov

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Grafana Mimir
 
+* [CHANGE] Store-gateway: lazy-loading concurrency limit default value is now 4. #6004
 * [ENHANCEMENT] Query-scheduler: add `cortex_query_scheduler_enqueue_duration_seconds` metric that records the time taken to enqueue or reject a query request. #5879
 * [ENHANCEMENT] Query-frontend: add `cortex_query_frontend_enqueue_duration_seconds` metric that records the time taken to enqueue or reject a query request when not using the query-scheduler. #5879
 * [ENHANCEMENT] Expose `/sync/mutex/wait/total:seconds` Go runtime metric as `go_sync_mutex_wait_total_seconds_total` from all components. #5879

cmd/mimir/config-descriptor.json

@@ -7721,7 +7721,7 @@
                   "required": false,
                   "desc": "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.",
                   "fieldValue": null,
-                  "fieldDefaultValue": 0,
+                  "fieldDefaultValue": 4,
                   "fieldFlag": "blocks-storage.bucket-store.index-header.lazy-loading-concurrency",
                   "fieldType": "int",
                   "fieldCategory": "experimental"

cmd/mimir/help-all.txt.tmpl

@@ -502,7 +502,7 @@ Usage of ./cmd/mimir/mimir:
   -blocks-storage.bucket-store.index-header.eager-loading-startup-enabled
     	[experimental] If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.
   -blocks-storage.bucket-store.index-header.lazy-loading-concurrency int
-    	[experimental] Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.
+    	[experimental] Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited. (default 4)
   -blocks-storage.bucket-store.index-header.lazy-loading-enabled
     	If enabled, store-gateway will lazy load an index-header only once required by a query. (default true)
   -blocks-storage.bucket-store.index-header.lazy-loading-idle-timeout duration

docs/sources/mimir/references/configuration-parameters/index.md

@@ -3492,7 +3492,7 @@ bucket_store:
     # (experimental) Maximum number of concurrent index header loads across all
     # tenants. If set to 0, concurrency is unlimited.
     # CLI flag: -blocks-storage.bucket-store.index-header.lazy-loading-concurrency
-    [lazy_loading_concurrency: <int> | default = 0]
+    [lazy_loading_concurrency: <int> | default = 4]
 
     # (experimental) If enabled, store-gateway will persist a sparse version of
     # the index-header to disk on construction and load sparse index-headers

pkg/storegateway/bucket_stores_test.go

@@ -116,10 +116,12 @@ func TestBucketStores_InitialSync(t *testing.T) {
 			# HELP cortex_bucket_stores_gate_queries_concurrent_max Number of maximum concurrent queries allowed.
 			# TYPE cortex_bucket_stores_gate_queries_concurrent_max gauge
 			cortex_bucket_stores_gate_queries_concurrent_max{gate="query"} 100
+			cortex_bucket_stores_gate_queries_concurrent_max{gate="index_header"} 4
 
 			# HELP cortex_bucket_stores_gate_queries_in_flight Number of queries that are currently in flight.
 			# TYPE cortex_bucket_stores_gate_queries_in_flight gauge
 			cortex_bucket_stores_gate_queries_in_flight{gate="query"} 0
+			cortex_bucket_stores_gate_queries_in_flight{gate="index_header"} 0
 	`),
 		"cortex_bucket_store_blocks_loaded",
 		"cortex_bucket_store_block_loads_total",
@@ -249,10 +251,12 @@ func TestBucketStores_SyncBlocks(t *testing.T) {
 			# HELP cortex_bucket_stores_gate_queries_concurrent_max Number of maximum concurrent queries allowed.
 			# TYPE cortex_bucket_stores_gate_queries_concurrent_max gauge
 			cortex_bucket_stores_gate_queries_concurrent_max{gate="query"} 100
+			cortex_bucket_stores_gate_queries_concurrent_max{gate="index_header"} 4
 
 			# HELP cortex_bucket_stores_gate_queries_in_flight Number of queries that are currently in flight.
 			# TYPE cortex_bucket_stores_gate_queries_in_flight gauge
 			cortex_bucket_stores_gate_queries_in_flight{gate="query"} 0
+			cortex_bucket_stores_gate_queries_in_flight{gate="index_header"} 0
 	`),
 		"cortex_bucket_store_blocks_loaded",
 		"cortex_bucket_store_block_loads_total",

pkg/storegateway/indexheader/header.go

@@ -81,7 +81,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) {
 	f.UintVar(&cfg.MaxIdleFileHandles, prefix+"max-idle-file-handles", 1, "Maximum number of idle file handles the store-gateway keeps open for each index-header file.")
 	f.BoolVar(&cfg.LazyLoadingEnabled, prefix+"lazy-loading-enabled", DefaultIndexHeaderLazyLoadingEnabled, "If enabled, store-gateway will lazy load an index-header only once required by a query.")
 	f.DurationVar(&cfg.LazyLoadingIdleTimeout, prefix+"lazy-loading-idle-timeout", DefaultIndexHeaderLazyLoadingIdleTimeout, "If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity.")
-	f.IntVar(&cfg.LazyLoadingConcurrency, prefix+"lazy-loading-concurrency", 0, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.")
+	f.IntVar(&cfg.LazyLoadingConcurrency, prefix+"lazy-loading-concurrency", 4, "Maximum number of concurrent index header loads across all tenants. If set to 0, concurrency is unlimited.")
 	f.BoolVar(&cfg.EagerLoadingStartupEnabled, prefix+"eager-loading-startup-enabled", false, "If enabled, store-gateway will periodically persist block IDs of lazy loaded index-headers and load them eagerly during startup. It is not valid to enable this if index-header lazy loading is disabled.")
 	f.BoolVar(&cfg.SparsePersistenceEnabled, prefix+"sparse-persistence-enabled", false, "If enabled, store-gateway will persist a sparse version of the index-header to disk on construction and load sparse index-headers from disk instead of the whole index-header.")
 	f.BoolVar(&cfg.VerifyOnLoad, prefix+"verify-on-load", false, "If true, verify the checksum of index headers upon loading them (either on startup or lazily when lazy loading is enabled). Setting to true helps detect disk corruption at the cost of slowing down index header loading.")