From 2fc2e233a68303af8d23df0e95e8e546d20e9123 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 31 Mar 2023 08:47:06 +0000 Subject: [PATCH 1/2] remote-write: raise default samples per send to 2,000 Larger messages cost less, because the per-message overheads at sender and receiver are spread over more samples. Example: scraping 1.5 million series every 15 seconds will send 50 messages per second. Previous default was 500. Signed-off-by: Bryan Boreham --- config/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index a29c98eed..9b0f4ad1e 100644 --- a/config/config.go +++ b/config/config.go @@ -174,10 +174,10 @@ var ( // DefaultQueueConfig is the default remote queue configuration. DefaultQueueConfig = QueueConfig{ // With a maximum of 200 shards, assuming an average of 100ms remote write - // time and 500 samples per batch, we will be able to push 1M samples/s. + // time and 2000 samples per batch, we will be able to push 4M samples/s. MaxShards: 200, MinShards: 1, - MaxSamplesPerSend: 500, + MaxSamplesPerSend: 2000, // Each shard will have a max of 2500 samples pending in its channel, plus the pending // samples that have been enqueued. Theoretically we should only ever have about 3000 samples @@ -194,7 +194,7 @@ var ( DefaultMetadataConfig = MetadataConfig{ Send: true, SendInterval: model.Duration(1 * time.Minute), - MaxSamplesPerSend: 500, + MaxSamplesPerSend: 2000, } // DefaultRemoteReadConfig is the default remote read configuration. From 889ef998f4f73d0d1fbd961e59494584e3616e42 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 2 Apr 2023 11:00:14 +0100 Subject: [PATCH 2/2] remote-write: adjust MaxShards and Capacity for larger MaxSamplesPerSend Keep the target throughput and total pending samples the same. Signed-off-by: Bryan Boreham --- config/config.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config.go b/config/config.go index 9b0f4ad1e..5c51d5a0d 100644 --- a/config/config.go +++ b/config/config.go @@ -173,16 +173,16 @@ var ( // DefaultQueueConfig is the default remote queue configuration. DefaultQueueConfig = QueueConfig{ - // With a maximum of 200 shards, assuming an average of 100ms remote write - // time and 2000 samples per batch, we will be able to push 4M samples/s. - MaxShards: 200, + // With a maximum of 50 shards, assuming an average of 100ms remote write + // time and 2000 samples per batch, we will be able to push 1M samples/s. + MaxShards: 50, MinShards: 1, MaxSamplesPerSend: 2000, - // Each shard will have a max of 2500 samples pending in its channel, plus the pending - // samples that have been enqueued. Theoretically we should only ever have about 3000 samples - // per shard pending. At 200 shards that's 600k. - Capacity: 2500, + // Each shard will have a max of 10,000 samples pending in its channel, plus the pending + // samples that have been enqueued. Theoretically we should only ever have about 12,000 samples + // per shard pending. At 50 shards that's 600k. + Capacity: 10000, BatchSendDeadline: model.Duration(5 * time.Second), // Backoff times for retrying a batch of samples on recoverable errors.