github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Introduce min_shards for remote write to set minimum number of shards. (#4924) 

Signed-off-by: Ryota Arai <ryota.arai@gmail.com>
pull/4962/head
Ryota Arai 2018-12-05 02:32:14 +09:00 committed by Tom Wilkie
parent e1d9bf77f1
commit 135d580ab2
3 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/config.go
View File

@ -111,6 +111,7 @@ var (
// With a maximum of 1000 shards, assuming an average of 100ms remote write // With a maximum of 1000 shards, assuming an average of 100ms remote write
// time and 100 samples per batch, we will be able to push 1M samples/s. // time and 100 samples per batch, we will be able to push 1M samples/s.
MaxShards: 1000, MaxShards: 1000,
MinShards: 1,
MaxSamplesPerSend: 100, MaxSamplesPerSend: 100,
// By default, buffer 100 batches, which at 100ms per batch is 10s. At // By default, buffer 100 batches, which at 100ms per batch is 10s. At
@ -706,6 +707,9 @@ type QueueConfig struct {
// Max number of shards, i.e. amount of concurrency. // Max number of shards, i.e. amount of concurrency.
MaxShards int `yaml:"max_shards,omitempty"` MaxShards int `yaml:"max_shards,omitempty"`
// Min number of shards, i.e. amount of concurrency.
MinShards int `yaml:"min_shards,omitempty"`
// Maximum number of samples per send. // Maximum number of samples per send.
MaxSamplesPerSend int `yaml:"max_samples_per_send,omitempty"` MaxSamplesPerSend int `yaml:"max_samples_per_send,omitempty"`

2
docs/configuration/configuration.md
View File

@ -1244,6 +1244,8 @@ queue_config:
[ capacity: <int> | default = 10000 ] [ capacity: <int> | default = 10000 ]
# Maximum number of shards, i.e. amount of concurrency. # Maximum number of shards, i.e. amount of concurrency.
[ max_shards: <int> | default = 1000 ] [ max_shards: <int> | default = 1000 ]
# Minimum number of shards, i.e. amount of concurrency.
[ min_shards: <int> | default = 1 ]
# Maximum number of samples per send. # Maximum number of samples per send.
[ max_samples_per_send: <int> | default = 100] [ max_samples_per_send: <int> | default = 100]
# Maximum time a sample will wait in buffer. # Maximum time a sample will wait in buffer.

6
storage/remote/queue_manager.go
View File

@ -177,7 +177,7 @@ func NewQueueManager(logger log.Logger, cfg config.QueueConfig, externalLabels m
queueName: client.Name(), queueName: client.Name(),
logLimiter: rate.NewLimiter(logRateLimit, logBurst), logLimiter: rate.NewLimiter(logRateLimit, logBurst),
numShards: 1, numShards: cfg.MinShards,
reshardChan: make(chan int), reshardChan: make(chan int),
quit: make(chan struct{}), quit: make(chan struct{}),
@ -326,8 +326,8 @@ func (t *QueueManager) calculateDesiredShards() {
numShards := int(math.Ceil(desiredShards)) numShards := int(math.Ceil(desiredShards))
if numShards > t.cfg.MaxShards { if numShards > t.cfg.MaxShards {
numShards = t.cfg.MaxShards numShards = t.cfg.MaxShards
} else if numShards < 1 { } else if numShards < t.cfg.MinShards {
numShards = 1 numShards = t.cfg.MinShards
} }
if numShards == t.numShards { if numShards == t.numShards {
return return