|
|
|
@ -16,6 +16,7 @@ package remote
|
|
|
|
|
import ( |
|
|
|
|
"context" |
|
|
|
|
"errors" |
|
|
|
|
"fmt" |
|
|
|
|
"math" |
|
|
|
|
"strconv" |
|
|
|
|
"sync" |
|
|
|
@ -1224,12 +1225,16 @@ func (s *shards) stop() {
|
|
|
|
|
// Force an unclean shutdown.
|
|
|
|
|
s.hardShutdown() |
|
|
|
|
<-s.done |
|
|
|
|
if dropped := s.samplesDroppedOnHardShutdown.Load(); dropped > 0 { |
|
|
|
|
level.Error(s.qm.logger).Log("msg", "Failed to flush all samples on shutdown", "count", dropped) |
|
|
|
|
|
|
|
|
|
// Log error for any dropped samples, exemplars, or histograms.
|
|
|
|
|
logDroppedError := func(t string, counter atomic.Uint32) { |
|
|
|
|
if dropped := counter.Load(); dropped > 0 { |
|
|
|
|
level.Error(s.qm.logger).Log("msg", fmt.Sprintf("Failed to flush all %s on shutdown", t), "count", dropped) |
|
|
|
|
} |
|
|
|
|
if dropped := s.exemplarsDroppedOnHardShutdown.Load(); dropped > 0 { |
|
|
|
|
level.Error(s.qm.logger).Log("msg", "Failed to flush all exemplars on shutdown", "count", dropped) |
|
|
|
|
} |
|
|
|
|
logDroppedError("samples", s.samplesDroppedOnHardShutdown) |
|
|
|
|
logDroppedError("exemplars", s.exemplarsDroppedOnHardShutdown) |
|
|
|
|
logDroppedError("histograms", s.histogramsDroppedOnHardShutdown) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// enqueue data (sample or exemplar). If the shard is full, shutting down, or
|
|
|
|
@ -1537,7 +1542,7 @@ func (s *shards) sendSamples(ctx context.Context, samples []prompb.TimeSeries, s
|
|
|
|
|
begin := time.Now() |
|
|
|
|
err := s.sendSamplesWithBackoff(ctx, samples, sampleCount, exemplarCount, histogramCount, pBuf, buf) |
|
|
|
|
if err != nil { |
|
|
|
|
level.Error(s.qm.logger).Log("msg", "non-recoverable error", "count", sampleCount, "exemplarCount", exemplarCount, "err", err) |
|
|
|
|
level.Error(s.qm.logger).Log("msg", "non-recoverable error", "count", sampleCount, "exemplarCount", exemplarCount, "histogramCount", histogramCount, "err", err) |
|
|
|
|
s.qm.metrics.failedSamplesTotal.Add(float64(sampleCount)) |
|
|
|
|
s.qm.metrics.failedExemplarsTotal.Add(float64(exemplarCount)) |
|
|
|
|
s.qm.metrics.failedHistogramsTotal.Add(float64(histogramCount)) |
|
|
|
|