From 175efada22fefe2e010cd9348e5859bd51027058 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Tue, 24 Aug 2021 09:01:04 +0200 Subject: [PATCH] Avoid deadlock when processing duplicate series record (#9170) (#8) * Avoid deadlock when processing duplicate series record `processWALSamples()` needs to be able to send on its output channel before it can read the input channel, so reads to allow this in case the output channel is full. Signed-off-by: Bryan Boreham * processWALSamples: update comment Previous text seems to relate to an earlier implementation. Signed-off-by: Bryan Boreham Co-authored-by: Bryan Boreham --- tsdb/head_wal.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tsdb/head_wal.go b/tsdb/head_wal.go index 056bd288a..9b2297080 100644 --- a/tsdb/head_wal.go +++ b/tsdb/head_wal.go @@ -218,9 +218,17 @@ Outer: // It is possible that some old sample is being processed in processWALSamples that // could cause race below. So we wait for the goroutine to empty input the buffer and finish // processing all old samples after emptying the buffer. + select { + case <-outputs[idx]: // allow output side to drain to avoid deadlock + default: + } inputs[idx] <- []record.RefSample{} for len(inputs[idx]) != 0 { time.Sleep(1 * time.Millisecond) + select { + case <-outputs[idx]: // allow output side to drain to avoid deadlock + default: + } } // Checking if the new m-mapped chunks overlap with the already existing ones. @@ -341,9 +349,9 @@ Outer: return nil } -// processWALSamples adds a partition of samples it receives to the head and passes -// them on to other workers. -// Samples before the mint timestamp are discarded. +// processWALSamples adds the samples it receives to the head and passes +// the buffer received to an output channel for reuse. +// Samples before the minValidTime timestamp are discarded. func (h *Head) processWALSamples( minValidTime int64, input <-chan []record.RefSample, output chan<- []record.RefSample,