[BUGFIX] TSDB: Fix race on stale values in headAppender (#15322)

* [BUGFIX] TSDB: Fix race on stale values in headAppender

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>

* Simplify

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>

---------

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>

tsdb/head_append.go

@@ -356,21 +356,21 @@ func (a *headAppender) Append(ref storage.SeriesRef, lset labels.Labels, t int64
 		}
 	}
 
+	s.Lock()
 	if value.IsStaleNaN(v) {
-		// This is not thread safe as we should be holding the lock for "s".
 		// TODO(krajorama): reorganize Commit() to handle samples in append order
 		// not floats first and then histograms. Then we could do this conversion
 		// in commit. This code should move into Commit().
 		switch {
 		case s.lastHistogramValue != nil:
+			s.Unlock()
 			return a.AppendHistogram(ref, lset, t, &histogram.Histogram{Sum: v}, nil)
 		case s.lastFloatHistogramValue != nil:
+			s.Unlock()
 			return a.AppendHistogram(ref, lset, t, nil, &histogram.FloatHistogram{Sum: v})
 		}
 	}
 
-	s.Lock()
-
 	defer s.Unlock()
 	// TODO(codesome): If we definitely know at this point that the sample is ooo, then optimise
 	// to skip that sample from the WAL and write only in the WBL.
@@ -1517,7 +1517,7 @@ type chunkOpts struct {
 // append adds the sample (t, v) to the series. The caller also has to provide
 // the appendID for isolation. (The appendID can be zero, which results in no
 // isolation for this append.)
-// It is unsafe to call this concurrently with s.iterator(...) without holding the series lock.
+// Series lock must be held when calling.
 func (s *memSeries) append(t int64, v float64, appendID uint64, o chunkOpts) (sampleInOrder, chunkCreated bool) {
 	c, sampleInOrder, chunkCreated := s.appendPreprocessor(t, chunkenc.EncXOR, o)
 	if !sampleInOrder {