Make the number of fingerprint mutexes configurable

With a lot of series accessed in a short timeframe (by a query, a
large scrape, checkpointing, ...), there is actually quite a
significant amount of lock contention if something similar is running
at the same time.

In those cases, the number of locks needs to be increased.

On the same front, as our fingerprints don't have a lot of entropy, I
introduced some additional shuffling. With the current state, anly
changes in the least singificant bits of a FP would matter.

cmd/prometheus/config.go

@@ -162,6 +162,10 @@ func init() {
 		&index.LabelPairFingerprintsCacheSize, "storage.local.index-cache-size.label-pair-to-fingerprints", index.LabelPairFingerprintsCacheSize,
 		"The size in bytes for the label pair to fingerprints index cache.",
 	)
+	cfg.fs.IntVar(
+		&cfg.storage.NumMutexes, "storage.local.num-fingerprint-mutexes", 4096,
+		"The number of mutexes used for fingerprint locking.",
+	)
 
 	// Remote storage.
 	cfg.fs.StringVar(

storage/local/locker.go

@@ -37,8 +37,12 @@ type fingerprintLocker struct {
 	numFpMtxs uint
 }
 
-// newFingerprintLocker returns a new fingerprintLocker ready for use.
+// newFingerprintLocker returns a new fingerprintLocker ready for use.  At least
+// 1024 preallocated mutexes are used, even if preallocatedMutexes is lower.
 func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker {
+	if preallocatedMutexes < 1024 {
+		preallocatedMutexes = 1024
+	}
 	return &fingerprintLocker{
 		make([]sync.Mutex, preallocatedMutexes),
 		uint(preallocatedMutexes),
@@ -47,10 +51,14 @@ func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker {
 
 // Lock locks the given fingerprint.
 func (l *fingerprintLocker) Lock(fp model.Fingerprint) {
-	l.fpMtxs[uint(fp)%l.numFpMtxs].Lock()
+	l.fpMtxs[hashFP(fp)%l.numFpMtxs].Lock()
 }
 
 // Unlock unlocks the given fingerprint.
 func (l *fingerprintLocker) Unlock(fp model.Fingerprint) {
-	l.fpMtxs[uint(fp)%l.numFpMtxs].Unlock()
+	l.fpMtxs[hashFP(fp)%l.numFpMtxs].Unlock()
+}
+
+func hashFP(fp model.Fingerprint) uint {
+	return uint(fp ^ (fp >> 32) ^ (fp >> 16))
 }

storage/local/storage.go

@@ -184,13 +184,14 @@ type MemorySeriesStorageOptions struct {
 	PedanticChecks             bool          // If dirty, perform crash-recovery checks on each series file.
 	SyncStrategy               SyncStrategy  // Which sync strategy to apply to series files.
 	MinShrinkRatio             float64       // Minimum ratio a series file has to shrink during truncation.
+	NumMutexes                 int           // Number of mutexes used for stochastic fingerprint locking.
 }
 
 // NewMemorySeriesStorage returns a newly allocated Storage. Storage.Serve still
 // has to be called to start the storage.
 func NewMemorySeriesStorage(o *MemorySeriesStorageOptions) Storage {
 	s := &memorySeriesStorage{
-		fpLocker: newFingerprintLocker(1024),
+		fpLocker: newFingerprintLocker(o.NumMutexes),
 
 		options: o,