mirror of https://github.com/prometheus/prometheus
Merge pull request #1555 from prometheus/beorn7/cd
Checkpoint fingerprint mappings only upon shutdownpull/1558/head
commit
096a2ef200
|
@ -72,13 +72,20 @@ func newFPMapper(fpToSeries *seriesMap, p *persistence) (*fpMapper, error) {
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkpoint persists the current mappings. The caller has to ensure that the
|
||||||
|
// provided mappings are not changed concurrently. This method is only called
|
||||||
|
// upon shutdown, when no samples are ingested anymore.
|
||||||
|
func (m *fpMapper) checkpoint() error {
|
||||||
|
return m.p.checkpointFPMappings(m.mappings)
|
||||||
|
}
|
||||||
|
|
||||||
// mapFP takes a raw fingerprint (as returned by Metrics.FastFingerprint) and
|
// mapFP takes a raw fingerprint (as returned by Metrics.FastFingerprint) and
|
||||||
// returns a truly unique fingerprint. The caller must have locked the raw
|
// returns a truly unique fingerprint. The caller must have locked the raw
|
||||||
// fingerprint.
|
// fingerprint.
|
||||||
//
|
//
|
||||||
// If an error is encountered, it is returned together with the unchanged raw
|
// If an error is encountered, it is returned together with the unchanged raw
|
||||||
// fingerprint.
|
// fingerprint.
|
||||||
func (m *fpMapper) mapFP(fp model.Fingerprint, metric model.Metric) (model.Fingerprint, error) {
|
func (m *fpMapper) mapFP(fp model.Fingerprint, metric model.Metric) model.Fingerprint {
|
||||||
// First check if we are in the reserved FP space, in which case this is
|
// First check if we are in the reserved FP space, in which case this is
|
||||||
// automatically a collision that has to be mapped.
|
// automatically a collision that has to be mapped.
|
||||||
if fp <= maxMappedFP {
|
if fp <= maxMappedFP {
|
||||||
|
@ -92,7 +99,7 @@ func (m *fpMapper) mapFP(fp model.Fingerprint, metric model.Metric) (model.Finge
|
||||||
// FP exists in memory, but is it for the same metric?
|
// FP exists in memory, but is it for the same metric?
|
||||||
if metric.Equal(s.metric) {
|
if metric.Equal(s.metric) {
|
||||||
// Yupp. We are done.
|
// Yupp. We are done.
|
||||||
return fp, nil
|
return fp
|
||||||
}
|
}
|
||||||
// Collision detected!
|
// Collision detected!
|
||||||
return m.maybeAddMapping(fp, metric)
|
return m.maybeAddMapping(fp, metric)
|
||||||
|
@ -110,28 +117,30 @@ func (m *fpMapper) mapFP(fp model.Fingerprint, metric model.Metric) (model.Finge
|
||||||
mappedFP, ok := mappedFPs[ms]
|
mappedFP, ok := mappedFPs[ms]
|
||||||
if ok {
|
if ok {
|
||||||
// Historical mapping found, return the mapped FP.
|
// Historical mapping found, return the mapped FP.
|
||||||
return mappedFP, nil
|
return mappedFP
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// If we are here, FP does not exist in memory and is either not mapped
|
// If we are here, FP does not exist in memory and is either not mapped
|
||||||
// at all, or existing mappings for FP are not for m. Check if we have
|
// at all, or existing mappings for FP are not for m. Check if we have
|
||||||
// something for FP in the archive.
|
// something for FP in the archive.
|
||||||
archivedMetric, err := m.p.archivedMetric(fp)
|
archivedMetric, err := m.p.archivedMetric(fp)
|
||||||
if err != nil {
|
if err != nil || archivedMetric == nil {
|
||||||
return fp, err
|
// Either the archive lookup has returend an error, or fp does
|
||||||
|
// not exist in the archive. In the former case, the storage has
|
||||||
|
// been marked as dirty already. We just carry on for as long as
|
||||||
|
// it goes, assuming that fp does not exist. In either case,
|
||||||
|
// since now we know (or assume) now that fp does not exist,
|
||||||
|
// neither in memory nor in archive, we can safely keep it
|
||||||
|
// unmapped.
|
||||||
|
return fp
|
||||||
}
|
}
|
||||||
if archivedMetric != nil {
|
// FP exists in archive, but is it for the same metric?
|
||||||
// FP exists in archive, but is it for the same metric?
|
if metric.Equal(archivedMetric) {
|
||||||
if metric.Equal(archivedMetric) {
|
// Yupp. We are done.
|
||||||
// Yupp. We are done.
|
return fp
|
||||||
return fp, nil
|
|
||||||
}
|
|
||||||
// Collision detected!
|
|
||||||
return m.maybeAddMapping(fp, metric)
|
|
||||||
}
|
}
|
||||||
// As fp does not exist, neither in memory nor in archive, we can safely
|
// Collision detected!
|
||||||
// keep it unmapped.
|
return m.maybeAddMapping(fp, metric)
|
||||||
return fp, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// maybeAddMapping is only used internally. It takes a detected collision and
|
// maybeAddMapping is only used internally. It takes a detected collision and
|
||||||
|
@ -140,7 +149,7 @@ func (m *fpMapper) mapFP(fp model.Fingerprint, metric model.Metric) (model.Finge
|
||||||
func (m *fpMapper) maybeAddMapping(
|
func (m *fpMapper) maybeAddMapping(
|
||||||
fp model.Fingerprint,
|
fp model.Fingerprint,
|
||||||
collidingMetric model.Metric,
|
collidingMetric model.Metric,
|
||||||
) (model.Fingerprint, error) {
|
) model.Fingerprint {
|
||||||
ms := metricToUniqueString(collidingMetric)
|
ms := metricToUniqueString(collidingMetric)
|
||||||
m.mtx.RLock()
|
m.mtx.RLock()
|
||||||
mappedFPs, ok := m.mappings[fp]
|
mappedFPs, ok := m.mappings[fp]
|
||||||
|
@ -149,20 +158,16 @@ func (m *fpMapper) maybeAddMapping(
|
||||||
// fp is locked by the caller, so no further locking required.
|
// fp is locked by the caller, so no further locking required.
|
||||||
mappedFP, ok := mappedFPs[ms]
|
mappedFP, ok := mappedFPs[ms]
|
||||||
if ok {
|
if ok {
|
||||||
return mappedFP, nil // Existing mapping.
|
return mappedFP // Existing mapping.
|
||||||
}
|
}
|
||||||
// A new mapping has to be created.
|
// A new mapping has to be created.
|
||||||
mappedFP = m.nextMappedFP()
|
mappedFP = m.nextMappedFP()
|
||||||
mappedFPs[ms] = mappedFP
|
mappedFPs[ms] = mappedFP
|
||||||
m.mtx.Lock()
|
|
||||||
// Checkpoint mappings after each change.
|
|
||||||
err := m.p.checkpointFPMappings(m.mappings)
|
|
||||||
m.mtx.Unlock()
|
|
||||||
log.Infof(
|
log.Infof(
|
||||||
"Collision detected for fingerprint %v, metric %v, mapping to new fingerprint %v.",
|
"Collision detected for fingerprint %v, metric %v, mapping to new fingerprint %v.",
|
||||||
fp, collidingMetric, mappedFP,
|
fp, collidingMetric, mappedFP,
|
||||||
)
|
)
|
||||||
return mappedFP, err
|
return mappedFP
|
||||||
}
|
}
|
||||||
// This is the first collision for fp.
|
// This is the first collision for fp.
|
||||||
mappedFP := m.nextMappedFP()
|
mappedFP := m.nextMappedFP()
|
||||||
|
@ -170,14 +175,12 @@ func (m *fpMapper) maybeAddMapping(
|
||||||
m.mtx.Lock()
|
m.mtx.Lock()
|
||||||
m.mappings[fp] = mappedFPs
|
m.mappings[fp] = mappedFPs
|
||||||
m.mappingsCounter.Inc()
|
m.mappingsCounter.Inc()
|
||||||
// Checkpoint mappings after each change.
|
|
||||||
err := m.p.checkpointFPMappings(m.mappings)
|
|
||||||
m.mtx.Unlock()
|
m.mtx.Unlock()
|
||||||
log.Infof(
|
log.Infof(
|
||||||
"Collision detected for fingerprint %v, metric %v, mapping to new fingerprint %v.",
|
"Collision detected for fingerprint %v, metric %v, mapping to new fingerprint %v.",
|
||||||
fp, collidingMetric, mappedFP,
|
fp, collidingMetric, mappedFP,
|
||||||
)
|
)
|
||||||
return mappedFP, err
|
return mappedFP
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *fpMapper) nextMappedFP() model.Fingerprint {
|
func (m *fpMapper) nextMappedFP() model.Fingerprint {
|
||||||
|
|
|
@ -63,22 +63,13 @@ func TestFPMapper(t *testing.T) {
|
||||||
defer closer.Close()
|
defer closer.Close()
|
||||||
|
|
||||||
mapper, err := newFPMapper(sm, p)
|
mapper, err := newFPMapper(sm, p)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Everything is empty, resolving a FP should do nothing.
|
// Everything is empty, resolving a FP should do nothing.
|
||||||
gotFP, err := mapper.mapFP(fp1, cm11)
|
gotFP := mapper.mapFP(fp1, cm11)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp1; gotFP != wantFP {
|
if wantFP := fp1; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm12)
|
gotFP = mapper.mapFP(fp1, cm12)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp1; gotFP != wantFP {
|
if wantFP := fp1; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
@ -86,184 +77,115 @@ func TestFPMapper(t *testing.T) {
|
||||||
// cm11 is in sm. Adding cm11 should do nothing. Mapping cm12 should resolve
|
// cm11 is in sm. Adding cm11 should do nothing. Mapping cm12 should resolve
|
||||||
// the collision.
|
// the collision.
|
||||||
sm.put(fp1, &memorySeries{metric: cm11})
|
sm.put(fp1, &memorySeries{metric: cm11})
|
||||||
gotFP, err = mapper.mapFP(fp1, cm11)
|
gotFP = mapper.mapFP(fp1, cm11)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp1; gotFP != wantFP {
|
if wantFP := fp1; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm12)
|
gotFP = mapper.mapFP(fp1, cm12)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The mapped cm12 is added to sm, too. That should not change the outcome.
|
// The mapped cm12 is added to sm, too. That should not change the outcome.
|
||||||
sm.put(model.Fingerprint(1), &memorySeries{metric: cm12})
|
sm.put(model.Fingerprint(1), &memorySeries{metric: cm12})
|
||||||
gotFP, err = mapper.mapFP(fp1, cm11)
|
gotFP = mapper.mapFP(fp1, cm11)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp1; gotFP != wantFP {
|
if wantFP := fp1; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm12)
|
gotFP = mapper.mapFP(fp1, cm12)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Now map cm13, should reproducibly result in the next mapped FP.
|
// Now map cm13, should reproducibly result in the next mapped FP.
|
||||||
gotFP, err = mapper.mapFP(fp1, cm13)
|
gotFP = mapper.mapFP(fp1, cm13)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm13)
|
gotFP = mapper.mapFP(fp1, cm13)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add cm13 to sm. Should not change anything.
|
// Add cm13 to sm. Should not change anything.
|
||||||
sm.put(model.Fingerprint(2), &memorySeries{metric: cm13})
|
sm.put(model.Fingerprint(2), &memorySeries{metric: cm13})
|
||||||
gotFP, err = mapper.mapFP(fp1, cm11)
|
gotFP = mapper.mapFP(fp1, cm11)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp1; gotFP != wantFP {
|
if wantFP := fp1; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm12)
|
gotFP = mapper.mapFP(fp1, cm12)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm13)
|
gotFP = mapper.mapFP(fp1, cm13)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Now add cm21 and cm22 in the same way, checking the mapped FPs.
|
// Now add cm21 and cm22 in the same way, checking the mapped FPs.
|
||||||
gotFP, err = mapper.mapFP(fp2, cm21)
|
gotFP = mapper.mapFP(fp2, cm21)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp2; gotFP != wantFP {
|
if wantFP := fp2; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
sm.put(fp2, &memorySeries{metric: cm21})
|
sm.put(fp2, &memorySeries{metric: cm21})
|
||||||
gotFP, err = mapper.mapFP(fp2, cm21)
|
gotFP = mapper.mapFP(fp2, cm21)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp2; gotFP != wantFP {
|
if wantFP := fp2; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp2, cm22)
|
gotFP = mapper.mapFP(fp2, cm22)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
sm.put(model.Fingerprint(3), &memorySeries{metric: cm22})
|
sm.put(model.Fingerprint(3), &memorySeries{metric: cm22})
|
||||||
gotFP, err = mapper.mapFP(fp2, cm21)
|
gotFP = mapper.mapFP(fp2, cm21)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp2; gotFP != wantFP {
|
if wantFP := fp2; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp2, cm22)
|
gotFP = mapper.mapFP(fp2, cm22)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Map cm31, resulting in a mapping straight away.
|
// Map cm31, resulting in a mapping straight away.
|
||||||
gotFP, err = mapper.mapFP(fp3, cm31)
|
gotFP = mapper.mapFP(fp3, cm31)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(4); gotFP != wantFP {
|
if wantFP := model.Fingerprint(4); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
sm.put(model.Fingerprint(4), &memorySeries{metric: cm31})
|
sm.put(model.Fingerprint(4), &memorySeries{metric: cm31})
|
||||||
|
|
||||||
// Map cm32, which is now mapped for two reasons...
|
// Map cm32, which is now mapped for two reasons...
|
||||||
gotFP, err = mapper.mapFP(fp3, cm32)
|
gotFP = mapper.mapFP(fp3, cm32)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(5); gotFP != wantFP {
|
if wantFP := model.Fingerprint(5); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
sm.put(model.Fingerprint(5), &memorySeries{metric: cm32})
|
sm.put(model.Fingerprint(5), &memorySeries{metric: cm32})
|
||||||
|
|
||||||
// Now check ALL the mappings, just to be sure.
|
// Now check ALL the mappings, just to be sure.
|
||||||
gotFP, err = mapper.mapFP(fp1, cm11)
|
gotFP = mapper.mapFP(fp1, cm11)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp1; gotFP != wantFP {
|
if wantFP := fp1; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm12)
|
gotFP = mapper.mapFP(fp1, cm12)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm13)
|
gotFP = mapper.mapFP(fp1, cm13)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp2, cm21)
|
gotFP = mapper.mapFP(fp2, cm21)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp2; gotFP != wantFP {
|
if wantFP := fp2; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp2, cm22)
|
gotFP = mapper.mapFP(fp2, cm22)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp3, cm31)
|
gotFP = mapper.mapFP(fp3, cm31)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(4); gotFP != wantFP {
|
if wantFP := model.Fingerprint(4); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp3, cm32)
|
gotFP = mapper.mapFP(fp3, cm32)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(5); gotFP != wantFP {
|
if wantFP := model.Fingerprint(5); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
@ -273,55 +195,38 @@ func TestFPMapper(t *testing.T) {
|
||||||
sm.del(fp1)
|
sm.del(fp1)
|
||||||
sm.del(fp2)
|
sm.del(fp2)
|
||||||
sm.del(fp3)
|
sm.del(fp3)
|
||||||
gotFP, err = mapper.mapFP(fp1, cm11)
|
gotFP = mapper.mapFP(fp1, cm11)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp1; gotFP != wantFP {
|
if wantFP := fp1; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm12)
|
gotFP = mapper.mapFP(fp1, cm12)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm13)
|
gotFP = mapper.mapFP(fp1, cm13)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp2, cm21)
|
gotFP = mapper.mapFP(fp2, cm21)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp2; gotFP != wantFP {
|
if wantFP := fp2; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp2, cm22)
|
gotFP = mapper.mapFP(fp2, cm22)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp3, cm31)
|
gotFP = mapper.mapFP(fp3, cm31)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(4); gotFP != wantFP {
|
if wantFP := model.Fingerprint(4); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp3, cm32)
|
gotFP = mapper.mapFP(fp3, cm32)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(5); gotFP != wantFP {
|
if wantFP := model.Fingerprint(5); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
err = mapper.checkpoint()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
// Load the mapper anew from disk and then check all the mappings again
|
// Load the mapper anew from disk and then check all the mappings again
|
||||||
// to make sure all changes have made it to disk.
|
// to make sure all changes have made it to disk.
|
||||||
|
@ -329,52 +234,32 @@ func TestFPMapper(t *testing.T) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm11)
|
|
||||||
if err != nil {
|
gotFP = mapper.mapFP(fp1, cm11)
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp1; gotFP != wantFP {
|
if wantFP := fp1; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm12)
|
gotFP = mapper.mapFP(fp1, cm12)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
if wantFP := model.Fingerprint(1); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp1, cm13)
|
gotFP = mapper.mapFP(fp1, cm13)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
if wantFP := model.Fingerprint(2); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp2, cm21)
|
gotFP = mapper.mapFP(fp2, cm21)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp2; gotFP != wantFP {
|
if wantFP := fp2; gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp2, cm22)
|
gotFP = mapper.mapFP(fp2, cm22)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
if wantFP := model.Fingerprint(3); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp3, cm31)
|
gotFP = mapper.mapFP(fp3, cm31)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(4); gotFP != wantFP {
|
if wantFP := model.Fingerprint(4); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp3, cm32)
|
gotFP = mapper.mapFP(fp3, cm32)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(5); gotFP != wantFP {
|
if wantFP := model.Fingerprint(5); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
@ -386,17 +271,11 @@ func TestFPMapper(t *testing.T) {
|
||||||
// the archive).
|
// the archive).
|
||||||
sm.put(fp1, &memorySeries{metric: cm12})
|
sm.put(fp1, &memorySeries{metric: cm12})
|
||||||
p.archiveMetric(fp2, cm22, 0, 0)
|
p.archiveMetric(fp2, cm22, 0, 0)
|
||||||
gotFP, err = mapper.mapFP(fp1, cm12)
|
gotFP = mapper.mapFP(fp1, cm12)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := fp1; gotFP != wantFP { // No mapping happened.
|
if wantFP := fp1; gotFP != wantFP { // No mapping happened.
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
gotFP, err = mapper.mapFP(fp2, cm22)
|
gotFP = mapper.mapFP(fp2, cm22)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(3); gotFP != wantFP { // Old mapping still applied.
|
if wantFP := model.Fingerprint(3); gotFP != wantFP { // Old mapping still applied.
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
@ -405,10 +284,7 @@ func TestFPMapper(t *testing.T) {
|
||||||
// archived metric is detected. Again, this is a pathological situation
|
// archived metric is detected. Again, this is a pathological situation
|
||||||
// that must never happen in real operations. It's just staged here to
|
// that must never happen in real operations. It's just staged here to
|
||||||
// test the expected behavior.
|
// test the expected behavior.
|
||||||
gotFP, err = mapper.mapFP(fp2, cm21)
|
gotFP = mapper.mapFP(fp2, cm21)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if wantFP := model.Fingerprint(6); gotFP != wantFP {
|
if wantFP := model.Fingerprint(6); gotFP != wantFP {
|
||||||
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP)
|
||||||
}
|
}
|
||||||
|
|
|
@ -1327,8 +1327,10 @@ loop:
|
||||||
close(p.indexingStopped)
|
close(p.indexingStopped)
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkpointFPMappings persists the fingerprint mappings. This method is not
|
// checkpointFPMappings persists the fingerprint mappings. The caller has to
|
||||||
// goroutine-safe.
|
// ensure that the provided mappings are not changed concurrently. This method
|
||||||
|
// is only called upon shutdown or during crash recovery, when no samples are
|
||||||
|
// ingested.
|
||||||
//
|
//
|
||||||
// Description of the file format, v1:
|
// Description of the file format, v1:
|
||||||
//
|
//
|
||||||
|
|
|
@ -356,6 +356,9 @@ func (s *memorySeriesStorage) Stop() error {
|
||||||
if err := s.persistence.checkpointSeriesMapAndHeads(s.fpToSeries, s.fpLocker); err != nil {
|
if err := s.persistence.checkpointSeriesMapAndHeads(s.fpToSeries, s.fpLocker); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := s.mapper.checkpoint(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
if err := s.persistence.close(); err != nil {
|
if err := s.persistence.close(); err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -587,14 +590,10 @@ func (s *memorySeriesStorage) Append(sample *model.Sample) error {
|
||||||
}
|
}
|
||||||
rawFP := sample.Metric.FastFingerprint()
|
rawFP := sample.Metric.FastFingerprint()
|
||||||
s.fpLocker.Lock(rawFP)
|
s.fpLocker.Lock(rawFP)
|
||||||
fp, err := s.mapper.mapFP(rawFP, sample.Metric)
|
fp := s.mapper.mapFP(rawFP, sample.Metric)
|
||||||
defer func() {
|
defer func() {
|
||||||
s.fpLocker.Unlock(fp)
|
s.fpLocker.Unlock(fp)
|
||||||
}() // Func wrapper because fp might change below.
|
}() // Func wrapper because fp might change below.
|
||||||
if err != nil {
|
|
||||||
s.persistence.setDirty(fmt.Errorf("error while mapping fingerprint %v: %s", rawFP, err))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if fp != rawFP {
|
if fp != rawFP {
|
||||||
// Switch locks.
|
// Switch locks.
|
||||||
s.fpLocker.Unlock(rawFP)
|
s.fpLocker.Unlock(rawFP)
|
||||||
|
|
|
@ -1683,10 +1683,7 @@ func verifyStorageRandom(t testing.TB, s *memorySeriesStorage, samples model.Sam
|
||||||
result := true
|
result := true
|
||||||
for _, i := range rand.Perm(len(samples)) {
|
for _, i := range rand.Perm(len(samples)) {
|
||||||
sample := samples[i]
|
sample := samples[i]
|
||||||
fp, err := s.mapper.mapFP(sample.Metric.FastFingerprint(), sample.Metric)
|
fp := s.mapper.mapFP(sample.Metric.FastFingerprint(), sample.Metric)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
p := s.NewPreloader()
|
p := s.NewPreloader()
|
||||||
it := p.PreloadInstant(fp, sample.Timestamp, 0)
|
it := p.PreloadInstant(fp, sample.Timestamp, 0)
|
||||||
found := it.ValueAtOrBeforeTime(sample.Timestamp)
|
found := it.ValueAtOrBeforeTime(sample.Timestamp)
|
||||||
|
@ -1726,10 +1723,7 @@ func verifyStorageSequential(t testing.TB, s *memorySeriesStorage, samples model
|
||||||
p.Close()
|
p.Close()
|
||||||
}()
|
}()
|
||||||
for i, sample := range samples {
|
for i, sample := range samples {
|
||||||
newFP, err := s.mapper.mapFP(sample.Metric.FastFingerprint(), sample.Metric)
|
newFP := s.mapper.mapFP(sample.Metric.FastFingerprint(), sample.Metric)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if it == nil || newFP != fp {
|
if it == nil || newFP != fp {
|
||||||
fp = newFP
|
fp = newFP
|
||||||
p.Close()
|
p.Close()
|
||||||
|
@ -1782,10 +1776,7 @@ func TestAppendOutOfOrder(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fp, err := s.mapper.mapFP(m.FastFingerprint(), m)
|
fp := s.mapper.mapFP(m.FastFingerprint(), m)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
pl := s.NewPreloader()
|
pl := s.NewPreloader()
|
||||||
defer pl.Close()
|
defer pl.Close()
|
||||||
|
|
Loading…
Reference in New Issue