From 42bdf921d1725ea8a362b1223b3d183c11947693 Mon Sep 17 00:00:00 2001 From: Julius Volz Date: Mon, 25 Mar 2013 13:04:47 +0100 Subject: [PATCH] Fetch integrated memory/disk data for simple Get* functions. --- storage/metric/tiered.go | 54 +++++++++++-- storage/metric/tiered_test.go | 138 ++++++++++++++++++++++++++++++---- 2 files changed, 169 insertions(+), 23 deletions(-) diff --git a/storage/metric/tiered.go b/storage/metric/tiered.go index 98e0fa263..28517f572 100644 --- a/storage/metric/tiered.go +++ b/storage/metric/tiered.go @@ -518,17 +518,55 @@ func (t *tieredStorage) loadChunkAroundTime(iterator leveldb.Iterator, frontier return } -func (t *tieredStorage) GetAllMetricNames() ([]string, error) { - // TODO: handle memory persistence as well. - return t.diskStorage.GetAllMetricNames() +func (t *tieredStorage) GetAllMetricNames() (metrics []string, err error) { + diskMetrics, err := t.diskStorage.GetAllMetricNames() + if err != nil { + return + } + memoryMetrics, err := t.memoryArena.GetAllMetricNames() + if err != nil { + return + } + + metricSet := map[string]bool{} + for _, metricName := range append(diskMetrics, memoryMetrics...) { + metricSet[metricName] = true + } + for metricName := range metricSet { + metrics = append(metrics, metricName) + } + sort.Strings(metrics) + + return } -func (t *tieredStorage) GetFingerprintsForLabelSet(labelSet model.LabelSet) (model.Fingerprints, error) { - // TODO: handle memory persistence as well. - return t.diskStorage.GetFingerprintsForLabelSet(labelSet) +func (t *tieredStorage) GetFingerprintsForLabelSet(labelSet model.LabelSet) (fingerprints model.Fingerprints, err error) { + memFingerprints, err := t.memoryArena.GetFingerprintsForLabelSet(labelSet) + if err != nil { + return + } + diskFingerprints, err := t.memoryArena.GetFingerprintsForLabelSet(labelSet) + if err != nil { + return + } + fingerprintSet := map[model.Fingerprint]bool{} + for _, fingerprint := range append(memFingerprints, diskFingerprints...) { + fingerprintSet[fingerprint] = true + } + for fingerprint := range fingerprintSet { + fingerprints = append(fingerprints, fingerprint) + } + + return } func (t *tieredStorage) GetMetricForFingerprint(f model.Fingerprint) (m *model.Metric, err error) { - // TODO: handle memory persistence as well. - return t.diskStorage.GetMetricForFingerprint(f) + m, err = t.memoryArena.GetMetricForFingerprint(f) + if err != nil { + return + } + if m == nil { + m, err = t.diskStorage.GetMetricForFingerprint(f) + } + return } diff --git a/storage/metric/tiered_test.go b/storage/metric/tiered_test.go index 5e6c7572f..e5a8c36d1 100644 --- a/storage/metric/tiered_test.go +++ b/storage/metric/tiered_test.go @@ -23,6 +23,32 @@ import ( "time" ) +type testTieredStorageCloser struct { + storage Storage + dirName string +} + +func (t *testTieredStorageCloser) Close() { + t.storage.Close() + os.RemoveAll(t.dirName) +} + +func newTestTieredStorage(t test.Tester) (storage Storage, closer *testTieredStorageCloser) { + tempDir, _ := ioutil.TempDir("", "test_tiered_storage") + storage = NewTieredStorage(5000000, 2500, 1000, 5*time.Second, 15*time.Second, 0*time.Second, tempDir) + + if storage == nil { + t.Fatalf("%d. storage == nil") + } + + go storage.Serve() + closer = &testTieredStorageCloser{ + storage: storage, + dirName: tempDir, + } + return +} + func buildSamples(from, to time.Time, interval time.Duration, m model.Metric) (v []model.Sample) { i := model.SampleValue(0) @@ -340,21 +366,8 @@ func testMakeView(t test.Tester) { ) for i, scenario := range scenarios { - var ( - temporary, _ = ioutil.TempDir("", "test_make_view") - tiered = NewTieredStorage(5000000, 2500, 1000, 5*time.Second, 15*time.Second, 0*time.Second, temporary) - ) - - if tiered == nil { - t.Fatalf("%d. tiered == nil", i) - } - - go tiered.Serve() - defer tiered.Drain() - - defer func() { - os.RemoveAll(temporary) - }() + tiered, closer := newTestTieredStorage(t) + defer closer.Close() for j, datum := range scenario.data { err := tiered.AppendSample(datum) @@ -417,3 +430,98 @@ func BenchmarkMakeView(b *testing.B) { testMakeView(b) } } + +func TestGetAllMetricNames(t *testing.T) { + type in struct { + metricName string + appendToMemory bool + appendToDisk bool + } + + scenarios := []struct { + in []in + out []string + }{ + { + // Empty case. + }, { + in: []in{ + { + metricName: "request_count", + appendToMemory: false, + appendToDisk: true, + }, + }, + out: []string{ + "request_count", + }, + }, { + in: []in{ + { + metricName: "request_count", + appendToMemory: true, + appendToDisk: false, + }, + { + metricName: "start_time", + appendToMemory: false, + appendToDisk: true, + }, + }, + out: []string{ + "request_count", + "start_time", + }, + }, { + in: []in{ + { + metricName: "request_count", + appendToMemory: true, + appendToDisk: true, + }, + { + metricName: "start_time", + appendToMemory: true, + appendToDisk: true, + }, + }, + out: []string{ + "request_count", + "start_time", + }, + }, + } + + for i, scenario := range scenarios { + tiered, closer := newTestTieredStorage(t) + defer closer.Close() + for j, metric := range scenario.in { + sample := model.Sample{ + Metric: model.Metric{"name": model.LabelValue(metric.metricName)}, + } + if metric.appendToMemory { + if err := tiered.(*tieredStorage).memoryArena.AppendSample(sample); err != nil { + t.Fatalf("%d.%d. failed to add fixture data: %s", i, j, err) + } + } + if metric.appendToDisk { + if err := tiered.(*tieredStorage).diskStorage.AppendSample(sample); err != nil { + t.Fatalf("%d.%d. failed to add fixture data: %s", i, j, err) + } + } + } + metricNames, err := tiered.GetAllMetricNames() + if err != nil { + t.Fatalf("%d. Error getting metric names: %s", i, err) + } + if len(metricNames) != len(scenario.out) { + t.Fatalf("%d. Expected metric count %d, got %d", i, len(scenario.out), len(metricNames)) + } + + for j, expected := range scenario.out { + if expected != metricNames[j] { + t.Fatalf("%d.%d. Expected metric %s, got %s", i, j, expected, metricNames[j]) + } + } + } +}