Add repair test

db.go

@@ -188,7 +188,7 @@ func Open(dir string, l log.Logger, r prometheus.Registerer, opts *Options) (db
 	if opts == nil {
 		opts = DefaultOptions
 	}
-	if err := repairBadIndexVersion(dir); err != nil {
+	if err := repairBadIndexVersion(l, dir); err != nil {
 		return nil, err
 	}
 

repair.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 
 	"github.com/go-kit/kit/log"
@@ -26,6 +27,7 @@ func repairBadIndexVersion(logger log.Logger, dir string) error {
 		if _, err := ulid.Parse(d); err != nil {
 			continue
 		}
+		d = path.Join(dir, d)
 
 		meta, err := readBogusMetaFile(d)
 		if err != nil {

repair_test.go

@@ -1,7 +1,117 @@
 package tsdb
 
-import "testing"
+import (
+	"reflect"
+	"testing"
+
+	"github.com/prometheus/tsdb/chunks"
+
+	"github.com/go-kit/kit/log"
+	"github.com/prometheus/tsdb/index"
+	"github.com/prometheus/tsdb/labels"
+)
 
 func TestRepairBadIndexVersion(t *testing.T) {
+	// The broken index used in this test was written by the following script
+	// at a broken revision.
+	//
+	// func main() {
+	// 	w, err := index.NewWriter("index")
+	// 	if err != nil {
+	// 		panic(err)
+	// 	}
+	// 	err = w.AddSymbols(map[string]struct{}{
+	// 		"a": struct{}{},
+	// 		"b": struct{}{},
+	// 		"1": struct{}{},
+	// 		"2": struct{}{},
+	// 	})
+	// 	if err != nil {
+	// 		panic(err)
+	// 	}
+	// 	err = w.AddSeries(1, labels.FromStrings("a", "1", "b", "1"))
+	// 	if err != nil {
+	// 		panic(err)
+	// 	}
+	// 	err = w.AddSeries(2, labels.FromStrings("a", "2", "b", "1"))
+	// 	if err != nil {
+	// 		panic(err)
+	// 	}
+	// 	err = w.WritePostings("b", "1", index.NewListPostings([]uint64{1, 2}))
+	// 	if err != nil {
+	// 		panic(err)
+	// 	}
+	// 	if err := w.Close(); err != nil {
+	// 		panic(err)
+	// 	}
+	// }
+
+	// In its current state, lookups should fail with the fixed code.
+	const dir = "testdata/repair_index_version/01BZJ9WJQPWHGNC2W4J9TA62KC/"
+	meta, err := readMetaFile(dir)
+	if err == nil {
+		t.Fatal("error expected but got none")
+	}
+
+	r, err := index.NewFileReader(dir + "index")
+	if err != nil {
+		t.Fatal(err)
+	}
+	p, err := r.Postings("b", "1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for p.Next() {
+		t.Logf("next ID %d", p.At())
+
+		var lset labels.Labels
+		if err := r.Series(p.At(), &lset, nil); err == nil {
+			t.Fatal("expected error but got none")
+		}
+	}
+	if p.Err() != nil {
+		t.Fatal(err)
+	}
+
+	if err := repairBadIndexVersion(log.NewNopLogger(), "testdata/repair_index_version"); err != nil {
+		t.Fatal(err)
+	}
+
+	r, err = index.NewFileReader(dir + "index")
+	if err != nil {
+		t.Fatal(err)
+	}
+	p, err = r.Postings("b", "1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	res := []labels.Labels{}
+
+	for p.Next() {
+		t.Logf("next ID %d", p.At())
+
+		var lset labels.Labels
+		var chks []chunks.Meta
+		if err := r.Series(p.At(), &lset, &chks); err != nil {
+			t.Fatal(err)
+		}
+		res = append(res, lset)
+	}
+	if p.Err() != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(res, []labels.Labels{
+		{{"a", "1"}, {"b", "1"}},
+		{{"a", "2"}, {"b", "1"}},
+	}) {
+		t.Fatalf("unexpected result %v", res)
+	}
 
+	meta, err = readMetaFile(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if meta.Version != 1 {
+		t.Fatalf("unexpected meta version %d", meta.Version)
+	}
 }

testdata/repair_index_version/01BZJ9WJQPWHGNC2W4J9TA62KC/meta.json

@@ -0,0 +1,17 @@
+{
+	"version": 2,
+	"ulid": "01BZJ9WJR6Z192734YNMD62F6M",
+	"minTime": 1511366400000,
+	"maxTime": 1511368200000,
+	"stats": {
+		"numSamples": 31897565,
+		"numSeries": 88910,
+		"numChunks": 266093
+	},
+	"compaction": {
+		"level": 1,
+		"sources": [
+			"01BZJ9WJR6Z192734YNMD62F6M"
+		]
+	}
+}