Simplify compaction and expose database sizes.

This commit simplifies the way that compactions across a database's
keyspace occur due to reading the LevelDB internals.  Secondarily it
introduces the database size estimation mechanisms.

storage/metric/leveldb.go

@@ -877,34 +877,57 @@ func (l *LevelDBMetricPersistence) ForEachSample(builder IteratorsForFingerprint
 	panic("not implemented")
 }
 
-// CompactKeyspace compacts each database's keyspace serially.  An error may
+// CompactKeyspace compacts each database's keyspace serially.
-// be returned if there are difficulties with the underlying database or if
-// it's empty.
 //
 // Beware that it would probably be imprudent to run this on a live user-facing
 // server due to latency implications.
-func (l *LevelDBMetricPersistence) CompactKeyspaces() error {
+func (l *LevelDBMetricPersistence) CompactKeyspaces() {
-	if err := l.CurationRemarks.CompactKeyspace(); err != nil {
+	l.CurationRemarks.CompactKeyspace()
-		return fmt.Errorf("Could not compact curation remarks: %s", err)
+	l.fingerprintToMetrics.CompactKeyspace()
+	l.labelNameToFingerprints.CompactKeyspace()
+	l.labelSetToFingerprints.CompactKeyspace()
+	l.MetricHighWatermarks.CompactKeyspace()
+	l.metricMembershipIndex.CompactKeyspace()
+	l.MetricSamples.CompactKeyspace()
+}
+
+func (l *LevelDBMetricPersistence) ApproximateSizes() (total uint64, err error) {
+	size := uint64(0)
+
+	if size, err = l.CurationRemarks.ApproximateSize(); err != nil {
+		return 0, err
 	}
-	if err := l.fingerprintToMetrics.CompactKeyspace(); err != nil {
+	total += size
-		return fmt.Errorf("Could not compact fingerprint to metric index: %s", err)
+
+	if size, err = l.fingerprintToMetrics.ApproximateSize(); err != nil {
+		return 0, err
 	}
-	if err := l.labelNameToFingerprints.CompactKeyspace(); err != nil {
+	total += size
-		return fmt.Errorf("Could not compact label name to fingerprint index: %s", err)
+
+	if size, err = l.labelNameToFingerprints.ApproximateSize(); err != nil {
+		return 0, err
 	}
-	if err := l.labelSetToFingerprints.CompactKeyspace(); err != nil {
+	total += size
-		return fmt.Errorf("Could not compact label pair to fingerprint index: %s", err)
+
+	if size, err = l.labelSetToFingerprints.ApproximateSize(); err != nil {
+		return 0, err
 	}
-	if err := l.MetricHighWatermarks.CompactKeyspace(); err != nil {
+	total += size
-		return fmt.Errorf("Could not compact metric high watermarks: %s", err)
+
+	if size, err = l.MetricHighWatermarks.ApproximateSize(); err != nil {
+		return 0, err
 	}
-	if err := l.metricMembershipIndex.CompactKeyspace(); err != nil {
+	total += size
-		return fmt.Errorf("Could not compact metric membership index: %s", err)
+
+	if size, err = l.metricMembershipIndex.ApproximateSize(); err != nil {
+		return 0, err
 	}
-	if err := l.MetricSamples.CompactKeyspace(); err != nil {
+	total += size
-		return fmt.Errorf("Could not compact metric samples database: %s", err)
+
+	if size, err = l.MetricSamples.ApproximateSize(); err != nil {
+		return 0, err
 	}
+	total += size
 
-	return nil
+	return total, nil
 }

storage/raw/index/leveldb/leveldb.go

@@ -62,12 +62,14 @@ func (l *LevelDBMembershipIndex) Commit(batch raw.Batch) error {
 	return l.persistence.Commit(batch)
 }
 
-// CompactKeyspace compacts the entire database's keyspace.  An error may be
+// CompactKeyspace compacts the entire database's keyspace.
-// returned if there are difficulties with the underlying database or if it's
-// empty.
 //
 // Beware that it would probably be imprudent to run this on a live user-facing
 // server due to latency implications.
-func (l *LevelDBMembershipIndex) CompactKeyspace() error {
+func (l *LevelDBMembershipIndex) CompactKeyspace() {
-	return l.persistence.CompactKeyspace()
+	l.persistence.CompactKeyspace()
+}
+
+func (l *LevelDBMembershipIndex) ApproximateSize() (uint64, error) {
+	return l.persistence.ApproximateSize()
 }

storage/raw/leveldb/leveldb.go

@@ -308,18 +308,27 @@ func (l *LevelDBPersistence) Commit(b raw.Batch) (err error) {
 	return l.storage.Write(l.writeOptions, batch.batch)
 }
 
-// CompactKeyspace compacts the entire database's keyspace.  An error may be
+// CompactKeyspace compacts the entire database's keyspace.
-// returned if there are difficulties with the underlying database or if it's
-// empty.
 //
 // Beware that it would probably be imprudent to run this on a live user-facing
 // server due to latency implications.
-func (l *LevelDBPersistence) CompactKeyspace() error {
+func (l *LevelDBPersistence) CompactKeyspace() {
+
+	// Magic values per https://code.google.com/p/leveldb/source/browse/include/leveldb/db.h#131.
+	keyspace := levigo.Range{
+		Start: nil,
+		Limit: nil,
+	}
+
+	l.storage.CompactRange(keyspace)
+}
+
+func (l *LevelDBPersistence) ApproximateSize() (uint64, error) {
 	iterator := l.NewIterator(false)
 	defer iterator.Close()
 
 	if !iterator.SeekToFirst() {
-		return fmt.Errorf("could not seek to first key")
+		return 0, fmt.Errorf("could not seek to first key")
 	}
 
 	keyspace := levigo.Range{}
@@ -327,14 +336,18 @@ func (l *LevelDBPersistence) CompactKeyspace() error {
 	keyspace.Start = iterator.Key()
 
 	if !iterator.SeekToLast() {
-		return fmt.Errorf("could not seek to last key")
+		return 0, fmt.Errorf("could not seek to last key")
 	}
 
 	keyspace.Limit = iterator.Key()
 
-	l.storage.CompactRange(keyspace)
+	sizes := l.storage.GetApproximateSizes([]levigo.Range{keyspace})
+	total := uint64(0)
+	for _, size := range sizes {
+		total += size
+	}
 
-	return nil
+	return total, nil
 }
 
 // NewIterator creates a new levigoIterator, which follows the Iterator

tools/pruner/main.go

@@ -42,8 +42,10 @@ func main() {
 
 	start := time.Now()
 	log.Printf("Starting compaction...")
-	if err := persistences.CompactKeyspaces(); err != nil {
+	size, _ := persistences.ApproximateSizes()
-		log.Fatalf("Abording after %s", time.Since(start))
+	log.Printf("Original Size: %d", size)
-	}
+	persistences.CompactKeyspaces()
 	log.Printf("Finished in %s", time.Since(start))
+	size, _ = persistences.ApproximateSizes()
+	log.Printf("New Size: %d", size)
 }