Merge pull request #59 from prometheus/feature/snappy

Include Snappy in Runtime.

Makefile

@@ -11,7 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-TEST_ARTIFACTS = prometheus search_index
+TEST_ARTIFACTS = prometheus prometheus.build search_index
 
 all: test
 
@@ -21,6 +21,7 @@ test: build
 build:
 	$(MAKE) -C model
 	go build ./...
+	go build -o prometheus.build
 
 clean:
 	rm -rf $(TEST_ARTIFACTS)

Makefile.TRAVIS

@@ -14,6 +14,7 @@
 GO_VERSION := 1.0.3
 LEVELDB_VERSION := 1.7.0
 PROTOCOL_BUFFERS_VERSION := 2.4.1
+SNAPPY_VERSION := 1.0.5
 
 OVERLAY_ROOT := ${HOME}/overlay_root
 
@@ -24,7 +25,7 @@ export CFLAGS := $(CFLAGS) -I$(OVERLAY_ROOT)/include
 export CXXFLAGS := $(CXXFLAGS) -I$(OVERLAY_ROOT)/include
 export CPPFLAGS := $(CPPFLAGS) -I$(OVERLAY_ROOT)/include
 export LDFLAGS := $(LDFLAGS) -L$(OVERLAY_ROOT)/lib
-export CGO_CFLAGS := $(CFLAGS)
+export CGO_CFLAGS := $(CFLAGS) -lsnappy
 export CGO_LDFLAGS := $(LDFLAGS)
 
 GO_GET := go get -u -v -x
@@ -114,7 +115,7 @@ instrumentation-stamp: go source
 
 leveldb: leveldb-stamp
 
-leveldb-stamp: cc rsync leveldb-$(LEVELDB_VERSION).tar.gz overlay
+leveldb-stamp: cc rsync leveldb-$(LEVELDB_VERSION).tar.gz snappy overlay
 	tar xzvf leveldb-$(LEVELDB_VERSION).tar.gz
 	$(MAKE) -C leveldb-$(LEVELDB_VERSION)
 	rsync -av "leveldb-$(LEVELDB_VERSION)/include/" "$(OVERLAY_ROOT)/include/"
@@ -128,7 +129,7 @@ leveldb-$(LEVELDB_VERSION).tar.gz: wget
 
 levigo: levigo-stamp
 
-levigo-stamp: leveldb go source
+levigo-stamp: leveldb go snappy source
 	$(GO_GET) github.com/jmhodges/levigo
 	touch $@
 
@@ -141,9 +142,23 @@ test: test-stamp
 
 test-stamp: preparation source
 	cd ${GOPATH}/src/github.com/prometheus
+	$(MAKE) build
 	$(MAKE) test
 	touch $@
 
+snappy-$(SNAPPY_VERSION).tar.gz: wget
+	$(WGET) http://snappy.googlecode.com/files/snappy-$(SNAPPY_VERSION).tar.gz
+	tar xzvf snappy-$(SNAPPY_VERSION).tar.gz
+	cd snappy-$(SNAPPY_VERSION) && ./configure --prefix="$(OVERLAY_ROOT)"
+	$(MAKE) -C snappy-$(SNAPPY_VERSION)
+	$(MAKE) -C snappy-$(SNAPPY_VERSION) install
+
+snappy: snappy-stamp
+	tar xzvf snappy-$(SNAPPY_VERSION).tar.gz
+
+snappy-stamp: cc overlay rsync snappy-$(SNAPPY_VERSION).tar.gz
+	touch $@
+
 source: source-stamp
 
 source-stamp:
@@ -156,8 +171,10 @@ clean:
 	-rm -rf "$(OVERLAY_ROOT)"
 	-rm -rf leveldb-$(LEVELDB_VERSION)
 	-rm -rf protobuf-$(PROTOCOL_BUFFERS_VERSION)
+	-rm -rf snappy-$(SNAPPY_VERSION)
 	-rm leveldb-$(LEVELDB_VERSION).tar.gz
 	-rm protobuf-$(PROTOCOL_BUFFERS_VERSION).tar.bz2
+	-rm snappy-$(SNAPPY_VERSION).tar.gz
 
 
-.PHONY: all bison build-dependencies cc clean go goprotobuf gorest instrumentation leveldb levigo mercurial overlay preparation protoc rsync source test wget
+.PHONY: all bison build-dependencies cc clean go goprotobuf gorest instrumentation leveldb levigo mercurial overlay preparation protoc rsync snappy source test wget

README.md

@@ -22,6 +22,7 @@ action if some condition is observed to be true.
   5. Levigo, a Go-wrapper around LevelDB's C library: (https://github.com/jmhodges/levigo).
   6. GoRest, a RESTful style web-services framework: (http://code.google.com/p/gorest/).
   7. Prometheus Client, Prometheus in Prometheus (https://github.com/matttproud/golang_instrumentation).
+  8. Snappy, a compression library for LevelDB and Levigo (code.google.com/p/snappy/).
 
 ## Getting started
 

storage/raw/leveldb/leveldb.go

@@ -22,7 +22,9 @@ import (
 )
 
 var (
-	leveldbFlushOnMutate = flag.Bool("leveldbFlushOnMutate", true, "Whether LevelDB should flush every operation to disk upon mutation before returning (bool).")
+	leveldbFlushOnMutate     = flag.Bool("leveldbFlushOnMutate", true, "Whether LevelDB should flush every operation to disk upon mutation before returning (bool).")
+	leveldbUseSnappy         = flag.Bool("leveldbUseSnappy", true, "Whether LevelDB attempts to use Snappy for compressing elements (bool).")
+	leveldbUseParanoidChecks = flag.Bool("leveldbUseParanoidChecks", true, "Whether LevelDB uses expensive checks (bool).")
 )
 
 type LevelDBPersistence struct {
@@ -44,7 +46,12 @@ type iteratorCloser struct {
 func NewLevelDBPersistence(storageRoot string, cacheCapacity, bitsPerBloomFilterEncoded int) (p *LevelDBPersistence, err error) {
 	options := levigo.NewOptions()
 	options.SetCreateIfMissing(true)
-	options.SetParanoidChecks(true)
+	options.SetParanoidChecks(*leveldbUseParanoidChecks)
+	compression := levigo.NoCompression
+	if *leveldbUseSnappy {
+		compression = levigo.SnappyCompression
+	}
+	options.SetCompression(compression)
 
 	cache := levigo.NewLRUCache(cacheCapacity)
 	options.SetCache(cache)