diff --git a/tsdb/chunkenc/bstream.go b/tsdb/chunkenc/bstream.go
index 6e13c4046..7b17f4686 100644
--- a/tsdb/chunkenc/bstream.go
+++ b/tsdb/chunkenc/bstream.go
@@ -113,16 +113,6 @@ func (b *bstream) writeBits(u uint64, nbits int) {
 	}
 }
 
-// wrapper for the standard library's PutUvarint to make it work
-// with our bstream.
-func (b *bstream) putUvarint(x uint64) {
-	buf := make([]byte, 2)
-	l := binary.PutUvarint(buf, x)
-	for i := 0; i < l; i++ {
-		b.writeByte(buf[i])
-	}
-}
-
 type bstreamReader struct {
 	stream       []byte
 	streamOffset int // The offset from which read the next byte from the stream.
@@ -267,9 +257,3 @@ func (b *bstreamReader) loadNextBuffer(nbits uint8) bool {
 
 	return true
 }
-
-// wrapper for the standard library's ReadUvarint to make it work
-// with our bstream.
-func (b *bstreamReader) readUvarint() (uint64, error) {
-	return binary.ReadUvarint(b)
-}
diff --git a/tsdb/chunkenc/histogram_meta.go b/tsdb/chunkenc/histogram_meta.go
index 9b65a05ba..ba83dabc4 100644
--- a/tsdb/chunkenc/histogram_meta.go
+++ b/tsdb/chunkenc/histogram_meta.go
@@ -183,29 +183,36 @@ func isWholeWhenMultiplied(in float64) bool {
 	return in == out
 }
 
-// putCustomBound writes the custom bound to the bstream. It stores values from 0 to
-// 16.382 (inclusive) that are multiples of 0.001 in an unsigned var int of up to 2 bytes,
-// but needs 1 bit + 8 bytes for other values like negative numbers, numbers greater than
-// 16.382, or numbers that are not a multiple of 0.001, on the assumption that they are
-// less common. In detail:
+// putCustomBound writes a custom bound to the bstream. It stores values from
+// 0 to 33554.430 (inclusive) that are multiples of 0.001 in unsigned varbit
+// encoding of up to 4 bytes, but needs 1 bit + 8 bytes for other values like
+// negative numbers, numbers greater than 33554.430, or numbers that are not
+// a multiple of 0.001, on the assumption that they are less common. In detail:
 //   - Multiply the bound by 1000, without rounding.
-//   - If the multiplied bound is >= 0, <= 16382 and a whole number, store it as an
-//     unsigned var int.
-//   - Otherwise, store 0 as an unsigned var int, followed by the 8 bytes of the original
+//   - If the multiplied bound is >= 0, <= 33554430 and a whole number,
+//     add 1 and store it in unsigned varbit encoding. All these numbers are
+//     greater than 0, so the leading bit of the varbit is always 1!
+//   - Otherwise, store a 0 bit, followed by the 8 bytes of the original
 //     bound as a float64.
+//
+// When reading the values, we can first decode a value as unsigned varbit,
+// if it's 0, then we read the next 8 bytes as a float64, otherwise
+// we can convert the value to a float64 by subtracting 1 and dividing by 1000.
 func putCustomBound(b *bstream, f float64) {
 	tf := f * 1000
-	if tf < 0 || tf > 16382 || !isWholeWhenMultiplied(f) {
-		b.putUvarint(0)
+	// 33554431-1 comes from the maximum that can be stored in a varint in 4
+	// bytes, other values are stored in 8 bytes anyway.
+	if tf < 0 || tf > 33554430 || !isWholeWhenMultiplied(f) {
+		b.writeBit(zero)
 		b.writeBits(math.Float64bits(f), 64)
 		return
 	}
-	b.putUvarint(uint64(math.Round(tf) + 1))
+	putVarbitUint(b, uint64(math.Round(tf))+1)
 }
 
 // readCustomBound reads the custom bound written with putCustomBound.
 func readCustomBound(br *bstreamReader) (float64, error) {
-	b, err := br.readUvarint()
+	b, err := readVarbitUint(br)
 	if err != nil {
 		return 0, err
 	}
diff --git a/tsdb/chunkenc/histogram_meta_test.go b/tsdb/chunkenc/histogram_meta_test.go
index c3ff4aabc..fdbd1825a 100644
--- a/tsdb/chunkenc/histogram_meta_test.go
+++ b/tsdb/chunkenc/histogram_meta_test.go
@@ -427,7 +427,7 @@ func TestWriteReadHistogramChunkLayout(t *testing.T) {
 			schema:        histogram.CustomBucketsSchema,
 			positiveSpans: []histogram.Span{{Offset: -4, Length: 3}, {Offset: 2, Length: 42}},
 			negativeSpans: nil,
-			customValues:  []float64{-5, -2.5, 0, 0.1, 0.25, 0.5, 1, 2, 5, 10, 25, 50, 100, 255, 500, 1000},
+			customValues:  []float64{-5, -2.5, 0, 0.1, 0.25, 0.5, 1, 2, 5, 10, 25, 50, 100, 255, 500, 1000, 50000, 1e7},
 		},
 		{
 			schema:        histogram.CustomBucketsSchema,