2017-04-10 18:59:45 +00:00
|
|
|
// Copyright 2017 The Prometheus Authors
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2017-05-02 23:32:09 +00:00
|
|
|
// The code in this file was largely written by Damian Gryski as part of
|
|
|
|
// https://github.com/dgryski/go-tsz and published under the license below.
|
2018-04-08 09:28:30 +00:00
|
|
|
// It was modified to accommodate reading from byte slices without modifying
|
2019-10-10 09:47:30 +00:00
|
|
|
// the underlying bytes, which would panic when reading from mmap'd
|
2017-05-02 23:32:09 +00:00
|
|
|
// read-only byte slices.
|
2017-04-28 09:34:35 +00:00
|
|
|
|
|
|
|
// Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
|
|
|
|
// All rights reserved.
|
|
|
|
|
|
|
|
// Redistribution and use in source and binary forms, with or without
|
|
|
|
// modification, are permitted provided that the following conditions are met:
|
|
|
|
|
|
|
|
// * Redistributions of source code must retain the above copyright notice,
|
|
|
|
// this list of conditions and the following disclaimer.
|
|
|
|
//
|
|
|
|
// * Redistributions in binary form must reproduce the above copyright notice,
|
|
|
|
// this list of conditions and the following disclaimer in the documentation
|
|
|
|
// and/or other materials provided with the distribution.
|
|
|
|
//
|
|
|
|
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
|
|
|
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
|
|
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
|
|
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
|
|
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
|
|
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
|
|
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
|
|
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
2017-11-30 14:34:49 +00:00
|
|
|
package chunkenc
|
2016-11-15 09:33:34 +00:00
|
|
|
|
|
|
|
import (
|
2016-11-29 21:02:58 +00:00
|
|
|
"encoding/binary"
|
2016-11-15 09:33:34 +00:00
|
|
|
"math"
|
2017-11-08 22:18:33 +00:00
|
|
|
"math/bits"
|
2021-06-29 08:37:41 +00:00
|
|
|
|
Style cleanup of all the changes in sparsehistogram so far
A lot of this code was hacked together, literally during a
hackathon. This commit intends not to change the code substantially,
but just make the code obey the usual style practices.
A (possibly incomplete) list of areas:
* Generally address linter warnings.
* The `pgk` directory is deprecated as per dev-summit. No new packages should
be added to it. I moved the new `pkg/histogram` package to `model`
anticipating what's proposed in #9478.
* Make the naming of the Sparse Histogram more consistent. Including
abbreviations, there were just too many names for it: SparseHistogram,
Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in
general. Only add "Sparse" if it is needed to avoid confusion with
conventional Histograms (which is rare because the TSDB really has no notion
of conventional Histograms). Use abbreviations only in local scope, and then
really abbreviate (not just removing three out of seven letters like in
"Histo"). This is in the spirit of
https://github.com/golang/go/wiki/CodeReviewComments#variable-names
* Several other minor name changes.
* A lot of formatting of doc comments. For one, following
https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences
, but also layout question, anticipating how things will look like
when rendered by `godoc` (even where `godoc` doesn't render them
right now because they are for unexported types or not a doc comment
at all but just a normal code comment - consistency is queen!).
* Re-enabled `TestQueryLog` and `TestEndopints` (they pass now,
leaving them disabled was presumably an oversight).
* Bucket iterator for histogram.Histogram is now created with a
method.
* HistogramChunk.iterator now allows iterator recycling. (I think
@dieterbe only commented it out because he was confused by the
question in the comment.)
* HistogramAppender.Append panics now because we decided to treat
staleness marker differently.
Signed-off-by: beorn7 <beorn@grafana.com>
2021-10-09 13:57:07 +00:00
|
|
|
"github.com/prometheus/prometheus/model/histogram"
|
2016-11-15 09:33:34 +00:00
|
|
|
)
|
|
|
|
|
2020-01-24 07:44:52 +00:00
|
|
|
const (
|
|
|
|
chunkCompactCapacityThreshold = 32
|
|
|
|
)
|
|
|
|
|
2016-11-15 09:33:34 +00:00
|
|
|
// XORChunk holds XOR encoded sample data.
|
|
|
|
type XORChunk struct {
|
2019-02-13 22:41:12 +00:00
|
|
|
b bstream
|
2016-11-15 09:33:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewXORChunk returns a new chunk with XOR encoding of the given size.
|
2016-12-31 09:10:27 +00:00
|
|
|
func NewXORChunk() *XORChunk {
|
2016-12-09 19:45:46 +00:00
|
|
|
b := make([]byte, 2, 128)
|
2019-02-13 22:41:12 +00:00
|
|
|
return &XORChunk{b: bstream{stream: b, count: 0}}
|
2016-11-20 15:14:21 +00:00
|
|
|
}
|
|
|
|
|
2024-01-17 17:28:06 +00:00
|
|
|
func (c *XORChunk) Reset(stream []byte) {
|
|
|
|
c.b.Reset(stream)
|
|
|
|
}
|
|
|
|
|
2017-03-19 16:05:01 +00:00
|
|
|
// Encoding returns the encoding type.
|
2016-12-09 19:45:46 +00:00
|
|
|
func (c *XORChunk) Encoding() Encoding {
|
|
|
|
return EncXOR
|
|
|
|
}
|
|
|
|
|
2016-11-29 21:02:58 +00:00
|
|
|
// Bytes returns the underlying byte slice of the chunk.
|
|
|
|
func (c *XORChunk) Bytes() []byte {
|
2017-01-02 09:34:55 +00:00
|
|
|
return c.b.bytes()
|
2016-11-15 09:33:34 +00:00
|
|
|
}
|
|
|
|
|
2017-07-12 16:31:26 +00:00
|
|
|
// NumSamples returns the number of samples in the chunk.
|
|
|
|
func (c *XORChunk) NumSamples() int {
|
|
|
|
return int(binary.BigEndian.Uint16(c.Bytes()))
|
|
|
|
}
|
|
|
|
|
2021-07-05 13:27:46 +00:00
|
|
|
// Compact implements the Chunk interface.
|
2020-01-24 07:44:52 +00:00
|
|
|
func (c *XORChunk) Compact() {
|
|
|
|
if l := len(c.b.stream); cap(c.b.stream) > l+chunkCompactCapacityThreshold {
|
|
|
|
buf := make([]byte, l)
|
|
|
|
copy(buf, c.b.stream)
|
|
|
|
c.b.stream = buf
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-11-15 09:33:34 +00:00
|
|
|
// Appender implements the Chunk interface.
|
2022-09-27 14:02:05 +00:00
|
|
|
// It is not valid to call Appender() multiple times concurrently or to use multiple
|
|
|
|
// Appenders on the same chunk.
|
2016-11-29 21:02:58 +00:00
|
|
|
func (c *XORChunk) Appender() (Appender, error) {
|
2019-07-09 09:49:34 +00:00
|
|
|
it := c.iterator(nil)
|
2016-11-29 21:02:58 +00:00
|
|
|
|
|
|
|
// To get an appender we must know the state it would have if we had
|
|
|
|
// appended all existing data from scratch.
|
|
|
|
// We iterate through the end and populate via the iterator's state.
|
2023-10-31 11:35:13 +00:00
|
|
|
for it.Next() != ValNone {
|
2016-11-29 21:02:58 +00:00
|
|
|
}
|
|
|
|
if err := it.Err(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2016-11-30 22:01:01 +00:00
|
|
|
a := &xorAppender{
|
2019-02-13 22:41:12 +00:00
|
|
|
b: &c.b,
|
2016-11-29 21:02:58 +00:00
|
|
|
t: it.t,
|
|
|
|
v: it.val,
|
|
|
|
tDelta: it.tDelta,
|
|
|
|
leading: it.leading,
|
|
|
|
trailing: it.trailing,
|
2016-11-30 22:01:01 +00:00
|
|
|
}
|
2021-10-15 18:33:14 +00:00
|
|
|
if it.numTotal == 0 {
|
2016-11-30 22:01:01 +00:00
|
|
|
a.leading = 0xff
|
|
|
|
}
|
|
|
|
return a, nil
|
2016-11-29 21:02:58 +00:00
|
|
|
}
|
|
|
|
|
2019-07-09 09:49:34 +00:00
|
|
|
func (c *XORChunk) iterator(it Iterator) *xorIterator {
|
|
|
|
if xorIter, ok := it.(*xorIterator); ok {
|
|
|
|
xorIter.Reset(c.b.bytes())
|
|
|
|
return xorIter
|
|
|
|
}
|
2016-11-29 21:02:58 +00:00
|
|
|
return &xorIterator{
|
2019-07-09 09:49:34 +00:00
|
|
|
// The first 2 bytes contain chunk headers.
|
|
|
|
// We skip that for actual samples.
|
2016-12-09 19:45:46 +00:00
|
|
|
br: newBReader(c.b.bytes()[2:]),
|
2017-01-02 09:34:55 +00:00
|
|
|
numTotal: binary.BigEndian.Uint16(c.b.bytes()),
|
2020-03-24 20:15:47 +00:00
|
|
|
t: math.MinInt64,
|
2016-11-29 21:02:58 +00:00
|
|
|
}
|
2016-11-15 09:33:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Iterator implements the Chunk interface.
|
2022-09-27 14:02:05 +00:00
|
|
|
// Iterator() must not be called concurrently with any modifications to the chunk,
|
|
|
|
// but after it returns you can use an Iterator concurrently with an Appender or
|
|
|
|
// other Iterators.
|
2019-07-09 09:49:34 +00:00
|
|
|
func (c *XORChunk) Iterator(it Iterator) Iterator {
|
|
|
|
return c.iterator(it)
|
2016-11-15 09:33:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type xorAppender struct {
|
2016-11-29 21:02:58 +00:00
|
|
|
b *bstream
|
2016-11-15 09:33:34 +00:00
|
|
|
|
2016-11-20 15:14:21 +00:00
|
|
|
t int64
|
|
|
|
v float64
|
|
|
|
tDelta uint64
|
2016-11-20 13:33:00 +00:00
|
|
|
|
|
|
|
leading uint8
|
|
|
|
trailing uint8
|
2016-11-15 09:33:34 +00:00
|
|
|
}
|
|
|
|
|
2016-12-31 09:10:27 +00:00
|
|
|
func (a *xorAppender) Append(t int64, v float64) {
|
2016-11-20 15:14:21 +00:00
|
|
|
var tDelta uint64
|
2017-01-02 09:34:55 +00:00
|
|
|
num := binary.BigEndian.Uint16(a.b.bytes())
|
style: Replace `else if` cascades with `switch`
Wiser coders than myself have come to the conclusion that a `switch`
statement is almost always superior to a statement that includes any
`else if`.
The exceptions that I have found in our codebase are just these two:
* The `if else` is followed by an additional statement before the next
condition (separated by a `;`).
* The whole thing is within a `for` loop and `break` statements are
used. In this case, using `switch` would require tagging the `for`
loop, which probably tips the balance.
Why are `switch` statements more readable?
For one, fewer curly braces. But more importantly, the conditions all
have the same alignment, so the whole thing follows the natural flow
of going down a list of conditions. With `else if`, in contrast, all
conditions but the first are "hidden" behind `} else if `, harder to
spot and (for no good reason) presented differently from the first
condition.
I'm sure the aforemention wise coders can list even more reasons.
In any case, I like it so much that I have found myself recommending
it in code reviews. I would like to make it a habit in our code base,
without making it a hard requirement that we would test on the CI. But
for that, there has to be a role model, so this commit eliminates all
`if else` occurrences, unless it is autogenerated code or fits one of
the exceptions above.
Signed-off-by: beorn7 <beorn@grafana.com>
2023-04-12 14:14:31 +00:00
|
|
|
switch num {
|
|
|
|
case 0:
|
2016-11-30 21:10:10 +00:00
|
|
|
buf := make([]byte, binary.MaxVarintLen64)
|
|
|
|
for _, b := range buf[:binary.PutVarint(buf, t)] {
|
|
|
|
a.b.writeByte(b)
|
|
|
|
}
|
2016-11-29 21:02:58 +00:00
|
|
|
a.b.writeBits(math.Float64bits(v), 64)
|
style: Replace `else if` cascades with `switch`
Wiser coders than myself have come to the conclusion that a `switch`
statement is almost always superior to a statement that includes any
`else if`.
The exceptions that I have found in our codebase are just these two:
* The `if else` is followed by an additional statement before the next
condition (separated by a `;`).
* The whole thing is within a `for` loop and `break` statements are
used. In this case, using `switch` would require tagging the `for`
loop, which probably tips the balance.
Why are `switch` statements more readable?
For one, fewer curly braces. But more importantly, the conditions all
have the same alignment, so the whole thing follows the natural flow
of going down a list of conditions. With `else if`, in contrast, all
conditions but the first are "hidden" behind `} else if `, harder to
spot and (for no good reason) presented differently from the first
condition.
I'm sure the aforemention wise coders can list even more reasons.
In any case, I like it so much that I have found myself recommending
it in code reviews. I would like to make it a habit in our code base,
without making it a hard requirement that we would test on the CI. But
for that, there has to be a role model, so this commit eliminates all
`if else` occurrences, unless it is autogenerated code or fits one of
the exceptions above.
Signed-off-by: beorn7 <beorn@grafana.com>
2023-04-12 14:14:31 +00:00
|
|
|
case 1:
|
2016-11-20 15:14:21 +00:00
|
|
|
tDelta = uint64(t - a.t)
|
2016-11-30 21:10:10 +00:00
|
|
|
|
|
|
|
buf := make([]byte, binary.MaxVarintLen64)
|
|
|
|
for _, b := range buf[:binary.PutUvarint(buf, tDelta)] {
|
|
|
|
a.b.writeByte(b)
|
|
|
|
}
|
|
|
|
|
2016-11-20 15:14:21 +00:00
|
|
|
a.writeVDelta(v)
|
2023-04-09 07:08:40 +00:00
|
|
|
default:
|
2016-11-20 15:14:21 +00:00
|
|
|
tDelta = uint64(t - a.t)
|
2016-11-20 13:33:00 +00:00
|
|
|
dod := int64(tDelta - a.tDelta)
|
|
|
|
|
|
|
|
// Gorilla has a max resolution of seconds, Prometheus milliseconds.
|
|
|
|
// Thus we use higher value range steps with larger bit size.
|
Style cleanup of all the changes in sparsehistogram so far
A lot of this code was hacked together, literally during a
hackathon. This commit intends not to change the code substantially,
but just make the code obey the usual style practices.
A (possibly incomplete) list of areas:
* Generally address linter warnings.
* The `pgk` directory is deprecated as per dev-summit. No new packages should
be added to it. I moved the new `pkg/histogram` package to `model`
anticipating what's proposed in #9478.
* Make the naming of the Sparse Histogram more consistent. Including
abbreviations, there were just too many names for it: SparseHistogram,
Histogram, Histo, hist, his, shs, h. The idea is to call it "Histogram" in
general. Only add "Sparse" if it is needed to avoid confusion with
conventional Histograms (which is rare because the TSDB really has no notion
of conventional Histograms). Use abbreviations only in local scope, and then
really abbreviate (not just removing three out of seven letters like in
"Histo"). This is in the spirit of
https://github.com/golang/go/wiki/CodeReviewComments#variable-names
* Several other minor name changes.
* A lot of formatting of doc comments. For one, following
https://github.com/golang/go/wiki/CodeReviewComments#comment-sentences
, but also layout question, anticipating how things will look like
when rendered by `godoc` (even where `godoc` doesn't render them
right now because they are for unexported types or not a doc comment
at all but just a normal code comment - consistency is queen!).
* Re-enabled `TestQueryLog` and `TestEndopints` (they pass now,
leaving them disabled was presumably an oversight).
* Bucket iterator for histogram.Histogram is now created with a
method.
* HistogramChunk.iterator now allows iterator recycling. (I think
@dieterbe only commented it out because he was confused by the
question in the comment.)
* HistogramAppender.Append panics now because we decided to treat
staleness marker differently.
Signed-off-by: beorn7 <beorn@grafana.com>
2021-10-09 13:57:07 +00:00
|
|
|
//
|
|
|
|
// TODO(beorn7): This seems to needlessly jump to large bit
|
|
|
|
// sizes even for very small deviations from zero. Timestamp
|
|
|
|
// compression can probably benefit from some smaller bit
|
|
|
|
// buckets. See also what was done for histogram encoding in
|
|
|
|
// varbit.go.
|
2016-11-20 13:33:00 +00:00
|
|
|
switch {
|
|
|
|
case dod == 0:
|
2016-11-29 21:02:58 +00:00
|
|
|
a.b.writeBit(zero)
|
2016-12-07 14:37:37 +00:00
|
|
|
case bitRange(dod, 14):
|
2021-07-05 14:39:24 +00:00
|
|
|
a.b.writeBits(0b10, 2)
|
2016-11-29 21:02:58 +00:00
|
|
|
a.b.writeBits(uint64(dod), 14)
|
2016-12-07 14:37:37 +00:00
|
|
|
case bitRange(dod, 17):
|
2021-07-05 14:39:24 +00:00
|
|
|
a.b.writeBits(0b110, 3)
|
2016-11-29 21:02:58 +00:00
|
|
|
a.b.writeBits(uint64(dod), 17)
|
2016-12-07 14:37:37 +00:00
|
|
|
case bitRange(dod, 20):
|
2021-07-05 14:39:24 +00:00
|
|
|
a.b.writeBits(0b1110, 4)
|
2016-11-29 21:02:58 +00:00
|
|
|
a.b.writeBits(uint64(dod), 20)
|
2016-11-20 13:33:00 +00:00
|
|
|
default:
|
2021-07-05 14:39:24 +00:00
|
|
|
a.b.writeBits(0b1111, 4)
|
2016-11-29 21:02:58 +00:00
|
|
|
a.b.writeBits(uint64(dod), 64)
|
2016-11-15 09:33:34 +00:00
|
|
|
}
|
2016-11-20 13:33:00 +00:00
|
|
|
|
|
|
|
a.writeVDelta(v)
|
2016-11-15 09:33:34 +00:00
|
|
|
}
|
|
|
|
|
2016-11-20 13:33:00 +00:00
|
|
|
a.t = t
|
|
|
|
a.v = v
|
2017-01-02 09:34:55 +00:00
|
|
|
binary.BigEndian.PutUint16(a.b.bytes(), num+1)
|
2016-11-20 15:14:21 +00:00
|
|
|
a.tDelta = tDelta
|
2016-11-15 09:33:34 +00:00
|
|
|
}
|
|
|
|
|
2021-10-25 13:14:15 +00:00
|
|
|
// bitRange returns whether the given integer can be represented by nbits.
|
|
|
|
// See docs/bstream.md.
|
2016-12-07 14:37:37 +00:00
|
|
|
func bitRange(x int64, nbits uint8) bool {
|
|
|
|
return -((1<<(nbits-1))-1) <= x && x <= 1<<(nbits-1)
|
|
|
|
}
|
|
|
|
|
2016-11-20 13:33:00 +00:00
|
|
|
func (a *xorAppender) writeVDelta(v float64) {
|
2022-10-20 09:38:01 +00:00
|
|
|
xorWrite(a.b, v, a.v, &a.leading, &a.trailing)
|
2021-10-18 16:10:12 +00:00
|
|
|
}
|
|
|
|
|
2023-07-26 13:08:16 +00:00
|
|
|
func (a *xorAppender) AppendHistogram(*HistogramAppender, int64, *histogram.Histogram, bool) (Chunk, bool, Appender, error) {
|
|
|
|
panic("appended a histogram sample to a float chunk")
|
|
|
|
}
|
|
|
|
|
|
|
|
func (a *xorAppender) AppendFloatHistogram(*FloatHistogramAppender, int64, *histogram.FloatHistogram, bool) (Chunk, bool, Appender, error) {
|
|
|
|
panic("appended a float histogram sample to a float chunk")
|
|
|
|
}
|
|
|
|
|
2016-11-15 09:33:34 +00:00
|
|
|
type xorIterator struct {
|
2020-06-15 15:44:40 +00:00
|
|
|
br bstreamReader
|
2016-11-20 13:33:00 +00:00
|
|
|
numTotal uint16
|
|
|
|
numRead uint16
|
|
|
|
|
|
|
|
t int64
|
|
|
|
val float64
|
|
|
|
|
|
|
|
leading uint8
|
|
|
|
trailing uint8
|
|
|
|
|
2016-11-29 21:02:58 +00:00
|
|
|
tDelta uint64
|
2016-11-20 13:33:00 +00:00
|
|
|
err error
|
|
|
|
}
|
|
|
|
|
2021-11-29 07:54:23 +00:00
|
|
|
func (it *xorIterator) Seek(t int64) ValueType {
|
2020-02-06 15:58:38 +00:00
|
|
|
if it.err != nil {
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValNone
|
2020-02-06 15:58:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for t > it.t || it.numRead == 0 {
|
2021-11-29 07:54:23 +00:00
|
|
|
if it.Next() == ValNone {
|
|
|
|
return ValNone
|
2020-02-06 15:58:38 +00:00
|
|
|
}
|
|
|
|
}
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValFloat
|
2020-02-06 15:58:38 +00:00
|
|
|
}
|
|
|
|
|
2017-01-02 12:27:52 +00:00
|
|
|
func (it *xorIterator) At() (int64, float64) {
|
2016-11-20 13:33:00 +00:00
|
|
|
return it.t, it.val
|
|
|
|
}
|
|
|
|
|
2024-01-23 16:02:14 +00:00
|
|
|
func (it *xorIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
|
2021-11-12 18:07:41 +00:00
|
|
|
panic("cannot call xorIterator.AtHistogram")
|
2021-06-29 14:38:46 +00:00
|
|
|
}
|
|
|
|
|
2024-01-23 16:02:14 +00:00
|
|
|
func (it *xorIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
|
2021-11-29 07:54:23 +00:00
|
|
|
panic("cannot call xorIterator.AtFloatHistogram")
|
|
|
|
}
|
|
|
|
|
|
|
|
func (it *xorIterator) AtT() int64 {
|
|
|
|
return it.t
|
2021-06-30 14:48:13 +00:00
|
|
|
}
|
|
|
|
|
2016-11-29 21:02:58 +00:00
|
|
|
func (it *xorIterator) Err() error {
|
|
|
|
return it.err
|
|
|
|
}
|
|
|
|
|
2019-07-09 09:49:34 +00:00
|
|
|
func (it *xorIterator) Reset(b []byte) {
|
|
|
|
// The first 2 bytes contain chunk headers.
|
|
|
|
// We skip that for actual samples.
|
|
|
|
it.br = newBReader(b[2:])
|
|
|
|
it.numTotal = binary.BigEndian.Uint16(b)
|
|
|
|
|
|
|
|
it.numRead = 0
|
|
|
|
it.t = 0
|
|
|
|
it.val = 0
|
|
|
|
it.leading = 0
|
|
|
|
it.trailing = 0
|
|
|
|
it.tDelta = 0
|
|
|
|
it.err = nil
|
|
|
|
}
|
|
|
|
|
2021-11-29 07:54:23 +00:00
|
|
|
func (it *xorIterator) Next() ValueType {
|
2016-11-20 13:33:00 +00:00
|
|
|
if it.err != nil || it.numRead == it.numTotal {
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValNone
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if it.numRead == 0 {
|
2018-11-22 07:51:57 +00:00
|
|
|
t, err := binary.ReadVarint(&it.br)
|
2016-11-20 13:33:00 +00:00
|
|
|
if err != nil {
|
|
|
|
it.err = err
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValNone
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
|
|
|
v, err := it.br.readBits(64)
|
|
|
|
if err != nil {
|
|
|
|
it.err = err
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValNone
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
2018-03-21 20:23:47 +00:00
|
|
|
it.t = t
|
2016-11-20 13:33:00 +00:00
|
|
|
it.val = math.Float64frombits(v)
|
|
|
|
|
|
|
|
it.numRead++
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValFloat
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
|
|
|
if it.numRead == 1 {
|
2018-11-22 07:51:57 +00:00
|
|
|
tDelta, err := binary.ReadUvarint(&it.br)
|
2016-11-20 13:33:00 +00:00
|
|
|
if err != nil {
|
|
|
|
it.err = err
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValNone
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
2016-11-29 21:02:58 +00:00
|
|
|
it.tDelta = tDelta
|
2023-04-09 07:08:40 +00:00
|
|
|
it.t += int64(it.tDelta)
|
2016-11-20 13:33:00 +00:00
|
|
|
|
2016-11-29 21:02:58 +00:00
|
|
|
return it.readValue()
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
|
|
|
|
2016-11-29 21:02:58 +00:00
|
|
|
var d byte
|
2016-11-20 13:33:00 +00:00
|
|
|
// read delta-of-delta
|
|
|
|
for i := 0; i < 4; i++ {
|
|
|
|
d <<= 1
|
2020-06-15 15:44:40 +00:00
|
|
|
bit, err := it.br.readBitFast()
|
|
|
|
if err != nil {
|
|
|
|
bit, err = it.br.readBit()
|
|
|
|
}
|
2016-11-20 13:33:00 +00:00
|
|
|
if err != nil {
|
|
|
|
it.err = err
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValNone
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
|
|
|
if bit == zero {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
d |= 1
|
|
|
|
}
|
2016-11-29 21:02:58 +00:00
|
|
|
var sz uint8
|
|
|
|
var dod int64
|
2016-11-20 13:33:00 +00:00
|
|
|
switch d {
|
2021-07-05 14:39:24 +00:00
|
|
|
case 0b0:
|
2016-11-20 13:33:00 +00:00
|
|
|
// dod == 0
|
2021-07-05 14:39:24 +00:00
|
|
|
case 0b10:
|
2016-11-20 13:33:00 +00:00
|
|
|
sz = 14
|
2021-07-05 14:39:24 +00:00
|
|
|
case 0b110:
|
2016-11-20 13:33:00 +00:00
|
|
|
sz = 17
|
2021-07-05 14:39:24 +00:00
|
|
|
case 0b1110:
|
2016-11-20 13:33:00 +00:00
|
|
|
sz = 20
|
2021-07-05 14:39:24 +00:00
|
|
|
case 0b1111:
|
2020-06-15 15:44:40 +00:00
|
|
|
// Do not use fast because it's very unlikely it will succeed.
|
2016-11-20 13:33:00 +00:00
|
|
|
bits, err := it.br.readBits(64)
|
|
|
|
if err != nil {
|
|
|
|
it.err = err
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValNone
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
|
|
|
|
2016-11-29 21:02:58 +00:00
|
|
|
dod = int64(bits)
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if sz != 0 {
|
2020-06-15 15:44:40 +00:00
|
|
|
bits, err := it.br.readBitsFast(sz)
|
|
|
|
if err != nil {
|
|
|
|
bits, err = it.br.readBits(sz)
|
|
|
|
}
|
2016-11-20 13:33:00 +00:00
|
|
|
if err != nil {
|
|
|
|
it.err = err
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValNone
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
2021-10-25 13:14:15 +00:00
|
|
|
|
|
|
|
// Account for negative numbers, which come back as high unsigned numbers.
|
|
|
|
// See docs/bstream.md.
|
2016-11-20 13:33:00 +00:00
|
|
|
if bits > (1 << (sz - 1)) {
|
2021-10-25 13:14:15 +00:00
|
|
|
bits -= 1 << sz
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
2016-11-29 21:02:58 +00:00
|
|
|
dod = int64(bits)
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
|
|
|
|
2016-11-29 21:02:58 +00:00
|
|
|
it.tDelta = uint64(int64(it.tDelta) + dod)
|
2023-04-09 07:08:40 +00:00
|
|
|
it.t += int64(it.tDelta)
|
2016-11-20 13:33:00 +00:00
|
|
|
|
2016-11-29 21:02:58 +00:00
|
|
|
return it.readValue()
|
|
|
|
}
|
2016-11-20 13:33:00 +00:00
|
|
|
|
2021-11-29 07:54:23 +00:00
|
|
|
func (it *xorIterator) readValue() ValueType {
|
2022-10-20 09:38:01 +00:00
|
|
|
err := xorRead(&it.br, &it.val, &it.leading, &it.trailing)
|
2016-11-20 13:33:00 +00:00
|
|
|
if err != nil {
|
|
|
|
it.err = err
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValNone
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
2021-10-18 16:46:08 +00:00
|
|
|
it.numRead++
|
2021-11-29 07:54:23 +00:00
|
|
|
return ValFloat
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
|
|
|
|
2022-10-20 09:38:01 +00:00
|
|
|
func xorWrite(b *bstream, newValue, currentValue float64, leading, trailing *uint8) {
|
2021-10-19 13:28:24 +00:00
|
|
|
delta := math.Float64bits(newValue) ^ math.Float64bits(currentValue)
|
2021-10-18 16:46:08 +00:00
|
|
|
|
|
|
|
if delta == 0 {
|
|
|
|
b.writeBit(zero)
|
2022-10-20 09:38:01 +00:00
|
|
|
return
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
|
|
|
b.writeBit(one)
|
|
|
|
|
2022-10-20 09:38:01 +00:00
|
|
|
newLeading := uint8(bits.LeadingZeros64(delta))
|
|
|
|
newTrailing := uint8(bits.TrailingZeros64(delta))
|
2021-10-18 16:46:08 +00:00
|
|
|
|
|
|
|
// Clamp number of leading zeros to avoid overflow when encoding.
|
2021-10-19 13:28:24 +00:00
|
|
|
if newLeading >= 32 {
|
|
|
|
newLeading = 31
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
|
|
|
|
2022-10-20 09:38:01 +00:00
|
|
|
if *leading != 0xff && newLeading >= *leading && newTrailing >= *trailing {
|
2021-10-19 13:28:24 +00:00
|
|
|
// In this case, we stick with the current leading/trailing.
|
2021-10-18 16:46:08 +00:00
|
|
|
b.writeBit(zero)
|
2022-10-20 09:38:01 +00:00
|
|
|
b.writeBits(delta>>*trailing, 64-int(*leading)-int(*trailing))
|
|
|
|
return
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
|
|
|
|
2022-10-20 09:38:01 +00:00
|
|
|
// Update leading/trailing for the caller.
|
|
|
|
*leading, *trailing = newLeading, newTrailing
|
|
|
|
|
2021-10-18 16:46:08 +00:00
|
|
|
b.writeBit(one)
|
2021-10-19 13:28:24 +00:00
|
|
|
b.writeBits(uint64(newLeading), 5)
|
|
|
|
|
|
|
|
// Note that if newLeading == newTrailing == 0, then sigbits == 64. But
|
|
|
|
// that value doesn't actually fit into the 6 bits we have. Luckily, we
|
|
|
|
// never need to encode 0 significant bits, since that would put us in
|
|
|
|
// the other case (vdelta == 0). So instead we write out a 0 and adjust
|
|
|
|
// it back to 64 on unpacking.
|
|
|
|
sigbits := 64 - newLeading - newTrailing
|
2021-10-18 16:46:08 +00:00
|
|
|
b.writeBits(uint64(sigbits), 6)
|
2021-10-19 13:28:24 +00:00
|
|
|
b.writeBits(delta>>newTrailing, int(sigbits))
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
|
|
|
|
2022-10-20 09:38:01 +00:00
|
|
|
func xorRead(br *bstreamReader, value *float64, leading, trailing *uint8) error {
|
|
|
|
bit, err := br.readBitFast()
|
2021-10-18 16:46:08 +00:00
|
|
|
if err != nil {
|
|
|
|
bit, err = br.readBit()
|
|
|
|
}
|
|
|
|
if err != nil {
|
2022-10-20 09:38:01 +00:00
|
|
|
return err
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
|
|
|
if bit == zero {
|
2022-10-20 09:38:01 +00:00
|
|
|
return nil
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
|
|
|
bit, err = br.readBitFast()
|
|
|
|
if err != nil {
|
|
|
|
bit, err = br.readBit()
|
|
|
|
}
|
|
|
|
if err != nil {
|
2022-10-20 09:38:01 +00:00
|
|
|
return err
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
2022-10-20 09:38:01 +00:00
|
|
|
|
|
|
|
var (
|
|
|
|
bits uint64
|
|
|
|
newLeading, newTrailing, mbits uint8
|
|
|
|
)
|
|
|
|
|
2016-11-20 13:33:00 +00:00
|
|
|
if bit == zero {
|
2021-10-18 16:46:08 +00:00
|
|
|
// Reuse leading/trailing zero bits.
|
2022-10-20 09:38:01 +00:00
|
|
|
newLeading, newTrailing = *leading, *trailing
|
|
|
|
mbits = 64 - newLeading - newTrailing
|
2016-11-20 13:33:00 +00:00
|
|
|
} else {
|
2021-10-18 16:46:08 +00:00
|
|
|
bits, err = br.readBitsFast(5)
|
2020-06-15 15:44:40 +00:00
|
|
|
if err != nil {
|
2021-10-18 16:46:08 +00:00
|
|
|
bits, err = br.readBits(5)
|
2020-06-15 15:44:40 +00:00
|
|
|
}
|
2016-11-29 21:02:58 +00:00
|
|
|
if err != nil {
|
2022-10-20 09:38:01 +00:00
|
|
|
return err
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
2021-10-18 16:46:08 +00:00
|
|
|
newLeading = uint8(bits)
|
2016-11-20 13:33:00 +00:00
|
|
|
|
2021-10-18 16:46:08 +00:00
|
|
|
bits, err = br.readBitsFast(6)
|
2020-06-15 15:44:40 +00:00
|
|
|
if err != nil {
|
2021-10-18 16:46:08 +00:00
|
|
|
bits, err = br.readBits(6)
|
2020-06-15 15:44:40 +00:00
|
|
|
}
|
2016-11-20 13:33:00 +00:00
|
|
|
if err != nil {
|
2022-10-20 09:38:01 +00:00
|
|
|
return err
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
2022-10-20 09:38:01 +00:00
|
|
|
mbits = uint8(bits)
|
2021-10-18 16:46:08 +00:00
|
|
|
// 0 significant bits here means we overflowed and we actually
|
|
|
|
// need 64; see comment in xrWrite.
|
|
|
|
if mbits == 0 {
|
|
|
|
mbits = 64
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
2021-10-18 16:46:08 +00:00
|
|
|
newTrailing = 64 - newLeading - mbits
|
2022-10-20 09:38:01 +00:00
|
|
|
// Update leading/trailing zero bits for the caller.
|
|
|
|
*leading, *trailing = newLeading, newTrailing
|
2016-11-20 13:33:00 +00:00
|
|
|
}
|
2021-10-18 16:46:08 +00:00
|
|
|
bits, err = br.readBitsFast(mbits)
|
|
|
|
if err != nil {
|
|
|
|
bits, err = br.readBits(mbits)
|
|
|
|
}
|
|
|
|
if err != nil {
|
2022-10-20 09:38:01 +00:00
|
|
|
return err
|
2021-10-18 16:46:08 +00:00
|
|
|
}
|
2022-10-20 09:38:01 +00:00
|
|
|
vbits := math.Float64bits(*value)
|
2021-10-18 16:46:08 +00:00
|
|
|
vbits ^= bits << newTrailing
|
2022-10-20 09:38:01 +00:00
|
|
|
*value = math.Float64frombits(vbits)
|
|
|
|
return nil
|
2016-11-15 09:33:34 +00:00
|
|
|
}
|