mirror of https://github.com/prometheus/prometheus
Fabian Reinartz
8 years ago
parent
2a825f6c28
commit
0b8c77361e
@ -0,0 +1,191 @@
|
||||
package tsdb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/coreos/etcd/pkg/fileutil"
|
||||
"github.com/fabxc/tsdb/labels"
|
||||
)
|
||||
|
||||
// WALEntryType indicates what data a WAL entry contains.
|
||||
type WALEntryType byte
|
||||
|
||||
// The valid WAL entry types.
|
||||
const (
|
||||
WALEntrySymbols = 1
|
||||
WALEntrySeries = 2
|
||||
WALEntrySamples = 3
|
||||
)
|
||||
|
||||
// WAL is a write ahead log for series data. It can only be written to.
|
||||
// Use WALReader to read back from a write ahead log.
|
||||
type WAL struct {
|
||||
f *fileutil.LockedFile
|
||||
enc *walEncoder
|
||||
|
||||
symbols map[string]uint32
|
||||
}
|
||||
|
||||
// CreateWAL creates a new write ahead log in the given directory.
|
||||
func CreateWAL(dir string) (*WAL, error) {
|
||||
if err := os.MkdirAll(dir, 0777); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
p := filepath.Join(dir, "wal")
|
||||
|
||||
f, err := fileutil.LockFile(p, os.O_WRONLY|os.O_CREATE, fileutil.PrivateFileMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = f.Seek(0, os.SEEK_END); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
w := &WAL{
|
||||
f: f,
|
||||
enc: newWALEncoder(f),
|
||||
symbols: map[string]uint32{},
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Log writes a batch of new series labels and samples to the log.
|
||||
func (w *WAL) Log(series []labels.Labels, samples []hashedSample) error {
|
||||
if err := w.enc.encodeSeries(series); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.enc.encodeSamples(samples); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *WAL) sync() error {
|
||||
return fileutil.Fdatasync(w.f.File)
|
||||
}
|
||||
|
||||
// Close sync all data and closes the underlying resources.
|
||||
func (w *WAL) Close() error {
|
||||
if err := w.sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.f.Close()
|
||||
}
|
||||
|
||||
// OpenWAL does things.
|
||||
func OpenWAL(dir string) (*WAL, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type walEncoder struct {
|
||||
w io.Writer
|
||||
|
||||
buf []byte
|
||||
}
|
||||
|
||||
func newWALEncoder(w io.Writer) *walEncoder {
|
||||
return &walEncoder{
|
||||
w: w,
|
||||
buf: make([]byte, 1024*1024),
|
||||
}
|
||||
}
|
||||
|
||||
func (e *walEncoder) entry(et WALEntryType, flag byte, n int) error {
|
||||
h := crc32.NewIEEE()
|
||||
w := io.MultiWriter(h, e.w)
|
||||
|
||||
b := make([]byte, 6)
|
||||
b[0] = byte(et)
|
||||
b[1] = flag
|
||||
|
||||
binary.BigEndian.PutUint32(b[2:], uint32(len(e.buf)))
|
||||
|
||||
if _, err := w.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write(e.buf[:n]); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := e.w.Write(h.Sum(nil)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
walSeriesSimple = 1
|
||||
walSamplesSimple = 1
|
||||
)
|
||||
|
||||
func (e *walEncoder) encodeSeries(series []labels.Labels) error {
|
||||
if len(series) == 0 {
|
||||
return nil
|
||||
}
|
||||
var (
|
||||
b = make([]byte, binary.MaxVarintLen32)
|
||||
buf = e.buf[:0]
|
||||
)
|
||||
|
||||
for _, lset := range series {
|
||||
n := binary.PutUvarint(b, uint64(len(lset)))
|
||||
buf = append(buf, b[:n]...)
|
||||
|
||||
for _, l := range lset {
|
||||
n = binary.PutUvarint(b, uint64(len(l.Name)))
|
||||
buf = append(buf, b[:n]...)
|
||||
|
||||
n = binary.PutUvarint(b, uint64(len(l.Value)))
|
||||
buf = append(buf, b[:n]...)
|
||||
}
|
||||
}
|
||||
|
||||
return e.entry(WALEntrySeries, walSeriesSimple, len(buf))
|
||||
}
|
||||
|
||||
func (e *walEncoder) encodeSamples(samples []hashedSample) error {
|
||||
if len(samples) == 0 {
|
||||
return nil
|
||||
}
|
||||
var (
|
||||
b = make([]byte, binary.MaxVarintLen64)
|
||||
buf = e.buf[:0]
|
||||
)
|
||||
|
||||
// Store base timestamp and base reference number of first sample.
|
||||
// All samples encode their timestamp and ref as delta to those.
|
||||
//
|
||||
// TODO(fabxc): optimize for all samples having the same timestamp.
|
||||
first := samples[0]
|
||||
|
||||
binary.BigEndian.PutUint32(b, first.ref)
|
||||
buf = append(buf, b[:4]...)
|
||||
binary.BigEndian.PutUint64(b, uint64(first.t))
|
||||
buf = append(buf, b[:8]...)
|
||||
|
||||
for _, s := range samples {
|
||||
n := binary.PutVarint(b, int64(s.ref)-int64(first.ref))
|
||||
buf = append(buf, b[:n]...)
|
||||
|
||||
n = binary.PutVarint(b, s.t-first.t)
|
||||
buf = append(buf, b[:n]...)
|
||||
|
||||
binary.BigEndian.PutUint64(b, math.Float64bits(s.v))
|
||||
buf = append(buf, b[:8]...)
|
||||
}
|
||||
|
||||
return e.entry(WALEntrySamples, walSamplesSimple, len(buf))
|
||||
}
|
||||
|
||||
type walDecoder struct {
|
||||
r io.Reader
|
||||
|
||||
handleSeries func(labels.Labels)
|
||||
handleSample func(hashedSample)
|
||||
}
|
||||
Loading…
Reference in new issue