prometheus/wal.go

package tsdb

import (
	"encoding/binary"
	"hash/crc32"
	"io"
	"math"
	"os"
	"path/filepath"

	"github.com/coreos/etcd/pkg/fileutil"
	"github.com/fabxc/tsdb/labels"
)

// WALEntryType indicates what data a WAL entry contains.
type WALEntryType byte

// The valid WAL entry types.
const (
	WALEntrySymbols = 1
	WALEntrySeries  = 2
	WALEntrySamples = 3
)

// WAL is a write ahead log for series data. It can only be written to.
// Use WALReader to read back from a write ahead log.
type WAL struct {
	f   *fileutil.LockedFile
	enc *walEncoder

	symbols map[string]uint32
}

// CreateWAL creates a new write ahead log in the given directory.
func CreateWAL(dir string) (*WAL, error) {
	if err := os.MkdirAll(dir, 0777); err != nil {
		return nil, err
	}

	p := filepath.Join(dir, "wal")

	f, err := fileutil.LockFile(p, os.O_WRONLY|os.O_CREATE, fileutil.PrivateFileMode)
	if err != nil {
		return nil, err
	}
	if _, err = f.Seek(0, os.SEEK_END); err != nil {
		return nil, err
	}

	w := &WAL{
		f:       f,
		enc:     newWALEncoder(f),
		symbols: map[string]uint32{},
	}
	return w, nil
}

// Log writes a batch of new series labels and samples to the log.
func (w *WAL) Log(series []labels.Labels, samples []hashedSample) error {
	if err := w.enc.encodeSeries(series); err != nil {
		return err
	}
	if err := w.enc.encodeSamples(samples); err != nil {
		return err
	}
	return nil
}

func (w *WAL) sync() error {
	return fileutil.Fdatasync(w.f.File)
}

// Close sync all data and closes the underlying resources.
func (w *WAL) Close() error {
	if err := w.sync(); err != nil {
		return err
	}
	return w.f.Close()
}

// OpenWAL does things.
func OpenWAL(dir string) (*WAL, error) {
	return nil, nil
}

type walEncoder struct {
	w io.Writer

	buf []byte
}

func newWALEncoder(w io.Writer) *walEncoder {
	return &walEncoder{
		w:   w,
		buf: make([]byte, 1024*1024),
	}
}

func (e *walEncoder) entry(et WALEntryType, flag byte, n int) error {
	h := crc32.NewIEEE()
	w := io.MultiWriter(h, e.w)

	b := make([]byte, 6)
	b[0] = byte(et)
	b[1] = flag

	binary.BigEndian.PutUint32(b[2:], uint32(len(e.buf)))

	if _, err := w.Write(b); err != nil {
		return err
	}
	if _, err := w.Write(e.buf[:n]); err != nil {
		return err
	}
	if _, err := e.w.Write(h.Sum(nil)); err != nil {
		return err
	}

	return nil
}

const (
	walSeriesSimple  = 1
	walSamplesSimple = 1
)

func (e *walEncoder) encodeSeries(series []labels.Labels) error {
	if len(series) == 0 {
		return nil
	}
	var (
		b   = make([]byte, binary.MaxVarintLen32)
		buf = e.buf[:0]
	)

	for _, lset := range series {
		n := binary.PutUvarint(b, uint64(len(lset)))
		buf = append(buf, b[:n]...)

		for _, l := range lset {
			n = binary.PutUvarint(b, uint64(len(l.Name)))
			buf = append(buf, b[:n]...)

			n = binary.PutUvarint(b, uint64(len(l.Value)))
			buf = append(buf, b[:n]...)
		}
	}

	return e.entry(WALEntrySeries, walSeriesSimple, len(buf))
}

func (e *walEncoder) encodeSamples(samples []hashedSample) error {
	if len(samples) == 0 {
		return nil
	}
	var (
		b   = make([]byte, binary.MaxVarintLen64)
		buf = e.buf[:0]
	)

	// Store base timestamp and base reference number of first sample.
	// All samples encode their timestamp and ref as delta to those.
	//
	// TODO(fabxc): optimize for all samples having the same timestamp.
	first := samples[0]

	binary.BigEndian.PutUint32(b, first.ref)
	buf = append(buf, b[:4]...)
	binary.BigEndian.PutUint64(b, uint64(first.t))
	buf = append(buf, b[:8]...)

	for _, s := range samples {
		n := binary.PutVarint(b, int64(s.ref)-int64(first.ref))
		buf = append(buf, b[:n]...)

		n = binary.PutVarint(b, s.t-first.t)
		buf = append(buf, b[:n]...)

		binary.BigEndian.PutUint64(b, math.Float64bits(s.v))
		buf = append(buf, b[:8]...)
	}

	return e.entry(WALEntrySamples, walSamplesSimple, len(buf))
}

type walDecoder struct {
	r io.Reader

	handleSeries func(labels.Labels)
	handleSample func(hashedSample)
}