Merge pull request #676 from prometheus/fabxc/promql

Query/rules refactoring

Godeps/Godeps.json

@@ -63,6 +63,10 @@
 		{
 			"ImportPath": "github.com/syndtr/gosnappy/snappy",
 			"Rev": "156a073208e131d7d2e212cb749feae7c339e846"
+		},
+		{
+			"ImportPath": "golang.org/x/net/context",
+			"Rev": "b6fdb7d8a4ccefede406f8fe0f017fb58265054c"
 		}
 	]
 }

Godeps/_workspace/src/golang.org/x/net/context/context.go

@@ -0,0 +1,447 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package context defines the Context type, which carries deadlines,
+// cancelation signals, and other request-scoped values across API boundaries
+// and between processes.
+//
+// Incoming requests to a server should create a Context, and outgoing calls to
+// servers should accept a Context.  The chain of function calls between must
+// propagate the Context, optionally replacing it with a modified copy created
+// using WithDeadline, WithTimeout, WithCancel, or WithValue.
+//
+// Programs that use Contexts should follow these rules to keep interfaces
+// consistent across packages and enable static analysis tools to check context
+// propagation:
+//
+// Do not store Contexts inside a struct type; instead, pass a Context
+// explicitly to each function that needs it.  The Context should be the first
+// parameter, typically named ctx:
+//
+// 	func DoSomething(ctx context.Context, arg Arg) error {
+// 		// ... use ctx ...
+// 	}
+//
+// Do not pass a nil Context, even if a function permits it.  Pass context.TODO
+// if you are unsure about which Context to use.
+//
+// Use context Values only for request-scoped data that transits processes and
+// APIs, not for passing optional parameters to functions.
+//
+// The same Context may be passed to functions running in different goroutines;
+// Contexts are safe for simultaneous use by multiple goroutines.
+//
+// See http://blog.golang.org/context for example code for a server that uses
+// Contexts.
+package context
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// A Context carries a deadline, a cancelation signal, and other values across
+// API boundaries.
+//
+// Context's methods may be called by multiple goroutines simultaneously.
+type Context interface {
+	// Deadline returns the time when work done on behalf of this context
+	// should be canceled.  Deadline returns ok==false when no deadline is
+	// set.  Successive calls to Deadline return the same results.
+	Deadline() (deadline time.Time, ok bool)
+
+	// Done returns a channel that's closed when work done on behalf of this
+	// context should be canceled.  Done may return nil if this context can
+	// never be canceled.  Successive calls to Done return the same value.
+	//
+	// WithCancel arranges for Done to be closed when cancel is called;
+	// WithDeadline arranges for Done to be closed when the deadline
+	// expires; WithTimeout arranges for Done to be closed when the timeout
+	// elapses.
+	//
+	// Done is provided for use in select statements:
+	//
+	//  // Stream generates values with DoSomething and sends them to out
+	//  // until DoSomething returns an error or ctx.Done is closed.
+	//  func Stream(ctx context.Context, out <-chan Value) error {
+	//  	for {
+	//  		v, err := DoSomething(ctx)
+	//  		if err != nil {
+	//  			return err
+	//  		}
+	//  		select {
+	//  		case <-ctx.Done():
+	//  			return ctx.Err()
+	//  		case out <- v:
+	//  		}
+	//  	}
+	//  }
+	//
+	// See http://blog.golang.org/pipelines for more examples of how to use
+	// a Done channel for cancelation.
+	Done() <-chan struct{}
+
+	// Err returns a non-nil error value after Done is closed.  Err returns
+	// Canceled if the context was canceled or DeadlineExceeded if the
+	// context's deadline passed.  No other values for Err are defined.
+	// After Done is closed, successive calls to Err return the same value.
+	Err() error
+
+	// Value returns the value associated with this context for key, or nil
+	// if no value is associated with key.  Successive calls to Value with
+	// the same key returns the same result.
+	//
+	// Use context values only for request-scoped data that transits
+	// processes and API boundaries, not for passing optional parameters to
+	// functions.
+	//
+	// A key identifies a specific value in a Context.  Functions that wish
+	// to store values in Context typically allocate a key in a global
+	// variable then use that key as the argument to context.WithValue and
+	// Context.Value.  A key can be any type that supports equality;
+	// packages should define keys as an unexported type to avoid
+	// collisions.
+	//
+	// Packages that define a Context key should provide type-safe accessors
+	// for the values stores using that key:
+	//
+	// 	// Package user defines a User type that's stored in Contexts.
+	// 	package user
+	//
+	// 	import "golang.org/x/net/context"
+	//
+	// 	// User is the type of value stored in the Contexts.
+	// 	type User struct {...}
+	//
+	// 	// key is an unexported type for keys defined in this package.
+	// 	// This prevents collisions with keys defined in other packages.
+	// 	type key int
+	//
+	// 	// userKey is the key for user.User values in Contexts.  It is
+	// 	// unexported; clients use user.NewContext and user.FromContext
+	// 	// instead of using this key directly.
+	// 	var userKey key = 0
+	//
+	// 	// NewContext returns a new Context that carries value u.
+	// 	func NewContext(ctx context.Context, u *User) context.Context {
+	// 		return context.WithValue(ctx, userKey, u)
+	// 	}
+	//
+	// 	// FromContext returns the User value stored in ctx, if any.
+	// 	func FromContext(ctx context.Context) (*User, bool) {
+	// 		u, ok := ctx.Value(userKey).(*User)
+	// 		return u, ok
+	// 	}
+	Value(key interface{}) interface{}
+}
+
+// Canceled is the error returned by Context.Err when the context is canceled.
+var Canceled = errors.New("context canceled")
+
+// DeadlineExceeded is the error returned by Context.Err when the context's
+// deadline passes.
+var DeadlineExceeded = errors.New("context deadline exceeded")
+
+// An emptyCtx is never canceled, has no values, and has no deadline.  It is not
+// struct{}, since vars of this type must have distinct addresses.
+type emptyCtx int
+
+func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
+	return
+}
+
+func (*emptyCtx) Done() <-chan struct{} {
+	return nil
+}
+
+func (*emptyCtx) Err() error {
+	return nil
+}
+
+func (*emptyCtx) Value(key interface{}) interface{} {
+	return nil
+}
+
+func (e *emptyCtx) String() string {
+	switch e {
+	case background:
+		return "context.Background"
+	case todo:
+		return "context.TODO"
+	}
+	return "unknown empty Context"
+}
+
+var (
+	background = new(emptyCtx)
+	todo       = new(emptyCtx)
+)
+
+// Background returns a non-nil, empty Context. It is never canceled, has no
+// values, and has no deadline.  It is typically used by the main function,
+// initialization, and tests, and as the top-level Context for incoming
+// requests.
+func Background() Context {
+	return background
+}
+
+// TODO returns a non-nil, empty Context.  Code should use context.TODO when
+// it's unclear which Context to use or it's is not yet available (because the
+// surrounding function has not yet been extended to accept a Context
+// parameter).  TODO is recognized by static analysis tools that determine
+// whether Contexts are propagated correctly in a program.
+func TODO() Context {
+	return todo
+}
+
+// A CancelFunc tells an operation to abandon its work.
+// A CancelFunc does not wait for the work to stop.
+// After the first call, subsequent calls to a CancelFunc do nothing.
+type CancelFunc func()
+
+// WithCancel returns a copy of parent with a new Done channel. The returned
+// context's Done channel is closed when the returned cancel function is called
+// or when the parent context's Done channel is closed, whichever happens first.
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete.
+func WithCancel(parent Context) (ctx Context, cancel CancelFunc) {
+	c := newCancelCtx(parent)
+	propagateCancel(parent, &c)
+	return &c, func() { c.cancel(true, Canceled) }
+}
+
+// newCancelCtx returns an initialized cancelCtx.
+func newCancelCtx(parent Context) cancelCtx {
+	return cancelCtx{
+		Context: parent,
+		done:    make(chan struct{}),
+	}
+}
+
+// propagateCancel arranges for child to be canceled when parent is.
+func propagateCancel(parent Context, child canceler) {
+	if parent.Done() == nil {
+		return // parent is never canceled
+	}
+	if p, ok := parentCancelCtx(parent); ok {
+		p.mu.Lock()
+		if p.err != nil {
+			// parent has already been canceled
+			child.cancel(false, p.err)
+		} else {
+			if p.children == nil {
+				p.children = make(map[canceler]bool)
+			}
+			p.children[child] = true
+		}
+		p.mu.Unlock()
+	} else {
+		go func() {
+			select {
+			case <-parent.Done():
+				child.cancel(false, parent.Err())
+			case <-child.Done():
+			}
+		}()
+	}
+}
+
+// parentCancelCtx follows a chain of parent references until it finds a
+// *cancelCtx.  This function understands how each of the concrete types in this
+// package represents its parent.
+func parentCancelCtx(parent Context) (*cancelCtx, bool) {
+	for {
+		switch c := parent.(type) {
+		case *cancelCtx:
+			return c, true
+		case *timerCtx:
+			return &c.cancelCtx, true
+		case *valueCtx:
+			parent = c.Context
+		default:
+			return nil, false
+		}
+	}
+}
+
+// removeChild removes a context from its parent.
+func removeChild(parent Context, child canceler) {
+	p, ok := parentCancelCtx(parent)
+	if !ok {
+		return
+	}
+	p.mu.Lock()
+	if p.children != nil {
+		delete(p.children, child)
+	}
+	p.mu.Unlock()
+}
+
+// A canceler is a context type that can be canceled directly.  The
+// implementations are *cancelCtx and *timerCtx.
+type canceler interface {
+	cancel(removeFromParent bool, err error)
+	Done() <-chan struct{}
+}
+
+// A cancelCtx can be canceled.  When canceled, it also cancels any children
+// that implement canceler.
+type cancelCtx struct {
+	Context
+
+	done chan struct{} // closed by the first cancel call.
+
+	mu       sync.Mutex
+	children map[canceler]bool // set to nil by the first cancel call
+	err      error             // set to non-nil by the first cancel call
+}
+
+func (c *cancelCtx) Done() <-chan struct{} {
+	return c.done
+}
+
+func (c *cancelCtx) Err() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.err
+}
+
+func (c *cancelCtx) String() string {
+	return fmt.Sprintf("%v.WithCancel", c.Context)
+}
+
+// cancel closes c.done, cancels each of c's children, and, if
+// removeFromParent is true, removes c from its parent's children.
+func (c *cancelCtx) cancel(removeFromParent bool, err error) {
+	if err == nil {
+		panic("context: internal error: missing cancel error")
+	}
+	c.mu.Lock()
+	if c.err != nil {
+		c.mu.Unlock()
+		return // already canceled
+	}
+	c.err = err
+	close(c.done)
+	for child := range c.children {
+		// NOTE: acquiring the child's lock while holding parent's lock.
+		child.cancel(false, err)
+	}
+	c.children = nil
+	c.mu.Unlock()
+
+	if removeFromParent {
+		removeChild(c.Context, c)
+	}
+}
+
+// WithDeadline returns a copy of the parent context with the deadline adjusted
+// to be no later than d.  If the parent's deadline is already earlier than d,
+// WithDeadline(parent, d) is semantically equivalent to parent.  The returned
+// context's Done channel is closed when the deadline expires, when the returned
+// cancel function is called, or when the parent context's Done channel is
+// closed, whichever happens first.
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete.
+func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) {
+	if cur, ok := parent.Deadline(); ok && cur.Before(deadline) {
+		// The current deadline is already sooner than the new one.
+		return WithCancel(parent)
+	}
+	c := &timerCtx{
+		cancelCtx: newCancelCtx(parent),
+		deadline:  deadline,
+	}
+	propagateCancel(parent, c)
+	d := deadline.Sub(time.Now())
+	if d <= 0 {
+		c.cancel(true, DeadlineExceeded) // deadline has already passed
+		return c, func() { c.cancel(true, Canceled) }
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.err == nil {
+		c.timer = time.AfterFunc(d, func() {
+			c.cancel(true, DeadlineExceeded)
+		})
+	}
+	return c, func() { c.cancel(true, Canceled) }
+}
+
+// A timerCtx carries a timer and a deadline.  It embeds a cancelCtx to
+// implement Done and Err.  It implements cancel by stopping its timer then
+// delegating to cancelCtx.cancel.
+type timerCtx struct {
+	cancelCtx
+	timer *time.Timer // Under cancelCtx.mu.
+
+	deadline time.Time
+}
+
+func (c *timerCtx) Deadline() (deadline time.Time, ok bool) {
+	return c.deadline, true
+}
+
+func (c *timerCtx) String() string {
+	return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now()))
+}
+
+func (c *timerCtx) cancel(removeFromParent bool, err error) {
+	c.cancelCtx.cancel(false, err)
+	if removeFromParent {
+		// Remove this timerCtx from its parent cancelCtx's children.
+		removeChild(c.cancelCtx.Context, c)
+	}
+	c.mu.Lock()
+	if c.timer != nil {
+		c.timer.Stop()
+		c.timer = nil
+	}
+	c.mu.Unlock()
+}
+
+// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)).
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete:
+//
+// 	func slowOperationWithTimeout(ctx context.Context) (Result, error) {
+// 		ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
+// 		defer cancel()  // releases resources if slowOperation completes before timeout elapses
+// 		return slowOperation(ctx)
+// 	}
+func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) {
+	return WithDeadline(parent, time.Now().Add(timeout))
+}
+
+// WithValue returns a copy of parent in which the value associated with key is
+// val.
+//
+// Use context Values only for request-scoped data that transits processes and
+// APIs, not for passing optional parameters to functions.
+func WithValue(parent Context, key interface{}, val interface{}) Context {
+	return &valueCtx{parent, key, val}
+}
+
+// A valueCtx carries a key-value pair.  It implements Value for that key and
+// delegates all other calls to the embedded Context.
+type valueCtx struct {
+	Context
+	key, val interface{}
+}
+
+func (c *valueCtx) String() string {
+	return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val)
+}
+
+func (c *valueCtx) Value(key interface{}) interface{} {
+	if c.key == key {
+		return c.val
+	}
+	return c.Context.Value(key)
+}

Godeps/_workspace/src/golang.org/x/net/context/context_test.go

@@ -0,0 +1,575 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package context
+
+import (
+	"fmt"
+	"math/rand"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// otherContext is a Context that's not one of the types defined in context.go.
+// This lets us test code paths that differ based on the underlying type of the
+// Context.
+type otherContext struct {
+	Context
+}
+
+func TestBackground(t *testing.T) {
+	c := Background()
+	if c == nil {
+		t.Fatalf("Background returned nil")
+	}
+	select {
+	case x := <-c.Done():
+		t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
+	default:
+	}
+	if got, want := fmt.Sprint(c), "context.Background"; got != want {
+		t.Errorf("Background().String() = %q want %q", got, want)
+	}
+}
+
+func TestTODO(t *testing.T) {
+	c := TODO()
+	if c == nil {
+		t.Fatalf("TODO returned nil")
+	}
+	select {
+	case x := <-c.Done():
+		t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
+	default:
+	}
+	if got, want := fmt.Sprint(c), "context.TODO"; got != want {
+		t.Errorf("TODO().String() = %q want %q", got, want)
+	}
+}
+
+func TestWithCancel(t *testing.T) {
+	c1, cancel := WithCancel(Background())
+
+	if got, want := fmt.Sprint(c1), "context.Background.WithCancel"; got != want {
+		t.Errorf("c1.String() = %q want %q", got, want)
+	}
+
+	o := otherContext{c1}
+	c2, _ := WithCancel(o)
+	contexts := []Context{c1, o, c2}
+
+	for i, c := range contexts {
+		if d := c.Done(); d == nil {
+			t.Errorf("c[%d].Done() == %v want non-nil", i, d)
+		}
+		if e := c.Err(); e != nil {
+			t.Errorf("c[%d].Err() == %v want nil", i, e)
+		}
+
+		select {
+		case x := <-c.Done():
+			t.Errorf("<-c.Done() == %v want nothing (it should block)", x)
+		default:
+		}
+	}
+
+	cancel()
+	time.Sleep(100 * time.Millisecond) // let cancelation propagate
+
+	for i, c := range contexts {
+		select {
+		case <-c.Done():
+		default:
+			t.Errorf("<-c[%d].Done() blocked, but shouldn't have", i)
+		}
+		if e := c.Err(); e != Canceled {
+			t.Errorf("c[%d].Err() == %v want %v", i, e, Canceled)
+		}
+	}
+}
+
+func TestParentFinishesChild(t *testing.T) {
+	// Context tree:
+	// parent -> cancelChild
+	// parent -> valueChild -> timerChild
+	parent, cancel := WithCancel(Background())
+	cancelChild, stop := WithCancel(parent)
+	defer stop()
+	valueChild := WithValue(parent, "key", "value")
+	timerChild, stop := WithTimeout(valueChild, 10000*time.Hour)
+	defer stop()
+
+	select {
+	case x := <-parent.Done():
+		t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
+	case x := <-cancelChild.Done():
+		t.Errorf("<-cancelChild.Done() == %v want nothing (it should block)", x)
+	case x := <-timerChild.Done():
+		t.Errorf("<-timerChild.Done() == %v want nothing (it should block)", x)
+	case x := <-valueChild.Done():
+		t.Errorf("<-valueChild.Done() == %v want nothing (it should block)", x)
+	default:
+	}
+
+	// The parent's children should contain the two cancelable children.
+	pc := parent.(*cancelCtx)
+	cc := cancelChild.(*cancelCtx)
+	tc := timerChild.(*timerCtx)
+	pc.mu.Lock()
+	if len(pc.children) != 2 || !pc.children[cc] || !pc.children[tc] {
+		t.Errorf("bad linkage: pc.children = %v, want %v and %v",
+			pc.children, cc, tc)
+	}
+	pc.mu.Unlock()
+
+	if p, ok := parentCancelCtx(cc.Context); !ok || p != pc {
+		t.Errorf("bad linkage: parentCancelCtx(cancelChild.Context) = %v, %v want %v, true", p, ok, pc)
+	}
+	if p, ok := parentCancelCtx(tc.Context); !ok || p != pc {
+		t.Errorf("bad linkage: parentCancelCtx(timerChild.Context) = %v, %v want %v, true", p, ok, pc)
+	}
+
+	cancel()
+
+	pc.mu.Lock()
+	if len(pc.children) != 0 {
+		t.Errorf("pc.cancel didn't clear pc.children = %v", pc.children)
+	}
+	pc.mu.Unlock()
+
+	// parent and children should all be finished.
+	check := func(ctx Context, name string) {
+		select {
+		case <-ctx.Done():
+		default:
+			t.Errorf("<-%s.Done() blocked, but shouldn't have", name)
+		}
+		if e := ctx.Err(); e != Canceled {
+			t.Errorf("%s.Err() == %v want %v", name, e, Canceled)
+		}
+	}
+	check(parent, "parent")
+	check(cancelChild, "cancelChild")
+	check(valueChild, "valueChild")
+	check(timerChild, "timerChild")
+
+	// WithCancel should return a canceled context on a canceled parent.
+	precanceledChild := WithValue(parent, "key", "value")
+	select {
+	case <-precanceledChild.Done():
+	default:
+		t.Errorf("<-precanceledChild.Done() blocked, but shouldn't have")
+	}
+	if e := precanceledChild.Err(); e != Canceled {
+		t.Errorf("precanceledChild.Err() == %v want %v", e, Canceled)
+	}
+}
+
+func TestChildFinishesFirst(t *testing.T) {
+	cancelable, stop := WithCancel(Background())
+	defer stop()
+	for _, parent := range []Context{Background(), cancelable} {
+		child, cancel := WithCancel(parent)
+
+		select {
+		case x := <-parent.Done():
+			t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
+		case x := <-child.Done():
+			t.Errorf("<-child.Done() == %v want nothing (it should block)", x)
+		default:
+		}
+
+		cc := child.(*cancelCtx)
+		pc, pcok := parent.(*cancelCtx) // pcok == false when parent == Background()
+		if p, ok := parentCancelCtx(cc.Context); ok != pcok || (ok && pc != p) {
+			t.Errorf("bad linkage: parentCancelCtx(cc.Context) = %v, %v want %v, %v", p, ok, pc, pcok)
+		}
+
+		if pcok {
+			pc.mu.Lock()
+			if len(pc.children) != 1 || !pc.children[cc] {
+				t.Errorf("bad linkage: pc.children = %v, cc = %v", pc.children, cc)
+			}
+			pc.mu.Unlock()
+		}
+
+		cancel()
+
+		if pcok {
+			pc.mu.Lock()
+			if len(pc.children) != 0 {
+				t.Errorf("child's cancel didn't remove self from pc.children = %v", pc.children)
+			}
+			pc.mu.Unlock()
+		}
+
+		// child should be finished.
+		select {
+		case <-child.Done():
+		default:
+			t.Errorf("<-child.Done() blocked, but shouldn't have")
+		}
+		if e := child.Err(); e != Canceled {
+			t.Errorf("child.Err() == %v want %v", e, Canceled)
+		}
+
+		// parent should not be finished.
+		select {
+		case x := <-parent.Done():
+			t.Errorf("<-parent.Done() == %v want nothing (it should block)", x)
+		default:
+		}
+		if e := parent.Err(); e != nil {
+			t.Errorf("parent.Err() == %v want nil", e)
+		}
+	}
+}
+
+func testDeadline(c Context, wait time.Duration, t *testing.T) {
+	select {
+	case <-time.After(wait):
+		t.Fatalf("context should have timed out")
+	case <-c.Done():
+	}
+	if e := c.Err(); e != DeadlineExceeded {
+		t.Errorf("c.Err() == %v want %v", e, DeadlineExceeded)
+	}
+}
+
+func TestDeadline(t *testing.T) {
+	c, _ := WithDeadline(Background(), time.Now().Add(100*time.Millisecond))
+	if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
+		t.Errorf("c.String() = %q want prefix %q", got, prefix)
+	}
+	testDeadline(c, 200*time.Millisecond, t)
+
+	c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond))
+	o := otherContext{c}
+	testDeadline(o, 200*time.Millisecond, t)
+
+	c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond))
+	o = otherContext{c}
+	c, _ = WithDeadline(o, time.Now().Add(300*time.Millisecond))
+	testDeadline(c, 200*time.Millisecond, t)
+}
+
+func TestTimeout(t *testing.T) {
+	c, _ := WithTimeout(Background(), 100*time.Millisecond)
+	if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) {
+		t.Errorf("c.String() = %q want prefix %q", got, prefix)
+	}
+	testDeadline(c, 200*time.Millisecond, t)
+
+	c, _ = WithTimeout(Background(), 100*time.Millisecond)
+	o := otherContext{c}
+	testDeadline(o, 200*time.Millisecond, t)
+
+	c, _ = WithTimeout(Background(), 100*time.Millisecond)
+	o = otherContext{c}
+	c, _ = WithTimeout(o, 300*time.Millisecond)
+	testDeadline(c, 200*time.Millisecond, t)
+}
+
+func TestCanceledTimeout(t *testing.T) {
+	c, _ := WithTimeout(Background(), 200*time.Millisecond)
+	o := otherContext{c}
+	c, cancel := WithTimeout(o, 400*time.Millisecond)
+	cancel()
+	time.Sleep(100 * time.Millisecond) // let cancelation propagate
+	select {
+	case <-c.Done():
+	default:
+		t.Errorf("<-c.Done() blocked, but shouldn't have")
+	}
+	if e := c.Err(); e != Canceled {
+		t.Errorf("c.Err() == %v want %v", e, Canceled)
+	}
+}
+
+type key1 int
+type key2 int
+
+var k1 = key1(1)
+var k2 = key2(1) // same int as k1, different type
+var k3 = key2(3) // same type as k2, different int
+
+func TestValues(t *testing.T) {
+	check := func(c Context, nm, v1, v2, v3 string) {
+		if v, ok := c.Value(k1).(string); ok == (len(v1) == 0) || v != v1 {
+			t.Errorf(`%s.Value(k1).(string) = %q, %t want %q, %t`, nm, v, ok, v1, len(v1) != 0)
+		}
+		if v, ok := c.Value(k2).(string); ok == (len(v2) == 0) || v != v2 {
+			t.Errorf(`%s.Value(k2).(string) = %q, %t want %q, %t`, nm, v, ok, v2, len(v2) != 0)
+		}
+		if v, ok := c.Value(k3).(string); ok == (len(v3) == 0) || v != v3 {
+			t.Errorf(`%s.Value(k3).(string) = %q, %t want %q, %t`, nm, v, ok, v3, len(v3) != 0)
+		}
+	}
+
+	c0 := Background()
+	check(c0, "c0", "", "", "")
+
+	c1 := WithValue(Background(), k1, "c1k1")
+	check(c1, "c1", "c1k1", "", "")
+
+	if got, want := fmt.Sprint(c1), `context.Background.WithValue(1, "c1k1")`; got != want {
+		t.Errorf("c.String() = %q want %q", got, want)
+	}
+
+	c2 := WithValue(c1, k2, "c2k2")
+	check(c2, "c2", "c1k1", "c2k2", "")
+
+	c3 := WithValue(c2, k3, "c3k3")
+	check(c3, "c2", "c1k1", "c2k2", "c3k3")
+
+	c4 := WithValue(c3, k1, nil)
+	check(c4, "c4", "", "c2k2", "c3k3")
+
+	o0 := otherContext{Background()}
+	check(o0, "o0", "", "", "")
+
+	o1 := otherContext{WithValue(Background(), k1, "c1k1")}
+	check(o1, "o1", "c1k1", "", "")
+
+	o2 := WithValue(o1, k2, "o2k2")
+	check(o2, "o2", "c1k1", "o2k2", "")
+
+	o3 := otherContext{c4}
+	check(o3, "o3", "", "c2k2", "c3k3")
+
+	o4 := WithValue(o3, k3, nil)
+	check(o4, "o4", "", "c2k2", "")
+}
+
+func TestAllocs(t *testing.T) {
+	bg := Background()
+	for _, test := range []struct {
+		desc       string
+		f          func()
+		limit      float64
+		gccgoLimit float64
+	}{
+		{
+			desc:       "Background()",
+			f:          func() { Background() },
+			limit:      0,
+			gccgoLimit: 0,
+		},
+		{
+			desc: fmt.Sprintf("WithValue(bg, %v, nil)", k1),
+			f: func() {
+				c := WithValue(bg, k1, nil)
+				c.Value(k1)
+			},
+			limit:      3,
+			gccgoLimit: 3,
+		},
+		{
+			desc: "WithTimeout(bg, 15*time.Millisecond)",
+			f: func() {
+				c, _ := WithTimeout(bg, 15*time.Millisecond)
+				<-c.Done()
+			},
+			limit:      8,
+			gccgoLimit: 13,
+		},
+		{
+			desc: "WithCancel(bg)",
+			f: func() {
+				c, cancel := WithCancel(bg)
+				cancel()
+				<-c.Done()
+			},
+			limit:      5,
+			gccgoLimit: 8,
+		},
+		{
+			desc: "WithTimeout(bg, 100*time.Millisecond)",
+			f: func() {
+				c, cancel := WithTimeout(bg, 100*time.Millisecond)
+				cancel()
+				<-c.Done()
+			},
+			limit:      8,
+			gccgoLimit: 25,
+		},
+	} {
+		limit := test.limit
+		if runtime.Compiler == "gccgo" {
+			// gccgo does not yet do escape analysis.
+			// TOOD(iant): Remove this when gccgo does do escape analysis.
+			limit = test.gccgoLimit
+		}
+		if n := testing.AllocsPerRun(100, test.f); n > limit {
+			t.Errorf("%s allocs = %f want %d", test.desc, n, int(limit))
+		}
+	}
+}
+
+func TestSimultaneousCancels(t *testing.T) {
+	root, cancel := WithCancel(Background())
+	m := map[Context]CancelFunc{root: cancel}
+	q := []Context{root}
+	// Create a tree of contexts.
+	for len(q) != 0 && len(m) < 100 {
+		parent := q[0]
+		q = q[1:]
+		for i := 0; i < 4; i++ {
+			ctx, cancel := WithCancel(parent)
+			m[ctx] = cancel
+			q = append(q, ctx)
+		}
+	}
+	// Start all the cancels in a random order.
+	var wg sync.WaitGroup
+	wg.Add(len(m))
+	for _, cancel := range m {
+		go func(cancel CancelFunc) {
+			cancel()
+			wg.Done()
+		}(cancel)
+	}
+	// Wait on all the contexts in a random order.
+	for ctx := range m {
+		select {
+		case <-ctx.Done():
+		case <-time.After(1 * time.Second):
+			buf := make([]byte, 10<<10)
+			n := runtime.Stack(buf, true)
+			t.Fatalf("timed out waiting for <-ctx.Done(); stacks:\n%s", buf[:n])
+		}
+	}
+	// Wait for all the cancel functions to return.
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(1 * time.Second):
+		buf := make([]byte, 10<<10)
+		n := runtime.Stack(buf, true)
+		t.Fatalf("timed out waiting for cancel functions; stacks:\n%s", buf[:n])
+	}
+}
+
+func TestInterlockedCancels(t *testing.T) {
+	parent, cancelParent := WithCancel(Background())
+	child, cancelChild := WithCancel(parent)
+	go func() {
+		parent.Done()
+		cancelChild()
+	}()
+	cancelParent()
+	select {
+	case <-child.Done():
+	case <-time.After(1 * time.Second):
+		buf := make([]byte, 10<<10)
+		n := runtime.Stack(buf, true)
+		t.Fatalf("timed out waiting for child.Done(); stacks:\n%s", buf[:n])
+	}
+}
+
+func TestLayersCancel(t *testing.T) {
+	testLayers(t, time.Now().UnixNano(), false)
+}
+
+func TestLayersTimeout(t *testing.T) {
+	testLayers(t, time.Now().UnixNano(), true)
+}
+
+func testLayers(t *testing.T, seed int64, testTimeout bool) {
+	rand.Seed(seed)
+	errorf := func(format string, a ...interface{}) {
+		t.Errorf(fmt.Sprintf("seed=%d: %s", seed, format), a...)
+	}
+	const (
+		timeout   = 200 * time.Millisecond
+		minLayers = 30
+	)
+	type value int
+	var (
+		vals      []*value
+		cancels   []CancelFunc
+		numTimers int
+		ctx       = Background()
+	)
+	for i := 0; i < minLayers || numTimers == 0 || len(cancels) == 0 || len(vals) == 0; i++ {
+		switch rand.Intn(3) {
+		case 0:
+			v := new(value)
+			ctx = WithValue(ctx, v, v)
+			vals = append(vals, v)
+		case 1:
+			var cancel CancelFunc
+			ctx, cancel = WithCancel(ctx)
+			cancels = append(cancels, cancel)
+		case 2:
+			var cancel CancelFunc
+			ctx, cancel = WithTimeout(ctx, timeout)
+			cancels = append(cancels, cancel)
+			numTimers++
+		}
+	}
+	checkValues := func(when string) {
+		for _, key := range vals {
+			if val := ctx.Value(key).(*value); key != val {
+				errorf("%s: ctx.Value(%p) = %p want %p", when, key, val, key)
+			}
+		}
+	}
+	select {
+	case <-ctx.Done():
+		errorf("ctx should not be canceled yet")
+	default:
+	}
+	if s, prefix := fmt.Sprint(ctx), "context.Background."; !strings.HasPrefix(s, prefix) {
+		t.Errorf("ctx.String() = %q want prefix %q", s, prefix)
+	}
+	t.Log(ctx)
+	checkValues("before cancel")
+	if testTimeout {
+		select {
+		case <-ctx.Done():
+		case <-time.After(timeout + timeout/10):
+			errorf("ctx should have timed out")
+		}
+		checkValues("after timeout")
+	} else {
+		cancel := cancels[rand.Intn(len(cancels))]
+		cancel()
+		select {
+		case <-ctx.Done():
+		default:
+			errorf("ctx should be canceled")
+		}
+		checkValues("after cancel")
+	}
+}
+
+func TestCancelRemoves(t *testing.T) {
+	checkChildren := func(when string, ctx Context, want int) {
+		if got := len(ctx.(*cancelCtx).children); got != want {
+			t.Errorf("%s: context has %d children, want %d", when, got, want)
+		}
+	}
+
+	ctx, _ := WithCancel(Background())
+	checkChildren("after creation", ctx, 0)
+	_, cancel := WithCancel(ctx)
+	checkChildren("with WithCancel child ", ctx, 1)
+	cancel()
+	checkChildren("after cancelling WithCancel child", ctx, 0)
+
+	ctx, _ = WithCancel(Background())
+	checkChildren("after creation", ctx, 0)
+	_, cancel = WithTimeout(ctx, 60*time.Minute)
+	checkChildren("with WithTimeout child ", ctx, 1)
+	cancel()
+	checkChildren("after cancelling WithTimeout child", ctx, 0)
+}

Godeps/_workspace/src/golang.org/x/net/context/withtimeout_test.go

@@ -0,0 +1,26 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package context_test
+
+import (
+	"fmt"
+	"time"
+
+	"golang.org/x/net/context"
+)
+
+func ExampleWithTimeout() {
+	// Pass a context with a timeout to tell a blocking function that it
+	// should abandon its work after the timeout elapses.
+	ctx, _ := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	select {
+	case <-time.After(200 * time.Millisecond):
+		fmt.Println("overslept")
+	case <-ctx.Done():
+		fmt.Println(ctx.Err()) // prints "context deadline exceeded"
+	}
+	// Output:
+	// context deadline exceeded
+}

main.go

@@ -32,8 +32,9 @@ import (
 
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/notification"
+	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/retrieval"
-	"github.com/prometheus/prometheus/rules/manager"
+	"github.com/prometheus/prometheus/rules"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/storage/local"
 	"github.com/prometheus/prometheus/storage/remote"
@@ -76,7 +77,8 @@ var (
 )
 
 type prometheus struct {
-	ruleManager         manager.RuleManager
+	queryEngine         *promql.Engine
+	ruleManager         *rules.Manager
 	targetManager       retrieval.TargetManager
 	notificationHandler *notification.NotificationHandler
 	storage             local.Storage
@@ -155,16 +157,18 @@ func NewPrometheus() *prometheus {
 	targetManager := retrieval.NewTargetManager(sampleAppender, conf.GlobalLabels())
 	targetManager.AddTargetsFromConfig(conf)
 
-	ruleManager := manager.NewRuleManager(&manager.RuleManagerOptions{
+	queryEngine := promql.NewEngine(memStorage)
+
+	ruleManager := rules.NewManager(&rules.ManagerOptions{
 		SampleAppender:      sampleAppender,
 		NotificationHandler: notificationHandler,
 		EvaluationInterval:  conf.EvaluationInterval(),
-		Storage:             memStorage,
+		QueryEngine:         queryEngine,
 		PrometheusURL:       web.MustBuildServerURL(*pathPrefix),
 		PathPrefix:          *pathPrefix,
 	})
-	if err := ruleManager.AddRulesFromConfig(conf); err != nil {
-		glog.Error("Error loading rule files: ", err)
+	if err := ruleManager.LoadRuleFiles(conf.Global.GetRuleFile()...); err != nil {
+		glog.Errorf("Error loading rule files: %s", err)
 		os.Exit(1)
 	}
 
@@ -188,8 +192,8 @@ func NewPrometheus() *prometheus {
 	}
 
 	consolesHandler := &web.ConsolesHandler{
-		Storage:    memStorage,
-		PathPrefix: *pathPrefix,
+		QueryEngine: queryEngine,
+		PathPrefix:  *pathPrefix,
 	}
 
 	graphsHandler := &web.GraphsHandler{
@@ -197,8 +201,9 @@ func NewPrometheus() *prometheus {
 	}
 
 	metricsService := &api.MetricsService{
-		Now:     clientmodel.Now,
-		Storage: memStorage,
+		Now:         clientmodel.Now,
+		Storage:     memStorage,
+		QueryEngine: queryEngine,
 	}
 
 	webService := &web.WebService{
@@ -210,6 +215,7 @@ func NewPrometheus() *prometheus {
 	}
 
 	p := &prometheus{
+		queryEngine:         queryEngine,
 		ruleManager:         ruleManager,
 		targetManager:       targetManager,
 		notificationHandler: notificationHandler,
@@ -252,6 +258,7 @@ func (p *prometheus) Serve() {
 
 	p.targetManager.Stop()
 	p.ruleManager.Stop()
+	p.queryEngine.Stop()
 
 	if err := p.storage.Stop(); err != nil {
 		glog.Error("Error stopping local storage: ", err)

promql/analyzer.go

@@ -0,0 +1,170 @@
+// Copyright 2013 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promql
+
+import (
+	"errors"
+	"time"
+
+	"golang.org/x/net/context"
+
+	clientmodel "github.com/prometheus/client_golang/model"
+
+	"github.com/prometheus/prometheus/storage/local"
+)
+
+// An Analyzer traverses an expression and determines which data has to be requested
+// from the storage. It is bound to a context that allows cancellation and timing out.
+type Analyzer struct {
+	// The storage from which to query data.
+	Storage local.Storage
+	// The expression being analyzed.
+	Expr Expr
+	// The time range for evaluation of Expr.
+	Start, End clientmodel.Timestamp
+
+	// The preload times for different query time offsets.
+	offsetPreloadTimes map[time.Duration]preloadTimes
+}
+
+// preloadTimes tracks which instants or ranges to preload for a set of
+// fingerprints. One of these structs is collected for each offset by the query
+// analyzer.
+type preloadTimes struct {
+	// Instants require single samples to be loaded along the entire query
+	// range, with intervals between the samples corresponding to the query
+	// resolution.
+	instants map[clientmodel.Fingerprint]struct{}
+	// Ranges require loading a range of samples at each resolution step,
+	// stretching backwards from the current evaluation timestamp. The length of
+	// the range into the past is given by the duration, as in "foo[5m]".
+	ranges map[clientmodel.Fingerprint]time.Duration
+}
+
+// Analyze the provided expression and attach metrics and fingerprints to data-selecting
+// AST nodes that are later used to preload the data from the storage.
+func (a *Analyzer) Analyze(ctx context.Context) error {
+	a.offsetPreloadTimes = map[time.Duration]preloadTimes{}
+
+	getPreloadTimes := func(offset time.Duration) preloadTimes {
+		if _, ok := a.offsetPreloadTimes[offset]; !ok {
+			a.offsetPreloadTimes[offset] = preloadTimes{
+				instants: map[clientmodel.Fingerprint]struct{}{},
+				ranges:   map[clientmodel.Fingerprint]time.Duration{},
+			}
+		}
+		return a.offsetPreloadTimes[offset]
+	}
+
+	// Retrieve fingerprints and metrics for the required time range for
+	// each metric or matrix selector node.
+	Inspect(a.Expr, func(node Node) bool {
+		switch n := node.(type) {
+		case *VectorSelector:
+			pt := getPreloadTimes(n.Offset)
+			fpts := a.Storage.GetFingerprintsForLabelMatchers(n.LabelMatchers)
+			n.fingerprints = fpts
+			n.metrics = map[clientmodel.Fingerprint]clientmodel.COWMetric{}
+			n.iterators = map[clientmodel.Fingerprint]local.SeriesIterator{}
+			for _, fp := range fpts {
+				// Only add the fingerprint to the instants if not yet present in the
+				// ranges. Ranges always contain more points and span more time than
+				// instants for the same offset.
+				if _, alreadyInRanges := pt.ranges[fp]; !alreadyInRanges {
+					pt.instants[fp] = struct{}{}
+				}
+				n.metrics[fp] = a.Storage.GetMetricForFingerprint(fp)
+			}
+		case *MatrixSelector:
+			pt := getPreloadTimes(n.Offset)
+			fpts := a.Storage.GetFingerprintsForLabelMatchers(n.LabelMatchers)
+			n.fingerprints = fpts
+			n.metrics = map[clientmodel.Fingerprint]clientmodel.COWMetric{}
+			n.iterators = map[clientmodel.Fingerprint]local.SeriesIterator{}
+			for _, fp := range fpts {
+				if pt.ranges[fp] < n.Range {
+					pt.ranges[fp] = n.Range
+					// Delete the fingerprint from the instants. Ranges always contain more
+					// points and span more time than instants, so we don't need to track
+					// an instant for the same fingerprint, should we have one.
+					delete(pt.instants, fp)
+				}
+				n.metrics[fp] = a.Storage.GetMetricForFingerprint(fp)
+			}
+		}
+		return true
+	})
+
+	// Currently we do not return an error but we might place a context check in here
+	// or extend the stage in some other way.
+	return nil
+}
+
+// Prepare the expression evaluation by preloading all required chunks from the storage
+// and setting the respective storage iterators in the AST nodes.
+func (a *Analyzer) Prepare(ctx context.Context) (local.Preloader, error) {
+	const env = "query preparation"
+
+	if a.offsetPreloadTimes == nil {
+		return nil, errors.New("analysis must be performed before preparing query")
+	}
+	var err error
+	// The preloader must not be closed unless an error ocurred as closing
+	// unpins the preloaded chunks.
+	p := a.Storage.NewPreloader()
+	defer func() {
+		if err != nil {
+			p.Close()
+		}
+	}()
+
+	// Preload all analyzed ranges.
+	for offset, pt := range a.offsetPreloadTimes {
+		if err = contextDone(ctx, env); err != nil {
+			return nil, err
+		}
+
+		start := a.Start.Add(-offset)
+		end := a.End.Add(-offset)
+		for fp, rangeDuration := range pt.ranges {
+			err = p.PreloadRange(fp, start.Add(-rangeDuration), end, *stalenessDelta)
+			if err != nil {
+				return nil, err
+			}
+		}
+		for fp := range pt.instants {
+			err = p.PreloadRange(fp, start, end, *stalenessDelta)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// Attach storage iterators to AST nodes.
+	Inspect(a.Expr, func(node Node) bool {
+		switch n := node.(type) {
+		case *VectorSelector:
+			for _, fp := range n.fingerprints {
+				n.iterators[fp] = a.Storage.NewIterator(fp)
+			}
+		case *MatrixSelector:
+			for _, fp := range n.fingerprints {
+				n.iterators[fp] = a.Storage.NewIterator(fp)
+			}
+		}
+		return true
+	})
+
+	return p, nil
+}

promql/ast.go

@@ -0,0 +1,345 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promql
+
+import (
+	"fmt"
+	"time"
+
+	clientmodel "github.com/prometheus/client_golang/model"
+
+	"github.com/prometheus/prometheus/storage/local"
+	"github.com/prometheus/prometheus/storage/metric"
+)
+
+// Node is a generic interface for all nodes in an AST.
+//
+// Whenever numerous nodes are listed such as in a switch-case statement
+// or a chain of function definitions (e.g. String(), expr(), etc.) convention is
+// to list them as follows:
+//
+// 	- Statements
+// 	- statement types (alphabetical)
+// 	- ...
+// 	- Expressions
+// 	- expression types (alphabetical)
+// 	- ...
+//
+type Node interface {
+	// String representation of the node that returns the given node when parsed
+	// as part of a valid query.
+	String() string
+	// DotGraph returns a dot graph representation of the node.
+	DotGraph() string
+}
+
+// Statement is a generic interface for all statements.
+type Statement interface {
+	Node
+
+	// stmt ensures that no other type accidentally implements the interface
+	stmt()
+}
+
+// Statements is a list of statement nodes that implements Node.
+type Statements []Statement
+
+// AlertStmt represents an added alert rule.
+type AlertStmt struct {
+	Name        string
+	Expr        Expr
+	Duration    time.Duration
+	Labels      clientmodel.LabelSet
+	Summary     string
+	Description string
+}
+
+// EvalStmt holds an expression and information on the range it should
+// be evaluated on.
+type EvalStmt struct {
+	Expr Expr // Expression to be evaluated.
+
+	// The time boundaries for the evaluation. If Start equals End an instant
+	// is evaluated.
+	Start, End clientmodel.Timestamp
+	// Time between two evaluated instants for the range [Start:End].
+	Interval time.Duration
+}
+
+// RecordStmt represents an added recording rule.
+type RecordStmt struct {
+	Name   string
+	Expr   Expr
+	Labels clientmodel.LabelSet
+}
+
+func (*AlertStmt) stmt()  {}
+func (*EvalStmt) stmt()   {}
+func (*RecordStmt) stmt() {}
+
+// ExprType is the type an evaluated expression returns.
+type ExprType int
+
+const (
+	ExprNone ExprType = iota
+	ExprScalar
+	ExprVector
+	ExprMatrix
+	ExprString
+)
+
+func (e ExprType) String() string {
+	switch e {
+	case ExprNone:
+		return "<ExprNone>"
+	case ExprScalar:
+		return "scalar"
+	case ExprVector:
+		return "vector"
+	case ExprMatrix:
+		return "matrix"
+	case ExprString:
+		return "string"
+	}
+	panic("promql.ExprType.String: unhandled expression type")
+}
+
+// Expr is a generic interface for all expression types.
+type Expr interface {
+	Node
+
+	// Type returns the type the expression evaluates to. It does not perform
+	// in-depth checks as this is done at parsing-time.
+	Type() ExprType
+	// expr ensures that no other types accidentally implement the interface.
+	expr()
+}
+
+// Expressions is a list of expression nodes that implements Node.
+type Expressions []Expr
+
+// AggregateExpr represents an aggregation operation on a vector.
+type AggregateExpr struct {
+	Op              itemType               // The used aggregation operation.
+	Expr            Expr                   // The vector expression over which is aggregated.
+	Grouping        clientmodel.LabelNames // The labels by which to group the vector.
+	KeepExtraLabels bool                   // Whether to keep extra labels common among result elements.
+}
+
+// BinaryExpr represents a binary expression between two child expressions.
+type BinaryExpr struct {
+	Op       itemType // The operation of the expression.
+	LHS, RHS Expr     // The operands on the respective sides of the operator.
+
+	// The matching behavior for the operation if both operands are vectors.
+	// If they are not this field is nil.
+	VectorMatching *VectorMatching
+}
+
+// Call represents a function call.
+type Call struct {
+	Func *Function   // The function that was called.
+	Args Expressions // Arguments used in the call.
+}
+
+// MatrixSelector represents a matrix selection.
+type MatrixSelector struct {
+	Name          string
+	Range         time.Duration
+	Offset        time.Duration
+	LabelMatchers metric.LabelMatchers
+
+	// The series iterators are populated at query analysis time.
+	iterators map[clientmodel.Fingerprint]local.SeriesIterator
+	metrics   map[clientmodel.Fingerprint]clientmodel.COWMetric
+	// Fingerprints are populated from label matchers at query analysis time.
+	fingerprints clientmodel.Fingerprints
+}
+
+// NumberLiteral represents a number.
+type NumberLiteral struct {
+	Val clientmodel.SampleValue
+}
+
+// ParenExpr wraps an expression so it cannot be disassembled as a consequence
+// of operator precendence.
+type ParenExpr struct {
+	Expr Expr
+}
+
+// StringLiteral represents a string.
+type StringLiteral struct {
+	Val string
+}
+
+// UnaryExpr represents a unary operation on another expression.
+// Currently unary operations are only supported for scalars.
+type UnaryExpr struct {
+	Op   itemType
+	Expr Expr
+}
+
+// VectorSelector represents a vector selection.
+type VectorSelector struct {
+	Name          string
+	Offset        time.Duration
+	LabelMatchers metric.LabelMatchers
+
+	// The series iterators are populated at query analysis time.
+	iterators map[clientmodel.Fingerprint]local.SeriesIterator
+	metrics   map[clientmodel.Fingerprint]clientmodel.COWMetric
+	// Fingerprints are populated from label matchers at query analysis time.
+	fingerprints clientmodel.Fingerprints
+}
+
+func (e *AggregateExpr) Type() ExprType  { return ExprVector }
+func (e *Call) Type() ExprType           { return e.Func.ReturnType }
+func (e *MatrixSelector) Type() ExprType { return ExprMatrix }
+func (e *NumberLiteral) Type() ExprType  { return ExprScalar }
+func (e *ParenExpr) Type() ExprType      { return e.Expr.Type() }
+func (e *StringLiteral) Type() ExprType  { return ExprString }
+func (e *UnaryExpr) Type() ExprType      { return e.Expr.Type() }
+func (e *VectorSelector) Type() ExprType { return ExprVector }
+
+func (e *BinaryExpr) Type() ExprType {
+	if e.LHS.Type() == ExprScalar && e.RHS.Type() == ExprScalar {
+		return ExprScalar
+	}
+	return ExprVector
+}
+
+func (*AggregateExpr) expr()  {}
+func (*BinaryExpr) expr()     {}
+func (*Call) expr()           {}
+func (*MatrixSelector) expr() {}
+func (*NumberLiteral) expr()  {}
+func (*ParenExpr) expr()      {}
+func (*StringLiteral) expr()  {}
+func (*UnaryExpr) expr()      {}
+func (*VectorSelector) expr() {}
+
+// VectorMatchCardinaly describes the cardinality relationship
+// of two vectors in a binary operation.
+type VectorMatchCardinality int
+
+const (
+	CardOneToOne VectorMatchCardinality = iota
+	CardManyToOne
+	CardOneToMany
+	CardManyToMany
+)
+
+func (vmc VectorMatchCardinality) String() string {
+	switch vmc {
+	case CardOneToOne:
+		return "one-to-one"
+	case CardManyToOne:
+		return "many-to-one"
+	case CardOneToMany:
+		return "one-to-many"
+	case CardManyToMany:
+		return "many-to-many"
+	}
+	panic("promql.VectorMatchCardinality.String: unknown match cardinality")
+}
+
+// VectorMatching describes how elements from two vectors in a binary
+// operation are supposed to be matched.
+type VectorMatching struct {
+	// The cardinality of the two vectors.
+	Card VectorMatchCardinality
+	// On contains the labels which define equality of a pair
+	// of elements from the vectors.
+	On clientmodel.LabelNames
+	// Include contains additional labels that should be included in
+	// the result from the side with the higher cardinality.
+	Include clientmodel.LabelNames
+}
+
+// A Visitor's Visit method is invoked for each node encountered by Walk.
+// If the result visitor w is not nil, Walk visits each of the children
+// of node with the visitor w, followed by a call of w.Visit(nil).
+type Visitor interface {
+	Visit(node Node) (w Visitor)
+}
+
+// Walk traverses an AST in depth-first order: It starts by calling
+// v.Visit(node); node must not be nil. If the visitor w returned by
+// v.Visit(node) is not nil, Walk is invoked recursively with visitor
+// w for each of the non-nil children of node, followed by a call of
+// w.Visit(nil).
+func Walk(v Visitor, node Node) {
+	if v = v.Visit(node); v == nil {
+		return
+	}
+
+	switch n := node.(type) {
+	case Statements:
+		for _, s := range n {
+			Walk(v, s)
+		}
+	case *AlertStmt:
+		Walk(v, n.Expr)
+
+	case *EvalStmt:
+		Walk(v, n.Expr)
+
+	case *RecordStmt:
+		Walk(v, n.Expr)
+
+	case Expressions:
+		for _, e := range n {
+			Walk(v, e)
+		}
+	case *AggregateExpr:
+		Walk(v, n.Expr)
+
+	case *BinaryExpr:
+		Walk(v, n.LHS)
+		Walk(v, n.RHS)
+
+	case *Call:
+		Walk(v, n.Args)
+
+	case *ParenExpr:
+		Walk(v, n.Expr)
+
+	case *UnaryExpr:
+		Walk(v, n.Expr)
+
+	case *MatrixSelector, *NumberLiteral, *StringLiteral, *VectorSelector:
+		// nothing to do
+
+	default:
+		panic(fmt.Errorf("promql.Walk: unhandled node type %T", node))
+	}
+
+	v.Visit(nil)
+}
+
+type inspector func(Node) bool
+
+func (f inspector) Visit(node Node) Visitor {
+	if f(node) {
+		return f
+	}
+	return nil
+}
+
+// Inspect traverses an AST in depth-first order: It starts by calling
+// f(node); node must not be nil. If f returns true, Inspect invokes f
+// for all the non-nil children of node, recursively.
+func Inspect(node Node, f func(Node) bool) {
+	Walk(inspector(f), node)
+}

promql/engine_test.go

@@ -0,0 +1,169 @@
+package promql
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/net/context"
+)
+
+var noop = testStmt(func(context.Context) error {
+	return nil
+})
+
+func TestQueryConcurreny(t *testing.T) {
+	engine := NewEngine(nil)
+	defer engine.Stop()
+
+	block := make(chan struct{})
+	processing := make(chan struct{})
+	f1 := testStmt(func(context.Context) error {
+		processing <- struct{}{}
+		<-block
+		return nil
+	})
+
+	for i := 0; i < *maxConcurrentQueries; i++ {
+		q := engine.newTestQuery(f1)
+		go q.Exec()
+		select {
+		case <-processing:
+			// Expected.
+		case <-time.After(5 * time.Millisecond):
+			t.Fatalf("Query within concurrency threshold not being executed")
+		}
+	}
+
+	q := engine.newTestQuery(f1)
+	go q.Exec()
+
+	select {
+	case <-processing:
+		t.Fatalf("Query above concurrency threhosld being executed")
+	case <-time.After(5 * time.Millisecond):
+		// Expected.
+	}
+
+	// Terminate a running query.
+	block <- struct{}{}
+
+	select {
+	case <-processing:
+		// Expected.
+	case <-time.After(5 * time.Millisecond):
+		t.Fatalf("Query within concurrency threshold not being executed")
+	}
+
+	// Terminate remaining queries.
+	for i := 0; i < *maxConcurrentQueries; i++ {
+		block <- struct{}{}
+	}
+}
+
+func TestQueryTimeout(t *testing.T) {
+	*defaultQueryTimeout = 5 * time.Millisecond
+	defer func() {
+		// Restore default query timeout
+		*defaultQueryTimeout = 2 * time.Minute
+	}()
+
+	engine := NewEngine(nil)
+	defer engine.Stop()
+
+	f1 := testStmt(func(context.Context) error {
+		time.Sleep(10 * time.Millisecond)
+		return nil
+	})
+
+	// Timeouts are not exact but checked in designated places. For example between
+	// invoking test statements.
+	query := engine.newTestQuery(f1, f1)
+
+	res := query.Exec()
+	if res.Err == nil {
+		t.Fatalf("expected timeout error but got none")
+	}
+	if _, ok := res.Err.(ErrQueryTimeout); res.Err != nil && !ok {
+		t.Fatalf("expected timeout error but got: %s", res.Err)
+	}
+}
+
+func TestQueryCancel(t *testing.T) {
+	engine := NewEngine(nil)
+	defer engine.Stop()
+
+	// As for timeouts, cancellation is only checked at designated points. We ensure
+	// that we reach one of those points using the same method.
+	f1 := testStmt(func(context.Context) error {
+		time.Sleep(2 * time.Millisecond)
+		return nil
+	})
+
+	query1 := engine.newTestQuery(f1, f1)
+	query2 := engine.newTestQuery(f1, f1)
+
+	// Cancel query after starting it.
+	var wg sync.WaitGroup
+	var res *Result
+
+	wg.Add(1)
+	go func() {
+		res = query1.Exec()
+		wg.Done()
+	}()
+	time.Sleep(1 * time.Millisecond)
+	query1.Cancel()
+	wg.Wait()
+
+	if res.Err == nil {
+		t.Fatalf("expected cancellation error for query1 but got none")
+	}
+	if _, ok := res.Err.(ErrQueryCanceled); res.Err != nil && !ok {
+		t.Fatalf("expected cancellation error for query1 but got: %s", res.Err)
+	}
+
+	// Canceling query before starting it must have no effect.
+	query2.Cancel()
+	res = query2.Exec()
+	if res.Err != nil {
+		t.Fatalf("unexpeceted error on executing query2: %s", res.Err)
+	}
+}
+
+func TestEngineShutdown(t *testing.T) {
+	engine := NewEngine(nil)
+
+	handlerExecutions := 0
+	// Shutdown engine on first handler execution. Should handler execution ever become
+	// concurrent this test has to be adjusted accordingly.
+	f1 := testStmt(func(context.Context) error {
+		handlerExecutions++
+		engine.Stop()
+		time.Sleep(10 * time.Millisecond)
+		return nil
+	})
+	query1 := engine.newTestQuery(f1, f1)
+	query2 := engine.newTestQuery(f1, f1)
+
+	// Stopping the engine must cancel the base context. While executing queries is
+	// still possible, their context is canceled from the beginning and execution should
+	// terminate immediately.
+
+	res := query1.Exec()
+	if res.Err == nil {
+		t.Fatalf("expected error on shutdown during query but got none")
+	}
+	if handlerExecutions != 1 {
+		t.Fatalf("expected only one handler to be executed before query cancellation but got %d executions", handlerExecutions)
+	}
+
+	res2 := query2.Exec()
+	if res2.Err == nil {
+		t.Fatalf("expected error on querying shutdown engine but got none")
+	}
+	if handlerExecutions != 1 {
+		t.Fatalf("expected no handler execution for query after engine shutdown")
+	}
+
+}

promql/functions.go

@@ -0,0 +1,716 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promql
+
+import (
+	"container/heap"
+	"math"
+	"sort"
+	"strconv"
+	"time"
+
+	clientmodel "github.com/prometheus/client_golang/model"
+
+	"github.com/prometheus/prometheus/storage/metric"
+)
+
+// Function represents a function of the expression language and is
+// used by function nodes.
+type Function struct {
+	Name         string
+	ArgTypes     []ExprType
+	OptionalArgs int
+	ReturnType   ExprType
+	Call         func(ev *evaluator, args Expressions) Value
+}
+
+// === time() clientmodel.SampleValue ===
+func funcTime(ev *evaluator, args Expressions) Value {
+	return &Scalar{
+		Value:     clientmodel.SampleValue(ev.Timestamp.Unix()),
+		Timestamp: ev.Timestamp,
+	}
+}
+
+// === delta(matrix ExprMatrix, isCounter=0 ExprScalar) Vector ===
+func funcDelta(ev *evaluator, args Expressions) Value {
+	isCounter := len(args) >= 2 && ev.evalInt(args[1]) > 0
+	resultVector := Vector{}
+
+	// If we treat these metrics as counters, we need to fetch all values
+	// in the interval to find breaks in the timeseries' monotonicity.
+	// I.e. if a counter resets, we want to ignore that reset.
+	var matrixValue Matrix
+	if isCounter {
+		matrixValue = ev.evalMatrix(args[0])
+	} else {
+		matrixValue = ev.evalMatrixBounds(args[0])
+	}
+	for _, samples := range matrixValue {
+		// No sense in trying to compute a delta without at least two points. Drop
+		// this vector element.
+		if len(samples.Values) < 2 {
+			continue
+		}
+
+		counterCorrection := clientmodel.SampleValue(0)
+		lastValue := clientmodel.SampleValue(0)
+		for _, sample := range samples.Values {
+			currentValue := sample.Value
+			if isCounter && currentValue < lastValue {
+				counterCorrection += lastValue - currentValue
+			}
+			lastValue = currentValue
+		}
+		resultValue := lastValue - samples.Values[0].Value + counterCorrection
+
+		targetInterval := args[0].(*MatrixSelector).Range
+		sampledInterval := samples.Values[len(samples.Values)-1].Timestamp.Sub(samples.Values[0].Timestamp)
+		if sampledInterval == 0 {
+			// Only found one sample. Cannot compute a rate from this.
+			continue
+		}
+		// Correct for differences in target vs. actual delta interval.
+		//
+		// Above, we didn't actually calculate the delta for the specified target
+		// interval, but for an interval between the first and last found samples
+		// under the target interval, which will usually have less time between
+		// them. Depending on how many samples are found under a target interval,
+		// the delta results are distorted and temporal aliasing occurs (ugly
+		// bumps). This effect is corrected for below.
+		intervalCorrection := clientmodel.SampleValue(targetInterval) / clientmodel.SampleValue(sampledInterval)
+		resultValue *= intervalCorrection
+
+		resultSample := &Sample{
+			Metric:    samples.Metric,
+			Value:     resultValue,
+			Timestamp: ev.Timestamp,
+		}
+		resultSample.Metric.Delete(clientmodel.MetricNameLabel)
+		resultVector = append(resultVector, resultSample)
+	}
+	return resultVector
+}
+
+// === rate(node ExprMatrix) Vector ===
+func funcRate(ev *evaluator, args Expressions) Value {
+	args = append(args, &NumberLiteral{1})
+	vector := funcDelta(ev, args).(Vector)
+
+	// TODO: could be other type of ExprMatrix in the future (right now, only
+	// MatrixSelector exists). Find a better way of getting the duration of a
+	// matrix, such as looking at the samples themselves.
+	interval := args[0].(*MatrixSelector).Range
+	for i := range vector {
+		vector[i].Value /= clientmodel.SampleValue(interval / time.Second)
+	}
+	return vector
+}
+
+// === sort(node ExprVector) Vector ===
+func funcSort(ev *evaluator, args Expressions) Value {
+	byValueSorter := vectorByValueHeap(ev.evalVector(args[0]))
+	sort.Sort(byValueSorter)
+	return Vector(byValueSorter)
+}
+
+// === sortDesc(node ExprVector) Vector ===
+func funcSortDesc(ev *evaluator, args Expressions) Value {
+	byValueSorter := vectorByValueHeap(ev.evalVector(args[0]))
+	sort.Sort(sort.Reverse(byValueSorter))
+	return Vector(byValueSorter)
+}
+
+// === topk(k ExprScalar, node ExprVector) Vector ===
+func funcTopk(ev *evaluator, args Expressions) Value {
+	k := ev.evalInt(args[0])
+	if k < 1 {
+		return Vector{}
+	}
+	vector := ev.evalVector(args[1])
+
+	topk := make(vectorByValueHeap, 0, k)
+
+	for _, el := range vector {
+		if len(topk) < k || topk[0].Value < el.Value {
+			if len(topk) == k {
+				heap.Pop(&topk)
+			}
+			heap.Push(&topk, el)
+		}
+	}
+	sort.Sort(sort.Reverse(topk))
+	return Vector(topk)
+}
+
+// === bottomk(k ExprScalar, node ExprVector) Vector ===
+func funcBottomk(ev *evaluator, args Expressions) Value {
+	k := ev.evalInt(args[0])
+	if k < 1 {
+		return Vector{}
+	}
+	vector := ev.evalVector(args[1])
+
+	bottomk := make(vectorByValueHeap, 0, k)
+	bkHeap := reverseHeap{Interface: &bottomk}
+
+	for _, el := range vector {
+		if len(bottomk) < k || bottomk[0].Value > el.Value {
+			if len(bottomk) == k {
+				heap.Pop(&bkHeap)
+			}
+			heap.Push(&bkHeap, el)
+		}
+	}
+	sort.Sort(bottomk)
+	return Vector(bottomk)
+}
+
+// === drop_common_labels(node ExprVector) Vector ===
+func funcDropCommonLabels(ev *evaluator, args Expressions) Value {
+	vector := ev.evalVector(args[0])
+	if len(vector) < 1 {
+		return Vector{}
+	}
+	common := clientmodel.LabelSet{}
+	for k, v := range vector[0].Metric.Metric {
+		// TODO(julius): Should we also drop common metric names?
+		if k == clientmodel.MetricNameLabel {
+			continue
+		}
+		common[k] = v
+	}
+
+	for _, el := range vector[1:] {
+		for k, v := range common {
+			if el.Metric.Metric[k] != v {
+				// Deletion of map entries while iterating over them is safe.
+				// From http://golang.org/ref/spec#For_statements:
+				// "If map entries that have not yet been reached are deleted during
+				// iteration, the corresponding iteration values will not be produced."
+				delete(common, k)
+			}
+		}
+	}
+
+	for _, el := range vector {
+		for k := range el.Metric.Metric {
+			if _, ok := common[k]; ok {
+				el.Metric.Delete(k)
+			}
+		}
+	}
+	return vector
+}
+
+// === round(vector ExprVector, toNearest=1 Scalar) Vector ===
+func funcRound(ev *evaluator, args Expressions) Value {
+	// round returns a number rounded to toNearest.
+	// Ties are solved by rounding up.
+	toNearest := float64(1)
+	if len(args) >= 2 {
+		toNearest = ev.evalFloat(args[1])
+	}
+	// Invert as it seems to cause fewer floating point accuracy issues.
+	toNearestInverse := 1.0 / toNearest
+
+	vector := ev.evalVector(args[0])
+	for _, el := range vector {
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		el.Value = clientmodel.SampleValue(math.Floor(float64(el.Value)*toNearestInverse+0.5) / toNearestInverse)
+	}
+	return vector
+}
+
+// === scalar(node ExprVector) Scalar ===
+func funcScalar(ev *evaluator, args Expressions) Value {
+	v := ev.evalVector(args[0])
+	if len(v) != 1 {
+		return &Scalar{clientmodel.SampleValue(math.NaN()), ev.Timestamp}
+	}
+	return &Scalar{clientmodel.SampleValue(v[0].Value), ev.Timestamp}
+}
+
+// === count_scalar(vector ExprVector) model.SampleValue ===
+func funcCountScalar(ev *evaluator, args Expressions) Value {
+	return &Scalar{
+		Value:     clientmodel.SampleValue(len(ev.evalVector(args[0]))),
+		Timestamp: ev.Timestamp,
+	}
+}
+
+func aggrOverTime(ev *evaluator, args Expressions, aggrFn func(metric.Values) clientmodel.SampleValue) Value {
+	matrix := ev.evalMatrix(args[0])
+	resultVector := Vector{}
+
+	for _, el := range matrix {
+		if len(el.Values) == 0 {
+			continue
+		}
+
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		resultVector = append(resultVector, &Sample{
+			Metric:    el.Metric,
+			Value:     aggrFn(el.Values),
+			Timestamp: ev.Timestamp,
+		})
+	}
+	return resultVector
+}
+
+// === avg_over_time(matrix ExprMatrix) Vector ===
+func funcAvgOverTime(ev *evaluator, args Expressions) Value {
+	return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue {
+		var sum clientmodel.SampleValue
+		for _, v := range values {
+			sum += v.Value
+		}
+		return sum / clientmodel.SampleValue(len(values))
+	})
+}
+
+// === count_over_time(matrix ExprMatrix) Vector ===
+func funcCountOverTime(ev *evaluator, args Expressions) Value {
+	return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue {
+		return clientmodel.SampleValue(len(values))
+	})
+}
+
+// === floor(vector ExprVector) Vector ===
+func funcFloor(ev *evaluator, args Expressions) Value {
+	vector := ev.evalVector(args[0])
+	for _, el := range vector {
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		el.Value = clientmodel.SampleValue(math.Floor(float64(el.Value)))
+	}
+	return vector
+}
+
+// === max_over_time(matrix ExprMatrix) Vector ===
+func funcMaxOverTime(ev *evaluator, args Expressions) Value {
+	return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue {
+		max := math.Inf(-1)
+		for _, v := range values {
+			max = math.Max(max, float64(v.Value))
+		}
+		return clientmodel.SampleValue(max)
+	})
+}
+
+// === min_over_time(matrix ExprMatrix) Vector ===
+func funcMinOverTime(ev *evaluator, args Expressions) Value {
+	return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue {
+		min := math.Inf(1)
+		for _, v := range values {
+			min = math.Min(min, float64(v.Value))
+		}
+		return clientmodel.SampleValue(min)
+	})
+}
+
+// === sum_over_time(matrix ExprMatrix) Vector ===
+func funcSumOverTime(ev *evaluator, args Expressions) Value {
+	return aggrOverTime(ev, args, func(values metric.Values) clientmodel.SampleValue {
+		var sum clientmodel.SampleValue
+		for _, v := range values {
+			sum += v.Value
+		}
+		return sum
+	})
+}
+
+// === abs(vector ExprVector) Vector ===
+func funcAbs(ev *evaluator, args Expressions) Value {
+	vector := ev.evalVector(args[0])
+	for _, el := range vector {
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		el.Value = clientmodel.SampleValue(math.Abs(float64(el.Value)))
+	}
+	return vector
+}
+
+// === absent(vector ExprVector) Vector ===
+func funcAbsent(ev *evaluator, args Expressions) Value {
+	if len(ev.evalVector(args[0])) > 0 {
+		return Vector{}
+	}
+	m := clientmodel.Metric{}
+	if vs, ok := args[0].(*VectorSelector); ok {
+		for _, matcher := range vs.LabelMatchers {
+			if matcher.Type == metric.Equal && matcher.Name != clientmodel.MetricNameLabel {
+				m[matcher.Name] = matcher.Value
+			}
+		}
+	}
+	return Vector{
+		&Sample{
+			Metric: clientmodel.COWMetric{
+				Metric: m,
+				Copied: true,
+			},
+			Value:     1,
+			Timestamp: ev.Timestamp,
+		},
+	}
+}
+
+// === ceil(vector ExprVector) Vector ===
+func funcCeil(ev *evaluator, args Expressions) Value {
+	vector := ev.evalVector(args[0])
+	for _, el := range vector {
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		el.Value = clientmodel.SampleValue(math.Ceil(float64(el.Value)))
+	}
+	return vector
+}
+
+// === exp(vector ExprVector) Vector ===
+func funcExp(ev *evaluator, args Expressions) Value {
+	vector := ev.evalVector(args[0])
+	for _, el := range vector {
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		el.Value = clientmodel.SampleValue(math.Exp(float64(el.Value)))
+	}
+	return vector
+}
+
+// === sqrt(vector VectorNode) Vector ===
+func funcSqrt(ev *evaluator, args Expressions) Value {
+	vector := ev.evalVector(args[0])
+	for _, el := range vector {
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		el.Value = clientmodel.SampleValue(math.Sqrt(float64(el.Value)))
+	}
+	return vector
+}
+
+// === ln(vector ExprVector) Vector ===
+func funcLn(ev *evaluator, args Expressions) Value {
+	vector := ev.evalVector(args[0])
+	for _, el := range vector {
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		el.Value = clientmodel.SampleValue(math.Log(float64(el.Value)))
+	}
+	return vector
+}
+
+// === log2(vector ExprVector) Vector ===
+func funcLog2(ev *evaluator, args Expressions) Value {
+	vector := ev.evalVector(args[0])
+	for _, el := range vector {
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		el.Value = clientmodel.SampleValue(math.Log2(float64(el.Value)))
+	}
+	return vector
+}
+
+// === log10(vector ExprVector) Vector ===
+func funcLog10(ev *evaluator, args Expressions) Value {
+	vector := ev.evalVector(args[0])
+	for _, el := range vector {
+		el.Metric.Delete(clientmodel.MetricNameLabel)
+		el.Value = clientmodel.SampleValue(math.Log10(float64(el.Value)))
+	}
+	return vector
+}
+
+// === deriv(node ExprMatrix) Vector ===
+func funcDeriv(ev *evaluator, args Expressions) Value {
+	resultVector := Vector{}
+	matrix := ev.evalMatrix(args[0])
+
+	for _, samples := range matrix {
+		// No sense in trying to compute a derivative without at least two points.
+		// Drop this vector element.
+		if len(samples.Values) < 2 {
+			continue
+		}
+
+		// Least squares.
+		n := clientmodel.SampleValue(0)
+		sumY := clientmodel.SampleValue(0)
+		sumX := clientmodel.SampleValue(0)
+		sumXY := clientmodel.SampleValue(0)
+		sumX2 := clientmodel.SampleValue(0)
+		for _, sample := range samples.Values {
+			x := clientmodel.SampleValue(sample.Timestamp.UnixNano() / 1e9)
+			n += 1.0
+			sumY += sample.Value
+			sumX += x
+			sumXY += x * sample.Value
+			sumX2 += x * x
+		}
+		numerator := sumXY - sumX*sumY/n
+		denominator := sumX2 - (sumX*sumX)/n
+
+		resultValue := numerator / denominator
+
+		resultSample := &Sample{
+			Metric:    samples.Metric,
+			Value:     resultValue,
+			Timestamp: ev.Timestamp,
+		}
+		resultSample.Metric.Delete(clientmodel.MetricNameLabel)
+		resultVector = append(resultVector, resultSample)
+	}
+	return resultVector
+}
+
+// === histogram_quantile(k ExprScalar, vector ExprVector) Vector ===
+func funcHistogramQuantile(ev *evaluator, args Expressions) Value {
+	q := clientmodel.SampleValue(ev.evalFloat(args[0]))
+	inVec := ev.evalVector(args[1])
+
+	outVec := Vector{}
+	signatureToMetricWithBuckets := map[uint64]*metricWithBuckets{}
+	for _, el := range inVec {
+		upperBound, err := strconv.ParseFloat(
+			string(el.Metric.Metric[clientmodel.BucketLabel]), 64,
+		)
+		if err != nil {
+			// Oops, no bucket label or malformed label value. Skip.
+			// TODO(beorn7): Issue a warning somehow.
+			continue
+		}
+		signature := clientmodel.SignatureWithoutLabels(el.Metric.Metric, excludedLabels)
+		mb, ok := signatureToMetricWithBuckets[signature]
+		if !ok {
+			el.Metric.Delete(clientmodel.BucketLabel)
+			el.Metric.Delete(clientmodel.MetricNameLabel)
+			mb = &metricWithBuckets{el.Metric, nil}
+			signatureToMetricWithBuckets[signature] = mb
+		}
+		mb.buckets = append(mb.buckets, bucket{upperBound, el.Value})
+	}
+
+	for _, mb := range signatureToMetricWithBuckets {
+		outVec = append(outVec, &Sample{
+			Metric:    mb.metric,
+			Value:     clientmodel.SampleValue(quantile(q, mb.buckets)),
+			Timestamp: ev.Timestamp,
+		})
+	}
+
+	return outVec
+}
+
+var functions = map[string]*Function{
+	"abs": {
+		Name:       "abs",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcAbs,
+	},
+	"absent": {
+		Name:       "absent",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcAbsent,
+	},
+	"avg_over_time": {
+		Name:       "avg_over_time",
+		ArgTypes:   []ExprType{ExprMatrix},
+		ReturnType: ExprVector,
+		Call:       funcAvgOverTime,
+	},
+	"bottomk": {
+		Name:       "bottomk",
+		ArgTypes:   []ExprType{ExprScalar, ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcBottomk,
+	},
+	"ceil": {
+		Name:       "ceil",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcCeil,
+	},
+	"count_over_time": {
+		Name:       "count_over_time",
+		ArgTypes:   []ExprType{ExprMatrix},
+		ReturnType: ExprVector,
+		Call:       funcCountOverTime,
+	},
+	"count_scalar": {
+		Name:       "count_scalar",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprScalar,
+		Call:       funcCountScalar,
+	},
+	"delta": {
+		Name:         "delta",
+		ArgTypes:     []ExprType{ExprMatrix, ExprScalar},
+		OptionalArgs: 1, // The 2nd argument is deprecated.
+		ReturnType:   ExprVector,
+		Call:         funcDelta,
+	},
+	"deriv": {
+		Name:       "deriv",
+		ArgTypes:   []ExprType{ExprMatrix},
+		ReturnType: ExprVector,
+		Call:       funcDeriv,
+	},
+	"drop_common_labels": {
+		Name:       "drop_common_labels",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcDropCommonLabels,
+	},
+	"exp": {
+		Name:       "exp",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcExp,
+	},
+	"floor": {
+		Name:       "floor",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcFloor,
+	},
+	"histogram_quantile": {
+		Name:       "histogram_quantile",
+		ArgTypes:   []ExprType{ExprScalar, ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcHistogramQuantile,
+	},
+	"ln": {
+		Name:       "ln",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcLn,
+	},
+	"log10": {
+		Name:       "log10",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcLog10,
+	},
+	"log2": {
+		Name:       "log2",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcLog2,
+	},
+	"max_over_time": {
+		Name:       "max_over_time",
+		ArgTypes:   []ExprType{ExprMatrix},
+		ReturnType: ExprVector,
+		Call:       funcMaxOverTime,
+	},
+	"min_over_time": {
+		Name:       "min_over_time",
+		ArgTypes:   []ExprType{ExprMatrix},
+		ReturnType: ExprVector,
+		Call:       funcMinOverTime,
+	},
+	"rate": {
+		Name:       "rate",
+		ArgTypes:   []ExprType{ExprMatrix},
+		ReturnType: ExprVector,
+		Call:       funcRate,
+	},
+	"round": {
+		Name:         "round",
+		ArgTypes:     []ExprType{ExprVector, ExprScalar},
+		OptionalArgs: 1,
+		ReturnType:   ExprVector,
+		Call:         funcRound,
+	},
+	"scalar": {
+		Name:       "scalar",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprScalar,
+		Call:       funcScalar,
+	},
+	"sort": {
+		Name:       "sort",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcSort,
+	},
+	"sort_desc": {
+		Name:       "sort_desc",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcSortDesc,
+	},
+	"sqrt": {
+		Name:       "sqrt",
+		ArgTypes:   []ExprType{ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcSqrt,
+	},
+	"sum_over_time": {
+		Name:       "sum_over_time",
+		ArgTypes:   []ExprType{ExprMatrix},
+		ReturnType: ExprVector,
+		Call:       funcSumOverTime,
+	},
+	"time": {
+		Name:       "time",
+		ArgTypes:   []ExprType{},
+		ReturnType: ExprScalar,
+		Call:       funcTime,
+	},
+	"topk": {
+		Name:       "topk",
+		ArgTypes:   []ExprType{ExprScalar, ExprVector},
+		ReturnType: ExprVector,
+		Call:       funcTopk,
+	},
+}
+
+// getFunction returns a predefined Function object for the given name.
+func getFunction(name string) (*Function, bool) {
+	function, ok := functions[name]
+	return function, ok
+}
+
+type vectorByValueHeap Vector
+
+func (s vectorByValueHeap) Len() int {
+	return len(s)
+}
+
+func (s vectorByValueHeap) Less(i, j int) bool {
+	if math.IsNaN(float64(s[i].Value)) {
+		return true
+	}
+	return s[i].Value < s[j].Value
+}
+
+func (s vectorByValueHeap) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s *vectorByValueHeap) Push(x interface{}) {
+	*s = append(*s, x.(*Sample))
+}
+
+func (s *vectorByValueHeap) Pop() interface{} {
+	old := *s
+	n := len(old)
+	el := old[n-1]
+	*s = old[0 : n-1]
+	return el
+}
+
+type reverseHeap struct {
+	heap.Interface
+}
+
+func (s reverseHeap) Less(i, j int) bool {
+	return s.Interface.Less(j, i)
+}

promql/lex.go

@@ -0,0 +1,707 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promql
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// item represents a token or text string returned from the scanner.
+type item struct {
+	typ itemType // The type of this item.
+	pos Pos      // The starting position, in bytes, of this item in the input string.
+	val string   // The value of this item.
+}
+
+// String returns a descriptive string for the item.
+func (i item) String() string {
+	switch {
+	case i.typ == itemEOF:
+		return "EOF"
+	case i.typ == itemError:
+		return i.val
+	case i.typ == itemIdentifier || i.typ == itemMetricIdentifier:
+		return fmt.Sprintf("%q", i.val)
+	case i.typ.isKeyword():
+		return fmt.Sprintf("<%s>", i.val)
+	case i.typ.isOperator():
+		return fmt.Sprintf("<op:%s>", i.val)
+	case i.typ.isAggregator():
+		return fmt.Sprintf("<aggr:%s>", i.val)
+	case len(i.val) > 10:
+		return fmt.Sprintf("%.10q...", i.val)
+	}
+	return fmt.Sprintf("%q", i.val)
+}
+
+// isOperator returns true if the item corresponds to a logical or arithmetic operator.
+// Returns false otherwise.
+func (i itemType) isOperator() bool { return i > operatorsStart && i < operatorsEnd }
+
+// isAggregator returns true if the item belongs to the aggregator functions.
+// Returns false otherwise
+func (i itemType) isAggregator() bool { return i > aggregatorsStart && i < aggregatorsEnd }
+
+// isKeyword returns true if the item corresponds to a keyword.
+// Returns false otherwise.
+func (i itemType) isKeyword() bool { return i > keywordsStart && i < keywordsEnd }
+
+// Constants for operator precedence in expressions.
+//
+const LowestPrec = 0 // Non-operators.
+
+// Precedence returns the operator precedence of the binary
+// operator op. If op is not a binary operator, the result
+// is LowestPrec.
+func (i itemType) precedence() int {
+	switch i {
+	case itemLOR:
+		return 1
+	case itemLAND:
+		return 2
+	case itemEQL, itemNEQ, itemLTE, itemLSS, itemGTE, itemGTR:
+		return 3
+	case itemADD, itemSUB:
+		return 4
+	case itemMUL, itemDIV, itemMOD:
+		return 5
+	default:
+		return LowestPrec
+	}
+}
+
+type itemType int
+
+const (
+	itemError itemType = iota // Error occurred, value is error message
+	itemEOF
+	itemComment
+	itemIdentifier
+	itemMetricIdentifier
+	itemLeftParen
+	itemRightParen
+	itemLeftBrace
+	itemRightBrace
+	itemLeftBracket
+	itemRightBracket
+	itemComma
+	itemAssign
+	itemSemicolon
+	itemString
+	itemNumber
+	itemDuration
+
+	operatorsStart
+	// Operators.
+	itemSUB
+	itemADD
+	itemMUL
+	itemMOD
+	itemDIV
+	itemLAND
+	itemLOR
+	itemEQL
+	itemNEQ
+	itemLTE
+	itemLSS
+	itemGTE
+	itemGTR
+	itemEQLRegex
+	itemNEQRegex
+	operatorsEnd
+
+	aggregatorsStart
+	// Aggregators.
+	itemAvg
+	itemCount
+	itemSum
+	itemMin
+	itemMax
+	itemStddev
+	itemStdvar
+	aggregatorsEnd
+
+	keywordsStart
+	// Keywords.
+	itemAlert
+	itemIf
+	itemFor
+	itemWith
+	itemSummary
+	itemDescription
+	itemKeepingExtra
+	itemOffset
+	itemBy
+	itemOn
+	itemGroupLeft
+	itemGroupRight
+	keywordsEnd
+)
+
+var key = map[string]itemType{
+	// Operators.
+	"and": itemLAND,
+	"or":  itemLOR,
+
+	// Aggregators.
+	"sum":    itemSum,
+	"avg":    itemAvg,
+	"count":  itemCount,
+	"min":    itemMin,
+	"max":    itemMax,
+	"stddev": itemStddev,
+	"stdvar": itemStdvar,
+
+	// Keywords.
+	"alert":         itemAlert,
+	"if":            itemIf,
+	"for":           itemFor,
+	"with":          itemWith,
+	"summary":       itemSummary,
+	"description":   itemDescription,
+	"offset":        itemOffset,
+	"by":            itemBy,
+	"keeping_extra": itemKeepingExtra,
+	"on":            itemOn,
+	"group_left":    itemGroupLeft,
+	"group_right":   itemGroupRight,
+}
+
+// These are the default string representations for common items. It does not
+// imply that those are the only character sequences that can be lexed to such an item.
+var itemTypeStr = map[itemType]string{
+	itemLeftParen:    "(",
+	itemRightParen:   ")",
+	itemLeftBrace:    "{",
+	itemRightBrace:   "}",
+	itemLeftBracket:  "[",
+	itemRightBracket: "]",
+	itemComma:        ",",
+	itemAssign:       "=",
+	itemSemicolon:    ";",
+
+	itemSUB:      "-",
+	itemADD:      "+",
+	itemMUL:      "*",
+	itemMOD:      "%",
+	itemDIV:      "/",
+	itemEQL:      "==",
+	itemNEQ:      "!=",
+	itemLTE:      "<=",
+	itemLSS:      "<",
+	itemGTE:      ">=",
+	itemGTR:      ">",
+	itemEQLRegex: "=~",
+	itemNEQRegex: "!~",
+}
+
+func init() {
+	// Add keywords to item type strings.
+	for s, ty := range key {
+		itemTypeStr[ty] = s
+	}
+}
+
+func (t itemType) String() string {
+	if s, ok := itemTypeStr[t]; ok {
+		return s
+	}
+	return fmt.Sprintf("<item %d>", t)
+}
+
+func (i item) desc() string {
+	if _, ok := itemTypeStr[i.typ]; ok {
+		return i.String()
+	}
+	if i.typ == itemEOF {
+		return i.typ.desc()
+	}
+	return fmt.Sprintf("%s %s", i.typ.desc(), i)
+}
+
+func (t itemType) desc() string {
+	switch t {
+	case itemError:
+		return "error"
+	case itemEOF:
+		return "end of input"
+	case itemComment:
+		return "comment"
+	case itemIdentifier:
+		return "identifier"
+	case itemMetricIdentifier:
+		return "metric identifier"
+	case itemString:
+		return "string"
+	case itemNumber:
+		return "number"
+	case itemDuration:
+		return "duration"
+	}
+	return fmt.Sprintf("%q", t)
+}
+
+const eof = -1
+
+// stateFn represents the state of the scanner as a function that returns the next state.
+type stateFn func(*lexer) stateFn
+
+// Pos is the position in a string.
+type Pos int
+
+// lexer holds the state of the scanner.
+type lexer struct {
+	input   string    // The string being scanned.
+	state   stateFn   // The next lexing function to enter.
+	pos     Pos       // Current position in the input.
+	start   Pos       // Start position of this item.
+	width   Pos       // Width of last rune read from input.
+	lastPos Pos       // Position of most recent item returned by nextItem.
+	items   chan item // Channel of scanned items.
+
+	parenDepth  int  // Nesting depth of ( ) exprs.
+	braceOpen   bool // Whether a { is opened.
+	bracketOpen bool // Whether a [ is opened.
+	stringOpen  rune // Quote rune of the string currently being read.
+}
+
+// next returns the next rune in the input.
+func (l *lexer) next() rune {
+	if int(l.pos) >= len(l.input) {
+		l.width = 0
+		return eof
+	}
+	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
+	l.width = Pos(w)
+	l.pos += l.width
+	return r
+}
+
+// peek returns but does not consume the next rune in the input.
+func (l *lexer) peek() rune {
+	r := l.next()
+	l.backup()
+	return r
+}
+
+// backup steps back one rune. Can only be called once per call of next.
+func (l *lexer) backup() {
+	l.pos -= l.width
+}
+
+// emit passes an item back to the client.
+func (l *lexer) emit(t itemType) {
+	l.items <- item{t, l.start, l.input[l.start:l.pos]}
+	l.start = l.pos
+}
+
+// ignore skips over the pending input before this point.
+func (l *lexer) ignore() {
+	l.start = l.pos
+}
+
+// accept consumes the next rune if it's from the valid set.
+func (l *lexer) accept(valid string) bool {
+	if strings.IndexRune(valid, l.next()) >= 0 {
+		return true
+	}
+	l.backup()
+	return false
+}
+
+// acceptRun consumes a run of runes from the valid set.
+func (l *lexer) acceptRun(valid string) {
+	for strings.IndexRune(valid, l.next()) >= 0 {
+		// consume
+	}
+	l.backup()
+}
+
+// lineNumber reports which line we're on, based on the position of
+// the previous item returned by nextItem. Doing it this way
+// means we don't have to worry about peek double counting.
+func (l *lexer) lineNumber() int {
+	return 1 + strings.Count(l.input[:l.lastPos], "\n")
+}
+
+// linePosition reports at which character in the current line
+// we are on.
+func (l *lexer) linePosition() int {
+	lb := strings.LastIndex(l.input[:l.lastPos], "\n")
+	if lb == -1 {
+		return 1 + int(l.lastPos)
+	}
+	return 1 + int(l.lastPos) - lb
+}
+
+// errorf returns an error token and terminates the scan by passing
+// back a nil pointer that will be the next state, terminating l.nextItem.
+func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+	l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)}
+	return nil
+}
+
+// nextItem returns the next item from the input.
+func (l *lexer) nextItem() item {
+	item := <-l.items
+	l.lastPos = item.pos
+	return item
+}
+
+// lex creates a new scanner for the input string.
+func lex(input string) *lexer {
+	l := &lexer{
+		input: input,
+		items: make(chan item),
+	}
+	go l.run()
+	return l
+}
+
+// run runs the state machine for the lexer.
+func (l *lexer) run() {
+	for l.state = lexStatements; l.state != nil; {
+		l.state = l.state(l)
+	}
+	close(l.items)
+}
+
+// lineComment is the character that starts a line comment.
+const lineComment = "#"
+
+// lexStatements is the top-level state for lexing.
+func lexStatements(l *lexer) stateFn {
+	if l.braceOpen {
+		return lexInsideBraces
+	}
+	if strings.HasPrefix(l.input[l.pos:], lineComment) {
+		return lexLineComment
+	}
+
+	switch r := l.next(); {
+	case r == eof:
+		if l.parenDepth != 0 {
+			return l.errorf("unclosed left parenthesis")
+		} else if l.bracketOpen {
+			return l.errorf("unclosed left bracket")
+		}
+		l.emit(itemEOF)
+		return nil
+	case r == ',':
+		l.emit(itemComma)
+	case isSpace(r):
+		return lexSpace
+	case r == '*':
+		l.emit(itemMUL)
+	case r == '/':
+		l.emit(itemDIV)
+	case r == '%':
+		l.emit(itemMOD)
+	case r == '+':
+		l.emit(itemADD)
+	case r == '-':
+		l.emit(itemSUB)
+	case r == '=':
+		if t := l.peek(); t == '=' {
+			l.next()
+			l.emit(itemEQL)
+		} else if t == '~' {
+			return l.errorf("unexpected character after '=': %q", t)
+		} else {
+			l.emit(itemAssign)
+		}
+	case r == '!':
+		if t := l.next(); t == '=' {
+			l.emit(itemNEQ)
+		} else {
+			return l.errorf("unexpected character after '!': %q", t)
+		}
+	case r == '<':
+		if t := l.peek(); t == '=' {
+			l.next()
+			l.emit(itemLTE)
+		} else {
+			l.emit(itemLSS)
+		}
+	case r == '>':
+		if t := l.peek(); t == '=' {
+			l.next()
+			l.emit(itemGTE)
+		} else {
+			l.emit(itemGTR)
+		}
+	case unicode.IsDigit(r) || (r == '.' && unicode.IsDigit(l.peek())):
+		l.backup()
+		return lexNumberOrDuration
+	case r == '"' || r == '\'':
+		l.stringOpen = r
+		return lexString
+	case r == 'N' || r == 'n' || r == 'I' || r == 'i':
+		n2 := strings.ToLower(l.input[l.pos:])
+		if len(n2) < 3 || !isAlphaNumeric(rune(n2[2])) {
+			if (r == 'N' || r == 'n') && strings.HasPrefix(n2, "an") {
+				l.pos += 2
+				l.emit(itemNumber)
+				break
+			}
+			if (r == 'I' || r == 'i') && strings.HasPrefix(n2, "nf") {
+				l.pos += 2
+				l.emit(itemNumber)
+				break
+			}
+		}
+		fallthrough
+	case isAlphaNumeric(r) || r == ':':
+		l.backup()
+		return lexKeywordOrIdentifier
+	case r == '(':
+		l.emit(itemLeftParen)
+		l.parenDepth++
+		return lexStatements
+	case r == ')':
+		l.emit(itemRightParen)
+		l.parenDepth--
+		if l.parenDepth < 0 {
+			return l.errorf("unexpected right parenthesis %q", r)
+		}
+		return lexStatements
+	case r == '{':
+		l.emit(itemLeftBrace)
+		l.braceOpen = true
+		return lexInsideBraces(l)
+	case r == '[':
+		if l.bracketOpen {
+			return l.errorf("unexpected left bracket %q", r)
+		}
+		l.emit(itemLeftBracket)
+		l.bracketOpen = true
+		return lexDuration
+	case r == ']':
+		if !l.bracketOpen {
+			return l.errorf("unexpected right bracket %q", r)
+		}
+		l.emit(itemRightBracket)
+		l.bracketOpen = false
+
+	default:
+		return l.errorf("unexpected character: %q", r)
+	}
+	return lexStatements
+}
+
+// lexInsideBraces scans the inside of a vector selector. Keywords are ignored and
+// scanned as identifiers.
+func lexInsideBraces(l *lexer) stateFn {
+	if strings.HasPrefix(l.input[l.pos:], lineComment) {
+		return lexLineComment
+	}
+
+	switch r := l.next(); {
+	case r == eof:
+		return l.errorf("unexpected end of input inside braces")
+	case isSpace(r):
+		return lexSpace
+	case unicode.IsLetter(r) || r == '_':
+		l.backup()
+		return lexIdentifier
+	case r == ',':
+		l.emit(itemComma)
+	case r == '"' || r == '\'':
+		l.stringOpen = r
+		return lexString
+	case r == '=':
+		if l.next() == '~' {
+			l.emit(itemEQLRegex)
+			break
+		}
+		l.backup()
+		l.emit(itemEQL)
+	case r == '!':
+		switch nr := l.next(); {
+		case nr == '~':
+			l.emit(itemNEQRegex)
+		case nr == '=':
+			l.emit(itemNEQ)
+		default:
+			return l.errorf("unexpected character after '!' inside braces: %q", nr)
+		}
+	case r == '{':
+		return l.errorf("unexpected left brace %q", r)
+	case r == '}':
+		l.emit(itemRightBrace)
+		l.braceOpen = false
+		return lexStatements
+	default:
+		return l.errorf("unexpected character inside braces: %q", r)
+	}
+	return lexInsideBraces
+}
+
+// lexString scans a quoted string. The initial quote has already been seen.
+func lexString(l *lexer) stateFn {
+Loop:
+	for {
+		switch l.next() {
+		case '\\':
+			if r := l.next(); r != eof && r != '\n' {
+				break
+			}
+			fallthrough
+		case eof, '\n':
+			return l.errorf("unterminated quoted string")
+		case l.stringOpen:
+			break Loop
+		}
+	}
+	l.emit(itemString)
+	return lexStatements
+}
+
+// lexSpace scans a run of space characters. One space has already been seen.
+func lexSpace(l *lexer) stateFn {
+	for isSpace(l.peek()) {
+		l.next()
+	}
+	l.ignore()
+	return lexStatements
+}
+
+// lexLineComment scans a line comment. Left comment marker is known to be present.
+func lexLineComment(l *lexer) stateFn {
+	l.pos += Pos(len(lineComment))
+	for r := l.next(); !isEndOfLine(r) && r != eof; {
+		r = l.next()
+	}
+	l.backup()
+	l.emit(itemComment)
+	return lexStatements
+}
+
+func lexDuration(l *lexer) stateFn {
+	if l.scanNumber() {
+		return l.errorf("missing unit character in duration")
+	}
+	// Next two chars must be a valid unit and a non-alphanumeric.
+	if l.accept("smhdwy") {
+		if isAlphaNumeric(l.next()) {
+			return l.errorf("bad duration syntax: %q", l.input[l.start:l.pos])
+		}
+		l.backup()
+		l.emit(itemDuration)
+		return lexStatements
+	}
+	return l.errorf("bad duration syntax: %q", l.input[l.start:l.pos])
+}
+
+// lexNumber scans a number: decimal, hex, oct or float.
+func lexNumber(l *lexer) stateFn {
+	if !l.scanNumber() {
+		return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
+	}
+	l.emit(itemNumber)
+	return lexStatements
+}
+
+// lexNumberOrDuration scans a number or a duration item.
+func lexNumberOrDuration(l *lexer) stateFn {
+	if l.scanNumber() {
+		l.emit(itemNumber)
+		return lexStatements
+	}
+	// Next two chars must be a valid unit and a non-alphanumeric.
+	if l.accept("smhdwy") {
+		if isAlphaNumeric(l.next()) {
+			return l.errorf("bad number or duration syntax: %q", l.input[l.start:l.pos])
+		}
+		l.backup()
+		l.emit(itemDuration)
+		return lexStatements
+	}
+	return l.errorf("bad number or duration syntax: %q", l.input[l.start:l.pos])
+}
+
+// scanNumber scans numbers of different formats. The scanned item is
+// not necessarily a valid number. This case is caught by the parser.
+func (l *lexer) scanNumber() bool {
+	digits := "0123456789"
+	if l.accept("0") && l.accept("xX") {
+		digits = "0123456789abcdefABCDEF"
+	}
+	l.acceptRun(digits)
+	if l.accept(".") {
+		l.acceptRun(digits)
+	}
+	if l.accept("eE") {
+		l.accept("+-")
+		l.acceptRun("0123456789")
+	}
+	// Next thing must not be alphanumeric.
+	if isAlphaNumeric(l.peek()) {
+		return false
+	}
+	return true
+}
+
+// lexIdentifier scans an alphanumeric identifier. The next character
+// is known to be a letter.
+func lexIdentifier(l *lexer) stateFn {
+	for isAlphaNumeric(l.next()) {
+		// absorb
+	}
+	l.backup()
+	l.emit(itemIdentifier)
+	return lexStatements
+}
+
+// lexKeywordOrIdentifier scans an alphanumeric identifier which may contain
+// a colon rune. If the identifier is a keyword the respective keyword item
+// is scanned.
+func lexKeywordOrIdentifier(l *lexer) stateFn {
+Loop:
+	for {
+		switch r := l.next(); {
+		case isAlphaNumeric(r) || r == ':':
+			// absorb.
+		default:
+			l.backup()
+			word := l.input[l.start:l.pos]
+			if kw, ok := key[strings.ToLower(word)]; ok {
+				l.emit(kw)
+			} else if !strings.Contains(word, ":") {
+				l.emit(itemIdentifier)
+			} else {
+				l.emit(itemMetricIdentifier)
+			}
+			break Loop
+		}
+	}
+	return lexStatements
+}
+
+func isSpace(r rune) bool {
+	return r == ' ' || r == '\t' || r == '\n'
+}
+
+// isEndOfLine reports whether r is an end-of-line character.
+func isEndOfLine(r rune) bool {
+	return r == '\r' || r == '\n'
+}
+
+// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
+func isAlphaNumeric(r rune) bool {
+	return r == '_' || ('a' <= r && r <= 'z') || ('A' <= r && r <= 'Z') || unicode.IsDigit(r)
+}

promql/lex_test.go

@@ -0,0 +1,359 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promql
+
+import (
+	"reflect"
+	"testing"
+)
+
+var tests = []struct {
+	input    string
+	expected []item
+	fail     bool
+}{
+	// Test common stuff.
+	{
+		input:    ",",
+		expected: []item{{itemComma, 0, ","}},
+	}, {
+		input:    "()",
+		expected: []item{{itemLeftParen, 0, `(`}, {itemRightParen, 1, `)`}},
+	}, {
+		input:    "{}",
+		expected: []item{{itemLeftBrace, 0, `{`}, {itemRightBrace, 1, `}`}},
+	}, {
+		input: "[5m]",
+		expected: []item{
+			{itemLeftBracket, 0, `[`},
+			{itemDuration, 1, `5m`},
+			{itemRightBracket, 3, `]`},
+		},
+	},
+	// Test numbers.
+	{
+		input:    "1",
+		expected: []item{{itemNumber, 0, "1"}},
+	}, {
+		input:    "4.23",
+		expected: []item{{itemNumber, 0, "4.23"}},
+	}, {
+		input:    ".3",
+		expected: []item{{itemNumber, 0, ".3"}},
+	}, {
+		input:    "5.",
+		expected: []item{{itemNumber, 0, "5."}},
+	}, {
+		input:    "NaN",
+		expected: []item{{itemNumber, 0, "NaN"}},
+	}, {
+		input:    "nAN",
+		expected: []item{{itemNumber, 0, "nAN"}},
+	}, {
+		input:    "NaN 123",
+		expected: []item{{itemNumber, 0, "NaN"}, {itemNumber, 4, "123"}},
+	}, {
+		input:    "NaN123",
+		expected: []item{{itemIdentifier, 0, "NaN123"}},
+	}, {
+		input:    "iNf",
+		expected: []item{{itemNumber, 0, "iNf"}},
+	}, {
+		input:    "Inf",
+		expected: []item{{itemNumber, 0, "Inf"}},
+	}, {
+		input:    "+Inf",
+		expected: []item{{itemADD, 0, "+"}, {itemNumber, 1, "Inf"}},
+	}, {
+		input:    "+Inf 123",
+		expected: []item{{itemADD, 0, "+"}, {itemNumber, 1, "Inf"}, {itemNumber, 5, "123"}},
+	}, {
+		input:    "-Inf",
+		expected: []item{{itemSUB, 0, "-"}, {itemNumber, 1, "Inf"}},
+	}, {
+		input:    "Infoo",
+		expected: []item{{itemIdentifier, 0, "Infoo"}},
+	}, {
+		input:    "-Infoo",
+		expected: []item{{itemSUB, 0, "-"}, {itemIdentifier, 1, "Infoo"}},
+	}, {
+		input:    "-Inf 123",
+		expected: []item{{itemSUB, 0, "-"}, {itemNumber, 1, "Inf"}, {itemNumber, 5, "123"}},
+	}, {
+		input:    "0x123",
+		expected: []item{{itemNumber, 0, "0x123"}},
+	},
+	// Test duration.
+	{
+		input:    "5s",
+		expected: []item{{itemDuration, 0, "5s"}},
+	}, {
+		input:    "123m",
+		expected: []item{{itemDuration, 0, "123m"}},
+	}, {
+		input:    "1h",
+		expected: []item{{itemDuration, 0, "1h"}},
+	}, {
+		input:    "3w",
+		expected: []item{{itemDuration, 0, "3w"}},
+	}, {
+		input:    "1y",
+		expected: []item{{itemDuration, 0, "1y"}},
+	},
+	// Test identifiers.
+	{
+		input:    "abc",
+		expected: []item{{itemIdentifier, 0, "abc"}},
+	}, {
+		input:    "a:bc",
+		expected: []item{{itemMetricIdentifier, 0, "a:bc"}},
+	}, {
+		input:    "abc d",
+		expected: []item{{itemIdentifier, 0, "abc"}, {itemIdentifier, 4, "d"}},
+	},
+	// Test comments.
+	{
+		input:    "# some comment",
+		expected: []item{{itemComment, 0, "# some comment"}},
+	}, {
+		input: "5 # 1+1\n5",
+		expected: []item{
+			{itemNumber, 0, "5"},
+			{itemComment, 2, "# 1+1"},
+			{itemNumber, 8, "5"},
+		},
+	},
+	// Test operators.
+	{
+		input:    `=`,
+		expected: []item{{itemAssign, 0, `=`}},
+	}, {
+		// Inside braces equality is a single '=' character.
+		input:    `{=}`,
+		expected: []item{{itemLeftBrace, 0, `{`}, {itemEQL, 1, `=`}, {itemRightBrace, 2, `}`}},
+	}, {
+		input:    `==`,
+		expected: []item{{itemEQL, 0, `==`}},
+	}, {
+		input:    `!=`,
+		expected: []item{{itemNEQ, 0, `!=`}},
+	}, {
+		input:    `<`,
+		expected: []item{{itemLSS, 0, `<`}},
+	}, {
+		input:    `>`,
+		expected: []item{{itemGTR, 0, `>`}},
+	}, {
+		input:    `>=`,
+		expected: []item{{itemGTE, 0, `>=`}},
+	}, {
+		input:    `<=`,
+		expected: []item{{itemLTE, 0, `<=`}},
+	}, {
+		input:    `+`,
+		expected: []item{{itemADD, 0, `+`}},
+	}, {
+		input:    `-`,
+		expected: []item{{itemSUB, 0, `-`}},
+	}, {
+		input:    `*`,
+		expected: []item{{itemMUL, 0, `*`}},
+	}, {
+		input:    `/`,
+		expected: []item{{itemDIV, 0, `/`}},
+	}, {
+		input:    `%`,
+		expected: []item{{itemMOD, 0, `%`}},
+	}, {
+		input:    `AND`,
+		expected: []item{{itemLAND, 0, `AND`}},
+	}, {
+		input:    `or`,
+		expected: []item{{itemLOR, 0, `or`}},
+	},
+	// Test aggregators.
+	{
+		input:    `sum`,
+		expected: []item{{itemSum, 0, `sum`}},
+	}, {
+		input:    `AVG`,
+		expected: []item{{itemAvg, 0, `AVG`}},
+	}, {
+		input:    `MAX`,
+		expected: []item{{itemMax, 0, `MAX`}},
+	}, {
+		input:    `min`,
+		expected: []item{{itemMin, 0, `min`}},
+	}, {
+		input:    `count`,
+		expected: []item{{itemCount, 0, `count`}},
+	}, {
+		input:    `stdvar`,
+		expected: []item{{itemStdvar, 0, `stdvar`}},
+	}, {
+		input:    `stddev`,
+		expected: []item{{itemStddev, 0, `stddev`}},
+	},
+	// Test keywords.
+	{
+		input:    "alert",
+		expected: []item{{itemAlert, 0, "alert"}},
+	}, {
+		input:    "keeping_extra",
+		expected: []item{{itemKeepingExtra, 0, "keeping_extra"}},
+	}, {
+		input:    "if",
+		expected: []item{{itemIf, 0, "if"}},
+	}, {
+		input:    "for",
+		expected: []item{{itemFor, 0, "for"}},
+	}, {
+		input:    "with",
+		expected: []item{{itemWith, 0, "with"}},
+	}, {
+		input:    "description",
+		expected: []item{{itemDescription, 0, "description"}},
+	}, {
+		input:    "summary",
+		expected: []item{{itemSummary, 0, "summary"}},
+	}, {
+		input:    "offset",
+		expected: []item{{itemOffset, 0, "offset"}},
+	}, {
+		input:    "by",
+		expected: []item{{itemBy, 0, "by"}},
+	}, {
+		input:    "on",
+		expected: []item{{itemOn, 0, "on"}},
+	}, {
+		input:    "group_left",
+		expected: []item{{itemGroupLeft, 0, "group_left"}},
+	}, {
+		input:    "group_right",
+		expected: []item{{itemGroupRight, 0, "group_right"}},
+	},
+	// Test Selector.
+	{
+		input: `台北`,
+		fail:  true,
+	}, {
+		input: `{foo="bar"}`,
+		expected: []item{
+			{itemLeftBrace, 0, `{`},
+			{itemIdentifier, 1, `foo`},
+			{itemEQL, 4, `=`},
+			{itemString, 5, `"bar"`},
+			{itemRightBrace, 10, `}`},
+		},
+	}, {
+		input: `{NaN	!= "bar" }`,
+		expected: []item{
+			{itemLeftBrace, 0, `{`},
+			{itemIdentifier, 1, `NaN`},
+			{itemNEQ, 5, `!=`},
+			{itemString, 8, `"bar"`},
+			{itemRightBrace, 14, `}`},
+		},
+	}, {
+		input: `{alert=~"bar" }`,
+		expected: []item{
+			{itemLeftBrace, 0, `{`},
+			{itemIdentifier, 1, `alert`},
+			{itemEQLRegex, 6, `=~`},
+			{itemString, 8, `"bar"`},
+			{itemRightBrace, 14, `}`},
+		},
+	}, {
+		input: `{on!~"bar"}`,
+		expected: []item{
+			{itemLeftBrace, 0, `{`},
+			{itemIdentifier, 1, `on`},
+			{itemNEQRegex, 3, `!~`},
+			{itemString, 5, `"bar"`},
+			{itemRightBrace, 10, `}`},
+		},
+	}, {
+		input: `{alert!#"bar"}`, fail: true,
+	}, {
+		input: `{foo:a="bar"}`, fail: true,
+	},
+	// Test common errors.
+	{
+		input: `=~`, fail: true,
+	}, {
+		input: `!~`, fail: true,
+	}, {
+		input: `!(`, fail: true,
+	}, {
+		input: "1a", fail: true,
+	},
+	// Test mismatched parens.
+	{
+		input: `(`, fail: true,
+	}, {
+		input: `())`, fail: true,
+	}, {
+		input: `(()`, fail: true,
+	}, {
+		input: `{`, fail: true,
+	}, {
+		input: `}`, fail: true,
+	}, {
+		input: "{{", fail: true,
+	}, {
+		input: "{{}}", fail: true,
+	}, {
+		input: `[`, fail: true,
+	}, {
+		input: `[[`, fail: true,
+	}, {
+		input: `[]]`, fail: true,
+	}, {
+		input: `[[]]`, fail: true,
+	}, {
+		input: `]`, fail: true,
+	},
+}
+
+// TestLexer tests basic functionality of the lexer. More elaborate tests are implemented
+// for the parser to avoid duplicated effort.
+func TestLexer(t *testing.T) {
+	for i, test := range tests {
+		l := lex(test.input)
+
+		out := []item{}
+		for it := range l.items {
+			out = append(out, it)
+		}
+
+		lastItem := out[len(out)-1]
+		if test.fail {
+			if lastItem.typ != itemError {
+				t.Fatalf("%d: expected lexing error but did not fail", i)
+			}
+			continue
+		}
+		if lastItem.typ == itemError {
+			t.Fatalf("%d: unexpected lexing error: %s", i, lastItem)
+		}
+
+		if !reflect.DeepEqual(lastItem, item{itemEOF, Pos(len(test.input)), ""}) {
+			t.Fatalf("%d: lexing error: expected output to end with EOF item", i)
+		}
+		out = out[:len(out)-1]
+		if !reflect.DeepEqual(out, test.expected) {
+			t.Errorf("%d: lexing mismatch:\nexpected: %#v\n-----\ngot: %#v", i, test.expected, out)
+		}
+	}
+}

promql/parse.go

@@ -0,0 +1,889 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promql
+
+import (
+	"fmt"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	clientmodel "github.com/prometheus/client_golang/model"
+	"github.com/prometheus/prometheus/storage/metric"
+	"github.com/prometheus/prometheus/utility"
+)
+
+type parser struct {
+	lex       *lexer
+	token     [3]item
+	peekCount int
+}
+
+// ParseErr wraps a parsing error with line and position context.
+// If the parsing input was a single line, line will be 0 and omitted
+// from the error string.
+type ParseErr struct {
+	Line, Pos int
+	Err       error
+}
+
+func (e *ParseErr) Error() string {
+	if e.Line == 0 {
+		return fmt.Sprintf("Parse error at char %d: %s", e.Pos, e.Err)
+	}
+	return fmt.Sprintf("Parse error at line %d, char %d: %s", e.Line, e.Pos, e.Err)
+}
+
+// ParseStmts parses the input and returns the resulting statements or any ocurring error.
+func ParseStmts(input string) (Statements, error) {
+	p := newParser(input)
+
+	stmts, err := p.parseStmts()
+	if err != nil {
+		return nil, err
+	}
+	err = p.typecheck(stmts)
+	return stmts, err
+}
+
+// ParseExpr returns the expression parsed from the input.
+func ParseExpr(input string) (Expr, error) {
+	p := newParser(input)
+
+	expr, err := p.parseExpr()
+	if err != nil {
+		return nil, err
+	}
+	err = p.typecheck(expr)
+	return expr, err
+}
+
+// newParser returns a new parser.
+func newParser(input string) *parser {
+	p := &parser{
+		lex: lex(input),
+	}
+	return p
+}
+
+// parseStmts parses a sequence of statements from the input.
+func (p *parser) parseStmts() (stmts Statements, err error) {
+	defer p.recover(&err)
+	stmts = Statements{}
+
+	for p.peek().typ != itemEOF {
+		if p.peek().typ == itemComment {
+			continue
+		}
+		stmts = append(stmts, p.stmt())
+	}
+	return
+}
+
+// parseExpr parses a single expression from the input.
+func (p *parser) parseExpr() (expr Expr, err error) {
+	defer p.recover(&err)
+
+	for p.peek().typ != itemEOF {
+		if p.peek().typ == itemComment {
+			continue
+		}
+		if expr != nil {
+			p.errorf("could not parse remaining input %.15q...", p.lex.input[p.lex.lastPos:])
+		}
+		expr = p.expr()
+	}
+
+	if expr == nil {
+		p.errorf("no expression found in input")
+	}
+	return
+}
+
+// typecheck checks correct typing of the parsed statements or expression.
+func (p *parser) typecheck(node Node) (err error) {
+	defer p.recover(&err)
+
+	p.checkType(node)
+	return nil
+}
+
+// next returns the next token.
+func (p *parser) next() item {
+	if p.peekCount > 0 {
+		p.peekCount--
+	} else {
+		t := p.lex.nextItem()
+		// Skip comments.
+		for t.typ == itemComment {
+			t = p.lex.nextItem()
+		}
+		p.token[0] = t
+	}
+	if p.token[p.peekCount].typ == itemError {
+		p.errorf("%s", p.token[p.peekCount].val)
+	}
+	return p.token[p.peekCount]
+}
+
+// peek returns but does not consume the next token.
+func (p *parser) peek() item {
+	if p.peekCount > 0 {
+		return p.token[p.peekCount-1]
+	}
+	p.peekCount = 1
+
+	t := p.lex.nextItem()
+	// Skip comments.
+	for t.typ == itemComment {
+		t = p.lex.nextItem()
+	}
+	p.token[0] = t
+	return p.token[0]
+}
+
+// backup backs the input stream up one token.
+func (p *parser) backup() {
+	p.peekCount++
+}
+
+// errorf formats the error and terminates processing.
+func (p *parser) errorf(format string, args ...interface{}) {
+	p.error(fmt.Errorf(format, args...))
+}
+
+// error terminates processing.
+func (p *parser) error(err error) {
+	perr := &ParseErr{
+		Line: p.lex.lineNumber(),
+		Pos:  p.lex.linePosition(),
+		Err:  err,
+	}
+	if strings.Count(strings.TrimSpace(p.lex.input), "\n") == 0 {
+		perr.Line = 0
+	}
+	panic(perr)
+}
+
+// expect consumes the next token and guarantees it has the required type.
+func (p *parser) expect(exp itemType, context string) item {
+	token := p.next()
+	if token.typ != exp {
+		p.errorf("unexpected %s in %s, expected %s", token.desc(), context, exp.desc())
+	}
+	return token
+}
+
+// expectOneOf consumes the next token and guarantees it has one of the required types.
+func (p *parser) expectOneOf(exp1, exp2 itemType, context string) item {
+	token := p.next()
+	if token.typ != exp1 && token.typ != exp2 {
+		p.errorf("unexpected %s in %s, expected %s or %s", token.desc(), context, exp1.desc(), exp2.desc())
+	}
+	return token
+}
+
+// recover is the handler that turns panics into returns from the top level of Parse.
+func (p *parser) recover(errp *error) {
+	e := recover()
+	if e != nil {
+		if _, ok := e.(runtime.Error); ok {
+			panic(e)
+		}
+		*errp = e.(error)
+	}
+	return
+}
+
+// stmt parses any statement.
+//
+// 		alertStatement | recordStatement
+//
+func (p *parser) stmt() Statement {
+	switch tok := p.peek(); tok.typ {
+	case itemAlert:
+		return p.alertStmt()
+	case itemIdentifier, itemMetricIdentifier:
+		return p.recordStmt()
+	}
+	p.errorf("no valid statement detected")
+	return nil
+}
+
+// alertStmt parses an alert rule.
+//
+//		ALERT name IF expr [FOR duration] [WITH label_set]
+//			SUMMARY "summary"
+//			DESCRIPTION "description"
+//
+func (p *parser) alertStmt() *AlertStmt {
+	const ctx = "alert statement"
+
+	p.expect(itemAlert, ctx)
+	name := p.expect(itemIdentifier, ctx)
+	// Alerts require a vector typed expression.
+	p.expect(itemIf, ctx)
+	expr := p.expr()
+
+	// Optional for clause.
+	var duration time.Duration
+	var err error
+
+	if p.peek().typ == itemFor {
+		p.next()
+		dur := p.expect(itemDuration, ctx)
+		duration, err = parseDuration(dur.val)
+		if err != nil {
+			p.error(err)
+		}
+	}
+
+	lset := clientmodel.LabelSet{}
+	if p.peek().typ == itemWith {
+		p.expect(itemWith, ctx)
+		lset = p.labelSet()
+	}
+
+	p.expect(itemSummary, ctx)
+	sum, err := strconv.Unquote(p.expect(itemString, ctx).val)
+	if err != nil {
+		p.error(err)
+	}
+
+	p.expect(itemDescription, ctx)
+	desc, err := strconv.Unquote(p.expect(itemString, ctx).val)
+	if err != nil {
+		p.error(err)
+	}
+
+	return &AlertStmt{
+		Name:        name.val,
+		Expr:        expr,
+		Duration:    duration,
+		Labels:      lset,
+		Summary:     sum,
+		Description: desc,
+	}
+}
+
+// recordStmt parses a recording rule.
+func (p *parser) recordStmt() *RecordStmt {
+	const ctx = "record statement"
+
+	name := p.expectOneOf(itemIdentifier, itemMetricIdentifier, ctx).val
+
+	var lset clientmodel.LabelSet
+	if p.peek().typ == itemLeftBrace {
+		lset = p.labelSet()
+	}
+
+	p.expect(itemAssign, ctx)
+	expr := p.expr()
+
+	return &RecordStmt{
+		Name:   name,
+		Labels: lset,
+		Expr:   expr,
+	}
+}
+
+// expr parses any expression.
+func (p *parser) expr() Expr {
+	// Parse the starting expression.
+	expr := p.unaryExpr()
+	if expr == nil {
+		p.errorf("no valid expression found")
+	}
+
+	// Loop through the operations and construct a binary operation tree based
+	// on the operators' precedence.
+	for {
+		// If the next token is not an operator the expression is done.
+		op := p.peek().typ
+		if !op.isOperator() {
+			return expr
+		}
+		p.next() // Consume operator.
+
+		// Parse optional operator matching options. Its validity
+		// is checked in the type-checking stage.
+		vecMatching := &VectorMatching{
+			Card: CardOneToOne,
+		}
+		if op == itemLAND || op == itemLOR {
+			vecMatching.Card = CardManyToMany
+		}
+
+		// Parse ON clause.
+		if p.peek().typ == itemOn {
+			p.next()
+			vecMatching.On = p.labels()
+
+			// Parse grouping.
+			if t := p.peek().typ; t == itemGroupLeft {
+				p.next()
+				vecMatching.Card = CardManyToOne
+				vecMatching.Include = p.labels()
+			} else if t == itemGroupRight {
+				p.next()
+				vecMatching.Card = CardOneToMany
+				vecMatching.Include = p.labels()
+			}
+		}
+
+		for _, ln := range vecMatching.On {
+			for _, ln2 := range vecMatching.Include {
+				if ln == ln2 {
+					p.errorf("label %q must not occur in ON and INCLUDE clause at once", ln)
+				}
+			}
+		}
+
+		// Parse the next operand.
+		rhs := p.unaryExpr()
+		if rhs == nil {
+			p.errorf("missing right-hand side in binary expression")
+		}
+
+		// Assign the new root based on the precendence of the LHS and RHS operators.
+		if lhs, ok := expr.(*BinaryExpr); ok && lhs.Op.precedence() < op.precedence() {
+			expr = &BinaryExpr{
+				Op:  lhs.Op,
+				LHS: lhs.LHS,
+				RHS: &BinaryExpr{
+					Op:             op,
+					LHS:            lhs.RHS,
+					RHS:            rhs,
+					VectorMatching: vecMatching,
+				},
+				VectorMatching: lhs.VectorMatching,
+			}
+		} else {
+			expr = &BinaryExpr{
+				Op:             op,
+				LHS:            expr,
+				RHS:            rhs,
+				VectorMatching: vecMatching,
+			}
+		}
+	}
+	return nil
+}
+
+// unaryExpr parses a unary expression.
+//
+//		<vector_selector> | <matrix_selector> | (+|-) <number_literal> | '(' <expr> ')'
+//
+func (p *parser) unaryExpr() Expr {
+	switch t := p.peek(); t.typ {
+	case itemADD, itemSUB:
+		p.next()
+		e := p.unaryExpr()
+		// Simplify unary expressions for number literals.
+		if nl, ok := e.(*NumberLiteral); ok {
+			if t.typ == itemSUB {
+				nl.Val *= -1
+			}
+			return nl
+		}
+		return &UnaryExpr{Op: t.typ, Expr: e}
+
+	case itemLeftParen:
+		p.next()
+		e := p.expr()
+		p.expect(itemRightParen, "paren expression")
+
+		return &ParenExpr{Expr: e}
+	}
+	e := p.primaryExpr()
+
+	// Expression might be followed by a range selector.
+	if p.peek().typ == itemLeftBracket {
+		vs, ok := e.(*VectorSelector)
+		if !ok {
+			p.errorf("range specification must be preceded by a metric selector, but follows a %T instead", e)
+		}
+		e = p.rangeSelector(vs)
+	}
+	return e
+}
+
+// rangeSelector parses a matrix selector based on a given vector selector.
+//
+//		<vector_selector> '[' <duration> ']'
+//
+func (p *parser) rangeSelector(vs *VectorSelector) *MatrixSelector {
+	const ctx = "matrix selector"
+	p.next()
+
+	var erange, offset time.Duration
+	var err error
+
+	erangeStr := p.expect(itemDuration, ctx).val
+	erange, err = parseDuration(erangeStr)
+	if err != nil {
+		p.error(err)
+	}
+
+	p.expect(itemRightBracket, ctx)
+
+	// Parse optional offset.
+	if p.peek().typ == itemOffset {
+		p.next()
+		offi := p.expect(itemDuration, ctx)
+
+		offset, err = parseDuration(offi.val)
+		if err != nil {
+			p.error(err)
+		}
+	}
+
+	e := &MatrixSelector{
+		Name:          vs.Name,
+		LabelMatchers: vs.LabelMatchers,
+		Range:         erange,
+		Offset:        offset,
+	}
+	return e
+}
+
+// primaryExpr parses a primary expression.
+//
+//		<metric_name> | <function_call> | <vector_aggregation> | <literal>
+//
+func (p *parser) primaryExpr() Expr {
+	switch t := p.next(); {
+	case t.typ == itemNumber:
+		n, err := strconv.ParseInt(t.val, 0, 64)
+		f := float64(n)
+		if err != nil {
+			f, err = strconv.ParseFloat(t.val, 64)
+		}
+		if err != nil {
+			p.errorf("error parsing number: %s", err)
+		}
+		return &NumberLiteral{clientmodel.SampleValue(f)}
+
+	case t.typ == itemString:
+		s := t.val[1 : len(t.val)-1]
+		return &StringLiteral{s}
+
+	case t.typ == itemLeftBrace:
+		// Metric selector without metric name.
+		p.backup()
+		return p.vectorSelector("")
+
+	case t.typ == itemIdentifier:
+		// Check for function call.
+		if p.peek().typ == itemLeftParen {
+			return p.call(t.val)
+		}
+		fallthrough // Else metric selector.
+
+	case t.typ == itemMetricIdentifier:
+		return p.vectorSelector(t.val)
+
+	case t.typ.isAggregator():
+		p.backup()
+		return p.aggrExpr()
+	}
+	return nil
+}
+
+// labels parses a list of labelnames.
+//
+//		'(' <label_name>, ... ')'
+//
+func (p *parser) labels() clientmodel.LabelNames {
+	const ctx = "grouping opts"
+
+	p.expect(itemLeftParen, ctx)
+
+	labels := clientmodel.LabelNames{}
+	for {
+		id := p.expect(itemIdentifier, ctx)
+		labels = append(labels, clientmodel.LabelName(id.val))
+
+		if p.peek().typ != itemComma {
+			break
+		}
+		p.next()
+	}
+	p.expect(itemRightParen, ctx)
+
+	return labels
+}
+
+// aggrExpr parses an aggregation expression.
+//
+//		<aggr_op> (<vector_expr>) [by <labels>] [keeping_extra]
+//		<aggr_op> [by <labels>] [keeping_extra] (<vector_expr>)
+//
+func (p *parser) aggrExpr() *AggregateExpr {
+	const ctx = "aggregation"
+
+	agop := p.next()
+	if !agop.typ.isAggregator() {
+		p.errorf("expected aggregation operator but got %s", agop)
+	}
+	var grouping clientmodel.LabelNames
+	var keepExtra bool
+
+	modifiersFirst := false
+
+	if p.peek().typ == itemBy {
+		p.next()
+		grouping = p.labels()
+		modifiersFirst = true
+	}
+	if p.peek().typ == itemKeepingExtra {
+		p.next()
+		keepExtra = true
+		modifiersFirst = true
+	}
+
+	p.expect(itemLeftParen, ctx)
+	e := p.expr()
+	p.expect(itemRightParen, ctx)
+
+	if !modifiersFirst {
+		if p.peek().typ == itemBy {
+			if len(grouping) > 0 {
+				p.errorf("aggregation must only contain one grouping clause")
+			}
+			p.next()
+			grouping = p.labels()
+		}
+		if p.peek().typ == itemKeepingExtra {
+			p.next()
+			keepExtra = true
+		}
+	}
+
+	return &AggregateExpr{
+		Op:              agop.typ,
+		Expr:            e,
+		Grouping:        grouping,
+		KeepExtraLabels: keepExtra,
+	}
+}
+
+// call parses a function call.
+//
+//		<func_name> '(' [ <arg_expr>, ...] ')'
+//
+func (p *parser) call(name string) *Call {
+	const ctx = "function call"
+
+	fn, exist := getFunction(name)
+	if !exist {
+		p.errorf("unknown function with name %q", name)
+	}
+
+	p.expect(itemLeftParen, ctx)
+	// Might be call without args.
+	if p.peek().typ == itemRightParen {
+		p.next() // Consume.
+		return &Call{fn, nil}
+	}
+
+	var args []Expr
+	for {
+		e := p.expr()
+		args = append(args, e)
+
+		// Terminate if no more arguments.
+		if p.peek().typ != itemComma {
+			break
+		}
+		p.next()
+	}
+
+	// Call must be closed.
+	p.expect(itemRightParen, ctx)
+
+	return &Call{Func: fn, Args: args}
+}
+
+// labelSet parses a set of label matchers
+//
+//		'{' [ <labelname> '=' <match_string>, ... ] '}'
+//
+func (p *parser) labelSet() clientmodel.LabelSet {
+	set := clientmodel.LabelSet{}
+	for _, lm := range p.labelMatchers(itemEQL) {
+		set[lm.Name] = lm.Value
+	}
+	return set
+}
+
+// labelMatchers parses a set of label matchers.
+//
+//		'{' [ <labelname> <match_op> <match_string>, ... ] '}'
+//
+func (p *parser) labelMatchers(operators ...itemType) metric.LabelMatchers {
+	const ctx = "label matching"
+
+	matchers := metric.LabelMatchers{}
+
+	p.expect(itemLeftBrace, ctx)
+
+	// Check if no matchers are provided.
+	if p.peek().typ == itemRightBrace {
+		p.next()
+		return matchers
+	}
+
+	for {
+		label := p.expect(itemIdentifier, ctx)
+
+		op := p.next().typ
+		if !op.isOperator() {
+			p.errorf("expected label matching operator but got %s", op)
+		}
+		var validOp = false
+		for _, allowedOp := range operators {
+			if op == allowedOp {
+				validOp = true
+			}
+		}
+		if !validOp {
+			p.errorf("operator must be one of %q, is %q", operators, op)
+		}
+
+		val, err := strconv.Unquote(p.expect(itemString, ctx).val)
+		if err != nil {
+			p.error(err)
+		}
+
+		// Map the item to the respective match type.
+		var matchType metric.MatchType
+		switch op {
+		case itemEQL:
+			matchType = metric.Equal
+		case itemNEQ:
+			matchType = metric.NotEqual
+		case itemEQLRegex:
+			matchType = metric.RegexMatch
+		case itemNEQRegex:
+			matchType = metric.RegexNoMatch
+		default:
+			p.errorf("item %q is not a metric match type", op)
+		}
+
+		m, err := metric.NewLabelMatcher(
+			matchType,
+			clientmodel.LabelName(label.val),
+			clientmodel.LabelValue(val),
+		)
+		if err != nil {
+			p.error(err)
+		}
+
+		matchers = append(matchers, m)
+
+		// Terminate list if last matcher.
+		if p.peek().typ != itemComma {
+			break
+		}
+		p.next()
+	}
+
+	p.expect(itemRightBrace, ctx)
+
+	return matchers
+}
+
+// metricSelector parses a new metric selector.
+//
+//		<metric_identifier> [<label_matchers>] [ offset <duration> ]
+//		[<metric_identifier>] <label_matchers> [ offset <duration> ]
+//
+func (p *parser) vectorSelector(name string) *VectorSelector {
+	const ctx = "metric selector"
+
+	var matchers metric.LabelMatchers
+	// Parse label matching if any.
+	if t := p.peek(); t.typ == itemLeftBrace {
+		matchers = p.labelMatchers(itemEQL, itemNEQ, itemEQLRegex, itemNEQRegex)
+	}
+	// Metric name must not be set in the label matchers and before at the same time.
+	if name != "" {
+		for _, m := range matchers {
+			if m.Name == clientmodel.MetricNameLabel {
+				p.errorf("metric name must not be set twice: %q or %q", name, m.Value)
+			}
+		}
+		// Set name label matching.
+		matchers = append(matchers, &metric.LabelMatcher{
+			Type:  metric.Equal,
+			Name:  clientmodel.MetricNameLabel,
+			Value: clientmodel.LabelValue(name),
+		})
+	}
+
+	if len(matchers) == 0 {
+		p.errorf("vector selector must contain label matchers or metric name")
+	}
+
+	var err error
+	var offset time.Duration
+	// Parse optional offset.
+	if p.peek().typ == itemOffset {
+		p.next()
+		offi := p.expect(itemDuration, ctx)
+
+		offset, err = parseDuration(offi.val)
+		if err != nil {
+			p.error(err)
+		}
+	}
+	return &VectorSelector{
+		Name:          name,
+		LabelMatchers: matchers,
+		Offset:        offset,
+	}
+}
+
+// expectType checks the type of the node and raises an error if it
+// is not of the expected type.
+func (p *parser) expectType(node Node, want ExprType, context string) {
+	t := p.checkType(node)
+	if t != want {
+		p.errorf("expected type %s in %s, got %s", want, context, t)
+	}
+}
+
+// check the types of the children of each node and raise an error
+// if they do not form a valid node.
+//
+// Some of these checks are redundant as the the parsing stage does not allow
+// them, but the costs are small and might reveal errors when making changes.
+func (p *parser) checkType(node Node) (typ ExprType) {
+	// For expressions the type is determined by their Type function.
+	// Statements and lists do not have a type but are not invalid either.
+	switch n := node.(type) {
+	case Statements, Expressions, Statement:
+		typ = ExprNone
+	case Expr:
+		typ = n.Type()
+	default:
+		p.errorf("unknown node type: %T", node)
+	}
+
+	// Recursively check correct typing for child nodes and raise
+	// errors in case of bad typing.
+	switch n := node.(type) {
+	case Statements:
+		for _, s := range n {
+			p.expectType(s, ExprNone, "statement list")
+		}
+	case *AlertStmt:
+		p.expectType(n.Expr, ExprVector, "alert statement")
+
+	case *EvalStmt:
+		ty := p.checkType(n.Expr)
+		if ty == ExprNone {
+			p.errorf("evaluation statement must have a valid expression type but got %s", ty)
+		}
+
+	case *RecordStmt:
+		p.expectType(n.Expr, ExprVector, "record statement")
+
+	case Expressions:
+		for _, e := range n {
+			ty := p.checkType(e)
+			if ty == ExprNone {
+				p.errorf("expression must have a valid expression type but got %s", ty)
+			}
+		}
+	case *AggregateExpr:
+		if !n.Op.isAggregator() {
+			p.errorf("aggregation operator expected in aggregation expression but got %q", n.Op)
+		}
+		p.expectType(n.Expr, ExprVector, "aggregation expression")
+
+	case *BinaryExpr:
+		lt := p.checkType(n.LHS)
+		rt := p.checkType(n.RHS)
+
+		if !n.Op.isOperator() {
+			p.errorf("only logical and arithmetic operators allowed in binary expression, got %q", n.Op)
+		}
+		if (lt != ExprScalar && lt != ExprVector) || (rt != ExprScalar && rt != ExprVector) {
+			p.errorf("binary expression must contain only scalar and vector types")
+		}
+
+		if (lt != ExprVector || rt != ExprVector) && n.VectorMatching != nil {
+			if len(n.VectorMatching.On) > 0 {
+				p.errorf("vector matching only allowed between vectors")
+			}
+			n.VectorMatching = nil
+		} else {
+			// Both operands are vectors.
+			if n.Op == itemLAND || n.Op == itemLOR {
+				if n.VectorMatching.Card == CardOneToMany || n.VectorMatching.Card == CardManyToOne {
+					p.errorf("no grouping allowed for AND and OR operations")
+				}
+				if n.VectorMatching.Card != CardManyToMany {
+					p.errorf("AND and OR operations must always be many-to-many")
+				}
+			}
+		}
+
+		if (lt == ExprScalar || rt == ExprScalar) && (n.Op == itemLAND || n.Op == itemLOR) {
+			p.errorf("AND and OR not allowed in binary scalar expression")
+		}
+
+	case *Call:
+		nargs := len(n.Func.ArgTypes)
+		if na := nargs - n.Func.OptionalArgs; na > len(n.Args) {
+			p.errorf("expected at least %d argument(s) in call to %q, got %d", na, n.Func.Name, len(n.Args))
+		}
+		if nargs < len(n.Args) {
+			p.errorf("expected at most %d argument(s) in call to %q, got %d", nargs, n.Func.Name, len(n.Args))
+		}
+		for i, arg := range n.Args {
+			p.expectType(arg, n.Func.ArgTypes[i], fmt.Sprintf("call to function %q", n.Func.Name))
+		}
+
+	case *ParenExpr:
+		p.checkType(n.Expr)
+
+	case *UnaryExpr:
+		if n.Op != itemADD && n.Op != itemSUB {
+			p.errorf("only + and - operators allowed for unary expressions")
+		}
+		p.expectType(n.Expr, ExprScalar, "unary expression")
+
+	case *NumberLiteral, *MatrixSelector, *StringLiteral, *VectorSelector:
+		// Nothing to do for terminals.
+
+	default:
+		p.errorf("unknown node type: %T", node)
+	}
+	return
+}
+
+func parseDuration(ds string) (time.Duration, error) {
+	dur, err := utility.StringToDuration(ds)
+	if err != nil {
+		return 0, err
+	}
+	if dur == 0 {
+		return 0, fmt.Errorf("duration must be greater than 0")
+	}
+	return dur, nil
+}

promql/printer.go

@@ -0,0 +1,399 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package promql
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+
+	clientmodel "github.com/prometheus/client_golang/model"
+
+	"github.com/prometheus/prometheus/storage/metric"
+	"github.com/prometheus/prometheus/utility"
+)
+
+func (matrix Matrix) String() string {
+	metricStrings := make([]string, 0, len(matrix))
+	for _, sampleStream := range matrix {
+		metricName, hasName := sampleStream.Metric.Metric[clientmodel.MetricNameLabel]
+		numLabels := len(sampleStream.Metric.Metric)
+		if hasName {
+			numLabels--
+		}
+		labelStrings := make([]string, 0, numLabels)
+		for label, value := range sampleStream.Metric.Metric {
+			if label != clientmodel.MetricNameLabel {
+				labelStrings = append(labelStrings, fmt.Sprintf("%s=%q", label, value))
+			}
+		}
+		sort.Strings(labelStrings)
+		valueStrings := make([]string, 0, len(sampleStream.Values))
+		for _, value := range sampleStream.Values {
+			valueStrings = append(valueStrings,
+				fmt.Sprintf("\n%v @[%v]", value.Value, value.Timestamp))
+		}
+		metricStrings = append(metricStrings,
+			fmt.Sprintf("%s{%s} => %s",
+				metricName,
+				strings.Join(labelStrings, ", "),
+				strings.Join(valueStrings, ", ")))
+	}
+	sort.Strings(metricStrings)
+	return strings.Join(metricStrings, "\n")
+}
+
+func (vector Vector) String() string {
+	metricStrings := make([]string, 0, len(vector))
+	for _, sample := range vector {
+		metricStrings = append(metricStrings,
+			fmt.Sprintf("%s => %v @[%v]",
+				sample.Metric,
+				sample.Value, sample.Timestamp))
+	}
+	return strings.Join(metricStrings, "\n")
+}
+
+// Tree returns a string of the tree structure of the given node.
+func Tree(node Node) string {
+	return tree(node, "")
+}
+
+func tree(node Node, level string) string {
+	if node == nil {
+		return fmt.Sprintf("%s |---- %T\n", level, node)
+	}
+	typs := strings.Split(fmt.Sprintf("%T", node), ".")[1]
+
+	var t string
+	// Only print the number of statements for readability.
+	if stmts, ok := node.(Statements); ok {
+		t = fmt.Sprintf("%s |---- %s :: %d\n", level, typs, len(stmts))
+	} else {
+		t = fmt.Sprintf("%s |---- %s :: %s\n", level, typs, node)
+	}
+
+	level += " · · ·"
+
+	switch n := node.(type) {
+	case Statements:
+		for _, s := range n {
+			t += tree(s, level)
+		}
+	case *AlertStmt:
+		t += tree(n.Expr, level)
+
+	case *EvalStmt:
+		t += tree(n.Expr, level)
+
+	case *RecordStmt:
+		t += tree(n.Expr, level)
+
+	case Expressions:
+		for _, e := range n {
+			t += tree(e, level)
+		}
+	case *AggregateExpr:
+		t += tree(n.Expr, level)
+
+	case *BinaryExpr:
+		t += tree(n.LHS, level)
+		t += tree(n.RHS, level)
+
+	case *Call:
+		t += tree(n.Args, level)
+
+	case *ParenExpr:
+		t += tree(n.Expr, level)
+
+	case *UnaryExpr:
+		t += tree(n.Expr, level)
+
+	case *MatrixSelector, *NumberLiteral, *StringLiteral, *VectorSelector:
+		// nothing to do
+
+	default:
+		panic("promql.Tree: not all node types covered")
+	}
+	return t
+}
+
+func (stmts Statements) String() (s string) {
+	if len(stmts) == 0 {
+		return ""
+	}
+	for _, stmt := range stmts {
+		s += stmt.String()
+		s += "\n\n"
+	}
+	return s[:len(s)-2]
+}
+
+func (node *AlertStmt) String() string {
+	s := fmt.Sprintf("ALERT %s", node.Name)
+	s += fmt.Sprintf("\n\tIF %s", node.Expr)
+	if node.Duration > 0 {
+		s += fmt.Sprintf("\n\tFOR %s", utility.DurationToString(node.Duration))
+	}
+	if len(node.Labels) > 0 {
+		s += fmt.Sprintf("\n\tWITH %s", node.Labels)
+	}
+	s += fmt.Sprintf("\n\tSUMMARY %q", node.Summary)
+	s += fmt.Sprintf("\n\tDESCRIPTION %q", node.Description)
+	return s
+}
+
+func (node *EvalStmt) String() string {
+	return "EVAL " + node.Expr.String()
+}
+
+func (node *RecordStmt) String() string {
+	s := fmt.Sprintf("%s%s = %s", node.Name, node.Labels, node.Expr)
+	return s
+}
+
+func (es Expressions) String() (s string) {
+	if len(es) == 0 {
+		return ""
+	}
+	for _, e := range es {
+		s += e.String()
+		s += ", "
+	}
+	return s[:len(s)-2]
+}
+
+func (node *AggregateExpr) String() string {
+	aggrString := fmt.Sprintf("%s(%s)", node.Op, node.Expr)
+	if len(node.Grouping) > 0 {
+		return fmt.Sprintf("%s BY (%s)", aggrString, node.Grouping)
+	}
+	return aggrString
+}
+
+func (node *BinaryExpr) String() string {
+	matching := ""
+	vm := node.VectorMatching
+	if vm != nil && len(vm.On) > 0 {
+		matching = fmt.Sprintf(" ON(%s)", vm.On)
+		if vm.Card == CardManyToOne {
+			matching += fmt.Sprintf(" GROUP_LEFT(%s)", vm.Include)
+		}
+		if vm.Card == CardOneToMany {
+			matching += fmt.Sprintf(" GROUP_RIGHT(%s)", vm.Include)
+		}
+	}
+	return fmt.Sprintf("%s %s%s %s", node.LHS, node.Op, matching, node.RHS)
+}
+
+func (node *Call) String() string {
+	return fmt.Sprintf("%s(%s)", node.Func.Name, node.Args)
+}
+
+func (node *MatrixSelector) String() string {
+	vecSelector := &VectorSelector{
+		Name:          node.Name,
+		LabelMatchers: node.LabelMatchers,
+	}
+	return fmt.Sprintf("%s[%s]", vecSelector.String(), utility.DurationToString(node.Range))
+}
+
+func (node *NumberLiteral) String() string {
+	return fmt.Sprint(node.Val)
+}
+
+func (node *ParenExpr) String() string {
+	return fmt.Sprintf("(%s)", node.Expr)
+}
+
+func (node *StringLiteral) String() string {
+	return fmt.Sprintf("%q", node.Val)
+}
+
+func (node *UnaryExpr) String() string {
+	return fmt.Sprintf("%s%s", node.Op, node.Expr)
+}
+
+func (node *VectorSelector) String() string {
+	labelStrings := make([]string, 0, len(node.LabelMatchers)-1)
+	for _, matcher := range node.LabelMatchers {
+		// Only include the __name__ label if its no equality matching.
+		if matcher.Name == clientmodel.MetricNameLabel && matcher.Type == metric.Equal {
+			continue
+		}
+		labelStrings = append(labelStrings, matcher.String())
+	}
+
+	if len(labelStrings) == 0 {
+		return node.Name
+	}
+	sort.Strings(labelStrings)
+	return fmt.Sprintf("%s{%s}", node.Name, strings.Join(labelStrings, ","))
+}
+
+// DotGraph returns a DOT representation of a statement list.
+func (ss Statements) DotGraph() string {
+	graph := ""
+	for _, stmt := range ss {
+		graph += stmt.DotGraph()
+	}
+	return graph
+}
+
+// DotGraph returns a DOT representation of the alert statement.
+func (node *AlertStmt) DotGraph() string {
+	graph := fmt.Sprintf(
+		`digraph "Alert Statement" {
+	  %#p[shape="box",label="ALERT %s IF FOR %s"];
+		%#p -> %x;
+		%s
+	}`,
+		node, node.Name, utility.DurationToString(node.Duration),
+		node, reflect.ValueOf(node.Expr).Pointer(),
+		node.Expr.DotGraph(),
+	)
+	return graph
+}
+
+// DotGraph returns a DOT representation of the eval statement.
+func (node *EvalStmt) DotGraph() string {
+	graph := fmt.Sprintf(
+		`%#p[shape="box",label="[%d:%s:%d]";
+		%#p -> %x;
+		%s
+	}`,
+		node, node.Start, node.End, node.Interval,
+		node, reflect.ValueOf(node.Expr).Pointer(),
+		node.Expr.DotGraph(),
+	)
+	return graph
+}
+
+// DotGraph returns a DOT representation of the record statement.
+func (node *RecordStmt) DotGraph() string {
+	graph := fmt.Sprintf(
+		`%#p[shape="box",label="%s = "];
+		%#p -> %x;
+		%s
+	}`,
+		node, node.Name,
+		node, reflect.ValueOf(node.Expr).Pointer(),
+		node.Expr.DotGraph(),
+	)
+	return graph
+}
+
+// DotGraph returns a DOT representation of // DotGraph returns a DOT representation of the record statement.
+// DotGraph returns a DOT representation of a statement list.
+func (es Expressions) DotGraph() string {
+	graph := ""
+	for _, expr := range es {
+		graph += expr.DotGraph()
+	}
+	return graph
+}
+
+// DotGraph returns a DOT representation of the vector aggregation.
+func (node *AggregateExpr) DotGraph() string {
+	groupByStrings := make([]string, 0, len(node.Grouping))
+	for _, label := range node.Grouping {
+		groupByStrings = append(groupByStrings, string(label))
+	}
+
+	graph := fmt.Sprintf("%#p[label=\"%s BY (%s)\"]\n",
+		node,
+		node.Op,
+		strings.Join(groupByStrings, ", "))
+	graph += fmt.Sprintf("%#p -> %x;\n", node, reflect.ValueOf(node.Expr).Pointer())
+	graph += node.Expr.DotGraph()
+	return graph
+}
+
+// DotGraph returns a DOT representation of the expression.
+func (node *BinaryExpr) DotGraph() string {
+	nodeAddr := reflect.ValueOf(node).Pointer()
+	graph := fmt.Sprintf(
+		`
+		%x[label="%s"];
+		%x -> %x;
+		%x -> %x;
+		%s
+		%s
+	}`,
+		nodeAddr, node.Op,
+		nodeAddr, reflect.ValueOf(node.LHS).Pointer(),
+		nodeAddr, reflect.ValueOf(node.RHS).Pointer(),
+		node.LHS.DotGraph(),
+		node.RHS.DotGraph(),
+	)
+	return graph
+}
+
+// DotGraph returns a DOT representation of the function call.
+func (node *Call) DotGraph() string {
+	graph := fmt.Sprintf("%#p[label=\"%s\"];\n", node, node.Func.Name)
+	graph += functionArgsToDotGraph(node, node.Args)
+	return graph
+}
+
+// DotGraph returns a DOT representation of the number literal.
+func (node *NumberLiteral) DotGraph() string {
+	return fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.Val)
+}
+
+// DotGraph returns a DOT representation of the encapsulated expression.
+func (node *ParenExpr) DotGraph() string {
+	return node.Expr.DotGraph()
+}
+
+// DotGraph returns a DOT representation of the matrix selector.
+func (node *MatrixSelector) DotGraph() string {
+	return fmt.Sprintf("%#p[label=\"%s\"];\n", node, node)
+}
+
+// DotGraph returns a DOT representation of the string literal.
+func (node *StringLiteral) DotGraph() string {
+	return fmt.Sprintf("%#p[label=\"'%q'\"];\n", node, node.Val)
+}
+
+// DotGraph returns a DOT representation of the unary expression.
+func (node *UnaryExpr) DotGraph() string {
+	nodeAddr := reflect.ValueOf(node).Pointer()
+	graph := fmt.Sprintf(
+		`
+		%x[label="%s"];
+		%x -> %x;
+		%s
+		%s
+	}`,
+		nodeAddr, node.Op,
+		nodeAddr, reflect.ValueOf(node.Expr).Pointer(),
+		node.Expr.DotGraph(),
+	)
+	return graph
+}
+
+// DotGraph returns a DOT representation of the vector selector.
+func (node *VectorSelector) DotGraph() string {
+	return fmt.Sprintf("%#p[label=\"%s\"];\n", node, node)
+}
+
+func functionArgsToDotGraph(node Node, args Expressions) string {
+	graph := args.DotGraph()
+	for _, arg := range args {
+		graph += fmt.Sprintf("%x -> %x;\n", reflect.ValueOf(node).Pointer(), reflect.ValueOf(arg).Pointer())
+	}
+	return graph
+}

promql/quantile.go

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package ast
+package promql
 
 import (
 	"math"
@@ -25,8 +25,8 @@ import (
 // excludedLabels are the labels to exclude from signature calculation for
 // quantiles.
 var excludedLabels = map[clientmodel.LabelName]struct{}{
-	clientmodel.MetricNameLabel: {},
-	clientmodel.BucketLabel:     {},
+	clientmodel.MetricNameLabel: struct{}{},
+	clientmodel.BucketLabel:     struct{}{},
 }
 
 type bucket struct {

promql/setup_test.go

@@ -11,14 +11,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package rules
+package promql
 
 import (
 	"time"
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
-	"github.com/prometheus/prometheus/rules/ast"
 	"github.com/prometheus/prometheus/storage/local"
 	"github.com/prometheus/prometheus/storage/metric"
 )
@@ -26,7 +25,7 @@ import (
 var testSampleInterval = time.Duration(5) * time.Minute
 var testStartTime = clientmodel.Timestamp(0)
 
-func getTestValueStream(startVal clientmodel.SampleValue, endVal clientmodel.SampleValue, stepVal clientmodel.SampleValue, startTime clientmodel.Timestamp) (resultValues metric.Values) {
+func getTestValueStream(startVal, endVal, stepVal clientmodel.SampleValue, startTime clientmodel.Timestamp) (resultValues metric.Values) {
 	currentTime := startTime
 	for currentVal := startVal; currentVal <= endVal; currentVal += stepVal {
 		sample := metric.SamplePair{
@@ -39,11 +38,11 @@ func getTestValueStream(startVal clientmodel.SampleValue, endVal clientmodel.Sam
 	return resultValues
 }
 
-func getTestVectorFromTestMatrix(matrix ast.Matrix) ast.Vector {
-	vector := ast.Vector{}
+func getTestVectorFromTestMatrix(matrix Matrix) Vector {
+	vector := Vector{}
 	for _, sampleStream := range matrix {
 		lastSample := sampleStream.Values[len(sampleStream.Values)-1]
-		vector = append(vector, &ast.Sample{
+		vector = append(vector, &Sample{
 			Metric:    sampleStream.Metric,
 			Value:     lastSample.Value,
 			Timestamp: lastSample.Timestamp,
@@ -52,7 +51,7 @@ func getTestVectorFromTestMatrix(matrix ast.Matrix) ast.Vector {
 	return vector
 }
 
-func storeMatrix(storage local.Storage, matrix ast.Matrix) {
+func storeMatrix(storage local.Storage, matrix Matrix) {
 	pendingSamples := clientmodel.Samples{}
 	for _, sampleStream := range matrix {
 		for _, sample := range sampleStream.Values {
@@ -69,7 +68,9 @@ func storeMatrix(storage local.Storage, matrix ast.Matrix) {
 	storage.WaitForIndexing()
 }
 
-var testMatrix = ast.Matrix{
+var testVector = getTestVectorFromTestMatrix(testMatrix)
+
+var testMatrix = Matrix{
 	{
 		Metric: clientmodel.COWMetric{
 			Metric: clientmodel.Metric{
@@ -483,5 +484,3 @@ var testMatrix = ast.Matrix{
 		Values: getTestValueStream(0, 200, 20, testStartTime),
 	},
 }
-
-var testVector = getTestVectorFromTestMatrix(testMatrix)

rules/Makefile

@@ -1,27 +0,0 @@
-# Copyright 2013 The Prometheus Authors
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-all: parser.y.go lexer.l.go
-
-include ../Makefile.INCLUDE
-
-parser.y.go: parser.y
-	$(GOCC) tool yacc -o parser.y.go -v "" parser.y
-
-lexer.l.go: parser.y.go lexer.l
-	# This is golex from https://github.com/cznic/golex.
-	$(GO_GET) github.com/cznic/golex
-	golex -o="lexer.l.go" lexer.l
-
-clean:
-	rm lexer.l.go parser.y.go

rules/alerting.go

@@ -22,9 +22,7 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
-	"github.com/prometheus/prometheus/rules/ast"
-	"github.com/prometheus/prometheus/stats"
-	"github.com/prometheus/prometheus/storage/local"
+	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/utility"
 )
 
@@ -80,7 +78,7 @@ type Alert struct {
 }
 
 // sample returns a Sample suitable for recording the alert.
-func (a Alert) sample(timestamp clientmodel.Timestamp, value clientmodel.SampleValue) *ast.Sample {
+func (a Alert) sample(timestamp clientmodel.Timestamp, value clientmodel.SampleValue) *promql.Sample {
 	recordedMetric := clientmodel.Metric{}
 	for label, value := range a.Labels {
 		recordedMetric[label] = value
@@ -90,7 +88,7 @@ func (a Alert) sample(timestamp clientmodel.Timestamp, value clientmodel.SampleV
 	recordedMetric[AlertNameLabel] = clientmodel.LabelValue(a.Name)
 	recordedMetric[AlertStateLabel] = clientmodel.LabelValue(a.State.String())
 
-	return &ast.Sample{
+	return &promql.Sample{
 		Metric: clientmodel.COWMetric{
 			Metric: recordedMetric,
 			Copied: true,
@@ -105,7 +103,7 @@ type AlertingRule struct {
 	// The name of the alert.
 	name string
 	// The vector expression from which to generate alerts.
-	Vector ast.VectorNode
+	Vector promql.Expr
 	// The duration for which a labelset needs to persist in the expression
 	// output vector before an alert transitions from Pending to Firing state.
 	holdDuration time.Duration
@@ -129,14 +127,18 @@ func (rule *AlertingRule) Name() string {
 }
 
 // EvalRaw returns the raw value of the rule expression, without creating alerts.
-func (rule *AlertingRule) EvalRaw(timestamp clientmodel.Timestamp, storage local.Storage) (ast.Vector, error) {
-	return ast.EvalVectorInstant(rule.Vector, timestamp, storage, stats.NewTimerGroup())
+func (rule *AlertingRule) EvalRaw(timestamp clientmodel.Timestamp, engine *promql.Engine) (promql.Vector, error) {
+	query, err := engine.NewInstantQuery(rule.Vector.String(), timestamp)
+	if err != nil {
+		return nil, err
+	}
+	return query.Exec().Vector()
 }
 
 // Eval evaluates the rule expression and then creates pending alerts and fires
 // or removes previously pending alerts accordingly.
-func (rule *AlertingRule) Eval(timestamp clientmodel.Timestamp, storage local.Storage) (ast.Vector, error) {
-	exprResult, err := rule.EvalRaw(timestamp, storage)
+func (rule *AlertingRule) Eval(timestamp clientmodel.Timestamp, engine *promql.Engine) (promql.Vector, error) {
+	exprResult, err := rule.EvalRaw(timestamp, engine)
 	if err != nil {
 		return nil, err
 	}
@@ -170,7 +172,7 @@ func (rule *AlertingRule) Eval(timestamp clientmodel.Timestamp, storage local.St
 		}
 	}
 
-	vector := ast.Vector{}
+	vector := promql.Vector{}
 
 	// Check if any pending alerts should be removed or fire now. Write out alert timeseries.
 	for fp, activeAlert := range rule.activeAlerts {
@@ -191,8 +193,8 @@ func (rule *AlertingRule) Eval(timestamp clientmodel.Timestamp, storage local.St
 	return vector, nil
 }
 
-// ToDotGraph returns the text representation of a dot graph.
-func (rule *AlertingRule) ToDotGraph() string {
+// DotGraph returns the text representation of a dot graph.
+func (rule *AlertingRule) DotGraph() string {
 	graph := fmt.Sprintf(
 		`digraph "Rules" {
 	  %#p[shape="box",label="ALERT %s IF FOR %s"];
@@ -201,7 +203,8 @@ func (rule *AlertingRule) ToDotGraph() string {
 	}`,
 		&rule, rule.name, utility.DurationToString(rule.holdDuration),
 		&rule, reflect.ValueOf(rule.Vector).Pointer(),
-		rule.Vector.NodeTreeToDotGraph())
+		rule.Vector.DotGraph(),
+	)
 	return graph
 }
 
@@ -217,9 +220,9 @@ func (rule *AlertingRule) HTMLSnippet() template.HTML {
 	}
 	return template.HTML(fmt.Sprintf(
 		`ALERT <a href="%s">%s</a> IF <a href="%s">%s</a> FOR %s WITH %s`,
-		GraphLinkForExpression(alertMetric.String()),
+		utility.GraphLinkForExpression(alertMetric.String()),
 		rule.name,
-		GraphLinkForExpression(rule.Vector.String()),
+		utility.GraphLinkForExpression(rule.Vector.String()),
 		rule.Vector,
 		utility.DurationToString(rule.holdDuration),
 		rule.Labels))
@@ -252,7 +255,7 @@ func (rule *AlertingRule) ActiveAlerts() []Alert {
 }
 
 // NewAlertingRule constructs a new AlertingRule.
-func NewAlertingRule(name string, vector ast.VectorNode, holdDuration time.Duration, labels clientmodel.LabelSet, summary string, description string) *AlertingRule {
+func NewAlertingRule(name string, vector promql.Expr, holdDuration time.Duration, labels clientmodel.LabelSet, summary string, description string) *AlertingRule {
 	return &AlertingRule{
 		name:         name,
 		Vector:       vector,

rules/ast/functions.go

@@ -1,772 +0,0 @@
-// Copyright 2013 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ast
-
-import (
-	"container/heap"
-	"fmt"
-	"math"
-	"sort"
-	"strconv"
-	"time"
-
-	clientmodel "github.com/prometheus/client_golang/model"
-
-	"github.com/prometheus/prometheus/storage/metric"
-)
-
-// Function represents a function of the expression language and is
-// used by function nodes.
-type Function struct {
-	name         string
-	argTypes     []ExprType
-	optionalArgs int
-	returnType   ExprType
-	callFn       func(timestamp clientmodel.Timestamp, args []Node) interface{}
-}
-
-// CheckArgTypes returns a non-nil error if the number or types of
-// passed in arg nodes do not match the function's expectations.
-func (function *Function) CheckArgTypes(args []Node) error {
-	if len(function.argTypes) < len(args) {
-		return fmt.Errorf(
-			"too many arguments to function %v(): %v expected at most, %v given",
-			function.name, len(function.argTypes), len(args),
-		)
-	}
-	if len(function.argTypes)-function.optionalArgs > len(args) {
-		return fmt.Errorf(
-			"too few arguments to function %v(): %v expected at least, %v given",
-			function.name, len(function.argTypes)-function.optionalArgs, len(args),
-		)
-	}
-	for idx, arg := range args {
-		invalidType := false
-		var expectedType string
-		if _, ok := arg.(ScalarNode); function.argTypes[idx] == ScalarType && !ok {
-			invalidType = true
-			expectedType = "scalar"
-		}
-		if _, ok := arg.(VectorNode); function.argTypes[idx] == VectorType && !ok {
-			invalidType = true
-			expectedType = "vector"
-		}
-		if _, ok := arg.(MatrixNode); function.argTypes[idx] == MatrixType && !ok {
-			invalidType = true
-			expectedType = "matrix"
-		}
-		if _, ok := arg.(StringNode); function.argTypes[idx] == StringType && !ok {
-			invalidType = true
-			expectedType = "string"
-		}
-
-		if invalidType {
-			return fmt.Errorf(
-				"wrong type for argument %v in function %v(), expected %v",
-				idx, function.name, expectedType,
-			)
-		}
-	}
-	return nil
-}
-
-// === time() clientmodel.SampleValue ===
-func timeImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	return clientmodel.SampleValue(timestamp.Unix())
-}
-
-// === delta(matrix MatrixNode, isCounter=0 ScalarNode) Vector ===
-func deltaImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	matrixNode := args[0].(MatrixNode)
-	isCounter := len(args) >= 2 && args[1].(ScalarNode).Eval(timestamp) > 0
-	resultVector := Vector{}
-
-	// If we treat these metrics as counters, we need to fetch all values
-	// in the interval to find breaks in the timeseries' monotonicity.
-	// I.e. if a counter resets, we want to ignore that reset.
-	var matrixValue Matrix
-	if isCounter {
-		matrixValue = matrixNode.Eval(timestamp)
-	} else {
-		matrixValue = matrixNode.EvalBoundaries(timestamp)
-	}
-	for _, samples := range matrixValue {
-		// No sense in trying to compute a delta without at least two points. Drop
-		// this vector element.
-		if len(samples.Values) < 2 {
-			continue
-		}
-
-		counterCorrection := clientmodel.SampleValue(0)
-		lastValue := clientmodel.SampleValue(0)
-		for _, sample := range samples.Values {
-			currentValue := sample.Value
-			if isCounter && currentValue < lastValue {
-				counterCorrection += lastValue - currentValue
-			}
-			lastValue = currentValue
-		}
-		resultValue := lastValue - samples.Values[0].Value + counterCorrection
-
-		targetInterval := args[0].(*MatrixSelector).interval
-		sampledInterval := samples.Values[len(samples.Values)-1].Timestamp.Sub(samples.Values[0].Timestamp)
-		if sampledInterval == 0 {
-			// Only found one sample. Cannot compute a rate from this.
-			continue
-		}
-		// Correct for differences in target vs. actual delta interval.
-		//
-		// Above, we didn't actually calculate the delta for the specified target
-		// interval, but for an interval between the first and last found samples
-		// under the target interval, which will usually have less time between
-		// them. Depending on how many samples are found under a target interval,
-		// the delta results are distorted and temporal aliasing occurs (ugly
-		// bumps). This effect is corrected for below.
-		intervalCorrection := clientmodel.SampleValue(targetInterval) / clientmodel.SampleValue(sampledInterval)
-		resultValue *= intervalCorrection
-
-		resultSample := &Sample{
-			Metric:    samples.Metric,
-			Value:     resultValue,
-			Timestamp: timestamp,
-		}
-		resultSample.Metric.Delete(clientmodel.MetricNameLabel)
-		resultVector = append(resultVector, resultSample)
-	}
-	return resultVector
-}
-
-// === rate(node MatrixNode) Vector ===
-func rateImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	args = append(args, &ScalarLiteral{value: 1})
-	vector := deltaImpl(timestamp, args).(Vector)
-
-	// TODO: could be other type of MatrixNode in the future (right now, only
-	// MatrixSelector exists). Find a better way of getting the duration of a
-	// matrix, such as looking at the samples themselves.
-	interval := args[0].(*MatrixSelector).interval
-	for i := range vector {
-		vector[i].Value /= clientmodel.SampleValue(interval / time.Second)
-	}
-	return vector
-}
-
-type vectorByValueHeap Vector
-
-func (s vectorByValueHeap) Len() int {
-	return len(s)
-}
-
-func (s vectorByValueHeap) Less(i, j int) bool {
-	if math.IsNaN(float64(s[i].Value)) {
-		return true
-	}
-	return s[i].Value < s[j].Value
-}
-
-func (s vectorByValueHeap) Swap(i, j int) {
-	s[i], s[j] = s[j], s[i]
-}
-
-func (s *vectorByValueHeap) Push(x interface{}) {
-	*s = append(*s, x.(*Sample))
-}
-
-func (s *vectorByValueHeap) Pop() interface{} {
-	old := *s
-	n := len(old)
-	el := old[n-1]
-	*s = old[0 : n-1]
-	return el
-}
-
-type reverseHeap struct {
-	heap.Interface
-}
-
-func (s reverseHeap) Less(i, j int) bool {
-	return s.Interface.Less(j, i)
-}
-
-// === sort(node VectorNode) Vector ===
-func sortImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	byValueSorter := vectorByValueHeap(args[0].(VectorNode).Eval(timestamp))
-	sort.Sort(byValueSorter)
-	return Vector(byValueSorter)
-}
-
-// === sortDesc(node VectorNode) Vector ===
-func sortDescImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	byValueSorter := vectorByValueHeap(args[0].(VectorNode).Eval(timestamp))
-	sort.Sort(sort.Reverse(byValueSorter))
-	return Vector(byValueSorter)
-}
-
-// === topk(k ScalarNode, node VectorNode) Vector ===
-func topkImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	k := int(args[0].(ScalarNode).Eval(timestamp))
-	if k < 1 {
-		return Vector{}
-	}
-
-	topk := make(vectorByValueHeap, 0, k)
-	vector := args[1].(VectorNode).Eval(timestamp)
-
-	for _, el := range vector {
-		if len(topk) < k || topk[0].Value < el.Value {
-			if len(topk) == k {
-				heap.Pop(&topk)
-			}
-			heap.Push(&topk, el)
-		}
-	}
-	sort.Sort(sort.Reverse(topk))
-	return Vector(topk)
-}
-
-// === bottomk(k ScalarNode, node VectorNode) Vector ===
-func bottomkImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	k := int(args[0].(ScalarNode).Eval(timestamp))
-	if k < 1 {
-		return Vector{}
-	}
-
-	bottomk := make(vectorByValueHeap, 0, k)
-	bkHeap := reverseHeap{Interface: &bottomk}
-	vector := args[1].(VectorNode).Eval(timestamp)
-
-	for _, el := range vector {
-		if len(bottomk) < k || bottomk[0].Value > el.Value {
-			if len(bottomk) == k {
-				heap.Pop(&bkHeap)
-			}
-			heap.Push(&bkHeap, el)
-		}
-	}
-	sort.Sort(bottomk)
-	return Vector(bottomk)
-}
-
-// === drop_common_labels(node VectorNode) Vector ===
-func dropCommonLabelsImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	vector := args[0].(VectorNode).Eval(timestamp)
-	if len(vector) < 1 {
-		return Vector{}
-	}
-	common := clientmodel.LabelSet{}
-	for k, v := range vector[0].Metric.Metric {
-		// TODO(julius): Should we also drop common metric names?
-		if k == clientmodel.MetricNameLabel {
-			continue
-		}
-		common[k] = v
-	}
-
-	for _, el := range vector[1:] {
-		for k, v := range common {
-			if el.Metric.Metric[k] != v {
-				// Deletion of map entries while iterating over them is safe.
-				// From http://golang.org/ref/spec#For_statements:
-				// "If map entries that have not yet been reached are deleted during
-				// iteration, the corresponding iteration values will not be produced."
-				delete(common, k)
-			}
-		}
-	}
-
-	for _, el := range vector {
-		for k := range el.Metric.Metric {
-			if _, ok := common[k]; ok {
-				el.Metric.Delete(k)
-			}
-		}
-	}
-	return vector
-}
-
-// === round(vector VectorNode, toNearest=1 Scalar) Vector ===
-func roundImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	// round returns a number rounded to toNearest.
-	// Ties are solved by rounding up.
-	toNearest := float64(1)
-	if len(args) >= 2 {
-		toNearest = float64(args[1].(ScalarNode).Eval(timestamp))
-	}
-	// Invert as it seems to cause fewer floating point accuracy issues.
-	toNearestInverse := 1.0 / toNearest
-
-	n := args[0].(VectorNode)
-	vector := n.Eval(timestamp)
-	for _, el := range vector {
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		el.Value = clientmodel.SampleValue(math.Floor(float64(el.Value)*toNearestInverse+0.5) / toNearestInverse)
-	}
-	return vector
-}
-
-// === scalar(node VectorNode) Scalar ===
-func scalarImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	v := args[0].(VectorNode).Eval(timestamp)
-	if len(v) != 1 {
-		return clientmodel.SampleValue(math.NaN())
-	}
-	return clientmodel.SampleValue(v[0].Value)
-}
-
-// === count_scalar(vector VectorNode) model.SampleValue ===
-func countScalarImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	return clientmodel.SampleValue(len(args[0].(VectorNode).Eval(timestamp)))
-}
-
-func aggrOverTime(timestamp clientmodel.Timestamp, args []Node, aggrFn func(metric.Values) clientmodel.SampleValue) interface{} {
-	n := args[0].(MatrixNode)
-	matrixVal := n.Eval(timestamp)
-	resultVector := Vector{}
-
-	for _, el := range matrixVal {
-		if len(el.Values) == 0 {
-			continue
-		}
-
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		resultVector = append(resultVector, &Sample{
-			Metric:    el.Metric,
-			Value:     aggrFn(el.Values),
-			Timestamp: timestamp,
-		})
-	}
-	return resultVector
-}
-
-// === avg_over_time(matrix MatrixNode) Vector ===
-func avgOverTimeImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	return aggrOverTime(timestamp, args, func(values metric.Values) clientmodel.SampleValue {
-		var sum clientmodel.SampleValue
-		for _, v := range values {
-			sum += v.Value
-		}
-		return sum / clientmodel.SampleValue(len(values))
-	})
-}
-
-// === count_over_time(matrix MatrixNode) Vector ===
-func countOverTimeImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	return aggrOverTime(timestamp, args, func(values metric.Values) clientmodel.SampleValue {
-		return clientmodel.SampleValue(len(values))
-	})
-}
-
-// === floor(vector VectorNode) Vector ===
-func floorImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	n := args[0].(VectorNode)
-	vector := n.Eval(timestamp)
-	for _, el := range vector {
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		el.Value = clientmodel.SampleValue(math.Floor(float64(el.Value)))
-	}
-	return vector
-}
-
-// === max_over_time(matrix MatrixNode) Vector ===
-func maxOverTimeImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	return aggrOverTime(timestamp, args, func(values metric.Values) clientmodel.SampleValue {
-		max := math.Inf(-1)
-		for _, v := range values {
-			max = math.Max(max, float64(v.Value))
-		}
-		return clientmodel.SampleValue(max)
-	})
-}
-
-// === min_over_time(matrix MatrixNode) Vector ===
-func minOverTimeImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	return aggrOverTime(timestamp, args, func(values metric.Values) clientmodel.SampleValue {
-		min := math.Inf(1)
-		for _, v := range values {
-			min = math.Min(min, float64(v.Value))
-		}
-		return clientmodel.SampleValue(min)
-	})
-}
-
-// === sum_over_time(matrix MatrixNode) Vector ===
-func sumOverTimeImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	return aggrOverTime(timestamp, args, func(values metric.Values) clientmodel.SampleValue {
-		var sum clientmodel.SampleValue
-		for _, v := range values {
-			sum += v.Value
-		}
-		return sum
-	})
-}
-
-// === abs(vector VectorNode) Vector ===
-func absImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	n := args[0].(VectorNode)
-	vector := n.Eval(timestamp)
-	for _, el := range vector {
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		el.Value = clientmodel.SampleValue(math.Abs(float64(el.Value)))
-	}
-	return vector
-}
-
-// === absent(vector VectorNode) Vector ===
-func absentImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	n := args[0].(VectorNode)
-	if len(n.Eval(timestamp)) > 0 {
-		return Vector{}
-	}
-	m := clientmodel.Metric{}
-	if vs, ok := n.(*VectorSelector); ok {
-		for _, matcher := range vs.labelMatchers {
-			if matcher.Type == metric.Equal && matcher.Name != clientmodel.MetricNameLabel {
-				m[matcher.Name] = matcher.Value
-			}
-		}
-	}
-	return Vector{
-		&Sample{
-			Metric: clientmodel.COWMetric{
-				Metric: m,
-				Copied: true,
-			},
-			Value:     1,
-			Timestamp: timestamp,
-		},
-	}
-}
-
-// === ceil(vector VectorNode) Vector ===
-func ceilImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	n := args[0].(VectorNode)
-	vector := n.Eval(timestamp)
-	for _, el := range vector {
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		el.Value = clientmodel.SampleValue(math.Ceil(float64(el.Value)))
-	}
-	return vector
-}
-
-// === exp(vector VectorNode) Vector ===
-func expImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	n := args[0].(VectorNode)
-	vector := n.Eval(timestamp)
-	for _, el := range vector {
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		el.Value = clientmodel.SampleValue(math.Exp(float64(el.Value)))
-	}
-	return vector
-}
-
-// === ln(vector VectorNode) Vector ===
-func lnImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	n := args[0].(VectorNode)
-	vector := n.Eval(timestamp)
-	for _, el := range vector {
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		el.Value = clientmodel.SampleValue(math.Log(float64(el.Value)))
-	}
-	return vector
-}
-
-// === log2(vector VectorNode) Vector ===
-func log2Impl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	n := args[0].(VectorNode)
-	vector := n.Eval(timestamp)
-	for _, el := range vector {
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		el.Value = clientmodel.SampleValue(math.Log2(float64(el.Value)))
-	}
-	return vector
-}
-
-// === log10(vector VectorNode) Vector ===
-func log10Impl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	n := args[0].(VectorNode)
-	vector := n.Eval(timestamp)
-	for _, el := range vector {
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		el.Value = clientmodel.SampleValue(math.Log10(float64(el.Value)))
-	}
-	return vector
-}
-
-// === sqrt(vector VectorNode) Vector ===
-func sqrtImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	n := args[0].(VectorNode)
-	vector := n.Eval(timestamp)
-	for _, el := range vector {
-		el.Metric.Delete(clientmodel.MetricNameLabel)
-		el.Value = clientmodel.SampleValue(math.Sqrt(float64(el.Value)))
-	}
-	return vector
-}
-
-// === deriv(node MatrixNode) Vector ===
-func derivImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	matrixNode := args[0].(MatrixNode)
-	resultVector := Vector{}
-
-	matrixValue := matrixNode.Eval(timestamp)
-	for _, samples := range matrixValue {
-		// No sense in trying to compute a derivative without at least two points.
-		// Drop this vector element.
-		if len(samples.Values) < 2 {
-			continue
-		}
-
-		// Least squares.
-		n := clientmodel.SampleValue(0)
-		sumY := clientmodel.SampleValue(0)
-		sumX := clientmodel.SampleValue(0)
-		sumXY := clientmodel.SampleValue(0)
-		sumX2 := clientmodel.SampleValue(0)
-		for _, sample := range samples.Values {
-			x := clientmodel.SampleValue(sample.Timestamp.UnixNano() / 1e9)
-			n += 1.0
-			sumY += sample.Value
-			sumX += x
-			sumXY += x * sample.Value
-			sumX2 += x * x
-		}
-		numerator := sumXY - sumX*sumY/n
-		denominator := sumX2 - (sumX*sumX)/n
-
-		resultValue := numerator / denominator
-
-		resultSample := &Sample{
-			Metric:    samples.Metric,
-			Value:     resultValue,
-			Timestamp: timestamp,
-		}
-		resultSample.Metric.Delete(clientmodel.MetricNameLabel)
-		resultVector = append(resultVector, resultSample)
-	}
-	return resultVector
-}
-
-// === histogram_quantile(k ScalarNode, vector VectorNode) Vector ===
-func histogramQuantileImpl(timestamp clientmodel.Timestamp, args []Node) interface{} {
-	q := args[0].(ScalarNode).Eval(timestamp)
-	inVec := args[1].(VectorNode).Eval(timestamp)
-	outVec := Vector{}
-	signatureToMetricWithBuckets := map[uint64]*metricWithBuckets{}
-	for _, el := range inVec {
-		upperBound, err := strconv.ParseFloat(
-			string(el.Metric.Metric[clientmodel.BucketLabel]), 64,
-		)
-		if err != nil {
-			// Oops, no bucket label or malformed label value. Skip.
-			// TODO(beorn7): Issue a warning somehow.
-			continue
-		}
-		signature := clientmodel.SignatureWithoutLabels(el.Metric.Metric, excludedLabels)
-		mb, ok := signatureToMetricWithBuckets[signature]
-		if !ok {
-			el.Metric.Delete(clientmodel.BucketLabel)
-			el.Metric.Delete(clientmodel.MetricNameLabel)
-			mb = &metricWithBuckets{el.Metric, nil}
-			signatureToMetricWithBuckets[signature] = mb
-		}
-		mb.buckets = append(mb.buckets, bucket{upperBound, el.Value})
-	}
-
-	for _, mb := range signatureToMetricWithBuckets {
-		outVec = append(outVec, &Sample{
-			Metric:    mb.metric,
-			Value:     clientmodel.SampleValue(quantile(q, mb.buckets)),
-			Timestamp: timestamp,
-		})
-	}
-
-	return outVec
-}
-
-var functions = map[string]*Function{
-	"abs": {
-		name:       "abs",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     absImpl,
-	},
-	"absent": {
-		name:       "absent",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     absentImpl,
-	},
-	"avg_over_time": {
-		name:       "avg_over_time",
-		argTypes:   []ExprType{MatrixType},
-		returnType: VectorType,
-		callFn:     avgOverTimeImpl,
-	},
-	"bottomk": {
-		name:       "bottomk",
-		argTypes:   []ExprType{ScalarType, VectorType},
-		returnType: VectorType,
-		callFn:     bottomkImpl,
-	},
-	"ceil": {
-		name:       "ceil",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     ceilImpl,
-	},
-	"count_over_time": {
-		name:       "count_over_time",
-		argTypes:   []ExprType{MatrixType},
-		returnType: VectorType,
-		callFn:     countOverTimeImpl,
-	},
-	"count_scalar": {
-		name:       "count_scalar",
-		argTypes:   []ExprType{VectorType},
-		returnType: ScalarType,
-		callFn:     countScalarImpl,
-	},
-	"delta": {
-		name:         "delta",
-		argTypes:     []ExprType{MatrixType, ScalarType},
-		optionalArgs: 1, // The 2nd argument is deprecated.
-		returnType:   VectorType,
-		callFn:       deltaImpl,
-	},
-	"deriv": {
-		name:       "deriv",
-		argTypes:   []ExprType{MatrixType},
-		returnType: VectorType,
-		callFn:     derivImpl,
-	},
-	"drop_common_labels": {
-		name:       "drop_common_labels",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     dropCommonLabelsImpl,
-	},
-	"exp": {
-		name:       "exp",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     expImpl,
-	},
-	"floor": {
-		name:       "floor",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     floorImpl,
-	},
-	"histogram_quantile": {
-		name:       "histogram_quantile",
-		argTypes:   []ExprType{ScalarType, VectorType},
-		returnType: VectorType,
-		callFn:     histogramQuantileImpl,
-	},
-	"ln": {
-		name:       "ln",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     lnImpl,
-	},
-	"log10": {
-		name:       "log10",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     log10Impl,
-	},
-	"log2": {
-		name:       "log2",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     log2Impl,
-	},
-	"max_over_time": {
-		name:       "max_over_time",
-		argTypes:   []ExprType{MatrixType},
-		returnType: VectorType,
-		callFn:     maxOverTimeImpl,
-	},
-	"min_over_time": {
-		name:       "min_over_time",
-		argTypes:   []ExprType{MatrixType},
-		returnType: VectorType,
-		callFn:     minOverTimeImpl,
-	},
-	"rate": {
-		name:       "rate",
-		argTypes:   []ExprType{MatrixType},
-		returnType: VectorType,
-		callFn:     rateImpl,
-	},
-	"round": {
-		name:         "round",
-		argTypes:     []ExprType{VectorType, ScalarType},
-		optionalArgs: 1,
-		returnType:   VectorType,
-		callFn:       roundImpl,
-	},
-	"scalar": {
-		name:       "scalar",
-		argTypes:   []ExprType{VectorType},
-		returnType: ScalarType,
-		callFn:     scalarImpl,
-	},
-	"sort": {
-		name:       "sort",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     sortImpl,
-	},
-	"sort_desc": {
-		name:       "sort_desc",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     sortDescImpl,
-	},
-	"sqrt": {
-		name:       "sqrt",
-		argTypes:   []ExprType{VectorType},
-		returnType: VectorType,
-		callFn:     sqrtImpl,
-	},
-	"sum_over_time": {
-		name:       "sum_over_time",
-		argTypes:   []ExprType{MatrixType},
-		returnType: VectorType,
-		callFn:     sumOverTimeImpl,
-	},
-	"time": {
-		name:       "time",
-		argTypes:   []ExprType{},
-		returnType: ScalarType,
-		callFn:     timeImpl,
-	},
-	"topk": {
-		name:       "topk",
-		argTypes:   []ExprType{ScalarType, VectorType},
-		returnType: VectorType,
-		callFn:     topkImpl,
-	},
-}
-
-// GetFunction returns a predefined Function object for the given
-// name.
-func GetFunction(name string) (*Function, error) {
-	function, ok := functions[name]
-	if !ok {
-		return nil, fmt.Errorf("couldn't find function %v()", name)
-	}
-	return function, nil
-}

rules/ast/functions_test.go

@@ -1,63 +0,0 @@
-// Copyright 2013 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ast
-
-import (
-	"testing"
-
-	clientmodel "github.com/prometheus/client_golang/model"
-
-	"github.com/prometheus/prometheus/storage/metric"
-)
-
-type emptyRangeNode struct{}
-
-func (node emptyRangeNode) Type() ExprType             { return MatrixType }
-func (node emptyRangeNode) NodeTreeToDotGraph() string { return "" }
-func (node emptyRangeNode) String() string             { return "" }
-func (node emptyRangeNode) Children() Nodes            { return Nodes{} }
-
-func (node emptyRangeNode) Eval(timestamp clientmodel.Timestamp) Matrix {
-	return Matrix{
-		SampleStream{
-			Metric: clientmodel.COWMetric{
-				Metric: clientmodel.Metric{clientmodel.MetricNameLabel: "empty_metric"},
-			},
-			Values: metric.Values{},
-		},
-	}
-}
-
-func (node emptyRangeNode) EvalBoundaries(timestamp clientmodel.Timestamp) Matrix {
-	return Matrix{
-		SampleStream{
-			Metric: clientmodel.COWMetric{
-				Metric: clientmodel.Metric{clientmodel.MetricNameLabel: "empty_metric"},
-			},
-			Values: metric.Values{},
-		},
-	}
-}
-
-func TestDeltaWithEmptyElementDoesNotCrash(t *testing.T) {
-	now := clientmodel.Now()
-	vector := deltaImpl(now, []Node{emptyRangeNode{}, &ScalarLiteral{value: 0}}).(Vector)
-	if len(vector) != 0 {
-		t.Fatalf("Expected empty result vector, got: %v", vector)
-	}
-	vector = deltaImpl(now, []Node{emptyRangeNode{}, &ScalarLiteral{value: 1}}).(Vector)
-	if len(vector) != 0 {
-		t.Fatalf("Expected empty result vector, got: %v", vector)
-	}
-}

rules/ast/printer.go

@@ -1,443 +0,0 @@
-// Copyright 2013 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ast
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"reflect"
-	"sort"
-	"strings"
-
-	clientmodel "github.com/prometheus/client_golang/model"
-
-	"github.com/prometheus/prometheus/stats"
-	"github.com/prometheus/prometheus/storage/local"
-	"github.com/prometheus/prometheus/utility"
-)
-
-// OutputFormat is an enum for the possible output formats.
-type OutputFormat int
-
-// Possible output formats.
-const (
-	Text OutputFormat = iota
-	JSON
-)
-
-const jsonFormatVersion = 1
-
-func (opType BinOpType) String() string {
-	opTypeMap := map[BinOpType]string{
-		Add: "+",
-		Sub: "-",
-		Mul: "*",
-		Div: "/",
-		Mod: "%",
-		GT:  ">",
-		LT:  "<",
-		EQ:  "==",
-		NE:  "!=",
-		GE:  ">=",
-		LE:  "<=",
-		And: "AND",
-		Or:  "OR",
-	}
-	return opTypeMap[opType]
-}
-
-func (aggrType AggrType) String() string {
-	aggrTypeMap := map[AggrType]string{
-		Sum:    "SUM",
-		Avg:    "AVG",
-		Min:    "MIN",
-		Max:    "MAX",
-		Count:  "COUNT",
-		Stdvar: "STDVAR",
-		Stddev: "STDDEV",
-	}
-	return aggrTypeMap[aggrType]
-}
-
-func (exprType ExprType) String() string {
-	exprTypeMap := map[ExprType]string{
-		ScalarType: "scalar",
-		VectorType: "vector",
-		MatrixType: "matrix",
-		StringType: "string",
-	}
-	return exprTypeMap[exprType]
-}
-
-func (vector Vector) String() string {
-	metricStrings := make([]string, 0, len(vector))
-	for _, sample := range vector {
-		metricStrings = append(metricStrings,
-			fmt.Sprintf("%s => %v @[%v]",
-				sample.Metric,
-				sample.Value, sample.Timestamp))
-	}
-	return strings.Join(metricStrings, "\n")
-}
-
-func (matrix Matrix) String() string {
-	metricStrings := make([]string, 0, len(matrix))
-	for _, sampleStream := range matrix {
-		metricName, hasName := sampleStream.Metric.Metric[clientmodel.MetricNameLabel]
-		numLabels := len(sampleStream.Metric.Metric)
-		if hasName {
-			numLabels--
-		}
-		labelStrings := make([]string, 0, numLabels)
-		for label, value := range sampleStream.Metric.Metric {
-			if label != clientmodel.MetricNameLabel {
-				labelStrings = append(labelStrings, fmt.Sprintf("%s=%q", label, value))
-			}
-		}
-		sort.Strings(labelStrings)
-		valueStrings := make([]string, 0, len(sampleStream.Values))
-		for _, value := range sampleStream.Values {
-			valueStrings = append(valueStrings,
-				fmt.Sprintf("\n%v @[%v]", value.Value, value.Timestamp))
-		}
-		metricStrings = append(metricStrings,
-			fmt.Sprintf("%s{%s} => %s",
-				metricName,
-				strings.Join(labelStrings, ", "),
-				strings.Join(valueStrings, ", ")))
-	}
-	sort.Strings(metricStrings)
-	return strings.Join(metricStrings, "\n")
-}
-
-// ErrorToJSON converts the given error into JSON.
-func ErrorToJSON(err error) string {
-	errorStruct := struct {
-		Type    string `json:"type"`
-		Value   string `json:"value"`
-		Version int    `json:"version"`
-	}{
-		Type:    "error",
-		Value:   err.Error(),
-		Version: jsonFormatVersion,
-	}
-
-	errorJSON, err := json.Marshal(errorStruct)
-	if err != nil {
-		return ""
-	}
-	return string(errorJSON)
-}
-
-// TypedValueToJSON converts the given data of type 'scalar',
-// 'vector', or 'matrix' into its JSON representation.
-func TypedValueToJSON(data interface{}, typeStr string) string {
-	dataStruct := struct {
-		Type    string      `json:"type"`
-		Value   interface{} `json:"value"`
-		Version int         `json:"version"`
-	}{
-		Type:    typeStr,
-		Value:   data,
-		Version: jsonFormatVersion,
-	}
-	dataJSON, err := json.Marshal(dataStruct)
-	if err != nil {
-		return ErrorToJSON(err)
-	}
-	return string(dataJSON)
-}
-
-// EvalToString evaluates the given node into a string of the given format.
-func EvalToString(node Node, timestamp clientmodel.Timestamp, format OutputFormat, storage local.Storage, queryStats *stats.TimerGroup) string {
-	totalEvalTimer := queryStats.GetTimer(stats.TotalEvalTime).Start()
-	defer totalEvalTimer.Stop()
-
-	prepareTimer := queryStats.GetTimer(stats.TotalQueryPreparationTime).Start()
-	closer, err := PrepareInstantQuery(node, timestamp, storage, queryStats)
-	prepareTimer.Stop()
-	if err != nil {
-		panic(err)
-	}
-	defer closer.Close()
-
-	evalTimer := queryStats.GetTimer(stats.InnerEvalTime).Start()
-	switch node.Type() {
-	case ScalarType:
-		scalar := node.(ScalarNode).Eval(timestamp)
-		evalTimer.Stop()
-		switch format {
-		case Text:
-			return fmt.Sprintf("scalar: %v @[%v]", scalar, timestamp)
-		case JSON:
-			return TypedValueToJSON(scalar, "scalar")
-		}
-	case VectorType:
-		vector := node.(VectorNode).Eval(timestamp)
-		evalTimer.Stop()
-		switch format {
-		case Text:
-			return vector.String()
-		case JSON:
-			return TypedValueToJSON(vector, "vector")
-		}
-	case MatrixType:
-		matrix := node.(MatrixNode).Eval(timestamp)
-		evalTimer.Stop()
-		switch format {
-		case Text:
-			return matrix.String()
-		case JSON:
-			return TypedValueToJSON(matrix, "matrix")
-		}
-	case StringType:
-		str := node.(StringNode).Eval(timestamp)
-		evalTimer.Stop()
-		switch format {
-		case Text:
-			return str
-		case JSON:
-			return TypedValueToJSON(str, "string")
-		}
-	}
-	panic("Switch didn't cover all node types")
-}
-
-// EvalToVector evaluates the given node into a Vector. Matrices aren't supported.
-func EvalToVector(node Node, timestamp clientmodel.Timestamp, storage local.Storage, queryStats *stats.TimerGroup) (Vector, error) {
-	totalEvalTimer := queryStats.GetTimer(stats.TotalEvalTime).Start()
-	defer totalEvalTimer.Stop()
-
-	prepareTimer := queryStats.GetTimer(stats.TotalQueryPreparationTime).Start()
-	closer, err := PrepareInstantQuery(node, timestamp, storage, queryStats)
-	prepareTimer.Stop()
-	if err != nil {
-		panic(err)
-	}
-	defer closer.Close()
-
-	evalTimer := queryStats.GetTimer(stats.InnerEvalTime).Start()
-	switch node.Type() {
-	case ScalarType:
-		scalar := node.(ScalarNode).Eval(timestamp)
-		evalTimer.Stop()
-		return Vector{&Sample{Value: scalar}}, nil
-	case VectorType:
-		vector := node.(VectorNode).Eval(timestamp)
-		evalTimer.Stop()
-		return vector, nil
-	case MatrixType:
-		return nil, errors.New("matrices not supported by EvalToVector")
-	case StringType:
-		str := node.(StringNode).Eval(timestamp)
-		evalTimer.Stop()
-		return Vector{
-			&Sample{
-				Metric: clientmodel.COWMetric{
-					Metric: clientmodel.Metric{
-						"__value__": clientmodel.LabelValue(str),
-					},
-					Copied: true,
-				},
-			},
-		}, nil
-	}
-	panic("Switch didn't cover all node types")
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the scalar
-// literal.
-func (node *ScalarLiteral) NodeTreeToDotGraph() string {
-	return fmt.Sprintf("%#p[label=\"%v\"];\n", node, node.value)
-}
-
-func functionArgsToDotGraph(node Node, args []Node) string {
-	graph := ""
-	for _, arg := range args {
-		graph += fmt.Sprintf("%x -> %x;\n", reflect.ValueOf(node).Pointer(), reflect.ValueOf(arg).Pointer())
-	}
-	for _, arg := range args {
-		graph += arg.NodeTreeToDotGraph()
-	}
-	return graph
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the function
-// call.
-func (node *ScalarFunctionCall) NodeTreeToDotGraph() string {
-	graph := fmt.Sprintf("%#p[label=\"%s\"];\n", node, node.function.name)
-	graph += functionArgsToDotGraph(node, node.args)
-	return graph
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the expression.
-func (node *ScalarArithExpr) NodeTreeToDotGraph() string {
-	nodeAddr := reflect.ValueOf(node).Pointer()
-	graph := fmt.Sprintf(
-		`
-		%x[label="%s"];
-		%x -> %x;
-		%x -> %x;
-		%s
-		%s
-	}`,
-		nodeAddr, node.opType,
-		nodeAddr, reflect.ValueOf(node.lhs).Pointer(),
-		nodeAddr, reflect.ValueOf(node.rhs).Pointer(),
-		node.lhs.NodeTreeToDotGraph(),
-		node.rhs.NodeTreeToDotGraph(),
-	)
-	return graph
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the vector selector.
-func (node *VectorSelector) NodeTreeToDotGraph() string {
-	return fmt.Sprintf("%#p[label=\"%s\"];\n", node, node)
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the function
-// call.
-func (node *VectorFunctionCall) NodeTreeToDotGraph() string {
-	graph := fmt.Sprintf("%#p[label=\"%s\"];\n", node, node.function.name)
-	graph += functionArgsToDotGraph(node, node.args)
-	return graph
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the vector
-// aggregation.
-func (node *VectorAggregation) NodeTreeToDotGraph() string {
-	groupByStrings := make([]string, 0, len(node.groupBy))
-	for _, label := range node.groupBy {
-		groupByStrings = append(groupByStrings, string(label))
-	}
-
-	graph := fmt.Sprintf("%#p[label=\"%s BY (%s)\"]\n",
-		node,
-		node.aggrType,
-		strings.Join(groupByStrings, ", "))
-	graph += fmt.Sprintf("%#p -> %x;\n", node, reflect.ValueOf(node.vector).Pointer())
-	graph += node.vector.NodeTreeToDotGraph()
-	return graph
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the expression.
-func (node *VectorArithExpr) NodeTreeToDotGraph() string {
-	nodeAddr := reflect.ValueOf(node).Pointer()
-	graph := fmt.Sprintf(
-		`
-		%x[label="%s"];
-		%x -> %x;
-		%x -> %x;
-		%s
-		%s
-	}`,
-		nodeAddr, node.opType,
-		nodeAddr, reflect.ValueOf(node.lhs).Pointer(),
-		nodeAddr, reflect.ValueOf(node.rhs).Pointer(),
-		node.lhs.NodeTreeToDotGraph(),
-		node.rhs.NodeTreeToDotGraph(),
-	)
-	return graph
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the matrix
-// selector.
-func (node *MatrixSelector) NodeTreeToDotGraph() string {
-	return fmt.Sprintf("%#p[label=\"%s\"];\n", node, node)
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the string
-// literal.
-func (node *StringLiteral) NodeTreeToDotGraph() string {
-	return fmt.Sprintf("%#p[label=\"'%q'\"];\n", node, node.str)
-}
-
-// NodeTreeToDotGraph returns a DOT representation of the function
-// call.
-func (node *StringFunctionCall) NodeTreeToDotGraph() string {
-	graph := fmt.Sprintf("%#p[label=\"%s\"];\n", node, node.function.name)
-	graph += functionArgsToDotGraph(node, node.args)
-	return graph
-}
-
-func (nodes Nodes) String() string {
-	nodeStrings := make([]string, 0, len(nodes))
-	for _, node := range nodes {
-		nodeStrings = append(nodeStrings, node.String())
-	}
-	return strings.Join(nodeStrings, ", ")
-}
-
-func (node *ScalarLiteral) String() string {
-	return fmt.Sprint(node.value)
-}
-
-func (node *ScalarFunctionCall) String() string {
-	return fmt.Sprintf("%s(%s)", node.function.name, node.args)
-}
-
-func (node *ScalarArithExpr) String() string {
-	return fmt.Sprintf("(%s %s %s)", node.lhs, node.opType, node.rhs)
-}
-
-func (node *VectorSelector) String() string {
-	labelStrings := make([]string, 0, len(node.labelMatchers)-1)
-	var metricName clientmodel.LabelValue
-	for _, matcher := range node.labelMatchers {
-		if matcher.Name != clientmodel.MetricNameLabel {
-			labelStrings = append(labelStrings, fmt.Sprintf("%s%s%q", matcher.Name, matcher.Type, matcher.Value))
-		} else {
-			metricName = matcher.Value
-		}
-	}
-
-	switch len(labelStrings) {
-	case 0:
-		return string(metricName)
-	default:
-		sort.Strings(labelStrings)
-		return fmt.Sprintf("%s{%s}", metricName, strings.Join(labelStrings, ","))
-	}
-}
-
-func (node *VectorFunctionCall) String() string {
-	return fmt.Sprintf("%s(%s)", node.function.name, node.args)
-}
-
-func (node *VectorAggregation) String() string {
-	aggrString := fmt.Sprintf("%s(%s)", node.aggrType, node.vector)
-	if len(node.groupBy) > 0 {
-		return fmt.Sprintf("%s BY (%s)", aggrString, node.groupBy)
-	}
-	return aggrString
-}
-
-func (node *VectorArithExpr) String() string {
-	return fmt.Sprintf("(%s %s %s)", node.lhs, node.opType, node.rhs)
-}
-
-func (node *MatrixSelector) String() string {
-	vectorString := (&VectorSelector{labelMatchers: node.labelMatchers}).String()
-	intervalString := fmt.Sprintf("[%s]", utility.DurationToString(node.interval))
-	return vectorString + intervalString
-}
-
-func (node *StringLiteral) String() string {
-	return fmt.Sprintf("%q", node.str)
-}
-
-func (node *StringFunctionCall) String() string {
-	return fmt.Sprintf("%s(%s)", node.function.name, node.args)
-}

rules/ast/query_analyzer.go

@@ -1,231 +0,0 @@
-// Copyright 2013 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ast
-
-import (
-	"time"
-
-	clientmodel "github.com/prometheus/client_golang/model"
-
-	"github.com/prometheus/prometheus/stats"
-	"github.com/prometheus/prometheus/storage/local"
-)
-
-// preloadTimes tracks which instants or ranges to preload for a set of
-// fingerprints. One of these structs is collected for each offset by the query
-// analyzer.
-type preloadTimes struct {
-	// Instants require single samples to be loaded along the entire query
-	// range, with intervals between the samples corresponding to the query
-	// resolution.
-	instants map[clientmodel.Fingerprint]struct{}
-	// Ranges require loading a range of samples at each resolution step,
-	// stretching backwards from the current evaluation timestamp. The length of
-	// the range into the past is given by the duration, as in "foo[5m]".
-	ranges map[clientmodel.Fingerprint]time.Duration
-}
-
-// A queryAnalyzer recursively traverses the AST to look for any nodes
-// which will need data from the datastore. Instantiate with
-// newQueryAnalyzer.
-type queryAnalyzer struct {
-	// Tracks one set of times to preload per offset that occurs in the query
-	// expression.
-	offsetPreloadTimes map[time.Duration]preloadTimes
-	// The underlying storage to which the query will be applied. Needed for
-	// extracting timeseries fingerprint information during query analysis.
-	storage local.Storage
-}
-
-// newQueryAnalyzer returns a pointer to a newly instantiated
-// queryAnalyzer. The storage is needed to extract timeseries
-// fingerprint information during query analysis.
-func newQueryAnalyzer(storage local.Storage) *queryAnalyzer {
-	return &queryAnalyzer{
-		offsetPreloadTimes: map[time.Duration]preloadTimes{},
-		storage:            storage,
-	}
-}
-
-func (analyzer *queryAnalyzer) getPreloadTimes(offset time.Duration) preloadTimes {
-	if _, ok := analyzer.offsetPreloadTimes[offset]; !ok {
-		analyzer.offsetPreloadTimes[offset] = preloadTimes{
-			instants: map[clientmodel.Fingerprint]struct{}{},
-			ranges:   map[clientmodel.Fingerprint]time.Duration{},
-		}
-	}
-	return analyzer.offsetPreloadTimes[offset]
-}
-
-// visit implements the visitor interface.
-func (analyzer *queryAnalyzer) visit(node Node) {
-	switch n := node.(type) {
-	case *VectorSelector:
-		pt := analyzer.getPreloadTimes(n.offset)
-		fingerprints := analyzer.storage.GetFingerprintsForLabelMatchers(n.labelMatchers)
-		n.fingerprints = fingerprints
-		for _, fp := range fingerprints {
-			// Only add the fingerprint to the instants if not yet present in the
-			// ranges. Ranges always contain more points and span more time than
-			// instants for the same offset.
-			if _, alreadyInRanges := pt.ranges[fp]; !alreadyInRanges {
-				pt.instants[fp] = struct{}{}
-			}
-
-			n.metrics[fp] = analyzer.storage.GetMetricForFingerprint(fp)
-		}
-	case *MatrixSelector:
-		pt := analyzer.getPreloadTimes(n.offset)
-		fingerprints := analyzer.storage.GetFingerprintsForLabelMatchers(n.labelMatchers)
-		n.fingerprints = fingerprints
-		for _, fp := range fingerprints {
-			if pt.ranges[fp] < n.interval {
-				pt.ranges[fp] = n.interval
-				// Delete the fingerprint from the instants. Ranges always contain more
-				// points and span more time than instants, so we don't need to track
-				// an instant for the same fingerprint, should we have one.
-				delete(pt.instants, fp)
-			}
-
-			n.metrics[fp] = analyzer.storage.GetMetricForFingerprint(fp)
-		}
-	}
-}
-
-type iteratorInitializer struct {
-	storage local.Storage
-}
-
-func (i *iteratorInitializer) visit(node Node) {
-	switch n := node.(type) {
-	case *VectorSelector:
-		for _, fp := range n.fingerprints {
-			n.iterators[fp] = i.storage.NewIterator(fp)
-		}
-	case *MatrixSelector:
-		for _, fp := range n.fingerprints {
-			n.iterators[fp] = i.storage.NewIterator(fp)
-		}
-	}
-}
-
-// PrepareInstantQuery analyzes the query and preloads the necessary time range for each series.
-func PrepareInstantQuery(node Node, timestamp clientmodel.Timestamp, storage local.Storage, queryStats *stats.TimerGroup) (local.Preloader, error) {
-	totalTimer := queryStats.GetTimer(stats.TotalEvalTime)
-
-	analyzeTimer := queryStats.GetTimer(stats.QueryAnalysisTime).Start()
-	analyzer := newQueryAnalyzer(storage)
-	Walk(analyzer, node)
-	analyzeTimer.Stop()
-
-	preloadTimer := queryStats.GetTimer(stats.PreloadTime).Start()
-	p := storage.NewPreloader()
-	for offset, pt := range analyzer.offsetPreloadTimes {
-		ts := timestamp.Add(-offset)
-		for fp, rangeDuration := range pt.ranges {
-			if et := totalTimer.ElapsedTime(); et > *queryTimeout {
-				preloadTimer.Stop()
-				p.Close()
-				return nil, queryTimeoutError{et}
-			}
-			if err := p.PreloadRange(fp, ts.Add(-rangeDuration), ts, *stalenessDelta); err != nil {
-				preloadTimer.Stop()
-				p.Close()
-				return nil, err
-			}
-		}
-		for fp := range pt.instants {
-			if et := totalTimer.ElapsedTime(); et > *queryTimeout {
-				preloadTimer.Stop()
-				p.Close()
-				return nil, queryTimeoutError{et}
-			}
-			if err := p.PreloadRange(fp, ts, ts, *stalenessDelta); err != nil {
-				preloadTimer.Stop()
-				p.Close()
-				return nil, err
-			}
-		}
-	}
-	preloadTimer.Stop()
-
-	ii := &iteratorInitializer{
-		storage: storage,
-	}
-	Walk(ii, node)
-
-	return p, nil
-}
-
-// PrepareRangeQuery analyzes the query and preloads the necessary time range for each series.
-func PrepareRangeQuery(node Node, start clientmodel.Timestamp, end clientmodel.Timestamp, interval time.Duration, storage local.Storage, queryStats *stats.TimerGroup) (local.Preloader, error) {
-	totalTimer := queryStats.GetTimer(stats.TotalEvalTime)
-
-	analyzeTimer := queryStats.GetTimer(stats.QueryAnalysisTime).Start()
-	analyzer := newQueryAnalyzer(storage)
-	Walk(analyzer, node)
-	analyzeTimer.Stop()
-
-	preloadTimer := queryStats.GetTimer(stats.PreloadTime).Start()
-	p := storage.NewPreloader()
-	for offset, pt := range analyzer.offsetPreloadTimes {
-		offsetStart := start.Add(-offset)
-		offsetEnd := end.Add(-offset)
-		for fp, rangeDuration := range pt.ranges {
-			if et := totalTimer.ElapsedTime(); et > *queryTimeout {
-				preloadTimer.Stop()
-				p.Close()
-				return nil, queryTimeoutError{et}
-			}
-			if err := p.PreloadRange(fp, offsetStart.Add(-rangeDuration), offsetEnd, *stalenessDelta); err != nil {
-				preloadTimer.Stop()
-				p.Close()
-				return nil, err
-			}
-			/*
-				if interval < rangeDuration {
-					if err := p.GetMetricRange(fp, offsetEnd, offsetEnd.Sub(offsetStart)+rangeDuration); err != nil {
-						p.Close()
-						return nil, err
-					}
-				} else {
-					if err := p.GetMetricRangeAtInterval(fp, offsetStart, offsetEnd, interval, rangeDuration); err != nil {
-						p.Close()
-						return nil, err
-					}
-				}
-			*/
-		}
-		for fp := range pt.instants {
-			if et := totalTimer.ElapsedTime(); et > *queryTimeout {
-				preloadTimer.Stop()
-				p.Close()
-				return nil, queryTimeoutError{et}
-			}
-			if err := p.PreloadRange(fp, offsetStart, offsetEnd, *stalenessDelta); err != nil {
-				preloadTimer.Stop()
-				p.Close()
-				return nil, err
-			}
-		}
-	}
-	preloadTimer.Stop()
-
-	ii := &iteratorInitializer{
-		storage: storage,
-	}
-	Walk(ii, node)
-
-	return p, nil
-}

rules/ast/walk.go

@@ -1,28 +0,0 @@
-// Copyright 2013 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package ast
-
-// visitor is the interface for a Node visitor.
-type visitor interface {
-	visit(node Node)
-}
-
-// Walk does a depth-first traversal of the AST, starting at node,
-// calling visitor.visit for each encountered Node in the tree.
-func Walk(v visitor, node Node) {
-	v.visit(node)
-	for _, childNode := range node.Children() {
-		Walk(v, childNode)
-	}
-}

rules/fixtures/mixed.rules

@@ -1,16 +0,0 @@
-// A simple test recording rule.
-dc_http_request_rate5m = sum(rate(http_request_count[5m])) by (dc)
-
-// A simple test alerting rule.
-ALERT GlobalRequestRateLow IF(dc_http_request_rate5m < 10000) FOR 5m WITH {
-    service = "testservice"
-    /* ... more fields here ... */
-  }
-  SUMMARY "Global request rate low"
-  DESCRIPTION "The global request rate is low"
-
-foo = bar{label1="value1"}
-
-ALERT BazAlert IF(foo > 10) WITH {}
-  SUMMARY "Baz"
-  DESCRIPTION "BazAlert"

rules/fixtures/non_vector.rules

@@ -1 +0,0 @@
-now = time()

rules/fixtures/syntax_error.rules

@@ -1,15 +0,0 @@
-// A simple test recording rule.
-dc_http_request_rate5m = sum(rate(http_request_count[5m])) by (dc)
-
-// A simple test alerting rule with a syntax error (invalid duration string "5").
-ALERT GlobalRequestRateLow IF(dc_http_request_rate5m < 10000) FOR 5 WITH {
-    description = "Global HTTP request rate low!",
-    summary = "Request rate low"
-    /* ... more fields here ... */
-  }
-  SUMMARY "summary"
-  DESCRIPTION "description"
-
-foo = bar{label1="value1"}
-
-ALERT BazAlert IF(foo > 10) WITH {} SUMMARY "summary" DESCRIPTION "description"

rules/helpers.go

@@ -1,223 +0,0 @@
-// Copyright 2013 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rules
-
-import (
-	"fmt"
-	"net/url"
-	"strings"
-
-	clientmodel "github.com/prometheus/client_golang/model"
-
-	"github.com/prometheus/prometheus/rules/ast"
-	"github.com/prometheus/prometheus/storage/metric"
-	"github.com/prometheus/prometheus/utility"
-)
-
-// CreateRecordingRule is a convenience function to create a recording rule.
-func CreateRecordingRule(name string, labels clientmodel.LabelSet, expr ast.Node, permanent bool) (*RecordingRule, error) {
-	if _, ok := expr.(ast.VectorNode); !ok {
-		return nil, fmt.Errorf("recording rule expression %v does not evaluate to vector type", expr)
-	}
-	return &RecordingRule{
-		name:      name,
-		labels:    labels,
-		vector:    expr.(ast.VectorNode),
-		permanent: permanent,
-	}, nil
-}
-
-// CreateAlertingRule is a convenience function to create a new alerting rule.
-func CreateAlertingRule(name string, expr ast.Node, holdDurationStr string, labels clientmodel.LabelSet, summary string, description string) (*AlertingRule, error) {
-	if _, ok := expr.(ast.VectorNode); !ok {
-		return nil, fmt.Errorf("alert rule expression %v does not evaluate to vector type", expr)
-	}
-	holdDuration, err := utility.StringToDuration(holdDurationStr)
-	if err != nil {
-		return nil, err
-	}
-	return NewAlertingRule(name, expr.(ast.VectorNode), holdDuration, labels, summary, description), nil
-}
-
-// NewScalarLiteral returns a ScalarLiteral with the given value. If sign is "-"
-// the value is negated.
-func NewScalarLiteral(value clientmodel.SampleValue, sign string) *ast.ScalarLiteral {
-	if sign == "-" {
-		value = -value
-	}
-	return ast.NewScalarLiteral(value)
-}
-
-// NewFunctionCall is a convenience function to create a new AST function-call node.
-func NewFunctionCall(name string, args []ast.Node) (ast.Node, error) {
-	function, err := ast.GetFunction(name)
-	if err != nil {
-		return nil, fmt.Errorf("unknown function %q", name)
-	}
-	functionCall, err := ast.NewFunctionCall(function, args)
-	if err != nil {
-		return nil, fmt.Errorf(err.Error())
-	}
-	return functionCall, nil
-}
-
-// NewVectorAggregation is a convenience function to create a new AST vector aggregation.
-func NewVectorAggregation(aggrTypeStr string, vector ast.Node, groupBy clientmodel.LabelNames, keepExtraLabels bool) (*ast.VectorAggregation, error) {
-	if _, ok := vector.(ast.VectorNode); !ok {
-		return nil, fmt.Errorf("operand of %v aggregation must be of vector type", aggrTypeStr)
-	}
-	var aggrTypes = map[string]ast.AggrType{
-		"SUM":    ast.Sum,
-		"MAX":    ast.Max,
-		"MIN":    ast.Min,
-		"AVG":    ast.Avg,
-		"COUNT":  ast.Count,
-		"STDVAR": ast.Stdvar,
-		"STDDEV": ast.Stddev,
-	}
-	aggrType, ok := aggrTypes[aggrTypeStr]
-	if !ok {
-		return nil, fmt.Errorf("unknown aggregation type %q", aggrTypeStr)
-	}
-	return ast.NewVectorAggregation(aggrType, vector.(ast.VectorNode), groupBy, keepExtraLabels), nil
-}
-
-// vectorMatching combines data used to match samples between vectors.
-type vectorMatching struct {
-	matchCardinality ast.VectorMatchCardinality
-	matchOn          clientmodel.LabelNames
-	includeLabels    clientmodel.LabelNames
-}
-
-// newVectorMatching is a convenience function to create a new vectorMatching.
-func newVectorMatching(card string, matchOn, include clientmodel.LabelNames) (*vectorMatching, error) {
-	var matchCardinalities = map[string]ast.VectorMatchCardinality{
-		"":            ast.MatchOneToOne,
-		"GROUP_LEFT":  ast.MatchManyToOne,
-		"GROUP_RIGHT": ast.MatchOneToMany,
-	}
-	matchCard, ok := matchCardinalities[card]
-	if !ok {
-		return nil, fmt.Errorf("invalid vector match cardinality %q", card)
-	}
-	if matchCard != ast.MatchOneToOne && len(include) == 0 {
-		return nil, fmt.Errorf("grouped vector matching must provide labels")
-	}
-	// There must be no overlap between both labelname lists.
-	for _, matchLabel := range matchOn {
-		for _, incLabel := range include {
-			if matchLabel == incLabel {
-				return nil, fmt.Errorf("use of label %s in ON and %s clauses not allowed", incLabel, card)
-			}
-		}
-	}
-	return &vectorMatching{matchCard, matchOn, include}, nil
-}
-
-// NewArithExpr is a convenience function to create a new AST arithmetic expression.
-func NewArithExpr(opTypeStr string, lhs ast.Node, rhs ast.Node, vecMatching *vectorMatching) (ast.Node, error) {
-	var opTypes = map[string]ast.BinOpType{
-		"+":   ast.Add,
-		"-":   ast.Sub,
-		"*":   ast.Mul,
-		"/":   ast.Div,
-		"%":   ast.Mod,
-		">":   ast.GT,
-		"<":   ast.LT,
-		"==":  ast.EQ,
-		"!=":  ast.NE,
-		">=":  ast.GE,
-		"<=":  ast.LE,
-		"AND": ast.And,
-		"OR":  ast.Or,
-	}
-	opType, ok := opTypes[opTypeStr]
-	if !ok {
-		return nil, fmt.Errorf("invalid binary operator %q", opTypeStr)
-	}
-	var vm vectorMatching
-	if vecMatching != nil {
-		vm = *vecMatching
-		// And/or always do many-to-many matching.
-		if opType == ast.And || opType == ast.Or {
-			vm.matchCardinality = ast.MatchManyToMany
-		}
-	}
-	expr, err := ast.NewArithExpr(opType, lhs, rhs, vm.matchCardinality, vm.matchOn, vm.includeLabels)
-	if err != nil {
-		return nil, fmt.Errorf(err.Error())
-	}
-	return expr, nil
-}
-
-// NewVectorSelector is a convenience function to create a new AST vector selector.
-func NewVectorSelector(m metric.LabelMatchers, offsetStr string) (ast.VectorNode, error) {
-	offset, err := utility.StringToDuration(offsetStr)
-	if err != nil {
-		return nil, err
-	}
-	return ast.NewVectorSelector(m, offset), nil
-}
-
-// NewMatrixSelector is a convenience function to create a new AST matrix selector.
-func NewMatrixSelector(vector ast.Node, intervalStr string, offsetStr string) (ast.MatrixNode, error) {
-	interval, err := utility.StringToDuration(intervalStr)
-	if err != nil {
-		return nil, err
-	}
-	offset, err := utility.StringToDuration(offsetStr)
-	if err != nil {
-		return nil, err
-	}
-	vectorSelector, ok := vector.(*ast.VectorSelector)
-	if !ok {
-		return nil, fmt.Errorf("intervals are currently only supported for vector selectors")
-	}
-	return ast.NewMatrixSelector(vectorSelector, interval, offset), nil
-}
-
-func newLabelMatcher(matchTypeStr string, name clientmodel.LabelName, value clientmodel.LabelValue) (*metric.LabelMatcher, error) {
-	matchTypes := map[string]metric.MatchType{
-		"=":  metric.Equal,
-		"!=": metric.NotEqual,
-		"=~": metric.RegexMatch,
-		"!~": metric.RegexNoMatch,
-	}
-	matchType, ok := matchTypes[matchTypeStr]
-	if !ok {
-		return nil, fmt.Errorf("invalid label matching operator %q", matchTypeStr)
-	}
-	return metric.NewLabelMatcher(matchType, name, value)
-}
-
-// TableLinkForExpression creates an escaped relative link to the table view of
-// the provided expression.
-func TableLinkForExpression(expr string) string {
-	// url.QueryEscape percent-escapes everything except spaces, for which it
-	// uses "+". However, in the non-query part of a URI, only percent-escaped
-	// spaces are legal, so we need to manually replace "+" with "%20" after
-	// query-escaping the string.
-	//
-	// See also:
-	// http://stackoverflow.com/questions/1634271/url-encoding-the-space-character-or-20.
-	urlData := url.QueryEscape(fmt.Sprintf(`[{"expr":%q,"tab":1}]`, expr))
-	return fmt.Sprintf("/graph#%s", strings.Replace(urlData, "+", "%20", -1))
-}
-
-// GraphLinkForExpression creates an escaped relative link to the graph view of
-// the provided expression.
-func GraphLinkForExpression(expr string) string {
-	urlData := url.QueryEscape(fmt.Sprintf(`[{"expr":%q,"tab":0}]`, expr))
-	return fmt.Sprintf("/graph#%s", strings.Replace(urlData, "+", "%20", -1))
-}

rules/lexer.l

@@ -1,118 +0,0 @@
-/* Copyright 2013 The Prometheus Authors
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. */
-
-%{
-package rules
-
-import (
-        "fmt"
-        "strconv"
-        "strings"
-
-        clientmodel "github.com/prometheus/client_golang/model"
-)
-
-// Lex is called by the parser generated by "go tool yacc" to obtain each
-// token. The method is opened before the matching rules block and closed at
-// the end of the file.
-func (lexer *RulesLexer) Lex(lval *yySymType) int {
-  // Internal lexer states.
-  const (
-    S_INITIAL = iota
-    S_COMMENTS
-  )
-
-  // We simulate multiple start symbols for closely-related grammars via dummy tokens. See
-  // http://www.gnu.org/software/bison/manual/html_node/Multiple-start_002dsymbols.html
-  // Reason: we want to be able to parse lists of named rules as well as single expressions.
-  if lexer.startToken != 0 {
-    startToken := lexer.startToken
-    lexer.startToken = 0
-    return startToken
-  }
-
-  c := lexer.current
-  currentState := 0
-
-  if lexer.empty {
-    c, lexer.empty = lexer.getChar(), false
-  }
-
-%}
-
-D                       [0-9]
-L                       [a-zA-Z_]
-M                       [a-zA-Z_:]
-U                       [smhdwy]
-
-FLOAT                   ({D}*\.?{D}+|{D}+\.?{D}*){EXPONENT}?|[+-]?[iI][nN][fF]|[nN][aA][nN]
-EXPONENT                [eE][-+]?[0-9]+
-
-STR                     \"(\\.|[^\\"])*\"|\'(\\.|[^\\'])*\'
-
-%x S_COMMENTS
-
-%yyc c
-%yyn c = lexer.getChar()
-%yyt currentState
-
-%%
-  lexer.buf = lexer.buf[:0]   // The code before the first rule executed before every scan cycle (rule #0 / state 0 action)
-
-"/*"                     currentState = S_COMMENTS
-<S_COMMENTS>"*/"         currentState = S_INITIAL
-<S_COMMENTS>.|\n         /* ignore chars within multi-line comments */
-
-\/\/[^\r\n]*\n           /* gobble up one-line comments */
-
-ALERT|alert              return ALERT
-IF|if                    return IF
-FOR|for                  return FOR
-WITH|with                return WITH
-SUMMARY|summary          return SUMMARY
-DESCRIPTION|description  return DESCRIPTION
-
-PERMANENT|permanent      return PERMANENT
-BY|by                    return GROUP_OP
-ON|on                    return MATCH_OP
-GROUP_LEFT|GROUP_RIGHT   lval.str = lexer.token(); return MATCH_MOD
-group_left|group_right   lval.str = strings.ToUpper(lexer.token()); return MATCH_MOD
-KEEPING_EXTRA|keeping_extra return KEEPING_EXTRA
-OFFSET|offset            return OFFSET
-AVG|SUM|MAX|MIN|COUNT|STDVAR|STDDEV    lval.str = lexer.token(); return AGGR_OP
-avg|sum|max|min|count|stdvar|stddev    lval.str = strings.ToUpper(lexer.token()); return AGGR_OP
-\<|>|AND|OR|and|or       lval.str = strings.ToUpper(lexer.token()); return CMP_OP
-==|!=|>=|<=|=~|!~        lval.str = lexer.token(); return CMP_OP
-[+\-]                    lval.str = lexer.token(); return ADDITIVE_OP
-[*/%]                    lval.str = lexer.token(); return MULT_OP
-
-{FLOAT}                  num, err := strconv.ParseFloat(lexer.token(), 64);
-                         if (err != nil && err.(*strconv.NumError).Err == strconv.ErrSyntax) {
-                           panic("Invalid float")
-                         }
-                         lval.num = clientmodel.SampleValue(num)
-                         return NUMBER
-
-{D}+{U}                  lval.str = lexer.token(); return DURATION
-{L}({L}|{D})*            lval.str = lexer.token(); return IDENTIFIER
-{M}({M}|{D})*            lval.str = lexer.token(); return METRICNAME
-
-{STR}                    lval.str = lexer.token()[1:len(lexer.token()) - 1]; return STRING
-
-[{}\[\]()=,]             return int(lexer.buf[0])
-[\t\n\r ]                /* gobble up any whitespace */
-%%
-
-  lexer.empty = true
-  return int(c)
-}

rules/load.go

@@ -1,164 +0,0 @@
-// Copyright 2013 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package rules
-
-import (
-	"bufio"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"strings"
-
-	"github.com/golang/glog"
-
-	"github.com/prometheus/prometheus/rules/ast"
-)
-
-// RulesLexer is the lexer for rule expressions.
-type RulesLexer struct {
-	// Errors encountered during parsing.
-	errors []string
-	// Dummy token to simulate multiple start symbols (see below).
-	startToken int
-	// Parsed full rules.
-	parsedRules []Rule
-	// Parsed single expression.
-	parsedExpr ast.Node
-
-	// Current character.
-	current byte
-	// Current token buffer.
-	buf []byte
-	// Input text.
-	src *bufio.Reader
-	// Whether we have a current char.
-	empty bool
-
-	// Current input line.
-	line int
-	// Current character position within the current input line.
-	pos int
-}
-
-func (lexer *RulesLexer) Error(errorStr string) {
-	err := fmt.Sprintf("Error parsing rules at line %v, char %v: %v", lexer.line, lexer.pos, errorStr)
-	lexer.errors = append(lexer.errors, err)
-}
-
-func (lexer *RulesLexer) getChar() byte {
-	if lexer.current != 0 {
-		lexer.buf = append(lexer.buf, lexer.current)
-	}
-	lexer.current = 0
-	if b, err := lexer.src.ReadByte(); err == nil {
-		if b == '\n' {
-			lexer.line++
-			lexer.pos = 0
-		} else {
-			lexer.pos++
-		}
-		lexer.current = b
-	} else if err != io.EOF {
-		glog.Fatal(err)
-	}
-	return lexer.current
-}
-
-func (lexer *RulesLexer) token() string {
-	return string(lexer.buf)
-}
-
-func newRulesLexer(src io.Reader, singleExpr bool) *RulesLexer {
-	lexer := &RulesLexer{
-		startToken: START_RULES,
-		src:        bufio.NewReader(src),
-		pos:        1,
-		line:       1,
-	}
-
-	if singleExpr {
-		lexer.startToken = START_EXPRESSION
-	}
-	lexer.getChar()
-	return lexer
-}
-
-func lexAndParse(rulesReader io.Reader, singleExpr bool) (*RulesLexer, error) {
-	lexer := newRulesLexer(rulesReader, singleExpr)
-	ret := yyParse(lexer)
-	if ret != 0 && len(lexer.errors) == 0 {
-		lexer.Error("unknown parser error")
-	}
-
-	if len(lexer.errors) > 0 {
-		err := errors.New(strings.Join(lexer.errors, "\n"))
-		return nil, err
-	}
-	return lexer, nil
-}
-
-// LoadRulesFromReader parses rules from the provided reader and returns them.
-func LoadRulesFromReader(rulesReader io.Reader) ([]Rule, error) {
-	lexer, err := lexAndParse(rulesReader, false)
-	if err != nil {
-		return nil, err
-	}
-	return lexer.parsedRules, err
-}
-
-// LoadRulesFromString parses rules from the provided string returns them.
-func LoadRulesFromString(rulesString string) ([]Rule, error) {
-	rulesReader := strings.NewReader(rulesString)
-	return LoadRulesFromReader(rulesReader)
-}
-
-// LoadRulesFromFile parses rules from the file of the provided name and returns
-// them.
-func LoadRulesFromFile(fileName string) ([]Rule, error) {
-	rulesReader, err := os.Open(fileName)
-	if err != nil {
-		return []Rule{}, err
-	}
-	defer rulesReader.Close()
-	return LoadRulesFromReader(rulesReader)
-}
-
-// LoadExprFromReader parses a single expression from the provided reader and
-// returns it as an AST node.
-func LoadExprFromReader(exprReader io.Reader) (ast.Node, error) {
-	lexer, err := lexAndParse(exprReader, true)
-	if err != nil {
-		return nil, err
-	}
-	return lexer.parsedExpr, err
-}
-
-// LoadExprFromString parses a single expression from the provided string and
-// returns it as an AST node.
-func LoadExprFromString(exprString string) (ast.Node, error) {
-	exprReader := strings.NewReader(exprString)
-	return LoadExprFromReader(exprReader)
-}
-
-// LoadExprFromFile parses a single expression from the file of the provided
-// name and returns it as an AST node.
-func LoadExprFromFile(fileName string) (ast.Node, error) {
-	exprReader, err := os.Open(fileName)
-	if err != nil {
-		return nil, err
-	}
-	defer exprReader.Close()
-	return LoadExprFromReader(exprReader)
-}

rules/manager.go

@@ -11,10 +11,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package manager
+package rules
 
 import (
 	"fmt"
+	"io/ioutil"
 	"sync"
 	"time"
 
@@ -23,12 +24,11 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
-	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/notification"
-	"github.com/prometheus/prometheus/rules"
+	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/storage"
-	"github.com/prometheus/prometheus/storage/local"
 	"github.com/prometheus/prometheus/templates"
+	"github.com/prometheus/prometheus/utility"
 )
 
 // Constants for instrumentation.
@@ -70,30 +70,16 @@ func init() {
 	prometheus.MustRegister(evalDuration)
 }
 
-// A RuleManager manages recording and alerting rules. Create instances with
-// NewRuleManager.
-type RuleManager interface {
-	// Load and add rules from rule files specified in the configuration.
-	AddRulesFromConfig(config config.Config) error
-	// Start the rule manager's periodic rule evaluation.
-	Run()
-	// Stop the rule manager's rule evaluation cycles.
-	Stop()
-	// Return all rules.
-	Rules() []rules.Rule
-	// Return all alerting rules.
-	AlertingRules() []*rules.AlertingRule
-}
-
-type ruleManager struct {
+// The Manager manages recording and alerting rules.
+type Manager struct {
 	// Protects the rules list.
 	sync.Mutex
-	rules []rules.Rule
+	rules []Rule
 
 	done chan bool
 
-	interval time.Duration
-	storage  local.Storage
+	interval    time.Duration
+	queryEngine *promql.Engine
 
 	sampleAppender      storage.SampleAppender
 	notificationHandler *notification.NotificationHandler
@@ -102,10 +88,10 @@ type ruleManager struct {
 	pathPrefix    string
 }
 
-// RuleManagerOptions bundles options for the RuleManager.
-type RuleManagerOptions struct {
+// ManagerOptions bundles options for the Manager.
+type ManagerOptions struct {
 	EvaluationInterval time.Duration
-	Storage            local.Storage
+	QueryEngine        *promql.Engine
 
 	NotificationHandler *notification.NotificationHandler
 	SampleAppender      storage.SampleAppender
@@ -114,23 +100,24 @@ type RuleManagerOptions struct {
 	PathPrefix    string
 }
 
-// NewRuleManager returns an implementation of RuleManager, ready to be started
+// NewManager returns an implementation of Manager, ready to be started
 // by calling the Run method.
-func NewRuleManager(o *RuleManagerOptions) RuleManager {
-	manager := &ruleManager{
-		rules: []rules.Rule{},
+func NewManager(o *ManagerOptions) *Manager {
+	manager := &Manager{
+		rules: []Rule{},
 		done:  make(chan bool),
 
 		interval:            o.EvaluationInterval,
-		storage:             o.Storage,
 		sampleAppender:      o.SampleAppender,
+		queryEngine:         o.QueryEngine,
 		notificationHandler: o.NotificationHandler,
 		prometheusURL:       o.PrometheusURL,
 	}
 	return manager
 }
 
-func (m *ruleManager) Run() {
+// Run the rule manager's periodic rule evaluation.
+func (m *Manager) Run() {
 	defer glog.Info("Rule manager stopped.")
 
 	ticker := time.NewTicker(m.interval)
@@ -157,12 +144,13 @@ func (m *ruleManager) Run() {
 	}
 }
 
-func (m *ruleManager) Stop() {
+// Stop the rule manager's rule evaluation cycles.
+func (m *Manager) Stop() {
 	glog.Info("Stopping rule manager...")
 	m.done <- true
 }
 
-func (m *ruleManager) queueAlertNotifications(rule *rules.AlertingRule, timestamp clientmodel.Timestamp) {
+func (m *Manager) queueAlertNotifications(rule *AlertingRule, timestamp clientmodel.Timestamp) {
 	activeAlerts := rule.ActiveAlerts()
 	if len(activeAlerts) == 0 {
 		return
@@ -170,7 +158,7 @@ func (m *ruleManager) queueAlertNotifications(rule *rules.AlertingRule, timestam
 
 	notifications := make(notification.NotificationReqs, 0, len(activeAlerts))
 	for _, aa := range activeAlerts {
-		if aa.State != rules.Firing {
+		if aa.State != Firing {
 			// BUG: In the future, make AlertManager support pending alerts?
 			continue
 		}
@@ -192,7 +180,7 @@ func (m *ruleManager) queueAlertNotifications(rule *rules.AlertingRule, timestam
 		defs := "{{$labels := .Labels}}{{$value := .Value}}"
 
 		expand := func(text string) string {
-			template := templates.NewTemplateExpander(defs+text, "__alert_"+rule.Name(), tmplData, timestamp, m.storage, m.pathPrefix)
+			template := templates.NewTemplateExpander(defs+text, "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.pathPrefix)
 			result, err := template.Expand()
 			if err != nil {
 				result = err.Error()
@@ -205,34 +193,34 @@ func (m *ruleManager) queueAlertNotifications(rule *rules.AlertingRule, timestam
 			Summary:     expand(rule.Summary),
 			Description: expand(rule.Description),
 			Labels: aa.Labels.Merge(clientmodel.LabelSet{
-				rules.AlertNameLabel: clientmodel.LabelValue(rule.Name()),
+				AlertNameLabel: clientmodel.LabelValue(rule.Name()),
 			}),
 			Value:        aa.Value,
 			ActiveSince:  aa.ActiveSince.Time(),
 			RuleString:   rule.String(),
-			GeneratorURL: m.prometheusURL + rules.GraphLinkForExpression(rule.Vector.String()),
+			GeneratorURL: m.prometheusURL + utility.GraphLinkForExpression(rule.Vector.String()),
 		})
 	}
 	m.notificationHandler.SubmitReqs(notifications)
 }
 
-func (m *ruleManager) runIteration() {
+func (m *Manager) runIteration() {
 	now := clientmodel.Now()
 	wg := sync.WaitGroup{}
 
 	m.Lock()
-	rulesSnapshot := make([]rules.Rule, len(m.rules))
+	rulesSnapshot := make([]Rule, len(m.rules))
 	copy(rulesSnapshot, m.rules)
 	m.Unlock()
 
 	for _, rule := range rulesSnapshot {
 		wg.Add(1)
 		// BUG(julius): Look at fixing thundering herd.
-		go func(rule rules.Rule) {
+		go func(rule Rule) {
 			defer wg.Done()
 
 			start := time.Now()
-			vector, err := rule.Eval(now, m.storage)
+			vector, err := rule.Eval(now, m.queryEngine)
 			duration := time.Since(start)
 
 			if err != nil {
@@ -242,17 +230,17 @@ func (m *ruleManager) runIteration() {
 			}
 
 			switch r := rule.(type) {
-			case *rules.AlertingRule:
+			case *AlertingRule:
 				m.queueAlertNotifications(r, now)
 				evalDuration.WithLabelValues(alertingRuleType).Observe(
 					float64(duration / time.Millisecond),
 				)
-			case *rules.RecordingRule:
+			case *RecordingRule:
 				evalDuration.WithLabelValues(recordingRuleType).Observe(
 					float64(duration / time.Millisecond),
 				)
 			default:
-				panic(fmt.Sprintf("Unknown rule type: %T", rule))
+				panic(fmt.Errorf("Unknown rule type: %T", rule))
 			}
 
 			for _, s := range vector {
@@ -267,35 +255,54 @@ func (m *ruleManager) runIteration() {
 	wg.Wait()
 }
 
-func (m *ruleManager) AddRulesFromConfig(config config.Config) error {
-	for _, ruleFile := range config.Global.RuleFile {
-		newRules, err := rules.LoadRulesFromFile(ruleFile)
+// LoadRuleFiles loads alerting and recording rules from the given files.
+func (m *Manager) LoadRuleFiles(filenames ...string) error {
+	m.Lock()
+	defer m.Unlock()
+
+	for _, fn := range filenames {
+		content, err := ioutil.ReadFile(fn)
 		if err != nil {
-			return fmt.Errorf("%s: %s", ruleFile, err)
+			return err
+		}
+		stmts, err := promql.ParseStmts(string(content))
+		if err != nil {
+			return fmt.Errorf("error parsing %s: %s", fn, err)
+		}
+		for _, stmt := range stmts {
+			switch r := stmt.(type) {
+			case *promql.AlertStmt:
+				rule := NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Summary, r.Description)
+				m.rules = append(m.rules, rule)
+			case *promql.RecordStmt:
+				rule := &RecordingRule{r.Name, r.Expr, r.Labels}
+				m.rules = append(m.rules, rule)
+			default:
+				panic("retrieval.Manager.LoadRuleFiles: unknown statement type")
+			}
 		}
-		m.Lock()
-		m.rules = append(m.rules, newRules...)
-		m.Unlock()
 	}
 	return nil
 }
 
-func (m *ruleManager) Rules() []rules.Rule {
+// Rules returns the list of the manager's rules.
+func (m *Manager) Rules() []Rule {
 	m.Lock()
 	defer m.Unlock()
 
-	rules := make([]rules.Rule, len(m.rules))
+	rules := make([]Rule, len(m.rules))
 	copy(rules, m.rules)
 	return rules
 }
 
-func (m *ruleManager) AlertingRules() []*rules.AlertingRule {
+// AlertingRules returns the list of the manager's alerting rules.
+func (m *Manager) AlertingRules() []*AlertingRule {
 	m.Lock()
 	defer m.Unlock()
 
-	alerts := []*rules.AlertingRule{}
+	alerts := []*AlertingRule{}
 	for _, rule := range m.rules {
-		if alertingRule, ok := rule.(*rules.AlertingRule); ok {
+		if alertingRule, ok := rule.(*AlertingRule); ok {
 			alerts = append(alerts, alertingRule)
 		}
 	}

rules/parser.y

@@ -1,281 +0,0 @@
-// Copyright 2013 The Prometheus Authors
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-%{
-        package rules
-
-        import (
-          clientmodel "github.com/prometheus/client_golang/model"
-
-          "github.com/prometheus/prometheus/rules/ast"
-          "github.com/prometheus/prometheus/storage/metric"
-        )
-%}
-
-%union {
-        num clientmodel.SampleValue
-        str string
-        ruleNode ast.Node
-        ruleNodeSlice []ast.Node
-        boolean bool
-        labelNameSlice clientmodel.LabelNames
-        labelSet clientmodel.LabelSet
-        labelMatcher *metric.LabelMatcher
-        labelMatchers metric.LabelMatchers
-        vectorMatching *vectorMatching
-}
-
-/* We simulate multiple start symbols for closely-related grammars via dummy tokens. See
-   http://www.gnu.org/software/bison/manual/html_node/Multiple-start_002dsymbols.html
-   Reason: we want to be able to parse lists of named rules as well as single expressions.
-   */
-%token START_RULES START_EXPRESSION
-
-%token <str> IDENTIFIER STRING DURATION METRICNAME
-%token <num> NUMBER
-%token PERMANENT GROUP_OP KEEPING_EXTRA OFFSET MATCH_OP
-%token <str> AGGR_OP CMP_OP ADDITIVE_OP MULT_OP MATCH_MOD
-%token ALERT IF FOR WITH SUMMARY DESCRIPTION
-
-%type <ruleNodeSlice> func_arg_list
-%type <labelNameSlice> label_list grouping_opts
-%type <labelSet> label_assign label_assign_list rule_labels
-%type <labelMatcher> label_match
-%type <labelMatchers> label_match_list label_matches
-%type <vectorMatching> vector_matching
-%type <ruleNode> rule_expr func_arg
-%type <boolean> qualifier extra_labels_opts
-%type <str> for_duration metric_name label_match_type offset_opts
-
-%right '='
-%left CMP_OP
-%left ADDITIVE_OP
-%left MULT_OP
-%start start
-
-%%
-start              : START_RULES rules_stat_list
-                   | START_EXPRESSION saved_rule_expr
-                   ;
-
-rules_stat_list    : /* empty */
-                   | rules_stat_list rules_stat
-                   ;
-
-saved_rule_expr    : rule_expr
-                     { yylex.(*RulesLexer).parsedExpr = $1 }
-                   ;
-
-
-rules_stat         : qualifier metric_name rule_labels '=' rule_expr
-                     {
-                       rule, err := CreateRecordingRule($2, $3, $5, $1)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                       yylex.(*RulesLexer).parsedRules = append(yylex.(*RulesLexer).parsedRules, rule)
-                     }
-                   | ALERT IDENTIFIER IF rule_expr for_duration WITH rule_labels SUMMARY STRING DESCRIPTION STRING
-                     {
-                       rule, err := CreateAlertingRule($2, $4, $5, $7, $9, $11)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                       yylex.(*RulesLexer).parsedRules = append(yylex.(*RulesLexer).parsedRules, rule)
-                     }
-                   ;
-
-for_duration       : /* empty */
-                     { $$ = "0s" }
-                   | FOR DURATION
-                     { $$ = $2 }
-                   ;
-
-qualifier          : /* empty */
-                     { $$ = false }
-                   | PERMANENT
-                     { $$ = true }
-                   ;
-
-metric_name        : METRICNAME
-                     { $$ = $1 }
-                   | IDENTIFIER
-                     { $$ = $1 }
-                   ;
-
-rule_labels        : /* empty */
-                     { $$ = clientmodel.LabelSet{} }
-                   | '{' label_assign_list '}'
-                     { $$ = $2  }
-                   | '{' '}'
-                     { $$ = clientmodel.LabelSet{} }
-
-label_assign_list  : label_assign
-                     { $$ = $1 }
-                   | label_assign_list ',' label_assign
-                     { for k, v := range $3 { $$[k] = v } }
-                   ;
-
-label_assign       : IDENTIFIER '=' STRING
-                     { $$ = clientmodel.LabelSet{ clientmodel.LabelName($1): clientmodel.LabelValue($3) } }
-                   ;
-
-label_matches      : /* empty */
-                     { $$ = metric.LabelMatchers{} }
-                   | '{' '}'
-                     { $$ = metric.LabelMatchers{} }
-                   | '{' label_match_list '}'
-                     { $$ = $2 }
-                   ;
-
-label_match_list   : label_match
-                     { $$ = metric.LabelMatchers{$1} }
-                   | label_match_list ',' label_match
-                     { $$ = append($$, $3) }
-                   ;
-
-label_match        : IDENTIFIER label_match_type STRING
-                     {
-                       var err error
-                       $$, err = newLabelMatcher($2, clientmodel.LabelName($1), clientmodel.LabelValue($3))
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   ;
-
-label_match_type   : '='
-                     { $$ = "=" }
-                   | CMP_OP
-                     { $$ = $1 }
-                   ;
-
-offset_opts        : /* empty */
-                     { $$ = "0s" }
-                   | OFFSET DURATION
-                     { $$ = $2 }
-                   ;
-
-rule_expr          : '(' rule_expr ')'
-                     { $$ = $2 }
-                   | '{' label_match_list '}' offset_opts
-                     {
-                       var err error
-                       $$, err = NewVectorSelector($2, $4)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | metric_name label_matches offset_opts
-                     {
-                       var err error
-                       m, err := metric.NewLabelMatcher(metric.Equal, clientmodel.MetricNameLabel, clientmodel.LabelValue($1))
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                       $2 = append($2, m)
-                       $$, err = NewVectorSelector($2, $3)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | IDENTIFIER '(' func_arg_list ')'
-                     {
-                       var err error
-                       $$, err = NewFunctionCall($1, $3)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | IDENTIFIER '(' ')'
-                     {
-                       var err error
-                       $$, err = NewFunctionCall($1, []ast.Node{})
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | rule_expr '[' DURATION ']' offset_opts
-                     {
-                       var err error
-                       $$, err = NewMatrixSelector($1, $3, $5)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | AGGR_OP '(' rule_expr ')' grouping_opts extra_labels_opts
-                     {
-                       var err error
-                       $$, err = NewVectorAggregation($1, $3, $5, $6)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | AGGR_OP grouping_opts extra_labels_opts '(' rule_expr ')'
-                     {
-                       var err error
-                       $$, err = NewVectorAggregation($1, $5, $2, $3)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   /* Yacc can only attach associativity to terminals, so we
-                    * have to list all operators here. */
-                   | rule_expr ADDITIVE_OP vector_matching rule_expr
-                     {
-                       var err error
-                       $$, err = NewArithExpr($2, $1, $4, $3)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | rule_expr MULT_OP vector_matching rule_expr
-                     {
-                       var err error
-                       $$, err = NewArithExpr($2, $1, $4, $3)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | rule_expr CMP_OP vector_matching rule_expr
-                     {
-                       var err error
-                       $$, err = NewArithExpr($2, $1, $4, $3)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | NUMBER
-                     { $$ = NewScalarLiteral($1, "+")}
-                   | ADDITIVE_OP NUMBER
-                     { $$ = NewScalarLiteral($2, $1)}
-                   ;
-
-extra_labels_opts  : /* empty */
-                     { $$ = false }
-                   | KEEPING_EXTRA
-                     { $$ = true }
-                   ;
-
-vector_matching    : /* empty */
-                     { $$ = nil }
-                   | MATCH_OP '(' label_list ')'
-                     {
-                       var err error
-                       $$, err = newVectorMatching("", $3, nil)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   | MATCH_OP '(' label_list ')' MATCH_MOD '(' label_list ')'
-                     {
-                       var err error
-                       $$, err = newVectorMatching($5, $3, $7)
-                       if err != nil { yylex.Error(err.Error()); return 1 }
-                     }
-                   ;
-
-grouping_opts      :
-                     { $$ = clientmodel.LabelNames{} }
-                   | GROUP_OP '(' label_list ')'
-                     { $$ = $3 }
-                   ;
-
-label_list         : IDENTIFIER
-                     { $$ = clientmodel.LabelNames{clientmodel.LabelName($1)} }
-                   | label_list ',' IDENTIFIER
-                     { $$ = append($$, clientmodel.LabelName($3)) }
-                   ;
-
-func_arg_list      : func_arg
-                     { $$ = []ast.Node{$1} }
-                   | func_arg_list ',' func_arg
-                     { $$ = append($$, $3) }
-                   ;
-
-func_arg           : rule_expr
-                     { $$ = $1 }
-                   | STRING
-                     { $$ = ast.NewStringLiteral($1) }
-                   ;
-%%

rules/parser.y.go

@@ -1,784 +0,0 @@
-//line parser.y:15
-package rules
-
-import __yyfmt__ "fmt"
-
-//line parser.y:15
-import (
-	clientmodel "github.com/prometheus/client_golang/model"
-
-	"github.com/prometheus/prometheus/rules/ast"
-	"github.com/prometheus/prometheus/storage/metric"
-)
-
-//line parser.y:25
-type yySymType struct {
-	yys            int
-	num            clientmodel.SampleValue
-	str            string
-	ruleNode       ast.Node
-	ruleNodeSlice  []ast.Node
-	boolean        bool
-	labelNameSlice clientmodel.LabelNames
-	labelSet       clientmodel.LabelSet
-	labelMatcher   *metric.LabelMatcher
-	labelMatchers  metric.LabelMatchers
-	vectorMatching *vectorMatching
-}
-
-const START_RULES = 57346
-const START_EXPRESSION = 57347
-const IDENTIFIER = 57348
-const STRING = 57349
-const DURATION = 57350
-const METRICNAME = 57351
-const NUMBER = 57352
-const PERMANENT = 57353
-const GROUP_OP = 57354
-const KEEPING_EXTRA = 57355
-const OFFSET = 57356
-const MATCH_OP = 57357
-const AGGR_OP = 57358
-const CMP_OP = 57359
-const ADDITIVE_OP = 57360
-const MULT_OP = 57361
-const MATCH_MOD = 57362
-const ALERT = 57363
-const IF = 57364
-const FOR = 57365
-const WITH = 57366
-const SUMMARY = 57367
-const DESCRIPTION = 57368
-
-var yyToknames = []string{
-	"START_RULES",
-	"START_EXPRESSION",
-	"IDENTIFIER",
-	"STRING",
-	"DURATION",
-	"METRICNAME",
-	"NUMBER",
-	"PERMANENT",
-	"GROUP_OP",
-	"KEEPING_EXTRA",
-	"OFFSET",
-	"MATCH_OP",
-	"AGGR_OP",
-	"CMP_OP",
-	"ADDITIVE_OP",
-	"MULT_OP",
-	"MATCH_MOD",
-	"ALERT",
-	"IF",
-	"FOR",
-	"WITH",
-	"SUMMARY",
-	"DESCRIPTION",
-	"'='",
-}
-var yyStatenames = []string{}
-
-const yyEofCode = 1
-const yyErrCode = 2
-const yyMaxDepth = 200
-
-//line parser.y:281
-
-//line yacctab:1
-var yyExca = []int{
-	-1, 1,
-	1, -1,
-	-2, 0,
-	-1, 4,
-	1, 1,
-	-2, 10,
-}
-
-const yyNprod = 56
-const yyPrivate = 57344
-
-var yyTokenNames []string
-var yyStates []string
-
-const yyLast = 159
-
-var yyAct = []int{
-
-	78, 61, 83, 58, 55, 54, 31, 48, 6, 25,
-	20, 21, 23, 21, 10, 56, 64, 14, 12, 10,
-	56, 19, 14, 12, 11, 19, 13, 19, 92, 11,
-	113, 13, 22, 20, 21, 57, 8, 32, 109, 7,
-	53, 8, 77, 65, 7, 67, 68, 101, 19, 22,
-	20, 21, 70, 69, 10, 98, 30, 14, 12, 22,
-	20, 21, 94, 95, 11, 19, 13, 87, 85, 92,
-	96, 99, 86, 84, 76, 19, 8, 66, 60, 7,
-	29, 88, 90, 89, 24, 93, 22, 20, 21, 22,
-	20, 21, 92, 100, 91, 75, 82, 74, 103, 73,
-	43, 42, 19, 44, 43, 19, 26, 108, 62, 47,
-	111, 28, 80, 51, 114, 110, 38, 105, 63, 46,
-	18, 107, 39, 9, 49, 59, 32, 33, 35, 50,
-	17, 14, 106, 72, 37, 115, 112, 104, 40, 41,
-	34, 71, 79, 84, 102, 26, 36, 2, 3, 15,
-	5, 4, 1, 45, 97, 16, 27, 81, 52,
-}
-var yyPact = []int{
-
-	143, -1000, -1000, 48, 109, -1000, 72, 48, 139, 83,
-	49, 25, -1000, 117, -1000, -1000, 122, 140, -1000, 126,
-	107, 107, 107, 69, 74, -1000, 92, 110, 100, 8,
-	48, 112, 47, -1000, 80, -1000, 96, -18, 48, 46,
-	48, 48, -1000, 139, 110, 134, -1000, -1000, -1000, 125,
-	-1000, 70, 65, -1000, -1000, 72, -1000, 42, 11, -1000,
-	136, 85, 67, 48, 110, -6, 136, -12, -8, -1000,
-	-1000, -1000, -1000, -1000, -1000, 13, 114, 48, 62, -1000,
-	48, 33, -1000, -1000, 43, 32, -1000, 39, -1000, 112,
-	15, -1000, 138, 72, -1000, 137, 130, 93, 124, 101,
-	-1000, -1000, -1000, -1000, -1000, 80, -1000, 7, 90, 136,
-	129, -2, 88, -1000, 128, -1000,
-}
-var yyPgo = []int{
-
-	0, 158, 0, 6, 2, 157, 1, 9, 84, 156,
-	116, 4, 5, 155, 3, 154, 123, 153, 7, 152,
-	151, 150, 149,
-}
-var yyR1 = []int{
-
-	0, 19, 19, 20, 20, 21, 22, 22, 15, 15,
-	13, 13, 16, 16, 6, 6, 6, 5, 5, 4,
-	9, 9, 9, 8, 8, 7, 17, 17, 18, 18,
-	11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
-	11, 11, 11, 14, 14, 10, 10, 10, 3, 3,
-	2, 2, 1, 1, 12, 12,
-}
-var yyR2 = []int{
-
-	0, 2, 2, 0, 2, 1, 5, 11, 0, 2,
-	0, 1, 1, 1, 0, 3, 2, 1, 3, 3,
-	0, 2, 3, 1, 3, 3, 1, 1, 0, 2,
-	3, 4, 3, 4, 3, 5, 6, 6, 4, 4,
-	4, 1, 2, 0, 1, 0, 4, 8, 0, 4,
-	1, 3, 1, 3, 1, 1,
-}
-var yyChk = []int{
-
-	-1000, -19, 4, 5, -20, -21, -11, 31, 28, -16,
-	6, 16, 10, 18, 9, -22, -13, 21, 11, 33,
-	18, 19, 17, -11, -8, -7, 6, -9, 28, 31,
-	31, -3, 12, 10, -16, 6, 6, 8, -10, 15,
-	-10, -10, 32, 30, 29, -17, 27, 17, -18, 14,
-	29, -8, -1, 32, -12, -11, 7, -11, -14, 13,
-	31, -6, 28, 22, 34, -11, 31, -11, -11, -7,
-	-18, 7, 8, 29, 32, 30, 32, 31, -2, 6,
-	27, -5, 29, -4, 6, -11, -18, -2, -12, -3,
-	-11, 32, 30, -11, 29, 30, 27, -15, 23, 32,
-	-14, 32, 6, -4, 7, 24, 8, 20, -6, 31,
-	25, -2, 7, 32, 26, 7,
-}
-var yyDef = []int{
-
-	0, -2, 3, 0, -2, 2, 5, 0, 0, 20,
-	13, 48, 41, 0, 12, 4, 0, 0, 11, 0,
-	45, 45, 45, 0, 0, 23, 0, 28, 0, 0,
-	0, 43, 0, 42, 14, 13, 0, 0, 0, 0,
-	0, 0, 30, 0, 28, 0, 26, 27, 32, 0,
-	21, 0, 0, 34, 52, 54, 55, 0, 0, 44,
-	0, 0, 0, 0, 28, 38, 0, 39, 40, 24,
-	31, 25, 29, 22, 33, 0, 48, 0, 0, 50,
-	0, 0, 16, 17, 0, 8, 35, 0, 53, 43,
-	0, 49, 0, 6, 15, 0, 0, 0, 0, 46,
-	36, 37, 51, 18, 19, 14, 9, 0, 0, 0,
-	0, 0, 0, 47, 0, 7,
-}
-var yyTok1 = []int{
-
-	1, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	31, 32, 3, 3, 30, 3, 3, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 27, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 33, 3, 34, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
-	3, 3, 3, 28, 3, 29,
-}
-var yyTok2 = []int{
-
-	2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
-	12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
-	22, 23, 24, 25, 26,
-}
-var yyTok3 = []int{
-	0,
-}
-
-//line yaccpar:1
-
-/*	parser for yacc output	*/
-
-var yyDebug = 0
-
-type yyLexer interface {
-	Lex(lval *yySymType) int
-	Error(s string)
-}
-
-const yyFlag = -1000
-
-func yyTokname(c int) string {
-	// 4 is TOKSTART above
-	if c >= 4 && c-4 < len(yyToknames) {
-		if yyToknames[c-4] != "" {
-			return yyToknames[c-4]
-		}
-	}
-	return __yyfmt__.Sprintf("tok-%v", c)
-}
-
-func yyStatname(s int) string {
-	if s >= 0 && s < len(yyStatenames) {
-		if yyStatenames[s] != "" {
-			return yyStatenames[s]
-		}
-	}
-	return __yyfmt__.Sprintf("state-%v", s)
-}
-
-func yylex1(lex yyLexer, lval *yySymType) int {
-	c := 0
-	char := lex.Lex(lval)
-	if char <= 0 {
-		c = yyTok1[0]
-		goto out
-	}
-	if char < len(yyTok1) {
-		c = yyTok1[char]
-		goto out
-	}
-	if char >= yyPrivate {
-		if char < yyPrivate+len(yyTok2) {
-			c = yyTok2[char-yyPrivate]
-			goto out
-		}
-	}
-	for i := 0; i < len(yyTok3); i += 2 {
-		c = yyTok3[i+0]
-		if c == char {
-			c = yyTok3[i+1]
-			goto out
-		}
-	}
-
-out:
-	if c == 0 {
-		c = yyTok2[1] /* unknown char */
-	}
-	if yyDebug >= 3 {
-		__yyfmt__.Printf("lex %s(%d)\n", yyTokname(c), uint(char))
-	}
-	return c
-}
-
-func yyParse(yylex yyLexer) int {
-	var yyn int
-	var yylval yySymType
-	var yyVAL yySymType
-	yyS := make([]yySymType, yyMaxDepth)
-
-	Nerrs := 0   /* number of errors */
-	Errflag := 0 /* error recovery flag */
-	yystate := 0
-	yychar := -1
-	yyp := -1
-	goto yystack
-
-ret0:
-	return 0
-
-ret1:
-	return 1
-
-yystack:
-	/* put a state and value onto the stack */
-	if yyDebug >= 4 {
-		__yyfmt__.Printf("char %v in %v\n", yyTokname(yychar), yyStatname(yystate))
-	}
-
-	yyp++
-	if yyp >= len(yyS) {
-		nyys := make([]yySymType, len(yyS)*2)
-		copy(nyys, yyS)
-		yyS = nyys
-	}
-	yyS[yyp] = yyVAL
-	yyS[yyp].yys = yystate
-
-yynewstate:
-	yyn = yyPact[yystate]
-	if yyn <= yyFlag {
-		goto yydefault /* simple state */
-	}
-	if yychar < 0 {
-		yychar = yylex1(yylex, &yylval)
-	}
-	yyn += yychar
-	if yyn < 0 || yyn >= yyLast {
-		goto yydefault
-	}
-	yyn = yyAct[yyn]
-	if yyChk[yyn] == yychar { /* valid shift */
-		yychar = -1
-		yyVAL = yylval
-		yystate = yyn
-		if Errflag > 0 {
-			Errflag--
-		}
-		goto yystack
-	}
-
-yydefault:
-	/* default state action */
-	yyn = yyDef[yystate]
-	if yyn == -2 {
-		if yychar < 0 {
-			yychar = yylex1(yylex, &yylval)
-		}
-
-		/* look through exception table */
-		xi := 0
-		for {
-			if yyExca[xi+0] == -1 && yyExca[xi+1] == yystate {
-				break
-			}
-			xi += 2
-		}
-		for xi += 2; ; xi += 2 {
-			yyn = yyExca[xi+0]
-			if yyn < 0 || yyn == yychar {
-				break
-			}
-		}
-		yyn = yyExca[xi+1]
-		if yyn < 0 {
-			goto ret0
-		}
-	}
-	if yyn == 0 {
-		/* error ... attempt to resume parsing */
-		switch Errflag {
-		case 0: /* brand new error */
-			yylex.Error("syntax error")
-			Nerrs++
-			if yyDebug >= 1 {
-				__yyfmt__.Printf("%s", yyStatname(yystate))
-				__yyfmt__.Printf(" saw %s\n", yyTokname(yychar))
-			}
-			fallthrough
-
-		case 1, 2: /* incompletely recovered error ... try again */
-			Errflag = 3
-
-			/* find a state where "error" is a legal shift action */
-			for yyp >= 0 {
-				yyn = yyPact[yyS[yyp].yys] + yyErrCode
-				if yyn >= 0 && yyn < yyLast {
-					yystate = yyAct[yyn] /* simulate a shift of "error" */
-					if yyChk[yystate] == yyErrCode {
-						goto yystack
-					}
-				}
-
-				/* the current p has no shift on "error", pop stack */
-				if yyDebug >= 2 {
-					__yyfmt__.Printf("error recovery pops state %d\n", yyS[yyp].yys)
-				}
-				yyp--
-			}
-			/* there is no state on the stack with an error shift ... abort */
-			goto ret1
-
-		case 3: /* no shift yet; clobber input char */
-			if yyDebug >= 2 {
-				__yyfmt__.Printf("error recovery discards %s\n", yyTokname(yychar))
-			}
-			if yychar == yyEofCode {
-				goto ret1
-			}
-			yychar = -1
-			goto yynewstate /* try again in the same state */
-		}
-	}
-
-	/* reduction by production yyn */
-	if yyDebug >= 2 {
-		__yyfmt__.Printf("reduce %v in:\n\t%v\n", yyn, yyStatname(yystate))
-	}
-
-	yynt := yyn
-	yypt := yyp
-	_ = yypt // guard against "declared and not used"
-
-	yyp -= yyR2[yyn]
-	yyVAL = yyS[yyp+1]
-
-	/* consult goto table to find next state */
-	yyn = yyR1[yyn]
-	yyg := yyPgo[yyn]
-	yyj := yyg + yyS[yyp].yys + 1
-
-	if yyj >= yyLast {
-		yystate = yyAct[yyg]
-	} else {
-		yystate = yyAct[yyj]
-		if yyChk[yystate] != -yyn {
-			yystate = yyAct[yyg]
-		}
-	}
-	// dummy call; replaced with literal code
-	switch yynt {
-
-	case 5:
-		//line parser.y:76
-		{
-			yylex.(*RulesLexer).parsedExpr = yyS[yypt-0].ruleNode
-		}
-	case 6:
-		//line parser.y:81
-		{
-			rule, err := CreateRecordingRule(yyS[yypt-3].str, yyS[yypt-2].labelSet, yyS[yypt-0].ruleNode, yyS[yypt-4].boolean)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-			yylex.(*RulesLexer).parsedRules = append(yylex.(*RulesLexer).parsedRules, rule)
-		}
-	case 7:
-		//line parser.y:87
-		{
-			rule, err := CreateAlertingRule(yyS[yypt-9].str, yyS[yypt-7].ruleNode, yyS[yypt-6].str, yyS[yypt-4].labelSet, yyS[yypt-2].str, yyS[yypt-0].str)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-			yylex.(*RulesLexer).parsedRules = append(yylex.(*RulesLexer).parsedRules, rule)
-		}
-	case 8:
-		//line parser.y:95
-		{
-			yyVAL.str = "0s"
-		}
-	case 9:
-		//line parser.y:97
-		{
-			yyVAL.str = yyS[yypt-0].str
-		}
-	case 10:
-		//line parser.y:101
-		{
-			yyVAL.boolean = false
-		}
-	case 11:
-		//line parser.y:103
-		{
-			yyVAL.boolean = true
-		}
-	case 12:
-		//line parser.y:107
-		{
-			yyVAL.str = yyS[yypt-0].str
-		}
-	case 13:
-		//line parser.y:109
-		{
-			yyVAL.str = yyS[yypt-0].str
-		}
-	case 14:
-		//line parser.y:113
-		{
-			yyVAL.labelSet = clientmodel.LabelSet{}
-		}
-	case 15:
-		//line parser.y:115
-		{
-			yyVAL.labelSet = yyS[yypt-1].labelSet
-		}
-	case 16:
-		//line parser.y:117
-		{
-			yyVAL.labelSet = clientmodel.LabelSet{}
-		}
-	case 17:
-		//line parser.y:120
-		{
-			yyVAL.labelSet = yyS[yypt-0].labelSet
-		}
-	case 18:
-		//line parser.y:122
-		{
-			for k, v := range yyS[yypt-0].labelSet {
-				yyVAL.labelSet[k] = v
-			}
-		}
-	case 19:
-		//line parser.y:126
-		{
-			yyVAL.labelSet = clientmodel.LabelSet{clientmodel.LabelName(yyS[yypt-2].str): clientmodel.LabelValue(yyS[yypt-0].str)}
-		}
-	case 20:
-		//line parser.y:130
-		{
-			yyVAL.labelMatchers = metric.LabelMatchers{}
-		}
-	case 21:
-		//line parser.y:132
-		{
-			yyVAL.labelMatchers = metric.LabelMatchers{}
-		}
-	case 22:
-		//line parser.y:134
-		{
-			yyVAL.labelMatchers = yyS[yypt-1].labelMatchers
-		}
-	case 23:
-		//line parser.y:138
-		{
-			yyVAL.labelMatchers = metric.LabelMatchers{yyS[yypt-0].labelMatcher}
-		}
-	case 24:
-		//line parser.y:140
-		{
-			yyVAL.labelMatchers = append(yyVAL.labelMatchers, yyS[yypt-0].labelMatcher)
-		}
-	case 25:
-		//line parser.y:144
-		{
-			var err error
-			yyVAL.labelMatcher, err = newLabelMatcher(yyS[yypt-1].str, clientmodel.LabelName(yyS[yypt-2].str), clientmodel.LabelValue(yyS[yypt-0].str))
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 26:
-		//line parser.y:152
-		{
-			yyVAL.str = "="
-		}
-	case 27:
-		//line parser.y:154
-		{
-			yyVAL.str = yyS[yypt-0].str
-		}
-	case 28:
-		//line parser.y:158
-		{
-			yyVAL.str = "0s"
-		}
-	case 29:
-		//line parser.y:160
-		{
-			yyVAL.str = yyS[yypt-0].str
-		}
-	case 30:
-		//line parser.y:164
-		{
-			yyVAL.ruleNode = yyS[yypt-1].ruleNode
-		}
-	case 31:
-		//line parser.y:166
-		{
-			var err error
-			yyVAL.ruleNode, err = NewVectorSelector(yyS[yypt-2].labelMatchers, yyS[yypt-0].str)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 32:
-		//line parser.y:172
-		{
-			var err error
-			m, err := metric.NewLabelMatcher(metric.Equal, clientmodel.MetricNameLabel, clientmodel.LabelValue(yyS[yypt-2].str))
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-			yyS[yypt-1].labelMatchers = append(yyS[yypt-1].labelMatchers, m)
-			yyVAL.ruleNode, err = NewVectorSelector(yyS[yypt-1].labelMatchers, yyS[yypt-0].str)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 33:
-		//line parser.y:181
-		{
-			var err error
-			yyVAL.ruleNode, err = NewFunctionCall(yyS[yypt-3].str, yyS[yypt-1].ruleNodeSlice)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 34:
-		//line parser.y:187
-		{
-			var err error
-			yyVAL.ruleNode, err = NewFunctionCall(yyS[yypt-2].str, []ast.Node{})
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 35:
-		//line parser.y:193
-		{
-			var err error
-			yyVAL.ruleNode, err = NewMatrixSelector(yyS[yypt-4].ruleNode, yyS[yypt-2].str, yyS[yypt-0].str)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 36:
-		//line parser.y:199
-		{
-			var err error
-			yyVAL.ruleNode, err = NewVectorAggregation(yyS[yypt-5].str, yyS[yypt-3].ruleNode, yyS[yypt-1].labelNameSlice, yyS[yypt-0].boolean)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 37:
-		//line parser.y:205
-		{
-			var err error
-			yyVAL.ruleNode, err = NewVectorAggregation(yyS[yypt-5].str, yyS[yypt-1].ruleNode, yyS[yypt-4].labelNameSlice, yyS[yypt-3].boolean)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 38:
-		//line parser.y:213
-		{
-			var err error
-			yyVAL.ruleNode, err = NewArithExpr(yyS[yypt-2].str, yyS[yypt-3].ruleNode, yyS[yypt-0].ruleNode, yyS[yypt-1].vectorMatching)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 39:
-		//line parser.y:219
-		{
-			var err error
-			yyVAL.ruleNode, err = NewArithExpr(yyS[yypt-2].str, yyS[yypt-3].ruleNode, yyS[yypt-0].ruleNode, yyS[yypt-1].vectorMatching)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 40:
-		//line parser.y:225
-		{
-			var err error
-			yyVAL.ruleNode, err = NewArithExpr(yyS[yypt-2].str, yyS[yypt-3].ruleNode, yyS[yypt-0].ruleNode, yyS[yypt-1].vectorMatching)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 41:
-		//line parser.y:231
-		{
-			yyVAL.ruleNode = NewScalarLiteral(yyS[yypt-0].num, "+")
-		}
-	case 42:
-		//line parser.y:233
-		{
-			yyVAL.ruleNode = NewScalarLiteral(yyS[yypt-0].num, yyS[yypt-1].str)
-		}
-	case 43:
-		//line parser.y:237
-		{
-			yyVAL.boolean = false
-		}
-	case 44:
-		//line parser.y:239
-		{
-			yyVAL.boolean = true
-		}
-	case 45:
-		//line parser.y:243
-		{
-			yyVAL.vectorMatching = nil
-		}
-	case 46:
-		//line parser.y:245
-		{
-			var err error
-			yyVAL.vectorMatching, err = newVectorMatching("", yyS[yypt-1].labelNameSlice, nil)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 47:
-		//line parser.y:251
-		{
-			var err error
-			yyVAL.vectorMatching, err = newVectorMatching(yyS[yypt-3].str, yyS[yypt-5].labelNameSlice, yyS[yypt-1].labelNameSlice)
-			if err != nil {
-				yylex.Error(err.Error())
-				return 1
-			}
-		}
-	case 48:
-		//line parser.y:259
-		{
-			yyVAL.labelNameSlice = clientmodel.LabelNames{}
-		}
-	case 49:
-		//line parser.y:261
-		{
-			yyVAL.labelNameSlice = yyS[yypt-1].labelNameSlice
-		}
-	case 50:
-		//line parser.y:265
-		{
-			yyVAL.labelNameSlice = clientmodel.LabelNames{clientmodel.LabelName(yyS[yypt-0].str)}
-		}
-	case 51:
-		//line parser.y:267
-		{
-			yyVAL.labelNameSlice = append(yyVAL.labelNameSlice, clientmodel.LabelName(yyS[yypt-0].str))
-		}
-	case 52:
-		//line parser.y:271
-		{
-			yyVAL.ruleNodeSlice = []ast.Node{yyS[yypt-0].ruleNode}
-		}
-	case 53:
-		//line parser.y:273
-		{
-			yyVAL.ruleNodeSlice = append(yyVAL.ruleNodeSlice, yyS[yypt-0].ruleNode)
-		}
-	case 54:
-		//line parser.y:277
-		{
-			yyVAL.ruleNode = yyS[yypt-0].ruleNode
-		}
-	case 55:
-		//line parser.y:279
-		{
-			yyVAL.ruleNode = ast.NewStringLiteral(yyS[yypt-0].str)
-		}
-	}
-	goto yystack /* stack new state and value */
-}

rules/recording.go

@@ -20,30 +20,32 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
-	"github.com/prometheus/prometheus/rules/ast"
-	"github.com/prometheus/prometheus/stats"
-	"github.com/prometheus/prometheus/storage/local"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/utility"
 )
 
 // A RecordingRule records its vector expression into new timeseries.
 type RecordingRule struct {
-	name      string
-	vector    ast.VectorNode
-	labels    clientmodel.LabelSet
-	permanent bool
+	name   string
+	vector promql.Expr
+	labels clientmodel.LabelSet
 }
 
 // Name returns the rule name.
 func (rule RecordingRule) Name() string { return rule.name }
 
 // EvalRaw returns the raw value of the rule expression.
-func (rule RecordingRule) EvalRaw(timestamp clientmodel.Timestamp, storage local.Storage) (ast.Vector, error) {
-	return ast.EvalVectorInstant(rule.vector, timestamp, storage, stats.NewTimerGroup())
+func (rule RecordingRule) EvalRaw(timestamp clientmodel.Timestamp, engine *promql.Engine) (promql.Vector, error) {
+	query, err := engine.NewInstantQuery(rule.vector.String(), timestamp)
+	if err != nil {
+		return nil, err
+	}
+	return query.Exec().Vector()
 }
 
 // Eval evaluates the rule and then overrides the metric names and labels accordingly.
-func (rule RecordingRule) Eval(timestamp clientmodel.Timestamp, storage local.Storage) (ast.Vector, error) {
-	vector, err := rule.EvalRaw(timestamp, storage)
+func (rule RecordingRule) Eval(timestamp clientmodel.Timestamp, engine *promql.Engine) (promql.Vector, error) {
+	vector, err := rule.EvalRaw(timestamp, engine)
 	if err != nil {
 		return nil, err
 	}
@@ -63,8 +65,8 @@ func (rule RecordingRule) Eval(timestamp clientmodel.Timestamp, storage local.St
 	return vector, nil
 }
 
-// ToDotGraph returns the text representation of a dot graph.
-func (rule RecordingRule) ToDotGraph() string {
+// DotGraph returns the text representation of a dot graph.
+func (rule RecordingRule) DotGraph() string {
 	graph := fmt.Sprintf(
 		`digraph "Rules" {
 	  %#p[shape="box",label="%s = "];
@@ -73,7 +75,7 @@ func (rule RecordingRule) ToDotGraph() string {
 	}`,
 		&rule, rule.name,
 		&rule, reflect.ValueOf(rule.vector).Pointer(),
-		rule.vector.NodeTreeToDotGraph(),
+		rule.vector.DotGraph(),
 	)
 	return graph
 }
@@ -87,9 +89,9 @@ func (rule RecordingRule) HTMLSnippet() template.HTML {
 	ruleExpr := rule.vector.String()
 	return template.HTML(fmt.Sprintf(
 		`<a href="%s">%s</a>%s = <a href="%s">%s</a>`,
-		GraphLinkForExpression(rule.name),
+		utility.GraphLinkForExpression(rule.name),
 		rule.name,
 		rule.labels,
-		GraphLinkForExpression(ruleExpr),
+		utility.GraphLinkForExpression(ruleExpr),
 		ruleExpr))
 }

rules/rules.go

@@ -18,8 +18,7 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
-	"github.com/prometheus/prometheus/rules/ast"
-	"github.com/prometheus/prometheus/storage/local"
+	"github.com/prometheus/prometheus/promql"
 )
 
 // A Rule encapsulates a vector expression which is evaluated at a specified
@@ -29,11 +28,11 @@ type Rule interface {
 	Name() string
 	// EvalRaw evaluates the rule's vector expression without triggering any
 	// other actions, like recording or alerting.
-	EvalRaw(timestamp clientmodel.Timestamp, storage local.Storage) (ast.Vector, error)
+	EvalRaw(clientmodel.Timestamp, *promql.Engine) (promql.Vector, error)
 	// Eval evaluates the rule, including any associated recording or alerting actions.
-	Eval(timestamp clientmodel.Timestamp, storage local.Storage) (ast.Vector, error)
-	// ToDotGraph returns a Graphviz dot graph of the rule.
-	ToDotGraph() string
+	Eval(clientmodel.Timestamp, *promql.Engine) (promql.Vector, error)
+	// DotGraph returns a Graphviz dot graph of the rule.
+	DotGraph() string
 	// String returns a human-readable string representation of the rule.
 	String() string
 	// HTMLSnippet returns a human-readable string representation of the rule,

stats/query_stats.go

@@ -31,7 +31,7 @@ const (
 	GetValueAtTimeTime
 	GetBoundaryValuesTime
 	GetRangeValuesTime
-	ViewQueueTime
+	ExecQueueTime
 	ViewDiskPreparationTime
 	ViewDataExtractionTime
 	ViewDiskExtractionTime
@@ -64,8 +64,8 @@ func (s QueryTiming) String() string {
 		return "GetBoundaryValues() time"
 	case GetRangeValuesTime:
 		return "GetRangeValues() time"
-	case ViewQueueTime:
-		return "View queue wait time"
+	case ExecQueueTime:
+		return "Exec queue wait time"
 	case ViewDiskPreparationTime:
 		return "View building disk preparation time"
 	case ViewDataExtractionTime:

storage/metric/matcher.go

@@ -14,6 +14,7 @@
 package metric
 
 import (
+	"fmt"
 	"regexp"
 
 	clientmodel "github.com/prometheus/client_golang/model"
@@ -71,6 +72,10 @@ func NewLabelMatcher(matchType MatchType, name clientmodel.LabelName, value clie
 	return m, nil
 }
 
+func (m *LabelMatcher) String() string {
+	return fmt.Sprintf("%s%s%q", m.Name, m.Type, m.Value)
+}
+
 // Match returns true if the label matcher matches the supplied label value.
 func (m *LabelMatcher) Match(v clientmodel.LabelValue) bool {
 	switch m.Type {

templates/templates.go

@@ -27,10 +27,8 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
-	"github.com/prometheus/prometheus/rules"
-	"github.com/prometheus/prometheus/rules/ast"
-	"github.com/prometheus/prometheus/stats"
-	"github.com/prometheus/prometheus/storage/local"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/utility"
 )
 
 // A version of vector that's easier to use from templates.
@@ -57,18 +55,17 @@ func (q queryResultByLabelSorter) Swap(i, j int) {
 	q.results[i], q.results[j] = q.results[j], q.results[i]
 }
 
-func query(q string, timestamp clientmodel.Timestamp, storage local.Storage) (queryResult, error) {
-	exprNode, err := rules.LoadExprFromString(q)
+func query(q string, timestamp clientmodel.Timestamp, queryEngine *promql.Engine) (queryResult, error) {
+	query, err := queryEngine.NewInstantQuery(q, timestamp)
 	if err != nil {
 		return nil, err
 	}
-	queryStats := stats.NewTimerGroup()
-	vector, err := ast.EvalToVector(exprNode, timestamp, storage, queryStats)
+	vector, err := query.Exec().Vector()
 	if err != nil {
 		return nil, err
 	}
 
-	// ast.Vector is hard to work with in templates, so convert to
+	// promql.Vector is hard to work with in templates, so convert to
 	// base data types.
 	var result = make(queryResult, len(vector))
 	for n, v := range vector {
@@ -92,14 +89,14 @@ type templateExpander struct {
 }
 
 // NewTemplateExpander returns a template expander ready to use.
-func NewTemplateExpander(text string, name string, data interface{}, timestamp clientmodel.Timestamp, storage local.Storage, pathPrefix string) *templateExpander {
+func NewTemplateExpander(text string, name string, data interface{}, timestamp clientmodel.Timestamp, queryEngine *promql.Engine, pathPrefix string) *templateExpander {
 	return &templateExpander{
 		text: text,
 		name: name,
 		data: data,
 		funcMap: text_template.FuncMap{
 			"query": func(q string) (queryResult, error) {
-				return query(q, timestamp, storage)
+				return query(q, timestamp, queryEngine)
 			},
 			"first": func(v queryResult) (*sample, error) {
 				if len(v) > 0 {
@@ -132,8 +129,8 @@ func NewTemplateExpander(text string, name string, data interface{}, timestamp c
 			},
 			"match":     regexp.MatchString,
 			"title":     strings.Title,
-			"graphLink": rules.GraphLinkForExpression,
-			"tableLink": rules.TableLinkForExpression,
+			"graphLink": utility.GraphLinkForExpression,
+			"tableLink": utility.TableLinkForExpression,
 			"sortByLabel": func(label string, v queryResult) queryResult {
 				sorter := queryResultByLabelSorter{v[:], label}
 				sort.Stable(sorter)

templates/templates_test.go

@@ -19,6 +19,7 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
+	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/storage/local"
 )
 
@@ -175,10 +176,12 @@ func TestTemplateExpansion(t *testing.T) {
 	})
 	storage.WaitForIndexing()
 
+	engine := promql.NewEngine(storage)
+
 	for i, s := range scenarios {
 		var result string
 		var err error
-		expander := NewTemplateExpander(s.text, "test", s.input, time, storage, "/")
+		expander := NewTemplateExpander(s.text, "test", s.input, time, engine, "/")
 		if s.html {
 			result, err = expander.ExpandHTML(nil)
 		} else {

tools/rule_checker/main.go

@@ -20,9 +20,10 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 
-	"github.com/prometheus/prometheus/rules"
+	"github.com/prometheus/prometheus/promql"
 )
 
 var (
@@ -33,7 +34,12 @@ var (
 // checkRules reads rules from in. Sucessfully read rules
 // are printed to out.
 func checkRules(filename string, in io.Reader, out io.Writer) error {
-	rules, err := rules.LoadRulesFromReader(in)
+	content, err := ioutil.ReadAll(in)
+	if err != nil {
+		return err
+	}
+
+	rules, err := promql.ParseStmts(string(content))
 	if err != nil {
 		return err
 	}

utility/strconv.go

@@ -15,8 +15,10 @@ package utility
 
 import (
 	"fmt"
+	"net/url"
 	"regexp"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -80,3 +82,24 @@ func StringToDuration(durationStr string) (duration time.Duration, err error) {
 	}
 	return
 }
+
+// TableLinkForExpression creates an escaped relative link to the table view of
+// the provided expression.
+func TableLinkForExpression(expr string) string {
+	// url.QueryEscape percent-escapes everything except spaces, for which it
+	// uses "+". However, in the non-query part of a URI, only percent-escaped
+	// spaces are legal, so we need to manually replace "+" with "%20" after
+	// query-escaping the string.
+	//
+	// See also:
+	// http://stackoverflow.com/questions/1634271/url-encoding-the-space-character-or-20.
+	urlData := url.QueryEscape(fmt.Sprintf(`[{"expr":%q,"tab":1}]`, expr))
+	return fmt.Sprintf("/graph#%s", strings.Replace(urlData, "+", "%20", -1))
+}
+
+// GraphLinkForExpression creates an escaped relative link to the graph view of
+// the provided expression.
+func GraphLinkForExpression(expr string) string {
+	urlData := url.QueryEscape(fmt.Sprintf(`[{"expr":%q,"tab":0}]`, expr))
+	return fmt.Sprintf("/graph#%s", strings.Replace(urlData, "+", "%20", -1))
+}

web/alerts.go

@@ -19,7 +19,6 @@ import (
 	"sync"
 
 	"github.com/prometheus/prometheus/rules"
-	"github.com/prometheus/prometheus/rules/manager"
 )
 
 // AlertStatus bundles alerting rules and the mapping of alert states to row
@@ -47,9 +46,10 @@ func (s byAlertStateSorter) Swap(i, j int) {
 
 // AlertsHandler implements http.Handler.
 type AlertsHandler struct {
-	mutex       sync.Mutex
-	RuleManager manager.RuleManager
+	RuleManager *rules.Manager
 	PathPrefix  string
+
+	mutex sync.Mutex
 }
 
 func (h *AlertsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

web/api/api.go

@@ -20,14 +20,16 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
+	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/storage/local"
 	"github.com/prometheus/prometheus/web/httputils"
 )
 
 // MetricsService manages the /api HTTP endpoint.
 type MetricsService struct {
-	Now     func() clientmodel.Timestamp
-	Storage local.Storage
+	Now         func() clientmodel.Timestamp
+	Storage     local.Storage
+	QueryEngine *promql.Engine
 }
 
 // RegisterHandler registers the handler for the various endpoints below /api.

web/api/api_test.go

@@ -23,6 +23,7 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
+	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/storage/local"
 )
 
@@ -51,7 +52,7 @@ func TestQuery(t *testing.T) {
 		{
 			queryStr: "",
 			status:   http.StatusOK,
-			bodyRe:   "syntax error",
+			bodyRe:   `{"type":"error","value":"Parse error at char 1: no expression found in input","version":1}`,
 		},
 		{
 			queryStr: "expr=testmetric",
@@ -76,7 +77,7 @@ func TestQuery(t *testing.T) {
 		{
 			queryStr: "expr=(badexpression",
 			status:   http.StatusOK,
-			bodyRe:   "syntax error",
+			bodyRe:   `{"type":"error","value":"Parse error at char 15: unclosed left parenthesis","version":1}`,
 		},
 	}
 
@@ -92,8 +93,9 @@ func TestQuery(t *testing.T) {
 	storage.WaitForIndexing()
 
 	api := MetricsService{
-		Now:     testNow,
-		Storage: storage,
+		Now:         testNow,
+		Storage:     storage,
+		QueryEngine: promql.NewEngine(storage),
 	}
 	api.RegisterHandler("/")
 

web/api/query.go

@@ -26,9 +26,6 @@ import (
 
 	clientmodel "github.com/prometheus/client_golang/model"
 
-	"github.com/prometheus/prometheus/rules"
-	"github.com/prometheus/prometheus/rules/ast"
-	"github.com/prometheus/prometheus/stats"
 	"github.com/prometheus/prometheus/web/httputils"
 )
 
@@ -41,9 +38,8 @@ func setAccessControlHeaders(w http.ResponseWriter) {
 }
 
 func httpJSONError(w http.ResponseWriter, err error, code int) {
-	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(code)
-	fmt.Fprintln(w, ast.ErrorToJSON(err))
+	httputils.ErrorJSON(w, err)
 }
 
 func parseTimestampOrNow(t string, now clientmodel.Timestamp) (clientmodel.Timestamp, error) {
@@ -80,16 +76,19 @@ func (serv MetricsService) Query(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	exprNode, err := rules.LoadExprFromString(expr)
+	query, err := serv.QueryEngine.NewInstantQuery(expr, timestamp)
 	if err != nil {
-		fmt.Fprint(w, ast.ErrorToJSON(err))
+		httpJSONError(w, err, http.StatusOK)
 		return
 	}
+	res := query.Exec()
+	if res.Err != nil {
+		httpJSONError(w, res.Err, http.StatusOK)
+		return
+	}
+	glog.V(1).Infof("Instant query: %s\nQuery stats:\n%s\n", expr, query.Stats())
 
-	queryStats := stats.NewTimerGroup()
-	result := ast.EvalToString(exprNode, timestamp, ast.JSON, serv.Storage, queryStats)
-	glog.V(1).Infof("Instant query: %s\nQuery stats:\n%s\n", expr, queryStats)
-	fmt.Fprint(w, result)
+	httputils.RespondJSON(w, res.Value)
 }
 
 // QueryRange handles the /api/query_range endpoint.
@@ -125,50 +124,31 @@ func (serv MetricsService) QueryRange(w http.ResponseWriter, r *http.Request) {
 		end = serv.Now()
 	}
 
-	exprNode, err := rules.LoadExprFromString(expr)
-	if err != nil {
-		fmt.Fprint(w, ast.ErrorToJSON(err))
-		return
-	}
-	if exprNode.Type() != ast.VectorType {
-		fmt.Fprint(w, ast.ErrorToJSON(errors.New("expression does not evaluate to vector type")))
-		return
-	}
-
 	// For safety, limit the number of returned points per timeseries.
 	// This is sufficient for 60s resolution for a week or 1h resolution for a year.
 	if duration/step > 11000 {
-		fmt.Fprint(w, ast.ErrorToJSON(errors.New("exceeded maximum resolution of 11,000 points per timeseries. Try decreasing the query resolution (?step=XX)")))
+		err := errors.New("exceeded maximum resolution of 11,000 points per timeseries. Try decreasing the query resolution (?step=XX)")
+		httpJSONError(w, err, http.StatusBadRequest)
 		return
 	}
 
 	// Align the start to step "tick" boundary.
 	end = end.Add(-time.Duration(end.UnixNano() % int64(step)))
+	start := end.Add(-duration)
 
-	queryStats := stats.NewTimerGroup()
-
-	matrix, err := ast.EvalVectorRange(
-		exprNode.(ast.VectorNode),
-		end.Add(-duration),
-		end,
-		step,
-		serv.Storage,
-		queryStats)
+	query, err := serv.QueryEngine.NewRangeQuery(expr, start, end, step)
 	if err != nil {
-		fmt.Fprint(w, ast.ErrorToJSON(err))
+		httpJSONError(w, err, http.StatusOK)
+		return
+	}
+	matrix, err := query.Exec().Matrix()
+	if err != nil {
+		httpJSONError(w, err, http.StatusOK)
 		return
 	}
 
-	sortTimer := queryStats.GetTimer(stats.ResultSortTime).Start()
-	sort.Sort(matrix)
-	sortTimer.Stop()
-
-	jsonTimer := queryStats.GetTimer(stats.JSONEncodeTime).Start()
-	result := ast.TypedValueToJSON(matrix, "matrix")
-	jsonTimer.Stop()
-
-	glog.V(1).Infof("Range query: %s\nQuery stats:\n%s\n", expr, queryStats)
-	fmt.Fprint(w, result)
+	glog.V(1).Infof("Range query: %s\nQuery stats:\n%s\n", expr, query.Stats())
+	httputils.RespondJSON(w, matrix)
 }
 
 // Metrics handles the /api/metrics endpoint.

web/consoles.go

@@ -22,7 +22,7 @@ import (
 	"path/filepath"
 
 	clientmodel "github.com/prometheus/client_golang/model"
-	"github.com/prometheus/prometheus/storage/local"
+	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/templates"
 )
 
@@ -33,8 +33,8 @@ var (
 
 // ConsolesHandler implements http.Handler.
 type ConsolesHandler struct {
-	Storage    local.Storage
-	PathPrefix string
+	QueryEngine *promql.Engine
+	PathPrefix  string
 }
 
 func (h *ConsolesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -70,7 +70,7 @@ func (h *ConsolesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		Path:      r.URL.Path,
 	}
 
-	template := templates.NewTemplateExpander(string(text), "__console_"+r.URL.Path, data, clientmodel.Now(), h.Storage, h.PathPrefix)
+	template := templates.NewTemplateExpander(string(text), "__console_"+r.URL.Path, data, clientmodel.Now(), h.QueryEngine, h.PathPrefix)
 	filenames, err := filepath.Glob(*consoleLibrariesPath + "/*.lib")
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)

web/httputils/httputils.go

@@ -14,8 +14,12 @@
 package httputils
 
 import (
+	"encoding/json"
+	"io"
 	"net/http"
 	"net/url"
+
+	"github.com/prometheus/prometheus/promql"
 )
 
 // GetQueryParams calls r.ParseForm and returns r.Form.
@@ -23,3 +27,39 @@ func GetQueryParams(r *http.Request) url.Values {
 	r.ParseForm()
 	return r.Form
 }
+
+var jsonFormatVersion = 1
+
+// ErrorJSON writes the given error JSON-formatted to w.
+func ErrorJSON(w io.Writer, err error) error {
+	data := struct {
+		Type    string `json:"type"`
+		Value   string `json:"value"`
+		Version int    `json:"version"`
+	}{
+		Type:    "error",
+		Value:   err.Error(),
+		Version: jsonFormatVersion,
+	}
+	enc := json.NewEncoder(w)
+	return enc.Encode(data)
+}
+
+// RespondJSON converts the given data value to JSON and writes it to w.
+func RespondJSON(w io.Writer, val promql.Value) error {
+	data := struct {
+		Type    string      `json:"type"`
+		Value   interface{} `json:"value"`
+		Version int         `json:"version"`
+	}{
+		Type:    val.Type().String(),
+		Value:   val,
+		Version: jsonFormatVersion,
+	}
+	// TODO(fabxc): Adding MarshalJSON to promql.Values might be a good idea.
+	if sc, ok := val.(*promql.Scalar); ok {
+		data.Value = sc.Value
+	}
+	enc := json.NewEncoder(w)
+	return enc.Encode(data)
+}

web/status.go

@@ -19,7 +19,7 @@ import (
 	"time"
 
 	"github.com/prometheus/prometheus/retrieval"
-	"github.com/prometheus/prometheus/rules/manager"
+	"github.com/prometheus/prometheus/rules"
 )
 
 // PrometheusStatusHandler implements http.Handler.
@@ -29,7 +29,7 @@ type PrometheusStatusHandler struct {
 	BuildInfo   map[string]string
 	Config      string
 	Flags       map[string]string
-	RuleManager manager.RuleManager
+	RuleManager *rules.Manager
 	TargetPools map[string]*retrieval.TargetPool
 
 	Birth      time.Time