github
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Add parsing and execution of new test format. 

This commit adds a new test structure that parses and executes
the new testing language.
pull/689/head
Fabian Reinartz 10 years ago
parent
a236c01457
commit
6321964738
5 changed files with 523 additions and 10 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 11
      promql/engine.go
  2. 2
      promql/parse_test.go
  3. 10
      promql/promql_test.go
  4. 3
      promql/setup_test.go
  5. 507
      promql/test.go

11
promql/engine.go
Unescape View File

@ -279,20 +279,25 @@ func (ng *Engine) NewRangeQuery(qs string, start, end clientmodel.Timestamp, int
if err != nil {
return nil, err
}
qry := ng.newQuery(expr, start, end, interval)
qry.q = qs
return qry, nil
}
func (ng *Engine) newQuery(expr Expr, start, end clientmodel.Timestamp, interval time.Duration) *query {
es := &EvalStmt{
Expr: expr,
Start: start,
End: end,
Interval: interval,
}
qry := &query{
q: qs,
stmts: Statements{es},
ng: ng,
stats: stats.NewTimerGroup(),
}
return qry, nil
return qry
}
// testStmt is an internal helper statement that allows execution

2
promql/parse_test.go
Unescape View File

@ -1272,7 +1272,7 @@ var testSeries = []struct {
},
}
// For these tests only, we use the samallest float64 to signal an omitted value.
// For these tests only, we use the smallest float64 to signal an omitted value.
const none = math.SmallestNonzeroFloat64
func newSeq(vals ...float64) (res []sequenceValue) {

10
promql/promql_test.go
Unescape View File

@ -33,13 +33,13 @@ var (
testEvalTime = testStartTime.Add(testSampleInterval * 10)
fixturesPath = "fixtures"
reSample = regexp.MustCompile(`^(.*)(?: \=\>|:) (\-?\d+\.?\d*(?:e-?\d+)?|[+-]Inf|NaN) \@\[(\d+)\]$`)
minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64.
reSample = regexp.MustCompile(`^(.*)(?: \=\>|:) (\-?\d+\.?\d*(?:e-?\d+)?|[+-]Inf|NaN) \@\[(\d+)\]$`)
// minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64.
)
const (
epsilon = 0.000001 // Relative error allowed for sample values.
)
// const (
// epsilon = 0.000001 // Relative error allowed for sample values.
// )
func annotateWithTime(lines []string, timestamp clientmodel.Timestamp) []string {
annotatedLines := []string{}

3
promql/setup_test.go
Unescape View File

@ -23,7 +23,8 @@ import (
)
var testSampleInterval = time.Duration(5) * time.Minute
var testStartTime = clientmodel.Timestamp(0)
// var testStartTime = clientmodel.Timestamp(0)
func getTestValueStream(startVal, endVal, stepVal clientmodel.SampleValue, startTime clientmodel.Timestamp) (resultValues metric.Values) {
currentTime := startTime

507
promql/test.go
Unescape View File

@ -0,0 +1,507 @@
// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package promql
import (
"fmt"
"io/ioutil"
"math"
"regexp"
"strconv"
"strings"
"testing"
"time"
clientmodel "github.com/prometheus/client_golang/model"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/storage/local"
"github.com/prometheus/prometheus/storage/metric"
"github.com/prometheus/prometheus/utility"
testutil "github.com/prometheus/prometheus/utility/test"
)
var (
minNormal = math.Float64frombits(0x0010000000000000) // The smallest positive normal value of type float64.
patSpace = regexp.MustCompile("[\t ]+")
patLoad = regexp.MustCompile(`^load\s+(.+?)$`)
patEvalInstant = regexp.MustCompile(`^eval(?:_(fail|ordered))?\s+instant\s+(?:at\s+(.+?))?\s+(.+)$`)
)
const (
testStartTime = clientmodel.Timestamp(0)
epsilon = 0.000001 // Relative error allowed for sample values.
maxErrorCount = 10
)
// Test is a sequence of read and write commands that are run
// against a test storage.
type Test struct {
*testing.T
cmds []testCommand
storage local.Storage
closeStorage func()
queryEngine *Engine
}
// NewTest returns an initialized empty Test.
func NewTest(t *testing.T, input string) (*Test, error) {
test := &Test{
T: t,
cmds: []testCommand{},
}
err := test.parse(input)
test.clear()
return test, err
}
func NewTestFromFile(t *testing.T, filename string) (*Test, error) {
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
return NewTest(t, string(content))
}
func raise(line int, format string, v ...interface{}) error {
return &ParseErr{
Line: line + 1,
Err: fmt.Errorf(format, v...),
}
}
func (t *Test) parseLoad(lines []string, i int) (int, *loadCmd, error) {
if !patLoad.MatchString(lines[i]) {
return i, nil, raise(i, "invalid load command. (load <step:duration>)")
}
parts := patLoad.FindStringSubmatch(lines[i])
gap, err := utility.StringToDuration(parts[1])
if err != nil {
return i, nil, raise(i, "invalid step definition %q: %s", parts[1], err)
}
cmd := newLoadCmd(gap)
for i+1 < len(lines) {
i++
defLine := lines[i]
if len(defLine) == 0 {
i--
break
}
metric, vals, err := parseSeriesDesc(defLine)
if err != nil {
perr := err.(*ParseErr)
perr.Line = i + 1
return i, nil, err
}
cmd.set(metric, vals...)
}
return i, cmd, nil
}
func (t *Test) parseEval(lines []string, i int) (int, *evalCmd, error) {
if !patEvalInstant.MatchString(lines[i]) {
return i, nil, raise(i, "invalid evaluation command. (eval[_fail|_ordered] instant [at <offset:duration>] <query>")
}
parts := patEvalInstant.FindStringSubmatch(lines[i])
var (
mod = parts[1]
at = parts[2]
qry = parts[3]
)
expr, err := ParseExpr(qry)
if err != nil {
perr := err.(*ParseErr)
perr.Line = i + 1
perr.Pos += strings.Index(lines[i], qry)
return i, nil, perr
}
offset, err := utility.StringToDuration(at)
if err != nil {
return i, nil, raise(i, "invalid step definition %q: %s", parts[1], err)
}
ts := testStartTime.Add(offset)
cmd := newEvalCmd(expr, ts, ts, 0)
switch mod {
case "ordered":
cmd.ordered = true
case "fail":
cmd.fail = true
}
for j := 1; i+1 < len(lines); j++ {
i++
defLine := lines[i]
if len(defLine) == 0 {
i--
break
}
if f, err := parseNumber(defLine); err == nil {
cmd.expect(0, nil, sequenceValue{value: clientmodel.SampleValue(f)})
break
}
metric, vals, err := parseSeriesDesc(defLine)
if err != nil {
perr := err.(*ParseErr)
perr.Line = i + 1
return i, nil, err
}
// Currently, we are not expecting any matrices.
if len(vals) > 1 {
return i, nil, raise(i, "expecting multiple values in instant evaluation not allowed")
}
cmd.expect(j, metric, vals...)
}
return i, cmd, nil
}
// parse the given command sequence and appends it to the test.
func (t *Test) parse(input string) error {
// Trim lines and remove comments.
lines := strings.Split(input, "\n")
for i, l := range lines {
l = strings.TrimSpace(l)
if strings.HasPrefix(l, "#") {
l = ""
}
lines[i] = l
}
var err error
// Scan for steps line by line.
for i := 0; i < len(lines); i++ {
l := lines[i]
if len(l) == 0 {
continue
}
var cmd testCommand
switch c := strings.ToLower(patSpace.Split(l, 2)[0]); {
case c == "clear":
cmd = &clearCmd{}
case c == "load":
i, cmd, err = t.parseLoad(lines, i)
case strings.HasPrefix(c, "eval"):
i, cmd, err = t.parseEval(lines, i)
default:
return raise(i, "invalid command %q", l)
}
if err != nil {
return err
}
t.cmds = append(t.cmds, cmd)
}
return nil
}
// testCommand is an interface that ensures that only the package internal
// types can be a valid command for a test.
type testCommand interface {
testCmd()
}
func (*clearCmd) testCmd() {}
func (*loadCmd) testCmd() {}
func (*evalCmd) testCmd() {}
// loadCmd is a command that loads sequences of sample values for specific
// metrics into the storage.
type loadCmd struct {
gap time.Duration
metrics map[clientmodel.Fingerprint]clientmodel.Metric
defs map[clientmodel.Fingerprint]metric.Values
}
func newLoadCmd(gap time.Duration) *loadCmd {
return &loadCmd{
gap: gap,
metrics: map[clientmodel.Fingerprint]clientmodel.Metric{},
defs: map[clientmodel.Fingerprint]metric.Values{},
}
}
func (cmd loadCmd) String() string {
return "load"
}
// set a sequence of sample values for the given metric.
func (cmd *loadCmd) set(m clientmodel.Metric, vals ...sequenceValue) {
fp := m.Fingerprint()
samples := make(metric.Values, 0, len(vals))
ts := testStartTime
for _, v := range vals {
if !v.omitted {
samples = append(samples, metric.SamplePair{
Timestamp: ts,
Value: v.value,
})
}
ts = ts.Add(cmd.gap)
}
cmd.defs[fp] = samples
cmd.metrics[fp] = m
}
// append the defined time series to the storage.
func (cmd *loadCmd) append(a storage.SampleAppender) {
for fp, samples := range cmd.defs {
met := cmd.metrics[fp]
for _, smpl := range samples {
s := &clientmodel.Sample{
Metric: met,
Value: smpl.Value,
Timestamp: smpl.Timestamp,
}
a.Append(s)
}
}
}
// evalCmd is a command that evaluates an expression for the given time (range)
// and expects a specific result.
type evalCmd struct {
expr Expr
start, end clientmodel.Timestamp
interval time.Duration
instant bool
fail, ordered bool
metrics map[clientmodel.Fingerprint]clientmodel.Metric
expected map[clientmodel.Fingerprint]entry
}
type entry struct {
pos int
vals []sequenceValue
}
func (e entry) String() string {
return fmt.Sprintf("%d: %s", e.pos, e.vals)
}
func newEvalCmd(expr Expr, start, end clientmodel.Timestamp, interval time.Duration) *evalCmd {
return &evalCmd{
expr: expr,
start: start,
end: end,
interval: interval,
instant: start == end && interval == 0,
metrics: map[clientmodel.Fingerprint]clientmodel.Metric{},
expected: map[clientmodel.Fingerprint]entry{},
}
}
func (ev *evalCmd) String() string {
return "eval"
}
// expect adds a new metric with a sequence of values to the set of expected
// results for the query.
func (ev *evalCmd) expect(pos int, m clientmodel.Metric, vals ...sequenceValue) {
if m == nil {
ev.expected[0] = entry{pos: pos, vals: vals}
return
}
fp := m.Fingerprint()
ev.metrics[fp] = m
ev.expected[fp] = entry{pos: pos, vals: vals}
}
// compareResult compares the result value with the defined expectation.
func (ev *evalCmd) compareResult(result Value) error {
switch val := result.(type) {
case Matrix:
if ev.instant {
return fmt.Errorf("received range result on instant evaluation")
}
seen := map[clientmodel.Fingerprint]bool{}
for pos, v := range val {
fp := v.Metric.Metric.Fingerprint()
if _, ok := ev.metrics[fp]; !ok {
return fmt.Errorf("unexpected metric %s in result", v.Metric.Metric)
}
exp := ev.expected[fp]
if ev.ordered && exp.pos != pos+1 {
return fmt.Errorf("expected metric %s with %v at position %d but was at %d", v.Metric.Metric, exp.vals, exp.pos, pos+1)
}
for i, expVal := range exp.vals {
if !almostEqual(float64(expVal.value), float64(v.Values[i].Value)) {
return fmt.Errorf("expected %v for %s but got %v", expVal, v.Metric.Metric, v.Values)
}
}
seen[fp] = true
}
for fp, expVals := range ev.expected {
if !seen[fp] {
return fmt.Errorf("expected metric %s with %v not found", ev.metrics[fp], expVals)
}
}
case Vector:
if !ev.instant {
fmt.Errorf("received instant result on range evaluation")
}
seen := map[clientmodel.Fingerprint]bool{}
for pos, v := range val {
fp := v.Metric.Metric.Fingerprint()
if _, ok := ev.metrics[fp]; !ok {
return fmt.Errorf("unexpected metric %s in result", v.Metric.Metric)
}
exp := ev.expected[fp]
if ev.ordered && exp.pos != pos+1 {
return fmt.Errorf("expected metric %s with %v at position %d but was at %d", v.Metric.Metric, exp.vals, exp.pos, pos+1)
}
if !almostEqual(float64(exp.vals[0].value), float64(v.Value)) {
return fmt.Errorf("expected %v for %s but got %v", exp.vals[0].value, v.Metric.Metric, v.Value)
}
seen[fp] = true
}
for fp, expVals := range ev.expected {
if !seen[fp] {
return fmt.Errorf("expected metric %s with %v not found", ev.metrics[fp], expVals)
}
}
case *Scalar:
if !almostEqual(float64(ev.expected[0].vals[0].value), float64(val.Value)) {
return fmt.Errorf("expected scalar %v but got %v", val.Value, ev.expected[0].vals[0].value)
}
default:
panic(fmt.Errorf("promql.Test.compareResult: unexpected result type %T", result))
}
return nil
}
// clearCmd is a command that wipes the test's storage state.
type clearCmd struct{}
func (cmd clearCmd) String() string {
return "clear"
}
// Run executes the command sequence of the test. Until the maximum error number
// is reached, evaluation errors do not terminate execution.
func (t *Test) Run() error {
for _, cmd := range t.cmds {
err := t.exec(cmd)
// TODO(fabxc): aggregate command errors, yield diffs for result
// comparison errors.
if err != nil {
return err
}
}
return nil
}
// exec processes a single step of the test
func (t *Test) exec(tc testCommand) error {
switch cmd := tc.(type) {
case *clearCmd:
t.clear()
case *loadCmd:
cmd.append(t.storage)
t.storage.WaitForIndexing()
case *evalCmd:
q := t.queryEngine.newQuery(cmd.expr, cmd.start, cmd.end, cmd.interval)
res := q.Exec()
if res.Err != nil {
if cmd.fail {
return nil
}
return fmt.Errorf("error evaluating query: %s", res.Err)
}
if res.Err == nil && cmd.fail {
return fmt.Errorf("expected error evaluating query but got none")
}
err := cmd.compareResult(res.Value)
if err != nil {
return fmt.Errorf("error in %s %s: %s", cmd, cmd.expr, err)
}
default:
panic("promql.Test.exec: unknown test command type")
}
return nil
}
// clear the current test storage of all inserted samples.
func (t *Test) clear() {
if t.closeStorage != nil {
t.closeStorage()
}
if t.queryEngine != nil {
t.queryEngine.Stop()
}
var closer testutil.Closer
t.storage, closer = local.NewTestStorage(t, 1)
t.closeStorage = closer.Close
t.queryEngine = NewEngine(t.storage)
}
func (t *Test) Close() {
t.queryEngine.Stop()
t.closeStorage()
}
// samplesAlmostEqual returns true if the two sample lines only differ by a
// small relative error in their sample value.
func almostEqual(a, b float64) bool {
// NaN has no equality but for testing we still want to know whether both values
// are NaN.
if math.IsNaN(a) && math.IsNaN(b) {
return true
}
// Cf. http://floating-point-gui.de/errors/comparison/
if a == b {
return true
}
diff := math.Abs(a - b)
if a == 0 || b == 0 || diff < minNormal {
return diff < epsilon*minNormal
}
return diff/(math.Abs(a)+math.Abs(b)) < epsilon
}
func parseNumber(s string) (float64, error) {
n, err := strconv.ParseInt(s, 0, 64)
f := float64(n)
if err != nil {
f, err = strconv.ParseFloat(s, 64)
}
if err != nil {
return 0, fmt.Errorf("error parsing number: %s", err)
}
return f, nil
}
Write Preview
Loading…
Cancel
Save