diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index e002cff31..108888d65 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -443,7 +443,11 @@ func main() { } files = append(files, fs...) } - return ruleManager.Update(time.Duration(cfg.GlobalConfig.EvaluationInterval), files) + return ruleManager.Update( + time.Duration(cfg.GlobalConfig.EvaluationInterval), + files, + cfg.GlobalConfig.ExternalLabels, + ) }, } @@ -747,6 +751,7 @@ func reloadConfig(filename string, logger log.Logger, rls ...func(*config.Config if failed { return errors.Errorf("one or more errors occurred while applying the new configuration (--config.file=%q)", filename) } + promql.SetDefaultEvaluationInterval(time.Duration(conf.GlobalConfig.EvaluationInterval)) level.Info(logger).Log("msg", "Completed loading of configuration file", "filename", filename) return nil diff --git a/cmd/promtool/http.go b/cmd/promtool/http.go index 43656ad67..e269a23ed 100644 --- a/cmd/promtool/http.go +++ b/cmd/promtool/http.go @@ -14,8 +14,6 @@ package main import ( - "context" - "net/http" "time" "github.com/pkg/errors" @@ -42,12 +40,6 @@ func newPrometheusHTTPClient(serverURL string) (*prometheusHTTPClient, error) { }, nil } -func (c *prometheusHTTPClient) do(req *http.Request) (*http.Response, []byte, error) { - ctx, cancel := context.WithTimeout(context.Background(), c.requestTimeout) - defer cancel() - return c.httpClient.Do(ctx, req) -} - func (c *prometheusHTTPClient) urlJoin(path string) string { return c.httpClient.URL(path, nil).String() } diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 7dbf41934..295ace2d7 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -163,7 +163,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou Logger: &dummyLogger{}, } m := rules.NewManager(opts) - groupsMap, ers := m.LoadGroups(tg.Interval, ruleFiles...) + // TODO(beorn7): Provide a way to pass in external labels. + groupsMap, ers := m.LoadGroups(tg.Interval, nil, ruleFiles...) if ers != nil { return ers } diff --git a/discovery/openstack/mock_test.go b/discovery/openstack/mock_test.go index 67b14f6f6..13c25dfbb 100644 --- a/discovery/openstack/mock_test.go +++ b/discovery/openstack/mock_test.go @@ -247,7 +247,7 @@ func (m *SDMock) HandleHypervisorListSuccessfully() { testHeader(m.t, r, "X-Auth-Token", tokenID) w.Header().Add("Content-Type", "application/json") - fmt.Fprintf(w, hypervisorListBody) + fmt.Fprint(w, hypervisorListBody) }) } @@ -544,7 +544,7 @@ func (m *SDMock) HandleServerListSuccessfully() { testHeader(m.t, r, "X-Auth-Token", tokenID) w.Header().Add("Content-Type", "application/json") - fmt.Fprintf(w, serverListBody) + fmt.Fprint(w, serverListBody) }) } @@ -583,6 +583,6 @@ func (m *SDMock) HandleFloatingIPListSuccessfully() { testHeader(m.t, r, "X-Auth-Token", tokenID) w.Header().Add("Content-Type", "application/json") - fmt.Fprintf(w, listOutput) + fmt.Fprint(w, listOutput) }) } diff --git a/docs/configuration/alerting_rules.md b/docs/configuration/alerting_rules.md index cc674a09b..1bc435beb 100644 --- a/docs/configuration/alerting_rules.md +++ b/docs/configuration/alerting_rules.md @@ -42,9 +42,11 @@ The `annotations` clause specifies a set of informational labels that can be use #### Templating -Label and annotation values can be templated using [console templates](https://prometheus.io/docs/visualization/consoles). -The `$labels` variable holds the label key/value pairs of an alert instance -and `$value` holds the evaluated value of an alert instance. +Label and annotation values can be templated using [console +templates](https://prometheus.io/docs/visualization/consoles). The `$labels` +variable holds the label key/value pairs of an alert instance. The configured +external labels can be accessed via the `$externalLabels` variable. The +`$value` variable holds the evaluated value of an alert instance. # To insert a firing element's label values: {{ $labels. }} diff --git a/docs/configuration/template_reference.md b/docs/configuration/template_reference.md index 2e10e4247..8791d68b3 100644 --- a/docs/configuration/template_reference.md +++ b/docs/configuration/template_reference.md @@ -89,8 +89,10 @@ parameterize templates, and have a few other differences. ### Alert field templates -`.Value` and `.Labels` contain the alert value and labels. They are also exposed -as the `$value` and `$labels` variables for convenience. +`.Value`, `.Labels`, and `ExternalLabels` contain the alert value, the alert +labels, and the globally configured external labels, respectively. They are +also exposed as the `$value`, `$labels`, and `$externalLabels` variables for +convenience. ### Console templates @@ -104,7 +106,9 @@ auto-escaping. To bypass the auto-escaping use the `safe*` functions., URL parameters are available as a map in `.Params`. To access multiple URL parameters by the same name, `.RawParams` is a map of the list values for each parameter. The URL path is available in `.Path`, excluding the `/consoles/` -prefix. +prefix. The globally configured external labels are available as +`.ExternalLabels`. There are also convenience variables for all four: +`$rawParams`, `$params`, `$path`, and `$externalLabels`. Consoles also have access to all the templates defined with `{{define "templateName"}}...{{end}}` found in `*.lib` files in the directory pointed to diff --git a/pkg/rulefmt/rulefmt.go b/pkg/rulefmt/rulefmt.go index 60faee085..b89470d38 100644 --- a/pkg/rulefmt/rulefmt.go +++ b/pkg/rulefmt/rulefmt.go @@ -16,6 +16,7 @@ package rulefmt import ( "context" "io/ioutil" + "strings" "time" "github.com/pkg/errors" @@ -155,12 +156,16 @@ func testTemplateParsing(rl *Rule) (errs []error) { } // Trying to parse templates. - tmplData := template.AlertTemplateData(make(map[string]string), 0) - defs := "{{$labels := .Labels}}{{$value := .Value}}" + tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0) + defs := []string{ + "{{$labels := .Labels}}", + "{{$externalLabels := .ExternalLabels}}", + "{{$value := .Value}}", + } parseTest := func(text string) error { tmpl := template.NewTemplateExpander( context.TODO(), - defs+text, + strings.Join(append(defs, text), ""), "__alert_"+rl.Alert, tmplData, model.Time(timestamp.FromTime(time.Now())), diff --git a/promql/parse.go b/promql/parse.go index 2a055fbbb..4bb4a11c6 100644 --- a/promql/parse.go +++ b/promql/parse.go @@ -326,17 +326,15 @@ var errUnexpected = errors.New("unexpected error") // recover is the handler that turns panics into returns from the top level of Parse. func (p *parser) recover(errp *error) { e := recover() - if e != nil { - if _, ok := e.(runtime.Error); ok { - // Print the stack trace but do not inhibit the running application. - buf := make([]byte, 64<<10) - buf = buf[:runtime.Stack(buf, false)] - - fmt.Fprintf(os.Stderr, "parser panic: %v\n%s", e, buf) - *errp = errUnexpected - } else { - *errp = e.(error) - } + if _, ok := e.(runtime.Error); ok { + // Print the stack trace but do not inhibit the running application. + buf := make([]byte, 64<<10) + buf = buf[:runtime.Stack(buf, false)] + + fmt.Fprintf(os.Stderr, "parser panic: %v\n%s", e, buf) + *errp = errUnexpected + } else if e != nil { + *errp = e.(error) } p.lex.close() } diff --git a/rules/alerting.go b/rules/alerting.go index 7292d287b..b66d6db88 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "net/url" + "strings" "sync" "time" @@ -119,6 +120,8 @@ type AlertingRule struct { labels labels.Labels // Non-identifying key/value pairs. annotations labels.Labels + // External labels from the global config. + externalLabels map[string]string // true if old state has been restored. We start persisting samples for ALERT_FOR_STATE // only after the restoration. restored bool @@ -140,17 +143,27 @@ type AlertingRule struct { } // NewAlertingRule constructs a new AlertingRule. -func NewAlertingRule(name string, vec promql.Expr, hold time.Duration, lbls, anns labels.Labels, restored bool, logger log.Logger) *AlertingRule { +func NewAlertingRule( + name string, vec promql.Expr, hold time.Duration, + labels, annotations, externalLabels labels.Labels, + restored bool, logger log.Logger, +) *AlertingRule { + el := make(map[string]string, len(externalLabels)) + for _, lbl := range externalLabels { + el[lbl.Name] = lbl.Value + } + return &AlertingRule{ - name: name, - vector: vec, - holdDuration: hold, - labels: lbls, - annotations: anns, - health: HealthUnknown, - active: map[uint64]*Alert{}, - logger: logger, - restored: restored, + name: name, + vector: vec, + holdDuration: hold, + labels: labels, + annotations: annotations, + externalLabels: el, + health: HealthUnknown, + active: map[uint64]*Alert{}, + logger: logger, + restored: restored, } } @@ -305,15 +318,19 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, l[lbl.Name] = lbl.Value } - tmplData := template.AlertTemplateData(l, smpl.V) + tmplData := template.AlertTemplateData(l, r.externalLabels, smpl.V) // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. - defs := "{{$labels := .Labels}}{{$value := .Value}}" + defs := []string{ + "{{$labels := .Labels}}", + "{{$externalLabels := .ExternalLabels}}", + "{{$value := .Value}}", + } expand := func(text string) string { tmpl := template.NewTemplateExpander( ctx, - defs+text, + strings.Join(append(defs, text), ""), "__alert_"+r.Name(), tmplData, model.Time(timestamp.FromTime(ts)), @@ -449,7 +466,7 @@ func (r *AlertingRule) ForEachActiveAlert(f func(*Alert)) { } func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay time.Duration, interval time.Duration, notifyFunc NotifyFunc) { - alerts := make([]*Alert, 0) + alerts := []*Alert{} r.ForEachActiveAlert(func(alert *Alert) { if alert.needsSending(ts, resendDelay) { alert.LastSentAt = ts diff --git a/rules/alerting_test.go b/rules/alerting_test.go index 884da30bf..8d611c3e4 100644 --- a/rules/alerting_test.go +++ b/rules/alerting_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + "github.com/go-kit/kit/log" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/promql" @@ -26,7 +27,7 @@ import ( func TestAlertingRuleHTMLSnippet(t *testing.T) { expr, err := promql.ParseExpr(`foo{html="BOLD"}`) testutil.Ok(t, err) - rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "BOLD"), labels.FromStrings("html", "BOLD"), false, nil) + rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "BOLD"), labels.FromStrings("html", "BOLD"), nil, false, nil) const want = `alert: testrule expr: foo{html="<b>BOLD<b>"} @@ -62,7 +63,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) { // If an alert is going back and forth between two label values it will never fire. // Instead, you should write two alerts with constant labels. labels.FromStrings("severity", "{{ if lt $value 80.0 }}critical{{ else }}warning{{ end }}"), - nil, true, nil, + nil, nil, true, nil, ) results := []promql.Vector{ @@ -142,3 +143,96 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) { testutil.Equals(t, result, filteredRes) } } + +func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) { + suite, err := promql.NewTest(t, ` + load 1m + http_requests{job="app-server", instance="0"} 75 85 70 70 + `) + testutil.Ok(t, err) + defer suite.Close() + + err = suite.Run() + testutil.Ok(t, err) + + expr, err := promql.ParseExpr(`http_requests < 100`) + testutil.Ok(t, err) + + ruleWithoutExternalLabels := NewAlertingRule( + "ExternalLabelDoesNotExist", + expr, + time.Minute, + labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."), + nil, + nil, + true, log.NewNopLogger(), + ) + ruleWithExternalLabels := NewAlertingRule( + "ExternalLabelExists", + expr, + time.Minute, + labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."), + nil, + labels.FromStrings("foo", "bar", "dings", "bums"), + true, log.NewNopLogger(), + ) + result := promql.Vector{ + { + Metric: labels.FromStrings( + "__name__", "ALERTS", + "alertname", "ExternalLabelDoesNotExist", + "alertstate", "pending", + "instance", "0", + "job", "app-server", + "templated_label", "There are 0 external Labels, of which foo is .", + ), + Point: promql.Point{V: 1}, + }, + { + Metric: labels.FromStrings( + "__name__", "ALERTS", + "alertname", "ExternalLabelExists", + "alertstate", "pending", + "instance", "0", + "job", "app-server", + "templated_label", "There are 2 external Labels, of which foo is bar.", + ), + Point: promql.Point{V: 1}, + }, + } + + evalTime := time.Unix(0, 0) + result[0].Point.T = timestamp.FromTime(evalTime) + result[1].Point.T = timestamp.FromTime(evalTime) + + var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples. + res, err := ruleWithoutExternalLabels.Eval( + suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, + ) + testutil.Ok(t, err) + for _, smpl := range res { + smplName := smpl.Metric.Get("__name__") + if smplName == "ALERTS" { + filteredRes = append(filteredRes, smpl) + } else { + // If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'. + testutil.Equals(t, smplName, "ALERTS_FOR_STATE") + } + } + + res, err = ruleWithExternalLabels.Eval( + suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, + ) + testutil.Ok(t, err) + for _, smpl := range res { + smplName := smpl.Metric.Get("__name__") + if smplName == "ALERTS" { + filteredRes = append(filteredRes, smpl) + } else { + // If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'. + testutil.Equals(t, smplName, "ALERTS_FOR_STATE") + } + } + + testutil.Equals(t, result, filteredRes) +} diff --git a/rules/manager.go b/rules/manager.go index 96766b2ab..77130489f 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -751,11 +751,11 @@ func (m *Manager) Stop() { // Update the rule manager's state as the config requires. If // loading the new rules failed the old rule set is restored. -func (m *Manager) Update(interval time.Duration, files []string) error { +func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels) error { m.mtx.Lock() defer m.mtx.Unlock() - groups, errs := m.LoadGroups(interval, files...) + groups, errs := m.LoadGroups(interval, externalLabels, files...) if errs != nil { for _, e := range errs { level.Error(m.logger).Log("msg", "loading groups failed", "err", e) @@ -803,7 +803,9 @@ func (m *Manager) Update(interval time.Duration, files []string) error { } // LoadGroups reads groups from a list of files. -func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[string]*Group, []error) { +func (m *Manager) LoadGroups( + interval time.Duration, externalLabels labels.Labels, filenames ...string, +) (map[string]*Group, []error) { groups := make(map[string]*Group) shouldRestore := !m.restored @@ -834,6 +836,7 @@ func (m *Manager) LoadGroups(interval time.Duration, filenames ...string) (map[s time.Duration(r.For), labels.FromMap(r.Labels), labels.FromMap(r.Annotations), + externalLabels, m.restored, log.With(m.logger, "alert", r.Alert), )) diff --git a/rules/manager_test.go b/rules/manager_test.go index 6dbff9c38..6298c0866 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -51,7 +51,7 @@ func TestAlertingRule(t *testing.T) { expr, time.Minute, labels.FromStrings("severity", "{{\"c\"}}ritical"), - nil, true, nil, + nil, nil, true, nil, ) result := promql.Vector{ { @@ -192,7 +192,7 @@ func TestForStateAddSamples(t *testing.T) { expr, time.Minute, labels.FromStrings("severity", "{{\"c\"}}ritical"), - nil, true, nil, + nil, nil, true, nil, ) result := promql.Vector{ { @@ -366,7 +366,7 @@ func TestForStateRestore(t *testing.T) { expr, alertForDuration, labels.FromStrings("severity", "critical"), - nil, true, nil, + nil, nil, true, nil, ) group := NewGroup("default", "", time.Second, []Rule{rule}, true, opts) @@ -426,7 +426,7 @@ func TestForStateRestore(t *testing.T) { expr, alertForDuration, labels.FromStrings("severity", "critical"), - nil, false, nil, + nil, nil, false, nil, ) newGroup := NewGroup("default", "", time.Second, []Rule{newRule}, true, opts) @@ -581,12 +581,12 @@ func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) { func TestCopyState(t *testing.T) { oldGroup := &Group{ rules: []Rule{ - NewAlertingRule("alert", nil, 0, nil, nil, true, nil), + NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil), NewRecordingRule("rule1", nil, nil), NewRecordingRule("rule2", nil, nil), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil), }, seriesInPreviousEval: []map[string]labels.Labels{ {"a": nil}, @@ -604,10 +604,10 @@ func TestCopyState(t *testing.T) { NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v0"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}), - NewAlertingRule("alert", nil, 0, nil, nil, true, nil), + NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil), NewRecordingRule("rule1", nil, nil), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, true, nil), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil), NewRecordingRule("rule4", nil, nil), }, seriesInPreviousEval: make([]map[string]labels.Labels, 8), @@ -654,7 +654,7 @@ func TestUpdate(t *testing.T) { ruleManager.Run() defer ruleManager.Stop() - err := ruleManager.Update(10*time.Second, files) + err := ruleManager.Update(10*time.Second, files, nil) testutil.Ok(t, err) testutil.Assert(t, len(ruleManager.groups) > 0, "expected non-empty rule groups") for _, g := range ruleManager.groups { @@ -663,7 +663,7 @@ func TestUpdate(t *testing.T) { } } - err = ruleManager.Update(10*time.Second, files) + err = ruleManager.Update(10*time.Second, files, nil) testutil.Ok(t, err) for _, g := range ruleManager.groups { for _, actual := range g.seriesInPreviousEval { @@ -699,7 +699,7 @@ func TestNotify(t *testing.T) { expr, err := promql.ParseExpr("a > 1") testutil.Ok(t, err) - rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, true, log.NewNopLogger()) + rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, true, log.NewNopLogger()) group := NewGroup("alert", "", time.Second, []Rule{rule}, true, opts) app, _ := storage.Appender() diff --git a/template/template.go b/template/template.go index ef6212780..de6ee5453 100644 --- a/template/template.go +++ b/template/template.go @@ -28,9 +28,9 @@ import ( text_template "text/template" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/util/strutil" ) @@ -261,13 +261,15 @@ func NewTemplateExpander( } // AlertTemplateData returns the interface to be used in expanding the template. -func AlertTemplateData(labels map[string]string, value float64) interface{} { +func AlertTemplateData(labels map[string]string, externalLabels map[string]string, value float64) interface{} { return struct { - Labels map[string]string - Value float64 + Labels map[string]string + ExternalLabels map[string]string + Value float64 }{ - Labels: labels, - Value: value, + Labels: labels, + ExternalLabels: externalLabels, + Value: value, } } diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 5082c1172..b2b8939de 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -142,6 +142,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule { time.Second, labels.Labels{}, labels.Labels{}, + labels.Labels{}, true, log.NewNopLogger(), ) @@ -151,6 +152,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule { time.Second, labels.Labels{}, labels.Labels{}, + labels.Labels{}, true, log.NewNopLogger(), ) diff --git a/web/federate_test.go b/web/federate_test.go index 03c4763eb..1476bec3f 100644 --- a/web/federate_test.go +++ b/web/federate_test.go @@ -28,7 +28,6 @@ import ( var scenarios = map[string]struct { params string - accept string externalLabels labels.Labels code int body string diff --git a/web/web.go b/web/web.go index 6a23e0254..77a059725 100644 --- a/web/web.go +++ b/web/web.go @@ -541,19 +541,39 @@ func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) { for k, v := range rawParams { params[k] = v[0] } + + externalLabels := map[string]string{} + h.mtx.RLock() + els := h.config.GlobalConfig.ExternalLabels + h.mtx.RUnlock() + for _, el := range els { + externalLabels[el.Name] = el.Value + } + + // Inject some convenience variables that are easier to remember for users + // who are not used to Go's templating system. + defs := []string{ + "{{$rawParams := .RawParams }}", + "{{$params := .Params}}", + "{{$path := .Path}}", + "{{$externalLabels := .ExternalLabels}}", + } + data := struct { - RawParams url.Values - Params map[string]string - Path string + RawParams url.Values + Params map[string]string + Path string + ExternalLabels map[string]string }{ - RawParams: rawParams, - Params: params, - Path: strings.TrimLeft(name, "/"), + RawParams: rawParams, + Params: params, + Path: strings.TrimLeft(name, "/"), + ExternalLabels: externalLabels, } tmpl := template.NewTemplateExpander( h.context, - string(text), + strings.Join(append(defs, string(text)), ""), "__console_"+name, data, h.now(),