diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 9fbd6e76b..f5b8cd3c5 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -527,6 +527,9 @@ func main() { conntrack.DialWithTracing(), ) + // This is passed to ruleManager.Update(). + var externalURL = cfg.web.ExternalURL.String() + reloaders := []reloader{ { name: "remote_storage", @@ -592,6 +595,7 @@ func main() { time.Duration(cfg.GlobalConfig.EvaluationInterval), files, cfg.GlobalConfig.ExternalLabels, + externalURL, ) }, }, diff --git a/cmd/promtool/rules.go b/cmd/promtool/rules.go index 5f2e9a33f..a97e45b68 100644 --- a/cmd/promtool/rules.go +++ b/cmd/promtool/rules.go @@ -68,7 +68,7 @@ func newRuleImporter(logger log.Logger, config ruleImporterConfig, apiClient que // loadGroups parses groups from a list of recording rule files. func (importer *ruleImporter) loadGroups(ctx context.Context, filenames []string) (errs []error) { - groups, errs := importer.ruleManager.LoadGroups(importer.config.evalInterval, labels.Labels{}, filenames...) + groups, errs := importer.ruleManager.LoadGroups(importer.config.evalInterval, labels.Labels{}, "", filenames...) if errs != nil { return errs } diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index c55db94da..624dd5ba5 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -146,6 +146,7 @@ type testGroup struct { AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"` PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"` ExternalLabels labels.Labels `yaml:"external_labels,omitempty"` + ExternalURL string `yaml:"external_url,omitempty"` TestGroupName string `yaml:"name,omitempty"` } @@ -168,7 +169,7 @@ func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]i Logger: log.NewNopLogger(), } m := rules.NewManager(opts) - groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, ruleFiles...) + groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, tg.ExternalURL, ruleFiles...) if ers != nil { return ers } diff --git a/docs/configuration/template_reference.md b/docs/configuration/template_reference.md index 92161f4c6..67e37e0d3 100644 --- a/docs/configuration/template_reference.md +++ b/docs/configuration/template_reference.md @@ -90,9 +90,9 @@ parameterize templates, and have a few other differences. ### Alert field templates -`.Value`, `.Labels`, and `.ExternalLabels` contain the alert value, the alert -labels, and the globally configured external labels, respectively. They are -also exposed as the `$value`, `$labels`, and `$externalLabels` variables for +`.Value`, `.Labels`, `.ExternalLabels`, and `.ExternalURL` contain the alert value, the alert +labels, the globally configured external labels, and the external URL (configured with `--web.external-url`) respectively. They are +also exposed as the `$value`, `$labels`, `$externalLabels`, and `$externalURL` variables for convenience. ### Console templates diff --git a/docs/configuration/unit_testing_rules.md b/docs/configuration/unit_testing_rules.md index 644b35a17..1cf87d6b3 100644 --- a/docs/configuration/unit_testing_rules.md +++ b/docs/configuration/unit_testing_rules.md @@ -59,6 +59,10 @@ promql_expr_test: # External labels accessible to the alert template. external_labels: [ : ... ] + +# External URL accessible to the alert template. +# Usually set using --web.external-url. + [ external_url: ] ``` ### `` diff --git a/pkg/rulefmt/rulefmt.go b/pkg/rulefmt/rulefmt.go index 8267bc90e..920c7a203 100644 --- a/pkg/rulefmt/rulefmt.go +++ b/pkg/rulefmt/rulefmt.go @@ -223,10 +223,11 @@ func testTemplateParsing(rl *RuleNode) (errs []error) { } // Trying to parse templates. - tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, 0) + tmplData := template.AlertTemplateData(map[string]string{}, map[string]string{}, "", 0) defs := []string{ "{{$labels := .Labels}}", "{{$externalLabels := .ExternalLabels}}", + "{{$externalURL := .ExternalURL}}", "{{$value := .Value}}", } parseTest := func(text string) error { diff --git a/rules/alerting.go b/rules/alerting.go index a4074221f..53da7586b 100644 --- a/rules/alerting.go +++ b/rules/alerting.go @@ -121,6 +121,8 @@ type AlertingRule struct { annotations labels.Labels // External labels from the global config. externalLabels map[string]string + // The external URL from the --web.external-url flag. + externalURL string // true if old state has been restored. We start persisting samples for ALERT_FOR_STATE // only after the restoration. restored bool @@ -144,7 +146,7 @@ type AlertingRule struct { // NewAlertingRule constructs a new AlertingRule. func NewAlertingRule( name string, vec parser.Expr, hold time.Duration, - labels, annotations, externalLabels labels.Labels, + labels, annotations, externalLabels labels.Labels, externalURL string, restored bool, logger log.Logger, ) *AlertingRule { el := make(map[string]string, len(externalLabels)) @@ -159,6 +161,7 @@ func NewAlertingRule( labels: labels, annotations: annotations, externalLabels: el, + externalURL: externalURL, health: HealthUnknown, active: map[uint64]*Alert{}, logger: logger, @@ -318,12 +321,13 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, l[lbl.Name] = lbl.Value } - tmplData := template.AlertTemplateData(l, r.externalLabels, smpl.V) + tmplData := template.AlertTemplateData(l, r.externalLabels, r.externalURL, smpl.V) // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := []string{ "{{$labels := .Labels}}", "{{$externalLabels := .ExternalLabels}}", + "{{$externalURL := .ExternalURL}}", "{{$value := .Value}}", } diff --git a/rules/alerting_test.go b/rules/alerting_test.go index c759deb65..f97f5718e 100644 --- a/rules/alerting_test.go +++ b/rules/alerting_test.go @@ -32,7 +32,7 @@ import ( func TestAlertingRuleHTMLSnippet(t *testing.T) { expr, err := parser.ParseExpr(`foo{html="BOLD"}`) require.NoError(t, err) - rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "BOLD"), labels.FromStrings("html", "BOLD"), nil, false, nil) + rule := NewAlertingRule("testrule", expr, 0, labels.FromStrings("html", "BOLD"), labels.FromStrings("html", "BOLD"), nil, "", false, nil) const want = template.HTML(`alert: testrule expr: foo{html="<b>BOLD<b>"} @@ -79,7 +79,7 @@ func TestAlertingRuleState(t *testing.T) { } for i, test := range tests { - rule := NewAlertingRule(test.name, nil, 0, nil, nil, nil, true, nil) + rule := NewAlertingRule(test.name, nil, 0, nil, nil, nil, "", true, nil) rule.active = test.active got := rule.State() require.Equal(t, test.want, got, "test case %d unexpected AlertState, want:%d got:%d", i, test.want, got) @@ -107,7 +107,7 @@ func TestAlertingRuleLabelsUpdate(t *testing.T) { // If an alert is going back and forth between two label values it will never fire. // Instead, you should write two alerts with constant labels. labels.FromStrings("severity", "{{ if lt $value 80.0 }}critical{{ else }}warning{{ end }}"), - nil, nil, true, nil, + nil, nil, "", true, nil, ) results := []promql.Vector{ @@ -208,6 +208,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) { labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."), nil, nil, + "", true, log.NewNopLogger(), ) ruleWithExternalLabels := NewAlertingRule( @@ -217,6 +218,7 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) { labels.FromStrings("templated_label", "There are {{ len $externalLabels }} external Labels, of which foo is {{ $externalLabels.foo }}."), nil, labels.FromStrings("foo", "bar", "dings", "bums"), + "", true, log.NewNopLogger(), ) result := promql.Vector{ @@ -280,6 +282,100 @@ func TestAlertingRuleExternalLabelsInTemplate(t *testing.T) { require.Equal(t, result, filteredRes) } +func TestAlertingRuleExternalURLInTemplate(t *testing.T) { + suite, err := promql.NewTest(t, ` + load 1m + http_requests{job="app-server", instance="0"} 75 85 70 70 + `) + require.NoError(t, err) + defer suite.Close() + + require.NoError(t, suite.Run()) + + expr, err := parser.ParseExpr(`http_requests < 100`) + require.NoError(t, err) + + ruleWithoutExternalURL := NewAlertingRule( + "ExternalURLDoesNotExist", + expr, + time.Minute, + labels.FromStrings("templated_label", "The external URL is {{ $externalURL }}."), + nil, + nil, + "", + true, log.NewNopLogger(), + ) + ruleWithExternalURL := NewAlertingRule( + "ExternalURLExists", + expr, + time.Minute, + labels.FromStrings("templated_label", "The external URL is {{ $externalURL }}."), + nil, + nil, + "http://localhost:1234", + true, log.NewNopLogger(), + ) + result := promql.Vector{ + { + Metric: labels.FromStrings( + "__name__", "ALERTS", + "alertname", "ExternalURLDoesNotExist", + "alertstate", "pending", + "instance", "0", + "job", "app-server", + "templated_label", "The external URL is .", + ), + Point: promql.Point{V: 1}, + }, + { + Metric: labels.FromStrings( + "__name__", "ALERTS", + "alertname", "ExternalURLExists", + "alertstate", "pending", + "instance", "0", + "job", "app-server", + "templated_label", "The external URL is http://localhost:1234.", + ), + Point: promql.Point{V: 1}, + }, + } + + evalTime := time.Unix(0, 0) + result[0].Point.T = timestamp.FromTime(evalTime) + result[1].Point.T = timestamp.FromTime(evalTime) + + var filteredRes promql.Vector // After removing 'ALERTS_FOR_STATE' samples. + res, err := ruleWithoutExternalURL.Eval( + suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, + ) + require.NoError(t, err) + for _, smpl := range res { + smplName := smpl.Metric.Get("__name__") + if smplName == "ALERTS" { + filteredRes = append(filteredRes, smpl) + } else { + // If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'. + require.Equal(t, "ALERTS_FOR_STATE", smplName) + } + } + + res, err = ruleWithExternalURL.Eval( + suite.Context(), evalTime, EngineQueryFunc(suite.QueryEngine(), suite.Storage()), nil, + ) + require.NoError(t, err) + for _, smpl := range res { + smplName := smpl.Metric.Get("__name__") + if smplName == "ALERTS" { + filteredRes = append(filteredRes, smpl) + } else { + // If not 'ALERTS', it has to be 'ALERTS_FOR_STATE'. + require.Equal(t, "ALERTS_FOR_STATE", smplName) + } + } + + require.Equal(t, result, filteredRes) +} + func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) { suite, err := promql.NewTest(t, ` load 1m @@ -300,6 +396,7 @@ func TestAlertingRuleEmptyLabelFromTemplate(t *testing.T) { labels.FromStrings("empty_label", ""), nil, nil, + "", true, log.NewNopLogger(), ) result := promql.Vector{ @@ -360,6 +457,7 @@ func TestAlertingRuleDuplicate(t *testing.T) { labels.FromStrings("test", "test"), nil, nil, + "", true, log.NewNopLogger(), ) _, err := rule.Eval(ctx, now, EngineQueryFunc(engine, storage), nil) diff --git a/rules/manager.go b/rules/manager.go index 0b38ec187..0cbcaa74f 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -946,11 +946,11 @@ func (m *Manager) Stop() { // Update the rule manager's state as the config requires. If // loading the new rules failed the old rule set is restored. -func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels) error { +func (m *Manager) Update(interval time.Duration, files []string, externalLabels labels.Labels, externalURL string) error { m.mtx.Lock() defer m.mtx.Unlock() - groups, errs := m.LoadGroups(interval, externalLabels, files...) + groups, errs := m.LoadGroups(interval, externalLabels, externalURL, files...) if errs != nil { for _, e := range errs { level.Error(m.logger).Log("msg", "loading groups failed", "err", e) @@ -1034,7 +1034,7 @@ func (FileLoader) Parse(query string) (parser.Expr, error) { return parser.Parse // LoadGroups reads groups from a list of files. func (m *Manager) LoadGroups( - interval time.Duration, externalLabels labels.Labels, filenames ...string, + interval time.Duration, externalLabels labels.Labels, externalURL string, filenames ...string, ) (map[string]*Group, []error) { groups := make(map[string]*Group) @@ -1067,6 +1067,7 @@ func (m *Manager) LoadGroups( labels.FromMap(r.Labels), labels.FromMap(r.Annotations), externalLabels, + externalURL, m.restored, log.With(m.logger, "alert", r.Alert), )) diff --git a/rules/manager_test.go b/rules/manager_test.go index a6af68361..ab92eda16 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -64,7 +64,7 @@ func TestAlertingRule(t *testing.T) { expr, time.Minute, labels.FromStrings("severity", "{{\"c\"}}ritical"), - nil, nil, true, nil, + nil, nil, "", true, nil, ) result := promql.Vector{ { @@ -205,7 +205,7 @@ func TestForStateAddSamples(t *testing.T) { expr, time.Minute, labels.FromStrings("severity", "{{\"c\"}}ritical"), - nil, nil, true, nil, + nil, nil, "", true, nil, ) result := promql.Vector{ { @@ -379,7 +379,7 @@ func TestForStateRestore(t *testing.T) { expr, alertForDuration, labels.FromStrings("severity", "critical"), - nil, nil, true, nil, + nil, nil, "", true, nil, ) group := NewGroup(GroupOptions{ @@ -445,7 +445,7 @@ func TestForStateRestore(t *testing.T) { expr, alertForDuration, labels.FromStrings("severity", "critical"), - nil, nil, false, nil, + nil, nil, "", false, nil, ) newGroup := NewGroup(GroupOptions{ Name: "default", @@ -609,13 +609,13 @@ func readSeriesSet(ss storage.SeriesSet) (map[string][]promql.Point, error) { func TestCopyState(t *testing.T) { oldGroup := &Group{ rules: []Rule{ - NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil), + NewAlertingRule("alert", nil, 0, nil, nil, nil, "", true, nil), NewRecordingRule("rule1", nil, nil), NewRecordingRule("rule2", nil, nil), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v3"}}), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, "", true, nil), }, seriesInPreviousEval: []map[string]labels.Labels{ {}, @@ -634,10 +634,10 @@ func TestCopyState(t *testing.T) { NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v0"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v1"}}), NewRecordingRule("rule3", nil, labels.Labels{{Name: "l1", Value: "v2"}}), - NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil), + NewAlertingRule("alert", nil, 0, nil, nil, nil, "", true, nil), NewRecordingRule("rule1", nil, nil), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, true, nil), - NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v0"}}, nil, nil, "", true, nil), + NewAlertingRule("alert2", nil, 0, labels.Labels{{Name: "l2", Value: "v1"}}, nil, nil, "", true, nil), NewRecordingRule("rule4", nil, nil), }, seriesInPreviousEval: make([]map[string]labels.Labels, 8), @@ -725,7 +725,7 @@ func TestUpdate(t *testing.T) { ruleManager.start() defer ruleManager.Stop() - err := ruleManager.Update(10*time.Second, files, nil) + err := ruleManager.Update(10*time.Second, files, nil, "") require.NoError(t, err) require.Greater(t, len(ruleManager.groups), 0, "expected non-empty rule groups") ogs := map[string]*Group{} @@ -736,7 +736,7 @@ func TestUpdate(t *testing.T) { ogs[h] = g } - err = ruleManager.Update(10*time.Second, files, nil) + err = ruleManager.Update(10*time.Second, files, nil, "") require.NoError(t, err) for h, g := range ruleManager.groups { for _, actual := range g.seriesInPreviousEval { @@ -755,7 +755,7 @@ func TestUpdate(t *testing.T) { defer os.Remove(tmpFile.Name()) defer tmpFile.Close() - err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil) + err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil, "") require.NoError(t, err) for h, g := range ruleManager.groups { @@ -826,7 +826,7 @@ func reloadAndValidate(rgs *rulefmt.RuleGroups, t *testing.T, tmpFile *os.File, tmpFile.Seek(0, 0) _, err = tmpFile.Write(bs) require.NoError(t, err) - err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil) + err = ruleManager.Update(10*time.Second, []string{tmpFile.Name()}, nil, "") require.NoError(t, err) for h, g := range ruleManager.groups { if ogs[h] == g { @@ -862,7 +862,7 @@ func TestNotify(t *testing.T) { expr, err := parser.ParseExpr("a > 1") require.NoError(t, err) - rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, true, log.NewNopLogger()) + rule := NewAlertingRule("aTooHigh", expr, 0, labels.Labels{}, labels.Labels{}, nil, "", true, log.NewNopLogger()) group := NewGroup(GroupOptions{ Name: "alert", Interval: time.Second, @@ -971,7 +971,7 @@ func TestMetricsUpdate(t *testing.T) { } for i, c := range cases { - err := ruleManager.Update(time.Second, c.files, nil) + err := ruleManager.Update(time.Second, c.files, nil, "") require.NoError(t, err) time.Sleep(2 * time.Second) require.Equal(t, c.metrics, countMetrics(), "test %d: invalid count of metrics", i) @@ -1045,7 +1045,7 @@ func TestGroupStalenessOnRemoval(t *testing.T) { var totalStaleNaN int for i, c := range cases { - err := ruleManager.Update(time.Second, c.files, nil) + err := ruleManager.Update(time.Second, c.files, nil, "") require.NoError(t, err) time.Sleep(3 * time.Second) totalStaleNaN += c.staleNaN @@ -1087,11 +1087,11 @@ func TestMetricsStalenessOnManagerShutdown(t *testing.T) { } }() - err := ruleManager.Update(2*time.Second, files, nil) + err := ruleManager.Update(2*time.Second, files, nil, "") time.Sleep(4 * time.Second) require.NoError(t, err) start := time.Now() - err = ruleManager.Update(3*time.Second, files[:0], nil) + err = ruleManager.Update(3*time.Second, files[:0], nil, "") require.NoError(t, err) ruleManager.Stop() stopped = true @@ -1134,7 +1134,7 @@ func TestGroupHasAlertingRules(t *testing.T) { group: &Group{ name: "HasAlertingRule", rules: []Rule{ - NewAlertingRule("alert", nil, 0, nil, nil, nil, true, nil), + NewAlertingRule("alert", nil, 0, nil, nil, nil, "", true, nil), NewRecordingRule("record", nil, nil), }, }, diff --git a/template/template.go b/template/template.go index ae427738e..b0d8f5cc0 100644 --- a/template/template.go +++ b/template/template.go @@ -295,14 +295,16 @@ func NewTemplateExpander( } // AlertTemplateData returns the interface to be used in expanding the template. -func AlertTemplateData(labels map[string]string, externalLabels map[string]string, value float64) interface{} { +func AlertTemplateData(labels map[string]string, externalLabels map[string]string, externalURL string, value float64) interface{} { return struct { Labels map[string]string ExternalLabels map[string]string + ExternalURL string Value float64 }{ Labels: labels, ExternalLabels: externalLabels, + ExternalURL: externalURL, Value: value, } } diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 691f6539f..1f4a0e027 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -211,6 +211,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule { labels.Labels{}, labels.Labels{}, labels.Labels{}, + "", true, log.NewNopLogger(), ) @@ -221,6 +222,7 @@ func (m rulesRetrieverMock) AlertingRules() []*rules.AlertingRule { labels.Labels{}, labels.Labels{}, labels.Labels{}, + "", true, log.NewNopLogger(), )