Merge pull request #11429 from prometheus/release-2.39

Merge back release branch

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 2.39.1 / 2022-10-07
+
+* [BUGFIX] Rules: Fix notifier relabel changing the labels on active alerts. #11427
+
 ## 2.39.0 / 2022-10-05
 
 * [FEATURE] **experimental** TSDB: Add support for ingesting out-of-order samples. This is configured via `out_of_order_time_window` field in the config file; check config file docs for more info. #11075

VERSION

@@ -1 +1 @@
-2.39.0
+2.39.1

cmd/prometheus/main.go

@@ -72,7 +72,6 @@ import (
 	"github.com/prometheus/prometheus/tsdb/agent"
 	"github.com/prometheus/prometheus/util/logging"
 	prom_runtime "github.com/prometheus/prometheus/util/runtime"
-	"github.com/prometheus/prometheus/util/strutil"
 	"github.com/prometheus/prometheus/web"
 )
 
@@ -619,7 +618,7 @@ func main() {
 			Appendable:      fanoutStorage,
 			Queryable:       localStorage,
 			QueryFunc:       rules.EngineQueryFunc(queryEngine, fanoutStorage),
-			NotifyFunc:      sendAlerts(notifierManager, cfg.web.ExternalURL.String()),
+			NotifyFunc:      rules.SendAlerts(notifierManager, cfg.web.ExternalURL.String()),
 			Context:         ctxRule,
 			ExternalURL:     cfg.web.ExternalURL,
 			Registerer:      prometheus.DefaultRegisterer,
@@ -1270,36 +1269,6 @@ func computeExternalURL(u, listenAddr string) (*url.URL, error) {
 	return eu, nil
 }
 
-type sender interface {
-	Send(alerts ...*notifier.Alert)
-}
-
-// sendAlerts implements the rules.NotifyFunc for a Notifier.
-func sendAlerts(s sender, externalURL string) rules.NotifyFunc {
-	return func(ctx context.Context, expr string, alerts ...*rules.Alert) {
-		var res []*notifier.Alert
-
-		for _, alert := range alerts {
-			a := &notifier.Alert{
-				StartsAt:     alert.FiredAt,
-				Labels:       alert.Labels,
-				Annotations:  alert.Annotations,
-				GeneratorURL: externalURL + strutil.TableLinkForExpression(expr),
-			}
-			if !alert.ResolvedAt.IsZero() {
-				a.EndsAt = alert.ResolvedAt
-			} else {
-				a.EndsAt = alert.ValidUntil
-			}
-			res = append(res, a)
-		}
-
-		if len(alerts) > 0 {
-			s.Send(res...)
-		}
-	}
-}
-
 // readyStorage implements the Storage interface while allowing to set the actual
 // storage at a later point in time.
 type readyStorage struct {

cmd/prometheus/main_test.go

@@ -198,7 +198,7 @@ func TestSendAlerts(t *testing.T) {
 				}
 				require.Equal(t, tc.exp, alerts)
 			})
-			sendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...)
+			rules.SendAlerts(senderFunc, "http://localhost:9090")(context.TODO(), "up", tc.in...)
 		})
 	}
 }

rules/alerting.go

@@ -507,6 +507,8 @@ func (r *AlertingRule) sendAlerts(ctx context.Context, ts time.Time, resendDelay
 			}
 			alert.ValidUntil = ts.Add(4 * delta)
 			anew := *alert
+			// The notifier re-uses the labels slice, hence make a copy.
+			anew.Labels = alert.Labels.Copy()
 			alerts = append(alerts, &anew)
 		}
 	})

rules/alerting_test.go

@@ -20,10 +20,13 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/model/labels"
+	"github.com/prometheus/prometheus/model/relabel"
 	"github.com/prometheus/prometheus/model/timestamp"
+	"github.com/prometheus/prometheus/notifier"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/prometheus/prometheus/storage"
@@ -659,3 +662,53 @@ func TestQueryForStateSeries(t *testing.T) {
 		testFunc(tst)
 	}
 }
+
+// TestSendAlertsDontAffectActiveAlerts tests a fix for https://github.com/prometheus/prometheus/issues/11424.
+func TestSendAlertsDontAffectActiveAlerts(t *testing.T) {
+	rule := NewAlertingRule(
+		"TestRule",
+		nil,
+		time.Minute,
+		labels.FromStrings("severity", "critical"),
+		labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
+	)
+
+	// Set an active alert.
+	lbls := labels.FromStrings("a1", "1")
+	h := lbls.Hash()
+	al := &Alert{State: StateFiring, Labels: lbls, ActiveAt: time.Now()}
+	rule.active[h] = al
+
+	expr, err := parser.ParseExpr("foo")
+	require.NoError(t, err)
+	rule.vector = expr
+
+	// The relabel rule reproduced the bug here.
+	opts := notifier.Options{
+		QueueCapacity: 1,
+		RelabelConfigs: []*relabel.Config{
+			{
+				SourceLabels: model.LabelNames{"a1"},
+				Regex:        relabel.MustNewRegexp("(.+)"),
+				TargetLabel:  "a1",
+				Replacement:  "bug",
+				Action:       "replace",
+			},
+		},
+	}
+	nm := notifier.NewManager(&opts, log.NewNopLogger())
+
+	f := SendAlerts(nm, "")
+	notifyFunc := func(ctx context.Context, expr string, alerts ...*Alert) {
+		require.Len(t, alerts, 1)
+		require.Equal(t, al, alerts[0])
+		f(ctx, expr, alerts...)
+	}
+
+	rule.sendAlerts(context.Background(), time.Now(), 0, 0, notifyFunc)
+	nm.Stop()
+
+	// The relabel rule changes a1=1 to a1=bug.
+	// But the labels with the AlertingRule should not be changed.
+	require.Equal(t, labels.FromStrings("a1", "1"), rule.active[h].Labels)
+}

rules/manager.go

@@ -35,9 +35,11 @@ import (
 	"github.com/prometheus/prometheus/model/rulefmt"
 	"github.com/prometheus/prometheus/model/timestamp"
 	"github.com/prometheus/prometheus/model/value"
+	"github.com/prometheus/prometheus/notifier"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
 	"github.com/prometheus/prometheus/storage"
+	"github.com/prometheus/prometheus/util/strutil"
 )
 
 // RuleHealth describes the health state of a rule.
@@ -1168,3 +1170,33 @@ func (m *Manager) AlertingRules() []*AlertingRule {
 
 	return alerts
 }
+
+type Sender interface {
+	Send(alerts ...*notifier.Alert)
+}
+
+// SendAlerts implements the rules.NotifyFunc for a Notifier.
+func SendAlerts(s Sender, externalURL string) NotifyFunc {
+	return func(ctx context.Context, expr string, alerts ...*Alert) {
+		var res []*notifier.Alert
+
+		for _, alert := range alerts {
+			a := &notifier.Alert{
+				StartsAt:     alert.FiredAt,
+				Labels:       alert.Labels,
+				Annotations:  alert.Annotations,
+				GeneratorURL: externalURL + strutil.TableLinkForExpression(expr),
+			}
+			if !alert.ResolvedAt.IsZero() {
+				a.EndsAt = alert.ResolvedAt
+			} else {
+				a.EndsAt = alert.ValidUntil
+			}
+			res = append(res, a)
+		}
+
+		if len(alerts) > 0 {
+			s.Send(res...)
+		}
+	}
+}

web/ui/module/codemirror-promql/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/codemirror-promql",
-  "version": "0.39.0",
+  "version": "0.39.1",
   "description": "a CodeMirror mode for the PromQL language",
   "types": "dist/esm/index.d.ts",
   "module": "dist/esm/index.js",
@@ -29,7 +29,7 @@
   },
   "homepage": "https://github.com/prometheus/prometheus/blob/main/web/ui/module/codemirror-promql/README.md",
   "dependencies": {
-    "@prometheus-io/lezer-promql": "^0.39.0",
+    "@prometheus-io/lezer-promql": "^0.39.1",
     "lru-cache": "^6.0.0"
   },
   "devDependencies": {

web/ui/module/lezer-promql/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/lezer-promql",
-  "version": "0.39.0",
+  "version": "0.39.1",
   "description": "lezer-based PromQL grammar",
   "main": "index.cjs",
   "type": "module",

web/ui/package-lock.json

@@ -28,10 +28,10 @@
     },
     "module/codemirror-promql": {
       "name": "@prometheus-io/codemirror-promql",
-      "version": "0.39.0",
+      "version": "0.39.1",
       "license": "Apache-2.0",
       "dependencies": {
-        "@prometheus-io/lezer-promql": "^0.39.0",
+        "@prometheus-io/lezer-promql": "^0.39.1",
         "lru-cache": "^6.0.0"
       },
       "devDependencies": {
@@ -61,7 +61,7 @@
     },
     "module/lezer-promql": {
       "name": "@prometheus-io/lezer-promql",
-      "version": "0.39.0",
+      "version": "0.39.1",
       "license": "Apache-2.0",
       "devDependencies": {
         "@lezer/generator": "^1.1.1",
@@ -17625,7 +17625,7 @@
     },
     "react-app": {
       "name": "@prometheus-io/app",
-      "version": "0.39.0",
+      "version": "0.39.1",
       "dependencies": {
         "@codemirror/autocomplete": "^6.2.0",
         "@codemirror/commands": "^6.1.0",
@@ -17643,7 +17643,7 @@
         "@lezer/lr": "^1.2.3",
         "@nexucis/fuzzy": "^0.4.1",
         "@nexucis/kvsearch": "^0.8.1",
-        "@prometheus-io/codemirror-promql": "^0.39.0",
+        "@prometheus-io/codemirror-promql": "^0.39.1",
         "bootstrap": "^4.6.2",
         "css.escape": "^1.5.1",
         "downshift": "^6.1.11",
@@ -19883,7 +19883,7 @@
         "@lezer/lr": "^1.2.3",
         "@nexucis/fuzzy": "^0.4.1",
         "@nexucis/kvsearch": "^0.8.1",
-        "@prometheus-io/codemirror-promql": "^0.39.0",
+        "@prometheus-io/codemirror-promql": "^0.39.1",
         "@testing-library/react-hooks": "^7.0.2",
         "@types/enzyme": "^3.10.12",
         "@types/flot": "0.0.32",
@@ -19935,7 +19935,7 @@
         "@lezer/common": "^1.0.1",
         "@lezer/highlight": "^1.1.0",
         "@lezer/lr": "^1.2.3",
-        "@prometheus-io/lezer-promql": "^0.39.0",
+        "@prometheus-io/lezer-promql": "^0.39.1",
         "@types/lru-cache": "^5.1.1",
         "isomorphic-fetch": "^3.0.0",
         "lru-cache": "^6.0.0",

web/ui/react-app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prometheus-io/app",
-  "version": "0.39.0",
+  "version": "0.39.1",
   "private": true,
   "dependencies": {
     "@codemirror/autocomplete": "^6.2.0",
@@ -19,7 +19,7 @@
     "@lezer/common": "^1.0.1",
     "@nexucis/fuzzy": "^0.4.1",
     "@nexucis/kvsearch": "^0.8.1",
-    "@prometheus-io/codemirror-promql": "^0.39.0",
+    "@prometheus-io/codemirror-promql": "^0.39.1",
     "bootstrap": "^4.6.2",
     "css.escape": "^1.5.1",
     "downshift": "^6.1.11",