Merge release-2.28 back into main (#9035)

* Cut v2.28.0-rc.0 (#8954)

* Cut v2.28.0-rc.0

Signed-off-by: Julius Volz <julius.volz@gmail.com>

* Changelog fixup

Signed-off-by: Julius Volz <julius.volz@gmail.com>

* Address review comments

Signed-off-by: Julius Volz <julius.volz@gmail.com>

* Downgrade some features to enhancements

Signed-off-by: Julius Volz <julius.volz@gmail.com>

* Adjust release date to today

Signed-off-by: Julius Volz <julius.volz@gmail.com>

* Migrate HTTP SD docs from docs repo (#8972)

See discussion in https://github.com/prometheus/docs/pull/1975

Signed-off-by: Julius Volz <julius.volz@gmail.com>

* Cut Prometheus v2.28.0 (#8973)

Signed-off-by: Julius Volz <julius.volz@gmail.com>

* HTTP SD: Allow charset in content type (#8981)

* Added content type regex

Signed-off-by: Levi Harrison <git@leviharrison.dev>
Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>

* fixed disappeared target groups in http_sd #9019

Signed-off-by: servak <fservak@gmail.com>

* Add a testcase for http-sd

Signed-off-by: servak <fservak@gmail.com>

* HTTP SD: Simplify logic of disappeared targetgroups (#9026)

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>

* Fix logging after the move to go-kit/log (#9021)

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>

* Cut Prometheus v2.28.1 (#9034)

Signed-off-by: Julius Volz <julius.volz@gmail.com>

Co-authored-by: Levi Harrison <git@leviharrison.dev>
Co-authored-by: Julien Pivotto <roidelapluie@inuits.eu>
Co-authored-by: servak <fservak@gmail.com>

.golangci.yml

@@ -22,6 +22,7 @@ linters-settings:
     packages-with-error-message:
       - sync/atomic: "Use go.uber.org/atomic instead of sync/atomic"
       - github.com/stretchr/testify/assert: "Use github.com/stretchr/testify/require instead of github.com/stretchr/testify/assert"
-      - github.com/go-kit/kit/log: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log"
+      # Temporarily commenting until a solution for removing it from klog is found.
+      # - github.com/go-kit/kit/log: "Use github.com/go-kit/log instead of github.com/go-kit/kit/log"
   errcheck:
     exclude: scripts/errcheck_excludes.txt

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.28.1 / 2021-07-01
+
+* [BUGFIX]: HTTP SD: Allow `charset` specification in `Content-Type` header. #8981
+* [BUGFIX]: HTTP SD: Fix handling of disappeared target groups. #9019
+* [BUGFIX]: Fix incorrect log-level handling after moving to go-kit/log. #9021
+
 ## 2.28.0 / 2021-06-21
 
 * [CHANGE] UI: Make the new experimental PromQL editor the default. #8925

cmd/prometheus/main.go

@@ -35,6 +35,7 @@ import (
 	"time"
 
 	"github.com/alecthomas/units"
+	kitloglevel "github.com/go-kit/kit/log/level"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	conntrack "github.com/mwitkow/go-conntrack"
@@ -417,11 +418,27 @@ func main() {
 	noStepSubqueryInterval := &safePromQLNoStepSubqueryInterval{}
 	noStepSubqueryInterval.Set(config.DefaultGlobalConfig.EvaluationInterval)
 
+	// FIXME: Temporary workaround until a proper solution is found. go-kit's
+	// level packages use private types to determine the level so we have to use
+	// the same level package as the underlying library.
+	var lvl kitloglevel.Option
+	switch cfg.promlogConfig.Level.String() {
+	case "debug":
+		lvl = kitloglevel.AllowDebug()
+	case "info":
+		lvl = kitloglevel.AllowInfo()
+	case "warn":
+		lvl = kitloglevel.AllowWarn()
+	case "error":
+		lvl = kitloglevel.AllowError()
+	}
+	kloglogger := kitloglevel.NewFilter(logger, lvl)
+
 	// Above level 6, the k8s client would log bearer tokens in clear-text.
 	klog.ClampLevel(6)
-	klog.SetLogger(log.With(logger, "component", "k8s_client_runtime"))
+	klog.SetLogger(log.With(kloglogger, "component", "k8s_client_runtime"))
 	klogv2.ClampLevel(6)
-	klogv2.SetLogger(log.With(logger, "component", "k8s_client_runtime"))
+	klogv2.SetLogger(log.With(kloglogger, "component", "k8s_client_runtime"))
 
 	level.Info(logger).Log("msg", "Starting Prometheus", "version", version.Info())
 	if bits.UintSize < 64 {

discovery/http/http.go

@@ -104,6 +104,7 @@ type Discovery struct {
 	url             string
 	client          *http.Client
 	refreshInterval time.Duration
+	tgLastLength    int
 }
 
 // NewDiscovery returns a new HTTP discovery for the given config.
@@ -183,6 +184,13 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 		tg.Labels[httpSDURLLabel] = model.LabelValue(d.url)
 	}
 
+	// Generate empty updates for sources that disappeared.
+	l := len(targetGroups)
+	for i := l; i < d.tgLastLength; i++ {
+		targetGroups = append(targetGroups, &targetgroup.Group{Source: urlSource(d.url, i)})
+	}
+	d.tgLastLength = l
+
 	return targetGroups, nil
 }
 

discovery/http/http_test.go

@@ -162,3 +162,241 @@ func TestContentTypeRegex(t *testing.T) {
 		})
 	}
 }
+
+func TestSourceDisappeared(t *testing.T) {
+	var stubResponse string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		fmt.Fprintln(w, stubResponse)
+	}))
+	t.Cleanup(ts.Close)
+
+	cases := []struct {
+		responses       []string
+		expectedTargets [][]*targetgroup.Group
+	}{
+		{
+			responses: []string{
+				`[]`,
+				`[]`,
+			},
+			expectedTargets: [][]*targetgroup.Group{{}, {}},
+		},
+		{
+			responses: []string{
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}]`,
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "2"}, "targets": ["127.0.0.1"]}]`,
+			},
+			expectedTargets: [][]*targetgroup.Group{
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+				},
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("2"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 1),
+					},
+				},
+			},
+		},
+		{
+			responses: []string{
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "2"}, "targets": ["127.0.0.1"]}]`,
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}]`,
+			},
+			expectedTargets: [][]*targetgroup.Group{
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("2"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 1),
+					},
+				},
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: nil,
+						Labels:  nil,
+						Source:  urlSource(ts.URL, 1),
+					},
+				},
+			},
+		},
+		{
+			responses: []string{
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "2"}, "targets": ["127.0.0.1"]}, {"labels": {"k": "3"}, "targets": ["127.0.0.1"]}]`,
+				`[{"labels": {"k": "1"}, "targets": ["127.0.0.1"]}]`,
+				`[{"labels": {"k": "v"}, "targets": ["127.0.0.2"]}, {"labels": {"k": "vv"}, "targets": ["127.0.0.3"]}]`,
+			},
+			expectedTargets: [][]*targetgroup.Group{
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("2"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 1),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("3"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 2),
+					},
+				},
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.1"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("1"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: nil,
+						Labels:  nil,
+						Source:  urlSource(ts.URL, 1),
+					},
+					{
+						Targets: nil,
+						Labels:  nil,
+						Source:  urlSource(ts.URL, 2),
+					},
+				},
+				{
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.2"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("v"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 0),
+					},
+					{
+						Targets: []model.LabelSet{
+							{
+								model.AddressLabel: model.LabelValue("127.0.0.3"),
+							},
+						},
+						Labels: model.LabelSet{
+							model.LabelName("k"):          model.LabelValue("vv"),
+							model.LabelName("__meta_url"): model.LabelValue(ts.URL),
+						},
+						Source: urlSource(ts.URL, 1),
+					},
+				},
+			},
+		},
+	}
+
+	cfg := SDConfig{
+		HTTPClientConfig: config.DefaultHTTPClientConfig,
+		URL:              ts.URL,
+		RefreshInterval:  model.Duration(1 * time.Second),
+	}
+	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	require.NoError(t, err)
+	for _, test := range cases {
+		ctx := context.Background()
+		for i, res := range test.responses {
+			stubResponse = res
+			tgs, err := d.refresh(ctx)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedTargets[i], tgs)
+		}
+	}
+
+}

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/docker/docker v20.10.7+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/edsrzf/mmap-go v1.0.0
+	github.com/go-kit/kit v0.10.0
 	github.com/go-kit/log v0.1.0
 	github.com/go-logfmt/logfmt v0.5.0
 	github.com/go-openapi/strfmt v0.20.1
@@ -49,7 +50,7 @@ require (
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.29.0
-	github.com/prometheus/exporter-toolkit v0.5.1
+	github.com/prometheus/exporter-toolkit v0.6.0
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546
@@ -61,7 +62,7 @@ require (
 	golang.org/x/net v0.0.0-20210610132358-84b48f89b13b
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210611083646-a4fc73990273
+	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
 	golang.org/x/tools v0.1.3
 	google.golang.org/api v0.48.0

go.sum

@@ -855,8 +855,9 @@ github.com/prometheus/common v0.23.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMD
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE=
 github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/exporter-toolkit v0.5.1 h1:9eqgis5er9xN613ZSADjypCJaDGj9ZlcWBvsIHa8/3c=
 github.com/prometheus/exporter-toolkit v0.5.1/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg=
+github.com/prometheus/exporter-toolkit v0.6.0 h1:rGoS9gIqj3sXaw+frvo0ozCs1CxBRqpOCGsbixC52UI=
+github.com/prometheus/exporter-toolkit v0.6.0/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -1033,8 +1034,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
+golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1239,8 +1241,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210611083646-a4fc73990273 h1:faDu4veV+8pcThn4fewv6TVlNCezafGoC1gM/mxQLbQ=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
-golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=