2016-04-13 14:08:22 +00:00
|
|
|
// Copyright 2016 The Prometheus Authors
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2015-07-03 12:47:52 +00:00
|
|
|
package web
|
|
|
|
|
|
|
|
import (
|
2017-08-10 14:28:49 +00:00
|
|
|
"context"
|
2020-08-29 10:22:00 +00:00
|
|
|
"encoding/json"
|
2018-01-13 10:37:38 +00:00
|
|
|
"fmt"
|
2020-08-29 10:22:00 +00:00
|
|
|
"io"
|
2020-08-17 08:50:32 +00:00
|
|
|
"net"
|
2017-07-26 00:47:45 +00:00
|
|
|
"net/http"
|
2017-09-01 11:12:51 +00:00
|
|
|
"net/http/httptest"
|
2015-07-03 12:47:52 +00:00
|
|
|
"net/url"
|
2017-10-05 05:38:07 +00:00
|
|
|
"os"
|
2020-08-29 10:22:00 +00:00
|
|
|
"path/filepath"
|
2019-06-19 15:56:04 +00:00
|
|
|
"strconv"
|
2017-10-05 05:38:07 +00:00
|
|
|
"strings"
|
2020-11-09 23:09:39 +00:00
|
|
|
"sync"
|
2015-07-03 12:47:52 +00:00
|
|
|
"testing"
|
2017-07-26 00:47:45 +00:00
|
|
|
"time"
|
2017-09-18 10:32:17 +00:00
|
|
|
|
2019-06-24 13:48:15 +00:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
2019-06-19 15:56:04 +00:00
|
|
|
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
|
2020-10-29 09:43:23 +00:00
|
|
|
"github.com/stretchr/testify/require"
|
2020-10-22 09:00:08 +00:00
|
|
|
|
2019-02-08 10:17:47 +00:00
|
|
|
"github.com/prometheus/prometheus/config"
|
|
|
|
"github.com/prometheus/prometheus/notifier"
|
|
|
|
"github.com/prometheus/prometheus/rules"
|
|
|
|
"github.com/prometheus/prometheus/scrape"
|
2020-02-06 15:58:38 +00:00
|
|
|
"github.com/prometheus/prometheus/tsdb"
|
2021-10-15 10:25:29 +00:00
|
|
|
"github.com/prometheus/prometheus/util/testutil"
|
2015-07-03 12:47:52 +00:00
|
|
|
)
|
|
|
|
|
2017-12-20 18:21:10 +00:00
|
|
|
func TestMain(m *testing.M) {
|
|
|
|
// On linux with a global proxy the tests will fail as the go client(http,grpc) tries to connect through the proxy.
|
|
|
|
os.Setenv("no_proxy", "localhost,127.0.0.1,0.0.0.0,:")
|
|
|
|
os.Exit(m.Run())
|
|
|
|
}
|
2020-04-29 16:16:14 +00:00
|
|
|
|
|
|
|
type dbAdapter struct {
|
|
|
|
*tsdb.DB
|
|
|
|
}
|
|
|
|
|
2023-05-22 12:37:07 +00:00
|
|
|
func (a *dbAdapter) Stats(statsByLabelName string, limit int) (*tsdb.Stats, error) {
|
|
|
|
return a.Head().Stats(statsByLabelName, limit), nil
|
2020-04-29 16:16:14 +00:00
|
|
|
}
|
|
|
|
|
2021-06-05 14:29:32 +00:00
|
|
|
func (a *dbAdapter) WALReplayStatus() (tsdb.WALReplayStatus, error) {
|
|
|
|
return tsdb.WALReplayStatus{}, nil
|
|
|
|
}
|
|
|
|
|
2017-07-26 00:47:45 +00:00
|
|
|
func TestReadyAndHealthy(t *testing.T) {
|
2017-09-06 12:41:07 +00:00
|
|
|
t.Parallel()
|
2020-05-06 15:30:00 +00:00
|
|
|
|
2021-12-08 22:14:50 +00:00
|
|
|
dbDir := t.TempDir()
|
2017-10-05 20:40:10 +00:00
|
|
|
|
2021-06-05 14:29:32 +00:00
|
|
|
db, err := tsdb.Open(dbDir, nil, nil, nil, nil)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
2021-12-20 12:07:53 +00:00
|
|
|
t.Cleanup(func() {
|
|
|
|
require.NoError(t, db.Close())
|
|
|
|
})
|
2021-10-15 10:25:29 +00:00
|
|
|
port := fmt.Sprintf(":%d", testutil.RandomUnprivilegedPort(t))
|
|
|
|
|
2017-07-26 00:47:45 +00:00
|
|
|
opts := &Options{
|
2024-08-13 12:34:26 +00:00
|
|
|
ListenAddresses: []string{port},
|
|
|
|
ReadTimeout: 30 * time.Second,
|
|
|
|
MaxConnections: 512,
|
|
|
|
Context: nil,
|
|
|
|
Storage: nil,
|
|
|
|
LocalStorage: &dbAdapter{db},
|
|
|
|
TSDBDir: dbDir,
|
|
|
|
QueryEngine: nil,
|
|
|
|
ScrapeManager: &scrape.Manager{},
|
|
|
|
RuleManager: &rules.Manager{},
|
|
|
|
Notifier: nil,
|
|
|
|
RoutePrefix: "/",
|
|
|
|
EnableAdminAPI: true,
|
2018-09-19 07:20:53 +00:00
|
|
|
ExternalURL: &url.URL{
|
|
|
|
Scheme: "http",
|
2021-10-15 10:25:29 +00:00
|
|
|
Host: "localhost" + port,
|
2018-09-19 07:20:53 +00:00
|
|
|
Path: "/",
|
|
|
|
},
|
2019-06-24 13:48:15 +00:00
|
|
|
Version: &PrometheusVersion{},
|
|
|
|
Gatherer: prometheus.DefaultGatherer,
|
2017-07-26 00:47:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
opts.Flags = map[string]string{}
|
|
|
|
|
2017-08-11 18:45:52 +00:00
|
|
|
webHandler := New(nil, opts)
|
2019-02-08 10:17:47 +00:00
|
|
|
|
|
|
|
webHandler.config = &config.Config{}
|
|
|
|
webHandler.notifier = ¬ifier.Manager{}
|
2024-08-13 12:34:26 +00:00
|
|
|
l, err := webHandler.Listeners()
|
2020-12-03 14:33:16 +00:00
|
|
|
if err != nil {
|
2024-08-13 12:34:26 +00:00
|
|
|
panic(fmt.Sprintf("Unable to start web listeners: %s", err))
|
2020-12-03 14:33:16 +00:00
|
|
|
}
|
2019-02-08 10:17:47 +00:00
|
|
|
|
2020-08-17 08:50:32 +00:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
defer cancel()
|
2017-12-14 13:45:36 +00:00
|
|
|
go func() {
|
2020-12-25 11:45:31 +00:00
|
|
|
err := webHandler.Run(ctx, l, "")
|
2017-12-14 13:45:36 +00:00
|
|
|
if err != nil {
|
2018-01-13 10:37:38 +00:00
|
|
|
panic(fmt.Sprintf("Can't start web handler:%s", err))
|
2017-12-14 13:45:36 +00:00
|
|
|
}
|
|
|
|
}()
|
2017-07-26 00:47:45 +00:00
|
|
|
|
|
|
|
// Give some time for the web goroutine to run since we need the server
|
|
|
|
// to be up before starting tests.
|
|
|
|
time.Sleep(5 * time.Second)
|
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
baseURL := "http://localhost" + port
|
|
|
|
|
|
|
|
resp, err := http.Get(baseURL + "/-/healthy")
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
|
|
|
|
2022-08-16 19:06:26 +00:00
|
|
|
resp, err = http.Head(baseURL + "/-/healthy")
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
|
|
|
cleanupTestResponse(t, resp)
|
|
|
|
|
2020-08-29 10:22:00 +00:00
|
|
|
for _, u := range []string{
|
2021-10-15 10:25:29 +00:00
|
|
|
baseURL + "/-/ready",
|
2020-08-29 10:22:00 +00:00
|
|
|
} {
|
|
|
|
resp, err = http.Get(u)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2022-08-16 19:06:26 +00:00
|
|
|
|
|
|
|
resp, err = http.Head(u)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
|
|
|
cleanupTestResponse(t, resp)
|
2020-08-29 10:22:00 +00:00
|
|
|
}
|
2018-09-19 07:20:53 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Post(baseURL+"/api/v1/admin/tsdb/snapshot", "", strings.NewReader(""))
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-10-05 05:38:07 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Post(baseURL+"/api/v1/admin/tsdb/delete_series", "", strings.NewReader("{}"))
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2019-02-08 10:17:47 +00:00
|
|
|
|
2017-07-26 00:47:45 +00:00
|
|
|
// Set to ready.
|
2024-09-16 15:17:50 +00:00
|
|
|
webHandler.SetReady(Ready)
|
2017-07-26 00:47:45 +00:00
|
|
|
|
2020-08-29 10:22:00 +00:00
|
|
|
for _, u := range []string{
|
2021-10-15 10:25:29 +00:00
|
|
|
baseURL + "/-/healthy",
|
|
|
|
baseURL + "/-/ready",
|
2020-08-29 10:22:00 +00:00
|
|
|
} {
|
|
|
|
resp, err = http.Get(u)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2022-08-16 19:06:26 +00:00
|
|
|
|
|
|
|
resp, err = http.Head(u)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
|
|
|
cleanupTestResponse(t, resp)
|
2020-08-29 10:22:00 +00:00
|
|
|
}
|
2018-09-19 07:20:53 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Post(baseURL+"/api/v1/admin/tsdb/snapshot", "", strings.NewReader(""))
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
2020-09-15 07:30:55 +00:00
|
|
|
cleanupSnapshot(t, dbDir, resp)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-10-05 05:38:07 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Post(baseURL+"/api/v1/admin/tsdb/delete_series?match[]=up", "", nil)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusNoContent, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-09-06 12:41:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestRoutePrefix(t *testing.T) {
|
|
|
|
t.Parallel()
|
2021-12-08 22:14:50 +00:00
|
|
|
dbDir := t.TempDir()
|
2017-10-05 20:40:10 +00:00
|
|
|
|
2021-06-05 14:29:32 +00:00
|
|
|
db, err := tsdb.Open(dbDir, nil, nil, nil, nil)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
2021-12-20 12:07:53 +00:00
|
|
|
t.Cleanup(func() {
|
|
|
|
require.NoError(t, db.Close())
|
|
|
|
})
|
2017-10-05 05:38:07 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
port := fmt.Sprintf(":%d", testutil.RandomUnprivilegedPort(t))
|
|
|
|
|
2017-09-06 12:41:07 +00:00
|
|
|
opts := &Options{
|
2024-08-13 12:34:26 +00:00
|
|
|
ListenAddresses: []string{port},
|
|
|
|
ReadTimeout: 30 * time.Second,
|
|
|
|
MaxConnections: 512,
|
|
|
|
Context: nil,
|
|
|
|
TSDBDir: dbDir,
|
|
|
|
LocalStorage: &dbAdapter{db},
|
|
|
|
Storage: nil,
|
|
|
|
QueryEngine: nil,
|
|
|
|
ScrapeManager: nil,
|
|
|
|
RuleManager: nil,
|
|
|
|
Notifier: nil,
|
|
|
|
RoutePrefix: "/prometheus",
|
|
|
|
EnableAdminAPI: true,
|
2020-02-17 17:19:15 +00:00
|
|
|
ExternalURL: &url.URL{
|
2021-10-15 10:25:29 +00:00
|
|
|
Host: "localhost.localdomain" + port,
|
2020-02-17 17:19:15 +00:00
|
|
|
Scheme: "http",
|
|
|
|
},
|
2017-09-06 12:41:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
opts.Flags = map[string]string{}
|
|
|
|
|
|
|
|
webHandler := New(nil, opts)
|
2024-08-13 12:34:26 +00:00
|
|
|
l, err := webHandler.Listeners()
|
2020-12-03 14:33:16 +00:00
|
|
|
if err != nil {
|
2024-08-13 12:34:26 +00:00
|
|
|
panic(fmt.Sprintf("Unable to start web listeners: %s", err))
|
2020-12-03 14:33:16 +00:00
|
|
|
}
|
2020-08-17 08:50:32 +00:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
defer cancel()
|
2017-09-06 12:41:07 +00:00
|
|
|
go func() {
|
2020-12-25 11:45:31 +00:00
|
|
|
err := webHandler.Run(ctx, l, "")
|
2017-09-06 12:41:07 +00:00
|
|
|
if err != nil {
|
2018-01-13 10:37:38 +00:00
|
|
|
panic(fmt.Sprintf("Can't start web handler:%s", err))
|
2017-09-06 12:41:07 +00:00
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
// Give some time for the web goroutine to run since we need the server
|
|
|
|
// to be up before starting tests.
|
|
|
|
time.Sleep(5 * time.Second)
|
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
baseURL := "http://localhost" + port
|
|
|
|
|
|
|
|
resp, err := http.Get(baseURL + opts.RoutePrefix + "/-/healthy")
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-09-06 12:41:07 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Get(baseURL + opts.RoutePrefix + "/-/ready")
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-09-06 12:41:07 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Post(baseURL+opts.RoutePrefix+"/api/v1/admin/tsdb/snapshot", "", strings.NewReader(""))
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-10-05 05:38:07 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Post(baseURL+opts.RoutePrefix+"/api/v1/admin/tsdb/delete_series", "", strings.NewReader("{}"))
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-09-06 12:41:07 +00:00
|
|
|
|
|
|
|
// Set to ready.
|
2024-09-16 15:17:50 +00:00
|
|
|
webHandler.SetReady(Ready)
|
2017-09-06 12:41:07 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Get(baseURL + opts.RoutePrefix + "/-/healthy")
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-07-26 00:47:45 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Get(baseURL + opts.RoutePrefix + "/-/ready")
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-09-06 12:41:07 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Post(baseURL+opts.RoutePrefix+"/api/v1/admin/tsdb/snapshot", "", strings.NewReader(""))
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
2020-09-15 07:30:55 +00:00
|
|
|
cleanupSnapshot(t, dbDir, resp)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-10-05 05:38:07 +00:00
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err = http.Post(baseURL+opts.RoutePrefix+"/api/v1/admin/tsdb/delete_series?match[]=up", "", nil)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusNoContent, resp.StatusCode)
|
2020-08-29 10:22:00 +00:00
|
|
|
cleanupTestResponse(t, resp)
|
2017-07-26 00:47:45 +00:00
|
|
|
}
|
2018-11-15 13:22:16 +00:00
|
|
|
|
2017-09-01 11:12:51 +00:00
|
|
|
func TestDebugHandler(t *testing.T) {
|
|
|
|
for _, tc := range []struct {
|
|
|
|
prefix, url string
|
|
|
|
code int
|
|
|
|
}{
|
|
|
|
{"/", "/debug/pprof/cmdline", 200},
|
|
|
|
{"/foo", "/foo/debug/pprof/cmdline", 200},
|
|
|
|
|
|
|
|
{"/", "/debug/pprof/goroutine", 200},
|
|
|
|
{"/foo", "/foo/debug/pprof/goroutine", 200},
|
|
|
|
|
|
|
|
{"/", "/debug/pprof/foo", 404},
|
|
|
|
{"/foo", "/bar/debug/pprof/goroutine", 404},
|
|
|
|
} {
|
|
|
|
opts := &Options{
|
2024-08-13 12:34:26 +00:00
|
|
|
RoutePrefix: tc.prefix,
|
|
|
|
ListenAddresses: []string{"somehost:9090"},
|
2020-02-17 17:19:15 +00:00
|
|
|
ExternalURL: &url.URL{
|
|
|
|
Host: "localhost.localdomain:9090",
|
|
|
|
Scheme: "http",
|
|
|
|
},
|
2017-09-01 11:12:51 +00:00
|
|
|
}
|
2017-10-05 10:16:15 +00:00
|
|
|
handler := New(nil, opts)
|
2024-09-16 15:17:50 +00:00
|
|
|
handler.SetReady(Ready)
|
2017-09-01 11:12:51 +00:00
|
|
|
|
|
|
|
w := httptest.NewRecorder()
|
2017-10-05 20:40:10 +00:00
|
|
|
|
2024-04-08 19:26:23 +00:00
|
|
|
req, err := http.NewRequest(http.MethodGet, tc.url, nil)
|
2017-10-05 20:40:10 +00:00
|
|
|
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
2017-09-01 11:12:51 +00:00
|
|
|
|
|
|
|
handler.router.ServeHTTP(w, req)
|
2017-10-05 20:40:10 +00:00
|
|
|
|
2020-10-29 09:43:23 +00:00
|
|
|
require.Equal(t, tc.code, w.Code)
|
2017-09-01 11:12:51 +00:00
|
|
|
}
|
|
|
|
}
|
2019-06-19 15:56:04 +00:00
|
|
|
|
|
|
|
func TestHTTPMetrics(t *testing.T) {
|
|
|
|
t.Parallel()
|
2020-02-17 17:19:15 +00:00
|
|
|
handler := New(nil, &Options{
|
2024-08-13 12:34:26 +00:00
|
|
|
RoutePrefix: "/",
|
|
|
|
ListenAddresses: []string{"somehost:9090"},
|
2020-02-17 17:19:15 +00:00
|
|
|
ExternalURL: &url.URL{
|
|
|
|
Host: "localhost.localdomain:9090",
|
|
|
|
Scheme: "http",
|
|
|
|
},
|
|
|
|
})
|
2019-06-19 15:56:04 +00:00
|
|
|
getReady := func() int {
|
|
|
|
t.Helper()
|
|
|
|
w := httptest.NewRecorder()
|
|
|
|
|
2024-04-08 19:26:23 +00:00
|
|
|
req, err := http.NewRequest(http.MethodGet, "/-/ready", nil)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
2019-06-19 15:56:04 +00:00
|
|
|
|
|
|
|
handler.router.ServeHTTP(w, req)
|
|
|
|
return w.Code
|
|
|
|
}
|
|
|
|
|
|
|
|
code := getReady()
|
2020-10-29 09:43:23 +00:00
|
|
|
require.Equal(t, http.StatusServiceUnavailable, code)
|
2022-05-23 14:00:59 +00:00
|
|
|
ready := handler.metrics.readyStatus
|
|
|
|
require.Equal(t, 0, int(prom_testutil.ToFloat64(ready)))
|
2019-06-24 13:48:15 +00:00
|
|
|
counter := handler.metrics.requestCounter
|
2020-10-29 09:43:23 +00:00
|
|
|
require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
|
2019-06-19 15:56:04 +00:00
|
|
|
|
2024-09-16 15:17:50 +00:00
|
|
|
handler.SetReady(Ready)
|
2019-06-19 15:56:04 +00:00
|
|
|
for range [2]int{} {
|
|
|
|
code = getReady()
|
2020-10-29 09:43:23 +00:00
|
|
|
require.Equal(t, http.StatusOK, code)
|
2019-06-19 15:56:04 +00:00
|
|
|
}
|
2022-05-23 14:00:59 +00:00
|
|
|
require.Equal(t, 1, int(prom_testutil.ToFloat64(ready)))
|
2020-10-29 09:43:23 +00:00
|
|
|
require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
|
|
|
|
require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
|
2022-05-23 14:00:59 +00:00
|
|
|
|
2024-09-16 15:17:50 +00:00
|
|
|
handler.SetReady(NotReady)
|
2022-05-23 14:00:59 +00:00
|
|
|
for range [2]int{} {
|
|
|
|
code = getReady()
|
|
|
|
require.Equal(t, http.StatusServiceUnavailable, code)
|
|
|
|
}
|
|
|
|
require.Equal(t, 0, int(prom_testutil.ToFloat64(ready)))
|
|
|
|
require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
|
|
|
|
require.Equal(t, 3, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
|
2019-06-19 15:56:04 +00:00
|
|
|
}
|
2020-08-17 08:50:32 +00:00
|
|
|
|
|
|
|
func TestShutdownWithStaleConnection(t *testing.T) {
|
2021-12-08 22:14:50 +00:00
|
|
|
dbDir := t.TempDir()
|
2020-08-17 08:50:32 +00:00
|
|
|
|
2021-06-05 14:29:32 +00:00
|
|
|
db, err := tsdb.Open(dbDir, nil, nil, nil, nil)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
2021-12-20 12:07:53 +00:00
|
|
|
t.Cleanup(func() {
|
|
|
|
require.NoError(t, db.Close())
|
|
|
|
})
|
2020-08-17 08:50:32 +00:00
|
|
|
timeout := 10 * time.Second
|
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
port := fmt.Sprintf(":%d", testutil.RandomUnprivilegedPort(t))
|
|
|
|
|
2020-08-17 08:50:32 +00:00
|
|
|
opts := &Options{
|
2024-08-13 12:34:26 +00:00
|
|
|
ListenAddresses: []string{port},
|
|
|
|
ReadTimeout: timeout,
|
|
|
|
MaxConnections: 512,
|
|
|
|
Context: nil,
|
|
|
|
Storage: nil,
|
|
|
|
LocalStorage: &dbAdapter{db},
|
|
|
|
TSDBDir: dbDir,
|
|
|
|
QueryEngine: nil,
|
|
|
|
ScrapeManager: &scrape.Manager{},
|
|
|
|
RuleManager: &rules.Manager{},
|
|
|
|
Notifier: nil,
|
|
|
|
RoutePrefix: "/",
|
2020-08-17 08:50:32 +00:00
|
|
|
ExternalURL: &url.URL{
|
|
|
|
Scheme: "http",
|
2021-10-15 10:25:29 +00:00
|
|
|
Host: "localhost" + port,
|
2020-08-17 08:50:32 +00:00
|
|
|
Path: "/",
|
|
|
|
},
|
|
|
|
Version: &PrometheusVersion{},
|
|
|
|
Gatherer: prometheus.DefaultGatherer,
|
|
|
|
}
|
|
|
|
|
|
|
|
opts.Flags = map[string]string{}
|
|
|
|
|
|
|
|
webHandler := New(nil, opts)
|
|
|
|
|
|
|
|
webHandler.config = &config.Config{}
|
|
|
|
webHandler.notifier = ¬ifier.Manager{}
|
2024-08-13 12:34:26 +00:00
|
|
|
l, err := webHandler.Listeners()
|
2020-12-03 14:33:16 +00:00
|
|
|
if err != nil {
|
2024-08-13 12:34:26 +00:00
|
|
|
panic(fmt.Sprintf("Unable to start web listeners: %s", err))
|
2020-12-03 14:33:16 +00:00
|
|
|
}
|
2020-08-17 08:50:32 +00:00
|
|
|
|
|
|
|
closed := make(chan struct{})
|
|
|
|
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
go func() {
|
2020-12-25 11:45:31 +00:00
|
|
|
err := webHandler.Run(ctx, l, "")
|
2020-08-17 08:50:32 +00:00
|
|
|
if err != nil {
|
|
|
|
panic(fmt.Sprintf("Can't start web handler:%s", err))
|
|
|
|
}
|
|
|
|
close(closed)
|
|
|
|
}()
|
|
|
|
|
|
|
|
// Give some time for the web goroutine to run since we need the server
|
|
|
|
// to be up before starting tests.
|
|
|
|
time.Sleep(5 * time.Second)
|
|
|
|
|
|
|
|
// Open a socket, and don't use it. This connection should then be closed
|
|
|
|
// after the ReadTimeout.
|
2021-10-15 10:25:29 +00:00
|
|
|
c, err := net.Dial("tcp", opts.ExternalURL.Host)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
t.Cleanup(func() { require.NoError(t, c.Close()) })
|
2020-08-17 08:50:32 +00:00
|
|
|
|
|
|
|
// Stop the web handler.
|
|
|
|
cancel()
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-closed:
|
|
|
|
case <-time.After(timeout + 5*time.Second):
|
2021-09-03 09:51:27 +00:00
|
|
|
require.FailNow(t, "Server still running after read timeout.")
|
2020-08-17 08:50:32 +00:00
|
|
|
}
|
|
|
|
}
|
2020-08-29 10:22:00 +00:00
|
|
|
|
2020-11-09 23:09:39 +00:00
|
|
|
func TestHandleMultipleQuitRequests(t *testing.T) {
|
2021-10-15 10:25:29 +00:00
|
|
|
port := fmt.Sprintf(":%d", testutil.RandomUnprivilegedPort(t))
|
|
|
|
|
2020-11-09 23:09:39 +00:00
|
|
|
opts := &Options{
|
2024-08-13 12:34:26 +00:00
|
|
|
ListenAddresses: []string{port},
|
2020-11-09 23:09:39 +00:00
|
|
|
MaxConnections: 512,
|
|
|
|
EnableLifecycle: true,
|
|
|
|
RoutePrefix: "/",
|
|
|
|
ExternalURL: &url.URL{
|
|
|
|
Scheme: "http",
|
2021-10-15 10:25:29 +00:00
|
|
|
Host: "localhost" + port,
|
2020-11-09 23:09:39 +00:00
|
|
|
Path: "/",
|
|
|
|
},
|
|
|
|
}
|
|
|
|
webHandler := New(nil, opts)
|
|
|
|
webHandler.config = &config.Config{}
|
|
|
|
webHandler.notifier = ¬ifier.Manager{}
|
2024-08-13 12:34:26 +00:00
|
|
|
l, err := webHandler.Listeners()
|
2020-12-03 14:33:16 +00:00
|
|
|
if err != nil {
|
2024-08-13 12:34:26 +00:00
|
|
|
panic(fmt.Sprintf("Unable to start web listeners: %s", err))
|
2020-12-03 14:33:16 +00:00
|
|
|
}
|
2020-11-09 23:09:39 +00:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
closed := make(chan struct{})
|
|
|
|
go func() {
|
2020-12-25 11:45:31 +00:00
|
|
|
err := webHandler.Run(ctx, l, "")
|
2020-11-09 23:09:39 +00:00
|
|
|
if err != nil {
|
|
|
|
panic(fmt.Sprintf("Can't start web handler:%s", err))
|
|
|
|
}
|
|
|
|
close(closed)
|
|
|
|
}()
|
|
|
|
|
|
|
|
// Give some time for the web goroutine to run since we need the server
|
|
|
|
// to be up before starting tests.
|
|
|
|
time.Sleep(5 * time.Second)
|
|
|
|
|
2021-10-15 10:25:29 +00:00
|
|
|
baseURL := opts.ExternalURL.Scheme + "://" + opts.ExternalURL.Host
|
|
|
|
|
2020-11-09 23:09:39 +00:00
|
|
|
start := make(chan struct{})
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
for i := 0; i < 3; i++ {
|
|
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
|
|
|
<-start
|
2021-10-15 10:25:29 +00:00
|
|
|
resp, err := http.Post(baseURL+"/-/quit", "", strings.NewReader(""))
|
2020-11-09 23:09:39 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
|
|
|
}()
|
|
|
|
}
|
|
|
|
close(start)
|
|
|
|
wg.Wait()
|
|
|
|
|
|
|
|
// Stop the web handler.
|
|
|
|
cancel()
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-closed:
|
|
|
|
case <-time.After(5 * time.Second):
|
2021-09-03 09:51:27 +00:00
|
|
|
require.FailNow(t, "Server still running after 5 seconds.")
|
2020-11-09 23:09:39 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-29 15:25:05 +00:00
|
|
|
// Test for availability of API endpoints in Prometheus Agent mode.
|
|
|
|
func TestAgentAPIEndPoints(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
|
2021-11-10 11:48:31 +00:00
|
|
|
port := fmt.Sprintf(":%d", testutil.RandomUnprivilegedPort(t))
|
|
|
|
|
2021-10-29 15:25:05 +00:00
|
|
|
opts := &Options{
|
2024-08-13 12:34:26 +00:00
|
|
|
ListenAddresses: []string{port},
|
|
|
|
ReadTimeout: 30 * time.Second,
|
|
|
|
MaxConnections: 512,
|
|
|
|
Context: nil,
|
|
|
|
Storage: nil,
|
|
|
|
QueryEngine: nil,
|
|
|
|
ScrapeManager: &scrape.Manager{},
|
|
|
|
RuleManager: &rules.Manager{},
|
|
|
|
Notifier: nil,
|
|
|
|
RoutePrefix: "/",
|
|
|
|
EnableAdminAPI: true,
|
2021-10-29 15:25:05 +00:00
|
|
|
ExternalURL: &url.URL{
|
|
|
|
Scheme: "http",
|
2021-11-10 11:48:31 +00:00
|
|
|
Host: "localhost" + port,
|
2021-10-29 15:25:05 +00:00
|
|
|
Path: "/",
|
|
|
|
},
|
|
|
|
Version: &PrometheusVersion{},
|
|
|
|
Gatherer: prometheus.DefaultGatherer,
|
|
|
|
IsAgent: true,
|
|
|
|
}
|
|
|
|
|
|
|
|
opts.Flags = map[string]string{}
|
|
|
|
|
|
|
|
webHandler := New(nil, opts)
|
2024-09-16 15:17:50 +00:00
|
|
|
webHandler.SetReady(Ready)
|
2021-12-17 14:36:15 +00:00
|
|
|
webHandler.config = &config.Config{}
|
|
|
|
webHandler.notifier = ¬ifier.Manager{}
|
2024-08-13 12:34:26 +00:00
|
|
|
l, err := webHandler.Listeners()
|
2021-12-17 14:36:15 +00:00
|
|
|
if err != nil {
|
2024-08-13 12:34:26 +00:00
|
|
|
panic(fmt.Sprintf("Unable to start web listeners: %s", err))
|
2021-12-17 14:36:15 +00:00
|
|
|
}
|
2021-10-29 15:25:05 +00:00
|
|
|
|
2021-12-17 14:36:15 +00:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
defer cancel()
|
|
|
|
go func() {
|
|
|
|
err := webHandler.Run(ctx, l, "")
|
|
|
|
if err != nil {
|
|
|
|
panic(fmt.Sprintf("Can't start web handler:%s", err))
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
// Give some time for the web goroutine to run since we need the server
|
|
|
|
// to be up before starting tests.
|
|
|
|
time.Sleep(5 * time.Second)
|
|
|
|
baseURL := "http://localhost" + port + "/api/v1"
|
2021-11-10 11:48:31 +00:00
|
|
|
|
2021-10-29 15:25:05 +00:00
|
|
|
// Test for non-available endpoints in the Agent mode.
|
2021-12-17 14:36:15 +00:00
|
|
|
for path, methods := range map[string][]string{
|
|
|
|
"/labels": {http.MethodGet, http.MethodPost},
|
|
|
|
"/label/:name/values": {http.MethodGet},
|
|
|
|
"/series": {http.MethodGet, http.MethodPost, http.MethodDelete},
|
|
|
|
"/alertmanagers": {http.MethodGet},
|
|
|
|
"/query": {http.MethodGet, http.MethodPost},
|
|
|
|
"/query_range": {http.MethodGet, http.MethodPost},
|
|
|
|
"/query_exemplars": {http.MethodGet, http.MethodPost},
|
|
|
|
"/status/tsdb": {http.MethodGet},
|
|
|
|
"/alerts": {http.MethodGet},
|
|
|
|
"/rules": {http.MethodGet},
|
|
|
|
"/admin/tsdb/delete_series": {http.MethodPost, http.MethodPut},
|
|
|
|
"/admin/tsdb/clean_tombstones": {http.MethodPost, http.MethodPut},
|
|
|
|
"/admin/tsdb/snapshot": {http.MethodPost, http.MethodPut},
|
2021-10-29 15:25:05 +00:00
|
|
|
} {
|
2021-12-17 14:36:15 +00:00
|
|
|
for _, m := range methods {
|
|
|
|
req, err := http.NewRequest(m, baseURL+path, nil)
|
|
|
|
require.NoError(t, err)
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode)
|
2022-06-17 07:55:52 +00:00
|
|
|
t.Cleanup(func() {
|
|
|
|
require.NoError(t, resp.Body.Close())
|
|
|
|
})
|
2021-12-17 14:36:15 +00:00
|
|
|
}
|
2021-10-29 15:25:05 +00:00
|
|
|
}
|
|
|
|
|
2021-12-17 14:36:15 +00:00
|
|
|
// Test for some of available endpoints in the Agent mode.
|
|
|
|
for path, methods := range map[string][]string{
|
|
|
|
"/targets": {http.MethodGet},
|
|
|
|
"/targets/metadata": {http.MethodGet},
|
|
|
|
"/metadata": {http.MethodGet},
|
|
|
|
"/status/config": {http.MethodGet},
|
|
|
|
"/status/runtimeinfo": {http.MethodGet},
|
|
|
|
"/status/flags": {http.MethodGet},
|
2021-10-29 15:25:05 +00:00
|
|
|
} {
|
2021-12-17 14:36:15 +00:00
|
|
|
for _, m := range methods {
|
|
|
|
req, err := http.NewRequest(m, baseURL+path, nil)
|
|
|
|
require.NoError(t, err)
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
2022-06-17 07:55:52 +00:00
|
|
|
t.Cleanup(func() {
|
|
|
|
require.NoError(t, resp.Body.Close())
|
|
|
|
})
|
2021-12-17 14:36:15 +00:00
|
|
|
}
|
2021-10-29 15:25:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-08-29 10:22:00 +00:00
|
|
|
func cleanupTestResponse(t *testing.T, resp *http.Response) {
|
2022-04-27 09:24:36 +00:00
|
|
|
_, err := io.Copy(io.Discard, resp.Body)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.NoError(t, resp.Body.Close())
|
2020-08-29 10:22:00 +00:00
|
|
|
}
|
|
|
|
|
2020-09-15 07:30:55 +00:00
|
|
|
func cleanupSnapshot(t *testing.T, dbDir string, resp *http.Response) {
|
2020-08-29 10:22:00 +00:00
|
|
|
snapshot := &struct {
|
2020-09-15 07:30:55 +00:00
|
|
|
Data struct {
|
|
|
|
Name string `json:"name"`
|
|
|
|
} `json:"data"`
|
2020-08-29 10:22:00 +00:00
|
|
|
}{}
|
2022-04-27 09:24:36 +00:00
|
|
|
b, err := io.ReadAll(resp.Body)
|
2020-10-29 09:43:23 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
require.NoError(t, json.Unmarshal(b, snapshot))
|
|
|
|
require.NotZero(t, snapshot.Data.Name, "snapshot directory not returned")
|
|
|
|
require.NoError(t, os.Remove(filepath.Join(dbDir, "snapshots", snapshot.Data.Name)))
|
|
|
|
require.NoError(t, os.Remove(filepath.Join(dbDir, "snapshots")))
|
2020-08-29 10:22:00 +00:00
|
|
|
}
|
2024-08-13 12:34:26 +00:00
|
|
|
|
|
|
|
func TestMultipleListenAddresses(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
|
|
|
|
dbDir := t.TempDir()
|
|
|
|
|
|
|
|
db, err := tsdb.Open(dbDir, nil, nil, nil, nil)
|
|
|
|
require.NoError(t, err)
|
|
|
|
t.Cleanup(func() {
|
|
|
|
require.NoError(t, db.Close())
|
|
|
|
})
|
|
|
|
|
|
|
|
// Create multiple ports for testing multiple ListenAddresses
|
|
|
|
port1 := fmt.Sprintf(":%d", testutil.RandomUnprivilegedPort(t))
|
|
|
|
port2 := fmt.Sprintf(":%d", testutil.RandomUnprivilegedPort(t))
|
|
|
|
|
|
|
|
opts := &Options{
|
|
|
|
ListenAddresses: []string{port1, port2},
|
|
|
|
ReadTimeout: 30 * time.Second,
|
|
|
|
MaxConnections: 512,
|
|
|
|
Context: nil,
|
|
|
|
Storage: nil,
|
|
|
|
LocalStorage: &dbAdapter{db},
|
|
|
|
TSDBDir: dbDir,
|
|
|
|
QueryEngine: nil,
|
|
|
|
ScrapeManager: &scrape.Manager{},
|
|
|
|
RuleManager: &rules.Manager{},
|
|
|
|
Notifier: nil,
|
|
|
|
RoutePrefix: "/",
|
|
|
|
EnableAdminAPI: true,
|
|
|
|
ExternalURL: &url.URL{
|
|
|
|
Scheme: "http",
|
|
|
|
Host: "localhost" + port1,
|
|
|
|
Path: "/",
|
|
|
|
},
|
|
|
|
Version: &PrometheusVersion{},
|
|
|
|
Gatherer: prometheus.DefaultGatherer,
|
|
|
|
}
|
|
|
|
|
|
|
|
opts.Flags = map[string]string{}
|
|
|
|
|
|
|
|
webHandler := New(nil, opts)
|
|
|
|
|
|
|
|
webHandler.config = &config.Config{}
|
|
|
|
webHandler.notifier = ¬ifier.Manager{}
|
|
|
|
l, err := webHandler.Listeners()
|
|
|
|
if err != nil {
|
|
|
|
panic(fmt.Sprintf("Unable to start web listener: %s", err))
|
|
|
|
}
|
|
|
|
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
defer cancel()
|
|
|
|
go func() {
|
|
|
|
err := webHandler.Run(ctx, l, "")
|
|
|
|
if err != nil {
|
|
|
|
panic(fmt.Sprintf("Can't start web handler:%s", err))
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
// Give some time for the web goroutine to run since we need the server
|
|
|
|
// to be up before starting tests.
|
|
|
|
time.Sleep(5 * time.Second)
|
|
|
|
|
|
|
|
// Set to ready.
|
2024-09-16 15:17:50 +00:00
|
|
|
webHandler.SetReady(Ready)
|
2024-08-13 12:34:26 +00:00
|
|
|
|
|
|
|
for _, port := range []string{port1, port2} {
|
|
|
|
baseURL := "http://localhost" + port
|
|
|
|
|
|
|
|
resp, err := http.Get(baseURL + "/-/healthy")
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
|
|
|
cleanupTestResponse(t, resp)
|
|
|
|
|
|
|
|
resp, err = http.Get(baseURL + "/-/ready")
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
|
|
|
cleanupTestResponse(t, resp)
|
|
|
|
}
|
|
|
|
}
|