mirror of https://github.com/prometheus/prometheus
Signed-off-by: Bram Vogelaar <bram@attachmentgenie.com>pull/10915/head
Bram Vogelaar
2 years ago
committed by
Julien Pivotto
parent
b93ce29674
commit
4456dcc26e
@ -0,0 +1,208 @@
|
|||||||
|
// Copyright 2022 The Prometheus Authors
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-kit/log"
|
||||||
|
nomad "github.com/hashicorp/nomad/api"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"github.com/prometheus/common/config"
|
||||||
|
"github.com/prometheus/common/model"
|
||||||
|
|
||||||
|
"github.com/prometheus/prometheus/discovery"
|
||||||
|
"github.com/prometheus/prometheus/discovery/refresh"
|
||||||
|
"github.com/prometheus/prometheus/discovery/targetgroup"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// nomadLabel is the name for the label containing a target.
|
||||||
|
nomadLabel = model.MetaLabelPrefix + "nomad_"
|
||||||
|
// serviceLabel is the name of the label containing the service name.
|
||||||
|
nomadAddress = nomadLabel + "address"
|
||||||
|
nomadService = nomadLabel + "service"
|
||||||
|
nomadNamespace = nomadLabel + "namespace"
|
||||||
|
nomadNodeID = nomadLabel + "node_id"
|
||||||
|
nomadDatacenter = nomadLabel + "dc"
|
||||||
|
nomadServiceAddress = nomadService + "_address"
|
||||||
|
nomadServicePort = nomadService + "_port"
|
||||||
|
nomadServiceID = nomadService + "_id"
|
||||||
|
nomadTags = nomadLabel + "tags"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultSDConfig is the default nomad SD configuration.
|
||||||
|
var (
|
||||||
|
DefaultSDConfig = SDConfig{
|
||||||
|
AllowStale: true,
|
||||||
|
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||||
|
Namespace: "default",
|
||||||
|
RefreshInterval: model.Duration(60 * time.Second),
|
||||||
|
Region: "global",
|
||||||
|
Server: "http://localhost:4646",
|
||||||
|
TagSeparator: ",",
|
||||||
|
}
|
||||||
|
|
||||||
|
failuresCount = prometheus.NewCounter(
|
||||||
|
prometheus.CounterOpts{
|
||||||
|
Name: "prometheus_sd_nomad_failures_total",
|
||||||
|
Help: "Number of nomad service discovery refresh failures.",
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
discovery.RegisterConfig(&SDConfig{})
|
||||||
|
prometheus.MustRegister(failuresCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SDConfig is the configuration for nomad based service discovery.
|
||||||
|
type SDConfig struct {
|
||||||
|
AllowStale bool `yaml:"allow_stale"`
|
||||||
|
HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
|
||||||
|
Namespace string `yaml:"namespace"`
|
||||||
|
RefreshInterval model.Duration `yaml:"refresh_interval"`
|
||||||
|
Region string `yaml:"region"`
|
||||||
|
Server string `yaml:"server"`
|
||||||
|
TagSeparator string `yaml:"tag_separator,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the name of the Config.
|
||||||
|
func (*SDConfig) Name() string { return "nomad" }
|
||||||
|
|
||||||
|
// NewDiscoverer returns a Discoverer for the Config.
|
||||||
|
func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
|
||||||
|
return NewDiscovery(c, opts.Logger)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetDirectory joins any relative file paths with dir.
|
||||||
|
func (c *SDConfig) SetDirectory(dir string) {
|
||||||
|
c.HTTPClientConfig.SetDirectory(dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalYAML implements the yaml.Unmarshaler interface.
|
||||||
|
func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
*c = DefaultSDConfig
|
||||||
|
type plain SDConfig
|
||||||
|
err := unmarshal((*plain)(c))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(c.Server) == "" {
|
||||||
|
return errors.New("nomad SD configuration requires a server address")
|
||||||
|
}
|
||||||
|
return c.HTTPClientConfig.Validate()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discovery periodically performs nomad requests. It implements
|
||||||
|
// the Discoverer interface.
|
||||||
|
type Discovery struct {
|
||||||
|
*refresh.Discovery
|
||||||
|
allowStale bool
|
||||||
|
client *nomad.Client
|
||||||
|
namespace string
|
||||||
|
refreshInterval time.Duration
|
||||||
|
region string
|
||||||
|
server string
|
||||||
|
tagSeparator string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDiscovery returns a new Discovery which periodically refreshes its targets.
|
||||||
|
func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
|
||||||
|
d := &Discovery{
|
||||||
|
allowStale: conf.AllowStale,
|
||||||
|
namespace: conf.Namespace,
|
||||||
|
refreshInterval: time.Duration(conf.RefreshInterval),
|
||||||
|
region: conf.Region,
|
||||||
|
server: conf.Server,
|
||||||
|
tagSeparator: conf.TagSeparator,
|
||||||
|
}
|
||||||
|
|
||||||
|
HTTPClient, err := config.NewClientFromConfig(conf.HTTPClientConfig, "nomad_sd")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
config := nomad.Config{
|
||||||
|
Address: conf.Server,
|
||||||
|
HttpClient: HTTPClient,
|
||||||
|
Namespace: conf.Namespace,
|
||||||
|
Region: conf.Region,
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := nomad.NewClient(&config)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
d.client = client
|
||||||
|
|
||||||
|
d.Discovery = refresh.NewDiscovery(
|
||||||
|
logger,
|
||||||
|
"nomad",
|
||||||
|
time.Duration(conf.RefreshInterval),
|
||||||
|
d.refresh,
|
||||||
|
)
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
|
||||||
|
opts := &nomad.QueryOptions{
|
||||||
|
AllowStale: d.allowStale,
|
||||||
|
}
|
||||||
|
stubs, _, err := d.client.Services().List(opts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
tg := &targetgroup.Group{
|
||||||
|
Source: "Nomad",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, stub := range stubs {
|
||||||
|
for _, service := range stub.Services {
|
||||||
|
instances, _, err := d.client.Services().Get(service.ServiceName, opts)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to fetch services: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, instance := range instances {
|
||||||
|
labels := model.LabelSet{
|
||||||
|
nomadAddress: model.LabelValue(instance.Address),
|
||||||
|
nomadDatacenter: model.LabelValue(instance.Datacenter),
|
||||||
|
nomadNodeID: model.LabelValue(instance.NodeID),
|
||||||
|
nomadNamespace: model.LabelValue(instance.Namespace),
|
||||||
|
nomadServiceAddress: model.LabelValue(instance.Address),
|
||||||
|
nomadServiceID: model.LabelValue(instance.ID),
|
||||||
|
nomadServicePort: model.LabelValue(strconv.Itoa(instance.Port)),
|
||||||
|
nomadService: model.LabelValue(instance.ServiceName),
|
||||||
|
}
|
||||||
|
addr := net.JoinHostPort(instance.Address, strconv.FormatInt(int64(instance.Port), 10))
|
||||||
|
labels[model.AddressLabel] = model.LabelValue(addr)
|
||||||
|
|
||||||
|
if len(instance.Tags) > 0 {
|
||||||
|
tags := d.tagSeparator + strings.Join(instance.Tags, d.tagSeparator) + d.tagSeparator
|
||||||
|
labels[nomadTags] = model.LabelValue(tags)
|
||||||
|
}
|
||||||
|
|
||||||
|
tg.Targets = append(tg.Targets, labels)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return []*targetgroup.Group{tg}, nil
|
||||||
|
}
|
||||||
@ -0,0 +1,172 @@
|
|||||||
|
// Copyright 2015 The Prometheus Authors
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package nomad
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/go-kit/log"
|
||||||
|
"github.com/prometheus/common/model"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
type NomadSDTestSuite struct {
|
||||||
|
Mock *SDMock
|
||||||
|
}
|
||||||
|
|
||||||
|
// SDMock is the interface for the nomad mock
|
||||||
|
type SDMock struct {
|
||||||
|
t *testing.T
|
||||||
|
Server *httptest.Server
|
||||||
|
Mux *http.ServeMux
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSDMock returns a new SDMock.
|
||||||
|
func NewSDMock(t *testing.T) *SDMock {
|
||||||
|
return &SDMock{
|
||||||
|
t: t,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Endpoint returns the URI to the mock server
|
||||||
|
func (m *SDMock) Endpoint() string {
|
||||||
|
return m.Server.URL + "/"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Setup creates the mock server
|
||||||
|
func (m *SDMock) Setup() {
|
||||||
|
m.Mux = http.NewServeMux()
|
||||||
|
m.Server = httptest.NewServer(m.Mux)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShutdownServer creates the mock server
|
||||||
|
func (m *SDMock) ShutdownServer() {
|
||||||
|
m.Server.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *NomadSDTestSuite) TearDownSuite() {
|
||||||
|
s.Mock.ShutdownServer()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *NomadSDTestSuite) SetupTest(t *testing.T) {
|
||||||
|
s.Mock = NewSDMock(t)
|
||||||
|
s.Mock.Setup()
|
||||||
|
|
||||||
|
s.Mock.HandleServicesList()
|
||||||
|
s.Mock.HandleServiceHashiCupsGet()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *SDMock) HandleServicesList() {
|
||||||
|
m.Mux.HandleFunc("/v1/services", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("content-type", "application/json; charset=utf-8")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
|
fmt.Fprint(w, `
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"Namespace": "default",
|
||||||
|
"Services": [
|
||||||
|
{
|
||||||
|
"ServiceName": "hashicups",
|
||||||
|
"Tags": [
|
||||||
|
"metrics"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]`,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *SDMock) HandleServiceHashiCupsGet() {
|
||||||
|
m.Mux.HandleFunc("/v1/service/hashicups", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("content-type", "application/json; charset=utf-8")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
|
fmt.Fprint(w, `
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"ID": "_nomad-task-6a1d5f0a-7362-3f5d-9baf-5ed438918e50-group-hashicups-hashicups-hashicups_ui",
|
||||||
|
"ServiceName": "hashicups",
|
||||||
|
"Namespace": "default",
|
||||||
|
"NodeID": "d92fdc3c-9c2b-298a-e8f4-c33f3a449f09",
|
||||||
|
"Datacenter": "dc1",
|
||||||
|
"JobID": "dashboard",
|
||||||
|
"AllocID": "6a1d5f0a-7362-3f5d-9baf-5ed438918e50",
|
||||||
|
"Tags": [
|
||||||
|
"metrics"
|
||||||
|
],
|
||||||
|
"Address": "127.0.0.1",
|
||||||
|
"Port": 30456,
|
||||||
|
"CreateIndex": 226,
|
||||||
|
"ModifyIndex": 226
|
||||||
|
}
|
||||||
|
]`,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfiguredService(t *testing.T) {
|
||||||
|
conf := &SDConfig{
|
||||||
|
Server: "http://localhost:4646",
|
||||||
|
}
|
||||||
|
_, err := NewDiscovery(conf, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Unexpected error when initializing discovery %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNomadSDRefresh(t *testing.T) {
|
||||||
|
sdmock := &NomadSDTestSuite{}
|
||||||
|
sdmock.SetupTest(t)
|
||||||
|
t.Cleanup(sdmock.TearDownSuite)
|
||||||
|
|
||||||
|
endpoint, err := url.Parse(sdmock.Mock.Endpoint())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
cfg := DefaultSDConfig
|
||||||
|
cfg.Server = endpoint.String()
|
||||||
|
d, err := NewDiscovery(&cfg, log.NewNopLogger())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
tgs, err := d.refresh(context.Background())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.Equal(t, 1, len(tgs))
|
||||||
|
|
||||||
|
tg := tgs[0]
|
||||||
|
require.NotNil(t, tg)
|
||||||
|
require.NotNil(t, tg.Targets)
|
||||||
|
require.Equal(t, 1, len(tg.Targets))
|
||||||
|
|
||||||
|
lbls := model.LabelSet{
|
||||||
|
"__address__": model.LabelValue("127.0.0.1:30456"),
|
||||||
|
"__meta_nomad_address": model.LabelValue("127.0.0.1"),
|
||||||
|
"__meta_nomad_dc": model.LabelValue("dc1"),
|
||||||
|
"__meta_nomad_namespace": model.LabelValue("default"),
|
||||||
|
"__meta_nomad_node_id": model.LabelValue("d92fdc3c-9c2b-298a-e8f4-c33f3a449f09"),
|
||||||
|
"__meta_nomad_service": model.LabelValue("hashicups"),
|
||||||
|
"__meta_nomad_service_address": model.LabelValue("127.0.0.1"),
|
||||||
|
"__meta_nomad_service_id": model.LabelValue("_nomad-task-6a1d5f0a-7362-3f5d-9baf-5ed438918e50-group-hashicups-hashicups-hashicups_ui"),
|
||||||
|
"__meta_nomad_service_port": model.LabelValue("30456"),
|
||||||
|
"__meta_nomad_tags": model.LabelValue(",metrics,"),
|
||||||
|
}
|
||||||
|
require.Equal(t, lbls, tg.Targets[0])
|
||||||
|
}
|
||||||
@ -0,0 +1,23 @@
|
|||||||
|
# An example scrape configuration for running Prometheus with
|
||||||
|
# Nomad build in service discovery.
|
||||||
|
#
|
||||||
|
# The following config can be used to monitor services running on
|
||||||
|
# a nomad that is started using the getting started tutorial [1]
|
||||||
|
#
|
||||||
|
# sudo nomad agent -dev -bind 0.0.0.0 -log-level INFO
|
||||||
|
#
|
||||||
|
# [1] https://learn.hashicorp.com/tutorials/nomad/get-started-run?in=nomad/get-started
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
# Make Prometheus scrape itself for metrics.
|
||||||
|
- job_name: "prometheus"
|
||||||
|
static_configs:
|
||||||
|
- targets: ["localhost:9090"]
|
||||||
|
|
||||||
|
# Discover Nomad services to scrape.
|
||||||
|
- job_name: 'nomad_sd'
|
||||||
|
nomad_sd_configs:
|
||||||
|
- server: 'http://localhost:4646'
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__meta_nomad_service]
|
||||||
|
target_label: job
|
||||||
Loading…
Reference in new issue