mirror of https://github.com/prometheus/prometheus
Julien Pivotto
2 years ago
committed by
GitHub
11 changed files with 529 additions and 2 deletions
@ -0,0 +1,210 @@
|
||||
// Copyright 2022 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package nomad |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"fmt" |
||||
"net" |
||||
"strconv" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/go-kit/log" |
||||
nomad "github.com/hashicorp/nomad/api" |
||||
"github.com/prometheus/client_golang/prometheus" |
||||
"github.com/prometheus/common/config" |
||||
"github.com/prometheus/common/model" |
||||
|
||||
"github.com/prometheus/prometheus/discovery" |
||||
"github.com/prometheus/prometheus/discovery/refresh" |
||||
"github.com/prometheus/prometheus/discovery/targetgroup" |
||||
) |
||||
|
||||
const ( |
||||
// nomadLabel is the name for the label containing a target.
|
||||
nomadLabel = model.MetaLabelPrefix + "nomad_" |
||||
// serviceLabel is the name of the label containing the service name.
|
||||
nomadAddress = nomadLabel + "address" |
||||
nomadService = nomadLabel + "service" |
||||
nomadNamespace = nomadLabel + "namespace" |
||||
nomadNodeID = nomadLabel + "node_id" |
||||
nomadDatacenter = nomadLabel + "dc" |
||||
nomadServiceAddress = nomadService + "_address" |
||||
nomadServicePort = nomadService + "_port" |
||||
nomadServiceID = nomadService + "_id" |
||||
nomadTags = nomadLabel + "tags" |
||||
) |
||||
|
||||
// DefaultSDConfig is the default nomad SD configuration.
|
||||
var ( |
||||
DefaultSDConfig = SDConfig{ |
||||
AllowStale: true, |
||||
HTTPClientConfig: config.DefaultHTTPClientConfig, |
||||
Namespace: "default", |
||||
RefreshInterval: model.Duration(60 * time.Second), |
||||
Region: "global", |
||||
Server: "http://localhost:4646", |
||||
TagSeparator: ",", |
||||
} |
||||
|
||||
failuresCount = prometheus.NewCounter( |
||||
prometheus.CounterOpts{ |
||||
Name: "prometheus_sd_nomad_failures_total", |
||||
Help: "Number of nomad service discovery refresh failures.", |
||||
}) |
||||
) |
||||
|
||||
func init() { |
||||
discovery.RegisterConfig(&SDConfig{}) |
||||
prometheus.MustRegister(failuresCount) |
||||
} |
||||
|
||||
// SDConfig is the configuration for nomad based service discovery.
|
||||
type SDConfig struct { |
||||
AllowStale bool `yaml:"allow_stale"` |
||||
HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` |
||||
Namespace string `yaml:"namespace"` |
||||
RefreshInterval model.Duration `yaml:"refresh_interval"` |
||||
Region string `yaml:"region"` |
||||
Server string `yaml:"server"` |
||||
TagSeparator string `yaml:"tag_separator,omitempty"` |
||||
} |
||||
|
||||
// Name returns the name of the Config.
|
||||
func (*SDConfig) Name() string { return "nomad" } |
||||
|
||||
// NewDiscoverer returns a Discoverer for the Config.
|
||||
func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { |
||||
return NewDiscovery(c, opts.Logger) |
||||
} |
||||
|
||||
// SetDirectory joins any relative file paths with dir.
|
||||
func (c *SDConfig) SetDirectory(dir string) { |
||||
c.HTTPClientConfig.SetDirectory(dir) |
||||
} |
||||
|
||||
// UnmarshalYAML implements the yaml.Unmarshaler interface.
|
||||
func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { |
||||
*c = DefaultSDConfig |
||||
type plain SDConfig |
||||
err := unmarshal((*plain)(c)) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
if strings.TrimSpace(c.Server) == "" { |
||||
return errors.New("nomad SD configuration requires a server address") |
||||
} |
||||
return c.HTTPClientConfig.Validate() |
||||
} |
||||
|
||||
// Discovery periodically performs nomad requests. It implements
|
||||
// the Discoverer interface.
|
||||
type Discovery struct { |
||||
*refresh.Discovery |
||||
allowStale bool |
||||
client *nomad.Client |
||||
namespace string |
||||
refreshInterval time.Duration |
||||
region string |
||||
server string |
||||
tagSeparator string |
||||
} |
||||
|
||||
// NewDiscovery returns a new Discovery which periodically refreshes its targets.
|
||||
func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { |
||||
d := &Discovery{ |
||||
allowStale: conf.AllowStale, |
||||
namespace: conf.Namespace, |
||||
refreshInterval: time.Duration(conf.RefreshInterval), |
||||
region: conf.Region, |
||||
server: conf.Server, |
||||
tagSeparator: conf.TagSeparator, |
||||
} |
||||
|
||||
HTTPClient, err := config.NewClientFromConfig(conf.HTTPClientConfig, "nomad_sd") |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
config := nomad.Config{ |
||||
Address: conf.Server, |
||||
HttpClient: HTTPClient, |
||||
Namespace: conf.Namespace, |
||||
Region: conf.Region, |
||||
} |
||||
|
||||
client, err := nomad.NewClient(&config) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
d.client = client |
||||
|
||||
d.Discovery = refresh.NewDiscovery( |
||||
logger, |
||||
"nomad", |
||||
time.Duration(conf.RefreshInterval), |
||||
d.refresh, |
||||
) |
||||
return d, nil |
||||
} |
||||
|
||||
func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { |
||||
opts := &nomad.QueryOptions{ |
||||
AllowStale: d.allowStale, |
||||
} |
||||
stubs, _, err := d.client.Services().List(opts) |
||||
if err != nil { |
||||
failuresCount.Inc() |
||||
return nil, err |
||||
} |
||||
|
||||
tg := &targetgroup.Group{ |
||||
Source: "Nomad", |
||||
} |
||||
|
||||
for _, stub := range stubs { |
||||
for _, service := range stub.Services { |
||||
instances, _, err := d.client.Services().Get(service.ServiceName, opts) |
||||
if err != nil { |
||||
failuresCount.Inc() |
||||
return nil, fmt.Errorf("failed to fetch services: %w", err) |
||||
} |
||||
|
||||
for _, instance := range instances { |
||||
labels := model.LabelSet{ |
||||
nomadAddress: model.LabelValue(instance.Address), |
||||
nomadDatacenter: model.LabelValue(instance.Datacenter), |
||||
nomadNodeID: model.LabelValue(instance.NodeID), |
||||
nomadNamespace: model.LabelValue(instance.Namespace), |
||||
nomadServiceAddress: model.LabelValue(instance.Address), |
||||
nomadServiceID: model.LabelValue(instance.ID), |
||||
nomadServicePort: model.LabelValue(strconv.Itoa(instance.Port)), |
||||
nomadService: model.LabelValue(instance.ServiceName), |
||||
} |
||||
addr := net.JoinHostPort(instance.Address, strconv.FormatInt(int64(instance.Port), 10)) |
||||
labels[model.AddressLabel] = model.LabelValue(addr) |
||||
|
||||
if len(instance.Tags) > 0 { |
||||
tags := d.tagSeparator + strings.Join(instance.Tags, d.tagSeparator) + d.tagSeparator |
||||
labels[nomadTags] = model.LabelValue(tags) |
||||
} |
||||
|
||||
tg.Targets = append(tg.Targets, labels) |
||||
} |
||||
} |
||||
} |
||||
return []*targetgroup.Group{tg}, nil |
||||
} |
||||
@ -0,0 +1,170 @@
|
||||
// Copyright 2022 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package nomad |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"net/http" |
||||
"net/http/httptest" |
||||
"net/url" |
||||
"testing" |
||||
|
||||
"github.com/go-kit/log" |
||||
"github.com/prometheus/common/model" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
type NomadSDTestSuite struct { |
||||
Mock *SDMock |
||||
} |
||||
|
||||
// SDMock is the interface for the nomad mock
|
||||
type SDMock struct { |
||||
t *testing.T |
||||
Server *httptest.Server |
||||
Mux *http.ServeMux |
||||
} |
||||
|
||||
// NewSDMock returns a new SDMock.
|
||||
func NewSDMock(t *testing.T) *SDMock { |
||||
return &SDMock{ |
||||
t: t, |
||||
} |
||||
} |
||||
|
||||
// Endpoint returns the URI to the mock server.
|
||||
func (m *SDMock) Endpoint() string { |
||||
return m.Server.URL + "/" |
||||
} |
||||
|
||||
// Setup creates the mock server.
|
||||
func (m *SDMock) Setup() { |
||||
m.Mux = http.NewServeMux() |
||||
m.Server = httptest.NewServer(m.Mux) |
||||
} |
||||
|
||||
// ShutdownServer creates the mock server.
|
||||
func (m *SDMock) ShutdownServer() { |
||||
m.Server.Close() |
||||
} |
||||
|
||||
func (s *NomadSDTestSuite) TearDownSuite() { |
||||
s.Mock.ShutdownServer() |
||||
} |
||||
|
||||
func (s *NomadSDTestSuite) SetupTest(t *testing.T) { |
||||
s.Mock = NewSDMock(t) |
||||
s.Mock.Setup() |
||||
|
||||
s.Mock.HandleServicesList() |
||||
s.Mock.HandleServiceHashiCupsGet() |
||||
} |
||||
|
||||
func (m *SDMock) HandleServicesList() { |
||||
m.Mux.HandleFunc("/v1/services", func(w http.ResponseWriter, r *http.Request) { |
||||
w.Header().Set("content-type", "application/json; charset=utf-8") |
||||
w.WriteHeader(http.StatusOK) |
||||
|
||||
fmt.Fprint(w, ` |
||||
[ |
||||
{ |
||||
"Namespace": "default", |
||||
"Services": [ |
||||
{ |
||||
"ServiceName": "hashicups", |
||||
"Tags": [ |
||||
"metrics" |
||||
] |
||||
} |
||||
] |
||||
} |
||||
]`, |
||||
) |
||||
}) |
||||
} |
||||
|
||||
func (m *SDMock) HandleServiceHashiCupsGet() { |
||||
m.Mux.HandleFunc("/v1/service/hashicups", func(w http.ResponseWriter, r *http.Request) { |
||||
w.Header().Set("content-type", "application/json; charset=utf-8") |
||||
w.WriteHeader(http.StatusOK) |
||||
|
||||
fmt.Fprint(w, ` |
||||
[ |
||||
{ |
||||
"ID": "_nomad-task-6a1d5f0a-7362-3f5d-9baf-5ed438918e50-group-hashicups-hashicups-hashicups_ui", |
||||
"ServiceName": "hashicups", |
||||
"Namespace": "default", |
||||
"NodeID": "d92fdc3c-9c2b-298a-e8f4-c33f3a449f09", |
||||
"Datacenter": "dc1", |
||||
"JobID": "dashboard", |
||||
"AllocID": "6a1d5f0a-7362-3f5d-9baf-5ed438918e50", |
||||
"Tags": [ |
||||
"metrics" |
||||
], |
||||
"Address": "127.0.0.1", |
||||
"Port": 30456, |
||||
"CreateIndex": 226, |
||||
"ModifyIndex": 226 |
||||
} |
||||
]`, |
||||
) |
||||
}) |
||||
} |
||||
|
||||
func TestConfiguredService(t *testing.T) { |
||||
conf := &SDConfig{ |
||||
Server: "http://localhost:4646", |
||||
} |
||||
_, err := NewDiscovery(conf, nil) |
||||
require.NoError(t, err) |
||||
} |
||||
|
||||
func TestNomadSDRefresh(t *testing.T) { |
||||
sdmock := &NomadSDTestSuite{} |
||||
sdmock.SetupTest(t) |
||||
t.Cleanup(sdmock.TearDownSuite) |
||||
|
||||
endpoint, err := url.Parse(sdmock.Mock.Endpoint()) |
||||
require.NoError(t, err) |
||||
|
||||
cfg := DefaultSDConfig |
||||
cfg.Server = endpoint.String() |
||||
d, err := NewDiscovery(&cfg, log.NewNopLogger()) |
||||
require.NoError(t, err) |
||||
|
||||
tgs, err := d.refresh(context.Background()) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, 1, len(tgs)) |
||||
|
||||
tg := tgs[0] |
||||
require.NotNil(t, tg) |
||||
require.NotNil(t, tg.Targets) |
||||
require.Equal(t, 1, len(tg.Targets)) |
||||
|
||||
lbls := model.LabelSet{ |
||||
"__address__": model.LabelValue("127.0.0.1:30456"), |
||||
"__meta_nomad_address": model.LabelValue("127.0.0.1"), |
||||
"__meta_nomad_dc": model.LabelValue("dc1"), |
||||
"__meta_nomad_namespace": model.LabelValue("default"), |
||||
"__meta_nomad_node_id": model.LabelValue("d92fdc3c-9c2b-298a-e8f4-c33f3a449f09"), |
||||
"__meta_nomad_service": model.LabelValue("hashicups"), |
||||
"__meta_nomad_service_address": model.LabelValue("127.0.0.1"), |
||||
"__meta_nomad_service_id": model.LabelValue("_nomad-task-6a1d5f0a-7362-3f5d-9baf-5ed438918e50-group-hashicups-hashicups-hashicups_ui"), |
||||
"__meta_nomad_service_port": model.LabelValue("30456"), |
||||
"__meta_nomad_tags": model.LabelValue(",metrics,"), |
||||
} |
||||
require.Equal(t, lbls, tg.Targets[0]) |
||||
} |
||||
@ -0,0 +1,23 @@
|
||||
# An example scrape configuration for running Prometheus with |
||||
# Nomad build in service discovery. |
||||
# |
||||
# The following config can be used to monitor services running on |
||||
# a nomad that is started using the getting started tutorial [1] |
||||
# |
||||
# sudo nomad agent -dev -bind 0.0.0.0 -log-level INFO |
||||
# |
||||
# [1] https://learn.hashicorp.com/tutorials/nomad/get-started-run?in=nomad/get-started |
||||
|
||||
scrape_configs: |
||||
# Make Prometheus scrape itself for metrics. |
||||
- job_name: "prometheus" |
||||
static_configs: |
||||
- targets: ["localhost:9090"] |
||||
|
||||
# Discover Nomad services to scrape. |
||||
- job_name: 'nomad_sd' |
||||
nomad_sd_configs: |
||||
- server: 'http://localhost:4646' |
||||
relabel_configs: |
||||
- source_labels: [__meta_nomad_service] |
||||
target_label: job |
||||
Loading…
Reference in new issue