From 0dbae36d36099a061d86d3484bc2ca50bc665a50 Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Fri, 12 Jun 2015 22:16:13 +0100 Subject: [PATCH] Allow ingested metrics to be relabeled. The main purpose of this is to allow for blacklisting of expensive metrics as a tactical option. It could also find uses for renaming and removing labels from federation. --- config/config.go | 4 +- config/config_test.go | 8 ++++ config/testdata/conf.good.yml | 6 ++- retrieval/target.go | 16 ++++++++ retrieval/target_test.go | 73 +++++++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 5b2356cff..47658e0ee 100644 --- a/config/config.go +++ b/config/config.go @@ -204,8 +204,10 @@ type ScrapeConfig struct { FileSDConfigs []*FileSDConfig `yaml:"file_sd_configs,omitempty"` // List of Consul service discovery configurations. ConsulSDConfigs []*ConsulSDConfig `yaml:"consul_sd_configs,omitempty"` - // List of relabel configurations. + // List of target relabel configurations. RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"` + // List of metric relabel configurations. + MetricRelabelConfigs []*RelabelConfig `yaml:"metric_relabel_configs,omitempty"` // Catches all undefined fields and must be empty after parsing. XXX map[string]interface{} `yaml:",inline"` diff --git a/config/config_test.go b/config/config_test.go index c65e5e715..16fe1e119 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -114,6 +114,14 @@ var expectedConf = &Config{ Action: RelabelDrop, }, }, + MetricRelabelConfigs: []*RelabelConfig{ + { + SourceLabels: clientmodel.LabelNames{"__name__"}, + Regex: &Regexp{*regexp.MustCompile("expensive_metric.*$")}, + Separator: ";", + Action: RelabelDrop, + }, + }, }, { JobName: "service-y", diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index 112540ec0..9b4921996 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -71,9 +71,13 @@ scrape_configs: regex: (.*)some-[regex]$ action: drop + metric_relabel_configs: + - source_labels: [__name__] + regex: expensive_metric.*$ + action: drop - job_name: service-y consul_sd_configs: - server: 'localhost:1234' - services: ['nginx', 'cache', 'mysql'] \ No newline at end of file + services: ['nginx', 'cache', 'mysql'] diff --git a/retrieval/target.go b/retrieval/target.go index 0365d5311..de219d9c0 100644 --- a/retrieval/target.go +++ b/retrieval/target.go @@ -151,6 +151,8 @@ type Target struct { scraperStopped chan struct{} // Channel to buffer ingested samples. ingestedSamples chan clientmodel.Samples + // Metric relabel configuration. + metricRelabelConfigs []*config.RelabelConfig // Mutex protects the members below. sync.RWMutex @@ -212,6 +214,7 @@ func (t *Target) Update(cfg *config.ScrapeConfig, baseLabels, metaLabels clientm if _, ok := t.baseLabels[clientmodel.InstanceLabel]; !ok { t.baseLabels[clientmodel.InstanceLabel] = clientmodel.LabelValue(t.InstanceIdentifier()) } + t.metricRelabelConfigs = cfg.MetricRelabelConfigs } func (t *Target) String() string { @@ -361,6 +364,19 @@ func (t *Target) scrape(sampleAppender storage.SampleAppender) (err error) { for samples := range t.ingestedSamples { for _, s := range samples { s.Metric.MergeFromLabelSet(baseLabels, clientmodel.ExporterLabelPrefix) + // Avoid the copy in Relabel if there are no configs. + if len(t.metricRelabelConfigs) > 0 { + labels, err := Relabel(clientmodel.LabelSet(s.Metric), t.metricRelabelConfigs...) + if err != nil { + log.Errorf("error while relabeling metric %s of instance %s: ", s.Metric, t.url, err) + continue + } + // Check if the timeseries was dropped. + if labels == nil { + continue + } + s.Metric = clientmodel.Metric(labels) + } sampleAppender.Append(s) } } diff --git a/retrieval/target_test.go b/retrieval/target_test.go index 5b4231ae1..87522ee0f 100644 --- a/retrieval/target_test.go +++ b/retrieval/target_test.go @@ -20,12 +20,14 @@ import ( "net/http/httptest" "net/url" "reflect" + "regexp" "strings" "testing" "time" clientmodel "github.com/prometheus/client_golang/model" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/util/httputil" ) @@ -77,6 +79,77 @@ func TestTargetScrapeWithFullChannel(t *testing.T) { } } +func TestTargetScrapeMetricRelabelConfigs(t *testing.T) { + server := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", `text/plain; version=0.0.4`) + w.Write([]byte("test_metric_drop 0\n")) + w.Write([]byte("test_metric_relabel 1\n")) + }, + ), + ) + defer server.Close() + testTarget := newTestTarget(server.URL, 10*time.Millisecond, clientmodel.LabelSet{}) + testTarget.metricRelabelConfigs = []*config.RelabelConfig{ + { + SourceLabels: clientmodel.LabelNames{"__name__"}, + Regex: &config.Regexp{*regexp.MustCompile(".*drop.*")}, + Action: config.RelabelDrop, + }, + { + SourceLabels: clientmodel.LabelNames{"__name__"}, + Regex: &config.Regexp{*regexp.MustCompile(".*(relabel|up).*")}, + TargetLabel: "foo", + Replacement: "bar", + Action: config.RelabelReplace, + }, + } + + appender := &collectResultAppender{} + testTarget.scrape(appender) + + // Remove variables part of result. + for _, sample := range appender.result { + sample.Timestamp = 0 + sample.Value = 0 + } + + expected := []*clientmodel.Sample{ + { + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: "test_metric_relabel", + "foo": "bar", + clientmodel.InstanceLabel: clientmodel.LabelValue(testTarget.url.Host), + }, + Timestamp: 0, + Value: 0, + }, + // The metrics about the scrape are not affected. + { + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: scrapeHealthMetricName, + clientmodel.InstanceLabel: clientmodel.LabelValue(testTarget.url.Host), + }, + Timestamp: 0, + Value: 0, + }, + { + Metric: clientmodel.Metric{ + clientmodel.MetricNameLabel: scrapeDurationMetricName, + clientmodel.InstanceLabel: clientmodel.LabelValue(testTarget.url.Host), + }, + Timestamp: 0, + Value: 0, + }, + } + + if !appender.result.Equal(expected) { + t.Fatalf("Expected and actual samples not equal. Expected: %s, actual: %s", expected, appender.result) + } + +} + func TestTargetRecordScrapeHealth(t *testing.T) { testTarget := newTestTarget("example.url:80", 0, clientmodel.LabelSet{clientmodel.JobLabel: "testjob"})