mirror of https://github.com/prometheus/prometheus
Browse Source
* Introduce a metadata watcher Similarly to the WAL watcher, its purpose is to observe the scrape manager and pull metadata. Then, send it to a remote storage. Signed-off-by: gotjosh <josue@grafana.com> * Additional fixes after rebasing. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Rework samples/metadata metrics. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Use more descriptive variable names in MetadataWatcher collect. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Fix issues caused during rebasing. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Fix missing metric add and unneeded config code. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Address some review comments. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Fix metrics and docs Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> * Replace assert with require Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> * Bring back max_samples_per_send metric Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> * Fix tests Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> Co-authored-by: Callum Styan <callumstyan@gmail.com> Co-authored-by: Ganesh Vernekar <cs15btech11018@iith.ac.in>pull/8209/head
gotjosh
4 years ago
committed by
GitHub
21 changed files with 1276 additions and 188 deletions
@ -0,0 +1,163 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package remote |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
|
||||
"github.com/go-kit/kit/log" |
||||
"github.com/go-kit/kit/log/level" |
||||
"github.com/pkg/errors" |
||||
"github.com/prometheus/common/model" |
||||
"github.com/prometheus/prometheus/scrape" |
||||
) |
||||
|
||||
// MetadataAppender is an interface used by the Metadata Watcher to send metadata, It is read from the scrape manager, on to somewhere else.
|
||||
type MetadataAppender interface { |
||||
AppendMetadata(context.Context, []scrape.MetricMetadata) |
||||
} |
||||
|
||||
// Watchable represents from where we fetch active targets for metadata.
|
||||
type Watchable interface { |
||||
TargetsActive() map[string][]*scrape.Target |
||||
} |
||||
|
||||
type noopScrapeManager struct{} |
||||
|
||||
func (noop *noopScrapeManager) Get() (*scrape.Manager, error) { |
||||
return nil, errors.New("Scrape manager not ready") |
||||
} |
||||
|
||||
// MetadataWatcher watches the Scrape Manager for a given WriteMetadataTo.
|
||||
type MetadataWatcher struct { |
||||
name string |
||||
logger log.Logger |
||||
|
||||
managerGetter ReadyScrapeManager |
||||
manager Watchable |
||||
writer MetadataAppender |
||||
|
||||
interval model.Duration |
||||
deadline time.Duration |
||||
|
||||
done chan struct{} |
||||
|
||||
softShutdownCtx context.Context |
||||
softShutdownCancel context.CancelFunc |
||||
hardShutdownCancel context.CancelFunc |
||||
hardShutdownCtx context.Context |
||||
} |
||||
|
||||
// NewMetadataWatcher builds a new MetadataWatcher.
|
||||
func NewMetadataWatcher(l log.Logger, mg ReadyScrapeManager, name string, w MetadataAppender, interval model.Duration, deadline time.Duration) *MetadataWatcher { |
||||
if l == nil { |
||||
l = log.NewNopLogger() |
||||
} |
||||
|
||||
if mg == nil { |
||||
mg = &noopScrapeManager{} |
||||
} |
||||
|
||||
return &MetadataWatcher{ |
||||
name: name, |
||||
logger: l, |
||||
|
||||
managerGetter: mg, |
||||
writer: w, |
||||
|
||||
interval: interval, |
||||
deadline: deadline, |
||||
|
||||
done: make(chan struct{}), |
||||
} |
||||
} |
||||
|
||||
// Start the MetadataWatcher.
|
||||
func (mw *MetadataWatcher) Start() { |
||||
level.Info(mw.logger).Log("msg", "Starting scraped metadata watcher") |
||||
mw.hardShutdownCtx, mw.hardShutdownCancel = context.WithCancel(context.Background()) |
||||
mw.softShutdownCtx, mw.softShutdownCancel = context.WithCancel(mw.hardShutdownCtx) |
||||
go mw.loop() |
||||
} |
||||
|
||||
// Stop the MetadataWatcher.
|
||||
func (mw *MetadataWatcher) Stop() { |
||||
level.Info(mw.logger).Log("msg", "Stopping metadata watcher...") |
||||
defer level.Info(mw.logger).Log("msg", "Scraped metadata watcher stopped") |
||||
|
||||
mw.softShutdownCancel() |
||||
select { |
||||
case <-mw.done: |
||||
return |
||||
case <-time.After(mw.deadline): |
||||
level.Error(mw.logger).Log("msg", "Failed to flush metadata") |
||||
} |
||||
|
||||
mw.hardShutdownCancel() |
||||
<-mw.done |
||||
} |
||||
|
||||
func (mw *MetadataWatcher) loop() { |
||||
ticker := time.NewTicker(time.Duration(mw.interval)) |
||||
defer ticker.Stop() |
||||
defer close(mw.done) |
||||
|
||||
for { |
||||
select { |
||||
case <-mw.softShutdownCtx.Done(): |
||||
return |
||||
case <-ticker.C: |
||||
mw.collect() |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (mw *MetadataWatcher) collect() { |
||||
if !mw.ready() { |
||||
return |
||||
} |
||||
|
||||
// We create a set of the metadata to help deduplicating based on the attributes of a
|
||||
// scrape.MetricMetadata. In this case, a combination of metric name, help, type, and unit.
|
||||
metadataSet := map[scrape.MetricMetadata]struct{}{} |
||||
metadata := []scrape.MetricMetadata{} |
||||
for _, tset := range mw.manager.TargetsActive() { |
||||
for _, target := range tset { |
||||
for _, entry := range target.MetadataList() { |
||||
if _, ok := metadataSet[entry]; !ok { |
||||
metadata = append(metadata, entry) |
||||
metadataSet[entry] = struct{}{} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Blocks until the metadata is sent to the remote write endpoint or hardShutdownContext is expired.
|
||||
mw.writer.AppendMetadata(mw.hardShutdownCtx, metadata) |
||||
} |
||||
|
||||
func (mw *MetadataWatcher) ready() bool { |
||||
if mw.manager != nil { |
||||
return true |
||||
} |
||||
|
||||
m, err := mw.managerGetter.Get() |
||||
if err != nil { |
||||
return false |
||||
} |
||||
|
||||
mw.manager = m |
||||
return true |
||||
} |
||||
@ -0,0 +1,155 @@
|
||||
// Copyright 2020 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package remote |
||||
|
||||
import ( |
||||
"context" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/pkg/errors" |
||||
"github.com/prometheus/common/model" |
||||
"github.com/prometheus/prometheus/pkg/textparse" |
||||
"github.com/prometheus/prometheus/scrape" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
var ( |
||||
interval = model.Duration(1 * time.Millisecond) |
||||
deadline = 1 * time.Millisecond |
||||
) |
||||
|
||||
// TestMetaStore satisfies the MetricMetadataStore interface.
|
||||
// It is used to inject specific metadata as part of a test case.
|
||||
type TestMetaStore struct { |
||||
Metadata []scrape.MetricMetadata |
||||
} |
||||
|
||||
func (s *TestMetaStore) ListMetadata() []scrape.MetricMetadata { |
||||
return s.Metadata |
||||
} |
||||
|
||||
func (s *TestMetaStore) GetMetadata(metric string) (scrape.MetricMetadata, bool) { |
||||
for _, m := range s.Metadata { |
||||
if metric == m.Metric { |
||||
return m, true |
||||
} |
||||
} |
||||
|
||||
return scrape.MetricMetadata{}, false |
||||
} |
||||
|
||||
func (s *TestMetaStore) SizeMetadata() int { return 0 } |
||||
func (s *TestMetaStore) LengthMetadata() int { return 0 } |
||||
|
||||
type writeMetadataToMock struct { |
||||
metadataAppended int |
||||
} |
||||
|
||||
func (mwtm *writeMetadataToMock) AppendMetadata(_ context.Context, m []scrape.MetricMetadata) { |
||||
mwtm.metadataAppended += len(m) |
||||
} |
||||
|
||||
func newMetadataWriteToMock() *writeMetadataToMock { |
||||
return &writeMetadataToMock{} |
||||
} |
||||
|
||||
type scrapeManagerMock struct { |
||||
manager *scrape.Manager |
||||
ready bool |
||||
} |
||||
|
||||
func (smm *scrapeManagerMock) Get() (*scrape.Manager, error) { |
||||
if smm.ready { |
||||
return smm.manager, nil |
||||
} |
||||
|
||||
return nil, errors.New("not ready") |
||||
} |
||||
|
||||
type fakeManager struct { |
||||
activeTargets map[string][]*scrape.Target |
||||
} |
||||
|
||||
func (fm *fakeManager) TargetsActive() map[string][]*scrape.Target { |
||||
return fm.activeTargets |
||||
} |
||||
|
||||
func TestWatchScrapeManager_NotReady(t *testing.T) { |
||||
wt := newMetadataWriteToMock() |
||||
smm := &scrapeManagerMock{ |
||||
ready: false, |
||||
} |
||||
|
||||
mw := NewMetadataWatcher(nil, smm, "", wt, interval, deadline) |
||||
require.Equal(t, false, mw.ready()) |
||||
|
||||
mw.collect() |
||||
|
||||
require.Equal(t, 0, wt.metadataAppended) |
||||
} |
||||
|
||||
func TestWatchScrapeManager_ReadyForCollection(t *testing.T) { |
||||
wt := newMetadataWriteToMock() |
||||
|
||||
metadata := &TestMetaStore{ |
||||
Metadata: []scrape.MetricMetadata{ |
||||
{ |
||||
Metric: "prometheus_tsdb_head_chunks_created_total", |
||||
Type: textparse.MetricTypeCounter, |
||||
Help: "Total number", |
||||
Unit: "", |
||||
}, |
||||
{ |
||||
Metric: "prometheus_remote_storage_retried_samples_total", |
||||
Type: textparse.MetricTypeCounter, |
||||
Help: "Total number", |
||||
Unit: "", |
||||
}, |
||||
}, |
||||
} |
||||
metadataDup := &TestMetaStore{ |
||||
Metadata: []scrape.MetricMetadata{ |
||||
{ |
||||
Metric: "prometheus_tsdb_head_chunks_created_total", |
||||
Type: textparse.MetricTypeCounter, |
||||
Help: "Total number", |
||||
Unit: "", |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
target := &scrape.Target{} |
||||
target.SetMetadataStore(metadata) |
||||
targetWithDup := &scrape.Target{} |
||||
targetWithDup.SetMetadataStore(metadataDup) |
||||
|
||||
manager := &fakeManager{ |
||||
activeTargets: map[string][]*scrape.Target{ |
||||
"job": []*scrape.Target{target}, |
||||
"dup": []*scrape.Target{targetWithDup}, |
||||
}, |
||||
} |
||||
|
||||
smm := &scrapeManagerMock{ |
||||
ready: true, |
||||
} |
||||
|
||||
mw := NewMetadataWatcher(nil, smm, "", wt, interval, deadline) |
||||
mw.manager = manager |
||||
|
||||
mw.collect() |
||||
|
||||
require.Equal(t, 2, wt.metadataAppended) |
||||
} |
||||
Loading…
Reference in new issue