mirror of https://github.com/prometheus/prometheus
Browse Source
* Introduce a metadata watcher Similarly to the WAL watcher, its purpose is to observe the scrape manager and pull metadata. Then, send it to a remote storage. Signed-off-by: gotjosh <josue@grafana.com> * Additional fixes after rebasing. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Rework samples/metadata metrics. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Use more descriptive variable names in MetadataWatcher collect. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Fix issues caused during rebasing. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Fix missing metric add and unneeded config code. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Address some review comments. Signed-off-by: Callum Styan <callumstyan@gmail.com> * Fix metrics and docs Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> * Replace assert with require Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> * Bring back max_samples_per_send metric Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> * Fix tests Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in> Co-authored-by: Callum Styan <callumstyan@gmail.com> Co-authored-by: Ganesh Vernekar <cs15btech11018@iith.ac.in>pull/8209/head
gotjosh
4 years ago
committed by
GitHub
21 changed files with 1276 additions and 188 deletions
@ -0,0 +1,163 @@ |
|||||||
|
// Copyright 2020 The Prometheus Authors
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package remote |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/go-kit/kit/log" |
||||||
|
"github.com/go-kit/kit/log/level" |
||||||
|
"github.com/pkg/errors" |
||||||
|
"github.com/prometheus/common/model" |
||||||
|
"github.com/prometheus/prometheus/scrape" |
||||||
|
) |
||||||
|
|
||||||
|
// MetadataAppender is an interface used by the Metadata Watcher to send metadata, It is read from the scrape manager, on to somewhere else.
|
||||||
|
type MetadataAppender interface { |
||||||
|
AppendMetadata(context.Context, []scrape.MetricMetadata) |
||||||
|
} |
||||||
|
|
||||||
|
// Watchable represents from where we fetch active targets for metadata.
|
||||||
|
type Watchable interface { |
||||||
|
TargetsActive() map[string][]*scrape.Target |
||||||
|
} |
||||||
|
|
||||||
|
type noopScrapeManager struct{} |
||||||
|
|
||||||
|
func (noop *noopScrapeManager) Get() (*scrape.Manager, error) { |
||||||
|
return nil, errors.New("Scrape manager not ready") |
||||||
|
} |
||||||
|
|
||||||
|
// MetadataWatcher watches the Scrape Manager for a given WriteMetadataTo.
|
||||||
|
type MetadataWatcher struct { |
||||||
|
name string |
||||||
|
logger log.Logger |
||||||
|
|
||||||
|
managerGetter ReadyScrapeManager |
||||||
|
manager Watchable |
||||||
|
writer MetadataAppender |
||||||
|
|
||||||
|
interval model.Duration |
||||||
|
deadline time.Duration |
||||||
|
|
||||||
|
done chan struct{} |
||||||
|
|
||||||
|
softShutdownCtx context.Context |
||||||
|
softShutdownCancel context.CancelFunc |
||||||
|
hardShutdownCancel context.CancelFunc |
||||||
|
hardShutdownCtx context.Context |
||||||
|
} |
||||||
|
|
||||||
|
// NewMetadataWatcher builds a new MetadataWatcher.
|
||||||
|
func NewMetadataWatcher(l log.Logger, mg ReadyScrapeManager, name string, w MetadataAppender, interval model.Duration, deadline time.Duration) *MetadataWatcher { |
||||||
|
if l == nil { |
||||||
|
l = log.NewNopLogger() |
||||||
|
} |
||||||
|
|
||||||
|
if mg == nil { |
||||||
|
mg = &noopScrapeManager{} |
||||||
|
} |
||||||
|
|
||||||
|
return &MetadataWatcher{ |
||||||
|
name: name, |
||||||
|
logger: l, |
||||||
|
|
||||||
|
managerGetter: mg, |
||||||
|
writer: w, |
||||||
|
|
||||||
|
interval: interval, |
||||||
|
deadline: deadline, |
||||||
|
|
||||||
|
done: make(chan struct{}), |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Start the MetadataWatcher.
|
||||||
|
func (mw *MetadataWatcher) Start() { |
||||||
|
level.Info(mw.logger).Log("msg", "Starting scraped metadata watcher") |
||||||
|
mw.hardShutdownCtx, mw.hardShutdownCancel = context.WithCancel(context.Background()) |
||||||
|
mw.softShutdownCtx, mw.softShutdownCancel = context.WithCancel(mw.hardShutdownCtx) |
||||||
|
go mw.loop() |
||||||
|
} |
||||||
|
|
||||||
|
// Stop the MetadataWatcher.
|
||||||
|
func (mw *MetadataWatcher) Stop() { |
||||||
|
level.Info(mw.logger).Log("msg", "Stopping metadata watcher...") |
||||||
|
defer level.Info(mw.logger).Log("msg", "Scraped metadata watcher stopped") |
||||||
|
|
||||||
|
mw.softShutdownCancel() |
||||||
|
select { |
||||||
|
case <-mw.done: |
||||||
|
return |
||||||
|
case <-time.After(mw.deadline): |
||||||
|
level.Error(mw.logger).Log("msg", "Failed to flush metadata") |
||||||
|
} |
||||||
|
|
||||||
|
mw.hardShutdownCancel() |
||||||
|
<-mw.done |
||||||
|
} |
||||||
|
|
||||||
|
func (mw *MetadataWatcher) loop() { |
||||||
|
ticker := time.NewTicker(time.Duration(mw.interval)) |
||||||
|
defer ticker.Stop() |
||||||
|
defer close(mw.done) |
||||||
|
|
||||||
|
for { |
||||||
|
select { |
||||||
|
case <-mw.softShutdownCtx.Done(): |
||||||
|
return |
||||||
|
case <-ticker.C: |
||||||
|
mw.collect() |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func (mw *MetadataWatcher) collect() { |
||||||
|
if !mw.ready() { |
||||||
|
return |
||||||
|
} |
||||||
|
|
||||||
|
// We create a set of the metadata to help deduplicating based on the attributes of a
|
||||||
|
// scrape.MetricMetadata. In this case, a combination of metric name, help, type, and unit.
|
||||||
|
metadataSet := map[scrape.MetricMetadata]struct{}{} |
||||||
|
metadata := []scrape.MetricMetadata{} |
||||||
|
for _, tset := range mw.manager.TargetsActive() { |
||||||
|
for _, target := range tset { |
||||||
|
for _, entry := range target.MetadataList() { |
||||||
|
if _, ok := metadataSet[entry]; !ok { |
||||||
|
metadata = append(metadata, entry) |
||||||
|
metadataSet[entry] = struct{}{} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Blocks until the metadata is sent to the remote write endpoint or hardShutdownContext is expired.
|
||||||
|
mw.writer.AppendMetadata(mw.hardShutdownCtx, metadata) |
||||||
|
} |
||||||
|
|
||||||
|
func (mw *MetadataWatcher) ready() bool { |
||||||
|
if mw.manager != nil { |
||||||
|
return true |
||||||
|
} |
||||||
|
|
||||||
|
m, err := mw.managerGetter.Get() |
||||||
|
if err != nil { |
||||||
|
return false |
||||||
|
} |
||||||
|
|
||||||
|
mw.manager = m |
||||||
|
return true |
||||||
|
} |
||||||
@ -0,0 +1,155 @@ |
|||||||
|
// Copyright 2020 The Prometheus Authors
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package remote |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"testing" |
||||||
|
"time" |
||||||
|
|
||||||
|
"github.com/pkg/errors" |
||||||
|
"github.com/prometheus/common/model" |
||||||
|
"github.com/prometheus/prometheus/pkg/textparse" |
||||||
|
"github.com/prometheus/prometheus/scrape" |
||||||
|
"github.com/stretchr/testify/require" |
||||||
|
) |
||||||
|
|
||||||
|
var ( |
||||||
|
interval = model.Duration(1 * time.Millisecond) |
||||||
|
deadline = 1 * time.Millisecond |
||||||
|
) |
||||||
|
|
||||||
|
// TestMetaStore satisfies the MetricMetadataStore interface.
|
||||||
|
// It is used to inject specific metadata as part of a test case.
|
||||||
|
type TestMetaStore struct { |
||||||
|
Metadata []scrape.MetricMetadata |
||||||
|
} |
||||||
|
|
||||||
|
func (s *TestMetaStore) ListMetadata() []scrape.MetricMetadata { |
||||||
|
return s.Metadata |
||||||
|
} |
||||||
|
|
||||||
|
func (s *TestMetaStore) GetMetadata(metric string) (scrape.MetricMetadata, bool) { |
||||||
|
for _, m := range s.Metadata { |
||||||
|
if metric == m.Metric { |
||||||
|
return m, true |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return scrape.MetricMetadata{}, false |
||||||
|
} |
||||||
|
|
||||||
|
func (s *TestMetaStore) SizeMetadata() int { return 0 } |
||||||
|
func (s *TestMetaStore) LengthMetadata() int { return 0 } |
||||||
|
|
||||||
|
type writeMetadataToMock struct { |
||||||
|
metadataAppended int |
||||||
|
} |
||||||
|
|
||||||
|
func (mwtm *writeMetadataToMock) AppendMetadata(_ context.Context, m []scrape.MetricMetadata) { |
||||||
|
mwtm.metadataAppended += len(m) |
||||||
|
} |
||||||
|
|
||||||
|
func newMetadataWriteToMock() *writeMetadataToMock { |
||||||
|
return &writeMetadataToMock{} |
||||||
|
} |
||||||
|
|
||||||
|
type scrapeManagerMock struct { |
||||||
|
manager *scrape.Manager |
||||||
|
ready bool |
||||||
|
} |
||||||
|
|
||||||
|
func (smm *scrapeManagerMock) Get() (*scrape.Manager, error) { |
||||||
|
if smm.ready { |
||||||
|
return smm.manager, nil |
||||||
|
} |
||||||
|
|
||||||
|
return nil, errors.New("not ready") |
||||||
|
} |
||||||
|
|
||||||
|
type fakeManager struct { |
||||||
|
activeTargets map[string][]*scrape.Target |
||||||
|
} |
||||||
|
|
||||||
|
func (fm *fakeManager) TargetsActive() map[string][]*scrape.Target { |
||||||
|
return fm.activeTargets |
||||||
|
} |
||||||
|
|
||||||
|
func TestWatchScrapeManager_NotReady(t *testing.T) { |
||||||
|
wt := newMetadataWriteToMock() |
||||||
|
smm := &scrapeManagerMock{ |
||||||
|
ready: false, |
||||||
|
} |
||||||
|
|
||||||
|
mw := NewMetadataWatcher(nil, smm, "", wt, interval, deadline) |
||||||
|
require.Equal(t, false, mw.ready()) |
||||||
|
|
||||||
|
mw.collect() |
||||||
|
|
||||||
|
require.Equal(t, 0, wt.metadataAppended) |
||||||
|
} |
||||||
|
|
||||||
|
func TestWatchScrapeManager_ReadyForCollection(t *testing.T) { |
||||||
|
wt := newMetadataWriteToMock() |
||||||
|
|
||||||
|
metadata := &TestMetaStore{ |
||||||
|
Metadata: []scrape.MetricMetadata{ |
||||||
|
{ |
||||||
|
Metric: "prometheus_tsdb_head_chunks_created_total", |
||||||
|
Type: textparse.MetricTypeCounter, |
||||||
|
Help: "Total number", |
||||||
|
Unit: "", |
||||||
|
}, |
||||||
|
{ |
||||||
|
Metric: "prometheus_remote_storage_retried_samples_total", |
||||||
|
Type: textparse.MetricTypeCounter, |
||||||
|
Help: "Total number", |
||||||
|
Unit: "", |
||||||
|
}, |
||||||
|
}, |
||||||
|
} |
||||||
|
metadataDup := &TestMetaStore{ |
||||||
|
Metadata: []scrape.MetricMetadata{ |
||||||
|
{ |
||||||
|
Metric: "prometheus_tsdb_head_chunks_created_total", |
||||||
|
Type: textparse.MetricTypeCounter, |
||||||
|
Help: "Total number", |
||||||
|
Unit: "", |
||||||
|
}, |
||||||
|
}, |
||||||
|
} |
||||||
|
|
||||||
|
target := &scrape.Target{} |
||||||
|
target.SetMetadataStore(metadata) |
||||||
|
targetWithDup := &scrape.Target{} |
||||||
|
targetWithDup.SetMetadataStore(metadataDup) |
||||||
|
|
||||||
|
manager := &fakeManager{ |
||||||
|
activeTargets: map[string][]*scrape.Target{ |
||||||
|
"job": []*scrape.Target{target}, |
||||||
|
"dup": []*scrape.Target{targetWithDup}, |
||||||
|
}, |
||||||
|
} |
||||||
|
|
||||||
|
smm := &scrapeManagerMock{ |
||||||
|
ready: true, |
||||||
|
} |
||||||
|
|
||||||
|
mw := NewMetadataWatcher(nil, smm, "", wt, interval, deadline) |
||||||
|
mw.manager = manager |
||||||
|
|
||||||
|
mw.collect() |
||||||
|
|
||||||
|
require.Equal(t, 2, wt.metadataAppended) |
||||||
|
} |
||||||
Loading…
Reference in new issue