From e1e30f12cd5b116abe0d0165e573bee1a924cad2 Mon Sep 17 00:00:00 2001 From: Dominik Schulz Date: Fri, 21 Oct 2016 10:48:28 +0200 Subject: [PATCH 1/3] Add Kubernetes-SD metrics. --- retrieval/discovery/kubernetes/endpoints.go | 3 ++ retrieval/discovery/kubernetes/kubernetes.go | 33 ++++++++++++++++++++ retrieval/discovery/kubernetes/node.go | 3 ++ retrieval/discovery/kubernetes/pod.go | 3 ++ retrieval/discovery/kubernetes/service.go | 3 ++ 5 files changed, 45 insertions(+) diff --git a/retrieval/discovery/kubernetes/endpoints.go b/retrieval/discovery/kubernetes/endpoints.go index cb7a9e551..696298dfb 100644 --- a/retrieval/discovery/kubernetes/endpoints.go +++ b/retrieval/discovery/kubernetes/endpoints.go @@ -83,12 +83,15 @@ func (e *Endpoints) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { e.endpointsInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { + eventCount.WithLabelValues("endpoint", "add").Inc() send(e.buildEndpoints(o.(*apiv1.Endpoints))) }, UpdateFunc: func(_, o interface{}) { + eventCount.WithLabelValues("endpoint", "update").Inc() send(e.buildEndpoints(o.(*apiv1.Endpoints))) }, DeleteFunc: func(o interface{}) { + eventCount.WithLabelValues("endpoint", "delete").Inc() send(&config.TargetGroup{Source: endpointsSource(o.(*apiv1.Endpoints).ObjectMeta)}) }, }) diff --git a/retrieval/discovery/kubernetes/kubernetes.go b/retrieval/discovery/kubernetes/kubernetes.go index ed228a5ad..b5cb59662 100644 --- a/retrieval/discovery/kubernetes/kubernetes.go +++ b/retrieval/discovery/kubernetes/kubernetes.go @@ -17,6 +17,7 @@ import ( "io/ioutil" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/config" "github.com/prometheus/common/log" @@ -35,8 +36,40 @@ const ( // in this discovery. metaLabelPrefix = model.MetaLabelPrefix + "kubernetes_" namespaceLabel = metaLabelPrefix + "namespace" + + // Constants for instrumentation. + namespace = "prometheus" +) + +var ( + eventCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "sd_kubernetes_events_total", + Help: "The number of Kubernetes events received.", + }, + []string{"type", "event"}, + ) ) +func init() { + prometheus.MustRegister(eventCount) + + // Initialize metric vectors. + eventCount.WithLabelValues("endpoint", "add") + eventCount.WithLabelValues("endpoint", "delete") + eventCount.WithLabelValues("endpoint", "update") + eventCount.WithLabelValues("node", "add") + eventCount.WithLabelValues("node", "delete") + eventCount.WithLabelValues("node", "update") + eventCount.WithLabelValues("pod", "add") + eventCount.WithLabelValues("pod", "delete") + eventCount.WithLabelValues("pod", "update") + eventCount.WithLabelValues("service", "add") + eventCount.WithLabelValues("service", "delete") + eventCount.WithLabelValues("service", "update") +} + // Kubernetes implements the TargetProvider interface for discovering // targets from Kubernetes. type Kubernetes struct { diff --git a/retrieval/discovery/kubernetes/node.go b/retrieval/discovery/kubernetes/node.go index e2d684b84..4b8faf730 100644 --- a/retrieval/discovery/kubernetes/node.go +++ b/retrieval/discovery/kubernetes/node.go @@ -63,12 +63,15 @@ func (n *Node) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } n.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { + eventCount.WithLabelValues("node", "add").Inc() send(n.buildNode(o.(*apiv1.Node))) }, DeleteFunc: func(o interface{}) { + eventCount.WithLabelValues("node", "delete").Inc() send(&config.TargetGroup{Source: nodeSource(o.(*apiv1.Node))}) }, UpdateFunc: func(_, o interface{}) { + eventCount.WithLabelValues("node", "update").Inc() send(n.buildNode(o.(*apiv1.Node))) }, }) diff --git a/retrieval/discovery/kubernetes/pod.go b/retrieval/discovery/kubernetes/pod.go index 3d16ea8b9..d2f6406a4 100644 --- a/retrieval/discovery/kubernetes/pod.go +++ b/retrieval/discovery/kubernetes/pod.go @@ -71,12 +71,15 @@ func (p *Pod) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } p.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { + eventCount.WithLabelValues("pod", "add").Inc() send(p.buildPod(o.(*apiv1.Pod))) }, DeleteFunc: func(o interface{}) { + eventCount.WithLabelValues("pod", "delete").Inc() send(&config.TargetGroup{Source: podSource(o.(*apiv1.Pod))}) }, UpdateFunc: func(_, o interface{}) { + eventCount.WithLabelValues("pod", "update").Inc() send(p.buildPod(o.(*apiv1.Pod))) }, }) diff --git a/retrieval/discovery/kubernetes/service.go b/retrieval/discovery/kubernetes/service.go index bb6cb6631..b483423d0 100644 --- a/retrieval/discovery/kubernetes/service.go +++ b/retrieval/discovery/kubernetes/service.go @@ -61,12 +61,15 @@ func (s *Service) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { } s.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(o interface{}) { + eventCount.WithLabelValues("service", "add").Inc() send(s.buildService(o.(*apiv1.Service))) }, DeleteFunc: func(o interface{}) { + eventCount.WithLabelValues("service", "delete").Inc() send(&config.TargetGroup{Source: serviceSource(o.(*apiv1.Service))}) }, UpdateFunc: func(_, o interface{}) { + eventCount.WithLabelValues("service", "update").Inc() send(s.buildService(o.(*apiv1.Service))) }, }) From 3d0fb0cf17145a57a96d29ddf345d89899b5b89b Mon Sep 17 00:00:00 2001 From: Dominik Schulz Date: Fri, 21 Oct 2016 12:11:15 +0200 Subject: [PATCH 2/3] Avoid too generic label type. --- retrieval/discovery/kubernetes/kubernetes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/retrieval/discovery/kubernetes/kubernetes.go b/retrieval/discovery/kubernetes/kubernetes.go index b5cb59662..7b02badd8 100644 --- a/retrieval/discovery/kubernetes/kubernetes.go +++ b/retrieval/discovery/kubernetes/kubernetes.go @@ -46,9 +46,9 @@ var ( prometheus.CounterOpts{ Namespace: namespace, Name: "sd_kubernetes_events_total", - Help: "The number of Kubernetes events received.", + Help: "The number of Kubernetes events handled.", }, - []string{"type", "event"}, + []string{"role", "event"}, ) ) From eb10ff987116bbf82d93b76b9ec9a23ced4362fd Mon Sep 17 00:00:00 2001 From: Dominik Schulz Date: Sun, 23 Oct 2016 13:33:54 +0200 Subject: [PATCH 3/3] Also handle service update in endpoints.go --- retrieval/discovery/kubernetes/endpoints.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/retrieval/discovery/kubernetes/endpoints.go b/retrieval/discovery/kubernetes/endpoints.go index 696298dfb..0ca93f837 100644 --- a/retrieval/discovery/kubernetes/endpoints.go +++ b/retrieval/discovery/kubernetes/endpoints.go @@ -111,9 +111,18 @@ func (e *Endpoints) Run(ctx context.Context, ch chan<- []*config.TargetGroup) { e.serviceInf.AddEventHandler(cache.ResourceEventHandlerFuncs{ // TODO(fabxc): potentially remove add and delete event handlers. Those should // be triggered via the endpoint handlers already. - AddFunc: func(o interface{}) { serviceUpdate(o.(*apiv1.Service)) }, - UpdateFunc: func(_, o interface{}) { serviceUpdate(o.(*apiv1.Service)) }, - DeleteFunc: func(o interface{}) { serviceUpdate(o.(*apiv1.Service)) }, + AddFunc: func(o interface{}) { + eventCount.WithLabelValues("service", "add").Inc() + serviceUpdate(o.(*apiv1.Service)) + }, + UpdateFunc: func(_, o interface{}) { + eventCount.WithLabelValues("service", "update").Inc() + serviceUpdate(o.(*apiv1.Service)) + }, + DeleteFunc: func(o interface{}) { + eventCount.WithLabelValues("service", "delete").Inc() + serviceUpdate(o.(*apiv1.Service)) + }, }) // Block until the target provider is explicitly canceled.