diff --git a/documentation/examples/prometheus-kubernetes.yml b/documentation/examples/prometheus-kubernetes.yml index e0f6df447..e91d0f9a3 100644 --- a/documentation/examples/prometheus-kubernetes.yml +++ b/documentation/examples/prometheus-kubernetes.yml @@ -28,14 +28,11 @@ scrape_configs: regex: (?:apiserver|node) - action: labelmap regex: __meta_kubernetes_node_label_(.+) - replacement: $1 - source_labels: [__meta_kubernetes_role] action: replace - regex: (.+) target_label: kubernetes_role - replacement: $1 -# Scrape config for services. +# Scrape config for service endpoints. # # The relabeling allows the actual service scrape endpoint to be configured # via the following annotations: @@ -46,7 +43,7 @@ scrape_configs: # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. # * `prometheus.io/port`: If the metrics are exposed on a different port to the # service then set this appropriately. -- job_name: 'kubernetes-services' +- job_name: 'kubernetes-service-endpoints' kubernetes_sd_configs: - api_servers: @@ -54,19 +51,16 @@ scrape_configs: in_cluster: true relabel_configs: - - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + - source_labels: [__meta_kubernetes_role, __meta_kubernetes_service_annotation_prometheus_io_scrape] action: keep - regex: true + regex: endpoint;true - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ regex: (https?) - replacement: $1 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ - regex: (.+) - replacement: $1 - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace target_label: __address__ @@ -74,19 +68,53 @@ scrape_configs: replacement: $1:$2 - action: labelmap regex: __meta_kubernetes_service_label_(.+) - replacement: $1 - source_labels: [__meta_kubernetes_role] action: replace - regex: (.+) target_label: kubernetes_role - replacement: $1 - source_labels: [__meta_kubernetes_service_namespace] action: replace - regex: (.+) target_label: kubernetes_namespace - replacement: $1 - source_labels: [__meta_kubernetes_service_name] action: replace - regex: (.+) target_label: kubernetes_name - replacement: $1 + +# Example scrape config for probing services via the Blackbox Exporter. +# +# The relabeling allows the actual service scrape endpoint to be configured +# via the following annotations: +# +# * `prometheus.io/probe`: Only probe services that have a value of `true` +- job_name: 'kubernetes-services' + + metrics_path: /probe + params: + module: [http_2xx] + + kubernetes_sd_configs: + - api_servers: + - 'https://kubernetes.default.svc' + in_cluster: true + + relabel_configs: + - source_labels: [__meta_kubernetes_role, __meta_kubernetes_service_annotation_prometheus_io_probe] + action: keep + regex: service;true + - source_labels: [] + target_label: __address__ + replacement: prom-blackbox.default.svc:9115 + - source_labels: [__address__] + regex: (.*)(:80)? + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_role] + action: replace + target_label: kubernetes_role + - source_labels: [__meta_kubernetes_service_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name diff --git a/retrieval/discovery/kubernetes/discovery.go b/retrieval/discovery/kubernetes/discovery.go index e45c4a48c..e4cf19944 100644 --- a/retrieval/discovery/kubernetes/discovery.go +++ b/retrieval/discovery/kubernetes/discovery.go @@ -501,7 +501,6 @@ func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints) Labels: model.LabelSet{ serviceNamespaceLabel: model.LabelValue(service.ObjectMeta.Namespace), serviceNameLabel: model.LabelValue(service.ObjectMeta.Name), - roleLabel: model.LabelValue("service"), }, } @@ -515,6 +514,12 @@ func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints) tg.Labels[model.LabelName(labelName)] = model.LabelValue(v) } + t := model.LabelSet{ + model.AddressLabel: model.LabelValue(service.ObjectMeta.Name + "." + service.ObjectMeta.Namespace + ".svc"), + roleLabel: model.LabelValue("service"), + } + tg.Targets = append(tg.Targets, t) + // Now let's loop through the endpoints & add them to the target group with appropriate labels. for _, ss := range eps.Subsets { epPort := ss.Ports[0].Port @@ -526,7 +531,10 @@ func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints) } address := fmt.Sprintf("%s:%d", ipAddr, epPort) - t := model.LabelSet{model.AddressLabel: model.LabelValue(address)} + t := model.LabelSet{ + model.AddressLabel: model.LabelValue(address), + roleLabel: model.LabelValue("endpoint"), + } tg.Targets = append(tg.Targets, t) }