diff --git a/config/config_test.go b/config/config_test.go index 4d9155a38..22e48e449 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -45,6 +45,7 @@ import ( "github.com/prometheus/prometheus/discovery/marathon" "github.com/prometheus/prometheus/discovery/moby" "github.com/prometheus/prometheus/discovery/openstack" + "github.com/prometheus/prometheus/discovery/puppetdb" "github.com/prometheus/prometheus/discovery/scaleway" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/discovery/triton" @@ -790,6 +791,34 @@ var expectedConf = &Config{ }}, }, }, + { + JobName: "service-puppetdb", + + HonorTimestamps: true, + ScrapeInterval: model.Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + + MetricsPath: DefaultScrapeConfig.MetricsPath, + Scheme: DefaultScrapeConfig.Scheme, + HTTPClientConfig: config.DefaultHTTPClientConfig, + + ServiceDiscoveryConfigs: discovery.Configs{&puppetdb.SDConfig{ + URL: "https://puppetserver/", + Query: "resources { type = \"Package\" and title = \"httpd\" }", + IncludeParameters: true, + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), + HTTPClientConfig: config.HTTPClientConfig{ + FollowRedirects: true, + TLSConfig: config.TLSConfig{ + CAFile: "testdata/valid_ca_file", + CertFile: "testdata/valid_cert_file", + KeyFile: "testdata/valid_key_file", + }, + }, + }, + }, + }, { JobName: "hetzner", HonorTimestamps: true, @@ -1262,6 +1291,22 @@ var expectedErrors = []struct { filename: "empty_static_config.bad.yml", errMsg: "empty or null section in static_configs", }, + { + filename: "puppetdb_no_query.bad.yml", + errMsg: "query missing", + }, + { + filename: "puppetdb_no_url.bad.yml", + errMsg: "url missing", + }, + { + filename: "puppetdb_bad_url.bad.yml", + errMsg: "host is missing in URL", + }, + { + filename: "puppetdb_no_scheme.bad.yml", + errMsg: "url scheme must be http or https", + }, { filename: "hetzner_role.bad.yml", errMsg: "unknown role", diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index 19fe0c9b2..a439bd0a0 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -307,6 +307,18 @@ scrape_configs: cert_file: valid_cert_file key_file: valid_key_file + - job_name: service-puppetdb + puppetdb_sd_configs: + - url: https://puppetserver/ + query: 'resources { type = "Package" and title = "httpd" }' + include_parameters: true + port: 80 + refresh_interval: 1m + tls_config: + ca_file: valid_ca_file + cert_file: valid_cert_file + key_file: valid_key_file + - job_name: hetzner hetzner_sd_configs: - role: hcloud diff --git a/config/testdata/puppetdb_bad_url.bad.yml b/config/testdata/puppetdb_bad_url.bad.yml new file mode 100644 index 000000000..8a82ae9ec --- /dev/null +++ b/config/testdata/puppetdb_bad_url.bad.yml @@ -0,0 +1,4 @@ +scrape_configs: + - puppetdb_sd_configs: + - url: http:// + query: 'resources { type = "Package" and title = "httpd" }' diff --git a/config/testdata/puppetdb_no_query.bad.yml b/config/testdata/puppetdb_no_query.bad.yml new file mode 100644 index 000000000..5d4c3f999 --- /dev/null +++ b/config/testdata/puppetdb_no_query.bad.yml @@ -0,0 +1,3 @@ +scrape_configs: + - puppetdb_sd_configs: + - url: http://puppetserver/ diff --git a/config/testdata/puppetdb_no_scheme.bad.yml b/config/testdata/puppetdb_no_scheme.bad.yml new file mode 100644 index 000000000..e980aa6b8 --- /dev/null +++ b/config/testdata/puppetdb_no_scheme.bad.yml @@ -0,0 +1,4 @@ +scrape_configs: + - puppetdb_sd_configs: + - url: ftp://puppet + query: 'resources { type = "Package" and title = "httpd" }' diff --git a/config/testdata/puppetdb_no_url.bad.yml b/config/testdata/puppetdb_no_url.bad.yml new file mode 100644 index 000000000..b806be16a --- /dev/null +++ b/config/testdata/puppetdb_no_url.bad.yml @@ -0,0 +1,3 @@ +scrape_configs: + - puppetdb_sd_configs: + - query: 'resources { type = "Package" and title = "httpd" }' diff --git a/discovery/install/install.go b/discovery/install/install.go index 34ccf3d0f..88cf67ca7 100644 --- a/discovery/install/install.go +++ b/discovery/install/install.go @@ -31,6 +31,7 @@ import ( _ "github.com/prometheus/prometheus/discovery/marathon" // register marathon _ "github.com/prometheus/prometheus/discovery/moby" // register moby _ "github.com/prometheus/prometheus/discovery/openstack" // register openstack + _ "github.com/prometheus/prometheus/discovery/puppetdb" // register puppetdb _ "github.com/prometheus/prometheus/discovery/scaleway" // register scaleway _ "github.com/prometheus/prometheus/discovery/triton" // register triton _ "github.com/prometheus/prometheus/discovery/xds" // register xds diff --git a/discovery/puppetdb/fixtures/vhosts.json b/discovery/puppetdb/fixtures/vhosts.json new file mode 100644 index 000000000..5ea7b05a2 --- /dev/null +++ b/discovery/puppetdb/fixtures/vhosts.json @@ -0,0 +1,49 @@ +[ + { + "certname": "edinburgh.example.com", + "environment": "prod", + "exported": false, + "file": "/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp", + "line": 384, + "parameters": { + "access_log": true, + "access_log_file": "ssl_access_log", + "additional_includes": [ ], + "directoryindex": "", + "docroot": "/var/www/html", + "ensure": "absent", + "options": [ + "Indexes", + "FollowSymLinks", + "MultiViews" + ], + "php_flags": { }, + "labels": { + "alias": "edinburgh" + }, + "scriptaliases": [ + { + "alias": "/cgi-bin", + "path": "/var/www/cgi-bin" + } + ] + }, + "resource": "49af83866dc5a1518968b68e58a25319107afe11", + "tags": [ + "roles::hypervisor", + "apache", + "apache::vhost", + "class", + "default-ssl", + "profile_hypervisor", + "vhost", + "profile_apache", + "hypervisor", + "__node_regexp__edinburgh", + "roles", + "node" + ], + "title": "default-ssl", + "type": "Apache::Vhost" + } +] diff --git a/discovery/puppetdb/puppetdb.go b/discovery/puppetdb/puppetdb.go new file mode 100644 index 000000000..cad07b33d --- /dev/null +++ b/discovery/puppetdb/puppetdb.go @@ -0,0 +1,252 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package puppetdb + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/go-kit/log" + "github.com/pkg/errors" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/version" + + "github.com/prometheus/prometheus/discovery" + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +const ( + pdbLabel = model.MetaLabelPrefix + "puppetdb_" + pdbLabelCertname = pdbLabel + "certname" + pdbLabelResource = pdbLabel + "resource" + pdbLabelType = pdbLabel + "type" + pdbLabelTitle = pdbLabel + "title" + pdbLabelExported = pdbLabel + "exported" + pdbLabelTags = pdbLabel + "tags" + pdbLabelFile = pdbLabel + "file" + pdbLabelEnvironment = pdbLabel + "environment" + pdbLabelParameter = pdbLabel + "parameter_" + separator = "," +) + +var ( + // DefaultSDConfig is the default PuppetDB SD configuration. + DefaultSDConfig = SDConfig{ + RefreshInterval: model.Duration(60 * time.Second), + Port: 80, + HTTPClientConfig: config.DefaultHTTPClientConfig, + } + matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`) + userAgent = fmt.Sprintf("Prometheus/%s", version.Version) +) + +func init() { + discovery.RegisterConfig(&SDConfig{}) +} + +// SDConfig is the configuration for PuppetDB based discovery. +type SDConfig struct { + HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + URL string `yaml:"url"` + Query string `yaml:"query"` + IncludeParameters bool `yaml:"include_parameters"` + Port int `yaml:"port"` +} + +// Name returns the name of the Config. +func (*SDConfig) Name() string { return "puppetdb" } + +// NewDiscoverer returns a Discoverer for the Config. +func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) { + return NewDiscovery(c, opts.Logger) +} + +// SetDirectory joins any relative file paths with dir. +func (c *SDConfig) SetDirectory(dir string) { + c.HTTPClientConfig.SetDirectory(dir) +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if c.URL == "" { + return fmt.Errorf("URL is missing") + } + parsedURL, err := url.Parse(c.URL) + if err != nil { + return err + } + if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { + return fmt.Errorf("URL scheme must be 'http' or 'https'") + } + if parsedURL.Host == "" { + return fmt.Errorf("host is missing in URL") + } + if c.Query == "" { + return fmt.Errorf("query missing") + } + return nil +} + +// Discovery provides service discovery functionality based +// on PuppetDB resources. +type Discovery struct { + *refresh.Discovery + url string + query string + port int + includeParameters bool + client *http.Client +} + +// NewDiscovery returns a new PuppetDB discovery for the given config. +func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { + if logger == nil { + logger = log.NewNopLogger() + } + + client, err := config.NewClientFromConfig(conf.HTTPClientConfig, "http", config.WithHTTP2Disabled()) + if err != nil { + return nil, err + } + client.Timeout = time.Duration(conf.RefreshInterval) + + u, err := url.Parse(conf.URL) + if err != nil { + return nil, err + } + u.Path = path.Join(u.Path, "pdb/query/v4") + + d := &Discovery{ + url: u.String(), + port: conf.Port, + query: conf.Query, + includeParameters: conf.IncludeParameters, + client: client, + } + + d.Discovery = refresh.NewDiscovery( + logger, + "http", + time.Duration(conf.RefreshInterval), + d.refresh, + ) + return d, nil +} + +func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + body := struct { + Query string `json:"query"` + }{d.query} + bodyBytes, err := json.Marshal(body) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", d.url, bytes.NewBuffer(bodyBytes)) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + + resp, err := d.client.Do(req.WithContext(ctx)) + if err != nil { + return nil, err + } + defer func() { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return nil, errors.Errorf("server returned HTTP status %s", resp.Status) + } + + if ct := resp.Header.Get("Content-Type"); !matchContentType.MatchString(ct) { + return nil, errors.Errorf("unsupported content type %s", resp.Header.Get("Content-Type")) + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var resources []Resource + + if err := json.Unmarshal(b, &resources); err != nil { + return nil, err + } + + tg := &targetgroup.Group{ + // Use a pseudo-URL as source. + Source: d.url + "?query=" + d.query, + } + + for _, resource := range resources { + labels := model.LabelSet{ + pdbLabelCertname: model.LabelValue(resource.Certname), + pdbLabelResource: model.LabelValue(resource.Resource), + pdbLabelType: model.LabelValue(resource.Type), + pdbLabelTitle: model.LabelValue(resource.Title), + pdbLabelExported: model.LabelValue(fmt.Sprintf("%t", resource.Exported)), + pdbLabelFile: model.LabelValue(resource.File), + pdbLabelEnvironment: model.LabelValue(resource.Environment), + } + + addr := net.JoinHostPort(resource.Certname, strconv.FormatUint(uint64(d.port), 10)) + labels[model.AddressLabel] = model.LabelValue(addr) + + if len(resource.Tags) > 0 { + // We surround the separated list with the separator as well. This way regular expressions + // in relabeling rules don't have to consider tag positions. + tags := separator + strings.Join(resource.Tags, separator) + separator + labels[pdbLabelTags] = model.LabelValue(tags) + } + + // Parameters are not included by default. This should only be enabled + // on select resources as it might expose secrets on the Prometheus UI + // for certain resources. + if d.includeParameters { + for k, v := range resource.Parameters.toLabels() { + labels[pdbLabelParameter+k] = v + } + } + + tg.Targets = append(tg.Targets, labels) + } + + return []*targetgroup.Group{tg}, nil +} diff --git a/discovery/puppetdb/puppetdb_test.go b/discovery/puppetdb/puppetdb_test.go new file mode 100644 index 000000000..3fcfab549 --- /dev/null +++ b/discovery/puppetdb/puppetdb_test.go @@ -0,0 +1,195 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package puppetdb + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/stretchr/testify/require" +) + +func mockServer(t *testing.T) *httptest.Server { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request struct { + Query string `json:"query"` + } + err := json.NewDecoder(r.Body).Decode(&request) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + http.ServeFile(w, r, "fixtures/"+request.Query+".json") + })) + t.Cleanup(ts.Close) + return ts +} + +func TestPuppetSlashInURL(t *testing.T) { + tests := map[string]string{ + "https://puppetserver": "https://puppetserver/pdb/query/v4", + "https://puppetserver/": "https://puppetserver/pdb/query/v4", + "http://puppetserver:8080/": "http://puppetserver:8080/pdb/query/v4", + "http://puppetserver:8080": "http://puppetserver:8080/pdb/query/v4", + } + + for serverURL, apiURL := range tests { + cfg := SDConfig{ + HTTPClientConfig: config.DefaultHTTPClientConfig, + URL: serverURL, + Query: "vhosts", // This is not a valid PuppetDB query, but it is used by the mock. + Port: 80, + RefreshInterval: model.Duration(30 * time.Second), + } + d, err := NewDiscovery(&cfg, log.NewNopLogger()) + require.NoError(t, err) + require.Equal(t, apiURL, d.url) + } +} + +func TestPuppetDBRefresh(t *testing.T) { + ts := mockServer(t) + + cfg := SDConfig{ + HTTPClientConfig: config.DefaultHTTPClientConfig, + URL: ts.URL, + Query: "vhosts", // This is not a valid PuppetDB query, but it is used by the mock. + Port: 80, + RefreshInterval: model.Duration(30 * time.Second), + } + + d, err := NewDiscovery(&cfg, log.NewNopLogger()) + require.NoError(t, err) + + ctx := context.Background() + tgs, err := d.refresh(ctx) + require.NoError(t, err) + + expectedTargets := []*targetgroup.Group{ + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("edinburgh.example.com:80"), + model.LabelName("__meta_puppetdb_certname"): model.LabelValue("edinburgh.example.com"), + model.LabelName("__meta_puppetdb_environment"): model.LabelValue("prod"), + model.LabelName("__meta_puppetdb_exported"): model.LabelValue("false"), + model.LabelName("__meta_puppetdb_file"): model.LabelValue("/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp"), + model.LabelName("__meta_puppetdb_resource"): model.LabelValue("49af83866dc5a1518968b68e58a25319107afe11"), + model.LabelName("__meta_puppetdb_tags"): model.LabelValue(",roles::hypervisor,apache,apache::vhost,class,default-ssl,profile_hypervisor,vhost,profile_apache,hypervisor,__node_regexp__edinburgh,roles,node,"), + model.LabelName("__meta_puppetdb_title"): model.LabelValue("default-ssl"), + model.LabelName("__meta_puppetdb_type"): model.LabelValue("Apache::Vhost"), + }, + }, + Source: ts.URL + "/pdb/query/v4?query=vhosts", + }, + } + require.Equal(t, tgs, expectedTargets) +} + +func TestPuppetDBRefreshWithParameters(t *testing.T) { + ts := mockServer(t) + + cfg := SDConfig{ + HTTPClientConfig: config.DefaultHTTPClientConfig, + URL: ts.URL, + Query: "vhosts", // This is not a valid PuppetDB query, but it is used by the mock. + Port: 80, + IncludeParameters: true, + RefreshInterval: model.Duration(30 * time.Second), + } + + d, err := NewDiscovery(&cfg, log.NewNopLogger()) + require.NoError(t, err) + + ctx := context.Background() + tgs, err := d.refresh(ctx) + require.NoError(t, err) + + expectedTargets := []*targetgroup.Group{ + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue("edinburgh.example.com:80"), + model.LabelName("__meta_puppetdb_certname"): model.LabelValue("edinburgh.example.com"), + model.LabelName("__meta_puppetdb_environment"): model.LabelValue("prod"), + model.LabelName("__meta_puppetdb_exported"): model.LabelValue("false"), + model.LabelName("__meta_puppetdb_file"): model.LabelValue("/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp"), + model.LabelName("__meta_puppetdb_parameter_access_log"): model.LabelValue("true"), + model.LabelName("__meta_puppetdb_parameter_access_log_file"): model.LabelValue("ssl_access_log"), + model.LabelName("__meta_puppetdb_parameter_docroot"): model.LabelValue("/var/www/html"), + model.LabelName("__meta_puppetdb_parameter_ensure"): model.LabelValue("absent"), + model.LabelName("__meta_puppetdb_parameter_labels_alias"): model.LabelValue("edinburgh"), + model.LabelName("__meta_puppetdb_parameter_options"): model.LabelValue("Indexes,FollowSymLinks,MultiViews"), + model.LabelName("__meta_puppetdb_resource"): model.LabelValue("49af83866dc5a1518968b68e58a25319107afe11"), + model.LabelName("__meta_puppetdb_tags"): model.LabelValue(",roles::hypervisor,apache,apache::vhost,class,default-ssl,profile_hypervisor,vhost,profile_apache,hypervisor,__node_regexp__edinburgh,roles,node,"), + model.LabelName("__meta_puppetdb_title"): model.LabelValue("default-ssl"), + model.LabelName("__meta_puppetdb_type"): model.LabelValue("Apache::Vhost"), + }, + }, + Source: ts.URL + "/pdb/query/v4?query=vhosts", + }, + } + require.Equal(t, tgs, expectedTargets) +} + +func TestPuppetDBInvalidCode(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + + t.Cleanup(ts.Close) + + cfg := SDConfig{ + HTTPClientConfig: config.DefaultHTTPClientConfig, + URL: ts.URL, + RefreshInterval: model.Duration(30 * time.Second), + } + + d, err := NewDiscovery(&cfg, log.NewNopLogger()) + require.NoError(t, err) + + ctx := context.Background() + _, err = d.refresh(ctx) + require.EqualError(t, err, "server returned HTTP status 400 Bad Request") +} + +func TestPuppetDBInvalidFormat(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "{}") + })) + + t.Cleanup(ts.Close) + + cfg := SDConfig{ + HTTPClientConfig: config.DefaultHTTPClientConfig, + URL: ts.URL, + RefreshInterval: model.Duration(30 * time.Second), + } + + d, err := NewDiscovery(&cfg, log.NewNopLogger()) + require.NoError(t, err) + + ctx := context.Background() + _, err = d.refresh(ctx) + require.EqualError(t, err, "unsupported content type text/plain; charset=utf-8") +} diff --git a/discovery/puppetdb/resources.go b/discovery/puppetdb/resources.go new file mode 100644 index 000000000..64b3a781e --- /dev/null +++ b/discovery/puppetdb/resources.go @@ -0,0 +1,82 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package puppetdb + +import ( + "strconv" + "strings" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/util/strutil" +) + +type Resource struct { + Certname string `json:"certname"` + Resource string `json:"resource"` + Type string `json:"type"` + Title string `json:"title"` + Exported bool `json:"exported"` + Tags []string `json:"tags"` + File string `json:"file"` + Environment string `json:"environment"` + Parameters Parameters `json:"parameters"` +} + +type Parameters map[string]interface{} + +func (p *Parameters) toLabels() model.LabelSet { + labels := model.LabelSet{} + + for k, v := range *p { + var labelValue string + switch value := v.(type) { + case string: + labelValue = value + case bool: + labelValue = strconv.FormatBool(value) + case []string: + labelValue = separator + strings.Join(value, separator) + separator + case []interface{}: + if len(value) == 0 { + continue + } + values := make([]string, len(value)) + for i, v := range value { + switch value := v.(type) { + case string: + values[i] = value + case bool: + values[i] = strconv.FormatBool(value) + case []string: + values[i] = separator + strings.Join(value, separator) + separator + } + } + labelValue = strings.Join(values, separator) + case map[string]interface{}: + subParameter := Parameters(value) + prefix := strutil.SanitizeLabelName(k + "_") + for subk, subv := range subParameter.toLabels() { + labels[model.LabelName(prefix)+subk] = subv + } + default: + continue + } + if labelValue == "" { + continue + } + name := strutil.SanitizeLabelName(k) + labels[model.LabelName(name)] = model.LabelValue(labelValue) + } + return labels +} diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index dd84d9ad7..9884a61ae 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -272,6 +272,10 @@ nerve_sd_configs: openstack_sd_configs: [ - ... ] +# List of PuppetDB service discovery configurations. +puppetdb_sd_configs: + [ - ... ] + # List of Scaleway service discovery configurations. scaleway_sd_configs: [ - ... ] @@ -1069,6 +1073,94 @@ tls_config: [ ] ``` +### `` + +PuppetDB SD configurations allow retrieving scrape targets from +[PuppetDB](https://puppet.com/docs/puppetdb/latest/index.html) resources. + +This SD discovers resources and will create a target for each resource returned +by the API. + +The resource address is the `certname` of the resource and can be changed during +[relabeling](#relabel_config). + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_puppetdb_certname`: the name of the node associated with the resource +* `__meta_puppetdb_resource`: a SHA-1 hash of the resource’s type, title, and parameters, for identification +* `__meta_puppetdb_type`: the resource type +* `__meta_puppetdb_title`: the resource title +* `__meta_puppetdb_exported`: whether the resource is exported (`"true"` or `"false"`) +* `__meta_puppetdb_tags`: comma separated list of resource tags +* `__meta_puppetdb_file`: the manifest file in which the resource was declared +* `__meta_puppetdb_environment`: the environment of the node associated with the resource +* `__meta_puppetdb_parameter_`: the parameters of the resource + + +See below for the configuration options for PuppetDB discovery: + +```yaml +# The URL of the PuppetDB root query endpoint. +url: + +# Puppet Query Language (PQL) query. Only resources are supported. +# https://puppet.com/docs/puppetdb/latest/api/query/v4/pql.html +query: + +# Whether to include the parameters as meta labels. +# Due to the differences between parameter types and Prometheus labels, +# some parameters might not be rendered. The format of the parameters might +# also change in future releases. +# +# Note: Enabling this exposes parameters in the Prometheus UI and API. Make sure +# that you don't have secrets exposed as parameters if you enable this. +[ include_parameters: | default = false ] + +# Refresh interval to re-read the resources list. +[ refresh_interval: | default = 60s ] + +# The port to scrape metrics from. +[ port: | default = 80 ] + +# TLS configuration to connect to the PuppetDB. +tls_config: + [ ] + +# basic_auth, authorization, and oauth2, are mutually exclusive. + +# Optional HTTP basic authentication information. +basic_auth: + [ username: ] + [ password: ] + [ password_file: ] + +# `Authorization` HTTP header configuration. +authorization: + # Sets the authentication type. + [ type: | default: Bearer ] + # Sets the credentials. It is mutually exclusive with + # `credentials_file`. + [ credentials: ] + # Sets the credentials with the credentials read from the configured file. + # It is mutually exclusive with `credentials`. + [ credentials_file: ] + +# Optional OAuth 2.0 configuration. +# Cannot be used at the same time as basic_auth or authorization. +oauth2: + [ ] + +# Optional proxy URL. +[ proxy_url: ] + +# Configure whether HTTP requests follow HTTP 3xx redirects. +[ follow_redirects: | default = true ] +``` + +See [this example Prometheus configuration file](/documentation/examples/prometheus-puppetdb.yml) +for a detailed example of configuring Prometheus with PuppetDB. + + ### `` File-based service discovery provides a more generic way to configure static targets @@ -2387,6 +2479,10 @@ nerve_sd_configs: openstack_sd_configs: [ - ... ] +# List of PuppetDB service discovery configurations. +puppetdb_sd_configs: + [ - ... ] + # List of Scaleway service discovery configurations. scaleway_sd_configs: [ - ... ] diff --git a/documentation/examples/prometheus-puppetdb.yml b/documentation/examples/prometheus-puppetdb.yml new file mode 100644 index 000000000..f7f4313e7 --- /dev/null +++ b/documentation/examples/prometheus-puppetdb.yml @@ -0,0 +1,40 @@ +# Prometheus example configuration to be used with PuppetDB. + +scrape_configs: + - job_name: 'puppetdb-node-exporter' + puppetdb_sd_configs: + # This example discovers the nodes which have the class Prometheus::Node_exporter. + - url: https://puppetdb.example.com + query: 'resources { type = "Class" and title = "Prometheus::Node_exporter" }' + port: 9100 + tls_config: + cert_file: prometheus-public.pem + key_file: prometheus-private.pem + ca_file: ca.pem + + - job_name: 'puppetdb-scrape-jobs' + puppetdb_sd_configs: + # This example uses the Prometheus::Scrape_job + # exported resources. + # https://github.com/camptocamp/prometheus-puppetdb-sd + # This examples is compatible with Prometheus-puppetdb-sd, + # if the exported Prometheus::Scrape_job only have at most one target. + - url: https://puppetdb.example.com + query: 'resources { type = "Prometheus::Scrape_job" and exported = true }' + include_parameters: true + tls_config: + cert_file: prometheus-public.pem + key_file: prometheus-private.pem + ca_file: ca.pem + relabel_configs: + - source_labels: [__meta_puppetdb_certname] + target_label: certname + - source_labels: [__meta_puppetdb_parameter_targets] + regex: '(.+),?.*' + replacement: $1 + target_label: __address__ + - source_labels: [__meta_puppetdb_parameter_job_name] + target_label: job + - regex: '__meta_puppetdb_parameter_labels_(.+)' + replacement: '$1' + action: labelmap