Merge pull request #2966 from alkalinecoffee/consul-node-metadata

Add support for consul's node metadata

discovery/consul/consul.go

@@ -27,6 +27,7 @@ import (
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/util/httputil"
+	"github.com/prometheus/prometheus/util/strutil"
 	"golang.org/x/net/context"
 )
 
@@ -38,6 +39,8 @@ const (
 	addressLabel = model.MetaLabelPrefix + "consul_address"
 	// nodeLabel is the name for the label containing a target's node name.
 	nodeLabel = model.MetaLabelPrefix + "consul_node"
+	// metaDataLabel is the prefix for the labels mapping to a target's metadata.
+	metaDataLabel = model.MetaLabelPrefix + "consul_metadata_"
 	// tagsLabel is the name of the label containing the tags assigned to the target.
 	tagsLabel = model.MetaLabelPrefix + "consul_tags"
 	// serviceLabel is the name of the label containing the service name.
@@ -294,7 +297,7 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*config.TargetG
 				addr = net.JoinHostPort(node.Address, fmt.Sprintf("%d", node.ServicePort))
 			}
 
-			tgroup.Targets = append(tgroup.Targets, model.LabelSet{
+			labels := model.LabelSet{
 				model.AddressLabel:  model.LabelValue(addr),
 				addressLabel:        model.LabelValue(node.Address),
 				nodeLabel:           model.LabelValue(node.Node),
@@ -302,7 +305,15 @@ func (srv *consulService) watch(ctx context.Context, ch chan<- []*config.TargetG
 				serviceAddressLabel: model.LabelValue(node.ServiceAddress),
 				servicePortLabel:    model.LabelValue(strconv.Itoa(node.ServicePort)),
 				serviceIDLabel:      model.LabelValue(node.ServiceID),
-			})
+			}
+
+			// Add all key/value pairs from the node's metadata as their own labels
+			for k, v := range node.NodeMeta {
+				name := strutil.SanitizeLabelName(k)
+				labels[metaDataLabel+model.LabelName(name)] = model.LabelValue(v)
+			}
+
+			tgroup.Targets = append(tgroup.Targets, labels)
 		}
 		// Check context twice to ensure we always catch cancelation.
 		select {

vendor/github.com/hashicorp/consul/api/agent.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"bufio"
 	"fmt"
 )
 
@@ -73,6 +74,8 @@ type AgentServiceCheck struct {
 	HTTP              string `json:",omitempty"`
 	TCP               string `json:",omitempty"`
 	Status            string `json:",omitempty"`
+	Notes             string `json:",omitempty"`
+	TLSSkipVerify     bool   `json:",omitempty"`
 
 	// In Consul 0.7 and later, checks that are associated with a service
 	// may also contain this optional DeregisterCriticalServiceAfter field,
@@ -114,6 +117,17 @@ func (a *Agent) Self() (map[string]map[string]interface{}, error) {
 	return out, nil
 }
 
+// Reload triggers a configuration reload for the agent we are connected to.
+func (a *Agent) Reload() error {
+	r := a.c.newRequest("PUT", "/v1/agent/reload")
+	_, resp, err := requireOK(a.c.doRequest(r))
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}
+
 // NodeName is used to get the node name of the agent
 func (a *Agent) NodeName() (string, error) {
 	if a.nodeName != "" {
@@ -345,6 +359,17 @@ func (a *Agent) Join(addr string, wan bool) error {
 	return nil
 }
 
+// Leave is used to have the agent gracefully leave the cluster and shutdown
+func (a *Agent) Leave() error {
+	r := a.c.newRequest("PUT", "/v1/agent/leave")
+	_, resp, err := requireOK(a.c.doRequest(r))
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}
+
 // ForceLeave is used to have the agent eject a failed node
 func (a *Agent) ForceLeave(node string) error {
 	r := a.c.newRequest("PUT", "/v1/agent/force-leave/"+node)
@@ -409,3 +434,38 @@ func (a *Agent) DisableNodeMaintenance() error {
 	resp.Body.Close()
 	return nil
 }
+
+// Monitor returns a channel which will receive streaming logs from the agent
+// Providing a non-nil stopCh can be used to close the connection and stop the
+// log stream
+func (a *Agent) Monitor(loglevel string, stopCh chan struct{}, q *QueryOptions) (chan string, error) {
+	r := a.c.newRequest("GET", "/v1/agent/monitor")
+	r.setQueryOptions(q)
+	if loglevel != "" {
+		r.params.Add("loglevel", loglevel)
+	}
+	_, resp, err := requireOK(a.c.doRequest(r))
+	if err != nil {
+		return nil, err
+	}
+
+	logCh := make(chan string, 64)
+	go func() {
+		defer resp.Body.Close()
+
+		scanner := bufio.NewScanner(resp.Body)
+		for {
+			select {
+			case <-stopCh:
+				close(logCh)
+				return
+			default:
+			}
+			if scanner.Scan() {
+				logCh <- scanner.Text()
+			}
+		}
+	}()
+
+	return logCh, nil
+}

vendor/github.com/hashicorp/consul/api/api.go

@@ -20,6 +20,28 @@ import (
 	"github.com/hashicorp/go-cleanhttp"
 )
 
+const (
+	// HTTPAddrEnvName defines an environment variable name which sets
+	// the HTTP address if there is no -http-addr specified.
+	HTTPAddrEnvName = "CONSUL_HTTP_ADDR"
+
+	// HTTPTokenEnvName defines an environment variable name which sets
+	// the HTTP token.
+	HTTPTokenEnvName = "CONSUL_HTTP_TOKEN"
+
+	// HTTPAuthEnvName defines an environment variable name which sets
+	// the HTTP authentication header.
+	HTTPAuthEnvName = "CONSUL_HTTP_AUTH"
+
+	// HTTPSSLEnvName defines an environment variable name which sets
+	// whether or not to use HTTPS.
+	HTTPSSLEnvName = "CONSUL_HTTP_SSL"
+
+	// HTTPSSLVerifyEnvName defines an environment variable name which sets
+	// whether or not to disable certificate checking.
+	HTTPSSLVerifyEnvName = "CONSUL_HTTP_SSL_VERIFY"
+)
+
 // QueryOptions are used to parameterize a query
 type QueryOptions struct {
 	// Providing a datacenter overwrites the DC provided
@@ -52,6 +74,11 @@ type QueryOptions struct {
 	// that node. Setting this to "_agent" will use the agent's node
 	// for the sort.
 	Near string
+
+	// NodeMeta is used to filter results by nodes with the given
+	// metadata key/value pairs. Currently, only one key/value pair can
+	// be provided for filtering.
+	NodeMeta map[string]string
 }
 
 // WriteOptions are used to parameterize a write
@@ -181,15 +208,15 @@ func defaultConfig(transportFn func() *http.Transport) *Config {
 		},
 	}
 
-	if addr := os.Getenv("CONSUL_HTTP_ADDR"); addr != "" {
+	if addr := os.Getenv(HTTPAddrEnvName); addr != "" {
 		config.Address = addr
 	}
 
-	if token := os.Getenv("CONSUL_HTTP_TOKEN"); token != "" {
+	if token := os.Getenv(HTTPTokenEnvName); token != "" {
 		config.Token = token
 	}
 
-	if auth := os.Getenv("CONSUL_HTTP_AUTH"); auth != "" {
+	if auth := os.Getenv(HTTPAuthEnvName); auth != "" {
 		var username, password string
 		if strings.Contains(auth, ":") {
 			split := strings.SplitN(auth, ":", 2)
@@ -205,10 +232,10 @@ func defaultConfig(transportFn func() *http.Transport) *Config {
 		}
 	}
 
-	if ssl := os.Getenv("CONSUL_HTTP_SSL"); ssl != "" {
+	if ssl := os.Getenv(HTTPSSLEnvName); ssl != "" {
 		enabled, err := strconv.ParseBool(ssl)
 		if err != nil {
-			log.Printf("[WARN] client: could not parse CONSUL_HTTP_SSL: %s", err)
+			log.Printf("[WARN] client: could not parse %s: %s", HTTPSSLEnvName, err)
 		}
 
 		if enabled {
@@ -216,10 +243,10 @@ func defaultConfig(transportFn func() *http.Transport) *Config {
 		}
 	}
 
-	if verify := os.Getenv("CONSUL_HTTP_SSL_VERIFY"); verify != "" {
+	if verify := os.Getenv(HTTPSSLVerifyEnvName); verify != "" {
 		doVerify, err := strconv.ParseBool(verify)
 		if err != nil {
-			log.Printf("[WARN] client: could not parse CONSUL_HTTP_SSL_VERIFY: %s", err)
+			log.Printf("[WARN] client: could not parse %s: %s", HTTPSSLVerifyEnvName, err)
 		}
 
 		if !doVerify {
@@ -364,6 +391,11 @@ func (r *request) setQueryOptions(q *QueryOptions) {
 	if q.Near != "" {
 		r.params.Set("near", q.Near)
 	}
+	if len(q.NodeMeta) > 0 {
+		for key, value := range q.NodeMeta {
+			r.params.Add("node-meta", key+":"+value)
+		}
+	}
 }
 
 // durToMsec converts a duration to a millisecond specified string. If the

vendor/github.com/hashicorp/consul/api/catalog.go

@@ -4,18 +4,22 @@ type Node struct {
 	Node            string
 	Address         string
 	TaggedAddresses map[string]string
+	Meta            map[string]string
 }
 
 type CatalogService struct {
 	Node                     string
 	Address                  string
 	TaggedAddresses          map[string]string
+	NodeMeta                 map[string]string
 	ServiceID                string
 	ServiceName              string
 	ServiceAddress           string
 	ServiceTags              []string
 	ServicePort              int
 	ServiceEnableTagOverride bool
+	CreateIndex              uint64
+	ModifyIndex              uint64
 }
 
 type CatalogNode struct {
@@ -27,6 +31,7 @@ type CatalogRegistration struct {
 	Node            string
 	Address         string
 	TaggedAddresses map[string]string
+	NodeMeta        map[string]string
 	Datacenter      string
 	Service         *AgentService
 	Check           *AgentCheck

vendor/github.com/hashicorp/consul/api/health.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"fmt"
+	"strings"
 )
 
 const (
@@ -11,6 +12,15 @@ const (
 	HealthPassing  = "passing"
 	HealthWarning  = "warning"
 	HealthCritical = "critical"
+	HealthMaint    = "maintenance"
+)
+
+const (
+	// NodeMaint is the special key set by a node in maintenance mode.
+	NodeMaint = "_node_maintenance"
+
+	// ServiceMaintPrefix is the prefix for a service in maintenance mode.
+	ServiceMaintPrefix = "_service_maintenance:"
 )
 
 // HealthCheck is used to represent a single check
@@ -25,11 +35,56 @@ type HealthCheck struct {
 	ServiceName string
 }
 
+// HealthChecks is a collection of HealthCheck structs.
+type HealthChecks []*HealthCheck
+
+// AggregatedStatus returns the "best" status for the list of health checks.
+// Because a given entry may have many service and node-level health checks
+// attached, this function determines the best representative of the status as
+// as single string using the following heuristic:
+//
+//  maintenance > critical > warning > passing
+//
+func (c HealthChecks) AggregatedStatus() string {
+	var passing, warning, critical, maintenance bool
+	for _, check := range c {
+		id := string(check.CheckID)
+		if id == NodeMaint || strings.HasPrefix(id, ServiceMaintPrefix) {
+			maintenance = true
+			continue
+		}
+
+		switch check.Status {
+		case HealthPassing:
+			passing = true
+		case HealthWarning:
+			warning = true
+		case HealthCritical:
+			critical = true
+		default:
+			return ""
+		}
+	}
+
+	switch {
+	case maintenance:
+		return HealthMaint
+	case critical:
+		return HealthCritical
+	case warning:
+		return HealthWarning
+	case passing:
+		return HealthPassing
+	default:
+		return HealthPassing
+	}
+}
+
 // ServiceEntry is used for the health service endpoint
 type ServiceEntry struct {
 	Node    *Node
 	Service *AgentService
-	Checks  []*HealthCheck
+	Checks  HealthChecks
 }
 
 // Health can be used to query the Health endpoints
@@ -43,7 +98,7 @@ func (c *Client) Health() *Health {
 }
 
 // Node is used to query for checks belonging to a given node
-func (h *Health) Node(node string, q *QueryOptions) ([]*HealthCheck, *QueryMeta, error) {
+func (h *Health) Node(node string, q *QueryOptions) (HealthChecks, *QueryMeta, error) {
 	r := h.c.newRequest("GET", "/v1/health/node/"+node)
 	r.setQueryOptions(q)
 	rtt, resp, err := requireOK(h.c.doRequest(r))
@@ -56,7 +111,7 @@ func (h *Health) Node(node string, q *QueryOptions) ([]*HealthCheck, *QueryMeta,
 	parseQueryMeta(resp, qm)
 	qm.RequestTime = rtt
 
-	var out []*HealthCheck
+	var out HealthChecks
 	if err := decodeBody(resp, &out); err != nil {
 		return nil, nil, err
 	}
@@ -64,7 +119,7 @@ func (h *Health) Node(node string, q *QueryOptions) ([]*HealthCheck, *QueryMeta,
 }
 
 // Checks is used to return the checks associated with a service
-func (h *Health) Checks(service string, q *QueryOptions) ([]*HealthCheck, *QueryMeta, error) {
+func (h *Health) Checks(service string, q *QueryOptions) (HealthChecks, *QueryMeta, error) {
 	r := h.c.newRequest("GET", "/v1/health/checks/"+service)
 	r.setQueryOptions(q)
 	rtt, resp, err := requireOK(h.c.doRequest(r))
@@ -77,7 +132,7 @@ func (h *Health) Checks(service string, q *QueryOptions) ([]*HealthCheck, *Query
 	parseQueryMeta(resp, qm)
 	qm.RequestTime = rtt
 
-	var out []*HealthCheck
+	var out HealthChecks
 	if err := decodeBody(resp, &out); err != nil {
 		return nil, nil, err
 	}
@@ -115,7 +170,7 @@ func (h *Health) Service(service, tag string, passingOnly bool, q *QueryOptions)
 
 // State is used to retrieve all the checks in a given state.
 // The wildcard "any" state can also be used for all checks.
-func (h *Health) State(state string, q *QueryOptions) ([]*HealthCheck, *QueryMeta, error) {
+func (h *Health) State(state string, q *QueryOptions) (HealthChecks, *QueryMeta, error) {
 	switch state {
 	case HealthAny:
 	case HealthWarning:
@@ -136,7 +191,7 @@ func (h *Health) State(state string, q *QueryOptions) ([]*HealthCheck, *QueryMet
 	parseQueryMeta(resp, qm)
 	qm.RequestTime = rtt
 
-	var out []*HealthCheck
+	var out HealthChecks
 	if err := decodeBody(resp, &out); err != nil {
 		return nil, nil, err
 	}

vendor/github.com/hashicorp/consul/api/kv.go

@@ -50,21 +50,21 @@ type KVOp string
 
 const (
 	KVSet          KVOp = "set"
-	KVDelete            = "delete"
-	KVDeleteCAS         = "delete-cas"
-	KVDeleteTree        = "delete-tree"
-	KVCAS               = "cas"
-	KVLock              = "lock"
-	KVUnlock            = "unlock"
-	KVGet               = "get"
-	KVGetTree           = "get-tree"
-	KVCheckSession      = "check-session"
-	KVCheckIndex        = "check-index"
+	KVDelete       KVOp = "delete"
+	KVDeleteCAS    KVOp = "delete-cas"
+	KVDeleteTree   KVOp = "delete-tree"
+	KVCAS          KVOp = "cas"
+	KVLock         KVOp = "lock"
+	KVUnlock       KVOp = "unlock"
+	KVGet          KVOp = "get"
+	KVGetTree      KVOp = "get-tree"
+	KVCheckSession KVOp = "check-session"
+	KVCheckIndex   KVOp = "check-index"
 )
 
 // KVTxnOp defines a single operation inside a transaction.
 type KVTxnOp struct {
-	Verb    string
+	Verb    KVOp
 	Key     string
 	Value   []byte
 	Flags   uint64
@@ -156,7 +156,7 @@ func (k *KV) Keys(prefix, separator string, q *QueryOptions) ([]string, *QueryMe
 }
 
 func (k *KV) getInternal(key string, params map[string]string, q *QueryOptions) (*http.Response, *QueryMeta, error) {
-	r := k.c.newRequest("GET", "/v1/kv/"+key)
+	r := k.c.newRequest("GET", "/v1/kv/"+strings.TrimPrefix(key, "/"))
 	r.setQueryOptions(q)
 	for param, val := range params {
 		r.params.Set(param, val)
@@ -277,7 +277,7 @@ func (k *KV) DeleteTree(prefix string, w *WriteOptions) (*WriteMeta, error) {
 }
 
 func (k *KV) deleteInternal(key string, params map[string]string, q *WriteOptions) (bool, *WriteMeta, error) {
-	r := k.c.newRequest("DELETE", "/v1/kv/"+key)
+	r := k.c.newRequest("DELETE", "/v1/kv/"+strings.TrimPrefix(key, "/"))
 	r.setWriteOptions(q)
 	for param, val := range params {
 		r.params.Set(param, val)

vendor/github.com/hashicorp/consul/api/operator.go

@@ -43,6 +43,26 @@ type RaftConfiguration struct {
 	Index uint64
 }
 
+// keyringRequest is used for performing Keyring operations
+type keyringRequest struct {
+	Key string
+}
+
+// KeyringResponse is returned when listing the gossip encryption keys
+type KeyringResponse struct {
+	// Whether this response is for a WAN ring
+	WAN bool
+
+	// The datacenter name this request corresponds to
+	Datacenter string
+
+	// A map of the encryption keys to the number of nodes they're installed on
+	Keys map[string]int
+
+	// The total number of nodes in this ring
+	NumNodes int
+}
+
 // RaftGetConfiguration is used to query the current Raft peer set.
 func (op *Operator) RaftGetConfiguration(q *QueryOptions) (*RaftConfiguration, error) {
 	r := op.c.newRequest("GET", "/v1/operator/raft/configuration")
@@ -79,3 +99,65 @@ func (op *Operator) RaftRemovePeerByAddress(address string, q *WriteOptions) err
 	resp.Body.Close()
 	return nil
 }
+
+// KeyringInstall is used to install a new gossip encryption key into the cluster
+func (op *Operator) KeyringInstall(key string, q *WriteOptions) error {
+	r := op.c.newRequest("POST", "/v1/operator/keyring")
+	r.setWriteOptions(q)
+	r.obj = keyringRequest{
+		Key: key,
+	}
+	_, resp, err := requireOK(op.c.doRequest(r))
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}
+
+// KeyringList is used to list the gossip keys installed in the cluster
+func (op *Operator) KeyringList(q *QueryOptions) ([]*KeyringResponse, error) {
+	r := op.c.newRequest("GET", "/v1/operator/keyring")
+	r.setQueryOptions(q)
+	_, resp, err := requireOK(op.c.doRequest(r))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var out []*KeyringResponse
+	if err := decodeBody(resp, &out); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// KeyringRemove is used to remove a gossip encryption key from the cluster
+func (op *Operator) KeyringRemove(key string, q *WriteOptions) error {
+	r := op.c.newRequest("DELETE", "/v1/operator/keyring")
+	r.setWriteOptions(q)
+	r.obj = keyringRequest{
+		Key: key,
+	}
+	_, resp, err := requireOK(op.c.doRequest(r))
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}
+
+// KeyringUse is used to change the active gossip encryption key
+func (op *Operator) KeyringUse(key string, q *WriteOptions) error {
+	r := op.c.newRequest("PUT", "/v1/operator/keyring")
+	r.setWriteOptions(q)
+	r.obj = keyringRequest{
+		Key: key,
+	}
+	_, resp, err := requireOK(op.c.doRequest(r))
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}

vendor/github.com/hashicorp/consul/api/prepared_query.go

@@ -167,19 +167,18 @@ func (c *PreparedQuery) Get(queryID string, q *QueryOptions) ([]*PreparedQueryDe
 }
 
 // Delete is used to delete a specific prepared query.
-func (c *PreparedQuery) Delete(queryID string, q *QueryOptions) (*QueryMeta, error) {
+func (c *PreparedQuery) Delete(queryID string, q *WriteOptions) (*WriteMeta, error) {
 	r := c.c.newRequest("DELETE", "/v1/query/"+queryID)
-	r.setQueryOptions(q)
+	r.setWriteOptions(q)
 	rtt, resp, err := requireOK(c.c.doRequest(r))
 	if err != nil {
 		return nil, err
 	}
 	defer resp.Body.Close()
 
-	qm := &QueryMeta{}
-	parseQueryMeta(resp, qm)
-	qm.RequestTime = rtt
-	return qm, nil
+	wm := &WriteMeta{}
+	wm.RequestTime = rtt
+	return wm, nil
 }
 
 // Execute is used to execute a specific prepared query. You can execute using

vendor/vendor.json

@@ -450,10 +450,10 @@
 			"revisionTime": "2017-06-07T03:48:29Z"
 		},
 		{
-			"checksumSHA1": "LclVLJYrBi03PBjsVPpgoMbUDQ8=",
+			"checksumSHA1": "/DReHn5j0caPm3thgFD9DmOmibQ=",
 			"path": "github.com/hashicorp/consul/api",
-			"revision": "daacc4be8bee214e3fc4b32a6dd385f5ef1b4c36",
-			"revisionTime": "2016-10-28T04:06:46Z"
+			"revision": "23ce10f8891369f4c7758474c7c808f4e0262701",
+			"revisionTime": "2017-01-12T01:29:24Z"
 		},
 		{
 			"checksumSHA1": "Uzyon2091lmwacNsl1hCytjhHtg=",