Add autopilot server health tracking

This adds two goroutines to perform autopilot tasks on the leader - one
to monitor the health of servers and another to periodically clean up
dead servers with a limit on removal count. Also adds a new http endpoint,
`/v1/operator/autopilot/health`, for querying this information through an
operator RPC endpoint.

api/operator.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"strconv"
 	"strings"
+	"time"
 )
 
 // Operator can be used to perform low-level operator tasks for Consul.
@@ -79,6 +80,19 @@ type AutopilotConfiguration struct {
 	// peer list when a new server joins
 	CleanupDeadServers bool
 
+	// LastContactThreshold is the limit on the amount of time a server can go
+	// without leader contact before being considered unhealthy.
+	LastContactThreshold time.Duration
+
+	// MaxTrailingLogs is the amount of entries in the Raft Log that a server can
+	// be behind before being considered unhealthy.
+	MaxTrailingLogs uint64
+
+	// ServerStabilizationTime is the minimum amount of time a server must be
+	// in a stable, healthy state before it can be added to the cluster. Only
+	// applicable with Raft protocol version 3 or higher.
+	ServerStabilizationTime time.Duration
+
 	// CreateIndex holds the index corresponding the creation of this configuration.
 	// This is a read-only field.
 	CreateIndex uint64

command/agent/agent.go

@@ -419,6 +419,15 @@ func (a *Agent) consulConfig() *consul.Config {
 	if a.config.Autopilot.CleanupDeadServers != nil {
 		base.AutopilotConfig.CleanupDeadServers = *a.config.Autopilot.CleanupDeadServers
 	}
+	if a.config.Autopilot.LastContactThreshold != nil {
+		base.AutopilotConfig.LastContactThreshold = *a.config.Autopilot.LastContactThreshold
+	}
+	if a.config.Autopilot.MaxTrailingLogs != nil {
+		base.AutopilotConfig.MaxTrailingLogs = *a.config.Autopilot.MaxTrailingLogs
+	}
+	if a.config.Autopilot.ServerStabilizationTime != nil {
+		base.AutopilotConfig.ServerStabilizationTime = *a.config.Autopilot.ServerStabilizationTime
+	}
 
 	// Format the build string
 	revision := a.config.Revision

command/agent/command.go

@@ -135,6 +135,8 @@ func (c *Command) readConfig() *Config {
 
 	f.IntVar(&cmdConfig.Protocol, "protocol", -1,
 		"Sets the protocol version. Defaults to latest.")
+	f.IntVar(&cmdConfig.RaftProtocol, "raft-protocol", -1,
+		"Sets the Raft protocol version. Defaults to latest.")
 
 	f.BoolVar(&cmdConfig.EnableSyslog, "syslog", false,
 		"Enables logging to syslog.")

command/agent/config.go

@@ -265,6 +265,21 @@ type Autopilot struct {
 	// CleanupDeadServers enables the automatic cleanup of dead servers when new ones
 	// are added to the peer list. Defaults to true.
 	CleanupDeadServers *bool `mapstructure:"cleanup_dead_servers"`
+
+	// LastContactThreshold is the limit on the amount of time a server can go
+	// without leader contact before being considered unhealthy.
+	LastContactThreshold    *time.Duration `mapstructure:"-" json:"-"`
+	LastContactThresholdRaw string         `mapstructure:"last_contact_threshold"`
+
+	// MaxTrailingLogs is the amount of entries in the Raft Log that a server can
+	// be behind before being considered unhealthy.
+	MaxTrailingLogs *uint64 `mapstructure:"max_trailing_logs"`
+
+	// ServerStabilizationTime is the minimum amount of time a server must be
+	// in a stable, healthy state before it can be added to the cluster. Only
+	// applicable with Raft protocol version 3 or higher.
+	ServerStabilizationTime    *time.Duration `mapstructure:"-" json:"-"`
+	ServerStabilizationTimeRaw string         `mapstructure:"server_stabilization_time"`
 }
 
 // Config is the configuration that can be set for an Agent.
@@ -692,6 +707,16 @@ func Bool(b bool) *bool {
 	return &b
 }
 
+// Uint64 is used to initialize uint64 pointers in struct literals.
+func Uint64(i uint64) *uint64 {
+	return &i
+}
+
+// Duration is used to initialize time.Duration pointers in struct literals.
+func Duration(d time.Duration) *time.Duration {
+	return &d
+}
+
 // UnixSocketPermissions contains information about a unix socket, and
 // implements the FilePermissions interface.
 type UnixSocketPermissions struct {
@@ -1041,6 +1066,21 @@ func DecodeConfig(r io.Reader) (*Config, error) {
 		result.ReconnectTimeoutWan = dur
 	}
 
+	if raw := result.Autopilot.LastContactThresholdRaw; raw != "" {
+		dur, err := time.ParseDuration(raw)
+		if err != nil {
+			return nil, fmt.Errorf("LastContactThreshold invalid: %v", err)
+		}
+		result.Autopilot.LastContactThreshold = &dur
+	}
+	if raw := result.Autopilot.ServerStabilizationTimeRaw; raw != "" {
+		dur, err := time.ParseDuration(raw)
+		if err != nil {
+			return nil, fmt.Errorf("ServerStabilizationTime invalid: %v", err)
+		}
+		result.Autopilot.ServerStabilizationTime = &dur
+	}
+
 	// Merge the single recursor
 	if result.DNSRecursor != "" {
 		result.DNSRecursors = append(result.DNSRecursors, result.DNSRecursor)
@@ -1293,7 +1333,7 @@ func MergeConfig(a, b *Config) *Config {
 	if b.Protocol > 0 {
 		result.Protocol = b.Protocol
 	}
-	if b.RaftProtocol != 0 {
+	if b.RaftProtocol > 0 {
 		result.RaftProtocol = b.RaftProtocol
 	}
 	if b.NodeID != "" {
@@ -1347,6 +1387,15 @@ func MergeConfig(a, b *Config) *Config {
 	if b.Autopilot.CleanupDeadServers != nil {
 		result.Autopilot.CleanupDeadServers = b.Autopilot.CleanupDeadServers
 	}
+	if b.Autopilot.LastContactThreshold != nil {
+		result.Autopilot.LastContactThreshold = b.Autopilot.LastContactThreshold
+	}
+	if b.Autopilot.MaxTrailingLogs != nil {
+		result.Autopilot.MaxTrailingLogs = b.Autopilot.MaxTrailingLogs
+	}
+	if b.Autopilot.ServerStabilizationTime != nil {
+		result.Autopilot.ServerStabilizationTime = b.Autopilot.ServerStabilizationTime
+	}
 	if b.Telemetry.DisableHostname == true {
 		result.Telemetry.DisableHostname = true
 	}

command/agent/config_test.go

@@ -1103,13 +1103,27 @@ func TestDecodeConfig_Performance(t *testing.T) {
 }
 
 func TestDecodeConfig_Autopilot(t *testing.T) {
-	input := `{"autopilot": { "cleanup_dead_servers": true }}`
+	input := `{"autopilot": {
+	  "cleanup_dead_servers": true,
+	  "last_contact_threshold": "100ms",
+	  "max_trailing_logs": 10,
+	  "server_stabilization_time": "10s"
+	 }}`
 	config, err := DecodeConfig(bytes.NewReader([]byte(input)))
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
 	if config.Autopilot.CleanupDeadServers == nil || !*config.Autopilot.CleanupDeadServers {
-		t.Fatalf("bad: cleanup_dead_servers isn't set: %#v", config)
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.Autopilot.LastContactThreshold == nil || *config.Autopilot.LastContactThreshold != 100*time.Millisecond {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.Autopilot.MaxTrailingLogs == nil || *config.Autopilot.MaxTrailingLogs != 10 {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.Autopilot.ServerStabilizationTime == nil || *config.Autopilot.ServerStabilizationTime != 10*time.Second {
+		t.Fatalf("bad: %#v", config)
 	}
 }
 
@@ -1630,6 +1644,9 @@ func TestMergeConfig(t *testing.T) {
 		RaftProtocol:   3,
 		Autopilot: Autopilot{
 			CleanupDeadServers:      Bool(true),
+			LastContactThreshold:    Duration(time.Duration(10)),
+			MaxTrailingLogs:         Uint64(10),
+			ServerStabilizationTime: Duration(time.Duration(100)),
 		},
 		EnableDebug:            true,
 		VerifyIncoming:         true,

command/agent/http.go

@@ -298,6 +298,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.handleFuncMetrics("/v1/operator/raft/peer", s.wrap(s.OperatorRaftPeer))
 	s.handleFuncMetrics("/v1/operator/keyring", s.wrap(s.OperatorKeyringEndpoint))
 	s.handleFuncMetrics("/v1/operator/autopilot/configuration", s.wrap(s.OperatorAutopilotConfiguration))
+	s.handleFuncMetrics("/v1/operator/autopilot/health", s.wrap(s.OperatorServerHealth))
 	s.handleFuncMetrics("/v1/query", s.wrap(s.PreparedQueryGeneral))
 	s.handleFuncMetrics("/v1/query/", s.wrap(s.PreparedQuerySpecific))
 	s.handleFuncMetrics("/v1/session/create", s.wrap(s.SessionCreate))

command/agent/operator_endpoint.go

@@ -224,3 +224,23 @@ func (s *HTTPServer) OperatorAutopilotConfiguration(resp http.ResponseWriter, re
 		return nil, nil
 	}
 }
+
+// OperatorServerHealth is used to get the health of the servers in the local DC
+func (s *HTTPServer) OperatorServerHealth(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	if req.Method != "GET" {
+		resp.WriteHeader(http.StatusMethodNotAllowed)
+		return nil, nil
+	}
+
+	var args structs.DCSpecificRequest
+	if done := s.parse(resp, req, &args.Datacenter, &args.QueryOptions); done {
+		return nil, nil
+	}
+
+	var reply structs.OperatorHealthReply
+	if err := s.agent.RPC("Operator.ServerHealth", &args, &reply); err != nil {
+		return nil, err
+	}
+
+	return reply, nil
+}

command/agent/operator_endpoint_test.go

@@ -8,7 +8,9 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/hashicorp/consul-enterprise/testutil"
 	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/serf/serf"
 )
 
 func TestOperator_OperatorRaftConfiguration(t *testing.T) {
@@ -420,3 +422,40 @@ func TestOperator_AutopilotCASConfiguration(t *testing.T) {
 		}
 	})
 }
+
+func TestOperator_OperatorServerHealth(t *testing.T) {
+	httpTest(t, func(srv *HTTPServer) {
+		body := bytes.NewBuffer(nil)
+		req, err := http.NewRequest("GET", "/v1/operator/autopilot/health", body)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		testutil.WaitForResult(func() (bool, error) {
+			resp := httptest.NewRecorder()
+			obj, err := srv.OperatorServerHealth(resp, req)
+			if err != nil {
+				return false, fmt.Errorf("err: %v", err)
+			}
+			if resp.Code != 200 {
+				return false, fmt.Errorf("bad code: %d", resp.Code)
+			}
+			out, ok := obj.(structs.OperatorHealthReply)
+			if !ok {
+				return false, fmt.Errorf("unexpected: %T", obj)
+			}
+			if len(out.Servers) != 1 ||
+				!out.Servers[0].Healthy ||
+				out.Servers[0].Name != srv.agent.config.NodeName ||
+				out.Servers[0].SerfStatusRaw != serf.StatusAlive ||
+				out.FailureTolerance != 0 {
+				return false, fmt.Errorf("bad: %v", out)
+			}
+
+			return true, nil
+		}, func(err error) {
+			t.Fatal(err)
+		})
+
+	})
+}

command/operator_autopilot_get.go

@@ -56,6 +56,9 @@ func (c *OperatorAutopilotGetCommand) Run(args []string) int {
 		return 1
 	}
 	c.Ui.Output(fmt.Sprintf("CleanupDeadServers = %v", config.CleanupDeadServers))
+	c.Ui.Output(fmt.Sprintf("LastContactThreshold = %v", config.LastContactThreshold.String()))
+	c.Ui.Output(fmt.Sprintf("MaxTrailingLogs = %v", config.MaxTrailingLogs))
+	c.Ui.Output(fmt.Sprintf("ServerStabilizationTime = %v", config.ServerStabilizationTime.String()))
 
 	return 0
 }

command/operator_autopilot_set.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/consul/command/base"
 )
@@ -29,12 +30,27 @@ func (c *OperatorAutopilotSetCommand) Synopsis() string {
 
 func (c *OperatorAutopilotSetCommand) Run(args []string) int {
 	var cleanupDeadServers base.BoolValue
+	var lastContactThresholdRaw string
+	var maxTrailingLogs base.UintValue
+	var serverStabilizationTimeRaw string
 
 	f := c.Command.NewFlagSet(c)
 
 	f.Var(&cleanupDeadServers, "cleanup-dead-servers",
 		"Controls whether Consul will automatically remove dead servers "+
 			"when new ones are successfully added. Must be one of `true|false`.")
+	f.Var(&maxTrailingLogs, "max-trailing-logs",
+		"Controls the maximum number of log entries that a server can trail the "+
+			"leader by before being considered unhealthy.")
+	f.StringVar(&lastContactThresholdRaw, "last-contact-threshold", "",
+		"Controls the maximum amount of time a server can go without contact "+
+			"from the leader before being considered unhealthy. Must be a duration value "+
+			"such as `10s`.")
+	f.StringVar(&serverStabilizationTimeRaw, "server-stabilization-time", "",
+		"Controls the minimum amount of time a server must be stable in the "+
+			"'healthy' state before being added to the cluster. Only takes effect if all "+
+			"servers are running Raft protocol version 3 or higher. Must be a duration "+
+			"value such as `10s`.")
 
 	if err := c.Command.Parse(args); err != nil {
 		if err == flag.ErrHelp {
@@ -59,8 +75,27 @@ func (c *OperatorAutopilotSetCommand) Run(args []string) int {
 		return 1
 	}
 
-	// Update the config values.
+	// Update the config values based on the set flags.
 	cleanupDeadServers.Merge(&conf.CleanupDeadServers)
+	trailing := uint(conf.MaxTrailingLogs)
+	maxTrailingLogs.Merge(&trailing)
+	conf.MaxTrailingLogs = uint64(trailing)
+
+	if lastContactThresholdRaw != "" {
+		dur, err := time.ParseDuration(lastContactThresholdRaw)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("invalid value for last-contact-threshold: %v", err))
+			return 1
+		}
+		conf.LastContactThreshold = dur
+	}
+	if serverStabilizationTimeRaw != "" {
+		dur, err := time.ParseDuration(serverStabilizationTimeRaw)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("invalid value for server-stabilization-time: %v", err))
+		}
+		conf.ServerStabilizationTime = dur
+	}
 
 	// Check-and-set the new configuration.
 	result, err := operator.AutopilotCASConfiguration(conf, nil)

consul/autopilot.go

@@ -0,0 +1,286 @@
+package consul
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/hashicorp/consul/consul/agent"
+	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/raft"
+	"github.com/hashicorp/serf/serf"
+)
+
+func (s *Server) startAutopilot() {
+	s.autopilotShutdownCh = make(chan struct{})
+
+	go s.serverHealthLoop()
+	go s.removeDeadLoop()
+}
+
+func (s *Server) stopAutopilot() {
+	close(s.autopilotShutdownCh)
+}
+
+// serverHealthLoop monitors the health of the servers in the cluster
+func (s *Server) serverHealthLoop() {
+	// Monitor server health until shutdown
+	ticker := time.NewTicker(s.config.ServerHealthInterval)
+	for {
+		select {
+		case <-s.autopilotShutdownCh:
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			serverHealths := make(map[string]*structs.ServerHealth)
+
+			state := s.fsm.State()
+			_, autopilotConf, err := state.AutopilotConfig()
+			if err != nil {
+				s.logger.Printf("[ERR] consul: error retrieving autopilot config: %s", err)
+			}
+
+			// Build an updated map of server healths
+			for _, member := range s.LANMembers() {
+				if member.Status == serf.StatusLeft {
+					continue
+				}
+
+				valid, parts := agent.IsConsulServer(member)
+				if valid {
+					health := s.queryServerHealth(member, parts, autopilotConf)
+					serverHealths[parts.Addr.String()] = health
+				}
+			}
+
+			s.autopilotLock.Lock()
+			s.autopilotHealth = serverHealths
+			s.autopilotLock.Unlock()
+
+			if err := s.promoteNonVoters(autopilotConf); err != nil {
+				s.logger.Printf("[ERR] consul: error checking for non-voters to promote: %s", err)
+			}
+		}
+	}
+}
+
+// removeDeadLoop checks for dead servers periodically, or when receiving on autopilotRemoveDeadCh
+func (s *Server) removeDeadLoop() {
+	ticker := time.NewTicker(s.config.RemoveDeadInterval)
+	for {
+		select {
+		case <-s.autopilotShutdownCh:
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			if err := s.pruneDeadServers(); err != nil {
+				s.logger.Printf("[ERR] consul: error checking for dead servers to remove: %s", err)
+			}
+		case <-s.autopilotRemoveDeadCh:
+			if err := s.pruneDeadServers(); err != nil {
+				s.logger.Printf("[ERR] consul: error checking for dead servers to remove: %s", err)
+			}
+		}
+	}
+}
+
+// pruneDeadServers removes up to numPeers/2 failed servers
+func (s *Server) pruneDeadServers() error {
+	state := s.fsm.State()
+	_, autopilotConf, err := state.AutopilotConfig()
+	if err != nil {
+		return err
+	}
+
+	// Look for dead servers to clean up
+	peers, err := s.numPeers()
+	if err != nil {
+		return err
+	}
+
+	removed := 0
+	if autopilotConf.CleanupDeadServers {
+		for _, member := range s.serfLAN.Members() {
+			// Exit early if we already removed the max amount of servers
+			if removed == peers/2 {
+				break
+			}
+
+			valid, _ := agent.IsConsulServer(member)
+			if valid && member.Status == serf.StatusFailed {
+				removed++
+				s.logger.Printf("[INFO] consul: Attempting removal of failed server: %v", member.Name)
+				go s.serfLAN.RemoveFailedNode(member.Name)
+			}
+		}
+	}
+
+	return nil
+}
+
+// promoteNonVoters promotes eligible non-voting servers to voters.
+func (s *Server) promoteNonVoters(autopilotConf *structs.AutopilotConfig) error {
+	minRaftProtocol, err := ServerMinRaftProtocol(s.LANMembers())
+	if err != nil {
+		return fmt.Errorf("error getting server raft protocol versions: %s", err)
+	}
+
+	if minRaftProtocol >= 3 {
+		future := s.raft.GetConfiguration()
+		if err := future.Error(); err != nil {
+			return fmt.Errorf("failed to get raft configuration: %v", err)
+		}
+
+		var promotions []raft.Server
+		raftServers := future.Configuration().Servers
+		voterCount := 0
+		for _, server := range raftServers {
+			// If this server has been stable and passing for long enough, promote it to a voter
+			if server.Suffrage == raft.Nonvoter {
+				health := s.getServerHealth(string(server.Address))
+				if health != nil && health.Healthy && time.Now().Sub(health.StableSince) >= autopilotConf.ServerStabilizationTime {
+					promotions = append(promotions, server)
+				}
+			} else {
+				voterCount++
+			}
+		}
+
+		// Exit early if there's nothing to promote
+		if len(promotions) == 0 {
+			return nil
+		}
+
+		// If there's currently an even number of servers, we can promote the first server in the list
+		// to get to an odd-sized quorum
+		newServers := false
+		if voterCount%2 == 0 {
+			addFuture := s.raft.AddVoter(promotions[0].ID, promotions[0].Address, 0, 0)
+			if err := addFuture.Error(); err != nil {
+				return fmt.Errorf("failed to add raft peer: %v", err)
+			}
+			promotions = promotions[1:]
+			newServers = true
+		}
+
+		// Promote remaining servers in twos to maintain an odd quorum size
+		for i := 0; i < len(promotions)-1; i += 2 {
+			addFirst := s.raft.AddVoter(promotions[i].ID, promotions[i].Address, 0, 0)
+			if err := addFirst.Error(); err != nil {
+				return fmt.Errorf("failed to add raft peer: %v", err)
+			}
+			addSecond := s.raft.AddVoter(promotions[i+1].ID, promotions[i+1].Address, 0, 0)
+			if err := addSecond.Error(); err != nil {
+				return fmt.Errorf("failed to add raft peer: %v", err)
+			}
+			newServers = true
+		}
+
+		// If we added a new server, trigger a check to remove dead servers
+		if newServers {
+			go func() {
+				s.autopilotRemoveDeadCh <- struct{}{}
+			}()
+		}
+	}
+
+	return nil
+}
+
+// queryServerHealth fetches the raft stats for the given server and uses them
+// to update its ServerHealth
+func (s *Server) queryServerHealth(member serf.Member, server *agent.Server, autopilotConf *structs.AutopilotConfig) *structs.ServerHealth {
+	health := &structs.ServerHealth{
+		ID:             server.ID,
+		Name:           server.Name,
+		SerfStatusRaw:  member.Status,
+		SerfStatus:     member.Status.String(),
+		LastContactRaw: -1,
+		LastContact:    "never",
+	}
+
+	stats, err := s.getServerStats(server)
+	if err != nil {
+		s.logger.Printf("[DEBUG] consul: error getting server's raft stats: %s", err)
+	}
+
+	if v, ok := stats["last_contact"]; ok && v != "never" {
+		health.LastContactRaw, err = time.ParseDuration(v)
+		if err != nil {
+			s.logger.Printf("[DEBUG] consul: error parsing server's last_contact value: %s", err)
+		}
+		health.LastContact = health.LastContactRaw.String()
+	}
+	// Set LastContact to 0 if we're the leader
+	if s.config.NodeName == member.Name {
+		health.LastContactRaw = 0
+		health.LastContact = "leader"
+	}
+
+	if v, ok := stats["last_log_index"]; ok {
+		health.LastIndex, err = strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			s.logger.Printf("[DEBUG] consul: error parsing server's last_log_index value: %s", err)
+		}
+	}
+
+	if v, ok := stats["last_log_term"]; ok {
+		health.LastTerm, err = strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			s.logger.Printf("[DEBUG] consul: error parsing server's last_log_term value: %s", err)
+		}
+	}
+
+	health.Healthy = s.isServerHealthy(health, autopilotConf)
+
+	// If this is a new server or the health changed, reset StableSince
+	lastHealth := s.getServerHealth(server.Addr.String())
+	if lastHealth == nil || lastHealth.Healthy != health.Healthy {
+		health.StableSince = time.Now()
+	} else {
+		health.StableSince = lastHealth.StableSince
+	}
+
+	return health
+}
+
+func (s *Server) getServerHealth(addr string) *structs.ServerHealth {
+	s.autopilotLock.RLock()
+	defer s.autopilotLock.RUnlock()
+	h, ok := s.autopilotHealth[addr]
+	if !ok {
+		return nil
+	}
+	return h
+}
+
+func (s *Server) getServerStats(server *agent.Server) (map[string]string, error) {
+	var args struct{}
+	var reply map[string]string
+	err := s.connPool.RPC(s.config.Datacenter, server.Addr, server.Version, "Operator.RaftStats", &args, &reply)
+	return reply, err
+}
+
+// isServerHealthy determines whether the given ServerHealth is healthy
+// based on the current Autopilot config
+func (s *Server) isServerHealthy(health *structs.ServerHealth, autopilotConf *structs.AutopilotConfig) bool {
+	if health.SerfStatusRaw != serf.StatusAlive {
+		return false
+	}
+
+	if health.LastContactRaw > autopilotConf.LastContactThreshold || health.LastContactRaw < 0 {
+		return false
+	}
+
+	lastTerm, _ := strconv.ParseUint(s.raft.Stats()["last_log_term"], 10, 64)
+	if health.LastTerm != lastTerm {
+		return false
+	}
+
+	if s.raft.LastIndex() > autopilotConf.MaxTrailingLogs &&
+		health.LastIndex < s.raft.LastIndex()-autopilotConf.MaxTrailingLogs {
+		return false
+	}
+
+	return true
+}

consul/autopilot_test.go

@@ -0,0 +1,235 @@
+package consul
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/consul/testutil"
+	"github.com/hashicorp/raft"
+	"github.com/hashicorp/serf/serf"
+)
+
+func TestAutopilot_CleanupDeadServer(t *testing.T) {
+	dir1, s1 := testServerDCBootstrap(t, "dc1", true)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	dir2, s2 := testServerDCBootstrap(t, "dc1", false)
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+
+	dir3, s3 := testServerDCBootstrap(t, "dc1", false)
+	defer os.RemoveAll(dir3)
+	defer s3.Shutdown()
+
+	servers := []*Server{s1, s2, s3}
+
+	// Try to join
+	addr := fmt.Sprintf("127.0.0.1:%d",
+		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
+	if _, err := s2.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if _, err := s3.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	for _, s := range servers {
+		testutil.WaitForResult(func() (bool, error) {
+			peers, _ := s.numPeers()
+			return peers == 3, nil
+		}, func(err error) {
+			t.Fatalf("should have 3 peers")
+		})
+	}
+
+	// Kill a non-leader server
+	s2.Shutdown()
+
+	testutil.WaitForResult(func() (bool, error) {
+		alive := 0
+		for _, m := range s1.LANMembers() {
+			if m.Status == serf.StatusAlive {
+				alive++
+			}
+		}
+		return alive == 2, nil
+	}, func(err error) {
+		t.Fatalf("should have 2 alive members")
+	})
+
+	// Bring up and join a new server
+	dir4, s4 := testServerDCBootstrap(t, "dc1", false)
+	defer os.RemoveAll(dir4)
+	defer s4.Shutdown()
+
+	if _, err := s4.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	servers[1] = s4
+
+	// Make sure the dead server is removed and we're back to 3 total peers
+	for _, s := range servers {
+		testutil.WaitForResult(func() (bool, error) {
+			peers, _ := s.numPeers()
+			return peers == 3, nil
+		}, func(err error) {
+			t.Fatalf("should have 3 peers")
+		})
+	}
+}
+
+func TestAutopilot_CleanupDeadServerPeriodic(t *testing.T) {
+	dir1, s1 := testServerWithConfig(t, func(c *Config) {
+		c.Datacenter = "dc1"
+		c.Bootstrap = true
+		c.RemoveDeadInterval = 100 * time.Millisecond
+	})
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	conf := func(c *Config) {
+		c.Datacenter = "dc1"
+		c.Bootstrap = false
+	}
+	dir2, s2 := testServerWithConfig(t, conf)
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+
+	dir3, s3 := testServerWithConfig(t, conf)
+	defer os.RemoveAll(dir3)
+	defer s3.Shutdown()
+
+	servers := []*Server{s1, s2, s3}
+
+	// Join the servers to s1
+	addr := fmt.Sprintf("127.0.0.1:%d",
+		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
+	if _, err := s2.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if _, err := s3.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	for _, s := range servers {
+		testutil.WaitForResult(func() (bool, error) {
+			peers, _ := s.numPeers()
+			return peers == 3, nil
+		}, func(err error) {
+			t.Fatalf("should have 3 peers")
+		})
+	}
+
+	// Kill a non-leader server
+	s3.Shutdown()
+
+	// Should be removed from the peers automatically
+	for _, s := range []*Server{s1, s2} {
+		testutil.WaitForResult(func() (bool, error) {
+			peers, _ := s.numPeers()
+			return peers == 2, nil
+		}, func(err error) {
+			t.Fatalf("should have 2 peers")
+		})
+	}
+}
+
+func TestAutopilot_PromoteNonVoter(t *testing.T) {
+	dir1, s1 := testServerWithConfig(t, func(c *Config) {
+		c.Datacenter = "dc1"
+		c.Bootstrap = true
+		c.RaftConfig.ProtocolVersion = 3
+		c.AutopilotConfig.ServerStabilizationTime = 200 * time.Millisecond
+		c.ServerHealthInterval = 100 * time.Millisecond
+	})
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	codec := rpcClient(t, s1)
+	defer codec.Close()
+
+	dir2, s2 := testServerWithConfig(t, func(c *Config) {
+		c.Datacenter = "dc1"
+		c.Bootstrap = false
+		c.RaftConfig.ProtocolVersion = 3
+	})
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+	addr := fmt.Sprintf("127.0.0.1:%d",
+		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
+	if _, err := s2.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	testutil.WaitForLeader(t, s1.RPC, "dc1")
+
+	// Wait for the new server to be added as a non-voter, but make sure
+	// it doesn't get promoted to a voter even after ServerStabilizationTime,
+	// because that would result in an even-numbered quorum count.
+	testutil.WaitForResult(func() (bool, error) {
+		future := s1.raft.GetConfiguration()
+		if err := future.Error(); err != nil {
+			return false, err
+		}
+
+		servers := future.Configuration().Servers
+
+		if len(servers) != 2 {
+			return false, fmt.Errorf("bad: %v", servers)
+		}
+		if servers[1].Suffrage != raft.Nonvoter {
+			return false, fmt.Errorf("bad: %v", servers)
+		}
+		health := s1.getServerHealth(string(servers[1].Address))
+		if health == nil {
+			return false, fmt.Errorf("nil health")
+		}
+		if !health.Healthy {
+			return false, fmt.Errorf("bad: %v", health)
+		}
+		if time.Now().Sub(health.StableSince) < s1.config.AutopilotConfig.ServerStabilizationTime {
+			return false, fmt.Errorf("stable period not elapsed")
+		}
+
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err)
+	})
+
+	// Now add another server and make sure they both get promoted to voters after stabilization
+	dir3, s3 := testServerWithConfig(t, func(c *Config) {
+		c.Datacenter = "dc1"
+		c.Bootstrap = false
+		c.RaftConfig.ProtocolVersion = 3
+	})
+	defer os.RemoveAll(dir3)
+	defer s3.Shutdown()
+	if _, err := s3.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	testutil.WaitForResult(func() (bool, error) {
+		future := s1.raft.GetConfiguration()
+		if err := future.Error(); err != nil {
+			return false, err
+		}
+
+		servers := future.Configuration().Servers
+
+		if len(servers) != 3 {
+			return false, fmt.Errorf("bad: %v", servers)
+		}
+		if servers[1].Suffrage != raft.Voter {
+			return false, fmt.Errorf("bad: %v", servers)
+		}
+		if servers[2].Suffrage != raft.Voter {
+			return false, fmt.Errorf("bad: %v", servers)
+		}
+
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err)
+	})
+}

consul/config.go

@@ -279,6 +279,14 @@ type Config struct {
 	// AutopilotConfig is used to apply the initial autopilot config when
 	// bootstrapping.
 	AutopilotConfig *structs.AutopilotConfig
+
+	// ServerHealthInterval is the frequency with which the leader will check
+	// the health of the servers in the cluster
+	ServerHealthInterval time.Duration
+
+	// RemoveDeadInterval is the frequency with which the leader will look for
+	// dead servers to remove from the cluster
+	RemoveDeadInterval time.Duration
 }
 
 // CheckVersion is used to check if the ProtocolVersion is valid
@@ -354,7 +362,12 @@ func DefaultConfig() *Config {
 
 		AutopilotConfig: &structs.AutopilotConfig{
 			CleanupDeadServers:      true,
+			LastContactThreshold:    200 * time.Millisecond,
+			MaxTrailingLogs:         250,
+			ServerStabilizationTime: 10 * time.Second,
 		},
+		ServerHealthInterval: 1 * time.Second,
+		RemoveDeadInterval:   30 * time.Second,
 	}
 
 	// Increase our reap interval to 3 days instead of 24h.

consul/leader.go

@@ -159,6 +159,8 @@ func (s *Server) establishLeadership() error {
 		return err
 	}
 
+	s.startAutopilot()
+
 	return nil
 }
 
@@ -174,6 +176,9 @@ func (s *Server) revokeLeadership() error {
 		s.logger.Printf("[ERR] consul: Clearing session timers failed: %v", err)
 		return err
 	}
+
+	s.stopAutopilot()
+
 	return nil
 }
 
@@ -597,13 +602,20 @@ func (s *Server) joinConsulServer(m serf.Member, parts *agent.Server) error {
 		return err
 	}
 
-	if minRaftProtocol >= 2 && parts.RaftVersion >= 3 {
+	switch {
+	case minRaftProtocol >= 3:
+		addFuture := s.raft.AddNonvoter(raft.ServerID(parts.ID), raft.ServerAddress(addr), 0, 0)
+		if err := addFuture.Error(); err != nil {
+			s.logger.Printf("[ERR] consul: failed to add raft peer: %v", err)
+			return err
+		}
+	case minRaftProtocol == 2 && parts.RaftVersion >= 3:
 		addFuture := s.raft.AddVoter(raft.ServerID(parts.ID), raft.ServerAddress(addr), 0, 0)
 		if err := addFuture.Error(); err != nil {
 			s.logger.Printf("[ERR] consul: failed to add raft peer: %v", err)
 			return err
 		}
-	} else {
+	default:
 		addFuture := s.raft.AddPeer(raft.ServerAddress(addr))
 		if err := addFuture.Error(); err != nil {
 			s.logger.Printf("[ERR] consul: failed to add raft peer: %v", err)
@@ -611,22 +623,10 @@ func (s *Server) joinConsulServer(m serf.Member, parts *agent.Server) error {
 		}
 	}
 
-	state := s.fsm.State()
+	// Trigger a check to remove dead servers
-	_, autopilotConf, err := state.AutopilotConfig()
+	go func() {
-	if err != nil {
+		s.autopilotRemoveDeadCh <- struct{}{}
-		return err
+	}()
-	}
-
-	// Look for dead servers to clean up
-	if autopilotConf.CleanupDeadServers {
-		for _, member := range s.serfLAN.Members() {
-			valid, _ := agent.IsConsulServer(member)
-			if valid && member.Name != m.Name && member.Status == serf.StatusFailed {
-				s.logger.Printf("[INFO] consul: Attempting removal of failed server: %v", member.Name)
-				go s.serfLAN.RemoveFailedNode(member.Name)
-			}
-		}
-	}
 
 	return nil
 }

consul/leader_test.go

@@ -623,76 +623,6 @@ func TestLeader_ReapTombstones(t *testing.T) {
 	})
 }
 
-func TestLeader_CleanupDeadServers(t *testing.T) {
-	dir1, s1 := testServerDCBootstrap(t, "dc1", true)
-	defer os.RemoveAll(dir1)
-	defer s1.Shutdown()
-
-	dir2, s2 := testServerDCBootstrap(t, "dc1", false)
-	defer os.RemoveAll(dir2)
-	defer s2.Shutdown()
-
-	dir3, s3 := testServerDCBootstrap(t, "dc1", false)
-	defer os.RemoveAll(dir3)
-	defer s3.Shutdown()
-
-	servers := []*Server{s1, s2, s3}
-
-	// Try to join
-	addr := fmt.Sprintf("127.0.0.1:%d",
-		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
-	if _, err := s2.JoinLAN([]string{addr}); err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	if _, err := s3.JoinLAN([]string{addr}); err != nil {
-		t.Fatalf("err: %v", err)
-	}
-
-	for _, s := range servers {
-		testutil.WaitForResult(func() (bool, error) {
-			peers, _ := s.numPeers()
-			return peers == 3, nil
-		}, func(err error) {
-			t.Fatalf("should have 3 peers")
-		})
-	}
-
-	// Kill a non-leader server
-	s2.Shutdown()
-
-	testutil.WaitForResult(func() (bool, error) {
-		alive := 0
-		for _, m := range s1.LANMembers() {
-			if m.Status == serf.StatusAlive {
-				alive++
-			}
-		}
-		return alive == 2, nil
-	}, func(err error) {
-		t.Fatalf("should have 2 alive members")
-	})
-
-	// Bring up and join a new server
-	dir4, s4 := testServerDCBootstrap(t, "dc1", false)
-	defer os.RemoveAll(dir4)
-	defer s4.Shutdown()
-
-	if _, err := s4.JoinLAN([]string{addr}); err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	servers[1] = s4
-
-	// Make sure the dead server is removed and we're back to 3 total peers
-	for _, s := range servers {
-		testutil.WaitForResult(func() (bool, error) {
-			peers, _ := s.numPeers()
-			return peers == 3, nil
-		}, func(err error) {
-			t.Fatalf("should have 3 peers")
-		})
-	}
-}
-
 func TestLeader_RollRaftServer(t *testing.T) {
 	dir1, s1 := testServerWithConfig(t, func(c *Config) {
 		c.Bootstrap = true

consul/operator_endpoint.go

@@ -8,6 +8,7 @@ import (
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/hashicorp/raft"
 	"github.com/hashicorp/serf/serf"
+	"time"
 )
 
 // Operator endpoint is used to perform low-level operator tasks for Consul.
@@ -183,3 +184,63 @@ func (op *Operator) AutopilotSetConfiguration(args *structs.AutopilotSetConfigRe
 	}
 	return nil
 }
+
+// Used by Autopilot to query the raft stats of the local server.
+func (op *Operator) RaftStats(args struct{}, reply *map[string]string) error {
+	*reply = op.srv.raft.Stats()
+	return nil
+}
+
+// ServerHealth is used to get the current health of the servers.
+func (op *Operator) ServerHealth(args *structs.DCSpecificRequest, reply *structs.OperatorHealthReply) error {
+	// This must be sent to the leader, so we fix the args since we are
+	// re-using a structure where we don't support all the options.
+	args.RequireConsistent = true
+	args.AllowStale = false
+	if done, err := op.srv.forward("Operator.ServerHealth", args, args, reply); done {
+		return err
+	}
+
+	// This action requires operator read access.
+	acl, err := op.srv.resolveToken(args.Token)
+	if err != nil {
+		return err
+	}
+	if acl != nil && !acl.OperatorRead() {
+		return permissionDeniedErr
+	}
+
+	status := structs.OperatorHealthReply{
+		Healthy: true,
+	}
+	future := op.srv.raft.GetConfiguration()
+	if err := future.Error(); err != nil {
+		return err
+	}
+
+	healthyCount := 0
+	servers := future.Configuration().Servers
+	for _, s := range servers {
+		health := op.srv.getServerHealth(string(s.Address))
+		if health != nil {
+			// Fix up StableSince to be more readable
+			health.StableSince = health.StableSince.Round(time.Second).UTC()
+
+			if !health.Healthy {
+				status.Healthy = false
+			} else {
+				healthyCount++
+			}
+			status.Servers = append(status.Servers, *health)
+		}
+	}
+
+	// If we have extra healthy servers, set FailureTolerance
+	if healthyCount > len(servers)/2+1 {
+		status.FailureTolerance = healthyCount - (len(servers)/2 + 1)
+	}
+
+	*reply = status
+
+	return nil
+}

consul/operator_endpoint_test.go

@@ -426,3 +426,79 @@ func TestOperator_Autopilot_SetConfiguration_ACLDeny(t *testing.T) {
 		t.Fatalf("bad: %#v", config)
 	}
 }
+
+func TestOperator_ServerHealth(t *testing.T) {
+	for i := 1; i <= 3; i++ {
+		testServerHealth(t, i)
+	}
+}
+
+func testServerHealth(t *testing.T, protocol int) {
+	dir1, s1 := testServerWithConfig(t, func(c *Config) {
+		c.Datacenter = "dc1"
+		c.Bootstrap = true
+		c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(protocol)
+	})
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	codec := rpcClient(t, s1)
+	defer codec.Close()
+
+	dir2, s2 := testServerWithConfig(t, func(c *Config) {
+		c.Datacenter = "dc1"
+		c.Bootstrap = false
+		c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(protocol)
+	})
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+	addr := fmt.Sprintf("127.0.0.1:%d",
+		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
+	if _, err := s2.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	dir3, s3 := testServerWithConfig(t, func(c *Config) {
+		c.Datacenter = "dc1"
+		c.Bootstrap = false
+		c.RaftConfig.ProtocolVersion = raft.ProtocolVersion(protocol)
+	})
+	defer os.RemoveAll(dir3)
+	defer s3.Shutdown()
+	if _, err := s3.JoinLAN([]string{addr}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	testutil.WaitForLeader(t, s1.RPC, "dc1")
+
+	testutil.WaitForResult(func() (bool, error) {
+		arg := structs.DCSpecificRequest{
+			Datacenter: "dc1",
+		}
+		var reply structs.OperatorHealthReply
+		err := msgpackrpc.CallWithCodec(codec, "Operator.ServerHealth", &arg, &reply)
+		if err != nil {
+			return false, fmt.Errorf("err: %v", err)
+		}
+		if !reply.Healthy {
+			return false, fmt.Errorf("bad: %v", reply)
+		}
+		if reply.FailureTolerance != 1 {
+			return false, fmt.Errorf("bad: %v", reply)
+		}
+		if len(reply.Servers) != 3 {
+			return false, fmt.Errorf("bad: %v", reply)
+		}
+		if reply.Servers[0].LastContact != "leader" {
+			return false, fmt.Errorf("bad: %v", reply)
+		}
+		if reply.Servers[1].LastContactRaw <= 0 {
+			return false, fmt.Errorf("bad: %v", reply)
+		}
+		if reply.Servers[2].LastContactRaw <= 0 {
+			return false, fmt.Errorf("bad: %v", reply)
+		}
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err)
+	})
+}

consul/server.go

@@ -76,6 +76,12 @@ type Server struct {
 	// aclCache is the non-authoritative ACL cache.
 	aclCache *aclCache
 
+	// autopilot
+	autopilotHealth       map[string]*structs.ServerHealth
+	autopilotLock         sync.RWMutex
+	autopilotShutdownCh   chan struct{}
+	autopilotRemoveDeadCh chan struct{}
+
 	// Consul configuration
 	config *Config
 
@@ -222,6 +228,8 @@ func NewServer(config *Config) (*Server, error) {
 
 	// Create server.
 	s := &Server{
+		autopilotRemoveDeadCh: make(chan struct{}),
+		autopilotShutdownCh:   make(chan struct{}),
 		config:                config,
 		connPool:              NewPool(config.LogOutput, serverRPCCache, serverMaxStreams, tlsWrap),
 		eventChLAN:            make(chan serf.Event, 256),

consul/structs/operator.go

@@ -1,15 +1,31 @@
 package structs
 
 import (
+	"time"
+
 	"github.com/hashicorp/raft"
+	"github.com/hashicorp/serf/serf"
 )
 
 type AutopilotConfig struct {
 	// CleanupDeadServers controls whether to remove dead servers when a new
-	// server is added to the Raft peers
+	// server is added to the Raft peers.
 	CleanupDeadServers bool
 
-	// RaftIndex stores the create/modify indexes of this configuration
+	// LastContactThreshold is the limit on the amount of time a server can go
+	// without leader contact before being considered unhealthy.
+	LastContactThreshold time.Duration
+
+	// MaxTrailingLogs is the amount of entries in the Raft Log that a server can
+	// be behind before being considered unhealthy.
+	MaxTrailingLogs uint64
+
+	// ServerStabilizationTime is the minimum amount of time a server must be
+	// in a stable, healthy state before it can be added to the cluster. Only
+	// applicable with Raft protocol version 3 or higher.
+	ServerStabilizationTime time.Duration
+
+	// RaftIndex stores the create/modify indexes of this configuration.
 	RaftIndex
 }
 
@@ -85,3 +101,46 @@ type AutopilotSetConfigRequest struct {
 func (op *AutopilotSetConfigRequest) RequestDatacenter() string {
 	return op.Datacenter
 }
+
+// ServerHealth is the health (from the leader's point of view) of a server.
+type ServerHealth struct {
+	// ID is the raft ID of the server.
+	ID string
+
+	// Name is the node name of the server.
+	Name string
+
+	// The status of the SerfHealth check for the server.
+	SerfStatusRaw serf.MemberStatus `json:"-"`
+	SerfStatus    string
+
+	// LastContact is the time since this node's last contact with the leader.
+	LastContactRaw time.Duration `json:"-"`
+	LastContact    string
+
+	// LastTerm is the highest leader term this server has a record of in its Raft log.
+	LastTerm uint64
+
+	// LastIndex is the last log index this server has a record of in its Raft log.
+	LastIndex uint64
+
+	// Healthy is whether or not the server is healthy according to the current
+	// Autopilot config.
+	Healthy bool
+
+	// StableSince is the amount of time since this server's Healthy value last changed.
+	StableSince time.Time
+}
+
+// OperatorHealthReply is a representation of the overall health of the cluster
+type OperatorHealthReply struct {
+	// Healthy is true if all the servers in the cluster are healthy.
+	Healthy bool
+
+	// FailureTolerance is the number of healthy servers that could be lost without
+	// an outage occurring.
+	FailureTolerance int
+
+	// Servers holds the health of each server.
+	Servers []ServerHealth
+}

website/source/docs/agent/http/operator.html.markdown

@@ -29,6 +29,7 @@ The following endpoints are supported:
 * [`/v1/operator/raft/peer`](#raft-peer): Operates on Raft peers
 * [`/v1/operator/keyring`](#keyring): Operates on gossip keyring
 * [`/v1/operator/autopilot/configuration`](#autopilot-configuration): Operates on the Autopilot configuration
+* [`/v1/operator/autopilot/health`](#autopilot-health): Returns the health of the servers
 
 Not all endpoints support blocking queries and all consistency modes,
 see details in the sections below.
@@ -288,13 +289,16 @@ A JSON body is returned that looks like this:
 ```javascript
 {
     "CleanupDeadServers": true,
+    "LastContactThreshold": 200000000,
+    "MaxTrailingLogs": 250,
+    "ServerStabilizationTime": 10000000000,
     "CreateIndex": 4,
     "ModifyIndex": 4
 }
 ```
 
-`CleanupDeadServers` is whether dead servers should be removed automatically when
+For more information about the Autopilot configuration options, see the agent configuration section
-a new server is added to the cluster.
+[here](/docs/agent/options.html#autopilot).
 
 #### PUT Method
 
@@ -313,11 +317,89 @@ body must look like:
 
 ```javascript
 {
-    "CleanupDeadServers": true
+    "CleanupDeadServers": true,
+    "LastContactThreshold": 200000000,
+    "MaxTrailingLogs": 250,
+    "ServerStabilizationTime": 10000000000,
+    "CreateIndex": 4,
+    "ModifyIndex": 4
 }
 ```
 
-`CleanupDeadServers` is whether dead servers should be removed automatically when
+For more information about the Autopilot configuration options, see the agent configuration section
-a new server is added to the cluster.
+[here](/docs/agent/options.html#autopilot).
 
 The return code will indicate success or failure.
+
+### <a name="autopilot-health"></a> /v1/operator/autopilot/health
+
+Available in Consul 0.8.0 and later, the autopilot health endpoint supports the
+`GET` method.
+
+This endpoint supports the use of ACL tokens using either the `X-CONSUL-TOKEN`
+header or the `?token=` query parameter.
+
+By default, the datacenter of the agent is queried; however, the `dc` can be
+provided using the `?dc=` query parameter.
+
+#### GET Method
+
+When using the `GET` method, the request will be forwarded to the cluster
+leader to retrieve its latest Autopilot configuration.
+
+If ACLs are enabled, the client will need to supply an ACL Token with
+[`operator`](/docs/internals/acl.html#operator) read privileges.
+
+A JSON body is returned that looks like this:
+
+```javascript
+{
+    "Healthy": true,
+    "FailureTolerance": 0,
+    "Servers": [
+        {
+            "ID": "e349749b-3303-3ddf-959c-b5885a0e1f6e",
+            "Name": "node1",
+            "SerfStatus": "alive",
+            "LastContact": "leader",
+            "LastTerm": 2,
+            "LastIndex": 46,
+            "Healthy": true,
+            "StableSince": "2017-03-06T22:07:51Z"
+        },
+        {
+            "ID": "e36ee410-cc3c-0a0c-c724-63817ab30303",
+            "Name": "node2",
+            "SerfStatus": "alive",
+            "LastContact": "27.291304ms",
+            "LastTerm": 2,
+            "LastIndex": 46,
+            "Healthy": true,
+            "StableSince": "2017-03-06T22:18:26Z"
+        }
+    ]
+}
+```
+
+`Healthy` is whether all the servers are currently heathly.
+
+`FailureTolerance` is the number of redundant healthy servers that could be fail
+without causing an outage (this would be 2 in a healthy cluster of 5 servers).
+
+The `Servers` list holds detailed health information on each server:
+
+- `ID` is the Raft ID of the server.
+
+- `Name` is the node name of the server.
+
+- `SerfStatus` is the SerfHealth check status for the server.
+
+- `LastContact` is the time elapsed since this server's last contact with the leader.
+
+- `LastTerm` is the server's last known Raft leader term.
+
+- `LastIndex` is the index of the server's last committed Raft log entry.
+
+- `Healthy` is whether the server is healthy according to the current Autopilot configuration.
+
+- `StableSince` is the time this server has been in its current `Healthy` state.

website/source/docs/agent/options.html.markdown

@@ -311,6 +311,11 @@ will exit with an error at startup.
   use. This defaults to the latest version. This should be set only when [upgrading](/docs/upgrading.html).
   You can view the protocol versions supported by Consul by running `consul -v`.
 
+* <a name="_raft_protocol"></a><a href="#_raft_protocol">`-raft_protocol`</a> - This controls the internal
+  version of the Raft consensus protocol used for server communications. This defaults to 2 but must
+  be set to 3 in order to gain access to Autopilot features, with the exception of
+  [`cleanup_dead_servers`](#cleanup_dead_servers).
+
 * <a name="_recursor"></a><a href="#_recursor">`-recursor`</a> - Specifies the address of an upstream DNS
   server. This option may be provided multiple times, and is functionally
   equivalent to the [`recursors` configuration option](#recursors).
@@ -556,14 +561,22 @@ Consul will not enable TLS for the HTTP API unless the `https` port has been ass
   <br><br>
   The following sub-keys are available:
 
-  * <a name="raft_protocol"></a><a href="#raft_protocol">`raft_protocol`</a> - This controls the internal
-  version of the Raft consensus protocol used for server communications. This defaults to 2 but must
-  be set to 3 in order to gain access to other Autopilot features, with the exception of
-  [`cleanup_dead_servers`](#cleanup_dead_servers).
-
   * <a name="cleanup_dead_servers"></a><a href="#cleanup_dead_servers">`cleanup_dead_servers`</a> - This controls
   the automatic removal of dead server nodes whenever a new server is added to the cluster. Defaults to `true`.
 
+  * <a name="last_contact_threshold"></a><a href="#last_contact_threshold">`last_contact_threshold`</a> - Controls
+  the maximum amount of time a server can go without contact from the leader before being considered unhealthy.
+  Must be a duration value such as `10s`. Defaults to `200ms`.
+
+  * <a name="max_trailing_threshold"></a><a href="#max_trailing_threshold">`max_trailing_threshold`</a> - Controls
+  the maximum number of log entries that a server can trail the leader by before being considered unhealthy. Defaults
+  to 250.
+
+  * <a name="server_stabilization_time"></a><a href="#server_stabilization_time">`server_stabilization_time`</a> -
+  Controls the minimum amount of time a server must be stable in the 'healthy' state before being added to the
+  cluster. Only takes effect if all servers are running Raft protocol version 3 or higher. Must be a duration value
+  such as `10s`. Defaults to `30s`.
+
 * <a name="bootstrap"></a><a href="#bootstrap">`bootstrap`</a> Equivalent to the
   [`-bootstrap` command-line flag](#_bootstrap).
 
@@ -769,6 +782,9 @@ Consul will not enable TLS for the HTTP API unless the `https` port has been ass
 * <a name="protocol"></a><a href="#protocol">`protocol`</a> Equivalent to the
   [`-protocol` command-line flag](#_protocol).
 
+* <a name="raft_protocol"></a><a href="#raft_protocol">`raft_protocol`</a> Equivalent to the
+  [`-raft-protocol` command-line flag](#_raft_protocol).
+
 * <a name="reap"></a><a href="#reap">`reap`</a> This controls Consul's automatic reaping of child processes,
   which is useful if Consul is running as PID 1 in a Docker container. If this isn't specified, then Consul will
   automatically reap child processes if it detects it is running as PID 1. If this is set to true or false, then

website/source/docs/commands/operator/autopilot.html.markdown.erb

@@ -40,6 +40,9 @@ The output looks like this:
 
 ```
 CleanupDeadServers = true
+LastContactThreshold = 200ms
+MaxTrailingLogs = 250
+ServerStabilizationTime = 10s
 ```
 
 ## set-config
@@ -58,6 +61,16 @@ Usage: `consul operator autopilot set-config [options]`
 * `-cleanup-dead-servers` - Specifies whether to enable automatic removal of dead servers
 upon the successful joining of new servers to the cluster. Must be one of `[true|false]`.
 
+* `last-contact-threshold` - Controls the maximum amount of time a server can go without contact
+from the leader before being considered unhealthy. Must be a duration value such as `10s`.
+
+* `max-trailing-logs` - Controls the maximum number of log entries that a server can trail
+the leader by before being considered unhealthy.
+
+* `server-stabilization-time` - Controls the minimum amount of time a server must be stable in
+the 'healthy' state before being added to the cluster. Only takes effect if all servers are
+running Raft protocol version 3 or higher. Must be a duration value such as `10s`.
+
 The output looks like this:
 
 ```