2016-03-29 23:22:53 +00:00
|
|
|
// Package servers provides a Manager interface for Manager managed
|
2017-07-06 10:48:37 +00:00
|
|
|
// metadata.Server objects. The servers package manages servers from a Consul
|
2016-03-30 00:39:19 +00:00
|
|
|
// client's perspective (i.e. a list of servers that a client talks with for
|
|
|
|
// RPCs). The servers package does not provide any API guarantees and should
|
|
|
|
// be called only by `hashicorp/consul`.
|
2017-07-06 10:40:54 +00:00
|
|
|
package router
|
2016-02-20 01:32:16 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"math/rand"
|
2017-06-15 13:16:16 +00:00
|
|
|
"net"
|
2016-02-20 01:32:16 +00:00
|
|
|
"sync"
|
|
|
|
"sync/atomic"
|
|
|
|
"time"
|
|
|
|
|
2017-07-06 10:48:37 +00:00
|
|
|
"github.com/hashicorp/consul/agent/metadata"
|
2016-02-20 01:32:16 +00:00
|
|
|
"github.com/hashicorp/consul/lib"
|
2020-01-28 23:50:41 +00:00
|
|
|
"github.com/hashicorp/consul/logging"
|
|
|
|
"github.com/hashicorp/go-hclog"
|
2016-02-20 01:32:16 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
// clientRPCJitterFraction determines the amount of jitter added to
|
|
|
|
// clientRPCMinReuseDuration before a connection is expired and a new
|
|
|
|
// connection is established in order to rebalance load across consul
|
|
|
|
// servers. The cluster-wide number of connections per second from
|
|
|
|
// rebalancing is applied after this jitter to ensure the CPU impact
|
|
|
|
// is always finite. See newRebalanceConnsPerSecPerServer's comment
|
|
|
|
// for additional commentary.
|
|
|
|
//
|
|
|
|
// For example, in a 10K consul cluster with 5x servers, this default
|
|
|
|
// averages out to ~13 new connections from rebalancing per server
|
|
|
|
// per second (each connection is reused for 120s to 180s).
|
|
|
|
clientRPCJitterFraction = 2
|
|
|
|
|
|
|
|
// clientRPCMinReuseDuration controls the minimum amount of time RPC
|
|
|
|
// queries are sent over an established connection to a single server
|
|
|
|
clientRPCMinReuseDuration = 120 * time.Second
|
|
|
|
|
|
|
|
// Limit the number of new connections a server receives per second
|
|
|
|
// for connection rebalancing. This limit caps the load caused by
|
|
|
|
// continual rebalancing efforts when a cluster is in equilibrium. A
|
|
|
|
// lower value comes at the cost of increased recovery time after a
|
|
|
|
// partition. This parameter begins to take effect when there are
|
|
|
|
// more than ~48K clients querying 5x servers or at lower server
|
|
|
|
// values when there is a partition.
|
|
|
|
//
|
|
|
|
// For example, in a 100K consul cluster with 5x servers, it will
|
|
|
|
// take ~5min for all servers to rebalance their connections. If
|
|
|
|
// 99,995 agents are in the minority talking to only one server, it
|
|
|
|
// will take ~26min for all servers to rebalance. A 10K cluster in
|
|
|
|
// the same scenario will take ~2.6min to rebalance.
|
|
|
|
newRebalanceConnsPerSecPerServer = 64
|
|
|
|
)
|
|
|
|
|
2017-03-14 01:54:34 +00:00
|
|
|
// ManagerSerfCluster is an interface wrapper around Serf in order to make this
|
|
|
|
// easier to unit test.
|
|
|
|
type ManagerSerfCluster interface {
|
2016-02-24 23:04:04 +00:00
|
|
|
NumNodes() int
|
|
|
|
}
|
|
|
|
|
2016-06-20 22:29:38 +00:00
|
|
|
// Pinger is an interface wrapping client.ConnPool to prevent a cyclic import
|
|
|
|
// dependency.
|
2016-03-28 20:46:01 +00:00
|
|
|
type Pinger interface {
|
2020-04-30 20:12:17 +00:00
|
|
|
Ping(dc, nodeName string, addr net.Addr, version int) (bool, error)
|
2016-03-27 02:28:13 +00:00
|
|
|
}
|
|
|
|
|
2016-03-29 23:17:16 +00:00
|
|
|
// serverList is a local copy of the struct used to maintain the list of
|
|
|
|
// Consul servers used by Manager.
|
2016-02-20 01:32:16 +00:00
|
|
|
//
|
2016-03-29 23:17:16 +00:00
|
|
|
// NOTE(sean@): We are explicitly relying on the fact that serverList will
|
2016-02-25 03:51:56 +00:00
|
|
|
// be copied onto the stack. Please keep this structure light.
|
2016-03-29 23:17:16 +00:00
|
|
|
type serverList struct {
|
2016-02-25 03:51:56 +00:00
|
|
|
// servers tracks the locally known servers. List membership is
|
|
|
|
// maintained by Serf.
|
2017-07-06 10:48:37 +00:00
|
|
|
servers []*metadata.Server
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2016-03-29 22:58:15 +00:00
|
|
|
type Manager struct {
|
2016-03-29 23:17:16 +00:00
|
|
|
// listValue manages the atomic load/store of a Manager's serverList
|
|
|
|
listValue atomic.Value
|
|
|
|
listLock sync.Mutex
|
2016-02-20 01:32:16 +00:00
|
|
|
|
2016-03-27 06:41:01 +00:00
|
|
|
// rebalanceTimer controls the duration of the rebalance interval
|
|
|
|
rebalanceTimer *time.Timer
|
|
|
|
|
2016-02-20 01:32:16 +00:00
|
|
|
// shutdownCh is a copy of the channel in consul.Client
|
|
|
|
shutdownCh chan struct{}
|
|
|
|
|
2020-01-28 23:50:41 +00:00
|
|
|
logger hclog.Logger
|
2016-02-24 18:55:04 +00:00
|
|
|
|
2016-03-25 19:38:40 +00:00
|
|
|
// clusterInfo is used to estimate the approximate number of nodes in
|
|
|
|
// a cluster and limit the rate at which it rebalances server
|
2017-03-14 01:54:34 +00:00
|
|
|
// connections. ManagerSerfCluster is an interface that wraps serf.
|
|
|
|
clusterInfo ManagerSerfCluster
|
2016-02-25 03:11:16 +00:00
|
|
|
|
2016-03-27 02:28:13 +00:00
|
|
|
// connPoolPinger is used to test the health of a server in the
|
2016-03-28 20:46:01 +00:00
|
|
|
// connection pool. Pinger is an interface that wraps
|
2016-03-27 02:28:13 +00:00
|
|
|
// client.ConnPool.
|
2016-03-28 20:46:01 +00:00
|
|
|
connPoolPinger Pinger
|
2016-03-27 02:28:13 +00:00
|
|
|
|
2020-04-30 20:12:17 +00:00
|
|
|
// serverName has the name of the managers's server. This is used to
|
|
|
|
// short-circuit pinging to itself.
|
|
|
|
serverName string
|
|
|
|
|
2016-03-28 19:32:30 +00:00
|
|
|
// notifyFailedBarrier is acts as a barrier to prevent queuing behind
|
2016-03-29 23:17:16 +00:00
|
|
|
// serverListLog and acts as a TryLock().
|
2016-02-25 03:11:16 +00:00
|
|
|
notifyFailedBarrier int32
|
2017-03-15 00:47:37 +00:00
|
|
|
|
|
|
|
// offline is used to indicate that there are no servers, or that all
|
|
|
|
// known servers have failed the ping test.
|
|
|
|
offline int32
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2016-02-20 02:57:15 +00:00
|
|
|
// AddServer takes out an internal write lock and adds a new server. If the
|
2016-02-25 03:51:56 +00:00
|
|
|
// server is not known, appends the server to the list. The new server will
|
|
|
|
// begin seeing use after the rebalance timer fires or enough servers fail
|
|
|
|
// organically. If the server is already known, merge the new server
|
|
|
|
// details.
|
2017-07-06 10:48:37 +00:00
|
|
|
func (m *Manager) AddServer(s *metadata.Server) {
|
2016-03-29 23:17:16 +00:00
|
|
|
m.listLock.Lock()
|
|
|
|
defer m.listLock.Unlock()
|
|
|
|
l := m.getServerList()
|
2016-02-20 01:32:16 +00:00
|
|
|
|
|
|
|
// Check if this server is known
|
|
|
|
found := false
|
2016-03-29 23:17:16 +00:00
|
|
|
for idx, existing := range l.servers {
|
2016-03-30 00:39:19 +00:00
|
|
|
if existing.Name == s.Name {
|
2017-07-06 10:48:37 +00:00
|
|
|
newServers := make([]*metadata.Server, len(l.servers))
|
2016-03-29 23:17:16 +00:00
|
|
|
copy(newServers, l.servers)
|
2016-02-20 03:01:46 +00:00
|
|
|
|
|
|
|
// Overwrite the existing server details in order to
|
|
|
|
// possibly update metadata (e.g. server version)
|
2016-03-30 00:39:19 +00:00
|
|
|
newServers[idx] = s
|
2016-02-20 03:01:46 +00:00
|
|
|
|
2016-03-29 23:17:16 +00:00
|
|
|
l.servers = newServers
|
2016-02-20 01:32:16 +00:00
|
|
|
found = true
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add to the list if not known
|
|
|
|
if !found {
|
2017-07-06 10:48:37 +00:00
|
|
|
newServers := make([]*metadata.Server, len(l.servers), len(l.servers)+1)
|
2016-03-29 23:17:16 +00:00
|
|
|
copy(newServers, l.servers)
|
2016-03-30 00:39:19 +00:00
|
|
|
newServers = append(newServers, s)
|
2016-03-29 23:17:16 +00:00
|
|
|
l.servers = newServers
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2017-03-15 00:47:37 +00:00
|
|
|
// Assume we are no longer offline since we've just seen a new server.
|
|
|
|
atomic.StoreInt32(&m.offline, 0)
|
|
|
|
|
|
|
|
// Start using this list of servers.
|
2016-03-29 23:17:16 +00:00
|
|
|
m.saveServerList(l)
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2020-01-29 18:21:38 +00:00
|
|
|
// UpdateTLS updates the TLS setting for the servers in this manager
|
|
|
|
func (m *Manager) UpdateTLS(useTLS bool) {
|
|
|
|
m.listLock.Lock()
|
|
|
|
defer m.listLock.Unlock()
|
|
|
|
|
|
|
|
list := m.getServerList()
|
|
|
|
for _, server := range list.servers {
|
|
|
|
server.UseTLS = useTLS
|
|
|
|
}
|
|
|
|
m.saveServerList(list)
|
|
|
|
}
|
|
|
|
|
2016-02-20 02:57:15 +00:00
|
|
|
// cycleServers returns a new list of servers that has dequeued the first
|
|
|
|
// server and enqueued it at the end of the list. cycleServers assumes the
|
2016-03-29 23:17:16 +00:00
|
|
|
// caller is holding the listLock. cycleServer does not test or ping
|
2016-03-27 01:53:13 +00:00
|
|
|
// the next server inline. cycleServer may be called when the environment
|
|
|
|
// has just entered an unhealthy situation and blocking on a server test is
|
|
|
|
// less desirable than just returning the next server in the firing line. If
|
|
|
|
// the next server fails, it will fail fast enough and cycleServer will be
|
|
|
|
// called again.
|
2017-07-06 10:48:37 +00:00
|
|
|
func (l *serverList) cycleServer() (servers []*metadata.Server) {
|
2016-03-29 23:17:16 +00:00
|
|
|
numServers := len(l.servers)
|
2016-02-20 01:32:16 +00:00
|
|
|
if numServers < 2 {
|
2016-02-25 03:51:56 +00:00
|
|
|
return servers // No action required
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2017-07-06 10:48:37 +00:00
|
|
|
newServers := make([]*metadata.Server, 0, numServers)
|
2016-03-29 23:17:16 +00:00
|
|
|
newServers = append(newServers, l.servers[1:]...)
|
|
|
|
newServers = append(newServers, l.servers[0])
|
2016-03-27 01:53:13 +00:00
|
|
|
|
2016-02-22 21:01:44 +00:00
|
|
|
return newServers
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2016-03-27 02:28:13 +00:00
|
|
|
// removeServerByKey performs an inline removal of the first matching server
|
2017-07-06 10:48:37 +00:00
|
|
|
func (l *serverList) removeServerByKey(targetKey *metadata.Key) {
|
2016-03-29 23:17:16 +00:00
|
|
|
for i, s := range l.servers {
|
2016-03-27 02:28:13 +00:00
|
|
|
if targetKey.Equal(s.Key()) {
|
2016-03-29 23:17:16 +00:00
|
|
|
copy(l.servers[i:], l.servers[i+1:])
|
|
|
|
l.servers[len(l.servers)-1] = nil
|
|
|
|
l.servers = l.servers[:len(l.servers)-1]
|
2016-03-27 02:28:13 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-03-27 02:19:04 +00:00
|
|
|
// shuffleServers shuffles the server list in place
|
2016-03-29 23:17:16 +00:00
|
|
|
func (l *serverList) shuffleServers() {
|
|
|
|
for i := len(l.servers) - 1; i > 0; i-- {
|
2016-03-27 02:19:04 +00:00
|
|
|
j := rand.Int31n(int32(i + 1))
|
2016-03-29 23:17:16 +00:00
|
|
|
l.servers[i], l.servers[j] = l.servers[j], l.servers[i]
|
2016-03-27 02:19:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-15 00:47:37 +00:00
|
|
|
// IsOffline checks to see if all the known servers have failed their ping
|
|
|
|
// test during the last rebalance.
|
|
|
|
func (m *Manager) IsOffline() bool {
|
|
|
|
offline := atomic.LoadInt32(&m.offline)
|
|
|
|
return offline == 1
|
|
|
|
}
|
|
|
|
|
2016-02-25 06:05:05 +00:00
|
|
|
// FindServer takes out an internal "read lock" and searches through the list
|
|
|
|
// of servers to find a "healthy" server. If the server is actually
|
2016-02-25 03:51:56 +00:00
|
|
|
// unhealthy, we rely on Serf to detect this and remove the node from the
|
|
|
|
// server list. If the server at the front of the list has failed or fails
|
|
|
|
// during an RPC call, it is rotated to the end of the list. If there are no
|
|
|
|
// servers available, return nil.
|
2017-07-06 10:48:37 +00:00
|
|
|
func (m *Manager) FindServer() *metadata.Server {
|
2016-03-29 23:17:16 +00:00
|
|
|
l := m.getServerList()
|
|
|
|
numServers := len(l.servers)
|
2016-02-20 01:32:16 +00:00
|
|
|
if numServers == 0 {
|
2020-01-28 23:50:41 +00:00
|
|
|
m.logger.Warn("No servers available")
|
2016-02-20 01:32:16 +00:00
|
|
|
return nil
|
|
|
|
}
|
2017-04-21 01:59:42 +00:00
|
|
|
|
|
|
|
// Return whatever is at the front of the list because it is
|
|
|
|
// assumed to be the oldest in the server list (unless -
|
|
|
|
// hypothetically - the server list was rotated right after a
|
|
|
|
// server was added).
|
|
|
|
return l.servers[0]
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2020-01-29 18:21:38 +00:00
|
|
|
func (m *Manager) checkServers(fn func(srv *metadata.Server) bool) bool {
|
2020-06-03 14:36:32 +00:00
|
|
|
if m == nil {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2020-01-29 18:21:38 +00:00
|
|
|
for _, srv := range m.getServerList().servers {
|
|
|
|
if !fn(srv) {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m *Manager) CheckServers(fn func(srv *metadata.Server) bool) {
|
|
|
|
_ = m.checkServers(fn)
|
|
|
|
}
|
|
|
|
|
2016-03-29 23:17:16 +00:00
|
|
|
// getServerList is a convenience method which hides the locking semantics
|
2016-02-22 21:01:44 +00:00
|
|
|
// of atomic.Value from the caller.
|
2016-03-29 23:17:16 +00:00
|
|
|
func (m *Manager) getServerList() serverList {
|
|
|
|
return m.listValue.Load().(serverList)
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2016-03-29 23:17:16 +00:00
|
|
|
// saveServerList is a convenience method which hides the locking semantics
|
2016-03-25 19:41:22 +00:00
|
|
|
// of atomic.Value from the caller.
|
2016-03-29 23:17:16 +00:00
|
|
|
func (m *Manager) saveServerList(l serverList) {
|
|
|
|
m.listValue.Store(l)
|
2016-03-25 19:41:22 +00:00
|
|
|
}
|
|
|
|
|
2016-03-29 22:58:15 +00:00
|
|
|
// New is the only way to safely create a new Manager struct.
|
2020-04-30 20:12:17 +00:00
|
|
|
func New(logger hclog.Logger, shutdownCh chan struct{}, clusterInfo ManagerSerfCluster, connPoolPinger Pinger, serverName string) (m *Manager) {
|
2020-01-28 23:50:41 +00:00
|
|
|
if logger == nil {
|
|
|
|
logger = hclog.New(&hclog.LoggerOptions{})
|
|
|
|
}
|
|
|
|
|
2016-03-29 22:58:15 +00:00
|
|
|
m = new(Manager)
|
2020-01-28 23:50:41 +00:00
|
|
|
m.logger = logger.Named(logging.Manager)
|
2016-03-29 22:58:15 +00:00
|
|
|
m.clusterInfo = clusterInfo // can't pass *consul.Client: import cycle
|
|
|
|
m.connPoolPinger = connPoolPinger // can't pass *consul.ConnPool: import cycle
|
|
|
|
m.rebalanceTimer = time.NewTimer(clientRPCMinReuseDuration)
|
|
|
|
m.shutdownCh = shutdownCh
|
2020-04-30 20:12:17 +00:00
|
|
|
m.serverName = serverName
|
2017-03-15 15:02:14 +00:00
|
|
|
atomic.StoreInt32(&m.offline, 1)
|
2016-02-22 21:01:44 +00:00
|
|
|
|
2016-03-29 23:17:16 +00:00
|
|
|
l := serverList{}
|
2017-07-06 10:48:37 +00:00
|
|
|
l.servers = make([]*metadata.Server, 0)
|
2016-03-29 23:17:16 +00:00
|
|
|
m.saveServerList(l)
|
2016-03-29 22:58:15 +00:00
|
|
|
return m
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2016-02-25 03:51:56 +00:00
|
|
|
// NotifyFailedServer marks the passed in server as "failed" by rotating it
|
|
|
|
// to the end of the server list.
|
2017-07-06 10:48:37 +00:00
|
|
|
func (m *Manager) NotifyFailedServer(s *metadata.Server) {
|
2016-03-29 23:17:16 +00:00
|
|
|
l := m.getServerList()
|
2016-02-24 22:48:04 +00:00
|
|
|
|
2016-02-25 03:51:56 +00:00
|
|
|
// If the server being failed is not the first server on the list,
|
|
|
|
// this is a noop. If, however, the server is failed and first on
|
|
|
|
// the list, acquire the lock, retest, and take the penalty of moving
|
|
|
|
// the server to the end of the list.
|
|
|
|
|
2016-03-25 19:54:36 +00:00
|
|
|
// Only rotate the server list when there is more than one server
|
2018-02-06 01:56:00 +00:00
|
|
|
if len(l.servers) > 1 && l.servers[0].Name == s.Name &&
|
2016-03-25 19:54:36 +00:00
|
|
|
// Use atomic.CAS to emulate a TryLock().
|
2016-03-29 22:58:15 +00:00
|
|
|
atomic.CompareAndSwapInt32(&m.notifyFailedBarrier, 0, 1) {
|
|
|
|
defer atomic.StoreInt32(&m.notifyFailedBarrier, 0)
|
2016-02-25 03:11:16 +00:00
|
|
|
|
2016-02-24 22:48:04 +00:00
|
|
|
// Grab a lock, retest, and take the hit of cycling the first
|
|
|
|
// server to the end.
|
2016-03-29 23:17:16 +00:00
|
|
|
m.listLock.Lock()
|
|
|
|
defer m.listLock.Unlock()
|
|
|
|
l = m.getServerList()
|
2016-02-24 22:48:04 +00:00
|
|
|
|
2018-02-06 01:56:00 +00:00
|
|
|
if len(l.servers) > 1 && l.servers[0].Name == s.Name {
|
2016-03-29 23:17:16 +00:00
|
|
|
l.servers = l.cycleServer()
|
|
|
|
m.saveServerList(l)
|
2020-01-28 23:50:41 +00:00
|
|
|
m.logger.Debug("cycled away from server", "server", s.String())
|
2016-02-24 22:48:04 +00:00
|
|
|
}
|
|
|
|
}
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2016-02-25 06:09:49 +00:00
|
|
|
// NumServers takes out an internal "read lock" and returns the number of
|
|
|
|
// servers. numServers includes both healthy and unhealthy servers.
|
2016-03-30 00:39:19 +00:00
|
|
|
func (m *Manager) NumServers() int {
|
2016-03-29 23:17:16 +00:00
|
|
|
l := m.getServerList()
|
2016-03-30 00:39:19 +00:00
|
|
|
return len(l.servers)
|
2016-02-25 06:09:49 +00:00
|
|
|
}
|
|
|
|
|
2017-07-06 10:48:37 +00:00
|
|
|
// RebalanceServers shuffles the list of servers on this metadata. The server
|
2016-03-27 02:28:13 +00:00
|
|
|
// at the front of the list is selected for the next RPC. RPC calls that
|
|
|
|
// fail for a particular server are rotated to the end of the list. This
|
|
|
|
// method reshuffles the list periodically in order to redistribute work
|
|
|
|
// across all known consul servers (i.e. guarantee that the order of servers
|
2016-06-20 22:29:38 +00:00
|
|
|
// in the server list is not positively correlated with the age of a server
|
|
|
|
// in the Consul cluster). Periodically shuffling the server list prevents
|
2016-03-27 02:28:13 +00:00
|
|
|
// long-lived clients from fixating on long-lived servers.
|
|
|
|
//
|
|
|
|
// Unhealthy servers are removed when serf notices the server has been
|
|
|
|
// deregistered. Before the newly shuffled server list is saved, the new
|
|
|
|
// remote endpoint is tested to ensure its responsive.
|
2016-03-29 22:58:15 +00:00
|
|
|
func (m *Manager) RebalanceServers() {
|
2016-03-29 23:17:16 +00:00
|
|
|
// Obtain a copy of the current serverList
|
|
|
|
l := m.getServerList()
|
2016-02-20 01:32:16 +00:00
|
|
|
|
2017-03-15 00:47:37 +00:00
|
|
|
// Shuffle servers so we have a chance of picking a new one.
|
2016-03-29 23:17:16 +00:00
|
|
|
l.shuffleServers()
|
2016-03-27 02:28:13 +00:00
|
|
|
|
2016-06-20 22:29:38 +00:00
|
|
|
// Iterate through the shuffled server list to find an assumed
|
|
|
|
// healthy server. NOTE: Do not iterate on the list directly because
|
|
|
|
// this loop mutates the server list in-place.
|
2016-03-27 02:28:13 +00:00
|
|
|
var foundHealthyServer bool
|
2016-03-29 23:17:16 +00:00
|
|
|
for i := 0; i < len(l.servers); i++ {
|
2016-03-27 02:28:13 +00:00
|
|
|
// Always test the first server. Failed servers are cycled
|
|
|
|
// while Serf detects the node has failed.
|
2017-06-15 13:16:16 +00:00
|
|
|
srv := l.servers[0]
|
2016-03-27 02:28:13 +00:00
|
|
|
|
2020-04-30 20:12:17 +00:00
|
|
|
// check to see if the manager is trying to ping itself,
|
|
|
|
// continue if that is the case.
|
|
|
|
if m.serverName != "" && srv.Name == m.serverName {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
ok, err := m.connPoolPinger.Ping(srv.Datacenter, srv.ShortName, srv.Addr, srv.Version)
|
2016-03-27 02:28:13 +00:00
|
|
|
if ok {
|
|
|
|
foundHealthyServer = true
|
|
|
|
break
|
|
|
|
}
|
2020-01-28 23:50:41 +00:00
|
|
|
m.logger.Debug("pinging server failed",
|
|
|
|
"server", srv.String(),
|
|
|
|
"error", err,
|
|
|
|
)
|
2017-11-08 02:13:23 +00:00
|
|
|
l.servers = l.cycleServer()
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2016-03-27 02:28:13 +00:00
|
|
|
// If no healthy servers were found, sleep and wait for Serf to make
|
2017-03-15 00:47:37 +00:00
|
|
|
// the world a happy place again. Update the offline status.
|
|
|
|
if foundHealthyServer {
|
|
|
|
atomic.StoreInt32(&m.offline, 0)
|
|
|
|
} else {
|
|
|
|
atomic.StoreInt32(&m.offline, 1)
|
2020-01-28 23:50:41 +00:00
|
|
|
m.logger.Debug("No healthy servers during rebalance, aborting")
|
2016-03-29 09:37:35 +00:00
|
|
|
return
|
|
|
|
}
|
2016-03-27 02:28:13 +00:00
|
|
|
|
2016-03-29 09:37:35 +00:00
|
|
|
// Verify that all servers are present
|
2016-03-29 23:17:16 +00:00
|
|
|
if m.reconcileServerList(&l) {
|
2020-01-28 23:50:41 +00:00
|
|
|
m.logger.Debug("Rebalanced servers, new active server",
|
|
|
|
"number_of_servers", len(l.servers),
|
|
|
|
"active_server", l.servers[0].String(),
|
|
|
|
)
|
2016-03-27 02:28:13 +00:00
|
|
|
}
|
2020-05-14 18:41:20 +00:00
|
|
|
// else {
|
|
|
|
// reconcileServerList failed because Serf removed the server
|
|
|
|
// that was at the front of the list that had successfully
|
|
|
|
// been Ping'ed. Between the Ping and reconcile, a Serf
|
|
|
|
// event had shown up removing the node.
|
|
|
|
//
|
|
|
|
// Instead of doing any heroics, "freeze in place" and
|
|
|
|
// continue to use the existing connection until the next
|
|
|
|
// rebalance occurs.
|
|
|
|
// }
|
2016-03-29 09:37:35 +00:00
|
|
|
}
|
2016-03-27 02:28:13 +00:00
|
|
|
|
2016-03-29 23:17:16 +00:00
|
|
|
// reconcileServerList returns true when the first server in serverList
|
2016-06-20 22:29:38 +00:00
|
|
|
// exists in the receiver's serverList. If true, the merged serverList is
|
|
|
|
// stored as the receiver's serverList. Returns false if the first server
|
|
|
|
// does not exist in the list (i.e. was removed by Serf during a
|
2016-03-29 09:37:35 +00:00
|
|
|
// PingConsulServer() call. Newly added servers are appended to the list and
|
|
|
|
// other missing servers are removed from the list.
|
2016-03-29 23:17:16 +00:00
|
|
|
func (m *Manager) reconcileServerList(l *serverList) bool {
|
|
|
|
m.listLock.Lock()
|
|
|
|
defer m.listLock.Unlock()
|
2016-03-27 02:28:13 +00:00
|
|
|
|
2016-03-29 23:17:16 +00:00
|
|
|
// newServerCfg is a serverList that has been kept up to date with
|
2016-03-29 09:37:35 +00:00
|
|
|
// Serf node join and node leave events.
|
2016-03-29 23:17:16 +00:00
|
|
|
newServerCfg := m.getServerList()
|
2016-03-27 02:28:13 +00:00
|
|
|
|
2016-03-29 09:37:35 +00:00
|
|
|
// If Serf has removed all nodes, or there is no selected server
|
2016-06-20 22:29:38 +00:00
|
|
|
// (zero nodes in serverList), abort early.
|
2016-03-29 23:17:16 +00:00
|
|
|
if len(newServerCfg.servers) == 0 || len(l.servers) == 0 {
|
2016-03-29 09:37:35 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
type targetServer struct {
|
2017-07-06 10:48:37 +00:00
|
|
|
server *metadata.Server
|
2016-03-27 02:28:13 +00:00
|
|
|
|
2016-03-29 09:37:35 +00:00
|
|
|
// 'b' == both
|
|
|
|
// 'o' == original
|
|
|
|
// 'n' == new
|
|
|
|
state byte
|
|
|
|
}
|
2017-07-06 10:48:37 +00:00
|
|
|
mergedList := make(map[metadata.Key]*targetServer, len(l.servers))
|
2016-03-29 23:17:16 +00:00
|
|
|
for _, s := range l.servers {
|
2016-03-29 09:37:35 +00:00
|
|
|
mergedList[*s.Key()] = &targetServer{server: s, state: 'o'}
|
|
|
|
}
|
|
|
|
for _, s := range newServerCfg.servers {
|
|
|
|
k := s.Key()
|
|
|
|
_, found := mergedList[*k]
|
|
|
|
if found {
|
|
|
|
mergedList[*k].state = 'b'
|
|
|
|
} else {
|
|
|
|
mergedList[*k] = &targetServer{server: s, state: 'n'}
|
2016-03-27 02:28:13 +00:00
|
|
|
}
|
2016-03-29 09:37:35 +00:00
|
|
|
}
|
2016-03-27 02:28:13 +00:00
|
|
|
|
2016-03-29 09:37:35 +00:00
|
|
|
// Ensure the selected server has not been removed by Serf
|
2016-03-29 23:17:16 +00:00
|
|
|
selectedServerKey := l.servers[0].Key()
|
2016-03-29 09:37:35 +00:00
|
|
|
if v, found := mergedList[*selectedServerKey]; found && v.state == 'o' {
|
|
|
|
return false
|
2016-03-27 02:28:13 +00:00
|
|
|
}
|
|
|
|
|
2016-03-29 09:37:35 +00:00
|
|
|
// Append any new servers and remove any old servers
|
|
|
|
for k, v := range mergedList {
|
|
|
|
switch v.state {
|
|
|
|
case 'b':
|
|
|
|
// Do nothing, server exists in both
|
|
|
|
case 'o':
|
|
|
|
// Server has been removed
|
2016-03-29 23:17:16 +00:00
|
|
|
l.removeServerByKey(&k)
|
2016-03-29 09:37:35 +00:00
|
|
|
case 'n':
|
|
|
|
// Server added
|
2016-03-29 23:17:16 +00:00
|
|
|
l.servers = append(l.servers, v.server)
|
2016-03-29 09:37:35 +00:00
|
|
|
default:
|
|
|
|
panic("unknown merge list state")
|
|
|
|
}
|
2016-03-27 02:28:13 +00:00
|
|
|
}
|
|
|
|
|
2016-03-29 23:17:16 +00:00
|
|
|
m.saveServerList(*l)
|
2016-03-29 09:37:35 +00:00
|
|
|
return true
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
|
2016-02-20 02:57:15 +00:00
|
|
|
// RemoveServer takes out an internal write lock and removes a server from
|
2016-02-25 03:51:56 +00:00
|
|
|
// the server list.
|
2017-07-06 10:48:37 +00:00
|
|
|
func (m *Manager) RemoveServer(s *metadata.Server) {
|
2016-03-29 23:17:16 +00:00
|
|
|
m.listLock.Lock()
|
|
|
|
defer m.listLock.Unlock()
|
|
|
|
l := m.getServerList()
|
2016-02-20 01:32:16 +00:00
|
|
|
|
|
|
|
// Remove the server if known
|
2017-04-20 18:42:22 +00:00
|
|
|
for i := range l.servers {
|
2016-03-30 00:39:19 +00:00
|
|
|
if l.servers[i].Name == s.Name {
|
2017-07-06 10:48:37 +00:00
|
|
|
newServers := make([]*metadata.Server, 0, len(l.servers)-1)
|
2016-03-29 23:17:16 +00:00
|
|
|
newServers = append(newServers, l.servers[:i]...)
|
|
|
|
newServers = append(newServers, l.servers[i+1:]...)
|
|
|
|
l.servers = newServers
|
|
|
|
|
|
|
|
m.saveServerList(l)
|
2016-02-20 03:01:46 +00:00
|
|
|
return
|
2016-02-20 01:32:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-03-29 22:58:15 +00:00
|
|
|
// refreshServerRebalanceTimer is only called once m.rebalanceTimer expires.
|
|
|
|
func (m *Manager) refreshServerRebalanceTimer() time.Duration {
|
2016-03-29 23:17:16 +00:00
|
|
|
l := m.getServerList()
|
2016-06-20 22:29:38 +00:00
|
|
|
numServers := len(l.servers)
|
2016-02-20 01:32:16 +00:00
|
|
|
// Limit this connection's life based on the size (and health) of the
|
|
|
|
// cluster. Never rebalance a connection more frequently than
|
|
|
|
// connReuseLowWatermarkDuration, and make sure we never exceed
|
|
|
|
// clusterWideRebalanceConnsPerSec operations/s across numLANMembers.
|
2016-06-20 22:29:38 +00:00
|
|
|
clusterWideRebalanceConnsPerSec := float64(numServers * newRebalanceConnsPerSecPerServer)
|
2016-02-20 01:32:16 +00:00
|
|
|
connReuseLowWatermarkDuration := clientRPCMinReuseDuration + lib.RandomStagger(clientRPCMinReuseDuration/clientRPCJitterFraction)
|
2016-03-29 22:58:15 +00:00
|
|
|
numLANMembers := m.clusterInfo.NumNodes()
|
2016-02-20 01:32:16 +00:00
|
|
|
connRebalanceTimeout := lib.RateScaledInterval(clusterWideRebalanceConnsPerSec, connReuseLowWatermarkDuration, numLANMembers)
|
|
|
|
|
2016-03-29 22:58:15 +00:00
|
|
|
m.rebalanceTimer.Reset(connRebalanceTimeout)
|
2016-02-25 16:05:15 +00:00
|
|
|
return connRebalanceTimeout
|
2016-02-24 18:55:04 +00:00
|
|
|
}
|
|
|
|
|
2016-06-20 22:29:38 +00:00
|
|
|
// ResetRebalanceTimer resets the rebalance timer. This method exists for
|
|
|
|
// testing and should not be used directly.
|
2016-03-29 22:58:15 +00:00
|
|
|
func (m *Manager) ResetRebalanceTimer() {
|
2016-03-29 23:17:16 +00:00
|
|
|
m.listLock.Lock()
|
|
|
|
defer m.listLock.Unlock()
|
2016-03-29 22:58:15 +00:00
|
|
|
m.rebalanceTimer.Reset(clientRPCMinReuseDuration)
|
2016-03-27 06:41:01 +00:00
|
|
|
}
|
|
|
|
|
2016-02-24 22:48:04 +00:00
|
|
|
// Start is used to start and manage the task of automatically shuffling and
|
2016-06-20 22:29:38 +00:00
|
|
|
// rebalancing the list of Consul servers. This maintenance only happens
|
2016-02-25 03:51:56 +00:00
|
|
|
// periodically based on the expiration of the timer. Failed servers are
|
|
|
|
// automatically cycled to the end of the list. New servers are appended to
|
|
|
|
// the list. The order of the server list must be shuffled periodically to
|
2016-06-20 22:29:38 +00:00
|
|
|
// distribute load across all known and available Consul servers.
|
2016-03-29 22:58:15 +00:00
|
|
|
func (m *Manager) Start() {
|
2016-02-20 01:32:16 +00:00
|
|
|
for {
|
|
|
|
select {
|
2016-03-29 22:58:15 +00:00
|
|
|
case <-m.rebalanceTimer.C:
|
|
|
|
m.RebalanceServers()
|
|
|
|
m.refreshServerRebalanceTimer()
|
2016-02-20 01:32:16 +00:00
|
|
|
|
2016-03-29 22:58:15 +00:00
|
|
|
case <-m.shutdownCh:
|
2020-01-28 23:50:41 +00:00
|
|
|
m.logger.Info("shutting down")
|
2016-02-20 01:32:16 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|