From eedeba682b8a31794be2cea9d105da4fd13c4de3 Mon Sep 17 00:00:00 2001 From: James Phillips Date: Sun, 10 Apr 2016 22:46:07 -0700 Subject: [PATCH] Makes reap time configurable for LAN and WAN. --- command/agent/agent.go | 6 +++ command/agent/agent_test.go | 37 +++++++++++++++++++ command/agent/config.go | 32 ++++++++++++++++ command/agent/config_test.go | 17 +++++++++ .../source/docs/agent/options.html.markdown | 11 ++++++ 5 files changed, 103 insertions(+) diff --git a/command/agent/agent.go b/command/agent/agent.go index fe495fcf91..0fe316c41e 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -299,6 +299,12 @@ func (a *Agent) consulConfig() *consul.Config { base.SerfWANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddrs.SerfWan.IP.String() base.SerfWANConfig.MemberlistConfig.AdvertisePort = a.config.AdvertiseAddrs.SerfWan.Port } + if a.config.ReconnectTimeoutLan != 0 { + base.SerfLANConfig.ReconnectTimeout = a.config.ReconnectTimeoutLan + } + if a.config.ReconnectTimeoutWan != 0 { + base.SerfWANConfig.ReconnectTimeout = a.config.ReconnectTimeoutWan + } if a.config.AdvertiseAddrs.RPC != nil { base.RPCAdvertise = a.config.AdvertiseAddrs.RPC } diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go index fb28fa3df5..32e5fc4a53 100644 --- a/command/agent/agent_test.go +++ b/command/agent/agent_test.go @@ -176,6 +176,43 @@ func TestAgent_CheckAdvertiseAddrsSettings(t *testing.T) { } } +func TestAgent_ReconnectConfigSettings(t *testing.T) { + c := nextConfig() + func() { + dir, agent := makeAgent(t, c) + defer os.RemoveAll(dir) + defer agent.Shutdown() + + lan := agent.consulConfig().SerfLANConfig.ReconnectTimeout + if lan != 3*24*time.Hour { + t.Fatalf("bad: %s", lan.String()) + } + + wan := agent.consulConfig().SerfWANConfig.ReconnectTimeout + if wan != 3*24*time.Hour { + t.Fatalf("bad: %s", wan.String()) + } + }() + + c.ReconnectTimeoutLan = 2 * time.Hour + c.ReconnectTimeoutWan = 3 * time.Hour + func() { + dir, agent := makeAgent(t, c) + defer os.RemoveAll(dir) + defer agent.Shutdown() + + lan := agent.consulConfig().SerfLANConfig.ReconnectTimeout + if lan != 2*time.Hour { + t.Fatalf("bad: %s", lan.String()) + } + + wan := agent.consulConfig().SerfWANConfig.ReconnectTimeout + if wan != 3*time.Hour { + t.Fatalf("bad: %s", wan.String()) + } + }() +} + func TestAgent_AddService(t *testing.T) { dir, agent := makeAgent(t, nextConfig()) defer os.RemoveAll(dir) diff --git a/command/agent/config.go b/command/agent/config.go index fd65d1b424..ce8287ce95 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -312,6 +312,14 @@ type Config struct { RetryIntervalWan time.Duration `mapstructure:"-" json:"-"` RetryIntervalWanRaw string `mapstructure:"retry_interval_wan"` + // ReconnectTimeout* specify the amount of time to wait to reconnect with + // another agent before deciding it's permanently gone. This can be used to + // control the time it takes to reap failed nodes from the cluster. + ReconnectTimeoutLan time.Duration `mapstructure:"-"` + ReconnectTimeoutLanRaw string `mapstructure:"reconnect_timeout"` + ReconnectTimeoutWan time.Duration `mapstructure:"-"` + ReconnectTimeoutWanRaw string `mapstructure:"reconnect_timeout_wan"` + // EnableUi enables the statically-compiled assets for the Consul web UI and // serves them at the default /ui/ endpoint automatically. EnableUi bool `mapstructure:"ui"` @@ -778,6 +786,22 @@ func DecodeConfig(r io.Reader) (*Config, error) { result.RetryIntervalWan = dur } + if raw := result.ReconnectTimeoutLanRaw; raw != "" { + dur, err := time.ParseDuration(raw) + if err != nil { + return nil, fmt.Errorf("ReconnectTimeoutLan invalid: %v", err) + } + result.ReconnectTimeoutLan = dur + } + + if raw := result.ReconnectTimeoutWanRaw; raw != "" { + dur, err := time.ParseDuration(raw) + if err != nil { + return nil, fmt.Errorf("ReconnectTimeoutWan invalid: %v", err) + } + result.ReconnectTimeoutWan = dur + } + // Merge the single recursor if result.DNSRecursor != "" { result.DNSRecursors = append(result.DNSRecursors, result.DNSRecursor) @@ -1131,6 +1155,14 @@ func MergeConfig(a, b *Config) *Config { if b.RetryIntervalWan != 0 { result.RetryIntervalWan = b.RetryIntervalWan } + if b.ReconnectTimeoutLan != 0 { + result.ReconnectTimeoutLan = b.ReconnectTimeoutLan + result.ReconnectTimeoutLanRaw = b.ReconnectTimeoutLanRaw + } + if b.ReconnectTimeoutWan != 0 { + result.ReconnectTimeoutWan = b.ReconnectTimeoutWan + result.ReconnectTimeoutWanRaw = b.ReconnectTimeoutWanRaw + } if b.DNSConfig.NodeTTL != 0 { result.DNSConfig.NodeTTL = b.DNSConfig.NodeTTL } diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 5674413804..09dad36b8b 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -462,6 +462,19 @@ func TestDecodeConfig(t *testing.T) { t.Fatalf("bad: %#v", config) } + // Reconnect timeout LAN and WAN + input = `{"reconnect_timeout": "1m", "reconnect_timeout_wan": "2m"}` + config, err = DecodeConfig(bytes.NewReader([]byte(input))) + if err != nil { + t.Fatalf("err: %s", err) + } + if config.ReconnectTimeoutLanRaw != "1m" || + config.ReconnectTimeoutLan.String() != "1m0s" || + config.ReconnectTimeoutWanRaw != "2m" || + config.ReconnectTimeoutWan.String() != "2m0s" { + t.Fatalf("bad: %#v", config) + } + // Static UI server input = `{"ui": true}` config, err = DecodeConfig(bytes.NewReader([]byte(input))) @@ -1351,6 +1364,10 @@ func TestMergeConfig(t *testing.T) { RetryJoinWan: []string{"1.1.1.1"}, RetryIntervalWanRaw: "10s", RetryIntervalWan: 10 * time.Second, + ReconnectTimeoutLanRaw: "1s", + ReconnectTimeoutLan: 1 * time.Second, + ReconnectTimeoutWanRaw: "2s", + ReconnectTimeoutWan: 2 * time.Second, CheckUpdateInterval: 8 * time.Minute, CheckUpdateIntervalRaw: "8m", ACLToken: "1234", diff --git a/website/source/docs/agent/options.html.markdown b/website/source/docs/agent/options.html.markdown index df90858970..fc21a7e2e8 100644 --- a/website/source/docs/agent/options.html.markdown +++ b/website/source/docs/agent/options.html.markdown @@ -580,6 +580,17 @@ Consul will not enable TLS for the HTTP API unless the `https` port has been ass automatically reap child processes if it detects it is running as PID 1. If this is set to true or false, then it controls reaping regardless of Consul's PID (forces reaping on or off, respectively). +* `reconnect_timeout` This controls + how long it takes for a failed node to be completely removed from the cluster. This defaults to + 72 hours and it is recommended that this is set to at least double the maximum expected recoverable + outage time for a node or network partition. The value is a time with a unit suffix, which can be + "s", "m", "h" for seconds, minutes, or hours. + +* `reconnect_timeout_wan` This + is the WAN equivalent of the `reconnect_timeout` parameter, which + controls how long it takes for a failed server to be completely removed from the WAN pool. This also + defaults to 72 hours. + * `recursor` Provides a single recursor address. This has been deprecated, and the value is appended to the [`recursors`](#recursors) list for backwards compatibility.