github
/
consul
mirror of https://github.com/hashicorp/consul
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

test: convert remaining WaitForResult tests

pull/3011/head
Frank Schroeder 2017-05-05 13:58:13 +02:00
parent ddfa57765c
commit 9c86d5c764
No known key found for this signature in database
GPG Key ID: 4D65C6EAEC87DECD
3 changed files with 220 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
command/exec_test.go
View File

@ -9,7 +9,7 @@ import (
consulapi "github.com/hashicorp/consul/api" consulapi "github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/command/agent" "github.com/hashicorp/consul/command/agent"
"github.com/hashicorp/consul/command/base" "github.com/hashicorp/consul/command/base"
"github.com/hashicorp/consul/testutil" "github.com/hashicorp/consul/testutil/retry"
"github.com/mitchellh/cli" "github.com/mitchellh/cli"
) )
@ -91,12 +91,15 @@ func waitForLeader(t *testing.T, httpAddr string) {
if err != nil { if err != nil {
t.Fatalf("err: %v", err) t.Fatalf("err: %v", err)
} }
if err := testutil.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, qm, err := client.Catalog().Nodes(nil) _, qm, err := client.Catalog().Nodes(nil)
return err == nil && qm.KnownLeader && qm.LastIndex > 0, err if err != nil {
}); err != nil { r.Fatal(err)
t.Fatal(err)
} }
if !qm.KnownLeader || qm.LastIndex == 0 {
r.Fatal("not leader")
}
})
} }
func httpClient(addr string) (*consulapi.Client, error) { func httpClient(addr string) (*consulapi.Client, error) {
@ -203,15 +206,15 @@ func TestExecCommand_Sessions_Foreign(t *testing.T) {
c.conf.localNode = "foo" c.conf.localNode = "foo"
var id string var id string
if err := testutil.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
id, err = c.createSession() id, err = c.createSession()
if err != nil && strings.Contains(err.Error(), "Failed to find Consul server") { if err != nil {
err = nil r.Fatal(err)
} }
return id != "", err if id == "" {
}); err != nil { r.Fatal("no id")
t.Fatal(err)
} }
})
se, _, err := client.Session().Info(id, nil) se, _, err := client.Session().Info(id, nil)
if err != nil { if err != nil {

109
consul/client_test.go
View File

@ -11,6 +11,7 @@ import (
"github.com/hashicorp/consul/consul/structs" "github.com/hashicorp/consul/consul/structs"
"github.com/hashicorp/consul/testrpc" "github.com/hashicorp/consul/testrpc"
"github.com/hashicorp/consul/testutil/retry"
"github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/net-rpc-msgpackrpc"
"github.com/hashicorp/serf/serf" "github.com/hashicorp/serf/serf"
) )
@ -79,32 +80,21 @@ func TestClient_JoinLAN(t *testing.T) {
defer c1.Shutdown() defer c1.Shutdown()
// Try to join // Try to join
addr := fmt.Sprintf("127.0.0.1:%d", addr := fmt.Sprintf("127.0.0.1:%d", s1.config.SerfLANConfig.MemberlistConfig.BindPort)
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := c1.JoinLAN([]string{addr}); err != nil { if _, err := c1.JoinLAN([]string{addr}); err != nil {
t.Fatalf("err: %v", err) t.Fatalf("err: %v", err)
} }
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
return c1.servers.NumServers() == 1, nil if got, want := c1.servers.NumServers(), 1; got != want {
}); err != nil { r.Fatal("got %d servers want %d", got, want)
t.Fatal("expected consul server")
} }
if got, want := len(s1.LANMembers()), 2; got != want {
// Check the members r.Fatalf("got %d server LAN members want %d", got, want)
if err := testrpc.WaitForResult(func() (bool, error) {
server_check := len(s1.LANMembers()) == 2
client_check := len(c1.LANMembers()) == 2
return server_check && client_check, nil
}); err != nil {
t.Fatal("bad len")
} }
if got, want := len(c1.LANMembers()), 2; got != want {
// Check we have a new consul r.Fatalf("got %d client LAN members want %d", got, want)
if err := testrpc.WaitForResult(func() (bool, error) {
return c1.servers.NumServers() == 1, nil
}); err != nil {
t.Fatal("expected consul server")
} }
})
} }
func TestClient_JoinLAN_Invalid(t *testing.T) { func TestClient_JoinLAN_Invalid(t *testing.T) {
@ -117,8 +107,7 @@ func TestClient_JoinLAN_Invalid(t *testing.T) {
defer c1.Shutdown() defer c1.Shutdown()
// Try to join // Try to join
addr := fmt.Sprintf("127.0.0.1:%d", addr := fmt.Sprintf("127.0.0.1:%d", s1.config.SerfLANConfig.MemberlistConfig.BindPort)
s1.config.SerfLANConfig.MemberlistConfig.BindPort)
if _, err := c1.JoinLAN([]string{addr}); err == nil { if _, err := c1.JoinLAN([]string{addr}); err == nil {
t.Fatalf("should error") t.Fatalf("should error")
} }
@ -189,12 +178,11 @@ func TestClient_RPC(t *testing.T) {
} }
// RPC should succeed // RPC should succeed
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
err := c1.RPC("Status.Ping", struct{}{}, &out) if err := c1.RPC("Status.Ping", struct{}{}, &out); err != nil {
return err == nil, err r.Fatal("ping failed", err)
}); err != nil {
t.Fatal(err)
} }
})
} }
func TestClient_RPC_Pool(t *testing.T) { func TestClient_RPC_Pool(t *testing.T) {
@ -214,12 +202,14 @@ func TestClient_RPC_Pool(t *testing.T) {
} }
// Wait for both agents to finish joining // Wait for both agents to finish joining
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
return len(s1.LANMembers()) == 2 && len(c1.LANMembers()) == 2, nil if got, want := len(s1.LANMembers()), 2; got != want {
}); err != nil { r.Fatalf("got %d server LAN members want %d", got, want)
t.Fatalf("Server has %v of %v expected members; Client has %v of %v expected members.",
len(s1.LANMembers()), 2, len(c1.LANMembers()), 2)
} }
if got, want := len(c1.LANMembers()), 2; got != want {
r.Fatalf("got %d client LAN members want %d", got, want)
}
})
// Blast out a bunch of RPC requests at the same time to try to get // Blast out a bunch of RPC requests at the same time to try to get
// contention opening new connections. // contention opening new connections.
@ -230,12 +220,11 @@ func TestClient_RPC_Pool(t *testing.T) {
go func() { go func() {
defer wg.Done() defer wg.Done()
var out struct{} var out struct{}
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
err := c1.RPC("Status.Ping", struct{}{}, &out) if err := c1.RPC("Status.Ping", struct{}{}, &out); err != nil {
return err == nil, err r.Fatal("ping failed", err)
}); err != nil {
t.Fatal(err)
} }
})
}() }()
} }
@ -345,20 +334,17 @@ func TestClient_RPC_TLS(t *testing.T) {
} }
// Wait for joins to finish/RPC to succeed // Wait for joins to finish/RPC to succeed
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
if len(s1.LANMembers()) != 2 { if got, want := len(s1.LANMembers()), 2; got != want {
return false, fmt.Errorf("bad len: %v", len(s1.LANMembers())) r.Fatalf("got %d server LAN members want %d", got, want)
} }
if got, want := len(c1.LANMembers()), 2; got != want {
if len(c1.LANMembers()) != 2 { r.Fatalf("got %d client LAN members want %d", got, want)
return false, fmt.Errorf("bad len: %v", len(c1.LANMembers()))
} }
if err := c1.RPC("Status.Ping", struct{}{}, &out); err != nil {
err := c1.RPC("Status.Ping", struct{}{}, &out) r.Fatal("ping failed", err)
return err == nil, err
}); err != nil {
t.Fatal(err)
} }
})
} }
func TestClient_SnapshotRPC(t *testing.T) { func TestClient_SnapshotRPC(t *testing.T) {
@ -384,11 +370,11 @@ func TestClient_SnapshotRPC(t *testing.T) {
} }
// Wait until we've got a healthy server. // Wait until we've got a healthy server.
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
return c1.servers.NumServers() == 1, nil if got, want := c1.servers.NumServers(), 1; got != want {
}); err != nil { r.Fatal("got %d servers want %d", got, want)
t.Fatal("expected consul server")
} }
})
// Take a snapshot. // Take a snapshot.
var snap bytes.Buffer var snap bytes.Buffer
@ -443,11 +429,11 @@ func TestClient_SnapshotRPC_TLS(t *testing.T) {
} }
// Wait until we've got a healthy server. // Wait until we've got a healthy server.
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
return c1.servers.NumServers() == 1, nil if got, want := c1.servers.NumServers(), 1; got != want {
}); err != nil { r.Fatal("got %d servers want %d", got, want)
t.Fatal("expected consul server")
} }
})
// Take a snapshot. // Take a snapshot.
var snap bytes.Buffer var snap bytes.Buffer
@ -496,11 +482,14 @@ func TestClientServer_UserEvent(t *testing.T) {
testrpc.WaitForLeader(t, s1.RPC, "dc1") testrpc.WaitForLeader(t, s1.RPC, "dc1")
// Check the members // Check the members
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
return len(c1.LANMembers()) == 2 && len(s1.LANMembers()) == 2, nil if got, want := len(s1.LANMembers()), 2; got != want {
}); err != nil { r.Fatalf("got %d server LAN members want %d", got, want)
t.Fatal("bad len")
} }
if got, want := len(c1.LANMembers()), 2; got != want {
r.Fatalf("got %d client LAN members want %d", got, want)
}
})
// Fire the user event // Fire the user event
codec := rpcClient(t, s1) codec := rpcClient(t, s1)

256
consul/leader_test.go
View File

@ -1,7 +1,6 @@
package consul package consul
import ( import (
"errors"
"fmt" "fmt"
"os" "os"
"testing" "testing"
@ -10,6 +9,7 @@ import (
"github.com/hashicorp/consul/api" "github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/consul/structs" "github.com/hashicorp/consul/consul/structs"
"github.com/hashicorp/consul/testrpc" "github.com/hashicorp/consul/testrpc"
"github.com/hashicorp/consul/testutil/retry"
"github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/net-rpc-msgpackrpc"
"github.com/hashicorp/serf/serf" "github.com/hashicorp/serf/serf"
) )
@ -39,15 +39,15 @@ func TestLeader_RegisterMember(t *testing.T) {
// Client should be registered // Client should be registered
state := s1.fsm.State() state := s1.fsm.State()
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, node, err := state.GetNode(c1.config.NodeName) _, node, err := state.GetNode(c1.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
return node != nil, nil if node == nil {
}); err != nil { r.Fatal("client not registered")
t.Fatal("client not registered")
} }
})
// Should have a check // Should have a check
_, checks, err := state.NodeChecks(nil, c1.config.NodeName) _, checks, err := state.NodeChecks(nil, c1.config.NodeName)
@ -114,15 +114,15 @@ func TestLeader_FailedMember(t *testing.T) {
// Should be registered // Should be registered
state := s1.fsm.State() state := s1.fsm.State()
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, node, err := state.GetNode(c1.config.NodeName) _, node, err := state.GetNode(c1.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
return node != nil, nil if node == nil {
}); err != nil { r.Fatal("client not registered")
t.Fatal("client not registered")
} }
})
// Should have a check // Should have a check
_, checks, err := state.NodeChecks(nil, c1.config.NodeName) _, checks, err := state.NodeChecks(nil, c1.config.NodeName)
@ -139,15 +139,15 @@ func TestLeader_FailedMember(t *testing.T) {
t.Fatalf("bad check: %v", checks[0]) t.Fatalf("bad check: %v", checks[0])
} }
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, checks, err = state.NodeChecks(nil, c1.config.NodeName) _, checks, err = state.NodeChecks(nil, c1.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
return checks[0].Status == api.HealthCritical, errors.New(checks[0].Status) if got, want := checks[0].Status, api.HealthCritical; got != want {
}); err != nil { r.Fatalf("got status %q want %q", got, want)
t.Fatalf("check status is %v, should be critical", err)
} }
})
} }
func TestLeader_LeftMember(t *testing.T) { func TestLeader_LeftMember(t *testing.T) {
@ -174,32 +174,31 @@ func TestLeader_LeftMember(t *testing.T) {
state := s1.fsm.State() state := s1.fsm.State()
// Should be registered // Should be registered
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, node, err := state.GetNode(c1.config.NodeName) _, node, err := state.GetNode(c1.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
return node != nil, nil if node == nil {
}); err != nil { r.Fatal("client not registered")
t.Fatal("client should be registered")
} }
})
// Node should leave // Node should leave
c1.Leave() c1.Leave()
c1.Shutdown() c1.Shutdown()
// Should be deregistered // Should be deregistered
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, node, err := state.GetNode(c1.config.NodeName) _, node, err := state.GetNode(c1.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
return node == nil, nil if node != nil {
}); err != nil { r.Fatal("client still registered")
t.Fatal("client should not be registered")
} }
})
} }
func TestLeader_ReapMember(t *testing.T) { func TestLeader_ReapMember(t *testing.T) {
dir1, s1 := testServerWithConfig(t, func(c *Config) { dir1, s1 := testServerWithConfig(t, func(c *Config) {
c.ACLDatacenter = "dc1" c.ACLDatacenter = "dc1"
@ -224,15 +223,15 @@ func TestLeader_ReapMember(t *testing.T) {
state := s1.fsm.State() state := s1.fsm.State()
// Should be registered // Should be registered
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, node, err := state.GetNode(c1.config.NodeName) _, node, err := state.GetNode(c1.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
return node != nil, nil if node == nil {
}); err != nil { r.Fatal("client not registered")
t.Fatal("client should be registered")
} }
})
// Simulate a node reaping // Simulate a node reaping
mems := s1.LANMembers() mems := s1.LANMembers()
@ -344,15 +343,15 @@ func TestLeader_Reconcile(t *testing.T) {
} }
// Should be registered // Should be registered
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, node, err = state.GetNode(c1.config.NodeName) _, node, err := state.GetNode(c1.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
return node != nil, nil if node == nil {
}); err != nil { r.Fatal("client not registered")
t.Fatal("client should be registered")
} }
})
} }
func TestLeader_Reconcile_Races(t *testing.T) { func TestLeader_Reconcile_Races(t *testing.T) {
@ -375,19 +374,16 @@ func TestLeader_Reconcile_Races(t *testing.T) {
// Wait for the server to reconcile the client and register it. // Wait for the server to reconcile the client and register it.
state := s1.fsm.State() state := s1.fsm.State()
var nodeAddr string var nodeAddr string
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, node, err := state.GetNode(c1.config.NodeName) _, node, err := state.GetNode(c1.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
}
if node == nil {
r.Fatal("client not registered")
} }
if node != nil {
nodeAddr = node.Address nodeAddr = node.Address
return true, nil })
}
return false, nil
}); err != nil {
t.Fatalf("client should be registered: %v", err)
}
// Add in some metadata via the catalog (as if the agent synced it // Add in some metadata via the catalog (as if the agent synced it
// there). We also set the serfHealth check to failing so the reconile // there). We also set the serfHealth check to failing so the reconile
@ -428,15 +424,15 @@ func TestLeader_Reconcile_Races(t *testing.T) {
// Fail the member and wait for the health to go critical. // Fail the member and wait for the health to go critical.
c1.Shutdown() c1.Shutdown()
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, checks, err := state.NodeChecks(nil, c1.config.NodeName) _, checks, err := state.NodeChecks(nil, c1.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
return checks[0].Status == api.HealthCritical, errors.New(checks[0].Status) if got, want := checks[0].Status, api.HealthCritical; got != want {
}); err != nil { r.Fatalf("got state %q want %q", got, want)
t.Fatalf("check status should be critical: %v", err)
} }
})
// Make sure the metadata didn't get clobbered. // Make sure the metadata didn't get clobbered.
_, node, err = state.GetNode(c1.config.NodeName) _, node, err = state.GetNode(c1.config.NodeName)
@ -476,32 +472,28 @@ func TestLeader_LeftServer(t *testing.T) {
} }
for _, s := range servers { for _, s := range servers {
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
peers, _ := s.numPeers() if got, want := numPeers(s), 3; got != want {
return peers == 3, nil r.Fatalf("got %d peers want %d", got, want)
}); err != nil {
t.Fatal("should have 3 peers")
} }
})
} }
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
// Kill any server // Kill any server
servers[0].Shutdown() servers[0].Shutdown()
// Force remove the non-leader (transition to left state) // Force remove the non-leader (transition to left state)
if err := servers[1].RemoveFailedNode(servers[0].config.NodeName); err != nil { if err := servers[1].RemoveFailedNode(servers[0].config.NodeName); err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
for _, s := range servers[1:] { for _, s := range servers[1:] {
peers, _ := s.numPeers() if got, want := numPeers(s), 2; got != want {
return peers == 2, fmt.Errorf("%d", peers) r.Fatalf("got %d peers want %d", got, want)
} }
return true, nil
}); err != nil {
t.Fatal(err)
} }
})
} }
func TestLeader_LeftLeader(t *testing.T) { func TestLeader_LeftLeader(t *testing.T) {
@ -529,12 +521,11 @@ func TestLeader_LeftLeader(t *testing.T) {
} }
for _, s := range servers { for _, s := range servers {
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
peers, _ := s.numPeers() if got, want := numPeers(s), 3; got != want {
return peers == 3, nil r.Fatalf("got %d peers want %d", got, want)
}); err != nil {
t.Fatal("should have 3 peers")
} }
})
} }
// Kill the leader! // Kill the leader!
@ -558,25 +549,24 @@ func TestLeader_LeftLeader(t *testing.T) {
continue continue
} }
remain = s remain = s
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
peers, _ := s.numPeers() if got, want := numPeers(s), 2; got != want {
return peers == 2, fmt.Errorf("%d", peers) r.Fatalf("got %d peers want %d", got, want)
}); err != nil {
t.Fatal("should have 2 peers")
} }
})
} }
// Verify the old leader is deregistered // Verify the old leader is deregistered
state := remain.fsm.State() state := remain.fsm.State()
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
_, node, err := state.GetNode(leader.config.NodeName) _, node, err := state.GetNode(leader.config.NodeName)
if err != nil { if err != nil {
t.Fatalf("err: %v", err) r.Fatalf("err: %v", err)
} }
return node == nil, nil if node != nil {
}); err != nil { r.Fatal("leader should be deregistered")
t.Fatal("should be deregistered")
} }
})
} }
func TestLeader_MultiBootstrap(t *testing.T) { func TestLeader_MultiBootstrap(t *testing.T) {
@ -598,12 +588,11 @@ func TestLeader_MultiBootstrap(t *testing.T) {
} }
for _, s := range servers { for _, s := range servers {
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
peers := s.serfLAN.Members() if got, want := len(s.serfLAN.Members()), 2; got != want {
return len(peers) == 2, nil r.Fatalf("got %d peers want %d", got, want)
}); err != nil {
t.Fatal("should have 2 peerss")
} }
})
} }
// Ensure we don't have multiple raft peers // Ensure we don't have multiple raft peers
@ -640,12 +629,11 @@ func TestLeader_TombstoneGC_Reset(t *testing.T) {
} }
for _, s := range servers { for _, s := range servers {
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
peers, _ := s.numPeers() if got, want := numPeers(s), 3; got != want {
return peers == 3, nil r.Fatalf("got %d peers want %d", got, want)
}); err != nil {
t.Fatal("should have 3 peers")
} }
})
} }
var leader *Server var leader *Server
@ -670,24 +658,21 @@ func TestLeader_TombstoneGC_Reset(t *testing.T) {
// Wait for a new leader // Wait for a new leader
leader = nil leader = nil
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
for _, s := range servers { for _, s := range servers {
if s.IsLeader() { if s.IsLeader() {
leader = s leader = s
return true, nil return
} }
} }
return false, nil r.Fatal("no leader")
}); err != nil { })
t.Fatal("should have leader")
}
// Check that the new leader has a pending GC expiration retry.Run(t, func(r *retry.R) {
if err := testrpc.WaitForResult(func() (bool, error) { if !leader.tombstoneGC.PendingExpiration() {
return leader.tombstoneGC.PendingExpiration(), nil r.Fatal("leader has no pending GC expiration")
}); err != nil {
t.Fatal("should have pending expiration")
} }
})
} }
func TestLeader_ReapTombstones(t *testing.T) { func TestLeader_ReapTombstones(t *testing.T) {
@ -746,17 +731,17 @@ func TestLeader_ReapTombstones(t *testing.T) {
// Check that the new leader has a pending GC expiration by // Check that the new leader has a pending GC expiration by
// watching for the tombstone to get removed. // watching for the tombstone to get removed.
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
snap := state.Snapshot() snap := state.Snapshot()
defer snap.Close() defer snap.Close()
stones, err := snap.Tombstones() stones, err := snap.Tombstones()
if err != nil { if err != nil {
return false, err r.Fatal(err)
} }
return stones.Next() == nil, nil if stones.Next() != nil {
}); err != nil { r.Fatal("should have no tombstones")
t.Fatal(err)
} }
})
} }
func TestLeader_RollRaftServer(t *testing.T) { func TestLeader_RollRaftServer(t *testing.T) {
@ -792,24 +777,26 @@ func TestLeader_RollRaftServer(t *testing.T) {
} }
for _, s := range servers { for _, s := range servers {
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
peers, _ := s.numPeers() if got, want := numPeers(s), 3; got != want {
return peers == 3, nil r.Fatalf("got %d peers want %d", got, want)
}); err != nil {
t.Fatal("should have 3 peers")
} }
})
} }
// Kill the v1 server // Kill the v1 server
s2.Shutdown() s2.Shutdown()
for _, s := range []*Server{s1, s3} { for _, s := range []*Server{s1, s3} {
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
minVer, err := ServerMinRaftProtocol(s.LANMembers()) minVer, err := ServerMinRaftProtocol(s.LANMembers())
return minVer == 2, err if err != nil {
}); err != nil { r.Fatal(err)
t.Fatalf("minimum protocol version among servers should be 2")
} }
if got, want := minVer, 2; got != want {
r.Fatalf("got min raft version %d want %d", got, want)
}
})
} }
// Replace the dead server with one running raft protocol v3 // Replace the dead server with one running raft protocol v3
@ -827,12 +814,12 @@ func TestLeader_RollRaftServer(t *testing.T) {
// Make sure the dead server is removed and we're back to 3 total peers // Make sure the dead server is removed and we're back to 3 total peers
for _, s := range servers { for _, s := range servers {
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
addrs := 0 addrs := 0
ids := 0 ids := 0
future := s.raft.GetConfiguration() future := s.raft.GetConfiguration()
if err := future.Error(); err != nil { if err := future.Error(); err != nil {
return false, err r.Fatal(err)
} }
for _, server := range future.Configuration().Servers { for _, server := range future.Configuration().Servers {
if string(server.ID) == string(server.Address) { if string(server.ID) == string(server.Address) {
@ -841,10 +828,13 @@ func TestLeader_RollRaftServer(t *testing.T) {
ids++ ids++
} }
} }
return addrs == 2 && ids == 1, nil if got, want := addrs, 2; got != want {
}); err != nil { r.Fatalf("got %d server addresses want %d", got, want)
t.Fatalf("should see 2 legacy IDs and 1 GUID")
} }
if got, want := ids, 1; got != want {
r.Fatalf("got %d server ids want %d", got, want)
}
})
} }
} }
@ -880,28 +870,27 @@ func TestLeader_ChangeServerID(t *testing.T) {
} }
for _, s := range servers { for _, s := range servers {
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
peers, _ := s.numPeers() if got, want := numPeers(s), 3; got != want {
return peers == 3, nil r.Fatalf("got %d peers want %d", got, want)
}); err != nil {
t.Fatal("should have 3 peers")
} }
})
} }
// Shut down a server, freeing up its address/port // Shut down a server, freeing up its address/port
s3.Shutdown() s3.Shutdown()
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
alive := 0 alive := 0
for _, m := range s1.LANMembers() { for _, m := range s1.LANMembers() {
if m.Status == serf.StatusAlive { if m.Status == serf.StatusAlive {
alive++ alive++
} }
} }
return alive == 2, nil if got, want := alive, 2; got != want {
}); err != nil { r.Fatalf("got %d alive members want %d", got, want)
t.Fatal("should have 2 alive members")
} }
})
// Bring up a new server with s3's address that will get a different ID // Bring up a new server with s3's address that will get a different ID
dir4, s4 := testServerWithConfig(t, func(c *Config) { dir4, s4 := testServerWithConfig(t, func(c *Config) {
@ -922,11 +911,10 @@ func TestLeader_ChangeServerID(t *testing.T) {
// Make sure the dead server is removed and we're back to 3 total peers // Make sure the dead server is removed and we're back to 3 total peers
for _, s := range servers { for _, s := range servers {
if err := testrpc.WaitForResult(func() (bool, error) { retry.Run(t, func(r *retry.R) {
peers, _ := s.numPeers() if got, want := numPeers(s), 3; got != want {
return peers == 3, nil r.Fatalf("got %d peers want %d", got, want)
}); err != nil {
t.Fatal("should have 3 peers")
} }
})
} }
} }