Add `testutils.WaitForLeader` and truly wait

command/agent/dns_test.go

@@ -2,12 +2,12 @@ package agent
 
 import (
 	"fmt"
+	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/miekg/dns"
 	"os"
 	"strings"
 	"testing"
-	"time"
 )
 
 func makeDNSServer(t *testing.T) (string, *DNSServer) {
@@ -65,15 +65,15 @@ func TestDNS_NodeLookup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo",
 		Address:    "127.0.0.1",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -129,15 +129,15 @@ func TestDNS_NodeLookup_PeriodName(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node with period in name
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "foo.bar",
 		Address:    "127.0.0.1",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -171,15 +171,15 @@ func TestDNS_NodeLookup_AAAA(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "bar",
 		Address:    "::4242:4242",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -213,15 +213,15 @@ func TestDNS_NodeLookup_CNAME(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
 		Node:       "google",
 		Address:    "www.google.com",
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -256,9 +256,6 @@ func TestDNS_ServiceLookup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -270,6 +267,9 @@ func TestDNS_ServiceLookup(t *testing.T) {
 			Port:    12345,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -317,9 +317,6 @@ func TestDNS_ServiceLookup_TagPeriod(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -331,6 +328,9 @@ func TestDNS_ServiceLookup_TagPeriod(t *testing.T) {
 			Port:    12345,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -378,9 +378,6 @@ func TestDNS_ServiceLookup_Dedup(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -392,6 +389,9 @@ func TestDNS_ServiceLookup_Dedup(t *testing.T) {
 			Port:    12345,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -455,9 +455,6 @@ func TestDNS_ServiceLookup_Dedup_SRV(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -469,6 +466,9 @@ func TestDNS_ServiceLookup_Dedup_SRV(t *testing.T) {
 			Port:    12345,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -584,9 +584,6 @@ func TestDNS_ServiceLookup_FilterCritical(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register nodes
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -603,6 +600,9 @@ func TestDNS_ServiceLookup_FilterCritical(t *testing.T) {
 			Status:  structs.HealthCritical,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)
@@ -648,9 +648,6 @@ func TestDNS_ServiceLookup_Randomize(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register nodes
 	for i := 0; i < 3*maxServiceResponses; i++ {
 		args := &structs.RegisterRequest{
@@ -662,6 +659,9 @@ func TestDNS_ServiceLookup_Randomize(t *testing.T) {
 				Port:    8000,
 			},
 		}
+
+		testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 		var out struct{}
 		if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 			t.Fatalf("err: %v", err)
@@ -712,9 +712,6 @@ func TestDNS_ServiceLookup_CNAME(t *testing.T) {
 	defer os.RemoveAll(dir)
 	defer srv.agent.Shutdown()
 
-	// Wait for leader
-	time.Sleep(100 * time.Millisecond)
-
 	// Register node
 	args := &structs.RegisterRequest{
 		Datacenter: "dc1",
@@ -725,6 +722,9 @@ func TestDNS_ServiceLookup_CNAME(t *testing.T) {
 			Port:    80,
 		},
 	}
+
+	testutil.WaitForLeader(t, srv.agent.RPC, args)
+
 	var out struct{}
 	if err := srv.agent.RPC("Catalog.Register", args, &out); err != nil {
 		t.Fatalf("err: %v", err)

testutil/wait.go

@@ -2,16 +2,18 @@ package testutil
 
 import (
 	"time"
+	"testing"
+	"github.com/hashicorp/consul/consul/structs"
 )
 
 type testFn func() (bool, error)
 type errorFn func(error)
 
 func WaitForResult(test testFn, error errorFn) {
-	retries := 500  // 5 seconds timeout
+	retries := 100  // 5 seconds timeout
 
 	for retries > 0 {
-		time.Sleep(10 * time.Millisecond)
+		time.Sleep(50 * time.Millisecond)
 		retries--
 
 		success, err := test()
@@ -24,3 +26,15 @@ func WaitForResult(test testFn, error errorFn) {
 		}
 	}
 }
+
+type clientRPC func(string, interface {}, interface {}) error
+
+func WaitForLeader(t *testing.T, rpc clientRPC, args interface{}) {
+	WaitForResult(func() (bool, error) {
+		var out structs.IndexedNodes
+		err := rpc("Catalog.ListNodes", args, &out)
+		return out.QueryMeta.KnownLeader, err
+	}, func(err error) {
+		t.Fatalf("failed to find leader: %v", err)
+	})
+}