From 8266f55fc4e564ec1ae0bc5d7e3a33a59b3d82ac Mon Sep 17 00:00:00 2001 From: Jerzy Szczepkowski Date: Thu, 6 Oct 2016 16:28:12 +0200 Subject: [PATCH] Added e2e test for HA master replicas in different zones. Added e2e test for HA master replicas in different zones. --- hack/e2e-internal/e2e-add-master.sh | 3 + hack/e2e-internal/e2e-remove-master.sh | 3 + test/e2e/ha_master.go | 97 ++++++++++++++++++++------ 3 files changed, 81 insertions(+), 22 deletions(-) diff --git a/hack/e2e-internal/e2e-add-master.sh b/hack/e2e-internal/e2e-add-master.sh index 7f04f84166..9d64a997c4 100755 --- a/hack/e2e-internal/e2e-add-master.sh +++ b/hack/e2e-internal/e2e-add-master.sh @@ -16,6 +16,9 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +if [[ ! -z "${1:-}" ]]; then + export KUBE_GCE_ZONE="${1}" +fi export KUBE_REPLICATE_EXISTING_MASTER=true source "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh" diff --git a/hack/e2e-internal/e2e-remove-master.sh b/hack/e2e-internal/e2e-remove-master.sh index 6f9b76022b..cb3310f30e 100755 --- a/hack/e2e-internal/e2e-remove-master.sh +++ b/hack/e2e-internal/e2e-remove-master.sh @@ -16,6 +16,9 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +if [[ ! -z "${1:-}" ]]; then + export KUBE_GCE_ZONE="${1}" +fi export KUBE_DELETE_NODES=false source "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh" diff --git a/test/e2e/ha_master.go b/test/e2e/ha_master.go index 30574c243c..932f92a90c 100644 --- a/test/e2e/ha_master.go +++ b/test/e2e/ha_master.go @@ -22,15 +22,16 @@ import ( "os/exec" "path" "strconv" + "strings" . "github.com/onsi/ginkgo" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/test/e2e/framework" ) -func addMasterReplica() error { - framework.Logf(fmt.Sprintf("Adding a new master replica:")) - v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-add-master.sh")) +func addMasterReplica(zone string) error { + framework.Logf(fmt.Sprintf("Adding a new master replica, zone: %s", zone)) + v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-add-master.sh"), zone) framework.Logf("%s", v) if err != nil { return err @@ -38,9 +39,9 @@ func addMasterReplica() error { return nil } -func removeMasterReplica() error { - framework.Logf(fmt.Sprintf("Removing an existing master replica:")) - v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-remove-master.sh")) +func removeMasterReplica(zone string) error { + framework.Logf(fmt.Sprintf("Removing an existing master replica, zone: %s", zone)) + v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-remove-master.sh"), zone) framework.Logf("%s", v) if err != nil { return err @@ -76,11 +77,44 @@ func verifyNumberOfMasterReplicas(expected int) { } } +func findRegionForZone(zone string) string { + region, err := exec.Command("gcloud", "compute", "zones", "list", zone, "--quiet", "--format=[no-heading](region)").CombinedOutput() + framework.ExpectNoError(err) + if string(region) == "" { + framework.Failf("Region not found; zone: %s", zone) + } + return string(region) +} + +func findZonesForRegion(region string) []string { + output, err := exec.Command("gcloud", "compute", "zones", "list", "--filter=region="+region, + "--quiet", "--format=[no-heading](name)").CombinedOutput() + framework.ExpectNoError(err) + zones := strings.Split(string(output), "\n") + return zones +} + +// removeZoneFromZones removes zone from zones slide. +// Please note that entries in zones can be repeated. In such situation only one replica is removed. +func removeZoneFromZones(zones []string, zone string) []string { + idx := -1 + for j, z := range zones { + if z == zone { + idx = j + break + } + } + if idx >= 0 { + return zones[:idx+copy(zones[idx:], zones[idx+1:])] + } + return zones +} + var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() { f := framework.NewDefaultFramework("ha-master") var c clientset.Interface var ns string - var additionalReplicas int + var additionalReplicaZones []string var existingRCs []string BeforeEach(func() { @@ -88,14 +122,14 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() { c = f.ClientSet ns = f.Namespace.Name verifyNumberOfMasterReplicas(1) - additionalReplicas = 0 + additionalReplicaZones = make([]string, 0) existingRCs = make([]string, 0) }) AfterEach(func() { // Clean-up additional master replicas if the test execution was broken. - for i := 0; i < additionalReplicas; i++ { - removeMasterReplica() + for _, zone := range additionalReplicaZones { + removeMasterReplica(zone) } }) @@ -106,17 +140,17 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() { RemoveReplica ) - step := func(action Action) { + step := func(action Action, zone string) { switch action { case None: case AddReplica: - framework.ExpectNoError(addMasterReplica()) - additionalReplicas++ + framework.ExpectNoError(addMasterReplica(zone)) + additionalReplicaZones = append(additionalReplicaZones, zone) case RemoveReplica: - framework.ExpectNoError(removeMasterReplica()) - additionalReplicas-- + framework.ExpectNoError(removeMasterReplica(zone)) + additionalReplicaZones = removeZoneFromZones(additionalReplicaZones, zone) } - verifyNumberOfMasterReplicas(additionalReplicas + 1) + verifyNumberOfMasterReplicas(len(additionalReplicaZones) + 1) // Verify that API server works correctly with HA master. rcName := "ha-master-" + strconv.Itoa(len(existingRCs)) @@ -125,11 +159,30 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() { verifyRCs(c, ns, existingRCs) } - It("pods survive addition/removal [Slow]", func() { - step(None) - step(AddReplica) - step(AddReplica) - step(RemoveReplica) - step(RemoveReplica) + It("pods survive addition/removal same zone [Slow]", func() { + zone := framework.TestContext.CloudConfig.Zone + step(None, "") + step(AddReplica, zone) + step(AddReplica, zone) + step(RemoveReplica, zone) + step(RemoveReplica, zone) + }) + + It("pods survive addition/removal different zones [Slow]", func() { + zone := framework.TestContext.CloudConfig.Zone + region := findRegionForZone(zone) + zones := findZonesForRegion(region) + zones = removeZoneFromZones(zones, zone) + + step(None, "") + // If numAdditionalReplicas is larger then the number of remaining zones in the region, + // we create a few masters in the same zone and zone entry is repeated in additionalReplicaZones. + numAdditionalReplicas := 2 + for i := 0; i < numAdditionalReplicas; i++ { + step(AddReplica, zones[i%len(zones)]) + } + for i := 0; i < numAdditionalReplicas; i++ { + step(RemoveReplica, zones[i%len(zones)]) + } }) })