From cdd8ce7b206b5c4fd3b935fb520305f2f4fa5882 Mon Sep 17 00:00:00 2001 From: Jerzy Szczepkowski Date: Wed, 28 Dec 2016 15:59:41 +0100 Subject: [PATCH] Added e2e test for HA master that creates multizone workers. Added e2e test for HA master that creates multizone workers. --- ...{e2e-add-master.sh => e2e-grow-cluster.sh} | 15 +++- ...remove-master.sh => e2e-shrink-cluster.sh} | 7 +- test/e2e/ha_master.go | 72 ++++++++++++++++++- 3 files changed, 88 insertions(+), 6 deletions(-) rename hack/e2e-internal/{e2e-add-master.sh => e2e-grow-cluster.sh} (73%) rename hack/e2e-internal/{e2e-remove-master.sh => e2e-shrink-cluster.sh} (85%) diff --git a/hack/e2e-internal/e2e-add-master.sh b/hack/e2e-internal/e2e-grow-cluster.sh similarity index 73% rename from hack/e2e-internal/e2e-add-master.sh rename to hack/e2e-internal/e2e-grow-cluster.sh index 039855726b..148ac3ed1f 100755 --- a/hack/e2e-internal/e2e-add-master.sh +++ b/hack/e2e-internal/e2e-grow-cluster.sh @@ -19,8 +19,17 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. if [[ ! -z "${1:-}" ]]; then export KUBE_GCE_ZONE="${1}" fi -export KUBE_REPLICATE_EXISTING_MASTER=true -export MULTIZONE=true +if [[ ! -z "${2:-}" ]]; then + export MULTIZONE="${2}" +fi +if [[ ! -z "${3:-}" ]]; then + export KUBE_REPLICATE_EXISTING_MASTER="${3}" +fi +if [[ ! -z "${4:-}" ]]; then + export KUBE_USE_EXISTING_MASTER="${4}" +fi +if [[ -z "${NUM_NODES:-}" ]]; then + export NUM_NODES=3 +fi source "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh" - diff --git a/hack/e2e-internal/e2e-remove-master.sh b/hack/e2e-internal/e2e-shrink-cluster.sh similarity index 85% rename from hack/e2e-internal/e2e-remove-master.sh rename to hack/e2e-internal/e2e-shrink-cluster.sh index 95242d1388..33e7e19450 100755 --- a/hack/e2e-internal/e2e-remove-master.sh +++ b/hack/e2e-internal/e2e-shrink-cluster.sh @@ -22,7 +22,12 @@ fi if [[ ! -z "${2:-}" ]]; then export MULTIZONE="${2}" fi -export KUBE_DELETE_NODES=false +if [[ ! -z "${3:-}" ]]; then + export KUBE_DELETE_NODES="${3}" +fi +if [[ ! -z "${4:-}" ]]; then + export KUBE_USE_EXISTING_MASTER="${4}" +fi source "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh" diff --git a/test/e2e/ha_master.go b/test/e2e/ha_master.go index fe81d520b2..654242f9c8 100644 --- a/test/e2e/ha_master.go +++ b/test/e2e/ha_master.go @@ -32,7 +32,7 @@ import ( func addMasterReplica(zone string) error { framework.Logf(fmt.Sprintf("Adding a new master replica, zone: %s", zone)) - v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-add-master.sh"), zone) + v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-grow-cluster.sh"), zone, "true", "true", "false") framework.Logf("%s", v) if err != nil { return err @@ -42,7 +42,27 @@ func addMasterReplica(zone string) error { func removeMasterReplica(zone string) error { framework.Logf(fmt.Sprintf("Removing an existing master replica, zone: %s", zone)) - v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-remove-master.sh"), zone, "true") + v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-shrink-cluster.sh"), zone, "true", "false", "false") + framework.Logf("%s", v) + if err != nil { + return err + } + return nil +} + +func addWorkerNodes(zone string) error { + framework.Logf(fmt.Sprintf("Adding worker nodes, zone: %s", zone)) + v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-grow-cluster.sh"), zone, "true", "false", "true") + framework.Logf("%s", v) + if err != nil { + return err + } + return nil +} + +func removeWorkerNodes(zone string) error { + framework.Logf(fmt.Sprintf("Removing worker nodes, zone: %s", zone)) + v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/e2e-internal/e2e-shrink-cluster.sh"), zone, "true", "true", "true") framework.Logf("%s", v) if err != nil { return err @@ -115,6 +135,7 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() { var c clientset.Interface var ns string var additionalReplicaZones []string + var additionalNodesZones []string var existingRCs []string BeforeEach(func() { @@ -127,6 +148,11 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() { }) AfterEach(func() { + // Clean-up additional worker nodes if the test execution was broken. + for _, zone := range additionalNodesZones { + removeWorkerNodes(zone) + } + // Clean-up additional master replicas if the test execution was broken. for _, zone := range additionalReplicaZones { removeMasterReplica(zone) @@ -140,6 +166,8 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() { None Action = iota AddReplica RemoveReplica + AddNodes + RemoveNodes ) step := func(action Action, zone string) { @@ -151,9 +179,16 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() { case RemoveReplica: framework.ExpectNoError(removeMasterReplica(zone)) additionalReplicaZones = removeZoneFromZones(additionalReplicaZones, zone) + case AddNodes: + framework.ExpectNoError(addWorkerNodes(zone)) + additionalNodesZones = append(additionalNodesZones, zone) + case RemoveNodes: + framework.ExpectNoError(removeWorkerNodes(zone)) + additionalNodesZones = removeZoneFromZones(additionalNodesZones, zone) } verifyNumberOfMasterReplicas(len(additionalReplicaZones) + 1) framework.WaitForMasters(framework.TestContext.CloudConfig.MasterName, c, len(additionalReplicaZones)+1, 10*time.Minute) + framework.AllNodesReady(c, 5*time.Minute) // Verify that API server works correctly with HA master. rcName := "ha-master-" + strconv.Itoa(len(existingRCs)) @@ -191,4 +226,37 @@ var _ = framework.KubeDescribe("HA-master [Feature:HAMaster]", func() { step(RemoveReplica, zones[i%len(zones)]) } }) + + It("survive addition/removal replicas multizone workers [Serial][Disruptive]", func() { + zone := framework.TestContext.CloudConfig.Zone + region := findRegionForZone(zone) + zones := findZonesForRegion(region) + zones = removeZoneFromZones(zones, zone) + + step(None, "") + numAdditionalReplicas := 2 + + // Add worker nodes. + for i := 0; i < numAdditionalReplicas && i < len(zones); i++ { + step(AddNodes, zones[i]) + } + + // Add master repilcas. + // + // If numAdditionalReplicas is larger then the number of remaining zones in the region, + // we create a few masters in the same zone and zone entry is repeated in additionalReplicaZones. + for i := 0; i < numAdditionalReplicas; i++ { + step(AddReplica, zones[i%len(zones)]) + } + + // Remove master repilcas. + for i := 0; i < numAdditionalReplicas; i++ { + step(RemoveReplica, zones[i%len(zones)]) + } + + // Remove worker nodes. + for i := 0; i < numAdditionalReplicas && i < len(zones); i++ { + step(RemoveNodes, zones[i]) + } + }) })