From 96818874f4fa958217c49b6736bce5c24f797a2c Mon Sep 17 00:00:00 2001 From: Jerzy Szczepkowski Date: Mon, 16 May 2016 13:27:18 +0200 Subject: [PATCH] Updated e2e tests for cluster autoscaling. Updated e2e tests for cluster autoscaling to work with the new cluster autoscaler. --- .../cluster-autoscaler.manifest | 2 +- test/e2e/cluster_size_autoscaling.go | 130 +++++------------- 2 files changed, 32 insertions(+), 100 deletions(-) diff --git a/cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest b/cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest index c40cb60b50..1d9fdb8cef 100644 --- a/cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest +++ b/cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest @@ -15,7 +15,7 @@ "containers": [ { "name": "cluster-autoscaler", - "image": "gcr.io/mwielgus-proj/cluster-autoscaler:v0.0.1-alpha2-4", + "image": "gcr.io/dev-jsz/cluster-autoscaler:0.1", "command": [ "./cluster-autoscaler", "--kubernetes=http://127.0.0.1:8080?inClusterConfig=f", diff --git a/test/e2e/cluster_size_autoscaling.go b/test/e2e/cluster_size_autoscaling.go index fd351dc76f..040ab9d2a5 100644 --- a/test/e2e/cluster_size_autoscaling.go +++ b/test/e2e/cluster_size_autoscaling.go @@ -18,7 +18,6 @@ package e2e import ( "fmt" - "os/exec" "time" "k8s.io/kubernetes/pkg/api" @@ -29,15 +28,9 @@ import ( ) const ( - scaleUpTimeout = 20 * time.Minute - scaleDownTimeout = 30 * time.Minute + scaleTimeout = 2 * time.Minute ) -// [Feature:ClusterSizeAutoscaling]: Cluster size autoscaling is experimental -// and require Google Cloud Monitoring to be enabled, so these tests are not -// run by default. -// -// These tests take ~20 minutes to run each. var _ = framework.KubeDescribe("Cluster size autoscaling [Feature:ClusterSizeAutoscaling] [Slow]", func() { f := framework.NewDefaultFramework("autoscaling") var nodeCount int @@ -56,117 +49,56 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Feature:ClusterSizeAut memCapacityMb = int((&mem).Value() / 1024 / 1024) }) - AfterEach(func() { - cleanUpAutoscaler() - }) - - It("Should scale cluster size based on cpu utilization", func() { - setUpAutoscaler("cpu/node_utilization", 0.4, nodeCount, nodeCount+1) - - // Consume 50% CPU - rcs := createConsumingRCs(f, "cpu-utilization", nodeCount*coresPerNode, 500, 0) - err := framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout) - for _, rc := range rcs { - rc.CleanUp() - } - framework.ExpectNoError(err) - - framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) - }) - - It("Should scale cluster size based on cpu reservation", func() { - setUpAutoscaler("cpu/node_reservation", 0.5, nodeCount, nodeCount+1) - - ReserveCpu(f, "cpu-reservation", 600*nodeCount*coresPerNode) - framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)) - - framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "cpu-reservation")) - framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) - }) - - It("Should scale cluster size based on memory utilization", func() { - setUpAutoscaler("memory/node_utilization", 0.6, nodeCount, nodeCount+1) - - // Consume 60% of total memory capacity - megabytesPerReplica := int(memCapacityMb * 6 / 10 / coresPerNode) - rcs := createConsumingRCs(f, "mem-utilization", nodeCount*coresPerNode, 0, megabytesPerReplica) - err := framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout) - for _, rc := range rcs { - rc.CleanUp() - } - framework.ExpectNoError(err) - - framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) - }) - - It("Should scale cluster size based on memory reservation", func() { - setUpAutoscaler("memory/node_reservation", 0.5, nodeCount, nodeCount+1) - - ReserveMemory(f, "memory-reservation", nodeCount*memCapacityMb*6/10) - framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)) + It("Should correctly handle pending pods", func() { + By("Too large pending pod does not increase cluster size") + ReserveMemory(f, "memory-reservation", 1, memCapacityMb, false) + // Verify, that cluster size is not changed. + // TODO: find a better way of verification that the cluster size will remain unchanged. + time.Sleep(scaleTimeout) + framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleTimeout)) framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "memory-reservation")) - framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) + framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleTimeout)) + + By("Small pending pods increase cluster size") + ReserveMemory(f, "memory-reservation", 100, nodeCount*memCapacityMb, false) + // Verify, that cluster size is increased + framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleTimeout)) + framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "memory-reservation")) + // TODO(jsz): Enable code bellow when scale down is implemented. + // framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout)) }) }) -func setUpAutoscaler(metric string, target float64, min, max int) { - // TODO integrate with kube-up.sh script once it will support autoscaler setup. - By("Setting up autoscaler to scale based on " + metric) - out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "set-autoscaling", - framework.TestContext.CloudConfig.NodeInstanceGroup, - "--project="+framework.TestContext.CloudConfig.ProjectID, - "--zone="+framework.TestContext.CloudConfig.Zone, - "--custom-metric-utilization=metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/"+metric+fmt.Sprintf(",utilization-target=%v", target)+",utilization-target-type=GAUGE", - fmt.Sprintf("--min-num-replicas=%v", min), - fmt.Sprintf("--max-num-replicas=%v", max), - ).CombinedOutput() - framework.ExpectNoError(err, "Output: "+string(out)) -} - -func createConsumingRCs(f *framework.Framework, name string, count, cpuPerReplica, memPerReplica int) []*ResourceConsumer { - var res []*ResourceConsumer - for i := 1; i <= count; i++ { - name := fmt.Sprintf("%s-%d", name, i) - res = append(res, NewStaticResourceConsumer(name, 1, cpuPerReplica, memPerReplica, 0, int64(cpuPerReplica), int64(memPerReplica+100), f)) - } - return res -} - -func cleanUpAutoscaler() { - By("Removing autoscaler") - out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "stop-autoscaling", - framework.TestContext.CloudConfig.NodeInstanceGroup, - "--project="+framework.TestContext.CloudConfig.ProjectID, - "--zone="+framework.TestContext.CloudConfig.Zone, - ).CombinedOutput() - framework.ExpectNoError(err, "Output: "+string(out)) -} - -func ReserveCpu(f *framework.Framework, id string, millicores int) { +func ReserveCpu(f *framework.Framework, id string, replicas, millicores int) { By(fmt.Sprintf("Running RC which reserves %v millicores", millicores)) + request := int64(millicores / replicas) config := &framework.RCConfig{ Client: f.Client, Name: id, Namespace: f.Namespace.Name, - Timeout: 10 * time.Minute, + Timeout: scaleTimeout, Image: "gcr.io/google_containers/pause-amd64:3.0", - Replicas: millicores / 100, - CpuRequest: 100, + Replicas: replicas, + CpuRequest: request, } framework.ExpectNoError(framework.RunRC(*config)) } -func ReserveMemory(f *framework.Framework, id string, megabytes int) { +func ReserveMemory(f *framework.Framework, id string, replicas, megabytes int, expectRunning bool) { By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes)) + request := int64(1024 * 1024 * megabytes / replicas) config := &framework.RCConfig{ Client: f.Client, Name: id, Namespace: f.Namespace.Name, - Timeout: 10 * time.Minute, + Timeout: scaleTimeout, Image: "gcr.io/google_containers/pause-amd64:3.0", - Replicas: megabytes / 500, - MemRequest: 500 * 1024 * 1024, + Replicas: replicas, + MemRequest: request, + } + err := framework.RunRC(*config) + if expectRunning { + framework.ExpectNoError(err) } - framework.ExpectNoError(framework.RunRC(*config)) }