mirror of https://github.com/k3s-io/k3s
Merge pull request #25658 from jszczepkowski/kac-test
Automatic merge from submit-queue Updated e2e tests for cluster autoscaling. []() Updated e2e tests for cluster autoscaling to work with the new cluster autoscaler.pull/6/head
k8s-merge-robot
commit
a0cc59f28a
|
|
@ -15,7 +15,7 @@
|
|||
"containers": [
|
||||
{
|
||||
"name": "cluster-autoscaler",
|
||||
"image": "gcr.io/mwielgus-proj/cluster-autoscaler:v0.0.1-alpha2-4",
|
||||
"image": "gcr.io/dev-jsz/cluster-autoscaler:0.1",
|
||||
"command": [
|
||||
"./cluster-autoscaler",
|
||||
"--kubernetes=http://127.0.0.1:8080?inClusterConfig=f",
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ package e2e
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"time"
|
||||
|
||||
"k8s.io/kubernetes/pkg/api"
|
||||
|
|
@ -29,15 +28,9 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
scaleUpTimeout = 20 * time.Minute
|
||||
scaleDownTimeout = 30 * time.Minute
|
||||
scaleTimeout = 2 * time.Minute
|
||||
)
|
||||
|
||||
// [Feature:ClusterSizeAutoscaling]: Cluster size autoscaling is experimental
|
||||
// and require Google Cloud Monitoring to be enabled, so these tests are not
|
||||
// run by default.
|
||||
//
|
||||
// These tests take ~20 minutes to run each.
|
||||
var _ = framework.KubeDescribe("Cluster size autoscaling [Feature:ClusterSizeAutoscaling] [Slow]", func() {
|
||||
f := framework.NewDefaultFramework("autoscaling")
|
||||
var nodeCount int
|
||||
|
|
@ -56,117 +49,56 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Feature:ClusterSizeAut
|
|||
memCapacityMb = int((&mem).Value() / 1024 / 1024)
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
cleanUpAutoscaler()
|
||||
})
|
||||
|
||||
It("Should scale cluster size based on cpu utilization", func() {
|
||||
setUpAutoscaler("cpu/node_utilization", 0.4, nodeCount, nodeCount+1)
|
||||
|
||||
// Consume 50% CPU
|
||||
rcs := createConsumingRCs(f, "cpu-utilization", nodeCount*coresPerNode, 500, 0)
|
||||
err := framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)
|
||||
for _, rc := range rcs {
|
||||
rc.CleanUp()
|
||||
}
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
|
||||
})
|
||||
|
||||
It("Should scale cluster size based on cpu reservation", func() {
|
||||
setUpAutoscaler("cpu/node_reservation", 0.5, nodeCount, nodeCount+1)
|
||||
|
||||
ReserveCpu(f, "cpu-reservation", 600*nodeCount*coresPerNode)
|
||||
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout))
|
||||
|
||||
framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "cpu-reservation"))
|
||||
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
|
||||
})
|
||||
|
||||
It("Should scale cluster size based on memory utilization", func() {
|
||||
setUpAutoscaler("memory/node_utilization", 0.6, nodeCount, nodeCount+1)
|
||||
|
||||
// Consume 60% of total memory capacity
|
||||
megabytesPerReplica := int(memCapacityMb * 6 / 10 / coresPerNode)
|
||||
rcs := createConsumingRCs(f, "mem-utilization", nodeCount*coresPerNode, 0, megabytesPerReplica)
|
||||
err := framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)
|
||||
for _, rc := range rcs {
|
||||
rc.CleanUp()
|
||||
}
|
||||
framework.ExpectNoError(err)
|
||||
|
||||
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
|
||||
})
|
||||
|
||||
It("Should scale cluster size based on memory reservation", func() {
|
||||
setUpAutoscaler("memory/node_reservation", 0.5, nodeCount, nodeCount+1)
|
||||
|
||||
ReserveMemory(f, "memory-reservation", nodeCount*memCapacityMb*6/10)
|
||||
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout))
|
||||
It("Should correctly handle pending pods", func() {
|
||||
By("Too large pending pod does not increase cluster size")
|
||||
ReserveMemory(f, "memory-reservation", 1, memCapacityMb, false)
|
||||
// Verify, that cluster size is not changed.
|
||||
// TODO: find a better way of verification that the cluster size will remain unchanged.
|
||||
time.Sleep(scaleTimeout)
|
||||
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleTimeout))
|
||||
|
||||
framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "memory-reservation"))
|
||||
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
|
||||
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleTimeout))
|
||||
|
||||
By("Small pending pods increase cluster size")
|
||||
ReserveMemory(f, "memory-reservation", 100, nodeCount*memCapacityMb, false)
|
||||
// Verify, that cluster size is increased
|
||||
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleTimeout))
|
||||
framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "memory-reservation"))
|
||||
// TODO(jsz): Enable code bellow when scale down is implemented.
|
||||
// framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
|
||||
})
|
||||
})
|
||||
|
||||
func setUpAutoscaler(metric string, target float64, min, max int) {
|
||||
// TODO integrate with kube-up.sh script once it will support autoscaler setup.
|
||||
By("Setting up autoscaler to scale based on " + metric)
|
||||
out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "set-autoscaling",
|
||||
framework.TestContext.CloudConfig.NodeInstanceGroup,
|
||||
"--project="+framework.TestContext.CloudConfig.ProjectID,
|
||||
"--zone="+framework.TestContext.CloudConfig.Zone,
|
||||
"--custom-metric-utilization=metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/"+metric+fmt.Sprintf(",utilization-target=%v", target)+",utilization-target-type=GAUGE",
|
||||
fmt.Sprintf("--min-num-replicas=%v", min),
|
||||
fmt.Sprintf("--max-num-replicas=%v", max),
|
||||
).CombinedOutput()
|
||||
framework.ExpectNoError(err, "Output: "+string(out))
|
||||
}
|
||||
|
||||
func createConsumingRCs(f *framework.Framework, name string, count, cpuPerReplica, memPerReplica int) []*ResourceConsumer {
|
||||
var res []*ResourceConsumer
|
||||
for i := 1; i <= count; i++ {
|
||||
name := fmt.Sprintf("%s-%d", name, i)
|
||||
res = append(res, NewStaticResourceConsumer(name, 1, cpuPerReplica, memPerReplica, 0, int64(cpuPerReplica), int64(memPerReplica+100), f))
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func cleanUpAutoscaler() {
|
||||
By("Removing autoscaler")
|
||||
out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "stop-autoscaling",
|
||||
framework.TestContext.CloudConfig.NodeInstanceGroup,
|
||||
"--project="+framework.TestContext.CloudConfig.ProjectID,
|
||||
"--zone="+framework.TestContext.CloudConfig.Zone,
|
||||
).CombinedOutput()
|
||||
framework.ExpectNoError(err, "Output: "+string(out))
|
||||
}
|
||||
|
||||
func ReserveCpu(f *framework.Framework, id string, millicores int) {
|
||||
func ReserveCpu(f *framework.Framework, id string, replicas, millicores int) {
|
||||
By(fmt.Sprintf("Running RC which reserves %v millicores", millicores))
|
||||
request := int64(millicores / replicas)
|
||||
config := &framework.RCConfig{
|
||||
Client: f.Client,
|
||||
Name: id,
|
||||
Namespace: f.Namespace.Name,
|
||||
Timeout: 10 * time.Minute,
|
||||
Timeout: scaleTimeout,
|
||||
Image: "gcr.io/google_containers/pause-amd64:3.0",
|
||||
Replicas: millicores / 100,
|
||||
CpuRequest: 100,
|
||||
Replicas: replicas,
|
||||
CpuRequest: request,
|
||||
}
|
||||
framework.ExpectNoError(framework.RunRC(*config))
|
||||
}
|
||||
|
||||
func ReserveMemory(f *framework.Framework, id string, megabytes int) {
|
||||
func ReserveMemory(f *framework.Framework, id string, replicas, megabytes int, expectRunning bool) {
|
||||
By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes))
|
||||
request := int64(1024 * 1024 * megabytes / replicas)
|
||||
config := &framework.RCConfig{
|
||||
Client: f.Client,
|
||||
Name: id,
|
||||
Namespace: f.Namespace.Name,
|
||||
Timeout: 10 * time.Minute,
|
||||
Timeout: scaleTimeout,
|
||||
Image: "gcr.io/google_containers/pause-amd64:3.0",
|
||||
Replicas: megabytes / 500,
|
||||
MemRequest: 500 * 1024 * 1024,
|
||||
Replicas: replicas,
|
||||
MemRequest: request,
|
||||
}
|
||||
err := framework.RunRC(*config)
|
||||
if expectRunning {
|
||||
framework.ExpectNoError(err)
|
||||
}
|
||||
framework.ExpectNoError(framework.RunRC(*config))
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue