mirror of https://github.com/k3s-io/k3s
remove soak test cauldron
Ruslan Gustomiasov
parent
b07f311322
commit
0069a6cf08
|
|
@ -21,7 +21,6 @@ filegroup(
|
||||||
"//test/integration:all-srcs",
|
"//test/integration:all-srcs",
|
||||||
"//test/kubemark:all-srcs",
|
"//test/kubemark:all-srcs",
|
||||||
"//test/list:all-srcs",
|
"//test/list:all-srcs",
|
||||||
"//test/soak/cauldron:all-srcs",
|
|
||||||
"//test/soak/serve_hostnames:all-srcs",
|
"//test/soak/serve_hostnames:all-srcs",
|
||||||
"//test/typecheck:all-srcs",
|
"//test/typecheck:all-srcs",
|
||||||
"//test/utils:all-srcs",
|
"//test/utils:all-srcs",
|
||||||
|
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
package(default_visibility = ["//visibility:public"])
|
|
||||||
|
|
||||||
load(
|
|
||||||
"@io_bazel_rules_go//go:def.bzl",
|
|
||||||
"go_binary",
|
|
||||||
"go_library",
|
|
||||||
)
|
|
||||||
|
|
||||||
go_binary(
|
|
||||||
name = "cauldron",
|
|
||||||
embed = [":go_default_library"],
|
|
||||||
)
|
|
||||||
|
|
||||||
go_library(
|
|
||||||
name = "go_default_library",
|
|
||||||
srcs = ["cauldron.go"],
|
|
||||||
importpath = "k8s.io/kubernetes/test/soak/cauldron",
|
|
||||||
deps = [
|
|
||||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
|
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
|
||||||
"//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
|
|
||||||
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
|
|
||||||
"//staging/src/k8s.io/client-go/rest:go_default_library",
|
|
||||||
"//test/e2e/framework:go_default_library",
|
|
||||||
"//vendor/k8s.io/klog:go_default_library",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
filegroup(
|
|
||||||
name = "package-srcs",
|
|
||||||
srcs = glob(["**"]),
|
|
||||||
tags = ["automanaged"],
|
|
||||||
visibility = ["//visibility:private"],
|
|
||||||
)
|
|
||||||
|
|
||||||
filegroup(
|
|
||||||
name = "all-srcs",
|
|
||||||
srcs = [":package-srcs"],
|
|
||||||
tags = ["automanaged"],
|
|
||||||
)
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
# Copyright 2016 The Kubernetes Authors.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
FROM busybox
|
|
||||||
ADD cauldron cauldron
|
|
||||||
ADD cauldron.go cauldron.go
|
|
||||||
ENTRYPOINT ["/cauldron"]
|
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
# Copyright 2016 The Kubernetes Authors.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
.PHONY: cauldron
|
|
||||||
|
|
||||||
TAG = 1.0
|
|
||||||
|
|
||||||
cauldron:
|
|
||||||
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -a -installsuffix cgo --ldflags '-w' cauldron.go
|
|
||||||
|
|
||||||
container: cauldron
|
|
||||||
docker build --pull -t kubernetes/cauldron:$(TAG) .
|
|
||||||
|
|
||||||
push:
|
|
||||||
docker push kubernetes/cauldron:$(TAG)
|
|
||||||
|
|
||||||
rc:
|
|
||||||
kubectl create --validate -f cauldron-rc.yaml
|
|
||||||
|
|
||||||
stop:
|
|
||||||
kubectl delete rc cauldron
|
|
||||||
|
|
||||||
get:
|
|
||||||
kubectl get rc,pods -l app=cauldron
|
|
||||||
|
|
||||||
scale20:
|
|
||||||
kubectl scale rc cauldron --replicas=20
|
|
||||||
|
|
||||||
scale1:
|
|
||||||
kubectl scale rc cauldron --replicas=1
|
|
||||||
|
|
||||||
clean:
|
|
||||||
rm -rf cauldron
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
apiVersion: v1
|
|
||||||
kind: ReplicationController
|
|
||||||
metadata:
|
|
||||||
name: cauldron
|
|
||||||
namespace: default
|
|
||||||
labels:
|
|
||||||
app: cauldron
|
|
||||||
spec:
|
|
||||||
replicas: 1
|
|
||||||
selector:
|
|
||||||
app: cauldron
|
|
||||||
template:
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: cauldron
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: cauldron
|
|
||||||
image: kubernetes/cauldron:1.0
|
|
||||||
imagePullPolicy: Always
|
|
||||||
args: ["--up_to=-1"]
|
|
||||||
|
|
||||||
|
|
@ -1,311 +0,0 @@
|
||||||
/*
|
|
||||||
Copyright 2015 The Kubernetes Authors.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/*
|
|
||||||
This soak tests places a specified number of pods on each node and then
|
|
||||||
repeatedly sends queries to a service running on these pods via
|
|
||||||
a service.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
corev1 "k8s.io/api/core/v1"
|
|
||||||
"k8s.io/apimachinery/pkg/api/errors"
|
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
||||||
"k8s.io/apimachinery/pkg/util/intstr"
|
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
|
||||||
restclient "k8s.io/client-go/rest"
|
|
||||||
"k8s.io/klog"
|
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
queriesAverage = flag.Int("queries", 100, "Number of hostname queries to make in each iteration per pod on average")
|
|
||||||
podsPerNode = flag.Int("pods_per_node", 1, "Number of serve_hostname pods per node")
|
|
||||||
upTo = flag.Int("up_to", 1, "Number of iterations or -1 for no limit")
|
|
||||||
maxPar = flag.Int("max_in_flight", 100, "Maximum number of queries in flight")
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
deleteTimeout = 2 * time.Minute
|
|
||||||
endpointTimeout = 5 * time.Minute
|
|
||||||
nodeListTimeout = 2 * time.Minute
|
|
||||||
podCreateTimeout = 2 * time.Minute
|
|
||||||
podStartTimeout = 30 * time.Minute
|
|
||||||
serviceCreateTimeout = 2 * time.Minute
|
|
||||||
namespaceDeleteTimeout = 5 * time.Minute
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
klog.Infof("Starting cauldron soak test with queries=%d podsPerNode=%d upTo=%d maxPar=%d",
|
|
||||||
*queriesAverage, *podsPerNode, *upTo, *maxPar)
|
|
||||||
|
|
||||||
cc, err := restclient.InClusterConfig()
|
|
||||||
if err != nil {
|
|
||||||
klog.Fatalf("Failed to make client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
client, err := clientset.NewForConfig(cc)
|
|
||||||
if err != nil {
|
|
||||||
klog.Fatalf("Failed to make client: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var nodes *corev1.NodeList
|
|
||||||
for start := time.Now(); time.Since(start) < nodeListTimeout; time.Sleep(2 * time.Second) {
|
|
||||||
nodes, err = client.CoreV1().Nodes().List(metav1.ListOptions{})
|
|
||||||
if err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
klog.Warningf("Failed to list nodes: %v", err)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
klog.Fatalf("Giving up trying to list nodes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(nodes.Items) == 0 {
|
|
||||||
klog.Fatalf("Failed to find any nodes.")
|
|
||||||
}
|
|
||||||
|
|
||||||
klog.Infof("Found %d nodes on this cluster:", len(nodes.Items))
|
|
||||||
for i, node := range nodes.Items {
|
|
||||||
klog.Infof("%d: %s", i, node.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
queries := *queriesAverage * len(nodes.Items) * *podsPerNode
|
|
||||||
|
|
||||||
// Create a uniquely named namespace.
|
|
||||||
got, err := client.CoreV1().Namespaces().Create(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "serve-hostnames-"}})
|
|
||||||
if err != nil {
|
|
||||||
klog.Fatalf("Failed to create namespace: %v", err)
|
|
||||||
}
|
|
||||||
ns := got.Name
|
|
||||||
defer func(ns string) {
|
|
||||||
if err := client.CoreV1().Namespaces().Delete(ns, nil); err != nil {
|
|
||||||
klog.Warningf("Failed to delete namespace %s: %v", ns, err)
|
|
||||||
} else {
|
|
||||||
// wait until the namespace disappears
|
|
||||||
for i := 0; i < int(namespaceDeleteTimeout/time.Second); i++ {
|
|
||||||
if _, err := client.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}); err != nil {
|
|
||||||
if errors.IsNotFound(err) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}(ns)
|
|
||||||
klog.Infof("Created namespace %s", ns)
|
|
||||||
|
|
||||||
// Create a service for these pods.
|
|
||||||
klog.Infof("Creating service %s/serve-hostnames", ns)
|
|
||||||
// Make several attempts to create a service.
|
|
||||||
var svc *corev1.Service
|
|
||||||
for start := time.Now(); time.Since(start) < serviceCreateTimeout; time.Sleep(2 * time.Second) {
|
|
||||||
t := time.Now()
|
|
||||||
svc, err = client.CoreV1().Services(ns).Create(&corev1.Service{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
|
||||||
Name: "serve-hostnames",
|
|
||||||
Labels: map[string]string{
|
|
||||||
"name": "serve-hostname",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Spec: corev1.ServiceSpec{
|
|
||||||
Ports: []corev1.ServicePort{{
|
|
||||||
Protocol: "TCP",
|
|
||||||
Port: 9376,
|
|
||||||
TargetPort: intstr.FromInt(9376),
|
|
||||||
}},
|
|
||||||
Selector: map[string]string{
|
|
||||||
"name": "serve-hostname",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
klog.V(4).Infof("Service create %s/server-hostnames took %v", ns, time.Since(t))
|
|
||||||
if err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
klog.Warningf("After %v failed to create service %s/serve-hostnames: %v", time.Since(start), ns, err)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
klog.Warningf("Unable to create service %s/%s: %v", ns, svc.Name, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Clean up service
|
|
||||||
defer func() {
|
|
||||||
klog.Infof("Cleaning up service %s/serve-hostnames", ns)
|
|
||||||
// Make several attempts to delete the service.
|
|
||||||
for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) {
|
|
||||||
if err := client.CoreV1().Services(ns).Delete(svc.Name, nil); err == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
klog.Warningf("After %v unable to delete service %s/%s: %v", time.Since(start), ns, svc.Name, err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Put serve-hostname pods on each node.
|
|
||||||
podNames := []string{}
|
|
||||||
for i, node := range nodes.Items {
|
|
||||||
for j := 0; j < *podsPerNode; j++ {
|
|
||||||
podName := fmt.Sprintf("serve-hostname-%d-%d", i, j)
|
|
||||||
podNames = append(podNames, podName)
|
|
||||||
// Make several attempts
|
|
||||||
for start := time.Now(); time.Since(start) < podCreateTimeout; time.Sleep(2 * time.Second) {
|
|
||||||
klog.Infof("Creating pod %s/%s on node %s", ns, podName, node.Name)
|
|
||||||
t := time.Now()
|
|
||||||
_, err = client.CoreV1().Pods(ns).Create(&corev1.Pod{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
|
||||||
Name: podName,
|
|
||||||
Labels: map[string]string{
|
|
||||||
"name": "serve-hostname",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Spec: corev1.PodSpec{
|
|
||||||
Containers: []corev1.Container{
|
|
||||||
{
|
|
||||||
Name: "serve-hostname",
|
|
||||||
Image: framework.ServeHostnameImage,
|
|
||||||
Ports: []corev1.ContainerPort{{ContainerPort: 9376}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
NodeName: node.Name,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
klog.V(4).Infof("Pod create %s/%s request took %v", ns, podName, time.Since(t))
|
|
||||||
if err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
klog.Warningf("After %s failed to create pod %s/%s: %v", time.Since(start), ns, podName, err)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
klog.Warningf("Failed to create pod %s/%s: %v", ns, podName, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Clean up the pods
|
|
||||||
defer func() {
|
|
||||||
klog.Info("Cleaning up pods")
|
|
||||||
// Make several attempts to delete the pods.
|
|
||||||
for _, podName := range podNames {
|
|
||||||
for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) {
|
|
||||||
if err = client.CoreV1().Pods(ns).Delete(podName, nil); err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
klog.Warningf("After %v failed to delete pod %s/%s: %v", time.Since(start), ns, podName, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
klog.Info("Waiting for the serve-hostname pods to be ready")
|
|
||||||
for _, podName := range podNames {
|
|
||||||
var pod *corev1.Pod
|
|
||||||
for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) {
|
|
||||||
pod, err = client.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
|
|
||||||
if err != nil {
|
|
||||||
klog.Warningf("Get pod %s/%s failed, ignoring for %v: %v", ns, podName, err, podStartTimeout)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if pod.Status.Phase == corev1.PodRunning {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if pod.Status.Phase != corev1.PodRunning {
|
|
||||||
klog.Warningf("Gave up waiting on pod %s/%s to be running (saw %v)", ns, podName, pod.Status.Phase)
|
|
||||||
} else {
|
|
||||||
klog.Infof("%s/%s is running", ns, podName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for the endpoints to propagate.
|
|
||||||
for start := time.Now(); time.Since(start) < endpointTimeout; time.Sleep(10 * time.Second) {
|
|
||||||
_, err = http.Get(fmt.Sprintf("http://serve-hostnames.%s:9376", ns))
|
|
||||||
if err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
klog.Infof("After %v while making a request got error %v", time.Since(start), err)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
klog.Errorf("Failed to get a response from service: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Repeatedly make requests.
|
|
||||||
for iteration := 0; iteration != *upTo; iteration++ {
|
|
||||||
responseChan := make(chan string, queries)
|
|
||||||
// Use a channel of size *maxPar to throttle the number
|
|
||||||
// of in-flight requests to avoid overloading the service.
|
|
||||||
inFlight := make(chan struct{}, *maxPar)
|
|
||||||
start := time.Now()
|
|
||||||
for q := 0; q < queries; q++ {
|
|
||||||
go func(i int, query int) {
|
|
||||||
inFlight <- struct{}{}
|
|
||||||
t := time.Now()
|
|
||||||
resp, err := http.Get(fmt.Sprintf("http://serve-hostnames.%s:9376", ns))
|
|
||||||
klog.V(4).Infof("Call to serve-hostnames in namespace %s took %v", ns, time.Since(t))
|
|
||||||
if err != nil {
|
|
||||||
klog.Warningf("Call failed during iteration %d query %d : %v", i, query, err)
|
|
||||||
// If the query failed return a string which starts with a character
|
|
||||||
// that can't be part of a hostname.
|
|
||||||
responseChan <- fmt.Sprintf("!failed in iteration %d to issue query %d: %v", i, query, err)
|
|
||||||
} else {
|
|
||||||
defer resp.Body.Close()
|
|
||||||
hostname, err := ioutil.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
responseChan <- fmt.Sprintf("!failed in iteration %d to read body of response: %v", i, err)
|
|
||||||
} else {
|
|
||||||
responseChan <- string(hostname)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
<-inFlight
|
|
||||||
}(iteration, q)
|
|
||||||
}
|
|
||||||
responses := make(map[string]int, *podsPerNode*len(nodes.Items))
|
|
||||||
missing := 0
|
|
||||||
for q := 0; q < queries; q++ {
|
|
||||||
r := <-responseChan
|
|
||||||
klog.V(4).Infof("Got response from %s", r)
|
|
||||||
responses[r]++
|
|
||||||
// If the returned hostname starts with '!' then it indicates
|
|
||||||
// an error response.
|
|
||||||
if len(r) > 0 && r[0] == '!' {
|
|
||||||
klog.V(3).Infof("Got response %s", r)
|
|
||||||
missing++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if missing > 0 {
|
|
||||||
klog.Warningf("Missing %d responses out of %d", missing, queries)
|
|
||||||
}
|
|
||||||
// Report any nodes that did not respond.
|
|
||||||
for n, node := range nodes.Items {
|
|
||||||
for i := 0; i < *podsPerNode; i++ {
|
|
||||||
name := fmt.Sprintf("serve-hostname-%d-%d", n, i)
|
|
||||||
if _, ok := responses[name]; !ok {
|
|
||||||
klog.Warningf("No response from pod %s on node %s at iteration %d", name, node.Name, iteration)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
klog.Infof("Iteration %d took %v for %d queries (%.2f QPS) with %d missing",
|
|
||||||
iteration, time.Since(start), queries-missing, float64(queries-missing)/time.Since(start).Seconds(), missing)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue