mirror of https://github.com/k3s-io/k3s
Merge pull request #7639 from jayunit100/e2e-k8bps
E2E: Soak test and Functional tests for K8Petstorepull/6/head
Nikhil Jindal
commit
2dbe659887
|
|
@ -16,16 +16,30 @@
|
|||
|
||||
echo "WRITING KUBE FILES , will overwrite the jsons, then testing pods. is kube clean ready to go?"
|
||||
|
||||
|
||||
#Args below can be overriden when calling from cmd line.
|
||||
#Just send all the args in order.
|
||||
#for dev/test you can use:
|
||||
#kubectl=$GOPATH/src/github.com/GoogleCloudPlatform/kubernetes/cluster/kubectl.sh"
|
||||
kubectl="kubectl"
|
||||
VERSION="r.2.8.19"
|
||||
PUBLIC_IP="10.1.4.89" # ip which we use to access the Web server.
|
||||
SECONDS=1000 # number of seconds to measure throughput.
|
||||
FE="1" # amount of Web server
|
||||
LG="1" # amount of load generators
|
||||
SLAVE="1" # amount of redis slaves
|
||||
TEST_SECONDS="1000" # 0 = Dont run tests, if > 0, run tests for n seconds.
|
||||
NS="k8petstore" # namespace
|
||||
|
||||
kubectl="${1:-$kubectl}"
|
||||
VERSION="${2:-$VERSION}"
|
||||
PUBLIC_IP="${3:-$PUBLIC_IP}"
|
||||
FE="${4:-$FE}"
|
||||
LG="${5:-$LG}"
|
||||
SLAVE="${6:-$SLAVE}"
|
||||
TEST_SECONDS="${7:-$TEST}"
|
||||
NS="${8:-$NS}"
|
||||
|
||||
echo "Running w/ args: kubectl $kubectl version $VERSION ip $PUBLIC_IP sec $_SECONDS fe $FE lg $LG slave $SLAVE test $TEST NAMESPACE $NS"
|
||||
function create {
|
||||
|
||||
cat << EOF > fe-rc.json
|
||||
|
|
@ -188,53 +202,74 @@ cat << EOF > slave-rc.json
|
|||
"labels": {"name": "redisslave"}
|
||||
}
|
||||
EOF
|
||||
$kubectl create -f rm.json --api-version=v1beta1
|
||||
$kubectl create -f rm-s.json --api-version=v1beta1
|
||||
$kubectl create -f rm.json --api-version=v1beta1 --namespace=$NS
|
||||
$kubectl create -f rm-s.json --api-version=v1beta1 --namespace=$NS
|
||||
sleep 3 # precaution to prevent fe from spinning up too soon.
|
||||
$kubectl create -f slave-rc.json --api-version=v1beta1
|
||||
$kubectl create -f rs-s.json --api-version=v1beta1
|
||||
$kubectl create -f slave-rc.json --api-version=v1beta1 --namespace=$NS
|
||||
$kubectl create -f rs-s.json --api-version=v1beta1 --namespace=$NS
|
||||
sleep 3 # see above comment.
|
||||
$kubectl create -f fe-rc.json --api-version=v1beta1
|
||||
$kubectl create -f fe-s.json --api-version=v1beta1
|
||||
$kubectl create -f bps-load-gen-rc.json --api-version=v1beta1
|
||||
$kubectl create -f fe-rc.json --api-version=v1beta1 --namespace=$NS
|
||||
$kubectl create -f fe-s.json --api-version=v1beta1 --namespace=$NS
|
||||
$kubectl create -f bps-load-gen-rc.json --api-version=v1beta1 --namespace=$NS
|
||||
}
|
||||
|
||||
function test {
|
||||
function pollfor {
|
||||
pass_http=0
|
||||
|
||||
### Test HTTP Server comes up.
|
||||
for i in `seq 1 150`;
|
||||
do
|
||||
### Just testing that the front end comes up. Not sure how to test total entries etc... (yet)
|
||||
echo "Trying curl ... $i . expect a few failures while pulling images... "
|
||||
echo "Trying curl ... $PUBLIC_IP:3000 , attempt $i . expect a few failures while pulling images... "
|
||||
curl "$PUBLIC_IP:3000" > result
|
||||
cat result
|
||||
cat result | grep -q "k8-bps"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "TEST PASSED after $i tries !"
|
||||
i=1000
|
||||
break
|
||||
echo "TEST PASSED after $i tries !"
|
||||
i=1000
|
||||
break
|
||||
else
|
||||
echo "the above RESULT didn't contain target string for trial $i"
|
||||
fi
|
||||
sleep 5
|
||||
sleep 3
|
||||
done
|
||||
|
||||
if [ $i -eq 1000 ]; then
|
||||
pass_http=-1
|
||||
pass_http=1
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
function tests {
|
||||
pass_load=0
|
||||
|
||||
### Print statistics of db size, every second, until $SECONDS are up.
|
||||
for i in `seq 1 $SECONDS`;
|
||||
do
|
||||
echo "curl : $i"
|
||||
curl "$PUBLIC_IP:3000/llen" >> result
|
||||
for i in `seq 1 $TEST_SECONDS`;
|
||||
do
|
||||
echo "curl : $PUBLIC_IP:3000 , $i of $TEST_SECONDS"
|
||||
curr_cnt="`curl "$PUBLIC_IP:3000/llen"`"
|
||||
### Write CSV File of # of trials / total transcations.
|
||||
echo "$i $curr_cnt" >> result
|
||||
echo "total transactions so far : $curr_cnt"
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
create
|
||||
|
||||
test
|
||||
pollfor
|
||||
|
||||
if [[ $pass_http -eq 1 ]]; then
|
||||
echo "Passed..."
|
||||
else
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ $TEST_SECONDS -eq 0 ]]; then
|
||||
echo "skipping tests, TEST_SECONDS value was 0"
|
||||
else
|
||||
echo "running polling tests now for $TEST_SECONDS"
|
||||
tests
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,193 @@
|
|||
/*
|
||||
Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
|
||||
"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var (
|
||||
root0 = absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), "..")))
|
||||
err error = nil
|
||||
namespace *api.Namespace = nil
|
||||
ns string = ""
|
||||
)
|
||||
|
||||
//This must run on a machine which is in the kube-proxy ring. Otherwise, the publicIP binding will fail.
|
||||
//The IP Below ~ letter->number cipher for k8petstore (most likely it won't be bound by any other process)
|
||||
var ip = "165.201.92.15"
|
||||
|
||||
// i.e. after 50 trials, expect 3000 transactions... minimum we settle for is 500.
|
||||
var minionCount int
|
||||
|
||||
// readTransactions reads # of transactions from the k8petstore web server endpoint.
|
||||
// for more details see the source of the k8petstore web server.
|
||||
func readTransactions(c *client.Client) int {
|
||||
resp, err := http.Get("http://165.201.92.15:3000/llen")
|
||||
if err != nil {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
totalTrans, err := strconv.Atoi(string(body))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
return totalTrans
|
||||
}
|
||||
|
||||
// runK8petstore runs the k8petstore application, bound to external ip "ip", and polls it to assert that "min_expected"
|
||||
// transactions are acquired in a maximum of "max_seconds".
|
||||
func runK8petstore(ip string, restServers int, loadGenerators int, c *client.Client, minExpected int, maxSeconds int64) {
|
||||
k8bps := filepath.Join(root0, "examples/k8petstore/k8petstore.sh")
|
||||
|
||||
//Get the count of minions. We'll use this to decide expected throughput and loadgen/REST server count.
|
||||
cmd := exec.Command(
|
||||
k8bps,
|
||||
"kubectl", //for dev, replace w/ "cluster/kubectl.sh"
|
||||
"r.2.8.19",
|
||||
ip,
|
||||
strconv.Itoa(restServers),
|
||||
strconv.Itoa(loadGenerators),
|
||||
"1", "0", ns) // 1= # slave, 0 = don't bother running the embedded test.
|
||||
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
Logf("Starting k8petstore application....")
|
||||
//Run the k8petstore app, and log / fail if it returns any errors.
|
||||
//This should return quickly, assuming containers are downloaded.
|
||||
if err = cmd.Start(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
//Make sure exit code != 0
|
||||
if err = cmd.Wait(); err != nil {
|
||||
if exiterr, ok := err.(*exec.ExitError); ok {
|
||||
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
|
||||
log.Printf("Exit Status: %d", status.ExitStatus())
|
||||
}
|
||||
}
|
||||
}
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Logf("... Done starting k8petstore successfully.... Now we will poll it...")
|
||||
|
||||
//By now, the original k8petstore app has run and spun up a k8petstore w/ load generators.
|
||||
//Lets poll until we reach min_expected transactions
|
||||
totalTransactions := 0
|
||||
Logf("Start polling, timeout is %v seconds", maxSeconds)
|
||||
|
||||
timeout := time.After(time.Duration(maxSeconds) * time.Second)
|
||||
tick := time.Tick(2 * time.Second)
|
||||
|
||||
T:
|
||||
for {
|
||||
select {
|
||||
case <-timeout:
|
||||
Logf("Timeout %v reached. Breaking!", tick)
|
||||
break T
|
||||
case <-tick:
|
||||
totalTransactions = readTransactions(c)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
if totalTransactions > minExpected {
|
||||
break T
|
||||
}
|
||||
Logf("%v == %v total petstore transactions stored into redis. ==", time.Now(), totalTransactions)
|
||||
}
|
||||
}
|
||||
|
||||
//Finally! We should have exceeded the min_expected num of transactions.
|
||||
//If this fails, but there are transactions being created, we may need to recalibrate
|
||||
//the min_expected value - or else - your cluster is broken/slow !
|
||||
Ω(totalTransactions).Should(BeNumerically(">", minExpected))
|
||||
}
|
||||
|
||||
var _ = Describe("k8bps", func() {
|
||||
BeforeEach(func() {
|
||||
By("Creating a kubernetes client")
|
||||
c, err = loadClient()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
By("Building a namespace api object")
|
||||
namespace, err = createTestingNS("k8petstore-soak", c)
|
||||
ns = namespace.Name
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
//Now get the # of minions and calibrate the test params to them...
|
||||
minions, err := c.Nodes().List(labels.Everything(), fields.Everything())
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
minionCount = len(minions.Items)
|
||||
|
||||
//simple calibration. TODO, make more ambitious (i.e. 10x density style load generators)...
|
||||
//load_generators = minionCount
|
||||
//rest_servers = minionCount
|
||||
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
By(fmt.Sprintf("Destroying namespace for this test %v", namespace.Name))
|
||||
if err := c.Namespaces().Delete(namespace.Name); err != nil {
|
||||
Failf("Couldn't delete ns %s", err)
|
||||
}
|
||||
})
|
||||
|
||||
//On a single node cluster, we expect about 10 transactions every second on average.
|
||||
//admittedly, this is a very rough estimate given that the ETL is done in batches...
|
||||
It(fmt.Sprintf("k8petstore-FUNCTIONAL : Should quickly acquire 500 petstore transactions."), func() {
|
||||
|
||||
//max number of trial before k8petstore.sh dies.
|
||||
var loadGenerators int = minionCount
|
||||
var restServers int = minionCount
|
||||
|
||||
fmt.Printf("load generators / rest generators [ %v / %v ] ", loadGenerators, restServers)
|
||||
|
||||
//At least 500 transactions should be acquired within 30 seconds after startup.
|
||||
runK8petstore(ip, restServers, loadGenerators, c, 500, 30)
|
||||
|
||||
})
|
||||
|
||||
//This test takes a few minutes... for simple CI setups testing pure functionality, filter it out.
|
||||
It(fmt.Sprintf("k8petstore-SCALE : Should support acquiring up to 5000 petstore transactions per minion"), func() {
|
||||
|
||||
//We double the load generators, to increase the load. Maybe parameterize this later,
|
||||
//the more generators -> the more transactions and the more bursty CPU used per minion.
|
||||
var loadGenerators int = minionCount * 2
|
||||
var restServers int = minionCount
|
||||
//var min_expected int = trials * 10 * minionCount // more minions --> higher expected # of transactions.
|
||||
|
||||
fmt.Printf("load generators / rest generators [ %v / %v ] ", loadGenerators, restServers)
|
||||
|
||||
//5000*M transactions in 6 minutes. That gives 6 minutes of cold-start time.
|
||||
runK8petstore(ip, restServers, loadGenerators, c, 5000*minionCount, 60*6)
|
||||
})
|
||||
})
|
||||
Loading…
Reference in New Issue