diff --git a/examples/k8petstore/k8petstore.sh b/examples/k8petstore/k8petstore.sh index 612640f21f..55abcea1bd 100755 --- a/examples/k8petstore/k8petstore.sh +++ b/examples/k8petstore/k8petstore.sh @@ -16,16 +16,30 @@ echo "WRITING KUBE FILES , will overwrite the jsons, then testing pods. is kube clean ready to go?" + +#Args below can be overriden when calling from cmd line. +#Just send all the args in order. #for dev/test you can use: #kubectl=$GOPATH/src/github.com/GoogleCloudPlatform/kubernetes/cluster/kubectl.sh" kubectl="kubectl" VERSION="r.2.8.19" PUBLIC_IP="10.1.4.89" # ip which we use to access the Web server. -SECONDS=1000 # number of seconds to measure throughput. FE="1" # amount of Web server LG="1" # amount of load generators SLAVE="1" # amount of redis slaves +TEST_SECONDS="1000" # 0 = Dont run tests, if > 0, run tests for n seconds. +NS="k8petstore" # namespace +kubectl="${1:-$kubectl}" +VERSION="${2:-$VERSION}" +PUBLIC_IP="${3:-$PUBLIC_IP}" +FE="${4:-$FE}" +LG="${5:-$LG}" +SLAVE="${6:-$SLAVE}" +TEST_SECONDS="${7:-$TEST}" +NS="${8:-$NS}" + +echo "Running w/ args: kubectl $kubectl version $VERSION ip $PUBLIC_IP sec $_SECONDS fe $FE lg $LG slave $SLAVE test $TEST NAMESPACE $NS" function create { cat << EOF > fe-rc.json @@ -188,53 +202,74 @@ cat << EOF > slave-rc.json "labels": {"name": "redisslave"} } EOF -$kubectl create -f rm.json --api-version=v1beta1 -$kubectl create -f rm-s.json --api-version=v1beta1 +$kubectl create -f rm.json --api-version=v1beta1 --namespace=$NS +$kubectl create -f rm-s.json --api-version=v1beta1 --namespace=$NS sleep 3 # precaution to prevent fe from spinning up too soon. -$kubectl create -f slave-rc.json --api-version=v1beta1 -$kubectl create -f rs-s.json --api-version=v1beta1 +$kubectl create -f slave-rc.json --api-version=v1beta1 --namespace=$NS +$kubectl create -f rs-s.json --api-version=v1beta1 --namespace=$NS sleep 3 # see above comment. -$kubectl create -f fe-rc.json --api-version=v1beta1 -$kubectl create -f fe-s.json --api-version=v1beta1 -$kubectl create -f bps-load-gen-rc.json --api-version=v1beta1 +$kubectl create -f fe-rc.json --api-version=v1beta1 --namespace=$NS +$kubectl create -f fe-s.json --api-version=v1beta1 --namespace=$NS +$kubectl create -f bps-load-gen-rc.json --api-version=v1beta1 --namespace=$NS } -function test { +function pollfor { pass_http=0 ### Test HTTP Server comes up. for i in `seq 1 150`; do ### Just testing that the front end comes up. Not sure how to test total entries etc... (yet) - echo "Trying curl ... $i . expect a few failures while pulling images... " + echo "Trying curl ... $PUBLIC_IP:3000 , attempt $i . expect a few failures while pulling images... " curl "$PUBLIC_IP:3000" > result cat result cat result | grep -q "k8-bps" if [ $? -eq 0 ]; then - echo "TEST PASSED after $i tries !" - i=1000 - break + echo "TEST PASSED after $i tries !" + i=1000 + break else echo "the above RESULT didn't contain target string for trial $i" fi - sleep 5 + sleep 3 done if [ $i -eq 1000 ]; then - pass_http=-1 + pass_http=1 fi + +} +function tests { pass_load=0 ### Print statistics of db size, every second, until $SECONDS are up. - for i in `seq 1 $SECONDS`; - do - echo "curl : $i" - curl "$PUBLIC_IP:3000/llen" >> result + for i in `seq 1 $TEST_SECONDS`; + do + echo "curl : $PUBLIC_IP:3000 , $i of $TEST_SECONDS" + curr_cnt="`curl "$PUBLIC_IP:3000/llen"`" + ### Write CSV File of # of trials / total transcations. + echo "$i $curr_cnt" >> result + echo "total transactions so far : $curr_cnt" sleep 1 done } create -test +pollfor + +if [[ $pass_http -eq 1 ]]; then + echo "Passed..." +else + exit 2 +fi + +if [[ $TEST_SECONDS -eq 0 ]]; then + echo "skipping tests, TEST_SECONDS value was 0" +else + echo "running polling tests now for $TEST_SECONDS" + tests +fi + +exit 0 diff --git a/test/e2e/soak_k8petstore.go b/test/e2e/soak_k8petstore.go new file mode 100644 index 0000000000..ffc0832281 --- /dev/null +++ b/test/e2e/soak_k8petstore.go @@ -0,0 +1,193 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "syscall" + "time" + + "github.com/GoogleCloudPlatform/kubernetes/pkg/api" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/fields" + "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var ( + root0 = absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), ".."))) + err error = nil + namespace *api.Namespace = nil + ns string = "" +) + +//This must run on a machine which is in the kube-proxy ring. Otherwise, the publicIP binding will fail. +//The IP Below ~ letter->number cipher for k8petstore (most likely it won't be bound by any other process) +var ip = "165.201.92.15" + +// i.e. after 50 trials, expect 3000 transactions... minimum we settle for is 500. +var minionCount int + +// readTransactions reads # of transactions from the k8petstore web server endpoint. +// for more details see the source of the k8petstore web server. +func readTransactions(c *client.Client) int { + resp, err := http.Get("http://165.201.92.15:3000/llen") + if err != nil { + Expect(err).NotTo(HaveOccurred()) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + totalTrans, err := strconv.Atoi(string(body)) + Expect(err).NotTo(HaveOccurred()) + return totalTrans +} + +// runK8petstore runs the k8petstore application, bound to external ip "ip", and polls it to assert that "min_expected" +// transactions are acquired in a maximum of "max_seconds". +func runK8petstore(ip string, restServers int, loadGenerators int, c *client.Client, minExpected int, maxSeconds int64) { + k8bps := filepath.Join(root0, "examples/k8petstore/k8petstore.sh") + + //Get the count of minions. We'll use this to decide expected throughput and loadgen/REST server count. + cmd := exec.Command( + k8bps, + "kubectl", //for dev, replace w/ "cluster/kubectl.sh" + "r.2.8.19", + ip, + strconv.Itoa(restServers), + strconv.Itoa(loadGenerators), + "1", "0", ns) // 1= # slave, 0 = don't bother running the embedded test. + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + Logf("Starting k8petstore application....") + //Run the k8petstore app, and log / fail if it returns any errors. + //This should return quickly, assuming containers are downloaded. + if err = cmd.Start(); err != nil { + log.Fatal(err) + } + //Make sure exit code != 0 + if err = cmd.Wait(); err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { + log.Printf("Exit Status: %d", status.ExitStatus()) + } + } + } + Expect(err).NotTo(HaveOccurred()) + Logf("... Done starting k8petstore successfully.... Now we will poll it...") + + //By now, the original k8petstore app has run and spun up a k8petstore w/ load generators. + //Lets poll until we reach min_expected transactions + totalTransactions := 0 + Logf("Start polling, timeout is %v seconds", maxSeconds) + + timeout := time.After(time.Duration(maxSeconds) * time.Second) + tick := time.Tick(2 * time.Second) + +T: + for { + select { + case <-timeout: + Logf("Timeout %v reached. Breaking!", tick) + break T + case <-tick: + totalTransactions = readTransactions(c) + Expect(err).NotTo(HaveOccurred()) + if totalTransactions > minExpected { + break T + } + Logf("%v == %v total petstore transactions stored into redis. ==", time.Now(), totalTransactions) + } + } + + //Finally! We should have exceeded the min_expected num of transactions. + //If this fails, but there are transactions being created, we may need to recalibrate + //the min_expected value - or else - your cluster is broken/slow ! + Ω(totalTransactions).Should(BeNumerically(">", minExpected)) +} + +var _ = Describe("k8bps", func() { + BeforeEach(func() { + By("Creating a kubernetes client") + c, err = loadClient() + Expect(err).NotTo(HaveOccurred()) + + By("Building a namespace api object") + namespace, err = createTestingNS("k8petstore-soak", c) + ns = namespace.Name + Expect(err).NotTo(HaveOccurred()) + + //Now get the # of minions and calibrate the test params to them... + minions, err := c.Nodes().List(labels.Everything(), fields.Everything()) + Expect(err).NotTo(HaveOccurred()) + minionCount = len(minions.Items) + + //simple calibration. TODO, make more ambitious (i.e. 10x density style load generators)... + //load_generators = minionCount + //rest_servers = minionCount + + }) + + AfterEach(func() { + By(fmt.Sprintf("Destroying namespace for this test %v", namespace.Name)) + if err := c.Namespaces().Delete(namespace.Name); err != nil { + Failf("Couldn't delete ns %s", err) + } + }) + + //On a single node cluster, we expect about 10 transactions every second on average. + //admittedly, this is a very rough estimate given that the ETL is done in batches... + It(fmt.Sprintf("k8petstore-FUNCTIONAL : Should quickly acquire 500 petstore transactions."), func() { + + //max number of trial before k8petstore.sh dies. + var loadGenerators int = minionCount + var restServers int = minionCount + + fmt.Printf("load generators / rest generators [ %v / %v ] ", loadGenerators, restServers) + + //At least 500 transactions should be acquired within 30 seconds after startup. + runK8petstore(ip, restServers, loadGenerators, c, 500, 30) + + }) + + //This test takes a few minutes... for simple CI setups testing pure functionality, filter it out. + It(fmt.Sprintf("k8petstore-SCALE : Should support acquiring up to 5000 petstore transactions per minion"), func() { + + //We double the load generators, to increase the load. Maybe parameterize this later, + //the more generators -> the more transactions and the more bursty CPU used per minion. + var loadGenerators int = minionCount * 2 + var restServers int = minionCount + //var min_expected int = trials * 10 * minionCount // more minions --> higher expected # of transactions. + + fmt.Printf("load generators / rest generators [ %v / %v ] ", loadGenerators, restServers) + + //5000*M transactions in 6 minutes. That gives 6 minutes of cold-start time. + runK8petstore(ip, restServers, loadGenerators, c, 5000*minionCount, 60*6) + }) +})