Node e2e test runner - run against images

- support allocating gce instances from images and running tests against them
- set --hostname-override to match node name
- add jenkins script to source to reproduce jenkins build locally

hack/jenkins/job-configs/node-e2e.yaml

@@ -57,7 +57,7 @@
 # owner: owner to be notified for job failures.  test results are published to owner email
 # repoName: github repo to checkout e.g. kubernetes/kubernetes or google/cadvisor
 # gitbasedir: directory under $WORKSPACE/go/src to checkout source repo to - e.g. k8s.io/kubernetes or github.com/google/cadvisor
-# shell: shell to execute from workspace
+# shell: bash command to execute from gitbasedir.  should be a single script such as {gitproject}-jenkins.sh
 - job-template:
     name: '{gitproject}-gce-e2e-ci'
     description: '{gitproject} continuous e2e tests.<br>Test Owner: {owner}.'
@@ -66,7 +66,12 @@
         numToKeep: 200
     node: node
     builders:
-         - shell: '{shell}'
+         - shell: |
+              #!/bin/bash
+              set -e
+              set -x
+              cd go/src/{gitbasedir}
+              {shell}
     publishers:
         - claim-build
         - gcs-uploader
@@ -125,11 +130,6 @@
             gitbasedir: 'github.com/google/cadvisor'
             owner: 'vishnuk@google.com'
             shell: |
-                #!/bin/bash
-                set -e
-                set -x
-                cd go/src/github.com/google/cadvisor
-
                 go get -u github.com/tools/godep
 
                 ./build/presubmit.sh
@@ -145,28 +145,11 @@
             repoName: 'kubernetes/heapster'
             gitbasedir: 'k8s.io/heapster'
             owner: 'pszczesniak@google.com'
-            shell: |
+            shell: 'make test-unit test-integration'
-                #!/bin/bash
-                set -e
-                set -x
-                cd go/src/k8s.io/heapster
-
-                make test-unit test-integration
         - 'kubelet':
             repoName: 'kubernetes/kubernetes'
             gitbasedir: 'k8s.io/kubernetes'
             owner: 'pwittroc@google.com'
-            shell: |
+            shell: 'test/e2e_node/jenkins/e2e-node-jenkins.sh test/e2e_node/jenkins/jenkins-ci.properties'
-                #!/bin/bash
-                set -e
-                set -x
-                cd go/src/k8s.io/kubernetes
-
-                go get -u github.com/tools/godep
-                go get -u github.com/onsi/ginkgo/ginkgo
-                go get -u github.com/onsi/gomega
-
-                godep go build test/e2e_node/environment/conformance.go
-                godep go run test/e2e_node/runner/run_e2e.go --zone us-central1-f --hosts e2e-node-container-vm-v20151215,e2e-node-coreos-beta.c.kubernetes-jenkins.internal,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10 --logtostderr -v 2
     jobs:
         - '{gitproject}-gce-e2e-ci'

hack/verify-flags/known-flags.txt

@@ -151,6 +151,7 @@ input-dirs
 insecure-bind-address
 insecure-port
 insecure-skip-tls-verify
+instance-name-prefix
 iptables-masquerade-bit
 iptables-sync-period
 ir-data-source

test/e2e_node/e2e_node_suite_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2015 The Kubernetes Authors All rights reserved.
+Copyright 2016 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -61,7 +61,7 @@ var _ = BeforeSuite(func() {
 	}
 
 	if *startServices {
-		e2es = newE2eService()
+		e2es = newE2eService(*nodeName)
 		if err := e2es.start(); err != nil {
 			Fail(fmt.Sprintf("Unable to start node services.\n%v", err))
 		}

test/e2e_node/e2e_remote.go

@@ -121,19 +121,19 @@ func CreateTestArchive() string {
 func RunRemote(archive string, host string) (string, error) {
 	// Create the temp staging directory
 	tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
-	_, err := runSshCommand("ssh", host, "--", "mkdir", tmp)
+	_, err := RunSshCommand("ssh", host, "--", "mkdir", tmp)
 	if err != nil {
 		return "", err
 	}
 	defer func() {
-		output, err := runSshCommand("ssh", host, "--", "rm", "-rf", tmp)
+		output, err := RunSshCommand("ssh", host, "--", "rm", "-rf", tmp)
 		if err != nil {
 			glog.Errorf("Failed to cleanup tmp directory %s on host %v.  Output:\n%s", tmp, err, output)
 		}
 	}()
 
 	// Copy the archive to the staging directory
-	_, err = runSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp))
+	_, err = RunSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp))
 	if err != nil {
 		return "", err
 	}
@@ -142,18 +142,20 @@ func RunRemote(archive string, host string) (string, error) {
 	cmd := getSshCommand(" ; ",
 		"sudo pkill kubelet",
 		"sudo pkill kube-apiserver",
-		"sudo pkill etcd")
+		"sudo pkill etcd",
+	)
 	// No need to log an error if pkill fails since pkill will fail if the commands are not running.
 	// If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd
 	// logs about failing to bind the required ports.
-	runSshCommand("ssh", host, "--", "sh", "-c", cmd)
+	RunSshCommand("ssh", host, "--", "sh", "-c", cmd)
 
 	// Extract the archive and run the tests
 	cmd = getSshCommand(" && ",
 		fmt.Sprintf("cd %s", tmp),
 		fmt.Sprintf("tar -xzvf ./%s", archiveName),
-		"./e2e_node.test --logtostderr --v 2 --build-services=false --node-name `hostname`")
+		fmt.Sprintf("./e2e_node.test --logtostderr --v 2 --build-services=false --node-name=%s", host),
-	output, err := runSshCommand("ssh", host, "--", "sh", "-c", cmd)
+	)
+	output, err := RunSshCommand("ssh", host, "--", "sh", "-c", cmd)
 	if err != nil {
 		return "", err
 	}
@@ -167,7 +169,7 @@ func getSshCommand(sep string, args ...string) string {
 }
 
 // runSshCommand executes the ssh or scp command, adding the flag provided --ssh-options
-func runSshCommand(cmd string, args ...string) (string, error) {
+func RunSshCommand(cmd string, args ...string) (string, error) {
 	if env, found := sshOptionsMap[*sshEnv]; found {
 		args = append(strings.Split(env, " "), args...)
 	}
@@ -176,7 +178,7 @@ func runSshCommand(cmd string, args ...string) (string, error) {
 	}
 	output, err := exec.Command(cmd, args...).CombinedOutput()
 	if err != nil {
-		return fmt.Sprintf("%s", output), fmt.Errorf("command %q %q failed with error: %v and output: %q", cmd, args, err, output)
+		return fmt.Sprintf("%s", output), fmt.Errorf("Command [%s %s] failed with error: %v and output:\n%s", cmd, strings.Join(args, " "), err, output)
 	}
 	return fmt.Sprintf("%s", output), nil
 }

test/e2e_node/e2e_service.go

@@ -40,10 +40,11 @@ type e2eService struct {
 	apiServerCombinedOut bytes.Buffer
 	kubeletCmd           *exec.Cmd
 	kubeletCombinedOut   bytes.Buffer
+	nodeName             string
 }
 
-func newE2eService() *e2eService {
+func newE2eService(nodeName string) *e2eService {
-	return &e2eService{}
+	return &e2eService{nodeName: nodeName}
 }
 
 func (es *e2eService) start() error {
@@ -141,7 +142,9 @@ func (es *e2eService) startKubeletServer() (*exec.Cmd, error) {
 				"--v", "2", "--logtostderr", "--log_dir", "./",
 				"--api-servers", "http://127.0.0.1:8080",
 				"--address", "0.0.0.0",
-				"--port", "10250"},
+				"--port", "10250",
+				"--hostname-override", es.nodeName, // Required because hostname is inconsistent across hosts
+			},
 		})
 }
 

test/e2e_node/jenkins/e2e-node-jenkins.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script executed by jenkins to run node e2e tests against gce
+# Usage: test/e2e_node/jenkins/e2e-node-jenkins.sh <path to properties>
+# Properties files:
+# - test/e2e_node/jenkins/jenkins-ci.properties : for running jenkins ci
+# - test/e2e_node/jenkins/jenkins-pull.properties : for running jenkins pull request builder
+# - test/e2e_node/jenkins/template.properties : template for creating a properties file to run locally
+
+set -e
+set -x
+
+: "${1:?Usage test/e2e_node/jenkins/e2e-node-jenkins.sh <path to properties>}"
+
+. $1
+
+if [ "$INSTALL_GODEP" = true ] ; then
+  go get -u github.com/tools/godep
+  go get -u github.com/onsi/ginkgo/ginkgo
+  go get -u github.com/onsi/gomega
+fi
+
+godep go build test/e2e_node/environment/conformance.go
+godep go run test/e2e_node/runner/run_e2e.go  --logtostderr --v="2" --ssh-env="gce" --zone="$GCE_ZONE" --project="$GCE_PROJECT"  --hosts="$GCE_HOSTS" --images="$GCE_IMAGES"

test/e2e_node/jenkins/jenkins-ci.properties

@@ -0,0 +1,5 @@
+GCE_HOSTS=e2e-node-container-vm-v20151215,e2e-node-coreos-beta,e2e-node-ubuntu-trusty,e2e-node-ubuntu-trusty-docker1-10
+GCE_IMAGES=
+GCE_ZONE=us-central1-f
+GCE_PROJECT=kubernetes-jenkins
+INSTALL_GODEP=true

test/e2e_node/jenkins/jenkins-pull.properties

@@ -0,0 +1,5 @@
+GCE_HOSTS=e2e-node-ubuntu-trusty-docker10
+GCE_IMAGES=e2e-node-ubuntu-trusty-docker10-image
+GCE_ZONE=us-central1-f
+GCE_PROJECT=kubernetes-jenkins-pull
+INSTALL_GODEP=true

test/e2e_node/jenkins/template.properties

@@ -0,0 +1,10 @@
+# Copy this file to your home directory and modify
+# Names of gce hosts to test against (must be resolvable) or empty
+GCE_HOSTS=
+# Names of gce images to test or empty
+GCE_IMAGES=
+# Gce zone to use - required when using GCE_IMAGES
+GCE_ZONE=
+# Gce project to use - required when using GCE_IMAGES
+GCE_PROJECT=
+INSTALL_GODEP=false

test/e2e_node/kubelet_test.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2015 The Kubernetes Authors All rights reserved.
+Copyright 2016 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

test/e2e_node/runner/run_e2e.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2015 The Kubernetes Authors All rights reserved.
+Copyright 2016 The Kubernetes Authors All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,22 +14,59 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// To run the e2e tests against one or more hosts on gce: $ go run run_e2e.go --hosts <comma separated hosts>
+// To run the e2e tests against one or more hosts on gce:
-// Requires gcloud compute ssh access to the hosts
+// $ go run run_e2e.go --logtostderr --v 2 --ssh-env gce --hosts <comma separated hosts>
+// To run the e2e tests against one or more images on gce and provision them:
+// $ go run run_e2e.go --logtostderr --v 2 --project <project> --zone <zone> --ssh-env gce --images <comma separated images>
 package main
 
 import (
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
+	"time"
 
 	"k8s.io/kubernetes/test/e2e_node"
+
+	"github.com/golang/glog"
+	"github.com/pborman/uuid"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/compute/v1"
 )
 
+var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names")
+var zone = flag.String("zone", "", "gce zone the hosts live in")
+var project = flag.String("project", "", "gce project the hosts live in")
+var images = flag.String("images", "", "images to test")
 var hosts = flag.String("hosts", "", "hosts to test")
 
+var computeService *compute.Service
+
+type TestResult struct {
+	output string
+	err    error
+	host   string
+}
+
 func main() {
+	flag.Parse()
+
+	if *hosts == "" && *images == "" {
+		glog.Fatalf("Must specify one of --images or --hosts flag.")
+	}
+	if *images != "" && *zone == "" {
+		glog.Fatal("Must specify --zone flag")
+	}
+	if *images != "" && *project == "" {
+		glog.Fatal("Must specify --project flag")
+	}
+	if *instanceNamePrefix == "" {
+		*instanceNamePrefix = "tmp-node-e2e-" + uuid.NewUUID().String()[:8]
+	}
+
 	// Setup coloring
 	stat, _ := os.Stdout.Stat()
 	useColor := (stat.Mode() & os.ModeCharDevice) != 0
@@ -40,38 +77,58 @@ func main() {
 		noColour = "\033[0m"
 	}
 
-	flag.Parse()
-	if *hosts == "" {
-		fmt.Printf("Must specific --hosts flag")
-	}
 	archive := e2e_node.CreateTestArchive()
 	defer os.Remove(archive)
 
 	results := make(chan *TestResult)
-	hs := strings.Split(*hosts, ",")
+	running := 0
-	for _, h := range hs {
+	if *images != "" {
-		fmt.Printf("Starting tests on host %s.", h)
+		// Setup the gce client for provisioning instances
-		go func(host string) {
+		// Getting credentials on gce jenkins is flaky, so try a couple times
-			output, err := e2e_node.RunRemote(archive, host)
+		var err error
-			results <- &TestResult{
+		for i := 0; i < 10; i++ {
-				output: output,
+			var client *http.Client
-				err:    err,
+			client, err = google.DefaultClient(oauth2.NoContext, compute.ComputeScope)
-				host:   host,
+			if err != nil {
+				continue
 			}
-		}(h)
+			computeService, err = compute.New(client)
+			if err != nil {
+				continue
+			}
+			time.Sleep(time.Second * 6)
+		}
+		if err != nil {
+			glog.Fatalf("Unable to create gcloud compute service using defaults.  Make sure you are authenticated. %v", err)
+		}
+
+		for _, image := range strings.Split(*images, ",") {
+			running++
+			fmt.Printf("Initializing e2e tests using image %s.\n", image)
+			go func(image string) { results <- testImage(image, archive) }(image)
+		}
+	}
+	if *hosts != "" {
+		for _, host := range strings.Split(*hosts, ",") {
+			fmt.Printf("Initializing e2e tests using host %s.\n", host)
+			running++
+			go func(host string) {
+				results <- testHost(host, archive)
+			}(host)
+		}
 	}
 
 	// Wait for all tests to complete and emit the results
 	errCount := 0
-	for i := 0; i < len(hs); i++ {
+	for i := 0; i < running; i++ {
 		tr := <-results
 		host := tr.host
 		fmt.Printf("%s================================================================%s\n", blue, noColour)
 		if tr.err != nil {
 			errCount++
-			fmt.Printf("Failure Finished Host %s Test Suite %s %v\n", host, tr.output, tr.err)
+			fmt.Printf("Failure Finished Host %s Test Suite\n%s\n%v\n", host, tr.output, tr.err)
 		} else {
-			fmt.Printf("Success Finished Host %s Test Suite %s\n", host, tr.output)
+			fmt.Printf("Success Finished Host %s Test Suite\n%s\n", host, tr.output)
 		}
 		fmt.Printf("%s================================================================%s\n", blue, noColour)
 	}
@@ -83,8 +140,107 @@ func main() {
 	}
 }
 
-type TestResult struct {
+// Run tests in archive against host
-	output string
+func testHost(host, archive string) *TestResult {
-	err    error
+	output, err := e2e_node.RunRemote(archive, host)
-	host   string
+	return &TestResult{
+		output: output,
+		err:    err,
+		host:   host,
+	}
+}
+
+// Provision a gce instance using image and run the tests in archive against the instance.
+// Delete the instance afterward.
+func testImage(image, archive string) *TestResult {
+	host, err := createInstance(image)
+	defer deleteInstance(image)
+	if err != nil {
+		return &TestResult{
+			err: fmt.Errorf("Unable to create gce instance with running docker daemon for image %s.  %v", image, err),
+		}
+	}
+	return testHost(host, archive)
+}
+
+// Provision a gce instance using image
+func createInstance(image string) (string, error) {
+	name := imageToInstanceName(image)
+	i := &compute.Instance{
+		Name:        name,
+		MachineType: machineType(),
+		NetworkInterfaces: []*compute.NetworkInterface{
+			{
+				AccessConfigs: []*compute.AccessConfig{
+					{
+						Type: "ONE_TO_ONE_NAT",
+						Name: "External NAT",
+					},
+				}},
+		},
+		Disks: []*compute.AttachedDisk{
+			{
+				AutoDelete: true,
+				Boot:       true,
+				Type:       "PERSISTENT",
+				InitializeParams: &compute.AttachedDiskInitializeParams{
+					SourceImage: sourceImage(image),
+				},
+			},
+		},
+	}
+	op, err := computeService.Instances.Insert(*project, *zone, i).Do()
+	if err != nil {
+		return "", err
+	}
+	if op.Error != nil {
+		return "", fmt.Errorf("Could not create instance %s: %+v", name, op.Error)
+	}
+
+	instanceRunning := false
+	for i := 0; i < 30 && !instanceRunning; i++ {
+		if i > 0 {
+			time.Sleep(time.Second * 20)
+		}
+		var instance *compute.Instance
+		instance, err = computeService.Instances.Get(*project, *zone, name).Do()
+		if err != nil {
+			continue
+		}
+		if strings.ToUpper(instance.Status) != "RUNNING" {
+			err = fmt.Errorf("Instance %s not in state RUNNING, was %s.", name, instance.Status)
+			continue
+		}
+		var output string
+		output, err = e2e_node.RunSshCommand("ssh", name, "--", "sudo", "docker", "version")
+		if err != nil {
+			err = fmt.Errorf("Instance %s not running docker daemon - Command failed: %s", name, output)
+			continue
+		}
+		if !strings.Contains(output, "Server") {
+			err = fmt.Errorf("Instance %s not running docker daemon - Server not found: %s", name, output)
+			continue
+		}
+		instanceRunning = true
+	}
+	return name, err
+}
+
+func deleteInstance(image string) {
+	_, err := computeService.Instances.Delete(*project, *zone, imageToInstanceName(image)).Do()
+	if err != nil {
+		glog.Infof("Error deleting instance %s", imageToInstanceName(image))
+	}
+}
+
+func imageToInstanceName(image string) string {
+	return *instanceNamePrefix + "-" + image
+}
+
+func sourceImage(image string) string {
+	return fmt.Sprintf("projects/%s/global/images/%s", *project, image)
+}
+
+func machineType() string {
+	return fmt.Sprintf("zones/%s/machineTypes/n1-standard-1", *zone)
 }