Node e2e export test artifacts to jenkins.

- Add junit test reported
- Write etcd.log, kubelet.log and kube-apiserver.log to files instead of stdout
- Scp artifacts to the jenkins WORKSPACE

Fixes #25966

hack/e2e-node-test.sh

@@ -19,6 +19,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 
 focus=${FOCUS:-""}
 skip=${SKIP:-""}
+report=${REPORT:-"/tmp/"}
 
 ginkgo=$(kube::util::find-binary "ginkgo")
 if [[ -z "${ginkgo}" ]]; then
@@ -27,6 +28,6 @@ if [[ -z "${ginkgo}" ]]; then
 fi
 
 # Provided for backwards compatibility
-"${ginkgo}" --focus=$focus --skip=$skip "${KUBE_ROOT}/test/e2e_node/" -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true --start-services=true --stop-services=true
+"${ginkgo}" --focus=$focus --skip=$skip "${KUBE_ROOT}/test/e2e_node/" --report-dir=${report} -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true --start-services=true --stop-services=true
 
 exit $?

hack/verify-flags/known-flags.txt

@@ -201,6 +201,7 @@ ir-password
 ir-user
 jenkins-host
 jenkins-jobs
+junit-file-number
 k8s-bin-dir
 k8s-build-output
 keep-gogoproto
@@ -366,6 +367,7 @@ resolv-conf
 resource-container
 resource-quota-sync-period
 resource-version
+results-dir
 retry_time
 rkt-api-endpoint
 rkt-path

test/e2e_node/container_list.go

@@ -53,7 +53,7 @@ func PrePullAllImages() error {
 	for _, image := range ImageRegistry {
 		output, err := exec.Command("docker", "pull", image).CombinedOutput()
 		if err != nil {
-			glog.Warning("Could not pre-pull image %s %v output:  %s", image, err, output)
+			glog.Warningf("Could not pre-pull image %s %v output:  %s", image, err, output)
 			return err
 		}
 	}

test/e2e_node/e2e_node_suite_test.go

@@ -24,27 +24,41 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"os"
 	"os/exec"
+	"path"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/golang/glog"
 	. "github.com/onsi/ginkgo"
-	"github.com/onsi/ginkgo/config"
-	"github.com/onsi/ginkgo/types"
+	more_reporters "github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
 )
 
 var e2es *e2eService
 
 var prePullImages = flag.Bool("prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
+var junitFileNumber = flag.Int("junit-file-number", 1, "Used to create junit filename - e.g. junit_01.xml.")
 
 func TestE2eNode(t *testing.T) {
 	flag.Parse()
+
 	rand.Seed(time.Now().UTC().UnixNano())
 	RegisterFailHandler(Fail)
-	reporters := []Reporter{&LogReporter{}}
+	reporters := []Reporter{}
+	if *reportDir != "" {
+		// Create the directory if it doesn't already exists
+		if err := os.MkdirAll(*reportDir, 0755); err != nil {
+			glog.Errorf("Failed creating report directory: %v", err)
+		} else {
+			// Configure a junit reporter to write to the directory
+			junitFile := fmt.Sprintf("junit_%02d.xml", *junitFileNumber)
+			junitPath := path.Join(*reportDir, junitFile)
+			reporters = append(reporters, more_reporters.NewJUnitReporter(junitPath))
+		}
+	}
 	RunSpecsWithDefaultAndCustomReporters(t, "E2eNode Suite", reporters)
 }
 
@@ -94,40 +108,6 @@ var _ = AfterSuite(func() {
 	glog.Infof("Tests Finished")
 })
 
-var _ Reporter = &LogReporter{}
-
-type LogReporter struct{}
-
-func (lr *LogReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) {
-	b := &bytes.Buffer{}
-	b.WriteString("******************************************************\n")
-	glog.Infof(b.String())
-}
-
-func (lr *LogReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {}
-
-func (lr *LogReporter) SpecWillRun(specSummary *types.SpecSummary) {}
-
-func (lr *LogReporter) SpecDidComplete(specSummary *types.SpecSummary) {}
-
-func (lr *LogReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {}
-
-func (lr *LogReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) {
-	// Only log the binary output if the suite failed.
-	b := &bytes.Buffer{}
-	if e2es != nil && !summary.SuiteSucceeded {
-		b.WriteString(fmt.Sprintf("Process Log For Failed Suite On %s\n", *nodeName))
-		b.WriteString("-------------------------------------------------------------\n")
-		b.WriteString(fmt.Sprintf("kubelet output:\n%s\n", e2es.kubeletCombinedOut.String()))
-		b.WriteString("-------------------------------------------------------------\n")
-		b.WriteString(fmt.Sprintf("apiserver output:\n%s\n", e2es.apiServerCombinedOut.String()))
-		b.WriteString("-------------------------------------------------------------\n")
-		b.WriteString(fmt.Sprintf("etcd output:\n%s\n", e2es.etcdCombinedOut.String()))
-	}
-	b.WriteString("******************************************************\n")
-	glog.Infof(b.String())
-}
-
 func maskLocksmithdOnCoreos() {
 	data, err := ioutil.ReadFile("/etc/os-release")
 	if err != nil {

test/e2e_node/e2e_remote.go

@@ -28,11 +28,13 @@ import (
 	"strings"
 
 	"github.com/golang/glog"
+	utilerrors "k8s.io/kubernetes/pkg/util/errors"
 )
 
 var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.")
 var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment.  Options: gce")
 var testTimeoutSeconds = flag.Int("test-timeout", 45*60, "How long (in seconds) to wait for ginkgo tests to complete.")
+var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test results to.")
 
 var sshOptionsMap map[string]string
 
@@ -119,7 +121,7 @@ func CreateTestArchive() string {
 }
 
 // RunRemote copies the archive file to a /tmp file on host, unpacks it, and runs the e2e_node.test
-func RunRemote(archive string, host string, cleanup bool) (string, error) {
+func RunRemote(archive string, host string, cleanup bool, junitFileNumber int) (string, error) {
 	// Create the temp staging directory
 	glog.Infof("Staging test binaries on %s", host)
 	tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
@@ -158,17 +160,39 @@ func RunRemote(archive string, host string, cleanup bool) (string, error) {
 	cmd = getSshCommand(" && ",
 		fmt.Sprintf("cd %s", tmp),
 		fmt.Sprintf("tar -xzvf ./%s", archiveName),
-		fmt.Sprintf("timeout -k 30s %ds ./e2e_node.test --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s", *testTimeoutSeconds, cleanup, host),
+		fmt.Sprintf("timeout -k 30s %ds ./e2e_node.test --logtostderr --v 2 --build-services=false --stop-services=%t --node-name=%s --report-dir=%s/results --junit-file-number=%d", *testTimeoutSeconds, cleanup, host, tmp, junitFileNumber),
 	)
 	glog.Infof("Starting tests on %s", host)
 	output, err := RunSshCommand("ssh", host, "--", "sh", "-c", cmd)
+
 	if err != nil {
+		scpErr := getTestArtifacts(host, tmp)
+
+		// Return both the testing and scp error
+		if scpErr != nil {
+			return "", utilerrors.NewAggregate([]error{err, scpErr})
+		}
 		return "", err
 	}
 
+	err = getTestArtifacts(host, tmp)
 	return output, nil
 }
 
+func getTestArtifacts(host, testDir string) error {
+	_, err := RunSshCommand("scp", "-r", fmt.Sprintf("%s:%s/results/", host, testDir), fmt.Sprintf("%s/%s", *resultsDir, host))
+	if err != nil {
+		return err
+	}
+
+	// Copy junit to the top of artifacts
+	_, err = RunSshCommand("scp", fmt.Sprintf("%s:%s/results/junit*", host, testDir), fmt.Sprintf("%s/", *resultsDir))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // getSshCommand handles proper quoting so that multiple commands are executed in the same shell over ssh
 func getSshCommand(sep string, args ...string) string {
 	return fmt.Sprintf("'%s'", strings.Join(args, sep))

test/e2e_node/e2e_service.go

@@ -17,13 +17,13 @@ limitations under the License.
 package e2e_node
 
 import (
-	"bytes"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
+	"path"
 	"strings"
 	"time"
 
@@ -31,17 +31,15 @@ import (
 )
 
 var serverStartTimeout = flag.Duration("server-start-timeout", time.Second*120, "Time to wait for each server to become healthy.")
+var reportDir = flag.String("report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
 
 type e2eService struct {
-	etcdCmd              *exec.Cmd
-	etcdCombinedOut      bytes.Buffer
-	etcdDataDir          string
-	apiServerCmd         *exec.Cmd
-	apiServerCombinedOut bytes.Buffer
-	kubeletCmd           *exec.Cmd
-	kubeletCombinedOut   bytes.Buffer
-	kubeletStaticPodDir  string
-	nodeName             string
+	etcdCmd             *exec.Cmd
+	etcdDataDir         string
+	apiServerCmd        *exec.Cmd
+	kubeletCmd          *exec.Cmd
+	kubeletStaticPodDir string
+	nodeName            string
 }
 
 func newE2eService(nodeName string) *e2eService {
@@ -123,23 +121,23 @@ func (es *e2eService) startEtcd() (*exec.Cmd, error) {
 	hcc := newHealthCheckCommand(
 		"http://127.0.0.1:4001/v2/keys/", // Trailing slash is required,
 		cmd,
-		&es.etcdCombinedOut)
+		"etcd.log")
 	return cmd, es.startServer(hcc)
 }
 
 func (es *e2eService) startApiServer() (*exec.Cmd, error) {
 	cmd := exec.Command("sudo", getApiServerBin(),
-		"--v", "2", "--logtostderr", "--log_dir", "./",
 		"--etcd-servers", "http://127.0.0.1:4001",
 		"--insecure-bind-address", "0.0.0.0",
 		"--service-cluster-ip-range", "10.0.0.1/24",
 		"--kubelet-port", "10250",
 		"--allow-privileged", "true",
+		"--v", "8", "--logtostderr",
 	)
 	hcc := newHealthCheckCommand(
 		"http://127.0.0.1:8080/healthz",
 		cmd,
-		&es.apiServerCombinedOut)
+		"kube-apiserver.log")
 	return cmd, es.startServer(hcc)
 }
 
@@ -150,7 +148,6 @@ func (es *e2eService) startKubeletServer() (*exec.Cmd, error) {
 	}
 	es.kubeletStaticPodDir = dataDir
 	cmd := exec.Command("sudo", getKubeletServerBin(),
-		"--v", "2", "--logtostderr", "--log_dir", "./",
 		"--api-servers", "http://127.0.0.1:8080",
 		"--address", "0.0.0.0",
 		"--port", "10250",
@@ -160,22 +157,40 @@ func (es *e2eService) startKubeletServer() (*exec.Cmd, error) {
 		"--serialize-image-pulls", "false",
 		"--config", es.kubeletStaticPodDir,
 		"--file-check-frequency", "10s", // Check file frequently so tests won't wait too long
+		"--v", "8", "--logtostderr",
 	)
 	hcc := newHealthCheckCommand(
 		"http://127.0.0.1:10255/healthz",
 		cmd,
-		&es.kubeletCombinedOut)
+		"kubelet.log")
 	return cmd, es.startServer(hcc)
 }
 
 func (es *e2eService) startServer(cmd *healthCheckCommand) error {
 	cmdErrorChan := make(chan error)
 	go func() {
-		err := cmd.Run()
+		defer close(cmdErrorChan)
+
+		// Create the output filename
+		outPath := path.Join(*reportDir, cmd.outputFilename)
+		outfile, err := os.Create(outPath)
 		if err != nil {
-			cmdErrorChan <- fmt.Errorf("%s Failed with error \"%v\".  Command output:\n%s", cmd, err, cmd.OutputBuffer)
+			cmdErrorChan <- fmt.Errorf("Failed to create file %s for `%s` %v.", outPath, cmd, err)
+			return
+		}
+		defer outfile.Close()
+		defer outfile.Sync()
+
+		// Set the command to write the output file
+		cmd.Cmd.Stdout = outfile
+		cmd.Cmd.Stderr = outfile
+
+		// Run the command
+		err = cmd.Run()
+		if err != nil {
+			cmdErrorChan <- fmt.Errorf("%s Failed with error \"%v\".  Output written to: %s", cmd, err, outPath)
+			return
 		}
-		close(cmdErrorChan)
 	}()
 
 	endTime := time.Now().Add(*serverStartTimeout)
@@ -196,16 +211,14 @@ func (es *e2eService) startServer(cmd *healthCheckCommand) error {
 type healthCheckCommand struct {
 	*exec.Cmd
 	HealthCheckUrl string
-	OutputBuffer   *bytes.Buffer
+	outputFilename string
 }
 
-func newHealthCheckCommand(healthCheckUrl string, cmd *exec.Cmd, combinedOutput *bytes.Buffer) *healthCheckCommand {
-	cmd.Stdout = combinedOutput
-	cmd.Stderr = combinedOutput
+func newHealthCheckCommand(healthCheckUrl string, cmd *exec.Cmd, filename string) *healthCheckCommand {
 	return &healthCheckCommand{
 		HealthCheckUrl: healthCheckUrl,
 		Cmd:            cmd,
-		OutputBuffer:   combinedOutput,
+		outputFilename: filename,
 	}
 }
 

test/e2e_node/jenkins/copy-e2e-image.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Usage: copy-e2e-image.sh <image-name> <from-project-name> <to-project-name>
+
+set -e
+set -x
+
+echo "Copying image $1 from project $2 to project $3..."
+gcloud compute --project $3 disks create $1 --image=https://www.googleapis.com/compute/v1/projects/$2/global/images/$1
+gcloud compute --project $3 images create $1 --source-disk=$1
+gcloud compute --project $3 disks delete $1

test/e2e_node/jenkins/e2e-node-jenkins.sh

@@ -35,6 +35,9 @@ if [ "$INSTALL_GODEP" = true ] ; then
 fi
 
 go build test/e2e_node/environment/conformance.go
+ARTIFACTS=${WORKSPACE}/_artifacts
+mkdir -p ${ARTIFACTS}
 go run test/e2e_node/runner/run_e2e.go  --logtostderr --vmodule=*=2 --ssh-env="gce" \
   --zone="$GCE_ZONE" --project="$GCE_PROJECT"  \
-  --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP"
+  --hosts="$GCE_HOSTS" --images="$GCE_IMAGES" --cleanup="$CLEANUP" \
+  --results-dir="$ARTIFACTS"

test/e2e_node/runner/run_e2e.go

@@ -114,16 +114,16 @@ func main() {
 		for _, image := range strings.Split(*images, ",") {
 			running++
 			fmt.Printf("Initializing e2e tests using image %s.\n", image)
-			go func(image string) { results <- testImage(image, archive) }(image)
+			go func(image string, junitFileNum int) { results <- testImage(image, archive, junitFileNum) }(image, running)
 		}
 	}
 	if *hosts != "" {
 		for _, host := range strings.Split(*hosts, ",") {
 			fmt.Printf("Initializing e2e tests using host %s.\n", host)
 			running++
-			go func(host string) {
-				results <- testHost(host, archive, *cleanup)
-			}(host)
+			go func(host string, junitFileNum int) {
+				results <- testHost(host, archive, *cleanup, junitFileNum)
+			}(host, running)
 		}
 	}
 
@@ -150,8 +150,8 @@ func main() {
 }
 
 // Run tests in archive against host
-func testHost(host, archive string, deleteFiles bool) *TestResult {
-	output, err := e2e_node.RunRemote(archive, host, deleteFiles)
+func testHost(host, archive string, deleteFiles bool, junitFileNum int) *TestResult {
+	output, err := e2e_node.RunRemote(archive, host, deleteFiles, junitFileNum)
 	return &TestResult{
 		output: output,
 		err:    err,
@@ -161,7 +161,7 @@ func testHost(host, archive string, deleteFiles bool) *TestResult {
 
 // Provision a gce instance using image and run the tests in archive against the instance.
 // Delete the instance afterward.
-func testImage(image, archive string) *TestResult {
+func testImage(image, archive string, junitFileNum int) *TestResult {
 	host, err := createInstance(image)
 	if *cleanup {
 		defer deleteInstance(image)
@@ -171,7 +171,7 @@ func testImage(image, archive string) *TestResult {
 			err: fmt.Errorf("Unable to create gce instance with running docker daemon for image %s.  %v", image, err),
 		}
 	}
-	return testHost(host, archive, false)
+	return testHost(host, archive, false, junitFileNum)
 }
 
 // Provision a gce instance using image