Merge pull request #58728 from dashpole/cadvisor_testing

Automatic merge from submit-queue (batch tested with PRs 57683, 59116, 58728, 59140, 58976). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Use node-e2e framework for testing cadvisor

**What this PR does / why we need it**:
With cadvisor checked out in your gopath, we can now run cadvisor integration tests: `make test-e2e-node TEST_SUITE=cadvisor`.  
This has a number of advantages:
 * we can use the same images to test both, configured the same way.
 * we will now get cadvisor logs from the integration test.
 * we can now use the familiar node-e2e arguments to specify images to test with cadvisor
 * no more managing snowflake VMs for cadvisor.

**Special notes for your reviewer**:
cadvisor doesnt currently produce junit* files, so I removed that as a requirement.
This wont actually work until https://github.com/google/cadvisor/pull/1868 is merged as well.

Related issue:
https://github.com/kubernetes/test-infra/issues/190

**Release note**:
```release-note
NONE
```

/assign @Random-Liu 
/sig node
/priority important-soon
/kind cleanup

hack/make-rules/test-e2e-node.sh

@@ -93,6 +93,7 @@ if [ $remote = true ] ; then
   instance_prefix=${INSTANCE_PREFIX:-"test"}
   cleanup=${CLEANUP:-"true"}
   delete_instances=${DELETE_INSTANCES:-"false"}
+  test_suite=${TEST_SUITE:-"default"}
 
   # Get the compute zone
   zone=$(gcloud info --format='value(config.properties.compute.zone)')
@@ -147,6 +148,7 @@ if [ $remote = true ] ; then
     --image-project="$image_project" --instance-name-prefix="$instance_prefix" \
     --delete-instances="$delete_instances" --test_args="$test_args" --instance-metadata="$metadata" \
     --image-config-file="$image_config_file" --system-spec-name="$system_spec_name" \
+    --test-suite="$test_suite" \
     2>&1 | tee -i "${artifacts}/build-log.txt"
   exit $?
 

test/e2e_node/builder/build.go

@@ -87,8 +87,27 @@ func getK8sBin(bin string) (string, error) {
 	return "", fmt.Errorf("Unable to locate %s.  Can be defined using --k8s-path.", bin)
 }
 
-// TODO: Dedup / merge this with comparable utilities in e2e/util.go
+// GetK8sRootDir returns the root directory for kubernetes, if present in the gopath.
 func GetK8sRootDir() (string, error) {
+	dir, err := RootDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, fmt.Sprintf("%s/", "k8s.io/kubernetes")), nil
+}
+
+// GetCAdvisorRootDir returns the root directory for cAdvisor, if present in the gopath.
+func GetCAdvisorRootDir() (string, error) {
+	dir, err := RootDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, fmt.Sprintf("%s/", "github.com/google/cadvisor")), nil
+}
+
+// TODO: Dedup / merge this with comparable utilities in e2e/util.go
+// RootDir returns the path to the directory containing the k8s.io directory
+func RootDir() (string, error) {
 	// Get the directory of the current executable
 	_, testExec, _, _ := runtime.Caller(0)
 	path := filepath.Dir(testExec)
@@ -96,7 +115,7 @@ func GetK8sRootDir() (string, error) {
 	// Look for the kubernetes source root directory
 	if strings.Contains(path, "k8s.io/kubernetes") {
 		splitPath := strings.Split(path, "k8s.io/kubernetes")
-		return filepath.Join(splitPath[0], "k8s.io/kubernetes/"), nil
+		return splitPath[0], nil
 	}
 
 	return "", fmt.Errorf("Could not find kubernetes source root directory.")

test/e2e_node/remote/BUILD

@@ -8,6 +8,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "cadvisor_e2e.go",
         "node_conformance.go",
         "node_e2e.go",
         "remote.go",

test/e2e_node/remote/cadvisor_e2e.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remote
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/kubernetes/test/e2e_node/builder"
+)
+
+// CAdvisorE2ERemote contains the specific functions in the cadvisor e2e test suite.
+type CAdvisorE2ERemote struct{}
+
+// InitCAdvisorE2ERemote performs initialization for cadvisor remote testing
+func InitCAdvisorE2ERemote() TestSuite {
+	return &CAdvisorE2ERemote{}
+}
+
+// SetupTestPackage implements TestSuite.SetupTestPackage
+func (n *CAdvisorE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
+	cadvisorRootDir, err := builder.GetCAdvisorRootDir()
+	if err != nil {
+		return err
+	}
+	// build the cadvisor binary and tests
+	if err := runCommand(fmt.Sprintf("%s/build/prow_e2e.sh", cadvisorRootDir)); err != nil {
+		return err
+	}
+	// transfer the entire directory to each node
+	if err := runCommand("cp", "-R", cadvisorRootDir, fmt.Sprintf("%s/", tardir)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func runCommand(command string, args ...string) error {
+	cmd := exec.Command(command, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("failed to run command %s. error: %v", command, err)
+	}
+	return nil
+}
+
+// RunTest implements TestSuite.RunTest
+func (n *CAdvisorE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName string, timeout time.Duration) (string, error) {
+	// Kill any running node processes
+	cleanupNodeProcesses(host)
+
+	glog.V(2).Infof("Starting tests on %q", host)
+	return SSH(host, "sh", "-c", getSSHCommand(" && ",
+		fmt.Sprintf("cd %s/cadvisor", workspace),
+		fmt.Sprintf("timeout -k 30s %fs ./build/integration.sh ../results/cadvisor.log",
+			timeout.Seconds()),
+	))
+}

test/e2e_node/remote/remote.go

@@ -139,21 +139,20 @@ func getTestArtifacts(host, testDir string) error {
 		return fmt.Errorf("failed to create log directory %q: %v", logPath, err)
 	}
 	// Copy logs to artifacts/hostname
-	_, err := runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/*.log", GetHostnameOrIp(host), testDir), logPath)
-	if err != nil {
+	if _, err := runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/*.log", GetHostnameOrIp(host), testDir), logPath); err != nil {
 		return err
 	}
 	// Copy json files (if any) to artifacts.
-	if _, err = SSH(host, "ls", fmt.Sprintf("%s/results/*.json", testDir)); err == nil {
-		_, err = runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/*.json", GetHostnameOrIp(host), testDir), *resultsDir)
-		if err != nil {
+	if _, err := SSH(host, "ls", fmt.Sprintf("%s/results/*.json", testDir)); err == nil {
+		if _, err = runSSHCommand("scp", "-r", fmt.Sprintf("%s:%s/results/*.json", GetHostnameOrIp(host), testDir), *resultsDir); err != nil {
 			return err
 		}
 	}
-	// Copy junit to the top of artifacts
-	_, err = runSSHCommand("scp", fmt.Sprintf("%s:%s/results/junit*", GetHostnameOrIp(host), testDir), *resultsDir)
-	if err != nil {
-		return err
+	if _, err := SSH(host, "ls", fmt.Sprintf("%s/results/junit*", testDir)); err == nil {
+		// Copy junit (if any) to the top of artifacts
+		if _, err = runSSHCommand("scp", fmt.Sprintf("%s:%s/results/junit*", GetHostnameOrIp(host), testDir), *resultsDir); err != nil {
+			return err
+		}
 	}
 	return nil
 }

test/e2e_node/remote/types.go

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package remote contains implementations of the TestSuite interface, which specify
+// how to run various node test suites remotely.
 package remote
 
 import (

test/e2e_node/runner/remote/run_remote.go

@@ -178,6 +178,8 @@ func main() {
 	switch *testSuite {
 	case "conformance":
 		suite = remote.InitConformanceRemote()
+	case "cadvisor":
+		suite = remote.InitCAdvisorE2ERemote()
 	// TODO: Add subcommand for node soaking, node conformance, cri validation.
 	case "default":
 		// Use node e2e suite by default if no subcommand is specified.