github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #21622 from pwittrock/nodee2essh 

Auto commit by PR queue bot
pull/6/head
k8s-merge-robot 2016-02-23 16:42:42 -08:00
parent a8c0ac88fc a3623c0c14
commit 067998e727
13 changed files with 623 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Makefile
View File

@ -101,6 +101,15 @@ test_e2e:
go run hack/e2e.go -v --build --up --test --down
.PHONY: test_e2e
# Build and run node end-to-end tests.
#
# Example:
# make test_e2e_node
test_e2e_node:
hack/e2e-node-test.sh
.PHONY: test_e2e_node
# Remove all build artifacts.
#
# Example:

21
hack/e2e-node-test.sh Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
# Copyright 2016 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Provided for backwards compatibility
sudo -v
ginkgo "$(dirname $0)/../test/e2e_node/" -- --alsologtostderr --v 2 --node-name $(hostname) --build-services=true --start-services=true --stop-services=true
exit $?

1
hack/lib/golang.sh
View File

@ -87,6 +87,7 @@ kube::golang::test_targets() {
examples/k8petstore/web-server/src
github.com/onsi/ginkgo/ginkgo
test/e2e/e2e.test
test/e2e_node/e2e_node.test
)
if [ -n "${KUBERNETES_CONTRIB:-}" ]; then
for contrib in "${KUBERNETES_CONTRIB}"; do

7
hack/verify-flags/known-flags.txt
View File

@ -28,6 +28,7 @@ bench-workers
bind-address
bind-pods-burst
bind-pods-qps
build-services
cadvisor-port
cert-dir
certificate-authority
@ -162,6 +163,7 @@ ir-hawkular
jenkins-host
jenkins-jobs
k8s-build-output
k8s-bin-dir
keep-gogoproto
km-path
kube-api-burst
@ -323,6 +325,7 @@ scheduler-name
schema-cache-dir
secure-port
serialize-image-pulls
server-start-timeout
service-account-key-file
service-account-lookup
service-account-private-key-file
@ -344,10 +347,14 @@ skip-generated-rewrite
skip-munges
sort-by
source-file
ssh-env
ssh-keyfile
ssh-options
ssh-user
start-services
static-pods-config
stats-port
stop-services
storage-version
storage-versions
streaming-connection-idle-timeout

2
test/e2e_node/doc.go
View File

@ -14,4 +14,6 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// e2e_node contains e2e tests specific to the node
// TODO: rename this package e2e-node
package e2e_node

131
test/e2e_node/e2e_build.go Normal file
View File

@ -0,0 +1,131 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"github.com/golang/glog"
)
var k8sBinDir = flag.String("k8s-bin-dir", "", "Directory containing k8s kubelet and kube-apiserver binaries.")
func buildGo() {
glog.Infof("Building k8s binaries...")
k8sRoot, err := getK8sRootDir()
if err != nil {
glog.Fatalf("Failed to locate kubernetes root directory %v.", err)
}
out, err := exec.Command(filepath.Join(k8sRoot, "hack/build-go.sh")).CombinedOutput()
if err != nil {
glog.Fatalf("Failed to build go packages %v. Output:\n%s", err, out)
}
}
func getK8sBin(bin string) (string, error) {
// Use commandline specified path
if *k8sBinDir != "" {
absPath, err := filepath.Abs(*k8sBinDir)
if err != nil {
return "", err
}
if _, err := os.Stat(filepath.Join(*k8sBinDir, bin)); err != nil {
return "", fmt.Errorf("Could not find kube-apiserver under directory %s.", absPath)
}
return filepath.Join(absPath, bin), nil
}
path, err := filepath.Abs(filepath.Dir(os.Args[0]))
if err != nil {
return "", fmt.Errorf("Could not find absolute path of directory containing the tests %s.", filepath.Dir(os.Args[0]))
}
if _, err := os.Stat(filepath.Join(path, bin)); err == nil {
return filepath.Join(path, bin), nil
}
buildOutputDir, err := getK8sBuildOutputDir()
if err != nil {
return "", err
}
if _, err := os.Stat(filepath.Join(buildOutputDir, bin)); err == nil {
return filepath.Join(buildOutputDir, bin), nil
}
// Give up with error
return "", fmt.Errorf("Unable to locate %s. Can be defined using --k8s-path.", bin)
}
// TODO: Dedup / merge this with comparable utilities in e2e/util.go
func getK8sRootDir() (string, error) {
// Get the directory of the current executable
_, testExec, _, _ := runtime.Caller(0)
path := filepath.Dir(testExec)
// Look for the kubernetes source root directory
if strings.Contains(path, "k8s.io/kubernetes") {
splitPath := strings.Split(path, "k8s.io/kubernetes")
return filepath.Join(splitPath[0], "k8s.io/kubernetes/"), nil
}
return "", fmt.Errorf("Could not find kubernetes source root directory.")
}
func getK8sBuildOutputDir() (string, error) {
k8sRoot, err := getK8sRootDir()
if err != nil {
return "", err
}
buildOutputDir := filepath.Join(k8sRoot, "_output/local/go/bin")
if _, err := os.Stat(buildOutputDir); err != nil {
return "", err
}
return buildOutputDir, nil
}
func getK8sNodeTestDir() (string, error) {
k8sRoot, err := getK8sRootDir()
if err != nil {
return "", err
}
buildOutputDir := filepath.Join(k8sRoot, "test/e2e_node")
if _, err := os.Stat(buildOutputDir); err != nil {
return "", err
}
return buildOutputDir, nil
}
func getKubeletServerBin() string {
bin, err := getK8sBin("kubelet")
if err != nil {
panic(fmt.Sprintf("Could not locate kubelet binary."))
}
return bin
}
func getApiServerBin() string {
bin, err := getK8sBin("kube-apiserver")
if err != nil {
panic(fmt.Sprintf("Could not locate kube-apiserver binary."))
}
return bin
}

56
test/e2e_node/e2e_node_suite_test.go
View File

@ -15,21 +15,31 @@ limitations under the License.
*/
// To run tests in this suite
// Local: `$ ginkgo -- --logtostderr -v 2`
// Remote: `$ ginkgo -- --node-name <hostname> --api-server-address=<hostname:api_port> --kubelet-address=<hostname=kubelet_port> --logtostderr -v 2`
// NOTE: This test suite requires sudo capabilities to run the kubelet and kube-apiserver.
// $ sudo -v && ginkgo test/e2e_node/ -- --logtostderr --v 2 --node-name `hostname` --start-services
package e2e_node
import (
"bytes"
"flag"
"fmt"
"os/exec"
"strings"
"testing"
"github.com/golang/glog"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"flag"
"testing"
)
var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "Host and port of the kubelet")
var apiServerAddress = flag.String("api-server-address", "http://127.0.0.1:8080", "Host and port of the api server")
var nodeName = flag.String("node-name", "127.0.0.1", "Name of the node")
var nodeName = flag.String("node-name", "", "Name of the node")
var buildServices = flag.Bool("build-services", true, "If true, build local executables")
var startServices = flag.Bool("start-services", true, "If true, start local node services")
var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tets")
var e2es *e2eService
func TestE2eNode(t *testing.T) {
flag.Parse()
@ -39,8 +49,42 @@ func TestE2eNode(t *testing.T) {
// Setup the kubelet on the node
var _ = BeforeSuite(func() {
if *buildServices {
buildGo()
}
if *nodeName == "" {
output, err := exec.Command("hostname").CombinedOutput()
if err != nil {
glog.Fatal("Could not get node name from hostname %v. Output:\n%s", err, output)
}
*nodeName = strings.TrimSpace(fmt.Sprintf("%s", output))
}
if *startServices {
e2es = newE2eService()
if err := e2es.start(); err != nil {
Fail(fmt.Sprintf("Unable to start node services.\n%v", err))
}
glog.Infof("Node services started. Running tests...")
} else {
glog.Infof("Running tests without starting services.")
}
})
// Tear down the kubelet on the node
var _ = AfterSuite(func() {
if e2es != nil && *startServices && *stopServices {
glog.Infof("Stopping node services...")
e2es.stop()
b := &bytes.Buffer{}
b.WriteString("-------------------------------------------------------------\n")
b.WriteString(fmt.Sprintf("kubelet output:\n%s\n", e2es.kubeletCombinedOut.String()))
b.WriteString("-------------------------------------------------------------\n")
b.WriteString(fmt.Sprintf("apiserver output:\n%s", e2es.apiServerCombinedOut.String()))
b.WriteString("-------------------------------------------------------------\n")
b.WriteString(fmt.Sprintf("etcd output:\n%s", e2es.etcdCombinedOut.String()))
b.WriteString("-------------------------------------------------------------\n")
glog.V(2).Infof(b.String())
}
})

182
test/e2e_node/e2e_remote.go Normal file
View File

@ -0,0 +1,182 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"flag"
"fmt"
"io/ioutil"
"math/rand"
"os"
"os/exec"
"os/user"
"path/filepath"
"strings"
"github.com/golang/glog"
)
var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.")
var sshEnv = flag.String("ssh-env", "", "Use predefined ssh options for environment. Options: gce")
var sshOptionsMap map[string]string
const archiveName = "e2e_node_test.tar.gz"
func init() {
usr, err := user.Current()
if err != nil {
glog.Fatal(err)
}
sshOptionsMap = map[string]string{
"gce": fmt.Sprintf("-i %s/.ssh/google_compute_engine -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o CheckHostIP=no -o StrictHostKeyChecking=no", usr.HomeDir),
}
}
// CreateTestArchive builds the local source and creates a tar archive e2e_node_test.tar.gz containing
// the binaries k8s required for node e2e tests
func CreateTestArchive() string {
// Build the executables
buildGo()
// Build the e2e tests into an executable
glog.Infof("Building ginkgo k8s test binaries...")
testDir, err := getK8sNodeTestDir()
if err != nil {
glog.Fatalf("Failed to locate test/e2e_node directory %v.", err)
}
out, err := exec.Command("ginkgo", "build", testDir).CombinedOutput()
if err != nil {
glog.Fatalf("Failed to build e2e tests under %s %v. Output:\n%s", testDir, err, out)
}
ginkgoTest := filepath.Join(testDir, "e2e_node.test")
if _, err := os.Stat(ginkgoTest); err != nil {
glog.Fatalf("Failed to locate test binary %s", ginkgoTest)
}
defer os.Remove(ginkgoTest)
// Make sure we can find the newly built binaries
buildOutputDir, err := getK8sBuildOutputDir()
if err != nil {
glog.Fatalf("Failed to locate kubernetes build output directory %v", err)
}
kubelet := filepath.Join(buildOutputDir, "kubelet")
if _, err := os.Stat(kubelet); err != nil {
glog.Fatalf("Failed to locate binary %s", kubelet)
}
apiserver := filepath.Join(buildOutputDir, "kube-apiserver")
if _, err := os.Stat(apiserver); err != nil {
glog.Fatalf("Failed to locate binary %s", apiserver)
}
glog.Infof("Building archive...")
tardir, err := ioutil.TempDir("", "node-e2e-archive")
if err != nil {
glog.Fatalf("Failed to create temporary directory %v.", err)
}
defer os.RemoveAll(tardir)
// Copy binaries
out, err = exec.Command("cp", ginkgoTest, filepath.Join(tardir, "e2e_node.test")).CombinedOutput()
if err != nil {
glog.Fatalf("Failed to copy e2e_node.test %v.", err)
}
out, err = exec.Command("cp", kubelet, filepath.Join(tardir, "kubelet")).CombinedOutput()
if err != nil {
glog.Fatalf("Failed to copy kubelet %v.", err)
}
out, err = exec.Command("cp", apiserver, filepath.Join(tardir, "kube-apiserver")).CombinedOutput()
if err != nil {
glog.Fatalf("Failed to copy kube-apiserver %v.", err)
}
// Build the tar
out, err = exec.Command("tar", "-zcvf", archiveName, "-C", tardir, ".").CombinedOutput()
if err != nil {
glog.Fatalf("Failed to build tar %v. Output:\n%s", err, out)
}
dir, err := os.Getwd()
if err != nil {
glog.Fatalf("Failed to get working directory %v.", err)
}
return filepath.Join(dir, archiveName)
}
// RunRemote copies the archive file to a /tmp file on host, unpacks it, and runs the e2e_node.test
func RunRemote(archive string, host string) (string, error) {
// Create the temp staging directory
tmp := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
_, err := runSshCommand("ssh", host, "--", "mkdir", tmp)
if err != nil {
return "", err
}
defer func() {
output, err := runSshCommand("ssh", host, "--", "rm", "-rf", tmp)
if err != nil {
glog.Errorf("Failed to cleanup tmp directory %s on host %v. Output:\n%s", tmp, err, output)
}
}()
// Copy the archive to the staging directory
_, err = runSshCommand("scp", archive, fmt.Sprintf("%s:%s/", host, tmp))
if err != nil {
return "", err
}
// Kill any running node processes
cmd := getSshCommand(" ; ",
"sudo pkill kubelet",
"sudo pkill kube-apiserver",
"sudo pkill etcd")
// No need to log an error if pkill fails since pkill will fail if the commands are not running.
// If we are unable to stop existing running k8s processes, we should see messages in the kubelet/apiserver/etcd
// logs about failing to bind the required ports.
runSshCommand("ssh", host, "--", "sh", "-c", cmd)
// Extract the archive and run the tests
cmd = getSshCommand(" && ",
fmt.Sprintf("cd %s", tmp),
fmt.Sprintf("tar -xzvf ./%s", archiveName),
"./e2e_node.test --logtostderr --v 2 --build-services=false --node-name `hostname`")
output, err := runSshCommand("ssh", host, "--", "sh", "-c", cmd)
if err != nil {
return "", err
}
return output, nil
}
// getSshCommand handles proper quoting so that multiple commands are executed in the same shell over ssh
func getSshCommand(sep string, args ...string) string {
return fmt.Sprintf("'%s'", strings.Join(args, sep))
}
// runSshCommand executes the ssh or scp command, adding the flag provided --ssh-options
func runSshCommand(cmd string, args ...string) (string, error) {
if env, found := sshOptionsMap[*sshEnv]; found {
args = append(strings.Split(env, " "), args...)
}
if *sshOptions != "" {
args = append(strings.Split(*sshOptions, " "), args...)
}
output, err := exec.Command(cmd, args...).CombinedOutput()
if err != nil {
return fmt.Sprintf("%s", output), fmt.Errorf("command %q %q failed with error: %v and output: %q", cmd, args, err, output)
}
return fmt.Sprintf("%s", output), nil
}

185
test/e2e_node/e2e_service.go Normal file
View File

@ -0,0 +1,185 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"net/http"
"os"
"os/exec"
"strings"
"time"
"github.com/golang/glog"
)
var serverStartTimeout = flag.Duration("server-start-timeout", time.Second*30, "Time to wait for each server to become healthy.")
type e2eService struct {
etcdCmd *exec.Cmd
etcdCombinedOut bytes.Buffer
etcdDataDir string
apiServerCmd *exec.Cmd
apiServerCombinedOut bytes.Buffer
kubeletCmd *exec.Cmd
kubeletCombinedOut bytes.Buffer
}
func newE2eService() *e2eService {
return &e2eService{}
}
func (es *e2eService) start() error {
if _, err := getK8sBin("kubelet"); err != nil {
return err
}
if _, err := getK8sBin("kube-apiserver"); err != nil {
return err
}
cmd, err := es.startEtcd()
if err != nil {
return err
}
es.etcdCmd = cmd
cmd, err = es.startApiServer()
if err != nil {
return err
}
es.apiServerCmd = cmd
cmd, err = es.startKubeletServer()
if err != nil {
return err
}
es.kubeletCmd = cmd
return nil
}
func (es *e2eService) stop() {
if es.kubeletCmd != nil {
err := es.kubeletCmd.Process.Kill()
if err != nil {
glog.Errorf("Failed to stop kubelet.\n%v", err)
}
}
if es.apiServerCmd != nil {
err := es.apiServerCmd.Process.Kill()
if err != nil {
glog.Errorf("Failed to stop be-apiserver.\n%v", err)
}
}
if es.etcdCmd != nil {
err := es.etcdCmd.Process.Kill()
if err != nil {
glog.Errorf("Failed to stop etcd.\n%v", err)
}
}
if es.etcdDataDir != "" {
err := os.RemoveAll(es.etcdDataDir)
if err != nil {
glog.Errorf("Failed to delete etcd data directory %s.\n%v", es.etcdDataDir, err)
}
}
}
func (es *e2eService) startEtcd() (*exec.Cmd, error) {
dataDir, err := ioutil.TempDir("", "node-e2e")
if err != nil {
return nil, err
}
es.etcdDataDir = dataDir
return es.startServer(healthCheckCommand{
combinedOut: &es.etcdCombinedOut,
healthCheckUrl: "http://127.0.0.1:4001/v2/keys",
command: "etcd",
args: []string{"--data-dir", dataDir, "--name", "e2e-node"},
})
}
func (es *e2eService) startApiServer() (*exec.Cmd, error) {
return es.startServer(
healthCheckCommand{
combinedOut: &es.apiServerCombinedOut,
healthCheckUrl: "http://127.0.0.1:8080/healthz",
command: "sudo",
args: []string{getApiServerBin(),
"--v", "2", "--logtostderr", "--log_dir", "./",
"--etcd-servers", "http://127.0.0.1:4001",
"--insecure-bind-address", "0.0.0.0",
"--service-cluster-ip-range", "10.0.0.1/24",
"--kubelet-port", "10250"},
})
}
func (es *e2eService) startKubeletServer() (*exec.Cmd, error) {
return es.startServer(
healthCheckCommand{
combinedOut: &es.kubeletCombinedOut,
healthCheckUrl: "http://127.0.0.1:10255/healthz",
command: "sudo",
args: []string{getKubeletServerBin(),
"--v", "2", "--logtostderr", "--log_dir", "./",
"--api-servers", "http://127.0.0.1:8080",
"--address", "0.0.0.0",
"--port", "10250"},
})
}
func (es *e2eService) startServer(hcc healthCheckCommand) (*exec.Cmd, error) {
cmdErrorChan := make(chan error)
cmd := exec.Command(hcc.command, hcc.args...)
cmd.Stdout = hcc.combinedOut
cmd.Stderr = hcc.combinedOut
go func() {
err := cmd.Run()
if err != nil {
cmdErrorChan <- fmt.Errorf("%v Exited with status %v. Output:\n%s", hcc, err, *hcc.combinedOut)
}
close(cmdErrorChan)
}()
endTime := time.Now().Add(*serverStartTimeout)
for endTime.After(time.Now()) {
select {
case err := <-cmdErrorChan:
return nil, err
case <-time.After(time.Second):
resp, err := http.Get(hcc.healthCheckUrl)
if err == nil && resp.StatusCode == http.StatusOK {
return cmd, nil
}
}
}
return nil, fmt.Errorf("Timeout waiting for service %v", hcc)
}
type healthCheckCommand struct {
healthCheckUrl string
command string
args []string
combinedOut *bytes.Buffer
}
func (hcc *healthCheckCommand) String() string {
return fmt.Sprintf("`%s %s` %s", hcc.command, strings.Join(hcc.args, " "), hcc.healthCheckUrl)
}

3
test/e2e_node/environment/conformance.go
View File

@ -28,8 +28,9 @@ import (
"strings"
"errors"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"os"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
)
const success = "\033[0;32mSUCESS\033[0m"

213
test/e2e_node/gcloud/gcloud.go
View File

@ -1,213 +0,0 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gcloud
import (
"bytes"
"errors"
"fmt"
"math/rand"
"net"
"net/http"
"os/exec"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/golang/glog"
)
var freePortRegexp = regexp.MustCompile(".+:([0-9]+)")
type TearDown func() *RunResult
type GCloudClient interface {
RunAndWaitTillHealthy(
sudo bool, copyBin bool, remotePort string,
timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error)
}
type gCloudClientImpl struct {
host string
zone string
}
type RunResult struct {
out []byte
err error
cmd string
}
type CmdHandle struct {
TearDown TearDown
CombinedOutput bytes.Buffer
Output chan RunResult
LPort string
}
func NewGCloudClient(host string, zone string) GCloudClient {
return &gCloudClientImpl{host, zone}
}
func (gc *gCloudClientImpl) Command(cmd string, moreargs ...string) ([]byte, error) {
args := append([]string{"compute", "ssh"})
if gc.zone != "" {
args = append(args, "--zone", gc.zone)
}
args = append(args, gc.host, "--", cmd)
args = append(args, moreargs...)
glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
return exec.Command("gcloud", args...).CombinedOutput()
}
func (gc *gCloudClientImpl) TunnelCommand(sudo bool, lPort string, rPort string, dir string, cmd string, moreargs ...string) *exec.Cmd {
tunnelStr := fmt.Sprintf("-L %s:localhost:%s", lPort, rPort)
args := []string{"compute", "ssh"}
if gc.zone != "" {
args = append(args, "--zone", gc.zone)
}
args = append(args, "--ssh-flag", tunnelStr, gc.host, "--")
args = append(args, "cd", dir, ";")
if sudo {
args = append(args, "sudo")
}
args = append(args, cmd)
args = append(args, moreargs...)
glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
return exec.Command("gcloud", args...)
}
func (gc *gCloudClientImpl) CopyToHost(from string, to string) ([]byte, error) {
rto := fmt.Sprintf("%s:%s", gc.host, to)
args := []string{"compute", "copy-files"}
if gc.zone != "" {
args = append(args, "--zone", gc.zone)
}
args = append(args, from, rto)
glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
return exec.Command("gcloud", args...).CombinedOutput()
}
func (gc *gCloudClientImpl) Run(
sudo bool, copyBin bool, remotePort string, bin string, args ...string) *CmdHandle {
h := &CmdHandle{}
h.Output = make(chan RunResult)
rand.Seed(time.Now().UnixNano())
// Define where we will copy the temp binary
tDir := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
_, f := filepath.Split(bin)
cmd := f
if copyBin {
cmd = filepath.Join(tDir, f)
}
h.LPort = getLocalPort()
h.TearDown = func() *RunResult {
out, err := gc.Command("sudo", "pkill", f)
if err != nil {
return &RunResult{out, err, fmt.Sprintf("pkill %s", f)}
}
out, err = gc.Command("rm", "-rf", tDir)
if err != nil {
return &RunResult{out, err, fmt.Sprintf("rm -rf %s", tDir)}
}
return &RunResult{}
}
// Run the commands in a Go fn so that this method doesn't block when writing to a channel
// to report an error
go func() {
// Create the tmp directory
out, err := gc.Command("mkdir", "-p", tDir)
// Work around for gcloud flakiness - TODO: debug why gcloud sometimes cannot find credentials for some hosts
// If there was an error about credentials, retry making the directory 6 times to see if it can be resolved
// This is to help debug if the credential issues are persistent for a given host on a given run, or transient
// And if downstream gcloud commands are also impacted
for i := 0; i < 6 && err != nil && strings.Contains(string(out), "does not have any valid credentials"); i++ {
glog.Warningf("mkdir failed on host %s due to credential issues, retrying in 5 seconds %v %s", gc.host, err, out)
time.Sleep(5 * time.Second)
out, err = gc.Command("mkdir", "-p", tDir)
}
if err != nil {
glog.Errorf("mkdir failed %v %s", err, out)
h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)}
return
}
// Copy the binary
if copyBin {
out, err = gc.CopyToHost(bin, tDir)
if err != nil {
glog.Errorf("copy-files failed %v %s", err, out)
h.Output <- RunResult{out, err, fmt.Sprintf("copy-files %s %s", bin, tDir)}
return
}
}
c := gc.TunnelCommand(sudo, h.LPort, remotePort, tDir, cmd, args...)
c.Stdout = &h.CombinedOutput
c.Stderr = &h.CombinedOutput
go func() {
// Start the process
err = c.Run()
if err != nil {
glog.Errorf("command failed %v %s", err, h.CombinedOutput.Bytes())
h.Output <- RunResult{h.CombinedOutput.Bytes(), err, fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))}
return
}
}()
}()
return h
}
func (gc *gCloudClientImpl) RunAndWaitTillHealthy(
sudo bool, copyBin bool,
remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) {
h := gc.Run(sudo, copyBin, remotePort, bin, args...)
eTime := time.Now().Add(timeout)
done := false
for eTime.After(time.Now()) && !done {
select {
case r := <-h.Output:
glog.V(2).Infof("Error running %s Output:\n%s Error:\n%v", r.cmd, r.out, r.err)
return h, r.err
case <-time.After(2 * time.Second):
resp, err := http.Get(fmt.Sprintf("http://localhost:%s/%s", h.LPort, healthUrl))
if err == nil && resp.StatusCode == http.StatusOK {
done = true
break
}
}
}
if !done {
return h, errors.New(fmt.Sprintf("Timeout waiting for service to be healthy at http://localhost:%s/%s", h.LPort, healthUrl))
}
glog.Info("Healthz Success")
return h, nil
}
// GetLocalPort returns a free local port that can be used for ssh tunneling
func getLocalPort() string {
l, _ := net.Listen("tcp", ":0")
defer l.Close()
return freePortRegexp.FindStringSubmatch(l.Addr().String())[1]
}

2
test/e2e_node/kubelet_test.go
View File

@ -249,7 +249,7 @@ var _ = Describe("Kubelet", func() {
Expect(*container.Logs.UsedBytes).NotTo(BeZero(), spew.Sdump(container))
}
Expect(podsList).To(ConsistOf(podNames))
Expect(podsList).To(ConsistOf(podNames), spew.Sdump(summary))
})
})

177
test/e2e_node/runner/run_e2e.go
View File

@ -19,189 +19,72 @@ limitations under the License.
package main
import (
"bufio"
"flag"
"fmt"
"os"
"os/exec"
"strings"
"sync"
"time"
"runtime"
"github.com/golang/glog"
"k8s.io/kubernetes/test/e2e_node/gcloud"
"path/filepath"
"k8s.io/kubernetes/test/e2e_node"
)
type RunFunc func(host string, port string) ([]byte, error)
type Result struct {
host string
output []byte
err error
}
const gray = "\033[1;30m"
const blue = "\033[0;34m"
const noColour = "\033[0m"
var u = sync.WaitGroup{}
var zone = flag.String("zone", "", "gce zone the hosts live in")
var hosts = flag.String("hosts", "", "hosts to test")
var wait = flag.Bool("wait", false, "if true, wait for input before running tests")
var kubeOutputRelPath = flag.String("k8s-build-output", "_output/local/bin/linux/amd64", "Where k8s binary files are written")
var kubeRoot = ""
const buildScriptRelPath = "hack/build-go.sh"
const ginkgoTestRelPath = "test/e2e_node"
const healthyTimeoutDuration = time.Minute * 3
func main() {
// Setup coloring
stat, _ := os.Stdout.Stat()
useColor := (stat.Mode() & os.ModeCharDevice) != 0
blue := ""
noColour := ""
if useColor {
blue = "\033[0;34m"
noColour = "\033[0m"
}
flag.Parse()
if *hosts == "" {
glog.Fatalf("Must specific --hosts flag")
}
// Figure out the kube root
_, path, _, _ := runtime.Caller(0)
kubeRoot, _ = filepath.Split(path)
kubeRoot = strings.Split(kubeRoot, "/test/e2e_node")[0]
// Build the go code
out, err := exec.Command(filepath.Join(kubeRoot, buildScriptRelPath)).CombinedOutput()
if err != nil {
glog.Fatalf("Failed to build go packages %s: %v", out, err)
}
// Copy kubelet to each host and run test
if *wait {
u.Add(1)
fmt.Printf("Must specific --hosts flag")
}
archive := e2e_node.CreateTestArchive()
defer os.Remove(archive)
results := make(chan *TestResult)
hs := strings.Split(*hosts, ",")
for _, h := range hs {
go func(host string) { results <- runTests(host) }(h)
fmt.Printf("Starting tests on host %s.", h)
go func(host string) {
output, err := e2e_node.RunRemote(archive, host)
results <- &TestResult{
output: output,
err: err,
host: host,
}
// Maybe wait for user input before running tests
if *wait {
WaitForUser()
}(h)
}
// Wait for all tests to complete and emit the results
errCount := 0
for i := 0; i < len(hs); i++ {
tr := <-results
host := tr.fullhost
host := tr.host
fmt.Printf("%s================================================================%s\n", blue, noColour)
if tr.err != nil {
errCount++
glog.Infof("%s================================================================%s", blue, noColour)
glog.Infof("Failure Finished Host %s Test Suite %s %v", host, tr.testCombinedOutput, tr.err)
glog.V(2).Infof("----------------------------------------------------------------")
glog.V(5).Infof("Host %s Etcd Logs\n%s%s%s", host, gray, tr.etcdCombinedOutput, noColour)
glog.V(5).Infof("----------------------------------------------------------------")
glog.V(5).Infof("Host %s Apiserver Logs\n%s%s%s", host, gray, tr.apiServerCombinedOutput, noColour)
glog.V(5).Infof("----------------------------------------------------------------")
glog.V(2).Infof("Host %s Kubelet Logs\n%s%s%s", host, gray, tr.kubeletCombinedOutput, noColour)
glog.Infof("%s================================================================%s", blue, noColour)
fmt.Printf("Failure Finished Host %s Test Suite %s %v\n", host, tr.output, tr.err)
} else {
glog.Infof("================================================================")
glog.Infof("Success Finished Host %s Test Suite %s", host, tr.testCombinedOutput)
glog.Infof("================================================================")
fmt.Printf("Success Finished Host %s Test Suite %s\n", host, tr.output)
}
fmt.Printf("%s================================================================%s\n", blue, noColour)
}
// Set the exit code if there were failures
if errCount > 0 {
glog.Errorf("Failure: %d errors encountered.", errCount)
fmt.Printf("Failure: %d errors encountered.", errCount)
os.Exit(1)
}
}
func WaitForUser() {
scanner := bufio.NewScanner(os.Stdin)
fmt.Printf("Enter \"y\" to run tests\n")
for scanner.Scan() {
if strings.ToUpper(scanner.Text()) != "Y\n" {
break
}
fmt.Printf("Enter \"y\" to run tests\n")
}
u.Done()
}
type TestResult struct {
fullhost string
output string
err error
testCombinedOutput string
etcdCombinedOutput string
apiServerCombinedOutput string
kubeletCombinedOutput string
}
func runTests(fullhost string) *TestResult {
result := &TestResult{fullhost: fullhost}
host := strings.Split(fullhost, ".")[0]
c := gcloud.NewGCloudClient(host, *zone)
// TODO(pwittrock): Come up with something better for bootstrapping the environment.
eh, err := c.RunAndWaitTillHealthy(
false, false, "4001", healthyTimeoutDuration, "v2/keys/", "etcd", "--data-dir", "./", "--name", "e2e-node")
defer func() {
eh.TearDown()
result.etcdCombinedOutput = fmt.Sprintf("%s", eh.CombinedOutput.Bytes())
}()
if err != nil {
result.err = fmt.Errorf("Host %s failed to run command %v", host, err)
return result
}
apiBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kube-apiserver")
ah, err := c.RunAndWaitTillHealthy(
true, true, "8080", healthyTimeoutDuration, "healthz", apiBin, "--service-cluster-ip-range",
"10.0.0.1/24", "--insecure-bind-address", "0.0.0.0", "--etcd-servers", "http://127.0.0.1:4001",
"--v", "2", "--alsologtostderr", "--kubelet-port", "10250")
defer func() {
ah.TearDown()
result.apiServerCombinedOutput = fmt.Sprintf("%s", ah.CombinedOutput.Bytes())
}()
if err != nil {
result.err = fmt.Errorf("Host %s failed to run command %v", host, err)
return result
}
kubeletBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kubelet")
// TODO: Used --v 4 or higher and upload to gcs instead of printing to the console
// TODO: Copy /var/log/messages and upload to GCS for failed tests
kh, err := c.RunAndWaitTillHealthy(
true, true, "10255", healthyTimeoutDuration, "healthz", kubeletBin, "--api-servers", "http://127.0.0.1:8080",
"--v", "2", "--alsologtostderr", "--address", "0.0.0.0", "--port", "10250")
defer func() {
kh.TearDown()
result.kubeletCombinedOutput = fmt.Sprintf("%s", kh.CombinedOutput.Bytes())
}()
if err != nil {
result.err = fmt.Errorf("Host %s failed to run command %v", host, err)
}
// Run the tests
glog.Infof("Kubelet healthy on host %s", host)
glog.Infof("Kubelet host %s tunnel running on port %s", host, ah.LPort)
u.Wait()
glog.Infof("Running ginkgo tests against host %s", host)
ginkgoTests := filepath.Join(kubeRoot, ginkgoTestRelPath)
out, err := exec.Command(
"ginkgo", ginkgoTests, "--",
"--kubelet-address", fmt.Sprintf("http://127.0.0.1:%s", kh.LPort),
"--api-server-address", fmt.Sprintf("http://127.0.0.1:%s", ah.LPort),
"--node-name", fullhost,
"--v", "2", "--alsologtostderr").CombinedOutput()
result.err = err
result.testCombinedOutput = fmt.Sprintf("%s", out)
return result
host string
}