github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Test runner harness for node e2e tests

pull/6/head
Phillip Wittrock 2015-11-10 15:42:07 -08:00
parent 2a92988392
commit 49e1baa313
7 changed files with 460 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hack/test-go.sh
View File

@ -37,6 +37,7 @@ kube::test::find_dirs() {
-o -path './release/*' \
-o -path './target/*' \
-o -path './test/e2e/*' \
-o -path './test/e2e_node/*' \
-o -path './test/integration/*' \
\) -prune \
\) -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^\./||' | sort -u

3
hack/verify-flags/known-flags.txt
View File

@ -13,6 +13,7 @@ allow-privileged
api-burst
api-prefix
api-rate
api-server-host
api-server-port
api-servers
api-token
@ -143,6 +144,7 @@ kubelet-certificate-authority
kubelet-client-certificate
kubelet-client-key
kubelet-docker-endpoint
kubelet-host
kubelet-host-network-sources
kubelet-https
kubelet-network-plugin
@ -154,6 +156,7 @@ kubelet-sync-frequency
kubelet-timeout
kube-master
kubernetes-service-node-port
k8s-build-output
label-columns
last-release-pr
leave-stdin-open

17
test/e2e_node/doc.go Normal file
View File

@ -0,0 +1,17 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node

45
test/e2e_node/e2e_node_suite_test.go Normal file
View File

@ -0,0 +1,45 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"flag"
"testing"
)
var kubeletHost = flag.String("kubelet-host", "localhost", "Host address of the kubelet")
var kubeletPort = flag.Int("kubelet-port", 10250, "Kubelet port")
var apiServerHost = flag.String("api-server-host", "localhost", "Host address of the api server")
var apiServerPort = flag.Int("api-server-port", 8080, "Api server port")
func TestE2eNode(t *testing.T) {
flag.Parse()
RegisterFailHandler(Fail)
RunSpecs(t, "E2eNode Suite")
}
// Setup the kubelet on the node
var _ = BeforeSuite(func() {
})
// Tear down the kubelet on the node
var _ = AfterSuite(func() {
})

187
test/e2e_node/gcloud/gcloud.go Normal file
View File

@ -0,0 +1,187 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gcloud
import (
"errors"
"fmt"
"math/rand"
"os/exec"
"net"
"net/http"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/golang/glog"
)
var freePortRegexp = regexp.MustCompile(".+:([0-9]+)")
type TearDown func()
type GCloudClient interface {
CopyAndWaitTillHealthy(sudo bool, remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error)
}
type gCloudClientImpl struct {
host string
zone string
}
type RunResult struct {
out []byte
err error
cmd string
}
type CmdHandle struct {
TearDown TearDown
Output chan RunResult
LPort string
}
func NewGCloudClient(host string, zone string) GCloudClient {
return &gCloudClientImpl{host, zone}
}
func (gc *gCloudClientImpl) Command(cmd string, moreargs ...string) ([]byte, error) {
args := append([]string{"compute", "ssh"})
if gc.zone != "" {
args = append(args, "--zone", gc.zone)
}
args = append(args, gc.host, "--", cmd)
args = append(args, moreargs...)
glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
return exec.Command("gcloud", args...).CombinedOutput()
}
func (gc *gCloudClientImpl) TunnelCommand(sudo bool, lPort string, rPort string, cmd string, moreargs ...string) ([]byte, error) {
tunnelStr := fmt.Sprintf("-L %s:localhost:%s", lPort, rPort)
args := []string{"compute", "ssh"}
if gc.zone != "" {
args = append(args, "--zone", gc.zone)
}
args = append(args, "--ssh-flag", tunnelStr, gc.host, "--")
if sudo {
args = append(args, "sudo")
}
args = append(args, cmd)
args = append(args, moreargs...)
glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
return exec.Command("gcloud", args...).CombinedOutput()
}
func (gc *gCloudClientImpl) CopyToHost(from string, to string) ([]byte, error) {
rto := fmt.Sprintf("%s:%s", gc.host, to)
args := []string{"compute", "copy-files"}
if gc.zone != "" {
args = append(args, "--zone", gc.zone)
}
args = append(args, from, rto)
glog.V(2).Infof("Command gcloud %s", strings.Join(args, " "))
return exec.Command("gcloud", args...).CombinedOutput()
}
func (gc *gCloudClientImpl) CopyAndRun(sudo bool, remotePort string, bin string, args ...string) *CmdHandle {
h := &CmdHandle{}
h.Output = make(chan RunResult)
rand.Seed(time.Now().UnixNano())
// Define where we will copy the temp binary
tDir := fmt.Sprintf("/tmp/gcloud-e2e-%d", rand.Int31())
_, f := filepath.Split(bin)
cmd := filepath.Join(tDir, f)
h.LPort = getLocalPort()
h.TearDown = func() {
out, err := gc.Command("sudo", "pkill", f)
if err != nil {
h.Output <- RunResult{out, err, fmt.Sprintf("pkill %s", cmd)}
return
}
out, err = gc.Command("rm", "-rf", tDir)
if err != nil {
h.Output <- RunResult{out, err, fmt.Sprintf("rm -rf %s", tDir)}
return
}
}
// Create the tmp directory
out, err := gc.Command("mkdir", "-p", tDir)
if err != nil {
glog.Errorf("mkdir failed %v", err)
h.Output <- RunResult{out, err, fmt.Sprintf("mkdir -p %s", tDir)}
return h
}
// Copy the binary
out, err = gc.CopyToHost(bin, tDir)
if err != nil {
glog.Errorf("copy-files failed %v", err)
h.Output <- RunResult{out, err, fmt.Sprintf("copy-files %s %s", bin, tDir)}
return h
}
// Do the setup
go func() {
// Start the process
out, err = gc.TunnelCommand(sudo, h.LPort, remotePort, cmd, args...)
if err != nil {
glog.Errorf("command failed %v", err)
h.Output <- RunResult{out, err, fmt.Sprintf("%s %s", cmd, strings.Join(args, " "))}
return
}
}()
return h
}
func (gc *gCloudClientImpl) CopyAndWaitTillHealthy(
sudo bool,
remotePort string, timeout time.Duration, healthUrl string, bin string, args ...string) (*CmdHandle, error) {
h := gc.CopyAndRun(sudo, remotePort, bin, args...)
eTime := time.Now().Add(timeout)
done := false
for eTime.After(time.Now()) && !done {
select {
case r := <-h.Output:
glog.V(2).Infof("Error running %s Output:\n%s Error:\n%v", r.cmd, r.out, r.err)
return h, r.err
case <-time.After(2 * time.Second):
resp, err := http.Get(fmt.Sprintf("http://localhost:%s/%s", h.LPort, healthUrl))
if err == nil && resp.StatusCode == http.StatusOK {
done = true
break
}
}
}
if !done {
return h, errors.New(fmt.Sprintf("Timeout waiting for service to be healthy at http://localhost:%s/%s", h.LPort, healthUrl))
}
glog.Info("Healthz Success")
return h, nil
}
// GetLocalPort returns a free local port that can be used for ssh tunneling
func getLocalPort() string {
l, _ := net.Listen("tcp", ":0")
defer l.Close()
return freePortRegexp.FindStringSubmatch(l.Addr().String())[1]
}

53
test/e2e_node/kubelet_test.go Normal file
View File

@ -0,0 +1,53 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e_node
import (
"fmt"
"io/ioutil"
"net/http"
"github.com/golang/glog"
. "github.com/onsi/ginkgo"
)
var _ = Describe("Kubelet", func() {
BeforeEach(func() {
// Setup the client to talk to the kubelet
})
Describe("checking kubelet status", func() {
Context("when retrieving the node status", func() {
It("should have the container version", func() {
// TODO: This is just a place holder, write a real test here
resp, err := http.Get(fmt.Sprintf("http://%s:%d/api/v2.0/attributes", *kubeletHost, *kubeletPort))
if err != nil {
glog.Errorf("Error: %v", err)
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
glog.Errorf("Error: %v", err)
return
}
glog.Infof("Resp: %s", body)
})
})
})
})

154
test/e2e_node/runner/run_e2e.go Normal file
View File

@ -0,0 +1,154 @@
/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"bufio"
"flag"
"fmt"
"os"
"os/exec"
"strings"
"sync"
"time"
"runtime"
"github.com/golang/glog"
"k8s.io/kubernetes/test/e2e_node/gcloud"
"path/filepath"
)
type RunFunc func(host string, port string) ([]byte, error)
type Result struct {
host string
output []byte
err error
}
var u = sync.WaitGroup{}
var zone = flag.String("zone", "", "gce zone the hosts live in")
var hosts = flag.String("hosts", "", "hosts to test")
var wait = flag.Bool("wait", false, "if true, wait for input before running tests")
var kubeOutputRelPath = flag.String("k8s-build-output", "_output/local/bin/linux/amd64", "Where k8s binary files are written")
var kubeRoot = ""
const buildScriptRelPath = "hack/build-go.sh"
const ginkoTestRelPath = "test/e2e_node"
const healthyTimeoutDuration = time.Minute * 3
func main() {
flag.Parse()
if *hosts == "" {
glog.Fatalf("Must specific --hosts flag")
}
// Figure out the kube root
_, path, _, _ := runtime.Caller(0)
kubeRoot, _ = filepath.Split(path)
kubeRoot = strings.Split(kubeRoot, "/test/e2e_node")[0]
// Build the go code
out, err := exec.Command(filepath.Join(kubeRoot, buildScriptRelPath)).CombinedOutput()
if err != nil {
glog.Fatalf("Failed to build go packages %s: %v", out, err)
}
// Copy kubelet to each host and run test
if *wait {
u.Add(1)
}
w := sync.WaitGroup{}
for _, h := range strings.Split(*hosts, ",") {
w.Add(1)
go func(host string) {
out, err := runTests(host)
if err != nil {
glog.Infof("Failure Finished Test Suite %s %v", out, err)
} else {
glog.Infof("Success Finished Test Suite %s", out)
}
w.Done()
}(h)
}
// Maybe wait for user input before running tests
if *wait {
WaitForUser()
}
// Wait for the tests to finish
w.Wait()
glog.Infof("All hosts finished")
}
func WaitForUser() {
scanner := bufio.NewScanner(os.Stdin)
fmt.Printf("Enter \"y\" to run tests\n")
for scanner.Scan() {
if strings.ToUpper(scanner.Text()) != "Y\n" {
break
}
fmt.Printf("Enter \"y\" to run tests\n")
}
u.Done()
}
func runTests(host string) ([]byte, error) {
c := gcloud.NewGCloudClient(host, *zone)
// TODO(pwittrock): Come up with something better for bootstrapping the environment.
etcdBin := filepath.Join(kubeRoot, "third_party/etcd/etcd")
eh, err := c.CopyAndWaitTillHealthy(false, "4001", healthyTimeoutDuration, "v2/keys/", etcdBin)
defer func() { eh.TearDown() }()
if err != nil {
return nil, fmt.Errorf("Host %s failed to run command %v", host, err)
}
apiBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kube-apiserver")
ah, err := c.CopyAndWaitTillHealthy(
true, "8080", healthyTimeoutDuration, "healthz", apiBin, "--service-cluster-ip-range",
"10.0.0.1/24", "--insecure-bind-address", "0.0.0.0", "--etcd-servers", "http://localhost:4001",
"--cluster-name", "kubernetes", "--v", "2", "--kubelet-port", "10250")
defer func() { ah.TearDown() }()
if err != nil {
return nil, fmt.Errorf("Host %s failed to run command %v", host, err)
}
kubeletBin := filepath.Join(kubeRoot, *kubeOutputRelPath, "kubelet")
kh, err := c.CopyAndWaitTillHealthy(
true, "4194", healthyTimeoutDuration, "healthz", kubeletBin, "--api-servers", "http://localhost:8080",
"--logtostderr", "--address", "0.0.0.0", "--port", "10250")
defer func() { kh.TearDown() }()
if err != nil {
return nil, fmt.Errorf("Host %s failed to run command %v", host, err)
}
// Run the tests
glog.Infof("Kubelet healthy on host %s", host)
glog.Infof("Kubelet host %s tunnel running on port %s", host, ah.LPort)
u.Wait()
glog.Infof("Running ginkgo tests against host %s", host)
ginkoTests := filepath.Join(kubeRoot, ginkoTestRelPath)
return exec.Command(
"ginkgo", ginkoTests, "--",
"--kubelet-host", "localhost", "--kubelet-port", kh.LPort,
"--api-server-host", "localhost", "--api-server-port", kh.LPort,
"-logtostderr").CombinedOutput()
}