2015-11-10 23:42:07 +00:00
|
|
|
/*
|
2016-06-03 00:25:58 +00:00
|
|
|
Copyright 2016 The Kubernetes Authors.
|
2015-11-10 23:42:07 +00:00
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2015-12-03 22:27:51 +00:00
|
|
|
// To run tests in this suite
|
2016-02-25 16:38:06 +00:00
|
|
|
// NOTE: This test suite requires password-less sudo capabilities to run the kubelet and kube-apiserver.
|
2015-11-10 23:42:07 +00:00
|
|
|
package e2e_node
|
|
|
|
|
|
|
|
import (
|
2016-02-22 18:52:20 +00:00
|
|
|
"bytes"
|
2016-07-18 07:52:39 +00:00
|
|
|
"encoding/json"
|
2015-11-10 23:42:07 +00:00
|
|
|
"flag"
|
2016-02-22 18:52:20 +00:00
|
|
|
"fmt"
|
2016-04-30 01:25:10 +00:00
|
|
|
"io/ioutil"
|
2016-02-29 19:37:17 +00:00
|
|
|
"math/rand"
|
2016-05-23 20:16:47 +00:00
|
|
|
"os"
|
2016-02-22 18:52:20 +00:00
|
|
|
"os/exec"
|
2016-05-23 20:16:47 +00:00
|
|
|
"path"
|
2016-08-19 05:43:20 +00:00
|
|
|
"syscall"
|
2015-11-10 23:42:07 +00:00
|
|
|
"testing"
|
2016-02-29 19:37:17 +00:00
|
|
|
"time"
|
2016-02-22 18:52:20 +00:00
|
|
|
|
2016-08-16 22:49:50 +00:00
|
|
|
"k8s.io/kubernetes/pkg/api"
|
|
|
|
clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
|
2016-07-08 05:56:46 +00:00
|
|
|
commontest "k8s.io/kubernetes/test/e2e/common"
|
2016-06-29 00:20:08 +00:00
|
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
2016-08-12 06:30:04 +00:00
|
|
|
"k8s.io/kubernetes/test/e2e_node/services"
|
2016-09-10 00:20:45 +00:00
|
|
|
"k8s.io/kubernetes/test/e2e_node/system"
|
2016-06-29 00:20:08 +00:00
|
|
|
|
2016-02-22 18:52:20 +00:00
|
|
|
"github.com/golang/glog"
|
2016-09-10 00:20:45 +00:00
|
|
|
"github.com/kardianos/osext"
|
2016-02-22 18:52:20 +00:00
|
|
|
. "github.com/onsi/ginkgo"
|
2016-08-04 22:50:35 +00:00
|
|
|
"github.com/onsi/ginkgo/config"
|
2016-05-23 20:16:47 +00:00
|
|
|
more_reporters "github.com/onsi/ginkgo/reporters"
|
2016-02-22 18:52:20 +00:00
|
|
|
. "github.com/onsi/gomega"
|
2016-08-04 23:51:11 +00:00
|
|
|
"github.com/spf13/pflag"
|
2015-11-10 23:42:07 +00:00
|
|
|
)
|
|
|
|
|
2016-08-12 06:30:04 +00:00
|
|
|
var e2es *services.E2EServices
|
2016-07-18 07:52:39 +00:00
|
|
|
|
2016-09-10 00:20:45 +00:00
|
|
|
// TODO(random-liu): Change the following modes to sub-command.
|
2016-08-04 23:51:11 +00:00
|
|
|
var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.")
|
2016-09-10 00:20:45 +00:00
|
|
|
var systemValidateMode = flag.Bool("system-validate-mode", false, "If true, only run system validation in current process, and not run test.")
|
2016-05-20 03:04:58 +00:00
|
|
|
|
2016-06-29 00:20:08 +00:00
|
|
|
func init() {
|
|
|
|
framework.RegisterCommonFlags()
|
|
|
|
framework.RegisterNodeFlags()
|
2016-08-04 23:51:11 +00:00
|
|
|
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
|
|
|
|
// Mark the run-services-mode flag as hidden to prevent user from using it.
|
|
|
|
pflag.CommandLine.MarkHidden("run-services-mode")
|
2016-09-19 23:25:22 +00:00
|
|
|
// It's weird that if I directly use pflag in TestContext, it will report error.
|
|
|
|
// It seems that someone is using flag.Parse() after init() and TestMain().
|
|
|
|
// TODO(random-liu): Find who is using flag.Parse() and cause errors and move the following logic
|
|
|
|
// into TestContext.
|
2016-11-06 17:20:57 +00:00
|
|
|
pflag.CommandLine.MarkHidden("enable-cri")
|
2016-06-29 00:20:08 +00:00
|
|
|
}
|
|
|
|
|
2016-09-19 23:25:22 +00:00
|
|
|
func TestMain(m *testing.M) {
|
2016-08-04 23:51:11 +00:00
|
|
|
pflag.Parse()
|
2016-09-19 23:25:22 +00:00
|
|
|
os.Exit(m.Run())
|
|
|
|
}
|
|
|
|
|
2016-08-19 05:43:20 +00:00
|
|
|
// When running the containerized conformance test, we'll mount the
|
|
|
|
// host root filesystem as readonly to /rootfs.
|
|
|
|
const rootfs = "/rootfs"
|
|
|
|
|
2016-09-19 23:25:22 +00:00
|
|
|
func TestE2eNode(t *testing.T) {
|
2016-08-04 23:51:11 +00:00
|
|
|
if *runServicesMode {
|
|
|
|
// If run-services-mode is specified, only run services in current process.
|
2016-08-12 06:30:04 +00:00
|
|
|
services.RunE2EServices()
|
2016-08-04 23:51:11 +00:00
|
|
|
return
|
|
|
|
}
|
2016-09-10 00:20:45 +00:00
|
|
|
if *systemValidateMode {
|
|
|
|
// If system-validate-mode is specified, only run system validation in current process.
|
2016-08-19 05:43:20 +00:00
|
|
|
if framework.TestContext.NodeConformance {
|
|
|
|
// Chroot to /rootfs to make system validation can check system
|
|
|
|
// as in the root filesystem.
|
|
|
|
// TODO(random-liu): Consider to chroot the whole test process to make writing
|
|
|
|
// test easier.
|
|
|
|
if err := syscall.Chroot(rootfs); err != nil {
|
|
|
|
glog.Exitf("chroot %q failed: %v", rootfs, err)
|
|
|
|
}
|
|
|
|
}
|
2016-09-10 00:20:45 +00:00
|
|
|
if err := system.Validate(); err != nil {
|
|
|
|
glog.Exitf("system validation failed: %v", err)
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
2016-08-04 23:51:11 +00:00
|
|
|
// If run-services-mode is not specified, run test.
|
2016-02-29 19:37:17 +00:00
|
|
|
rand.Seed(time.Now().UTC().UnixNano())
|
2015-11-10 23:42:07 +00:00
|
|
|
RegisterFailHandler(Fail)
|
2016-05-23 20:16:47 +00:00
|
|
|
reporters := []Reporter{}
|
2016-08-05 00:17:57 +00:00
|
|
|
reportDir := framework.TestContext.ReportDir
|
|
|
|
if reportDir != "" {
|
2016-05-23 20:16:47 +00:00
|
|
|
// Create the directory if it doesn't already exists
|
2016-08-05 00:17:57 +00:00
|
|
|
if err := os.MkdirAll(reportDir, 0755); err != nil {
|
2016-05-23 20:16:47 +00:00
|
|
|
glog.Errorf("Failed creating report directory: %v", err)
|
|
|
|
} else {
|
|
|
|
// Configure a junit reporter to write to the directory
|
2016-08-04 22:50:35 +00:00
|
|
|
junitFile := fmt.Sprintf("junit_%s%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode)
|
2016-08-05 00:17:57 +00:00
|
|
|
junitPath := path.Join(reportDir, junitFile)
|
2016-05-23 20:16:47 +00:00
|
|
|
reporters = append(reporters, more_reporters.NewJUnitReporter(junitPath))
|
|
|
|
}
|
|
|
|
}
|
2016-04-14 19:04:36 +00:00
|
|
|
RunSpecsWithDefaultAndCustomReporters(t, "E2eNode Suite", reporters)
|
2015-11-10 23:42:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Setup the kubelet on the node
|
2016-07-18 07:52:39 +00:00
|
|
|
var _ = SynchronizedBeforeSuite(func() []byte {
|
2016-08-04 23:51:11 +00:00
|
|
|
// Initialize node name here, so that the following code can get right node name.
|
2016-08-04 01:14:53 +00:00
|
|
|
if framework.TestContext.NodeName == "" {
|
|
|
|
hostname, err := os.Hostname()
|
2016-08-18 23:03:02 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred(), "should be able to get node name")
|
2016-08-04 01:14:53 +00:00
|
|
|
framework.TestContext.NodeName = hostname
|
|
|
|
}
|
2016-09-10 00:20:45 +00:00
|
|
|
|
|
|
|
// Run system validation test.
|
|
|
|
Expect(validateSystem()).To(Succeed(), "system validation")
|
|
|
|
|
2016-05-20 03:04:58 +00:00
|
|
|
// Pre-pull the images tests depend on so we can fail immediately if there is an image pull issue
|
|
|
|
// This helps with debugging test flakes since it is hard to tell when a test failure is due to image pulling.
|
2016-09-15 21:49:14 +00:00
|
|
|
if framework.TestContext.PrepullImages {
|
2016-05-20 03:04:58 +00:00
|
|
|
glog.Infof("Pre-pulling images so that they are cached for the tests.")
|
|
|
|
err := PrePullAllImages()
|
|
|
|
Expect(err).ShouldNot(HaveOccurred())
|
|
|
|
}
|
|
|
|
|
2016-04-30 01:25:10 +00:00
|
|
|
// TODO(yifan): Temporary workaround to disable coreos from auto restart
|
|
|
|
// by masking the locksmithd.
|
|
|
|
// We should mask locksmithd when provisioning the machine.
|
|
|
|
maskLocksmithdOnCoreos()
|
|
|
|
|
2016-02-22 18:52:20 +00:00
|
|
|
if *startServices {
|
2016-09-13 23:47:45 +00:00
|
|
|
// If the services are expected to stop after test, they should monitor the test process.
|
|
|
|
// If the services are expected to keep running after test, they should not monitor the test process.
|
|
|
|
e2es = services.NewE2EServices(*stopServices)
|
2016-08-18 23:03:02 +00:00
|
|
|
Expect(e2es.Start()).To(Succeed(), "should be able to start node services.")
|
2016-02-22 18:52:20 +00:00
|
|
|
glog.Infof("Node services started. Running tests...")
|
|
|
|
} else {
|
|
|
|
glog.Infof("Running tests without starting services.")
|
|
|
|
}
|
2016-07-08 05:56:46 +00:00
|
|
|
|
2016-08-16 22:49:50 +00:00
|
|
|
glog.Infof("Wait for the node to be ready")
|
|
|
|
waitForNodeReady()
|
|
|
|
|
2016-07-08 05:56:46 +00:00
|
|
|
// Reference common test to make the import valid.
|
|
|
|
commontest.CurrentSuite = commontest.NodeE2E
|
2016-07-18 07:52:39 +00:00
|
|
|
|
2016-08-04 23:51:11 +00:00
|
|
|
data, err := json.Marshal(&framework.TestContext.NodeTestContextType)
|
2016-08-18 23:03:02 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred(), "should be able to serialize node test context.")
|
|
|
|
|
2016-07-18 07:52:39 +00:00
|
|
|
return data
|
|
|
|
}, func(data []byte) {
|
2016-08-04 23:51:11 +00:00
|
|
|
// The node test context is updated in the first function, update it on every test node.
|
|
|
|
err := json.Unmarshal(data, &framework.TestContext.NodeTestContextType)
|
2016-08-18 23:03:02 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred(), "should be able to deserialize node test context.")
|
2015-11-10 23:42:07 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
// Tear down the kubelet on the node
|
2016-07-18 07:52:39 +00:00
|
|
|
var _ = SynchronizedAfterSuite(func() {}, func() {
|
2016-05-25 22:04:02 +00:00
|
|
|
if e2es != nil {
|
|
|
|
if *startServices && *stopServices {
|
|
|
|
glog.Infof("Stopping node services...")
|
2016-08-04 23:51:11 +00:00
|
|
|
e2es.Stop()
|
2016-05-25 22:04:02 +00:00
|
|
|
}
|
2016-04-14 19:04:36 +00:00
|
|
|
}
|
2016-05-26 22:21:38 +00:00
|
|
|
|
2016-04-18 20:56:07 +00:00
|
|
|
glog.Infof("Tests Finished")
|
2016-04-14 19:04:36 +00:00
|
|
|
})
|
|
|
|
|
2016-09-10 00:20:45 +00:00
|
|
|
// validateSystem runs system validation in a separate process and returns error if validation fails.
|
|
|
|
func validateSystem() error {
|
|
|
|
testBin, err := osext.Executable()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("can't get current binary: %v", err)
|
|
|
|
}
|
2016-08-19 05:43:20 +00:00
|
|
|
// Pass all flags into the child process, so that it will see the same flag set.
|
2016-08-19 04:50:33 +00:00
|
|
|
output, err := exec.Command(testBin, append([]string{"--system-validate-mode"}, os.Args[1:]...)...).CombinedOutput()
|
2016-09-10 00:20:45 +00:00
|
|
|
// The output of system validation should have been formatted, directly print here.
|
|
|
|
fmt.Print(string(output))
|
|
|
|
if err != nil {
|
2016-08-19 04:50:33 +00:00
|
|
|
return fmt.Errorf("system validation failed: %v", err)
|
2016-09-10 00:20:45 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-04-30 01:25:10 +00:00
|
|
|
func maskLocksmithdOnCoreos() {
|
|
|
|
data, err := ioutil.ReadFile("/etc/os-release")
|
|
|
|
if err != nil {
|
2016-07-23 00:24:21 +00:00
|
|
|
// Not all distros contain this file.
|
|
|
|
glog.Infof("Could not read /etc/os-release: %v", err)
|
|
|
|
return
|
2016-04-30 01:25:10 +00:00
|
|
|
}
|
|
|
|
if bytes.Contains(data, []byte("ID=coreos")) {
|
2016-08-19 04:50:33 +00:00
|
|
|
output, err := exec.Command("systemctl", "mask", "--now", "locksmithd").CombinedOutput()
|
2016-08-18 23:03:02 +00:00
|
|
|
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("should be able to mask locksmithd - output: %q", string(output)))
|
2016-05-25 22:04:02 +00:00
|
|
|
glog.Infof("Locksmithd is masked successfully")
|
2016-04-30 01:25:10 +00:00
|
|
|
}
|
|
|
|
}
|
2016-08-16 22:49:50 +00:00
|
|
|
|
|
|
|
func waitForNodeReady() {
|
|
|
|
const (
|
|
|
|
// nodeReadyTimeout is the time to wait for node to become ready.
|
|
|
|
nodeReadyTimeout = 2 * time.Minute
|
|
|
|
// nodeReadyPollInterval is the interval to check node ready.
|
|
|
|
nodeReadyPollInterval = 1 * time.Second
|
|
|
|
)
|
|
|
|
config, err := framework.LoadConfig()
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
client, err := clientset.NewForConfig(config)
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
Eventually(func() error {
|
|
|
|
nodes, err := client.Nodes().List(api.ListOptions{})
|
|
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
if nodes == nil {
|
|
|
|
return fmt.Errorf("the node list is nil.")
|
|
|
|
}
|
|
|
|
Expect(len(nodes.Items) > 1).NotTo(BeTrue())
|
|
|
|
if len(nodes.Items) == 0 {
|
|
|
|
return fmt.Errorf("empty node list: %+v", nodes)
|
|
|
|
}
|
|
|
|
node := nodes.Items[0]
|
|
|
|
if !api.IsNodeReady(&node) {
|
|
|
|
return fmt.Errorf("node is not ready: %+v", node)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}, nodeReadyTimeout, nodeReadyPollInterval).Should(Succeed())
|
|
|
|
}
|