From 22c9e23202ab654e5cff902a8789044e0b6cb659 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Fri, 2 Jun 2017 17:00:46 -0700 Subject: [PATCH] Supports customized system spec in the node conformance test and creates the GKE system spec --- build/root/Makefile | 5 + hack/make-rules/test-e2e-node.sh | 6 +- hack/verify-flags/known-flags.txt | 2 + test/e2e_node/BUILD | 1 + test/e2e_node/conformance/build/Dockerfile | 5 + test/e2e_node/conformance/build/Makefile | 30 +- test/e2e_node/e2e_node_suite_test.go | 31 +- .../conformance/conformance-jenkins.sh | 3 +- test/e2e_node/jenkins/e2e-node-jenkins.sh | 2 +- test/e2e_node/remote/node_conformance.go | 27 +- test/e2e_node/remote/node_e2e.go | 37 ++- test/e2e_node/remote/remote.go | 8 +- test/e2e_node/remote/types.go | 6 +- test/e2e_node/runner/local/run_local.go | 18 +- test/e2e_node/runner/remote/run_remote.go | 9 +- test/e2e_node/system/specs/gke.yaml | 270 ++++++++++++++++++ test/e2e_node/system/types.go | 49 ++-- test/e2e_node/system/validators.go | 6 +- 18 files changed, 452 insertions(+), 63 deletions(-) create mode 100644 test/e2e_node/system/specs/gke.yaml diff --git a/build/root/Makefile b/build/root/Makefile index 401b8c44f4..f693b71254 100644 --- a/build/root/Makefile +++ b/build/root/Makefile @@ -240,6 +240,11 @@ define TEST_E2E_NODE_HELP_INFO # IMAGE_SERVICE_ENDPOINT: remote image endpoint to connect to, to prepull images. # Used when RUNTIME is set to "remote". # IMAGE_CONFIG_FILE: path to a file containing image configuration. +# SYSTEM_SPEC_NAME: The name of the system spec to be used for validating the +# image in the node conformance test. The specs are located at +# test/e2e_node/system/specs/. For example, "SYSTEM_SPEC_NAME=gke" will use +# the spec at test/e2e_node/system/specs/gke.yaml. If unspecified, the +# default built-in spec (system.DefaultSpec) will be used. # # Example: # make test-e2e-node FOCUS=Kubelet SKIP=container diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh index 54a8233c7e..8c1e2e1158 100755 --- a/hack/make-rules/test-e2e-node.sh +++ b/hack/make-rules/test-e2e-node.sh @@ -33,6 +33,7 @@ container_runtime_endpoint=${CONTAINER_RUNTIME_ENDPOINT:-""} image_service_endpoint=${IMAGE_SERVICE_ENDPOINT:-""} run_until_failure=${RUN_UNTIL_FAILURE:-"false"} test_args=${TEST_ARGS:-""} +system_spec_name=${SYSTEM_SPEC_NAME:-} # Parse the flags to pass to ginkgo ginkgoflags="" @@ -135,7 +136,7 @@ if [ $remote = true ] ; then --results-dir="$artifacts" --ginkgo-flags="$ginkgoflags" \ --image-project="$image_project" --instance-name-prefix="$instance_prefix" \ --delete-instances="$delete_instances" --test_args="$test_args" --instance-metadata="$metadata" \ - --image-config-file="$image_config_file" \ + --image-config-file="$image_config_file" --system-spec-name="$system_spec_name" \ 2>&1 | tee -i "${artifacts}/build-log.txt" exit $? @@ -163,7 +164,8 @@ else # Test using the host the script was run on # Provided for backwards compatibility - go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \ + go run test/e2e_node/runner/local/run_local.go \ + --system-spec-name="$system_spec_name" --ginkgo-flags="$ginkgoflags" \ --test-flags="--container-runtime=${runtime} \ --container-runtime-endpoint=${container_runtime_endpoint} \ --image-service-endpoint=${image_service_endpoint} \ diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 088c086c09..29aa63828c 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -673,6 +673,8 @@ system-cgroups system-pods-startup-timeout system-reserved system-reserved-cgroup +system-spec-file +system-spec-name system-validate-mode target-port target-ram-mb diff --git a/test/e2e_node/BUILD b/test/e2e_node/BUILD index 0872b85278..ef01c8c4ec 100644 --- a/test/e2e_node/BUILD +++ b/test/e2e_node/BUILD @@ -140,6 +140,7 @@ go_test( "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/tools/cache:go_default_library", diff --git a/test/e2e_node/conformance/build/Dockerfile b/test/e2e_node/conformance/build/Dockerfile index 7424cc9e72..75d40cf914 100644 --- a/test/e2e_node/conformance/build/Dockerfile +++ b/test/e2e_node/conformance/build/Dockerfile @@ -17,6 +17,9 @@ FROM BASEIMAGE COPY ginkgo /usr/local/bin/ COPY e2e_node.test /usr/local/bin +# This is a placeholder that will be substituted in the Makefile. +COPY_SYSTEM_SPEC_FILE + # The following environment variables can be override when starting the container. # FOCUS is regex matching test to run. By default run all conformance test. # SKIP is regex matching test to skip. By default skip flaky and serial test. @@ -39,4 +42,6 @@ ENTRYPOINT ginkgo --focus="$FOCUS" \ -- --conformance=true \ --prepull-images=false \ --report-dir="$REPORT_PATH" \ + # This is a placeholder that will be substituted in the Makefile. + --system-spec-file=SYSTEM_SPEC_FILE_PATH \ $TEST_ARGS diff --git a/test/e2e_node/conformance/build/Makefile b/test/e2e_node/conformance/build/Makefile index f726c5f363..15022197a3 100644 --- a/test/e2e_node/conformance/build/Makefile +++ b/test/e2e_node/conformance/build/Makefile @@ -17,6 +17,11 @@ # Usage: # [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] [BIN_DIR="../../../../_output/bin"] make (build|push) VERSION={some_version_number e.g. 0.1} +# SYSTEM_SPEC_NAME is the name of the system spec used for the node conformance +# test. The specs are expected to be in SYSTEM_SPEC_DIR. +SYSTEM_SPEC_NAME?= +SYSTEM_SPEC_DIR?=../../system/specs + # TODO(random-liu): Add this into release progress. REGISTRY?=gcr.io/google_containers ARCH?=amd64 @@ -32,6 +37,15 @@ BASEIMAGE_ppc64le=ppc64le/debian:jessie BASEIMAGE?=${BASEIMAGE_${ARCH}} +IMAGE_NAME:=${REGISTRY}/node-test +COPY_SYSTEM_SPEC_FILE= +SYSTEM_SPEC_FILE_PATH= +ifneq ($(strip $(SYSTEM_SPEC_NAME)),) + IMAGE_NAME:=${IMAGE_NAME}-${SYSTEM_SPEC_NAME} + COPY_SYSTEM_SPEC_FILE="'COPY system-spec.yaml /usr/local/etc/'" + SYSTEM_SPEC_FILE_PATH="'/usr/local/etc/system-spec.yaml'" +endif + all: build build: @@ -43,8 +57,14 @@ endif cp ${BIN_DIR}/ginkgo ${TEMP_DIR} cp ${BIN_DIR}/e2e_node.test ${TEMP_DIR} +ifneq ($(strip $(SYSTEM_SPEC_NAME)),) + cp ${SYSTEM_SPEC_DIR}/${SYSTEM_SPEC_NAME}.yaml ${TEMP_DIR}/system-spec.yaml +endif - cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile + cd ${TEMP_DIR} && sed -i.back \ + "s|BASEIMAGE|${BASEIMAGE}|g;\ + s|COPY_SYSTEM_SPEC_FILE|${COPY_SYSTEM_SPEC_FILE}|g;\ + s|SYSTEM_SPEC_FILE_PATH|${SYSTEM_SPEC_FILE_PATH}|g" Dockerfile # Make scripts executable before they are copied into the Docker image. If we make them executable later, in another layer # they'll take up twice the space because the new executable binary differs from the old one, but everything is cached in layers. @@ -52,13 +72,13 @@ endif e2e_node.test \ ginkgo - docker build --pull -t ${REGISTRY}/node-test-${ARCH}:${VERSION} ${TEMP_DIR} + docker build --pull -t ${IMAGE_NAME}-${ARCH}:${VERSION} ${TEMP_DIR} push: build - gcloud docker -- push ${REGISTRY}/node-test-${ARCH}:${VERSION} + gcloud docker -- push ${IMAGE_NAME}-${ARCH}:${VERSION} ifeq ($(ARCH),amd64) - docker tag ${REGISTRY}/node-test-${ARCH}:${VERSION} ${REGISTRY}/node-test:${VERSION} - gcloud docker -- push ${REGISTRY}/node-test:${VERSION} + docker tag ${IMAGE_NAME}-${ARCH}:${VERSION} ${IMAGE_NAME}:${VERSION} + gcloud docker -- push ${IMAGE_NAME}:${VERSION} endif .PHONY: all diff --git a/test/e2e_node/e2e_node_suite_test.go b/test/e2e_node/e2e_node_suite_test.go index a640e6523d..ec52393a59 100644 --- a/test/e2e_node/e2e_node_suite_test.go +++ b/test/e2e_node/e2e_node_suite_test.go @@ -20,6 +20,7 @@ package e2e_node import ( "bytes" + "encoding/json" "flag" "fmt" "io/ioutil" @@ -33,6 +34,7 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" nodeutil "k8s.io/kubernetes/pkg/api/v1/node" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" commontest "k8s.io/kubernetes/test/e2e/common" @@ -55,6 +57,7 @@ var e2es *services.E2EServices var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.") var runKubeletMode = flag.Bool("run-kubelet-mode", false, "If true, only start kubelet, and not run test.") var systemValidateMode = flag.Bool("system-validate-mode", false, "If true, only run system validation in current process, and not run test.") +var systemSpecFile = flag.String("system-spec-file", "", "The name of the system spec file that will be used for node conformance test. If it's unspecified or empty, the default system spec (system.DefaultSysSpec) will be used.") func init() { framework.RegisterCommonFlags() @@ -91,6 +94,14 @@ func TestE2eNode(t *testing.T) { } if *systemValidateMode { // If system-validate-mode is specified, only run system validation in current process. + spec := &system.DefaultSysSpec + if *systemSpecFile != "" { + var err error + spec, err = loadSystemSpecFromFile(*systemSpecFile) + if err != nil { + glog.Exitf("Failed to load system spec: %v", err) + } + } if framework.TestContext.NodeConformance { // Chroot to /rootfs to make system validation can check system // as in the root filesystem. @@ -100,7 +111,7 @@ func TestE2eNode(t *testing.T) { glog.Exitf("chroot %q failed: %v", rootfs, err) } } - if _, err := system.ValidateDefault(framework.TestContext.ContainerRuntime); err != nil { + if _, err := system.ValidateSpec(*spec, framework.TestContext.ContainerRuntime); err != nil { glog.Exitf("system validation failed: %v", err) } return @@ -278,3 +289,21 @@ func getAPIServerClient() (*clientset.Clientset, error) { } return client, nil } + +// loadSystemSpecFromFile returns the system spec from the file with the +// filename. +func loadSystemSpecFromFile(filename string) (*system.SysSpec, error) { + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + data, err := utilyaml.ToJSON(b) + if err != nil { + return nil, err + } + spec := new(system.SysSpec) + if err := json.Unmarshal(data, spec); err != nil { + return nil, err + } + return spec, nil +} diff --git a/test/e2e_node/jenkins/conformance/conformance-jenkins.sh b/test/e2e_node/jenkins/conformance/conformance-jenkins.sh index bcf66d79cb..b4dce50776 100755 --- a/test/e2e_node/jenkins/conformance/conformance-jenkins.sh +++ b/test/e2e_node/jenkins/conformance/conformance-jenkins.sh @@ -39,4 +39,5 @@ go run test/e2e_node/runner/remote/run_remote.go conformance \ --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \ --results-dir="$ARTIFACTS" --test-timeout="$TIMEOUT" \ --test_args="--kubelet-flags=\"$KUBELET_ARGS\"" \ - --instance-metadata="$GCE_INSTANCE_METADATA" + --instance-metadata="$GCE_INSTANCE_METADATA" \ + --system-spec-name="$SYSTEM_SPEC_NAME" diff --git a/test/e2e_node/jenkins/e2e-node-jenkins.sh b/test/e2e_node/jenkins/e2e-node-jenkins.sh index fd2dbef2ff..c0f83e59e7 100755 --- a/test/e2e_node/jenkins/e2e-node-jenkins.sh +++ b/test/e2e_node/jenkins/e2e-node-jenkins.sh @@ -47,4 +47,4 @@ go run test/e2e_node/runner/remote/run_remote.go --logtostderr --vmodule=*=4 \ --image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \ --results-dir="$ARTIFACTS" --ginkgo-flags="--nodes=$PARALLELISM $GINKGO_FLAGS" \ --test-timeout="$TIMEOUT" --test_args="$TEST_ARGS --kubelet-flags=\"$KUBELET_ARGS\"" \ - --instance-metadata="$GCE_INSTANCE_METADATA" + --instance-metadata="$GCE_INSTANCE_METADATA" --system-spec-name="$SYSTEM_SPEC_NAME" diff --git a/test/e2e_node/remote/node_conformance.go b/test/e2e_node/remote/node_conformance.go index f1a955ca40..59e322a77b 100644 --- a/test/e2e_node/remote/node_conformance.go +++ b/test/e2e_node/remote/node_conformance.go @@ -63,13 +63,17 @@ const ( // timestamp is used as an unique id of current test. var timestamp = getTimestamp() -// getConformanceImageRepo returns conformance image full repo name. -func getConformanceImageRepo() string { - return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp) +// getConformanceTestImageName returns name of the conformance test image given the system spec name. +func getConformanceTestImageName(systemSpecName string) string { + if systemSpecName == "" { + return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp) + } else { + return fmt.Sprintf("%s/node-test-%s-%s:%s", conformanceRegistry, systemSpecName, conformanceArch, timestamp) + } } // buildConformanceTest builds node conformance test image tarball into binDir. -func buildConformanceTest(binDir string) error { +func buildConformanceTest(binDir, systemSpecName string) error { // Get node conformance directory. conformancePath, err := getConformanceDirectory() if err != nil { @@ -79,13 +83,14 @@ func buildConformanceTest(binDir string) error { cmd := exec.Command("make", "-C", conformancePath, "BIN_DIR="+binDir, "REGISTRY="+conformanceRegistry, "ARCH="+conformanceArch, - "VERSION="+timestamp) + "VERSION="+timestamp, + "SYSTEM_SPEC_NAME="+systemSpecName) if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to build node conformance docker image: command - %q, error - %v, output - %q", commandToString(cmd), err, output) } // Save docker image into tar file. - cmd = exec.Command("docker", "save", "-o", filepath.Join(binDir, conformanceTarfile), getConformanceImageRepo()) + cmd = exec.Command("docker", "save", "-o", filepath.Join(binDir, conformanceTarfile), getConformanceTestImageName(systemSpecName)) if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to save node conformance docker image into tar file: command - %q, error - %v, output - %q", commandToString(cmd), err, output) @@ -94,7 +99,7 @@ func buildConformanceTest(binDir string) error { } // SetupTestPackage sets up the test package with binaries k8s required for node conformance test -func (c *ConformanceRemote) SetupTestPackage(tardir string) error { +func (c *ConformanceRemote) SetupTestPackage(tardir, systemSpecName string) error { // Build the executables if err := builder.BuildGo(); err != nil { return fmt.Errorf("failed to build the depedencies: %v", err) @@ -107,8 +112,8 @@ func (c *ConformanceRemote) SetupTestPackage(tardir string) error { } // Build node conformance tarball. - if err := buildConformanceTest(buildOutputDir); err != nil { - return fmt.Errorf("failed to build node conformance test %v", err) + if err := buildConformanceTest(buildOutputDir, systemSpecName); err != nil { + return fmt.Errorf("failed to build node conformance test: %v", err) } // Copy files @@ -253,7 +258,7 @@ func stopKubelet(host, workspace string) error { } // RunTest runs test on the node. -func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, _ string, timeout time.Duration) (string, error) { +func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, _, systemSpecName string, timeout time.Duration) (string, error) { // Install the cni plugins and add a basic CNI configuration. if err := setupCNI(host, workspace); err != nil { return "", err @@ -288,7 +293,7 @@ func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFi glog.V(2).Infof("Starting tests on %q", host) podManifestPath := getPodManifestPath(workspace) cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result -e TEST_ARGS=--report-prefix=%s %s'", - timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, getConformanceImageRepo()) + timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, getConformanceTestImageName(systemSpecName)) testOutput, err := SSH(host, "sh", "-c", cmd) if err != nil { return testOutput, err diff --git a/test/e2e_node/remote/node_e2e.go b/test/e2e_node/remote/node_e2e.go index 832896234b..134098077d 100644 --- a/test/e2e_node/remote/node_e2e.go +++ b/test/e2e_node/remote/node_e2e.go @@ -29,7 +29,10 @@ import ( "k8s.io/kubernetes/test/e2e_node/builder" ) -const localCOSMounterPath = "cluster/gce/gci/mounter/mounter" +const ( + localCOSMounterPath = "cluster/gce/gci/mounter/mounter" + systemSpecPath = "test/e2e_node/system/specs" +) // NodeE2ERemote contains the specific functions in the node e2e test suite. type NodeE2ERemote struct{} @@ -40,7 +43,7 @@ func InitNodeE2ERemote() TestSuite { } // SetupTestPackage sets up the test package with binaries k8s required for node e2e tests -func (n *NodeE2ERemote) SetupTestPackage(tardir string) error { +func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error { // Build the executables if err := builder.BuildGo(); err != nil { return fmt.Errorf("failed to build the depedencies: %v", err) @@ -49,7 +52,12 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir string) error { // Make sure we can find the newly built binaries buildOutputDir, err := builder.GetK8sBuildOutputDir() if err != nil { - return fmt.Errorf("failed to locate kubernetes build output directory %v", err) + return fmt.Errorf("failed to locate kubernetes build output directory: %v", err) + } + + rootDir, err := builder.GetK8sRootDir() + if err != nil { + return fmt.Errorf("failed to locate kubernetes root directory: %v", err) } // Copy binaries @@ -65,6 +73,18 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir string) error { } } + if systemSpecName != "" { + // Copy system spec file + source := filepath.Join(rootDir, systemSpecPath, systemSpecName+".yaml") + if _, err := os.Stat(source); err != nil { + return fmt.Errorf("failed to locate system spec %q: %v", source, err) + } + out, err := exec.Command("cp", source, tardir).CombinedOutput() + if err != nil { + return fmt.Errorf("failed to copy system spec %q: %v, output: %q", source, err, out) + } + } + // Include the GCI/COS mounter artifacts in the deployed tarball err = tarAddCOSMounter(tardir) if err != nil { @@ -163,7 +183,7 @@ func updateOSSpecificKubeletFlags(args, host, workspace string) (string, error) } // RunTest runs test on the node. -func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) { +func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName string, timeout time.Duration) (string, error) { // Install the cni plugins and add a basic CNI configuration. if err := setupCNI(host, workspace); err != nil { return "", err @@ -182,12 +202,17 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePr return "", err } + systemSpecFile := "" + if systemSpecName != "" { + systemSpecFile = systemSpecName + ".yaml" + } + // Run the tests glog.V(2).Infof("Starting tests on %q", host) cmd := getSSHCommand(" && ", fmt.Sprintf("cd %s", workspace), - fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=%s %s", - timeout.Seconds(), ginkgoArgs, host, results, junitFilePrefix, imageDesc, testArgs), + fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-file=%s --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=%s %s", + timeout.Seconds(), ginkgoArgs, systemSpecFile, host, results, junitFilePrefix, imageDesc, testArgs), ) return SSH(host, "sh", "-c", cmd) } diff --git a/test/e2e_node/remote/remote.go b/test/e2e_node/remote/remote.go index 7b44839611..721fc30e8f 100644 --- a/test/e2e_node/remote/remote.go +++ b/test/e2e_node/remote/remote.go @@ -34,7 +34,7 @@ var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test resu const archiveName = "e2e_node_test.tar.gz" -func CreateTestArchive(suite TestSuite) (string, error) { +func CreateTestArchive(suite TestSuite, systemSpecName string) (string, error) { glog.V(2).Infof("Building archive...") tardir, err := ioutil.TempDir("", "node-e2e-archive") if err != nil { @@ -43,7 +43,7 @@ func CreateTestArchive(suite TestSuite) (string, error) { defer os.RemoveAll(tardir) // Call the suite function to setup the test package. - err = suite.SetupTestPackage(tardir) + err = suite.SetupTestPackage(tardir, systemSpecName) if err != nil { return "", fmt.Errorf("failed to setup test package %q: %v", tardir, err) } @@ -63,7 +63,7 @@ func CreateTestArchive(suite TestSuite) (string, error) { // Returns the command output, whether the exit was ok, and any errors // TODO(random-liu): junitFilePrefix is not prefix actually, the file name is junit-junitFilePrefix.xml. Change the variable name. -func RunRemote(suite TestSuite, archive string, host string, cleanup bool, imageDesc, junitFilePrefix, testArgs, ginkgoArgs string) (string, bool, error) { +func RunRemote(suite TestSuite, archive string, host string, cleanup bool, imageDesc, junitFilePrefix string, testArgs string, ginkgoArgs string, systemSpecName string) (string, bool, error) { // Create the temp staging directory glog.V(2).Infof("Staging test binaries on %q", host) workspace := fmt.Sprintf("/tmp/node-e2e-%s", getTimestamp()) @@ -108,7 +108,7 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, image } glog.V(2).Infof("Running test on %q", host) - output, err := suite.RunTest(host, workspace, resultDir, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds) + output, err := suite.RunTest(host, workspace, resultDir, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, *testTimeoutSeconds) aggErrs := []error{} // Do not log the output here, let the caller deal with the test output. diff --git a/test/e2e_node/remote/types.go b/test/e2e_node/remote/types.go index a8c0302d0e..110405ac3b 100644 --- a/test/e2e_node/remote/types.go +++ b/test/e2e_node/remote/types.go @@ -29,7 +29,7 @@ type TestSuite interface { // * create a tarball with the directory. // * deploy the tarball to the testing host. // * untar the tarball to the testing workspace on the testing host. - SetupTestPackage(path string) error + SetupTestPackage(path, systemSpecName string) error // RunTest runs test on the node in the given workspace and returns test output // and test error if there is any. // * host is the target node to run the test. @@ -42,6 +42,8 @@ type TestSuite interface { // * junitFilePrefix is the prefix of output junit file. // * testArgs is the arguments passed to test. // * ginkgoArgs is the arguments passed to ginkgo. + // * systemSpecName is the name of the system spec used for validating the + // image on which the test runs. // * timeout is the test timeout. - RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) + RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName string, timeout time.Duration) (string, error) } diff --git a/test/e2e_node/runner/local/run_local.go b/test/e2e_node/runner/local/run_local.go index 3371f53351..be80f28808 100644 --- a/test/e2e_node/runner/local/run_local.go +++ b/test/e2e_node/runner/local/run_local.go @@ -18,6 +18,7 @@ package main import ( "flag" + "fmt" "os" "os/exec" "path/filepath" @@ -31,6 +32,11 @@ import ( var buildDependencies = flag.Bool("build-dependencies", true, "If true, build all dependencies.") var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.") var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.") +var systemSpecName = flag.String("system-spec-name", "", "The name of the system spec used for validating the image in the node conformance test. The specs are at test/e2e_node/system/specs/. If unspecified, the default built-in spec (system.DefaultSpec) will be used.") + +const ( + systemSpecPath = "test/e2e_node/system/specs" +) func main() { flag.Parse() @@ -50,7 +56,17 @@ func main() { glog.Infof("Got build output dir: %v", outputDir) ginkgo := filepath.Join(outputDir, "ginkgo") test := filepath.Join(outputDir, "e2e_node.test") - runCommand(ginkgo, *ginkgoFlags, test, "--", *testFlags) + + if *systemSpecName == "" { + runCommand(ginkgo, *ginkgoFlags, test, "--", *testFlags) + return + } + rootDir, err := builder.GetK8sRootDir() + if err != nil { + glog.Fatalf("Failed to get k8s root directory: %v", err) + } + systemSpecFile := filepath.Join(rootDir, systemSpecPath, *systemSpecName+".yaml") + runCommand(ginkgo, *ginkgoFlags, test, "--", fmt.Sprintf("--system-spec-file=%s", systemSpecFile), *testFlags) return } diff --git a/test/e2e_node/runner/remote/run_remote.go b/test/e2e_node/runner/remote/run_remote.go index d2be8c0901..84db8c6225 100644 --- a/test/e2e_node/runner/remote/run_remote.go +++ b/test/e2e_node/runner/remote/run_remote.go @@ -58,6 +58,7 @@ var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k=2.10.1' +- name: apparmor-profiles + versionRange: '>=2.10.1' +- name: audit + versionRange: '>=2.5.0' +- name: autofs + versionRange: '>=5.0.7' +- name: bash + versionRange: '>=4.3' +- name: bridge-utils + versionRange: '>=1.5' +- name: cloud-init + versionRange: '>=0.7.6' +- name: coreutils + versionRange: '>=8.24' +- name: dbus + versionRange: '>=1.6.8' +- name: e2fsprogs + versionRange: '>=1.4.3' +- name: ebtables + versionRange: '>=2.0.10' +- name: ethtool + versionRange: '>=3.18' +- name: iproute2 + versionRange: '>=4.2.0' +- name: less + versionRange: '>=481' +- name: linux-headers-${KERNEL_RELEASE} +- name: netcat-openbsd + versionRange: '>=1.10' +- name: python + versionRange: '>=2.7.10' +- name: pv + versionRange: '>=1.3.4' +- name: sudo + versionRange: '>=1.8.12' +- name: systemd + versionRange: '>=225' +- name: tar + versionRange: '>=1.28' +- name: util-linux + versionRange: '>=2.27.1' +- name: vim + versionRange: '>=7.4.712' +- name: wget + versionRange: '>=1.18' +- name: gce-compute-image-packages + versionRange: '>=20170227' +# TODO(yguo0905): Figure out whether watchdog is required. + +# packageSpecOverrides contains the OS distro specific package requirements. +packageSpecOverrides: +# The following overrides apply to all Ubuntu images. +- osDistro: ubuntu + subtractions: + - name: apparmor-profiles + description: 'On Ubuntu the apparmor profiles are shipped with individual + application package, so the "apparmor-profiles" package is not required.' + - name: audit + description: 'On Ubuntu the equivalent package is called "auditd", so the + "audit" package is not required and "auditd" exists in the additions.' + - name: wget + description: 'The Ubuntu 1604-xenial image includes wget 1.17.1, which does + not satisfy the spec (>=1.18), but meets the functionality requirements. + Therefore, it is removed from the base spec. See wget in the additions.' + additions: + - name: auditd + versionRange: '>=2.4.5' + description: 'auditd 2.4.5 currently satisfies the requirements because the + GKE features that require auditd 2.5 are not yet available.' + - name: grub-common + versionRange: '>=2.2' + description: 'grub is the bootloader on Ubuntu.' + - name: wget + versionRange: '>=1.17.1' + description: 'wget 1.17.1 satisfies the functionality requirements but does + not meet the spec, which is fine' diff --git a/test/e2e_node/system/types.go b/test/e2e_node/system/types.go index f119e92ff7..07c15f0451 100644 --- a/test/e2e_node/system/types.go +++ b/test/e2e_node/system/types.go @@ -20,16 +20,19 @@ package system type KernelConfig struct { // Name is the general name of the kernel configuration. It is used to // match kernel configuration. - Name string + Name string `json:"name,omitempty"` + // TODO(yguo0905): Support the "or" operation, which will be the same + // as the "aliases". + // // Aliases are aliases of the kernel configuration. Some configuration // has different names in different kernel version. Names of different // versions will be treated as aliases. - Aliases []string + Aliases []string `json:"aliases,omitempty"` // Description is the description of the kernel configuration, for example: // * What is it used for? // * Why is it needed? // * Who needs it? - Description string + Description string `json:"description,omitempty"` } // KernelSpec defines the specification for the kernel. Currently, it contains @@ -38,31 +41,31 @@ type KernelConfig struct { // * Kernel Configuration type KernelSpec struct { // Versions define supported kernel version. It is a group of regexps. - Versions []string + Versions []string `json:"versions,omitempty"` // Required contains all kernel configurations required to be enabled // (built in or as module). - Required []KernelConfig + Required []KernelConfig `json:"required,omitempty"` // Optional contains all kernel configurations are required for optional // features. - Optional []KernelConfig + Optional []KernelConfig `json:"optional,omitempty"` // Forbidden contains all kernel configurations which areforbidden (disabled // or not set) - Forbidden []KernelConfig + Forbidden []KernelConfig `json:"forbidden,omitempty"` } // DockerSpec defines the requirement configuration for docker. Currently, it only // contains spec for graph driver. type DockerSpec struct { // Version is a group of regex matching supported docker versions. - Version []string + Version []string `json:"version,omitempty"` // GraphDriver is the graph drivers supported by kubelet. - GraphDriver []string + GraphDriver []string `json:"graphDriver,omitempty"` } // RuntimeSpec is the abstract layer for different runtimes. Different runtimes // should put their spec inside the RuntimeSpec. type RuntimeSpec struct { - *DockerSpec + *DockerSpec `json:",inline"` } // PackageSpec defines the required packages and their versions. @@ -72,7 +75,7 @@ type RuntimeSpec struct { // either "foo (>=1.0)" or "bar (>=2.0)" is required. type PackageSpec struct { // Name is the name of the package to be checked. - Name string + Name string `json:"name,omitempty"` // VersionRange represents a range of versions that the package must // satisfy. Note that the version requirement will not be enforced if // the version range is empty. For example, @@ -81,9 +84,11 @@ type PackageSpec struct { // - ">1.0 <2.0" would match between both ranges, so "1.1.1" and "1.8.7" // but not "1.0.0" or "2.0.0". // - "<2.0.0 || >=3.0.0" would match "1.0.0" and "3.0.0" but not "2.0.0". - VersionRange string + VersionRange string `json:"versionRange,omitempty"` // Description explains the reason behind this package requirements. - Description string + // + // TODO(yguo0905): Print the description where necessary. + Description string `json:"description,omitempty"` } // PackageSpecOverride defines the overrides on the PackageSpec for an OS @@ -91,31 +96,31 @@ type PackageSpec struct { type PackageSpecOverride struct { // OSDistro identifies to which OS distro this override applies. // Must be "ubuntu", "cos" or "coreos". - OSDistro string + OSDistro string `json:"osDistro,omitempty"` // Subtractions is a list of package names that are excluded from the // package spec. - Subtractions []PackageSpec + Subtractions []PackageSpec `json:"subtractions,omitempty"` // Additions is a list of additional package requirements included the // package spec. - Additions []PackageSpec + Additions []PackageSpec `json:"additions,omitempty"` } // SysSpec defines the requirement of supported system. Currently, it only contains // spec for OS, Kernel and Cgroups. type SysSpec struct { // OS is the operating system of the SysSpec. - OS string + OS string `json:"os,omitempty"` // KernelConfig defines the spec for kernel. - KernelSpec KernelSpec + KernelSpec KernelSpec `json:"kernelSpec,omitempty"` // Cgroups is the required cgroups. - Cgroups []string + Cgroups []string `json:"cgroups,omitempty"` // RuntimeSpec defines the spec for runtime. - RuntimeSpec RuntimeSpec + RuntimeSpec RuntimeSpec `json:"runtimeSpec,omitempty"` // PackageSpec defines the required packages and their versions. - PackageSpecs []PackageSpec + PackageSpecs []PackageSpec `json:"packageSpecs,omitempty"` // PackageSpec defines the overrides of the required packages and their // versions for an OS distro. - PackageSpecOverrides []PackageSpecOverride + PackageSpecOverrides []PackageSpecOverride `json:"packageSpecOverrides,omitempty"` } // DefaultSysSpec is the default SysSpec. diff --git a/test/e2e_node/system/validators.go b/test/e2e_node/system/validators.go index 8f6d9c1c41..7819da7b4a 100644 --- a/test/e2e_node/system/validators.go +++ b/test/e2e_node/system/validators.go @@ -49,8 +49,8 @@ func Validate(spec SysSpec, validators []Validator) (error, error) { return errors.NewAggregate(warns), errors.NewAggregate(errs) } -// ValidateDefault uses all default validators to validate the system and writes to stdout. -func ValidateDefault(runtime string) (error, error) { +// ValidateSpec uses all default validators to validate the system and writes to stdout. +func ValidateSpec(spec SysSpec, runtime string) (error, error) { // OS-level validators. var osValidators = []Validator{ &OSValidator{Reporter: DefaultReporter}, @@ -68,5 +68,5 @@ func ValidateDefault(runtime string) (error, error) { case "docker": validators = append(validators, dockerValidators...) } - return Validate(DefaultSysSpec, validators) + return Validate(spec, validators) }