Supports customized system spec in the node conformance test and creates the GKE system spec

pull/6/head
Yang Guo 2017-06-02 17:00:46 -07:00
parent cb712e41d4
commit 22c9e23202
18 changed files with 452 additions and 63 deletions

View File

@ -240,6 +240,11 @@ define TEST_E2E_NODE_HELP_INFO
# IMAGE_SERVICE_ENDPOINT: remote image endpoint to connect to, to prepull images.
# Used when RUNTIME is set to "remote".
# IMAGE_CONFIG_FILE: path to a file containing image configuration.
# SYSTEM_SPEC_NAME: The name of the system spec to be used for validating the
# image in the node conformance test. The specs are located at
# test/e2e_node/system/specs/. For example, "SYSTEM_SPEC_NAME=gke" will use
# the spec at test/e2e_node/system/specs/gke.yaml. If unspecified, the
# default built-in spec (system.DefaultSpec) will be used.
#
# Example:
# make test-e2e-node FOCUS=Kubelet SKIP=container

View File

@ -33,6 +33,7 @@ container_runtime_endpoint=${CONTAINER_RUNTIME_ENDPOINT:-""}
image_service_endpoint=${IMAGE_SERVICE_ENDPOINT:-""}
run_until_failure=${RUN_UNTIL_FAILURE:-"false"}
test_args=${TEST_ARGS:-""}
system_spec_name=${SYSTEM_SPEC_NAME:-}
# Parse the flags to pass to ginkgo
ginkgoflags=""
@ -135,7 +136,7 @@ if [ $remote = true ] ; then
--results-dir="$artifacts" --ginkgo-flags="$ginkgoflags" \
--image-project="$image_project" --instance-name-prefix="$instance_prefix" \
--delete-instances="$delete_instances" --test_args="$test_args" --instance-metadata="$metadata" \
--image-config-file="$image_config_file" \
--image-config-file="$image_config_file" --system-spec-name="$system_spec_name" \
2>&1 | tee -i "${artifacts}/build-log.txt"
exit $?
@ -163,7 +164,8 @@ else
# Test using the host the script was run on
# Provided for backwards compatibility
go run test/e2e_node/runner/local/run_local.go --ginkgo-flags="$ginkgoflags" \
go run test/e2e_node/runner/local/run_local.go \
--system-spec-name="$system_spec_name" --ginkgo-flags="$ginkgoflags" \
--test-flags="--container-runtime=${runtime} \
--container-runtime-endpoint=${container_runtime_endpoint} \
--image-service-endpoint=${image_service_endpoint} \

View File

@ -673,6 +673,8 @@ system-cgroups
system-pods-startup-timeout
system-reserved
system-reserved-cgroup
system-spec-file
system-spec-name
system-validate-mode
target-port
target-ram-mb

View File

@ -140,6 +140,7 @@ go_test(
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",

View File

@ -17,6 +17,9 @@ FROM BASEIMAGE
COPY ginkgo /usr/local/bin/
COPY e2e_node.test /usr/local/bin
# This is a placeholder that will be substituted in the Makefile.
COPY_SYSTEM_SPEC_FILE
# The following environment variables can be override when starting the container.
# FOCUS is regex matching test to run. By default run all conformance test.
# SKIP is regex matching test to skip. By default skip flaky and serial test.
@ -39,4 +42,6 @@ ENTRYPOINT ginkgo --focus="$FOCUS" \
-- --conformance=true \
--prepull-images=false \
--report-dir="$REPORT_PATH" \
# This is a placeholder that will be substituted in the Makefile.
--system-spec-file=SYSTEM_SPEC_FILE_PATH \
$TEST_ARGS

View File

@ -17,6 +17,11 @@
# Usage:
# [ARCH=amd64] [REGISTRY="gcr.io/google_containers"] [BIN_DIR="../../../../_output/bin"] make (build|push) VERSION={some_version_number e.g. 0.1}
# SYSTEM_SPEC_NAME is the name of the system spec used for the node conformance
# test. The specs are expected to be in SYSTEM_SPEC_DIR.
SYSTEM_SPEC_NAME?=
SYSTEM_SPEC_DIR?=../../system/specs
# TODO(random-liu): Add this into release progress.
REGISTRY?=gcr.io/google_containers
ARCH?=amd64
@ -32,6 +37,15 @@ BASEIMAGE_ppc64le=ppc64le/debian:jessie
BASEIMAGE?=${BASEIMAGE_${ARCH}}
IMAGE_NAME:=${REGISTRY}/node-test
COPY_SYSTEM_SPEC_FILE=
SYSTEM_SPEC_FILE_PATH=
ifneq ($(strip $(SYSTEM_SPEC_NAME)),)
IMAGE_NAME:=${IMAGE_NAME}-${SYSTEM_SPEC_NAME}
COPY_SYSTEM_SPEC_FILE="'COPY system-spec.yaml /usr/local/etc/'"
SYSTEM_SPEC_FILE_PATH="'/usr/local/etc/system-spec.yaml'"
endif
all: build
build:
@ -43,8 +57,14 @@ endif
cp ${BIN_DIR}/ginkgo ${TEMP_DIR}
cp ${BIN_DIR}/e2e_node.test ${TEMP_DIR}
ifneq ($(strip $(SYSTEM_SPEC_NAME)),)
cp ${SYSTEM_SPEC_DIR}/${SYSTEM_SPEC_NAME}.yaml ${TEMP_DIR}/system-spec.yaml
endif
cd ${TEMP_DIR} && sed -i.back "s|BASEIMAGE|${BASEIMAGE}|g" Dockerfile
cd ${TEMP_DIR} && sed -i.back \
"s|BASEIMAGE|${BASEIMAGE}|g;\
s|COPY_SYSTEM_SPEC_FILE|${COPY_SYSTEM_SPEC_FILE}|g;\
s|SYSTEM_SPEC_FILE_PATH|${SYSTEM_SPEC_FILE_PATH}|g" Dockerfile
# Make scripts executable before they are copied into the Docker image. If we make them executable later, in another layer
# they'll take up twice the space because the new executable binary differs from the old one, but everything is cached in layers.
@ -52,13 +72,13 @@ endif
e2e_node.test \
ginkgo
docker build --pull -t ${REGISTRY}/node-test-${ARCH}:${VERSION} ${TEMP_DIR}
docker build --pull -t ${IMAGE_NAME}-${ARCH}:${VERSION} ${TEMP_DIR}
push: build
gcloud docker -- push ${REGISTRY}/node-test-${ARCH}:${VERSION}
gcloud docker -- push ${IMAGE_NAME}-${ARCH}:${VERSION}
ifeq ($(ARCH),amd64)
docker tag ${REGISTRY}/node-test-${ARCH}:${VERSION} ${REGISTRY}/node-test:${VERSION}
gcloud docker -- push ${REGISTRY}/node-test:${VERSION}
docker tag ${IMAGE_NAME}-${ARCH}:${VERSION} ${IMAGE_NAME}:${VERSION}
gcloud docker -- push ${IMAGE_NAME}:${VERSION}
endif
.PHONY: all

View File

@ -20,6 +20,7 @@ package e2e_node
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
@ -33,6 +34,7 @@ import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilyaml "k8s.io/apimachinery/pkg/util/yaml"
nodeutil "k8s.io/kubernetes/pkg/api/v1/node"
"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
commontest "k8s.io/kubernetes/test/e2e/common"
@ -55,6 +57,7 @@ var e2es *services.E2EServices
var runServicesMode = flag.Bool("run-services-mode", false, "If true, only run services (etcd, apiserver) in current process, and not run test.")
var runKubeletMode = flag.Bool("run-kubelet-mode", false, "If true, only start kubelet, and not run test.")
var systemValidateMode = flag.Bool("system-validate-mode", false, "If true, only run system validation in current process, and not run test.")
var systemSpecFile = flag.String("system-spec-file", "", "The name of the system spec file that will be used for node conformance test. If it's unspecified or empty, the default system spec (system.DefaultSysSpec) will be used.")
func init() {
framework.RegisterCommonFlags()
@ -91,6 +94,14 @@ func TestE2eNode(t *testing.T) {
}
if *systemValidateMode {
// If system-validate-mode is specified, only run system validation in current process.
spec := &system.DefaultSysSpec
if *systemSpecFile != "" {
var err error
spec, err = loadSystemSpecFromFile(*systemSpecFile)
if err != nil {
glog.Exitf("Failed to load system spec: %v", err)
}
}
if framework.TestContext.NodeConformance {
// Chroot to /rootfs to make system validation can check system
// as in the root filesystem.
@ -100,7 +111,7 @@ func TestE2eNode(t *testing.T) {
glog.Exitf("chroot %q failed: %v", rootfs, err)
}
}
if _, err := system.ValidateDefault(framework.TestContext.ContainerRuntime); err != nil {
if _, err := system.ValidateSpec(*spec, framework.TestContext.ContainerRuntime); err != nil {
glog.Exitf("system validation failed: %v", err)
}
return
@ -278,3 +289,21 @@ func getAPIServerClient() (*clientset.Clientset, error) {
}
return client, nil
}
// loadSystemSpecFromFile returns the system spec from the file with the
// filename.
func loadSystemSpecFromFile(filename string) (*system.SysSpec, error) {
b, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
data, err := utilyaml.ToJSON(b)
if err != nil {
return nil, err
}
spec := new(system.SysSpec)
if err := json.Unmarshal(data, spec); err != nil {
return nil, err
}
return spec, nil
}

View File

@ -39,4 +39,5 @@ go run test/e2e_node/runner/remote/run_remote.go conformance \
--image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \
--results-dir="$ARTIFACTS" --test-timeout="$TIMEOUT" \
--test_args="--kubelet-flags=\"$KUBELET_ARGS\"" \
--instance-metadata="$GCE_INSTANCE_METADATA"
--instance-metadata="$GCE_INSTANCE_METADATA" \
--system-spec-name="$SYSTEM_SPEC_NAME"

View File

@ -47,4 +47,4 @@ go run test/e2e_node/runner/remote/run_remote.go --logtostderr --vmodule=*=4 \
--image-config-file="$GCE_IMAGE_CONFIG_PATH" --cleanup="$CLEANUP" \
--results-dir="$ARTIFACTS" --ginkgo-flags="--nodes=$PARALLELISM $GINKGO_FLAGS" \
--test-timeout="$TIMEOUT" --test_args="$TEST_ARGS --kubelet-flags=\"$KUBELET_ARGS\"" \
--instance-metadata="$GCE_INSTANCE_METADATA"
--instance-metadata="$GCE_INSTANCE_METADATA" --system-spec-name="$SYSTEM_SPEC_NAME"

View File

@ -63,13 +63,17 @@ const (
// timestamp is used as an unique id of current test.
var timestamp = getTimestamp()
// getConformanceImageRepo returns conformance image full repo name.
func getConformanceImageRepo() string {
return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp)
// getConformanceTestImageName returns name of the conformance test image given the system spec name.
func getConformanceTestImageName(systemSpecName string) string {
if systemSpecName == "" {
return fmt.Sprintf("%s/node-test-%s:%s", conformanceRegistry, conformanceArch, timestamp)
} else {
return fmt.Sprintf("%s/node-test-%s-%s:%s", conformanceRegistry, systemSpecName, conformanceArch, timestamp)
}
}
// buildConformanceTest builds node conformance test image tarball into binDir.
func buildConformanceTest(binDir string) error {
func buildConformanceTest(binDir, systemSpecName string) error {
// Get node conformance directory.
conformancePath, err := getConformanceDirectory()
if err != nil {
@ -79,13 +83,14 @@ func buildConformanceTest(binDir string) error {
cmd := exec.Command("make", "-C", conformancePath, "BIN_DIR="+binDir,
"REGISTRY="+conformanceRegistry,
"ARCH="+conformanceArch,
"VERSION="+timestamp)
"VERSION="+timestamp,
"SYSTEM_SPEC_NAME="+systemSpecName)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to build node conformance docker image: command - %q, error - %v, output - %q",
commandToString(cmd), err, output)
}
// Save docker image into tar file.
cmd = exec.Command("docker", "save", "-o", filepath.Join(binDir, conformanceTarfile), getConformanceImageRepo())
cmd = exec.Command("docker", "save", "-o", filepath.Join(binDir, conformanceTarfile), getConformanceTestImageName(systemSpecName))
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to save node conformance docker image into tar file: command - %q, error - %v, output - %q",
commandToString(cmd), err, output)
@ -94,7 +99,7 @@ func buildConformanceTest(binDir string) error {
}
// SetupTestPackage sets up the test package with binaries k8s required for node conformance test
func (c *ConformanceRemote) SetupTestPackage(tardir string) error {
func (c *ConformanceRemote) SetupTestPackage(tardir, systemSpecName string) error {
// Build the executables
if err := builder.BuildGo(); err != nil {
return fmt.Errorf("failed to build the depedencies: %v", err)
@ -107,8 +112,8 @@ func (c *ConformanceRemote) SetupTestPackage(tardir string) error {
}
// Build node conformance tarball.
if err := buildConformanceTest(buildOutputDir); err != nil {
return fmt.Errorf("failed to build node conformance test %v", err)
if err := buildConformanceTest(buildOutputDir, systemSpecName); err != nil {
return fmt.Errorf("failed to build node conformance test: %v", err)
}
// Copy files
@ -253,7 +258,7 @@ func stopKubelet(host, workspace string) error {
}
// RunTest runs test on the node.
func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, _ string, timeout time.Duration) (string, error) {
func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, _, systemSpecName string, timeout time.Duration) (string, error) {
// Install the cni plugins and add a basic CNI configuration.
if err := setupCNI(host, workspace); err != nil {
return "", err
@ -288,7 +293,7 @@ func (c *ConformanceRemote) RunTest(host, workspace, results, imageDesc, junitFi
glog.V(2).Infof("Starting tests on %q", host)
podManifestPath := getPodManifestPath(workspace)
cmd := fmt.Sprintf("'timeout -k 30s %fs docker run --rm --privileged=true --net=host -v /:/rootfs -v %s:%s -v %s:/var/result -e TEST_ARGS=--report-prefix=%s %s'",
timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, getConformanceImageRepo())
timeout.Seconds(), podManifestPath, podManifestPath, results, junitFilePrefix, getConformanceTestImageName(systemSpecName))
testOutput, err := SSH(host, "sh", "-c", cmd)
if err != nil {
return testOutput, err

View File

@ -29,7 +29,10 @@ import (
"k8s.io/kubernetes/test/e2e_node/builder"
)
const localCOSMounterPath = "cluster/gce/gci/mounter/mounter"
const (
localCOSMounterPath = "cluster/gce/gci/mounter/mounter"
systemSpecPath = "test/e2e_node/system/specs"
)
// NodeE2ERemote contains the specific functions in the node e2e test suite.
type NodeE2ERemote struct{}
@ -40,7 +43,7 @@ func InitNodeE2ERemote() TestSuite {
}
// SetupTestPackage sets up the test package with binaries k8s required for node e2e tests
func (n *NodeE2ERemote) SetupTestPackage(tardir string) error {
func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
// Build the executables
if err := builder.BuildGo(); err != nil {
return fmt.Errorf("failed to build the depedencies: %v", err)
@ -49,7 +52,12 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir string) error {
// Make sure we can find the newly built binaries
buildOutputDir, err := builder.GetK8sBuildOutputDir()
if err != nil {
return fmt.Errorf("failed to locate kubernetes build output directory %v", err)
return fmt.Errorf("failed to locate kubernetes build output directory: %v", err)
}
rootDir, err := builder.GetK8sRootDir()
if err != nil {
return fmt.Errorf("failed to locate kubernetes root directory: %v", err)
}
// Copy binaries
@ -65,6 +73,18 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir string) error {
}
}
if systemSpecName != "" {
// Copy system spec file
source := filepath.Join(rootDir, systemSpecPath, systemSpecName+".yaml")
if _, err := os.Stat(source); err != nil {
return fmt.Errorf("failed to locate system spec %q: %v", source, err)
}
out, err := exec.Command("cp", source, tardir).CombinedOutput()
if err != nil {
return fmt.Errorf("failed to copy system spec %q: %v, output: %q", source, err, out)
}
}
// Include the GCI/COS mounter artifacts in the deployed tarball
err = tarAddCOSMounter(tardir)
if err != nil {
@ -163,7 +183,7 @@ func updateOSSpecificKubeletFlags(args, host, workspace string) (string, error)
}
// RunTest runs test on the node.
func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error) {
func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName string, timeout time.Duration) (string, error) {
// Install the cni plugins and add a basic CNI configuration.
if err := setupCNI(host, workspace); err != nil {
return "", err
@ -182,12 +202,17 @@ func (n *NodeE2ERemote) RunTest(host, workspace, results, imageDesc, junitFilePr
return "", err
}
systemSpecFile := ""
if systemSpecName != "" {
systemSpecFile = systemSpecName + ".yaml"
}
// Run the tests
glog.V(2).Infof("Starting tests on %q", host)
cmd := getSSHCommand(" && ",
fmt.Sprintf("cd %s", workspace),
fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=%s %s",
timeout.Seconds(), ginkgoArgs, host, results, junitFilePrefix, imageDesc, testArgs),
fmt.Sprintf("timeout -k 30s %fs ./ginkgo %s ./e2e_node.test -- --system-spec-file=%s --logtostderr --v 4 --node-name=%s --report-dir=%s --report-prefix=%s --image-description=%s %s",
timeout.Seconds(), ginkgoArgs, systemSpecFile, host, results, junitFilePrefix, imageDesc, testArgs),
)
return SSH(host, "sh", "-c", cmd)
}

View File

@ -34,7 +34,7 @@ var resultsDir = flag.String("results-dir", "/tmp/", "Directory to scp test resu
const archiveName = "e2e_node_test.tar.gz"
func CreateTestArchive(suite TestSuite) (string, error) {
func CreateTestArchive(suite TestSuite, systemSpecName string) (string, error) {
glog.V(2).Infof("Building archive...")
tardir, err := ioutil.TempDir("", "node-e2e-archive")
if err != nil {
@ -43,7 +43,7 @@ func CreateTestArchive(suite TestSuite) (string, error) {
defer os.RemoveAll(tardir)
// Call the suite function to setup the test package.
err = suite.SetupTestPackage(tardir)
err = suite.SetupTestPackage(tardir, systemSpecName)
if err != nil {
return "", fmt.Errorf("failed to setup test package %q: %v", tardir, err)
}
@ -63,7 +63,7 @@ func CreateTestArchive(suite TestSuite) (string, error) {
// Returns the command output, whether the exit was ok, and any errors
// TODO(random-liu): junitFilePrefix is not prefix actually, the file name is junit-junitFilePrefix.xml. Change the variable name.
func RunRemote(suite TestSuite, archive string, host string, cleanup bool, imageDesc, junitFilePrefix, testArgs, ginkgoArgs string) (string, bool, error) {
func RunRemote(suite TestSuite, archive string, host string, cleanup bool, imageDesc, junitFilePrefix string, testArgs string, ginkgoArgs string, systemSpecName string) (string, bool, error) {
// Create the temp staging directory
glog.V(2).Infof("Staging test binaries on %q", host)
workspace := fmt.Sprintf("/tmp/node-e2e-%s", getTimestamp())
@ -108,7 +108,7 @@ func RunRemote(suite TestSuite, archive string, host string, cleanup bool, image
}
glog.V(2).Infof("Running test on %q", host)
output, err := suite.RunTest(host, workspace, resultDir, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, *testTimeoutSeconds)
output, err := suite.RunTest(host, workspace, resultDir, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName, *testTimeoutSeconds)
aggErrs := []error{}
// Do not log the output here, let the caller deal with the test output.

View File

@ -29,7 +29,7 @@ type TestSuite interface {
// * create a tarball with the directory.
// * deploy the tarball to the testing host.
// * untar the tarball to the testing workspace on the testing host.
SetupTestPackage(path string) error
SetupTestPackage(path, systemSpecName string) error
// RunTest runs test on the node in the given workspace and returns test output
// and test error if there is any.
// * host is the target node to run the test.
@ -42,6 +42,8 @@ type TestSuite interface {
// * junitFilePrefix is the prefix of output junit file.
// * testArgs is the arguments passed to test.
// * ginkgoArgs is the arguments passed to ginkgo.
// * systemSpecName is the name of the system spec used for validating the
// image on which the test runs.
// * timeout is the test timeout.
RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs string, timeout time.Duration) (string, error)
RunTest(host, workspace, results, imageDesc, junitFilePrefix, testArgs, ginkgoArgs, systemSpecName string, timeout time.Duration) (string, error)
}

View File

@ -18,6 +18,7 @@ package main
import (
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
@ -31,6 +32,11 @@ import (
var buildDependencies = flag.Bool("build-dependencies", true, "If true, build all dependencies.")
var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.")
var systemSpecName = flag.String("system-spec-name", "", "The name of the system spec used for validating the image in the node conformance test. The specs are at test/e2e_node/system/specs/. If unspecified, the default built-in spec (system.DefaultSpec) will be used.")
const (
systemSpecPath = "test/e2e_node/system/specs"
)
func main() {
flag.Parse()
@ -50,7 +56,17 @@ func main() {
glog.Infof("Got build output dir: %v", outputDir)
ginkgo := filepath.Join(outputDir, "ginkgo")
test := filepath.Join(outputDir, "e2e_node.test")
runCommand(ginkgo, *ginkgoFlags, test, "--", *testFlags)
if *systemSpecName == "" {
runCommand(ginkgo, *ginkgoFlags, test, "--", *testFlags)
return
}
rootDir, err := builder.GetK8sRootDir()
if err != nil {
glog.Fatalf("Failed to get k8s root directory: %v", err)
}
systemSpecFile := filepath.Join(rootDir, systemSpecPath, *systemSpecName+".yaml")
runCommand(ginkgo, *ginkgoFlags, test, "--", fmt.Sprintf("--system-spec-file=%s", systemSpecFile), *testFlags)
return
}

View File

@ -58,6 +58,7 @@ var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar
var instanceMetadata = flag.String("instance-metadata", "", "key/value metadata for instances separated by '=' or '<', 'k=v' means the key is 'k' and the value is 'v'; 'k<p' means the key is 'k' and the value is extracted from the local path 'p', e.g. k1=v1,k2<p2")
var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs")
var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.")
var systemSpecName = flag.String("system-spec-name", "", "The name of the system spec used for validating the image in the node conformance test. The specs are at test/e2e_node/system/specs/. If unspecified, the default built-in spec (system.DefaultSpec) will be used.")
const (
defaultMachine = "n1-standard-1"
@ -163,7 +164,7 @@ func main() {
rand.Seed(time.Now().UTC().UnixNano())
if *buildOnly {
// Build the archive and exit
remote.CreateTestArchive(suite)
remote.CreateTestArchive(suite, *systemSpecName)
return
}
@ -336,7 +337,7 @@ func callGubernator(gubernator bool) {
}
func (a *Archive) getArchive() (string, error) {
a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite) })
a.Do(func() { a.path, a.err = remote.CreateTestArchive(suite, *systemSpecName) })
return a.path, a.err
}
@ -394,11 +395,11 @@ func testHost(host string, deleteFiles bool, imageDesc, junitFilePrefix, ginkgoF
if err != nil {
// Don't log fatal because we need to do any needed cleanup contained in "defer" statements
return &TestResult{
err: fmt.Errorf("unable to create test archive %v.", err),
err: fmt.Errorf("unable to create test archive: %v.", err),
}
}
output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, imageDesc, junitFilePrefix, *testArgs, ginkgoFlagsStr)
output, exitOk, err := remote.RunRemote(suite, path, host, deleteFiles, imageDesc, junitFilePrefix, *testArgs, ginkgoFlagsStr, *systemSpecName)
return &TestResult{
output: output,
err: err,

View File

@ -0,0 +1,270 @@
# This is the system spec that must be satisfied by the images running on GKE.
os: Linux
kernelSpec:
versions:
# GKE requires kernel version 4.4+.
- 4\.[4-9].*
# Required kernel configurations -- the configuration must be set to "y" or
# "m".
required:
# The configurations required by virtual machine or cloud provider.
- name: BOOTPARAM_HARDLOCKUP_PANIC
description: 'Enable the kernel to panic on "hard lockups".'
- name: BOOTPARAM_SOFTLOCKUP_PANIC
description: 'Enable the kernel to panic on "soft lockups".'
- name: PANIC_ON_OOPS
description: 'Enable the kernel to panic when it oops.'
- name: PVPANIC
description: 'Enable the VM (guest) to communicate panic events with the
host.'
- name: DMIID
description: 'Make sure /sys/class/dmi is exported - cAdvisor currently
uses this to determine which the cloud provider it is: aws, azure, or
gce, etc'
- name: ACPI_BUTTON
description: 'Enable the software-controlled power management, and required
by reset or stop button of GCE console.'
# The configurations required by network.
- name: INET
description: 'Enable TCP/IP networking.'
- name: VXLAN
description: 'Required by the overlay networking in Kubernetes.'
- name: IP_SET
description: 'Required by Kubernetes network policy.'
- name: IP_SET_HASH_IP
description: 'This introduces hash:ip set type support, which is required
by Kubernetes Calico networking.'
- name: IPVLAN
description: 'Required by IPVLAN feature.'
- name: IPV6
description: 'Required by IPVLAN feature.'
- name: IP6_NF_IPTABLES
description: 'Required by kube-proxy.'
- name: IP_NF_TARGET_REDIRECT
alias:
- NETFILTER_XT_TARGET_REDIRECT
description: 'Enabled REDIRECT: all incoming connections are mapped onto
the incoming interface''s address, causing the packets to come to the
local machine instead of passing through. This is required by
kube-proxy.'
- name: NETFILTER_XT_MATCH_COMMENT
description: 'This option adds a "comment" dummy-match, which allows you to
put comments in your iptables ruleset. Today''s kube-proxy implementation
depends on this feature.'
# This is not critical, but debian-based container-vm kernel module study
# shows that many customers' nodes have loaded those kernel modules. We
# suspect sysdig module depends on these set of kernel modules for
# monitoring.
- name: PACKET_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: UNIX_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: INET_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: INET_TCP_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: INET_UDP_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
- name: NETLINK_DIAG
description: 'Required by ss (similar to netstat) tools to display Linux
TCP / UDP network and socket information.'
# The configurations are required by filesystem.
- name: EXT4_FS
- name: DEBUG_FS
- name: PROC_FS
- name: XFS_FS
- name: SCSI_PROC_FS
# Currently Kubelet supports three docker graph drivers: overlay, aufs, and
# devicemapper due to the legacy reason. But for GKE, we plan to only support
# overlayfs.
- name: OVERLAY_FS
description: 'Enable OverlayFS, which will be the only docker graph driver
supported on GKE.'
- name: NFS_FS
description: 'Required by NFS support.'
- name: AUTOFS4_FS
description: 'Required by NFS support.'
- name: NFS_FSCACHE
description: 'Required by NFS support.'
- name: FSCACHE
description: 'Required by NFS support.'
- name: CACHEFILES
description: 'Required by NFS support.'
- name: FUSE_FS
description: 'Required by GlusterFS support.'
- name: BCACHE
# TODO(yguo0905): Add a description for BCACHE.
# The configuration required by the resource isolation, accounting, and
# management.
- name: NAMESPACES
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: IPC_NS
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: NET_NS
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: PID_NS
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: UTS_NS
description: 'Required by kubelet and docker. Enabling it allows the
processes within a pod or a container to have their own view of the
system.'
- name: CGROUPS
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: CGROUP_CPUACCT
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: CGROUP_DEVICE
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: CGROUP_SCHED
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: CPUSETS
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: MEMCG
description: 'Required by kubelet and docker. The resource usage of the
processes within a pod or a container can be monitored, accounted, and
controlled.'
- name: QUOTA
description: 'Required by kubelet to have an accurate and efficient disk
space and inode accounting, and eventually to limit the usage.'
# The security-related configurations
- name: SECCOMP
description: 'Enabled the SECCOMP application API.'
- name: SECURITY_APPARMOR
description: 'Enable for AppArmor support.'
- name: CC_STACKPROTECTOR_STRONG
alias:
- CONFIG_CC_STACKPROTECTOR_REGULAR
CONFIG_CC_STACKPROTECTOR_ALL
description: 'Add the stack buffer overflow protections.'
- name: STRICT_DEVMEM
description: 'Required for blocking the direct physical memory access.'
- name: IMA
description: 'Required for security-related logging and auditing.'
- name: AUDIT
description: 'Required for security-related logging and auditing.'
- name: AUDITSYSCALL
description: 'Required for security-related logging and auditing.'
# Misc. configurations
- name: MODULES
description: 'Required for loadable module support.'
- name: PRINTK
description: 'Required for kernel logging message.'
- name: MMU
description: 'Required for memory management hardware and mmap() system
call.'
packageSpecs:
- name: apparmor
versionRange: '>=2.10.1'
- name: apparmor-profiles
versionRange: '>=2.10.1'
- name: audit
versionRange: '>=2.5.0'
- name: autofs
versionRange: '>=5.0.7'
- name: bash
versionRange: '>=4.3'
- name: bridge-utils
versionRange: '>=1.5'
- name: cloud-init
versionRange: '>=0.7.6'
- name: coreutils
versionRange: '>=8.24'
- name: dbus
versionRange: '>=1.6.8'
- name: e2fsprogs
versionRange: '>=1.4.3'
- name: ebtables
versionRange: '>=2.0.10'
- name: ethtool
versionRange: '>=3.18'
- name: iproute2
versionRange: '>=4.2.0'
- name: less
versionRange: '>=481'
- name: linux-headers-${KERNEL_RELEASE}
- name: netcat-openbsd
versionRange: '>=1.10'
- name: python
versionRange: '>=2.7.10'
- name: pv
versionRange: '>=1.3.4'
- name: sudo
versionRange: '>=1.8.12'
- name: systemd
versionRange: '>=225'
- name: tar
versionRange: '>=1.28'
- name: util-linux
versionRange: '>=2.27.1'
- name: vim
versionRange: '>=7.4.712'
- name: wget
versionRange: '>=1.18'
- name: gce-compute-image-packages
versionRange: '>=20170227'
# TODO(yguo0905): Figure out whether watchdog is required.
# packageSpecOverrides contains the OS distro specific package requirements.
packageSpecOverrides:
# The following overrides apply to all Ubuntu images.
- osDistro: ubuntu
subtractions:
- name: apparmor-profiles
description: 'On Ubuntu the apparmor profiles are shipped with individual
application package, so the "apparmor-profiles" package is not required.'
- name: audit
description: 'On Ubuntu the equivalent package is called "auditd", so the
"audit" package is not required and "auditd" exists in the additions.'
- name: wget
description: 'The Ubuntu 1604-xenial image includes wget 1.17.1, which does
not satisfy the spec (>=1.18), but meets the functionality requirements.
Therefore, it is removed from the base spec. See wget in the additions.'
additions:
- name: auditd
versionRange: '>=2.4.5'
description: 'auditd 2.4.5 currently satisfies the requirements because the
GKE features that require auditd 2.5 are not yet available.'
- name: grub-common
versionRange: '>=2.2'
description: 'grub is the bootloader on Ubuntu.'
- name: wget
versionRange: '>=1.17.1'
description: 'wget 1.17.1 satisfies the functionality requirements but does
not meet the spec, which is fine'

View File

@ -20,16 +20,19 @@ package system
type KernelConfig struct {
// Name is the general name of the kernel configuration. It is used to
// match kernel configuration.
Name string
Name string `json:"name,omitempty"`
// TODO(yguo0905): Support the "or" operation, which will be the same
// as the "aliases".
//
// Aliases are aliases of the kernel configuration. Some configuration
// has different names in different kernel version. Names of different
// versions will be treated as aliases.
Aliases []string
Aliases []string `json:"aliases,omitempty"`
// Description is the description of the kernel configuration, for example:
// * What is it used for?
// * Why is it needed?
// * Who needs it?
Description string
Description string `json:"description,omitempty"`
}
// KernelSpec defines the specification for the kernel. Currently, it contains
@ -38,31 +41,31 @@ type KernelConfig struct {
// * Kernel Configuration
type KernelSpec struct {
// Versions define supported kernel version. It is a group of regexps.
Versions []string
Versions []string `json:"versions,omitempty"`
// Required contains all kernel configurations required to be enabled
// (built in or as module).
Required []KernelConfig
Required []KernelConfig `json:"required,omitempty"`
// Optional contains all kernel configurations are required for optional
// features.
Optional []KernelConfig
Optional []KernelConfig `json:"optional,omitempty"`
// Forbidden contains all kernel configurations which areforbidden (disabled
// or not set)
Forbidden []KernelConfig
Forbidden []KernelConfig `json:"forbidden,omitempty"`
}
// DockerSpec defines the requirement configuration for docker. Currently, it only
// contains spec for graph driver.
type DockerSpec struct {
// Version is a group of regex matching supported docker versions.
Version []string
Version []string `json:"version,omitempty"`
// GraphDriver is the graph drivers supported by kubelet.
GraphDriver []string
GraphDriver []string `json:"graphDriver,omitempty"`
}
// RuntimeSpec is the abstract layer for different runtimes. Different runtimes
// should put their spec inside the RuntimeSpec.
type RuntimeSpec struct {
*DockerSpec
*DockerSpec `json:",inline"`
}
// PackageSpec defines the required packages and their versions.
@ -72,7 +75,7 @@ type RuntimeSpec struct {
// either "foo (>=1.0)" or "bar (>=2.0)" is required.
type PackageSpec struct {
// Name is the name of the package to be checked.
Name string
Name string `json:"name,omitempty"`
// VersionRange represents a range of versions that the package must
// satisfy. Note that the version requirement will not be enforced if
// the version range is empty. For example,
@ -81,9 +84,11 @@ type PackageSpec struct {
// - ">1.0 <2.0" would match between both ranges, so "1.1.1" and "1.8.7"
// but not "1.0.0" or "2.0.0".
// - "<2.0.0 || >=3.0.0" would match "1.0.0" and "3.0.0" but not "2.0.0".
VersionRange string
VersionRange string `json:"versionRange,omitempty"`
// Description explains the reason behind this package requirements.
Description string
//
// TODO(yguo0905): Print the description where necessary.
Description string `json:"description,omitempty"`
}
// PackageSpecOverride defines the overrides on the PackageSpec for an OS
@ -91,31 +96,31 @@ type PackageSpec struct {
type PackageSpecOverride struct {
// OSDistro identifies to which OS distro this override applies.
// Must be "ubuntu", "cos" or "coreos".
OSDistro string
OSDistro string `json:"osDistro,omitempty"`
// Subtractions is a list of package names that are excluded from the
// package spec.
Subtractions []PackageSpec
Subtractions []PackageSpec `json:"subtractions,omitempty"`
// Additions is a list of additional package requirements included the
// package spec.
Additions []PackageSpec
Additions []PackageSpec `json:"additions,omitempty"`
}
// SysSpec defines the requirement of supported system. Currently, it only contains
// spec for OS, Kernel and Cgroups.
type SysSpec struct {
// OS is the operating system of the SysSpec.
OS string
OS string `json:"os,omitempty"`
// KernelConfig defines the spec for kernel.
KernelSpec KernelSpec
KernelSpec KernelSpec `json:"kernelSpec,omitempty"`
// Cgroups is the required cgroups.
Cgroups []string
Cgroups []string `json:"cgroups,omitempty"`
// RuntimeSpec defines the spec for runtime.
RuntimeSpec RuntimeSpec
RuntimeSpec RuntimeSpec `json:"runtimeSpec,omitempty"`
// PackageSpec defines the required packages and their versions.
PackageSpecs []PackageSpec
PackageSpecs []PackageSpec `json:"packageSpecs,omitempty"`
// PackageSpec defines the overrides of the required packages and their
// versions for an OS distro.
PackageSpecOverrides []PackageSpecOverride
PackageSpecOverrides []PackageSpecOverride `json:"packageSpecOverrides,omitempty"`
}
// DefaultSysSpec is the default SysSpec.

View File

@ -49,8 +49,8 @@ func Validate(spec SysSpec, validators []Validator) (error, error) {
return errors.NewAggregate(warns), errors.NewAggregate(errs)
}
// ValidateDefault uses all default validators to validate the system and writes to stdout.
func ValidateDefault(runtime string) (error, error) {
// ValidateSpec uses all default validators to validate the system and writes to stdout.
func ValidateSpec(spec SysSpec, runtime string) (error, error) {
// OS-level validators.
var osValidators = []Validator{
&OSValidator{Reporter: DefaultReporter},
@ -68,5 +68,5 @@ func ValidateDefault(runtime string) (error, error) {
case "docker":
validators = append(validators, dockerValidators...)
}
return Validate(DefaultSysSpec, validators)
return Validate(spec, validators)
}