From cc84e0895abc26fee3977abb5807cc880cd4c49f Mon Sep 17 00:00:00 2001 From: Kris Date: Wed, 8 Mar 2017 14:13:07 -0800 Subject: [PATCH] Implement automated downgrade testing. Node version cannot be higher than the master version, so we must switch the node version first. Also, we must use the upgrade script from the appropriate version for GCE. --- hack/verify-flags/known-flags.txt | 1 + test/e2e/cluster_upgrade.go | 33 ++++++++++++++++++++++++++++++ test/e2e/framework/nodes_util.go | 15 ++++++++++---- test/e2e/framework/test_context.go | 2 ++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 3927af4e76..6ce481577f 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -282,6 +282,7 @@ gather-metrics-at-teardown gather-resource-usage gce-project gce-service-account +gce-upgrade-script gce-zone ginkgo-flags gke-cluster diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index 5382d4ff9d..d494cb6aa6 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -113,6 +113,39 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { }) }) +var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() { + f := framework.NewDefaultFramework("cluster-downgrade") + + // Create the frameworks here because we can only create them + // in a "Describe". + testFrameworks := map[string]*framework.Framework{} + for _, t := range upgradeTests { + testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name()) + } + + framework.KubeDescribe("cluster downgrade", func() { + It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() { + cm := chaosmonkey.New(func() { + // Yes this really is a downgrade. And nodes must downgrade first. + v, err := realVersion(framework.TestContext.UpgradeTarget) + framework.ExpectNoError(err) + framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage)) + framework.ExpectNoError(checkNodesVersions(f.ClientSet, v)) + framework.ExpectNoError(framework.MasterUpgrade(v)) + framework.ExpectNoError(checkMasterVersion(f.ClientSet, v)) + }) + for _, t := range upgradeTests { + cm.RegisterInterface(&chaosMonkeyAdapter{ + test: t, + framework: testFrameworks[t.Name()], + upgradeType: upgrades.ClusterUpgrade, + }) + } + cm.Do() + }) + }) +}) + var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() { // Create the frameworks here because we can only create them // in a "Describe". diff --git a/test/e2e/framework/nodes_util.go b/test/e2e/framework/nodes_util.go index 3b073ba475..57337401c4 100644 --- a/test/e2e/framework/nodes_util.go +++ b/test/e2e/framework/nodes_util.go @@ -57,13 +57,13 @@ func etcdUpgradeGCE(target_storage, target_version string) error { "STORAGE_BACKEND="+target_storage, "TEST_ETCD_IMAGE=3.0.17") - _, _, err := RunCmdEnv(env, path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-l", "-M") + _, _, err := RunCmdEnv(env, gceUpgradeScript(), "-l", "-M") return err } func masterUpgradeGCE(rawV string) error { v := "v" + rawV - _, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", v) + _, _, err := RunCmd(gceUpgradeScript(), "-M", v) return err } @@ -111,10 +111,10 @@ func nodeUpgradeGCE(rawV, img string) error { v := "v" + rawV if img != "" { env := append(os.Environ(), "KUBE_NODE_OS_DISTRIBUTION="+img) - _, _, err := RunCmdEnv(env, path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-N", "-o", v) + _, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v) return err } - _, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-N", v) + _, _, err := RunCmd(gceUpgradeScript(), "-N", v) return err } @@ -254,3 +254,10 @@ func MigTemplate() (string, error) { } return templ, nil } + +func gceUpgradeScript() string { + if len(TestContext.GCEUpgradeScript) == 0 { + return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh") + } + return TestContext.GCEUpgradeScript +} diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index f68fc4e41d..8597965b01 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -52,6 +52,7 @@ type TestContextType struct { EtcdUpgradeStorage string EtcdUpgradeVersion string UpgradeImage string + GCEUpgradeScript string PrometheusPushGateway string ContainerRuntime string MasterOSDistro string @@ -200,6 +201,7 @@ func RegisterClusterFlags() { flag.StringVar(&TestContext.EtcdUpgradeStorage, "etcd-upgrade-storage", "", "The storage version to upgrade to (either 'etcdv2' or 'etcdv3') if doing an etcd upgrade test.") flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.") flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.") + flag.StringVar(&TestContext.GCEUpgradeScript, "gce-upgrade-script", "", "Script to use to upgrade a GCE cluster.") flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.") flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.") flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", true, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")