Merge pull request #50940 from MrHohn/kube-proxy-ds-upgrade-tests

Automatic merge from submit-queue (batch tested with PRs 51228, 50185, 50940, 51544, 51543)

Add upgrades tests for kube-proxy daemonset migration path

**What this PR does / why we need it**:
From #23225, this is a part of setting up CIs to validate the kube-proxy migration path (static pods -> daemonset and reverse).
The other part of the works (adding real CIs that run these tests) will be in a separate PR against [kubernetes/test-infra](https://github.com/kubernetes/test-infra).

Though this is currently blocked by #50705.

**Special notes for your reviewer**:
cc @roberthbailey  @pwittrock 

**Release note**:

```release-note
NONE
```

test/e2e/framework/nodes_util.go

@@ -42,7 +42,7 @@ func EtcdUpgrade(target_storage, target_version string) error {
 func MasterUpgrade(v string) error {
 	switch TestContext.Provider {
 	case "gce":
-		return masterUpgradeGCE(v)
+		return masterUpgradeGCE(v, false)
 	case "gke":
 		return masterUpgradeGKE(v)
 	default:
@@ -61,8 +61,14 @@ func etcdUpgradeGCE(target_storage, target_version string) error {
 	return err
 }
 
-func masterUpgradeGCE(rawV string) error {
-	env := os.Environ()
+// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
+func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error {
+	return masterUpgradeGCE(v, enableKubeProxyDaemonSet)
+}
+
+// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default.
+func masterUpgradeGCE(rawV string, enableKubeProxyDaemonSet bool) error {
+	env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet))
 	// TODO: Remove these variables when they're no longer needed for downgrades.
 	if TestContext.EtcdUpgradeVersion != "" && TestContext.EtcdUpgradeStorage != "" {
 		env = append(env,
@@ -101,7 +107,7 @@ func NodeUpgrade(f *Framework, v string, img string) error {
 	var err error
 	switch TestContext.Provider {
 	case "gce":
-		err = nodeUpgradeGCE(v, img)
+		err = nodeUpgradeGCE(v, img, false)
 	case "gke":
 		err = nodeUpgradeGKE(v, img)
 	default:
@@ -122,14 +128,30 @@ func NodeUpgrade(f *Framework, v string, img string) error {
 	return nil
 }
 
-func nodeUpgradeGCE(rawV, img string) error {
+// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
+func NodeUpgradeGCEWithKubeProxyDaemonSet(f *Framework, v string, enableKubeProxyDaemonSet bool) error {
+	// Perform the upgrade.
+	if err := nodeUpgradeGCE(v, "", enableKubeProxyDaemonSet); err != nil {
+		return err
+	}
+	// Wait for it to complete and validate nodes are healthy.
+	Logf("Waiting up to %v for all nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout)
+	if _, err := CheckNodesReady(f.ClientSet, RestartNodeReadyAgainTimeout, TestContext.CloudConfig.NumNodes); err != nil {
+		return err
+	}
+	return nil
+}
+
+// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default.
+func nodeUpgradeGCE(rawV, img string, enableKubeProxyDaemonSet bool) error {
 	v := "v" + rawV
+	env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet))
 	if img != "" {
-		env := append(os.Environ(), "KUBE_NODE_OS_DISTRIBUTION="+img)
+		env = append(env, "KUBE_NODE_OS_DISTRIBUTION="+img)
 		_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v)
 		return err
 	}
-	_, _, err := RunCmd(gceUpgradeScript(), "-N", v)
+	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", v)
 	return err
 }
 

test/e2e/lifecycle/cluster_upgrade.go

@@ -216,6 +216,69 @@ var _ = Describe("[sig-apps] stateful Upgrade [Feature:StatefulUpgrade]", func()
 	})
 })
 
+var _ = SIGDescribe("kube-proxy migration [Feature:KubeProxyDaemonSetMigration]", func() {
+	f := framework.NewDefaultFramework("kube-proxy-ds-migration")
+
+	// Create the frameworks here because we can only create them
+	// in a "Describe".
+	testFrameworks := createUpgradeFrameworks(upgradeTests)
+
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce")
+	})
+
+	Describe("Upgrade kube-proxy from static pods to a DaemonSet", func() {
+		It("should maintain a functioning cluster [Feature:KubeProxyDaemonSetUpgrade]", func() {
+			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
+			framework.ExpectNoError(err)
+
+			testSuite := &junit.TestSuite{Name: "kube-proxy upgrade"}
+			kubeProxyUpgradeTest := &junit.TestCase{
+				Name:      "kube-proxy-ds-upgrade",
+				Classname: "upgrade_tests",
+			}
+			testSuite.TestCases = append(testSuite.TestCases, kubeProxyUpgradeTest)
+
+			upgradeFunc := func() {
+				start := time.Now()
+				defer finalizeUpgradeTest(start, kubeProxyUpgradeTest)
+				target := upgCtx.Versions[1].Version.String()
+				framework.ExpectNoError(framework.MasterUpgradeGCEWithKubeProxyDaemonSet(target, true))
+				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
+				framework.ExpectNoError(framework.NodeUpgradeGCEWithKubeProxyDaemonSet(f, target, true))
+				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
+			}
+			runUpgradeSuite(f, upgradeTests, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc)
+		})
+	})
+
+	Describe("Downgrade kube-proxy from a DaemonSet to static pods", func() {
+		It("should maintain a functioning cluster [Feature:KubeProxyDaemonSetDowngrade]", func() {
+			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
+			framework.ExpectNoError(err)
+
+			testSuite := &junit.TestSuite{Name: "kube-proxy downgrade"}
+			kubeProxyDowngradeTest := &junit.TestCase{
+				Name:      "kube-proxy-ds-downgrade",
+				Classname: "upgrade_tests",
+			}
+			testSuite.TestCases = append(testSuite.TestCases, kubeProxyDowngradeTest)
+
+			upgradeFunc := func() {
+				start := time.Now()
+				defer finalizeUpgradeTest(start, kubeProxyDowngradeTest)
+				// Yes this really is a downgrade. And nodes must downgrade first.
+				target := upgCtx.Versions[1].Version.String()
+				framework.ExpectNoError(framework.NodeUpgradeGCEWithKubeProxyDaemonSet(f, target, false))
+				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
+				framework.ExpectNoError(framework.MasterUpgradeGCEWithKubeProxyDaemonSet(target, false))
+				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
+			}
+			runUpgradeSuite(f, upgradeTests, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc)
+		})
+	})
+})
+
 type chaosMonkeyAdapter struct {
 	test        upgrades.Test
 	testReport  *junit.TestCase