Merge pull request #58442 from m1093782566/ipvs-ga

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Declare IPVS-based kube-proxy GA and remove feature gateway

**What this PR does / why we need it**:

As discussed in sig-network meeting, we decided to target IPVS-based kube-proxy GA in 1.10.

This PR declares IPVS-based kube-proxy GA and remove feature gateway.

**Which issue(s) this PR fixes**:

Fixes: #58443

issue in feature repo: https://github.com/kubernetes/features/issues/265

**Special notes for your reviewer**:

**Release note**:

```release-note
Declare IPVS-based kube-proxy GA
```

@quinton-hoole

cmd/kube-proxy/app/BUILD

@@ -114,70 +114,60 @@ go_library(
         "//vendor/k8s.io/utils/exec:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:android": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:darwin": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:dragonfly": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:freebsd": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:linux": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:netbsd": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:openbsd": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:plan9": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:solaris": [
-            "//pkg/features:go_default_library",
             "//pkg/proxy/metrics:go_default_library",
             "//pkg/util/dbus:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
@@ -235,7 +225,6 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",
-        "//pkg/features:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig:go_default_library",
         "//pkg/util/configz:go_default_library",
         "//pkg/util/pointer:go_default_library",
@@ -246,52 +235,42 @@ go_test(
         "@io_bazel_rules_go//go/platform:android": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:darwin": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:dragonfly": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:freebsd": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:linux": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:netbsd": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:openbsd": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:plan9": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:solaris": [
             "//pkg/proxy/ipvs:go_default_library",
             "//pkg/util/iptables:go_default_library",
-            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         ],
         "//conditions:default": [],
     }),

cmd/kube-proxy/app/server_others.go

@@ -30,9 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/record"
-	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/proxy"
 	proxyconfigapi "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
 	proxyconfig "k8s.io/kubernetes/pkg/proxy/config"
@@ -276,21 +274,13 @@ func newProxyServer(
 }
 
 func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
-	if proxyMode == proxyModeUserspace {
+	switch proxyMode {
+	case proxyModeUserspace:
 		return proxyModeUserspace
-	}
+	case proxyModeIPTables:
-
-	if len(proxyMode) > 0 && proxyMode == proxyModeIPTables {
 		return tryIPTablesProxy(iptver, kcompat)
-	}
+	case proxyModeIPVS:
-
+		return tryIPVSProxy(iptver, khandle, ipsetver, kcompat)
-	if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) {
-		if proxyMode == proxyModeIPVS {
-			return tryIPVSProxy(iptver, khandle, ipsetver, kcompat)
-		} else {
-			glog.Warningf("Can't use ipvs proxier, trying iptables proxier")
-			return tryIPTablesProxy(iptver, kcompat)
-		}
 	}
 	glog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode)
 	return tryIPTablesProxy(iptver, kcompat)

cmd/kube-proxy/app/server_others_test.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"testing"
 
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/proxy/ipvs"
 	"k8s.io/kubernetes/pkg/util/iptables"
 )
@@ -88,19 +87,48 @@ func Test_getProxyMode(t *testing.T) {
 			kernelCompat:    true,
 			expected:        proxyModeIPTables,
 		},
-		{ // specify ipvs, feature gateway disabled, iptables version ok, kernel is compatible
+		{ // flag says ipvs, ipset version ok, kernel modules installed
+			flag:         "ipvs",
+			kmods:        []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
+			ipsetVersion: ipvs.MinIPSetCheckVersion,
+			expected:     proxyModeIPVS,
+		},
+		{ // flag says ipvs, ipset version too low, fallback on iptables mode
 			flag:            "ipvs",
+			kmods:           []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
+			ipsetVersion:    "0.0",
 			iptablesVersion: iptables.MinCheckVersion,
 			kernelCompat:    true,
 			expected:        proxyModeIPTables,
 		},
-		{ // specify ipvs, feature gateway disabled, iptables version too low
+		{ // flag says ipvs, bad ipset version, fallback on iptables mode
 			flag:            "ipvs",
+			kmods:           []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
+			ipsetVersion:    "a.b.c",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // flag says ipvs, required kernel modules are not installed, fallback on iptables mode
+			flag:            "ipvs",
+			kmods:           []string{"foo", "bar", "baz"},
+			ipsetVersion:    ipvs.MinIPSetCheckVersion,
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // flag says ipvs, required kernel modules are not installed, iptables version too old, fallback on userspace mode
+			flag:            "ipvs",
+			kmods:           []string{"foo", "bar", "baz"},
+			ipsetVersion:    ipvs.MinIPSetCheckVersion,
 			iptablesVersion: "0.0.0",
+			kernelCompat:    true,
 			expected:        proxyModeUserspace,
 		},
-		{ // specify ipvs, feature gateway disabled, iptables version ok, kernel is not compatible
+		{ // flag says ipvs, ipset version too low, iptables version too old, kernel not compatible, fallback on userspace mode
 			flag:            "ipvs",
+			kmods:           []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
+			ipsetVersion:    "0.0",
 			iptablesVersion: iptables.MinCheckVersion,
 			kernelCompat:    false,
 			expected:        proxyModeUserspace,
@@ -117,131 +145,3 @@ func Test_getProxyMode(t *testing.T) {
 		}
 	}
 }
-
-// This is a coarse test, but it offers some modicum of confidence as the code is evolved.
-func Test_getProxyModeEnableFeatureGateway(t *testing.T) {
-	// enable IPVS feature gateway
-	utilfeature.DefaultFeatureGate.Set("SupportIPVSProxyMode=true")
-
-	var cases = []struct {
-		flag            string
-		iptablesVersion string
-		ipsetVersion    string
-		kernelCompat    bool
-		iptablesError   error
-		ipsetError      error
-		mods            []string
-		expected        string
-	}{
-		{ // flag says userspace
-			flag:     "userspace",
-			expected: proxyModeUserspace,
-		},
-		{ // flag says iptables, error detecting version
-			flag:          "iptables",
-			iptablesError: fmt.Errorf("oops!"),
-			expected:      proxyModeUserspace,
-		},
-		{ // flag says iptables, version too low
-			flag:            "iptables",
-			iptablesVersion: "0.0.0",
-			expected:        proxyModeUserspace,
-		},
-		{ // flag says iptables, version ok, kernel not compatible
-			flag:            "iptables",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    false,
-			expected:        proxyModeUserspace,
-		},
-		{ // flag says iptables, version ok, kernel is compatible
-			flag:            "iptables",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    true,
-			expected:        proxyModeIPTables,
-		},
-		{ // detect, error
-			flag:          "",
-			iptablesError: fmt.Errorf("oops!"),
-			expected:      proxyModeUserspace,
-		},
-		{ // detect, version too low
-			flag:            "",
-			iptablesVersion: "0.0.0",
-			expected:        proxyModeUserspace,
-		},
-		{ // detect, version ok, kernel not compatible
-			flag:            "",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    false,
-			expected:        proxyModeUserspace,
-		},
-		{ // detect, version ok, kernel is compatible
-			flag:            "",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    true,
-			expected:        proxyModeIPTables,
-		},
-		{ // detect, version ok, kernel is compatible
-			flag:            "",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    true,
-			expected:        proxyModeIPTables,
-		},
-		{ // flag says ipvs, ipset version ok, kernel modules installed
-			flag:         "ipvs",
-			mods:         []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
-			ipsetVersion: ipvs.MinIPSetCheckVersion,
-			expected:     proxyModeIPVS,
-		},
-		{ // flag says ipvs, ipset version too low, fallback on iptables mode
-			flag:            "ipvs",
-			mods:            []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
-			ipsetVersion:    "0.0",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    true,
-			expected:        proxyModeIPTables,
-		},
-		{ // flag says ipvs, bad ipset version, fallback on iptables mode
-			flag:            "ipvs",
-			mods:            []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
-			ipsetVersion:    "a.b.c",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    true,
-			expected:        proxyModeIPTables,
-		},
-		{ // flag says ipvs, required kernel modules are not installed, fallback on iptables mode
-			flag:            "ipvs",
-			mods:            []string{"foo", "bar", "baz"},
-			ipsetVersion:    ipvs.MinIPSetCheckVersion,
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    true,
-			expected:        proxyModeIPTables,
-		},
-		{ // flag says ipvs, required kernel modules are not installed, iptables version too old, fallback on userspace mode
-			flag:            "ipvs",
-			mods:            []string{"foo", "bar", "baz"},
-			ipsetVersion:    ipvs.MinIPSetCheckVersion,
-			iptablesVersion: "0.0.0",
-			kernelCompat:    true,
-			expected:        proxyModeUserspace,
-		},
-		{ // flag says ipvs, ipset version too low, iptables version too old, kernel not compatible, fallback on userspace mode
-			flag:            "ipvs",
-			mods:            []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
-			ipsetVersion:    "0.0",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    false,
-			expected:        proxyModeUserspace,
-		},
-	}
-	for i, c := range cases {
-		versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
-		kcompater := &fakeKernelCompatTester{c.kernelCompat}
-		ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
-		khandle := &fakeKernelHandler{c.mods}
-		r := getProxyMode(c.flag, versioner, khandle, ipsetver, kcompater)
-		if r != c.expected {
-			t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
-		}
-	}
-}

cmd/kube-proxy/app/server_test.go

@@ -29,7 +29,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
 	api "k8s.io/kubernetes/pkg/apis/core"
-	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
 	"k8s.io/kubernetes/pkg/util/configz"
 	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
@@ -189,8 +188,6 @@ conntrack:
   min: 1
   tcpCloseWaitTimeout: 10s
   tcpEstablishedTimeout: 20s
-featureGates:
-  SupportIPVSProxyMode: true
 healthzBindAddress: "%s"
 hostnameOverride: "foo"
 iptables:
@@ -307,7 +304,7 @@ nodePortAddresses:
 				TCPCloseWaitTimeout:   &metav1.Duration{Duration: 10 * time.Second},
 				TCPEstablishedTimeout: &metav1.Duration{Duration: 20 * time.Second},
 			},
-			FeatureGates:       map[string]bool{string(features.SupportIPVSProxyMode): true},
+			FeatureGates:       map[string]bool{},
 			HealthzBindAddress: tc.healthzBindAddress,
 			HostnameOverride:   "foo",
 			IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{

hack/local-up-cluster.sh

@@ -126,13 +126,11 @@ if [ "${CLOUD_PROVIDER}" == "openstack" ]; then
     fi
 fi
 
-# set feature gates if using ipvs mode
+# load required kernel modules if proxy mode is set to "ipvs".
 if [ "${KUBE_PROXY_MODE}" == "ipvs" ]; then
     # If required kernel modules are not available, fall back to iptables.
     sudo modprobe -a ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack_ipv4
-    if [[ $? -eq 0 ]]; then
+    if [[ $? -ne 0 ]]; then
-      FEATURE_GATES="${FEATURE_GATES},SupportIPVSProxyMode=true"
-    else
       echo "Required kernel modules for ipvs not found. Falling back to iptables mode."
       KUBE_PROXY_MODE=iptables
     fi

pkg/features/kube_features.go

@@ -212,7 +212,7 @@ const (
 	ResourceLimitsPriorityFunction utilfeature.Feature = "ResourceLimitsPriorityFunction"
 
 	// owner: @m1093782566
-	// beta: v1.9
+	// GA: v1.11
 	//
 	// Implement IPVS-based in-cluster service load balancing
 	SupportIPVSProxyMode utilfeature.Feature = "SupportIPVSProxyMode"
@@ -331,7 +331,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	BlockVolume:                                 {Default: false, PreRelease: utilfeature.Alpha},
 	StorageObjectInUseProtection:                {Default: true, PreRelease: utilfeature.GA},
 	ResourceLimitsPriorityFunction:              {Default: false, PreRelease: utilfeature.Alpha},
-	SupportIPVSProxyMode:                        {Default: true, PreRelease: utilfeature.Beta},
+	SupportIPVSProxyMode:                        {Default: true, PreRelease: utilfeature.GA},
 	SupportPodPidsLimit:                         {Default: false, PreRelease: utilfeature.Alpha},
 	HyperVContainer:                             {Default: false, PreRelease: utilfeature.Alpha},
 	ScheduleDaemonSetPods:                       {Default: false, PreRelease: utilfeature.Alpha},