From c7071ed09a7d5a9fb7e25370693f2fdc6785f15f Mon Sep 17 00:00:00 2001 From: m1093782566 Date: Thu, 12 Oct 2017 16:20:05 +0800 Subject: [PATCH] try ipset in ipvs proxy mode --- .../kubeproxyconfig/validation/validation.go | 26 +++- .../validation/validation_test.go | 116 +++++++++++++++++- pkg/proxy/iptables/proxier.go | 8 -- pkg/proxy/ipvs/proxier.go | 8 -- pkg/proxy/winkernel/proxier.go | 9 -- 5 files changed, 140 insertions(+), 27 deletions(-) diff --git a/pkg/proxy/apis/kubeproxyconfig/validation/validation.go b/pkg/proxy/apis/kubeproxyconfig/validation/validation.go index c85152eb74..657bc58bc9 100644 --- a/pkg/proxy/apis/kubeproxyconfig/validation/validation.go +++ b/pkg/proxy/apis/kubeproxyconfig/validation/validation.go @@ -35,10 +35,10 @@ func Validate(config *kubeproxyconfig.KubeProxyConfiguration) field.ErrorList { newPath := field.NewPath("KubeProxyConfiguration") allErrs = append(allErrs, validateKubeProxyIPTablesConfiguration(config.IPTables, newPath.Child("KubeProxyIPTablesConfiguration"))...) + allErrs = append(allErrs, validateKubeProxyIPVSConfiguration(config.IPVS, newPath.Child("KubeProxyIPVSConfiguration"))...) allErrs = append(allErrs, validateKubeProxyConntrackConfiguration(config.Conntrack, newPath.Child("KubeProxyConntrackConfiguration"))...) allErrs = append(allErrs, validateProxyMode(config.Mode, newPath.Child("Mode"))...) allErrs = append(allErrs, validateClientConnectionConfiguration(config.ClientConnection, newPath.Child("ClientConnection"))...) - allErrs = append(allErrs, validateIPVSSchedulerMethod(kubeproxyconfig.IPVSSchedulerMethod(config.IPVS.Scheduler), newPath.Child("KubeProxyIPVSConfiguration").Child("Scheduler"))...) if config.OOMScoreAdj != nil && (*config.OOMScoreAdj < -1000 || *config.OOMScoreAdj > 1000) { allErrs = append(allErrs, field.Invalid(newPath.Child("OOMScoreAdj"), *config.OOMScoreAdj, "must be within the range [-1000, 1000]")) @@ -87,6 +87,30 @@ func validateKubeProxyIPTablesConfiguration(config kubeproxyconfig.KubeProxyIPTa allErrs = append(allErrs, field.Invalid(fldPath.Child("MinSyncPeriod"), config.MinSyncPeriod, "must be greater than or equal to 0")) } + if config.MinSyncPeriod.Duration > config.SyncPeriod.Duration { + allErrs = append(allErrs, field.Invalid(fldPath.Child("SyncPeriod"), config.MinSyncPeriod, fmt.Sprintf("must be greater than or equal to %s", fldPath.Child("MinSyncPeriod").String()))) + } + + return allErrs +} + +func validateKubeProxyIPVSConfiguration(config kubeproxyconfig.KubeProxyIPVSConfiguration, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if config.SyncPeriod.Duration <= 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("SyncPeriod"), config.SyncPeriod, "must be greater than 0")) + } + + if config.MinSyncPeriod.Duration < 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("MinSyncPeriod"), config.MinSyncPeriod, "must be greater than or equal to 0")) + } + + if config.MinSyncPeriod.Duration > config.SyncPeriod.Duration { + allErrs = append(allErrs, field.Invalid(fldPath.Child("SyncPeriod"), config.MinSyncPeriod, fmt.Sprintf("must be greater than or equal to %s", fldPath.Child("MinSyncPeriod").String()))) + } + + allErrs = append(allErrs, validateIPVSSchedulerMethod(kubeproxyconfig.IPVSSchedulerMethod(config.Scheduler), fldPath.Child("Scheduler"))...) + return allErrs } diff --git a/pkg/proxy/apis/kubeproxyconfig/validation/validation_test.go b/pkg/proxy/apis/kubeproxyconfig/validation/validation_test.go index 1fe01e64b8..033a6fb798 100644 --- a/pkg/proxy/apis/kubeproxyconfig/validation/validation_test.go +++ b/pkg/proxy/apis/kubeproxyconfig/validation/validation_test.go @@ -17,6 +17,7 @@ limitations under the License. package validation import ( + "fmt" "strings" "testing" "time" @@ -40,6 +41,10 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, }, + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ Max: int32(2), MaxPerCore: int32(1), @@ -74,6 +79,10 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, }, + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ Max: int32(2), MaxPerCore: int32(1), @@ -98,6 +107,10 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, }, + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ Max: int32(2), MaxPerCore: int32(1), @@ -122,6 +135,10 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, }, + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ Max: int32(2), MaxPerCore: int32(1), @@ -146,6 +163,10 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, }, + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ Max: int32(2), MaxPerCore: int32(1), @@ -170,6 +191,10 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, }, + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ Max: int32(2), MaxPerCore: int32(1), @@ -194,6 +219,10 @@ func TestValidateKubeProxyConfiguration(t *testing.T) { SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, }, + IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{ Max: int32(2), MaxPerCore: int32(1), @@ -268,6 +297,16 @@ func TestValidateKubeProxyIPTablesConfiguration(t *testing.T) { }, msg: "must be within the range [0, 31]", }, + // SyncPeriod must be >= MinSyncPeriod + { + config: kubeproxyconfig.KubeProxyIPTablesConfiguration{ + MasqueradeBit: &valid, + MasqueradeAll: true, + SyncPeriod: metav1.Duration{Duration: 1 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, + msg: fmt.Sprintf("must be greater than or equal to %s", newPath.Child("KubeProxyIPTablesConfiguration").Child("MinSyncPeriod").String()), + }, } for _, errorCase := range errorCases { @@ -279,6 +318,82 @@ func TestValidateKubeProxyIPTablesConfiguration(t *testing.T) { } } +func TestValidateKubeProxyIPVSConfiguration(t *testing.T) { + newPath := field.NewPath("KubeProxyConfiguration") + testCases := []struct { + config kubeproxyconfig.KubeProxyIPVSConfiguration + expectErr bool + reason string + }{ + { + config: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: -5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 2 * time.Second}, + }, + expectErr: true, + reason: "SyncPeriod must be greater than 0", + }, + { + config: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 0 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + }, + expectErr: true, + reason: "SyncPeriod must be greater than 0", + }, + { + config: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: -1 * time.Second}, + }, + expectErr: true, + reason: "MinSyncPeriod must be greater than or equal to 0", + }, + { + config: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 1 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, + expectErr: true, + reason: "SyncPeriod must be greater than or equal to MinSyncPeriod", + }, + // SyncPeriod == MinSyncPeriod + { + config: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + }, + expectErr: false, + }, + // SyncPeriod > MinSyncPeriod + { + config: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 10 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + }, + expectErr: false, + }, + // SyncPeriod can be 0 + { + config: kubeproxyconfig.KubeProxyIPVSConfiguration{ + SyncPeriod: metav1.Duration{Duration: 5 * time.Second}, + MinSyncPeriod: metav1.Duration{Duration: 0 * time.Second}, + }, + expectErr: false, + }, + } + + for _, test := range testCases { + errs := validateKubeProxyIPVSConfiguration(test.config, newPath.Child("KubeProxyIPVSConfiguration")) + if len(errs) == 0 && test.expectErr { + t.Errorf("Expect error, got nil, reason: %s", test.reason) + } + if len(errs) > 0 && !test.expectErr { + t.Errorf("Unexpected error: %v", errs) + } + } +} + func TestValidateKubeProxyConntrackConfiguration(t *testing.T) { successCases := []kubeproxyconfig.KubeProxyConntrackConfiguration{ { @@ -370,7 +485,6 @@ func TestValidateKubeProxyConntrackConfiguration(t *testing.T) { func TestValidateProxyMode(t *testing.T) { newPath := field.NewPath("KubeProxyConfiguration") - successCases := []kubeproxyconfig.ProxyMode{ kubeproxyconfig.ProxyModeUserspace, kubeproxyconfig.ProxyModeIPTables, diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 786434f7c1..a3aed018f4 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -440,11 +440,6 @@ func NewProxier(ipt utiliptables.Interface, recorder record.EventRecorder, healthzServer healthcheck.HealthzUpdater, ) (*Proxier, error) { - // check valid user input - if minSyncPeriod > syncPeriod { - return nil, fmt.Errorf("minSyncPeriod (%v) must be <= syncPeriod (%v)", minSyncPeriod, syncPeriod) - } - // Set the route_localnet sysctl we need for if err := sysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil { return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlRouteLocalnet, err) @@ -458,9 +453,6 @@ func NewProxier(ipt utiliptables.Interface, } // Generate the masquerade mark to use for SNAT rules. - if masqueradeBit < 0 || masqueradeBit > 31 { - return nil, fmt.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", masqueradeBit) - } masqueradeValue := 1 << uint(masqueradeBit) masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 6d06d0053a..566153012e 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -198,11 +198,6 @@ func NewProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, healthzServer healthcheck.HealthzUpdater, scheduler string, ) (*Proxier, error) { - // check valid user input - if minSyncPeriod > syncPeriod { - return nil, fmt.Errorf("min-sync (%v) must be < sync(%v)", minSyncPeriod, syncPeriod) - } - // Set the route_localnet sysctl we need for if err := sysctl.SetSysctl(sysctlRouteLocalnet, 1); err != nil { return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlRouteLocalnet, err) @@ -226,9 +221,6 @@ func NewProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, } // Generate the masquerade mark to use for SNAT rules. - if masqueradeBit < 0 || masqueradeBit > 31 { - return nil, fmt.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", masqueradeBit) - } masqueradeValue := 1 << uint(masqueradeBit) masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue) diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index c984183df6..d4f856f9b5 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -453,15 +453,6 @@ func NewProxier( recorder record.EventRecorder, healthzServer healthcheck.HealthzUpdater, ) (*Proxier, error) { - // check valid user input - if minSyncPeriod > syncPeriod { - return nil, fmt.Errorf("min-sync (%v) must be < sync(%v)", minSyncPeriod, syncPeriod) - } - - // Generate the masquerade mark to use for SNAT rules. - if masqueradeBit < 0 || masqueradeBit > 31 { - return nil, fmt.Errorf("invalid iptables-masquerade-bit %v not in [0, 31]", masqueradeBit) - } masqueradeValue := 1 << uint(masqueradeBit) masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue)