diff --git a/cmd/hyperkube/BUILD b/cmd/hyperkube/BUILD index 21067af04a..830eb17c26 100644 --- a/cmd/hyperkube/BUILD +++ b/cmd/hyperkube/BUILD @@ -50,7 +50,6 @@ go_library( "//cmd/kube-controller-manager/app:go_default_library", "//cmd/kube-controller-manager/app/options:go_default_library", "//cmd/kube-proxy/app:go_default_library", - "//cmd/kube-proxy/app/options:go_default_library", "//cmd/kubelet/app:go_default_library", "//cmd/kubelet/app/options:go_default_library", "//federation/cmd/federation-apiserver/app:go_default_library", diff --git a/cmd/hyperkube/kube-proxy.go b/cmd/hyperkube/kube-proxy.go index 46083e49b9..76bc2dc927 100644 --- a/cmd/hyperkube/kube-proxy.go +++ b/cmd/hyperkube/kube-proxy.go @@ -17,39 +17,37 @@ limitations under the License. package main import ( + "flag" + "k8s.io/apiserver/pkg/server/healthz" "k8s.io/kubernetes/cmd/kube-proxy/app" - "k8s.io/kubernetes/cmd/kube-proxy/app/options" ) -func init() { - healthz.DefaultHealthz() -} - // NewKubeProxy creates a new hyperkube Server object that includes the // description and flags. func NewKubeProxy() *Server { - config := options.NewProxyConfig() + healthz.DefaultHealthz() + + command := app.NewProxyCommand() hks := Server{ name: "proxy", AlternativeName: "kube-proxy", SimpleUsage: "proxy", - Long: `The Kubernetes proxy server is responsible for taking traffic directed at - services and forwarding it to the appropriate pods. It generally runs on - nodes next to the Kubelet and proxies traffic from local pods to remote pods. - It is also used when handling incoming external traffic.`, + Long: command.Long, } - config.AddFlags(hks.Flags()) + serverFlags := hks.Flags() + serverFlags.AddFlagSet(command.Flags()) - hks.Run = func(_ *Server, _ []string) error { - s, err := app.NewProxyServerDefault(config) - if err != nil { - return err - } + // FIXME this is here because hyperkube does its own flag parsing, and we need + // the command to know about the go flag set. Remove this once hyperkube is + // refactored to use cobra throughout. + command.Flags().AddGoFlagSet(flag.CommandLine) - return s.Run() + hks.Run = func(_ *Server, args []string) error { + command.SetArgs(args) + return command.Execute() } return &hks diff --git a/cmd/kube-proxy/BUILD b/cmd/kube-proxy/BUILD index 639aadf1b6..27170c26a7 100644 --- a/cmd/kube-proxy/BUILD +++ b/cmd/kube-proxy/BUILD @@ -27,7 +27,6 @@ go_library( tags = ["automanaged"], deps = [ "//cmd/kube-proxy/app:go_default_library", - "//cmd/kube-proxy/app/options:go_default_library", "//pkg/client/metrics/prometheus:go_default_library", "//pkg/version/prometheus:go_default_library", "//pkg/version/verflag:go_default_library", diff --git a/cmd/kube-proxy/app/BUILD b/cmd/kube-proxy/app/BUILD index 111153a4a0..b7872bd801 100644 --- a/cmd/kube-proxy/app/BUILD +++ b/cmd/kube-proxy/app/BUILD @@ -16,10 +16,13 @@ go_library( ], tags = ["automanaged"], deps = [ - "//cmd/kube-proxy/app/options:go_default_library", "//pkg/api:go_default_library", + "//pkg/apis/componentconfig:go_default_library", + "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/informers/informers_generated/internalversion:go_default_library", + "//pkg/kubectl/cmd/util:go_default_library", + "//pkg/kubelet/qos:go_default_library", "//pkg/proxy:go_default_library", "//pkg/proxy/config:go_default_library", "//pkg/proxy/iptables:go_default_library", @@ -42,10 +45,13 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/pkg/api/v1:go_default_library", + "//vendor/k8s.io/client-go/pkg/util:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library", "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", @@ -58,7 +64,6 @@ go_test( library = ":go_default_library", tags = ["automanaged"], deps = [ - "//cmd/kube-proxy/app/options:go_default_library", "//pkg/api:go_default_library", "//pkg/apis/componentconfig:go_default_library", "//pkg/util/iptables:go_default_library", @@ -76,9 +81,6 @@ filegroup( filegroup( name = "all-srcs", - srcs = [ - ":package-srcs", - "//cmd/kube-proxy/app/options:all-srcs", - ], + srcs = [":package-srcs"], tags = ["automanaged"], ) diff --git a/cmd/kube-proxy/app/options/BUILD b/cmd/kube-proxy/app/options/BUILD deleted file mode 100644 index 86d9146f02..0000000000 --- a/cmd/kube-proxy/app/options/BUILD +++ /dev/null @@ -1,38 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) - -load( - "@io_bazel_rules_go//go:def.bzl", - "go_library", -) - -go_library( - name = "go_default_library", - srcs = ["options.go"], - tags = ["automanaged"], - deps = [ - "//pkg/api:go_default_library", - "//pkg/apis/componentconfig:go_default_library", - "//pkg/apis/componentconfig/v1alpha1:go_default_library", - "//pkg/features:go_default_library", - "//pkg/kubelet/qos:go_default_library", - "//pkg/util:go_default_library", - "//vendor/github.com/spf13/pflag:go_default_library", - "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//vendor/k8s.io/client-go/pkg/api/v1:go_default_library", - ], -) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kube-proxy/app/options/options.go b/cmd/kube-proxy/app/options/options.go deleted file mode 100644 index 9ce8fff94e..0000000000 --- a/cmd/kube-proxy/app/options/options.go +++ /dev/null @@ -1,105 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package options contains flags for initializing a proxy. -package options - -import ( - _ "net/http/pprof" - "time" - - utilfeature "k8s.io/apiserver/pkg/util/feature" - clientv1 "k8s.io/client-go/pkg/api/v1" - "k8s.io/kubernetes/pkg/api" - "k8s.io/kubernetes/pkg/apis/componentconfig" - "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" - _ "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/kubelet/qos" - "k8s.io/kubernetes/pkg/util" - - // add the kubernetes feature gates - _ "k8s.io/kubernetes/pkg/features" - - "github.com/spf13/pflag" -) - -// ProxyServerConfig configures and runs a Kubernetes proxy server -type ProxyServerConfig struct { - componentconfig.KubeProxyConfiguration - ResourceContainer string - ContentType string - KubeAPIQPS float32 - KubeAPIBurst int32 - ConfigSyncPeriod time.Duration - CleanupAndExit bool - NodeRef *clientv1.ObjectReference - Master string - Kubeconfig string -} - -func NewProxyConfig() *ProxyServerConfig { - versioned := &v1alpha1.KubeProxyConfiguration{} - api.Scheme.Default(versioned) - cfg := componentconfig.KubeProxyConfiguration{} - api.Scheme.Convert(versioned, &cfg, nil) - return &ProxyServerConfig{ - KubeProxyConfiguration: cfg, - ContentType: "application/vnd.kubernetes.protobuf", - KubeAPIQPS: 5.0, - KubeAPIBurst: 10, - ConfigSyncPeriod: 15 * time.Minute, - } -} - -// AddFlags adds flags for a specific ProxyServer to the specified FlagSet -func (s *ProxyServerConfig) AddFlags(fs *pflag.FlagSet) { - fs.Var(componentconfig.IPVar{Val: &s.BindAddress}, "bind-address", "The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)") - fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") - fs.Int32Var(&s.HealthzPort, "healthz-port", s.HealthzPort, "The port to bind the health check server. Use 0 to disable.") - fs.Var(componentconfig.IPVar{Val: &s.HealthzBindAddress}, "healthz-bind-address", "The IP address for the health check server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)") - fs.Int32Var(s.OOMScoreAdj, "oom-score-adj", util.Int32PtrDerefOr(s.OOMScoreAdj, int32(qos.KubeProxyOOMScoreAdj)), "The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]") - fs.StringVar(&s.ResourceContainer, "resource-container", s.ResourceContainer, "Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).") - fs.MarkDeprecated("resource-container", "This feature will be removed in a later release.") - fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") - fs.Var(componentconfig.PortRangeVar{Val: &s.PortRange}, "proxy-port-range", "Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.") - fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") - fs.Var(&s.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If blank, use the best-available proxy (currently iptables). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.") - fs.Int32Var(s.IPTablesMasqueradeBit, "iptables-masquerade-bit", util.Int32PtrDerefOr(s.IPTablesMasqueradeBit, 14), "If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within the range [0, 31].") - fs.DurationVar(&s.IPTablesSyncPeriod.Duration, "iptables-sync-period", s.IPTablesSyncPeriod.Duration, "The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") - fs.DurationVar(&s.IPTablesMinSyncPeriod.Duration, "iptables-min-sync-period", s.IPTablesMinSyncPeriod.Duration, "The minimum interval of how often the iptables rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').") - fs.DurationVar(&s.ConfigSyncPeriod, "config-sync-period", s.ConfigSyncPeriod, "How often configuration from the apiserver is refreshed. Must be greater than 0.") - fs.BoolVar(&s.MasqueradeAll, "masquerade-all", s.MasqueradeAll, "If using the pure iptables proxy, SNAT everything") - fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "The CIDR range of pods in the cluster. It is used to bridge traffic coming from outside of the cluster. If not provided, no off-cluster bridging will be performed.") - fs.BoolVar(&s.CleanupAndExit, "cleanup-iptables", s.CleanupAndExit, "If true cleanup iptables rules and exit.") - fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.") - fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver") - fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver") - fs.DurationVar(&s.UDPIdleTimeout.Duration, "udp-timeout", s.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace") - fs.Int32Var(&s.ConntrackMax, "conntrack-max", s.ConntrackMax, - "Maximum number of NAT connections to track (0 to leave as-is). This overrides conntrack-max-per-core and conntrack-min.") - fs.MarkDeprecated("conntrack-max", "This feature will be removed in a later release.") - fs.Int32Var(&s.ConntrackMaxPerCore, "conntrack-max-per-core", s.ConntrackMaxPerCore, - "Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore conntrack-min).") - fs.Int32Var(&s.ConntrackMin, "conntrack-min", s.ConntrackMin, - "Minimum number of conntrack entries to allocate, regardless of conntrack-max-per-core (set conntrack-max-per-core=0 to leave the limit as-is).") - fs.DurationVar(&s.ConntrackTCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", s.ConntrackTCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)") - fs.DurationVar( - &s.ConntrackTCPCloseWaitTimeout.Duration, "conntrack-tcp-timeout-close-wait", - s.ConntrackTCPCloseWaitTimeout.Duration, - "NAT timeout for TCP connections in the CLOSE_WAIT state") - - utilfeature.DefaultFeatureGate.AddFlag(fs) -} diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index f64ee729e6..0b753163a4 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -21,27 +21,34 @@ package app import ( "errors" "fmt" + "io/ioutil" "net" "net/http" _ "net/http/pprof" "runtime" - "strconv" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + utilfeature "k8s.io/apiserver/pkg/util/feature" clientgoclientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" clientv1 "k8s.io/client-go/pkg/api/v1" + "k8s.io/client-go/pkg/util" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/client-go/tools/record" - "k8s.io/kubernetes/cmd/kube-proxy/app/options" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/componentconfig" + "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" + cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubelet/qos" "k8s.io/kubernetes/pkg/proxy" proxyconfig "k8s.io/kubernetes/pkg/proxy/config" "k8s.io/kubernetes/pkg/proxy/iptables" @@ -63,23 +70,12 @@ import ( "github.com/spf13/pflag" ) -type ProxyServer struct { - Client clientset.Interface - EventClient v1core.EventsGetter - Config *options.ProxyServerConfig - IptInterface utiliptables.Interface - Proxier proxy.ProxyProvider - Broadcaster record.EventBroadcaster - Recorder record.EventRecorder - Conntracker Conntracker // if nil, ignored - ProxyMode string -} - const ( proxyModeUserspace = "userspace" proxyModeIPTables = "iptables" ) +// checkKnownProxyMode returns true if proxyMode is valid. func checkKnownProxyMode(proxyMode string) bool { switch proxyMode { case "", proxyModeUserspace, proxyModeIPTables: @@ -88,34 +84,174 @@ func checkKnownProxyMode(proxyMode string) bool { return false } -func NewProxyServer( - client clientset.Interface, - eventClient v1core.EventsGetter, - config *options.ProxyServerConfig, - iptInterface utiliptables.Interface, - proxier proxy.ProxyProvider, - broadcaster record.EventBroadcaster, - recorder record.EventRecorder, - conntracker Conntracker, - proxyMode string, -) (*ProxyServer, error) { - return &ProxyServer{ - Client: client, - EventClient: eventClient, - Config: config, - IptInterface: iptInterface, - Proxier: proxier, - Broadcaster: broadcaster, - Recorder: recorder, - Conntracker: conntracker, - ProxyMode: proxyMode, - }, nil +// Options contains everything necessary to create and run a proxy server. +type Options struct { + // ConfigFile is the location of the proxy server's configuration file. + ConfigFile string + // CleanupAndExit, when true, makes the proxy server clean up iptables rules, then exit. + CleanupAndExit bool + + // config is the proxy server's configuration object. + config *componentconfig.KubeProxyConfiguration + + // The fields below here are placeholders for flags that can't be directly mapped into + // componentconfig.KubeProxyConfiguration. + // + // TODO remove these fields once the deprecated flags are removed. + + // master is used to override the kubeconfig's URL to the apiserver. + master string + // healthzPort is the port to be used by the healthz server. + healthzPort int32 +} + +// AddFlags adds flags to fs and binds them to options. +func AddFlags(options *Options, fs *pflag.FlagSet) { + fs.StringVar(&options.ConfigFile, "config", options.ConfigFile, "The path to the configuration file.") + fs.BoolVar(&options.CleanupAndExit, "cleanup-iptables", options.CleanupAndExit, "If true cleanup iptables rules and exit.") + + // All flags below here are deprecated and will eventually be removed. + + fs.Var(componentconfig.IPVar{Val: &options.config.BindAddress}, "bind-address", "The IP address for the proxy server to serve on (set to 0.0.0.0 for all interfaces)") + fs.StringVar(&options.master, "master", options.master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") + fs.Int32Var(&options.healthzPort, "healthz-port", options.healthzPort, "The port to bind the health check server. Use 0 to disable.") + fs.Var(componentconfig.IPVar{Val: &options.config.HealthzBindAddress}, "healthz-bind-address", "The IP address and port for the health check server to serve on (set to 0.0.0.0 for all interfaces)") + fs.Int32Var(options.config.OOMScoreAdj, "oom-score-adj", util.Int32PtrDerefOr(options.config.OOMScoreAdj, int32(qos.KubeProxyOOMScoreAdj)), "The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]") + fs.StringVar(&options.config.ResourceContainer, "resource-container", options.config.ResourceContainer, "Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).") + fs.MarkDeprecated("resource-container", "This feature will be removed in a later release.") + fs.StringVar(&options.config.ClientConnection.KubeConfigFile, "kubeconfig", options.config.ClientConnection.KubeConfigFile, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") + fs.Var(componentconfig.PortRangeVar{Val: &options.config.PortRange}, "proxy-port-range", "Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.") + fs.StringVar(&options.config.HostnameOverride, "hostname-override", options.config.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.") + fs.Var(&options.config.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster). If blank, use the best-available proxy (currently iptables). If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.") + fs.Int32Var(options.config.IPTables.MasqueradeBit, "iptables-masquerade-bit", util.Int32PtrDerefOr(options.config.IPTables.MasqueradeBit, 14), "If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with. Must be within the range [0, 31].") + fs.DurationVar(&options.config.IPTables.SyncPeriod.Duration, "iptables-sync-period", options.config.IPTables.SyncPeriod.Duration, "The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") + fs.DurationVar(&options.config.IPTables.MinSyncPeriod.Duration, "iptables-min-sync-period", options.config.IPTables.MinSyncPeriod.Duration, "The minimum interval of how often the iptables rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').") + fs.DurationVar(&options.config.ConfigSyncPeriod.Duration, "config-sync-period", options.config.ConfigSyncPeriod.Duration, "How often configuration from the apiserver is refreshed. Must be greater than 0.") + fs.BoolVar(&options.config.IPTables.MasqueradeAll, "masquerade-all", options.config.IPTables.MasqueradeAll, "If using the pure iptables proxy, SNAT everything") + fs.StringVar(&options.config.ClusterCIDR, "cluster-cidr", options.config.ClusterCIDR, "The CIDR range of pods in the cluster. It is used to bridge traffic coming from outside of the cluster. If not provided, no off-cluster bridging will be performed.") + fs.StringVar(&options.config.ClientConnection.ContentType, "kube-api-content-type", options.config.ClientConnection.ContentType, "Content type of requests sent to apiserver.") + fs.Float32Var(&options.config.ClientConnection.QPS, "kube-api-qps", options.config.ClientConnection.QPS, "QPS to use while talking with kubernetes apiserver") + fs.IntVar(&options.config.ClientConnection.Burst, "kube-api-burst", options.config.ClientConnection.Burst, "Burst to use while talking with kubernetes apiserver") + fs.DurationVar(&options.config.UDPIdleTimeout.Duration, "udp-timeout", options.config.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s'). Must be greater than 0. Only applicable for proxy-mode=userspace") + fs.Int32Var(&options.config.Conntrack.Max, "conntrack-max", options.config.Conntrack.Max, + "Maximum number of NAT connections to track (0 to leave as-is). This overrides conntrack-max-per-core and conntrack-min.") + fs.MarkDeprecated("conntrack-max", "This feature will be removed in a later release.") + fs.Int32Var(&options.config.Conntrack.MaxPerCore, "conntrack-max-per-core", options.config.Conntrack.MaxPerCore, + "Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore conntrack-min).") + fs.Int32Var(&options.config.Conntrack.Min, "conntrack-min", options.config.Conntrack.Min, + "Minimum number of conntrack entries to allocate, regardless of conntrack-max-per-core (set conntrack-max-per-core=0 to leave the limit as-is).") + fs.DurationVar(&options.config.Conntrack.TCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", options.config.Conntrack.TCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)") + fs.DurationVar( + &options.config.Conntrack.TCPCloseWaitTimeout.Duration, "conntrack-tcp-timeout-close-wait", + options.config.Conntrack.TCPCloseWaitTimeout.Duration, + "NAT timeout for TCP connections in the CLOSE_WAIT state") + + utilfeature.DefaultFeatureGate.AddFlag(fs) +} + +// Complete completes all the required options. +func (o Options) Complete() error { + if len(o.ConfigFile) == 0 { + glog.Warning("WARNING: all flags other than --config and --cleanup-iptables are deprecated. Please begin using a config file ASAP.") + o.applyDeprecatedHealthzPortToConfig() + } + return nil +} + +// Validate validates all the required options. +func (o Options) Validate(args []string) error { + if len(args) != 0 { + return errors.New("no arguments are supported") + } + + return nil +} + +func (o Options) Run() error { + config := o.config + + if len(o.ConfigFile) > 0 { + if c, err := loadConfigFromFile(o.ConfigFile); err != nil { + return err + } else { + config = c + // Make sure we apply the feature gate settings in the config file. + utilfeature.DefaultFeatureGate.Set(config.FeatureGates) + } + } + + proxyServer, err := NewProxyServer(config, o.CleanupAndExit, o.master) + if err != nil { + return err + } + + return proxyServer.Run() +} + +// applyDeprecatedHealthzPortToConfig sets o.config.HealthzBindAddress from +// flags passed on the command line based on the following rules: +// +// 1. If --healthz-port is 0, disable the healthz server. +// 2. Otherwise, use the value of --healthz-port for the port portion of +// o.config.HealthzBindAddress +func (o Options) applyDeprecatedHealthzPortToConfig() { + if o.healthzPort == 0 { + o.config.HealthzBindAddress = "" + return + } + + index := strings.Index(o.config.HealthzBindAddress, ":") + if index != -1 { + o.config.HealthzBindAddress = o.config.HealthzBindAddress[0:index] + } + + o.config.HealthzBindAddress = fmt.Sprintf("%s:%d", o.config.HealthzBindAddress, o.healthzPort) +} + +// loadConfigFromFile loads the contents of file and decodes it as a +// KubeProxyConfiguration object. +func loadConfigFromFile(file string) (*componentconfig.KubeProxyConfiguration, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + + configObj, gvk, err := api.Codecs.UniversalDecoder().Decode(data, nil, nil) + if err != nil { + return nil, err + } + config, ok := configObj.(*componentconfig.KubeProxyConfiguration) + if !ok { + return nil, fmt.Errorf("got unexpected config type: %v", gvk) + } + return config, nil +} + +func applyDefaults(in *componentconfig.KubeProxyConfiguration) (*componentconfig.KubeProxyConfiguration, error) { + external, err := api.Scheme.ConvertToVersion(in, v1alpha1.SchemeGroupVersion) + if err != nil { + return nil, err + } + + api.Scheme.Default(external) + + internal, err := api.Scheme.ConvertToVersion(external, componentconfig.SchemeGroupVersion) + if err != nil { + return nil, err + } + + out := internal.(*componentconfig.KubeProxyConfiguration) + + return out, nil } // NewProxyCommand creates a *cobra.Command object with default parameters func NewProxyCommand() *cobra.Command { - s := options.NewProxyConfig() - s.AddFlags(pflag.CommandLine) + opts := Options{ + config: new(componentconfig.KubeProxyConfiguration), + healthzPort: 10249, + } + cmd := &cobra.Command{ Use: "kube-proxy", Long: `The Kubernetes network proxy runs on each node. This @@ -126,19 +262,98 @@ environment variables specifying ports opened by the service proxy. There is an addon that provides cluster DNS for these cluster IPs. The user must create a service with the apiserver API to configure the proxy.`, Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(opts.Complete()) + cmdutil.CheckErr(opts.Validate(args)) + cmdutil.CheckErr(opts.Run()) }, } + var err error + opts.config, err = applyDefaults(opts.config) + if err != nil { + glog.Fatalf("unable to create flag defaults: %v", err) + } + + flags := cmd.Flags() + AddFlags(&opts, flags) + + cmd.MarkFlagFilename("config", "yaml", "yml", "json") + return cmd } -// NewProxyServerDefault creates a new ProxyServer object with default parameters. -func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, error) { - if c, err := configz.New("componentconfig"); err == nil { - c.Set(config.KubeProxyConfiguration) - } else { - glog.Errorf("unable to register configz: %s", err) +// ProxyServer represents all the parameters required to start the Kubernetes proxy server. All +// fields are required. +type ProxyServer struct { + Client clientset.Interface + EventClient v1core.EventsGetter + IptInterface utiliptables.Interface + Proxier proxy.ProxyProvider + Broadcaster record.EventBroadcaster + Recorder record.EventRecorder + ConntrackConfiguration componentconfig.KubeProxyConntrackConfiguration + Conntracker Conntracker // if nil, ignored + ProxyMode string + NodeRef *clientv1.ObjectReference + CleanupAndExit bool + HealthzBindAddress string + OOMScoreAdj *int32 + ResourceContainer string + ConfigSyncPeriod time.Duration + ServiceEventHandler proxyconfig.ServiceHandler + // TODO: Migrate all handlers to ServiceHandler types and + // get rid of this one. + ServiceHandler proxyconfig.ServiceConfigHandler + EndpointsEventHandler proxyconfig.EndpointsHandler +} + +// createClients creates a kube client and an event client from the given config and masterOverride. +// TODO remove masterOverride when CLI flags are removed. +func createClients(config componentconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, v1core.EventsGetter, error) { + if len(config.KubeConfigFile) == 0 && len(masterOverride) == 0 { + glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") } + + // This creates a client, first loading any specified kubeconfig + // file, and then overriding the Master flag, if non-empty. + kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: config.KubeConfigFile}, + &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterOverride}}).ClientConfig() + if err != nil { + return nil, nil, err + } + + kubeConfig.AcceptContentTypes = config.AcceptContentTypes + kubeConfig.ContentType = config.ContentType + kubeConfig.QPS = config.QPS + //TODO make config struct use int instead of int32? + kubeConfig.Burst = int(config.Burst) + + client, err := clientset.NewForConfig(kubeConfig) + if err != nil { + return nil, nil, err + } + + eventClient, err := clientgoclientset.NewForConfig(kubeConfig) + if err != nil { + return nil, nil, err + } + + return client, eventClient, nil +} + +// NewProxyServer returns a new ProxyServer. +func NewProxyServer(config *componentconfig.KubeProxyConfiguration, cleanupAndExit bool, master string) (*ProxyServer, error) { + if config == nil { + return nil, errors.New("config is required") + } + + if c, err := configz.New("componentconfig"); err == nil { + c.Set(config) + } else { + return nil, fmt.Errorf("unable to register configz: %s", err) + } + protocol := utiliptables.ProtocolIpv4 if net.ParseIP(config.BindAddress).To4() == nil { protocol = utiliptables.ProtocolIpv6 @@ -159,56 +374,11 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err } // We omit creation of pretty much everything if we run in cleanup mode - if config.CleanupAndExit { - return &ProxyServer{ - Config: config, - IptInterface: iptInterface, - }, nil + if cleanupAndExit { + return &ProxyServer{IptInterface: iptInterface}, nil } - // TODO(vmarmol): Use container config for this. - var oomAdjuster *oom.OOMAdjuster - if config.OOMScoreAdj != nil { - oomAdjuster = oom.NewOOMAdjuster() - if err := oomAdjuster.ApplyOOMScoreAdj(0, int(*config.OOMScoreAdj)); err != nil { - glog.V(2).Info(err) - } - } - - if config.ResourceContainer != "" { - // Run in its own container. - if err := resourcecontainer.RunInResourceContainer(config.ResourceContainer); err != nil { - glog.Warningf("Failed to start in resource-only container %q: %v", config.ResourceContainer, err) - } else { - glog.V(2).Infof("Running in resource-only container %q", config.ResourceContainer) - } - } - - // Create a Kube Client - // define api config source - if config.Kubeconfig == "" && config.Master == "" { - glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") - } - // This creates a client, first loading any specified kubeconfig - // file, and then overriding the Master flag, if non-empty. - kubeconfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: config.Kubeconfig}, - &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: config.Master}}).ClientConfig() - if err != nil { - return nil, err - } - - kubeconfig.ContentType = config.ContentType - // Override kubeconfig qps/burst settings from flags - kubeconfig.QPS = config.KubeAPIQPS - kubeconfig.Burst = int(config.KubeAPIBurst) - - client, err := clientset.NewForConfig(kubeconfig) - if err != nil { - glog.Fatalf("Invalid API configuration: %v", err) - } - - eventClient, err := clientgoclientset.NewForConfig(kubeconfig) + client, eventClient, err := createClients(config.ClientConnection, master) if err != nil { return nil, err } @@ -228,31 +398,34 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err proxyMode := getProxyMode(string(config.Mode), iptInterface, iptables.LinuxKernelCompatTester{}) if proxyMode == proxyModeIPTables { glog.V(0).Info("Using iptables Proxier.") - if config.IPTablesMasqueradeBit == nil { - // IPTablesMasqueradeBit must be specified or defaulted. - return nil, fmt.Errorf("Unable to read IPTablesMasqueradeBit from config") + if config.IPTables.MasqueradeBit == nil { + // MasqueradeBit must be specified or defaulted. + return nil, fmt.Errorf("unable to read IPTables MasqueradeBit from config") } + + // TODO this has side effects that should only happen when Run() is invoked. proxierIPTables, err := iptables.NewProxier( iptInterface, utilsysctl.New(), execer, - config.IPTablesSyncPeriod.Duration, - config.IPTablesMinSyncPeriod.Duration, - config.MasqueradeAll, - int(*config.IPTablesMasqueradeBit), + config.IPTables.SyncPeriod.Duration, + config.IPTables.MinSyncPeriod.Duration, + config.IPTables.MasqueradeAll, + int(*config.IPTables.MasqueradeBit), config.ClusterCIDR, hostname, getNodeIP(client, hostname), recorder, ) if err != nil { - glog.Fatalf("Unable to create proxier: %v", err) + return nil, fmt.Errorf("unable to create proxier: %v", err) } proxier = proxierIPTables serviceEventHandler = proxierIPTables endpointsEventHandler = proxierIPTables // No turning back. Remove artifacts that might still exist from the userspace Proxier. glog.V(0).Info("Tearing down userspace rules.") + // TODO this has side effects that should only happen when Run() is invoked. userspace.CleanupLeftovers(iptInterface) } else { glog.V(0).Info("Using userspace Proxier.") @@ -268,11 +441,11 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err netshInterface, *utilnet.ParsePortRangeOrDie(config.PortRange), // TODO @pires replace below with default values, if applicable - config.IPTablesSyncPeriod.Duration, + config.IPTables.SyncPeriod.Duration, config.UDPIdleTimeout.Duration, ) if err != nil { - glog.Fatalf("Unable to create proxier: %v", err) + return nil, fmt.Errorf("unable to create proxier: %v", err) } serviceHandler = proxierUserspace proxier = proxierUserspace @@ -282,18 +455,20 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err loadBalancer := userspace.NewLoadBalancerRR() // set EndpointsConfigHandler to our loadBalancer endpointsEventHandler = loadBalancer + + // TODO this has side effects that should only happen when Run() is invoked. proxierUserspace, err := userspace.NewProxier( loadBalancer, net.ParseIP(config.BindAddress), iptInterface, execer, *utilnet.ParsePortRangeOrDie(config.PortRange), - config.IPTablesSyncPeriod.Duration, - config.IPTablesMinSyncPeriod.Duration, + config.IPTables.SyncPeriod.Duration, + config.IPTables.MinSyncPeriod.Duration, config.UDPIdleTimeout.Duration, ) if err != nil { - glog.Fatalf("Unable to create proxier: %v", err) + return nil, fmt.Errorf("unable to create proxier: %v", err) } serviceHandler = proxierUserspace proxier = proxierUserspace @@ -301,6 +476,7 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err // Remove artifacts from the pure-iptables Proxier, if not on Windows. if runtime.GOOS != "windows" { glog.V(0).Info("Tearing down pure-iptables proxy rules.") + // TODO this has side effects that should only happen when Run() is invoked. iptables.CleanupLeftovers(iptInterface) } } @@ -310,73 +486,84 @@ func NewProxyServerDefault(config *options.ProxyServerConfig) (*ProxyServer, err iptInterface.AddReloadFunc(proxier.Sync) } - informerFactory := informers.NewSharedInformerFactory(client, config.ConfigSyncPeriod) - - // Create configs (i.e. Watches for Services and Endpoints) - // Note: RegisterHandler() calls need to happen before creation of Sources because sources - // only notify on changes, and the initial update (on process start) may be lost if no handlers - // are registered yet. - serviceConfig := proxyconfig.NewServiceConfig(informerFactory.Core().InternalVersion().Services(), config.ConfigSyncPeriod) - if serviceHandler != nil { - serviceConfig.RegisterHandler(serviceHandler) - } - if serviceEventHandler != nil { - serviceConfig.RegisterEventHandler(serviceEventHandler) - } - go serviceConfig.Run(wait.NeverStop) - - endpointsConfig := proxyconfig.NewEndpointsConfig(informerFactory.Core().InternalVersion().Endpoints(), config.ConfigSyncPeriod) - endpointsConfig.RegisterEventHandler(endpointsEventHandler) - go endpointsConfig.Run(wait.NeverStop) - - // This has to start after the calls to NewServiceConfig and NewEndpointsConfig because those - // functions must configure their shared informer event handlers first. - go informerFactory.Start(wait.NeverStop) - - config.NodeRef = &clientv1.ObjectReference{ + nodeRef := &clientv1.ObjectReference{ Kind: "Node", Name: hostname, UID: types.UID(hostname), Namespace: "", } - conntracker := realConntracker{} - - return NewProxyServer(client, eventClient, config, iptInterface, proxier, eventBroadcaster, recorder, conntracker, proxyMode) + return &ProxyServer{ + Client: client, + EventClient: eventClient, + IptInterface: iptInterface, + Proxier: proxier, + Broadcaster: eventBroadcaster, + Recorder: recorder, + ConntrackConfiguration: config.Conntrack, + Conntracker: &realConntracker{}, + ProxyMode: proxyMode, + NodeRef: nodeRef, + HealthzBindAddress: config.HealthzBindAddress, + OOMScoreAdj: config.OOMScoreAdj, + ResourceContainer: config.ResourceContainer, + ConfigSyncPeriod: config.ConfigSyncPeriod.Duration, + ServiceEventHandler: serviceEventHandler, + ServiceHandler: serviceHandler, + EndpointsEventHandler: endpointsEventHandler, + }, nil } // Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set). func (s *ProxyServer) Run() error { // remove iptables rules and exit - if s.Config.CleanupAndExit { + if s.CleanupAndExit { encounteredError := userspace.CleanupLeftovers(s.IptInterface) encounteredError = iptables.CleanupLeftovers(s.IptInterface) || encounteredError if encounteredError { - return errors.New("Encountered an error while tearing down rules.") + return errors.New("encountered an error while tearing down rules.") } return nil } + // TODO(vmarmol): Use container config for this. + var oomAdjuster *oom.OOMAdjuster + if s.OOMScoreAdj != nil { + oomAdjuster = oom.NewOOMAdjuster() + if err := oomAdjuster.ApplyOOMScoreAdj(0, int(*s.OOMScoreAdj)); err != nil { + glog.V(2).Info(err) + } + } + + if len(s.ResourceContainer) != 0 { + // Run in its own container. + if err := resourcecontainer.RunInResourceContainer(s.ResourceContainer); err != nil { + glog.Warningf("Failed to start in resource-only container %q: %v", s.ResourceContainer, err) + } else { + glog.V(2).Infof("Running in resource-only container %q", s.ResourceContainer) + } + } + s.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.EventClient.Events("")}) // Start up a webserver if requested - if s.Config.HealthzPort > 0 { + if len(s.HealthzBindAddress) > 0 { http.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "%s", s.ProxyMode) }) http.Handle("/metrics", prometheus.Handler()) configz.InstallHandler(http.DefaultServeMux) go wait.Until(func() { - err := http.ListenAndServe(s.Config.HealthzBindAddress+":"+strconv.Itoa(int(s.Config.HealthzPort)), nil) + err := http.ListenAndServe(s.HealthzBindAddress, nil) if err != nil { - glog.Errorf("Starting health server failed: %v", err) + utilruntime.HandleError(fmt.Errorf("starting health server failed: %v", err)) } }, 5*time.Second, wait.NeverStop) } // Tune conntrack, if requested if s.Conntracker != nil && runtime.GOOS != "windows" { - max, err := getConntrackMax(s.Config) + max, err := getConntrackMax(s.ConntrackConfiguration) if err != nil { return err } @@ -394,25 +581,48 @@ func (s *ProxyServer) Run() error { // TODO(random-liu): Remove this when the docker bug is fixed. const message = "DOCKER RESTART NEEDED (docker issue #24000): /sys is read-only: " + "cannot modify conntrack limits, problems may arise later." - s.Recorder.Eventf(s.Config.NodeRef, api.EventTypeWarning, err.Error(), message) + s.Recorder.Eventf(s.NodeRef, api.EventTypeWarning, err.Error(), message) } } - if s.Config.ConntrackTCPEstablishedTimeout.Duration > 0 { - timeout := int(s.Config.ConntrackTCPEstablishedTimeout.Duration / time.Second) + if s.ConntrackConfiguration.TCPEstablishedTimeout.Duration > 0 { + timeout := int(s.ConntrackConfiguration.TCPEstablishedTimeout.Duration / time.Second) if err := s.Conntracker.SetTCPEstablishedTimeout(timeout); err != nil { return err } } - if s.Config.ConntrackTCPCloseWaitTimeout.Duration > 0 { - timeout := int(s.Config.ConntrackTCPCloseWaitTimeout.Duration / time.Second) + if s.ConntrackConfiguration.TCPCloseWaitTimeout.Duration > 0 { + timeout := int(s.ConntrackConfiguration.TCPCloseWaitTimeout.Duration / time.Second) if err := s.Conntracker.SetTCPCloseWaitTimeout(timeout); err != nil { return err } } } + informerFactory := informers.NewSharedInformerFactory(s.Client, s.ConfigSyncPeriod) + + // Create configs (i.e. Watches for Services and Endpoints) + // Note: RegisterHandler() calls need to happen before creation of Sources because sources + // only notify on changes, and the initial update (on process start) may be lost if no handlers + // are registered yet. + serviceConfig := proxyconfig.NewServiceConfig(informerFactory.Core().InternalVersion().Services(), s.ConfigSyncPeriod) + if s.ServiceHandler != nil { + serviceConfig.RegisterHandler(s.ServiceHandler) + } + if s.ServiceEventHandler != nil { + serviceConfig.RegisterEventHandler(s.ServiceEventHandler) + } + go serviceConfig.Run(wait.NeverStop) + + endpointsConfig := proxyconfig.NewEndpointsConfig(informerFactory.Core().InternalVersion().Endpoints(), s.ConfigSyncPeriod) + endpointsConfig.RegisterEventHandler(s.EndpointsEventHandler) + go endpointsConfig.Run(wait.NeverStop) + + // This has to start after the calls to NewServiceConfig and NewEndpointsConfig because those + // functions must configure their shared informer event handlers first. + go informerFactory.Start(wait.NeverStop) + // Birth Cry after the birth is successful s.birthCry() @@ -421,17 +631,17 @@ func (s *ProxyServer) Run() error { return nil } -func getConntrackMax(config *options.ProxyServerConfig) (int, error) { - if config.ConntrackMax > 0 { - if config.ConntrackMaxPerCore > 0 { - return -1, fmt.Errorf("invalid config: ConntrackMax and ConntrackMaxPerCore are mutually exclusive") +func getConntrackMax(config componentconfig.KubeProxyConntrackConfiguration) (int, error) { + if config.Max > 0 { + if config.MaxPerCore > 0 { + return -1, fmt.Errorf("invalid config: Conntrack Max and Conntrack MaxPerCore are mutually exclusive") } glog.V(3).Infof("getConntrackMax: using absolute conntrack-max (deprecated)") - return int(config.ConntrackMax), nil + return int(config.Max), nil } - if config.ConntrackMaxPerCore > 0 { - floor := int(config.ConntrackMin) - scaled := int(config.ConntrackMaxPerCore) * runtime.NumCPU() + if config.MaxPerCore > 0 { + floor := int(config.Min) + scaled := int(config.MaxPerCore) * runtime.NumCPU() if scaled > floor { glog.V(3).Infof("getConntrackMax: using scaled conntrack-max-per-core") return scaled, nil @@ -447,7 +657,7 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, kcompat i return proxyModeUserspace } - if proxyMode != "" && proxyMode != proxyModeIPTables { + if len(proxyMode) > 0 && proxyMode != proxyModeIPTables { glog.Warningf("Flag proxy-mode=%q unknown, assuming iptables proxy", proxyMode) } @@ -458,7 +668,7 @@ func tryIPTablesProxy(iptver iptables.IPTablesVersioner, kcompat iptables.Kernel // guaranteed false on error, error only necessary for debugging useIPTablesProxy, err := iptables.CanUseIPTablesProxier(iptver, kcompat) if err != nil { - glog.Errorf("Can't determine whether to use iptables proxy, using userspace proxier: %v", err) + utilruntime.HandleError(fmt.Errorf("can't determine whether to use iptables proxy, using userspace proxier: %v", err)) return proxyModeUserspace } if useIPTablesProxy { @@ -470,7 +680,7 @@ func tryIPTablesProxy(iptver iptables.IPTablesVersioner, kcompat iptables.Kernel } func (s *ProxyServer) birthCry() { - s.Recorder.Eventf(s.Config.NodeRef, api.EventTypeNormal, "Starting", "Starting kube-proxy.") + s.Recorder.Eventf(s.NodeRef, api.EventTypeNormal, "Starting", "Starting kube-proxy.") } func getNodeIP(client clientset.Interface, hostname string) net.IP { diff --git a/cmd/kube-proxy/app/server_test.go b/cmd/kube-proxy/app/server_test.go index c2bb857e50..846894eeb1 100644 --- a/cmd/kube-proxy/app/server_test.go +++ b/cmd/kube-proxy/app/server_test.go @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/cmd/kube-proxy/app/options" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/util/iptables" @@ -131,77 +130,68 @@ func Test_getProxyMode(t *testing.T) { } } -// This test verifies that Proxy Server does not crash that means -// Config and iptinterface are not nil when CleanupAndExit is true. -// To avoid proxy crash: https://github.com/kubernetes/kubernetes/pull/14736 +// This test verifies that Proxy Server does not crash when CleanupAndExit is true. func TestProxyServerWithCleanupAndExit(t *testing.T) { - // creates default config - config := options.NewProxyConfig() + options := Options{ + config: &componentconfig.KubeProxyConfiguration{ + BindAddress: "0.0.0.0", + }, + CleanupAndExit: true, + } - // sets CleanupAndExit manually - config.CleanupAndExit = true + proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.master) - // creates new proxy server - proxyserver, err := NewProxyServerDefault(config) - - // verifies that nothing is nill except error assert.Nil(t, err) assert.NotNil(t, proxyserver) - assert.NotNil(t, proxyserver.Config) assert.NotNil(t, proxyserver.IptInterface) } func TestGetConntrackMax(t *testing.T) { ncores := runtime.NumCPU() testCases := []struct { - config componentconfig.KubeProxyConfiguration - expected int - err string + min int32 + max int32 + maxPerCore int32 + expected int + err string }{ { - config: componentconfig.KubeProxyConfiguration{}, expected: 0, }, { - config: componentconfig.KubeProxyConfiguration{ - ConntrackMax: 12345, - }, + max: 12345, expected: 12345, }, { - config: componentconfig.KubeProxyConfiguration{ - ConntrackMax: 12345, - ConntrackMaxPerCore: 67890, - }, - expected: -1, - err: "mutually exclusive", + max: 12345, + maxPerCore: 67890, + expected: -1, + err: "mutually exclusive", }, { - config: componentconfig.KubeProxyConfiguration{ - ConntrackMaxPerCore: 67890, // use this if Max is 0 - ConntrackMin: 1, // avoid 0 default - }, - expected: 67890 * ncores, + maxPerCore: 67890, // use this if Max is 0 + min: 1, // avoid 0 default + expected: 67890 * ncores, }, { - config: componentconfig.KubeProxyConfiguration{ - ConntrackMaxPerCore: 1, // ensure that Min is considered - ConntrackMin: 123456, - }, - expected: 123456, + maxPerCore: 1, // ensure that Min is considered + min: 123456, + expected: 123456, }, { - config: componentconfig.KubeProxyConfiguration{ - ConntrackMaxPerCore: 0, // leave system setting - ConntrackMin: 123456, - }, - expected: 0, + maxPerCore: 0, // leave system setting + min: 123456, + expected: 0, }, } for i, tc := range testCases { - cfg := options.ProxyServerConfig{KubeProxyConfiguration: tc.config} - x, e := getConntrackMax(&cfg) + cfg := componentconfig.KubeProxyConntrackConfiguration{ + Min: tc.min, + Max: tc.max, + MaxPerCore: tc.maxPerCore, + } + x, e := getConntrackMax(cfg) if e != nil { if tc.err == "" { t.Errorf("[%d] unexpected error: %v", i, e) diff --git a/cmd/kube-proxy/proxy.go b/cmd/kube-proxy/proxy.go index 6a5772a6fb..ab1d5df90d 100644 --- a/cmd/kube-proxy/proxy.go +++ b/cmd/kube-proxy/proxy.go @@ -17,43 +17,37 @@ limitations under the License. package main import ( - "fmt" + goflag "flag" "os" + "github.com/spf13/pflag" + "k8s.io/apiserver/pkg/server/healthz" - "k8s.io/apiserver/pkg/util/flag" + utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/cmd/kube-proxy/app" - "k8s.io/kubernetes/cmd/kube-proxy/app/options" _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration "k8s.io/kubernetes/pkg/version/verflag" - - "github.com/spf13/pflag" ) -func init() { - healthz.DefaultHealthz() -} - func main() { - config := options.NewProxyConfig() - config.AddFlags(pflag.CommandLine) + healthz.DefaultHealthz() - flag.InitFlags() + command := app.NewProxyCommand() + + // TODO: once we switch everything over to Cobra commands, we can go back to calling + // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the + // normalize func and add the go flag set by hand. + pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + // utilflag.InitFlags() logs.InitLogs() defer logs.FlushLogs() verflag.PrintAndExitIfRequested() - s, err := app.NewProxyServerDefault(config) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) - } - - if err = s.Run(); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) + if err := command.Execute(); err != nil { os.Exit(1) } } diff --git a/cmd/kubemark/BUILD b/cmd/kubemark/BUILD index 6ae97a6dc2..cde439603c 100644 --- a/cmd/kubemark/BUILD +++ b/cmd/kubemark/BUILD @@ -22,13 +22,11 @@ go_library( "//pkg/api:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", - "//pkg/client/informers/informers_generated/internalversion:go_default_library", "//pkg/client/metrics/prometheus:go_default_library", "//pkg/kubelet/cadvisor/testing:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/dockertools:go_default_library", "//pkg/kubemark:go_default_library", - "//pkg/proxy/config:go_default_library", "//pkg/util/iptables/testing:go_default_library", "//pkg/version/prometheus:go_default_library", "//vendor/github.com/golang/glog:go_default_library", diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go index 5c422d10db..1ef163dfe7 100644 --- a/cmd/kubemark/hollow-node.go +++ b/cmd/kubemark/hollow-node.go @@ -30,13 +30,11 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/client/clientset_generated/clientset" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/dockertools" "k8s.io/kubernetes/pkg/kubemark" - proxyconfig "k8s.io/kubernetes/pkg/proxy/config" fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing" _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration @@ -138,13 +136,6 @@ func main() { iptInterface := fakeiptables.NewFake() - informerFactory := informers.NewSharedInformerFactory(internalClientset, configResyncPeriod) - serviceConfig := proxyconfig.NewServiceConfig(informerFactory.Core().InternalVersion().Services(), configResyncPeriod) - serviceConfig.RegisterHandler(&kubemark.FakeProxyHandler{}) - - endpointsConfig := proxyconfig.NewEndpointsConfig(informerFactory.Core().InternalVersion().Endpoints(), configResyncPeriod) - endpointsConfig.RegisterEventHandler(&kubemark.FakeProxyHandler{}) - eventClient, err := clientgoclientset.NewForConfig(clientConfig) if err != nil { glog.Fatalf("Failed to create API Server client: %v", err) @@ -154,9 +145,6 @@ func main() { config.NodeName, internalClientset, eventClient, - endpointsConfig, - serviceConfig, - informerFactory, iptInterface, eventBroadcaster, recorder, diff --git a/hack/local-up-cluster.sh b/hack/local-up-cluster.sh index e7ac7f6389..b770a75b97 100755 --- a/hack/local-up-cluster.sh +++ b/hack/local-up-cluster.sh @@ -689,12 +689,20 @@ function start_kubelet { function start_kubeproxy { PROXY_LOG=${LOG_DIR}/kube-proxy.log + + cat < /tmp/kube-proxy.yaml +apiVersion: componentconfig/v1alpha1 +kind: KubeProxyConfiguration +clientConnection: + kubeconfig: ${CERT_DIR}/kube-proxy.kubeconfig +hostnameOverride: ${HOSTNAME_OVERRIDE} +featureGates: ${FEATURE_GATES} +EOF + sudo "${GO_OUT}/hyperkube" proxy \ - --v=${LOG_LEVEL} \ - --hostname-override="${HOSTNAME_OVERRIDE}" \ - --feature-gates="${FEATURE_GATES}" \ - --kubeconfig "$CERT_DIR"/kube-proxy.kubeconfig \ - --master="https://${API_HOST}:${API_SECURE_PORT}" >"${PROXY_LOG}" 2>&1 & + --config=/tmp/kube-proxy.yaml \ + --master="https://${API_HOST}:${API_SECURE_PORT}" >"${PROXY_LOG}" \ + --v=${LOG_LEVEL} 2>&1 & PROXY_PID=$! SCHEDULER_LOG=${LOG_DIR}/kube-scheduler.log diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go index 6ca58411e6..91ab2c5784 100644 --- a/pkg/apis/componentconfig/types.go +++ b/pkg/apis/componentconfig/types.go @@ -25,39 +25,90 @@ import ( "k8s.io/kubernetes/pkg/api" ) +// ClientConnectionConfiguration contains details for constructing a client. +type ClientConnectionConfiguration struct { + // kubeConfigFile is the path to a kubeconfig file. + KubeConfigFile string + // acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the + // default value of 'application/json'. This field will control all connections to the server used by a particular + // client. + AcceptContentTypes string + // contentType is the content type used when sending data to the server from this client. + ContentType string + // qps controls the number of queries per second allowed for this connection. + QPS float32 + // burst allows extra queries to accumulate when a client is exceeding its rate. + Burst int +} + +// KubeProxyIPTablesConfiguration contains iptables-related configuration +// details for the Kubernetes proxy server. +type KubeProxyIPTablesConfiguration struct { + // masqueradeBit is the bit of the iptables fwmark space to use for SNAT if using + // the pure iptables proxy mode. Values must be within the range [0, 31]. + MasqueradeBit *int32 + // masqueradeAll tells kube-proxy to SNAT everything if using the pure iptables proxy mode. + MasqueradeAll bool + // syncPeriod is the period that iptables rules are refreshed (e.g. '5s', '1m', + // '2h22m'). Must be greater than 0. + SyncPeriod metav1.Duration + // minSyncPeriod is the minimum period that iptables rules are refreshed (e.g. '5s', '1m', + // '2h22m'). + MinSyncPeriod metav1.Duration +} + +// KubeProxyConntrackConfiguration contains conntrack settings for +// the Kubernetes proxy server. +type KubeProxyConntrackConfiguration struct { + // max is the maximum number of NAT connections to track (0 to + // leave as-is). This takes precedence over conntrackMaxPerCore and conntrackMin. + Max int32 + // maxPerCore is the maximum number of NAT connections to track + // per CPU core (0 to leave the limit as-is and ignore conntrackMin). + MaxPerCore int32 + // min is the minimum value of connect-tracking records to allocate, + // regardless of conntrackMaxPerCore (set conntrackMaxPerCore=0 to leave the limit as-is). + Min int32 + // tcpEstablishedTimeout is how long an idle TCP connection will be kept open + // (e.g. '2s'). Must be greater than 0. + TCPEstablishedTimeout metav1.Duration + // tcpCloseWaitTimeout is how long an idle conntrack entry + // in CLOSE_WAIT state will remain in the conntrack + // table. (e.g. '60s'). Must be greater than 0 to set. + TCPCloseWaitTimeout metav1.Duration +} + +// KubeProxyConfiguration contains everything necessary to configure the +// Kubernetes proxy server. type KubeProxyConfiguration struct { metav1.TypeMeta + // featureGates is a comma-separated list of key=value pairs that control + // which alpha/beta features are enabled. + // + // TODO this really should be a map but that requires refactoring all + // components to use config files because local-up-cluster.sh only supports + // the --feature-gates flag right now, which is comma-separated key=value + // pairs. + FeatureGates string + // bindAddress is the IP address for the proxy server to serve on (set to 0.0.0.0 // for all interfaces) BindAddress string + // healthzBindAddress is the IP address and port for the health check server to serve on, + // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) + HealthzBindAddress string // clusterCIDR is the CIDR range of the pods in the cluster. It is used to // bridge traffic coming from outside of the cluster. If not provided, // no off-cluster bridging will be performed. ClusterCIDR string - // healthzBindAddress is the IP address for the health check server to serve on, - // defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces) - HealthzBindAddress string - // healthzPort is the port to bind the health check server. Use 0 to disable. - HealthzPort int32 // hostnameOverride, if non-empty, will be used as the identity instead of the actual hostname. HostnameOverride string - // iptablesMasqueradeBit is the bit of the iptables fwmark space to use for SNAT if using - // the pure iptables proxy mode. Values must be within the range [0, 31]. - IPTablesMasqueradeBit *int32 - // iptablesSyncPeriod is the period that iptables rules are refreshed (e.g. '5s', '1m', - // '2h22m'). Must be greater than 0. - IPTablesSyncPeriod metav1.Duration - // iptablesMinSyncPeriod is the minimum period that iptables rules are refreshed (e.g. '5s', '1m', - // '2h22m'). - IPTablesMinSyncPeriod metav1.Duration - // kubeconfigPath is the path to the kubeconfig file with authorization information (the - // master location is set by the master flag). - KubeconfigPath string - // masqueradeAll tells kube-proxy to SNAT everything if using the pure iptables proxy mode. - MasqueradeAll bool - // master is the address of the Kubernetes API server (overrides any value in kubeconfig) - Master string + // clientConnection specifies the kubeconfig file and client connection settings for the proxy + // server to use when communicating with the apiserver. + ClientConnection ClientConnectionConfiguration + // iptables contains iptables-related configuration options. + IPTables KubeProxyIPTablesConfiguration // oomScoreAdj is the oom-score-adj value for kube-proxy process. Values must be within // the range [-1000, 1000] OOMScoreAdj *int32 @@ -72,22 +123,11 @@ type KubeProxyConfiguration struct { // udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s'). // Must be greater than 0. Only applicable for proxyMode=userspace. UDPIdleTimeout metav1.Duration - // conntrackMax is the maximum number of NAT connections to track (0 to - // leave as-is). This takes precedence over conntrackMaxPerCore and conntrackMin. - ConntrackMax int32 - // conntrackMaxPerCore is the maximum number of NAT connections to track - // per CPU core (0 to leave the limit as-is and ignore conntrackMin). - ConntrackMaxPerCore int32 - // conntrackMin is the minimum value of connect-tracking records to allocate, - // regardless of conntrackMaxPerCore (set conntrackMaxPerCore=0 to leave the limit as-is). - ConntrackMin int32 - // conntrackTCPEstablishedTimeout is how long an idle TCP connection will be kept open - // (e.g. '2s'). Must be greater than 0. - ConntrackTCPEstablishedTimeout metav1.Duration - // conntrackTCPCloseWaitTimeout is how long an idle conntrack entry - // in CLOSE_WAIT state will remain in the conntrack - // table. (e.g. '60s'). Must be greater than 0 to set. - ConntrackTCPCloseWaitTimeout metav1.Duration + // conntrack contains conntrack-related configuration options. + Conntrack KubeProxyConntrackConfiguration + // configSyncPeriod is how often configuration from the apiserver is refreshed. Must be greater + // than 0. + ConfigSyncPeriod metav1.Duration } // Currently two modes of proxying are available: 'userspace' (older, stable) or 'iptables' diff --git a/pkg/apis/componentconfig/v1alpha1/defaults.go b/pkg/apis/componentconfig/v1alpha1/defaults.go index 63812f59a3..b6c15e8696 100644 --- a/pkg/apis/componentconfig/v1alpha1/defaults.go +++ b/pkg/apis/componentconfig/v1alpha1/defaults.go @@ -19,6 +19,7 @@ package v1alpha1 import ( "path/filepath" "runtime" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -59,14 +60,13 @@ func addDefaultingFuncs(scheme *kruntime.Scheme) error { } func SetDefaults_KubeProxyConfiguration(obj *KubeProxyConfiguration) { - if obj.BindAddress == "" { + if len(obj.BindAddress) == 0 { obj.BindAddress = "0.0.0.0" } - if obj.HealthzPort == 0 { - obj.HealthzPort = 10249 - } - if obj.HealthzBindAddress == "" { - obj.HealthzBindAddress = "127.0.0.1" + if len(obj.HealthzBindAddress) == 0 { + obj.HealthzBindAddress = "127.0.0.1:10249" + } else if !strings.Contains(obj.HealthzBindAddress, ":") { + obj.HealthzBindAddress = ":10249" } if obj.OOMScoreAdj == nil { temp := int32(qos.KubeProxyOOMScoreAdj) @@ -75,31 +75,31 @@ func SetDefaults_KubeProxyConfiguration(obj *KubeProxyConfiguration) { if obj.ResourceContainer == "" { obj.ResourceContainer = "/kube-proxy" } - if obj.IPTablesSyncPeriod.Duration == 0 { - obj.IPTablesSyncPeriod = metav1.Duration{Duration: 30 * time.Second} + if obj.IPTables.SyncPeriod.Duration == 0 { + obj.IPTables.SyncPeriod = metav1.Duration{Duration: 30 * time.Second} } zero := metav1.Duration{} if obj.UDPIdleTimeout == zero { obj.UDPIdleTimeout = metav1.Duration{Duration: 250 * time.Millisecond} } // If ConntrackMax is set, respect it. - if obj.ConntrackMax == 0 { + if obj.Conntrack.Max == 0 { // If ConntrackMax is *not* set, use per-core scaling. - if obj.ConntrackMaxPerCore == 0 { - obj.ConntrackMaxPerCore = 32 * 1024 + if obj.Conntrack.MaxPerCore == 0 { + obj.Conntrack.MaxPerCore = 32 * 1024 } - if obj.ConntrackMin == 0 { - obj.ConntrackMin = 128 * 1024 + if obj.Conntrack.Min == 0 { + obj.Conntrack.Min = 128 * 1024 } } - if obj.IPTablesMasqueradeBit == nil { + if obj.IPTables.MasqueradeBit == nil { temp := int32(14) - obj.IPTablesMasqueradeBit = &temp + obj.IPTables.MasqueradeBit = &temp } - if obj.ConntrackTCPEstablishedTimeout == zero { - obj.ConntrackTCPEstablishedTimeout = metav1.Duration{Duration: 24 * time.Hour} // 1 day (1/5 default) + if obj.Conntrack.TCPEstablishedTimeout == zero { + obj.Conntrack.TCPEstablishedTimeout = metav1.Duration{Duration: 24 * time.Hour} // 1 day (1/5 default) } - if obj.ConntrackTCPCloseWaitTimeout == zero { + if obj.Conntrack.TCPCloseWaitTimeout == zero { // See https://github.com/kubernetes/kubernetes/issues/32551. // // CLOSE_WAIT conntrack state occurs when the the Linux kernel @@ -120,7 +120,20 @@ func SetDefaults_KubeProxyConfiguration(obj *KubeProxyConfiguration) { // // We set CLOSE_WAIT to one hour by default to better match // typical server timeouts. - obj.ConntrackTCPCloseWaitTimeout = metav1.Duration{Duration: 1 * time.Hour} + obj.Conntrack.TCPCloseWaitTimeout = metav1.Duration{Duration: 1 * time.Hour} + } + if obj.ConfigSyncPeriod.Duration == 0 { + obj.ConfigSyncPeriod.Duration = 15 * time.Minute + } + + if len(obj.ClientConnection.ContentType) == 0 { + obj.ClientConnection.ContentType = "application/vnd.kubernetes.protobuf" + } + if obj.ClientConnection.QPS == 0.0 { + obj.ClientConnection.QPS = 5.0 + } + if obj.ClientConnection.Burst == 0 { + obj.ClientConnection.Burst = 10 } } diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go index a85cc657d9..325aea45c1 100644 --- a/pkg/apis/componentconfig/v1alpha1/types.go +++ b/pkg/apis/componentconfig/v1alpha1/types.go @@ -21,39 +21,90 @@ import ( "k8s.io/kubernetes/pkg/api/v1" ) +// ClientConnectionConfiguration contains details for constructing a client. +type ClientConnectionConfiguration struct { + // kubeConfigFile is the path to a kubeconfig file. + KubeConfigFile string `json:"kubeconfig"` + // acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the + // default value of 'application/json'. This field will control all connections to the server used by a particular + // client. + AcceptContentTypes string `json:"acceptContentTypes"` + // contentType is the content type used when sending data to the server from this client. + ContentType string `json:"contentType"` + // cps controls the number of queries per second allowed for this connection. + QPS float32 `json:"qps"` + // burst allows extra queries to accumulate when a client is exceeding its rate. + Burst int `json:"burst"` +} + +// KubeProxyIPTablesConfiguration contains iptables-related configuration +// details for the Kubernetes proxy server. +type KubeProxyIPTablesConfiguration struct { + // masqueradeBit is the bit of the iptables fwmark space to use for SNAT if using + // the pure iptables proxy mode. Values must be within the range [0, 31]. + MasqueradeBit *int32 `json:"masqueradeBit"` + // masqueradeAll tells kube-proxy to SNAT everything if using the pure iptables proxy mode. + MasqueradeAll bool `json:"masqueradeAll"` + // syncPeriod is the period that iptables rules are refreshed (e.g. '5s', '1m', + // '2h22m'). Must be greater than 0. + SyncPeriod metav1.Duration `json:"syncPeriod"` + // minSyncPeriod is the minimum period that iptables rules are refreshed (e.g. '5s', '1m', + // '2h22m'). + MinSyncPeriod metav1.Duration `json:"minSyncPeriod"` +} + +// KubeProxyConntrackConfiguration contains conntrack settings for +// the Kubernetes proxy server. +type KubeProxyConntrackConfiguration struct { + // max is the maximum number of NAT connections to track (0 to + // leave as-is). This takes precedence over conntrackMaxPerCore and conntrackMin. + Max int32 `json:"max"` + // maxPerCore is the maximum number of NAT connections to track + // per CPU core (0 to leave the limit as-is and ignore conntrackMin). + MaxPerCore int32 `json:"maxPerCore"` + // min is the minimum value of connect-tracking records to allocate, + // regardless of conntrackMaxPerCore (set conntrackMaxPerCore=0 to leave the limit as-is). + Min int32 `json:"min"` + // tcpEstablishedTimeout is how long an idle TCP connection will be kept open + // (e.g. '2s'). Must be greater than 0. + TCPEstablishedTimeout metav1.Duration `json:"tcpEstablishedTimeout"` + // tcpCloseWaitTimeout is how long an idle conntrack entry + // in CLOSE_WAIT state will remain in the conntrack + // table. (e.g. '60s'). Must be greater than 0 to set. + TCPCloseWaitTimeout metav1.Duration `json:"tcpCloseWaitTimeout"` +} + +// KubeProxyConfiguration contains everything necessary to configure the +// Kubernetes proxy server. type KubeProxyConfiguration struct { metav1.TypeMeta `json:",inline"` + // featureGates is a comma-separated list of key=value pairs that control + // which alpha/beta features are enabled. + // + // TODO this really should be a map but that requires refactoring all + // components to use config files because local-up-cluster.sh only supports + // the --feature-gates flag right now, which is comma-separated key=value + // pairs. + FeatureGates string `json:"featureGates"` + // bindAddress is the IP address for the proxy server to serve on (set to 0.0.0.0 // for all interfaces) BindAddress string `json:"bindAddress"` + // healthzBindAddress is the IP address and port for the health check server to serve on, + // defaulting to 127.0.0.1:10249 (set to 0.0.0.0 for all interfaces) + HealthzBindAddress string `json:"healthzBindAddress"` // clusterCIDR is the CIDR range of the pods in the cluster. It is used to // bridge traffic coming from outside of the cluster. If not provided, // no off-cluster bridging will be performed. ClusterCIDR string `json:"clusterCIDR"` - // healthzBindAddress is the IP address for the health check server to serve on, - // defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces) - HealthzBindAddress string `json:"healthzBindAddress"` - // healthzPort is the port to bind the health check server. Use 0 to disable. - HealthzPort int32 `json:"healthzPort"` // hostnameOverride, if non-empty, will be used as the identity instead of the actual hostname. HostnameOverride string `json:"hostnameOverride"` - // iptablesMasqueradeBit is the bit of the iptables fwmark space to use for SNAT if using - // the pure iptables proxy mode. Values must be within the range [0, 31]. - IPTablesMasqueradeBit *int32 `json:"iptablesMasqueradeBit"` - // iptablesSyncPeriod is the period that iptables rules are refreshed (e.g. '5s', '1m', - // '2h22m'). Must be greater than 0. - IPTablesSyncPeriod metav1.Duration `json:"iptablesSyncPeriodSeconds"` - // iptablesMinSyncPeriod is the minimum period that iptables rules are refreshed (e.g. '5s', '1m', - // '2h22m'). - IPTablesMinSyncPeriod metav1.Duration `json:"iptablesMinSyncPeriodSeconds"` - // kubeconfigPath is the path to the kubeconfig file with authorization information (the - // master location is set by the master flag). - KubeconfigPath string `json:"kubeconfigPath"` - // masqueradeAll tells kube-proxy to SNAT everything if using the pure iptables proxy mode. - MasqueradeAll bool `json:"masqueradeAll"` - // master is the address of the Kubernetes API server (overrides any value in kubeconfig) - Master string `json:"master"` + // clientConnection specifies the kubeconfig file and client connection settings for the proxy + // server to use when communicating with the apiserver. + ClientConnection ClientConnectionConfiguration `json:"clientConnection"` + // iptables contains iptables-related configuration options. + IPTables KubeProxyIPTablesConfiguration `json:"iptables"` // oomScoreAdj is the oom-score-adj value for kube-proxy process. Values must be within // the range [-1000, 1000] OOMScoreAdj *int32 `json:"oomScoreAdj"` @@ -68,22 +119,11 @@ type KubeProxyConfiguration struct { // udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s'). // Must be greater than 0. Only applicable for proxyMode=userspace. UDPIdleTimeout metav1.Duration `json:"udpTimeoutMilliseconds"` - // conntrackMax is the maximum number of NAT connections to track (0 to - // leave as-is). This takes precedence over conntrackMaxPerCore and conntrackMin. - ConntrackMax int32 `json:"conntrackMax"` - // conntrackMaxPerCore is the maximum number of NAT connections to track - // per CPU core (0 to leave the limit as-is and ignore conntrackMin). - ConntrackMaxPerCore int32 `json:"conntrackMaxPerCore"` - // conntrackMin is the minimum value of connect-tracking records to allocate, - // regardless of conntrackMaxPerCore (set conntrackMaxPerCore=0 to leave the limit as-is). - ConntrackMin int32 `json:"conntrackMin"` - // conntrackTCPEstablishedTimeout is how long an idle TCP connection - // will be kept open (e.g. '2s'). Must be greater than 0. - ConntrackTCPEstablishedTimeout metav1.Duration `json:"conntrackTCPEstablishedTimeout"` - // conntrackTCPCloseWaitTimeout is how long an idle conntrack entry - // in CLOSE_WAIT state will remain in the conntrack - // table. (e.g. '60s'). Must be greater than 0 to set. - ConntrackTCPCloseWaitTimeout metav1.Duration `json:"conntrackTCPCloseWaitTimeout"` + // conntrack contains conntrack-related configuration options. + Conntrack KubeProxyConntrackConfiguration `json:"conntrack"` + // configSyncPeriod is how often configuration from the apiserver is refreshed. Must be greater + // than 0. + ConfigSyncPeriod metav1.Duration `json:"configSyncPeriod"` } // Currently two modes of proxying are available: 'userspace' (older, stable) or 'iptables' diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go index 932f8d9638..d0bbf4aeb1 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go @@ -38,8 +38,14 @@ func init() { // Public to allow building arbitrary schemes. func RegisterConversions(scheme *runtime.Scheme) error { return scheme.AddGeneratedConversionFuncs( + Convert_v1alpha1_ClientConnectionConfiguration_To_componentconfig_ClientConnectionConfiguration, + Convert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientConnectionConfiguration, Convert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyConfiguration, Convert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyConfiguration, + Convert_v1alpha1_KubeProxyConntrackConfiguration_To_componentconfig_KubeProxyConntrackConfiguration, + Convert_componentconfig_KubeProxyConntrackConfiguration_To_v1alpha1_KubeProxyConntrackConfiguration, + Convert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration, + Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration, Convert_v1alpha1_KubeSchedulerConfiguration_To_componentconfig_KubeSchedulerConfiguration, Convert_componentconfig_KubeSchedulerConfiguration_To_v1alpha1_KubeSchedulerConfiguration, Convert_v1alpha1_KubeletAnonymousAuthentication_To_componentconfig_KubeletAnonymousAuthentication, @@ -61,28 +67,55 @@ func RegisterConversions(scheme *runtime.Scheme) error { ) } +func autoConvert_v1alpha1_ClientConnectionConfiguration_To_componentconfig_ClientConnectionConfiguration(in *ClientConnectionConfiguration, out *componentconfig.ClientConnectionConfiguration, s conversion.Scope) error { + out.KubeConfigFile = in.KubeConfigFile + out.AcceptContentTypes = in.AcceptContentTypes + out.ContentType = in.ContentType + out.QPS = in.QPS + out.Burst = in.Burst + return nil +} + +// Convert_v1alpha1_ClientConnectionConfiguration_To_componentconfig_ClientConnectionConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_ClientConnectionConfiguration_To_componentconfig_ClientConnectionConfiguration(in *ClientConnectionConfiguration, out *componentconfig.ClientConnectionConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_ClientConnectionConfiguration_To_componentconfig_ClientConnectionConfiguration(in, out, s) +} + +func autoConvert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientConnectionConfiguration(in *componentconfig.ClientConnectionConfiguration, out *ClientConnectionConfiguration, s conversion.Scope) error { + out.KubeConfigFile = in.KubeConfigFile + out.AcceptContentTypes = in.AcceptContentTypes + out.ContentType = in.ContentType + out.QPS = in.QPS + out.Burst = in.Burst + return nil +} + +// Convert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientConnectionConfiguration is an autogenerated conversion function. +func Convert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientConnectionConfiguration(in *componentconfig.ClientConnectionConfiguration, out *ClientConnectionConfiguration, s conversion.Scope) error { + return autoConvert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientConnectionConfiguration(in, out, s) +} + func autoConvert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyConfiguration(in *KubeProxyConfiguration, out *componentconfig.KubeProxyConfiguration, s conversion.Scope) error { + out.FeatureGates = in.FeatureGates out.BindAddress = in.BindAddress - out.ClusterCIDR = in.ClusterCIDR out.HealthzBindAddress = in.HealthzBindAddress - out.HealthzPort = in.HealthzPort + out.ClusterCIDR = in.ClusterCIDR out.HostnameOverride = in.HostnameOverride - out.IPTablesMasqueradeBit = (*int32)(unsafe.Pointer(in.IPTablesMasqueradeBit)) - out.IPTablesSyncPeriod = in.IPTablesSyncPeriod - out.IPTablesMinSyncPeriod = in.IPTablesMinSyncPeriod - out.KubeconfigPath = in.KubeconfigPath - out.MasqueradeAll = in.MasqueradeAll - out.Master = in.Master + if err := Convert_v1alpha1_ClientConnectionConfiguration_To_componentconfig_ClientConnectionConfiguration(&in.ClientConnection, &out.ClientConnection, s); err != nil { + return err + } + if err := Convert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration(&in.IPTables, &out.IPTables, s); err != nil { + return err + } out.OOMScoreAdj = (*int32)(unsafe.Pointer(in.OOMScoreAdj)) out.Mode = componentconfig.ProxyMode(in.Mode) out.PortRange = in.PortRange out.ResourceContainer = in.ResourceContainer out.UDPIdleTimeout = in.UDPIdleTimeout - out.ConntrackMax = in.ConntrackMax - out.ConntrackMaxPerCore = in.ConntrackMaxPerCore - out.ConntrackMin = in.ConntrackMin - out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout - out.ConntrackTCPCloseWaitTimeout = in.ConntrackTCPCloseWaitTimeout + if err := Convert_v1alpha1_KubeProxyConntrackConfiguration_To_componentconfig_KubeProxyConntrackConfiguration(&in.Conntrack, &out.Conntrack, s); err != nil { + return err + } + out.ConfigSyncPeriod = in.ConfigSyncPeriod return nil } @@ -92,27 +125,26 @@ func Convert_v1alpha1_KubeProxyConfiguration_To_componentconfig_KubeProxyConfigu } func autoConvert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyConfiguration(in *componentconfig.KubeProxyConfiguration, out *KubeProxyConfiguration, s conversion.Scope) error { + out.FeatureGates = in.FeatureGates out.BindAddress = in.BindAddress - out.ClusterCIDR = in.ClusterCIDR out.HealthzBindAddress = in.HealthzBindAddress - out.HealthzPort = in.HealthzPort + out.ClusterCIDR = in.ClusterCIDR out.HostnameOverride = in.HostnameOverride - out.IPTablesMasqueradeBit = (*int32)(unsafe.Pointer(in.IPTablesMasqueradeBit)) - out.IPTablesSyncPeriod = in.IPTablesSyncPeriod - out.IPTablesMinSyncPeriod = in.IPTablesMinSyncPeriod - out.KubeconfigPath = in.KubeconfigPath - out.MasqueradeAll = in.MasqueradeAll - out.Master = in.Master + if err := Convert_componentconfig_ClientConnectionConfiguration_To_v1alpha1_ClientConnectionConfiguration(&in.ClientConnection, &out.ClientConnection, s); err != nil { + return err + } + if err := Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration(&in.IPTables, &out.IPTables, s); err != nil { + return err + } out.OOMScoreAdj = (*int32)(unsafe.Pointer(in.OOMScoreAdj)) out.Mode = ProxyMode(in.Mode) out.PortRange = in.PortRange out.ResourceContainer = in.ResourceContainer out.UDPIdleTimeout = in.UDPIdleTimeout - out.ConntrackMax = in.ConntrackMax - out.ConntrackMaxPerCore = in.ConntrackMaxPerCore - out.ConntrackMin = in.ConntrackMin - out.ConntrackTCPEstablishedTimeout = in.ConntrackTCPEstablishedTimeout - out.ConntrackTCPCloseWaitTimeout = in.ConntrackTCPCloseWaitTimeout + if err := Convert_componentconfig_KubeProxyConntrackConfiguration_To_v1alpha1_KubeProxyConntrackConfiguration(&in.Conntrack, &out.Conntrack, s); err != nil { + return err + } + out.ConfigSyncPeriod = in.ConfigSyncPeriod return nil } @@ -121,6 +153,60 @@ func Convert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyConfigu return autoConvert_componentconfig_KubeProxyConfiguration_To_v1alpha1_KubeProxyConfiguration(in, out, s) } +func autoConvert_v1alpha1_KubeProxyConntrackConfiguration_To_componentconfig_KubeProxyConntrackConfiguration(in *KubeProxyConntrackConfiguration, out *componentconfig.KubeProxyConntrackConfiguration, s conversion.Scope) error { + out.Max = in.Max + out.MaxPerCore = in.MaxPerCore + out.Min = in.Min + out.TCPEstablishedTimeout = in.TCPEstablishedTimeout + out.TCPCloseWaitTimeout = in.TCPCloseWaitTimeout + return nil +} + +// Convert_v1alpha1_KubeProxyConntrackConfiguration_To_componentconfig_KubeProxyConntrackConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_KubeProxyConntrackConfiguration_To_componentconfig_KubeProxyConntrackConfiguration(in *KubeProxyConntrackConfiguration, out *componentconfig.KubeProxyConntrackConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_KubeProxyConntrackConfiguration_To_componentconfig_KubeProxyConntrackConfiguration(in, out, s) +} + +func autoConvert_componentconfig_KubeProxyConntrackConfiguration_To_v1alpha1_KubeProxyConntrackConfiguration(in *componentconfig.KubeProxyConntrackConfiguration, out *KubeProxyConntrackConfiguration, s conversion.Scope) error { + out.Max = in.Max + out.MaxPerCore = in.MaxPerCore + out.Min = in.Min + out.TCPEstablishedTimeout = in.TCPEstablishedTimeout + out.TCPCloseWaitTimeout = in.TCPCloseWaitTimeout + return nil +} + +// Convert_componentconfig_KubeProxyConntrackConfiguration_To_v1alpha1_KubeProxyConntrackConfiguration is an autogenerated conversion function. +func Convert_componentconfig_KubeProxyConntrackConfiguration_To_v1alpha1_KubeProxyConntrackConfiguration(in *componentconfig.KubeProxyConntrackConfiguration, out *KubeProxyConntrackConfiguration, s conversion.Scope) error { + return autoConvert_componentconfig_KubeProxyConntrackConfiguration_To_v1alpha1_KubeProxyConntrackConfiguration(in, out, s) +} + +func autoConvert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration(in *KubeProxyIPTablesConfiguration, out *componentconfig.KubeProxyIPTablesConfiguration, s conversion.Scope) error { + out.MasqueradeBit = (*int32)(unsafe.Pointer(in.MasqueradeBit)) + out.MasqueradeAll = in.MasqueradeAll + out.SyncPeriod = in.SyncPeriod + out.MinSyncPeriod = in.MinSyncPeriod + return nil +} + +// Convert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration is an autogenerated conversion function. +func Convert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration(in *KubeProxyIPTablesConfiguration, out *componentconfig.KubeProxyIPTablesConfiguration, s conversion.Scope) error { + return autoConvert_v1alpha1_KubeProxyIPTablesConfiguration_To_componentconfig_KubeProxyIPTablesConfiguration(in, out, s) +} + +func autoConvert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration(in *componentconfig.KubeProxyIPTablesConfiguration, out *KubeProxyIPTablesConfiguration, s conversion.Scope) error { + out.MasqueradeBit = (*int32)(unsafe.Pointer(in.MasqueradeBit)) + out.MasqueradeAll = in.MasqueradeAll + out.SyncPeriod = in.SyncPeriod + out.MinSyncPeriod = in.MinSyncPeriod + return nil +} + +// Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration is an autogenerated conversion function. +func Convert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration(in *componentconfig.KubeProxyIPTablesConfiguration, out *KubeProxyIPTablesConfiguration, s conversion.Scope) error { + return autoConvert_componentconfig_KubeProxyIPTablesConfiguration_To_v1alpha1_KubeProxyIPTablesConfiguration(in, out, s) +} + func autoConvert_v1alpha1_KubeSchedulerConfiguration_To_componentconfig_KubeSchedulerConfiguration(in *KubeSchedulerConfiguration, out *componentconfig.KubeSchedulerConfiguration, s conversion.Scope) error { out.Port = int32(in.Port) out.Address = in.Address diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go index 4ca4c4d306..e4c717473b 100644 --- a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go @@ -35,7 +35,10 @@ func init() { // to allow building arbitrary schemes. func RegisterDeepCopies(scheme *runtime.Scheme) error { return scheme.AddGeneratedDeepCopyFuncs( + conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_v1alpha1_ClientConnectionConfiguration, InType: reflect.TypeOf(&ClientConnectionConfiguration{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_v1alpha1_KubeProxyConfiguration, InType: reflect.TypeOf(&KubeProxyConfiguration{})}, + conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_v1alpha1_KubeProxyConntrackConfiguration, InType: reflect.TypeOf(&KubeProxyConntrackConfiguration{})}, + conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_v1alpha1_KubeProxyIPTablesConfiguration, InType: reflect.TypeOf(&KubeProxyIPTablesConfiguration{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_v1alpha1_KubeSchedulerConfiguration, InType: reflect.TypeOf(&KubeSchedulerConfiguration{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_v1alpha1_KubeletAnonymousAuthentication, InType: reflect.TypeOf(&KubeletAnonymousAuthentication{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_v1alpha1_KubeletAuthentication, InType: reflect.TypeOf(&KubeletAuthentication{})}, @@ -48,15 +51,22 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { ) } +func DeepCopy_v1alpha1_ClientConnectionConfiguration(in interface{}, out interface{}, c *conversion.Cloner) error { + { + in := in.(*ClientConnectionConfiguration) + out := out.(*ClientConnectionConfiguration) + *out = *in + return nil + } +} + func DeepCopy_v1alpha1_KubeProxyConfiguration(in interface{}, out interface{}, c *conversion.Cloner) error { { in := in.(*KubeProxyConfiguration) out := out.(*KubeProxyConfiguration) *out = *in - if in.IPTablesMasqueradeBit != nil { - in, out := &in.IPTablesMasqueradeBit, &out.IPTablesMasqueradeBit - *out = new(int32) - **out = **in + if err := DeepCopy_v1alpha1_KubeProxyIPTablesConfiguration(&in.IPTables, &out.IPTables, c); err != nil { + return err } if in.OOMScoreAdj != nil { in, out := &in.OOMScoreAdj, &out.OOMScoreAdj @@ -67,6 +77,29 @@ func DeepCopy_v1alpha1_KubeProxyConfiguration(in interface{}, out interface{}, c } } +func DeepCopy_v1alpha1_KubeProxyConntrackConfiguration(in interface{}, out interface{}, c *conversion.Cloner) error { + { + in := in.(*KubeProxyConntrackConfiguration) + out := out.(*KubeProxyConntrackConfiguration) + *out = *in + return nil + } +} + +func DeepCopy_v1alpha1_KubeProxyIPTablesConfiguration(in interface{}, out interface{}, c *conversion.Cloner) error { + { + in := in.(*KubeProxyIPTablesConfiguration) + out := out.(*KubeProxyIPTablesConfiguration) + *out = *in + if in.MasqueradeBit != nil { + in, out := &in.MasqueradeBit, &out.MasqueradeBit + *out = new(int32) + **out = **in + } + return nil + } +} + func DeepCopy_v1alpha1_KubeSchedulerConfiguration(in interface{}, out interface{}, c *conversion.Cloner) error { { in := in.(*KubeSchedulerConfiguration) diff --git a/pkg/apis/componentconfig/zz_generated.deepcopy.go b/pkg/apis/componentconfig/zz_generated.deepcopy.go index 733137549d..5d562f43b2 100644 --- a/pkg/apis/componentconfig/zz_generated.deepcopy.go +++ b/pkg/apis/componentconfig/zz_generated.deepcopy.go @@ -35,9 +35,12 @@ func init() { // to allow building arbitrary schemes. func RegisterDeepCopies(scheme *runtime.Scheme) error { return scheme.AddGeneratedDeepCopyFuncs( + conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_componentconfig_ClientConnectionConfiguration, InType: reflect.TypeOf(&ClientConnectionConfiguration{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_componentconfig_IPVar, InType: reflect.TypeOf(&IPVar{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_componentconfig_KubeControllerManagerConfiguration, InType: reflect.TypeOf(&KubeControllerManagerConfiguration{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_componentconfig_KubeProxyConfiguration, InType: reflect.TypeOf(&KubeProxyConfiguration{})}, + conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_componentconfig_KubeProxyConntrackConfiguration, InType: reflect.TypeOf(&KubeProxyConntrackConfiguration{})}, + conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_componentconfig_KubeProxyIPTablesConfiguration, InType: reflect.TypeOf(&KubeProxyIPTablesConfiguration{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_componentconfig_KubeSchedulerConfiguration, InType: reflect.TypeOf(&KubeSchedulerConfiguration{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_componentconfig_KubeletAnonymousAuthentication, InType: reflect.TypeOf(&KubeletAnonymousAuthentication{})}, conversion.GeneratedDeepCopyFunc{Fn: DeepCopy_componentconfig_KubeletAuthentication, InType: reflect.TypeOf(&KubeletAuthentication{})}, @@ -53,6 +56,15 @@ func RegisterDeepCopies(scheme *runtime.Scheme) error { ) } +func DeepCopy_componentconfig_ClientConnectionConfiguration(in interface{}, out interface{}, c *conversion.Cloner) error { + { + in := in.(*ClientConnectionConfiguration) + out := out.(*ClientConnectionConfiguration) + *out = *in + return nil + } +} + func DeepCopy_componentconfig_IPVar(in interface{}, out interface{}, c *conversion.Cloner) error { { in := in.(*IPVar) @@ -86,10 +98,8 @@ func DeepCopy_componentconfig_KubeProxyConfiguration(in interface{}, out interfa in := in.(*KubeProxyConfiguration) out := out.(*KubeProxyConfiguration) *out = *in - if in.IPTablesMasqueradeBit != nil { - in, out := &in.IPTablesMasqueradeBit, &out.IPTablesMasqueradeBit - *out = new(int32) - **out = **in + if err := DeepCopy_componentconfig_KubeProxyIPTablesConfiguration(&in.IPTables, &out.IPTables, c); err != nil { + return err } if in.OOMScoreAdj != nil { in, out := &in.OOMScoreAdj, &out.OOMScoreAdj @@ -100,6 +110,29 @@ func DeepCopy_componentconfig_KubeProxyConfiguration(in interface{}, out interfa } } +func DeepCopy_componentconfig_KubeProxyConntrackConfiguration(in interface{}, out interface{}, c *conversion.Cloner) error { + { + in := in.(*KubeProxyConntrackConfiguration) + out := out.(*KubeProxyConntrackConfiguration) + *out = *in + return nil + } +} + +func DeepCopy_componentconfig_KubeProxyIPTablesConfiguration(in interface{}, out interface{}, c *conversion.Cloner) error { + { + in := in.(*KubeProxyIPTablesConfiguration) + out := out.(*KubeProxyIPTablesConfiguration) + *out = *in + if in.MasqueradeBit != nil { + in, out := &in.MasqueradeBit, &out.MasqueradeBit + *out = new(int32) + **out = **in + } + return nil + } +} + func DeepCopy_componentconfig_KubeSchedulerConfiguration(in interface{}, out interface{}, c *conversion.Cloner) error { { in := in.(*KubeSchedulerConfiguration) diff --git a/pkg/kubemark/BUILD b/pkg/kubemark/BUILD index 2a508c8af9..cb3e44aea1 100644 --- a/pkg/kubemark/BUILD +++ b/pkg/kubemark/BUILD @@ -16,7 +16,6 @@ go_library( tags = ["automanaged"], deps = [ "//cmd/kube-proxy/app:go_default_library", - "//cmd/kube-proxy/app/options:go_default_library", "//cmd/kubelet/app:go_default_library", "//cmd/kubelet/app/options:go_default_library", "//pkg/api:go_default_library", @@ -24,14 +23,12 @@ go_library( "//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/client/clientset_generated/clientset:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", - "//pkg/client/informers/informers_generated/internalversion:go_default_library", "//pkg/kubelet:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", "//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/container/testing:go_default_library", "//pkg/kubelet/dockertools:go_default_library", "//pkg/kubelet/types:go_default_library", - "//pkg/proxy/config:go_default_library", "//pkg/util:go_default_library", "//pkg/util/io:go_default_library", "//pkg/util/iptables:go_default_library", @@ -43,7 +40,6 @@ go_library( "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/pkg/api/v1:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", diff --git a/pkg/kubemark/hollow_proxy.go b/pkg/kubemark/hollow_proxy.go index 4e5fecb361..24a56943c0 100644 --- a/pkg/kubemark/hollow_proxy.go +++ b/pkg/kubemark/hollow_proxy.go @@ -17,17 +17,15 @@ limitations under the License. package kubemark import ( + "time" + "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" v1core "k8s.io/client-go/kubernetes/typed/core/v1" clientv1 "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/tools/record" proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app" - "k8s.io/kubernetes/cmd/kube-proxy/app/options" "k8s.io/kubernetes/pkg/api" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" - informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" - proxyconfig "k8s.io/kubernetes/pkg/proxy/config" "k8s.io/kubernetes/pkg/util" utiliptables "k8s.io/kubernetes/pkg/util/iptables" @@ -58,34 +56,32 @@ func NewHollowProxyOrDie( nodeName string, client clientset.Interface, eventClient v1core.EventsGetter, - endpointsConfig *proxyconfig.EndpointsConfig, - serviceConfig *proxyconfig.ServiceConfig, - informerFactory informers.SharedInformerFactory, iptInterface utiliptables.Interface, broadcaster record.EventBroadcaster, recorder record.EventRecorder, ) *HollowProxy { // Create and start Hollow Proxy - config := options.NewProxyConfig() - config.OOMScoreAdj = util.Int32Ptr(0) - config.ResourceContainer = "" - config.NodeRef = &clientv1.ObjectReference{ + nodeRef := &clientv1.ObjectReference{ Kind: "Node", Name: nodeName, UID: types.UID(nodeName), Namespace: "", } - go endpointsConfig.Run(wait.NeverStop) - go serviceConfig.Run(wait.NeverStop) - go informerFactory.Start(wait.NeverStop) - - hollowProxy, err := proxyapp.NewProxyServer(client, eventClient, config, iptInterface, &FakeProxier{}, broadcaster, recorder, nil, "fake") - if err != nil { - glog.Fatalf("Error while creating ProxyServer: %v\n", err) - } return &HollowProxy{ - ProxyServer: hollowProxy, + ProxyServer: &proxyapp.ProxyServer{ + Client: client, + EventClient: eventClient, + IptInterface: iptInterface, + Proxier: &FakeProxier{}, + Broadcaster: broadcaster, + Recorder: recorder, + ProxyMode: "fake", + NodeRef: nodeRef, + OOMScoreAdj: util.Int32Ptr(0), + ResourceContainer: "", + ConfigSyncPeriod: 30 * time.Second, + }, } } diff --git a/pkg/util/iptables/iptables.go b/pkg/util/iptables/iptables.go index c7166bc7ea..9c8800f999 100644 --- a/pkg/util/iptables/iptables.go +++ b/pkg/util/iptables/iptables.go @@ -152,6 +152,8 @@ func New(exec utilexec.Interface, dbus utildbus.Interface, protocol Protocol) In waitFlag: getIPTablesWaitFlag(vstring), restoreWaitFlag: getIPTablesRestoreWaitFlag(exec), } + // TODO this needs to be moved to a separate Start() or Run() function so that New() has zero side + // effects. runner.connectToFirewallD() return runner }