From fa60bbe8e6b1b50bfc82f13a8c1e7ed059d8ea99 Mon Sep 17 00:00:00 2001 From: Mike Danese Date: Tue, 15 Sep 2015 21:53:33 -0700 Subject: [PATCH] add flag to kubelet to ignore the cidr passed down by the apiserver on the master --- cluster/saltbase/salt/kubelet/default | 5 +- cmd/kubelet/app/server.go | 55 ++++++++++++------- contrib/mesos/pkg/executor/service/service.go | 2 + hack/verify-flags/known-flags.txt | 2 + pkg/kubelet/kubelet.go | 14 ++++- 5 files changed, 55 insertions(+), 23 deletions(-) diff --git a/cluster/saltbase/salt/kubelet/default b/cluster/saltbase/salt/kubelet/default index 8a20a774e7..35242a7169 100644 --- a/cluster/saltbase/salt/kubelet/default +++ b/cluster/saltbase/salt/kubelet/default @@ -22,6 +22,8 @@ {% set api_servers_with_port = api_servers + ":6443" -%} {% endif -%} +{% set master_kubelet_args = "" %} + {% set debugging_handlers = "--enable-debugging-handlers=true" -%} {% if grains['roles'][0] == 'kubernetes-master' -%} @@ -31,6 +33,7 @@ # running on the master. {% if grains.kubelet_api_servers is defined -%} {% set api_servers_with_port = "--api-servers=https://" + grains.kubelet_api_servers -%} + {% set master_kubelet_args = master_kubelet_args + "--register-schedulable=false --reconcile-cidr=false" -%} {% else -%} {% set api_servers_with_port = "" -%} {% endif -%} @@ -114,4 +117,4 @@ {% endif -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration -DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{cpu_cfs_quota}} {{network_plugin}} {{test_args}}" +DAEMON_ARGS="{{daemon_args}} {{api_servers_with_port}} {{debugging_handlers}} {{hostname_override}} {{cloud_provider}} {{config}} {{manifest_url}} --allow-privileged={{pillar['allow_privileged']}} {{pillar['log_level']}} {{cluster_dns}} {{cluster_domain}} {{docker_root}} {{kubelet_root}} {{configure_cbr0}} {{cgroup_root}} {{system_container}} {{pod_cidr}} {{ master_kubelet_args }} {{cpu_cfs_quota}} {{network_plugin}} {{test_args}}" diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go index cd4d0cd4fc..1e517e91a4 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -119,6 +119,7 @@ type KubeletServer struct { Port uint ReadOnlyPort uint RegisterNode bool + RegisterSchedulable bool RegistryBurst int RegistryPullQPS float64 ResolverConfig string @@ -133,6 +134,7 @@ type KubeletServer struct { SystemContainer string TLSCertFile string TLSPrivateKeyFile string + ReconcileCIDR bool // Flags intended for testing // Is the kubelet containerized? @@ -192,16 +194,18 @@ func NewKubeletServer() *KubeletServer { NodeStatusUpdateFrequency: 10 * time.Second, OOMScoreAdj: qos.KubeletOOMScoreAdj, PodInfraContainerImage: dockertools.PodInfraContainerImage, - Port: ports.KubeletPort, - ReadOnlyPort: ports.KubeletReadOnlyPort, - RegisterNode: true, // will be ignored if no apiserver is configured - RegistryBurst: 10, - ResourceContainer: "/kubelet", - RktPath: "", - RktStage1Image: "", - RootDirectory: defaultRootDir, - SyncFrequency: 10 * time.Second, - SystemContainer: "", + Port: ports.KubeletPort, + ReadOnlyPort: ports.KubeletReadOnlyPort, + RegisterNode: true, // will be ignored if no apiserver is configured + RegisterSchedulable: true, + RegistryBurst: 10, + ResourceContainer: "/kubelet", + RktPath: "", + RktStage1Image: "", + RootDirectory: defaultRootDir, + SyncFrequency: 10 * time.Second, + SystemContainer: "", + ReconcileCIDR: true, } } @@ -279,6 +283,8 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) { fs.Float64Var(&s.ChaosChance, "chaos-chance", s.ChaosChance, "If > 0.0, introduce random client errors and latency. Intended for testing. [default=0.0]") fs.BoolVar(&s.Containerized, "containerized", s.Containerized, "Experimental support for running kubelet in a container. Intended for testing. [default=false]") fs.Uint64Var(&s.MaxOpenFiles, "max-open-files", 1000000, "Number of files that can be opened by Kubelet process. [default=1000000]") + fs.BoolVar(&s.ReconcileCIDR, "reconcile-cidr", s.ReconcileCIDR, "Reconcile node CIDR with the CIDR specified by the API server. No-op if register-node or configure-cbr0 is false. [default=true]") + fs.BoolVar(&s.RegisterSchedulable, "register-schedulable", s.RegisterSchedulable, "Register the node as schedulable. No-op if register-node is false. [default=true]") } // KubeletConfig returns a KubeletConfig suitable for being run, or an error if the server setup @@ -385,10 +391,12 @@ func (s *KubeletServer) KubeletConfig() (*KubeletConfig, error) { OOMAdjuster: oom.NewOOMAdjuster(), OSInterface: kubecontainer.RealOS{}, PodCIDR: s.PodCIDR, + ReconcileCIDR: s.ReconcileCIDR, PodInfraContainerImage: s.PodInfraContainerImage, Port: s.Port, ReadOnlyPort: s.ReadOnlyPort, RegisterNode: s.RegisterNode, + RegisterSchedulable: s.RegisterSchedulable, RegistryBurst: s.RegistryBurst, RegistryPullQPS: s.RegistryPullQPS, ResolverConfig: s.ResolverConfig, @@ -644,17 +652,18 @@ func SimpleKubelet(client *client.Client, OOMAdjuster: oom.NewFakeOOMAdjuster(), OSInterface: osInterface, PodInfraContainerImage: dockertools.PodInfraContainerImage, - Port: port, - ReadOnlyPort: readOnlyPort, - RegisterNode: true, - ResolverConfig: kubelet.ResolvConfDefault, - ResourceContainer: "/kubelet", - RootDirectory: rootDir, - SyncFrequency: syncFrequency, - SystemContainer: "", - TLSOptions: tlsOptions, - Writer: &io.StdWriter{}, - VolumePlugins: volumePlugins, + Port: port, + ReadOnlyPort: readOnlyPort, + RegisterNode: true, + RegisterSchedulable: true, + ResolverConfig: kubelet.ResolvConfDefault, + ResourceContainer: "/kubelet", + RootDirectory: rootDir, + SyncFrequency: syncFrequency, + SystemContainer: "", + TLSOptions: tlsOptions, + Writer: &io.StdWriter{}, + VolumePlugins: volumePlugins, } return &kcfg } @@ -827,12 +836,14 @@ type KubeletConfig struct { OOMAdjuster *oom.OOMAdjuster OSInterface kubecontainer.OSInterface PodCIDR string + ReconcileCIDR bool PodConfig *config.PodConfig PodInfraContainerImage string Port uint ReadOnlyPort uint Recorder record.EventRecorder RegisterNode bool + RegisterSchedulable bool RegistryBurst int RegistryPullQPS float64 ResolverConfig string @@ -890,6 +901,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod gcPolicy, pc.SeenAllSources, kc.RegisterNode, + kc.RegisterSchedulable, kc.StandaloneMode, kc.ClusterDomain, kc.ClusterDNS, @@ -916,6 +928,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod kc.SystemContainer, kc.ConfigureCBR0, kc.PodCIDR, + kc.ReconcileCIDR, kc.MaxPods, kc.DockerExecHandler, kc.ResolverConfig, diff --git a/contrib/mesos/pkg/executor/service/service.go b/contrib/mesos/pkg/executor/service/service.go index c473e0a1b1..8b681310e6 100644 --- a/contrib/mesos/pkg/executor/service/service.go +++ b/contrib/mesos/pkg/executor/service/service.go @@ -322,6 +322,7 @@ func (ks *KubeletExecutorServer) createAndInitKubelet( gcPolicy, pc.SeenAllSources, kc.RegisterNode, + kc.RegisterSchedulable, kc.StandaloneMode, kc.ClusterDomain, net.IP(kc.ClusterDNS), @@ -348,6 +349,7 @@ func (ks *KubeletExecutorServer) createAndInitKubelet( kc.SystemContainer, kc.ConfigureCBR0, kc.PodCIDR, + kc.ReconcileCIDR, kc.MaxPods, kc.DockerExecHandler, kc.ResolverConfig, diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 9b07feb220..3db0d4117f 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -301,4 +301,6 @@ retry_time file_content_in_loop cpu-cfs-quota terminated-pod-gc-threshold +reconcile-cidr +register-schedulable diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index ff1b3cbca4..a403de511f 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -150,6 +150,7 @@ func NewMainKubelet( containerGCPolicy ContainerGCPolicy, sourcesReady SourcesReadyFn, registerNode bool, + registerSchedulable bool, standaloneMode bool, clusterDomain string, clusterDNS net.IP, @@ -176,6 +177,7 @@ func NewMainKubelet( systemContainer string, configureCBR0 bool, podCIDR string, + reconcileCIDR bool, pods int, dockerExecHandler dockertools.ExecHandler, resolverConfig string, @@ -263,6 +265,7 @@ func NewMainKubelet( httpClient: &http.Client{}, sourcesReady: sourcesReady, registerNode: registerNode, + registerSchedulable: registerSchedulable, standaloneMode: standaloneMode, clusterDomain: clusterDomain, clusterDNS: clusterDNS, @@ -290,6 +293,7 @@ func NewMainKubelet( writer: writer, configureCBR0: configureCBR0, podCIDR: podCIDR, + reconcileCIDR: reconcileCIDR, pods: pods, syncLoopMonitor: util.AtomicValue{}, resolverConfig: resolverConfig, @@ -465,6 +469,8 @@ type Kubelet struct { // Set to true to have the node register itself with the apiserver. registerNode bool + // Set to true to have the node register itself as schedulable. + registerSchedulable bool // for internal book keeping; access only from within registerWithApiserver registrationCompleted bool @@ -574,6 +580,7 @@ type Kubelet struct { // the correct state. configureCBR0 bool podCIDR string + reconcileCIDR bool // Number of Pods which can be run by this Kubelet pods int @@ -819,6 +826,9 @@ func (kl *Kubelet) initialNodeStatus() (*api.Node, error) { Name: kl.nodeName, Labels: map[string]string{"kubernetes.io/hostname": kl.hostname}, }, + Spec: api.NodeSpec{ + Unschedulable: !kl.registerSchedulable, + }, } if kl.cloud != nil { instances, ok := kl.cloud.Instances() @@ -2453,7 +2463,9 @@ func (kl *Kubelet) tryUpdateNodeStatus() error { return fmt.Errorf("no node instance returned for %q", kl.nodeName) } kl.networkConfigMutex.Lock() - kl.podCIDR = node.Spec.PodCIDR + if kl.reconcileCIDR { + kl.podCIDR = node.Spec.PodCIDR + } kl.networkConfigMutex.Unlock() if err := kl.setNodeStatus(node); err != nil {