From 7c6213e9221265a0e3e062d3094d1c6319884625 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Thu, 14 Jun 2018 17:55:29 +0800 Subject: [PATCH 1/2] set leader election client timeout --- cmd/kube-controller-manager/app/options/options.go | 5 ++++- cmd/kube-scheduler/app/options/options.go | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/kube-controller-manager/app/options/options.go b/cmd/kube-controller-manager/app/options/options.go index d4149aa5ba..56593fdea6 100644 --- a/cmd/kube-controller-manager/app/options/options.go +++ b/cmd/kube-controller-manager/app/options/options.go @@ -432,7 +432,10 @@ func (s KubeControllerManagerOptions) Config(allControllers []string, disabledBy return nil, err } - leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election")) + // shallow copy, do not modify the kubeconfig.Timeout. + config := *kubeconfig + config.Timeout = s.GenericComponent.LeaderElection.RenewDeadline.Duration + leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "leader-election")) eventRecorder := createRecorder(client, KubeControllerManagerUserAgent) diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 431354d61c..9023fef445 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -21,6 +21,7 @@ import ( "net" "os" "strconv" + "time" "github.com/golang/glog" "github.com/spf13/pflag" @@ -203,7 +204,7 @@ func (o *Options) Config() (*schedulerappconfig.Config, error) { } // prepare kube clients. - client, leaderElectionClient, eventClient, err := createClients(c.ComponentConfig.ClientConnection, o.Master) + client, leaderElectionClient, eventClient, err := createClients(o.ComponentConfig.ClientConnection, o.Master, o.ComponentConfig.LeaderElection.RenewDeadline.Duration) if err != nil { return nil, err } @@ -264,7 +265,7 @@ func makeLeaderElectionConfig(config componentconfig.KubeSchedulerLeaderElection // createClients creates a kube client and an event client from the given config and masterOverride. // TODO remove masterOverride when CLI flags are removed. -func createClients(config componentconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, clientset.Interface, v1core.EventsGetter, error) { +func createClients(config componentconfig.ClientConnectionConfiguration, masterOverride string, timeout time.Duration) (clientset.Interface, clientset.Interface, v1core.EventsGetter, error) { if len(config.KubeConfigFile) == 0 && len(masterOverride) == 0 { glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") } @@ -289,7 +290,10 @@ func createClients(config componentconfig.ClientConnectionConfiguration, masterO return nil, nil, nil, err } - leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeConfig, "leader-election")) + // shallow copy, do not modify the kubeConfig.Timeout. + restConfig := *kubeConfig + restConfig.Timeout = timeout + leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(&restConfig, "leader-election")) if err != nil { return nil, nil, nil, err } From 90b287c12d921e4e08e522dbe214f5587f147956 Mon Sep 17 00:00:00 2001 From: xuzhonghu Date: Wed, 20 Jun 2018 15:44:31 +0800 Subject: [PATCH 2/2] leaderelection: set timeout for tryAcquireOrRenew --- cmd/kube-scheduler/app/options/options.go | 2 +- .../tools/leaderelection/leaderelection.go | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cmd/kube-scheduler/app/options/options.go b/cmd/kube-scheduler/app/options/options.go index 9023fef445..9aa852c223 100644 --- a/cmd/kube-scheduler/app/options/options.go +++ b/cmd/kube-scheduler/app/options/options.go @@ -204,7 +204,7 @@ func (o *Options) Config() (*schedulerappconfig.Config, error) { } // prepare kube clients. - client, leaderElectionClient, eventClient, err := createClients(o.ComponentConfig.ClientConnection, o.Master, o.ComponentConfig.LeaderElection.RenewDeadline.Duration) + client, leaderElectionClient, eventClient, err := createClients(c.ComponentConfig.ClientConnection, o.Master, c.ComponentConfig.LeaderElection.RenewDeadline.Duration) if err != nil { return nil, err } diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go index a7aa60e76d..9f8913af57 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/leaderelection.go @@ -207,8 +207,20 @@ func (le *LeaderElector) renew(ctx context.Context) { timeoutCtx, timeoutCancel := context.WithTimeout(ctx, le.config.RenewDeadline) defer timeoutCancel() err := wait.PollImmediateUntil(le.config.RetryPeriod, func() (bool, error) { - return le.tryAcquireOrRenew(), nil + done := make(chan bool, 1) + go func() { + defer close(done) + done <- le.tryAcquireOrRenew() + }() + + select { + case <-timeoutCtx.Done(): + return false, fmt.Errorf("failed to tryAcquireOrRenew %s", timeoutCtx.Err()) + case result := <-done: + return result, nil + } }, timeoutCtx.Done()) + le.maybeReportTransition() desc := le.config.Lock.Describe() if err == nil {