github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add watchdog for leader election logic getting wedged

pull/58/head
Bobby (Babak) Salamat 2018-11-15 11:02:14 -08:00
parent 076adbf4a2
commit 34b4a5bbee
2 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/kube-scheduler/app/options/options.go
View File

@ -286,6 +286,8 @@ func makeLeaderElectionConfig(config kubeschedulerconfig.KubeSchedulerLeaderElec
LeaseDuration: config.LeaseDuration.Duration, LeaseDuration: config.LeaseDuration.Duration,
RenewDeadline: config.RenewDeadline.Duration, RenewDeadline: config.RenewDeadline.Duration,
RetryPeriod: config.RetryPeriod.Duration, RetryPeriod: config.RetryPeriod.Duration,
WatchDog: leaderelection.NewLeaderHealthzAdaptor(time.Second * 20),
Name: "kube-scheduler",
}, nil }, nil
} }

14
cmd/kube-scheduler/app/server.go
View File

@ -200,10 +200,16 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error
cc.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: cc.EventClient.Events("")}) cc.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: cc.EventClient.Events("")})
} }
// Setup healthz checks.
var checks []healthz.HealthzChecker
if cc.ComponentConfig.LeaderElection.LeaderElect {
checks = append(checks, cc.LeaderElection.WatchDog)
}
// Start up the healthz server. // Start up the healthz server.
if cc.InsecureServing != nil { if cc.InsecureServing != nil {
separateMetrics := cc.InsecureMetricsServing != nil separateMetrics := cc.InsecureMetricsServing != nil
handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, separateMetrics), nil, nil) handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, separateMetrics, checks...), nil, nil)
if err := cc.InsecureServing.Serve(handler, 0, stopCh); err != nil { if err := cc.InsecureServing.Serve(handler, 0, stopCh); err != nil {
return fmt.Errorf("failed to start healthz server: %v", err) return fmt.Errorf("failed to start healthz server: %v", err)
} }
@ -215,7 +221,7 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error
} }
} }
if cc.SecureServing != nil { if cc.SecureServing != nil {
handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, false), cc.Authentication.Authenticator, cc.Authorization.Authorizer) handler := buildHandlerChain(newHealthzHandler(&cc.ComponentConfig, false, checks...), cc.Authentication.Authenticator, cc.Authorization.Authorizer)
if err := cc.SecureServing.Serve(handler, 0, stopCh); err != nil { if err := cc.SecureServing.Serve(handler, 0, stopCh); err != nil {
// fail early for secure handlers, removing the old error loop from above // fail early for secure handlers, removing the old error loop from above
return fmt.Errorf("failed to start healthz server: %v", err) return fmt.Errorf("failed to start healthz server: %v", err)
@ -313,9 +319,9 @@ func newMetricsHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration) h
// newHealthzServer creates a healthz server from the config, and will also // newHealthzServer creates a healthz server from the config, and will also
// embed the metrics handler if the healthz and metrics address configurations // embed the metrics handler if the healthz and metrics address configurations
// are the same. // are the same.
func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, separateMetrics bool) http.Handler { func newHealthzHandler(config *kubeschedulerconfig.KubeSchedulerConfiguration, separateMetrics bool, checks ...healthz.HealthzChecker) http.Handler {
pathRecorderMux := mux.NewPathRecorderMux("kube-scheduler") pathRecorderMux := mux.NewPathRecorderMux("kube-scheduler")
healthz.InstallHandler(pathRecorderMux) healthz.InstallHandler(pathRecorderMux, checks...)
if !separateMetrics { if !separateMetrics {
installMetricHandler(pathRecorderMux) installMetricHandler(pathRecorderMux)
} }