From 29dd7e3bb80e8a80bb9f37437066cab1575ceb8d Mon Sep 17 00:00:00 2001 From: David Oppenheimer Date: Thu, 1 Oct 2015 02:16:08 -0700 Subject: [PATCH] Fix race condition in DaemonSet controller. Fixes #14693. --- pkg/controller/daemon/controller.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/controller/daemon/controller.go b/pkg/controller/daemon/controller.go index f55307432a..19d95a5984 100644 --- a/pkg/controller/daemon/controller.go +++ b/pkg/controller/daemon/controller.go @@ -40,10 +40,17 @@ import ( const ( // Daemon sets will periodically check that their daemon pods are running as expected. FullDaemonSetResyncPeriod = 30 * time.Second // TODO: Figure out if this time seems reasonable. + // Nodes don't need relisting. FullNodeResyncPeriod = 0 + // Daemon pods don't need relisting. FullDaemonPodResyncPeriod = 0 + + // We must avoid counting pods until the pod store has synced. If it hasn't synced, to + // avoid a hot loop, we'll wait this long between checks. + PodStoreSyncedPollPeriod = 100 * time.Millisecond + // If sending a status upate to API server fails, we retry a finite number of times. StatusUpdateRetries = 1 ) @@ -70,6 +77,10 @@ type DaemonSetsController struct { podController *framework.Controller // Watches changes to all nodes. nodeController *framework.Controller + // podStoreSynced returns true if the pod store has been synced at least once. + // Added as a member to the struct to allow injection for testing. + podStoreSynced func() bool + // Daemon sets that need to be synced. queue *workqueue.Type } @@ -155,6 +166,7 @@ func NewDaemonSetsController(kubeClient client.Interface) *DaemonSetsController }, ) dsc.syncHandler = dsc.syncDaemonSet + dsc.podStoreSynced = dsc.podController.HasSynced return dsc } @@ -467,6 +479,13 @@ func (dsc *DaemonSetsController) syncDaemonSet(key string) error { return nil } ds := obj.(*experimental.DaemonSet) + if !dsc.podStoreSynced() { + // Sleep so we give the pod reflector goroutine a chance to run. + time.Sleep(PodStoreSyncedPollPeriod) + glog.Infof("Waiting for pods controller to sync, requeuing ds %v", ds.Name) + dsc.enqueueDaemonSet(&ds) + return nil + } // Don't process a daemon set until all its creations and deletions have been processed. // For example if daemon set foo asked for 3 new daemon pods in the previous call to manage,