github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #56446 from ironcladlou/gc-test-flakes 

Automatic merge from submit-queue (batch tested with PRs 56446, 56437). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Fix GC sync race condition

Remove faulty diff detection logic from GC sync which leads to a race
condition: If the GC's discovery client is returning a fully up to date
view of server resources during the very first GC sync, the sync
function will never sync monitors or reset the REST mapper unless
discovery changes again. This causes REST mapping to fail for any custom
types already present in discovery.

Fixes https://github.com/kubernetes/kubernetes/issues/56262.

```release-note
NONE
```

/cc @liggitt @caesarxuchao
pull/6/head
Kubernetes Submit Queue 2017-11-28 13:08:10 -08:00 committed by GitHub
parent 4034e6fbfd a62d07ce2a
commit 7ce780d52e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
pkg/controller/garbagecollector/garbagecollector.go
View File

@ -173,21 +173,14 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.DiscoveryInterface, p
// Get the current resource list from discovery.
newResources := GetDeletableResources(discoveryClient)
// Detect first or abnormal sync and try again later.
if oldResources == nil || len(oldResources) == 0 {
oldResources = newResources
return
}
// Decide whether discovery has reported a change.
if reflect.DeepEqual(oldResources, newResources) {
glog.V(5).Infof("no resource updates from discovery, skipping garbage collector sync")
return
}
// Something has changed, so track the new state and perform a sync.
// Something has changed, time to sync.
glog.V(2).Infof("syncing garbage collector with updated resources from discovery: %v", newResources)
oldResources = newResources
// Ensure workers are paused to avoid processing events before informers
// have resynced.
@ -212,9 +205,19 @@ func (gc *GarbageCollector) Sync(discoveryClient discovery.DiscoveryInterface, p
utilruntime.HandleError(fmt.Errorf("failed to sync resource monitors: %v", err))
return
}
// TODO: WaitForCacheSync can block forever during normal operation. Could
// pass a timeout channel, but we have to consider the implications of
// un-pausing the GC with a partially synced graph builder.
if !controller.WaitForCacheSync("garbage collector", stopCh, gc.dependencyGraphBuilder.IsSynced) {
utilruntime.HandleError(fmt.Errorf("timed out waiting for dependency graph builder sync during GC sync"))
return
}
// Finally, keep track of our new state. Do this after all preceding steps
// have succeeded to ensure we'll retry on subsequent syncs if an error
// occured.
oldResources = newResources
glog.V(2).Infof("synced garbage collector")
}, period, stopCh)
}