From 9b068706209c5133e65e9c64bb791a86d6507cbc Mon Sep 17 00:00:00 2001 From: Jonathan Basseri Date: Tue, 8 May 2018 15:10:40 -0700 Subject: [PATCH] Clean up names and comments in equivalence cache. --- pkg/scheduler/core/equivalence_cache.go | 111 +++++++++++-------- pkg/scheduler/core/equivalence_cache_test.go | 26 ++--- pkg/scheduler/core/generic_scheduler.go | 6 +- 3 files changed, 78 insertions(+), 65 deletions(-) diff --git a/pkg/scheduler/core/equivalence_cache.go b/pkg/scheduler/core/equivalence_cache.go index da50d2a81f..6e080a256f 100644 --- a/pkg/scheduler/core/equivalence_cache.go +++ b/pkg/scheduler/core/equivalence_cache.go @@ -31,22 +31,30 @@ import ( "github.com/golang/glog" ) -// EquivalenceCache holds: -// 1. a map of AlgorithmCache with node name as key -// 2. function to get equivalence pod +// EquivalenceCache saves and reuses the output of predicate functions. Use +// RunPredicate to get or update the cached results. An appropriate Invalidate* +// function should be called when some predicate results are no longer valid. +// +// Internally, results are keyed by node name, predicate name, and "equivalence +// class". (Equivalence class is defined in the `EquivalenceClassInfo` type.) +// Saved results will be reused until an appropriate invalidation function is +// called. type EquivalenceCache struct { - mu sync.RWMutex - algorithmCache map[string]AlgorithmCache + mu sync.RWMutex + cache nodeMap } -// The AlgorithmCache stores PredicateMap with predicate name as key, PredicateMap as value. -type AlgorithmCache map[string]PredicateMap +// nodeMap stores PredicateCaches with node name as the key. +type nodeMap map[string]predicateMap -// PredicateMap stores HostPrediacte with equivalence hash as key -type PredicateMap map[uint64]HostPredicate +// predicateMap stores resultMaps with predicate name as the key. +type predicateMap map[string]resultMap -// HostPredicate is the cached predicate result -type HostPredicate struct { +// resultMap stores PredicateResult with pod equivalence hash as the key. +type resultMap map[uint64]predicateResult + +// predicateResult stores the output of a FitPredicate. +type predicateResult struct { Fit bool FailReasons []algorithm.PredicateFailureReason } @@ -55,12 +63,12 @@ type HostPredicate struct { // result from previous scheduling. func NewEquivalenceCache() *EquivalenceCache { return &EquivalenceCache{ - algorithmCache: make(map[string]AlgorithmCache), + cache: make(nodeMap), } } -// RunPredicate will return a cached predicate result. In case of a cache miss, the predicate will -// be run and its results cached for the next call. +// RunPredicate returns a cached predicate result. In case of a cache miss, the predicate will be +// run and its results cached for the next call. // // NOTE: RunPredicate will not update the equivalence cache if the given NodeInfo is stale. func (ec *EquivalenceCache) RunPredicate( @@ -69,7 +77,7 @@ func (ec *EquivalenceCache) RunPredicate( pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *schedulercache.NodeInfo, - equivClassInfo *equivalenceClassInfo, + equivClassInfo *EquivalenceClassInfo, cache schedulercache.Cache, ) (bool, []algorithm.PredicateFailureReason, error) { if nodeInfo == nil || nodeInfo.Node() == nil { @@ -111,20 +119,20 @@ func (ec *EquivalenceCache) updateResult( return } nodeName := nodeInfo.Node().GetName() - if _, exist := ec.algorithmCache[nodeName]; !exist { - ec.algorithmCache[nodeName] = AlgorithmCache{} + if _, exist := ec.cache[nodeName]; !exist { + ec.cache[nodeName] = make(predicateMap) } - predicateItem := HostPredicate{ + predicateItem := predicateResult{ Fit: fit, FailReasons: reasons, } // if cached predicate map already exists, just update the predicate by key - if predicateMap, ok := ec.algorithmCache[nodeName][predicateKey]; ok { + if predicates, ok := ec.cache[nodeName][predicateKey]; ok { // maps in golang are references, no need to add them back - predicateMap[equivalenceHash] = predicateItem + predicates[equivalenceHash] = predicateItem } else { - ec.algorithmCache[nodeName][predicateKey] = - PredicateMap{ + ec.cache[nodeName][predicateKey] = + resultMap{ equivalenceHash: predicateItem, } } @@ -143,11 +151,11 @@ func (ec *EquivalenceCache) lookupResult( defer ec.mu.RUnlock() glog.V(5).Infof("Begin to calculate predicate: %v for pod: %s on node: %s based on equivalence cache", predicateKey, podName, nodeName) - if hostPredicate, exist := ec.algorithmCache[nodeName][predicateKey][equivalenceHash]; exist { - if hostPredicate.Fit { + if result, exist := ec.cache[nodeName][predicateKey][equivalenceHash]; exist { + if result.Fit { return true, []algorithm.PredicateFailureReason{}, false } - return false, hostPredicate.FailReasons, false + return false, result.FailReasons, false } // is invalid return false, []algorithm.PredicateFailureReason{}, true @@ -161,40 +169,43 @@ func (ec *EquivalenceCache) InvalidateCachedPredicateItem(nodeName string, predi ec.mu.Lock() defer ec.mu.Unlock() for predicateKey := range predicateKeys { - delete(ec.algorithmCache[nodeName], predicateKey) + delete(ec.cache[nodeName], predicateKey) } glog.V(5).Infof("Done invalidating cached predicates: %v on node: %s", predicateKeys, nodeName) } -// InvalidateCachedPredicateItemOfAllNodes marks all items of given predicateKeys, of all pods, on all node as invalid +// InvalidateCachedPredicateItemOfAllNodes marks all items of given +// predicateKeys, of all pods, on all node as invalid func (ec *EquivalenceCache) InvalidateCachedPredicateItemOfAllNodes(predicateKeys sets.String) { if len(predicateKeys) == 0 { return } ec.mu.Lock() defer ec.mu.Unlock() - // algorithmCache uses nodeName as key, so we just iterate it and invalid given predicates - for _, algorithmCache := range ec.algorithmCache { + // ec.cache uses nodeName as key, so we just iterate it and invalid given predicates + for _, predicates := range ec.cache { for predicateKey := range predicateKeys { - delete(algorithmCache, predicateKey) + delete(predicates, predicateKey) } } glog.V(5).Infof("Done invalidating cached predicates: %v on all node", predicateKeys) } -// InvalidateAllCachedPredicateItemOfNode marks all cached items on given node as invalid +// InvalidateAllCachedPredicateItemOfNode marks all cached items on given node +// as invalid func (ec *EquivalenceCache) InvalidateAllCachedPredicateItemOfNode(nodeName string) { ec.mu.Lock() defer ec.mu.Unlock() - delete(ec.algorithmCache, nodeName) + delete(ec.cache, nodeName) glog.V(5).Infof("Done invalidating all cached predicates on node: %s", nodeName) } -// InvalidateCachedPredicateItemForPodAdd is a wrapper of InvalidateCachedPredicateItem for pod add case +// InvalidateCachedPredicateItemForPodAdd is a wrapper of +// InvalidateCachedPredicateItem for pod add case func (ec *EquivalenceCache) InvalidateCachedPredicateItemForPodAdd(pod *v1.Pod, nodeName string) { // MatchInterPodAffinity: we assume scheduler can make sure newly bound pod - // will not break the existing inter pod affinity. So we does not need to invalidate - // MatchInterPodAffinity when pod added. + // will not break the existing inter pod affinity. So we does not need to + // invalidate MatchInterPodAffinity when pod added. // // But when a pod is deleted, existing inter pod affinity may become invalid. // (e.g. this pod was preferred by some else, or vice versa) @@ -227,22 +238,23 @@ func (ec *EquivalenceCache) InvalidateCachedPredicateItemForPodAdd(pod *v1.Pod, ec.InvalidateCachedPredicateItem(nodeName, invalidPredicates) } -// equivalenceClassInfo holds equivalence hash which is used for checking equivalence cache. -// We will pass this to podFitsOnNode to ensure equivalence hash is only calculated per schedule. -type equivalenceClassInfo struct { +// EquivalenceClassInfo holds equivalence hash which is used for checking +// equivalence cache. We will pass this to podFitsOnNode to ensure equivalence +// hash is only calculated per schedule. +type EquivalenceClassInfo struct { // Equivalence hash. hash uint64 } -// getEquivalenceClassInfo returns a hash of the given pod. -// The hashing function returns the same value for any two pods that are -// equivalent from the perspective of scheduling. -func (ec *EquivalenceCache) getEquivalenceClassInfo(pod *v1.Pod) *equivalenceClassInfo { - equivalencePod := getEquivalenceHash(pod) +// GetEquivalenceClassInfo returns a hash of the given pod. The hashing function +// returns the same value for any two pods that are equivalent from the +// perspective of scheduling. +func (ec *EquivalenceCache) GetEquivalenceClassInfo(pod *v1.Pod) *EquivalenceClassInfo { + equivalencePod := getEquivalencePod(pod) if equivalencePod != nil { hash := fnv.New32a() hashutil.DeepHashObject(hash, equivalencePod) - return &equivalenceClassInfo{ + return &EquivalenceClassInfo{ hash: uint64(hash.Sum32()), } } @@ -254,9 +266,9 @@ func (ec *EquivalenceCache) getEquivalenceClassInfo(pod *v1.Pod) *equivalenceCla // include any Pod field which is used by a FitPredicate. // // NOTE: For equivalence hash to be formally correct, lists and maps in the -// equivalencePod should be normalized. (e.g. by sorting them) However, the -// vast majority of equivalent pod classes are expected to be created from a -// single pod template, so they will all have the same ordering. +// equivalencePod should be normalized. (e.g. by sorting them) However, the vast +// majority of equivalent pod classes are expected to be created from a single +// pod template, so they will all have the same ordering. type equivalencePod struct { Namespace *string Labels map[string]string @@ -269,8 +281,9 @@ type equivalencePod struct { Volumes []v1.Volume // See note about ordering } -// getEquivalenceHash returns the equivalencePod for a Pod. -func getEquivalenceHash(pod *v1.Pod) *equivalencePod { +// getEquivalencePod returns a normalized representation of a pod so that two +// "equivalent" pods will hash to the same value. +func getEquivalencePod(pod *v1.Pod) *equivalencePod { ep := &equivalencePod{ Namespace: &pod.Namespace, Labels: pod.Labels, diff --git a/pkg/scheduler/core/equivalence_cache_test.go b/pkg/scheduler/core/equivalence_cache_test.go index 412e258510..efbcf43e9a 100644 --- a/pkg/scheduler/core/equivalence_cache_test.go +++ b/pkg/scheduler/core/equivalence_cache_test.go @@ -251,7 +251,7 @@ func TestRunPredicate(t *testing.T) { meta := algorithm.EmptyPredicateMetadataProducer(nil, nil) ecache := NewEquivalenceCache() - equivClass := ecache.getEquivalenceClassInfo(pod) + equivClass := ecache.GetEquivalenceClassInfo(pod) if test.expectCacheHit { ecache.updateResult(pod.Name, "testPredicate", test.expectFit, test.expectedReasons, equivClass.hash, test.cache, node) } @@ -311,7 +311,7 @@ func TestUpdateResult(t *testing.T) { reasons []algorithm.PredicateFailureReason equivalenceHash uint64 expectPredicateMap bool - expectCacheItem HostPredicate + expectCacheItem predicateResult cache schedulercache.Cache }{ { @@ -322,7 +322,7 @@ func TestUpdateResult(t *testing.T) { fit: true, equivalenceHash: 123, expectPredicateMap: false, - expectCacheItem: HostPredicate{ + expectCacheItem: predicateResult{ Fit: true, }, cache: &upToDateCache{}, @@ -335,7 +335,7 @@ func TestUpdateResult(t *testing.T) { fit: false, equivalenceHash: 123, expectPredicateMap: true, - expectCacheItem: HostPredicate{ + expectCacheItem: predicateResult{ Fit: false, }, cache: &upToDateCache{}, @@ -344,12 +344,12 @@ func TestUpdateResult(t *testing.T) { for _, test := range tests { ecache := NewEquivalenceCache() if test.expectPredicateMap { - ecache.algorithmCache[test.nodeName] = AlgorithmCache{} - predicateItem := HostPredicate{ + ecache.cache[test.nodeName] = make(predicateMap) + predicateItem := predicateResult{ Fit: true, } - ecache.algorithmCache[test.nodeName][test.predicateKey] = - PredicateMap{ + ecache.cache[test.nodeName][test.predicateKey] = + resultMap{ test.equivalenceHash: predicateItem, } } @@ -366,7 +366,7 @@ func TestUpdateResult(t *testing.T) { node, ) - cachedMapItem, ok := ecache.algorithmCache[test.nodeName][test.predicateKey] + cachedMapItem, ok := ecache.cache[test.nodeName][test.predicateKey] if !ok { t.Errorf("Failed: %s, can't find expected cache item: %v", test.name, test.expectCacheItem) @@ -618,7 +618,7 @@ func TestGetEquivalenceHash(t *testing.T) { t.Run(test.name, func(t *testing.T) { for i, podInfo := range test.podInfoList { testPod := podInfo.pod - eclassInfo := ecache.getEquivalenceClassInfo(testPod) + eclassInfo := ecache.GetEquivalenceClassInfo(testPod) if eclassInfo == nil && podInfo.hashIsValid { t.Errorf("Failed: pod %v is expected to have valid hash", testPod) } @@ -708,7 +708,7 @@ func TestInvalidateCachedPredicateItemOfAllNodes(t *testing.T) { // there should be no cached predicate any more for _, test := range tests { - if algorithmCache, exist := ecache.algorithmCache[test.nodeName]; exist { + if algorithmCache, exist := ecache.cache[test.nodeName]; exist { if _, exist := algorithmCache[testPredicate]; exist { t.Errorf("Failed: cached item for predicate key: %v on node: %v should be invalidated", testPredicate, test.nodeName) @@ -778,7 +778,7 @@ func TestInvalidateAllCachedPredicateItemOfNode(t *testing.T) { for _, test := range tests { // invalidate cached predicate for all nodes ecache.InvalidateAllCachedPredicateItemOfNode(test.nodeName) - if _, exist := ecache.algorithmCache[test.nodeName]; exist { + if _, exist := ecache.cache[test.nodeName]; exist { t.Errorf("Failed: cached item for node: %v should be invalidated", test.nodeName) break } @@ -788,7 +788,7 @@ func TestInvalidateAllCachedPredicateItemOfNode(t *testing.T) { func BenchmarkEquivalenceHash(b *testing.B) { pod := makeBasicPod("test") for i := 0; i < b.N; i++ { - getEquivalenceHash(pod) + getEquivalencePod(pod) } } diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index 960054b0d1..83c21f45c3 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -342,10 +342,10 @@ func (g *genericScheduler) findNodesThatFit(pod *v1.Pod, nodes []*v1.Node) ([]*v // We can use the same metadata producer for all nodes. meta := g.predicateMetaProducer(pod, g.cachedNodeInfoMap) - var equivCacheInfo *equivalenceClassInfo + var equivCacheInfo *EquivalenceClassInfo if g.equivalenceCache != nil { // getEquivalenceClassInfo will return immediately if no equivalence pod found - equivCacheInfo = g.equivalenceCache.getEquivalenceClassInfo(pod) + equivCacheInfo = g.equivalenceCache.GetEquivalenceClassInfo(pod) } checkNode := func(i int) { @@ -462,7 +462,7 @@ func podFitsOnNode( ecache *EquivalenceCache, queue SchedulingQueue, alwaysCheckAllPredicates bool, - equivCacheInfo *equivalenceClassInfo, + equivCacheInfo *EquivalenceClassInfo, ) (bool, []algorithm.PredicateFailureReason, error) { var ( eCacheAvailable bool