github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Clean up names and comments in equivalence cache.

pull/8/head
Jonathan Basseri 2018-05-08 15:10:40 -07:00
parent 0c837a36ac
commit 9b06870620
3 changed files with 78 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
pkg/scheduler/core/equivalence_cache.go
View File

@ -31,22 +31,30 @@ import (
"github.com/golang/glog"
)
// EquivalenceCache holds:
// 1. a map of AlgorithmCache with node name as key
// 2. function to get equivalence pod
// EquivalenceCache saves and reuses the output of predicate functions. Use
// RunPredicate to get or update the cached results. An appropriate Invalidate*
// function should be called when some predicate results are no longer valid.
//
// Internally, results are keyed by node name, predicate name, and "equivalence
// class". (Equivalence class is defined in the `EquivalenceClassInfo` type.)
// Saved results will be reused until an appropriate invalidation function is
// called.
type EquivalenceCache struct {
mu sync.RWMutex
algorithmCache map[string]AlgorithmCache
cache nodeMap
}
// The AlgorithmCache stores PredicateMap with predicate name as key, PredicateMap as value.
type AlgorithmCache map[string]PredicateMap
// nodeMap stores PredicateCaches with node name as the key.
type nodeMap map[string]predicateMap
// PredicateMap stores HostPrediacte with equivalence hash as key
type PredicateMap map[uint64]HostPredicate
// predicateMap stores resultMaps with predicate name as the key.
type predicateMap map[string]resultMap
// HostPredicate is the cached predicate result
type HostPredicate struct {
// resultMap stores PredicateResult with pod equivalence hash as the key.
type resultMap map[uint64]predicateResult
// predicateResult stores the output of a FitPredicate.
type predicateResult struct {
Fit bool
FailReasons []algorithm.PredicateFailureReason
}
@ -55,12 +63,12 @@ type HostPredicate struct {
// result from previous scheduling.
func NewEquivalenceCache() *EquivalenceCache {
return &EquivalenceCache{
algorithmCache: make(map[string]AlgorithmCache),
cache: make(nodeMap),
}
}
// RunPredicate will return a cached predicate result. In case of a cache miss, the predicate will
// be run and its results cached for the next call.
// RunPredicate returns a cached predicate result. In case of a cache miss, the predicate will be
// run and its results cached for the next call.
//
// NOTE: RunPredicate will not update the equivalence cache if the given NodeInfo is stale.
func (ec *EquivalenceCache) RunPredicate(
@ -69,7 +77,7 @@ func (ec *EquivalenceCache) RunPredicate(
pod *v1.Pod,
meta algorithm.PredicateMetadata,
nodeInfo *schedulercache.NodeInfo,
equivClassInfo *equivalenceClassInfo,
equivClassInfo *EquivalenceClassInfo,
cache schedulercache.Cache,
) (bool, []algorithm.PredicateFailureReason, error) {
if nodeInfo == nil || nodeInfo.Node() == nil {
@ -111,20 +119,20 @@ func (ec *EquivalenceCache) updateResult(
return
}
nodeName := nodeInfo.Node().GetName()
if _, exist := ec.algorithmCache[nodeName]; !exist {
ec.algorithmCache[nodeName] = AlgorithmCache{}
if _, exist := ec.cache[nodeName]; !exist {
ec.cache[nodeName] = make(predicateMap)
}
predicateItem := HostPredicate{
predicateItem := predicateResult{
Fit: fit,
FailReasons: reasons,
}
// if cached predicate map already exists, just update the predicate by key
if predicateMap, ok := ec.algorithmCache[nodeName][predicateKey]; ok {
if predicates, ok := ec.cache[nodeName][predicateKey]; ok {
// maps in golang are references, no need to add them back
predicateMap[equivalenceHash] = predicateItem
predicates[equivalenceHash] = predicateItem
} else {
ec.algorithmCache[nodeName][predicateKey] =
PredicateMap{
ec.cache[nodeName][predicateKey] =
resultMap{
equivalenceHash: predicateItem,
}
}
@ -143,11 +151,11 @@ func (ec *EquivalenceCache) lookupResult(
defer ec.mu.RUnlock()
glog.V(5).Infof("Begin to calculate predicate: %v for pod: %s on node: %s based on equivalence cache",
predicateKey, podName, nodeName)
if hostPredicate, exist := ec.algorithmCache[nodeName][predicateKey][equivalenceHash]; exist {
if hostPredicate.Fit {
if result, exist := ec.cache[nodeName][predicateKey][equivalenceHash]; exist {
if result.Fit {
return true, []algorithm.PredicateFailureReason{}, false
}
return false, hostPredicate.FailReasons, false
return false, result.FailReasons, false
}
// is invalid
return false, []algorithm.PredicateFailureReason{}, true
@ -161,40 +169,43 @@ func (ec *EquivalenceCache) InvalidateCachedPredicateItem(nodeName string, predi
ec.mu.Lock()
defer ec.mu.Unlock()
for predicateKey := range predicateKeys {
delete(ec.algorithmCache[nodeName], predicateKey)
delete(ec.cache[nodeName], predicateKey)
}
glog.V(5).Infof("Done invalidating cached predicates: %v on node: %s", predicateKeys, nodeName)
}
// InvalidateCachedPredicateItemOfAllNodes marks all items of given predicateKeys, of all pods, on all node as invalid
// InvalidateCachedPredicateItemOfAllNodes marks all items of given
// predicateKeys, of all pods, on all node as invalid
func (ec *EquivalenceCache) InvalidateCachedPredicateItemOfAllNodes(predicateKeys sets.String) {
if len(predicateKeys) == 0 {
return
}
ec.mu.Lock()
defer ec.mu.Unlock()
// algorithmCache uses nodeName as key, so we just iterate it and invalid given predicates
for _, algorithmCache := range ec.algorithmCache {
// ec.cache uses nodeName as key, so we just iterate it and invalid given predicates
for _, predicates := range ec.cache {
for predicateKey := range predicateKeys {
delete(algorithmCache, predicateKey)
delete(predicates, predicateKey)
}
}
glog.V(5).Infof("Done invalidating cached predicates: %v on all node", predicateKeys)
}
// InvalidateAllCachedPredicateItemOfNode marks all cached items on given node as invalid
// InvalidateAllCachedPredicateItemOfNode marks all cached items on given node
// as invalid
func (ec *EquivalenceCache) InvalidateAllCachedPredicateItemOfNode(nodeName string) {
ec.mu.Lock()
defer ec.mu.Unlock()
delete(ec.algorithmCache, nodeName)
delete(ec.cache, nodeName)
glog.V(5).Infof("Done invalidating all cached predicates on node: %s", nodeName)
}
// InvalidateCachedPredicateItemForPodAdd is a wrapper of InvalidateCachedPredicateItem for pod add case
// InvalidateCachedPredicateItemForPodAdd is a wrapper of
// InvalidateCachedPredicateItem for pod add case
func (ec *EquivalenceCache) InvalidateCachedPredicateItemForPodAdd(pod *v1.Pod, nodeName string) {
// MatchInterPodAffinity: we assume scheduler can make sure newly bound pod
// will not break the existing inter pod affinity. So we does not need to invalidate
// MatchInterPodAffinity when pod added.
// will not break the existing inter pod affinity. So we does not need to
// invalidate MatchInterPodAffinity when pod added.
//
// But when a pod is deleted, existing inter pod affinity may become invalid.
// (e.g. this pod was preferred by some else, or vice versa)
@ -227,22 +238,23 @@ func (ec *EquivalenceCache) InvalidateCachedPredicateItemForPodAdd(pod *v1.Pod,
ec.InvalidateCachedPredicateItem(nodeName, invalidPredicates)
}
// equivalenceClassInfo holds equivalence hash which is used for checking equivalence cache.
// We will pass this to podFitsOnNode to ensure equivalence hash is only calculated per schedule.
type equivalenceClassInfo struct {
// EquivalenceClassInfo holds equivalence hash which is used for checking
// equivalence cache. We will pass this to podFitsOnNode to ensure equivalence
// hash is only calculated per schedule.
type EquivalenceClassInfo struct {
// Equivalence hash.
hash uint64
}
// getEquivalenceClassInfo returns a hash of the given pod.
// The hashing function returns the same value for any two pods that are
// equivalent from the perspective of scheduling.
func (ec *EquivalenceCache) getEquivalenceClassInfo(pod *v1.Pod) *equivalenceClassInfo {
equivalencePod := getEquivalenceHash(pod)
// GetEquivalenceClassInfo returns a hash of the given pod. The hashing function
// returns the same value for any two pods that are equivalent from the
// perspective of scheduling.
func (ec *EquivalenceCache) GetEquivalenceClassInfo(pod *v1.Pod) *EquivalenceClassInfo {
equivalencePod := getEquivalencePod(pod)
if equivalencePod != nil {
hash := fnv.New32a()
hashutil.DeepHashObject(hash, equivalencePod)
return &equivalenceClassInfo{
return &EquivalenceClassInfo{
hash: uint64(hash.Sum32()),
}
}
@ -254,9 +266,9 @@ func (ec *EquivalenceCache) getEquivalenceClassInfo(pod *v1.Pod) *equivalenceCla
// include any Pod field which is used by a FitPredicate.
//
// NOTE: For equivalence hash to be formally correct, lists and maps in the
// equivalencePod should be normalized. (e.g. by sorting them) However, the
// vast majority of equivalent pod classes are expected to be created from a
// single pod template, so they will all have the same ordering.
// equivalencePod should be normalized. (e.g. by sorting them) However, the vast
// majority of equivalent pod classes are expected to be created from a single
// pod template, so they will all have the same ordering.
type equivalencePod struct {
Namespace *string
Labels map[string]string
@ -269,8 +281,9 @@ type equivalencePod struct {
Volumes []v1.Volume // See note about ordering
}
// getEquivalenceHash returns the equivalencePod for a Pod.
func getEquivalenceHash(pod *v1.Pod) *equivalencePod {
// getEquivalencePod returns a normalized representation of a pod so that two
// "equivalent" pods will hash to the same value.
func getEquivalencePod(pod *v1.Pod) *equivalencePod {
ep := &equivalencePod{
Namespace: &pod.Namespace,
Labels: pod.Labels,

26
pkg/scheduler/core/equivalence_cache_test.go
View File

@ -251,7 +251,7 @@ func TestRunPredicate(t *testing.T) {
meta := algorithm.EmptyPredicateMetadataProducer(nil, nil)
ecache := NewEquivalenceCache()
equivClass := ecache.getEquivalenceClassInfo(pod)
equivClass := ecache.GetEquivalenceClassInfo(pod)
if test.expectCacheHit {
ecache.updateResult(pod.Name, "testPredicate", test.expectFit, test.expectedReasons, equivClass.hash, test.cache, node)
}
@ -311,7 +311,7 @@ func TestUpdateResult(t *testing.T) {
reasons []algorithm.PredicateFailureReason
equivalenceHash uint64
expectPredicateMap bool
expectCacheItem HostPredicate
expectCacheItem predicateResult
cache schedulercache.Cache
}{
{
@ -322,7 +322,7 @@ func TestUpdateResult(t *testing.T) {
fit: true,
equivalenceHash: 123,
expectPredicateMap: false,
expectCacheItem: HostPredicate{
expectCacheItem: predicateResult{
Fit: true,
},
cache: &upToDateCache{},
@ -335,7 +335,7 @@ func TestUpdateResult(t *testing.T) {
fit: false,
equivalenceHash: 123,
expectPredicateMap: true,
expectCacheItem: HostPredicate{
expectCacheItem: predicateResult{
Fit: false,
},
cache: &upToDateCache{},
@ -344,12 +344,12 @@ func TestUpdateResult(t *testing.T) {
for _, test := range tests {
ecache := NewEquivalenceCache()
if test.expectPredicateMap {
ecache.algorithmCache[test.nodeName] = AlgorithmCache{}
predicateItem := HostPredicate{
ecache.cache[test.nodeName] = make(predicateMap)
predicateItem := predicateResult{
Fit: true,
}
ecache.algorithmCache[test.nodeName][test.predicateKey] =
PredicateMap{
ecache.cache[test.nodeName][test.predicateKey] =
resultMap{
test.equivalenceHash: predicateItem,
}
}
@ -366,7 +366,7 @@ func TestUpdateResult(t *testing.T) {
node,
)
cachedMapItem, ok := ecache.algorithmCache[test.nodeName][test.predicateKey]
cachedMapItem, ok := ecache.cache[test.nodeName][test.predicateKey]
if !ok {
t.Errorf("Failed: %s, can't find expected cache item: %v",
test.name, test.expectCacheItem)
@ -618,7 +618,7 @@ func TestGetEquivalenceHash(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
for i, podInfo := range test.podInfoList {
testPod := podInfo.pod
eclassInfo := ecache.getEquivalenceClassInfo(testPod)
eclassInfo := ecache.GetEquivalenceClassInfo(testPod)
if eclassInfo == nil && podInfo.hashIsValid {
t.Errorf("Failed: pod %v is expected to have valid hash", testPod)
}
@ -708,7 +708,7 @@ func TestInvalidateCachedPredicateItemOfAllNodes(t *testing.T) {
// there should be no cached predicate any more
for _, test := range tests {
if algorithmCache, exist := ecache.algorithmCache[test.nodeName]; exist {
if algorithmCache, exist := ecache.cache[test.nodeName]; exist {
if _, exist := algorithmCache[testPredicate]; exist {
t.Errorf("Failed: cached item for predicate key: %v on node: %v should be invalidated",
testPredicate, test.nodeName)
@ -778,7 +778,7 @@ func TestInvalidateAllCachedPredicateItemOfNode(t *testing.T) {
for _, test := range tests {
// invalidate cached predicate for all nodes
ecache.InvalidateAllCachedPredicateItemOfNode(test.nodeName)
if _, exist := ecache.algorithmCache[test.nodeName]; exist {
if _, exist := ecache.cache[test.nodeName]; exist {
t.Errorf("Failed: cached item for node: %v should be invalidated", test.nodeName)
break
}
@ -788,7 +788,7 @@ func TestInvalidateAllCachedPredicateItemOfNode(t *testing.T) {
func BenchmarkEquivalenceHash(b *testing.B) {
pod := makeBasicPod("test")
for i := 0; i < b.N; i++ {
getEquivalenceHash(pod)
getEquivalencePod(pod)
}
}

6
pkg/scheduler/core/generic_scheduler.go
View File

@ -342,10 +342,10 @@ func (g *genericScheduler) findNodesThatFit(pod *v1.Pod, nodes []*v1.Node) ([]*v
// We can use the same metadata producer for all nodes.
meta := g.predicateMetaProducer(pod, g.cachedNodeInfoMap)
var equivCacheInfo *equivalenceClassInfo
var equivCacheInfo *EquivalenceClassInfo
if g.equivalenceCache != nil {
// getEquivalenceClassInfo will return immediately if no equivalence pod found
equivCacheInfo = g.equivalenceCache.getEquivalenceClassInfo(pod)
equivCacheInfo = g.equivalenceCache.GetEquivalenceClassInfo(pod)
}
checkNode := func(i int) {
@ -462,7 +462,7 @@ func podFitsOnNode(
ecache *EquivalenceCache,
queue SchedulingQueue,
alwaysCheckAllPredicates bool,
equivCacheInfo *equivalenceClassInfo,
equivCacheInfo *EquivalenceClassInfo,
) (bool, []algorithm.PredicateFailureReason, error) {
var (
eCacheAvailable bool