github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move equivalence class hash code. 

This moves the equivalence hashing code from
algorithm/predicates/utils.go to core/equivalence_cache.go.

In the process, making the hashing function and hashing function factory
both injectable dependencies is removed.
pull/6/head
Jonathan Basseri 2018-01-18 16:05:55 -08:00
parent 5ab4714520
commit 466a499fcb
9 changed files with 59 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/scheduler/algorithm/predicates/BUILD
View File

@ -34,7 +34,6 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/listers/storage/v1:go_default_library",

45
pkg/scheduler/algorithm/predicates/utils.go
View File

@ -19,7 +19,6 @@ package predicates
import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/kubernetes/pkg/scheduler/algorithm"
schedutil "k8s.io/kubernetes/pkg/scheduler/util"
)
@ -67,50 +66,6 @@ func CreateSelectorFromLabels(aL map[string]string) labels.Selector {
return labels.Set(aL).AsSelector()
}
// EquivalencePodGenerator is a generator of equivalence class for pod with consideration of PVC info.
type EquivalencePodGenerator struct {
pvcInfo PersistentVolumeClaimInfo
}
// NewEquivalencePodGenerator returns a getEquivalencePod method with consideration of PVC info.
func NewEquivalencePodGenerator(pvcInfo PersistentVolumeClaimInfo) algorithm.GetEquivalencePodFunc {
g := &EquivalencePodGenerator{
pvcInfo: pvcInfo,
}
return g.getEquivalencePod
}
// GetEquivalencePod returns a equivalencePod which contains a group of pod attributes which can be reused.
func (e *EquivalencePodGenerator) getEquivalencePod(pod *v1.Pod) interface{} {
// For equivalence hash to be formally correct, lists and maps
// in the equivalencePod should be normalized. (e.g. by sorting
// them) However, the vast majority of equivalent pod classes
// are expected to be created from a single pod template, so
// they will all have the same ordering.
return &equivalencePod{
Labels: pod.Labels,
Affinity: pod.Spec.Affinity,
Containers: pod.Spec.Containers,
InitContainers: pod.Spec.InitContainers,
NodeName: &pod.Spec.NodeName,
NodeSelector: pod.Spec.NodeSelector,
Tolerations: pod.Spec.Tolerations,
Volumes: pod.Spec.Volumes,
}
}
// equivalencePod is a group of pod attributes which can be reused as equivalence to schedule other pods.
type equivalencePod struct {
Labels map[string]string
Affinity *v1.Affinity
Containers []v1.Container // note about ordering
InitContainers []v1.Container // note about ordering
NodeName *string
NodeSelector map[string]string
Tolerations []v1.Toleration
Volumes []v1.Volume // note about ordering
}
// portsConflict check whether existingPorts and wantPorts conflict with each other
// return true if we have a conflict
func portsConflict(existingPorts schedutil.HostPortInfo, wantPorts []*v1.ContainerPort) bool {

2
pkg/scheduler/algorithm/types.go
View File

@ -75,8 +75,6 @@ type PredicateFailureReason interface {
GetReason() string
}
type GetEquivalencePodFunc func(pod *v1.Pod) interface{}
// NodeLister interface represents anything that can list nodes for a scheduler.
type NodeLister interface {
// We explicitly return []*v1.Node, instead of v1.NodeList, to avoid

7
pkg/scheduler/algorithmprovider/defaults/defaults.go
View File

@ -77,13 +77,6 @@ func init() {
// Fit is determined by node selector query.
factory.RegisterFitPredicate(predicates.MatchNodeSelectorPred, predicates.PodMatchNodeSelector)
// Use equivalence class to speed up heavy predicates phase.
factory.RegisterGetEquivalencePodFunction(
func(args factory.PluginFactoryArgs) algorithm.GetEquivalencePodFunc {
return predicates.NewEquivalencePodGenerator(args.PVCInfo)
},
)
// ServiceSpreadingPriority is a priority config factory that spreads pods by minimizing
// the number of pods (belonging to the same service) on the same node.
// Register the factory so that it's available, but do not include it as part of the default priorities

62
pkg/scheduler/core/equivalence_cache.go
View File

@ -37,8 +37,7 @@ const maxCacheEntries = 100
// 2. function to get equivalence pod
type EquivalenceCache struct {
sync.RWMutex
getEquivalencePod algorithm.GetEquivalencePodFunc
algorithmCache map[string]AlgorithmCache
algorithmCache map[string]AlgorithmCache
}
// The AlgorithmCache stores PredicateMap with predicate name as key
@ -62,10 +61,9 @@ func newAlgorithmCache() AlgorithmCache {
}
}
func NewEquivalenceCache(getEquivalencePodFunc algorithm.GetEquivalencePodFunc) *EquivalenceCache {
func NewEquivalenceCache() *EquivalenceCache {
return &EquivalenceCache{
getEquivalencePod: getEquivalencePodFunc,
algorithmCache: make(map[string]AlgorithmCache),
algorithmCache: make(map[string]AlgorithmCache),
}
}
@ -208,15 +206,47 @@ func (ec *EquivalenceCache) InvalidateCachedPredicateItemForPodAdd(pod *v1.Pod,
ec.InvalidateCachedPredicateItem(nodeName, invalidPredicates)
}
// getHashEquivalencePod returns the hash of equivalence pod.
// 1. equivalenceHash
// 2. if equivalence hash is valid
func (ec *EquivalenceCache) getHashEquivalencePod(pod *v1.Pod) (uint64, bool) {
equivalencePod := ec.getEquivalencePod(pod)
if equivalencePod != nil {
hash := fnv.New32a()
hashutil.DeepHashObject(hash, equivalencePod)
return uint64(hash.Sum32()), true
}
return 0, false
// getEquivalenceHash computes a hash of the given pod.
// The hashing function returns the same value for any two pods that are
// equivalent from the perspective of scheduling.
func (ec *EquivalenceCache) getEquivalenceHash(pod *v1.Pod) (uint64, bool) {
equivalencePod := getEquivalencePod(pod)
hash := fnv.New32a()
hashutil.DeepHashObject(hash, equivalencePod)
return uint64(hash.Sum32()), true
}
// equivalencePod is the set of pod attributes which must match for two pods to
// be considered equivalent for scheduling purposes. For correctness, this must
// include any Pod field which is used by a FitPredicate.
//
// NOTE: For equivalence hash to be formally correct, lists and maps in the
// equivalencePod should be normalized. (e.g. by sorting them) However, the
// vast majority of equivalent pod classes are expected to be created from a
// single pod template, so they will all have the same ordering.
type equivalencePod struct {
Namespace *string
Labels map[string]string
Affinity *v1.Affinity
Containers []v1.Container // See note about ordering
InitContainers []v1.Container // See note about ordering
NodeName *string
NodeSelector map[string]string
Tolerations []v1.Toleration
Volumes []v1.Volume // See note about ordering
}
// getEquivalencePod returns the equivalencePod for a Pod.
func getEquivalencePod(pod *v1.Pod) *equivalencePod {
return &equivalencePod{
Namespace: &pod.Namespace,
Labels: pod.Labels,
Affinity: pod.Spec.Affinity,
Containers: pod.Spec.Containers,
InitContainers: pod.Spec.InitContainers,
NodeName: &pod.Spec.NodeName,
NodeSelector: pod.Spec.NodeSelector,
Tolerations: pod.Spec.Tolerations,
Volumes: pod.Spec.Volumes,
}
}

66
pkg/scheduler/core/equivalence_cache_test.go
View File

@ -190,9 +190,7 @@ func TestUpdateCachedPredicateItem(t *testing.T) {
},
}
for _, test := range tests {
// this case does not need to calculate equivalence hash, just pass an empty function
fakeGetEquivalencePodFunc := func(pod *v1.Pod) interface{} { return nil }
ecache := NewEquivalenceCache(fakeGetEquivalencePodFunc)
ecache := NewEquivalenceCache()
if test.expectPredicateMap {
ecache.algorithmCache[test.nodeName] = newAlgorithmCache()
predicateItem := HostPredicate{
@ -310,9 +308,7 @@ func TestPredicateWithECache(t *testing.T) {
}
for _, test := range tests {
// this case does not need to calculate equivalence hash, just pass an empty function
fakeGetEquivalencePodFunc := func(pod *v1.Pod) interface{} { return nil }
ecache := NewEquivalenceCache(fakeGetEquivalencePodFunc)
ecache := NewEquivalenceCache()
// set cached item to equivalence cache
ecache.UpdateCachedPredicateItem(
test.podName,
@ -361,27 +357,7 @@ func TestGetHashEquivalencePod(t *testing.T) {
testNamespace := "test"
pvcInfo := predicates.FakePersistentVolumeClaimInfo{
{
ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol1", Name: "someEBSVol1", Namespace: testNamespace},
Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someEBSVol1"},
},
{
ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol2", Name: "someEBSVol2", Namespace: testNamespace},
Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someNonEBSVol"},
},
{
ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol3-0", Name: "someEBSVol3-0", Namespace: testNamespace},
Spec: v1.PersistentVolumeClaimSpec{VolumeName: "pvcWithDeletedPV"},
},
{
ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol3-1", Name: "someEBSVol3-1", Namespace: testNamespace},
Spec: v1.PersistentVolumeClaimSpec{VolumeName: "anotherPVCWithDeletedPV"},
},
}
// use default equivalence class generator
ecache := NewEquivalenceCache(predicates.NewEquivalencePodGenerator(pvcInfo))
ecache := NewEquivalenceCache()
isController := true
@ -607,7 +583,7 @@ func TestGetHashEquivalencePod(t *testing.T) {
for _, test := range tests {
for i, podInfo := range test.podInfoList {
testPod := podInfo.pod
hash, isValid := ecache.getHashEquivalencePod(testPod)
hash, isValid := ecache.getEquivalenceHash(testPod)
if isValid != podInfo.hashIsValid {
t.Errorf("Failed: pod %v is expected to have valid hash", testPod)
}
@ -668,9 +644,7 @@ func TestInvalidateCachedPredicateItemOfAllNodes(t *testing.T) {
},
},
}
// this case does not need to calculate equivalence hash, just pass an empty function
fakeGetEquivalencePodFunc := func(pod *v1.Pod) interface{} { return nil }
ecache := NewEquivalenceCache(fakeGetEquivalencePodFunc)
ecache := NewEquivalenceCache()
for _, test := range tests {
// set cached item to equivalence cache
@ -736,9 +710,7 @@ func TestInvalidateAllCachedPredicateItemOfNode(t *testing.T) {
},
},
}
// this case does not need to calculate equivalence hash, just pass an empty function
fakeGetEquivalencePodFunc := func(pod *v1.Pod) interface{} { return nil }
ecache := NewEquivalenceCache(fakeGetEquivalencePodFunc)
ecache := NewEquivalenceCache()
for _, test := range tests {
// set cached item to equivalence cache
@ -763,31 +735,9 @@ func TestInvalidateAllCachedPredicateItemOfNode(t *testing.T) {
}
func BenchmarkEquivalenceHash(b *testing.B) {
testNamespace := "test"
pvcInfo := predicates.FakePersistentVolumeClaimInfo{
{
ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol1", Name: "someEBSVol1", Namespace: testNamespace},
Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someEBSVol1"},
},
{
ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol2", Name: "someEBSVol2", Namespace: testNamespace},
Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someNonEBSVol"},
},
{
ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol3-0", Name: "someEBSVol3-0", Namespace: testNamespace},
Spec: v1.PersistentVolumeClaimSpec{VolumeName: "pvcWithDeletedPV"},
},
{
ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol3-1", Name: "someEBSVol3-1", Namespace: testNamespace},
Spec: v1.PersistentVolumeClaimSpec{VolumeName: "anotherPVCWithDeletedPV"},
},
}
// use default equivalence class generator
cache := NewEquivalenceCache(predicates.NewEquivalencePodGenerator(pvcInfo))
cache := NewEquivalenceCache()
pod := makeBasicPod()
for i := 0; i < b.N; i++ {
cache.getHashEquivalencePod(pod)
cache.getEquivalenceHash(pod)
}
}

2
pkg/scheduler/core/generic_scheduler.go
View File

@ -418,7 +418,7 @@ func podFitsOnNode(
if ecache != nil {
// getHashEquivalencePod will return immediately if no equivalence pod found
equivalenceHash, eCacheAvailable = ecache.getHashEquivalencePod(pod)
equivalenceHash, eCacheAvailable = ecache.getEquivalenceHash(pod)
}
podsAdded := false
// We run predicates twice in some cases. If the node has greater or equal priority

12
pkg/scheduler/factory/factory.go
View File

@ -135,6 +135,8 @@ type configFactory struct {
alwaysCheckAllPredicates bool
}
var _ scheduler.Configurator = &configFactory{}
// NewConfigFactory initializes the default implementation of a Configurator To encourage eventual privatization of the struct type, we only
// return the interface.
func NewConfigFactory(
@ -967,14 +969,8 @@ func (f *configFactory) CreateFromKeys(predicateKeys, priorityKeys sets.String,
}
// Init equivalence class cache
if f.enableEquivalenceClassCache && getEquivalencePodFuncFactory != nil {
pluginArgs, err := f.getPluginArgs()
if err != nil {
return nil, err
}
f.equivalencePodCache = core.NewEquivalenceCache(
getEquivalencePodFuncFactory(*pluginArgs),
)
if f.enableEquivalenceClassCache {
f.equivalencePodCache = core.NewEquivalenceCache()
glog.Info("Created equivalence class cache")
}

11
pkg/scheduler/factory/plugins.go
View File

@ -75,9 +75,6 @@ type PriorityConfigFactory struct {
Weight int
}
// EquivalencePodFuncFactory produces a function to get equivalence class for given pod.
type EquivalencePodFuncFactory func(PluginFactoryArgs) algorithm.GetEquivalencePodFunc
var (
schedulerFactoryMutex sync.Mutex
@ -90,9 +87,6 @@ var (
// Registered metadata producers
priorityMetadataProducer PriorityMetadataProducerFactory
predicateMetadataProducer PredicateMetadataProducerFactory
// get equivalence pod function
getEquivalencePodFuncFactory EquivalencePodFuncFactory
)
const (
@ -341,11 +335,6 @@ func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string {
return RegisterPriorityConfigFactory(policy.Name, *pcf)
}
// RegisterGetEquivalencePodFunction registers equivalenceFuncFactory to produce equivalence class for given pod.
func RegisterGetEquivalencePodFunction(equivalenceFuncFactory EquivalencePodFuncFactory) {
getEquivalencePodFuncFactory = equivalenceFuncFactory
}
// IsPriorityFunctionRegistered is useful for testing providers.
func IsPriorityFunctionRegistered(name string) bool {
schedulerFactoryMutex.Lock()