diff --git a/pkg/scheduler/algorithm/predicates/BUILD b/pkg/scheduler/algorithm/predicates/BUILD index 85b38d8681..a028dac625 100644 --- a/pkg/scheduler/algorithm/predicates/BUILD +++ b/pkg/scheduler/algorithm/predicates/BUILD @@ -34,6 +34,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/client-go/listers/core/v1:go_default_library", "//vendor/k8s.io/client-go/listers/storage/v1:go_default_library", diff --git a/pkg/scheduler/algorithm/predicates/utils.go b/pkg/scheduler/algorithm/predicates/utils.go index 2b9c94f9dc..ce3e9d5888 100644 --- a/pkg/scheduler/algorithm/predicates/utils.go +++ b/pkg/scheduler/algorithm/predicates/utils.go @@ -17,8 +17,12 @@ limitations under the License. package predicates import ( + "github.com/golang/glog" "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kubernetes/pkg/scheduler/algorithm" schedutil "k8s.io/kubernetes/pkg/scheduler/util" ) @@ -66,6 +70,68 @@ func CreateSelectorFromLabels(aL map[string]string) labels.Selector { return labels.Set(aL).AsSelector() } +// EquivalencePodGenerator is a generator of equivalence class for pod with consideration of PVC info. +type EquivalencePodGenerator struct { + pvcInfo PersistentVolumeClaimInfo +} + +// NewEquivalencePodGenerator returns a getEquivalencePod method with consideration of PVC info. +func NewEquivalencePodGenerator(pvcInfo PersistentVolumeClaimInfo) algorithm.GetEquivalencePodFunc { + g := &EquivalencePodGenerator{ + pvcInfo: pvcInfo, + } + return g.getEquivalencePod +} + +// GetEquivalencePod returns a EquivalencePod which contains a group of pod attributes which can be reused. +func (e *EquivalencePodGenerator) getEquivalencePod(pod *v1.Pod) interface{} { + // For now we only consider pods: + // 1. OwnerReferences is Controller + // 2. with same OwnerReferences + // 3. with same PVC claim + // to be equivalent + for _, ref := range pod.OwnerReferences { + if ref.Controller != nil && *ref.Controller { + if pvcSet, err := e.getPVCSet(pod); err == nil { + // A pod can only belongs to one controller, so let's return. + return &EquivalencePod{ + ControllerRef: ref, + PVCSet: pvcSet, + } + } else { + // If error encountered, log warning and return nil (i.e. no equivalent pod found) + glog.Warningf("[EquivalencePodGenerator] for pod: %v failed due to: %v", pod.GetName(), err) + return nil + } + } + } + return nil +} + +// getPVCSet returns a set of PVC UIDs of given pod. +func (e *EquivalencePodGenerator) getPVCSet(pod *v1.Pod) (sets.String, error) { + result := sets.NewString() + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim == nil { + continue + } + pvcName := volume.PersistentVolumeClaim.ClaimName + pvc, err := e.pvcInfo.GetPersistentVolumeClaimInfo(pod.GetNamespace(), pvcName) + if err != nil { + return nil, err + } + result.Insert(string(pvc.UID)) + } + + return result, nil +} + +// EquivalencePod is a group of pod attributes which can be reused as equivalence to schedule other pods. +type EquivalencePod struct { + ControllerRef metav1.OwnerReference + PVCSet sets.String +} + // portsConflict check whether existingPorts and wantPorts conflict with each other // return true if we have a conflict func portsConflict(existingPorts schedutil.HostPortInfo, wantPorts []*v1.ContainerPort) bool { diff --git a/pkg/scheduler/algorithm/types.go b/pkg/scheduler/algorithm/types.go index 8a40bfd692..f6ff3b4942 100644 --- a/pkg/scheduler/algorithm/types.go +++ b/pkg/scheduler/algorithm/types.go @@ -75,6 +75,8 @@ type PredicateFailureReason interface { GetReason() string } +type GetEquivalencePodFunc func(pod *v1.Pod) interface{} + // NodeLister interface represents anything that can list nodes for a scheduler. type NodeLister interface { // We explicitly return []*v1.Node, instead of v1.NodeList, to avoid diff --git a/pkg/scheduler/algorithmprovider/defaults/defaults.go b/pkg/scheduler/algorithmprovider/defaults/defaults.go index 016a4fcb05..11b0a54042 100644 --- a/pkg/scheduler/algorithmprovider/defaults/defaults.go +++ b/pkg/scheduler/algorithmprovider/defaults/defaults.go @@ -77,6 +77,13 @@ func init() { // Fit is determined by node selector query. factory.RegisterFitPredicate(predicates.MatchNodeSelectorPred, predicates.PodMatchNodeSelector) + // Use equivalence class to speed up heavy predicates phase. + factory.RegisterGetEquivalencePodFunction( + func(args factory.PluginFactoryArgs) algorithm.GetEquivalencePodFunc { + return predicates.NewEquivalencePodGenerator(args.PVCInfo) + }, + ) + // ServiceSpreadingPriority is a priority config factory that spreads pods by minimizing // the number of pods (belonging to the same service) on the same node. // Register the factory so that it's available, but do not include it as part of the default priorities diff --git a/pkg/scheduler/core/equivalence_cache.go b/pkg/scheduler/core/equivalence_cache.go index f6dc084d2e..5d9bda7eaf 100644 --- a/pkg/scheduler/core/equivalence_cache.go +++ b/pkg/scheduler/core/equivalence_cache.go @@ -37,7 +37,8 @@ const maxCacheEntries = 100 // 2. function to get equivalence pod type EquivalenceCache struct { sync.RWMutex - algorithmCache map[string]AlgorithmCache + getEquivalencePod algorithm.GetEquivalencePodFunc + algorithmCache map[string]AlgorithmCache } // The AlgorithmCache stores PredicateMap with predicate name as key @@ -61,9 +62,10 @@ func newAlgorithmCache() AlgorithmCache { } } -func NewEquivalenceCache() *EquivalenceCache { +func NewEquivalenceCache(getEquivalencePodFunc algorithm.GetEquivalencePodFunc) *EquivalenceCache { return &EquivalenceCache{ - algorithmCache: make(map[string]AlgorithmCache), + getEquivalencePod: getEquivalencePodFunc, + algorithmCache: make(map[string]AlgorithmCache), } } @@ -206,69 +208,15 @@ func (ec *EquivalenceCache) InvalidateCachedPredicateItemForPodAdd(pod *v1.Pod, ec.InvalidateCachedPredicateItem(nodeName, invalidPredicates) } -// getEquivalenceHash computes a hash of the given pod. -// The hashing function returns the same value for any two pods that are -// equivalent from the perspective of scheduling. -func (ec *EquivalenceCache) getEquivalenceHash(pod *v1.Pod) (uint64, bool) { - equivalencePod := getEquivalencePod(pod) - hash := fnv.New32a() - hashutil.DeepHashObject(hash, equivalencePod) - return uint64(hash.Sum32()), true -} - -// equivalencePod is the set of pod attributes which must match for two pods to -// be considered equivalent for scheduling purposes. For correctness, this must -// include any Pod field which is used by a FitPredicate. -// -// NOTE: For equivalence hash to be formally correct, lists and maps in the -// equivalencePod should be normalized. (e.g. by sorting them) However, the -// vast majority of equivalent pod classes are expected to be created from a -// single pod template, so they will all have the same ordering. -type equivalencePod struct { - Namespace *string - Labels map[string]string - Affinity *v1.Affinity - Containers []v1.Container // See note about ordering - InitContainers []v1.Container // See note about ordering - NodeName *string - NodeSelector map[string]string - Tolerations []v1.Toleration - Volumes []v1.Volume // See note about ordering -} - -// getEquivalencePod returns the equivalencePod for a Pod. -func getEquivalencePod(pod *v1.Pod) *equivalencePod { - ep := &equivalencePod{ - Namespace: &pod.Namespace, - Labels: pod.Labels, - Affinity: pod.Spec.Affinity, - Containers: pod.Spec.Containers, - InitContainers: pod.Spec.InitContainers, - NodeName: &pod.Spec.NodeName, - NodeSelector: pod.Spec.NodeSelector, - Tolerations: pod.Spec.Tolerations, - Volumes: pod.Spec.Volumes, - } - // DeepHashObject considers nil and empy slices to be different. Normalize them. - if len(ep.Containers) == 0 { - ep.Containers = nil - } - if len(ep.InitContainers) == 0 { - ep.InitContainers = nil - } - if len(ep.Tolerations) == 0 { - ep.Tolerations = nil - } - if len(ep.Volumes) == 0 { - ep.Volumes = nil - } - // Normalize empty maps also. - if len(ep.Labels) == 0 { - ep.Labels = nil - } - if len(ep.NodeSelector) == 0 { - ep.NodeSelector = nil - } - // TODO: Also normalize nested maps and slices. - return ep +// getHashEquivalencePod returns the hash of equivalence pod. +// 1. equivalenceHash +// 2. if equivalence hash is valid +func (ec *EquivalenceCache) getHashEquivalencePod(pod *v1.Pod) (uint64, bool) { + equivalencePod := ec.getEquivalencePod(pod) + if equivalencePod != nil { + hash := fnv.New32a() + hashutil.DeepHashObject(hash, equivalencePod) + return uint64(hash.Sum32()), true + } + return 0, false } diff --git a/pkg/scheduler/core/equivalence_cache_test.go b/pkg/scheduler/core/equivalence_cache_test.go index 4501e870b0..54b903e2fb 100644 --- a/pkg/scheduler/core/equivalence_cache_test.go +++ b/pkg/scheduler/core/equivalence_cache_test.go @@ -21,132 +21,12 @@ import ( "testing" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/scheduler/algorithm" "k8s.io/kubernetes/pkg/scheduler/algorithm/predicates" ) -// makeBasicPod returns a Pod object with many of the fields populated. -func makeBasicPod(name string) *v1.Pod { - isController := true - return &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "test-ns", - Labels: map[string]string{"app": "web", "env": "prod"}, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "v1", - Kind: "ReplicationController", - Name: "rc", - UID: "123", - Controller: &isController, - }, - }, - }, - Spec: v1.PodSpec{ - Affinity: &v1.Affinity{ - NodeAffinity: &v1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ - NodeSelectorTerms: []v1.NodeSelectorTerm{ - { - MatchExpressions: []v1.NodeSelectorRequirement{ - { - Key: "failure-domain.beta.kubernetes.io/zone", - Operator: "Exists", - }, - }, - }, - }, - }, - }, - PodAffinity: &v1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "db"}}, - TopologyKey: "kubernetes.io/hostname", - }, - }, - }, - PodAntiAffinity: &v1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": "web"}}, - TopologyKey: "kubernetes.io/hostname", - }, - }, - }, - }, - InitContainers: []v1.Container{ - { - Name: "init-pause", - Image: "gcr.io/google_containers/pause", - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - "cpu": resource.MustParse("1"), - "mem": resource.MustParse("100Mi"), - }, - }, - }, - }, - Containers: []v1.Container{ - { - Name: "pause", - Image: "gcr.io/google_containers/pause", - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - "cpu": resource.MustParse("1"), - "mem": resource.MustParse("100Mi"), - }, - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "nfs", - MountPath: "/srv/data", - }, - }, - }, - }, - NodeSelector: map[string]string{"node-type": "awesome"}, - Tolerations: []v1.Toleration{ - { - Effect: "NoSchedule", - Key: "experimental", - Operator: "Exists", - }, - }, - Volumes: []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "someEBSVol1", - }, - }, - }, - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "someEBSVol2", - }, - }, - }, - { - Name: "nfs", - VolumeSource: v1.VolumeSource{ - NFS: &v1.NFSVolumeSource{ - Server: "nfs.corp.example.com", - }, - }, - }, - }, - }, - } -} - type predicateItemType struct { fit bool reasons []algorithm.PredicateFailureReason @@ -190,7 +70,9 @@ func TestUpdateCachedPredicateItem(t *testing.T) { }, } for _, test := range tests { - ecache := NewEquivalenceCache() + // this case does not need to calculate equivalence hash, just pass an empty function + fakeGetEquivalencePodFunc := func(pod *v1.Pod) interface{} { return nil } + ecache := NewEquivalenceCache(fakeGetEquivalencePodFunc) if test.expectPredicateMap { ecache.algorithmCache[test.nodeName] = newAlgorithmCache() predicateItem := HostPredicate{ @@ -308,7 +190,9 @@ func TestPredicateWithECache(t *testing.T) { } for _, test := range tests { - ecache := NewEquivalenceCache() + // this case does not need to calculate equivalence hash, just pass an empty function + fakeGetEquivalencePodFunc := func(pod *v1.Pod) interface{} { return nil } + ecache := NewEquivalenceCache(fakeGetEquivalencePodFunc) // set cached item to equivalence cache ecache.UpdateCachedPredicateItem( test.podName, @@ -353,46 +237,205 @@ func TestPredicateWithECache(t *testing.T) { } } -func TestGetEquivalenceHash(t *testing.T) { +func TestGetHashEquivalencePod(t *testing.T) { - ecache := NewEquivalenceCache() + testNamespace := "test" - pod1 := makeBasicPod("pod1") - pod2 := makeBasicPod("pod2") - - pod3 := makeBasicPod("pod3") - pod3.Spec.Volumes = []v1.Volume{ + pvcInfo := predicates.FakePersistentVolumeClaimInfo{ { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "someEBSVol111", + ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol1", Name: "someEBSVol1", Namespace: testNamespace}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someEBSVol1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol2", Name: "someEBSVol2", Namespace: testNamespace}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "someNonEBSVol"}, + }, + { + ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol3-0", Name: "someEBSVol3-0", Namespace: testNamespace}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "pvcWithDeletedPV"}, + }, + { + ObjectMeta: metav1.ObjectMeta{UID: "someEBSVol3-1", Name: "someEBSVol3-1", Namespace: testNamespace}, + Spec: v1.PersistentVolumeClaimSpec{VolumeName: "anotherPVCWithDeletedPV"}, + }, + } + + // use default equivalence class generator + ecache := NewEquivalenceCache(predicates.NewEquivalencePodGenerator(pvcInfo)) + + isController := true + + pod1 := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "ReplicationController", + Name: "rc", + UID: "123", + Controller: &isController, + }, + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "someEBSVol1", + }, + }, + }, + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "someEBSVol2", + }, + }, }, }, }, } - pod4 := makeBasicPod("pod4") - pod4.Spec.Volumes = []v1.Volume{ - { - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: "someEBSVol222", + pod2 := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "ReplicationController", + Name: "rc", + UID: "123", + Controller: &isController, + }, + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "someEBSVol2", + }, + }, + }, + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "someEBSVol1", + }, + }, }, }, }, } - pod5 := makeBasicPod("pod5") - pod5.Spec.Volumes = []v1.Volume{} + pod3 := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod3", + Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "ReplicationController", + Name: "rc", + UID: "567", + Controller: &isController, + }, + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "someEBSVol3-1", + }, + }, + }, + }, + }, + } - pod6 := makeBasicPod("pod6") - pod6.Spec.Volumes = nil + pod4 := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod4", + Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "ReplicationController", + Name: "rc", + UID: "567", + Controller: &isController, + }, + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "someEBSVol3-0", + }, + }, + }, + }, + }, + } - pod7 := makeBasicPod("pod7") - pod7.Spec.NodeSelector = nil + pod5 := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod5", + Namespace: testNamespace, + }, + } - pod8 := makeBasicPod("pod8") - pod8.Spec.NodeSelector = make(map[string]string) + pod6 := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod6", + Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "ReplicationController", + Name: "rc", + UID: "567", + Controller: &isController, + }, + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "no-exists-pvc", + }, + }, + }, + }, + }, + } + + pod7 := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod7", + Namespace: testNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "ReplicationController", + Name: "rc", + UID: "567", + Controller: &isController, + }, + }, + }, + } type podInfo struct { pod *v1.Pod @@ -400,41 +443,39 @@ func TestGetEquivalenceHash(t *testing.T) { } tests := []struct { - name string podInfoList []podInfo isEquivalent bool }{ + // pods with same controllerRef and same pvc claim { - name: "pods with everything the same except name", podInfoList: []podInfo{ {pod: pod1, hashIsValid: true}, {pod: pod2, hashIsValid: true}, }, isEquivalent: true, }, + // pods with same controllerRef but different pvc claim { - name: "pods that only differ in their PVC volume sources", podInfoList: []podInfo{ {pod: pod3, hashIsValid: true}, {pod: pod4, hashIsValid: true}, }, isEquivalent: false, }, + // pod without controllerRef { - name: "pods that have no volumes, but one uses nil and one uses an empty slice", podInfoList: []podInfo{ - {pod: pod5, hashIsValid: true}, - {pod: pod6, hashIsValid: true}, + {pod: pod5, hashIsValid: false}, }, - isEquivalent: true, + isEquivalent: false, }, + // pods with same controllerRef but one has non-exists pvc claim { - name: "pods that have no NodeSelector, but one uses nil and one uses an empty map", podInfoList: []podInfo{ + {pod: pod6, hashIsValid: false}, {pod: pod7, hashIsValid: true}, - {pod: pod8, hashIsValid: true}, }, - isEquivalent: true, + isEquivalent: false, }, } @@ -444,27 +485,25 @@ func TestGetEquivalenceHash(t *testing.T) { ) for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - for i, podInfo := range test.podInfoList { - testPod := podInfo.pod - hash, isValid := ecache.getEquivalenceHash(testPod) - if isValid != podInfo.hashIsValid { - t.Errorf("Failed: pod %v is expected to have valid hash", testPod) - } - // NOTE(harry): the first element will be used as target so - // this logic can't verify more than two inequivalent pods - if i == 0 { - targetHash = hash - targetPodInfo = podInfo - } else { - if targetHash != hash { - if test.isEquivalent { - t.Errorf("Failed: pod: %v is expected to be equivalent to: %v", testPod, targetPodInfo.pod) - } + for i, podInfo := range test.podInfoList { + testPod := podInfo.pod + hash, isValid := ecache.getHashEquivalencePod(testPod) + if isValid != podInfo.hashIsValid { + t.Errorf("Failed: pod %v is expected to have valid hash", testPod) + } + // NOTE(harry): the first element will be used as target so + // this logic can't verify more than two inequivalent pods + if i == 0 { + targetHash = hash + targetPodInfo = podInfo + } else { + if targetHash != hash { + if test.isEquivalent { + t.Errorf("Failed: pod: %v is expected to be equivalent to: %v", testPod, targetPodInfo.pod) } } } - }) + } } } @@ -509,7 +548,9 @@ func TestInvalidateCachedPredicateItemOfAllNodes(t *testing.T) { }, }, } - ecache := NewEquivalenceCache() + // this case does not need to calculate equivalence hash, just pass an empty function + fakeGetEquivalencePodFunc := func(pod *v1.Pod) interface{} { return nil } + ecache := NewEquivalenceCache(fakeGetEquivalencePodFunc) for _, test := range tests { // set cached item to equivalence cache @@ -575,7 +616,9 @@ func TestInvalidateAllCachedPredicateItemOfNode(t *testing.T) { }, }, } - ecache := NewEquivalenceCache() + // this case does not need to calculate equivalence hash, just pass an empty function + fakeGetEquivalencePodFunc := func(pod *v1.Pod) interface{} { return nil } + ecache := NewEquivalenceCache(fakeGetEquivalencePodFunc) for _, test := range tests { // set cached item to equivalence cache @@ -598,11 +641,3 @@ func TestInvalidateAllCachedPredicateItemOfNode(t *testing.T) { } } } - -func BenchmarkEquivalenceHash(b *testing.B) { - cache := NewEquivalenceCache() - pod := makeBasicPod("test") - for i := 0; i < b.N; i++ { - cache.getEquivalenceHash(pod) - } -} diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index e3f47a387f..da04ff45ad 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -418,7 +418,7 @@ func podFitsOnNode( if ecache != nil { // getHashEquivalencePod will return immediately if no equivalence pod found - equivalenceHash, eCacheAvailable = ecache.getEquivalenceHash(pod) + equivalenceHash, eCacheAvailable = ecache.getHashEquivalencePod(pod) } podsAdded := false // We run predicates twice in some cases. If the node has greater or equal priority diff --git a/pkg/scheduler/factory/factory.go b/pkg/scheduler/factory/factory.go index 3f6e6e815e..dfc5d6c3db 100644 --- a/pkg/scheduler/factory/factory.go +++ b/pkg/scheduler/factory/factory.go @@ -135,8 +135,6 @@ type configFactory struct { alwaysCheckAllPredicates bool } -var _ scheduler.Configurator = &configFactory{} - // NewConfigFactory initializes the default implementation of a Configurator To encourage eventual privatization of the struct type, we only // return the interface. func NewConfigFactory( @@ -969,8 +967,14 @@ func (f *configFactory) CreateFromKeys(predicateKeys, priorityKeys sets.String, } // Init equivalence class cache - if f.enableEquivalenceClassCache { - f.equivalencePodCache = core.NewEquivalenceCache() + if f.enableEquivalenceClassCache && getEquivalencePodFuncFactory != nil { + pluginArgs, err := f.getPluginArgs() + if err != nil { + return nil, err + } + f.equivalencePodCache = core.NewEquivalenceCache( + getEquivalencePodFuncFactory(*pluginArgs), + ) glog.Info("Created equivalence class cache") } diff --git a/pkg/scheduler/factory/plugins.go b/pkg/scheduler/factory/plugins.go index 9ec28df06f..1447d9487c 100644 --- a/pkg/scheduler/factory/plugins.go +++ b/pkg/scheduler/factory/plugins.go @@ -75,6 +75,9 @@ type PriorityConfigFactory struct { Weight int } +// EquivalencePodFuncFactory produces a function to get equivalence class for given pod. +type EquivalencePodFuncFactory func(PluginFactoryArgs) algorithm.GetEquivalencePodFunc + var ( schedulerFactoryMutex sync.Mutex @@ -87,6 +90,9 @@ var ( // Registered metadata producers priorityMetadataProducer PriorityMetadataProducerFactory predicateMetadataProducer PredicateMetadataProducerFactory + + // get equivalence pod function + getEquivalencePodFuncFactory EquivalencePodFuncFactory ) const ( @@ -335,6 +341,11 @@ func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string { return RegisterPriorityConfigFactory(policy.Name, *pcf) } +// RegisterGetEquivalencePodFunction registers equivalenceFuncFactory to produce equivalence class for given pod. +func RegisterGetEquivalencePodFunction(equivalenceFuncFactory EquivalencePodFuncFactory) { + getEquivalencePodFuncFactory = equivalenceFuncFactory +} + // IsPriorityFunctionRegistered is useful for testing providers. func IsPriorityFunctionRegistered(name string) bool { schedulerFactoryMutex.Lock()