github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Enable PodTolerateNodeTaints predicate in DaemonSet controller

pull/6/head
Janet Kuo 2017-02-08 17:33:59 -08:00
parent 95badd95ce
commit b593427105
3 changed files with 102 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/controller/daemon/BUILD
View File

@ -27,6 +27,7 @@ go_library(
"//pkg/client/listers/extensions/v1beta1:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/util/metrics:go_default_library",
"//plugin/pkg/scheduler/algorithm:go_default_library",
"//plugin/pkg/scheduler/algorithm/predicates:go_default_library",
"//plugin/pkg/scheduler/schedulercache:go_default_library",
"//vendor:github.com/golang/glog",

42
pkg/controller/daemon/daemoncontroller.go
View File

@ -45,6 +45,7 @@ import (
extensionslisters "k8s.io/kubernetes/pkg/client/listers/extensions/v1beta1"
"k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/util/metrics"
"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
@ -779,13 +780,14 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *exten
nodeInfo := schedulercache.NewNodeInfo(pods...)
nodeInfo.SetNode(node)
_, reasons, err := predicates.GeneralPredicates(newPod, nil, nodeInfo)
_, reasons, err := daemonSetPredicates(newPod, nodeInfo)
if err != nil {
glog.Warningf("GeneralPredicates failed on ds '%s/%s' due to unexpected error: %v", ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, err)
glog.Warningf("daemonSetPredicates failed on ds '%s/%s' due to unexpected error: %v", ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, err)
return false, false, false, err
}
for _, r := range reasons {
glog.V(4).Infof("GeneralPredicates failed on ds '%s/%s' for reason: %v", ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, r.GetReason())
glog.V(4).Infof("daemonSetPredicates failed on ds '%s/%s' for reason: %v", ds.ObjectMeta.Namespace, ds.ObjectMeta.Name, r.GetReason())
switch reason := r.(type) {
case *predicates.InsufficientResourceError:
dsc.eventRecorder.Eventf(ds, v1.EventTypeNormal, FailedPlacementReason, "failed to place pod on %q: %s", node.ObjectMeta.Name, reason.Error())
@ -801,7 +803,9 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *exten
predicates.ErrNodeLabelPresenceViolated,
// this one is probably intentional since it's a workaround for not having
// pod hard anti affinity.
predicates.ErrPodNotFitsHostPorts:
predicates.ErrPodNotFitsHostPorts,
// DaemonSet is expected to respect taints and tolerations
predicates.ErrTaintsTolerationsNotMatch:
wantToRun, shouldSchedule, shouldContinueRunning = false, false, false
// unintentional
case
@ -818,9 +822,9 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *exten
// unexpected
case
predicates.ErrPodAffinityNotMatch,
predicates.ErrServiceAffinityViolated,
predicates.ErrTaintsTolerationsNotMatch:
return false, false, false, fmt.Errorf("unexpected reason: GeneralPredicates should not return reason %s", reason.GetReason())
predicates.ErrServiceAffinityViolated:
glog.Warningf("unexpected predicate failure reason: %s", reason.GetReason())
return false, false, false, fmt.Errorf("unexpected reason: daemonSetPredicates should not return reason %s", reason.GetReason())
default:
glog.V(4).Infof("unknown predicate failure reason: %s", reason.GetReason())
wantToRun, shouldSchedule, shouldContinueRunning = false, false, false
@ -834,6 +838,30 @@ func (dsc *DaemonSetsController) nodeShouldRunDaemonPod(node *v1.Node, ds *exten
return
}
// daemonSetPredicates checks if a DaemonSet's pod can be scheduled on a node using GeneralPredicates
// and PodToleratesNodeTaints predicate
func daemonSetPredicates(pod *v1.Pod, nodeInfo *schedulercache.NodeInfo) (bool, []algorithm.PredicateFailureReason, error) {
var predicateFails []algorithm.PredicateFailureReason
fit, reasons, err := predicates.GeneralPredicates(pod, nil, nodeInfo)
if err != nil {
return false, predicateFails, err
}
if !fit {
predicateFails = append(predicateFails, reasons...)
}
fit, reasons, err = predicates.PodToleratesNodeTaints(pod, nil, nodeInfo)
if err != nil {
return false, predicateFails, err
}
if !fit {
predicateFails = append(predicateFails, reasons...)
}
return len(predicateFails) == 0, predicateFails, nil
}
// byCreationTimestamp sorts a list by creation timestamp, using their names as a tie breaker.
type byCreationTimestamp []*extensions.DaemonSet

66
pkg/controller/daemon/daemoncontroller_test.go
View File

@ -45,6 +45,15 @@ var (
alwaysReady = func() bool { return true }
)
const (
noSchedule = `
[{
"key": "dedicated",
"value": "user1",
"effect": "NoSchedule"
}]`
)
func getKey(ds *extensions.DaemonSet, t *testing.T) string {
if key, err := controller.KeyFunc(ds); err != nil {
t.Errorf("Unexpected error getting key for ds %v: %v", ds.Name, err)
@ -708,6 +717,63 @@ func TestDaemonKillFailedPods(t *testing.T) {
}
}
// DaemonSet should not launch a pod on a tainted node when the pod doesn't tolerate that taint.
func TestTaintedNodeDaemonDoesNotLaunchUntoleratePod(t *testing.T) {
manager, podControl, _ := newTestController()
node := newNode("tainted", nil)
setNodeTaint(node, noSchedule)
manager.nodeStore.Add(node)
ds := newDaemonSet("untolerate")
manager.dsStore.Add(ds)
syncAndValidateDaemonSets(t, manager, ds, podControl, 0, 0)
}
// DaemonSet should launch a pod on a tainted node when the pod can tolerate that taint.
func TestTaintedNodeDaemonLaunchesToleratePod(t *testing.T) {
manager, podControl, _ := newTestController()
node := newNode("tainted", nil)
setNodeTaint(node, noSchedule)
manager.nodeStore.Add(node)
ds := newDaemonSet("tolerate")
setDaemonSetToleration(ds, noSchedule)
manager.dsStore.Add(ds)
syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0)
}
// DaemonSet should launch a pod on an untainted node when the pod has tolerations.
func TestNodeDaemonLaunchesToleratePod(t *testing.T) {
manager, podControl, _ := newTestController()
node := newNode("untainted", nil)
manager.nodeStore.Add(node)
ds := newDaemonSet("tolerate")
setDaemonSetToleration(ds, noSchedule)
manager.dsStore.Add(ds)
syncAndValidateDaemonSets(t, manager, ds, podControl, 1, 0)
}
func setNodeTaint(node *v1.Node, taint string) {
if node.ObjectMeta.Annotations == nil {
node.ObjectMeta.Annotations = make(map[string]string)
}
node.ObjectMeta.Annotations[v1.TaintsAnnotationKey] = taint
}
func setDaemonSetToleration(ds *extensions.DaemonSet, toleration string) {
if ds.Spec.Template.ObjectMeta.Annotations == nil {
ds.Spec.Template.ObjectMeta.Annotations = make(map[string]string)
}
ds.Spec.Template.ObjectMeta.Annotations[v1.TolerationsAnnotationKey] = toleration
}
func TestNodeShouldRunDaemonPod(t *testing.T) {
cases := []struct {
podsOnNode []*v1.Pod