Improve clarity around PodFitsResource by showing pods limits in `kubectl describe node`

pkg/kubectl/describe.go

@@ -878,10 +878,33 @@ func describeNode(node *api.Node, pods []*api.Pod, events *api.EventList) (strin
 			fmt.Fprintf(out, "ExternalID:\t%s\n", node.Spec.ExternalID)
 		}
 		fmt.Fprintf(out, "Pods:\t(%d in total)\n", len(pods))
-		fmt.Fprint(out, "  Namespace\tName\n")
+		fmt.Fprint(out, "  Namespace\tName\t\tCPU(milliCPU)\t\tMemory(bytes)\n")
+		totalMilliCPU := int64(0)
+		totalMemory := int64(0)
+		fractionPodCPU := float64(0)
+		fractionPodMemory := float64(0)
+		fractionTotalCPU := float64(0)
+		fractionTotalMemory := float64(0)
 		for _, pod := range pods {
-			fmt.Fprintf(out, "  %s\t%s\n", pod.Namespace, pod.Name)
+			podTotalMilliCPU := int64(0)
+			podTotalMemory := int64(0)
+
+			for ix := range pod.Spec.Containers {
+				limits := pod.Spec.Containers[ix].Resources.Limits
+				podTotalMilliCPU += limits.Cpu().MilliValue()
+				podTotalMemory += limits.Memory().Value()
 			}
+			totalMilliCPU += podTotalMilliCPU
+			totalMemory += podTotalMemory
+			fractionPodCPU = float64(podTotalMilliCPU) / float64(node.Status.Capacity.Cpu().MilliValue()) * 100
+			fractionPodMemory = float64(podTotalMemory) / float64(node.Status.Capacity.Memory().Value()) * 100
+			fmt.Fprintf(out, "  %s\t%s\t\t%d (%d%% of total)\t\t%d (%d%% of total)\n", pod.Namespace, pod.Name, podTotalMilliCPU, int64(fractionPodCPU), podTotalMemory, int64(fractionPodMemory))
+		}
+		fmt.Fprint(out, "TotalResourceLimits:\n")
+		fractionTotalCPU = float64(totalMilliCPU) / float64(node.Status.Capacity.Cpu().MilliValue()) * 100
+		fractionTotalMemory = float64(totalMemory) / float64(node.Status.Capacity.Memory().Value()) * 100
+		fmt.Fprintf(out, "  CPU(milliCPU):\t\t%d (%d%% of total)\n", totalMilliCPU, int64(fractionTotalCPU))
+		fmt.Fprintf(out, "  Memory(bytes):\t\t%d (%d%% of total)\n", totalMemory, int64(fractionTotalMemory))
 		if events != nil {
 			DescribeEvents(events, out)
 		}

pkg/kubelet/kubelet.go

@@ -1568,19 +1568,19 @@ func checkHostPortConflicts(pods []*api.Pod) (fitting []*api.Pod, notFitting []*
 	return
 }
 
-// checkCapacityExceeded detects pods that exceeds node's resources.
+// checkSufficientfFreeResources detects pods that exceeds node's resources.
-func (kl *Kubelet) checkCapacityExceeded(pods []*api.Pod) (fitting []*api.Pod, notFitting []*api.Pod) {
+func (kl *Kubelet) checkSufficientfFreeResources(pods []*api.Pod) (fitting []*api.Pod, notFittingCPU, notFittingMemory []*api.Pod) {
 	info, err := kl.GetCachedMachineInfo()
 	if err != nil {
 		glog.Errorf("error getting machine info: %v", err)
-		return pods, nil
+		return pods, nil, nil
 	}
 
 	// Respect the pod creation order when resolving conflicts.
 	sort.Sort(podsByCreationTime(pods))
 
 	capacity := CapacityFromMachineInfo(info)
-	return predicates.CheckPodsExceedingCapacity(pods, capacity)
+	return predicates.CheckPodsExceedingFreeResources(pods, capacity)
 }
 
 // handleOutOfDisk detects if pods can't fit due to lack of disk space.
@@ -1673,14 +1673,22 @@ func (kl *Kubelet) handleNotFittingPods(pods []*api.Pod) []*api.Pod {
 			Reason:  reason,
 			Message: "Pod cannot be started due to node selector mismatch"})
 	}
-	fitting, notFitting = kl.checkCapacityExceeded(fitting)
+	fitting, notFittingCPU, notFittingMemory := kl.checkSufficientfFreeResources(fitting)
-	for _, pod := range notFitting {
+	for _, pod := range notFittingCPU {
-		reason := "CapacityExceeded"
+		reason := "InsufficientFreeCPU"
-		kl.recorder.Eventf(pod, reason, "Cannot start the pod due to exceeded capacity.")
+		kl.recorder.Eventf(pod, reason, "Cannot start the pod due to insufficient free CPU.")
 		kl.statusManager.SetPodStatus(pod, api.PodStatus{
 			Phase:   api.PodFailed,
 			Reason:  reason,
-			Message: "Pod cannot be started due to exceeded capacity"})
+			Message: "Pod cannot be started due to insufficient free CPU"})
+	}
+	for _, pod := range notFittingMemory {
+		reason := "InsufficientFreeMemory"
+		kl.recorder.Eventf(pod, reason, "Cannot start the pod due to insufficient free memory.")
+		kl.statusManager.SetPodStatus(pod, api.PodStatus{
+			Phase:   api.PodFailed,
+			Reason:  reason,
+			Message: "Pod cannot be started due to insufficient free memory"})
 	}
 	return fitting
 }

plugin/pkg/scheduler/algorithm/predicates/predicates.go

@@ -105,6 +105,8 @@ type resourceRequest struct {
 	memory   int64
 }
 
+var FailedResourceType string
+
 func getResourceRequest(pod *api.Pod) resourceRequest {
 	result := resourceRequest{}
 	for ix := range pod.Spec.Containers {
@@ -115,7 +117,7 @@ func getResourceRequest(pod *api.Pod) resourceRequest {
 	return result
 }
 
-func CheckPodsExceedingCapacity(pods []*api.Pod, capacity api.ResourceList) (fitting []*api.Pod, notFitting []*api.Pod) {
+func CheckPodsExceedingFreeResources(pods []*api.Pod, capacity api.ResourceList) (fitting []*api.Pod, notFittingCPU, notFittingMemory []*api.Pod) {
 	totalMilliCPU := capacity.Cpu().MilliValue()
 	totalMemory := capacity.Memory().Value()
 	milliCPURequested := int64(0)
@@ -124,9 +126,14 @@ func CheckPodsExceedingCapacity(pods []*api.Pod, capacity api.ResourceList) (fit
 		podRequest := getResourceRequest(pod)
 		fitsCPU := totalMilliCPU == 0 || (totalMilliCPU-milliCPURequested) >= podRequest.milliCPU
 		fitsMemory := totalMemory == 0 || (totalMemory-memoryRequested) >= podRequest.memory
-		if !fitsCPU || !fitsMemory {
+		if !fitsCPU {
-			// the pod doesn't fit
+			// the pod doesn't fit due to CPU limit
-			notFitting = append(notFitting, pod)
+			notFittingCPU = append(notFittingCPU, pod)
+			continue
+		}
+		if !fitsMemory {
+			// the pod doesn't fit due to Memory limit
+			notFittingMemory = append(notFittingMemory, pod)
 			continue
 		}
 		// the pod fits
@@ -150,9 +157,20 @@ func (r *ResourceFit) PodFitsResources(pod *api.Pod, existingPods []*api.Pod, no
 	pods := []*api.Pod{}
 	copy(pods, existingPods)
 	pods = append(existingPods, pod)
-	_, exceeding := CheckPodsExceedingCapacity(pods, info.Status.Capacity)
+	_, exceedingCPU, exceedingMemory := CheckPodsExceedingFreeResources(pods, info.Status.Capacity)
-	if len(exceeding) > 0 || int64(len(pods)) > info.Status.Capacity.Pods().Value() {
+	if int64(len(pods)) > info.Status.Capacity.Pods().Value() {
 		glog.V(4).Infof("Cannot schedule Pod %v, because Node %v is full, running %v out of %v Pods.", pod, node, len(pods)-1, info.Status.Capacity.Pods().Value())
+		FailedResourceType = "PodExceedsMaxPodNumber"
+		return false, nil
+	}
+	if len(exceedingCPU) > 0 {
+		glog.V(4).Infof("Cannot schedule Pod %v, because Node does not have sufficient CPU", pod)
+		FailedResourceType = "PodExceedsFreeCPU"
+		return false, nil
+	}
+	if len(exceedingMemory) > 0 {
+		glog.V(4).Infof("Cannot schedule Pod %v, because Node does not have sufficient Memory", pod)
+		FailedResourceType = "PodExceedsFreeMemory"
 		return false, nil
 	}
 	glog.V(4).Infof("Schedule Pod %v on Node %v is allowed, Node is running only %v out of %v Pods.", pod, node, len(pods)-1, info.Status.Capacity.Pods().Value())

plugin/pkg/scheduler/generic_scheduler.go

@@ -116,6 +116,7 @@ func findNodesThatFit(pod *api.Pod, podLister algorithm.PodLister, predicateFunc
 	for _, node := range nodes.Items {
 		fits := true
 		for name, predicate := range predicateFuncs {
+			predicates.FailedResourceType = ""
 			fit, err := predicate(pod, machineToPods[node.Name], node.Name)
 			if err != nil {
 				return api.NodeList{}, FailedPredicateMap{}, err
@@ -125,6 +126,10 @@ func findNodesThatFit(pod *api.Pod, podLister algorithm.PodLister, predicateFunc
 				if _, found := failedPredicateMap[node.Name]; !found {
 					failedPredicateMap[node.Name] = util.StringSet{}
 				}
+				if predicates.FailedResourceType != "" {
+					failedPredicateMap[node.Name].Insert(predicates.FailedResourceType)
+					break
+				}
 				failedPredicateMap[node.Name].Insert(name)
 				break
 			}