Merge pull request #70500 from bsalamat/scheduler_debuger

Add a scheduler cache dumper for debugging purposes

pkg/scheduler/factory/BUILD

@@ -23,7 +23,7 @@ go_library(
         "//pkg/scheduler/core:go_default_library",
         "//pkg/scheduler/core/equivalence:go_default_library",
         "//pkg/scheduler/internal/cache:go_default_library",
-        "//pkg/scheduler/internal/cache/comparer:go_default_library",
+        "//pkg/scheduler/internal/cache/debugger:go_default_library",
         "//pkg/scheduler/internal/queue:go_default_library",
         "//pkg/scheduler/util:go_default_library",
         "//pkg/scheduler/volumebinder:go_default_library",

pkg/scheduler/factory/factory.go

@@ -60,7 +60,7 @@ import (
 	"k8s.io/kubernetes/pkg/scheduler/core"
 	"k8s.io/kubernetes/pkg/scheduler/core/equivalence"
 	schedulerinternalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
-	cachecomparer "k8s.io/kubernetes/pkg/scheduler/internal/cache/comparer"
+	cachedebugger "k8s.io/kubernetes/pkg/scheduler/internal/cache/debugger"
 	internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue"
 	"k8s.io/kubernetes/pkg/scheduler/util"
 	"k8s.io/kubernetes/pkg/scheduler/volumebinder"
@@ -398,7 +398,7 @@ func NewConfigFactory(args *ConfigFactoryArgs) Configurator {
 	}
 
 	// Setup cache comparer
-	comparer := cachecomparer.New(
+	debugger := cachedebugger.New(
 		args.NodeInformer.Lister(),
 		args.PodInformer.Lister(),
 		c.schedulerCache,
@@ -415,7 +415,8 @@ func NewConfigFactory(args *ConfigFactoryArgs) Configurator {
 				c.podQueue.Close()
 				return
 			case <-ch:
-				comparer.Compare()
+				debugger.Comparer.Compare()
+				debugger.Dumper.DumpAll()
 			}
 		}
 	}()

pkg/scheduler/internal/cache/BUILD

@@ -54,7 +54,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
-        "//pkg/scheduler/internal/cache/comparer:all-srcs",
+        "//pkg/scheduler/internal/cache/debugger:all-srcs",
         "//pkg/scheduler/internal/cache/fake:all-srcs",
     ],
     tags = ["automanaged"],

pkg/scheduler/internal/cache/debugger/BUILD

@@ -2,8 +2,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["comparer.go"],
-    importpath = "k8s.io/kubernetes/pkg/scheduler/internal/cache/comparer",
+    srcs = [
+        "comparer.go",
+        "debugger.go",
+        "dumper.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/scheduler/internal/cache/debugger",
     visibility = ["//pkg/scheduler:__subpackages__"],
     deps = [
         "//pkg/scheduler/cache:go_default_library",

pkg/scheduler/internal/cache/debugger/comparer.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package comparer
+package debugger
 
 import (
 	"sort"
@@ -37,21 +37,6 @@ type CacheComparer struct {
 	PodQueue   internalqueue.SchedulingQueue
 }
 
-// New creates a CacheComparer.
-func New(
-	nodeLister corelisters.NodeLister,
-	podLister corelisters.PodLister,
-	cache schedulerinternalcache.Cache,
-	podQueue internalqueue.SchedulingQueue,
-) *CacheComparer {
-	return &CacheComparer{
-		NodeLister: nodeLister,
-		PodLister:  podLister,
-		Cache:      cache,
-		PodQueue:   podQueue,
-	}
-}
-
 // Compare compares the nodes and pods of NodeLister with Cache.Snapshot.
 func (c *CacheComparer) Compare() error {
 	glog.V(3).Info("cache comparer started")

pkg/scheduler/internal/cache/debugger/comparer_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package comparer
+package debugger
 
 import (
 	"reflect"

pkg/scheduler/internal/cache/debugger/debugger.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package debugger
+
+import (
+	corelisters "k8s.io/client-go/listers/core/v1"
+	internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
+	internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue"
+)
+
+// CacheDebugger provides ways to check and write cache information for debugging.
+type CacheDebugger struct {
+	Comparer CacheComparer
+	Dumper   CacheDumper
+}
+
+// New creates a CacheDebugger.
+func New(
+	nodeLister corelisters.NodeLister,
+	podLister corelisters.PodLister,
+	cache internalcache.Cache,
+	podQueue internalqueue.SchedulingQueue,
+) *CacheDebugger {
+	return &CacheDebugger{
+		Comparer: CacheComparer{
+			NodeLister: nodeLister,
+			PodLister:  podLister,
+			Cache:      cache,
+			PodQueue:   podQueue,
+		},
+		Dumper: CacheDumper{
+			cache:    cache,
+			podQueue: podQueue,
+		},
+	}
+}

pkg/scheduler/internal/cache/debugger/dumper.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package debugger
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/golang/glog"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/scheduler/cache"
+	internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
+	"k8s.io/kubernetes/pkg/scheduler/internal/queue"
+)
+
+// CacheDumper writes some information from the scheduler cache and the scheduling queue to the
+// scheduler logs for debugging purposes.
+type CacheDumper struct {
+	cache    internalcache.Cache
+	podQueue queue.SchedulingQueue
+}
+
+// DumpAll writes cached nodes and scheduling queue information to the scheduler logs.
+func (d *CacheDumper) DumpAll() {
+	d.dumpNodes()
+	d.dumpSchedulingQueue()
+}
+
+// dumpNodes writes NodeInfo to the scheduler logs.
+func (d *CacheDumper) dumpNodes() {
+	snapshot := d.cache.Snapshot()
+	glog.Info("Dump of cached NodeInfo")
+	for _, nodeInfo := range snapshot.Nodes {
+		glog.Info(printNodeInfo(nodeInfo))
+	}
+}
+
+// dumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs.
+func (d *CacheDumper) dumpSchedulingQueue() {
+	waitingPods := d.podQueue.WaitingPods()
+	var podData strings.Builder
+	for _, p := range waitingPods {
+		podData.WriteString(printPod(p))
+	}
+	glog.Infof("Dump of scheduling queue:\n%s", podData.String())
+}
+
+// printNodeInfo writes parts of NodeInfo to a string.
+func printNodeInfo(n *cache.NodeInfo) string {
+	var nodeData strings.Builder
+	nodeData.WriteString(fmt.Sprintf("\nNode name: %+v\nRequested Resources: %+v\nAllocatable Resources:%+v\nNumber of Pods: %v\nPods:\n",
+		n.Node().Name, n.RequestedResource(), n.AllocatableResource(), len(n.Pods())))
+	// Dumping Pod Info
+	for _, p := range n.Pods() {
+		nodeData.WriteString(printPod(p))
+	}
+	return nodeData.String()
+}
+
+// printPod writes parts of a Pod object to a string.
+func printPod(p *v1.Pod) string {
+	return fmt.Sprintf("name: %v, namespace: %v, uid: %v, phase: %v, nominated node: %v\n", p.Name, p.Namespace, p.UID, p.Status.Phase, p.Status.NominatedNodeName)
+}