From 7ce3245ca9bc9bfe541101dc2e8983e736f21065 Mon Sep 17 00:00:00 2001 From: "Bobby (Babak) Salamat" Date: Wed, 31 Oct 2018 11:45:15 -0700 Subject: [PATCH 1/3] Add a scheduler cache dumper --- pkg/scheduler/factory/factory.go | 7 +- .../cache/{comparer => debugger}/BUILD | 0 .../cache/{comparer => debugger}/comparer.go | 17 +--- .../{comparer => debugger}/comparer_test.go | 2 +- .../internal/cache/debugger/debugger.go | 50 ++++++++++++ .../internal/cache/debugger/dumper.go | 78 +++++++++++++++++++ 6 files changed, 134 insertions(+), 20 deletions(-) rename pkg/scheduler/internal/cache/{comparer => debugger}/BUILD (100%) rename pkg/scheduler/internal/cache/{comparer => debugger}/comparer.go (91%) rename pkg/scheduler/internal/cache/{comparer => debugger}/comparer_test.go (99%) create mode 100644 pkg/scheduler/internal/cache/debugger/debugger.go create mode 100644 pkg/scheduler/internal/cache/debugger/dumper.go diff --git a/pkg/scheduler/factory/factory.go b/pkg/scheduler/factory/factory.go index 2d0801480f..f77362465b 100644 --- a/pkg/scheduler/factory/factory.go +++ b/pkg/scheduler/factory/factory.go @@ -60,7 +60,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/core" "k8s.io/kubernetes/pkg/scheduler/core/equivalence" schedulerinternalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" - cachecomparer "k8s.io/kubernetes/pkg/scheduler/internal/cache/comparer" + cachedebugger "k8s.io/kubernetes/pkg/scheduler/internal/cache/debugger" internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" "k8s.io/kubernetes/pkg/scheduler/util" "k8s.io/kubernetes/pkg/scheduler/volumebinder" @@ -398,7 +398,7 @@ func NewConfigFactory(args *ConfigFactoryArgs) Configurator { } // Setup cache comparer - comparer := cachecomparer.New( + debugger := cachedebugger.New( args.NodeInformer.Lister(), args.PodInformer.Lister(), c.schedulerCache, @@ -415,7 +415,8 @@ func NewConfigFactory(args *ConfigFactoryArgs) Configurator { c.podQueue.Close() return case <-ch: - comparer.Compare() + debugger.Comparer.Compare() + debugger.Dumper.DumpAll() } } }() diff --git a/pkg/scheduler/internal/cache/comparer/BUILD b/pkg/scheduler/internal/cache/debugger/BUILD similarity index 100% rename from pkg/scheduler/internal/cache/comparer/BUILD rename to pkg/scheduler/internal/cache/debugger/BUILD diff --git a/pkg/scheduler/internal/cache/comparer/comparer.go b/pkg/scheduler/internal/cache/debugger/comparer.go similarity index 91% rename from pkg/scheduler/internal/cache/comparer/comparer.go rename to pkg/scheduler/internal/cache/debugger/comparer.go index bf043fa3f9..00a1b0c3d6 100644 --- a/pkg/scheduler/internal/cache/comparer/comparer.go +++ b/pkg/scheduler/internal/cache/debugger/comparer.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package comparer +package debugger import ( "sort" @@ -37,21 +37,6 @@ type CacheComparer struct { PodQueue internalqueue.SchedulingQueue } -// New creates a CacheComparer. -func New( - nodeLister corelisters.NodeLister, - podLister corelisters.PodLister, - cache schedulerinternalcache.Cache, - podQueue internalqueue.SchedulingQueue, -) *CacheComparer { - return &CacheComparer{ - NodeLister: nodeLister, - PodLister: podLister, - Cache: cache, - PodQueue: podQueue, - } -} - // Compare compares the nodes and pods of NodeLister with Cache.Snapshot. func (c *CacheComparer) Compare() error { glog.V(3).Info("cache comparer started") diff --git a/pkg/scheduler/internal/cache/comparer/comparer_test.go b/pkg/scheduler/internal/cache/debugger/comparer_test.go similarity index 99% rename from pkg/scheduler/internal/cache/comparer/comparer_test.go rename to pkg/scheduler/internal/cache/debugger/comparer_test.go index 701bd87ada..967b4027b5 100644 --- a/pkg/scheduler/internal/cache/comparer/comparer_test.go +++ b/pkg/scheduler/internal/cache/debugger/comparer_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package comparer +package debugger import ( "reflect" diff --git a/pkg/scheduler/internal/cache/debugger/debugger.go b/pkg/scheduler/internal/cache/debugger/debugger.go new file mode 100644 index 0000000000..64428d5693 --- /dev/null +++ b/pkg/scheduler/internal/cache/debugger/debugger.go @@ -0,0 +1,50 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debugger + +import ( + corelisters "k8s.io/client-go/listers/core/v1" + internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" + internalqueue "k8s.io/kubernetes/pkg/scheduler/internal/queue" +) + +// CacheDebugger provides ways to check and write cache information for debugging. +type CacheDebugger struct { + Comparer CacheComparer + Dumper CacheDumper +} + +// New creates a CacheDebugger. +func New( + nodeLister corelisters.NodeLister, + podLister corelisters.PodLister, + cache internalcache.Cache, + podQueue internalqueue.SchedulingQueue, +) *CacheDebugger { + return &CacheDebugger{ + Comparer: CacheComparer{ + NodeLister: nodeLister, + PodLister: podLister, + Cache: cache, + PodQueue: podQueue, + }, + Dumper: CacheDumper{ + cache: cache, + podQueue: podQueue, + }, + } +} diff --git a/pkg/scheduler/internal/cache/debugger/dumper.go b/pkg/scheduler/internal/cache/debugger/dumper.go new file mode 100644 index 0000000000..2890054d80 --- /dev/null +++ b/pkg/scheduler/internal/cache/debugger/dumper.go @@ -0,0 +1,78 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debugger + +import ( + "fmt" + "strings" + + "github.com/golang/glog" + + "k8s.io/api/core/v1" + "k8s.io/kubernetes/pkg/scheduler/cache" + internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache" + "k8s.io/kubernetes/pkg/scheduler/internal/queue" +) + +// CacheDumper writes some information from the scheduler cache and the scheduling queue to the +// scheduler logs for debugging purposes. +type CacheDumper struct { + cache internalcache.Cache + podQueue queue.SchedulingQueue +} + +// DumpAll writes cached nodes and scheduling queue information to the scheduler logs. +func (d *CacheDumper) DumpAll() { + d.DumpNodes() + d.DumpSchedulingQueue() +} + +// DumpNodes writes NodeInfo to the scheduler logs. +func (d *CacheDumper) DumpNodes() { + snapshot := d.cache.Snapshot() + glog.Info("Dump of cached NodeInfo") + for _, info := range snapshot.Nodes { + glog.Info(printNodeInfo(info)) + } +} + +// DumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs. +func (d *CacheDumper) DumpSchedulingQueue() { + waitingPods := d.podQueue.WaitingPods() + var podData strings.Builder + for _, p := range waitingPods { + podData.WriteString(printPod(p)) + } + glog.Infof("Dump of scheduling queue:\n%s", podData.String()) +} + +// printNodeInfo writes parts of NodeInfo to a string. +func printNodeInfo(n *cache.NodeInfo) string { + var nodeData strings.Builder + nodeData.WriteString(fmt.Sprintf("\nNode name: %+v\nRequested Resources: %+v\nAllocatable Resources:%+v\nNumber of Pods: %v\nPods:\n", + n.Node().Name, n.RequestedResource(), n.AllocatableResource(), len(n.Pods()))) + // Dumping Pod Info + for _, p := range n.Pods() { + nodeData.WriteString(printPod(p)) + } + return nodeData.String() +} + +// printPod writes parts of a Pod object to a string. +func printPod(p *v1.Pod) string { + return fmt.Sprintf("name: %v, namespace: %v, uid: %v, phase: %v, nominated node: %v\n", p.Name, p.Namespace, p.UID, p.Status.Phase, p.Status.NominatedNodeName) +} From 4bb57c440eb1b8d138994644f546302636c33c20 Mon Sep 17 00:00:00 2001 From: "Bobby (Babak) Salamat" Date: Wed, 31 Oct 2018 11:51:38 -0700 Subject: [PATCH 2/3] Autogenerated files --- pkg/scheduler/factory/BUILD | 2 +- pkg/scheduler/internal/cache/BUILD | 2 +- pkg/scheduler/internal/cache/debugger/BUILD | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/scheduler/factory/BUILD b/pkg/scheduler/factory/BUILD index dfade874b3..6530261291 100644 --- a/pkg/scheduler/factory/BUILD +++ b/pkg/scheduler/factory/BUILD @@ -23,7 +23,7 @@ go_library( "//pkg/scheduler/core:go_default_library", "//pkg/scheduler/core/equivalence:go_default_library", "//pkg/scheduler/internal/cache:go_default_library", - "//pkg/scheduler/internal/cache/comparer:go_default_library", + "//pkg/scheduler/internal/cache/debugger:go_default_library", "//pkg/scheduler/internal/queue:go_default_library", "//pkg/scheduler/util:go_default_library", "//pkg/scheduler/volumebinder:go_default_library", diff --git a/pkg/scheduler/internal/cache/BUILD b/pkg/scheduler/internal/cache/BUILD index cce14ec53b..53f7681782 100644 --- a/pkg/scheduler/internal/cache/BUILD +++ b/pkg/scheduler/internal/cache/BUILD @@ -54,7 +54,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", - "//pkg/scheduler/internal/cache/comparer:all-srcs", + "//pkg/scheduler/internal/cache/debugger:all-srcs", "//pkg/scheduler/internal/cache/fake:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/scheduler/internal/cache/debugger/BUILD b/pkg/scheduler/internal/cache/debugger/BUILD index c35831fbb8..42c9c06061 100644 --- a/pkg/scheduler/internal/cache/debugger/BUILD +++ b/pkg/scheduler/internal/cache/debugger/BUILD @@ -2,8 +2,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["comparer.go"], - importpath = "k8s.io/kubernetes/pkg/scheduler/internal/cache/comparer", + srcs = [ + "comparer.go", + "debugger.go", + "dumper.go", + ], + importpath = "k8s.io/kubernetes/pkg/scheduler/internal/cache/debugger", visibility = ["//pkg/scheduler:__subpackages__"], deps = [ "//pkg/scheduler/cache:go_default_library", From 48557a163a2659c4e675c4ad50623dcd5e132768 Mon Sep 17 00:00:00 2001 From: "Bobby (Babak) Salamat" Date: Tue, 6 Nov 2018 10:08:22 -0800 Subject: [PATCH 3/3] fixup! Add a scheduler cache dumper --- pkg/scheduler/internal/cache/debugger/dumper.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/scheduler/internal/cache/debugger/dumper.go b/pkg/scheduler/internal/cache/debugger/dumper.go index 2890054d80..213468c789 100644 --- a/pkg/scheduler/internal/cache/debugger/dumper.go +++ b/pkg/scheduler/internal/cache/debugger/dumper.go @@ -37,21 +37,21 @@ type CacheDumper struct { // DumpAll writes cached nodes and scheduling queue information to the scheduler logs. func (d *CacheDumper) DumpAll() { - d.DumpNodes() - d.DumpSchedulingQueue() + d.dumpNodes() + d.dumpSchedulingQueue() } -// DumpNodes writes NodeInfo to the scheduler logs. -func (d *CacheDumper) DumpNodes() { +// dumpNodes writes NodeInfo to the scheduler logs. +func (d *CacheDumper) dumpNodes() { snapshot := d.cache.Snapshot() glog.Info("Dump of cached NodeInfo") - for _, info := range snapshot.Nodes { - glog.Info(printNodeInfo(info)) + for _, nodeInfo := range snapshot.Nodes { + glog.Info(printNodeInfo(nodeInfo)) } } -// DumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs. -func (d *CacheDumper) DumpSchedulingQueue() { +// dumpSchedulingQueue writes pods in the scheduling queue to the scheduler logs. +func (d *CacheDumper) dumpSchedulingQueue() { waitingPods := d.podQueue.WaitingPods() var podData strings.Builder for _, p := range waitingPods {