From 9a7ff8dbe5f4f962b5edf2bab168ada635c66083 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Mon, 9 Apr 2018 13:16:33 +0800 Subject: [PATCH 1/4] Add log stats for Windows containers --- pkg/volume/util/fs/BUILD | 3 +- pkg/volume/util/fs/fs_unsupported.go | 2 +- pkg/volume/util/fs/fs_windows.go | 76 ++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 pkg/volume/util/fs/fs_windows.go diff --git a/pkg/volume/util/fs/BUILD b/pkg/volume/util/fs/BUILD index 205ddaaaeb..0e9be3016b 100644 --- a/pkg/volume/util/fs/BUILD +++ b/pkg/volume/util/fs/BUILD @@ -34,7 +34,7 @@ go_library( "fs_unsupported.go", ], "@io_bazel_rules_go//go/platform:windows": [ - "fs_unsupported.go", + "fs_windows.go", ], "//conditions:default": [], }), @@ -74,6 +74,7 @@ go_library( "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", ], "@io_bazel_rules_go//go/platform:windows": [ + "//vendor/golang.org/x/sys/windows:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", ], "//conditions:default": [], diff --git a/pkg/volume/util/fs/fs_unsupported.go b/pkg/volume/util/fs/fs_unsupported.go index da41fc8eee..b48e4fd20c 100644 --- a/pkg/volume/util/fs/fs_unsupported.go +++ b/pkg/volume/util/fs/fs_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux,!darwin +// +build !linux,!darwin,!windows /* Copyright 2014 The Kubernetes Authors. diff --git a/pkg/volume/util/fs/fs_windows.go b/pkg/volume/util/fs/fs_windows.go new file mode 100644 index 0000000000..534604d2b2 --- /dev/null +++ b/pkg/volume/util/fs/fs_windows.go @@ -0,0 +1,76 @@ +// +build windows + +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fs + +import ( + "fmt" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" + + "k8s.io/apimachinery/pkg/api/resource" +) + +var ( + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW") +) + +// FSInfo returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error) +// for the filesystem that path resides upon. +func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) { + var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes int64 + var err error + + ret, _, err := syscall.Syscall6( + procGetDiskFreeSpaceEx.Addr(), + 4, + uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))), + uintptr(unsafe.Pointer(&freeBytesAvailable)), + uintptr(unsafe.Pointer(&totalNumberOfBytes)), + uintptr(unsafe.Pointer(&totalNumberOfFreeBytes)), + 0, + 0, + ) + if ret == 0 { + return 0, 0, 0, 0, 0, 0, err + } + + return freeBytesAvailable, totalNumberOfBytes, totalNumberOfBytes - freeBytesAvailable, 0, 0, 0, nil +} + +func Du(path string) (*resource.Quantity, error) { + _, _, usage, _, _, _, err := FsInfo(path) + if err != nil { + return nil, err + } + + used, err := resource.ParseQuantity(fmt.Sprintf("%d", usage)) + if err != nil { + return nil, fmt.Errorf("failed to parse fs usage %d due to %v", usage, err) + } + used.Format = resource.BinarySI + return &used, nil +} + +// Always return zero since inodes is not supported on Windows. +func Find(path string) (int64, error) { + return 0, nil +} From 196381c0391cf9a7b356d4a9197b8d8affa08ffe Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Thu, 17 May 2018 14:21:49 +0800 Subject: [PATCH 2/4] Add fs status for Windows containers --- pkg/kubelet/cadvisor/cadvisor_windows.go | 8 ++++++-- pkg/kubelet/dockershim/docker_stats_windows.go | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/cadvisor/cadvisor_windows.go b/pkg/kubelet/cadvisor/cadvisor_windows.go index 597721cd17..eb416a141d 100644 --- a/pkg/kubelet/cadvisor/cadvisor_windows.go +++ b/pkg/kubelet/cadvisor/cadvisor_windows.go @@ -26,6 +26,7 @@ import ( ) type cadvisorClient struct { + rootPath string winStatsClient winstats.Client } @@ -34,7 +35,10 @@ var _ Interface = new(cadvisorClient) // New creates a cAdvisor and exports its API on the specified port if port > 0. func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string, usingLegacyStats bool) (Interface, error) { client, err := winstats.NewPerfCounterClient() - return &cadvisorClient{winStatsClient: client}, err + return &cadvisorClient{ + rootPath: rootPath, + winStatsClient: client, + }, err } func (cu *cadvisorClient) Start() error { @@ -70,7 +74,7 @@ func (cu *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) { } func (cu *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) { - return cadvisorapiv2.FsInfo{}, nil + return cu.GetDirFsInfo(cu.rootPath) } func (cu *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) { diff --git a/pkg/kubelet/dockershim/docker_stats_windows.go b/pkg/kubelet/dockershim/docker_stats_windows.go index a67a0511a0..a690f883b6 100644 --- a/pkg/kubelet/dockershim/docker_stats_windows.go +++ b/pkg/kubelet/dockershim/docker_stats_windows.go @@ -64,6 +64,11 @@ func (ds *dockerService) ListContainerStats(ctx context.Context, r *runtimeapi.L } func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.ContainerStats, error) { + info, err := ds.client.Info() + if err != nil { + return nil, err + } + statsJSON, err := ds.client.GetContainerStats(containerID) if err != nil { return nil, err @@ -101,6 +106,7 @@ func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.Cont }, WritableLayer: &runtimeapi.FilesystemUsage{ Timestamp: timestamp, + FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: info.DockerRootDir}, UsedBytes: &runtimeapi.UInt64Value{Value: uint64(*containerJSON.SizeRw)}, }, } From 7cbee06a00ffc93d7178634242deb0a9968eb916 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Thu, 17 May 2018 15:28:46 +0800 Subject: [PATCH 3/4] Add Pod stats for Windows containers --- pkg/kubelet/stats/cri_stats_provider.go | 56 +++++++++++++++++++++---- pkg/kubelet/stats/helper.go | 8 ++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index e82ca97a0d..1af0a0cbfb 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -142,12 +142,14 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { ps, found := sandboxIDToPodStats[podSandboxID] if !found { ps = buildPodStats(podSandbox) - // Fill stats from cadvisor is available for full set of required pod stats - p.addCadvisorPodNetworkStats(ps, podSandboxID, caInfos) - p.addCadvisorPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos) sandboxIDToPodStats[podSandboxID] = ps } + + // Fill available stats for full set of required pod stats cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata().GetUid()) + p.addPodNetworkStats(ps, podSandboxID, caInfos, cs) + p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs) + // If cadvisor stats is available for the container, use it to populate // container stats caStats, caFound := caInfos[containerID] @@ -263,29 +265,69 @@ func (p *criStatsProvider) makePodStorageStats(s *statsapi.PodStats, rootFsInfo return s } -func (p *criStatsProvider) addCadvisorPodNetworkStats( +func (p *criStatsProvider) addPodNetworkStats( ps *statsapi.PodStats, podSandboxID string, caInfos map[string]cadvisorapiv2.ContainerInfo, + cs *statsapi.ContainerStats, ) { caPodSandbox, found := caInfos[podSandboxID] + // try get network stats from cadvisor first. if found { ps.Network = cadvisorInfoToNetworkStats(ps.PodRef.Name, &caPodSandbox) - } else { - glog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID) + return } + + // TODO: sum Pod network stats from container stats. + glog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID) } -func (p *criStatsProvider) addCadvisorPodCPUMemoryStats( +func (p *criStatsProvider) addPodCPUMemoryStats( ps *statsapi.PodStats, podUID types.UID, allInfos map[string]cadvisorapiv2.ContainerInfo, + cs *statsapi.ContainerStats, ) { + // try get cpu and memory stats from cadvisor first. podCgroupInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos) if podCgroupInfo != nil { cpu, memory := cadvisorInfoToCPUandMemoryStats(podCgroupInfo) ps.CPU = cpu ps.Memory = memory + return + } + + // Sum Pod cpu and memory stats from containers stats. + if cs.CPU != nil { + if ps.CPU == nil { + ps.CPU = &statsapi.CPUStats{} + } + + ps.CPU.Time = cs.StartTime + usageCoreNanoSeconds := getUint64Value(cs.CPU.UsageCoreNanoSeconds) + getUint64Value(ps.CPU.UsageCoreNanoSeconds) + usageNanoCores := getUint64Value(cs.CPU.UsageNanoCores) + getUint64Value(ps.CPU.UsageNanoCores) + ps.CPU.UsageCoreNanoSeconds = &usageCoreNanoSeconds + ps.CPU.UsageNanoCores = &usageNanoCores + } + + if cs.Memory != nil { + if ps.Memory == nil { + ps.Memory = &statsapi.MemoryStats{} + } + + ps.Memory.Time = cs.Memory.Time + availableBytes := getUint64Value(cs.Memory.AvailableBytes) + getUint64Value(ps.Memory.AvailableBytes) + usageBytes := getUint64Value(cs.Memory.UsageBytes) + getUint64Value(ps.Memory.UsageBytes) + workingSetBytes := getUint64Value(cs.Memory.WorkingSetBytes) + getUint64Value(ps.Memory.WorkingSetBytes) + rSSBytes := getUint64Value(cs.Memory.RSSBytes) + getUint64Value(ps.Memory.RSSBytes) + pageFaults := getUint64Value(cs.Memory.PageFaults) + getUint64Value(ps.Memory.PageFaults) + majorPageFaults := getUint64Value(cs.Memory.MajorPageFaults) + getUint64Value(ps.Memory.MajorPageFaults) + ps.Memory.AvailableBytes = &availableBytes + ps.Memory.UsageBytes = &usageBytes + ps.Memory.WorkingSetBytes = &workingSetBytes + ps.Memory.RSSBytes = &rSSBytes + ps.Memory.PageFaults = &pageFaults + ps.Memory.MajorPageFaults = &majorPageFaults } } diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index 77f3a6569e..a195b9c634 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -303,3 +303,11 @@ func buildRootfsStats(cstat *cadvisorapiv2.ContainerStats, imageFs *cadvisorapiv Inodes: imageFs.Inodes, } } + +func getUint64Value(value *uint64) uint64 { + if value == nil { + return 0 + } + + return *value +} From 66da2ddcd03c236bd7a9db72fee08b3391fc9269 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Tue, 22 May 2018 10:51:26 +0800 Subject: [PATCH 4/4] Rename Du() to DiskUsage() for more expressive --- pkg/volume/metrics_du.go | 10 +++++----- pkg/volume/util/fs/fs.go | 3 ++- pkg/volume/util/fs/fs_unsupported.go | 3 ++- pkg/volume/util/fs/fs_windows.go | 3 ++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/volume/metrics_du.go b/pkg/volume/metrics_du.go index 88a985d5ac..1cae99c107 100644 --- a/pkg/volume/metrics_du.go +++ b/pkg/volume/metrics_du.go @@ -25,7 +25,7 @@ import ( var _ MetricsProvider = &metricsDu{} // metricsDu represents a MetricsProvider that calculates the used and -// available Volume space by executing the "du" command and gathering +// available Volume space by calling fs.DiskUsage() and gathering // filesystem info for the Volume path. type metricsDu struct { // the directory path the volume is mounted to. @@ -46,7 +46,7 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) { return metrics, NewNoPathDefinedError() } - err := md.runDu(metrics) + err := md.runDiskUsage(metrics) if err != nil { return metrics, err } @@ -64,9 +64,9 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) { return metrics, nil } -// runDu executes the "du" command and writes the results to metrics.Used -func (md *metricsDu) runDu(metrics *Metrics) error { - used, err := fs.Du(md.path) +// runDiskUsage gets disk usage of md.path and writes the results to metrics.Used +func (md *metricsDu) runDiskUsage(metrics *Metrics) error { + used, err := fs.DiskUsage(md.path) if err != nil { return err } diff --git a/pkg/volume/util/fs/fs.go b/pkg/volume/util/fs/fs.go index bbb4b0105c..a80a167eea 100644 --- a/pkg/volume/util/fs/fs.go +++ b/pkg/volume/util/fs/fs.go @@ -54,7 +54,8 @@ func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) { return available, capacity, usage, inodes, inodesFree, inodesUsed, nil } -func Du(path string) (*resource.Quantity, error) { +// DiskUsage gets disk usage of specified path. +func DiskUsage(path string) (*resource.Quantity, error) { // Uses the same niceness level as cadvisor.fs does when running du // Uses -B 1 to always scale to a blocksize of 1 byte out, err := exec.Command("nice", "-n", "19", "du", "-s", "-B", "1", path).CombinedOutput() diff --git a/pkg/volume/util/fs/fs_unsupported.go b/pkg/volume/util/fs/fs_unsupported.go index b48e4fd20c..340b4fdc22 100644 --- a/pkg/volume/util/fs/fs_unsupported.go +++ b/pkg/volume/util/fs/fs_unsupported.go @@ -29,7 +29,8 @@ func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) { return 0, 0, 0, 0, 0, 0, fmt.Errorf("FsInfo not supported for this build.") } -func Du(path string) (*resource.Quantity, error) { +// DiskUsage gets disk usage of specified path. +func DiskUsage(path string) (*resource.Quantity, error) { return nil, fmt.Errorf("Du not supported for this build.") } diff --git a/pkg/volume/util/fs/fs_windows.go b/pkg/volume/util/fs/fs_windows.go index 534604d2b2..ea8c4784ae 100644 --- a/pkg/volume/util/fs/fs_windows.go +++ b/pkg/volume/util/fs/fs_windows.go @@ -56,7 +56,8 @@ func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) { return freeBytesAvailable, totalNumberOfBytes, totalNumberOfBytes - freeBytesAvailable, 0, 0, 0, nil } -func Du(path string) (*resource.Quantity, error) { +// DiskUsage gets disk usage of specified path. +func DiskUsage(path string) (*resource.Quantity, error) { _, _, usage, _, _, _, err := FsInfo(path) if err != nil { return nil, err