Merge pull request #62266 from feiskyer/win-log-stats

Automatic merge from submit-queue (batch tested with PRs 62266, 64351, 64366, 64235, 64560). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Add log and fs stats for Windows containers

**What this PR does / why we need it**:

Add log and fs stats for Windows containers.

Without this, kubelet will report errors continuously:

```
Unable to fetch container log stats for path \var\log\pods\2a70ed65-37ae-11e8-8730-000d3a14b1a0\echo: Du not supported for this build.
```

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #60180 #62047

**Special notes for your reviewer**:

**Release note**:

```release-note
Add log and fs stats for Windows containers
```

pkg/kubelet/cadvisor/cadvisor_windows.go

@@ -26,6 +26,7 @@ import (
 )
 
 type cadvisorClient struct {
+	rootPath       string
 	winStatsClient winstats.Client
 }
 
@@ -34,7 +35,10 @@ var _ Interface = new(cadvisorClient)
 // New creates a cAdvisor and exports its API on the specified port if port > 0.
 func New(address string, port uint, imageFsInfoProvider ImageFsInfoProvider, rootPath string, usingLegacyStats bool) (Interface, error) {
 	client, err := winstats.NewPerfCounterClient()
-	return &cadvisorClient{winStatsClient: client}, err
+	return &cadvisorClient{
+		rootPath:       rootPath,
+		winStatsClient: client,
+	}, err
 }
 
 func (cu *cadvisorClient) Start() error {
@@ -70,7 +74,7 @@ func (cu *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
 }
 
 func (cu *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
-	return cadvisorapiv2.FsInfo{}, nil
+	return cu.GetDirFsInfo(cu.rootPath)
 }
 
 func (cu *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) {

pkg/kubelet/dockershim/docker_stats_windows.go

@@ -64,6 +64,11 @@ func (ds *dockerService) ListContainerStats(ctx context.Context, r *runtimeapi.L
 }
 
 func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.ContainerStats, error) {
+	info, err := ds.client.Info()
+	if err != nil {
+		return nil, err
+	}
+
 	statsJSON, err := ds.client.GetContainerStats(containerID)
 	if err != nil {
 		return nil, err
@@ -101,6 +106,7 @@ func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.Cont
 		},
 		WritableLayer: &runtimeapi.FilesystemUsage{
 			Timestamp: timestamp,
+			FsId:      &runtimeapi.FilesystemIdentifier{Mountpoint: info.DockerRootDir},
 			UsedBytes: &runtimeapi.UInt64Value{Value: uint64(*containerJSON.SizeRw)},
 		},
 	}

pkg/kubelet/stats/cri_stats_provider.go

@@ -142,12 +142,14 @@ func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
 		ps, found := sandboxIDToPodStats[podSandboxID]
 		if !found {
 			ps = buildPodStats(podSandbox)
-			// Fill stats from cadvisor is available for full set of required pod stats
-			p.addCadvisorPodNetworkStats(ps, podSandboxID, caInfos)
-			p.addCadvisorPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos)
 			sandboxIDToPodStats[podSandboxID] = ps
 		}
+
+		// Fill available stats for full set of required pod stats
 		cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata().GetUid())
+		p.addPodNetworkStats(ps, podSandboxID, caInfos, cs)
+		p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs)
+
 		// If cadvisor stats is available for the container, use it to populate
 		// container stats
 		caStats, caFound := caInfos[containerID]
@@ -263,29 +265,69 @@ func (p *criStatsProvider) makePodStorageStats(s *statsapi.PodStats, rootFsInfo
 	return s
 }
 
-func (p *criStatsProvider) addCadvisorPodNetworkStats(
+func (p *criStatsProvider) addPodNetworkStats(
 	ps *statsapi.PodStats,
 	podSandboxID string,
 	caInfos map[string]cadvisorapiv2.ContainerInfo,
+	cs *statsapi.ContainerStats,
 ) {
 	caPodSandbox, found := caInfos[podSandboxID]
+	// try get network stats from cadvisor first.
 	if found {
 		ps.Network = cadvisorInfoToNetworkStats(ps.PodRef.Name, &caPodSandbox)
-	} else {
+		return
-		glog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID)
 	}
+
+	// TODO: sum Pod network stats from container stats.
+	glog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID)
 }
 
-func (p *criStatsProvider) addCadvisorPodCPUMemoryStats(
+func (p *criStatsProvider) addPodCPUMemoryStats(
 	ps *statsapi.PodStats,
 	podUID types.UID,
 	allInfos map[string]cadvisorapiv2.ContainerInfo,
+	cs *statsapi.ContainerStats,
 ) {
+	// try get cpu and memory stats from cadvisor first.
 	podCgroupInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos)
 	if podCgroupInfo != nil {
 		cpu, memory := cadvisorInfoToCPUandMemoryStats(podCgroupInfo)
 		ps.CPU = cpu
 		ps.Memory = memory
+		return
+	}
+
+	// Sum Pod cpu and memory stats from containers stats.
+	if cs.CPU != nil {
+		if ps.CPU == nil {
+			ps.CPU = &statsapi.CPUStats{}
+		}
+
+		ps.CPU.Time = cs.StartTime
+		usageCoreNanoSeconds := getUint64Value(cs.CPU.UsageCoreNanoSeconds) + getUint64Value(ps.CPU.UsageCoreNanoSeconds)
+		usageNanoCores := getUint64Value(cs.CPU.UsageNanoCores) + getUint64Value(ps.CPU.UsageNanoCores)
+		ps.CPU.UsageCoreNanoSeconds = &usageCoreNanoSeconds
+		ps.CPU.UsageNanoCores = &usageNanoCores
+	}
+
+	if cs.Memory != nil {
+		if ps.Memory == nil {
+			ps.Memory = &statsapi.MemoryStats{}
+		}
+
+		ps.Memory.Time = cs.Memory.Time
+		availableBytes := getUint64Value(cs.Memory.AvailableBytes) + getUint64Value(ps.Memory.AvailableBytes)
+		usageBytes := getUint64Value(cs.Memory.UsageBytes) + getUint64Value(ps.Memory.UsageBytes)
+		workingSetBytes := getUint64Value(cs.Memory.WorkingSetBytes) + getUint64Value(ps.Memory.WorkingSetBytes)
+		rSSBytes := getUint64Value(cs.Memory.RSSBytes) + getUint64Value(ps.Memory.RSSBytes)
+		pageFaults := getUint64Value(cs.Memory.PageFaults) + getUint64Value(ps.Memory.PageFaults)
+		majorPageFaults := getUint64Value(cs.Memory.MajorPageFaults) + getUint64Value(ps.Memory.MajorPageFaults)
+		ps.Memory.AvailableBytes = &availableBytes
+		ps.Memory.UsageBytes = &usageBytes
+		ps.Memory.WorkingSetBytes = &workingSetBytes
+		ps.Memory.RSSBytes = &rSSBytes
+		ps.Memory.PageFaults = &pageFaults
+		ps.Memory.MajorPageFaults = &majorPageFaults
 	}
 }
 

pkg/kubelet/stats/helper.go

@@ -303,3 +303,11 @@ func buildRootfsStats(cstat *cadvisorapiv2.ContainerStats, imageFs *cadvisorapiv
 		Inodes:         imageFs.Inodes,
 	}
 }
+
+func getUint64Value(value *uint64) uint64 {
+	if value == nil {
+		return 0
+	}
+
+	return *value
+}

pkg/volume/metrics_du.go

@@ -25,7 +25,7 @@ import (
 var _ MetricsProvider = &metricsDu{}
 
 // metricsDu represents a MetricsProvider that calculates the used and
-// available Volume space by executing the "du" command and gathering
+// available Volume space by calling fs.DiskUsage() and gathering
 // filesystem info for the Volume path.
 type metricsDu struct {
 	// the directory path the volume is mounted to.
@@ -46,7 +46,7 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) {
 		return metrics, NewNoPathDefinedError()
 	}
 
-	err := md.runDu(metrics)
+	err := md.runDiskUsage(metrics)
 	if err != nil {
 		return metrics, err
 	}
@@ -64,9 +64,9 @@ func (md *metricsDu) GetMetrics() (*Metrics, error) {
 	return metrics, nil
 }
 
-// runDu executes the "du" command and writes the results to metrics.Used
+// runDiskUsage gets disk usage of md.path and writes the results to metrics.Used
-func (md *metricsDu) runDu(metrics *Metrics) error {
+func (md *metricsDu) runDiskUsage(metrics *Metrics) error {
-	used, err := fs.Du(md.path)
+	used, err := fs.DiskUsage(md.path)
 	if err != nil {
 		return err
 	}

pkg/volume/util/fs/BUILD

@@ -34,7 +34,7 @@ go_library(
             "fs_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:windows": [
-            "fs_unsupported.go",
+            "fs_windows.go",
         ],
         "//conditions:default": [],
     }),
@@ -74,6 +74,7 @@ go_library(
             "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:windows": [
+            "//vendor/golang.org/x/sys/windows:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         ],
         "//conditions:default": [],

pkg/volume/util/fs/fs.go

@@ -54,7 +54,8 @@ func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
 	return available, capacity, usage, inodes, inodesFree, inodesUsed, nil
 }
 
-func Du(path string) (*resource.Quantity, error) {
+// DiskUsage gets disk usage of specified path.
+func DiskUsage(path string) (*resource.Quantity, error) {
 	// Uses the same niceness level as cadvisor.fs does when running du
 	// Uses -B 1 to always scale to a blocksize of 1 byte
 	out, err := exec.Command("nice", "-n", "19", "du", "-s", "-B", "1", path).CombinedOutput()

pkg/volume/util/fs/fs_unsupported.go

@@ -1,4 +1,4 @@
-// +build !linux,!darwin
+// +build !linux,!darwin,!windows
 
 /*
 Copyright 2014 The Kubernetes Authors.
@@ -29,7 +29,8 @@ func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
 	return 0, 0, 0, 0, 0, 0, fmt.Errorf("FsInfo not supported for this build.")
 }
 
-func Du(path string) (*resource.Quantity, error) {
+// DiskUsage gets disk usage of specified path.
+func DiskUsage(path string) (*resource.Quantity, error) {
 	return nil, fmt.Errorf("Du not supported for this build.")
 }
 

pkg/volume/util/fs/fs_windows.go

@@ -0,0 +1,77 @@
+// +build windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fs
+
+import (
+	"fmt"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+var (
+	modkernel32            = windows.NewLazySystemDLL("kernel32.dll")
+	procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW")
+)
+
+// FSInfo returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
+// for the filesystem that path resides upon.
+func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
+	var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes int64
+	var err error
+
+	ret, _, err := syscall.Syscall6(
+		procGetDiskFreeSpaceEx.Addr(),
+		4,
+		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
+		uintptr(unsafe.Pointer(&freeBytesAvailable)),
+		uintptr(unsafe.Pointer(&totalNumberOfBytes)),
+		uintptr(unsafe.Pointer(&totalNumberOfFreeBytes)),
+		0,
+		0,
+	)
+	if ret == 0 {
+		return 0, 0, 0, 0, 0, 0, err
+	}
+
+	return freeBytesAvailable, totalNumberOfBytes, totalNumberOfBytes - freeBytesAvailable, 0, 0, 0, nil
+}
+
+// DiskUsage gets disk usage of specified path.
+func DiskUsage(path string) (*resource.Quantity, error) {
+	_, _, usage, _, _, _, err := FsInfo(path)
+	if err != nil {
+		return nil, err
+	}
+
+	used, err := resource.ParseQuantity(fmt.Sprintf("%d", usage))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse fs usage %d due to %v", usage, err)
+	}
+	used.Format = resource.BinarySI
+	return &used, nil
+}
+
+// Always return zero since inodes is not supported on Windows.
+func Find(path string) (int64, error) {
+	return 0, nil
+}