From 84a7f48cf763213b60ee1614a7b4d63fa0d864c8 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Thu, 7 Mar 2019 09:54:31 +0800 Subject: [PATCH] Add network stats for Windows containers --- pkg/kubelet/stats/cri_stats_provider.go | 26 ++++- .../stats/cri_stats_provider_others.go | 29 +++++ .../stats/cri_stats_provider_windows.go | 107 ++++++++++++++++++ pkg/kubelet/stats/helper.go | 7 +- 4 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 pkg/kubelet/stats/cri_stats_provider_others.go create mode 100644 pkg/kubelet/stats/cri_stats_provider_windows.go diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index 83cc892501..f1dbc34ff0 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -26,11 +26,10 @@ import ( "time" cadvisorfs "github.com/google/cadvisor/fs" - "k8s.io/klog" - cadvisorapiv2 "github.com/google/cadvisor/info/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" @@ -161,6 +160,13 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi } caInfos := getCRICadvisorStats(allInfos) + // get network stats for containers. + // This is only used on Windows. For other platforms, (nil, nil) should be returned. + containerNetworkStats, err := p.listContainerNetworkStats() + if err != nil { + return nil, fmt.Errorf("failed to list container network stats: %v", err) + } + for _, stats := range resp { containerID := stats.Attributes.Id container, found := containerMap[containerID] @@ -184,7 +190,7 @@ func (p *criStatsProvider) listPodStats(updateCPUNanoCoreUsage bool) ([]statsapi // Fill available stats for full set of required pod stats cs := p.makeContainerStats(stats, container, &rootFsInfo, fsIDtoInfo, podSandbox.GetMetadata().GetUid(), updateCPUNanoCoreUsage) - p.addPodNetworkStats(ps, podSandboxID, caInfos, cs) + p.addPodNetworkStats(ps, podSandboxID, caInfos, cs, containerNetworkStats[podSandboxID]) p.addPodCPUMemoryStats(ps, types.UID(podSandbox.Metadata.Uid), allInfos, cs) // If cadvisor stats is available for the container, use it to populate @@ -394,16 +400,26 @@ func (p *criStatsProvider) addPodNetworkStats( podSandboxID string, caInfos map[string]cadvisorapiv2.ContainerInfo, cs *statsapi.ContainerStats, + netStats *statsapi.NetworkStats, ) { caPodSandbox, found := caInfos[podSandboxID] // try get network stats from cadvisor first. if found { - ps.Network = cadvisorInfoToNetworkStats(ps.PodRef.Name, &caPodSandbox) + networkStats := cadvisorInfoToNetworkStats(ps.PodRef.Name, &caPodSandbox) + if networkStats != nil { + ps.Network = networkStats + return + } + } + + // Not found from cadvisor, get from netStats. + if netStats != nil { + ps.Network = netStats return } // TODO: sum Pod network stats from container stats. - klog.V(4).Infof("Unable to find cadvisor stats for sandbox %q", podSandboxID) + klog.V(4).Infof("Unable to find network stats for sandbox %q", podSandboxID) } func (p *criStatsProvider) addPodCPUMemoryStats( diff --git a/pkg/kubelet/stats/cri_stats_provider_others.go b/pkg/kubelet/stats/cri_stats_provider_others.go new file mode 100644 index 0000000000..78a2c2beff --- /dev/null +++ b/pkg/kubelet/stats/cri_stats_provider_others.go @@ -0,0 +1,29 @@ +// +build !windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" +) + +// listContainerNetworkStats returns the network stats of all the running containers. +// It should return (nil, nil) for platforms other than Windows. +func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) { + return nil, nil +} diff --git a/pkg/kubelet/stats/cri_stats_provider_windows.go b/pkg/kubelet/stats/cri_stats_provider_windows.go new file mode 100644 index 0000000000..cc9934e782 --- /dev/null +++ b/pkg/kubelet/stats/cri_stats_provider_windows.go @@ -0,0 +1,107 @@ +// +build windows + +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "time" + + "github.com/Microsoft/hcsshim" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" +) + +// listContainerNetworkStats returns the network stats of all the running containers. +func (p *criStatsProvider) listContainerNetworkStats() (map[string]*statsapi.NetworkStats, error) { + containers, err := hcsshim.GetContainers(hcsshim.ComputeSystemQuery{ + Types: []string{"Container"}, + }) + if err != nil { + return nil, err + } + + stats := make(map[string]*statsapi.NetworkStats) + for _, c := range containers { + container, err := hcsshim.OpenContainer(c.ID) + if err != nil { + klog.V(4).Infof("Failed to open container %q with error '%v', continue to get stats for other containers", c.ID, err) + continue + } + + cstats, err := container.Statistics() + if err != nil { + klog.V(4).Infof("Failed to get statistics for container %q with error '%v', continue to get stats for other containers", c.ID, err) + continue + } + + if len(cstats.Network) > 0 { + stats[c.ID] = hcsStatsToNetworkStats(cstats.Timestamp, cstats.Network) + } + } + + return stats, nil +} + +// hcsStatsToNetworkStats converts hcsshim.Statistics.Network to statsapi.NetworkStats +func hcsStatsToNetworkStats(timestamp time.Time, hcsStats []hcsshim.NetworkStats) *statsapi.NetworkStats { + result := &statsapi.NetworkStats{ + Time: metav1.NewTime(timestamp), + Interfaces: make([]statsapi.InterfaceStats, 0), + } + + adapters := sets.NewString() + for _, stat := range hcsStats { + iStat, err := hcsStatsToInterfaceStats(stat) + if err != nil { + klog.Warningf("Failed to get HNS endpoint %q with error '%v', continue to get stats for other endpoints", stat.EndpointId, err) + continue + } + + // Only count each adapter once. + if adapters.Has(iStat.Name) { + continue + } + + result.Interfaces = append(result.Interfaces, *iStat) + adapters.Insert(iStat.Name) + } + + // TODO(feiskyer): add support of multiple interfaces for getting default interface. + if len(result.Interfaces) > 0 { + result.InterfaceStats = result.Interfaces[0] + } + + return result +} + +// hcsStatsToInterfaceStats converts hcsshim.NetworkStats to statsapi.InterfaceStats. +func hcsStatsToInterfaceStats(stat hcsshim.NetworkStats) (*statsapi.InterfaceStats, error) { + endpoint, err := hcsshim.GetHNSEndpointByID(stat.EndpointId) + if err != nil { + return nil, err + } + + return &statsapi.InterfaceStats{ + Name: endpoint.Name, + RxBytes: &stat.BytesReceived, + TxBytes: &stat.BytesSent, + }, nil +} diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index a26790a5ec..b46c23c98f 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -20,11 +20,10 @@ import ( "fmt" "time" - "k8s.io/klog" - cadvisorapiv1 "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" ) @@ -158,6 +157,10 @@ func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) return nil } + if cstat.Network == nil { + return nil + } + iStats := statsapi.NetworkStats{ Time: metav1.NewTime(cstat.Timestamp), }