diff --git a/pkg/kubelet/eviction/eviction_manager.go b/pkg/kubelet/eviction/eviction_manager.go index 007c204f7f..7811f3bea5 100644 --- a/pkg/kubelet/eviction/eviction_manager.go +++ b/pkg/kubelet/eviction/eviction_manager.go @@ -455,7 +455,9 @@ func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceNam // localStorageEviction checks the EmptyDir volume usage for each pod and determine whether it exceeds the specified limit and needs // to be evicted. It also checks every container in the pod, if the container overlay usage exceeds the limit, the pod will be evicted too. func (m *managerImpl) localStorageEviction(pods []*v1.Pod) []*v1.Pod { - summary, err := m.summaryProvider.Get() + // do not update node-level stats as local storage evictions do not utilize them. + forceStatsUpdate := false + summary, err := m.summaryProvider.Get(forceStatsUpdate) if err != nil { glog.Errorf("Could not get summary provider") return nil diff --git a/pkg/kubelet/eviction/helpers.go b/pkg/kubelet/eviction/helpers.go index 3ae9c0306a..4376c84288 100644 --- a/pkg/kubelet/eviction/helpers.go +++ b/pkg/kubelet/eviction/helpers.go @@ -712,7 +712,8 @@ func (a byEvictionPriority) Less(i, j int) bool { // makeSignalObservations derives observations using the specified summary provider. func makeSignalObservations(summaryProvider stats.SummaryProvider, capacityProvider CapacityProvider, pods []*v1.Pod) (signalObservations, statsFunc, error) { - summary, err := summaryProvider.Get() + updateStats := true + summary, err := summaryProvider.Get(updateStats) if err != nil { return nil, nil, err } diff --git a/pkg/kubelet/eviction/helpers_test.go b/pkg/kubelet/eviction/helpers_test.go index 5a79411205..727e0be14b 100644 --- a/pkg/kubelet/eviction/helpers_test.go +++ b/pkg/kubelet/eviction/helpers_test.go @@ -920,7 +920,7 @@ type fakeSummaryProvider struct { result *statsapi.Summary } -func (f *fakeSummaryProvider) Get() (*statsapi.Summary, error) { +func (f *fakeSummaryProvider) Get(updateStats bool) (*statsapi.Summary, error) { return f.result, nil } diff --git a/pkg/kubelet/server/server_test.go b/pkg/kubelet/server/server_test.go index c1b068f231..230b7df43a 100644 --- a/pkg/kubelet/server/server_test.go +++ b/pkg/kubelet/server/server_test.go @@ -176,7 +176,7 @@ func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Vo func (_ *fakeKubelet) RootFsStats() (*statsapi.FsStats, error) { return nil, nil } func (_ *fakeKubelet) ListPodStats() ([]statsapi.PodStats, error) { return nil, nil } func (_ *fakeKubelet) ImageFsStats() (*statsapi.FsStats, error) { return nil, nil } -func (_ *fakeKubelet) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) { +func (_ *fakeKubelet) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) { return nil, nil, nil } diff --git a/pkg/kubelet/server/stats/handler.go b/pkg/kubelet/server/stats/handler.go index 969b25f3de..85d441e9f7 100644 --- a/pkg/kubelet/server/stats/handler.go +++ b/pkg/kubelet/server/stats/handler.go @@ -50,7 +50,7 @@ type StatsProvider interface { // // GetCgroupStats returns the stats and the networking usage of the cgroup // with the specified cgroupName. - GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) + GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) // RootFsStats returns the stats of the node root filesystem. RootFsStats() (*statsapi.FsStats, error) @@ -183,7 +183,9 @@ func (h *handler) handleStats(request *restful.Request, response *restful.Respon // Handles stats summary requests to /stats/summary func (h *handler) handleSummary(request *restful.Request, response *restful.Response) { - summary, err := h.summaryProvider.Get() + // external calls to the summary API use cached stats + forceStatsUpdate := false + summary, err := h.summaryProvider.Get(forceStatsUpdate) if err != nil { handleError(response, "/stats/summary", err) } else { diff --git a/pkg/kubelet/server/stats/summary.go b/pkg/kubelet/server/stats/summary.go index 3e59d77247..a1ffc22ea4 100644 --- a/pkg/kubelet/server/stats/summary.go +++ b/pkg/kubelet/server/stats/summary.go @@ -25,7 +25,9 @@ import ( ) type SummaryProvider interface { - Get() (*statsapi.Summary, error) + // Get provides a new Summary with the stats from Kubelet, + // and will update some stats if updateStats is true + Get(updateStats bool) (*statsapi.Summary, error) } // summaryProviderImpl implements the SummaryProvider interface. @@ -41,8 +43,7 @@ func NewSummaryProvider(statsProvider StatsProvider) SummaryProvider { return &summaryProviderImpl{statsProvider} } -// Get provides a new Summary with the stats from Kubelet. -func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) { +func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error) { // TODO(timstclair): Consider returning a best-effort response if any of // the following errors occur. node, err := sp.provider.GetNode() @@ -50,7 +51,7 @@ func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) { return nil, fmt.Errorf("failed to get node info: %v", err) } nodeConfig := sp.provider.GetNodeConfig() - rootStats, networkStats, err := sp.provider.GetCgroupStats("/") + rootStats, networkStats, err := sp.provider.GetCgroupStats("/", updateStats) if err != nil { return nil, fmt.Errorf("failed to get root cgroup stats: %v", err) } @@ -87,7 +88,7 @@ func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) { if name == "" { continue } - s, _, err := sp.provider.GetCgroupStats(name) + s, _, err := sp.provider.GetCgroupStats(name, false) if err != nil { glog.Errorf("Failed to get system container stats for %q: %v", name, err) continue diff --git a/pkg/kubelet/server/stats/summary_test.go b/pkg/kubelet/server/stats/summary_test.go index 422688446f..49b4ebebba 100644 --- a/pkg/kubelet/server/stats/summary_test.go +++ b/pkg/kubelet/server/stats/summary_test.go @@ -69,13 +69,13 @@ func TestSummaryProvider(t *testing.T) { On("ListPodStats").Return(podStats, nil). On("ImageFsStats").Return(imageFsStats, nil). On("RootFsStats").Return(rootFsStats, nil). - On("GetCgroupStats", "/").Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil). - On("GetCgroupStats", "/runtime").Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil). - On("GetCgroupStats", "/misc").Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil). - On("GetCgroupStats", "/kubelet").Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil) + On("GetCgroupStats", "/", true).Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil). + On("GetCgroupStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil). + On("GetCgroupStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil). + On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil) provider := NewSummaryProvider(mockStatsProvider) - summary, err := provider.Get() + summary, err := provider.Get(true) assert.NoError(err) assert.Equal(summary.Node.NodeName, "test-node") diff --git a/pkg/kubelet/server/stats/testing/mock_stats_provider.go b/pkg/kubelet/server/stats/testing/mock_stats_provider.go index befa19b0bf..0da5d872f3 100644 --- a/pkg/kubelet/server/stats/testing/mock_stats_provider.go +++ b/pkg/kubelet/server/stats/testing/mock_stats_provider.go @@ -33,13 +33,13 @@ type StatsProvider struct { mock.Mock } -// GetCgroupStats provides a mock function with given fields: cgroupName -func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerStats, *v1alpha1.NetworkStats, error) { - ret := _m.Called(cgroupName) +// GetCgroupStats provides a mock function with given fields: cgroupName, updateStats +func (_m *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*v1alpha1.ContainerStats, *v1alpha1.NetworkStats, error) { + ret := _m.Called(cgroupName, updateStats) var r0 *v1alpha1.ContainerStats - if rf, ok := ret.Get(0).(func(string) *v1alpha1.ContainerStats); ok { - r0 = rf(cgroupName) + if rf, ok := ret.Get(0).(func(string, bool) *v1alpha1.ContainerStats); ok { + r0 = rf(cgroupName, updateStats) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*v1alpha1.ContainerStats) @@ -47,8 +47,8 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerS } var r1 *v1alpha1.NetworkStats - if rf, ok := ret.Get(1).(func(string) *v1alpha1.NetworkStats); ok { - r1 = rf(cgroupName) + if rf, ok := ret.Get(1).(func(string, bool) *v1alpha1.NetworkStats); ok { + r1 = rf(cgroupName, updateStats) } else { if ret.Get(1) != nil { r1 = ret.Get(1).(*v1alpha1.NetworkStats) @@ -56,8 +56,8 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerS } var r2 error - if rf, ok := ret.Get(2).(func(string) error); ok { - r2 = rf(cgroupName) + if rf, ok := ret.Get(2).(func(string, bool) error); ok { + r2 = rf(cgroupName, updateStats) } else { r2 = ret.Error(2) } diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index 6856a8c76d..cee923722a 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -238,11 +238,17 @@ func isMemoryUnlimited(v uint64) bool { // getCgroupInfo returns the information of the container with the specified // containerName from cadvisor. -func getCgroupInfo(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerInfo, error) { +func getCgroupInfo(cadvisor cadvisor.Interface, containerName string, updateStats bool) (*cadvisorapiv2.ContainerInfo, error) { + var maxAge *time.Duration + if updateStats { + age := 0 * time.Second + maxAge = &age + } infoMap, err := cadvisor.ContainerInfoV2(containerName, cadvisorapiv2.RequestOptions{ IdType: cadvisorapiv2.TypeName, Count: 2, // 2 samples are needed to compute "instantaneous" CPU Recursive: false, + MaxAge: maxAge, }) if err != nil { return nil, fmt.Errorf("failed to get container info for %q: %v", containerName, err) @@ -256,8 +262,8 @@ func getCgroupInfo(cadvisor cadvisor.Interface, containerName string) (*cadvisor // getCgroupStats returns the latest stats of the container having the // specified containerName from cadvisor. -func getCgroupStats(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerStats, error) { - info, err := getCgroupInfo(cadvisor, containerName) +func getCgroupStats(cadvisor cadvisor.Interface, containerName string, updateStats bool) (*cadvisorapiv2.ContainerStats, error) { + info, err := getCgroupInfo(cadvisor, containerName, updateStats) if err != nil { return nil, err } diff --git a/pkg/kubelet/stats/stats_provider.go b/pkg/kubelet/stats/stats_provider.go index b61a410794..235f1a46d2 100644 --- a/pkg/kubelet/stats/stats_provider.go +++ b/pkg/kubelet/stats/stats_provider.go @@ -88,8 +88,8 @@ type containerStatsProvider interface { // GetCgroupStats returns the stats of the cgroup with the cgroupName. Note that // this function doesn't generate filesystem stats. -func (p *StatsProvider) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) { - info, err := getCgroupInfo(p.cadvisor, cgroupName) +func (p *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) { + info, err := getCgroupInfo(p.cadvisor, cgroupName, updateStats) if err != nil { return nil, nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err) } @@ -113,8 +113,8 @@ func (p *StatsProvider) RootFsStats() (*statsapi.FsStats, error) { } // Get the root container stats's timestamp, which will be used as the - // imageFs stats timestamp. - rootStats, err := getCgroupStats(p.cadvisor, "/") + // imageFs stats timestamp. Dont force a stats update, as we only want the timestamp. + rootStats, err := getCgroupStats(p.cadvisor, "/", false) if err != nil { return nil, fmt.Errorf("failed to get root container stats: %v", err) } diff --git a/pkg/kubelet/stats/stats_provider_test.go b/pkg/kubelet/stats/stats_provider_test.go index 20616f3902..6f02704ed5 100644 --- a/pkg/kubelet/stats/stats_provider_test.go +++ b/pkg/kubelet/stats/stats_provider_test.go @@ -71,6 +71,7 @@ func TestGetCgroupStats(t *testing.T) { const ( cgroupName = "test-cgroup-name" containerInfoSeed = 1000 + updateStats = false ) var ( mockCadvisor = new(cadvisortest.Mock) @@ -87,7 +88,7 @@ func TestGetCgroupStats(t *testing.T) { mockCadvisor.On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil) provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{}) - cs, ns, err := provider.GetCgroupStats(cgroupName) + cs, ns, err := provider.GetCgroupStats(cgroupName, updateStats) assert.NoError(err) checkCPUStats(t, "", containerInfoSeed, cs.CPU) @@ -599,8 +600,8 @@ type fakeResourceAnalyzer struct { podVolumeStats serverstats.PodVolumeStats } -func (o *fakeResourceAnalyzer) Start() {} -func (o *fakeResourceAnalyzer) Get() (*statsapi.Summary, error) { return nil, nil } +func (o *fakeResourceAnalyzer) Start() {} +func (o *fakeResourceAnalyzer) Get(bool) (*statsapi.Summary, error) { return nil, nil } func (o *fakeResourceAnalyzer) GetPodVolumeStats(uid types.UID) (serverstats.PodVolumeStats, bool) { return o.podVolumeStats, true }