github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

enable on-demand metrics for eviction

pull/6/head
David Ashpole 2018-01-08 10:20:02 -08:00
parent 6244ff3644
commit f6721480f4
11 changed files with 48 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/kubelet/eviction/eviction_manager.go
View File

@ -455,7 +455,9 @@ func (m *managerImpl) reclaimNodeLevelResources(resourceToReclaim v1.ResourceNam
// localStorageEviction checks the EmptyDir volume usage for each pod and determine whether it exceeds the specified limit and needs
// to be evicted. It also checks every container in the pod, if the container overlay usage exceeds the limit, the pod will be evicted too.
func (m *managerImpl) localStorageEviction(pods []*v1.Pod) []*v1.Pod {
summary, err := m.summaryProvider.Get()
// do not update node-level stats as local storage evictions do not utilize them.
forceStatsUpdate := false
summary, err := m.summaryProvider.Get(forceStatsUpdate)
if err != nil {
glog.Errorf("Could not get summary provider")
return nil

3
pkg/kubelet/eviction/helpers.go
View File

@ -712,7 +712,8 @@ func (a byEvictionPriority) Less(i, j int) bool {
// makeSignalObservations derives observations using the specified summary provider.
func makeSignalObservations(summaryProvider stats.SummaryProvider, capacityProvider CapacityProvider, pods []*v1.Pod) (signalObservations, statsFunc, error) {
summary, err := summaryProvider.Get()
updateStats := true
summary, err := summaryProvider.Get(updateStats)
if err != nil {
return nil, nil, err
}

2
pkg/kubelet/eviction/helpers_test.go
View File

@ -920,7 +920,7 @@ type fakeSummaryProvider struct {
result *statsapi.Summary
}
func (f *fakeSummaryProvider) Get() (*statsapi.Summary, error) {
func (f *fakeSummaryProvider) Get(updateStats bool) (*statsapi.Summary, error) {
return f.result, nil
}

2
pkg/kubelet/server/server_test.go
View File

@ -176,7 +176,7 @@ func (fk *fakeKubelet) ListVolumesForPod(podUID types.UID) (map[string]volume.Vo
func (_ *fakeKubelet) RootFsStats() (*statsapi.FsStats, error) { return nil, nil }
func (_ *fakeKubelet) ListPodStats() ([]statsapi.PodStats, error) { return nil, nil }
func (_ *fakeKubelet) ImageFsStats() (*statsapi.FsStats, error) { return nil, nil }
func (_ *fakeKubelet) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
func (_ *fakeKubelet) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
return nil, nil, nil
}

6
pkg/kubelet/server/stats/handler.go
View File

@ -50,7 +50,7 @@ type StatsProvider interface {
//
// GetCgroupStats returns the stats and the networking usage of the cgroup
// with the specified cgroupName.
GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error)
GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error)
// RootFsStats returns the stats of the node root filesystem.
RootFsStats() (*statsapi.FsStats, error)
@ -183,7 +183,9 @@ func (h *handler) handleStats(request *restful.Request, response *restful.Respon
// Handles stats summary requests to /stats/summary
func (h *handler) handleSummary(request *restful.Request, response *restful.Response) {
summary, err := h.summaryProvider.Get()
// external calls to the summary API use cached stats
forceStatsUpdate := false
summary, err := h.summaryProvider.Get(forceStatsUpdate)
if err != nil {
handleError(response, "/stats/summary", err)
} else {

11
pkg/kubelet/server/stats/summary.go
View File

@ -25,7 +25,9 @@ import (
)
type SummaryProvider interface {
Get() (*statsapi.Summary, error)
// Get provides a new Summary with the stats from Kubelet,
// and will update some stats if updateStats is true
Get(updateStats bool) (*statsapi.Summary, error)
}
// summaryProviderImpl implements the SummaryProvider interface.
@ -41,8 +43,7 @@ func NewSummaryProvider(statsProvider StatsProvider) SummaryProvider {
return &summaryProviderImpl{statsProvider}
}
// Get provides a new Summary with the stats from Kubelet.
func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) {
func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error) {
// TODO(timstclair): Consider returning a best-effort response if any of
// the following errors occur.
node, err := sp.provider.GetNode()
@ -50,7 +51,7 @@ func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) {
return nil, fmt.Errorf("failed to get node info: %v", err)
}
nodeConfig := sp.provider.GetNodeConfig()
rootStats, networkStats, err := sp.provider.GetCgroupStats("/")
rootStats, networkStats, err := sp.provider.GetCgroupStats("/", updateStats)
if err != nil {
return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
}
@ -87,7 +88,7 @@ func (sp *summaryProviderImpl) Get() (*statsapi.Summary, error) {
if name == "" {
continue
}
s, _, err := sp.provider.GetCgroupStats(name)
s, _, err := sp.provider.GetCgroupStats(name, false)
if err != nil {
glog.Errorf("Failed to get system container stats for %q: %v", name, err)
continue

10
pkg/kubelet/server/stats/summary_test.go
View File

@ -69,13 +69,13 @@ func TestSummaryProvider(t *testing.T) {
On("ListPodStats").Return(podStats, nil).
On("ImageFsStats").Return(imageFsStats, nil).
On("RootFsStats").Return(rootFsStats, nil).
On("GetCgroupStats", "/").Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil).
On("GetCgroupStats", "/runtime").Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil).
On("GetCgroupStats", "/misc").Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil).
On("GetCgroupStats", "/kubelet").Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil)
On("GetCgroupStats", "/", true).Return(cgroupStatsMap["/"].cs, cgroupStatsMap["/"].ns, nil).
On("GetCgroupStats", "/runtime", false).Return(cgroupStatsMap["/runtime"].cs, cgroupStatsMap["/runtime"].ns, nil).
On("GetCgroupStats", "/misc", false).Return(cgroupStatsMap["/misc"].cs, cgroupStatsMap["/misc"].ns, nil).
On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil)
provider := NewSummaryProvider(mockStatsProvider)
summary, err := provider.Get()
summary, err := provider.Get(true)
assert.NoError(err)
assert.Equal(summary.Node.NodeName, "test-node")

18
pkg/kubelet/server/stats/testing/mock_stats_provider.go
View File

@ -33,13 +33,13 @@ type StatsProvider struct {
mock.Mock
}
// GetCgroupStats provides a mock function with given fields: cgroupName
func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerStats, *v1alpha1.NetworkStats, error) {
ret := _m.Called(cgroupName)
// GetCgroupStats provides a mock function with given fields: cgroupName, updateStats
func (_m *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*v1alpha1.ContainerStats, *v1alpha1.NetworkStats, error) {
ret := _m.Called(cgroupName, updateStats)
var r0 *v1alpha1.ContainerStats
if rf, ok := ret.Get(0).(func(string) *v1alpha1.ContainerStats); ok {
r0 = rf(cgroupName)
if rf, ok := ret.Get(0).(func(string, bool) *v1alpha1.ContainerStats); ok {
r0 = rf(cgroupName, updateStats)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*v1alpha1.ContainerStats)
@ -47,8 +47,8 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerS
}
var r1 *v1alpha1.NetworkStats
if rf, ok := ret.Get(1).(func(string) *v1alpha1.NetworkStats); ok {
r1 = rf(cgroupName)
if rf, ok := ret.Get(1).(func(string, bool) *v1alpha1.NetworkStats); ok {
r1 = rf(cgroupName, updateStats)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(*v1alpha1.NetworkStats)
@ -56,8 +56,8 @@ func (_m *StatsProvider) GetCgroupStats(cgroupName string) (*v1alpha1.ContainerS
}
var r2 error
if rf, ok := ret.Get(2).(func(string) error); ok {
r2 = rf(cgroupName)
if rf, ok := ret.Get(2).(func(string, bool) error); ok {
r2 = rf(cgroupName, updateStats)
} else {
r2 = ret.Error(2)
}

12
pkg/kubelet/stats/helper.go
View File

@ -238,11 +238,17 @@ func isMemoryUnlimited(v uint64) bool {
// getCgroupInfo returns the information of the container with the specified
// containerName from cadvisor.
func getCgroupInfo(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerInfo, error) {
func getCgroupInfo(cadvisor cadvisor.Interface, containerName string, updateStats bool) (*cadvisorapiv2.ContainerInfo, error) {
var maxAge *time.Duration
if updateStats {
age := 0 * time.Second
maxAge = &age
}
infoMap, err := cadvisor.ContainerInfoV2(containerName, cadvisorapiv2.RequestOptions{
IdType: cadvisorapiv2.TypeName,
Count: 2, // 2 samples are needed to compute "instantaneous" CPU
Recursive: false,
MaxAge: maxAge,
})
if err != nil {
return nil, fmt.Errorf("failed to get container info for %q: %v", containerName, err)
@ -256,8 +262,8 @@ func getCgroupInfo(cadvisor cadvisor.Interface, containerName string) (*cadvisor
// getCgroupStats returns the latest stats of the container having the
// specified containerName from cadvisor.
func getCgroupStats(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerStats, error) {
info, err := getCgroupInfo(cadvisor, containerName)
func getCgroupStats(cadvisor cadvisor.Interface, containerName string, updateStats bool) (*cadvisorapiv2.ContainerStats, error) {
info, err := getCgroupInfo(cadvisor, containerName, updateStats)
if err != nil {
return nil, err
}

8
pkg/kubelet/stats/stats_provider.go
View File

@ -88,8 +88,8 @@ type containerStatsProvider interface {
// GetCgroupStats returns the stats of the cgroup with the cgroupName. Note that
// this function doesn't generate filesystem stats.
func (p *StatsProvider) GetCgroupStats(cgroupName string) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
info, err := getCgroupInfo(p.cadvisor, cgroupName)
func (p *StatsProvider) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
info, err := getCgroupInfo(p.cadvisor, cgroupName, updateStats)
if err != nil {
return nil, nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err)
}
@ -113,8 +113,8 @@ func (p *StatsProvider) RootFsStats() (*statsapi.FsStats, error) {
}
// Get the root container stats's timestamp, which will be used as the
// imageFs stats timestamp.
rootStats, err := getCgroupStats(p.cadvisor, "/")
// imageFs stats timestamp. Dont force a stats update, as we only want the timestamp.
rootStats, err := getCgroupStats(p.cadvisor, "/", false)
if err != nil {
return nil, fmt.Errorf("failed to get root container stats: %v", err)
}

7
pkg/kubelet/stats/stats_provider_test.go
View File

@ -71,6 +71,7 @@ func TestGetCgroupStats(t *testing.T) {
const (
cgroupName = "test-cgroup-name"
containerInfoSeed = 1000
updateStats = false
)
var (
mockCadvisor = new(cadvisortest.Mock)
@ -87,7 +88,7 @@ func TestGetCgroupStats(t *testing.T) {
mockCadvisor.On("ContainerInfoV2", cgroupName, options).Return(containerInfoMap, nil)
provider := newStatsProvider(mockCadvisor, mockPodManager, mockRuntimeCache, fakeContainerStatsProvider{})
cs, ns, err := provider.GetCgroupStats(cgroupName)
cs, ns, err := provider.GetCgroupStats(cgroupName, updateStats)
assert.NoError(err)
checkCPUStats(t, "", containerInfoSeed, cs.CPU)
@ -599,8 +600,8 @@ type fakeResourceAnalyzer struct {
podVolumeStats serverstats.PodVolumeStats
}
func (o *fakeResourceAnalyzer) Start() {}
func (o *fakeResourceAnalyzer) Get() (*statsapi.Summary, error) { return nil, nil }
func (o *fakeResourceAnalyzer) Start() {}
func (o *fakeResourceAnalyzer) Get(bool) (*statsapi.Summary, error) { return nil, nil }
func (o *fakeResourceAnalyzer) GetPodVolumeStats(uid types.UID) (serverstats.PodVolumeStats, bool) {
return o.podVolumeStats, true
}