diff --git a/pkg/kubelet/apis/cri/testing/fake_image_service.go b/pkg/kubelet/apis/cri/testing/fake_image_service.go index 992268d27d..23fa9c77ea 100644 --- a/pkg/kubelet/apis/cri/testing/fake_image_service.go +++ b/pkg/kubelet/apis/cri/testing/fake_image_service.go @@ -34,6 +34,8 @@ type FakeImageService struct { Images map[string]*runtimeapi.Image pulledImages []*pulledImage + + FakeFilesystemUsage []*runtimeapi.FilesystemUsage } func (r *FakeImageService) SetFakeImages(images []string) { @@ -53,6 +55,13 @@ func (r *FakeImageService) SetFakeImageSize(size uint64) { r.FakeImageSize = size } +func (r *FakeImageService) SetFakeFilesystemUsage(usage []*runtimeapi.FilesystemUsage) { + r.Lock() + defer r.Unlock() + + r.FakeFilesystemUsage = usage +} + func NewFakeImageService() *FakeImageService { return &FakeImageService{ Called: make([]string, 0), @@ -132,7 +141,9 @@ func (r *FakeImageService) ImageFsInfo(req *runtimeapi.ImageFsInfoRequest) (*run r.Called = append(r.Called, "ImageFsInfo") - return nil, nil + return &runtimeapi.ImageFsInfoResponse{ + ImageFilesystems: r.FakeFilesystemUsage, + }, nil } func (r *FakeImageService) AssertImagePulledWithAuth(t *testing.T, image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig, failMsg string) { diff --git a/pkg/kubelet/apis/cri/testing/fake_runtime_service.go b/pkg/kubelet/apis/cri/testing/fake_runtime_service.go index b245641844..cdff814d2c 100644 --- a/pkg/kubelet/apis/cri/testing/fake_runtime_service.go +++ b/pkg/kubelet/apis/cri/testing/fake_runtime_service.go @@ -50,9 +50,10 @@ type FakeRuntimeService struct { Called []string - FakeStatus *runtimeapi.RuntimeStatus - Containers map[string]*FakeContainer - Sandboxes map[string]*FakePodSandbox + FakeStatus *runtimeapi.RuntimeStatus + Containers map[string]*FakeContainer + Sandboxes map[string]*FakePodSandbox + FakeContainerStats map[string]*runtimeapi.ContainerStats } func (r *FakeRuntimeService) GetContainerID(sandboxID, name string, attempt uint32) (string, error) { @@ -102,9 +103,10 @@ func (r *FakeRuntimeService) AssertCalls(calls []string) error { func NewFakeRuntimeService() *FakeRuntimeService { return &FakeRuntimeService{ - Called: make([]string, 0), - Containers: make(map[string]*FakeContainer), - Sandboxes: make(map[string]*FakePodSandbox), + Called: make([]string, 0), + Containers: make(map[string]*FakeContainer), + Sandboxes: make(map[string]*FakePodSandbox), + FakeContainerStats: make(map[string]*runtimeapi.ContainerStats), } } @@ -406,10 +408,54 @@ func (r *FakeRuntimeService) UpdateRuntimeConfig(runtimeCOnfig *runtimeapi.Runti return nil } +func (r *FakeRuntimeService) SetFakeContainerStats(containerStats []*runtimeapi.ContainerStats) { + r.Lock() + defer r.Unlock() + + r.FakeContainerStats = make(map[string]*runtimeapi.ContainerStats) + for _, s := range containerStats { + r.FakeContainerStats[s.Attributes.Id] = s + } +} + func (r *FakeRuntimeService) ContainerStats(req *runtimeapi.ContainerStatsRequest) (*runtimeapi.ContainerStatsResponse, error) { - return nil, fmt.Errorf("Not implemented") + r.Lock() + defer r.Unlock() + + r.Called = append(r.Called, "ContainerStats") + + s, found := r.FakeContainerStats[req.ContainerId] + if !found { + return nil, fmt.Errorf("no stats for container %q", req.ContainerId) + } + return &runtimeapi.ContainerStatsResponse{Stats: s}, nil } func (r *FakeRuntimeService) ListContainerStats(req *runtimeapi.ListContainerStatsRequest) (*runtimeapi.ListContainerStatsResponse, error) { - return nil, fmt.Errorf("Not implemented") + r.Lock() + defer r.Unlock() + + r.Called = append(r.Called, "ListContainerStats") + + var result []*runtimeapi.ContainerStats + for _, c := range r.Containers { + if req.Filter != nil { + if req.Filter.Id != "" && req.Filter.Id != c.Id { + continue + } + if req.Filter.PodSandboxId != "" && req.Filter.PodSandboxId != c.SandboxID { + continue + } + if req.Filter.LabelSelector != nil && !filterInLabels(req.Filter.LabelSelector, c.GetLabels()) { + continue + } + } + s, found := r.FakeContainerStats[c.Id] + if !found { + continue + } + result = append(result, s) + } + + return &runtimeapi.ListContainerStatsResponse{Stats: result}, nil } diff --git a/pkg/kubelet/cadvisor/testing/cadvisor_fake.go b/pkg/kubelet/cadvisor/testing/cadvisor_fake.go index c9d0f2ebb5..9ab24f07ac 100644 --- a/pkg/kubelet/cadvisor/testing/cadvisor_fake.go +++ b/pkg/kubelet/cadvisor/testing/cadvisor_fake.go @@ -79,3 +79,7 @@ func (c *Fake) WatchEvents(request *events.Request) (*events.EventChannel, error func (c *Fake) HasDedicatedImageFs() (bool, error) { return false, nil } + +func (c *Fake) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) { + return cadvisorapiv2.FsInfo{}, nil +} diff --git a/pkg/kubelet/cadvisor/testing/cadvisor_mock.go b/pkg/kubelet/cadvisor/testing/cadvisor_mock.go index 7848039178..d192c16594 100644 --- a/pkg/kubelet/cadvisor/testing/cadvisor_mock.go +++ b/pkg/kubelet/cadvisor/testing/cadvisor_mock.go @@ -88,3 +88,8 @@ func (c *Mock) HasDedicatedImageFs() (bool, error) { args := c.Called() return args.Get(0).(bool), args.Error(1) } + +func (c *Mock) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) { + args := c.Called(uuid) + return args.Get(0).(cadvisorapiv2.FsInfo), args.Error(1) +} diff --git a/pkg/kubelet/cadvisor/types.go b/pkg/kubelet/cadvisor/types.go index 2a97ba3523..89cbcbf27d 100644 --- a/pkg/kubelet/cadvisor/types.go +++ b/pkg/kubelet/cadvisor/types.go @@ -44,4 +44,8 @@ type Interface interface { // HasDedicatedImageFs returns true iff a dedicated image filesystem exists for storing images. HasDedicatedImageFs() (bool, error) + + // GetFsInfoByFsUUID returns the stats of the filesystem with the specified + // uuid. + GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) } diff --git a/pkg/kubelet/stats/BUILD b/pkg/kubelet/stats/BUILD index e96170a6bd..2d027fc31a 100644 --- a/pkg/kubelet/stats/BUILD +++ b/pkg/kubelet/stats/BUILD @@ -11,6 +11,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/kubelet/apis/cri:go_default_library", + "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", "//pkg/kubelet/container:go_default_library", @@ -20,6 +21,7 @@ go_library( "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/types:go_default_library", "//vendor/github.com/golang/glog:go_default_library", + "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -45,11 +47,14 @@ go_test( name = "go_default_test", srcs = [ "cadvisor_stats_provider_test.go", + "cri_stats_provider_test.go", "helper_test.go", "stats_provider_test.go", ], library = ":go_default_library", deps = [ + "//pkg/kubelet/apis/cri/testing:go_default_library", + "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library", "//pkg/kubelet/apis/stats/v1alpha1:go_default_library", "//pkg/kubelet/cadvisor/testing:go_default_library", "//pkg/kubelet/container:go_default_library", @@ -58,6 +63,7 @@ go_test( "//pkg/kubelet/pod/testing:go_default_library", "//pkg/kubelet/server/stats:go_default_library", "//pkg/kubelet/types:go_default_library", + "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", diff --git a/pkg/kubelet/stats/cadvisor_stats_provider.go b/pkg/kubelet/stats/cadvisor_stats_provider.go index 39a6b8aac6..27538240fd 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider.go @@ -160,15 +160,8 @@ func (p *cadvisorStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { imageFsInodesUsed = &imageFsIU } - // Get the root container stats's timestamp, which will be used as the - // imageFs stats timestamp. - rootStats, err := getCgroupStats(p.cadvisor, "/") - if err != nil { - return nil, fmt.Errorf("failed to get root container stats: %v", err) - } - return &statsapi.FsStats{ - Time: metav1.NewTime(rootStats.Timestamp), + Time: metav1.NewTime(imageFsInfo.Timestamp), AvailableBytes: &imageFsInfo.Available, CapacityBytes: &imageFsInfo.Capacity, UsedBytes: &imageStats.TotalStorageBytes, diff --git a/pkg/kubelet/stats/cadvisor_stats_provider_test.go b/pkg/kubelet/stats/cadvisor_stats_provider_test.go index 06b60a64f5..bb3df5f431 100644 --- a/pkg/kubelet/stats/cadvisor_stats_provider_test.go +++ b/pkg/kubelet/stats/cadvisor_stats_provider_test.go @@ -22,7 +22,6 @@ import ( cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" @@ -74,7 +73,7 @@ func TestRemoveTerminatedContainerInfo(t *testing.T) { } } -func TestListPodStats(t *testing.T) { +func TestCadvisorListPodStats(t *testing.T) { const ( namespace0 = "test0" namespace2 = "test2" @@ -236,36 +235,31 @@ func TestListPodStats(t *testing.T) { checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network) } -func TestImagesFsStats(t *testing.T) { +func TestCadvisorImagesFsStats(t *testing.T) { var ( assert = assert.New(t) mockCadvisor = new(cadvisortest.Mock) mockRuntime = new(containertest.Mock) - seed = 100 - options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false} - imageFsInfo = getTestFsInfo(100) - containerInfo = map[string]cadvisorapiv2.ContainerInfo{"/": getTestContainerInfo(seed, "test-pod", "test-ns", "test-container")} - imageStats = &kubecontainer.ImageStats{TotalStorageBytes: 100} + seed = 1000 + imageFsInfo = getTestFsInfo(seed) + imageStats = &kubecontainer.ImageStats{TotalStorageBytes: 100} ) - mockCadvisor. - On("ImagesFsInfo").Return(imageFsInfo, nil). - On("ContainerInfoV2", "/", options).Return(containerInfo, nil) - mockRuntime. - On("ImageStats").Return(imageStats, nil) + mockCadvisor.On("ImagesFsInfo").Return(imageFsInfo, nil) + mockRuntime.On("ImageStats").Return(imageStats, nil) provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime) stats, err := provider.ImageFsStats() assert.NoError(err) - assert.Equal(stats.Time, metav1.NewTime(containerInfo["/"].Stats[0].Timestamp)) - assert.Equal(*stats.AvailableBytes, imageFsInfo.Available) - assert.Equal(*stats.CapacityBytes, imageFsInfo.Capacity) - assert.Equal(*stats.UsedBytes, imageStats.TotalStorageBytes) - assert.Equal(stats.InodesFree, imageFsInfo.InodesFree) - assert.Equal(stats.Inodes, imageFsInfo.Inodes) - assert.Equal(*stats.InodesUsed, *imageFsInfo.Inodes-*imageFsInfo.InodesFree) + assert.Equal(imageFsInfo.Timestamp, stats.Time.Time) + assert.Equal(imageFsInfo.Available, *stats.AvailableBytes) + assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes) + assert.Equal(imageStats.TotalStorageBytes, *stats.UsedBytes) + assert.Equal(imageFsInfo.InodesFree, stats.InodesFree) + assert.Equal(imageFsInfo.Inodes, stats.Inodes) + assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *stats.InodesUsed) mockCadvisor.AssertExpectations(t) } diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index 105fa853f9..b340416eb2 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -18,8 +18,16 @@ package stats import ( "fmt" + "time" + "github.com/golang/glog" + cadvisorfs "github.com/google/cadvisor/fs" + + cadvisorapiv2 "github.com/google/cadvisor/info/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/server/stats" @@ -57,10 +65,215 @@ func newCRIStatsProvider( } } +// ListPodStats returns the stats of all the pod-managed containers. func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) { - return nil, fmt.Errorf("not implemented") + // Gets node root filesystem information, which will be used to populate + // the available and capacity bytes/inodes in container stats. + rootFsInfo, err := p.cadvisor.RootFsInfo() + if err != nil { + return nil, fmt.Errorf("failed to get rootFs info: %v", err) + } + + // Creates container map. + containerMap := make(map[string]*runtimeapi.Container) + containers, err := p.runtimeService.ListContainers(&runtimeapi.ContainerFilter{}) + if err != nil { + return nil, fmt.Errorf("failed to list all containers: %v", err) + } + for _, c := range containers { + containerMap[c.Id] = c + } + + // Creates pod sandbox map. + podSandboxMap := make(map[string]*runtimeapi.PodSandbox) + podSandboxes, err := p.runtimeService.ListPodSandbox(&runtimeapi.PodSandboxFilter{}) + if err != nil { + return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err) + } + for _, s := range podSandboxes { + podSandboxMap[s.Id] = s + } + + // uuidToFsInfo is a map from filesystem UUID to its stats. This will be + // used as a cache to avoid querying cAdvisor for the filesystem stats with + // the same UUID many times. + uuidToFsInfo := make(map[runtimeapi.StorageIdentifier]*cadvisorapiv2.FsInfo) + + // sandboxIDToPodStats is a temporary map from sandbox ID to its pod stats. + sandboxIDToPodStats := make(map[string]*statsapi.PodStats) + + resp, err := p.runtimeService.ListContainerStats(&runtimeapi.ListContainerStatsRequest{}) + if err != nil { + return nil, fmt.Errorf("failed to list all container stats: %v", err) + } + for _, stats := range resp.Stats { + containerID := stats.Attributes.Id + container, found := containerMap[containerID] + if !found { + glog.Errorf("Unknown id %q in container map.", containerID) + continue + } + + podSandboxID := container.PodSandboxId + podSandbox, found := podSandboxMap[podSandboxID] + if !found { + glog.Errorf("Unknown id %q in pod sandbox map.", podSandboxID) + continue + } + + // Creates the stats of the pod (if not created yet) which the + // container belongs to. + ps, found := sandboxIDToPodStats[podSandboxID] + if !found { + ps = p.makePodStats(podSandbox) + sandboxIDToPodStats[podSandboxID] = ps + } + ps.Containers = append(ps.Containers, *p.makeContainerStats(stats, container, &rootFsInfo, uuidToFsInfo)) + } + + result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats)) + for _, s := range sandboxIDToPodStats { + result = append(result, *s) + } + return result, nil } +// ImageFsStats returns the stats of the image filesystem. func (p *criStatsProvider) ImageFsStats() (*statsapi.FsStats, error) { - return nil, fmt.Errorf("not implemented") + resp, err := p.imageService.ImageFsInfo(&runtimeapi.ImageFsInfoRequest{}) + if err != nil { + return nil, err + } + + // CRI may return the stats of multiple image filesystems but we only + // return the first one. + // + // TODO(yguo0905): Support returning stats of multiple image filesystems. + for _, fs := range resp.ImageFilesystems { + s := &statsapi.FsStats{ + Time: metav1.NewTime(time.Unix(0, fs.Timestamp)), + UsedBytes: &fs.UsedBytes.Value, + InodesUsed: &fs.InodesUsed.Value, + } + imageFsInfo := p.getFsInfo(fs.StorageId) + if imageFsInfo != nil { + // The image filesystem UUID is unknown to the local node or + // there's an error on retrieving the stats. In these cases, we + // omit those stats and return the best-effort partial result. See + // https://github.com/kubernetes/heapster/issues/1793. + s.AvailableBytes = &imageFsInfo.Available + s.CapacityBytes = &imageFsInfo.Capacity + s.InodesFree = imageFsInfo.InodesFree + s.Inodes = imageFsInfo.Inodes + } + return s, nil + } + + return nil, fmt.Errorf("imageFs information is unavailable") +} + +// getFsInfo returns the information of the filesystem with the specified +// storageID. If any error occurs, this function logs the error and returns +// nil. +func (p *criStatsProvider) getFsInfo(storageID *runtimeapi.StorageIdentifier) *cadvisorapiv2.FsInfo { + if storageID == nil { + glog.V(2).Infof("Failed to get filesystem info: storageID is nil.") + return nil + } + fsInfo, err := p.cadvisor.GetFsInfoByFsUUID(storageID.Uuid) + if err != nil { + msg := fmt.Sprintf("Failed to get the info of the filesystem with id %q: %v.", storageID.Uuid, err) + if err == cadvisorfs.ErrNoSuchDevice { + glog.V(2).Info(msg) + } else { + glog.Error(msg) + } + return nil + } + return &fsInfo +} + +func (p *criStatsProvider) makePodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats { + s := &statsapi.PodStats{ + PodRef: statsapi.PodReference{ + Name: podSandbox.Metadata.Name, + UID: podSandbox.Metadata.Uid, + Namespace: podSandbox.Metadata.Namespace, + }, + // The StartTime in the summary API is the pod creation time. + StartTime: metav1.NewTime(time.Unix(0, podSandbox.CreatedAt)), + // Network stats are not supported by CRI. + } + podUID := types.UID(s.PodRef.UID) + if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found { + s.VolumeStats = vstats.Volumes + } + return s +} + +func (p *criStatsProvider) makeContainerStats( + stats *runtimeapi.ContainerStats, + container *runtimeapi.Container, + rootFsInfo *cadvisorapiv2.FsInfo, + uuidToFsInfo map[runtimeapi.StorageIdentifier]*cadvisorapiv2.FsInfo, +) *statsapi.ContainerStats { + result := &statsapi.ContainerStats{ + Name: stats.Attributes.Metadata.Name, + // The StartTime in the summary API is the container creation time. + StartTime: metav1.NewTime(time.Unix(0, container.CreatedAt)), + CPU: &statsapi.CPUStats{}, + Memory: &statsapi.MemoryStats{}, + Rootfs: &statsapi.FsStats{}, + Logs: &statsapi.FsStats{ + Time: metav1.NewTime(rootFsInfo.Timestamp), + AvailableBytes: &rootFsInfo.Available, + CapacityBytes: &rootFsInfo.Capacity, + InodesFree: rootFsInfo.InodesFree, + Inodes: rootFsInfo.Inodes, + // UsedBytes and InodesUsed are unavailable from CRI stats. + // + // TODO(yguo0905): Get this information from kubelet and + // populate the two fields here. + }, + // UserDefinedMetrics is not supported by CRI. + } + if stats.Cpu != nil { + result.CPU.Time = metav1.NewTime(time.Unix(0, stats.Cpu.Timestamp)) + if stats.Cpu.UsageCoreNanoSeconds != nil { + result.CPU.UsageCoreNanoSeconds = &stats.Cpu.UsageCoreNanoSeconds.Value + } + } + if stats.Memory != nil { + result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp)) + if stats.Memory.WorkingSetBytes != nil { + result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value + } + } + if stats.WritableLayer != nil { + result.Rootfs.Time = metav1.NewTime(time.Unix(0, stats.WritableLayer.Timestamp)) + if stats.WritableLayer.UsedBytes != nil { + result.Rootfs.UsedBytes = &stats.WritableLayer.UsedBytes.Value + } + if stats.WritableLayer.InodesUsed != nil { + result.Rootfs.InodesUsed = &stats.WritableLayer.InodesUsed.Value + } + } + storageID := stats.WritableLayer.StorageId + imageFsInfo, found := uuidToFsInfo[*storageID] + if !found { + imageFsInfo = p.getFsInfo(storageID) + uuidToFsInfo[*storageID] = imageFsInfo + } + if imageFsInfo != nil { + // The image filesystem UUID is unknown to the local node or there's an + // error on retrieving the stats. In these cases, we omit those stats + // and return the best-effort partial result. See + // https://github.com/kubernetes/heapster/issues/1793. + result.Rootfs.AvailableBytes = &imageFsInfo.Available + result.Rootfs.CapacityBytes = &imageFsInfo.Capacity + result.Rootfs.InodesFree = imageFsInfo.InodesFree + result.Rootfs.Inodes = imageFsInfo.Inodes + } + + return result } diff --git a/pkg/kubelet/stats/cri_stats_provider_test.go b/pkg/kubelet/stats/cri_stats_provider_test.go new file mode 100644 index 0000000000..8e4fdd881c --- /dev/null +++ b/pkg/kubelet/stats/cri_stats_provider_test.go @@ -0,0 +1,274 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stats + +import ( + "math/rand" + "testing" + "time" + + cadvisorfs "github.com/google/cadvisor/fs" + "github.com/stretchr/testify/assert" + + cadvisorapiv2 "github.com/google/cadvisor/info/v2" + critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" + statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" + cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" + kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing" + kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing" +) + +func TestCRIListPodStats(t *testing.T) { + var ( + imageFsStorageUUID = "imagefs-storage-uuid" + unknownStorageUUID = "unknown-storage-uuid" + imageFsInfo = getTestFsInfo(2000) + rootFsInfo = getTestFsInfo(1000) + + sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns") + container0 = makeFakeContainer(sandbox0, "container0-name") + containerStats0 = makeFakeContainerStats(container0, imageFsStorageUUID) + container1 = makeFakeContainer(sandbox0, "container1-name") + containerStats1 = makeFakeContainerStats(container1, unknownStorageUUID) + + sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns") + container2 = makeFakeContainer(sandbox1, "container2-name") + containerStats2 = makeFakeContainerStats(container2, imageFsStorageUUID) + ) + + var ( + mockCadvisor = new(cadvisortest.Mock) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + mockPodManager = new(kubepodtest.MockManager) + resourceAnalyzer = new(fakeResourceAnalyzer) + fakeRuntimeService = critest.NewFakeRuntimeService() + fakeImageService = critest.NewFakeImageService() + ) + + mockCadvisor. + On("RootFsInfo").Return(rootFsInfo, nil). + On("GetFsInfoByFsUUID", imageFsStorageUUID).Return(imageFsInfo, nil). + On("GetFsInfoByFsUUID", unknownStorageUUID).Return(cadvisorapiv2.FsInfo{}, cadvisorfs.ErrNoSuchDevice) + fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{ + sandbox0, sandbox1, + }) + fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{ + container0, container1, container2, + }) + fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{ + containerStats0, containerStats1, containerStats2, + }) + + provider := NewCRIStatsProvider( + mockCadvisor, + resourceAnalyzer, + mockPodManager, + mockRuntimeCache, + fakeRuntimeService, + fakeImageService) + + stats, err := provider.ListPodStats() + assert := assert.New(t) + assert.NoError(err) + assert.Equal(2, len(stats)) + + podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats) + for _, s := range stats { + podStatsMap[s.PodRef] = s + } + + p0 := podStatsMap[statsapi.PodReference{Name: "sandbox0-name", UID: "sandbox0-uid", Namespace: "sandbox0-ns"}] + assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano()) + assert.Equal(2, len(p0.Containers)) + + containerStatsMap := make(map[string]statsapi.ContainerStats) + for _, s := range p0.Containers { + containerStatsMap[s.Name] = s + } + c1 := containerStatsMap["container0-name"] + assert.Equal(container0.CreatedAt, c1.StartTime.UnixNano()) + checkCRICPUAndMemoryStats(assert, c1, containerStats0) + checkCRIRootfsStats(assert, c1, containerStats0, &imageFsInfo) + checkCRILogsStats(assert, c1, &rootFsInfo) + c2 := containerStatsMap["container1-name"] + assert.Equal(container1.CreatedAt, c2.StartTime.UnixNano()) + checkCRICPUAndMemoryStats(assert, c2, containerStats1) + checkCRIRootfsStats(assert, c2, containerStats1, nil) + checkCRILogsStats(assert, c2, &rootFsInfo) + + p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}] + assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano()) + assert.Equal(1, len(p1.Containers)) + + c3 := p1.Containers[0] + assert.Equal("container2-name", c3.Name) + assert.Equal(container2.CreatedAt, c3.StartTime.UnixNano()) + checkCRICPUAndMemoryStats(assert, c3, containerStats2) + checkCRIRootfsStats(assert, c3, containerStats2, &imageFsInfo) + checkCRILogsStats(assert, c3, &rootFsInfo) + + mockCadvisor.AssertExpectations(t) +} + +func TestCRIImagesFsStats(t *testing.T) { + var ( + imageFsStorageUUID = "imagefs-storage-uuid" + imageFsInfo = getTestFsInfo(2000) + imageFsUsage = makeFakeImageFsUsage(imageFsStorageUUID) + ) + var ( + mockCadvisor = new(cadvisortest.Mock) + mockRuntimeCache = new(kubecontainertest.MockRuntimeCache) + mockPodManager = new(kubepodtest.MockManager) + resourceAnalyzer = new(fakeResourceAnalyzer) + fakeRuntimeService = critest.NewFakeRuntimeService() + fakeImageService = critest.NewFakeImageService() + ) + + mockCadvisor.On("GetFsInfoByFsUUID", imageFsStorageUUID).Return(imageFsInfo, nil) + fakeImageService.SetFakeFilesystemUsage([]*runtimeapi.FilesystemUsage{ + imageFsUsage, + }) + + provider := NewCRIStatsProvider( + mockCadvisor, + resourceAnalyzer, + mockPodManager, + mockRuntimeCache, + fakeRuntimeService, + fakeImageService) + + stats, err := provider.ImageFsStats() + assert := assert.New(t) + assert.NoError(err) + + assert.Equal(imageFsUsage.Timestamp, stats.Time.UnixNano()) + assert.Equal(imageFsInfo.Available, *stats.AvailableBytes) + assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes) + assert.Equal(imageFsInfo.InodesFree, stats.InodesFree) + assert.Equal(imageFsInfo.Inodes, stats.Inodes) + assert.Equal(imageFsUsage.UsedBytes.Value, *stats.UsedBytes) + assert.Equal(imageFsUsage.InodesUsed.Value, *stats.InodesUsed) + + mockCadvisor.AssertExpectations(t) +} + +func makeFakePodSandbox(name, uid, namespace string) *critest.FakePodSandbox { + p := &critest.FakePodSandbox{ + PodSandboxStatus: runtimeapi.PodSandboxStatus{ + Metadata: &runtimeapi.PodSandboxMetadata{ + Name: name, + Uid: uid, + Namespace: namespace, + }, + State: runtimeapi.PodSandboxState_SANDBOX_READY, + CreatedAt: time.Now().UnixNano(), + }, + } + p.PodSandboxStatus.Id = critest.BuildSandboxName(p.PodSandboxStatus.Metadata) + return p +} + +func makeFakeContainer(sandbox *critest.FakePodSandbox, name string) *critest.FakeContainer { + sandboxID := sandbox.PodSandboxStatus.Id + c := &critest.FakeContainer{ + SandboxID: sandboxID, + ContainerStatus: runtimeapi.ContainerStatus{ + Metadata: &runtimeapi.ContainerMetadata{Name: name}, + Image: &runtimeapi.ImageSpec{}, + ImageRef: "fake-image-ref", + CreatedAt: time.Now().UnixNano(), + State: runtimeapi.ContainerState_CONTAINER_RUNNING, + }, + } + c.ContainerStatus.Id = critest.BuildContainerName(c.ContainerStatus.Metadata, sandboxID) + return c +} + +func makeFakeContainerStats(container *critest.FakeContainer, imageFsUUID string) *runtimeapi.ContainerStats { + return &runtimeapi.ContainerStats{ + Attributes: &runtimeapi.ContainerAttributes{ + Id: container.ContainerStatus.Id, + Metadata: container.ContainerStatus.Metadata, + }, + Cpu: &runtimeapi.CpuUsage{ + Timestamp: time.Now().UnixNano(), + UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: rand.Uint64()}, + }, + Memory: &runtimeapi.MemoryUsage{ + Timestamp: time.Now().UnixNano(), + WorkingSetBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()}, + }, + WritableLayer: &runtimeapi.FilesystemUsage{ + Timestamp: time.Now().UnixNano(), + StorageId: &runtimeapi.StorageIdentifier{Uuid: imageFsUUID}, + UsedBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()}, + InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64()}, + }, + } +} + +func makeFakeImageFsUsage(fsUUID string) *runtimeapi.FilesystemUsage { + return &runtimeapi.FilesystemUsage{ + Timestamp: time.Now().UnixNano(), + StorageId: &runtimeapi.StorageIdentifier{Uuid: fsUUID}, + UsedBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()}, + InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64()}, + } +} + +func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats) { + assert.Equal(cs.Cpu.Timestamp, actual.CPU.Time.UnixNano()) + assert.Equal(cs.Cpu.UsageCoreNanoSeconds.Value, *actual.CPU.UsageCoreNanoSeconds) + assert.Nil(actual.CPU.UsageNanoCores) + + assert.Equal(cs.Memory.Timestamp, actual.Memory.Time.UnixNano()) + assert.Nil(actual.Memory.AvailableBytes) + assert.Nil(actual.Memory.UsageBytes) + assert.Equal(cs.Memory.WorkingSetBytes.Value, *actual.Memory.WorkingSetBytes) + assert.Nil(actual.Memory.RSSBytes) + assert.Nil(actual.Memory.PageFaults) + assert.Nil(actual.Memory.MajorPageFaults) +} + +func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats, imageFsInfo *cadvisorapiv2.FsInfo) { + assert.Equal(cs.WritableLayer.Timestamp, actual.Rootfs.Time.UnixNano()) + if imageFsInfo != nil { + assert.Equal(imageFsInfo.Available, *actual.Rootfs.AvailableBytes) + assert.Equal(imageFsInfo.Capacity, *actual.Rootfs.CapacityBytes) + assert.Equal(*imageFsInfo.InodesFree, *actual.Rootfs.InodesFree) + assert.Equal(*imageFsInfo.Inodes, *actual.Rootfs.Inodes) + } else { + assert.Nil(actual.Rootfs.AvailableBytes) + assert.Nil(actual.Rootfs.CapacityBytes) + assert.Nil(actual.Rootfs.InodesFree) + assert.Nil(actual.Rootfs.Inodes) + } + assert.Equal(cs.WritableLayer.UsedBytes.Value, *actual.Rootfs.UsedBytes) + assert.Equal(cs.WritableLayer.InodesUsed.Value, *actual.Rootfs.InodesUsed) +} + +func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo) { + assert.Equal(rootFsInfo.Timestamp, actual.Logs.Time.Time) + assert.Equal(rootFsInfo.Available, *actual.Logs.AvailableBytes) + assert.Equal(rootFsInfo.Capacity, *actual.Logs.CapacityBytes) + assert.Equal(*rootFsInfo.InodesFree, *actual.Logs.InodesFree) + assert.Equal(*rootFsInfo.Inodes, *actual.Logs.Inodes) + assert.Nil(actual.Logs.UsedBytes) + assert.Nil(actual.Logs.InodesUsed) +} diff --git a/pkg/kubelet/stats/stats_provider_test.go b/pkg/kubelet/stats/stats_provider_test.go index 532a43292e..69cb50c443 100644 --- a/pkg/kubelet/stats/stats_provider_test.go +++ b/pkg/kubelet/stats/stats_provider_test.go @@ -102,16 +102,16 @@ func TestGetCgroupStats(t *testing.T) { checkFsStats(t, "", rootFsInfoSeed, cs.Logs) checkNetworkStats(t, "", containerInfoSeed, ns) - assert.Equal(cs.Name, cgroupName) - assert.Equal(cs.StartTime, metav1.NewTime(containerInfo.Spec.CreationTime)) + assert.Equal(cgroupName, cs.Name) + assert.Equal(metav1.NewTime(containerInfo.Spec.CreationTime), cs.StartTime) - assert.Equal(cs.Rootfs.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp)) - assert.Equal(*cs.Rootfs.UsedBytes, *containerInfo.Stats[0].Filesystem.BaseUsageBytes) - assert.Equal(*cs.Rootfs.InodesUsed, *containerInfo.Stats[0].Filesystem.InodeUsage) + assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Rootfs.Time) + assert.Equal(*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Rootfs.UsedBytes) + assert.Equal(*containerInfo.Stats[0].Filesystem.InodeUsage, *cs.Rootfs.InodesUsed) - assert.Equal(cs.Logs.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp)) - assert.Equal(*cs.Logs.UsedBytes, *containerInfo.Stats[0].Filesystem.TotalUsageBytes-*containerInfo.Stats[0].Filesystem.BaseUsageBytes) - assert.Equal(*cs.Logs.InodesUsed, *rootFsInfo.Inodes-*rootFsInfo.InodesFree) + assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Logs.Time) + assert.Equal(*containerInfo.Stats[0].Filesystem.TotalUsageBytes-*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Logs.UsedBytes) + assert.Equal(*rootFsInfo.Inodes-*rootFsInfo.InodesFree, *cs.Logs.InodesUsed) mockCadvisor.AssertExpectations(t) } @@ -144,9 +144,9 @@ func TestRootFsStats(t *testing.T) { checkFsStats(t, "", rootFsInfoSeed, stats) - assert.Equal(stats.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp)) - assert.Equal(*stats.UsedBytes, rootFsInfo.Usage) - assert.Equal(*stats.InodesUsed, *rootFsInfo.Inodes-*rootFsInfo.InodesFree) + assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), stats.Time) + assert.Equal(rootFsInfo.Usage, *stats.UsedBytes) + assert.Equal(*rootFsInfo.Inodes-*rootFsInfo.InodesFree, *stats.InodesUsed) mockCadvisor.AssertExpectations(t) } @@ -462,6 +462,7 @@ func getTestFsInfo(seed int) cadvisorapiv2.FsInfo { inodesFree = uint64(seed + offsetFsInodesFree) ) return cadvisorapiv2.FsInfo{ + Timestamp: time.Now(), Device: "test-device", Mountpoint: "test-mount-point", Capacity: uint64(seed + offsetFsCapacity),