mirror of https://github.com/k3s-io/k3s
Merge pull request #51557 from yguo0905/stats-cri
Automatic merge from submit-queue (batch tested with PRs 49133, 51557, 51749, 50842, 52018) Implement StatsProvider interface using CRI stats Ref: https://github.com/kubernetes/kubernetes/issues/46984 This is the follow up of https://github.com/kubernetes/kubernetes/pull/50932 - I include the cadvisor dependency changes in this PR for now to make it build. @dashpole will update the cadvisor dependency very soon, and I will remove the change once it's updated. - Please take a closer look at the implementation in `cri_stats_provider.go` since we currently don't have any runtime implementing the CRI stats interface and the changes here cannot be enabled in e2e tests. - Pod level network stats and container level logs stats are not provided. - In `cadvisor_stats_provider.go`, we are able to remove the call to `getCgroupStats` in `ImageFsStats` for getting the timestamp of the stats, given that we've changed cadvisor to include the timestamp in `FsInfo`. - Fixed the usage of `assert.Equal` in unit tests. **Release note**: ``` Support getting container stats from CRI. ``` /assign @yujuhong /assign @Random-Liupull/6/head
Kubernetes Submit Queue
GitHub
commit
3b2e32e064
|
|
@ -34,6 +34,8 @@ type FakeImageService struct {
|
|||
Images map[string]*runtimeapi.Image
|
||||
|
||||
pulledImages []*pulledImage
|
||||
|
||||
FakeFilesystemUsage []*runtimeapi.FilesystemUsage
|
||||
}
|
||||
|
||||
func (r *FakeImageService) SetFakeImages(images []string) {
|
||||
|
|
@ -53,6 +55,13 @@ func (r *FakeImageService) SetFakeImageSize(size uint64) {
|
|||
r.FakeImageSize = size
|
||||
}
|
||||
|
||||
func (r *FakeImageService) SetFakeFilesystemUsage(usage []*runtimeapi.FilesystemUsage) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
|
||||
r.FakeFilesystemUsage = usage
|
||||
}
|
||||
|
||||
func NewFakeImageService() *FakeImageService {
|
||||
return &FakeImageService{
|
||||
Called: make([]string, 0),
|
||||
|
|
@ -132,7 +141,9 @@ func (r *FakeImageService) ImageFsInfo(req *runtimeapi.ImageFsInfoRequest) (*run
|
|||
|
||||
r.Called = append(r.Called, "ImageFsInfo")
|
||||
|
||||
return nil, nil
|
||||
return &runtimeapi.ImageFsInfoResponse{
|
||||
ImageFilesystems: r.FakeFilesystemUsage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *FakeImageService) AssertImagePulledWithAuth(t *testing.T, image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig, failMsg string) {
|
||||
|
|
|
|||
|
|
@ -50,9 +50,10 @@ type FakeRuntimeService struct {
|
|||
|
||||
Called []string
|
||||
|
||||
FakeStatus *runtimeapi.RuntimeStatus
|
||||
Containers map[string]*FakeContainer
|
||||
Sandboxes map[string]*FakePodSandbox
|
||||
FakeStatus *runtimeapi.RuntimeStatus
|
||||
Containers map[string]*FakeContainer
|
||||
Sandboxes map[string]*FakePodSandbox
|
||||
FakeContainerStats map[string]*runtimeapi.ContainerStats
|
||||
}
|
||||
|
||||
func (r *FakeRuntimeService) GetContainerID(sandboxID, name string, attempt uint32) (string, error) {
|
||||
|
|
@ -102,9 +103,10 @@ func (r *FakeRuntimeService) AssertCalls(calls []string) error {
|
|||
|
||||
func NewFakeRuntimeService() *FakeRuntimeService {
|
||||
return &FakeRuntimeService{
|
||||
Called: make([]string, 0),
|
||||
Containers: make(map[string]*FakeContainer),
|
||||
Sandboxes: make(map[string]*FakePodSandbox),
|
||||
Called: make([]string, 0),
|
||||
Containers: make(map[string]*FakeContainer),
|
||||
Sandboxes: make(map[string]*FakePodSandbox),
|
||||
FakeContainerStats: make(map[string]*runtimeapi.ContainerStats),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -406,10 +408,54 @@ func (r *FakeRuntimeService) UpdateRuntimeConfig(runtimeCOnfig *runtimeapi.Runti
|
|||
return nil
|
||||
}
|
||||
|
||||
func (r *FakeRuntimeService) SetFakeContainerStats(containerStats []*runtimeapi.ContainerStats) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
|
||||
r.FakeContainerStats = make(map[string]*runtimeapi.ContainerStats)
|
||||
for _, s := range containerStats {
|
||||
r.FakeContainerStats[s.Attributes.Id] = s
|
||||
}
|
||||
}
|
||||
|
||||
func (r *FakeRuntimeService) ContainerStats(req *runtimeapi.ContainerStatsRequest) (*runtimeapi.ContainerStatsResponse, error) {
|
||||
return nil, fmt.Errorf("Not implemented")
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
|
||||
r.Called = append(r.Called, "ContainerStats")
|
||||
|
||||
s, found := r.FakeContainerStats[req.ContainerId]
|
||||
if !found {
|
||||
return nil, fmt.Errorf("no stats for container %q", req.ContainerId)
|
||||
}
|
||||
return &runtimeapi.ContainerStatsResponse{Stats: s}, nil
|
||||
}
|
||||
|
||||
func (r *FakeRuntimeService) ListContainerStats(req *runtimeapi.ListContainerStatsRequest) (*runtimeapi.ListContainerStatsResponse, error) {
|
||||
return nil, fmt.Errorf("Not implemented")
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
|
||||
r.Called = append(r.Called, "ListContainerStats")
|
||||
|
||||
var result []*runtimeapi.ContainerStats
|
||||
for _, c := range r.Containers {
|
||||
if req.Filter != nil {
|
||||
if req.Filter.Id != "" && req.Filter.Id != c.Id {
|
||||
continue
|
||||
}
|
||||
if req.Filter.PodSandboxId != "" && req.Filter.PodSandboxId != c.SandboxID {
|
||||
continue
|
||||
}
|
||||
if req.Filter.LabelSelector != nil && !filterInLabels(req.Filter.LabelSelector, c.GetLabels()) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
s, found := r.FakeContainerStats[c.Id]
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
result = append(result, s)
|
||||
}
|
||||
|
||||
return &runtimeapi.ListContainerStatsResponse{Stats: result}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -79,3 +79,7 @@ func (c *Fake) WatchEvents(request *events.Request) (*events.EventChannel, error
|
|||
func (c *Fake) HasDedicatedImageFs() (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (c *Fake) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) {
|
||||
return cadvisorapiv2.FsInfo{}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -88,3 +88,8 @@ func (c *Mock) HasDedicatedImageFs() (bool, error) {
|
|||
args := c.Called()
|
||||
return args.Get(0).(bool), args.Error(1)
|
||||
}
|
||||
|
||||
func (c *Mock) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) {
|
||||
args := c.Called(uuid)
|
||||
return args.Get(0).(cadvisorapiv2.FsInfo), args.Error(1)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,4 +44,8 @@ type Interface interface {
|
|||
|
||||
// HasDedicatedImageFs returns true iff a dedicated image filesystem exists for storing images.
|
||||
HasDedicatedImageFs() (bool, error)
|
||||
|
||||
// GetFsInfoByFsUUID returns the stats of the filesystem with the specified
|
||||
// uuid.
|
||||
GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ go_library(
|
|||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//pkg/kubelet/apis/cri:go_default_library",
|
||||
"//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
|
||||
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
|
||||
"//pkg/kubelet/cadvisor:go_default_library",
|
||||
"//pkg/kubelet/container:go_default_library",
|
||||
|
|
@ -20,6 +21,7 @@ go_library(
|
|||
"//pkg/kubelet/server/stats:go_default_library",
|
||||
"//pkg/kubelet/types:go_default_library",
|
||||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
"//vendor/github.com/google/cadvisor/fs:go_default_library",
|
||||
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
||||
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
|
|
@ -45,11 +47,14 @@ go_test(
|
|||
name = "go_default_test",
|
||||
srcs = [
|
||||
"cadvisor_stats_provider_test.go",
|
||||
"cri_stats_provider_test.go",
|
||||
"helper_test.go",
|
||||
"stats_provider_test.go",
|
||||
],
|
||||
library = ":go_default_library",
|
||||
deps = [
|
||||
"//pkg/kubelet/apis/cri/testing:go_default_library",
|
||||
"//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
|
||||
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
|
||||
"//pkg/kubelet/cadvisor/testing:go_default_library",
|
||||
"//pkg/kubelet/container:go_default_library",
|
||||
|
|
@ -58,6 +63,7 @@ go_test(
|
|||
"//pkg/kubelet/pod/testing:go_default_library",
|
||||
"//pkg/kubelet/server/stats:go_default_library",
|
||||
"//pkg/kubelet/types:go_default_library",
|
||||
"//vendor/github.com/google/cadvisor/fs:go_default_library",
|
||||
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
||||
"//vendor/github.com/google/cadvisor/info/v2:go_default_library",
|
||||
"//vendor/github.com/google/gofuzz:go_default_library",
|
||||
|
|
|
|||
|
|
@ -160,15 +160,8 @@ func (p *cadvisorStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
|
|||
imageFsInodesUsed = &imageFsIU
|
||||
}
|
||||
|
||||
// Get the root container stats's timestamp, which will be used as the
|
||||
// imageFs stats timestamp.
|
||||
rootStats, err := getCgroupStats(p.cadvisor, "/")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get root container stats: %v", err)
|
||||
}
|
||||
|
||||
return &statsapi.FsStats{
|
||||
Time: metav1.NewTime(rootStats.Timestamp),
|
||||
Time: metav1.NewTime(imageFsInfo.Timestamp),
|
||||
AvailableBytes: &imageFsInfo.Available,
|
||||
CapacityBytes: &imageFsInfo.Capacity,
|
||||
UsedBytes: &imageStats.TotalStorageBytes,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import (
|
|||
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
||||
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
|
|
@ -74,7 +73,7 @@ func TestRemoveTerminatedContainerInfo(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestListPodStats(t *testing.T) {
|
||||
func TestCadvisorListPodStats(t *testing.T) {
|
||||
const (
|
||||
namespace0 = "test0"
|
||||
namespace2 = "test2"
|
||||
|
|
@ -236,36 +235,31 @@ func TestListPodStats(t *testing.T) {
|
|||
checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network)
|
||||
}
|
||||
|
||||
func TestImagesFsStats(t *testing.T) {
|
||||
func TestCadvisorImagesFsStats(t *testing.T) {
|
||||
var (
|
||||
assert = assert.New(t)
|
||||
mockCadvisor = new(cadvisortest.Mock)
|
||||
mockRuntime = new(containertest.Mock)
|
||||
|
||||
seed = 100
|
||||
options = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false}
|
||||
imageFsInfo = getTestFsInfo(100)
|
||||
containerInfo = map[string]cadvisorapiv2.ContainerInfo{"/": getTestContainerInfo(seed, "test-pod", "test-ns", "test-container")}
|
||||
imageStats = &kubecontainer.ImageStats{TotalStorageBytes: 100}
|
||||
seed = 1000
|
||||
imageFsInfo = getTestFsInfo(seed)
|
||||
imageStats = &kubecontainer.ImageStats{TotalStorageBytes: 100}
|
||||
)
|
||||
|
||||
mockCadvisor.
|
||||
On("ImagesFsInfo").Return(imageFsInfo, nil).
|
||||
On("ContainerInfoV2", "/", options).Return(containerInfo, nil)
|
||||
mockRuntime.
|
||||
On("ImageStats").Return(imageStats, nil)
|
||||
mockCadvisor.On("ImagesFsInfo").Return(imageFsInfo, nil)
|
||||
mockRuntime.On("ImageStats").Return(imageStats, nil)
|
||||
|
||||
provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime)
|
||||
stats, err := provider.ImageFsStats()
|
||||
assert.NoError(err)
|
||||
|
||||
assert.Equal(stats.Time, metav1.NewTime(containerInfo["/"].Stats[0].Timestamp))
|
||||
assert.Equal(*stats.AvailableBytes, imageFsInfo.Available)
|
||||
assert.Equal(*stats.CapacityBytes, imageFsInfo.Capacity)
|
||||
assert.Equal(*stats.UsedBytes, imageStats.TotalStorageBytes)
|
||||
assert.Equal(stats.InodesFree, imageFsInfo.InodesFree)
|
||||
assert.Equal(stats.Inodes, imageFsInfo.Inodes)
|
||||
assert.Equal(*stats.InodesUsed, *imageFsInfo.Inodes-*imageFsInfo.InodesFree)
|
||||
assert.Equal(imageFsInfo.Timestamp, stats.Time.Time)
|
||||
assert.Equal(imageFsInfo.Available, *stats.AvailableBytes)
|
||||
assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes)
|
||||
assert.Equal(imageStats.TotalStorageBytes, *stats.UsedBytes)
|
||||
assert.Equal(imageFsInfo.InodesFree, stats.InodesFree)
|
||||
assert.Equal(imageFsInfo.Inodes, stats.Inodes)
|
||||
assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *stats.InodesUsed)
|
||||
|
||||
mockCadvisor.AssertExpectations(t)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,16 @@ package stats
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
cadvisorfs "github.com/google/cadvisor/fs"
|
||||
|
||||
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
|
||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
|
||||
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
||||
"k8s.io/kubernetes/pkg/kubelet/server/stats"
|
||||
|
|
@ -57,10 +65,215 @@ func newCRIStatsProvider(
|
|||
}
|
||||
}
|
||||
|
||||
// ListPodStats returns the stats of all the pod-managed containers.
|
||||
func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
// Gets node root filesystem information, which will be used to populate
|
||||
// the available and capacity bytes/inodes in container stats.
|
||||
rootFsInfo, err := p.cadvisor.RootFsInfo()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get rootFs info: %v", err)
|
||||
}
|
||||
|
||||
// Creates container map.
|
||||
containerMap := make(map[string]*runtimeapi.Container)
|
||||
containers, err := p.runtimeService.ListContainers(&runtimeapi.ContainerFilter{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list all containers: %v", err)
|
||||
}
|
||||
for _, c := range containers {
|
||||
containerMap[c.Id] = c
|
||||
}
|
||||
|
||||
// Creates pod sandbox map.
|
||||
podSandboxMap := make(map[string]*runtimeapi.PodSandbox)
|
||||
podSandboxes, err := p.runtimeService.ListPodSandbox(&runtimeapi.PodSandboxFilter{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err)
|
||||
}
|
||||
for _, s := range podSandboxes {
|
||||
podSandboxMap[s.Id] = s
|
||||
}
|
||||
|
||||
// uuidToFsInfo is a map from filesystem UUID to its stats. This will be
|
||||
// used as a cache to avoid querying cAdvisor for the filesystem stats with
|
||||
// the same UUID many times.
|
||||
uuidToFsInfo := make(map[runtimeapi.StorageIdentifier]*cadvisorapiv2.FsInfo)
|
||||
|
||||
// sandboxIDToPodStats is a temporary map from sandbox ID to its pod stats.
|
||||
sandboxIDToPodStats := make(map[string]*statsapi.PodStats)
|
||||
|
||||
resp, err := p.runtimeService.ListContainerStats(&runtimeapi.ListContainerStatsRequest{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list all container stats: %v", err)
|
||||
}
|
||||
for _, stats := range resp.Stats {
|
||||
containerID := stats.Attributes.Id
|
||||
container, found := containerMap[containerID]
|
||||
if !found {
|
||||
glog.Errorf("Unknown id %q in container map.", containerID)
|
||||
continue
|
||||
}
|
||||
|
||||
podSandboxID := container.PodSandboxId
|
||||
podSandbox, found := podSandboxMap[podSandboxID]
|
||||
if !found {
|
||||
glog.Errorf("Unknown id %q in pod sandbox map.", podSandboxID)
|
||||
continue
|
||||
}
|
||||
|
||||
// Creates the stats of the pod (if not created yet) which the
|
||||
// container belongs to.
|
||||
ps, found := sandboxIDToPodStats[podSandboxID]
|
||||
if !found {
|
||||
ps = p.makePodStats(podSandbox)
|
||||
sandboxIDToPodStats[podSandboxID] = ps
|
||||
}
|
||||
ps.Containers = append(ps.Containers, *p.makeContainerStats(stats, container, &rootFsInfo, uuidToFsInfo))
|
||||
}
|
||||
|
||||
result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats))
|
||||
for _, s := range sandboxIDToPodStats {
|
||||
result = append(result, *s)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ImageFsStats returns the stats of the image filesystem.
|
||||
func (p *criStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
resp, err := p.imageService.ImageFsInfo(&runtimeapi.ImageFsInfoRequest{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// CRI may return the stats of multiple image filesystems but we only
|
||||
// return the first one.
|
||||
//
|
||||
// TODO(yguo0905): Support returning stats of multiple image filesystems.
|
||||
for _, fs := range resp.ImageFilesystems {
|
||||
s := &statsapi.FsStats{
|
||||
Time: metav1.NewTime(time.Unix(0, fs.Timestamp)),
|
||||
UsedBytes: &fs.UsedBytes.Value,
|
||||
InodesUsed: &fs.InodesUsed.Value,
|
||||
}
|
||||
imageFsInfo := p.getFsInfo(fs.StorageId)
|
||||
if imageFsInfo != nil {
|
||||
// The image filesystem UUID is unknown to the local node or
|
||||
// there's an error on retrieving the stats. In these cases, we
|
||||
// omit those stats and return the best-effort partial result. See
|
||||
// https://github.com/kubernetes/heapster/issues/1793.
|
||||
s.AvailableBytes = &imageFsInfo.Available
|
||||
s.CapacityBytes = &imageFsInfo.Capacity
|
||||
s.InodesFree = imageFsInfo.InodesFree
|
||||
s.Inodes = imageFsInfo.Inodes
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("imageFs information is unavailable")
|
||||
}
|
||||
|
||||
// getFsInfo returns the information of the filesystem with the specified
|
||||
// storageID. If any error occurs, this function logs the error and returns
|
||||
// nil.
|
||||
func (p *criStatsProvider) getFsInfo(storageID *runtimeapi.StorageIdentifier) *cadvisorapiv2.FsInfo {
|
||||
if storageID == nil {
|
||||
glog.V(2).Infof("Failed to get filesystem info: storageID is nil.")
|
||||
return nil
|
||||
}
|
||||
fsInfo, err := p.cadvisor.GetFsInfoByFsUUID(storageID.Uuid)
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("Failed to get the info of the filesystem with id %q: %v.", storageID.Uuid, err)
|
||||
if err == cadvisorfs.ErrNoSuchDevice {
|
||||
glog.V(2).Info(msg)
|
||||
} else {
|
||||
glog.Error(msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return &fsInfo
|
||||
}
|
||||
|
||||
func (p *criStatsProvider) makePodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats {
|
||||
s := &statsapi.PodStats{
|
||||
PodRef: statsapi.PodReference{
|
||||
Name: podSandbox.Metadata.Name,
|
||||
UID: podSandbox.Metadata.Uid,
|
||||
Namespace: podSandbox.Metadata.Namespace,
|
||||
},
|
||||
// The StartTime in the summary API is the pod creation time.
|
||||
StartTime: metav1.NewTime(time.Unix(0, podSandbox.CreatedAt)),
|
||||
// Network stats are not supported by CRI.
|
||||
}
|
||||
podUID := types.UID(s.PodRef.UID)
|
||||
if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found {
|
||||
s.VolumeStats = vstats.Volumes
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (p *criStatsProvider) makeContainerStats(
|
||||
stats *runtimeapi.ContainerStats,
|
||||
container *runtimeapi.Container,
|
||||
rootFsInfo *cadvisorapiv2.FsInfo,
|
||||
uuidToFsInfo map[runtimeapi.StorageIdentifier]*cadvisorapiv2.FsInfo,
|
||||
) *statsapi.ContainerStats {
|
||||
result := &statsapi.ContainerStats{
|
||||
Name: stats.Attributes.Metadata.Name,
|
||||
// The StartTime in the summary API is the container creation time.
|
||||
StartTime: metav1.NewTime(time.Unix(0, container.CreatedAt)),
|
||||
CPU: &statsapi.CPUStats{},
|
||||
Memory: &statsapi.MemoryStats{},
|
||||
Rootfs: &statsapi.FsStats{},
|
||||
Logs: &statsapi.FsStats{
|
||||
Time: metav1.NewTime(rootFsInfo.Timestamp),
|
||||
AvailableBytes: &rootFsInfo.Available,
|
||||
CapacityBytes: &rootFsInfo.Capacity,
|
||||
InodesFree: rootFsInfo.InodesFree,
|
||||
Inodes: rootFsInfo.Inodes,
|
||||
// UsedBytes and InodesUsed are unavailable from CRI stats.
|
||||
//
|
||||
// TODO(yguo0905): Get this information from kubelet and
|
||||
// populate the two fields here.
|
||||
},
|
||||
// UserDefinedMetrics is not supported by CRI.
|
||||
}
|
||||
if stats.Cpu != nil {
|
||||
result.CPU.Time = metav1.NewTime(time.Unix(0, stats.Cpu.Timestamp))
|
||||
if stats.Cpu.UsageCoreNanoSeconds != nil {
|
||||
result.CPU.UsageCoreNanoSeconds = &stats.Cpu.UsageCoreNanoSeconds.Value
|
||||
}
|
||||
}
|
||||
if stats.Memory != nil {
|
||||
result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp))
|
||||
if stats.Memory.WorkingSetBytes != nil {
|
||||
result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value
|
||||
}
|
||||
}
|
||||
if stats.WritableLayer != nil {
|
||||
result.Rootfs.Time = metav1.NewTime(time.Unix(0, stats.WritableLayer.Timestamp))
|
||||
if stats.WritableLayer.UsedBytes != nil {
|
||||
result.Rootfs.UsedBytes = &stats.WritableLayer.UsedBytes.Value
|
||||
}
|
||||
if stats.WritableLayer.InodesUsed != nil {
|
||||
result.Rootfs.InodesUsed = &stats.WritableLayer.InodesUsed.Value
|
||||
}
|
||||
}
|
||||
storageID := stats.WritableLayer.StorageId
|
||||
imageFsInfo, found := uuidToFsInfo[*storageID]
|
||||
if !found {
|
||||
imageFsInfo = p.getFsInfo(storageID)
|
||||
uuidToFsInfo[*storageID] = imageFsInfo
|
||||
}
|
||||
if imageFsInfo != nil {
|
||||
// The image filesystem UUID is unknown to the local node or there's an
|
||||
// error on retrieving the stats. In these cases, we omit those stats
|
||||
// and return the best-effort partial result. See
|
||||
// https://github.com/kubernetes/heapster/issues/1793.
|
||||
result.Rootfs.AvailableBytes = &imageFsInfo.Available
|
||||
result.Rootfs.CapacityBytes = &imageFsInfo.Capacity
|
||||
result.Rootfs.InodesFree = imageFsInfo.InodesFree
|
||||
result.Rootfs.Inodes = imageFsInfo.Inodes
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,274 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cadvisorfs "github.com/google/cadvisor/fs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
|
||||
critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
|
||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
|
||||
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
|
||||
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
||||
kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
|
||||
kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
|
||||
)
|
||||
|
||||
func TestCRIListPodStats(t *testing.T) {
|
||||
var (
|
||||
imageFsStorageUUID = "imagefs-storage-uuid"
|
||||
unknownStorageUUID = "unknown-storage-uuid"
|
||||
imageFsInfo = getTestFsInfo(2000)
|
||||
rootFsInfo = getTestFsInfo(1000)
|
||||
|
||||
sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
|
||||
container0 = makeFakeContainer(sandbox0, "container0-name")
|
||||
containerStats0 = makeFakeContainerStats(container0, imageFsStorageUUID)
|
||||
container1 = makeFakeContainer(sandbox0, "container1-name")
|
||||
containerStats1 = makeFakeContainerStats(container1, unknownStorageUUID)
|
||||
|
||||
sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
|
||||
container2 = makeFakeContainer(sandbox1, "container2-name")
|
||||
containerStats2 = makeFakeContainerStats(container2, imageFsStorageUUID)
|
||||
)
|
||||
|
||||
var (
|
||||
mockCadvisor = new(cadvisortest.Mock)
|
||||
mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
|
||||
mockPodManager = new(kubepodtest.MockManager)
|
||||
resourceAnalyzer = new(fakeResourceAnalyzer)
|
||||
fakeRuntimeService = critest.NewFakeRuntimeService()
|
||||
fakeImageService = critest.NewFakeImageService()
|
||||
)
|
||||
|
||||
mockCadvisor.
|
||||
On("RootFsInfo").Return(rootFsInfo, nil).
|
||||
On("GetFsInfoByFsUUID", imageFsStorageUUID).Return(imageFsInfo, nil).
|
||||
On("GetFsInfoByFsUUID", unknownStorageUUID).Return(cadvisorapiv2.FsInfo{}, cadvisorfs.ErrNoSuchDevice)
|
||||
fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{
|
||||
sandbox0, sandbox1,
|
||||
})
|
||||
fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
|
||||
container0, container1, container2,
|
||||
})
|
||||
fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
|
||||
containerStats0, containerStats1, containerStats2,
|
||||
})
|
||||
|
||||
provider := NewCRIStatsProvider(
|
||||
mockCadvisor,
|
||||
resourceAnalyzer,
|
||||
mockPodManager,
|
||||
mockRuntimeCache,
|
||||
fakeRuntimeService,
|
||||
fakeImageService)
|
||||
|
||||
stats, err := provider.ListPodStats()
|
||||
assert := assert.New(t)
|
||||
assert.NoError(err)
|
||||
assert.Equal(2, len(stats))
|
||||
|
||||
podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats)
|
||||
for _, s := range stats {
|
||||
podStatsMap[s.PodRef] = s
|
||||
}
|
||||
|
||||
p0 := podStatsMap[statsapi.PodReference{Name: "sandbox0-name", UID: "sandbox0-uid", Namespace: "sandbox0-ns"}]
|
||||
assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
|
||||
assert.Equal(2, len(p0.Containers))
|
||||
|
||||
containerStatsMap := make(map[string]statsapi.ContainerStats)
|
||||
for _, s := range p0.Containers {
|
||||
containerStatsMap[s.Name] = s
|
||||
}
|
||||
c1 := containerStatsMap["container0-name"]
|
||||
assert.Equal(container0.CreatedAt, c1.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c1, containerStats0)
|
||||
checkCRIRootfsStats(assert, c1, containerStats0, &imageFsInfo)
|
||||
checkCRILogsStats(assert, c1, &rootFsInfo)
|
||||
c2 := containerStatsMap["container1-name"]
|
||||
assert.Equal(container1.CreatedAt, c2.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c2, containerStats1)
|
||||
checkCRIRootfsStats(assert, c2, containerStats1, nil)
|
||||
checkCRILogsStats(assert, c2, &rootFsInfo)
|
||||
|
||||
p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
|
||||
assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
|
||||
assert.Equal(1, len(p1.Containers))
|
||||
|
||||
c3 := p1.Containers[0]
|
||||
assert.Equal("container2-name", c3.Name)
|
||||
assert.Equal(container2.CreatedAt, c3.StartTime.UnixNano())
|
||||
checkCRICPUAndMemoryStats(assert, c3, containerStats2)
|
||||
checkCRIRootfsStats(assert, c3, containerStats2, &imageFsInfo)
|
||||
checkCRILogsStats(assert, c3, &rootFsInfo)
|
||||
|
||||
mockCadvisor.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestCRIImagesFsStats(t *testing.T) {
|
||||
var (
|
||||
imageFsStorageUUID = "imagefs-storage-uuid"
|
||||
imageFsInfo = getTestFsInfo(2000)
|
||||
imageFsUsage = makeFakeImageFsUsage(imageFsStorageUUID)
|
||||
)
|
||||
var (
|
||||
mockCadvisor = new(cadvisortest.Mock)
|
||||
mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
|
||||
mockPodManager = new(kubepodtest.MockManager)
|
||||
resourceAnalyzer = new(fakeResourceAnalyzer)
|
||||
fakeRuntimeService = critest.NewFakeRuntimeService()
|
||||
fakeImageService = critest.NewFakeImageService()
|
||||
)
|
||||
|
||||
mockCadvisor.On("GetFsInfoByFsUUID", imageFsStorageUUID).Return(imageFsInfo, nil)
|
||||
fakeImageService.SetFakeFilesystemUsage([]*runtimeapi.FilesystemUsage{
|
||||
imageFsUsage,
|
||||
})
|
||||
|
||||
provider := NewCRIStatsProvider(
|
||||
mockCadvisor,
|
||||
resourceAnalyzer,
|
||||
mockPodManager,
|
||||
mockRuntimeCache,
|
||||
fakeRuntimeService,
|
||||
fakeImageService)
|
||||
|
||||
stats, err := provider.ImageFsStats()
|
||||
assert := assert.New(t)
|
||||
assert.NoError(err)
|
||||
|
||||
assert.Equal(imageFsUsage.Timestamp, stats.Time.UnixNano())
|
||||
assert.Equal(imageFsInfo.Available, *stats.AvailableBytes)
|
||||
assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes)
|
||||
assert.Equal(imageFsInfo.InodesFree, stats.InodesFree)
|
||||
assert.Equal(imageFsInfo.Inodes, stats.Inodes)
|
||||
assert.Equal(imageFsUsage.UsedBytes.Value, *stats.UsedBytes)
|
||||
assert.Equal(imageFsUsage.InodesUsed.Value, *stats.InodesUsed)
|
||||
|
||||
mockCadvisor.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func makeFakePodSandbox(name, uid, namespace string) *critest.FakePodSandbox {
|
||||
p := &critest.FakePodSandbox{
|
||||
PodSandboxStatus: runtimeapi.PodSandboxStatus{
|
||||
Metadata: &runtimeapi.PodSandboxMetadata{
|
||||
Name: name,
|
||||
Uid: uid,
|
||||
Namespace: namespace,
|
||||
},
|
||||
State: runtimeapi.PodSandboxState_SANDBOX_READY,
|
||||
CreatedAt: time.Now().UnixNano(),
|
||||
},
|
||||
}
|
||||
p.PodSandboxStatus.Id = critest.BuildSandboxName(p.PodSandboxStatus.Metadata)
|
||||
return p
|
||||
}
|
||||
|
||||
func makeFakeContainer(sandbox *critest.FakePodSandbox, name string) *critest.FakeContainer {
|
||||
sandboxID := sandbox.PodSandboxStatus.Id
|
||||
c := &critest.FakeContainer{
|
||||
SandboxID: sandboxID,
|
||||
ContainerStatus: runtimeapi.ContainerStatus{
|
||||
Metadata: &runtimeapi.ContainerMetadata{Name: name},
|
||||
Image: &runtimeapi.ImageSpec{},
|
||||
ImageRef: "fake-image-ref",
|
||||
CreatedAt: time.Now().UnixNano(),
|
||||
State: runtimeapi.ContainerState_CONTAINER_RUNNING,
|
||||
},
|
||||
}
|
||||
c.ContainerStatus.Id = critest.BuildContainerName(c.ContainerStatus.Metadata, sandboxID)
|
||||
return c
|
||||
}
|
||||
|
||||
func makeFakeContainerStats(container *critest.FakeContainer, imageFsUUID string) *runtimeapi.ContainerStats {
|
||||
return &runtimeapi.ContainerStats{
|
||||
Attributes: &runtimeapi.ContainerAttributes{
|
||||
Id: container.ContainerStatus.Id,
|
||||
Metadata: container.ContainerStatus.Metadata,
|
||||
},
|
||||
Cpu: &runtimeapi.CpuUsage{
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
},
|
||||
Memory: &runtimeapi.MemoryUsage{
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
WorkingSetBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
},
|
||||
WritableLayer: &runtimeapi.FilesystemUsage{
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
StorageId: &runtimeapi.StorageIdentifier{Uuid: imageFsUUID},
|
||||
UsedBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func makeFakeImageFsUsage(fsUUID string) *runtimeapi.FilesystemUsage {
|
||||
return &runtimeapi.FilesystemUsage{
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
StorageId: &runtimeapi.StorageIdentifier{Uuid: fsUUID},
|
||||
UsedBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64()},
|
||||
}
|
||||
}
|
||||
|
||||
func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats) {
|
||||
assert.Equal(cs.Cpu.Timestamp, actual.CPU.Time.UnixNano())
|
||||
assert.Equal(cs.Cpu.UsageCoreNanoSeconds.Value, *actual.CPU.UsageCoreNanoSeconds)
|
||||
assert.Nil(actual.CPU.UsageNanoCores)
|
||||
|
||||
assert.Equal(cs.Memory.Timestamp, actual.Memory.Time.UnixNano())
|
||||
assert.Nil(actual.Memory.AvailableBytes)
|
||||
assert.Nil(actual.Memory.UsageBytes)
|
||||
assert.Equal(cs.Memory.WorkingSetBytes.Value, *actual.Memory.WorkingSetBytes)
|
||||
assert.Nil(actual.Memory.RSSBytes)
|
||||
assert.Nil(actual.Memory.PageFaults)
|
||||
assert.Nil(actual.Memory.MajorPageFaults)
|
||||
}
|
||||
|
||||
func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats, imageFsInfo *cadvisorapiv2.FsInfo) {
|
||||
assert.Equal(cs.WritableLayer.Timestamp, actual.Rootfs.Time.UnixNano())
|
||||
if imageFsInfo != nil {
|
||||
assert.Equal(imageFsInfo.Available, *actual.Rootfs.AvailableBytes)
|
||||
assert.Equal(imageFsInfo.Capacity, *actual.Rootfs.CapacityBytes)
|
||||
assert.Equal(*imageFsInfo.InodesFree, *actual.Rootfs.InodesFree)
|
||||
assert.Equal(*imageFsInfo.Inodes, *actual.Rootfs.Inodes)
|
||||
} else {
|
||||
assert.Nil(actual.Rootfs.AvailableBytes)
|
||||
assert.Nil(actual.Rootfs.CapacityBytes)
|
||||
assert.Nil(actual.Rootfs.InodesFree)
|
||||
assert.Nil(actual.Rootfs.Inodes)
|
||||
}
|
||||
assert.Equal(cs.WritableLayer.UsedBytes.Value, *actual.Rootfs.UsedBytes)
|
||||
assert.Equal(cs.WritableLayer.InodesUsed.Value, *actual.Rootfs.InodesUsed)
|
||||
}
|
||||
|
||||
func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo) {
|
||||
assert.Equal(rootFsInfo.Timestamp, actual.Logs.Time.Time)
|
||||
assert.Equal(rootFsInfo.Available, *actual.Logs.AvailableBytes)
|
||||
assert.Equal(rootFsInfo.Capacity, *actual.Logs.CapacityBytes)
|
||||
assert.Equal(*rootFsInfo.InodesFree, *actual.Logs.InodesFree)
|
||||
assert.Equal(*rootFsInfo.Inodes, *actual.Logs.Inodes)
|
||||
assert.Nil(actual.Logs.UsedBytes)
|
||||
assert.Nil(actual.Logs.InodesUsed)
|
||||
}
|
||||
|
|
@ -102,16 +102,16 @@ func TestGetCgroupStats(t *testing.T) {
|
|||
checkFsStats(t, "", rootFsInfoSeed, cs.Logs)
|
||||
checkNetworkStats(t, "", containerInfoSeed, ns)
|
||||
|
||||
assert.Equal(cs.Name, cgroupName)
|
||||
assert.Equal(cs.StartTime, metav1.NewTime(containerInfo.Spec.CreationTime))
|
||||
assert.Equal(cgroupName, cs.Name)
|
||||
assert.Equal(metav1.NewTime(containerInfo.Spec.CreationTime), cs.StartTime)
|
||||
|
||||
assert.Equal(cs.Rootfs.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp))
|
||||
assert.Equal(*cs.Rootfs.UsedBytes, *containerInfo.Stats[0].Filesystem.BaseUsageBytes)
|
||||
assert.Equal(*cs.Rootfs.InodesUsed, *containerInfo.Stats[0].Filesystem.InodeUsage)
|
||||
assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Rootfs.Time)
|
||||
assert.Equal(*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Rootfs.UsedBytes)
|
||||
assert.Equal(*containerInfo.Stats[0].Filesystem.InodeUsage, *cs.Rootfs.InodesUsed)
|
||||
|
||||
assert.Equal(cs.Logs.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp))
|
||||
assert.Equal(*cs.Logs.UsedBytes, *containerInfo.Stats[0].Filesystem.TotalUsageBytes-*containerInfo.Stats[0].Filesystem.BaseUsageBytes)
|
||||
assert.Equal(*cs.Logs.InodesUsed, *rootFsInfo.Inodes-*rootFsInfo.InodesFree)
|
||||
assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Logs.Time)
|
||||
assert.Equal(*containerInfo.Stats[0].Filesystem.TotalUsageBytes-*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Logs.UsedBytes)
|
||||
assert.Equal(*rootFsInfo.Inodes-*rootFsInfo.InodesFree, *cs.Logs.InodesUsed)
|
||||
|
||||
mockCadvisor.AssertExpectations(t)
|
||||
}
|
||||
|
|
@ -144,9 +144,9 @@ func TestRootFsStats(t *testing.T) {
|
|||
|
||||
checkFsStats(t, "", rootFsInfoSeed, stats)
|
||||
|
||||
assert.Equal(stats.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp))
|
||||
assert.Equal(*stats.UsedBytes, rootFsInfo.Usage)
|
||||
assert.Equal(*stats.InodesUsed, *rootFsInfo.Inodes-*rootFsInfo.InodesFree)
|
||||
assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), stats.Time)
|
||||
assert.Equal(rootFsInfo.Usage, *stats.UsedBytes)
|
||||
assert.Equal(*rootFsInfo.Inodes-*rootFsInfo.InodesFree, *stats.InodesUsed)
|
||||
|
||||
mockCadvisor.AssertExpectations(t)
|
||||
}
|
||||
|
|
@ -462,6 +462,7 @@ func getTestFsInfo(seed int) cadvisorapiv2.FsInfo {
|
|||
inodesFree = uint64(seed + offsetFsInodesFree)
|
||||
)
|
||||
return cadvisorapiv2.FsInfo{
|
||||
Timestamp: time.Now(),
|
||||
Device: "test-device",
|
||||
Mountpoint: "test-mount-point",
|
||||
Capacity: uint64(seed + offsetFsCapacity),
|
||||
|
|
|
|||
Loading…
Reference in New Issue