Merge pull request #51557 from yguo0905/stats-cri

Automatic merge from submit-queue (batch tested with PRs 49133, 51557, 51749, 50842, 52018)

Implement StatsProvider interface using CRI stats

Ref: https://github.com/kubernetes/kubernetes/issues/46984

This is the follow up of https://github.com/kubernetes/kubernetes/pull/50932

- I include the cadvisor dependency changes in this PR for now to make it build. @dashpole will update the cadvisor dependency very soon, and I will remove the change once it's updated.
- Please take a closer look at the implementation in `cri_stats_provider.go` since we currently don't have any runtime implementing the CRI stats interface and the changes here cannot be enabled in e2e tests.
- Pod level network stats and container level logs stats are not provided.
- In `cadvisor_stats_provider.go`, we are able to remove the call to `getCgroupStats` in `ImageFsStats` for getting the timestamp of the stats, given that we've changed cadvisor to include the timestamp in `FsInfo`.
- Fixed the usage of `assert.Equal` in unit tests.

**Release note**:
```
Support getting container stats from CRI.
```

/assign @yujuhong 
/assign @Random-Liu

pkg/kubelet/apis/cri/testing/fake_image_service.go

@@ -34,6 +34,8 @@ type FakeImageService struct {
 	Images        map[string]*runtimeapi.Image
 
 	pulledImages []*pulledImage
+
+	FakeFilesystemUsage []*runtimeapi.FilesystemUsage
 }
 
 func (r *FakeImageService) SetFakeImages(images []string) {
@@ -53,6 +55,13 @@ func (r *FakeImageService) SetFakeImageSize(size uint64) {
 	r.FakeImageSize = size
 }
 
+func (r *FakeImageService) SetFakeFilesystemUsage(usage []*runtimeapi.FilesystemUsage) {
+	r.Lock()
+	defer r.Unlock()
+
+	r.FakeFilesystemUsage = usage
+}
+
 func NewFakeImageService() *FakeImageService {
 	return &FakeImageService{
 		Called: make([]string, 0),
@@ -132,7 +141,9 @@ func (r *FakeImageService) ImageFsInfo(req *runtimeapi.ImageFsInfoRequest) (*run
 
 	r.Called = append(r.Called, "ImageFsInfo")
 
-	return nil, nil
+	return &runtimeapi.ImageFsInfoResponse{
+		ImageFilesystems: r.FakeFilesystemUsage,
+	}, nil
 }
 
 func (r *FakeImageService) AssertImagePulledWithAuth(t *testing.T, image *runtimeapi.ImageSpec, auth *runtimeapi.AuthConfig, failMsg string) {

pkg/kubelet/apis/cri/testing/fake_runtime_service.go

@@ -50,9 +50,10 @@ type FakeRuntimeService struct {
 
 	Called []string
 
-	FakeStatus *runtimeapi.RuntimeStatus
+	FakeStatus         *runtimeapi.RuntimeStatus
-	Containers map[string]*FakeContainer
+	Containers         map[string]*FakeContainer
-	Sandboxes  map[string]*FakePodSandbox
+	Sandboxes          map[string]*FakePodSandbox
+	FakeContainerStats map[string]*runtimeapi.ContainerStats
 }
 
 func (r *FakeRuntimeService) GetContainerID(sandboxID, name string, attempt uint32) (string, error) {
@@ -102,9 +103,10 @@ func (r *FakeRuntimeService) AssertCalls(calls []string) error {
 
 func NewFakeRuntimeService() *FakeRuntimeService {
 	return &FakeRuntimeService{
-		Called:     make([]string, 0),
+		Called:             make([]string, 0),
-		Containers: make(map[string]*FakeContainer),
+		Containers:         make(map[string]*FakeContainer),
-		Sandboxes:  make(map[string]*FakePodSandbox),
+		Sandboxes:          make(map[string]*FakePodSandbox),
+		FakeContainerStats: make(map[string]*runtimeapi.ContainerStats),
 	}
 }
 
@@ -406,10 +408,54 @@ func (r *FakeRuntimeService) UpdateRuntimeConfig(runtimeCOnfig *runtimeapi.Runti
 	return nil
 }
 
+func (r *FakeRuntimeService) SetFakeContainerStats(containerStats []*runtimeapi.ContainerStats) {
+	r.Lock()
+	defer r.Unlock()
+
+	r.FakeContainerStats = make(map[string]*runtimeapi.ContainerStats)
+	for _, s := range containerStats {
+		r.FakeContainerStats[s.Attributes.Id] = s
+	}
+}
+
 func (r *FakeRuntimeService) ContainerStats(req *runtimeapi.ContainerStatsRequest) (*runtimeapi.ContainerStatsResponse, error) {
-	return nil, fmt.Errorf("Not implemented")
+	r.Lock()
+	defer r.Unlock()
+
+	r.Called = append(r.Called, "ContainerStats")
+
+	s, found := r.FakeContainerStats[req.ContainerId]
+	if !found {
+		return nil, fmt.Errorf("no stats for container %q", req.ContainerId)
+	}
+	return &runtimeapi.ContainerStatsResponse{Stats: s}, nil
 }
 
 func (r *FakeRuntimeService) ListContainerStats(req *runtimeapi.ListContainerStatsRequest) (*runtimeapi.ListContainerStatsResponse, error) {
-	return nil, fmt.Errorf("Not implemented")
+	r.Lock()
+	defer r.Unlock()
+
+	r.Called = append(r.Called, "ListContainerStats")
+
+	var result []*runtimeapi.ContainerStats
+	for _, c := range r.Containers {
+		if req.Filter != nil {
+			if req.Filter.Id != "" && req.Filter.Id != c.Id {
+				continue
+			}
+			if req.Filter.PodSandboxId != "" && req.Filter.PodSandboxId != c.SandboxID {
+				continue
+			}
+			if req.Filter.LabelSelector != nil && !filterInLabels(req.Filter.LabelSelector, c.GetLabels()) {
+				continue
+			}
+		}
+		s, found := r.FakeContainerStats[c.Id]
+		if !found {
+			continue
+		}
+		result = append(result, s)
+	}
+
+	return &runtimeapi.ListContainerStatsResponse{Stats: result}, nil
 }

pkg/kubelet/cadvisor/testing/cadvisor_fake.go

@@ -79,3 +79,7 @@ func (c *Fake) WatchEvents(request *events.Request) (*events.EventChannel, error
 func (c *Fake) HasDedicatedImageFs() (bool, error) {
 	return false, nil
 }
+
+func (c *Fake) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) {
+	return cadvisorapiv2.FsInfo{}, nil
+}

pkg/kubelet/cadvisor/testing/cadvisor_mock.go

@@ -88,3 +88,8 @@ func (c *Mock) HasDedicatedImageFs() (bool, error) {
 	args := c.Called()
 	return args.Get(0).(bool), args.Error(1)
 }
+
+func (c *Mock) GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error) {
+	args := c.Called(uuid)
+	return args.Get(0).(cadvisorapiv2.FsInfo), args.Error(1)
+}

pkg/kubelet/cadvisor/types.go

@@ -44,4 +44,8 @@ type Interface interface {
 
 	// HasDedicatedImageFs returns true iff a dedicated image filesystem exists for storing images.
 	HasDedicatedImageFs() (bool, error)
+
+	// GetFsInfoByFsUUID returns the stats of the filesystem with the specified
+	// uuid.
+	GetFsInfoByFsUUID(uuid string) (cadvisorapiv2.FsInfo, error)
 }

pkg/kubelet/stats/BUILD

@@ -11,6 +11,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/kubelet/apis/cri:go_default_library",
+        "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
         "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
         "//pkg/kubelet/cadvisor:go_default_library",
         "//pkg/kubelet/container:go_default_library",
@@ -20,6 +21,7 @@ go_library(
         "//pkg/kubelet/server/stats:go_default_library",
         "//pkg/kubelet/types:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/google/cadvisor/fs:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -45,11 +47,14 @@ go_test(
     name = "go_default_test",
     srcs = [
         "cadvisor_stats_provider_test.go",
+        "cri_stats_provider_test.go",
         "helper_test.go",
         "stats_provider_test.go",
     ],
     library = ":go_default_library",
     deps = [
+        "//pkg/kubelet/apis/cri/testing:go_default_library",
+        "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
         "//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
         "//pkg/kubelet/cadvisor/testing:go_default_library",
         "//pkg/kubelet/container:go_default_library",
@@ -58,6 +63,7 @@ go_test(
         "//pkg/kubelet/pod/testing:go_default_library",
         "//pkg/kubelet/server/stats:go_default_library",
         "//pkg/kubelet/types:go_default_library",
+        "//vendor/github.com/google/cadvisor/fs:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v1:go_default_library",
         "//vendor/github.com/google/cadvisor/info/v2:go_default_library",
         "//vendor/github.com/google/gofuzz:go_default_library",

pkg/kubelet/stats/cadvisor_stats_provider.go

@@ -160,15 +160,8 @@ func (p *cadvisorStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
 		imageFsInodesUsed = &imageFsIU
 	}
 
-	// Get the root container stats's timestamp, which will be used as the
-	// imageFs stats timestamp.
-	rootStats, err := getCgroupStats(p.cadvisor, "/")
-	if err != nil {
-		return nil, fmt.Errorf("failed to get root container stats: %v", err)
-	}
-
 	return &statsapi.FsStats{
-		Time:           metav1.NewTime(rootStats.Timestamp),
+		Time:           metav1.NewTime(imageFsInfo.Timestamp),
 		AvailableBytes: &imageFsInfo.Available,
 		CapacityBytes:  &imageFsInfo.Capacity,
 		UsedBytes:      &imageStats.TotalStorageBytes,

pkg/kubelet/stats/cadvisor_stats_provider_test.go

@@ -22,7 +22,6 @@ import (
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
 	"github.com/stretchr/testify/assert"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@@ -74,7 +73,7 @@ func TestRemoveTerminatedContainerInfo(t *testing.T) {
 	}
 }
 
-func TestListPodStats(t *testing.T) {
+func TestCadvisorListPodStats(t *testing.T) {
 	const (
 		namespace0 = "test0"
 		namespace2 = "test2"
@@ -236,36 +235,31 @@ func TestListPodStats(t *testing.T) {
 	checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network)
 }
 
-func TestImagesFsStats(t *testing.T) {
+func TestCadvisorImagesFsStats(t *testing.T) {
 	var (
 		assert       = assert.New(t)
 		mockCadvisor = new(cadvisortest.Mock)
 		mockRuntime  = new(containertest.Mock)
 
-		seed          = 100
+		seed        = 1000
-		options       = cadvisorapiv2.RequestOptions{IdType: cadvisorapiv2.TypeName, Count: 2, Recursive: false}
+		imageFsInfo = getTestFsInfo(seed)
-		imageFsInfo   = getTestFsInfo(100)
+		imageStats  = &kubecontainer.ImageStats{TotalStorageBytes: 100}
-		containerInfo = map[string]cadvisorapiv2.ContainerInfo{"/": getTestContainerInfo(seed, "test-pod", "test-ns", "test-container")}
-		imageStats    = &kubecontainer.ImageStats{TotalStorageBytes: 100}
 	)
 
-	mockCadvisor.
+	mockCadvisor.On("ImagesFsInfo").Return(imageFsInfo, nil)
-		On("ImagesFsInfo").Return(imageFsInfo, nil).
+	mockRuntime.On("ImageStats").Return(imageStats, nil)
-		On("ContainerInfoV2", "/", options).Return(containerInfo, nil)
-	mockRuntime.
-		On("ImageStats").Return(imageStats, nil)
 
 	provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime)
 	stats, err := provider.ImageFsStats()
 	assert.NoError(err)
 
-	assert.Equal(stats.Time, metav1.NewTime(containerInfo["/"].Stats[0].Timestamp))
+	assert.Equal(imageFsInfo.Timestamp, stats.Time.Time)
-	assert.Equal(*stats.AvailableBytes, imageFsInfo.Available)
+	assert.Equal(imageFsInfo.Available, *stats.AvailableBytes)
-	assert.Equal(*stats.CapacityBytes, imageFsInfo.Capacity)
+	assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes)
-	assert.Equal(*stats.UsedBytes, imageStats.TotalStorageBytes)
+	assert.Equal(imageStats.TotalStorageBytes, *stats.UsedBytes)
-	assert.Equal(stats.InodesFree, imageFsInfo.InodesFree)
+	assert.Equal(imageFsInfo.InodesFree, stats.InodesFree)
-	assert.Equal(stats.Inodes, imageFsInfo.Inodes)
+	assert.Equal(imageFsInfo.Inodes, stats.Inodes)
-	assert.Equal(*stats.InodesUsed, *imageFsInfo.Inodes-*imageFsInfo.InodesFree)
+	assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *stats.InodesUsed)
 
 	mockCadvisor.AssertExpectations(t)
 }

pkg/kubelet/stats/cri_stats_provider.go

@@ -18,8 +18,16 @@ package stats
 
 import (
 	"fmt"
+	"time"
 
+	"github.com/golang/glog"
+	cadvisorfs "github.com/google/cadvisor/fs"
+
+	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
 	"k8s.io/kubernetes/pkg/kubelet/server/stats"
@@ -57,10 +65,215 @@ func newCRIStatsProvider(
 	}
 }
 
+// ListPodStats returns the stats of all the pod-managed containers.
 func (p *criStatsProvider) ListPodStats() ([]statsapi.PodStats, error) {
-	return nil, fmt.Errorf("not implemented")
+	// Gets node root filesystem information, which will be used to populate
+	// the available and capacity bytes/inodes in container stats.
+	rootFsInfo, err := p.cadvisor.RootFsInfo()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get rootFs info: %v", err)
+	}
+
+	// Creates container map.
+	containerMap := make(map[string]*runtimeapi.Container)
+	containers, err := p.runtimeService.ListContainers(&runtimeapi.ContainerFilter{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list all containers: %v", err)
+	}
+	for _, c := range containers {
+		containerMap[c.Id] = c
+	}
+
+	// Creates pod sandbox map.
+	podSandboxMap := make(map[string]*runtimeapi.PodSandbox)
+	podSandboxes, err := p.runtimeService.ListPodSandbox(&runtimeapi.PodSandboxFilter{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list all pod sandboxes: %v", err)
+	}
+	for _, s := range podSandboxes {
+		podSandboxMap[s.Id] = s
+	}
+
+	// uuidToFsInfo is a map from filesystem UUID to its stats. This will be
+	// used as a cache to avoid querying cAdvisor for the filesystem stats with
+	// the same UUID many times.
+	uuidToFsInfo := make(map[runtimeapi.StorageIdentifier]*cadvisorapiv2.FsInfo)
+
+	// sandboxIDToPodStats is a temporary map from sandbox ID to its pod stats.
+	sandboxIDToPodStats := make(map[string]*statsapi.PodStats)
+
+	resp, err := p.runtimeService.ListContainerStats(&runtimeapi.ListContainerStatsRequest{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list all container stats: %v", err)
+	}
+	for _, stats := range resp.Stats {
+		containerID := stats.Attributes.Id
+		container, found := containerMap[containerID]
+		if !found {
+			glog.Errorf("Unknown id %q in container map.", containerID)
+			continue
+		}
+
+		podSandboxID := container.PodSandboxId
+		podSandbox, found := podSandboxMap[podSandboxID]
+		if !found {
+			glog.Errorf("Unknown id %q in pod sandbox map.", podSandboxID)
+			continue
+		}
+
+		// Creates the stats of the pod (if not created yet) which the
+		// container belongs to.
+		ps, found := sandboxIDToPodStats[podSandboxID]
+		if !found {
+			ps = p.makePodStats(podSandbox)
+			sandboxIDToPodStats[podSandboxID] = ps
+		}
+		ps.Containers = append(ps.Containers, *p.makeContainerStats(stats, container, &rootFsInfo, uuidToFsInfo))
+	}
+
+	result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats))
+	for _, s := range sandboxIDToPodStats {
+		result = append(result, *s)
+	}
+	return result, nil
 }
 
+// ImageFsStats returns the stats of the image filesystem.
 func (p *criStatsProvider) ImageFsStats() (*statsapi.FsStats, error) {
-	return nil, fmt.Errorf("not implemented")
+	resp, err := p.imageService.ImageFsInfo(&runtimeapi.ImageFsInfoRequest{})
+	if err != nil {
+		return nil, err
+	}
+
+	// CRI may return the stats of multiple image filesystems but we only
+	// return the first one.
+	//
+	// TODO(yguo0905): Support returning stats of multiple image filesystems.
+	for _, fs := range resp.ImageFilesystems {
+		s := &statsapi.FsStats{
+			Time:       metav1.NewTime(time.Unix(0, fs.Timestamp)),
+			UsedBytes:  &fs.UsedBytes.Value,
+			InodesUsed: &fs.InodesUsed.Value,
+		}
+		imageFsInfo := p.getFsInfo(fs.StorageId)
+		if imageFsInfo != nil {
+			// The image filesystem UUID is unknown to the local node or
+			// there's an error on retrieving the stats. In these cases, we
+			// omit those stats and return the best-effort partial result. See
+			// https://github.com/kubernetes/heapster/issues/1793.
+			s.AvailableBytes = &imageFsInfo.Available
+			s.CapacityBytes = &imageFsInfo.Capacity
+			s.InodesFree = imageFsInfo.InodesFree
+			s.Inodes = imageFsInfo.Inodes
+		}
+		return s, nil
+	}
+
+	return nil, fmt.Errorf("imageFs information is unavailable")
+}
+
+// getFsInfo returns the information of the filesystem with the specified
+// storageID. If any error occurs, this function logs the error and returns
+// nil.
+func (p *criStatsProvider) getFsInfo(storageID *runtimeapi.StorageIdentifier) *cadvisorapiv2.FsInfo {
+	if storageID == nil {
+		glog.V(2).Infof("Failed to get filesystem info: storageID is nil.")
+		return nil
+	}
+	fsInfo, err := p.cadvisor.GetFsInfoByFsUUID(storageID.Uuid)
+	if err != nil {
+		msg := fmt.Sprintf("Failed to get the info of the filesystem with id %q: %v.", storageID.Uuid, err)
+		if err == cadvisorfs.ErrNoSuchDevice {
+			glog.V(2).Info(msg)
+		} else {
+			glog.Error(msg)
+		}
+		return nil
+	}
+	return &fsInfo
+}
+
+func (p *criStatsProvider) makePodStats(podSandbox *runtimeapi.PodSandbox) *statsapi.PodStats {
+	s := &statsapi.PodStats{
+		PodRef: statsapi.PodReference{
+			Name:      podSandbox.Metadata.Name,
+			UID:       podSandbox.Metadata.Uid,
+			Namespace: podSandbox.Metadata.Namespace,
+		},
+		// The StartTime in the summary API is the pod creation time.
+		StartTime: metav1.NewTime(time.Unix(0, podSandbox.CreatedAt)),
+		// Network stats are not supported by CRI.
+	}
+	podUID := types.UID(s.PodRef.UID)
+	if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found {
+		s.VolumeStats = vstats.Volumes
+	}
+	return s
+}
+
+func (p *criStatsProvider) makeContainerStats(
+	stats *runtimeapi.ContainerStats,
+	container *runtimeapi.Container,
+	rootFsInfo *cadvisorapiv2.FsInfo,
+	uuidToFsInfo map[runtimeapi.StorageIdentifier]*cadvisorapiv2.FsInfo,
+) *statsapi.ContainerStats {
+	result := &statsapi.ContainerStats{
+		Name: stats.Attributes.Metadata.Name,
+		// The StartTime in the summary API is the container creation time.
+		StartTime: metav1.NewTime(time.Unix(0, container.CreatedAt)),
+		CPU:       &statsapi.CPUStats{},
+		Memory:    &statsapi.MemoryStats{},
+		Rootfs:    &statsapi.FsStats{},
+		Logs: &statsapi.FsStats{
+			Time:           metav1.NewTime(rootFsInfo.Timestamp),
+			AvailableBytes: &rootFsInfo.Available,
+			CapacityBytes:  &rootFsInfo.Capacity,
+			InodesFree:     rootFsInfo.InodesFree,
+			Inodes:         rootFsInfo.Inodes,
+			// UsedBytes and InodesUsed are unavailable from CRI stats.
+			//
+			// TODO(yguo0905): Get this information from kubelet and
+			// populate the two fields here.
+		},
+		// UserDefinedMetrics is not supported by CRI.
+	}
+	if stats.Cpu != nil {
+		result.CPU.Time = metav1.NewTime(time.Unix(0, stats.Cpu.Timestamp))
+		if stats.Cpu.UsageCoreNanoSeconds != nil {
+			result.CPU.UsageCoreNanoSeconds = &stats.Cpu.UsageCoreNanoSeconds.Value
+		}
+	}
+	if stats.Memory != nil {
+		result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp))
+		if stats.Memory.WorkingSetBytes != nil {
+			result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value
+		}
+	}
+	if stats.WritableLayer != nil {
+		result.Rootfs.Time = metav1.NewTime(time.Unix(0, stats.WritableLayer.Timestamp))
+		if stats.WritableLayer.UsedBytes != nil {
+			result.Rootfs.UsedBytes = &stats.WritableLayer.UsedBytes.Value
+		}
+		if stats.WritableLayer.InodesUsed != nil {
+			result.Rootfs.InodesUsed = &stats.WritableLayer.InodesUsed.Value
+		}
+	}
+	storageID := stats.WritableLayer.StorageId
+	imageFsInfo, found := uuidToFsInfo[*storageID]
+	if !found {
+		imageFsInfo = p.getFsInfo(storageID)
+		uuidToFsInfo[*storageID] = imageFsInfo
+	}
+	if imageFsInfo != nil {
+		// The image filesystem UUID is unknown to the local node or there's an
+		// error on retrieving the stats. In these cases, we omit those stats
+		// and return the best-effort partial result. See
+		// https://github.com/kubernetes/heapster/issues/1793.
+		result.Rootfs.AvailableBytes = &imageFsInfo.Available
+		result.Rootfs.CapacityBytes = &imageFsInfo.Capacity
+		result.Rootfs.InodesFree = imageFsInfo.InodesFree
+		result.Rootfs.Inodes = imageFsInfo.Inodes
+	}
+
+	return result
 }

pkg/kubelet/stats/cri_stats_provider_test.go

@@ -0,0 +1,274 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stats
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+
+	cadvisorfs "github.com/google/cadvisor/fs"
+	"github.com/stretchr/testify/assert"
+
+	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+	critest "k8s.io/kubernetes/pkg/kubelet/apis/cri/testing"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
+	kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+	kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
+)
+
+func TestCRIListPodStats(t *testing.T) {
+	var (
+		imageFsStorageUUID = "imagefs-storage-uuid"
+		unknownStorageUUID = "unknown-storage-uuid"
+		imageFsInfo        = getTestFsInfo(2000)
+		rootFsInfo         = getTestFsInfo(1000)
+
+		sandbox0        = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns")
+		container0      = makeFakeContainer(sandbox0, "container0-name")
+		containerStats0 = makeFakeContainerStats(container0, imageFsStorageUUID)
+		container1      = makeFakeContainer(sandbox0, "container1-name")
+		containerStats1 = makeFakeContainerStats(container1, unknownStorageUUID)
+
+		sandbox1        = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns")
+		container2      = makeFakeContainer(sandbox1, "container2-name")
+		containerStats2 = makeFakeContainerStats(container2, imageFsStorageUUID)
+	)
+
+	var (
+		mockCadvisor       = new(cadvisortest.Mock)
+		mockRuntimeCache   = new(kubecontainertest.MockRuntimeCache)
+		mockPodManager     = new(kubepodtest.MockManager)
+		resourceAnalyzer   = new(fakeResourceAnalyzer)
+		fakeRuntimeService = critest.NewFakeRuntimeService()
+		fakeImageService   = critest.NewFakeImageService()
+	)
+
+	mockCadvisor.
+		On("RootFsInfo").Return(rootFsInfo, nil).
+		On("GetFsInfoByFsUUID", imageFsStorageUUID).Return(imageFsInfo, nil).
+		On("GetFsInfoByFsUUID", unknownStorageUUID).Return(cadvisorapiv2.FsInfo{}, cadvisorfs.ErrNoSuchDevice)
+	fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{
+		sandbox0, sandbox1,
+	})
+	fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
+		container0, container1, container2,
+	})
+	fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
+		containerStats0, containerStats1, containerStats2,
+	})
+
+	provider := NewCRIStatsProvider(
+		mockCadvisor,
+		resourceAnalyzer,
+		mockPodManager,
+		mockRuntimeCache,
+		fakeRuntimeService,
+		fakeImageService)
+
+	stats, err := provider.ListPodStats()
+	assert := assert.New(t)
+	assert.NoError(err)
+	assert.Equal(2, len(stats))
+
+	podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats)
+	for _, s := range stats {
+		podStatsMap[s.PodRef] = s
+	}
+
+	p0 := podStatsMap[statsapi.PodReference{Name: "sandbox0-name", UID: "sandbox0-uid", Namespace: "sandbox0-ns"}]
+	assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
+	assert.Equal(2, len(p0.Containers))
+
+	containerStatsMap := make(map[string]statsapi.ContainerStats)
+	for _, s := range p0.Containers {
+		containerStatsMap[s.Name] = s
+	}
+	c1 := containerStatsMap["container0-name"]
+	assert.Equal(container0.CreatedAt, c1.StartTime.UnixNano())
+	checkCRICPUAndMemoryStats(assert, c1, containerStats0)
+	checkCRIRootfsStats(assert, c1, containerStats0, &imageFsInfo)
+	checkCRILogsStats(assert, c1, &rootFsInfo)
+	c2 := containerStatsMap["container1-name"]
+	assert.Equal(container1.CreatedAt, c2.StartTime.UnixNano())
+	checkCRICPUAndMemoryStats(assert, c2, containerStats1)
+	checkCRIRootfsStats(assert, c2, containerStats1, nil)
+	checkCRILogsStats(assert, c2, &rootFsInfo)
+
+	p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
+	assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
+	assert.Equal(1, len(p1.Containers))
+
+	c3 := p1.Containers[0]
+	assert.Equal("container2-name", c3.Name)
+	assert.Equal(container2.CreatedAt, c3.StartTime.UnixNano())
+	checkCRICPUAndMemoryStats(assert, c3, containerStats2)
+	checkCRIRootfsStats(assert, c3, containerStats2, &imageFsInfo)
+	checkCRILogsStats(assert, c3, &rootFsInfo)
+
+	mockCadvisor.AssertExpectations(t)
+}
+
+func TestCRIImagesFsStats(t *testing.T) {
+	var (
+		imageFsStorageUUID = "imagefs-storage-uuid"
+		imageFsInfo        = getTestFsInfo(2000)
+		imageFsUsage       = makeFakeImageFsUsage(imageFsStorageUUID)
+	)
+	var (
+		mockCadvisor       = new(cadvisortest.Mock)
+		mockRuntimeCache   = new(kubecontainertest.MockRuntimeCache)
+		mockPodManager     = new(kubepodtest.MockManager)
+		resourceAnalyzer   = new(fakeResourceAnalyzer)
+		fakeRuntimeService = critest.NewFakeRuntimeService()
+		fakeImageService   = critest.NewFakeImageService()
+	)
+
+	mockCadvisor.On("GetFsInfoByFsUUID", imageFsStorageUUID).Return(imageFsInfo, nil)
+	fakeImageService.SetFakeFilesystemUsage([]*runtimeapi.FilesystemUsage{
+		imageFsUsage,
+	})
+
+	provider := NewCRIStatsProvider(
+		mockCadvisor,
+		resourceAnalyzer,
+		mockPodManager,
+		mockRuntimeCache,
+		fakeRuntimeService,
+		fakeImageService)
+
+	stats, err := provider.ImageFsStats()
+	assert := assert.New(t)
+	assert.NoError(err)
+
+	assert.Equal(imageFsUsage.Timestamp, stats.Time.UnixNano())
+	assert.Equal(imageFsInfo.Available, *stats.AvailableBytes)
+	assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes)
+	assert.Equal(imageFsInfo.InodesFree, stats.InodesFree)
+	assert.Equal(imageFsInfo.Inodes, stats.Inodes)
+	assert.Equal(imageFsUsage.UsedBytes.Value, *stats.UsedBytes)
+	assert.Equal(imageFsUsage.InodesUsed.Value, *stats.InodesUsed)
+
+	mockCadvisor.AssertExpectations(t)
+}
+
+func makeFakePodSandbox(name, uid, namespace string) *critest.FakePodSandbox {
+	p := &critest.FakePodSandbox{
+		PodSandboxStatus: runtimeapi.PodSandboxStatus{
+			Metadata: &runtimeapi.PodSandboxMetadata{
+				Name:      name,
+				Uid:       uid,
+				Namespace: namespace,
+			},
+			State:     runtimeapi.PodSandboxState_SANDBOX_READY,
+			CreatedAt: time.Now().UnixNano(),
+		},
+	}
+	p.PodSandboxStatus.Id = critest.BuildSandboxName(p.PodSandboxStatus.Metadata)
+	return p
+}
+
+func makeFakeContainer(sandbox *critest.FakePodSandbox, name string) *critest.FakeContainer {
+	sandboxID := sandbox.PodSandboxStatus.Id
+	c := &critest.FakeContainer{
+		SandboxID: sandboxID,
+		ContainerStatus: runtimeapi.ContainerStatus{
+			Metadata:  &runtimeapi.ContainerMetadata{Name: name},
+			Image:     &runtimeapi.ImageSpec{},
+			ImageRef:  "fake-image-ref",
+			CreatedAt: time.Now().UnixNano(),
+			State:     runtimeapi.ContainerState_CONTAINER_RUNNING,
+		},
+	}
+	c.ContainerStatus.Id = critest.BuildContainerName(c.ContainerStatus.Metadata, sandboxID)
+	return c
+}
+
+func makeFakeContainerStats(container *critest.FakeContainer, imageFsUUID string) *runtimeapi.ContainerStats {
+	return &runtimeapi.ContainerStats{
+		Attributes: &runtimeapi.ContainerAttributes{
+			Id:       container.ContainerStatus.Id,
+			Metadata: container.ContainerStatus.Metadata,
+		},
+		Cpu: &runtimeapi.CpuUsage{
+			Timestamp:            time.Now().UnixNano(),
+			UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: rand.Uint64()},
+		},
+		Memory: &runtimeapi.MemoryUsage{
+			Timestamp:       time.Now().UnixNano(),
+			WorkingSetBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()},
+		},
+		WritableLayer: &runtimeapi.FilesystemUsage{
+			Timestamp:  time.Now().UnixNano(),
+			StorageId:  &runtimeapi.StorageIdentifier{Uuid: imageFsUUID},
+			UsedBytes:  &runtimeapi.UInt64Value{Value: rand.Uint64()},
+			InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64()},
+		},
+	}
+}
+
+func makeFakeImageFsUsage(fsUUID string) *runtimeapi.FilesystemUsage {
+	return &runtimeapi.FilesystemUsage{
+		Timestamp:  time.Now().UnixNano(),
+		StorageId:  &runtimeapi.StorageIdentifier{Uuid: fsUUID},
+		UsedBytes:  &runtimeapi.UInt64Value{Value: rand.Uint64()},
+		InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64()},
+	}
+}
+
+func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats) {
+	assert.Equal(cs.Cpu.Timestamp, actual.CPU.Time.UnixNano())
+	assert.Equal(cs.Cpu.UsageCoreNanoSeconds.Value, *actual.CPU.UsageCoreNanoSeconds)
+	assert.Nil(actual.CPU.UsageNanoCores)
+
+	assert.Equal(cs.Memory.Timestamp, actual.Memory.Time.UnixNano())
+	assert.Nil(actual.Memory.AvailableBytes)
+	assert.Nil(actual.Memory.UsageBytes)
+	assert.Equal(cs.Memory.WorkingSetBytes.Value, *actual.Memory.WorkingSetBytes)
+	assert.Nil(actual.Memory.RSSBytes)
+	assert.Nil(actual.Memory.PageFaults)
+	assert.Nil(actual.Memory.MajorPageFaults)
+}
+
+func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats, imageFsInfo *cadvisorapiv2.FsInfo) {
+	assert.Equal(cs.WritableLayer.Timestamp, actual.Rootfs.Time.UnixNano())
+	if imageFsInfo != nil {
+		assert.Equal(imageFsInfo.Available, *actual.Rootfs.AvailableBytes)
+		assert.Equal(imageFsInfo.Capacity, *actual.Rootfs.CapacityBytes)
+		assert.Equal(*imageFsInfo.InodesFree, *actual.Rootfs.InodesFree)
+		assert.Equal(*imageFsInfo.Inodes, *actual.Rootfs.Inodes)
+	} else {
+		assert.Nil(actual.Rootfs.AvailableBytes)
+		assert.Nil(actual.Rootfs.CapacityBytes)
+		assert.Nil(actual.Rootfs.InodesFree)
+		assert.Nil(actual.Rootfs.Inodes)
+	}
+	assert.Equal(cs.WritableLayer.UsedBytes.Value, *actual.Rootfs.UsedBytes)
+	assert.Equal(cs.WritableLayer.InodesUsed.Value, *actual.Rootfs.InodesUsed)
+}
+
+func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo) {
+	assert.Equal(rootFsInfo.Timestamp, actual.Logs.Time.Time)
+	assert.Equal(rootFsInfo.Available, *actual.Logs.AvailableBytes)
+	assert.Equal(rootFsInfo.Capacity, *actual.Logs.CapacityBytes)
+	assert.Equal(*rootFsInfo.InodesFree, *actual.Logs.InodesFree)
+	assert.Equal(*rootFsInfo.Inodes, *actual.Logs.Inodes)
+	assert.Nil(actual.Logs.UsedBytes)
+	assert.Nil(actual.Logs.InodesUsed)
+}

pkg/kubelet/stats/stats_provider_test.go

@@ -102,16 +102,16 @@ func TestGetCgroupStats(t *testing.T) {
 	checkFsStats(t, "", rootFsInfoSeed, cs.Logs)
 	checkNetworkStats(t, "", containerInfoSeed, ns)
 
-	assert.Equal(cs.Name, cgroupName)
+	assert.Equal(cgroupName, cs.Name)
-	assert.Equal(cs.StartTime, metav1.NewTime(containerInfo.Spec.CreationTime))
+	assert.Equal(metav1.NewTime(containerInfo.Spec.CreationTime), cs.StartTime)
 
-	assert.Equal(cs.Rootfs.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp))
+	assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Rootfs.Time)
-	assert.Equal(*cs.Rootfs.UsedBytes, *containerInfo.Stats[0].Filesystem.BaseUsageBytes)
+	assert.Equal(*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Rootfs.UsedBytes)
-	assert.Equal(*cs.Rootfs.InodesUsed, *containerInfo.Stats[0].Filesystem.InodeUsage)
+	assert.Equal(*containerInfo.Stats[0].Filesystem.InodeUsage, *cs.Rootfs.InodesUsed)
 
-	assert.Equal(cs.Logs.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp))
+	assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), cs.Logs.Time)
-	assert.Equal(*cs.Logs.UsedBytes, *containerInfo.Stats[0].Filesystem.TotalUsageBytes-*containerInfo.Stats[0].Filesystem.BaseUsageBytes)
+	assert.Equal(*containerInfo.Stats[0].Filesystem.TotalUsageBytes-*containerInfo.Stats[0].Filesystem.BaseUsageBytes, *cs.Logs.UsedBytes)
-	assert.Equal(*cs.Logs.InodesUsed, *rootFsInfo.Inodes-*rootFsInfo.InodesFree)
+	assert.Equal(*rootFsInfo.Inodes-*rootFsInfo.InodesFree, *cs.Logs.InodesUsed)
 
 	mockCadvisor.AssertExpectations(t)
 }
@@ -144,9 +144,9 @@ func TestRootFsStats(t *testing.T) {
 
 	checkFsStats(t, "", rootFsInfoSeed, stats)
 
-	assert.Equal(stats.Time, metav1.NewTime(containerInfo.Stats[0].Timestamp))
+	assert.Equal(metav1.NewTime(containerInfo.Stats[0].Timestamp), stats.Time)
-	assert.Equal(*stats.UsedBytes, rootFsInfo.Usage)
+	assert.Equal(rootFsInfo.Usage, *stats.UsedBytes)
-	assert.Equal(*stats.InodesUsed, *rootFsInfo.Inodes-*rootFsInfo.InodesFree)
+	assert.Equal(*rootFsInfo.Inodes-*rootFsInfo.InodesFree, *stats.InodesUsed)
 
 	mockCadvisor.AssertExpectations(t)
 }
@@ -462,6 +462,7 @@ func getTestFsInfo(seed int) cadvisorapiv2.FsInfo {
 		inodesFree = uint64(seed + offsetFsInodesFree)
 	)
 	return cadvisorapiv2.FsInfo{
+		Timestamp:  time.Now(),
 		Device:     "test-device",
 		Mountpoint: "test-mount-point",
 		Capacity:   uint64(seed + offsetFsCapacity),