Kubelet: Add /stats/container endpoint.

This endpoint exposes container stats for all raw containers on the
machine. The addition is backwards compatible.

pkg/kubelet/kubelet.go

@@ -2200,9 +2200,19 @@ func (kl *Kubelet) GetContainerInfo(podFullName string, uid types.UID, container
 	return &ci, nil
 }
 
-// GetRootInfo returns stats (from Cadvisor) of current machine (root container).
+// Returns stats (from Cadvisor) for a non-Kubernetes container.
-func (kl *Kubelet) GetRootInfo(req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) {
+func (kl *Kubelet) GetRawContainerInfo(containerName string, req *cadvisorApi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorApi.ContainerInfo, error) {
-	return kl.cadvisor.ContainerInfo("/", req)
+	if subcontainers {
+		return kl.cadvisor.SubcontainerInfo(containerName, req)
+	} else {
+		containerInfo, err := kl.cadvisor.ContainerInfo(containerName, req)
+		if err != nil {
+			return nil, err
+		}
+		return map[string]*cadvisorApi.ContainerInfo{
+			containerInfo.Name: containerInfo,
+		}, nil
+	}
 }
 
 // GetCachedMachineInfo assumes that the machine info can't change without a reboot

pkg/kubelet/kubelet_test.go

@@ -1441,7 +1441,7 @@ func TestGetContainerInfo(t *testing.T) {
 	mockCadvisor.AssertExpectations(t)
 }
 
-func TestGetRootInfo(t *testing.T) {
+func TestGetRawContainerInfoRoot(t *testing.T) {
 	containerPath := "/"
 	containerInfo := &cadvisorApi.ContainerInfo{
 		ContainerReference: cadvisorApi.ContainerReference{
@@ -1459,14 +1459,48 @@ func TestGetRootInfo(t *testing.T) {
 		cadvisor:     mockCadvisor,
 	}
 
-	// If the container name is an empty string, then it means the root container.
+	_, err := kubelet.GetRawContainerInfo(containerPath, cadvisorReq, false)
-	_, err := kubelet.GetRootInfo(cadvisorReq)
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
 	mockCadvisor.AssertExpectations(t)
 }
 
+func TestGetRawContainerInfoSubcontainers(t *testing.T) {
+	containerPath := "/kubelet"
+	containerInfo := map[string]*cadvisorApi.ContainerInfo{
+		containerPath: {
+			ContainerReference: cadvisorApi.ContainerReference{
+				Name: containerPath,
+			},
+		},
+		"/kubelet/sub": {
+			ContainerReference: cadvisorApi.ContainerReference{
+				Name: "/kubelet/sub",
+			},
+		},
+	}
+	fakeDocker := dockertools.FakeDockerClient{}
+
+	mockCadvisor := &cadvisor.Mock{}
+	cadvisorReq := &cadvisorApi.ContainerInfoRequest{}
+	mockCadvisor.On("SubcontainerInfo", containerPath, cadvisorReq).Return(containerInfo, nil)
+
+	kubelet := Kubelet{
+		dockerClient: &fakeDocker,
+		cadvisor:     mockCadvisor,
+	}
+
+	result, err := kubelet.GetRawContainerInfo(containerPath, cadvisorReq, true)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if len(result) != 2 {
+		t.Errorf("Expected 2 elements, received: %+v", result)
+	}
+	mockCadvisor.AssertExpectations(t)
+}
+
 func TestGetContainerInfoWhenCadvisorFailed(t *testing.T) {
 	containerID := "ab2cdf"
 

pkg/kubelet/server.go

@@ -99,8 +99,8 @@ func ListenAndServeKubeletReadOnlyServer(host HostInterface, address net.IP, por
 // For testablitiy.
 type HostInterface interface {
 	GetContainerInfo(podFullName string, uid types.UID, containerName string, req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error)
-	GetRootInfo(req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error)
 	GetContainerRuntimeVersion() (kubecontainer.Version, error)
+	GetRawContainerInfo(containerName string, req *cadvisorApi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorApi.ContainerInfo, error)
 	GetCachedMachineInfo() (*cadvisorApi.MachineInfo, error)
 	GetPods() []*api.Pod
 	GetPodByName(namespace, name string) (*api.Pod, bool)
@@ -634,26 +634,68 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	s.mux.ServeHTTP(w, req)
 }
 
+type StatsRequest struct {
+	// The name of the container for which to request stats.
+	// Default: /
+	ContainerName string `json:"containerName,omitempty"`
+
+	// Max number of stats to return.
+	// If start and end time are specified this limit is ignored.
+	// Default: 60
+	NumStats int `json:"num_stats,omitempty"`
+
+	// Start time for which to query information.
+	// If ommitted, the beginning of time is assumed.
+	Start time.Time `json:"start,omitempty"`
+
+	// End time for which to query information.
+	// If ommitted, current time is assumed.
+	End time.Time `json:"end,omitempty"`
+
+	// Whether to also include information from subcontainers.
+	// Default: false.
+	Subcontainers bool `json:"subcontainers,omitempty"`
+}
+
 // serveStats implements stats logic.
 func (s *Server) serveStats(w http.ResponseWriter, req *http.Request) {
-	// /stats/<pod name>/<container name> or /stats/<namespace>/<pod name>/<uid>/<container name>
+	// Stats requests are in the following forms:
+	//
+	// /stats/                                              : Root container stats
+	// /stats/container/                                    : Non-Kubernetes container stats (returns a map)
+	// /stats/<pod name>/<container name>                   : Stats for Kubernetes pod/container
+	// /stats/<namespace>/<pod name>/<uid>/<container name> : Stats for Kubernetes namespace/pod/uid/container
 	components := strings.Split(strings.TrimPrefix(path.Clean(req.URL.Path), "/"), "/")
-	var stats *cadvisorApi.ContainerInfo
+	var stats interface{}
 	var err error
-	query := cadvisorApi.DefaultContainerInfoRequest()
+	var query StatsRequest
+	query.NumStats = 60
+
 	err = json.NewDecoder(req.Body).Decode(&query)
 	if err != nil && err != io.EOF {
 		s.error(w, err)
 		return
 	}
+	cadvisorRequest := cadvisorApi.ContainerInfoRequest{
+		NumStats: query.NumStats,
+		Start:    query.Start,
+		End:      query.End,
+	}
+
 	switch len(components) {
 	case 1:
-		// Machine stats
+		// Root container stats.
-		stats, err = s.host.GetRootInfo(&query)
+		var statsMap map[string]*cadvisorApi.ContainerInfo
+		statsMap, err = s.host.GetRawContainerInfo("/", &cadvisorRequest, false)
+		stats = statsMap["/"]
 	case 2:
-		// pod stats
+		// Non-Kubernetes container stats.
-		// TODO(monnand) Implement this
+		if components[1] != "container" {
-		err = errors.New("pod level status currently unimplemented")
+			http.Error(w, fmt.Sprintf("unknown stats request type %q", components[1]), http.StatusNotFound)
+			return
+		}
+		containerName := path.Join("/", query.ContainerName)
+		stats, err = s.host.GetRawContainerInfo(containerName, &cadvisorRequest, query.Subcontainers)
 	case 3:
 		// Backward compatibility without uid information, does not support namespace
 		pod, ok := s.host.GetPodByName(api.NamespaceDefault, components[1])
@@ -661,16 +703,16 @@ func (s *Server) serveStats(w http.ResponseWriter, req *http.Request) {
 			http.Error(w, "Pod does not exist", http.StatusNotFound)
 			return
 		}
-		stats, err = s.host.GetContainerInfo(kubecontainer.GetPodFullName(pod), "", components[2], &query)
+		stats, err = s.host.GetContainerInfo(kubecontainer.GetPodFullName(pod), "", components[2], &cadvisorRequest)
 	case 5:
 		pod, ok := s.host.GetPodByName(components[1], components[2])
 		if !ok {
 			http.Error(w, "Pod does not exist", http.StatusNotFound)
 			return
 		}
-		stats, err = s.host.GetContainerInfo(kubecontainer.GetPodFullName(pod), types.UID(components[3]), components[4], &query)
+		stats, err = s.host.GetContainerInfo(kubecontainer.GetPodFullName(pod), types.UID(components[3]), components[4], &cadvisorRequest)
 	default:
-		http.Error(w, "unknown resource.", http.StatusNotFound)
+		http.Error(w, fmt.Sprintf("Unknown resource: %v", components), http.StatusNotFound)
 		return
 	}
 	switch err {

pkg/kubelet/server_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package kubelet
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -44,7 +45,7 @@ type fakeKubelet struct {
 	podByNameFunc                      func(namespace, name string) (*api.Pod, bool)
 	statusFunc                         func(name string) (api.PodStatus, error)
 	containerInfoFunc                  func(podFullName string, uid types.UID, containerName string, req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error)
-	rootInfoFunc                       func(query *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error)
+	rawInfoFunc                        func(query *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error)
 	machineInfoFunc                    func() (*cadvisorApi.MachineInfo, error)
 	podsFunc                           func() []*api.Pod
 	logFunc                            func(w http.ResponseWriter, req *http.Request)
@@ -69,8 +70,8 @@ func (fk *fakeKubelet) GetContainerInfo(podFullName string, uid types.UID, conta
 	return fk.containerInfoFunc(podFullName, uid, containerName, req)
 }
 
-func (fk *fakeKubelet) GetRootInfo(req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) {
+func (fk *fakeKubelet) GetRawContainerInfo(containerName string, req *cadvisorApi.ContainerInfoRequest, subcontainers bool) (map[string]*cadvisorApi.ContainerInfo, error) {
-	return fk.rootInfoFunc(req)
+	return fk.rawInfoFunc(req)
 }
 
 func (fk *fakeKubelet) GetContainerRuntimeVersion() (kubecontainer.Version, error) {
@@ -269,9 +270,15 @@ func TestContainerNotFound(t *testing.T) {
 
 func TestRootInfo(t *testing.T) {
 	fw := newServerTest()
-	expectedInfo := &cadvisorApi.ContainerInfo{}
+	expectedInfo := &cadvisorApi.ContainerInfo{
-	fw.fakeKubelet.rootInfoFunc = func(req *cadvisorApi.ContainerInfoRequest) (*cadvisorApi.ContainerInfo, error) {
+		ContainerReference: cadvisorApi.ContainerReference{
-		return expectedInfo, nil
+			Name: "/",
+		},
+	}
+	fw.fakeKubelet.rawInfoFunc = func(req *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) {
+		return map[string]*cadvisorApi.ContainerInfo{
+			expectedInfo.Name: expectedInfo,
+		}, nil
 	}
 
 	resp, err := http.Get(fw.testHTTPServer.URL + "/stats")
@@ -285,7 +292,52 @@ func TestRootInfo(t *testing.T) {
 		t.Fatalf("received invalid json data: %v", err)
 	}
 	if !receivedInfo.Eq(expectedInfo) {
-		t.Errorf("received wrong data: %#v", receivedInfo)
+		t.Errorf("received wrong data: %#v, expected %#v", receivedInfo, expectedInfo)
+	}
+}
+
+func TestSubcontainerContainerInfo(t *testing.T) {
+	fw := newServerTest()
+	const kubeletContainer = "/kubelet"
+	const kubeletSubContainer = "/kubelet/sub"
+	expectedInfo := map[string]*cadvisorApi.ContainerInfo{
+		kubeletContainer: {
+			ContainerReference: cadvisorApi.ContainerReference{
+				Name: kubeletContainer,
+			},
+		},
+		kubeletSubContainer: {
+			ContainerReference: cadvisorApi.ContainerReference{
+				Name: kubeletSubContainer,
+			},
+		},
+	}
+	fw.fakeKubelet.rawInfoFunc = func(req *cadvisorApi.ContainerInfoRequest) (map[string]*cadvisorApi.ContainerInfo, error) {
+		return expectedInfo, nil
+	}
+
+	request := fmt.Sprintf("{\"containerName\":%q, \"subcontainers\": true}", kubeletContainer)
+	resp, err := http.Post(fw.testHTTPServer.URL+"/stats/container", "application/json", bytes.NewBuffer([]byte(request)))
+	if err != nil {
+		t.Fatalf("Got error GETing: %v", err)
+	}
+	defer resp.Body.Close()
+	var receivedInfo map[string]*cadvisorApi.ContainerInfo
+	err = json.NewDecoder(resp.Body).Decode(&receivedInfo)
+	if err != nil {
+		t.Fatalf("Received invalid json data: %v", err)
+	}
+	if len(receivedInfo) != len(expectedInfo) {
+		t.Errorf("Received wrong data: %#v, expected %#v", receivedInfo, expectedInfo)
+	}
+
+	for _, containerName := range []string{kubeletContainer, kubeletSubContainer} {
+		if _, ok := receivedInfo[containerName]; !ok {
+			t.Errorf("Expected container %q to be present in result: %#v", containerName, receivedInfo)
+		}
+		if !receivedInfo[containerName].Eq(expectedInfo[containerName]) {
+			t.Errorf("Invalid result for %q: Expected %#v, received %#v", containerName, expectedInfo[containerName], receivedInfo[containerName])
+		}
 	}
 }
 
@@ -426,29 +478,6 @@ func TestServeRunInContainerWithUID(t *testing.T) {
 	}
 }
 
-// TODO: fix me when pod level stats get implemented
-func TestPodsInfo(t *testing.T) {
-	fw := newServerTest()
-
-	resp, err := http.Get(fw.testHTTPServer.URL + "/stats/goodpod")
-	if err != nil {
-		t.Fatalf("Got error GETing: %v", err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusInternalServerError {
-		t.Errorf("expected status code %d, got %d", http.StatusInternalServerError, resp.StatusCode)
-	}
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		// copying the response body did not work
-		t.Fatalf("Cannot copy resp: %#v", err)
-	}
-	result := string(body)
-	if !strings.Contains(result, "pod level status currently unimplemented") {
-		t.Errorf("expected body contains pod level status currently unimplemented, got %s", result)
-	}
-}
-
 func TestHealthCheck(t *testing.T) {
 	fw := newServerTest()
 	fw.fakeKubelet.containerVersionFunc = func() (kubecontainer.Version, error) {