Add RuntimeStatus in container/runtime.go

pkg/kubelet/container/runtime.go

@@ -70,8 +70,9 @@ type Runtime interface {
 	// This may be different from the runtime engine's version.
 	// TODO(random-liu): We should fold this into Version()
 	APIVersion() (Version, error)
-	// Status returns error if the runtime is unhealthy; nil otherwise.
+	// Status returns the status of the runtime. An error is returned if the Status
-	Status() error
+	// function itself fails, nil otherwise.
+	Status() (*RuntimeStatus, error)
 	// GetPods returns a list of containers grouped by pods. The boolean parameter
 	// specifies whether the runtime returns all containers including those already
 	// exited and dead containers (used for garbage collection).
@@ -446,6 +447,59 @@ type VolumeInfo struct {
 
 type VolumeMap map[string]VolumeInfo
 
+// RuntimeConditionType is the types of required runtime conditions.
+type RuntimeConditionType string
+
+const (
+	// RuntimeReady means the runtime is up and ready to accept basic containers.
+	RuntimeReady RuntimeConditionType = "RuntimeReady"
+	// NetworkReady means the runtime network is up and ready to accept containers which require network.
+	NetworkReady RuntimeConditionType = "NetworkReady"
+)
+
+// RuntimeStatus contains the status of the runtime.
+type RuntimeStatus struct {
+	// Conditions is an array of current observed runtime conditions.
+	Conditions []RuntimeCondition
+}
+
+// GetRuntimeCondition gets a specified runtime condition from the runtime status.
+func (r *RuntimeStatus) GetRuntimeCondition(t RuntimeConditionType) *RuntimeCondition {
+	for i := range r.Conditions {
+		c := &r.Conditions[i]
+		if c.Type == t {
+			return c
+		}
+	}
+	return nil
+}
+
+// String formats the runtime status into human readable string.
+func (s *RuntimeStatus) String() string {
+	var ss []string
+	for _, c := range s.Conditions {
+		ss = append(ss, c.String())
+	}
+	return fmt.Sprintf("Runtime Conditions: %s", strings.Join(ss, ", "))
+}
+
+// RuntimeCondition contains condition information for the runtime.
+type RuntimeCondition struct {
+	// Type of runtime condition.
+	Type RuntimeConditionType
+	// Status of the condition, one of true/false.
+	Status bool
+	// Reason is brief reason for the condition's last transition.
+	Reason string
+	// Message is human readable message indicating details about last transition.
+	Message string
+}
+
+// String formats the runtime condition into human readable string.
+func (c *RuntimeCondition) String() string {
+	return fmt.Sprintf("%s=%t reason:%s message:%s", c.Type, c.Status, c.Reason, c.Message)
+}
+
 type Pods []*Pod
 
 // FindPodByID finds and returns a pod in the pod list by UID. It will return an empty pod

pkg/kubelet/container/testing/fake_runtime.go

@@ -50,6 +50,7 @@ type FakeRuntime struct {
 	KilledPods        []string
 	StartedContainers []string
 	KilledContainers  []string
+	RuntimeStatus     *RuntimeStatus
 	VersionInfo       string
 	APIVersionInfo    string
 	RuntimeType       string
@@ -137,6 +138,7 @@ func (f *FakeRuntime) ClearCalls() {
 	f.KilledPods = []string{}
 	f.StartedContainers = []string{}
 	f.KilledContainers = []string{}
+	f.RuntimeStatus = nil
 	f.VersionInfo = ""
 	f.RuntimeType = ""
 	f.Err = nil
@@ -207,12 +209,12 @@ func (f *FakeRuntime) APIVersion() (Version, error) {
 	return &FakeVersion{Version: f.APIVersionInfo}, f.Err
 }
 
-func (f *FakeRuntime) Status() error {
+func (f *FakeRuntime) Status() (*RuntimeStatus, error) {
 	f.Lock()
 	defer f.Unlock()
 
 	f.CalledFunctions = append(f.CalledFunctions, "Status")
-	return f.StatusErr
+	return f.RuntimeStatus, f.StatusErr
 }
 
 func (f *FakeRuntime) GetPods(all bool) ([]*Pod, error) {

pkg/kubelet/container/testing/runtime_mock.go

@@ -54,9 +54,9 @@ func (r *Mock) APIVersion() (Version, error) {
 	return args.Get(0).(Version), args.Error(1)
 }
 
-func (r *Mock) Status() error {
+func (r *Mock) Status() (*RuntimeStatus, error) {
 	args := r.Called()
-	return args.Error(0)
+	return args.Get(0).(*RuntimeStatus), args.Error(0)
 }
 
 func (r *Mock) GetPods(all bool) ([]*Pod, error) {

pkg/kubelet/dockershim/docker_service.go

@@ -60,7 +60,7 @@ const (
 	sandboxIDLabelKey           = "io.kubernetes.sandbox.id"
 )
 
-// NetworkPluginArgs is the subset of kubelet runtime args we pass
+// NetworkPluginSettings is the subset of kubelet runtime args we pass
 // to the container runtime shim so it can probe for network plugins.
 // In the future we will feed these directly to a standalone container
 // runtime process.

pkg/kubelet/dockertools/docker_manager.go

@@ -1107,8 +1107,8 @@ func (dm *DockerManager) APIVersion() (kubecontainer.Version, error) {
 // Now we do this by checking whether:
 // 1) `docker version` works
 // 2) docker version is compatible with minimum requirement
-func (dm *DockerManager) Status() error {
+func (dm *DockerManager) Status() (*kubecontainer.RuntimeStatus, error) {
-	return dm.checkVersionCompatibility()
+	return nil, dm.checkVersionCompatibility()
 }
 
 func (dm *DockerManager) checkVersionCompatibility() error {

pkg/kubelet/kubelet.go

@@ -1944,10 +1944,34 @@ func (kl *Kubelet) PLEGHealthCheck() (bool, error) {
 // and returns an error if the status check fails.  If the status check is OK,
 // update the container runtime uptime in the kubelet runtimeState.
 func (kl *Kubelet) updateRuntimeUp() {
-	if err := kl.containerRuntime.Status(); err != nil {
+	s, err := kl.containerRuntime.Status()
+	if err != nil {
 		glog.Errorf("Container runtime sanity check failed: %v", err)
 		return
 	}
+	// Only check specific conditions when runtime integration type is cri,
+	// because the old integration doesn't populate any runtime condition.
+	// TODO(random-liu): Add runtime error in runtimeState, and update it
+	// when when runtime is not ready, so that the information in RuntimeReady
+	// condition will be propagated to NodeReady condition.
+	// TODO(random-liu): Update network state according to NetworkReady runtime
+	// condition once it is implemented in dockershim.
+	if kl.kubeletConfiguration.ExperimentalRuntimeIntegrationType == "cri" {
+		if s == nil {
+			glog.Errorf("Container runtime status is nil")
+			return
+		}
+		runtimeReady := s.GetRuntimeCondition(kubecontainer.RuntimeReady)
+		// Periodically log the whole runtime status for debugging.
+		// TODO(random-liu): Consider to send node event when optional
+		// condition is unmet.
+		glog.V(4).Infof("Container runtime status: %v", s)
+		// If RuntimeReady is not set or is false, report an error.
+		if runtimeReady == nil || !runtimeReady.Status {
+			glog.Errorf("Container runtime not ready: %v", runtimeReady)
+			return
+		}
+	}
 	kl.oneTimeInitializer.Do(kl.initializeRuntimeDependentModules)
 	kl.runtimeState.setRuntimeSync(kl.clock.Now())
 }

pkg/kubelet/kubelet_node_status_test.go

@@ -865,6 +865,38 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
 	clock.SetTime(time.Now())
 	kubelet.updateRuntimeUp()
 	checkNodeStatus(api.ConditionFalse, "KubeletNotReady", downMessage)
+
+	// Test cri integration.
+	kubelet.kubeletConfiguration.ExperimentalRuntimeIntegrationType = "cri"
+	fakeRuntime.StatusErr = nil
+
+	// Should report node not ready if runtime status is nil.
+	fakeRuntime.RuntimeStatus = nil
+	kubelet.updateRuntimeUp()
+	checkNodeStatus(api.ConditionFalse, "KubeletNotReady", downMessage)
+
+	// Should report node not ready if runtime status is empty.
+	fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{}
+	kubelet.updateRuntimeUp()
+	checkNodeStatus(api.ConditionFalse, "KubeletNotReady", downMessage)
+
+	// Should report node not ready if RuntimeReady is false.
+	fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{
+		Conditions: []kubecontainer.RuntimeCondition{
+			{Type: kubecontainer.RuntimeReady, Status: false},
+		},
+	}
+	kubelet.updateRuntimeUp()
+	checkNodeStatus(api.ConditionFalse, "KubeletNotReady", downMessage)
+
+	// Should report node ready if RuntimeReady is true.
+	fakeRuntime.RuntimeStatus = &kubecontainer.RuntimeStatus{
+		Conditions: []kubecontainer.RuntimeCondition{
+			{Type: kubecontainer.RuntimeReady, Status: true},
+		},
+	}
+	kubelet.updateRuntimeUp()
+	checkNodeStatus(api.ConditionTrue, "KubeletReady", readyMessage)
 }
 
 func TestUpdateNodeStatusError(t *testing.T) {

pkg/kubelet/kuberuntime/helpers.go

@@ -217,13 +217,16 @@ func buildPodLogsDirectory(podUID types.UID) string {
 	return filepath.Join(podLogsRootDirectory, string(podUID))
 }
 
-// getRuntimeCondition gets specified runtime condition from the runtime status.
+// toKubeRuntimeStatus converts the runtimeApi.RuntimeStatus to kubecontainer.RuntimeStatus.
-func getRuntimeCondition(status *runtimeApi.RuntimeStatus, t string) *runtimeApi.RuntimeCondition {
+func toKubeRuntimeStatus(status *runtimeApi.RuntimeStatus) *kubecontainer.RuntimeStatus {
-	conditions := status.GetConditions()
+	conditions := []kubecontainer.RuntimeCondition{}
-	for _, condition := range conditions {
+	for _, c := range status.GetConditions() {
-		if condition.GetType() == t {
+		conditions = append(conditions, kubecontainer.RuntimeCondition{
-			return condition
+			Type:    kubecontainer.RuntimeConditionType(c.GetType()),
-		}
+			Status:  c.GetStatus(),
+			Reason:  c.GetReason(),
+			Message: c.GetMessage(),
+		})
 	}
-	return nil
+	return &kubecontainer.RuntimeStatus{Conditions: conditions}
 }

pkg/kubelet/kuberuntime/kuberuntime_manager.go

@@ -268,25 +268,14 @@ func (m *kubeGenericRuntimeManager) APIVersion() (kubecontainer.Version, error)
 	return newRuntimeVersion(typedVersion.GetRuntimeApiVersion())
 }
 
-// Status returns error if the runtime is unhealthy; nil otherwise.
+// Status returns the status of the runtime. An error is returned if the Status
-// TODO(random-liu): Change the Status in runtime interface to return runtime status.
+// function itself fails, nil otherwise.
-// TODO(random-liu): Add unit test for this function after addressing the TODO above.
+func (m *kubeGenericRuntimeManager) Status() (*kubecontainer.RuntimeStatus, error) {
-func (m *kubeGenericRuntimeManager) Status() error {
 	status, err := m.runtimeService.Status()
 	if err != nil {
-		return fmt.Errorf("failed to checkout runtime status: %v", err)
+		return nil, err
 	}
-	networkReady := getRuntimeCondition(status, runtimeApi.NetworkReady)
+	return toKubeRuntimeStatus(status), nil
-	if networkReady == nil || !networkReady.GetStatus() {
-		return fmt.Errorf("runtime network not ready: reason: %q, message: %q",
-			networkReady.GetReason(), networkReady.GetMessage())
-	}
-	runtimeReady := getRuntimeCondition(status, runtimeApi.RuntimeReady)
-	if runtimeReady == nil || !runtimeReady.GetStatus() {
-		return fmt.Errorf("runtime not ready: reason: %q, message: %q",
-			runtimeReady.GetReason(), runtimeReady.GetMessage())
-	}
-	return nil
 }
 
 // GetPods returns a list of containers grouped by pods. The boolean parameter

pkg/kubelet/rkt/rkt.go

@@ -1701,8 +1701,8 @@ func (r *Runtime) APIVersion() (kubecontainer.Version, error) {
 }
 
 // Status returns error if rkt is unhealthy, nil otherwise.
-func (r *Runtime) Status() error {
+func (r *Runtime) Status() (*kubecontainer.RuntimeStatus, error) {
-	return r.checkVersion(minimumRktBinVersion, minimumRktApiVersion, minimumSystemdVersion)
+	return nil, r.checkVersion(minimumRktBinVersion, minimumRktApiVersion, minimumSystemdVersion)
 }
 
 // SyncPod syncs the running pod to match the specified desired pod.