Merge pull request #4376 from dchen1107/podstatus

Fix podstatus issue caused by docker's resource temporarily unavailable issue

cluster/saltbase/salt/docker/docker-defaults

@@ -3,3 +3,4 @@ DOCKER_OPTS=""
 DOCKER_OPTS="${DOCKER_OPTS} {{grains.docker_opts}}"
 {% endif %}
 DOCKER_OPTS="${DOCKER_OPTS} --bridge cbr0 --iptables=false --ip-masq=false -r=false"
+DOCKER_NOFILE=1000000

pkg/kubelet/kubelet.go

@@ -1463,10 +1463,6 @@ func (kl *Kubelet) GetPodByName(namespace, name string) (*api.BoundPod, bool) {
 
 // getPhase returns the phase of a pod given its container info.
 func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
-	if info == nil {
-		return api.PodPending
-	}
-
 	running := 0
 	waiting := 0
 	stopped := 0
@@ -1495,6 +1491,7 @@ func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
 	}
 	switch {
 	case waiting > 0:
+		glog.V(5).Infof("pod waiting > 0, pending")
 		// One or more containers has not been started
 		return api.PodPending
 	case running > 0 && unknown == 0:
@@ -1521,6 +1518,7 @@ func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
 		// and in the process of restarting
 		return api.PodRunning
 	default:
+		glog.V(5).Infof("pod default case, pending")
 		return api.PodPending
 	}
 }
@@ -1569,10 +1567,19 @@ func (kl *Kubelet) GetPodStatus(podFullName string, uid types.UID) (api.PodStatu
 	info, err := dockertools.GetDockerPodInfo(kl.dockerClient, spec, podFullName, uid)
 
 	if err != nil {
-		glog.Infof("Query docker container info failed with error: %v", err)
+		// Error handling
+		glog.Infof("Query docker container info for pod %s failed with error (%v)", podFullName, err)
+		if strings.Contains(err.Error(), "resource temporarily unavailable") {
+			// Leave upstream layer to decide what to do
 			return podStatus, err
+		} else {
+			podStatus.Phase = api.PodPending
+			podStatus.Message = fmt.Sprintf("Query docker container info failed with error (%v)", err)
+			return podStatus, nil
+		}
 	}
 
+	// Assume info is ready to process
 	podStatus.Phase = getPhase(&spec, info)
 	for _, c := range spec.Containers {
 		containerStatus := info[c.Name]
@@ -1589,7 +1596,7 @@ func (kl *Kubelet) GetPodStatus(podFullName string, uid types.UID) (api.PodStatu
 	// TODO(dchen1107): Change Info to list from map
 	podStatus.Info = info
 
-	return podStatus, err
+	return podStatus, nil
 }
 
 // Returns logs of current machine.

pkg/master/pod_cache.go

@@ -148,7 +148,9 @@ func (p *PodCache) updatePodStatus(pod *api.Pod) error {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 	// Map accesses must be locked.
+	if err == nil {
 		p.podStatus[objKey{pod.Namespace, pod.Name}] = newStatus
+	}
 
 	return err
 }
@@ -187,13 +189,11 @@ func (p *PodCache) computePodStatus(pod *api.Pod) (api.PodStatus, error) {
 	}
 
 	result, err := p.containerInfo.GetPodStatus(pod.Status.Host, pod.Namespace, pod.Name)
-	newStatus.HostIP = nodeStatus.HostIP
 
 	if err != nil {
-		glog.Errorf("error getting pod status: %v, setting status to unknown", err)
-		newStatus.Phase = api.PodUnknown
-		newStatus.Conditions = append(newStatus.Conditions, pod.Status.Conditions...)
+		glog.Infof("error getting pod %s status: %v, retry later", pod.Name, err)
 	} else {
+		newStatus.HostIP = nodeStatus.HostIP
 		newStatus.Info = result.Status.Info
 		newStatus.PodIP = result.Status.PodIP
 		if newStatus.Info == nil {