Install and use crictl in gce kube-up.sh

Signed-off-by: Lantao Liu <lantaol@google.com>

cluster/gce/gci/configure-helper.sh

@@ -2089,10 +2089,16 @@ function start-fluentd-resource-update {
   wait-for-apiserver-and-update-fluentd &
 }
 
-# Update {{ container-runtime }} with actual container runtime name.
+# Update {{ container-runtime }} with actual container runtime name,
+# and {{ container-runtime-endpoint }} with actual container runtime
+# endpoint.
 function update-container-runtime {
-  local -r configmap_yaml="$1"
-  sed -i -e "s@{{ *container_runtime *}}@${CONTAINER_RUNTIME_NAME:-docker}@g" "${configmap_yaml}"
+  local -r file="$1"
+  local -r container_runtime_endpoint="${CONTAINER_RUNTIME_ENDPOINT:-unix:///var/run/dockershim.sock}"
+  sed -i \
+    -e "s@{{ *container_runtime *}}@${CONTAINER_RUNTIME_NAME:-docker}@g" \
+    -e "s@{{ *container_runtime_endpoint *}}@${container_runtime_endpoint#unix://}@g" \
+    "${file}"
 }
 
 # Remove configuration in yaml file if node journal is not enabled.
@@ -2375,8 +2381,9 @@ EOF
 # Starts an image-puller - used in test clusters.
 function start-image-puller {
   echo "Start image-puller"
-  cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/e2e-image-puller.manifest" \
-    /etc/kubernetes/manifests/
+  local -r e2e_image_puller_manifest="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/e2e-image-puller.manifest"
+  update-container-runtime "${e2e_image_puller_manifest}"
+  cp "${e2e_image_puller_manifest}" /etc/kubernetes/manifests/
 }
 
 # Setups manifests for ingress controller and gce-specific policies for service controller.
@@ -2590,4 +2597,4 @@ if [[ "$#" -eq 1 && "${1}" == "--source-only" ]]; then
    :
 else
    main "${@}"
-fi
+fi

cluster/gce/gci/configure.sh

@@ -28,6 +28,8 @@ DEFAULT_CNI_VERSION="v0.6.0"
 DEFAULT_CNI_SHA1="d595d3ded6499a64e8dac02466e2f5f2ce257c9f"
 DEFAULT_NPD_VERSION="v0.4.1"
 DEFAULT_NPD_SHA1="a57a3fe64cab8a18ec654f5cef0aec59dae62568"
+DEFAULT_CRICTL_VERSION="v1.0.0-beta.1"
+DEFAULT_CRICTL_SHA1="6816982ea1b83506945ce02949199171fee17b0b"
 DEFAULT_MOUNTER_TAR_SHA="8003b798cf33c7f91320cd6ee5cec4fa22244571"
 ###
 
@@ -234,6 +236,34 @@ function install-cni-binaries {
   rm -f "${KUBE_HOME}/${cni_tar}"
 }
 
+# Install crictl binary.
+function install-crictl {
+  if [[ -n "${CRICTL_VERSION:-}" ]]; then
+    local -r crictl_version="${CRICTL_VERSION}"
+    local -r crictl_sha1="${CRICTL_TAR_HASH}"
+  else
+    local -r crictl_version="${DEFAULT_CRICTL_VERSION}"
+    local -r crictl_sha1="${DEFAULT_CRICTL_SHA1}"
+  fi
+  local -r crictl="crictl-${crictl_version}-linux-amd64"
+
+  if is-preloaded "${crictl}" "${crictl_sha1}"; then
+    echo "crictl is preloaded"
+    return
+  fi
+
+  echo "Downloading crictl"
+  local -r crictl_path="https://storage.googleapis.com/kubernetes-release/crictl"
+  download-or-bust "${crictl_sha1}" "${crictl_path}/${crictl}"
+  mv "${KUBE_HOME}/${crictl}" "${KUBE_BIN}/crictl"
+  chmod a+x "${KUBE_BIN}/crictl"
+
+  # Create crictl config file.
+  cat > /etc/crictl.yaml <<EOF
+runtime-endpoint: ${CONTAINER_RUNTIME_ENDPOINT:-unix:///var/run/dockershim.sock}
+EOF
+}
+
 function install-kube-manifests {
   # Put kube-system pods manifests in ${KUBE_HOME}/kube-manifests/.
   local dst_dir="${KUBE_HOME}/kube-manifests"
@@ -370,6 +400,9 @@ function install-kube-binary-config {
     remount-flexvolume-directory "${VOLUME_PLUGIN_DIR}"
   fi
 
+  # Install crictl on each node.
+  install-crictl
+
   # Clean up.
   rm -rf "${KUBE_HOME}/kubernetes"
   rm -f "${KUBE_HOME}/${server_binary_tar}"

cluster/gce/gci/health-monitor.sh

@@ -24,11 +24,25 @@ set -o pipefail
 
 # We simply kill the process when there is a failure. Another systemd service will
 # automatically restart the process.
-function docker_monitoring {
-  while [ 1 ]; do
-    if ! timeout 60 docker ps > /dev/null; then
-      echo "Docker daemon failed!"
-      pkill docker
+function container_runtime_monitoring {
+  # Container runtime startup takes time. Make initial attempts before starting
+  # killing the container runtime.
+  local -r max_attempts=5
+  local attempt=1
+  local -r crictl="${KUBE_HOME}/bin/crictl"
+  local -r container_runtime="${CONTAINER_RUNTIME_NAME:-docker}"
+  until timeout 60 "${crictl}" pods > /dev/null; do
+    if (( attempt == max_attempts )); then
+      echo "Max attempt ${max_attempts} reached! Proceeding to monitor container runtime healthiness."
+      break
+    fi
+    echo "$attempt initial attempt \"${crictl} pods\"! Trying again in $attempt seconds..."
+    sleep "$(( attempt++ ))"
+  done
+  while true; do
+    if ! timeout 60 "${crictl}" pods > /dev/null; then
+      echo "Container runtime ${container_runtime} failed!"
+      systemctl kill --kill-who=main "${container_runtime}"
       # Wait for a while, as we don't want to kill it again before it is really up.
       sleep 120
     else
@@ -48,7 +62,7 @@ function kubelet_monitoring {
       # Print the response and/or errors.
       echo $output
       echo "Kubelet is unhealthy!"
-      pkill kubelet
+      systemctl kill kubelet
       # Wait for a while, as we don't want to kill it again before it is really up.
       sleep 60
     else
@@ -60,11 +74,12 @@ function kubelet_monitoring {
 
 ############## Main Function ################
 if [[ "$#" -ne 1 ]]; then
-  echo "Usage: health-monitor.sh <docker/kubelet>"
+  echo "Usage: health-monitor.sh <container-runtime/kubelet>"
   exit 1
 fi
 
-KUBE_ENV="/home/kubernetes/kube-env"
+KUBE_HOME="/home/kubernetes"
+KUBE_ENV="${KUBE_HOME}/kube-env"
 if [[ ! -e "${KUBE_ENV}" ]]; then
   echo "The ${KUBE_ENV} file does not exist!! Terminate health monitoring"
   exit 1
@@ -74,8 +89,8 @@ SLEEP_SECONDS=10
 component=$1
 echo "Start kubernetes health monitoring for ${component}"
 source "${KUBE_ENV}"
-if [[ "${component}" == "docker" ]]; then
-  docker_monitoring 
+if [[ "${component}" == "container-runtime" ]]; then
+  container_runtime_monitoring
 elif [[ "${component}" == "kubelet" ]]; then
   kubelet_monitoring
 else

cluster/gce/gci/master.yaml

@@ -40,12 +40,12 @@ write_files:
       [Install]
       WantedBy=kubernetes.target
 
-  - path: /etc/systemd/system/kube-docker-monitor.service
+  - path: /etc/systemd/system/kube-container-runtime-monitor.service
     permissions: 0644
     owner: root
     content: |
       [Unit]
-      Description=Kubernetes health monitoring for docker
+      Description=Kubernetes health monitoring for container runtime
       After=kube-master-configuration.service
 
       [Service]
@@ -54,7 +54,7 @@ write_files:
       RemainAfterExit=yes
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
-      ExecStart=/home/kubernetes/bin/health-monitor.sh docker
+      ExecStart=/home/kubernetes/bin/health-monitor.sh container-runtime
 
       [Install]
       WantedBy=kubernetes.target
@@ -120,7 +120,7 @@ runcmd:
  - systemctl daemon-reload
  - systemctl enable kube-master-installation.service
  - systemctl enable kube-master-configuration.service
- - systemctl enable kube-docker-monitor.service
+ - systemctl enable kube-container-runtime-monitor.service
  - systemctl enable kubelet-monitor.service
  - systemctl enable kube-logrotate.timer
  - systemctl enable kube-logrotate.service

cluster/gce/gci/node.yaml

@@ -40,12 +40,12 @@ write_files:
       [Install]
       WantedBy=kubernetes.target
 
-  - path: /etc/systemd/system/kube-docker-monitor.service
+  - path: /etc/systemd/system/kube-container-runtime-monitor.service
     permissions: 0644
     owner: root
     content: |
       [Unit]
-      Description=Kubernetes health monitoring for docker
+      Description=Kubernetes health monitoring for container runtime
       After=kube-node-configuration.service
 
       [Service]
@@ -54,7 +54,7 @@ write_files:
       RemainAfterExit=yes
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
-      ExecStart=/home/kubernetes/bin/health-monitor.sh docker
+      ExecStart=/home/kubernetes/bin/health-monitor.sh container-runtime
 
       [Install]
       WantedBy=kubernetes.target
@@ -120,7 +120,7 @@ runcmd:
  - systemctl daemon-reload
  - systemctl enable kube-node-installation.service
  - systemctl enable kube-node-configuration.service
- - systemctl enable kube-docker-monitor.service
+ - systemctl enable kube-container-runtime-monitor.service
  - systemctl enable kubelet-monitor.service
  - systemctl enable kube-logrotate.timer
  - systemctl enable kube-logrotate.service

cluster/gce/manifests/e2e-image-puller.manifest

@@ -76,14 +76,16 @@ spec:
       gcr.io/kubernetes-e2e-test-images/volume-rbd:0.1
       k8s.gcr.io/zookeeper-install-3.5.0-alpha:e2e
       gcr.io/google_samples/gb-redisslave:nonexistent
-      ; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;
+      ; do echo $(date '+%X') pulling $i; crictl pull $i 1>/dev/null; done; exit 0;
     securityContext:
       privileged: true
     volumeMounts:
-    - mountPath: /var/run/docker.sock
+    - mountPath: {{ container_runtime_endpoint }}
       name: socket
-    - mountPath: /usr/bin/docker
-      name: docker
+    - mountPath: /usr/bin/crictl
+      name: crictl
+    - mountPath: /etc/crictl.yaml
+      name: config
   # Add a container that runs a health-check
   - name: nethealth-check
     resources:
@@ -98,13 +100,17 @@ spec:
     - "/usr/bin/nethealth || true"
   volumes:
   - hostPath:
-      path: /var/run/docker.sock
+      path: {{ container_runtime_endpoint }}
       type: Socket
     name: socket
   - hostPath:
-      path: /usr/bin/docker
+      path: /home/kubernetes/bin/crictl
       type: File
-    name: docker
+    name: crictl
+  - hostPath:
+      path: /etc/crictl.yaml
+      type: File
+    name: config
   # This pod is really fire-and-forget.
   restartPolicy: OnFailure
   # This pod needs hostNetworking for true VM perf measurement as well as avoiding cbr0 issues

cluster/gce/util.sh

@@ -673,7 +673,6 @@ function construct-kubelet-flags {
   if [[ -n "${CONTAINER_RUNTIME:-}" ]]; then
     flags+=" --container-runtime=${CONTAINER_RUNTIME}"
   fi
-  # TODO(mtaufen): CONTAINER_RUNTIME_ENDPOINT seems unused; delete it?
   if [[ -n "${CONTAINER_RUNTIME_ENDPOINT:-}" ]]; then
     flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}"
   fi