Merge pull request #6191 from roberthbailey/kubelet-ssl

Configure the kubelet to bind a simple healthz server to a localhost port for monitoring by monit

cluster/saltbase/salt/monit/docker

@@ -1,8 +1,9 @@
 check process docker with pidfile /var/run/docker.pid
-group docker 
+group docker
 start program = "/etc/init.d/docker start"
 stop program = "/etc/init.d/docker stop"
 if does not exist then restart
-if failed unixsocket /var/run/docker.sock
+if failed
+  unixsocket /var/run/docker.sock
   protocol HTTP request "/version"
-  then restart
+then restart

cluster/saltbase/salt/monit/etcd

@@ -1,5 +1,5 @@
 check process etcd with pidfile /var/run/etcd.pid
-group etcd 
+group etcd
 start program = "/etc/init.d/etcd start"
 stop program = "/etc/init.d/etcd stop"
 if failed

cluster/saltbase/salt/monit/kube-proxy

@@ -3,7 +3,9 @@ group kube-proxy
 start program = "/etc/init.d/kube-proxy start"
 stop program = "/etc/init.d/kube-proxy stop"
 if does not exist then restart
-if failed port 10249
-  protocol HTTP request "/healthz"
-  with timeout 10 seconds
-  then restart
+if failed
+  host 127.0.0.1
+  port 10249
+  protocol HTTP
+  request "/healthz"
+then restart

cluster/saltbase/salt/monit/kubelet

@@ -1,9 +1,11 @@
 check process kubelet with pidfile /var/run/kubelet.pid
-group kubelet 
+group kubelet
 start program = "/etc/init.d/kubelet start"
 stop program = "/etc/init.d/kubelet stop"
 if does not exist then restart
-if failed port 10250
-  protocol HTTP request "/healthz"
-  with timeout 10 seconds
-  then restart
+if failed
+  host 127.0.0.1
+  port 10248
+  protocol HTTP
+  request "/healthz"
+then restart

cmd/kubelet/app/server.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
+	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -29,7 +31,7 @@ import (
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/credentialprovider"
-	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/cadvisor"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
@@ -72,6 +74,8 @@ type KubeletServer struct {
 	MaxContainerCount              int
 	AuthPath                       string
 	CadvisorPort                   uint
+	HealthzPort                    int
+	HealthzBindAddress             util.IP
 	OOMScoreAdj                    int
 	APIServerList                  util.StringList
 	ClusterDomain                  string
@@ -103,6 +107,8 @@ func NewKubeletServer() *KubeletServer {
 		MaxPerPodContainerCount:     5,
 		MaxContainerCount:           100,
 		CadvisorPort:                4194,
+		HealthzPort:                 10248,
+		HealthzBindAddress:          util.IP(net.ParseIP("127.0.0.1")),
 		OOMScoreAdj:                 -900,
 		MasterServiceNamespace:      api.NamespaceDefault,
 		ImageGCHighThresholdPercent: 90,
@@ -137,6 +143,8 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&s.MaxContainerCount, "maximum_dead_containers", s.MaxContainerCount, "Maximum number of old instances of a containers to retain globally.  Each container takes up some disk space.  Default: 100.")
 	fs.StringVar(&s.AuthPath, "auth_path", s.AuthPath, "Path to .kubernetes_auth file, specifying how to authenticate to API server.")
 	fs.UintVar(&s.CadvisorPort, "cadvisor_port", s.CadvisorPort, "The port of the localhost cAdvisor endpoint")
+	fs.IntVar(&s.HealthzPort, "healthz_port", s.HealthzPort, "The port of the localhost healthz endpoint")
+	fs.Var(&s.HealthzBindAddress, "healthz_bind_address", "The IP address for the healthz server to serve on, defaulting to 127.0.0.1 (set to 0.0.0.0 for all interfaces)")
 	fs.IntVar(&s.OOMScoreAdj, "oom_score_adj", s.OOMScoreAdj, "The oom_score_adj value for kubelet process. Values must be within the range [-1000, 1000]")
 	fs.Var(&s.APIServerList, "api_servers", "List of Kubernetes API servers for publishing events, and reading pods and services. (ip:port), comma separated.")
 	fs.StringVar(&s.ClusterDomain, "cluster_domain", s.ClusterDomain, "Domain for this cluster.  If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
@@ -223,6 +231,16 @@ func (s *KubeletServer) Run(_ []string) error {
 
 	RunKubelet(&kcfg)
 
+	if s.HealthzPort > 0 {
+		healthz.DefaultHealthz()
+		go util.Forever(func() {
+			err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress.String(), strconv.Itoa(s.HealthzPort)), nil)
+			if err != nil {
+				glog.Errorf("Starting health server failed: %v", err)
+			}
+		}, 5*time.Second)
+	}
+
 	// runs forever
 	select {}
 

pkg/master/ports/ports.go

@@ -17,7 +17,13 @@ limitations under the License.
 package ports
 
 const (
-	// KubeletPort is the default port for the kubelet status server on each host machine.
+	// KubeletStatusPort is the default port for the kubelet healthz server.
+	// May be overridden by a flag at startup.
+	KubeletStatusPort = 10248
+	// ProxyPort is the default port for the proxy healthz server.
+	// May be overriden by a flag at startup.
+	ProxyStatusPort = 10249
+	// KubeletPort is the default port for the kubelet server on each host machine.
 	// May be overridden by a flag at startup.
 	KubeletPort = 10250
 	// SchedulerPort is the default port for the scheduler status server.
@@ -26,7 +32,4 @@ const (
 	// ControllerManagerPort is the default port for the controller manager status server.
 	// May be overridden by a flag at startup.
 	ControllerManagerPort = 10252
-	// ProxyPort is the default port for the proxy status server.
-	// May be overriden by a flag at startup.
-	ProxyPort = 10249
 )