Merge pull request #6380 from roberthbailey/kubelet-ssl

Configure the kubelet to use HTTPS (take 2)

cluster/addons/cluster-monitoring/heapster-controller.yaml

@@ -1,9 +1,9 @@
 apiVersion: v1beta1
 id: monitoring-heapster-controller
 kind: ReplicationController
-desiredState: 
+desiredState:
   replicas: 1
-  replicaSelector:    
+  replicaSelector:
     name: heapster
   podTemplate:
     desiredState:
@@ -13,11 +13,13 @@ desiredState:
         containers:
           - name: heapster
             image: gcr.io/google_containers/heapster:v0.10.0
-            env: 
+            env:
               - name: "INFLUXDB_HOST"
                 value: "monitoring-influxdb"
               - name: "SINK"
                 value: "influxdb"
+              - name: "FLAGS"
+                value: "--kubelet_port=10255"
             volumeMounts:
               - name: ssl-certs
                 mountPath: /etc/ssl/certs
@@ -27,10 +29,10 @@ desiredState:
             source:
               hostDir:
                 path: /etc/ssl/certs
-    labels: 
+    labels:
       name: heapster
       uses: monitoring-influxdb
       kubernetes.io/cluster-service: "true"
-labels: 
+labels:
   name: heapster
   kubernetes.io/cluster-service: "true"

cmd/kube-apiserver/app/server.go

@@ -103,7 +103,7 @@ func NewAPIServer() *APIServer {
 		RuntimeConfig: make(util.ConfigurationMap),
 		KubeletConfig: client.KubeletConfig{
 			Port:        10250,
-			EnableHttps: false,
+			EnableHttps: true,
 			HTTPTimeout: time.Duration(5) * time.Second,
 		},
 	}

cmd/kube-controller-manager/app/controllermanager.go

@@ -85,7 +85,7 @@ func NewCMServer() *CMServer {
 		SyncNodeStatus:          false,
 		KubeletConfig: client.KubeletConfig{
 			Port:        ports.KubeletPort,
-			EnableHttps: false,
+			EnableHttps: true,
 			HTTPTimeout: time.Duration(5) * time.Second,
 		},
 	}

cmd/kubelet/app/server.go

@@ -18,10 +18,12 @@ limitations under the License.
 package app
 
 import (
+	"crypto/tls"
 	"fmt"
 	"math/rand"
 	"net"
 	"net/http"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -59,6 +61,7 @@ type KubeletServer struct {
 	EnableServer                   bool
 	Address                        util.IP
 	Port                           uint
+	ReadOnlyPort                   uint
 	HostnameOverride               string
 	PodInfraContainerImage         string
 	DockerEndpoint                 string
@@ -88,17 +91,21 @@ type KubeletServer struct {
 	NetworkPluginName              string
 	CloudProvider                  string
 	CloudConfigFile                string
+	TLSCertFile                    string
+	TLSPrivateKeyFile              string
+	CertDirectory                  string
 }
 
 // NewKubeletServer will create a new KubeletServer with default values.
 func NewKubeletServer() *KubeletServer {
 	return &KubeletServer{
-		SyncFrequency:      10 * time.Second,
+		SyncFrequency:               10 * time.Second,
-		FileCheckFrequency: 20 * time.Second,
+		FileCheckFrequency:          20 * time.Second,
-		HTTPCheckFrequency: 20 * time.Second,
+		HTTPCheckFrequency:          20 * time.Second,
-		EnableServer:       true,
+		EnableServer:                true,
-		Address:            util.IP(net.ParseIP("0.0.0.0")),
+		Address:                     util.IP(net.ParseIP("0.0.0.0")),
-		Port:               ports.KubeletPort,
+		Port:                        ports.KubeletPort,
+		ReadOnlyPort:                ports.KubeletReadOnlyPort,
 		PodInfraContainerImage:      kubelet.PodInfraContainerImage,
 		RootDirectory:               defaultRootDir,
 		RegistryBurst:               10,
@@ -116,6 +123,7 @@ func NewKubeletServer() *KubeletServer {
 		ImageGCLowThresholdPercent:  80,
 		NetworkPluginName:           "",
 		HostNetworkSources:          kubelet.FileSource,
+		CertDirectory:               "/var/run/kubernetes",
 	}
 }
 
@@ -129,6 +137,13 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.EnableServer, "enable_server", s.EnableServer, "Enable the info server")
 	fs.Var(&s.Address, "address", "The IP address for the info server to serve on (set to 0.0.0.0 for all interfaces)")
 	fs.UintVar(&s.Port, "port", s.Port, "The port for the info server to serve on")
+	fs.UintVar(&s.ReadOnlyPort, "read_only_port", s.ReadOnlyPort, "The read-only port for the info server to serve on (set to 0 to disable)")
+	fs.StringVar(&s.TLSCertFile, "tls_cert_file", s.TLSCertFile, ""+
+		"File containing x509 Certificate for HTTPS.  (CA cert, if any, concatenated after server cert). "+
+		"If --tls_cert_file and --tls_private_key_file are not provided, a self-signed certificate and key "+
+		"are generated for the public address and saved to the directory passed to --cert_dir.")
+	fs.StringVar(&s.TLSPrivateKeyFile, "tls_private_key_file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls_cert_file.")
+	fs.StringVar(&s.CertDirectory, "cert_dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes)")
 	fs.StringVar(&s.HostnameOverride, "hostname_override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
 	fs.StringVar(&s.PodInfraContainerImage, "pod_infra_container_image", s.PodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.")
 	fs.StringVar(&s.DockerEndpoint, "docker_endpoint", s.DockerEndpoint, "If non-empty, use this for the docker endpoint to communicate with")
@@ -195,6 +210,26 @@ func (s *KubeletServer) Run(_ []string) error {
 	if err != nil {
 		return err
 	}
+
+	if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
+		s.TLSCertFile = path.Join(s.CertDirectory, "kubelet.crt")
+		s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "kubelet.key")
+		if err := util.GenerateSelfSignedCert(util.GetHostname(s.HostnameOverride), s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
+			glog.Fatalf("Unable to generate self signed cert: %v", err)
+		}
+		glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
+	}
+	tlsOptions := &kubelet.TLSOptions{
+		Config: &tls.Config{
+			// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability).
+			MinVersion: tls.VersionTLS10,
+			// Populate PeerCertificates in requests, but don't yet reject connections without certificates.
+			ClientAuth: tls.RequestClientCert,
+		},
+		CertFile: s.TLSCertFile,
+		KeyFile:  s.TLSPrivateKeyFile,
+	}
+
 	kcfg := KubeletConfig{
 		Address:                        s.Address,
 		AllowPrivileged:                s.AllowPrivileged,
@@ -216,6 +251,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		ClusterDNS:                     s.ClusterDNS,
 		Runonce:                        s.RunOnce,
 		Port:                           s.Port,
+		ReadOnlyPort:                   s.ReadOnlyPort,
 		CadvisorInterface:              cadvisorInterface,
 		EnableServer:                   s.EnableServer,
 		EnableDebuggingHandlers:        s.EnableDebuggingHandlers,
@@ -226,6 +262,7 @@ func (s *KubeletServer) Run(_ []string) error {
 		NetworkPlugins:                 ProbeNetworkPlugins(),
 		NetworkPluginName:              s.NetworkPluginName,
 		StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout,
+		TLSOptions:                     tlsOptions,
 		ImageGCPolicy:                  imageGCPolicy,
 		Cloud:                          cloud,
 	}
@@ -381,6 +418,11 @@ func startKubelet(k *kubelet.Kubelet, podCfg *config.PodConfig, kc *KubeletConfi
 			kubelet.ListenAndServeKubeletServer(k, net.IP(kc.Address), kc.Port, kc.TLSOptions, kc.EnableDebuggingHandlers)
 		}, 0)
 	}
+	if kc.ReadOnlyPort > 0 {
+		go util.Forever(func() {
+			kubelet.ListenAndServeKubeletReadOnlyServer(k, net.IP(kc.Address), kc.ReadOnlyPort)
+		}, 0)
+	}
 }
 
 func makePodSourceConfig(kc *KubeletConfig) *config.PodConfig {
@@ -433,6 +475,7 @@ type KubeletConfig struct {
 	EnableServer                   bool
 	EnableDebuggingHandlers        bool
 	Port                           uint
+	ReadOnlyPort                   uint
 	Runonce                        bool
 	MasterServiceNamespace         string
 	VolumePlugins                  []volume.VolumePlugin

hack/test-cmd.sh

@@ -48,6 +48,7 @@ ETCD_PORT=${ETCD_PORT:-4001}
 API_PORT=${API_PORT:-8080}
 API_HOST=${API_HOST:-127.0.0.1}
 KUBELET_PORT=${KUBELET_PORT:-10250}
+KUBELET_HEALTHZ_PORT=${KUBELET_HEALTHZ_PORT:-10248}
 CTLRMGR_PORT=${CTLRMGR_PORT:-10252}
 
 # Check kubectl
@@ -58,27 +59,31 @@ kube::log::status "Starting kubelet in masterless mode"
 "${KUBE_OUTPUT_HOSTBIN}/kubelet" \
   --really_crash_for_testing=true \
   --root_dir=/tmp/kubelet.$$ \
+  --cert_dir="${TMPDIR:-/tmp/}" \
   --docker_endpoint="fake://" \
   --hostname_override="127.0.0.1" \
   --address="127.0.0.1" \
-  --port="$KUBELET_PORT" 1>&2 &
+  --port="$KUBELET_PORT" \
+  --healthz_port="${KUBELET_HEALTHZ_PORT}" 1>&2 &
 KUBELET_PID=$!
-kube::util::wait_for_url "http://127.0.0.1:${KUBELET_PORT}/healthz" "kubelet: " 0.2 25
+kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet: " 0.2 25
 kill ${KUBELET_PID} 1>&2 2>/dev/null
 
 kube::log::status "Starting kubelet in masterful mode"
 "${KUBE_OUTPUT_HOSTBIN}/kubelet" \
   --really_crash_for_testing=true \
   --root_dir=/tmp/kubelet.$$ \
+  --cert_dir="${TMPDIR:-/tmp/}" \
   --docker_endpoint="fake://" \
   --hostname_override="127.0.0.1" \
   --address="127.0.0.1" \
   --api_servers="${API_HOST}:${API_PORT}" \
   --auth_path="${KUBE_ROOT}/hack/.test-cmd-auth" \
-  --port="$KUBELET_PORT" 1>&2 &
+  --port="$KUBELET_PORT" \
+  --healthz_port="${KUBELET_HEALTHZ_PORT}" 1>&2 &
 KUBELET_PID=$!
 
-kube::util::wait_for_url "http://127.0.0.1:${KUBELET_PORT}/healthz" "kubelet: " 0.2 25
+kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet: " 0.2 25
 
 # Start kube-apiserver
 kube::log::status "Starting kube-apiserver"

pkg/apiserver/apiserver.go

@@ -139,14 +139,7 @@ func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
 
 // TODO: Convert to go-restful
 func InstallValidator(mux Mux, servers func() map[string]Server) {
-	validator, err := NewValidator(servers)
+	mux.Handle("/validate", NewValidator(servers))
-	if err != nil {
-		glog.Errorf("failed to set up validator: %v", err)
-		return
-	}
-	if validator != nil {
-		mux.Handle("/validate", validator)
-	}
 }
 
 // TODO: document all handlers

pkg/apiserver/validator.go

@@ -17,6 +17,7 @@ limitations under the License.
 package apiserver
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -34,21 +35,26 @@ type httpGet interface {
 }
 
 type Server struct {
-	Addr string
+	Addr        string
-	Port int
+	Port        int
-	Path string
+	Path        string
+	EnableHTTPS bool
 }
 
 // validator is responsible for validating the cluster and serving
 type validator struct {
 	// a list of servers to health check
 	servers func() map[string]Server
-	client  httpGet
+	rt      http.RoundTripper
 }
 
 // TODO: can this use pkg/probe/http
 func (s *Server) check(client httpGet) (probe.Result, string, error) {
-	resp, err := client.Get("http://" + net.JoinHostPort(s.Addr, strconv.Itoa(s.Port)) + s.Path)
+	scheme := "http://"
+	if s.EnableHTTPS {
+		scheme = "https://"
+	}
+	resp, err := client.Get(scheme + net.JoinHostPort(s.Addr, strconv.Itoa(s.Port)) + s.Path)
 	if err != nil {
 		return probe.Unknown, "", err
 	}
@@ -81,7 +87,22 @@ func (v *validator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	reply := []ServerStatus{}
 	for name, server := range v.servers() {
-		status, msg, err := server.check(v.client)
+		transport := v.rt
+		if server.EnableHTTPS {
+			// TODO(roberthbailey): The servers that use HTTPS are currently the
+			// kubelets, and we should be using a standard kubelet client library
+			// to talk to them rather than a separate http client.
+			transport = &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+				Dial: (&net.Dialer{
+					Timeout:   30 * time.Second,
+					KeepAlive: 30 * time.Second,
+				}).Dial,
+				TLSHandshakeTimeout: 10 * time.Second,
+				TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
+			}
+		}
+		status, msg, err := server.check(&http.Client{Transport: transport})
 		var errorMsg string
 		if err != nil {
 			errorMsg = err.Error()
@@ -103,30 +124,6 @@ func (v *validator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 // NewValidator creates a validator for a set of servers.
-func NewValidator(servers func() map[string]Server) (http.Handler, error) {
+func NewValidator(servers func() map[string]Server) http.Handler {
-	return &validator{
+	return &validator{servers: servers, rt: http.DefaultTransport}
-		servers: servers,
-		client:  &http.Client{},
-	}, nil
-}
-
-func makeTestValidator(servers map[string]string, get httpGet) (http.Handler, error) {
-	result := map[string]Server{}
-	for name, value := range servers {
-		host, port, err := net.SplitHostPort(value)
-		if err != nil {
-			return nil, fmt.Errorf("invalid server spec: %s (%v)", value, err)
-		}
-		val, err := strconv.Atoi(port)
-		if err != nil {
-			return nil, fmt.Errorf("invalid server spec: %s (%v)", port, err)
-		}
-		result[name] = Server{Addr: host, Port: val, Path: "/healthz"}
-	}
-
-	v, e := NewValidator(func() map[string]Server { return result })
-	if e == nil {
-		v.(*validator).client = get
-	}
-	return v, e
 }

pkg/apiserver/validator_test.go

@@ -21,35 +21,27 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"testing"
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/probe"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
 )
 
-type fakeHttpGet struct {
+type fakeRoundTripper struct {
 	err  error
 	resp *http.Response
 	url  string
 }
 
-func (f *fakeHttpGet) Get(url string) (*http.Response, error) {
+func (f *fakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
-	f.url = url
+	f.url = req.URL.String()
 	return f.resp, f.err
 }
 
-func makeFake(data string, statusCode int, err error) *fakeHttpGet {
-	return &fakeHttpGet{
-		err: err,
-		resp: &http.Response{
-			Body:       ioutil.NopCloser(bytes.NewBufferString(data)),
-			StatusCode: statusCode,
-		},
-	}
-}
-
 func TestValidate(t *testing.T) {
 	tests := []struct {
 		err            error
@@ -66,11 +58,18 @@ func TestValidate(t *testing.T) {
 	s := Server{Addr: "foo.com", Port: 8080, Path: "/healthz"}
 
 	for _, test := range tests {
-		fake := makeFake(test.data, test.code, test.err)
+		fakeRT := &fakeRoundTripper{
+			err: test.err,
+			resp: &http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(test.data)),
+				StatusCode: test.code,
+			},
+		}
+		fake := &http.Client{Transport: fakeRT}
 		status, data, err := s.check(fake)
 		expect := fmt.Sprintf("http://%s:%d/healthz", s.Addr, s.Port)
-		if fake.url != expect {
+		if fakeRT.url != expect {
-			t.Errorf("expected %s, got %s", expect, fake.url)
+			t.Errorf("expected %s, got %s", expect, fakeRT.url)
 		}
 		if test.expectErr && err == nil {
 			t.Errorf("unexpected non-error")
@@ -87,8 +86,30 @@ func TestValidate(t *testing.T) {
 	}
 }
 
+func makeTestValidator(servers map[string]string, rt http.RoundTripper) (http.Handler, error) {
+	result := map[string]Server{}
+	for name, value := range servers {
+		host, port, err := net.SplitHostPort(value)
+		if err != nil {
+			return nil, fmt.Errorf("invalid server spec: %s (%v)", value, err)
+		}
+		val, err := strconv.Atoi(port)
+		if err != nil {
+			return nil, fmt.Errorf("invalid server spec: %s (%v)", port, err)
+		}
+		result[name] = Server{Addr: host, Port: val, Path: "/healthz"}
+	}
+
+	return &validator{servers: func() map[string]Server { return result }, rt: rt}, nil
+}
+
 func TestValidator(t *testing.T) {
-	fake := makeFake("foo", 200, nil)
+	fake := &fakeRoundTripper{
+		resp: &http.Response{
+			Body:       ioutil.NopCloser(bytes.NewBufferString("foo")),
+			StatusCode: 200,
+		},
+	}
 	validator, err := makeTestValidator(map[string]string{
 		"foo": "foo.com:80",
 		"bar": "bar.com:8080",
@@ -101,7 +122,6 @@ func TestValidator(t *testing.T) {
 	defer testServer.Close()
 
 	resp, err := http.Get(testServer.URL + "/validatez")
-
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
@@ -113,13 +133,15 @@ func TestValidator(t *testing.T) {
 	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
-	status := []ServerStatus{}
+	var status []ServerStatus
-	err = json.Unmarshal(data, &status)
+	if err := json.Unmarshal(data, &status); err != nil {
-	if err != nil {
 		t.Errorf("unexpected error: %v", err)
 	}
 	components := util.StringSet{}
 	for _, s := range status {
+		if s.Err != "nil" {
+			t.Errorf("Component %v is unhealthy: %v", s.Component, s.Err)
+		}
 		components.Insert(s.Component)
 	}
 	if len(status) != 2 || !components.Has("foo") || !components.Has("bar") {

pkg/client/kubelet.go

@@ -75,9 +75,14 @@ type HTTPKubeletClient struct {
 func NewKubeletClient(config *KubeletConfig) (KubeletClient, error) {
 	transport := http.DefaultTransport
 
-	tlsConfig, err := TLSConfigFor(&Config{
+	cfg := &Config{TLSClientConfig: config.TLSClientConfig}
-		TLSClientConfig: config.TLSClientConfig,
+	if config.EnableHttps {
-	})
+		hasCA := len(config.CAFile) > 0 || len(config.CAData) > 0
+		if !hasCA {
+			cfg.Insecure = true
+		}
+	}
+	tlsConfig, err := TLSConfigFor(cfg)
 	if err != nil {
 		return nil, err
 	}

pkg/kubelet/server.go

@@ -76,6 +76,24 @@ func ListenAndServeKubeletServer(host HostInterface, address net.IP, port uint,
 	}
 }
 
+// ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet.
+func ListenAndServeKubeletReadOnlyServer(host HostInterface, address net.IP, port uint) {
+	glog.V(1).Infof("Starting to listen read-only on %s:%d", address, port)
+	s := &Server{host, http.NewServeMux()}
+	healthz.InstallHandler(s.mux)
+	s.mux.HandleFunc("/stats/", s.handleStats)
+	s.mux.Handle("/metrics", prometheus.Handler())
+
+	server := &http.Server{
+		Addr:           net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
+		Handler:        s,
+		ReadTimeout:    5 * time.Minute,
+		WriteTimeout:   5 * time.Minute,
+		MaxHeaderBytes: 1 << 20,
+	}
+	glog.Fatal(server.ListenAndServe())
+}
+
 // HostInterface contains all the kubelet methods required by the server.
 // For testablitiy.
 type HostInterface interface {

pkg/master/master.go

@@ -561,7 +561,7 @@ func (m *Master) getServersToValidate(c *Config) map[string]apiserver.Server {
 		glog.Errorf("Failed to list minions: %v", err)
 	}
 	for ix, node := range nodes.Items {
-		serversToValidate[fmt.Sprintf("node-%d", ix)] = apiserver.Server{Addr: node.Name, Port: ports.KubeletPort, Path: "/healthz"}
+		serversToValidate[fmt.Sprintf("node-%d", ix)] = apiserver.Server{Addr: node.Name, Port: ports.KubeletPort, Path: "/healthz", EnableHTTPS: true}
 	}
 	return serversToValidate
 }

pkg/master/ports/ports.go

@@ -32,4 +32,10 @@ const (
 	// ControllerManagerPort is the default port for the controller manager status server.
 	// May be overridden by a flag at startup.
 	ControllerManagerPort = 10252
+	// KubeletReadOnlyPort exposes basic read-only services from the kubelet.
+	// May be overridden by a flag at startup.
+	// This is necessary for heapster to collect monitoring stats from the kubelet
+	// until heapster can transition to using the SSL endpoint.
+	// TODO(roberthbailey): Remove this once we have a better solution for heapster.
+	KubeletReadOnlyPort = 10255
 )