github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #6380 from roberthbailey/kubelet-ssl 

Configure the kubelet to use HTTPS (take 2)
pull/6/head
Eric Tune 2015-04-03 13:43:00 -07:00
parent 3d1dfd47ee 4488ff95a2
commit e49424785e
12 changed files with 173 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cluster/addons/cluster-monitoring/heapster-controller.yaml
View File

@ -1,9 +1,9 @@
apiVersion: v1beta1
id: monitoring-heapster-controller
kind: ReplicationController
desiredState:
desiredState:
replicas: 1
replicaSelector:
replicaSelector:
name: heapster
podTemplate:
desiredState:
@ -13,11 +13,13 @@ desiredState:
containers:
- name: heapster
image: gcr.io/google_containers/heapster:v0.10.0
env:
env:
- name: "INFLUXDB_HOST"
value: "monitoring-influxdb"
- name: "SINK"
value: "influxdb"
- name: "FLAGS"
value: "--kubelet_port=10255"
volumeMounts:
- name: ssl-certs
mountPath: /etc/ssl/certs
@ -27,10 +29,10 @@ desiredState:
source:
hostDir:
path: /etc/ssl/certs
labels:
labels:
name: heapster
uses: monitoring-influxdb
kubernetes.io/cluster-service: "true"
labels:
labels:
name: heapster
kubernetes.io/cluster-service: "true"

2
cmd/kube-apiserver/app/server.go
View File

@ -103,7 +103,7 @@ func NewAPIServer() *APIServer {
RuntimeConfig: make(util.ConfigurationMap),
KubeletConfig: client.KubeletConfig{
Port: 10250,
EnableHttps: false,
EnableHttps: true,
HTTPTimeout: time.Duration(5) * time.Second,
},
}

2
cmd/kube-controller-manager/app/controllermanager.go
View File

@ -85,7 +85,7 @@ func NewCMServer() *CMServer {
SyncNodeStatus: false,
KubeletConfig: client.KubeletConfig{
Port: ports.KubeletPort,
EnableHttps: false,
EnableHttps: true,
HTTPTimeout: time.Duration(5) * time.Second,
},
}

55
cmd/kubelet/app/server.go
View File

@ -18,10 +18,12 @@ limitations under the License.
package app
import (
"crypto/tls"
"fmt"
"math/rand"
"net"
"net/http"
"path"
"strconv"
"strings"
"time"
@ -59,6 +61,7 @@ type KubeletServer struct {
EnableServer bool
Address util.IP
Port uint
ReadOnlyPort uint
HostnameOverride string
PodInfraContainerImage string
DockerEndpoint string
@ -88,17 +91,21 @@ type KubeletServer struct {
NetworkPluginName string
CloudProvider string
CloudConfigFile string
TLSCertFile string
TLSPrivateKeyFile string
CertDirectory string
}
// NewKubeletServer will create a new KubeletServer with default values.
func NewKubeletServer() *KubeletServer {
return &KubeletServer{
SyncFrequency: 10 * time.Second,
FileCheckFrequency: 20 * time.Second,
HTTPCheckFrequency: 20 * time.Second,
EnableServer: true,
Address: util.IP(net.ParseIP("0.0.0.0")),
Port: ports.KubeletPort,
SyncFrequency: 10 * time.Second,
FileCheckFrequency: 20 * time.Second,
HTTPCheckFrequency: 20 * time.Second,
EnableServer: true,
Address: util.IP(net.ParseIP("0.0.0.0")),
Port: ports.KubeletPort,
ReadOnlyPort: ports.KubeletReadOnlyPort,
PodInfraContainerImage: kubelet.PodInfraContainerImage,
RootDirectory: defaultRootDir,
RegistryBurst: 10,
@ -116,6 +123,7 @@ func NewKubeletServer() *KubeletServer {
ImageGCLowThresholdPercent: 80,
NetworkPluginName: "",
HostNetworkSources: kubelet.FileSource,
CertDirectory: "/var/run/kubernetes",
}
}
@ -129,6 +137,13 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&s.EnableServer, "enable_server", s.EnableServer, "Enable the info server")
fs.Var(&s.Address, "address", "The IP address for the info server to serve on (set to 0.0.0.0 for all interfaces)")
fs.UintVar(&s.Port, "port", s.Port, "The port for the info server to serve on")
fs.UintVar(&s.ReadOnlyPort, "read_only_port", s.ReadOnlyPort, "The read-only port for the info server to serve on (set to 0 to disable)")
fs.StringVar(&s.TLSCertFile, "tls_cert_file", s.TLSCertFile, ""+
"File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated after server cert). "+
"If --tls_cert_file and --tls_private_key_file are not provided, a self-signed certificate and key "+
"are generated for the public address and saved to the directory passed to --cert_dir.")
fs.StringVar(&s.TLSPrivateKeyFile, "tls_private_key_file", s.TLSPrivateKeyFile, "File containing x509 private key matching --tls_cert_file.")
fs.StringVar(&s.CertDirectory, "cert_dir", s.CertDirectory, "The directory where the TLS certs are located (by default /var/run/kubernetes)")
fs.StringVar(&s.HostnameOverride, "hostname_override", s.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
fs.StringVar(&s.PodInfraContainerImage, "pod_infra_container_image", s.PodInfraContainerImage, "The image whose network/ipc namespaces containers in each pod will use.")
fs.StringVar(&s.DockerEndpoint, "docker_endpoint", s.DockerEndpoint, "If non-empty, use this for the docker endpoint to communicate with")
@ -195,6 +210,26 @@ func (s *KubeletServer) Run(_ []string) error {
if err != nil {
return err
}
if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
s.TLSCertFile = path.Join(s.CertDirectory, "kubelet.crt")
s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "kubelet.key")
if err := util.GenerateSelfSignedCert(util.GetHostname(s.HostnameOverride), s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
glog.Fatalf("Unable to generate self signed cert: %v", err)
}
glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
}
tlsOptions := &kubelet.TLSOptions{
Config: &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability).
MinVersion: tls.VersionTLS10,
// Populate PeerCertificates in requests, but don't yet reject connections without certificates.
ClientAuth: tls.RequestClientCert,
},
CertFile: s.TLSCertFile,
KeyFile: s.TLSPrivateKeyFile,
}
kcfg := KubeletConfig{
Address: s.Address,
AllowPrivileged: s.AllowPrivileged,
@ -216,6 +251,7 @@ func (s *KubeletServer) Run(_ []string) error {
ClusterDNS: s.ClusterDNS,
Runonce: s.RunOnce,
Port: s.Port,
ReadOnlyPort: s.ReadOnlyPort,
CadvisorInterface: cadvisorInterface,
EnableServer: s.EnableServer,
EnableDebuggingHandlers: s.EnableDebuggingHandlers,
@ -226,6 +262,7 @@ func (s *KubeletServer) Run(_ []string) error {
NetworkPlugins: ProbeNetworkPlugins(),
NetworkPluginName: s.NetworkPluginName,
StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout,
TLSOptions: tlsOptions,
ImageGCPolicy: imageGCPolicy,
Cloud: cloud,
}
@ -381,6 +418,11 @@ func startKubelet(k *kubelet.Kubelet, podCfg *config.PodConfig, kc *KubeletConfi
kubelet.ListenAndServeKubeletServer(k, net.IP(kc.Address), kc.Port, kc.TLSOptions, kc.EnableDebuggingHandlers)
}, 0)
}
if kc.ReadOnlyPort > 0 {
go util.Forever(func() {
kubelet.ListenAndServeKubeletReadOnlyServer(k, net.IP(kc.Address), kc.ReadOnlyPort)
}, 0)
}
}
func makePodSourceConfig(kc *KubeletConfig) *config.PodConfig {
@ -433,6 +475,7 @@ type KubeletConfig struct {
EnableServer bool
EnableDebuggingHandlers bool
Port uint
ReadOnlyPort uint
Runonce bool
MasterServiceNamespace string
VolumePlugins []volume.VolumePlugin

13
hack/test-cmd.sh
View File

@ -48,6 +48,7 @@ ETCD_PORT=${ETCD_PORT:-4001}
API_PORT=${API_PORT:-8080}
API_HOST=${API_HOST:-127.0.0.1}
KUBELET_PORT=${KUBELET_PORT:-10250}
KUBELET_HEALTHZ_PORT=${KUBELET_HEALTHZ_PORT:-10248}
CTLRMGR_PORT=${CTLRMGR_PORT:-10252}
# Check kubectl
@ -58,27 +59,31 @@ kube::log::status "Starting kubelet in masterless mode"
"${KUBE_OUTPUT_HOSTBIN}/kubelet" \
--really_crash_for_testing=true \
--root_dir=/tmp/kubelet.$$ \
--cert_dir="${TMPDIR:-/tmp/}" \
--docker_endpoint="fake://" \
--hostname_override="127.0.0.1" \
--address="127.0.0.1" \
--port="$KUBELET_PORT" 1>&2 &
--port="$KUBELET_PORT" \
--healthz_port="${KUBELET_HEALTHZ_PORT}" 1>&2 &
KUBELET_PID=$!
kube::util::wait_for_url "http://127.0.0.1:${KUBELET_PORT}/healthz" "kubelet: " 0.2 25
kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet: " 0.2 25
kill ${KUBELET_PID} 1>&2 2>/dev/null
kube::log::status "Starting kubelet in masterful mode"
"${KUBE_OUTPUT_HOSTBIN}/kubelet" \
--really_crash_for_testing=true \
--root_dir=/tmp/kubelet.$$ \
--cert_dir="${TMPDIR:-/tmp/}" \
--docker_endpoint="fake://" \
--hostname_override="127.0.0.1" \
--address="127.0.0.1" \
--api_servers="${API_HOST}:${API_PORT}" \
--auth_path="${KUBE_ROOT}/hack/.test-cmd-auth" \
--port="$KUBELET_PORT" 1>&2 &
--port="$KUBELET_PORT" \
--healthz_port="${KUBELET_HEALTHZ_PORT}" 1>&2 &
KUBELET_PID=$!
kube::util::wait_for_url "http://127.0.0.1:${KUBELET_PORT}/healthz" "kubelet: " 0.2 25
kube::util::wait_for_url "http://127.0.0.1:${KUBELET_HEALTHZ_PORT}/healthz" "kubelet: " 0.2 25
# Start kube-apiserver
kube::log::status "Starting kube-apiserver"

9
pkg/apiserver/apiserver.go
View File

@ -139,14 +139,7 @@ func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
// TODO: Convert to go-restful
func InstallValidator(mux Mux, servers func() map[string]Server) {
validator, err := NewValidator(servers)
if err != nil {
glog.Errorf("failed to set up validator: %v", err)
return
}
if validator != nil {
mux.Handle("/validate", validator)
}
mux.Handle("/validate", NewValidator(servers))
}
// TODO: document all handlers

61
pkg/apiserver/validator.go
View File

@ -17,6 +17,7 @@ limitations under the License.
package apiserver
import (
"crypto/tls"
"encoding/json"
"fmt"
"io/ioutil"
@ -34,21 +35,26 @@ type httpGet interface {
}
type Server struct {
Addr string
Port int
Path string
Addr string
Port int
Path string
EnableHTTPS bool
}
// validator is responsible for validating the cluster and serving
type validator struct {
// a list of servers to health check
servers func() map[string]Server
client httpGet
rt http.RoundTripper
}
// TODO: can this use pkg/probe/http
func (s *Server) check(client httpGet) (probe.Result, string, error) {
resp, err := client.Get("http://" + net.JoinHostPort(s.Addr, strconv.Itoa(s.Port)) + s.Path)
scheme := "http://"
if s.EnableHTTPS {
scheme = "https://"
}
resp, err := client.Get(scheme + net.JoinHostPort(s.Addr, strconv.Itoa(s.Port)) + s.Path)
if err != nil {
return probe.Unknown, "", err
}
@ -81,7 +87,22 @@ func (v *validator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
reply := []ServerStatus{}
for name, server := range v.servers() {
status, msg, err := server.check(v.client)
transport := v.rt
if server.EnableHTTPS {
// TODO(roberthbailey): The servers that use HTTPS are currently the
// kubelets, and we should be using a standard kubelet client library
// to talk to them rather than a separate http client.
transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
status, msg, err := server.check(&http.Client{Transport: transport})
var errorMsg string
if err != nil {
errorMsg = err.Error()
@ -103,30 +124,6 @@ func (v *validator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
// NewValidator creates a validator for a set of servers.
func NewValidator(servers func() map[string]Server) (http.Handler, error) {
return &validator{
servers: servers,
client: &http.Client{},
}, nil
}
func makeTestValidator(servers map[string]string, get httpGet) (http.Handler, error) {
result := map[string]Server{}
for name, value := range servers {
host, port, err := net.SplitHostPort(value)
if err != nil {
return nil, fmt.Errorf("invalid server spec: %s (%v)", value, err)
}
val, err := strconv.Atoi(port)
if err != nil {
return nil, fmt.Errorf("invalid server spec: %s (%v)", port, err)
}
result[name] = Server{Addr: host, Port: val, Path: "/healthz"}
}
v, e := NewValidator(func() map[string]Server { return result })
if e == nil {
v.(*validator).client = get
}
return v, e
func NewValidator(servers func() map[string]Server) http.Handler {
return &validator{servers: servers, rt: http.DefaultTransport}
}

64
pkg/apiserver/validator_test.go
View File

@ -21,35 +21,27 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"github.com/GoogleCloudPlatform/kubernetes/pkg/probe"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)
type fakeHttpGet struct {
type fakeRoundTripper struct {
err error
resp *http.Response
url string
}
func (f *fakeHttpGet) Get(url string) (*http.Response, error) {
f.url = url
func (f *fakeRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
f.url = req.URL.String()
return f.resp, f.err
}
func makeFake(data string, statusCode int, err error) *fakeHttpGet {
return &fakeHttpGet{
err: err,
resp: &http.Response{
Body: ioutil.NopCloser(bytes.NewBufferString(data)),
StatusCode: statusCode,
},
}
}
func TestValidate(t *testing.T) {
tests := []struct {
err error
@ -66,11 +58,18 @@ func TestValidate(t *testing.T) {
s := Server{Addr: "foo.com", Port: 8080, Path: "/healthz"}
for _, test := range tests {
fake := makeFake(test.data, test.code, test.err)
fakeRT := &fakeRoundTripper{
err: test.err,
resp: &http.Response{
Body: ioutil.NopCloser(bytes.NewBufferString(test.data)),
StatusCode: test.code,
},
}
fake := &http.Client{Transport: fakeRT}
status, data, err := s.check(fake)
expect := fmt.Sprintf("http://%s:%d/healthz", s.Addr, s.Port)
if fake.url != expect {
t.Errorf("expected %s, got %s", expect, fake.url)
if fakeRT.url != expect {
t.Errorf("expected %s, got %s", expect, fakeRT.url)
}
if test.expectErr && err == nil {
t.Errorf("unexpected non-error")
@ -87,8 +86,30 @@ func TestValidate(t *testing.T) {
}
}
func makeTestValidator(servers map[string]string, rt http.RoundTripper) (http.Handler, error) {
result := map[string]Server{}
for name, value := range servers {
host, port, err := net.SplitHostPort(value)
if err != nil {
return nil, fmt.Errorf("invalid server spec: %s (%v)", value, err)
}
val, err := strconv.Atoi(port)
if err != nil {
return nil, fmt.Errorf("invalid server spec: %s (%v)", port, err)
}
result[name] = Server{Addr: host, Port: val, Path: "/healthz"}
}
return &validator{servers: func() map[string]Server { return result }, rt: rt}, nil
}
func TestValidator(t *testing.T) {
fake := makeFake("foo", 200, nil)
fake := &fakeRoundTripper{
resp: &http.Response{
Body: ioutil.NopCloser(bytes.NewBufferString("foo")),
StatusCode: 200,
},
}
validator, err := makeTestValidator(map[string]string{
"foo": "foo.com:80",
"bar": "bar.com:8080",
@ -101,7 +122,6 @@ func TestValidator(t *testing.T) {
defer testServer.Close()
resp, err := http.Get(testServer.URL + "/validatez")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
@ -113,13 +133,15 @@ func TestValidator(t *testing.T) {
if err != nil {
t.Errorf("unexpected error: %v", err)
}
status := []ServerStatus{}
err = json.Unmarshal(data, &status)
if err != nil {
var status []ServerStatus
if err := json.Unmarshal(data, &status); err != nil {
t.Errorf("unexpected error: %v", err)
}
components := util.StringSet{}
for _, s := range status {
if s.Err != "nil" {
t.Errorf("Component %v is unhealthy: %v", s.Component, s.Err)
}
components.Insert(s.Component)
}
if len(status) != 2 || !components.Has("foo") || !components.Has("bar") {

11
pkg/client/kubelet.go
View File

@ -75,9 +75,14 @@ type HTTPKubeletClient struct {
func NewKubeletClient(config *KubeletConfig) (KubeletClient, error) {
transport := http.DefaultTransport
tlsConfig, err := TLSConfigFor(&Config{
TLSClientConfig: config.TLSClientConfig,
})
cfg := &Config{TLSClientConfig: config.TLSClientConfig}
if config.EnableHttps {
hasCA := len(config.CAFile) > 0 || len(config.CAData) > 0
if !hasCA {
cfg.Insecure = true
}
}
tlsConfig, err := TLSConfigFor(cfg)
if err != nil {
return nil, err
}

18
pkg/kubelet/server.go
View File

@ -76,6 +76,24 @@ func ListenAndServeKubeletServer(host HostInterface, address net.IP, port uint,
}
}
// ListenAndServeKubeletReadOnlyServer initializes a server to respond to HTTP network requests on the Kubelet.
func ListenAndServeKubeletReadOnlyServer(host HostInterface, address net.IP, port uint) {
glog.V(1).Infof("Starting to listen read-only on %s:%d", address, port)
s := &Server{host, http.NewServeMux()}
healthz.InstallHandler(s.mux)
s.mux.HandleFunc("/stats/", s.handleStats)
s.mux.Handle("/metrics", prometheus.Handler())
server := &http.Server{
Addr: net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),
Handler: s,
ReadTimeout: 5 * time.Minute,
WriteTimeout: 5 * time.Minute,
MaxHeaderBytes: 1 << 20,
}
glog.Fatal(server.ListenAndServe())
}
// HostInterface contains all the kubelet methods required by the server.
// For testablitiy.
type HostInterface interface {

2
pkg/master/master.go
View File

@ -561,7 +561,7 @@ func (m *Master) getServersToValidate(c *Config) map[string]apiserver.Server {
glog.Errorf("Failed to list minions: %v", err)
}
for ix, node := range nodes.Items {
serversToValidate[fmt.Sprintf("node-%d", ix)] = apiserver.Server{Addr: node.Name, Port: ports.KubeletPort, Path: "/healthz"}
serversToValidate[fmt.Sprintf("node-%d", ix)] = apiserver.Server{Addr: node.Name, Port: ports.KubeletPort, Path: "/healthz", EnableHTTPS: true}
}
return serversToValidate
}

6
pkg/master/ports/ports.go
View File

@ -32,4 +32,10 @@ const (
// ControllerManagerPort is the default port for the controller manager status server.
// May be overridden by a flag at startup.
ControllerManagerPort = 10252
// KubeletReadOnlyPort exposes basic read-only services from the kubelet.
// May be overridden by a flag at startup.
// This is necessary for heapster to collect monitoring stats from the kubelet
// until heapster can transition to using the SSL endpoint.
// TODO(roberthbailey): Remove this once we have a better solution for heapster.
KubeletReadOnlyPort = 10255
)