From 31cf2bc9ee0667fe37993c151b86b8904699d634 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Tue, 16 Apr 2019 16:42:23 -0700 Subject: [PATCH 1/8] Add coredns entries for nodes --- manifests/coredns.yaml | 6 +++ pkg/node/controller.go | 115 +++++++++++++++++++++++++++++++++++++++++ pkg/server/server.go | 2 + 3 files changed, 123 insertions(+) create mode 100644 pkg/node/controller.go diff --git a/manifests/coredns.yaml b/manifests/coredns.yaml index fb0ee759f0..4d584f6741 100644 --- a/manifests/coredns.yaml +++ b/manifests/coredns.yaml @@ -60,6 +60,10 @@ data: upstream fallthrough in-addr.arpa ip6.arpa } + hosts /etc/coredns/NodeHosts { + reload 1s + fallthrough + } prometheus :9153 proxy . /etc/resolv.conf cache 30 @@ -146,6 +150,8 @@ spec: items: - key: Corefile path: Corefile + - key: NodeHosts + path: NodeHosts --- apiVersion: v1 kind: Service diff --git a/pkg/node/controller.go b/pkg/node/controller.go new file mode 100644 index 0000000000..10728a9a12 --- /dev/null +++ b/pkg/node/controller.go @@ -0,0 +1,115 @@ +package node + +import ( + "context" + "strings" + + "github.com/pkg/errors" + coreclient "github.com/rancher/k3s/types/apis/core/v1" + "github.com/sirupsen/logrus" + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func Register(ctx context.Context) error { + clients := coreclient.ClientsFrom(ctx) + h := &handler{ + configCache: clients.ConfigMap.Cache(), + configClient: clients.ConfigMap, + } + clients.Node.OnCreate(ctx, "node", h.onChange) + clients.Node.OnChange(ctx, "node", h.onChange) + clients.Node.OnRemove(ctx, "node", h.onRemove) + + return nil +} + +type handler struct { + configCache coreclient.ConfigMapClientCache + configClient coreclient.ConfigMapClient +} + +func (h *handler) onChange(node *core.Node) (runtime.Object, error) { + return h.updateHosts(node, false) +} + +func (h *handler) onRemove(node *core.Node) (runtime.Object, error) { + return h.updateHosts(node, true) +} + +func (h *handler) updateHosts(node *core.Node, removed bool) (runtime.Object, error) { + var ( + newHosts string + nodeUID string + nodeAddress string + nodeEntry string + uidHostsMap map[string]string + ) + nodeUID = string(node.UID) + uidHostsMap = make(map[string]string) + + for _, address := range node.Status.Addresses { + if address.Type == "InternalIP" { + nodeAddress = address.Address + break + } + } + if nodeAddress == "" { + logrus.Errorf("No InternalIP found for node %s", node.Name) + return nil, nil + } + nodeEntry = nodeAddress + " " + node.Name + + configMap, err := h.configCache.Get("kube-system", "coredns") + if err != nil || configMap == nil { + logrus.Warn(errors.Wrap(err, "Unable to fetch coredns config map")) + return nil, nil + } + + hosts := configMap.Data["NodeHosts"] + for _, line := range strings.Split(hosts, "\n") { + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) != 4 || fields[2] != "#" { + logrus.Warnf("Unknown format for hosts line [%s]", line) + continue + } + ip := fields[0] + host := fields[1] + uid := fields[3] + hostEntry := ip + " " + host + if uid == nodeUID { + if removed { + continue + } + if hostEntry == nodeEntry { + return nil, nil + } + } + uidHostsMap[uid] = hostEntry + } + + if !removed { + uidHostsMap[nodeUID] = nodeEntry + } + for uid, hostEntry := range uidHostsMap { + newHosts += hostEntry + " # " + uid + "\n" + } + configMap.Data["NodeHosts"] = newHosts + + if _, err := h.configClient.Update(configMap); err != nil { + return nil, err + } + + var actionType string + if removed { + actionType = "Removed" + } else { + actionType = "Updated" + } + logrus.Infof("%s coredns node hosts entry [%s]", actionType, nodeEntry) + + return nil, nil +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 995a808f28..b44ab34138 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -18,6 +18,7 @@ import ( "github.com/rancher/k3s/pkg/datadir" "github.com/rancher/k3s/pkg/deploy" "github.com/rancher/k3s/pkg/helm" + "github.com/rancher/k3s/pkg/node" "github.com/rancher/k3s/pkg/rootlessports" "github.com/rancher/k3s/pkg/servicelb" "github.com/rancher/k3s/pkg/static" @@ -112,6 +113,7 @@ func startNorman(ctx context.Context, config *Config) (string, error) { }, DisableLeaderElection: true, MasterControllers: []norman.ControllerRegister{ + node.Register, helm.Register, func(ctx context.Context) error { return servicelb.Register(ctx, norman.GetServer(ctx).K8sClient, !config.DisableServiceLB, From 37dd5cbfd2e4607e3668ed2da6aba352cfa8b1dc Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Wed, 17 Apr 2019 22:24:38 +0000 Subject: [PATCH 2/8] Generated data --- pkg/deploy/zz_generated_bindata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index ff26aaf71d..e9876bb5b4 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -69,7 +69,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _corednsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\x4d\x6f\xdb\x38\x13\xbe\xfb\x57\x10\x7a\xd1\xdb\x2b\xc7\x46\xd0\x6e\x96\xb7\xd6\xce\x76\x03\x34\xae\x11\x27\xbd\x2c\x16\x05\x4d\x8d\x2d\x6e\x28\x0e\x97\x1c\xb9\xf1\x76\xf3\xdf\x17\xd4\x57\x28\x45\x2e\xda\xa0\xba\x88\xe4\x70\x9e\x21\xe7\xe3\x19\x0a\xab\x3e\x81\xf3\x0a\x0d\x67\x87\xf9\xe4\x5e\x99\x8c\xb3\x0d\xb8\x83\x92\xf0\x56\x4a\x2c\x0d\x4d\x0a\x20\x91\x09\x12\x7c\xc2\x98\x11\x05\x70\x26\xd1\x41\x66\x7c\x33\xf7\x56\x48\xe0\xec\xbe\xdc\x42\xea\x8f\x9e\xa0\x98\xa4\x69\x3a\x89\xa1\xdd\x56\xc8\xa9\x28\x29\x47\xa7\xfe\x11\xa4\xd0\x4c\xef\x2f\xfc\x54\xe1\xd9\x61\xbe\x05\x12\xad\xe5\x85\x2e\x3d\x81\xbb\x41\x0d\x3d\xb3\x5a\x6c\x41\xfb\x30\x62\x95\x1d\x67\x80\xa0\xd2\xdf\x22\x92\x27\x27\xac\x55\x66\x5f\x1b\x4a\x33\xd8\x89\x52\x93\xef\xce\x5b\x9f\x8a\xb7\xc7\x76\xa5\x06\xcf\x27\x29\x13\x56\xbd\x77\x58\xda\x0a\x39\x65\x49\x32\x61\xcc\x81\xc7\xd2\x49\x68\xd6\xc0\x64\x16\x95\xa9\xc0\x52\xe6\x6b\xcf\xd4\x13\x8b\x59\x3d\xe8\x9c\x10\xa6\x07\x70\xdb\x46\x57\x2b\x4f\xd5\xe0\x8b\x20\x99\x7f\x9f\x3d\x83\xd9\x10\x66\x0f\xf4\x33\x1c\xfa\x4e\x99\x4c\x99\x7d\xcf\xaf\xc2\x18\xa4\x4a\xbd\x71\xee\x18\x6e\xcf\xdf\xa2\x24\x2c\x6d\x26\x08\x38\x4b\xc8\x95\x90\xfc\xfc\xf0\xa0\x86\x1b\xd8\x55\xe7\x6b\x1c\xf6\x8d\x0b\x4f\x18\x7b\x9e\x3b\x27\x90\x7d\xb9\xfd\x0b\x24\x55\xb1\x1f\x4d\xf5\x17\x27\x78\x57\x3b\x0b\x34\x3b\xb5\xbf\x16\xf6\x25\x65\xd3\x6e\x5f\xa0\x83\x9d\xd2\xc0\xd9\xbf\x95\x4f\xa7\xfc\xf5\x39\xfb\x5a\x0d\xc3\x07\xce\xa1\xf3\xdd\x34\x07\xa1\x29\xef\xa6\x4f\x01\x60\xaf\xbe\x2e\x3e\xdc\x6d\x6e\x2f\x6f\x3e\x2f\x3f\x5e\xbf\xbd\x5a\x3d\xbe\x62\xca\xa4\x22\xcb\xdc\x54\x38\x2b\x98\xb2\x6f\xea\xc1\x13\x36\xab\xd2\x9a\x29\xe3\x41\x96\x0e\xa2\xf5\xd2\x7a\x72\x20\x8a\x68\x69\x27\xb4\xa6\xdc\x61\xb9\xcf\xc7\x81\xbb\xbd\x8f\xdd\xc8\x3a\x2c\x80\x72\x28\x3d\xe3\xbf\xce\x5f\x9f\xc7\x82\x87\x23\x9b\xb2\x33\x20\x79\x16\xaa\x42\x1f\xa6\x12\xcd\xae\xdb\x20\x85\xcc\x81\x9d\xcf\xba\x05\x8d\x68\xbb\x89\x03\x8d\x22\x8b\x64\x22\xdb\x0a\x2d\x8c\xac\xef\xf0\xf8\x2c\x64\xf0\x40\x60\xc2\xd0\x0f\x6a\x66\x09\x56\xe3\xb1\x80\x97\x51\xdf\xa0\x1a\x2e\x7c\x2a\xac\x6d\xb6\xd4\x8a\xc3\x1a\xa9\x81\x93\x10\xf4\xe5\x6a\x93\x4c\xbc\x05\x19\xb4\xff\xe7\xc0\x6a\x25\x85\xe7\x6c\x3e\x61\x2c\x94\x11\xc1\xfe\x58\x03\xd3\xd1\x02\x67\x37\xa8\xb5\x32\xfb\xbb\xaa\x20\xeb\x02\x8e\x57\x78\xe3\x8e\x42\x3c\xdc\x19\x71\x10\x4a\x8b\x6d\xc8\xaa\x0a\x0e\x34\x48\x42\x57\xef\x29\x02\x43\x7d\x88\x0e\x3e\x7e\x74\x82\xc2\xea\x0e\x38\xf6\x4e\xe5\xf3\x9e\xfe\xa9\xcb\xb7\xd7\xab\xc6\xbd\xf2\x5b\x0d\x3c\x5c\xdd\x13\x35\xb8\x98\xa1\xc2\x97\xb2\x7b\x38\x06\x97\x39\x45\x4a\x0a\xfd\x36\xcb\xd0\xf8\x8f\x46\x1f\x93\x28\x3b\xd1\x06\x4d\x74\x9c\x25\x97\x0f\xca\x93\x6f\x85\x81\x63\x37\xbd\xeb\x87\x2f\xa4\xc0\x80\xec\xd0\x73\xa6\x95\x29\x1f\x9a\x4d\x12\x0d\x09\x65\xc0\x75\x67\x49\x9f\xa5\x45\xfd\xa9\x42\xec\x9f\x96\xcf\x9a\x3f\x9f\x4f\xcf\xa7\xb3\xfe\xa6\x75\xa9\xf5\x1a\xb5\x92\x47\xce\xae\x76\x2b\xa4\xb5\x03\x0f\x15\x17\xb5\x89\x1d\x35\x88\x2e\xbd\x55\xa1\xa8\xb7\x12\xc2\x51\xa0\x3b\x72\x36\xff\x65\x76\xad\x22\x89\x83\xbf\x4b\xf0\xc3\xdd\xd2\x96\x9c\xcd\x67\xb3\x62\x14\xa3\x07\x21\xdc\xde\x73\xf6\x07\x4b\xd2\x50\x8f\xc9\xff\x59\x52\xd5\x68\x7b\xb9\x96\xad\x12\xf6\x67\xa7\x72\x40\x5d\x16\x70\x1d\xa2\xda\x8b\x5b\xeb\xad\x40\x92\x69\xbd\x29\xb2\x5f\x84\xfd\x6b\x41\x39\x67\xb1\x85\xde\x5d\x44\x16\xe2\xcc\x59\xe8\x3d\x4f\xdc\x81\xae\x6f\xa7\x8b\xd4\x1a\x1d\x71\x16\xd1\x4c\x5b\xc8\x7d\x5c\xeb\x90\x50\xa2\xe6\xec\x6e\xb9\xfe\x51\x9c\x94\xa4\x1d\xc5\xba\x5d\x7c\x03\xab\x47\x7e\x2d\x5a\x01\xe4\x94\x1c\x3f\x59\x8c\x56\xb1\xb3\xa2\xe3\x02\x0d\xc1\x03\xc5\xa1\x15\x5a\xe3\x97\xb5\x53\x07\xa5\x61\x0f\x97\x5e\x0a\x5d\xd5\x0f\x0f\x74\xed\x63\x77\x4b\x61\xc5\x56\x69\x45\x0a\x06\xc9\x21\xb2\xac\xbf\x90\xb2\xd5\xe5\xed\xe7\x77\x57\xab\xe5\xe7\xcd\xe5\xcd\xa7\xab\xc5\x65\x4f\x9c\x39\xb4\x43\x05\xa1\xf5\x48\xe0\x6e\x10\xe9\x37\xa5\xa1\xe9\xcc\xfd\x30\x6a\x75\x00\x03\xde\xaf\x1d\x6e\x21\xc6\xcb\x89\xec\x7b\xa0\xbe\x09\x5b\x27\xca\xa0\xfd\xb1\x26\x1d\x38\xbb\x98\x5d\xcc\x7a\xcb\x5e\xe6\x10\x9c\xfc\xfb\xed\xed\x3a\x12\x28\xa3\x48\x09\xbd\x04\x2d\x8e\x1b\x90\x68\x32\xcf\xd9\x9b\x58\x95\x54\x01\x58\x52\x27\x7c\x1d\xc9\x7c\x29\x25\x78\x7f\x9b\x3b\xf0\x39\xea\xac\x66\xd7\xf6\xdb\x09\xa5\x4b\x07\x91\xb4\xd5\xcd\x8c\x6f\xcb\x7e\x59\x3f\x88\x1a\x41\x5d\x15\x3f\x50\x35\xb2\x7d\x72\xf4\xdd\x33\x4e\x4c\xd5\x85\x09\x0a\x3f\x0c\x57\xc5\xa8\x6d\x29\xf7\x64\xad\xa7\x3b\xe1\xc9\xc7\x4f\xf3\x9a\x1a\x69\x9b\x51\x07\x38\xd9\x37\x9f\x3d\x46\x9f\x9e\x0a\x81\x8c\xeb\xa0\x26\xa1\x6c\x92\x11\xb1\x97\x4e\xd8\x93\x8f\xd2\xef\x68\xc3\xb2\x7e\x3f\xa6\x4d\x4f\x8a\x90\xbe\xb7\x61\xf7\x5b\xea\x98\xcd\xc6\xc6\xd5\x9a\xc7\x6f\xb3\xd5\xe6\xf1\xd5\x24\x22\xb1\x74\x40\x51\x36\xe6\x9e\x21\x53\xa5\x23\x3c\x74\x42\xa1\x26\x90\x74\x84\x6a\x6c\x9f\x91\xfa\x2a\xff\x05\x00\x00\xff\xff\xcd\x5f\xed\xd4\x24\x0e\x00\x00") +var _corednsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x56\xcd\x6e\x1b\x37\x10\xbe\xeb\x29\x88\x2d\x72\xeb\xca\x12\x8c\xa4\x2e\x6f\x89\xe4\x26\x06\x62\x45\xb0\xec\x5c\x8a\x22\xa0\xb8\x23\x89\x35\x97\xc3\x92\xb3\x8a\xd5\xd4\xef\x5e\x70\xff\x44\xca\xeb\x20\x09\xb2\x27\x2e\x87\xf3\xcd\x70\x7e\xbe\xa1\xb0\xea\x23\x38\xaf\xd0\x70\xb6\x9f\x8e\xee\x95\x29\x38\x5b\x81\xdb\x2b\x09\xaf\xa5\xc4\xca\xd0\xa8\x04\x12\x85\x20\xc1\x47\x8c\x19\x51\x02\x67\x12\x1d\x14\xc6\xb7\xff\xde\x0a\x09\x9c\xdd\x57\x6b\xc8\xfd\xc1\x13\x94\xa3\x3c\xcf\x47\x31\xb4\x5b\x0b\x39\x16\x15\xed\xd0\xa9\x7f\x05\x29\x34\xe3\xfb\x0b\x3f\x56\x78\xb6\x9f\xae\x81\x44\x67\x79\xa6\x2b\x4f\xe0\x6e\x50\x43\x62\x56\x8b\x35\x68\x1f\x56\xac\xb6\xe3\x0c\x10\xd4\xfa\x6b\x44\xf2\xe4\x84\xb5\xca\x6c\x1b\x43\x79\x01\x1b\x51\x69\xf2\xbd\xbf\x8d\x57\xbc\x73\xdb\x55\x1a\x3c\x1f\xe5\x4c\x58\xf5\xd6\x61\x65\x6b\xe4\x9c\x65\xd9\x88\x31\x07\x1e\x2b\x27\xa1\xdd\x03\x53\x58\x54\xa6\x06\xcb\x99\x6f\x22\xd3\xfc\x58\x2c\x9a\x45\x1f\x84\xf0\xbb\x07\xb7\x6e\x75\xb5\xf2\x54\x2f\x3e\x0b\x92\xbb\x6f\xb3\x67\xb0\x38\x85\xd9\x02\xfd\x8c\x80\xbe\x51\xa6\x50\x66\x9b\xc4\x55\x18\x83\x54\xab\xb7\xc1\x1d\xc2\x4d\xe2\x2d\x2a\xc2\xca\x16\x82\x80\xb3\x8c\x5c\x05\xd9\xcf\x4f\x0f\x6a\xb8\x81\x4d\xed\x5f\x1b\xb0\xaf\x5c\x78\xc4\xd8\xd3\xda\x79\x06\xd9\x57\xeb\xbf\x41\x52\x9d\xfb\xc1\x52\xff\xe1\x02\xef\x7b\x67\x86\x66\xa3\xb6\xd7\xc2\xfe\x48\xdb\x74\xc7\x67\xe8\x60\xa3\x34\x70\xf6\x5f\x1d\xd3\x31\x7f\x79\xce\xbe\xd4\xcb\xf0\x81\x73\xe8\x7c\xff\xbb\x03\xa1\x69\xd7\xff\x1e\x13\xc0\x5e\x7c\x99\xbd\xbf\x5b\xdd\x5e\xde\x7c\x9a\x7f\xb8\x7e\x7d\xb5\x78\x7c\xc1\x94\xc9\x45\x51\xb8\xb1\x70\x56\x30\x65\x5f\x35\x8b\x23\x36\xab\xcb\x9a\x29\xe3\x41\x56\x0e\xa2\xfd\xca\x7a\x72\x20\xca\x68\x6b\x23\xb4\xa6\x9d\xc3\x6a\xbb\x1b\x06\xee\xcf\x3e\x1e\xbd\x45\x4f\x9e\x9d\x01\xc9\xb3\x36\x1e\x67\x0b\x2c\xe0\x5d\xbd\x1d\xfb\xe1\x40\xa3\x28\xd8\xd4\x0f\x1b\x1c\x80\xb6\x0e\x4b\xa0\x1d\x54\x9e\xf1\xdf\xa7\x2f\xcf\x63\xc1\xc3\x81\x8d\x1b\xab\xa1\xe1\xf4\x7e\x2c\xd1\x6c\xfa\x03\x52\xc8\x1d\xb0\xf3\x49\xbf\xa1\x11\xed\x28\xf5\x24\x92\x89\x62\x2d\xb4\x30\xb2\x09\xcf\xe3\x93\x6a\x80\x07\x02\x13\x96\xfe\xa4\x1d\xe7\x60\x35\x1e\x4a\xf8\x31\x56\x3d\x69\xb4\x0b\x9f\x0b\x6b\xdb\x23\x8d\xe2\x69\xfb\x35\xc0\x59\xa8\xa7\xf9\x62\x95\x8d\xbc\x05\x19\xb4\x7f\x71\x60\xb5\x92\xc2\x73\x36\x1d\x31\x16\x3a\x94\x60\x7b\x68\x80\xe9\x60\x81\xb3\x1b\xd4\x5a\x99\xed\x5d\xdd\xeb\x0d\x37\xc4\x3b\xbc\x0d\x47\x29\x1e\xee\x8c\xd8\x0b\xa5\xc5\x3a\x14\x6c\x0d\x07\x1a\x24\xa1\x6b\xce\x94\x81\xfc\xde\x47\x8e\x0f\xbb\x4e\x50\x5a\xdd\x03\xc7\xd1\xa9\x63\x9e\xe8\x3f\x77\xf9\xee\x7a\xf5\x3a\xe9\xec\xc5\x49\x84\xeb\x7b\xa2\x06\x17\x93\x5f\xf8\x72\x76\x0f\x87\x10\x32\xa7\x48\x49\xa1\x5f\x17\x05\x1a\xff\xc1\xe8\x43\x16\xd5\x21\xda\xa0\x89\x8e\xb3\xec\xf2\x41\x79\xf2\x9d\x30\xd0\xf7\x2a\xb9\x7e\xf8\x42\x09\x9c\xf0\x28\x7a\xce\xb4\x32\xd5\x43\x7b\x48\xa2\x21\xa1\x0c\xb8\xde\x97\xfc\x49\x59\x34\x9f\x2a\xc5\xf6\xb8\xdd\x35\x11\x9f\x8e\xcf\xc7\x93\xf4\xd0\xb2\xd2\x7a\x89\x5a\xc9\x03\x67\x57\x9b\x05\xd2\xd2\x81\x87\x9a\xe6\xba\xc2\x8e\x66\x4f\x5f\xde\xaa\x54\x94\xec\x84\x74\x94\xe8\x0e\x9c\x4d\x7f\x9b\x5c\xab\xa4\x47\xff\xa9\xc0\x9f\x9e\x96\xb6\xe2\x6c\x3a\x99\x94\x83\x18\x09\x84\x70\x5b\xcf\xd9\x9f\x2c\xcb\x43\x3f\x66\xbf\xb2\x2c\x61\x86\x8e\x08\x33\xf6\x57\xaf\xb2\x47\x5d\x95\x70\x1d\xb2\x9a\xe4\xad\x8b\x56\xe0\xdf\xbc\x39\x14\xd9\x2f\xc3\xf9\xa5\xa0\x1d\x4f\xb8\x27\xb9\x8b\x28\x42\x9e\x39\x0b\x63\xed\xc8\x1d\xe8\x52\x3b\x7d\xa6\x96\xe8\x88\xb3\x88\x66\xba\x46\x4e\x71\xad\x43\x42\x89\x9a\xb3\xbb\xf9\xf2\x7b\x71\x72\x92\x76\x10\xeb\x76\xf6\x15\xac\x84\xfc\x3a\xb4\x12\xc8\x29\x39\xec\x59\x8c\x56\x13\xbf\xa2\xc3\x0c\x0d\xc1\x03\xc5\xa9\x15\x5a\xe3\xe7\xa5\x53\x7b\xa5\x61\x0b\x97\x5e\x0a\x5d\xf7\x0f\x0f\xc4\xec\xe3\x70\x4b\x61\xc5\x5a\x69\x45\x0a\x4e\x8a\x43\x14\x45\xba\x91\xb3\xc5\xe5\xed\xa7\x37\x57\x8b\xf9\xa7\xd5\xe5\xcd\xc7\xab\xd9\x65\x22\x2e\x1c\xda\x53\x05\xa1\xf5\x40\xe2\x6e\x10\xe9\x0f\xa5\xa1\x1d\xfa\x69\x1a\xb5\xda\x83\x01\xef\x97\x0e\xd7\x10\xe3\xed\x88\xec\x5b\xa0\xd4\x84\x6d\x0a\xe5\x64\xb2\xb2\xb6\x1c\x38\xbb\x98\x5c\x4c\x92\x6d\x2f\x77\x10\x82\xfc\xee\xf6\x76\x19\x09\x94\x51\xa4\x84\x9e\x83\x16\x87\x15\x48\x34\x85\xe7\xec\x55\xac\x4a\xaa\x04\xac\xa8\x17\xbe\x8c\x64\xbe\x92\x12\xbc\xbf\xdd\x39\xf0\x3b\xd4\x45\xc3\xae\xdd\xb7\x11\x4a\x57\x0e\x22\x69\xa7\x5b\x18\xdf\xb5\xfd\xbc\x79\x6b\xb5\x82\xa6\x2b\xbe\xa3\x6b\x64\xf7\x9a\x49\xc3\x33\x4c\x4c\xf5\x85\x09\x4a\x7f\x9a\xae\x9a\x51\xbb\x56\x4e\x64\x5d\xa4\x07\x85\xad\x62\xff\x3a\x18\xd4\x3c\x4a\x9f\x7d\x92\xb5\x6f\xbc\x81\x89\x1b\x0d\x8f\x67\x47\xee\x93\x27\xf2\xf1\x95\x11\x78\xbc\xa9\x87\x2c\x74\x5c\x36\x20\xf6\xd2\x09\xfb\xec\x53\xf9\x1b\x26\xb8\x6c\x5e\xb5\x79\x3b\xce\x22\xa4\x6f\x9d\xf5\xe9\x34\x1e\xb2\xd9\xda\xb8\x5a\xf2\xf8\xc5\xb8\x58\x3d\xbe\x18\x45\xfc\x97\x9f\xb0\x9b\x8d\x69\xeb\x94\xe4\xf2\x01\x0a\x7b\x46\xa1\xe1\x9e\x7c\x80\xa5\x6c\x4a\x66\xa9\xca\xff\x01\x00\x00\xff\xff\x51\xb6\xd2\x56\xba\x0e\x00\x00") func corednsYamlBytes() ([]byte, error) { return bindataRead( From 1b2db423defcc92be894e0dd0989fd5d6a1c9416 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Fri, 19 Apr 2019 18:20:34 +0000 Subject: [PATCH 3/8] Add node name to node cert generation --- pkg/agent/config/config.go | 65 ++++++++++++++++++++++++++---- pkg/daemons/agent/agent.go | 4 ++ pkg/daemons/config/types.go | 2 + pkg/server/router.go | 79 ++++++++++++++++++++++++++++++++++++- 4 files changed, 141 insertions(+), 9 deletions(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index a7d6d7a19f..1c0ba40144 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -8,6 +8,7 @@ import ( "fmt" "io/ioutil" sysnet "net" + "net/http" "net/url" "os" "os/exec" @@ -43,16 +44,60 @@ func Get(ctx context.Context, agent cmds.Agent) *config.Node { } } -func getNodeCert(info *clientaccess.Info) (*tls.Certificate, error) { - nodeCert, err := clientaccess.Get("/v1-k3s/node.crt", info) +type HTTPRequester func(u string, client *http.Client, username, password string) ([]byte, error) + +func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]byte, error) { + u, err := url.Parse(info.URL) if err != nil { return nil, err } + u.Path = path + username, password, _ := clientaccess.ParseUsernamePassword(info.Token) + return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts), username, password) +} + +func getNodeNamedCrt(nodeName string) HTTPRequester { + return func(u string, client *http.Client, username, password string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return nil, err + } + + if username != "" { + req.SetBasicAuth(username, password) + } + + req.Header.Set("K3s-Node-Name", nodeName) + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s: %s", u, resp.Status) + } + + return ioutil.ReadAll(resp.Body) + } +} + +func getNodeCert(nodeName, nodeCertFile, nodeKeyFile string, info *clientaccess.Info) (*tls.Certificate, error) { + nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName)) + if err != nil { + return nil, err + } + if err := ioutil.WriteFile(nodeCertFile, nodeCert, 0600); err != nil { + return nil, errors.Wrapf(err, "failed to write node cert") + } nodeKey, err := clientaccess.Get("/v1-k3s/node.key", info) if err != nil { return nil, err } + if err := ioutil.WriteFile(nodeKeyFile, nodeKey, 0600); err != nil { + return nil, errors.Wrapf(err, "failed to write node key") + } cert, err := tls.X509KeyPair(nodeCert, nodeKey) if err != nil { @@ -181,7 +226,14 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } - nodeCert, err := getNodeCert(info) + nodeName, nodeIP, err := getHostnameAndIP(*envInfo) + if err != nil { + return nil, err + } + + nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt") + nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key") + nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, info) if err != nil { return nil, err } @@ -191,11 +243,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } - nodeName, nodeIP, err := getHostnameAndIP(*envInfo) - if err != nil { - return nil, err - } - kubeConfig, err := writeKubeConfig(envInfo, *info, controlConfig, nodeCert) if err != nil { return nil, err @@ -224,6 +271,8 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.Images = filepath.Join(envInfo.DataDir, "images") nodeConfig.AgentConfig.NodeIP = nodeIP nodeConfig.AgentConfig.NodeName = nodeName + nodeConfig.AgentConfig.NodeCertFile = nodeCertFile + nodeConfig.AgentConfig.NodeKeyFile = nodeKeyFile nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 3417dc206a..9e8014beb2 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -94,6 +94,10 @@ func kubelet(cfg *config.Agent) { argsMap["anonymous-auth"] = "false" argsMap["client-ca-file"] = cfg.CACertPath } + if cfg.NodeCertFile != "" && cfg.NodeKeyFile != "" { + argsMap["tls-cert-file"] = cfg.NodeCertFile + argsMap["tls-private-key-file"] = cfg.NodeKeyFile + } if cfg.NodeName != "" { argsMap["hostname-override"] = cfg.NodeName } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 577f26eb3b..ddbe3474f5 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -36,6 +36,8 @@ type Containerd struct { type Agent struct { NodeName string + NodeCertFile string + NodeKeyFile string ClusterCIDR net.IPNet ClusterDNS net.IP ClusterDomain string diff --git a/pkg/server/router.go b/pkg/server/router.go index 0626cae997..7665e78693 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -1,6 +1,10 @@ package server import ( + "crypto/rsa" + "crypto/x509" + "io/ioutil" + "net" "net/http" "path/filepath" "strconv" @@ -8,7 +12,10 @@ import ( "github.com/gorilla/mux" "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/openapi" + certutil "github.com/rancher/norman/pkg/cert" + "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" + "k8s.io/kubernetes/pkg/master" ) const ( @@ -59,7 +66,71 @@ func nodeCrt(server *config.Control) http.Handler { resp.WriteHeader(http.StatusNotFound) return } - http.ServeFile(resp, req, server.Runtime.NodeCert) + + var nodeName string + nodeNames := req.Header["K3s-Node-Name"] + if len(nodeNames) == 1 { + nodeName = nodeNames[0] + } + + nodeKey, err := ioutil.ReadFile(server.Runtime.NodeKey) + if err != nil { + sendError(err, resp) + return + } + + key, err := certutil.ParsePrivateKeyPEM(nodeKey) + if err != nil { + sendError(err, resp) + return + } + + caKeyBytes, err := ioutil.ReadFile(server.Runtime.TokenCAKey) + if err != nil { + sendError(err, resp) + return + } + + caBytes, err := ioutil.ReadFile(server.Runtime.TokenCA) + if err != nil { + sendError(err, resp) + return + } + + caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) + if err != nil { + sendError(err, resp) + return + } + + caCert, err := certutil.ParseCertsPEM(caBytes) + if err != nil { + sendError(err, resp) + return + } + + _, apiServerServiceIP, err := master.DefaultServiceIPRange(*server.ServiceIPRange) + if err != nil { + sendError(err, resp) + return + } + + cfg := certutil.Config{ + CommonName: "kubernetes", + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + AltNames: certutil.AltNames{ + DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeName}, + IPs: []net.IP{apiServerServiceIP, net.ParseIP("127.0.0.1")}, + }, + } + + cert, err := certutil.NewSignedCert(cfg, key.(*rsa.PrivateKey), caCert[0], caKey.(*rsa.PrivateKey)) + if err != nil { + sendError(err, resp) + return + } + + resp.Write(append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...)) }) } @@ -118,3 +189,9 @@ func ping() http.Handler { func serveStatic(urlPrefix, staticDir string) http.Handler { return http.StripPrefix(urlPrefix, http.FileServer(http.Dir(staticDir))) } + +func sendError(err error, resp http.ResponseWriter) { + logrus.Error(err) + resp.WriteHeader(http.StatusInternalServerError) + resp.Write([]byte(err.Error())) +} From 055a574fee024e05c0cd022efe77374a12aad8ea Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Mon, 22 Apr 2019 16:13:16 -0700 Subject: [PATCH 4/8] Simplify DNS hosts creation --- pkg/node/controller.go | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/pkg/node/controller.go b/pkg/node/controller.go index 10728a9a12..32e851295c 100644 --- a/pkg/node/controller.go +++ b/pkg/node/controller.go @@ -40,13 +40,10 @@ func (h *handler) onRemove(node *core.Node) (runtime.Object, error) { func (h *handler) updateHosts(node *core.Node, removed bool) (runtime.Object, error) { var ( newHosts string - nodeUID string nodeAddress string - nodeEntry string - uidHostsMap map[string]string + hostsMap map[string]string ) - nodeUID = string(node.UID) - uidHostsMap = make(map[string]string) + hostsMap = make(map[string]string) for _, address := range node.Status.Addresses { if address.Type == "InternalIP" { @@ -58,7 +55,6 @@ func (h *handler) updateHosts(node *core.Node, removed bool) (runtime.Object, er logrus.Errorf("No InternalIP found for node %s", node.Name) return nil, nil } - nodeEntry = nodeAddress + " " + node.Name configMap, err := h.configCache.Get("kube-system", "coredns") if err != nil || configMap == nil { @@ -72,30 +68,28 @@ func (h *handler) updateHosts(node *core.Node, removed bool) (runtime.Object, er continue } fields := strings.Fields(line) - if len(fields) != 4 || fields[2] != "#" { + if len(fields) != 2 { logrus.Warnf("Unknown format for hosts line [%s]", line) continue } ip := fields[0] host := fields[1] - uid := fields[3] - hostEntry := ip + " " + host - if uid == nodeUID { + if host == node.Name { if removed { continue } - if hostEntry == nodeEntry { + if ip == nodeAddress { return nil, nil } } - uidHostsMap[uid] = hostEntry + hostsMap[host] = ip } if !removed { - uidHostsMap[nodeUID] = nodeEntry + hostsMap[node.Name] = nodeAddress } - for uid, hostEntry := range uidHostsMap { - newHosts += hostEntry + " # " + uid + "\n" + for host, ip := range hostsMap { + newHosts += ip + " " + host + "\n" } configMap.Data["NodeHosts"] = newHosts @@ -109,7 +103,7 @@ func (h *handler) updateHosts(node *core.Node, removed bool) (runtime.Object, er } else { actionType = "Updated" } - logrus.Infof("%s coredns node hosts entry [%s]", actionType, nodeEntry) + logrus.Infof("%s coredns node hosts entry [%s]", actionType, nodeAddress+" "+node.Name) return nil, nil } From e64c0298f27961bcf7466030c28be6ee796ab680 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Tue, 23 Apr 2019 10:53:06 -0700 Subject: [PATCH 5/8] Add cert per-node password authentication --- pkg/agent/config/config.go | 30 ++++++++++++++--- pkg/server/router.go | 68 ++++++++++++++++++++++++++++++++++---- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 1c0ba40144..4acf6d6d5b 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -3,7 +3,9 @@ package config import ( "bufio" "context" + cryptorand "crypto/rand" "crypto/tls" + "encoding/hex" "encoding/pem" "fmt" "io/ioutil" @@ -56,7 +58,7 @@ func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]b return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts), username, password) } -func getNodeNamedCrt(nodeName string) HTTPRequester { +func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester { return func(u string, client *http.Client, username, password string) ([]byte, error) { req, err := http.NewRequest(http.MethodGet, u, nil) if err != nil { @@ -68,6 +70,12 @@ func getNodeNamedCrt(nodeName string) HTTPRequester { } req.Header.Set("K3s-Node-Name", nodeName) + nodePassword, err := ensureNodePassword(nodePasswordFile) + if err != nil { + return nil, err + } + req.Header.Set("K3s-Node-Password", hex.EncodeToString(nodePassword)) + resp, err := client.Do(req) if err != nil { return nil, err @@ -82,8 +90,20 @@ func getNodeNamedCrt(nodeName string) HTTPRequester { } } -func getNodeCert(nodeName, nodeCertFile, nodeKeyFile string, info *clientaccess.Info) (*tls.Certificate, error) { - nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName)) +func ensureNodePassword(nodePasswordFile string) ([]byte, error) { + if _, err := os.Stat(nodePasswordFile); err == nil { + return ioutil.ReadFile(nodePasswordFile) + } + password := make([]byte, 16, 16) + _, err := cryptorand.Read(password) + if err != nil { + return nil, err + } + return password, ioutil.WriteFile(nodePasswordFile, password, 0600) +} + +func getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) { + nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile)) if err != nil { return nil, err } @@ -233,7 +253,9 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt") nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key") - nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, info) + nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.bin") + + nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile, info) if err != nil { return nil, err } diff --git a/pkg/server/router.go b/pkg/server/router.go index 7665e78693..84c3a739aa 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -1,13 +1,18 @@ package server import ( + "bufio" "crypto/rsa" "crypto/x509" + "errors" + "fmt" "io/ioutil" "net" "net/http" + "os" "path/filepath" "strconv" + "strings" "github.com/gorilla/mux" "github.com/rancher/k3s/pkg/daemons/config" @@ -67,10 +72,21 @@ func nodeCrt(server *config.Control) http.Handler { return } - var nodeName string nodeNames := req.Header["K3s-Node-Name"] - if len(nodeNames) == 1 { - nodeName = nodeNames[0] + if len(nodeNames) != 1 || nodeNames[0] == "" { + sendError(errors.New("node name not set"), resp) + return + } + + nodePasswords := req.Header["K3s-Node-Password"] + if len(nodePasswords) != 1 || nodePasswords[0] == "" { + sendError(errors.New("node password not set"), resp) + return + } + + if err := ensureNodePassword(server.Runtime.PasswdFile, nodeNames[0], nodePasswords[0]); err != nil { + sendError(err, resp, http.StatusForbidden) + return } nodeKey, err := ioutil.ReadFile(server.Runtime.NodeKey) @@ -119,7 +135,7 @@ func nodeCrt(server *config.Control) http.Handler { CommonName: "kubernetes", Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, AltNames: certutil.AltNames{ - DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeName}, + DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeNames[0]}, IPs: []net.IP{apiServerServiceIP, net.ParseIP("127.0.0.1")}, }, } @@ -190,8 +206,48 @@ func serveStatic(urlPrefix, staticDir string) http.Handler { return http.StripPrefix(urlPrefix, http.FileServer(http.Dir(staticDir))) } -func sendError(err error, resp http.ResponseWriter) { +func sendError(err error, resp http.ResponseWriter, status ...int) { + code := http.StatusInternalServerError + if len(status) == 1 { + code = status[0] + } + logrus.Error(err) - resp.WriteHeader(http.StatusInternalServerError) + resp.WriteHeader(code) resp.Write([]byte(err.Error())) } + +func ensureNodePassword(passwdFile, nodeName, passwd string) error { + f, err := os.Open(passwdFile) + if err != nil { + return err + } + defer f.Close() + user := strings.ToLower("node:" + nodeName) + + buf := &strings.Builder{} + scan := bufio.NewScanner(f) + for scan.Scan() { + line := scan.Text() + parts := strings.Split(line, ",") + if len(parts) < 4 { + continue + } + if parts[1] == user { + if parts[0] == passwd { + return nil + } + return fmt.Errorf("Node password validation failed for [%s]", nodeName) + } + buf.WriteString(line) + buf.WriteString("\n") + } + buf.WriteString(fmt.Sprintf("%s,%s,%s,system:masters\n", passwd, user, user)) + + if scan.Err() != nil { + return scan.Err() + } + + f.Close() + return ioutil.WriteFile(passwdFile, []byte(buf.String()), 0600) +} From f584197bbae07ba17d58126ac3b95075a6446f1c Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Thu, 25 Apr 2019 10:53:21 -0700 Subject: [PATCH 6/8] Save password as text file --- pkg/agent/config/config.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 4acf6d6d5b..6f075fd37e 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -74,7 +74,7 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester { if err != nil { return nil, err } - req.Header.Set("K3s-Node-Password", hex.EncodeToString(nodePassword)) + req.Header.Set("K3s-Node-Password", nodePassword) resp, err := client.Do(req) if err != nil { @@ -90,16 +90,18 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester { } } -func ensureNodePassword(nodePasswordFile string) ([]byte, error) { +func ensureNodePassword(nodePasswordFile string) (string, error) { if _, err := os.Stat(nodePasswordFile); err == nil { - return ioutil.ReadFile(nodePasswordFile) + password, err := ioutil.ReadFile(nodePasswordFile) + return strings.TrimSpace(string(password)), err } password := make([]byte, 16, 16) _, err := cryptorand.Read(password) if err != nil { - return nil, err + return "", err } - return password, ioutil.WriteFile(nodePasswordFile, password, 0600) + nodePassword := hex.EncodeToString(password) + return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword), 0600) } func getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) { @@ -253,7 +255,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt") nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key") - nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.bin") + nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt") nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile, info) if err != nil { From 305b5967450901d6719d51fa598f2a267b4500c0 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Fri, 26 Apr 2019 10:50:33 -0700 Subject: [PATCH 7/8] Remove node OnCreate --- pkg/node/controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/node/controller.go b/pkg/node/controller.go index 32e851295c..c3bf3e2428 100644 --- a/pkg/node/controller.go +++ b/pkg/node/controller.go @@ -17,7 +17,6 @@ func Register(ctx context.Context) error { configCache: clients.ConfigMap.Cache(), configClient: clients.ConfigMap, } - clients.Node.OnCreate(ctx, "node", h.onChange) clients.Node.OnChange(ctx, "node", h.onChange) clients.Node.OnRemove(ctx, "node", h.onRemove) From c9941895d6d7acd7802d3db731c4245f6f810bb8 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Fri, 26 Apr 2019 15:02:30 -0700 Subject: [PATCH 8/8] Bind kubelet to all interfaces and use webhook auth --- pkg/agent/config/config.go | 2 +- pkg/daemons/agent/agent.go | 2 ++ pkg/daemons/control/server.go | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 6f075fd37e..4cca2476af 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -301,7 +301,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) nodeConfig.AgentConfig.CACertPath = clientCA - nodeConfig.AgentConfig.ListenAddress = "127.0.0.1" + nodeConfig.AgentConfig.ListenAddress = "0.0.0.0" nodeConfig.AgentConfig.KubeConfig = kubeConfig nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet") nodeConfig.CACerts = info.CACerts diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 9e8014beb2..b4f2f635dc 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -16,6 +16,7 @@ import ( "k8s.io/component-base/logs" app2 "k8s.io/kubernetes/cmd/kube-proxy/app" "k8s.io/kubernetes/cmd/kubelet/app" + "k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes" _ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration _ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration @@ -64,6 +65,7 @@ func kubelet(cfg *config.Agent) { //"cgroup-root": "/k3s", "cgroup-driver": "cgroupfs", "authentication-token-webhook": "true", + "authorization-mode": modes.ModeWebhook, } if cfg.RootDir != "" { argsMap["root-dir"] = cfg.RootDir diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 4bd21bae02..58d5d5fc5b 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -455,8 +455,8 @@ func genTokenCerts(config *config.Control, runtime *config.ControlRuntime) error return err } - if err := createClientCertKey(regen, "kubernetes", - nil, &certutil.AltNames{ + if err := createClientCertKey(regen, "kubernetes", []string{"system:masters"}, + &certutil.AltNames{ DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}, IPs: []net.IP{apiServerServiceIP, localhostIP}, }, x509KeyClientUsage,