From 28d9d83be2cedf2e4f7a9284872c74fafc8ae037 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Thu, 27 Jun 2019 21:00:43 +0200 Subject: [PATCH] Add k3s HA bootstrap --- pkg/cli/server/server.go | 5 +- pkg/daemons/control/ha.go | 151 ++++++++++++++++++++++++++++++++++ pkg/daemons/control/server.go | 8 ++ 3 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 pkg/daemons/control/ha.go diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index fa4f8f1141..90eb338996 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -157,8 +157,9 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.ClusterDNS = net2.ParseIP(cfg.ClusterDNS) } - // TODO: support etcd - serverConfig.ControlConfig.NoLeaderElect = true + if serverConfig.ControlConfig.StorageBackend != "etcd3" { + serverConfig.ControlConfig.NoLeaderElect = true + } for _, noDeploy := range app.StringSlice("no-deploy") { if noDeploy == "servicelb" { diff --git a/pkg/daemons/control/ha.go b/pkg/daemons/control/ha.go new file mode 100644 index 0000000000..036f697476 --- /dev/null +++ b/pkg/daemons/control/ha.go @@ -0,0 +1,151 @@ +package control + +import ( + "context" + "encoding/json" + "io/ioutil" + "os" + "strings" + "time" + + "encoding/base64" + + "github.com/rancher/k3s/pkg/daemons/config" + "go.etcd.io/etcd/clientv3" +) + +const ( + etcdDialTimeout = 5 * time.Second + k3sRuntimeEtcdPath = "/k3s/runtime" +) + +type serverHA struct { + ServerCAData string `json:"serverCAData,omitempty"` + ServerCAKeyData string `json:"serverCAKeyData,omitempty"` + ClientCAData string `json:"clientCAData,omitempty"` + ClientCAKeyData string `json:"clientCAKeyData,omitempty"` + ServiceKeyData string `json:"serviceKeyData,omitempty"` + PasswdFileData string `json:"passwdFileData,omitempty"` + RequestHeaderCAData string `json:"requestHeaderCAData,omitempty"` + RequestHeaderCAKeyData string `json:"requestHeaderCAKeyData,omitempty"` +} + +func setHAData(cfg *config.Control) error { + if cfg.StorageBackend != "etcd3" { + return nil + } + endpoints := strings.Split(cfg.StorageEndpoint, ",") + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: etcdDialTimeout, + }) + if err != nil { + return err + } + defer cli.Close() + + gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) + if err != nil { + return err + } + if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" { + return nil + } + certData, err := readRuntimeCertData(cfg.Runtime) + if err != nil { + return err + } + + runtimeBase64 := base64.StdEncoding.EncodeToString(certData) + _, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64) + if err != nil { + return err + } + + return nil +} + +func getHAData(cfg *config.Control) error { + serverRuntime := &serverHA{} + if cfg.StorageBackend != "etcd3" { + return nil + } + endpoints := strings.Split(cfg.StorageEndpoint, ",") + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: etcdDialTimeout, + }) + if err != nil { + return err + } + defer cli.Close() + + gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) + if err != nil { + return err + } + if len(gr.Kvs) == 0 { + return nil + } + + runtimeJSON, err := base64.URLEncoding.DecodeString(string(gr.Kvs[0].Value)) + if err != nil { + return err + } + if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil { + return err + } + return writeRuntimeCertData(cfg.Runtime, serverRuntime) +} + +func readRuntimeCertData(runtime *config.ControlRuntime) ([]byte, error) { + serverHACerts := map[string]string{ + runtime.ServerCA: "", + runtime.ServerCAKey: "", + runtime.ClientCA: "", + runtime.ClientCAKey: "", + runtime.ServiceKey: "", + runtime.PasswdFile: "", + runtime.RequestHeaderCA: "", + runtime.RequestHeaderCAKey: "", + } + for k := range serverHACerts { + data, err := ioutil.ReadFile(k) + if err != nil { + return nil, err + } + serverHACerts[k] = string(data) + } + serverHACertsData := &serverHA{ + ServerCAData: serverHACerts[runtime.ServerCA], + ServerCAKeyData: serverHACerts[runtime.ServerCAKey], + ClientCAData: serverHACerts[runtime.ClientCA], + ClientCAKeyData: serverHACerts[runtime.ClientCAKey], + ServiceKeyData: serverHACerts[runtime.ServiceKey], + PasswdFileData: serverHACerts[runtime.PasswdFile], + RequestHeaderCAData: serverHACerts[runtime.RequestHeaderCA], + RequestHeaderCAKeyData: serverHACerts[runtime.RequestHeaderCAKey], + } + return json.Marshal(serverHACertsData) +} + +func writeRuntimeCertData(runtime *config.ControlRuntime, runtimeData *serverHA) error { + runtimePathValue := map[string]string{ + runtime.ServerCA: runtimeData.ServerCAData, + runtime.ServerCAKey: runtimeData.ServerCAKeyData, + runtime.ClientCA: runtimeData.ClientCAData, + runtime.ClientCAKey: runtimeData.ClientCAKeyData, + runtime.ServiceKey: runtimeData.ServiceKeyData, + runtime.PasswdFile: runtimeData.PasswdFileData, + runtime.RequestHeaderCA: runtimeData.RequestHeaderCAData, + runtime.RequestHeaderCAKey: runtimeData.RequestHeaderCAKeyData, + } + for k, v := range runtimePathValue { + if _, err := os.Stat(k); os.IsNotExist(err) { + if err := ioutil.WriteFile(k, []byte(v), 600); err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 88e1ed269d..7b0c8be358 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -290,6 +290,10 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt") runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") + if err := getHAData(config); err != nil { + return err + } + if err := genCerts(config, runtime); err != nil { return err } @@ -302,6 +306,10 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { return err } + if err := setHAData(config); err != nil { + return err + } + return readTokens(runtime) }