From 8d979d675ead4a470cbc6752fbc75ed278dd6b5f Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Sun, 30 Jun 2019 08:28:42 -0700 Subject: [PATCH 1/2] Add tls support for etcd cert storage backend --- pkg/daemons/control/ha.go | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pkg/daemons/control/ha.go b/pkg/daemons/control/ha.go index 6f1784c616..759461e735 100644 --- a/pkg/daemons/control/ha.go +++ b/pkg/daemons/control/ha.go @@ -2,6 +2,8 @@ package control import ( "context" + "crypto/tls" + "crypto/x509" "encoding/json" "io/ioutil" "os" @@ -34,11 +36,16 @@ func setHAData(cfg *config.Control) error { if cfg.StorageBackend != "etcd3" || cfg.CertStorageBackend != "etcd3" { return nil } + tlsConfig, err := genTLSConfig(cfg) + if err != nil { + return err + } endpoints := strings.Split(cfg.StorageEndpoint, ",") cli, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, DialTimeout: etcdDialTimeout, + TLS: tlsConfig, }) if err != nil { return err @@ -71,10 +78,16 @@ func getHAData(cfg *config.Control) error { if cfg.StorageBackend != "etcd3" || cfg.CertStorageBackend != "etcd3" { return nil } + tlsConfig, err := genTLSConfig(cfg) + if err != nil { + return err + } + endpoints := strings.Split(cfg.StorageEndpoint, ",") cli, err := clientv3.New(clientv3.Config{ Endpoints: endpoints, DialTimeout: etcdDialTimeout, + TLS: tlsConfig, }) if err != nil { return err @@ -99,6 +112,35 @@ func getHAData(cfg *config.Control) error { return writeRuntimeCertData(cfg.Runtime, serverRuntime) } +func genTLSConfig(cfg *config.Control) (*tls.Config, error) { + tlsConfig := &tls.Config{} + if cfg.StorageCertFile != "" && cfg.StorageKeyFile != "" { + certPem, err := ioutil.ReadFile(cfg.StorageCertFile) + if err != nil { + return nil, err + } + keyPem, err := ioutil.ReadFile(cfg.StorageKeyFile) + if err != nil { + return nil, err + } + tlsCert, err := tls.X509KeyPair(certPem, keyPem) + if err != nil { + return nil, err + } + tlsConfig.Certificates = []tls.Certificate{tlsCert} + } + if cfg.StorageCAFile != "" { + caData, err := ioutil.ReadFile(cfg.StorageCAFile) + if err != nil { + return nil, err + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(caData) + tlsConfig.RootCAs = certPool + } + return tlsConfig, nil +} + func readRuntimeCertData(runtime *config.ControlRuntime) ([]byte, error) { serverHACerts := map[string]string{ runtime.ServerCA: "", From 24b73403c79a65defc99f4c19d2952ef75129d91 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Sun, 30 Jun 2019 12:39:54 -0700 Subject: [PATCH 2/2] Cleanup bootstrap --- pkg/cli/cmds/server.go | 8 +- pkg/cli/server/server.go | 2 +- pkg/daemons/config/types.go | 2 +- pkg/daemons/control/{ha.go => bootstrap.go} | 165 ++++++++++++-------- pkg/daemons/control/server.go | 4 +- 5 files changed, 112 insertions(+), 69 deletions(-) rename pkg/daemons/control/{ha.go => bootstrap.go} (59%) diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 6826c9b007..52bb444d71 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -23,7 +23,7 @@ type Server struct { ExtraSchedulerArgs cli.StringSlice ExtraControllerArgs cli.StringSlice Rootless bool - CertStorageBackend string + BootstrapType string StorageBackend string StorageEndpoint string StorageCAFile string @@ -146,9 +146,9 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Destination: &ServerConfig.Rootless, }, cli.StringFlag{ - Name: "cert-storage-backend", - Usage: "(experimental) Specify storage type for storing certificate information", - Destination: &ServerConfig.CertStorageBackend, + Name: "bootstrap", + Usage: "(experimental) Specify data bootstrap behavior (one of: none, read, write, or full), etcd3 only", + Destination: &ServerConfig.BootstrapType, }, cli.StringFlag{ Name: "storage-backend", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 297ed73cee..05d06fed94 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -124,7 +124,7 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort - serverConfig.ControlConfig.CertStorageBackend = cfg.CertStorageBackend + serverConfig.ControlConfig.BootstrapType = cfg.BootstrapType if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" { serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index ae7f39de02..7a0d4104ef 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -81,7 +81,7 @@ type Control struct { KubeConfigMode string DataDir string Skips []string - CertStorageBackend string + BootstrapType string StorageBackend string StorageEndpoint string StorageCAFile string diff --git a/pkg/daemons/control/ha.go b/pkg/daemons/control/bootstrap.go similarity index 59% rename from pkg/daemons/control/ha.go rename to pkg/daemons/control/bootstrap.go index 759461e735..b5ab233942 100644 --- a/pkg/daemons/control/ha.go +++ b/pkg/daemons/control/bootstrap.go @@ -5,6 +5,8 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" + "errors" + "fmt" "io/ioutil" "os" "strings" @@ -13,15 +15,20 @@ import ( "encoding/base64" "github.com/rancher/k3s/pkg/daemons/config" + "github.com/sirupsen/logrus" "go.etcd.io/etcd/clientv3" ) const ( etcdDialTimeout = 5 * time.Second k3sRuntimeEtcdPath = "/k3s/runtime" + bootstrapTypeNone = "none" + bootstrapTypeRead = "read" + bootstrapTypeWrite = "write" + bootstrapTypeFull = "full" ) -type serverHA struct { +type serverBootstrap struct { ServerCAData string `json:"serverCAData,omitempty"` ServerCAKeyData string `json:"serverCAKeyData,omitempty"` ClientCAData string `json:"clientCAData,omitempty"` @@ -32,53 +39,24 @@ type serverHA struct { RequestHeaderCAKeyData string `json:"requestHeaderCAKeyData,omitempty"` } -func setHAData(cfg *config.Control) error { - if cfg.StorageBackend != "etcd3" || cfg.CertStorageBackend != "etcd3" { - return nil - } - tlsConfig, err := genTLSConfig(cfg) - if err != nil { - return err - } - - endpoints := strings.Split(cfg.StorageEndpoint, ",") - cli, err := clientv3.New(clientv3.Config{ - Endpoints: endpoints, - DialTimeout: etcdDialTimeout, - TLS: tlsConfig, - }) - if err != nil { - return err - } - defer cli.Close() - - gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) - if err != nil { - return err - } - if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" { - return nil - } - certData, err := readRuntimeCertData(cfg.Runtime) - if err != nil { - return err - } - - runtimeBase64 := base64.StdEncoding.EncodeToString(certData) - _, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64) - if err != nil { - return err - } - - return nil +var validBootstrapTypes = map[string]bool{ + bootstrapTypeRead: true, + bootstrapTypeWrite: true, + bootstrapTypeFull: true, } -func getHAData(cfg *config.Control) error { - serverRuntime := &serverHA{} - if cfg.StorageBackend != "etcd3" || cfg.CertStorageBackend != "etcd3" { +func fetchBootstrapData(cfg *config.Control) error { + if valid, err := checkBootstrapArgs(cfg, map[string]bool{ + bootstrapTypeFull: true, + bootstrapTypeRead: true, + }); !valid { + if err != nil { + logrus.Warnf("Not fetching bootstrap data: %v", err) + } return nil } - tlsConfig, err := genTLSConfig(cfg) + + tlsConfig, err := genBootstrapTLSConfig(cfg) if err != nil { return err } @@ -106,13 +84,78 @@ func getHAData(cfg *config.Control) error { if err != nil { return err } + serverRuntime := &serverBootstrap{} if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil { return err } - return writeRuntimeCertData(cfg.Runtime, serverRuntime) + return writeRuntimeBootstrapData(cfg.Runtime, serverRuntime) } -func genTLSConfig(cfg *config.Control) (*tls.Config, error) { +func storeBootstrapData(cfg *config.Control) error { + if valid, err := checkBootstrapArgs(cfg, map[string]bool{ + bootstrapTypeFull: true, + bootstrapTypeWrite: true, + }); !valid { + if err != nil { + logrus.Warnf("Not storing boostrap data: %v", err) + } + return nil + } + + tlsConfig, err := genBootstrapTLSConfig(cfg) + if err != nil { + return err + } + + endpoints := strings.Split(cfg.StorageEndpoint, ",") + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: etcdDialTimeout, + TLS: tlsConfig, + }) + if err != nil { + return err + } + defer cli.Close() + + gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) + if err != nil { + return err + } + if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" { + return nil + } + certData, err := readRuntimeBootstrapData(cfg.Runtime) + if err != nil { + return err + } + + runtimeBase64 := base64.StdEncoding.EncodeToString(certData) + _, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64) + if err != nil { + return err + } + + return nil +} + +func checkBootstrapArgs(cfg *config.Control, accepted map[string]bool) (bool, error) { + if cfg.BootstrapType == "" || cfg.BootstrapType == bootstrapTypeNone { + return false, nil + } + if !validBootstrapTypes[cfg.BootstrapType] { + return false, fmt.Errorf("unsupported bootstrap type [%s]", cfg.BootstrapType) + } + if cfg.StorageBackend != "etcd3" { + return false, errors.New("bootstrap only supported with etcd3 as storage backend") + } + if !accepted[cfg.BootstrapType] { + return false, nil + } + return true, nil +} + +func genBootstrapTLSConfig(cfg *config.Control) (*tls.Config, error) { tlsConfig := &tls.Config{} if cfg.StorageCertFile != "" && cfg.StorageKeyFile != "" { certPem, err := ioutil.ReadFile(cfg.StorageCertFile) @@ -141,8 +184,8 @@ func genTLSConfig(cfg *config.Control) (*tls.Config, error) { return tlsConfig, nil } -func readRuntimeCertData(runtime *config.ControlRuntime) ([]byte, error) { - serverHACerts := map[string]string{ +func readRuntimeBootstrapData(runtime *config.ControlRuntime) ([]byte, error) { + serverBootstrapFiles := map[string]string{ runtime.ServerCA: "", runtime.ServerCAKey: "", runtime.ClientCA: "", @@ -152,27 +195,27 @@ func readRuntimeCertData(runtime *config.ControlRuntime) ([]byte, error) { runtime.RequestHeaderCA: "", runtime.RequestHeaderCAKey: "", } - for k := range serverHACerts { + for k := range serverBootstrapFiles { data, err := ioutil.ReadFile(k) if err != nil { return nil, err } - serverHACerts[k] = string(data) + serverBootstrapFiles[k] = string(data) } - serverHACertsData := &serverHA{ - ServerCAData: serverHACerts[runtime.ServerCA], - ServerCAKeyData: serverHACerts[runtime.ServerCAKey], - ClientCAData: serverHACerts[runtime.ClientCA], - ClientCAKeyData: serverHACerts[runtime.ClientCAKey], - ServiceKeyData: serverHACerts[runtime.ServiceKey], - PasswdFileData: serverHACerts[runtime.PasswdFile], - RequestHeaderCAData: serverHACerts[runtime.RequestHeaderCA], - RequestHeaderCAKeyData: serverHACerts[runtime.RequestHeaderCAKey], + serverBootstrapFileData := &serverBootstrap{ + ServerCAData: serverBootstrapFiles[runtime.ServerCA], + ServerCAKeyData: serverBootstrapFiles[runtime.ServerCAKey], + ClientCAData: serverBootstrapFiles[runtime.ClientCA], + ClientCAKeyData: serverBootstrapFiles[runtime.ClientCAKey], + ServiceKeyData: serverBootstrapFiles[runtime.ServiceKey], + PasswdFileData: serverBootstrapFiles[runtime.PasswdFile], + RequestHeaderCAData: serverBootstrapFiles[runtime.RequestHeaderCA], + RequestHeaderCAKeyData: serverBootstrapFiles[runtime.RequestHeaderCAKey], } - return json.Marshal(serverHACertsData) + return json.Marshal(serverBootstrapFileData) } -func writeRuntimeCertData(runtime *config.ControlRuntime, runtimeData *serverHA) error { +func writeRuntimeBootstrapData(runtime *config.ControlRuntime, runtimeData *serverBootstrap) error { runtimePathValue := map[string]string{ runtime.ServerCA: runtimeData.ServerCAData, runtime.ServerCAKey: runtimeData.ServerCAKeyData, diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 7b0c8be358..2c9ffdb78a 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -290,7 +290,7 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt") runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") - if err := getHAData(config); err != nil { + if err := fetchBootstrapData(config); err != nil { return err } @@ -306,7 +306,7 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { return err } - if err := setHAData(config); err != nil { + if err := storeBootstrapData(config); err != nil { return err }