diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 43616fb7d8..e3dc3e046e 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -22,8 +22,6 @@ type Server struct { ExtraSchedulerArgs cli.StringSlice ExtraControllerArgs cli.StringSlice Rootless bool - BootstrapType string - StorageBackend string StorageEndpoint string StorageCAFile string StorageCertFile string @@ -144,17 +142,6 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Usage: "(experimental) Run rootless", Destination: &ServerConfig.Rootless, }, - cli.StringFlag{ - Name: "bootstrap", - Usage: "(experimental) Specify data bootstrap behavior (one of: none, read, write, or full), etcd3 only", - Destination: &ServerConfig.BootstrapType, - }, - cli.StringFlag{ - Name: "storage-backend", - Usage: "Specify storage type etcd3 or kvsql", - Destination: &ServerConfig.StorageBackend, - EnvVar: "K3S_STORAGE_BACKEND", - }, cli.StringFlag{ Name: "storage-endpoint", Usage: "Specify etcd, Mysql, Postgres, or Sqlite (default) data source name", diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index ded7c8be4a..ebe7c57fe6 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -8,13 +8,12 @@ import ( "path/filepath" "strings" - "github.com/rancher/k3s/pkg/netutil" - systemd "github.com/coreos/go-systemd/daemon" "github.com/pkg/errors" "github.com/rancher/k3s/pkg/agent" "github.com/rancher/k3s/pkg/cli/cmds" "github.com/rancher/k3s/pkg/datadir" + "github.com/rancher/k3s/pkg/netutil" "github.com/rancher/k3s/pkg/rootless" "github.com/rancher/k3s/pkg/server" "github.com/rancher/wrangler/pkg/signals" @@ -82,14 +81,12 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs serverConfig.ControlConfig.ClusterDomain = cfg.ClusterDomain - serverConfig.ControlConfig.StorageEndpoint = cfg.StorageEndpoint - serverConfig.ControlConfig.StorageBackend = cfg.StorageBackend - serverConfig.ControlConfig.StorageCAFile = cfg.StorageCAFile - serverConfig.ControlConfig.StorageCertFile = cfg.StorageCertFile - serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile + serverConfig.ControlConfig.Storage.Endpoint = cfg.StorageEndpoint + serverConfig.ControlConfig.Storage.CAFile = cfg.StorageCAFile + serverConfig.ControlConfig.Storage.CertFile = cfg.StorageCertFile + serverConfig.ControlConfig.Storage.KeyFile = cfg.StorageKeyFile serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort - serverConfig.ControlConfig.BootstrapType = cfg.BootstrapType if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" { cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface) @@ -127,10 +124,6 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.ClusterDNS = net2.ParseIP(cfg.ClusterDNS) } - if serverConfig.ControlConfig.StorageBackend != "etcd3" { - serverConfig.ControlConfig.NoLeaderElect = true - } - for _, noDeploy := range app.StringSlice("no-deploy") { if noDeploy == "servicelb" { serverConfig.DisableServiceLB = true diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index bf552938a6..975c2647a1 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -8,6 +8,8 @@ import ( "sort" "strings" + "github.com/rancher/kine/pkg/endpoint" + "k8s.io/apiserver/pkg/authentication/authenticator" ) @@ -66,44 +68,51 @@ type Agent struct { } type Control struct { - AdvertisePort int - AdvertiseIP string - ListenPort int - HTTPSPort int - ClusterSecret string - ClusterIPRange *net.IPNet - ServiceIPRange *net.IPNet - ClusterDNS net.IP - ClusterDomain string - NoCoreDNS bool - KubeConfigOutput string - KubeConfigMode string - DataDir string - Skips []string - BootstrapType string - StorageBackend string - StorageEndpoint string - StorageCAFile string - StorageCertFile string - StorageKeyFile string - NoScheduler bool - ExtraAPIArgs []string - ExtraControllerArgs []string - ExtraSchedulerAPIArgs []string - NoLeaderElect bool + AdvertisePort int + AdvertiseIP string + ListenPort int + HTTPSPort int + ClusterSecret string + ClusterIPRange *net.IPNet + ServiceIPRange *net.IPNet + ClusterDNS net.IP + ClusterDomain string + NoCoreDNS bool + KubeConfigOutput string + KubeConfigMode string + DataDir string + Skips []string + BootstrapReadOnly bool + BootstrapOverwriteLocal bool + Storage endpoint.Config + NoScheduler bool + ExtraAPIArgs []string + ExtraControllerArgs []string + ExtraSchedulerAPIArgs []string + NoLeaderElect bool Runtime *ControlRuntime `json:"-"` } +type ControlRuntimeBootstrap struct { + ServerCA string + ServerCAKey string + ClientCA string + ClientCAKey string + ServiceKey string + PasswdFile string + RequestHeaderCA string + RequestHeaderCAKey string + ClientKubeletKey string + ClientKubeProxyKey string + ServingKubeletKey string +} + type ControlRuntime struct { + ControlRuntimeBootstrap + ClientKubeAPICert string ClientKubeAPIKey string - ClientCA string - ClientCAKey string - ServerCA string - ServerCAKey string - ServiceKey string - PasswdFile string NodePasswdFile string KubeConfigAdmin string @@ -119,8 +128,6 @@ type ControlRuntime struct { Tunnel http.Handler Authenticator authenticator.Request - RequestHeaderCA string - RequestHeaderCAKey string ClientAuthProxyCert string ClientAuthProxyKey string @@ -131,10 +138,6 @@ type ControlRuntime struct { ClientSchedulerCert string ClientSchedulerKey string ClientKubeProxyCert string - ClientKubeProxyKey string - - ServingKubeletKey string - ClientKubeletKey string } type ArgString []string diff --git a/pkg/daemons/control/bootstrap.go b/pkg/daemons/control/bootstrap.go index 8ac114c3bc..9297a6243f 100644 --- a/pkg/daemons/control/bootstrap.go +++ b/pkg/daemons/control/bootstrap.go @@ -2,264 +2,105 @@ package control import ( "context" - "crypto/tls" - "crypto/x509" "encoding/json" - "errors" - "fmt" "io/ioutil" "os" - "strings" - "time" - - "encoding/base64" + "path/filepath" + "github.com/pkg/errors" "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/kine/pkg/client" "github.com/sirupsen/logrus" - "go.etcd.io/etcd/clientv3" ) const ( - etcdDialTimeout = 5 * time.Second k3sRuntimeEtcdPath = "/k3s/runtime" - bootstrapTypeNone = "none" - bootstrapTypeRead = "read" - bootstrapTypeWrite = "write" - bootstrapTypeFull = "full" ) -type serverBootstrap struct { - ServerCAData string `json:"serverCAData,omitempty"` - ServerCAKeyData string `json:"serverCAKeyData,omitempty"` - ClientCAData string `json:"clientCAData,omitempty"` - ClientCAKeyData string `json:"clientCAKeyData,omitempty"` - ServiceKeyData string `json:"serviceKeyData,omitempty"` - PasswdFileData string `json:"passwdFileData,omitempty"` - RequestHeaderCAData string `json:"requestHeaderCAData,omitempty"` - RequestHeaderCAKeyData string `json:"requestHeaderCAKeyData,omitempty"` - ClientKubeletKey string `json:"clientKubeletKey,omitempty"` - ClientKubeProxyKey string `json:"clientKubeProxyKey,omitempty"` - ServingKubeletKey string `json:"servingKubeletKey,omitempty"` -} - -var validBootstrapTypes = map[string]bool{ - bootstrapTypeRead: true, - bootstrapTypeWrite: true, - bootstrapTypeFull: true, -} - // fetchBootstrapData copies the bootstrap data (certs, keys, passwords) // from etcd to inidividual files specified by cfg.Runtime. -func fetchBootstrapData(cfg *config.Control) error { - if valid, err := checkBootstrapArgs(cfg, map[string]bool{ - bootstrapTypeFull: true, - bootstrapTypeRead: true, - }); !valid { - if err != nil { - logrus.Warnf("Not fetching bootstrap data: %v", err) - } - return nil - } - - tlsConfig, err := genBootstrapTLSConfig(cfg) - if err != nil { - return err - } - - endpoints := strings.Split(cfg.StorageEndpoint, ",") - cli, err := clientv3.New(clientv3.Config{ - Endpoints: endpoints, - DialTimeout: etcdDialTimeout, - TLS: tlsConfig, - }) - if err != nil { - return err - } - defer cli.Close() - +func fetchBootstrapData(ctx context.Context, cfg *config.Control, c client.Client) error { logrus.Info("Fetching bootstrap data from etcd") - gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) + gr, err := c.Get(ctx, k3sRuntimeEtcdPath) if err != nil { return err } - if len(gr.Kvs) == 0 { - if cfg.BootstrapType != bootstrapTypeRead { - return nil - } - return errors.New("Unable to read bootstrap data from server") - } - - runtimeJSON, err := base64.URLEncoding.DecodeString(string(gr.Kvs[0].Value)) - if err != nil { - return err - } - serverRuntime := &serverBootstrap{} - if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil { - return err - } - return writeRuntimeBootstrapData(cfg.Runtime, serverRuntime) -} - -// storeBootstrapData copies the bootstrap data in the opposite direction to -// fetchBootstrapData. -func storeBootstrapData(cfg *config.Control) error { - if valid, err := checkBootstrapArgs(cfg, map[string]bool{ - bootstrapTypeFull: true, - bootstrapTypeWrite: true, - }); !valid { - if err != nil { - logrus.Warnf("Not storing boostrap data: %v", err) - } + if gr.Modified == 0 { return nil } - tlsConfig, err := genBootstrapTLSConfig(cfg) + paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap) if err != nil { return err } - endpoints := strings.Split(cfg.StorageEndpoint, ",") - cli, err := clientv3.New(clientv3.Config{ - Endpoints: endpoints, - DialTimeout: etcdDialTimeout, - TLS: tlsConfig, - }) - if err != nil { + files := map[string][]byte{} + if err := json.Unmarshal(gr.Data, &files); err != nil { return err } - defer cli.Close() - if cfg.BootstrapType != bootstrapTypeWrite { - gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath) - if err != nil { - return err + for pathKey, data := range files { + path, ok := paths[pathKey] + if !ok { + continue } - if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" { - return nil + + if !cfg.BootstrapOverwriteLocal { + if _, err := os.Stat(path); err == nil { + continue + } } - } - certData, err := readRuntimeBootstrapData(cfg.Runtime) - if err != nil { - return err - } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path)) + } - logrus.Info("Storing bootstrap data to etcd") - runtimeBase64 := base64.StdEncoding.EncodeToString(certData) - _, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64) - if err != nil { - return err + if err := ioutil.WriteFile(path, data, 0700); err != nil { + return errors.Wrapf(err, "failed to write to %s", path) + } } return nil } -func checkBootstrapArgs(cfg *config.Control, accepted map[string]bool) (bool, error) { - if cfg.BootstrapType == "" || cfg.BootstrapType == bootstrapTypeNone { - return false, nil - } - if !validBootstrapTypes[cfg.BootstrapType] { - return false, fmt.Errorf("unsupported bootstrap type [%s]", cfg.BootstrapType) - } - if cfg.StorageBackend != "etcd3" { - return false, errors.New("bootstrap only supported with etcd3 as storage backend") +// storeBootstrapData copies the bootstrap data in the opposite direction to +// fetchBootstrapData. +func storeBootstrapData(ctx context.Context, cfg *config.Control, client client.Client) error { + if cfg.BootstrapReadOnly { + return nil } - if !accepted[cfg.BootstrapType] { - return false, nil + + paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap) + if err != nil { + return nil } - return true, nil -} -func genBootstrapTLSConfig(cfg *config.Control) (*tls.Config, error) { - secureTLSConfig := &tls.Config{} - // Note: clientv3 excepts nil for non-tls - var tlsConfig *tls.Config - if cfg.StorageCertFile != "" && cfg.StorageKeyFile != "" { - certPem, err := ioutil.ReadFile(cfg.StorageCertFile) - if err != nil { - return nil, err - } - keyPem, err := ioutil.ReadFile(cfg.StorageKeyFile) - if err != nil { - return nil, err + dataMap := map[string][]byte{} + for pathKey, path := range paths { + if path == "" { + continue } - tlsCert, err := tls.X509KeyPair(certPem, keyPem) + data, err := ioutil.ReadFile(path) if err != nil { - return nil, err + return errors.Wrapf(err, "failed to read %s", path) } - tlsConfig = secureTLSConfig - tlsConfig.Certificates = []tls.Certificate{tlsCert} - } - if cfg.StorageCAFile != "" { - caData, err := ioutil.ReadFile(cfg.StorageCAFile) - if err != nil { - return nil, err - } - certPool := x509.NewCertPool() - certPool.AppendCertsFromPEM(caData) - tlsConfig = secureTLSConfig - tlsConfig.RootCAs = certPool - } - return tlsConfig, nil -} -func readRuntimeBootstrapData(runtime *config.ControlRuntime) ([]byte, error) { - serverBootstrapFiles := map[string]string{ - runtime.ServerCA: "", - runtime.ServerCAKey: "", - runtime.ClientCA: "", - runtime.ClientCAKey: "", - runtime.ServiceKey: "", - runtime.PasswdFile: "", - runtime.RequestHeaderCA: "", - runtime.RequestHeaderCAKey: "", - runtime.ClientKubeletKey: "", - runtime.ClientKubeProxyKey: "", - runtime.ServingKubeletKey: "", - } - for k := range serverBootstrapFiles { - data, err := ioutil.ReadFile(k) - if err != nil { - return nil, err - } - serverBootstrapFiles[k] = string(data) + dataMap[pathKey] = data } - serverBootstrapFileData := &serverBootstrap{ - ServerCAData: serverBootstrapFiles[runtime.ServerCA], - ServerCAKeyData: serverBootstrapFiles[runtime.ServerCAKey], - ClientCAData: serverBootstrapFiles[runtime.ClientCA], - ClientCAKeyData: serverBootstrapFiles[runtime.ClientCAKey], - ServiceKeyData: serverBootstrapFiles[runtime.ServiceKey], - PasswdFileData: serverBootstrapFiles[runtime.PasswdFile], - RequestHeaderCAData: serverBootstrapFiles[runtime.RequestHeaderCA], - RequestHeaderCAKeyData: serverBootstrapFiles[runtime.RequestHeaderCAKey], - ClientKubeletKey: serverBootstrapFiles[runtime.ClientKubeletKey], - ClientKubeProxyKey: serverBootstrapFiles[runtime.ClientKubeProxyKey], - ServingKubeletKey: serverBootstrapFiles[runtime.ServingKubeletKey], + + bytes, err := json.Marshal(dataMap) + if err != nil { + return err } - return json.Marshal(serverBootstrapFileData) + + return client.Put(ctx, k3sRuntimeEtcdPath, bytes) } -func writeRuntimeBootstrapData(runtime *config.ControlRuntime, runtimeData *serverBootstrap) error { - runtimePathValue := map[string]string{ - runtime.ServerCA: runtimeData.ServerCAData, - runtime.ServerCAKey: runtimeData.ServerCAKeyData, - runtime.ClientCA: runtimeData.ClientCAData, - runtime.ClientCAKey: runtimeData.ClientCAKeyData, - runtime.ServiceKey: runtimeData.ServiceKeyData, - runtime.PasswdFile: runtimeData.PasswdFileData, - runtime.RequestHeaderCA: runtimeData.RequestHeaderCAData, - runtime.RequestHeaderCAKey: runtimeData.RequestHeaderCAKeyData, - runtime.ClientKubeletKey: runtimeData.ClientKubeletKey, - runtime.ClientKubeProxyKey: runtimeData.ClientKubeProxyKey, - runtime.ServingKubeletKey: runtimeData.ServingKubeletKey, - } - for k, v := range runtimePathValue { - if _, err := os.Stat(k); os.IsNotExist(err) { - if err := ioutil.WriteFile(k, []byte(v), 0600); err != nil { - return err - } - } +func objToMap(obj interface{}) (map[string]string, error) { + bytes, err := json.Marshal(obj) + if err != nil { + return nil, err } - return nil + data := map[string]string{} + return data, json.Unmarshal(bytes, &data) } diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 280c296d61..82d300d979 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -21,6 +21,9 @@ import ( "strings" "time" + "github.com/rancher/kine/pkg/client" + "github.com/rancher/kine/pkg/endpoint" + certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/k3s/pkg/daemons/config" "github.com/sirupsen/logrus" @@ -69,7 +72,7 @@ func Server(ctx context.Context, cfg *config.Control) error { runtime := &config.ControlRuntime{} cfg.Runtime = runtime - if err := prepare(cfg, runtime); err != nil { + if err := prepare(ctx, cfg, runtime); err != nil { return err } @@ -147,9 +150,6 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control argsMap := make(map[string]string) setupStorageBackend(argsMap, cfg) - if len(cfg.StorageEndpoint) > 0 { - argsMap["etcd-servers"] = cfg.StorageEndpoint - } certDir := filepath.Join(cfg.DataDir, "tls/temporary-certs") os.MkdirAll(certDir, 0700) @@ -227,16 +227,12 @@ func defaults(config *config.Control) { } } -func prepare(config *config.Control, runtime *config.ControlRuntime) error { +func prepare(ctx context.Context, config *config.Control, runtime *config.ControlRuntime) error { var err error defaults(config) - if _, err := os.Stat(config.DataDir); os.IsNotExist(err) { - if err := os.MkdirAll(config.DataDir, 0700); err != nil { - return err - } - } else if err != nil { + if err := os.MkdirAll(config.DataDir, 0700); err != nil { return err } @@ -284,7 +280,13 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt") runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") - if err := fetchBootstrapData(config); err != nil { + etcdClient, err := prepareStorageBackend(ctx, config) + if err != nil { + return err + } + defer etcdClient.Close() + + if err := fetchBootstrapData(ctx, config, etcdClient); err != nil { return err } @@ -300,13 +302,25 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { return err } - if err := storeBootstrapData(config); err != nil { + if err := storeBootstrapData(ctx, config, etcdClient); err != nil { return err } return readTokens(runtime) } +func prepareStorageBackend(ctx context.Context, config *config.Control) (client.Client, error) { + etcdConfig, err := endpoint.Listen(ctx, config.Storage) + if err != nil { + return nil, err + } + + config.Storage.Config = etcdConfig.TLSConfig + config.Storage.Endpoint = strings.Join(etcdConfig.Endpoints, ",") + config.NoLeaderElect = !etcdConfig.LeaderElect + return client.New(etcdConfig) +} + func readTokenFile(passwdFile string) (map[string]string, error) { f, err := os.Open(passwdFile) if err != nil { @@ -717,22 +731,19 @@ func KubeConfig(dest, url, caCert, clientCert, clientKey string) error { } func setupStorageBackend(argsMap map[string]string, cfg *config.Control) { - // setup the storage backend - if len(cfg.StorageBackend) > 0 { - argsMap["storage-backend"] = cfg.StorageBackend - } + argsMap["storage-backend"] = "etcd3" // specify the endpoints - if len(cfg.StorageEndpoint) > 0 { - argsMap["etcd-servers"] = cfg.StorageEndpoint + if len(cfg.Storage.Endpoint) > 0 { + argsMap["etcd-servers"] = cfg.Storage.Endpoint } // storage backend tls configuration - if len(cfg.StorageCAFile) > 0 { - argsMap["etcd-cafile"] = cfg.StorageCAFile + if len(cfg.Storage.CAFile) > 0 { + argsMap["etcd-cafile"] = cfg.Storage.CAFile } - if len(cfg.StorageCertFile) > 0 { - argsMap["etcd-certfile"] = cfg.StorageCertFile + if len(cfg.Storage.CertFile) > 0 { + argsMap["etcd-certfile"] = cfg.Storage.CertFile } - if len(cfg.StorageKeyFile) > 0 { - argsMap["etcd-keyfile"] = cfg.StorageKeyFile + if len(cfg.Storage.KeyFile) > 0 { + argsMap["etcd-keyfile"] = cfg.Storage.KeyFile } }