From 215fb157ff4cc0b378e8a71c90115761b2615006 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Sat, 10 Dec 2022 00:20:51 +0000 Subject: [PATCH] Add `certificate rotate-ca` to write updated CA certs to datastore This command must be run on a server while the service is running. After this command completes, all the servers in the cluster should be restarted to load the new CA files. Signed-off-by: Brad Davidson --- cmd/cert/main.go | 4 +- cmd/k3s/main.go | 4 +- cmd/server/main.go | 4 +- main.go | 4 +- pkg/cli/cert/cert.go | 88 ++++++++++++--- pkg/cli/cmds/certs.go | 47 ++++++-- pkg/clientaccess/token.go | 22 +++- pkg/server/cert.go | 206 ++++++++++++++++++++++++++++++++++ pkg/server/router.go | 1 + pkg/server/secrets-encrypt.go | 12 +- 10 files changed, 355 insertions(+), 37 deletions(-) create mode 100644 pkg/server/cert.go diff --git a/cmd/cert/main.go b/cmd/cert/main.go index c64070c7bc..1053056eeb 100644 --- a/cmd/cert/main.go +++ b/cmd/cert/main.go @@ -17,7 +17,9 @@ func main() { app.Commands = []cli.Command{ cmds.NewCertCommand( cmds.NewCertSubcommands( - cert.Run), + cert.Rotate, + cert.RotateCA, + ), ), } diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 07364f9d4e..f6db6e37aa 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -68,7 +68,9 @@ func main() { ), cmds.NewCertCommand( cmds.NewCertSubcommands( - certCommand), + certCommand, + certCommand, + ), ), cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)), } diff --git a/cmd/server/main.go b/cmd/server/main.go index 47dce8222b..0f0ca4a3c2 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -65,7 +65,9 @@ func main() { ), cmds.NewCertCommand( cmds.NewCertSubcommands( - cert.Run), + cert.Rotate, + cert.RotateCA, + ), ), cmds.NewCompletionCommand(completion.Run), } diff --git a/main.go b/main.go index a2b5109eb9..e0e86b3ea2 100644 --- a/main.go +++ b/main.go @@ -49,7 +49,9 @@ func main() { ), cmds.NewCertCommand( cmds.NewCertSubcommands( - cert.Run), + cert.Rotate, + cert.RotateCA, + ), ), cmds.NewCompletionCommand(completion.Run), } diff --git a/pkg/cli/cert/cert.go b/pkg/cli/cert/cert.go index c206a505a7..afbbe8a07a 100644 --- a/pkg/cli/cert/cert.go +++ b/pkg/cli/cert/cert.go @@ -1,20 +1,24 @@ package cert import ( - "errors" + "bytes" + "fmt" "os" "path/filepath" "strconv" "time" "github.com/erikdubbelboer/gspt" + "github.com/k3s-io/k3s/pkg/bootstrap" "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/daemons/control/deps" "github.com/k3s-io/k3s/pkg/datadir" "github.com/k3s-io/k3s/pkg/server" "github.com/k3s-io/k3s/pkg/version" "github.com/otiai10/copy" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -47,19 +51,31 @@ var services = []string{ version.Program + k3sServerService, } -func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) (string, string, error) { +func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) (string, error) { gspt.SetProcTitle(os.Args[0]) - sc.ControlConfig.DataDir = cfg.DataDir - sc.ControlConfig.Runtime = &config.ControlRuntime{} dataDir, err := datadir.Resolve(cfg.DataDir) if err != nil { - return "", "", err + return "", err } - return filepath.Join(dataDir, "server"), filepath.Join(dataDir, "agent"), err + sc.ControlConfig.DataDir = filepath.Join(dataDir, "server") + + if cfg.Token == "" { + fp := filepath.Join(sc.ControlConfig.DataDir, "token") + tokenByte, err := os.ReadFile(fp) + if err != nil { + return "", err + } + cfg.Token = string(bytes.TrimRight(tokenByte, "\n")) + } + sc.ControlConfig.Token = cfg.Token + + sc.ControlConfig.Runtime = &config.ControlRuntime{} + + return dataDir, nil } -func Run(app *cli.Context) error { +func Rotate(app *cli.Context) error { if err := cmds.InitLogging(); err != nil { return err } @@ -69,27 +85,26 @@ func Run(app *cli.Context) error { func rotate(app *cli.Context, cfg *cmds.Server) error { var serverConfig server.Config - serverDataDir, agentDataDir, err := commandSetup(app, cfg, &serverConfig) + dataDir, err := commandSetup(app, cfg, &serverConfig) if err != nil { return err } - serverConfig.ControlConfig.DataDir = serverDataDir - serverConfig.ControlConfig.Runtime = &config.ControlRuntime{} deps.CreateRuntimeCertFiles(&serverConfig.ControlConfig) if err := validateCertConfig(); err != nil { return err } - tlsBackupDir, err := backupCertificates(serverDataDir, agentDataDir) + agentDataDir := filepath.Join(dataDir, "agent") + tlsBackupDir, err := backupCertificates(serverConfig.ControlConfig.DataDir, agentDataDir) if err != nil { return err } if len(cmds.ServicesList) == 0 { - // detecting if the service is an agent or server - _, err := os.Stat(serverDataDir) + // detecting if the command is being run on an agent or server + _, err := os.Stat(serverConfig.ControlConfig.DataDir) if err != nil { if !os.IsNotExist(err) { return err @@ -152,7 +167,7 @@ func rotate(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.Runtime.ClientCloudControllerCert, serverConfig.ControlConfig.Runtime.ClientCloudControllerKey) case version.Program + k3sServerService: - dynamicListenerRegenFilePath := filepath.Join(serverDataDir, "tls", "dynamic-cert-regenerate") + dynamicListenerRegenFilePath := filepath.Join(serverConfig.ControlConfig.DataDir, "tls", "dynamic-cert-regenerate") if err := os.WriteFile(dynamicListenerRegenFilePath, []byte{}, 0600); err != nil { return err } @@ -254,3 +269,48 @@ func validateCertConfig() error { } return nil } + +func RotateCA(app *cli.Context) error { + if err := cmds.InitLogging(); err != nil { + return err + } + return rotateCA(app, &cmds.ServerConfig, &cmds.CertRotateCAConfig) +} + +func rotateCA(app *cli.Context, cfg *cmds.Server, sync *cmds.CertRotateCA) error { + var serverConfig server.Config + + _, err := commandSetup(app, cfg, &serverConfig) + if err != nil { + return err + } + + info, err := clientaccess.ParseAndValidateTokenForUser(cmds.ServerConfig.ServerURL, serverConfig.ControlConfig.Token, "server") + if err != nil { + return err + } + + // Set up dummy server config for reading new bootstrap data from disk. + tmpServer := &config.Control{ + Runtime: &config.ControlRuntime{}, + DataDir: filepath.Dir(sync.CACertPath), + } + deps.CreateRuntimeCertFiles(tmpServer) + + // Override these paths so that we don't get warnings when they don't exist, as the user is not expected to provide them. + tmpServer.Runtime.PasswdFile = "/dev/null" + tmpServer.Runtime.IPSECKey = "/dev/null" + + buf := &bytes.Buffer{} + if err := bootstrap.ReadFromDisk(buf, &tmpServer.Runtime.ControlRuntimeBootstrap); err != nil { + return err + } + + url := fmt.Sprintf("/v1-%s/cert/cacerts?force=%t", version.Program, sync.Force) + if err = info.Put(url, buf.Bytes()); err != nil { + return errors.Wrap(err, "see server log for details") + } + + fmt.Println("certificates saved to datastore") + return nil +} diff --git a/pkg/cli/cmds/certs.go b/pkg/cli/cmds/certs.go index 266fffc6cd..27d02a4a71 100644 --- a/pkg/cli/cmds/certs.go +++ b/pkg/cli/cmds/certs.go @@ -7,9 +7,15 @@ import ( const CertCommand = "certificate" +type CertRotateCA struct { + CACertPath string + Force bool +} + var ( - ServicesList cli.StringSlice - CertCommandFlags = []cli.Flag{ + ServicesList cli.StringSlice + CertRotateCAConfig CertRotateCA + CertRotateCommandFlags = []cli.Flag{ DebugFlag, ConfigFlag, LogFile, @@ -21,28 +27,55 @@ var ( Value: &ServicesList, }, } + CertRotateCACommandFlags = []cli.Flag{ + cli.StringFlag{ + Name: "server,s", + Usage: "(cluster) Server to connect to", + EnvVar: version.ProgramUpper + "_URL", + Value: "https://127.0.0.1:6443", + Destination: &ServerConfig.ServerURL, + }, + cli.StringFlag{ + Name: "path", + Usage: "Path to directory containing new CA certificates", + Destination: &CertRotateCAConfig.CACertPath, + Required: true, + }, + cli.BoolFlag{ + Name: "force", + Usage: "Force certificate replacement, even if consistency checks fail", + Destination: &CertRotateCAConfig.Force, + }, + } ) func NewCertCommand(subcommands []cli.Command) cli.Command { return cli.Command{ Name: CertCommand, - Usage: "Certificates management", + Usage: "Manage K3s certificates", SkipFlagParsing: false, SkipArgReorder: true, Subcommands: subcommands, - Flags: CertCommandFlags, } } -func NewCertSubcommands(rotate func(ctx *cli.Context) error) []cli.Command { +func NewCertSubcommands(rotate, rotateCA func(ctx *cli.Context) error) []cli.Command { return []cli.Command{ { Name: "rotate", - Usage: "Certificate rotation", + Usage: "Rotate " + version.Program + " component certificates on disk", SkipFlagParsing: false, SkipArgReorder: true, Action: rotate, - Flags: CertCommandFlags, + Flags: CertRotateCommandFlags, + }, + { + Name: "rotate-ca", + Usage: "Write updated " + version.Program + " CA certificates to the datastore", + SkipFlagParsing: false, + SkipArgReorder: true, + Action: rotateCA, + Flags: CertRotateCACommandFlags, }, } } diff --git a/pkg/clientaccess/token.go b/pkg/clientaccess/token.go index 1dce962e86..b0031b5143 100644 --- a/pkg/clientaccess/token.go +++ b/pkg/clientaccess/token.go @@ -217,8 +217,13 @@ func (i *Info) Get(path string) ([]byte, error) { if err != nil { return nil, err } - u.Path = path - return get(u.String(), GetHTTPClient(i.CACerts), i.Username, i.Password) + p, err := url.Parse(path) + if err != nil { + return nil, err + } + p.Scheme = u.Scheme + p.Host = u.Host + return get(p.String(), GetHTTPClient(i.CACerts), i.Username, i.Password) } // Put makes a request to a subpath of info's BaseURL @@ -227,8 +232,13 @@ func (i *Info) Put(path string, body []byte) error { if err != nil { return err } - u.Path = path - return put(u.String(), body, GetHTTPClient(i.CACerts), i.Username, i.Password) + p, err := url.Parse(path) + if err != nil { + return err + } + p.Scheme = u.Scheme + p.Host = u.Host + return put(p.String(), body, GetHTTPClient(i.CACerts), i.Username, i.Password) } // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server @@ -326,7 +336,7 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { + if resp.StatusCode < 200 || resp.StatusCode > 299 { return nil, fmt.Errorf("%s: %s", u, resp.Status) } @@ -352,7 +362,7 @@ func put(u string, body []byte, client *http.Client, username, password string) defer resp.Body.Close() respBody, _ := io.ReadAll(resp.Body) - if resp.StatusCode != http.StatusOK { + if resp.StatusCode < 200 || resp.StatusCode > 299 { return fmt.Errorf("%s: %s %s", u, resp.Status, string(respBody)) } diff --git a/pkg/server/cert.go b/pkg/server/cert.go new file mode 100644 index 0000000000..5ef4520bb1 --- /dev/null +++ b/pkg/server/cert.go @@ -0,0 +1,206 @@ +package server + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "reflect" + "strconv" + "strings" + + "github.com/k3s-io/k3s/pkg/bootstrap" + "github.com/k3s-io/k3s/pkg/cluster" + "github.com/k3s-io/k3s/pkg/daemons/config" + "github.com/k3s-io/k3s/pkg/daemons/control/deps" + "github.com/k3s-io/k3s/pkg/version" + "github.com/pkg/errors" + certutil "github.com/rancher/dynamiclistener/cert" + "github.com/rancher/wrangler/pkg/merr" + "github.com/sirupsen/logrus" + "k8s.io/client-go/util/keyutil" +) + +func caCertReplaceHandler(server *config.Control) http.HandlerFunc { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if req.TLS == nil || req.Method != http.MethodPut { + resp.WriteHeader(http.StatusNotFound) + return + } + force, _ := strconv.ParseBool(req.FormValue("force")) + if err := caCertReplace(server, req.Body, force); err != nil { + genErrorMessage(resp, http.StatusInternalServerError, err, "certificate") + return + } + logrus.Infof("certificate: Cluster Certificate Authority data has been updated, %s must be restarted.", version.Program) + resp.WriteHeader(http.StatusNoContent) + }) +} + +// caCertReplace stores new CA Certificate data from the client. The data is temporarily written out to disk, +// validated to confirm that the new certs share a common root with the existing certs, and if so are saved to +// the datastore. If the functions succeeds, servers should be restarted immediately to load the new certs +// from the bootstrap data. +func caCertReplace(server *config.Control, buf io.ReadCloser, force bool) error { + tmpdir, err := os.MkdirTemp("", "cacerts") + if err != nil { + return err + } + defer os.RemoveAll(tmpdir) + + tmpServer := &config.Control{ + Runtime: &config.ControlRuntime{ + EtcdConfig: server.Runtime.EtcdConfig, + ServerToken: server.Runtime.ServerToken, + }, + Token: server.Token, + DataDir: tmpdir, + } + deps.CreateRuntimeCertFiles(tmpServer) + + bootstrapData := bootstrap.PathsDataformat{} + if err := json.NewDecoder(buf).Decode(&bootstrapData); err != nil { + return err + } + + if err := bootstrap.WriteToDiskFromStorage(bootstrapData, &tmpServer.Runtime.ControlRuntimeBootstrap); err != nil { + return err + } + + if err := validateBootstrap(server, tmpServer); err != nil { + if !force { + return errors.Wrap(err, "failed to validate new CA certificates and keys") + } + logrus.Warnf("Save of CA certificates and keys forced, ignoring validation errors: %v", err) + } + + return cluster.Save(context.TODO(), tmpServer, true) +} + +// validateBootstrap checks the new certs and keys to ensure that the cluster would function properly were they to be used. +// - The new leaf CA certificates must be verifiable using the same root and intermediate certs as the current leaf CA certificates. +// - The new service account signing key bundle must include the currently active signing key. +func validateBootstrap(oldServer, newServer *config.Control) error { + errs := []error{} + + // Use reflection to iterate over all of the bootstrap fields, checking files at each of the new paths. + oldMeta := reflect.ValueOf(&oldServer.Runtime.ControlRuntimeBootstrap).Elem() + newMeta := reflect.ValueOf(&newServer.Runtime.ControlRuntimeBootstrap).Elem() + for _, field := range reflect.VisibleFields(oldMeta.Type()) { + oldVal := oldMeta.FieldByName(field.Name) + newVal := newMeta.FieldByName(field.Name) + + info, err := os.Stat(newVal.String()) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + errs = append(errs, errors.Wrap(err, field.Name)) + continue + } + + if info == nil || info.Size() == 0 { + if newVal.CanSet() { + logrus.Infof("certificate: %s not provided; using current value", field.Name) + newVal.Set(oldVal) + } else { + errs = append(errs, fmt.Errorf("cannot use current data for %s; field is not settable", field.Name)) + } + } + + // Check CA chain consistency and cert/key agreement + if strings.HasSuffix(field.Name, "CA") { + if err := validateCA(oldVal.String(), newVal.String()); err != nil { + errs = append(errs, errors.Wrap(err, field.Name)) + } + newKeyVal := newMeta.FieldByName(field.Name + "Key") + if err := validateCAKey(newVal.String(), newKeyVal.String()); err != nil { + errs = append(errs, errors.Wrap(err, field.Name+"Key")) + } + } + + // Check signing key rotation + if field.Name == "ServiceKey" { + if err := validateServiceKey(oldVal.String(), newVal.String()); err != nil { + errs = append(errs, errors.Wrap(err, field.Name)) + } + } + } + + if len(errs) > 0 { + return merr.NewErrors(errs...) + } + return nil +} + +func validateCA(oldCAPath, newCAPath string) error { + oldCerts, err := certutil.CertsFromFile(oldCAPath) + if err != nil { + return err + } + + if len(oldCerts) == 1 { + return errors.New("old CA is self-signed") + } + + newCerts, err := certutil.CertsFromFile(newCAPath) + if err != nil { + return err + } + + if len(newCerts) == 1 { + return errors.New("new CA is self-signed") + } + + roots := x509.NewCertPool() + intermediates := x509.NewCertPool() + for i, cert := range oldCerts { + if i > 0 { + if len(cert.AuthorityKeyId) == 0 || bytes.Equal(cert.AuthorityKeyId, cert.SubjectKeyId) { + roots.AddCert(cert) + } else { + intermediates.AddCert(cert) + } + } + } + + _, err = newCerts[0].Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates}) + if err != nil { + err = errors.Wrap(err, "new CA cert cannot be verified using old CA chain") + } + return err +} + +// validateCAKey confirms that the private key is valid for the certificate +func validateCAKey(newCAPath, newCAKeyPath string) error { + _, err := tls.LoadX509KeyPair(newCAPath, newCAKeyPath) + if err != nil { + err = errors.Wrap(err, "new CA cert and key cannot be loaded as X590KeyPair") + } + return err +} + +// validateServiceKey ensures that the first key from the old serviceaccount signing key list +// is also present in the new key list, to ensure that old signatures can still be validated. +func validateServiceKey(oldKeyPath, newKeyPath string) error { + oldKeys, err := keyutil.PublicKeysFromFile(oldKeyPath) + if err != nil { + return err + } + + newKeys, err := keyutil.PublicKeysFromFile(newKeyPath) + if err != nil { + return err + } + + for _, key := range newKeys { + if reflect.DeepEqual(oldKeys[0], key) { + return nil + } + } + + return errors.New("old ServiceAccount signing key not in new ServiceAccount key list") +} diff --git a/pkg/server/router.go b/pkg/server/router.go index 5a1a4ae5b6..b1105de399 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -71,6 +71,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler serverAuthed.Use(authMiddleware(serverConfig, version.Program+":server")) serverAuthed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig)) serverAuthed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig)) + serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig)) serverAuthed.Path("/db/info").Handler(nodeAuthed) serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime)) diff --git a/pkg/server/secrets-encrypt.go b/pkg/server/secrets-encrypt.go index 38e9c83030..0a4b20ae7c 100644 --- a/pkg/server/secrets-encrypt.go +++ b/pkg/server/secrets-encrypt.go @@ -60,12 +60,12 @@ func encryptionStatusHandler(server *config.Control) http.Handler { } status, err := encryptionStatus(server) if err != nil { - genErrorMessage(resp, http.StatusInternalServerError, err) + genErrorMessage(resp, http.StatusInternalServerError, err, "secrets-encrypt") return } b, err := json.Marshal(status) if err != nil { - genErrorMessage(resp, http.StatusInternalServerError, err) + genErrorMessage(resp, http.StatusInternalServerError, err, "secrets-encrypt") return } resp.Write(b) @@ -183,7 +183,7 @@ func encryptionConfigHandler(ctx context.Context, server *config.Control) http.H } if err != nil { - genErrorMessage(resp, http.StatusBadRequest, err) + genErrorMessage(resp, http.StatusBadRequest, err, "secrets-encrypt") return } // If a user kills the k3s server immediately after this call, we run into issues where the files @@ -364,14 +364,14 @@ func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.In // genErrorMessage sends and logs a random error ID so that logs can be correlated // between the REST API (which does not provide any detailed error output, to avoid // information disclosure) and the server logs. -func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error) { +func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error, component string) { errID, err := rand.Int(rand.Reader, big.NewInt(99999)) if err != nil { resp.WriteHeader(http.StatusInternalServerError) resp.Write([]byte(err.Error())) return } - logrus.Warnf("secrets-encrypt error ID %05d: %s", errID, passedErr.Error()) + logrus.Warnf("%s error ID %05d: %s", component, errID, passedErr.Error()) resp.WriteHeader(statusCode) - resp.Write([]byte(fmt.Sprintf("secrets-encrypt error ID %05d", errID))) + resp.Write([]byte(fmt.Sprintf("%s error ID %05d", component, errID))) }