diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index d956507ace..adaafba280 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -2,9 +2,11 @@ package config import ( "bufio" + "bytes" "context" cryptorand "crypto/rand" "crypto/tls" + "crypto/x509" "encoding/hex" "encoding/pem" "fmt" @@ -32,6 +34,7 @@ import ( "github.com/k3s-io/k3s/pkg/version" "github.com/k3s-io/k3s/pkg/vpn" "github.com/pkg/errors" + certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/wharfie/pkg/registries" "github.com/rancher/wrangler/v3/pkg/slice" "github.com/sirupsen/logrus" @@ -133,9 +136,9 @@ func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]b return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts, info.CertFile, info.KeyFile), info.Username, info.Password, info.Token()) } -func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string) HTTPRequester { +func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string, csr []byte) HTTPRequester { return func(u string, client *http.Client, username, password, token string) ([]byte, error) { - req, err := http.NewRequest(http.MethodGet, u, nil) + req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(csr)) if err != nil { return nil, err } @@ -238,47 +241,93 @@ func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string) } } -func getServingCert(nodeName string, nodeIPs []net.IP, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) error { - body, err := Request("/v1-"+version.Program+"/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile)) +// getKubeletServingCert fills the kubelet server certificate with content returned +// from the server. We attempt to POST a CSR to the server, in hopes that it will +// sign the cert using our locally generated key. If the server does not support CSR +// signing, the key generated by the server is used instead. +func getKubeletServingCert(nodeName string, nodeIPs []net.IP, certFile, keyFile, nodePasswordFile string, info *clientaccess.Info) error { + csr, err := getCSRBytes(keyFile) + if err != nil { + return errors.Wrapf(err, "failed to create certificate request %s", certFile) + } + + basename := filepath.Base(certFile) + body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr)) if err != nil { return err } - servingCert, servingKey := splitCertKeyPEM(body) - - if err := os.WriteFile(servingCertFile, servingCert, 0600); err != nil { - return errors.Wrapf(err, "failed to write node cert") + // Always split the response, as down-level servers may send back a cert+key + // instead of signing a new cert with our key. If the response includes a key it + // must be used instead of the one we signed the CSR with. + certBytes, keyBytes := splitCertKeyPEM(body) + if err := os.WriteFile(certFile, certBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write cert %s", certFile) } - - if err := os.WriteFile(servingKeyFile, servingKey, 0600); err != nil { - return errors.Wrapf(err, "failed to write node key") + if len(keyBytes) > 0 { + if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write key %s", keyFile) + } } - return nil } -func getHostFile(filename, keyFile string, info *clientaccess.Info) error { +// getHostFile fills a file with content returned from the server. +func getHostFile(filename string, info *clientaccess.Info) error { basename := filepath.Base(filename) fileBytes, err := info.Get("/v1-" + version.Program + "/" + basename) if err != nil { return err } - if keyFile == "" { - if err := os.WriteFile(filename, fileBytes, 0600); err != nil { - return errors.Wrapf(err, "failed to write cert %s", filename) - } - } else { - fileBytes, keyBytes := splitCertKeyPEM(fileBytes) - if err := os.WriteFile(filename, fileBytes, 0600); err != nil { - return errors.Wrapf(err, "failed to write cert %s", filename) - } + if err := os.WriteFile(filename, fileBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write cert %s", filename) + } + return nil +} + +// getClientCert fills a client certificate with content returned from the server. +// We attempt to POST a CSR to the server, in hopes that it will sign the cert using +// our locally generated key. If the server does not support CSR signing, the key +// generated by the server is used instead. +func getClientCert(certFile, keyFile string, info *clientaccess.Info) error { + csr, err := getCSRBytes(keyFile) + if err != nil { + return errors.Wrapf(err, "failed to create certificate request %s", certFile) + } + + basename := filepath.Base(certFile) + fileBytes, err := info.Post("/v1-"+version.Program+"/"+basename, csr) + if err != nil { + return err + } + + // Always split the response, as down-level servers may send back a cert+key + // instead of signing a new cert with our key. If the response includes a key it + // must be used instead of the one we signed the CSR with. + certBytes, keyBytes := splitCertKeyPEM(fileBytes) + if err := os.WriteFile(certFile, certBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write cert %s", certFile) + } + if len(keyBytes) > 0 { if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil { - return errors.Wrapf(err, "failed to write key %s", filename) + return errors.Wrapf(err, "failed to write key %s", keyFile) } } return nil } +func getCSRBytes(keyFile string) ([]byte, error) { + keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false) + if err != nil { + return nil, err + } + key, err := certutil.ParsePrivateKeyPEM(keyBytes) + if err != nil { + return nil, err + } + return x509.CreateCertificateRequest(cryptorand.Reader, &x509.CertificateRequest{}, key) +} + func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) { for { b, rest := pem.Decode(bytes) @@ -297,19 +346,33 @@ func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) { return } -func getNodeNamedHostFile(filename, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error { - basename := filepath.Base(filename) - body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile)) +// getKubeletClientCert fills the kubelet client certificate with content returned +// from the server. We attempt to POST a CSR to the server, in hopes that it will +// sign the cert using our locally generated key. If the server does not support CSR +// signing, the key generated by the server is used instead. +func getKubeletClientCert(certFile, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error { + csr, err := getCSRBytes(keyFile) + if err != nil { + return errors.Wrapf(err, "failed to create certificate request %s", certFile) + } + + basename := filepath.Base(certFile) + body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr)) if err != nil { return err } - fileBytes, keyBytes := splitCertKeyPEM(body) - if err := os.WriteFile(filename, fileBytes, 0600); err != nil { - return errors.Wrapf(err, "failed to write cert %s", filename) + // Always split the response, as down-level servers may send back a cert+key + // instead of signing a new cert with our key. If the response includes a key it + // must be used instead of the one we signed the CSR with. + certBytes, keyBytes := splitCertKeyPEM(body) + if err := os.WriteFile(certFile, certBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write cert %s", certFile) } - if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil { - return errors.Wrapf(err, "failed to write key %s", filename) + if len(keyBytes) > 0 { + if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write key %s", keyFile) + } } return nil } @@ -395,12 +458,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N } clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt") - if err := getHostFile(clientCAFile, "", info); err != nil { + if err := getHostFile(clientCAFile, info); err != nil { return nil, err } serverCAFile := filepath.Join(envInfo.DataDir, "agent", "server-ca.crt") - if err := getHostFile(serverCAFile, "", info); err != nil { + if err := getHostFile(serverCAFile, info); err != nil { return nil, err } @@ -494,13 +557,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N // that the cert will not be valid for, as they are not present in the list collected here. nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...) - // Ask the server to generate a kubelet server cert+key. These files are unique to this node. - if err := getServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil { + // Ask the server to sign our kubelet server cert. + if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil { return nil, errors.Wrap(err, servingKubeletCert) } - // Ask the server to genrate a kubelet client cert+key. These files are unique to this node. - if err := getNodeNamedHostFile(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil { + // Ask the server to sign our kubelet client cert. + if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil { return nil, errors.Wrap(err, clientKubeletCert) } @@ -513,8 +576,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt") clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key") - // Ask the server to send us its kube-proxy client cert+key. These files are not unique to this node. - if err := getHostFile(clientKubeProxyCert, clientKubeProxyKey, info); err != nil { + // Ask the server to sign our kube-proxy client cert. + if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil { return nil, errors.Wrap(err, clientKubeProxyCert) } @@ -527,8 +590,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt") clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key") - // Ask the server to send us its agent controller client cert+key. These files are not unique to this node. - if err := getHostFile(clientK3sControllerCert, clientK3sControllerKey, info); err != nil { + // Ask the server to sign our agent controller client cert. + if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil { return nil, errors.Wrap(err, clientK3sControllerCert) }