github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add client-side certificate generation support 

Clients now generate keys client-side and send CSRs. If the server is down-level and sends a cert+key instead of just responding with a cert signed with the client's public key, we use the key from the server instead.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit caeebc52b7)
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
pull/11561/head
Brad Davidson 2024-12-15 23:49:47 +00:00 committed by Brad Davidson
parent fcc5f32cfe
commit fb7b765383
1 changed files with 104 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
pkg/agent/config/config.go
View File

@ -2,9 +2,11 @@ package config
import (
"bufio"
"bytes"
"context"
cryptorand "crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/pem"
"fmt"
@ -32,6 +34,7 @@ import (
"github.com/k3s-io/k3s/pkg/version"
"github.com/k3s-io/k3s/pkg/vpn"
"github.com/pkg/errors"
certutil "github.com/rancher/dynamiclistener/cert"
"github.com/rancher/wharfie/pkg/registries"
"github.com/rancher/wrangler/v3/pkg/slice"
"github.com/sirupsen/logrus"
@ -133,9 +136,9 @@ func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]b
return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts, info.CertFile, info.KeyFile), info.Username, info.Password, info.Token())
}
func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string) HTTPRequester {
func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string, csr []byte) HTTPRequester {
return func(u string, client *http.Client, username, password, token string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, u, nil)
req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(csr))
if err != nil {
return nil, err
}
@ -238,47 +241,93 @@ func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string)
}
}
func getServingCert(nodeName string, nodeIPs []net.IP, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) error {
body, err := Request("/v1-"+version.Program+"/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
// getKubeletServingCert fills the kubelet server certificate with content returned
// from the server. We attempt to POST a CSR to the server, in hopes that it will
// sign the cert using our locally generated key. If the server does not support CSR
// signing, the key generated by the server is used instead.
func getKubeletServingCert(nodeName string, nodeIPs []net.IP, certFile, keyFile, nodePasswordFile string, info *clientaccess.Info) error {
csr, err := getCSRBytes(keyFile)
if err != nil {
return errors.Wrapf(err, "failed to create certificate request %s", certFile)
}
basename := filepath.Base(certFile)
body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
if err != nil {
return err
}
servingCert, servingKey := splitCertKeyPEM(body)
if err := os.WriteFile(servingCertFile, servingCert, 0600); err != nil {
return errors.Wrapf(err, "failed to write node cert")
// Always split the response, as down-level servers may send back a cert+key
// instead of signing a new cert with our key. If the response includes a key it
// must be used instead of the one we signed the CSR with.
certBytes, keyBytes := splitCertKeyPEM(body)
if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", certFile)
}
if err := os.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
return errors.Wrapf(err, "failed to write node key")
if len(keyBytes) > 0 {
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", keyFile)
}
}
return nil
}
func getHostFile(filename, keyFile string, info *clientaccess.Info) error {
// getHostFile fills a file with content returned from the server.
func getHostFile(filename string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
fileBytes, err := info.Get("/v1-" + version.Program + "/" + basename)
if err != nil {
return err
}
if keyFile == "" {
if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
} else {
fileBytes, keyBytes := splitCertKeyPEM(fileBytes)
if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
return nil
}
// getClientCert fills a client certificate with content returned from the server.
// We attempt to POST a CSR to the server, in hopes that it will sign the cert using
// our locally generated key. If the server does not support CSR signing, the key
// generated by the server is used instead.
func getClientCert(certFile, keyFile string, info *clientaccess.Info) error {
csr, err := getCSRBytes(keyFile)
if err != nil {
return errors.Wrapf(err, "failed to create certificate request %s", certFile)
}
basename := filepath.Base(certFile)
fileBytes, err := info.Post("/v1-"+version.Program+"/"+basename, csr)
if err != nil {
return err
}
// Always split the response, as down-level servers may send back a cert+key
// instead of signing a new cert with our key. If the response includes a key it
// must be used instead of the one we signed the CSR with.
certBytes, keyBytes := splitCertKeyPEM(fileBytes)
if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", certFile)
}
if len(keyBytes) > 0 {
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", filename)
return errors.Wrapf(err, "failed to write key %s", keyFile)
}
}
return nil
}
func getCSRBytes(keyFile string) ([]byte, error) {
keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false)
if err != nil {
return nil, err
}
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
if err != nil {
return nil, err
}
return x509.CreateCertificateRequest(cryptorand.Reader, &x509.CertificateRequest{}, key)
}
func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
for {
b, rest := pem.Decode(bytes)
@ -297,19 +346,33 @@ func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
return
}
func getNodeNamedHostFile(filename, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile))
// getKubeletClientCert fills the kubelet client certificate with content returned
// from the server. We attempt to POST a CSR to the server, in hopes that it will
// sign the cert using our locally generated key. If the server does not support CSR
// signing, the key generated by the server is used instead.
func getKubeletClientCert(certFile, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
csr, err := getCSRBytes(keyFile)
if err != nil {
return errors.Wrapf(err, "failed to create certificate request %s", certFile)
}
basename := filepath.Base(certFile)
body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
if err != nil {
return err
}
fileBytes, keyBytes := splitCertKeyPEM(body)
if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
// Always split the response, as down-level servers may send back a cert+key
// instead of signing a new cert with our key. If the response includes a key it
// must be used instead of the one we signed the CSR with.
certBytes, keyBytes := splitCertKeyPEM(body)
if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", certFile)
}
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", filename)
if len(keyBytes) > 0 {
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", keyFile)
}
}
return nil
}
@ -395,12 +458,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
}
clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt")
if err := getHostFile(clientCAFile, "", info); err != nil {
if err := getHostFile(clientCAFile, info); err != nil {
return nil, err
}
serverCAFile := filepath.Join(envInfo.DataDir, "agent", "server-ca.crt")
if err := getHostFile(serverCAFile, "", info); err != nil {
if err := getHostFile(serverCAFile, info); err != nil {
return nil, err
}
@ -494,13 +557,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
// that the cert will not be valid for, as they are not present in the list collected here.
nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...)
// Ask the server to generate a kubelet server cert+key. These files are unique to this node.
if err := getServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
// Ask the server to sign our kubelet server cert.
if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
return nil, errors.Wrap(err, servingKubeletCert)
}
// Ask the server to genrate a kubelet client cert+key. These files are unique to this node.
if err := getNodeNamedHostFile(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
// Ask the server to sign our kubelet client cert.
if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
return nil, errors.Wrap(err, clientKubeletCert)
}
@ -513,8 +576,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt")
clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key")
// Ask the server to send us its kube-proxy client cert+key. These files are not unique to this node.
if err := getHostFile(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
// Ask the server to sign our kube-proxy client cert.
if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
return nil, errors.Wrap(err, clientKubeProxyCert)
}
@ -527,8 +590,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt")
clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key")
// Ask the server to send us its agent controller client cert+key. These files are not unique to this node.
if err := getHostFile(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
// Ask the server to sign our agent controller client cert.
if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
return nil, errors.Wrap(err, clientK3sControllerCert)
}