github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add client-side certificate generation support 

Clients now generate keys client-side and send CSRs. If the server is down-level and sends a cert+key instead of just responding with a cert signed with the client's public key, we use the key from the server instead.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit caeebc52b7)
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
pull/11561/head
Brad Davidson 2024-12-15 23:49:47 +00:00 committed by Brad Davidson
parent fcc5f32cfe
commit fb7b765383
1 changed files with 104 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
pkg/agent/config/config.go
View File

@ -2,9 +2,11 @@ package config
import ( import (
"bufio" "bufio"
"bytes"
"context" "context"
cryptorand "crypto/rand" cryptorand "crypto/rand"
"crypto/tls" "crypto/tls"
"crypto/x509"
"encoding/hex" "encoding/hex"
"encoding/pem" "encoding/pem"
"fmt" "fmt"
@ -32,6 +34,7 @@ import (
"github.com/k3s-io/k3s/pkg/version" "github.com/k3s-io/k3s/pkg/version"
"github.com/k3s-io/k3s/pkg/vpn" "github.com/k3s-io/k3s/pkg/vpn"
"github.com/pkg/errors" "github.com/pkg/errors"
certutil "github.com/rancher/dynamiclistener/cert"
"github.com/rancher/wharfie/pkg/registries" "github.com/rancher/wharfie/pkg/registries"
"github.com/rancher/wrangler/v3/pkg/slice" "github.com/rancher/wrangler/v3/pkg/slice"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
@ -133,9 +136,9 @@ func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]b
return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts, info.CertFile, info.KeyFile), info.Username, info.Password, info.Token()) return requester(u.String(), clientaccess.GetHTTPClient(info.CACerts, info.CertFile, info.KeyFile), info.Username, info.Password, info.Token())
} }
func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string) HTTPRequester { func getNodeNamedCrt(nodeName string, nodeIPs []net.IP, nodePasswordFile string, csr []byte) HTTPRequester {
return func(u string, client *http.Client, username, password, token string) ([]byte, error) { return func(u string, client *http.Client, username, password, token string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, u, nil) req, err := http.NewRequest(http.MethodPost, u, bytes.NewReader(csr))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -238,47 +241,93 @@ func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string)
} }
} }
func getServingCert(nodeName string, nodeIPs []net.IP, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) error { // getKubeletServingCert fills the kubelet server certificate with content returned
body, err := Request("/v1-"+version.Program+"/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile)) // from the server. We attempt to POST a CSR to the server, in hopes that it will
// sign the cert using our locally generated key. If the server does not support CSR
// signing, the key generated by the server is used instead.
func getKubeletServingCert(nodeName string, nodeIPs []net.IP, certFile, keyFile, nodePasswordFile string, info *clientaccess.Info) error {
csr, err := getCSRBytes(keyFile)
if err != nil {
return errors.Wrapf(err, "failed to create certificate request %s", certFile)
}
basename := filepath.Base(certFile)
body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
if err != nil { if err != nil {
return err return err
} }
servingCert, servingKey := splitCertKeyPEM(body) // Always split the response, as down-level servers may send back a cert+key
// instead of signing a new cert with our key. If the response includes a key it
if err := os.WriteFile(servingCertFile, servingCert, 0600); err != nil { // must be used instead of the one we signed the CSR with.
return errors.Wrapf(err, "failed to write node cert") certBytes, keyBytes := splitCertKeyPEM(body)
if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", certFile)
} }
if len(keyBytes) > 0 {
if err := os.WriteFile(servingKeyFile, servingKey, 0600); err != nil { if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write node key") return errors.Wrapf(err, "failed to write key %s", keyFile)
}
} }
return nil return nil
} }
func getHostFile(filename, keyFile string, info *clientaccess.Info) error { // getHostFile fills a file with content returned from the server.
func getHostFile(filename string, info *clientaccess.Info) error {
basename := filepath.Base(filename) basename := filepath.Base(filename)
fileBytes, err := info.Get("/v1-" + version.Program + "/" + basename) fileBytes, err := info.Get("/v1-" + version.Program + "/" + basename)
if err != nil { if err != nil {
return err return err
} }
if keyFile == "" { if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
if err := os.WriteFile(filename, fileBytes, 0600); err != nil { return errors.Wrapf(err, "failed to write cert %s", filename)
return errors.Wrapf(err, "failed to write cert %s", filename) }
} return nil
} else { }
fileBytes, keyBytes := splitCertKeyPEM(fileBytes)
if err := os.WriteFile(filename, fileBytes, 0600); err != nil { // getClientCert fills a client certificate with content returned from the server.
return errors.Wrapf(err, "failed to write cert %s", filename) // We attempt to POST a CSR to the server, in hopes that it will sign the cert using
} // our locally generated key. If the server does not support CSR signing, the key
// generated by the server is used instead.
func getClientCert(certFile, keyFile string, info *clientaccess.Info) error {
csr, err := getCSRBytes(keyFile)
if err != nil {
return errors.Wrapf(err, "failed to create certificate request %s", certFile)
}
basename := filepath.Base(certFile)
fileBytes, err := info.Post("/v1-"+version.Program+"/"+basename, csr)
if err != nil {
return err
}
// Always split the response, as down-level servers may send back a cert+key
// instead of signing a new cert with our key. If the response includes a key it
// must be used instead of the one we signed the CSR with.
certBytes, keyBytes := splitCertKeyPEM(fileBytes)
if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", certFile)
}
if len(keyBytes) > 0 {
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil { if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", filename) return errors.Wrapf(err, "failed to write key %s", keyFile)
} }
} }
return nil return nil
} }
func getCSRBytes(keyFile string) ([]byte, error) {
keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile, false)
if err != nil {
return nil, err
}
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
if err != nil {
return nil, err
}
return x509.CreateCertificateRequest(cryptorand.Reader, &x509.CertificateRequest{}, key)
}
func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) { func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
for { for {
b, rest := pem.Decode(bytes) b, rest := pem.Decode(bytes)
@ -297,19 +346,33 @@ func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
return return
} }
func getNodeNamedHostFile(filename, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error { // getKubeletClientCert fills the kubelet client certificate with content returned
basename := filepath.Base(filename) // from the server. We attempt to POST a CSR to the server, in hopes that it will
body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile)) // sign the cert using our locally generated key. If the server does not support CSR
// signing, the key generated by the server is used instead.
func getKubeletClientCert(certFile, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
csr, err := getCSRBytes(keyFile)
if err != nil {
return errors.Wrapf(err, "failed to create certificate request %s", certFile)
}
basename := filepath.Base(certFile)
body, err := Request("/v1-"+version.Program+"/"+basename, info, getNodeNamedCrt(nodeName, nodeIPs, nodePasswordFile, csr))
if err != nil { if err != nil {
return err return err
} }
fileBytes, keyBytes := splitCertKeyPEM(body)
if err := os.WriteFile(filename, fileBytes, 0600); err != nil { // Always split the response, as down-level servers may send back a cert+key
return errors.Wrapf(err, "failed to write cert %s", filename) // instead of signing a new cert with our key. If the response includes a key it
// must be used instead of the one we signed the CSR with.
certBytes, keyBytes := splitCertKeyPEM(body)
if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", certFile)
} }
if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil { if len(keyBytes) > 0 {
return errors.Wrapf(err, "failed to write key %s", filename) if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write key %s", keyFile)
}
} }
return nil return nil
} }
@ -395,12 +458,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
} }
clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt") clientCAFile := filepath.Join(envInfo.DataDir, "agent", "client-ca.crt")
if err := getHostFile(clientCAFile, "", info); err != nil { if err := getHostFile(clientCAFile, info); err != nil {
return nil, err return nil, err
} }
serverCAFile := filepath.Join(envInfo.DataDir, "agent", "server-ca.crt") serverCAFile := filepath.Join(envInfo.DataDir, "agent", "server-ca.crt")
if err := getHostFile(serverCAFile, "", info); err != nil { if err := getHostFile(serverCAFile, info); err != nil {
return nil, err return nil, err
} }
@ -494,13 +557,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
// that the cert will not be valid for, as they are not present in the list collected here. // that the cert will not be valid for, as they are not present in the list collected here.
nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...) nodeExternalAndInternalIPs := append(nodeIPs, nodeExternalIPs...)
// Ask the server to generate a kubelet server cert+key. These files are unique to this node. // Ask the server to sign our kubelet server cert.
if err := getServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil { if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
return nil, errors.Wrap(err, servingKubeletCert) return nil, errors.Wrap(err, servingKubeletCert)
} }
// Ask the server to genrate a kubelet client cert+key. These files are unique to this node. // Ask the server to sign our kubelet client cert.
if err := getNodeNamedHostFile(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil { if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
return nil, errors.Wrap(err, clientKubeletCert) return nil, errors.Wrap(err, clientKubeletCert)
} }
@ -513,8 +576,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt") clientKubeProxyCert := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.crt")
clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key") clientKubeProxyKey := filepath.Join(envInfo.DataDir, "agent", "client-kube-proxy.key")
// Ask the server to send us its kube-proxy client cert+key. These files are not unique to this node. // Ask the server to sign our kube-proxy client cert.
if err := getHostFile(clientKubeProxyCert, clientKubeProxyKey, info); err != nil { if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
return nil, errors.Wrap(err, clientKubeProxyCert) return nil, errors.Wrap(err, clientKubeProxyCert)
} }
@ -527,8 +590,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt") clientK3sControllerCert := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.crt")
clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key") clientK3sControllerKey := filepath.Join(envInfo.DataDir, "agent", "client-"+version.Program+"-controller.key")
// Ask the server to send us its agent controller client cert+key. These files are not unique to this node. // Ask the server to sign our agent controller client cert.
if err := getHostFile(clientK3sControllerCert, clientK3sControllerKey, info); err != nil { if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
return nil, errors.Wrap(err, clientK3sControllerCert) return nil, errors.Wrap(err, clientK3sControllerCert)
} }