github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Rotate kubelet client certificate. 

Changes the kubelet so it bootstraps off the cert/key specified in the
config file and uses those to request new cert/key pairs from the
Certificate Signing Request API, as well as rotating client certificates
when they approach expiration.
pull/6/head
Jacob Simpson 2017-02-06 09:26:32 -08:00
parent 14a1cdd208
commit 1519bb94dc
5 changed files with 130 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/kubelet/app/BUILD
View File

@ -37,6 +37,7 @@ go_library(
"//cmd/kubelet/app/options:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/apis/certificates/v1beta1:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/componentconfig/v1alpha1:go_default_library",
"//pkg/capabilities:go_default_library",
@ -52,6 +53,7 @@ go_library(
"//pkg/features:go_default_library",
"//pkg/kubelet:go_default_library",
"//pkg/kubelet/cadvisor:go_default_library",
"//pkg/kubelet/certificate:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/config:go_default_library",
"//pkg/kubelet/container:go_default_library",
@ -110,6 +112,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",

109
cmd/kubelet/app/server.go
View File

@ -19,6 +19,8 @@ package app
import (
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"errors"
"fmt"
"io/ioutil"
@ -41,6 +43,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilnet "k8s.io/apimachinery/pkg/util/net"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
@ -58,6 +61,7 @@ import (
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
"k8s.io/kubernetes/pkg/apis/componentconfig"
componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
"k8s.io/kubernetes/pkg/capabilities"
@ -68,6 +72,7 @@ import (
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/certificate"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
@ -446,10 +451,30 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
}
clientConfig, err := CreateAPIServerClientConfig(s)
var clientCertificateManager certificate.Manager
if err == nil {
if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) {
nodeName, err := getNodeName(cloud, nodeutil.GetHostname(s.HostnameOverride))
if err != nil {
return err
}
clientCertificateManager, err = initializeClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData)
if err != nil {
return err
}
if err := updateTransport(clientConfig, clientCertificateManager); err != nil {
return err
}
}
kubeClient, err = clientset.NewForConfig(clientConfig)
if err != nil {
glog.Warningf("New kubeClient from clientConfig error: %v", err)
} else if kubeClient.Certificates() != nil && clientCertificateManager != nil {
glog.V(2).Info("Starting client certificate rotation.")
clientCertificateManager.SetCertificateSigningRequestClient(kubeClient.Certificates().CertificateSigningRequests())
clientCertificateManager.Start()
}
externalKubeClient, err = clientgoclientset.NewForConfig(clientConfig)
if err != nil {
@ -597,6 +622,90 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
return nil
}
func updateTransport(clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error {
if clientConfig.Transport != nil {
return fmt.Errorf("there is already a transport configured")
}
tlsConfig, err := restclient.TLSConfigFor(clientConfig)
if err != nil {
return fmt.Errorf("unable to configure TLS for the rest client: %v", err)
}
if tlsConfig == nil {
tlsConfig = &tls.Config{}
}
tlsConfig.Certificates = nil
tlsConfig.GetClientCertificate = func(requestInfo *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert := clientCertificateManager.Current()
if cert == nil {
return &tls.Certificate{Certificate: nil}, nil
}
return cert, nil
}
clientConfig.Transport = utilnet.SetTransportDefaults(&http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: tlsConfig,
MaxIdleConnsPerHost: 25,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
})
clientConfig.CertData = nil
clientConfig.KeyData = nil
clientConfig.CertFile = ""
clientConfig.KeyFile = ""
clientConfig.CAData = nil
clientConfig.CAFile = ""
return nil
}
// initializeClientCertificateManager sets up a certificate manager without a
// client that can be used to sign new certificates (or rotate). It answers with
// whatever certificate it is initialized with. If a CSR client is set later, it
// may begin rotating/renewing the client cert
func initializeClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte) (certificate.Manager, error) {
certificateStore, err := certificate.NewFileStore(
"kubelet-client",
certDirectory,
certDirectory,
"",
"")
if err != nil {
return nil, fmt.Errorf("failed to initialize certificate store: %v", err)
}
clientCertificateManager, err := certificate.NewManager(&certificate.Config{
Template: &x509.CertificateRequest{
Subject: pkix.Name{
Organization: []string{"system:nodes"},
CommonName: fmt.Sprintf("system:node:%s", nodeName),
},
},
Usages: []certificates.KeyUsage{
// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
//
// DigitalSignature allows the certificate to be used to verify
// digital signatures including signatures used during TLS
// negotiation.
certificates.UsageDigitalSignature,
// KeyEncipherment allows the cert/key pair to be used to encrypt
// keys, including the symetric keys negotiated during TLS setup
// and used for data transfer..
certificates.UsageKeyEncipherment,
// ClientAuth allows the cert to be used by a TLS client to
// authenticate itself to the TLS server.
certificates.UsageClientAuth,
},
CertificateStore: certificateStore,
BootstrapCertificatePEM: certData,
BootstrapKeyPEM: keyData,
})
if err != nil {
return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
}
return clientCertificateManager, nil
}
// getNodeName returns the node name according to the cloud provider
// if cloud provider is specified. Otherwise, returns the hostname of the node.
func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) {

8
pkg/features/kube_features.go
View File

@ -97,6 +97,13 @@ const (
// certificate as expiration approaches.
RotateKubeletServerCertificate utilfeature.Feature = "RotateKubeletServerCertificate"
// owner: @jcbsmpsn
// alpha: v1.7
//
// Automatically renews the client certificate used for communicating with
// the API server as the certificate approaches expiration.
RotateKubeletClientCertificate utilfeature.Feature = "RotateKubeletClientCertificate"
// owner: @msau
// alpha: v1.7
//
@ -128,6 +135,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
Accelerators: {Default: false, PreRelease: utilfeature.Alpha},
TaintBasedEvictions: {Default: false, PreRelease: utilfeature.Alpha},
RotateKubeletServerCertificate: {Default: false, PreRelease: utilfeature.Alpha},
RotateKubeletClientCertificate: {Default: false, PreRelease: utilfeature.Alpha},
PersistentLocalVolumes: {Default: false, PreRelease: utilfeature.Alpha},
LocalStorageCapacityIsolation: {Default: false, PreRelease: utilfeature.Alpha},

10
pkg/kubelet/certificate/certificate_manager.go
View File

@ -52,14 +52,17 @@ type Manager interface {
// Start the API server status sync loop.
Start()
// Current returns the currently selected certificate from the
// certificate manager.
// certificate manager, as well as the associated certificate and key data
// in PEM format.
Current() *tls.Certificate
}
// Config is the set of configuration parameters available for a new Manager.
type Config struct {
// CertificateSigningRequestClient will be used for signing new certificate
// requests generated when a key rotation occurs.
// requests generated when a key rotation occurs. It must be set either at
// initialization or by using CertificateSigningRequestClient before
// Manager.Start() is called.
CertificateSigningRequestClient certificatesclient.CertificateSigningRequestInterface
// Template is the CertificateRequest that will be used as a template for
// generating certificate signing requests for all new keys generated as
@ -99,7 +102,8 @@ type Config struct {
// Depending on the concrete implementation, the backing store for this
// behavior may vary.
type Store interface {
// Current returns the currently selected certificate. If the Store doesn't
// Current returns the currently selected certificate, as well as the
// associated certificate and key data in PEM format. If the Store doesn't
// have a cert/key pair currently, it should return a NoCertKeyError so
// that the Manager can recover by using bootstrap certificates to request
// a new cert/key pair.

8
pkg/kubelet/certificate/certificate_manager_test.go
View File

@ -173,7 +173,7 @@ func TestShouldRotate(t *testing.T) {
}
m.setRotationDeadline()
if m.shouldRotate() != test.shouldRotate {
t.Errorf("For time %v, a certificate issued for (%v, %v) should rotate should be %t.",
t.Errorf("Time %v, a certificate issued for (%v, %v) should rotate should be %t.",
now,
m.cert.Leaf.NotBefore,
m.cert.Leaf.NotAfter,
@ -296,7 +296,6 @@ func TestNewManagerBootstrap(t *testing.T) {
if cert == nil {
t.Errorf("Certificate was nil, expected something.")
}
if m, ok := cm.(*manager); !ok {
t.Errorf("Expected a '*manager' from 'NewManager'")
} else if !m.shouldRotate() {
@ -335,7 +334,6 @@ func TestNewManagerNoBootstrap(t *testing.T) {
if currentCert == nil {
t.Errorf("Certificate was nil, expected something.")
}
if m, ok := cm.(*manager); !ok {
t.Errorf("Expected a '*manager' from 'NewManager'")
} else {
@ -386,8 +384,8 @@ func TestGetCurrentCertificateOrBootstrap(t *testing.T) {
store,
tc.bootstrapCertData,
tc.bootstrapKeyData)
if certResult == nil || tc.expectedCert == nil {
if certResult != tc.expectedCert {
if certResult == nil || certResult.Certificate == nil || tc.expectedCert == nil {
if certResult != nil && tc.expectedCert != nil {
t.Errorf("Got certificate %v, wanted %v", certResult, tc.expectedCert)
}
} else {