mirror of https://github.com/k3s-io/k3s
Merge pull request #49654 from jcbsmpsn/move-certificate-manager
Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Move certificate manager to client. Fixes https://github.com/kubernetes/kubernetes/issues/53452 **What this PR does / why we need it**: Migrate the certificate_manager to a location where it can be shared. ```release-note NONE ```pull/6/head
Kubernetes Submit Queue
GitHub
commit
f321a16af4
|
|
@ -121,6 +121,7 @@ go_library(
|
|||
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
|
||||
"//vendor/k8s.io/client-go/tools/record:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/cert:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/certificate:go_default_library",
|
||||
] + select({
|
||||
"@io_bazel_rules_go//go/platform:linux_amd64": [
|
||||
"//vendor/golang.org/x/exp/inotify:go_default_library",
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ import (
|
|||
"k8s.io/client-go/tools/clientcmd"
|
||||
"k8s.io/client-go/tools/record"
|
||||
certutil "k8s.io/client-go/util/cert"
|
||||
"k8s.io/client-go/util/certificate"
|
||||
"k8s.io/kubernetes/cmd/kubelet/app/options"
|
||||
"k8s.io/kubernetes/pkg/api"
|
||||
"k8s.io/kubernetes/pkg/capabilities"
|
||||
|
|
@ -64,7 +65,7 @@ import (
|
|||
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
|
||||
kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
||||
"k8s.io/kubernetes/pkg/kubelet/certificate"
|
||||
kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate"
|
||||
"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cm"
|
||||
"k8s.io/kubernetes/pkg/kubelet/config"
|
||||
|
|
@ -336,11 +337,11 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) {
|
|||
var clientCertificateManager certificate.Manager
|
||||
if err == nil {
|
||||
if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) {
|
||||
clientCertificateManager, err = certificate.NewKubeletClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData, clientConfig.CertFile, clientConfig.KeyFile)
|
||||
clientCertificateManager, err = kubeletcertificate.NewKubeletClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData, clientConfig.CertFile, clientConfig.KeyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := certificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager); err != nil {
|
||||
if err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -216,7 +216,6 @@ pkg/kubelet/apis/kubeletconfig
|
|||
pkg/kubelet/apis/kubeletconfig/v1alpha1
|
||||
pkg/kubelet/cadvisor
|
||||
pkg/kubelet/cadvisor/testing
|
||||
pkg/kubelet/certificate
|
||||
pkg/kubelet/client
|
||||
pkg/kubelet/cm
|
||||
pkg/kubelet/cm/util
|
||||
|
|
|
|||
|
|
@ -136,6 +136,7 @@ go_library(
|
|||
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
|
||||
"//vendor/k8s.io/client-go/tools/record:go_default_library",
|
||||
"//vendor/k8s.io/client-go/tools/remotecommand:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/certificate:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/integer:go_default_library",
|
||||
"//vendor/k8s.io/utils/exec:go_default_library",
|
||||
|
|
|
|||
|
|
@ -9,49 +9,34 @@ load(
|
|||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"certificate_manager.go",
|
||||
"certificate_store.go",
|
||||
"kubelet.go",
|
||||
"transport.go",
|
||||
],
|
||||
deps = [
|
||||
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
|
||||
"//pkg/kubelet/metrics:go_default_library",
|
||||
"//pkg/util/file:go_default_library",
|
||||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
|
||||
"//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/client-go/rest:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/cert:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/certificate:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"certificate_manager_test.go",
|
||||
"certificate_store_test.go",
|
||||
"transport_test.go",
|
||||
],
|
||||
srcs = ["transport_test.go"],
|
||||
library = ":go_default_library",
|
||||
deps = [
|
||||
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
|
||||
"//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/client-go/rest:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/cert:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,21 +22,25 @@ import (
|
|||
"fmt"
|
||||
"net"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
certificates "k8s.io/api/certificates/v1beta1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
clientcertificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
|
||||
"k8s.io/client-go/util/certificate"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
"k8s.io/kubernetes/pkg/kubelet/metrics"
|
||||
)
|
||||
|
||||
// NewKubeletServerCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate
|
||||
// or returns an error.
|
||||
func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string, certDirectory string) (Manager, error) {
|
||||
func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string, certDirectory string) (certificate.Manager, error) {
|
||||
var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface
|
||||
if kubeClient != nil && kubeClient.Certificates() != nil {
|
||||
certSigningRequestClient = kubeClient.Certificates().CertificateSigningRequests()
|
||||
}
|
||||
certificateStore, err := NewFileStore(
|
||||
certificateStore, err := certificate.NewFileStore(
|
||||
"kubelet-server",
|
||||
certDirectory,
|
||||
certDirectory,
|
||||
|
|
@ -45,8 +49,17 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize server certificate store: %v", err)
|
||||
}
|
||||
m, err := NewManager(&Config{
|
||||
Name: "server",
|
||||
var certificateExpiration = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: metrics.KubeletSubsystem,
|
||||
Subsystem: "certificate_manager",
|
||||
Name: "server_expiration_seconds",
|
||||
Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
|
||||
},
|
||||
)
|
||||
prometheus.MustRegister(certificateExpiration)
|
||||
|
||||
m, err := certificate.NewManager(&certificate.Config{
|
||||
CertificateSigningRequestClient: certSigningRequestClient,
|
||||
Template: &x509.CertificateRequest{
|
||||
Subject: pkix.Name{
|
||||
|
|
@ -70,7 +83,8 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
|
|||
// authenticate itself to a TLS client.
|
||||
certificates.UsageServerAuth,
|
||||
},
|
||||
CertificateStore: certificateStore,
|
||||
CertificateStore: certificateStore,
|
||||
CertificateExpiration: certificateExpiration,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize server certificate manager: %v", err)
|
||||
|
|
@ -82,8 +96,8 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
|
|||
// client that can be used to sign new certificates (or rotate). It answers with
|
||||
// whatever certificate it is initialized with. If a CSR client is set later, it
|
||||
// may begin rotating/renewing the client cert
|
||||
func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte, certFile string, keyFile string) (Manager, error) {
|
||||
certificateStore, err := NewFileStore(
|
||||
func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte, certFile string, keyFile string) (certificate.Manager, error) {
|
||||
certificateStore, err := certificate.NewFileStore(
|
||||
"kubelet-client",
|
||||
certDirectory,
|
||||
certDirectory,
|
||||
|
|
@ -92,8 +106,17 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
|
|||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize client certificate store: %v", err)
|
||||
}
|
||||
m, err := NewManager(&Config{
|
||||
Name: "client",
|
||||
var certificateExpiration = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: metrics.KubeletSubsystem,
|
||||
Subsystem: "certificate_manager",
|
||||
Name: "client_expiration_seconds",
|
||||
Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
|
||||
},
|
||||
)
|
||||
prometheus.MustRegister(certificateExpiration)
|
||||
|
||||
m, err := certificate.NewManager(&certificate.Config{
|
||||
Template: &x509.CertificateRequest{
|
||||
Subject: pkix.Name{
|
||||
CommonName: fmt.Sprintf("system:node:%s", nodeName),
|
||||
|
|
@ -118,6 +141,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
|
|||
CertificateStore: certificateStore,
|
||||
BootstrapCertificatePEM: certData,
|
||||
BootstrapKeyPEM: keyData,
|
||||
CertificateExpiration: certificateExpiration,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize client certificate manager: %v", err)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import (
|
|||
utilnet "k8s.io/apimachinery/pkg/util/net"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
restclient "k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/util/certificate"
|
||||
)
|
||||
|
||||
// UpdateTransport instruments a restconfig with a transport that dynamically uses
|
||||
|
|
@ -44,13 +45,13 @@ import (
|
|||
//
|
||||
// stopCh should be used to indicate when the transport is unused and doesn't need
|
||||
// to continue checking the manager.
|
||||
func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager Manager) error {
|
||||
func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error {
|
||||
return updateTransport(stopCh, 10*time.Second, clientConfig, clientCertificateManager)
|
||||
}
|
||||
|
||||
// updateTransport is an internal method that exposes how often this method checks that the
|
||||
// client cert has changed. Intended for testing.
|
||||
func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager Manager) error {
|
||||
func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error {
|
||||
if clientConfig.Transport != nil {
|
||||
return fmt.Errorf("there is already a transport configured")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package certificate
|
|||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -89,6 +90,29 @@ uC6Jo2eLcSV1sSdzTjaaWdM6XeYj6yHOAm8ZBIQs7m6V
|
|||
-----END RSA PRIVATE KEY-----`)
|
||||
)
|
||||
|
||||
type certificateData struct {
|
||||
keyPEM []byte
|
||||
certificatePEM []byte
|
||||
certificate *tls.Certificate
|
||||
}
|
||||
|
||||
func newCertificateData(certificatePEM string, keyPEM string) *certificateData {
|
||||
certificate, err := tls.X509KeyPair([]byte(certificatePEM), []byte(keyPEM))
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Unable to initialize certificate: %v", err))
|
||||
}
|
||||
certs, err := x509.ParseCertificates(certificate.Certificate[0])
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Unable to initialize certificate leaf: %v", err))
|
||||
}
|
||||
certificate.Leaf = certs[0]
|
||||
return &certificateData{
|
||||
keyPEM: []byte(keyPEM),
|
||||
certificatePEM: []byte(certificatePEM),
|
||||
certificate: &certificate,
|
||||
}
|
||||
}
|
||||
|
||||
type fakeManager struct {
|
||||
cert atomic.Value // Always a *tls.Certificate
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ import (
|
|||
corelisters "k8s.io/client-go/listers/core/v1"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"k8s.io/client-go/util/certificate"
|
||||
"k8s.io/client-go/util/flowcontrol"
|
||||
"k8s.io/client-go/util/integer"
|
||||
"k8s.io/kubernetes/cmd/kubelet/app/options"
|
||||
|
|
@ -64,7 +65,7 @@ import (
|
|||
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
||||
"k8s.io/kubernetes/pkg/kubelet/certificate"
|
||||
kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cm"
|
||||
"k8s.io/kubernetes/pkg/kubelet/config"
|
||||
"k8s.io/kubernetes/pkg/kubelet/configmap"
|
||||
|
|
@ -743,7 +744,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
|
|||
|
||||
ips = append(ips, cloudIPs...)
|
||||
names := append([]string{klet.GetHostname(), hostnameOverride}, cloudNames...)
|
||||
klet.serverCertificateManager, err = certificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names, certDirectory)
|
||||
klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names, certDirectory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -182,6 +182,7 @@ filegroup(
|
|||
"//staging/src/k8s.io/client-go/transport:all-srcs",
|
||||
"//staging/src/k8s.io/client-go/util/buffer:all-srcs",
|
||||
"//staging/src/k8s.io/client-go/util/cert:all-srcs",
|
||||
"//staging/src/k8s.io/client-go/util/certificate:all-srcs",
|
||||
"//staging/src/k8s.io/client-go/util/exec:all-srcs",
|
||||
"//staging/src/k8s.io/client-go/util/flowcontrol:all-srcs",
|
||||
"//staging/src/k8s.io/client-go/util/homedir:all-srcs",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
load(
|
||||
"@io_bazel_rules_go//go:def.bzl",
|
||||
"go_library",
|
||||
"go_test",
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"certificate_manager_test.go",
|
||||
"certificate_store_test.go",
|
||||
],
|
||||
library = ":go_default_library",
|
||||
tags = ["automanaged"],
|
||||
deps = [
|
||||
"//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/cert:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"certificate_manager.go",
|
||||
"certificate_store.go",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
deps = [
|
||||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
"//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
|
||||
"//vendor/k8s.io/client-go/util/cert:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
reviewers:
|
||||
- mikedanese
|
||||
- liggit
|
||||
- smarterclayton
|
||||
approvers:
|
||||
- mikedanese
|
||||
- liggit
|
||||
- smarterclayton
|
||||
|
|
@ -28,7 +28,6 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
certificates "k8s.io/api/certificates/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
|
@ -37,12 +36,6 @@ import (
|
|||
"k8s.io/apimachinery/pkg/watch"
|
||||
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
|
||||
"k8s.io/client-go/util/cert"
|
||||
"k8s.io/kubernetes/pkg/kubelet/metrics"
|
||||
)
|
||||
|
||||
const (
|
||||
certificateManagerSubsystem = "certificate_manager"
|
||||
certificateExpirationKey = "expiration_seconds"
|
||||
)
|
||||
|
||||
// Manager maintains and updates the certificates in use by this certificate
|
||||
|
|
@ -62,10 +55,6 @@ type Manager interface {
|
|||
|
||||
// Config is the set of configuration parameters available for a new Manager.
|
||||
type Config struct {
|
||||
// Name is a name describing the certificate being managed by this
|
||||
// certificate manager. It will be used for recording metrics relevant to
|
||||
// the certificate.
|
||||
Name string
|
||||
// CertificateSigningRequestClient will be used for signing new certificate
|
||||
// requests generated when a key rotation occurs. It must be set either at
|
||||
// initialization or by using CertificateSigningRequestClient before
|
||||
|
|
@ -103,6 +92,9 @@ type Config struct {
|
|||
// initialized using a generic, multi-use cert/key pair which will be
|
||||
// quickly replaced with a unique cert/key pair.
|
||||
BootstrapKeyPEM []byte
|
||||
// CertificateExpiration will record a metric that shows the remaining
|
||||
// lifetime of the certificate.
|
||||
CertificateExpiration Gauge
|
||||
}
|
||||
|
||||
// Store is responsible for getting and updating the current certificate.
|
||||
|
|
@ -121,6 +113,12 @@ type Store interface {
|
|||
Update(cert, key []byte) (*tls.Certificate, error)
|
||||
}
|
||||
|
||||
// Gauge will record the remaining lifetime of the certificate each time it is
|
||||
// updated.
|
||||
type Gauge interface {
|
||||
Set(float64)
|
||||
}
|
||||
|
||||
// NoCertKeyError indicates there is no cert/key currently available.
|
||||
type NoCertKeyError string
|
||||
|
||||
|
|
@ -135,17 +133,13 @@ type manager struct {
|
|||
cert *tls.Certificate
|
||||
rotationDeadline time.Time
|
||||
forceRotation bool
|
||||
certificateExpiration prometheus.Gauge
|
||||
certificateExpiration Gauge
|
||||
}
|
||||
|
||||
// NewManager returns a new certificate manager. A certificate manager is
|
||||
// responsible for being the authoritative source of certificates in the
|
||||
// Kubelet and handling updates due to rotation.
|
||||
func NewManager(config *Config) (Manager, error) {
|
||||
if config.Name == "" {
|
||||
return nil, fmt.Errorf("the 'Name' is required to disambiguate metric values of different certificate manager instances")
|
||||
}
|
||||
|
||||
cert, forceRotation, err := getCurrentCertificateOrBootstrap(
|
||||
config.CertificateStore,
|
||||
config.BootstrapCertificatePEM,
|
||||
|
|
@ -154,17 +148,6 @@ func NewManager(config *Config) (Manager, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
var certificateExpiration = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: metrics.KubeletSubsystem,
|
||||
Subsystem: certificateManagerSubsystem,
|
||||
Name: fmt.Sprintf("%s_%s", config.Name, certificateExpirationKey),
|
||||
Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
|
||||
},
|
||||
)
|
||||
|
||||
prometheus.MustRegister(certificateExpiration)
|
||||
|
||||
m := manager{
|
||||
certSigningRequestClient: config.CertificateSigningRequestClient,
|
||||
template: config.Template,
|
||||
|
|
@ -172,7 +155,7 @@ func NewManager(config *Config) (Manager, error) {
|
|||
certStore: config.CertificateStore,
|
||||
cert: cert,
|
||||
forceRotation: forceRotation,
|
||||
certificateExpiration: certificateExpiration,
|
||||
certificateExpiration: config.CertificateExpiration,
|
||||
}
|
||||
|
||||
return &m, nil
|
||||
|
|
@ -199,7 +182,7 @@ func (m *manager) SetCertificateSigningRequestClient(certSigningRequestClient ce
|
|||
m.certSigningRequestClient = certSigningRequestClient
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("CertificateSigningRequestClient is already set.")
|
||||
return fmt.Errorf("property CertificateSigningRequestClient is already set")
|
||||
}
|
||||
|
||||
// Start will start the background work of rotating the certificates.
|
||||
|
|
@ -335,16 +318,23 @@ func (m *manager) setRotationDeadline() {
|
|||
notAfter := m.cert.Leaf.NotAfter
|
||||
totalDuration := float64(notAfter.Sub(m.cert.Leaf.NotBefore))
|
||||
|
||||
// Use some jitter to set the rotation threshold so each node will rotate
|
||||
// at approximately 70-90% of the total lifetime of the certificate. With
|
||||
// jitter, if a number of nodes are added to a cluster at approximately the
|
||||
// same time (such as cluster creation time), they won't all try to rotate
|
||||
// certificates at the same time for the rest of the life of the cluster.
|
||||
jitteryDuration := wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3)
|
||||
|
||||
m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration)
|
||||
m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration))
|
||||
glog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, m.rotationDeadline)
|
||||
m.certificateExpiration.Set(float64(notAfter.Unix()))
|
||||
if m.certificateExpiration != nil {
|
||||
m.certificateExpiration.Set(float64(notAfter.Unix()))
|
||||
}
|
||||
}
|
||||
|
||||
// jitteryDuration uses some jitter to set the rotation threshold so each node
|
||||
// will rotate at approximately 70-90% of the total lifetime of the
|
||||
// certificate. With jitter, if a number of nodes are added to a cluster at
|
||||
// approximately the same time (such as cluster creation time), they won't all
|
||||
// try to rotate certificates at the same time for the rest of the life of the
|
||||
// cluster.
|
||||
//
|
||||
// This function is represented as a variable to allow replacement during testing.
|
||||
var jitteryDuration = func(totalDuration float64) time.Duration {
|
||||
return wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3)
|
||||
}
|
||||
|
||||
func (m *manager) updateCached(cert *tls.Certificate) {
|
||||
|
|
@ -26,20 +26,12 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
certificates "k8s.io/api/certificates/v1beta1"
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
watch "k8s.io/apimachinery/pkg/watch"
|
||||
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
|
||||
)
|
||||
|
||||
type certificateData struct {
|
||||
keyPEM []byte
|
||||
certificatePEM []byte
|
||||
certificate *tls.Certificate
|
||||
}
|
||||
|
||||
var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE-----
|
||||
MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV
|
||||
BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE
|
||||
|
|
@ -115,6 +107,12 @@ bDQT1r8Q3Gx+h9LRqQeHgPBQ3F5ylqqBAiBaJ0hkYvrIdWxNlcLqD3065bJpHQ4S
|
|||
WQkuZUQN1M/Xvg==
|
||||
-----END RSA PRIVATE KEY-----`)
|
||||
|
||||
type certificateData struct {
|
||||
keyPEM []byte
|
||||
certificatePEM []byte
|
||||
certificate *tls.Certificate
|
||||
}
|
||||
|
||||
func newCertificateData(certificatePEM string, keyPEM string) *certificateData {
|
||||
certificate, err := tls.X509KeyPair([]byte(certificatePEM), []byte(keyPEM))
|
||||
if err != nil {
|
||||
|
|
@ -137,7 +135,6 @@ func TestNewManagerNoRotation(t *testing.T) {
|
|||
cert: storeCertData.certificate,
|
||||
}
|
||||
if _, err := NewManager(&Config{
|
||||
Name: "test_no_rotation",
|
||||
Template: &x509.CertificateRequest{},
|
||||
Usages: []certificates.KeyUsage{},
|
||||
CertificateStore: store,
|
||||
|
|
@ -173,11 +170,6 @@ func TestShouldRotate(t *testing.T) {
|
|||
},
|
||||
template: &x509.CertificateRequest{},
|
||||
usages: []certificates.KeyUsage{},
|
||||
certificateExpiration: prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "test_gauge_name",
|
||||
},
|
||||
),
|
||||
}
|
||||
m.setRotationDeadline()
|
||||
if m.shouldRotate() != test.shouldRotate {
|
||||
|
|
@ -191,7 +183,19 @@ func TestShouldRotate(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
type gaugeMock struct {
|
||||
calls int
|
||||
lastValue float64
|
||||
}
|
||||
|
||||
func (g *gaugeMock) Set(v float64) {
|
||||
g.calls++
|
||||
g.lastValue = v
|
||||
}
|
||||
|
||||
func TestSetRotationDeadline(t *testing.T) {
|
||||
defer func(original func(float64) time.Duration) { jitteryDuration = original }(jitteryDuration)
|
||||
|
||||
now := time.Now()
|
||||
testCases := []struct {
|
||||
name string
|
||||
|
|
@ -211,6 +215,7 @@ func TestSetRotationDeadline(t *testing.T) {
|
|||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g := gaugeMock{}
|
||||
m := manager{
|
||||
cert: &tls.Certificate{
|
||||
Leaf: &x509.Certificate{
|
||||
|
|
@ -218,27 +223,27 @@ func TestSetRotationDeadline(t *testing.T) {
|
|||
NotAfter: tc.notAfter,
|
||||
},
|
||||
},
|
||||
template: &x509.CertificateRequest{},
|
||||
usages: []certificates.KeyUsage{},
|
||||
certificateExpiration: prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "test_gauge_name",
|
||||
},
|
||||
),
|
||||
template: &x509.CertificateRequest{},
|
||||
usages: []certificates.KeyUsage{},
|
||||
certificateExpiration: &g,
|
||||
}
|
||||
jitteryDuration = func(float64) time.Duration { return time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7) }
|
||||
lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7))
|
||||
upperBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.9))
|
||||
for i := 0; i < 1000; i++ {
|
||||
// setRotationDeadline includes jitter, so this needs to run many times for validation.
|
||||
m.setRotationDeadline()
|
||||
if m.rotationDeadline.Before(lowerBound) || m.rotationDeadline.After(upperBound) {
|
||||
t.Errorf("For notBefore %v, notAfter %v, the rotationDeadline %v should be between %v and %v.",
|
||||
tc.notBefore,
|
||||
tc.notAfter,
|
||||
m.rotationDeadline,
|
||||
lowerBound,
|
||||
upperBound)
|
||||
}
|
||||
|
||||
m.setRotationDeadline()
|
||||
|
||||
if !m.rotationDeadline.Equal(lowerBound) {
|
||||
t.Errorf("For notBefore %v, notAfter %v, the rotationDeadline %v should be %v.",
|
||||
tc.notBefore,
|
||||
tc.notAfter,
|
||||
m.rotationDeadline,
|
||||
lowerBound)
|
||||
}
|
||||
if g.calls != 1 {
|
||||
t.Errorf("%d metrics were recorded, wanted %d", g.calls, 1)
|
||||
}
|
||||
if g.lastValue != float64(tc.notAfter.Unix()) {
|
||||
t.Errorf("%d value for metric was recorded, wanted %d", g.lastValue, tc.notAfter.Unix())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -295,7 +300,6 @@ func TestNewManagerBootstrap(t *testing.T) {
|
|||
|
||||
var cm Manager
|
||||
cm, err := NewManager(&Config{
|
||||
Name: "test_bootstrap",
|
||||
Template: &x509.CertificateRequest{},
|
||||
Usages: []certificates.KeyUsage{},
|
||||
CertificateStore: store,
|
||||
|
|
@ -333,7 +337,6 @@ func TestNewManagerNoBootstrap(t *testing.T) {
|
|||
}
|
||||
|
||||
cm, err := NewManager(&Config{
|
||||
Name: "test_no_bootstrap",
|
||||
Template: &x509.CertificateRequest{},
|
||||
Usages: []certificates.KeyUsage{},
|
||||
CertificateStore: store,
|
||||
|
|
@ -469,14 +472,13 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.description, func(t *testing.T) {
|
||||
certificateStore := &fakeStore{
|
||||
cert: tc.storeCert.certificate,
|
||||
}
|
||||
|
||||
certificateManager, err := NewManager(&Config{
|
||||
Name: fmt.Sprintf("test_initialize_client_%d", i),
|
||||
Template: &x509.CertificateRequest{
|
||||
Subject: pkix.Name{
|
||||
Organization: []string{"system:nodes"},
|
||||
|
|
@ -571,14 +573,13 @@ func TestInitializeOtherRESTClients(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.description, func(t *testing.T) {
|
||||
certificateStore := &fakeStore{
|
||||
cert: tc.storeCert.certificate,
|
||||
}
|
||||
|
||||
certificateManager, err := NewManager(&Config{
|
||||
Name: fmt.Sprintf("test_initialize_other_rest_clients_%d", i),
|
||||
Template: &x509.CertificateRequest{
|
||||
Subject: pkix.Name{
|
||||
Organization: []string{"system:nodes"},
|
||||
|
|
@ -28,8 +28,6 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
utilfile "k8s.io/kubernetes/pkg/util/file"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -86,7 +84,7 @@ func NewFileStore(
|
|||
func (s *fileStore) recover() error {
|
||||
// If the 'current' file doesn't exist, continue on with the recovery process.
|
||||
currentPath := filepath.Join(s.certDirectory, s.filename(currentPair))
|
||||
if exists, err := utilfile.FileExists(currentPath); err != nil {
|
||||
if exists, err := fileExists(currentPath); err != nil {
|
||||
return err
|
||||
} else if exists {
|
||||
return nil
|
||||
|
|
@ -113,18 +111,18 @@ func (s *fileStore) recover() error {
|
|||
|
||||
func (s *fileStore) Current() (*tls.Certificate, error) {
|
||||
pairFile := filepath.Join(s.certDirectory, s.filename(currentPair))
|
||||
if pairFileExists, err := utilfile.FileExists(pairFile); err != nil {
|
||||
if pairFileExists, err := fileExists(pairFile); err != nil {
|
||||
return nil, err
|
||||
} else if pairFileExists {
|
||||
glog.Infof("Loading cert/key pair from %q.", pairFile)
|
||||
return loadFile(pairFile)
|
||||
}
|
||||
|
||||
certFileExists, err := utilfile.FileExists(s.certFile)
|
||||
certFileExists, err := fileExists(s.certFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keyFileExists, err := utilfile.FileExists(s.keyFile)
|
||||
keyFileExists, err := fileExists(s.keyFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -135,11 +133,11 @@ func (s *fileStore) Current() (*tls.Certificate, error) {
|
|||
|
||||
c := filepath.Join(s.certDirectory, s.pairNamePrefix+certExtension)
|
||||
k := filepath.Join(s.keyDirectory, s.pairNamePrefix+keyExtension)
|
||||
certFileExists, err = utilfile.FileExists(c)
|
||||
certFileExists, err = fileExists(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keyFileExists, err = utilfile.FileExists(k)
|
||||
keyFileExists, err = fileExists(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -240,7 +238,7 @@ func (s *fileStore) updateSymlink(filename string) error {
|
|||
return err
|
||||
}
|
||||
} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
||||
return fmt.Errorf("expected %q to be a symlink but it is a file.", currentPath)
|
||||
return fmt.Errorf("expected %q to be a symlink but it is a file", currentPath)
|
||||
} else {
|
||||
currentPathExists = true
|
||||
}
|
||||
|
|
@ -253,7 +251,7 @@ func (s *fileStore) updateSymlink(filename string) error {
|
|||
return err
|
||||
}
|
||||
} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
|
||||
return fmt.Errorf("expected %q to be a symlink but it is a file.", updatedPath)
|
||||
return fmt.Errorf("expected %q to be a symlink but it is a file", updatedPath)
|
||||
} else {
|
||||
if err := os.Remove(updatedPath); err != nil {
|
||||
return fmt.Errorf("unable to remove %q: %v", updatedPath, err)
|
||||
|
|
@ -262,7 +260,7 @@ func (s *fileStore) updateSymlink(filename string) error {
|
|||
|
||||
// Check that the new cert/key pair file exists to avoid rotating to an
|
||||
// invalid cert/key.
|
||||
if filenameExists, err := utilfile.FileExists(filename); err != nil {
|
||||
if filenameExists, err := fileExists(filename); err != nil {
|
||||
return err
|
||||
} else if !filenameExists {
|
||||
return fmt.Errorf("file %q does not exist so it can not be used as the currently selected cert/key", filename)
|
||||
|
|
@ -307,3 +305,13 @@ func loadX509KeyPair(certFile, keyFile string) (*tls.Certificate, error) {
|
|||
cert.Leaf = certs[0]
|
||||
return &cert, nil
|
||||
}
|
||||
|
||||
// FileExists checks if specified file exists.
|
||||
func fileExists(filename string) (bool, error) {
|
||||
if _, err := os.Stat(filename); os.IsNotExist(err) {
|
||||
return false, nil
|
||||
} else if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
Loading…
Reference in New Issue