Merge pull request #49654 from jcbsmpsn/move-certificate-manager

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Move certificate manager to client.

Fixes https://github.com/kubernetes/kubernetes/issues/53452

**What this PR does / why we need it**:
Migrate the certificate_manager to a location where it can be shared.

```release-note
NONE
```

cmd/kubelet/app/BUILD

@@ -121,6 +121,7 @@ go_library(
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/k8s.io/client-go/tools/record:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
+        "//vendor/k8s.io/client-go/util/certificate:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:linux_amd64": [
             "//vendor/golang.org/x/exp/inotify:go_default_library",

cmd/kubelet/app/server.go

@@ -52,6 +52,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/certificate"
 	"k8s.io/kubernetes/cmd/kubelet/app/options"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/capabilities"
@@ -64,7 +65,7 @@ import (
 	kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
-	"k8s.io/kubernetes/pkg/kubelet/certificate"
+	kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate"
 	"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	"k8s.io/kubernetes/pkg/kubelet/config"
@@ -336,11 +337,11 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies) (err error) {
 		var clientCertificateManager certificate.Manager
 		if err == nil {
 			if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) {
-				clientCertificateManager, err = certificate.NewKubeletClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData, clientConfig.CertFile, clientConfig.KeyFile)
+				clientCertificateManager, err = kubeletcertificate.NewKubeletClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData, clientConfig.CertFile, clientConfig.KeyFile)
 				if err != nil {
 					return err
 				}
-				if err := certificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager); err != nil {
+				if err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager); err != nil {
 					return err
 				}
 			}

hack/.golint_failures

@@ -216,7 +216,6 @@ pkg/kubelet/apis/kubeletconfig
 pkg/kubelet/apis/kubeletconfig/v1alpha1
 pkg/kubelet/cadvisor
 pkg/kubelet/cadvisor/testing
-pkg/kubelet/certificate
 pkg/kubelet/client
 pkg/kubelet/cm
 pkg/kubelet/cm/util

pkg/kubelet/BUILD

@@ -136,6 +136,7 @@ go_library(
         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
         "//vendor/k8s.io/client-go/tools/record:go_default_library",
         "//vendor/k8s.io/client-go/tools/remotecommand:go_default_library",
+        "//vendor/k8s.io/client-go/util/certificate:go_default_library",
         "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library",
         "//vendor/k8s.io/client-go/util/integer:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",

pkg/kubelet/certificate/BUILD

@@ -9,49 +9,34 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
-        "certificate_manager.go",
-        "certificate_store.go",
         "kubelet.go",
         "transport.go",
     ],
     deps = [
         "//pkg/kubelet/apis/kubeletconfig:go_default_library",
         "//pkg/kubelet/metrics:go_default_library",
-        "//pkg/util/file:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/util/cert:go_default_library",
+        "//vendor/k8s.io/client-go/util/certificate:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["transport_test.go"],
-        "certificate_manager_test.go",
-        "certificate_store_test.go",
-        "transport_test.go",
-    ],
     library = ":go_default_library",
     deps = [
-        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
-        "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/util/cert:go_default_library",
     ],
 )
 

pkg/kubelet/certificate/kubelet.go

@@ -22,21 +22,25 @@ import (
 	"fmt"
 	"net"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	certificates "k8s.io/api/certificates/v1beta1"
 	"k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
 	clientcertificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
+	"k8s.io/client-go/util/certificate"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	"k8s.io/kubernetes/pkg/kubelet/metrics"
 )
 
 // NewKubeletServerCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate
 // or returns an error.
-func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string, certDirectory string) (Manager, error) {
+func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *kubeletconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string, certDirectory string) (certificate.Manager, error) {
 	var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface
 	if kubeClient != nil && kubeClient.Certificates() != nil {
 		certSigningRequestClient = kubeClient.Certificates().CertificateSigningRequests()
 	}
-	certificateStore, err := NewFileStore(
+	certificateStore, err := certificate.NewFileStore(
 		"kubelet-server",
 		certDirectory,
 		certDirectory,
@@ -45,8 +49,17 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize server certificate store: %v", err)
 	}
-	m, err := NewManager(&Config{
+	var certificateExpiration = prometheus.NewGauge(
-		Name: "server",
+		prometheus.GaugeOpts{
+			Namespace: metrics.KubeletSubsystem,
+			Subsystem: "certificate_manager",
+			Name:      "server_expiration_seconds",
+			Help:      "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
+		},
+	)
+	prometheus.MustRegister(certificateExpiration)
+
+	m, err := certificate.NewManager(&certificate.Config{
 		CertificateSigningRequestClient: certSigningRequestClient,
 		Template: &x509.CertificateRequest{
 			Subject: pkix.Name{
@@ -70,7 +83,8 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
 			// authenticate itself to a TLS client.
 			certificates.UsageServerAuth,
 		},
-		CertificateStore: certificateStore,
+		CertificateStore:      certificateStore,
+		CertificateExpiration: certificateExpiration,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize server certificate manager: %v", err)
@@ -82,8 +96,8 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
 // client that can be used to sign new certificates (or rotate). It answers with
 // whatever certificate it is initialized with. If a CSR client is set later, it
 // may begin rotating/renewing the client cert
-func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte, certFile string, keyFile string) (Manager, error) {
+func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte, certFile string, keyFile string) (certificate.Manager, error) {
-	certificateStore, err := NewFileStore(
+	certificateStore, err := certificate.NewFileStore(
 		"kubelet-client",
 		certDirectory,
 		certDirectory,
@@ -92,8 +106,17 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize client certificate store: %v", err)
 	}
-	m, err := NewManager(&Config{
+	var certificateExpiration = prometheus.NewGauge(
-		Name: "client",
+		prometheus.GaugeOpts{
+			Namespace: metrics.KubeletSubsystem,
+			Subsystem: "certificate_manager",
+			Name:      "client_expiration_seconds",
+			Help:      "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
+		},
+	)
+	prometheus.MustRegister(certificateExpiration)
+
+	m, err := certificate.NewManager(&certificate.Config{
 		Template: &x509.CertificateRequest{
 			Subject: pkix.Name{
 				CommonName:   fmt.Sprintf("system:node:%s", nodeName),
@@ -118,6 +141,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
 		CertificateStore:        certificateStore,
 		BootstrapCertificatePEM: certData,
 		BootstrapKeyPEM:         keyData,
+		CertificateExpiration:   certificateExpiration,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize client certificate manager: %v", err)

pkg/kubelet/certificate/transport.go

@@ -30,6 +30,7 @@ import (
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/util/certificate"
 )
 
 // UpdateTransport instruments a restconfig with a transport that dynamically uses
@@ -44,13 +45,13 @@ import (
 //
 // stopCh should be used to indicate when the transport is unused and doesn't need
 // to continue checking the manager.
-func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager Manager) error {
+func UpdateTransport(stopCh <-chan struct{}, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error {
 	return updateTransport(stopCh, 10*time.Second, clientConfig, clientCertificateManager)
 }
 
 // updateTransport is an internal method that exposes how often this method checks that the
 // client cert has changed. Intended for testing.
-func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager Manager) error {
+func updateTransport(stopCh <-chan struct{}, period time.Duration, clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error {
 	if clientConfig.Transport != nil {
 		return fmt.Errorf("there is already a transport configured")
 	}

pkg/kubelet/certificate/transport_test.go

@@ -19,6 +19,7 @@ package certificate
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"fmt"
 	"math/big"
 	"net/http"
 	"net/http/httptest"
@@ -89,6 +90,29 @@ uC6Jo2eLcSV1sSdzTjaaWdM6XeYj6yHOAm8ZBIQs7m6V
 -----END RSA PRIVATE KEY-----`)
 )
 
+type certificateData struct {
+	keyPEM         []byte
+	certificatePEM []byte
+	certificate    *tls.Certificate
+}
+
+func newCertificateData(certificatePEM string, keyPEM string) *certificateData {
+	certificate, err := tls.X509KeyPair([]byte(certificatePEM), []byte(keyPEM))
+	if err != nil {
+		panic(fmt.Sprintf("Unable to initialize certificate: %v", err))
+	}
+	certs, err := x509.ParseCertificates(certificate.Certificate[0])
+	if err != nil {
+		panic(fmt.Sprintf("Unable to initialize certificate leaf: %v", err))
+	}
+	certificate.Leaf = certs[0]
+	return &certificateData{
+		keyPEM:         []byte(keyPEM),
+		certificatePEM: []byte(certificatePEM),
+		certificate:    &certificate,
+	}
+}
+
 type fakeManager struct {
 	cert atomic.Value // Always a *tls.Certificate
 }

pkg/kubelet/kubelet.go

@@ -54,6 +54,7 @@ import (
 	corelisters "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/certificate"
 	"k8s.io/client-go/util/flowcontrol"
 	"k8s.io/client-go/util/integer"
 	"k8s.io/kubernetes/cmd/kubelet/app/options"
@@ -64,7 +65,7 @@ import (
 	kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
 	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
-	"k8s.io/kubernetes/pkg/kubelet/certificate"
+	kubeletcertificate "k8s.io/kubernetes/pkg/kubelet/certificate"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	"k8s.io/kubernetes/pkg/kubelet/configmap"
@@ -743,7 +744,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
 
 		ips = append(ips, cloudIPs...)
 		names := append([]string{klet.GetHostname(), hostnameOverride}, cloudNames...)
-		klet.serverCertificateManager, err = certificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names, certDirectory)
+		klet.serverCertificateManager, err = kubeletcertificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names, certDirectory)
 		if err != nil {
 			return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
 		}

staging/BUILD

@@ -182,6 +182,7 @@ filegroup(
         "//staging/src/k8s.io/client-go/transport:all-srcs",
         "//staging/src/k8s.io/client-go/util/buffer:all-srcs",
         "//staging/src/k8s.io/client-go/util/cert:all-srcs",
+        "//staging/src/k8s.io/client-go/util/certificate:all-srcs",
         "//staging/src/k8s.io/client-go/util/exec:all-srcs",
         "//staging/src/k8s.io/client-go/util/flowcontrol:all-srcs",
         "//staging/src/k8s.io/client-go/util/homedir:all-srcs",

staging/src/k8s.io/client-go/util/certificate/BUILD

@@ -0,0 +1,59 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "certificate_manager_test.go",
+        "certificate_store_test.go",
+    ],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
+        "//vendor/k8s.io/client-go/util/cert:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "certificate_manager.go",
+        "certificate_store.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
+        "//vendor/k8s.io/client-go/util/cert:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

staging/src/k8s.io/client-go/util/certificate/OWNERS

@@ -0,0 +1,8 @@
+reviewers:
+- mikedanese
+- liggit
+- smarterclayton
+approvers:
+- mikedanese
+- liggit
+- smarterclayton

staging/src/k8s.io/client-go/util/certificate/certificate_manager.go

@@ -28,7 +28,6 @@ import (
 	"time"
 
 	"github.com/golang/glog"
-	"github.com/prometheus/client_golang/prometheus"
 
 	certificates "k8s.io/api/certificates/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -37,12 +36,6 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	"k8s.io/client-go/util/cert"
-	"k8s.io/kubernetes/pkg/kubelet/metrics"
-)
-
-const (
-	certificateManagerSubsystem = "certificate_manager"
-	certificateExpirationKey    = "expiration_seconds"
 )
 
 // Manager maintains and updates the certificates in use by this certificate
@@ -62,10 +55,6 @@ type Manager interface {
 
 // Config is the set of configuration parameters available for a new Manager.
 type Config struct {
-	// Name is a name describing the certificate being managed by this
-	// certificate manager. It will be used for recording metrics relevant to
-	// the certificate.
-	Name string
 	// CertificateSigningRequestClient will be used for signing new certificate
 	// requests generated when a key rotation occurs. It must be set either at
 	// initialization or by using CertificateSigningRequestClient before
@@ -103,6 +92,9 @@ type Config struct {
 	// initialized using a generic, multi-use cert/key pair which will be
 	// quickly replaced with a unique cert/key pair.
 	BootstrapKeyPEM []byte
+	// CertificateExpiration will record a metric that shows the remaining
+	// lifetime of the certificate.
+	CertificateExpiration Gauge
 }
 
 // Store is responsible for getting and updating the current certificate.
@@ -121,6 +113,12 @@ type Store interface {
 	Update(cert, key []byte) (*tls.Certificate, error)
 }
 
+// Gauge will record the remaining lifetime of the certificate each time it is
+// updated.
+type Gauge interface {
+	Set(float64)
+}
+
 // NoCertKeyError indicates there is no cert/key currently available.
 type NoCertKeyError string
 
@@ -135,17 +133,13 @@ type manager struct {
 	cert                     *tls.Certificate
 	rotationDeadline         time.Time
 	forceRotation            bool
-	certificateExpiration    prometheus.Gauge
+	certificateExpiration    Gauge
 }
 
 // NewManager returns a new certificate manager. A certificate manager is
 // responsible for being the authoritative source of certificates in the
 // Kubelet and handling updates due to rotation.
 func NewManager(config *Config) (Manager, error) {
-	if config.Name == "" {
-		return nil, fmt.Errorf("the 'Name' is required to disambiguate metric values of different certificate manager instances")
-	}
-
 	cert, forceRotation, err := getCurrentCertificateOrBootstrap(
 		config.CertificateStore,
 		config.BootstrapCertificatePEM,
@@ -154,17 +148,6 @@ func NewManager(config *Config) (Manager, error) {
 		return nil, err
 	}
 
-	var certificateExpiration = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: metrics.KubeletSubsystem,
-			Subsystem: certificateManagerSubsystem,
-			Name:      fmt.Sprintf("%s_%s", config.Name, certificateExpirationKey),
-			Help:      "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
-		},
-	)
-
-	prometheus.MustRegister(certificateExpiration)
-
 	m := manager{
 		certSigningRequestClient: config.CertificateSigningRequestClient,
 		template:                 config.Template,
@@ -172,7 +155,7 @@ func NewManager(config *Config) (Manager, error) {
 		certStore:                config.CertificateStore,
 		cert:                     cert,
 		forceRotation:            forceRotation,
-		certificateExpiration:    certificateExpiration,
+		certificateExpiration:    config.CertificateExpiration,
 	}
 
 	return &m, nil
@@ -199,7 +182,7 @@ func (m *manager) SetCertificateSigningRequestClient(certSigningRequestClient ce
 		m.certSigningRequestClient = certSigningRequestClient
 		return nil
 	}
-	return fmt.Errorf("CertificateSigningRequestClient is already set.")
+	return fmt.Errorf("property CertificateSigningRequestClient is already set")
 }
 
 // Start will start the background work of rotating the certificates.
@@ -335,16 +318,23 @@ func (m *manager) setRotationDeadline() {
 	notAfter := m.cert.Leaf.NotAfter
 	totalDuration := float64(notAfter.Sub(m.cert.Leaf.NotBefore))
 
-	// Use some jitter to set the rotation threshold so each node will rotate
+	m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration(totalDuration))
-	// at approximately 70-90% of the total lifetime of the certificate.  With
-	// jitter, if a number of nodes are added to a cluster at approximately the
-	// same time (such as cluster creation time), they won't all try to rotate
-	// certificates at the same time for the rest of the life of the cluster.
-	jitteryDuration := wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3)
-
-	m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration)
 	glog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, m.rotationDeadline)
-	m.certificateExpiration.Set(float64(notAfter.Unix()))
+	if m.certificateExpiration != nil {
+		m.certificateExpiration.Set(float64(notAfter.Unix()))
+	}
+}
+
+// jitteryDuration uses some jitter to set the rotation threshold so each node
+// will rotate at approximately 70-90% of the total lifetime of the
+// certificate.  With jitter, if a number of nodes are added to a cluster at
+// approximately the same time (such as cluster creation time), they won't all
+// try to rotate certificates at the same time for the rest of the life of the
+// cluster.
+//
+// This function is represented as a variable to allow replacement during testing.
+var jitteryDuration = func(totalDuration float64) time.Duration {
+	return wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3)
 }
 
 func (m *manager) updateCached(cert *tls.Certificate) {

staging/src/k8s.io/client-go/util/certificate/certificate_manager_test.go

@@ -26,20 +26,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
-
 	certificates "k8s.io/api/certificates/v1beta1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	watch "k8s.io/apimachinery/pkg/watch"
 	certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 )
 
-type certificateData struct {
-	keyPEM         []byte
-	certificatePEM []byte
-	certificate    *tls.Certificate
-}
-
 var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE-----
 MIICRzCCAfGgAwIBAgIJALMb7ecMIk3MMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV
 BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE
@@ -115,6 +107,12 @@ bDQT1r8Q3Gx+h9LRqQeHgPBQ3F5ylqqBAiBaJ0hkYvrIdWxNlcLqD3065bJpHQ4S
 WQkuZUQN1M/Xvg==
 -----END RSA PRIVATE KEY-----`)
 
+type certificateData struct {
+	keyPEM         []byte
+	certificatePEM []byte
+	certificate    *tls.Certificate
+}
+
 func newCertificateData(certificatePEM string, keyPEM string) *certificateData {
 	certificate, err := tls.X509KeyPair([]byte(certificatePEM), []byte(keyPEM))
 	if err != nil {
@@ -137,7 +135,6 @@ func TestNewManagerNoRotation(t *testing.T) {
 		cert: storeCertData.certificate,
 	}
 	if _, err := NewManager(&Config{
-		Name:             "test_no_rotation",
 		Template:         &x509.CertificateRequest{},
 		Usages:           []certificates.KeyUsage{},
 		CertificateStore: store,
@@ -173,11 +170,6 @@ func TestShouldRotate(t *testing.T) {
 				},
 				template: &x509.CertificateRequest{},
 				usages:   []certificates.KeyUsage{},
-				certificateExpiration: prometheus.NewGauge(
-					prometheus.GaugeOpts{
-						Name: "test_gauge_name",
-					},
-				),
 			}
 			m.setRotationDeadline()
 			if m.shouldRotate() != test.shouldRotate {
@@ -191,7 +183,19 @@ func TestShouldRotate(t *testing.T) {
 	}
 }
 
+type gaugeMock struct {
+	calls     int
+	lastValue float64
+}
+
+func (g *gaugeMock) Set(v float64) {
+	g.calls++
+	g.lastValue = v
+}
+
 func TestSetRotationDeadline(t *testing.T) {
+	defer func(original func(float64) time.Duration) { jitteryDuration = original }(jitteryDuration)
+
 	now := time.Now()
 	testCases := []struct {
 		name         string
@@ -211,6 +215,7 @@ func TestSetRotationDeadline(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
+			g := gaugeMock{}
 			m := manager{
 				cert: &tls.Certificate{
 					Leaf: &x509.Certificate{
@@ -218,27 +223,27 @@ func TestSetRotationDeadline(t *testing.T) {
 						NotAfter:  tc.notAfter,
 					},
 				},
-				template: &x509.CertificateRequest{},
+				template:              &x509.CertificateRequest{},
-				usages:   []certificates.KeyUsage{},
+				usages:                []certificates.KeyUsage{},
-				certificateExpiration: prometheus.NewGauge(
+				certificateExpiration: &g,
-					prometheus.GaugeOpts{
-						Name: "test_gauge_name",
-					},
-				),
 			}
+			jitteryDuration = func(float64) time.Duration { return time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7) }
 			lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7))
-			upperBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.9))
+
-			for i := 0; i < 1000; i++ {
+			m.setRotationDeadline()
-				// setRotationDeadline includes jitter, so this needs to run many times for validation.
+
-				m.setRotationDeadline()
+			if !m.rotationDeadline.Equal(lowerBound) {
-				if m.rotationDeadline.Before(lowerBound) || m.rotationDeadline.After(upperBound) {
+				t.Errorf("For notBefore %v, notAfter %v, the rotationDeadline %v should be %v.",
-					t.Errorf("For notBefore %v, notAfter %v, the rotationDeadline %v should be between %v and %v.",
+					tc.notBefore,
-						tc.notBefore,
+					tc.notAfter,
-						tc.notAfter,
+					m.rotationDeadline,
-						m.rotationDeadline,
+					lowerBound)
-						lowerBound,
+			}
-						upperBound)
+			if g.calls != 1 {
-				}
+				t.Errorf("%d metrics were recorded, wanted %d", g.calls, 1)
+			}
+			if g.lastValue != float64(tc.notAfter.Unix()) {
+				t.Errorf("%d value for metric was recorded, wanted %d", g.lastValue, tc.notAfter.Unix())
 			}
 		})
 	}
@@ -295,7 +300,6 @@ func TestNewManagerBootstrap(t *testing.T) {
 
 	var cm Manager
 	cm, err := NewManager(&Config{
-		Name:                    "test_bootstrap",
 		Template:                &x509.CertificateRequest{},
 		Usages:                  []certificates.KeyUsage{},
 		CertificateStore:        store,
@@ -333,7 +337,6 @@ func TestNewManagerNoBootstrap(t *testing.T) {
 	}
 
 	cm, err := NewManager(&Config{
-		Name:                    "test_no_bootstrap",
 		Template:                &x509.CertificateRequest{},
 		Usages:                  []certificates.KeyUsage{},
 		CertificateStore:        store,
@@ -469,14 +472,13 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
 		},
 	}
 
-	for i, tc := range testCases {
+	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
 			certificateStore := &fakeStore{
 				cert: tc.storeCert.certificate,
 			}
 
 			certificateManager, err := NewManager(&Config{
-				Name: fmt.Sprintf("test_initialize_client_%d", i),
 				Template: &x509.CertificateRequest{
 					Subject: pkix.Name{
 						Organization: []string{"system:nodes"},
@@ -571,14 +573,13 @@ func TestInitializeOtherRESTClients(t *testing.T) {
 		},
 	}
 
-	for i, tc := range testCases {
+	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
 			certificateStore := &fakeStore{
 				cert: tc.storeCert.certificate,
 			}
 
 			certificateManager, err := NewManager(&Config{
-				Name: fmt.Sprintf("test_initialize_other_rest_clients_%d", i),
 				Template: &x509.CertificateRequest{
 					Subject: pkix.Name{
 						Organization: []string{"system:nodes"},

staging/src/k8s.io/client-go/util/certificate/certificate_store.go

@@ -28,8 +28,6 @@ import (
 	"time"
 
 	"github.com/golang/glog"
-
-	utilfile "k8s.io/kubernetes/pkg/util/file"
 )
 
 const (
@@ -86,7 +84,7 @@ func NewFileStore(
 func (s *fileStore) recover() error {
 	// If the 'current' file doesn't exist, continue on with the recovery process.
 	currentPath := filepath.Join(s.certDirectory, s.filename(currentPair))
-	if exists, err := utilfile.FileExists(currentPath); err != nil {
+	if exists, err := fileExists(currentPath); err != nil {
 		return err
 	} else if exists {
 		return nil
@@ -113,18 +111,18 @@ func (s *fileStore) recover() error {
 
 func (s *fileStore) Current() (*tls.Certificate, error) {
 	pairFile := filepath.Join(s.certDirectory, s.filename(currentPair))
-	if pairFileExists, err := utilfile.FileExists(pairFile); err != nil {
+	if pairFileExists, err := fileExists(pairFile); err != nil {
 		return nil, err
 	} else if pairFileExists {
 		glog.Infof("Loading cert/key pair from %q.", pairFile)
 		return loadFile(pairFile)
 	}
 
-	certFileExists, err := utilfile.FileExists(s.certFile)
+	certFileExists, err := fileExists(s.certFile)
 	if err != nil {
 		return nil, err
 	}
-	keyFileExists, err := utilfile.FileExists(s.keyFile)
+	keyFileExists, err := fileExists(s.keyFile)
 	if err != nil {
 		return nil, err
 	}
@@ -135,11 +133,11 @@ func (s *fileStore) Current() (*tls.Certificate, error) {
 
 	c := filepath.Join(s.certDirectory, s.pairNamePrefix+certExtension)
 	k := filepath.Join(s.keyDirectory, s.pairNamePrefix+keyExtension)
-	certFileExists, err = utilfile.FileExists(c)
+	certFileExists, err = fileExists(c)
 	if err != nil {
 		return nil, err
 	}
-	keyFileExists, err = utilfile.FileExists(k)
+	keyFileExists, err = fileExists(k)
 	if err != nil {
 		return nil, err
 	}
@@ -240,7 +238,7 @@ func (s *fileStore) updateSymlink(filename string) error {
 			return err
 		}
 	} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
-		return fmt.Errorf("expected %q to be a symlink but it is a file.", currentPath)
+		return fmt.Errorf("expected %q to be a symlink but it is a file", currentPath)
 	} else {
 		currentPathExists = true
 	}
@@ -253,7 +251,7 @@ func (s *fileStore) updateSymlink(filename string) error {
 			return err
 		}
 	} else if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
-		return fmt.Errorf("expected %q to be a symlink but it is a file.", updatedPath)
+		return fmt.Errorf("expected %q to be a symlink but it is a file", updatedPath)
 	} else {
 		if err := os.Remove(updatedPath); err != nil {
 			return fmt.Errorf("unable to remove %q: %v", updatedPath, err)
@@ -262,7 +260,7 @@ func (s *fileStore) updateSymlink(filename string) error {
 
 	// Check that the new cert/key pair file exists to avoid rotating to an
 	// invalid cert/key.
-	if filenameExists, err := utilfile.FileExists(filename); err != nil {
+	if filenameExists, err := fileExists(filename); err != nil {
 		return err
 	} else if !filenameExists {
 		return fmt.Errorf("file %q does not exist so it can not be used as the currently selected cert/key", filename)
@@ -307,3 +305,13 @@ func loadX509KeyPair(certFile, keyFile string) (*tls.Certificate, error) {
 	cert.Leaf = certs[0]
 	return &cert, nil
 }
+
+// FileExists checks if specified file exists.
+func fileExists(filename string) (bool, error) {
+	if _, err := os.Stat(filename); os.IsNotExist(err) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	return true, nil
+}