Merge pull request #51031 from jcbsmpsn/metric-certificate-expiration-on-kubelet

Automatic merge from submit-queue (batch tested with PRs 51031, 51705, 51888, 51727, 51684). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>..

Add a kubelet metric to track certificate expiration.

Fix https://github.com/kubernetes/kubernetes/issues/51964

```release-note
Add a metric to the kubelet to monitor remaining lifetime of the certificate that
authenticates the kubelet to the API server.
```

pkg/kubelet/certificate/BUILD

@@ -16,8 +16,10 @@ go_library(
     ],
     deps = [
         "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+        "//pkg/kubelet/metrics:go_default_library",
         "//pkg/util/file:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
@@ -41,6 +43,7 @@ go_test(
     ],
     library = ":go_default_library",
     deps = [
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",

pkg/kubelet/certificate/certificate_manager.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus"
 
 	certificates "k8s.io/api/certificates/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -36,10 +37,13 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	"k8s.io/client-go/util/cert"
+	"k8s.io/kubernetes/pkg/kubelet/metrics"
 )
 
 const (
-	syncPeriod = 1 * time.Hour
+	syncPeriod                  = 1 * time.Hour
+	certificateManagerSubsystem = "certificate_manager"
+	certificateExpirationKey    = "expiration_seconds"
 )
 
 // Manager maintains and updates the certificates in use by this certificate
@@ -59,6 +63,10 @@ type Manager interface {
 
 // Config is the set of configuration parameters available for a new Manager.
 type Config struct {
+	// Name is a name describing the certificate being managed by this
+	// certificate manager. It will be used for recording metrics relevant to
+	// the certificate.
+	Name string
 	// CertificateSigningRequestClient will be used for signing new certificate
 	// requests generated when a key rotation occurs. It must be set either at
 	// initialization or by using CertificateSigningRequestClient before
@@ -128,12 +136,17 @@ type manager struct {
 	cert                     *tls.Certificate
 	rotationDeadline         time.Time
 	forceRotation            bool
+	certificateExpiration    prometheus.Gauge
 }
 
 // NewManager returns a new certificate manager. A certificate manager is
 // responsible for being the authoritative source of certificates in the
 // Kubelet and handling updates due to rotation.
 func NewManager(config *Config) (Manager, error) {
+	if config.Name == "" {
+		return nil, fmt.Errorf("the 'Name' is required to disambiguate metric values of different certificate manager instances")
+	}
+
 	cert, forceRotation, err := getCurrentCertificateOrBootstrap(
 		config.CertificateStore,
 		config.BootstrapCertificatePEM,
@@ -142,6 +155,17 @@ func NewManager(config *Config) (Manager, error) {
 		return nil, err
 	}
 
+	var certificateExpiration = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: metrics.KubeletSubsystem,
+			Subsystem: certificateManagerSubsystem,
+			Name:      fmt.Sprintf("%s_%s", config.Name, certificateExpirationKey),
+			Help:      "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
+		},
+	)
+
+	prometheus.MustRegister(certificateExpiration)
+
 	m := manager{
 		certSigningRequestClient: config.CertificateSigningRequestClient,
 		template:                 config.Template,
@@ -149,6 +173,7 @@ func NewManager(config *Config) (Manager, error) {
 		certStore:                config.CertificateStore,
 		cert:                     cert,
 		forceRotation:            forceRotation,
+		certificateExpiration:    certificateExpiration,
 	}
 
 	return &m, nil
@@ -319,7 +344,8 @@ func (m *manager) setRotationDeadline() {
 	jitteryDuration := wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3)
 
 	m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration)
-	glog.V(2).Infof("Certificate rotation deadline is %v", m.rotationDeadline)
+	glog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, m.rotationDeadline)
+	m.certificateExpiration.Set(float64(notAfter.Unix()))
 }
 
 func (m *manager) updateCached(cert *tls.Certificate) {

pkg/kubelet/certificate/certificate_manager_test.go

@@ -26,6 +26,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	certificates "k8s.io/api/certificates/v1beta1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	watch "k8s.io/apimachinery/pkg/watch"
@@ -135,6 +137,7 @@ func TestNewManagerNoRotation(t *testing.T) {
 		cert: storeCertData.certificate,
 	}
 	if _, err := NewManager(&Config{
+		Name:             "test_no_rotation",
 		Template:         &x509.CertificateRequest{},
 		Usages:           []certificates.KeyUsage{},
 		CertificateStore: store,
@@ -170,6 +173,11 @@ func TestShouldRotate(t *testing.T) {
 				},
 				template: &x509.CertificateRequest{},
 				usages:   []certificates.KeyUsage{},
+				certificateExpiration: prometheus.NewGauge(
+					prometheus.GaugeOpts{
+						Name: "test_gauge_name",
+					},
+				),
 			}
 			m.setRotationDeadline()
 			if m.shouldRotate() != test.shouldRotate {
@@ -212,6 +220,11 @@ func TestSetRotationDeadline(t *testing.T) {
 				},
 				template: &x509.CertificateRequest{},
 				usages:   []certificates.KeyUsage{},
+				certificateExpiration: prometheus.NewGauge(
+					prometheus.GaugeOpts{
+						Name: "test_gauge_name",
+					},
+				),
 			}
 			lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7))
 			upperBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.9))
@@ -282,6 +295,7 @@ func TestNewManagerBootstrap(t *testing.T) {
 
 	var cm Manager
 	cm, err := NewManager(&Config{
+		Name:                    "test_bootstrap",
 		Template:                &x509.CertificateRequest{},
 		Usages:                  []certificates.KeyUsage{},
 		CertificateStore:        store,
@@ -319,6 +333,7 @@ func TestNewManagerNoBootstrap(t *testing.T) {
 	}
 
 	cm, err := NewManager(&Config{
+		Name:                    "test_no_bootstrap",
 		Template:                &x509.CertificateRequest{},
 		Usages:                  []certificates.KeyUsage{},
 		CertificateStore:        store,
@@ -454,13 +469,14 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
 		},
 	}
 
-	for _, tc := range testCases {
+	for i, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
 			certificateStore := &fakeStore{
 				cert: tc.storeCert.certificate,
 			}
 
 			certificateManager, err := NewManager(&Config{
+				Name: fmt.Sprintf("test_initialize_client_%d", i),
 				Template: &x509.CertificateRequest{
 					Subject: pkix.Name{
 						Organization: []string{"system:nodes"},
@@ -555,13 +571,14 @@ func TestInitializeOtherRESTClients(t *testing.T) {
 		},
 	}
 
-	for _, tc := range testCases {
+	for i, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
 			certificateStore := &fakeStore{
 				cert: tc.storeCert.certificate,
 			}
 
 			certificateManager, err := NewManager(&Config{
+				Name: fmt.Sprintf("test_initialize_other_rest_clients_%d", i),
 				Template: &x509.CertificateRequest{
 					Subject: pkix.Name{
 						Organization: []string{"system:nodes"},

pkg/kubelet/certificate/kubelet.go

@@ -46,6 +46,7 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
 		return nil, fmt.Errorf("failed to initialize server certificate store: %v", err)
 	}
 	m, err := NewManager(&Config{
+		Name: "server",
 		CertificateSigningRequestClient: certSigningRequestClient,
 		Template: &x509.CertificateRequest{
 			Subject: pkix.Name{
@@ -92,6 +93,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
 		return nil, fmt.Errorf("failed to initialize client certificate store: %v", err)
 	}
 	m, err := NewManager(&Config{
+		Name: "client",
 		Template: &x509.CertificateRequest{
 			Subject: pkix.Name{
 				CommonName:   fmt.Sprintf("system:node:%s", nodeName),