Verify the bootstrap client cert before using it

Before the bootstrap client is used, check a number of conditions that
ensure it can be safely loaded by the server. If any of those conditions
are invalid, re-bootstrap the node. This is primarily to force
bootstrapping without human intervention when a certificate is expired,
but also handles partial file corruption.

pkg/kubelet/certificate/bootstrap/BUILD

@@ -25,10 +25,12 @@ go_library(
         "//pkg/kubelet/util/csr:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
+        "//vendor/k8s.io/client-go/transport:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
     ],
 )

pkg/kubelet/certificate/bootstrap/bootstrap.go

@@ -20,14 +20,17 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/golang/glog"
 
 	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	certificates "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/transport"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/kubernetes/pkg/kubelet/util/csr"
 )
@@ -42,17 +45,15 @@ const (
 // On success, a kubeconfig file referencing the generated key and obtained certificate is written to kubeconfigPath.
 // The certificate and key file are stored in certDir.
 func LoadClientCert(kubeconfigPath string, bootstrapPath string, certDir string, nodeName types.NodeName) error {
-	// Short-circuit if the kubeconfig file already exists.
+	// Short-circuit if the kubeconfig file exists and is valid.
-	// TODO: inspect the kubeconfig, ensure a rest client can be built from it, verify client cert expiration, etc.
+	ok, err := verifyBootstrapClientConfig(kubeconfigPath)
-	_, err := os.Stat(kubeconfigPath)
+	if err != nil {
-	if err == nil {
-		glog.V(2).Infof("Kubeconfig %s exists, skipping bootstrap", kubeconfigPath)
-		return nil
-	}
-	if !os.IsNotExist(err) {
-		glog.Errorf("Error reading kubeconfig %s, skipping bootstrap: %v", kubeconfigPath, err)
 		return err
 	}
+	if ok {
+		glog.V(2).Infof("Kubeconfig %s exists and is valid, skipping bootstrap", kubeconfigPath)
+		return nil
+	}
 
 	glog.V(2).Info("Using bootstrap kubeconfig to generate TLS client cert, key and kubeconfig file")
 
@@ -150,3 +151,48 @@ func loadRESTClientConfig(kubeconfig string) (*restclient.Config, error) {
 		loader,
 	).ClientConfig()
 }
+
+// verifyBootstrapClientConfig checks the provided kubeconfig to see if it has a valid
+// client certificate. It returns true if the kubeconfig is valid, or an error if bootstrapping
+// should stop immediately.
+func verifyBootstrapClientConfig(kubeconfigPath string) (bool, error) {
+	_, err := os.Stat(kubeconfigPath)
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("error reading existing bootstrap kubeconfig %s: %v", kubeconfigPath, err)
+	}
+	bootstrapClientConfig, err := loadRESTClientConfig(kubeconfigPath)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("Unable to read existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	transportConfig, err := bootstrapClientConfig.TransportConfig()
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("Unable to load transport configuration from existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	// has side effect of populating transport config data fields
+	if _, err := transport.TLSConfigFor(transportConfig); err != nil {
+		utilruntime.HandleError(fmt.Errorf("Unable to load TLS configuration from existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	certs, err := certutil.ParseCertsPEM(transportConfig.TLS.CertData)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("Unable to load TLS certificates from existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	if len(certs) == 0 {
+		utilruntime.HandleError(fmt.Errorf("Unable to read TLS certificates from existing bootstrap client config: %v", err))
+		return false, nil
+	}
+	now := time.Now()
+	for _, cert := range certs {
+		if now.After(cert.NotAfter) {
+			utilruntime.HandleError(fmt.Errorf("Part of the existing bootstrap client certificate is expired: %s", cert.NotAfter))
+			return false, nil
+		}
+	}
+	return true, nil
+}