Fix HA server initialization

Signed-off-by: Erik Wilson <Erik.E.Wilson@gmail.com>

pkg/server/router.go

@@ -26,18 +26,13 @@ const (
 	staticURL = "/static/"
 )
 
-func router(serverConfig *config.Control, tunnel http.Handler, secretClient coreclient.SecretClient) (http.Handler, error) {
+func router(serverConfig *config.Control) http.Handler {
-	ca, err := ioutil.ReadFile(serverConfig.Runtime.ServerCA)
-	if err != nil {
-		return nil, err
-	}
-
 	prefix := "/v1-" + version.Program
 	authed := mux.NewRouter()
 	authed.Use(authMiddleware(serverConfig, version.Program+":agent"))
 	authed.NotFoundHandler = serverConfig.Runtime.Handler
-	authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, secretClient))
+	authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, serverConfig.Runtime))
-	authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, secretClient))
+	authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, serverConfig.Runtime))
 	authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey))
 	authed.Path(prefix + "/client-" + version.Program + "-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert, serverConfig.Runtime.ClientK3sControllerKey))
 	authed.Path(prefix + "/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
@@ -46,7 +41,7 @@ func router(serverConfig *config.Control, tunnel http.Handler, secretClient core
 
 	nodeAuthed := mux.NewRouter()
 	nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes"))
-	nodeAuthed.Path(prefix + "/connect").Handler(tunnel)
+	nodeAuthed.Path(prefix + "/connect").Handler(serverConfig.Runtime.Tunnel)
 	nodeAuthed.NotFoundHandler = authed
 
 	serverAuthed := mux.NewRouter()
@@ -61,14 +56,23 @@ func router(serverConfig *config.Control, tunnel http.Handler, secretClient core
 	router := mux.NewRouter()
 	router.NotFoundHandler = serverAuthed
 	router.PathPrefix(staticURL).Handler(serveStatic(staticURL, staticDir))
-	router.Path("/cacerts").Handler(cacerts(ca))
+	router.Path("/cacerts").Handler(cacerts(serverConfig.Runtime.ServerCA))
 	router.Path("/ping").Handler(ping())
 
-	return router, nil
+	return router
 }
 
-func cacerts(ca []byte) http.Handler {
+func cacerts(serverCA string) http.Handler {
+	var ca []byte
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if ca == nil {
+			var err error
+			ca, err = ioutil.ReadFile(serverCA)
+			if err != nil {
+				sendError(err, resp)
+				return
+			}
+		}
 		resp.Header().Set("content-type", "text/plain")
 		resp.Write(ca)
 	})
@@ -122,8 +126,17 @@ func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Cer
 	return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil
 }
 
-func servingKubeletCert(server *config.Control, keyFile string, secretClient coreclient.SecretClient) http.Handler {
+func servingKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler {
+	var secretClient coreclient.SecretClient
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if secretClient == nil {
+			if runtime.Core == nil {
+				sendError(errors.New("runtime core not ready"), resp)
+				return
+			}
+			secretClient = runtime.Core.Core().V1().Secret()
+		}
+
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
 			return
@@ -175,8 +188,17 @@ func servingKubeletCert(server *config.Control, keyFile string, secretClient cor
 	})
 }
 
-func clientKubeletCert(server *config.Control, keyFile string, secretClient coreclient.SecretClient) http.Handler {
+func clientKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler {
+	var secretClient coreclient.SecretClient
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if secretClient == nil {
+			if runtime.Core == nil {
+				sendError(errors.New("runtime core not ready"), resp)
+				return
+			}
+			secretClient = runtime.Core.Core().V1().Secret()
+		}
+
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
 			return

pkg/server/server.go

@@ -60,6 +60,8 @@ func StartServer(ctx context.Context, config *Config) error {
 		return errors.Wrap(err, "starting kubernetes")
 	}
 
+	config.ControlConfig.Runtime.Handler = router(&config.ControlConfig)
+
 	go startOnAPIServerReady(ctx, config)
 
 	for _, hook := range config.StartupHooks {
@@ -108,9 +110,17 @@ func runControllers(ctx context.Context, config *Config) error {
 		return err
 	}
 
+	// run migration before we set controlConfig.Runtime.Core
+	if err := nodepassword.MigrateFile(
+		sc.Core.Core().V1().Secret(),
+		sc.Core.Core().V1().Node(),
+		controlConfig.Runtime.NodePasswdFile); err != nil {
+		logrus.Warn(errors.Wrapf(err, "error migrating node-password file"))
+	}
 	controlConfig.Runtime.Core = sc.Core
-	if config.ControlConfig.Runtime.ClusterControllerStart != nil {
+
-		if err := config.ControlConfig.Runtime.ClusterControllerStart(ctx); err != nil {
+	if controlConfig.Runtime.ClusterControllerStart != nil {
+		if err := controlConfig.Runtime.ClusterControllerStart(ctx); err != nil {
 			return errors.Wrapf(err, "starting cluster controllers")
 		}
 	}
@@ -126,11 +136,6 @@ func runControllers(ctx context.Context, config *Config) error {
 		if err := sc.Start(ctx); err != nil {
 			panic(err)
 		}
-		handler, err := router(controlConfig, controlConfig.Runtime.Tunnel, sc.Core.Core().V1().Secret())
-		if err != nil {
-			panic(errors.Wrap(err, "starting router"))
-		}
-		controlConfig.Runtime.Handler = handler
 	}
 	if !config.DisableAgent {
 		go setControlPlaneRoleLabel(ctx, sc.Core.Core().V1().Node())
@@ -152,13 +157,6 @@ func runControllers(ctx context.Context, config *Config) error {
 }
 
 func coreControllers(ctx context.Context, sc *Context, config *Config) error {
-	if err := nodepassword.MigrateFile(
-		sc.Core.Core().V1().Secret(),
-		sc.Core.Core().V1().Node(),
-		config.ControlConfig.Runtime.NodePasswdFile); err != nil {
-		logrus.Warn(errors.Wrapf(err, "error migrating node-password file"))
-	}
-
 	if err := node.Register(ctx,
 		!config.ControlConfig.Skips["coredns"],
 		sc.Core.Core().V1().Secret(),
@@ -425,6 +423,11 @@ func isSymlink(config string) bool {
 func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
 	for {
 		nodeName := os.Getenv("NODE_NAME")
+		if nodeName == "" {
+			logrus.Info("Waiting for control-plane node agent startup")
+			time.Sleep(1 * time.Second)
+			continue
+		}
 		node, err := nodes.Get(nodeName, metav1.GetOptions{})
 		if err != nil {
 			logrus.Infof("Waiting for control-plane node %s startup: %v", nodeName, err)
@@ -454,7 +457,6 @@ func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
 }
 
 func setClusterDNSConfig(ctx context.Context, controlConfig *Config, configMap v1.ConfigMapClient) error {
-	nodeName := os.Getenv("NODE_NAME")
 	// check if configmap already exists
 	_, err := configMap.Get("kube-system", "cluster-dns", metav1.GetOptions{})
 	if err == nil {
@@ -483,7 +485,7 @@ func setClusterDNSConfig(ctx context.Context, controlConfig *Config, configMap v
 			logrus.Infof("Cluster dns configmap has been set successfully")
 			break
 		}
-		logrus.Infof("Waiting for control-plane node %s startup: %v", nodeName, err)
+		logrus.Infof("Waiting for control-plane dns startup: %v", err)
 
 		select {
 		case <-ctx.Done():