From 1230d7b7dfae1d28fa4106b8795f25ee97323666 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Thu, 10 Dec 2020 11:26:16 -0700 Subject: [PATCH] Fix HA server initialization Signed-off-by: Erik Wilson --- pkg/server/router.go | 50 +++++++++++++++++++++++++++++++------------- pkg/server/server.go | 34 ++++++++++++++++-------------- 2 files changed, 54 insertions(+), 30 deletions(-) diff --git a/pkg/server/router.go b/pkg/server/router.go index 662d74f54c..2d5a6a68a5 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -26,18 +26,13 @@ const ( staticURL = "/static/" ) -func router(serverConfig *config.Control, tunnel http.Handler, secretClient coreclient.SecretClient) (http.Handler, error) { - ca, err := ioutil.ReadFile(serverConfig.Runtime.ServerCA) - if err != nil { - return nil, err - } - +func router(serverConfig *config.Control) http.Handler { prefix := "/v1-" + version.Program authed := mux.NewRouter() authed.Use(authMiddleware(serverConfig, version.Program+":agent")) authed.NotFoundHandler = serverConfig.Runtime.Handler - authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, secretClient)) - authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, secretClient)) + authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, serverConfig.Runtime)) + authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, serverConfig.Runtime)) authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey)) authed.Path(prefix + "/client-" + version.Program + "-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert, serverConfig.Runtime.ClientK3sControllerKey)) authed.Path(prefix + "/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA)) @@ -46,7 +41,7 @@ func router(serverConfig *config.Control, tunnel http.Handler, secretClient core nodeAuthed := mux.NewRouter() nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes")) - nodeAuthed.Path(prefix + "/connect").Handler(tunnel) + nodeAuthed.Path(prefix + "/connect").Handler(serverConfig.Runtime.Tunnel) nodeAuthed.NotFoundHandler = authed serverAuthed := mux.NewRouter() @@ -61,14 +56,23 @@ func router(serverConfig *config.Control, tunnel http.Handler, secretClient core router := mux.NewRouter() router.NotFoundHandler = serverAuthed router.PathPrefix(staticURL).Handler(serveStatic(staticURL, staticDir)) - router.Path("/cacerts").Handler(cacerts(ca)) + router.Path("/cacerts").Handler(cacerts(serverConfig.Runtime.ServerCA)) router.Path("/ping").Handler(ping()) - return router, nil + return router } -func cacerts(ca []byte) http.Handler { +func cacerts(serverCA string) http.Handler { + var ca []byte return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if ca == nil { + var err error + ca, err = ioutil.ReadFile(serverCA) + if err != nil { + sendError(err, resp) + return + } + } resp.Header().Set("content-type", "text/plain") resp.Write(ca) }) @@ -122,8 +126,17 @@ func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Cer return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil } -func servingKubeletCert(server *config.Control, keyFile string, secretClient coreclient.SecretClient) http.Handler { +func servingKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler { + var secretClient coreclient.SecretClient return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if secretClient == nil { + if runtime.Core == nil { + sendError(errors.New("runtime core not ready"), resp) + return + } + secretClient = runtime.Core.Core().V1().Secret() + } + if req.TLS == nil { resp.WriteHeader(http.StatusNotFound) return @@ -175,8 +188,17 @@ func servingKubeletCert(server *config.Control, keyFile string, secretClient cor }) } -func clientKubeletCert(server *config.Control, keyFile string, secretClient coreclient.SecretClient) http.Handler { +func clientKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler { + var secretClient coreclient.SecretClient return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if secretClient == nil { + if runtime.Core == nil { + sendError(errors.New("runtime core not ready"), resp) + return + } + secretClient = runtime.Core.Core().V1().Secret() + } + if req.TLS == nil { resp.WriteHeader(http.StatusNotFound) return diff --git a/pkg/server/server.go b/pkg/server/server.go index 160a43bccd..bf31470caf 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -60,6 +60,8 @@ func StartServer(ctx context.Context, config *Config) error { return errors.Wrap(err, "starting kubernetes") } + config.ControlConfig.Runtime.Handler = router(&config.ControlConfig) + go startOnAPIServerReady(ctx, config) for _, hook := range config.StartupHooks { @@ -108,9 +110,17 @@ func runControllers(ctx context.Context, config *Config) error { return err } + // run migration before we set controlConfig.Runtime.Core + if err := nodepassword.MigrateFile( + sc.Core.Core().V1().Secret(), + sc.Core.Core().V1().Node(), + controlConfig.Runtime.NodePasswdFile); err != nil { + logrus.Warn(errors.Wrapf(err, "error migrating node-password file")) + } controlConfig.Runtime.Core = sc.Core - if config.ControlConfig.Runtime.ClusterControllerStart != nil { - if err := config.ControlConfig.Runtime.ClusterControllerStart(ctx); err != nil { + + if controlConfig.Runtime.ClusterControllerStart != nil { + if err := controlConfig.Runtime.ClusterControllerStart(ctx); err != nil { return errors.Wrapf(err, "starting cluster controllers") } } @@ -126,11 +136,6 @@ func runControllers(ctx context.Context, config *Config) error { if err := sc.Start(ctx); err != nil { panic(err) } - handler, err := router(controlConfig, controlConfig.Runtime.Tunnel, sc.Core.Core().V1().Secret()) - if err != nil { - panic(errors.Wrap(err, "starting router")) - } - controlConfig.Runtime.Handler = handler } if !config.DisableAgent { go setControlPlaneRoleLabel(ctx, sc.Core.Core().V1().Node()) @@ -152,13 +157,6 @@ func runControllers(ctx context.Context, config *Config) error { } func coreControllers(ctx context.Context, sc *Context, config *Config) error { - if err := nodepassword.MigrateFile( - sc.Core.Core().V1().Secret(), - sc.Core.Core().V1().Node(), - config.ControlConfig.Runtime.NodePasswdFile); err != nil { - logrus.Warn(errors.Wrapf(err, "error migrating node-password file")) - } - if err := node.Register(ctx, !config.ControlConfig.Skips["coredns"], sc.Core.Core().V1().Secret(), @@ -425,6 +423,11 @@ func isSymlink(config string) bool { func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient) error { for { nodeName := os.Getenv("NODE_NAME") + if nodeName == "" { + logrus.Info("Waiting for control-plane node agent startup") + time.Sleep(1 * time.Second) + continue + } node, err := nodes.Get(nodeName, metav1.GetOptions{}) if err != nil { logrus.Infof("Waiting for control-plane node %s startup: %v", nodeName, err) @@ -454,7 +457,6 @@ func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient) error { } func setClusterDNSConfig(ctx context.Context, controlConfig *Config, configMap v1.ConfigMapClient) error { - nodeName := os.Getenv("NODE_NAME") // check if configmap already exists _, err := configMap.Get("kube-system", "cluster-dns", metav1.GetOptions{}) if err == nil { @@ -483,7 +485,7 @@ func setClusterDNSConfig(ctx context.Context, controlConfig *Config, configMap v logrus.Infof("Cluster dns configmap has been set successfully") break } - logrus.Infof("Waiting for control-plane node %s startup: %v", nodeName, err) + logrus.Infof("Waiting for control-plane dns startup: %v", err) select { case <-ctx.Done():