github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix HA server initialization 

Signed-off-by: Erik Wilson <Erik.E.Wilson@gmail.com>
pull/2714/head v1.20.0-rc2+k3s1
Erik Wilson 2020-12-10 11:26:16 -07:00 committed by Brad Davidson
parent 8e4d3e645b
commit 1230d7b7df
2 changed files with 54 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
pkg/server/router.go
View File

@ -26,18 +26,13 @@ const (
staticURL = "/static/"
)
func router(serverConfig *config.Control, tunnel http.Handler, secretClient coreclient.SecretClient) (http.Handler, error) {
ca, err := ioutil.ReadFile(serverConfig.Runtime.ServerCA)
if err != nil {
return nil, err
}
func router(serverConfig *config.Control) http.Handler {
prefix := "/v1-" + version.Program
authed := mux.NewRouter()
authed.Use(authMiddleware(serverConfig, version.Program+":agent"))
authed.NotFoundHandler = serverConfig.Runtime.Handler
authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, secretClient))
authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, secretClient))
authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, serverConfig.Runtime))
authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, serverConfig.Runtime))
authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey))
authed.Path(prefix + "/client-" + version.Program + "-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert, serverConfig.Runtime.ClientK3sControllerKey))
authed.Path(prefix + "/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
@ -46,7 +41,7 @@ func router(serverConfig *config.Control, tunnel http.Handler, secretClient core
nodeAuthed := mux.NewRouter()
nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes"))
nodeAuthed.Path(prefix + "/connect").Handler(tunnel)
nodeAuthed.Path(prefix + "/connect").Handler(serverConfig.Runtime.Tunnel)
nodeAuthed.NotFoundHandler = authed
serverAuthed := mux.NewRouter()
@ -61,14 +56,23 @@ func router(serverConfig *config.Control, tunnel http.Handler, secretClient core
router := mux.NewRouter()
router.NotFoundHandler = serverAuthed
router.PathPrefix(staticURL).Handler(serveStatic(staticURL, staticDir))
router.Path("/cacerts").Handler(cacerts(ca))
router.Path("/cacerts").Handler(cacerts(serverConfig.Runtime.ServerCA))
router.Path("/ping").Handler(ping())
return router, nil
return router
}
func cacerts(ca []byte) http.Handler {
func cacerts(serverCA string) http.Handler {
var ca []byte
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if ca == nil {
var err error
ca, err = ioutil.ReadFile(serverCA)
if err != nil {
sendError(err, resp)
return
}
}
resp.Header().Set("content-type", "text/plain")
resp.Write(ca)
})
@ -122,8 +126,17 @@ func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Cer
return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil
}
func servingKubeletCert(server *config.Control, keyFile string, secretClient coreclient.SecretClient) http.Handler {
func servingKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler {
var secretClient coreclient.SecretClient
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if secretClient == nil {
if runtime.Core == nil {
sendError(errors.New("runtime core not ready"), resp)
return
}
secretClient = runtime.Core.Core().V1().Secret()
}
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return
@ -175,8 +188,17 @@ func servingKubeletCert(server *config.Control, keyFile string, secretClient cor
})
}
func clientKubeletCert(server *config.Control, keyFile string, secretClient coreclient.SecretClient) http.Handler {
func clientKubeletCert(server *config.Control, keyFile string, runtime *config.ControlRuntime) http.Handler {
var secretClient coreclient.SecretClient
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if secretClient == nil {
if runtime.Core == nil {
sendError(errors.New("runtime core not ready"), resp)
return
}
secretClient = runtime.Core.Core().V1().Secret()
}
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return

34
pkg/server/server.go
View File

@ -60,6 +60,8 @@ func StartServer(ctx context.Context, config *Config) error {
return errors.Wrap(err, "starting kubernetes")
}
config.ControlConfig.Runtime.Handler = router(&config.ControlConfig)
go startOnAPIServerReady(ctx, config)
for _, hook := range config.StartupHooks {
@ -108,9 +110,17 @@ func runControllers(ctx context.Context, config *Config) error {
return err
}
// run migration before we set controlConfig.Runtime.Core
if err := nodepassword.MigrateFile(
sc.Core.Core().V1().Secret(),
sc.Core.Core().V1().Node(),
controlConfig.Runtime.NodePasswdFile); err != nil {
logrus.Warn(errors.Wrapf(err, "error migrating node-password file"))
}
controlConfig.Runtime.Core = sc.Core
if config.ControlConfig.Runtime.ClusterControllerStart != nil {
if err := config.ControlConfig.Runtime.ClusterControllerStart(ctx); err != nil {
if controlConfig.Runtime.ClusterControllerStart != nil {
if err := controlConfig.Runtime.ClusterControllerStart(ctx); err != nil {
return errors.Wrapf(err, "starting cluster controllers")
}
}
@ -126,11 +136,6 @@ func runControllers(ctx context.Context, config *Config) error {
if err := sc.Start(ctx); err != nil {
panic(err)
}
handler, err := router(controlConfig, controlConfig.Runtime.Tunnel, sc.Core.Core().V1().Secret())
if err != nil {
panic(errors.Wrap(err, "starting router"))
}
controlConfig.Runtime.Handler = handler
}
if !config.DisableAgent {
go setControlPlaneRoleLabel(ctx, sc.Core.Core().V1().Node())
@ -152,13 +157,6 @@ func runControllers(ctx context.Context, config *Config) error {
}
func coreControllers(ctx context.Context, sc *Context, config *Config) error {
if err := nodepassword.MigrateFile(
sc.Core.Core().V1().Secret(),
sc.Core.Core().V1().Node(),
config.ControlConfig.Runtime.NodePasswdFile); err != nil {
logrus.Warn(errors.Wrapf(err, "error migrating node-password file"))
}
if err := node.Register(ctx,
!config.ControlConfig.Skips["coredns"],
sc.Core.Core().V1().Secret(),
@ -425,6 +423,11 @@ func isSymlink(config string) bool {
func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
for {
nodeName := os.Getenv("NODE_NAME")
if nodeName == "" {
logrus.Info("Waiting for control-plane node agent startup")
time.Sleep(1 * time.Second)
continue
}
node, err := nodes.Get(nodeName, metav1.GetOptions{})
if err != nil {
logrus.Infof("Waiting for control-plane node %s startup: %v", nodeName, err)
@ -454,7 +457,6 @@ func setControlPlaneRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
}
func setClusterDNSConfig(ctx context.Context, controlConfig *Config, configMap v1.ConfigMapClient) error {
nodeName := os.Getenv("NODE_NAME")
// check if configmap already exists
_, err := configMap.Get("kube-system", "cluster-dns", metav1.GetOptions{})
if err == nil {
@ -483,7 +485,7 @@ func setClusterDNSConfig(ctx context.Context, controlConfig *Config, configMap v
logrus.Infof("Cluster dns configmap has been set successfully")
break
}
logrus.Infof("Waiting for control-plane node %s startup: %v", nodeName, err)
logrus.Infof("Waiting for control-plane dns startup: %v", err)
select {
case <-ctx.Done():