Allow agents to query non-apiserver supervisors for apiserver endpoints

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit 49544e0d49)

pkg/agent/config/config.go

@@ -77,6 +77,29 @@ RETRY:
 	}
 }
 
+// APIServers returns a list of apiserver endpoints, suitable for seeding client loadbalancer configurations.
+// This function will block until it can return a populated list of apiservers, or if the remote server returns
+// an error (indicating that it does not support this functionality).
+func APIServers(ctx context.Context, node *config.Node, proxy proxy.Proxy) []string {
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+RETRY:
+	for {
+		addresses, err := getAPIServers(ctx, node, proxy)
+		if err != nil {
+			logrus.Infof("Failed to retrieve list of apiservers from server: %v", err)
+			return nil
+		}
+		if len(addresses) == 0 {
+			logrus.Infof("Waiting for apiserver addresses")
+			for range ticker.C {
+				continue RETRY
+			}
+		}
+		return addresses
+	}
+}
+
 type HTTPRequester func(u string, client *http.Client, username, password string) ([]byte, error)
 
 func Request(path string, info *clientaccess.Info, requester HTTPRequester) ([]byte, error) {
@@ -576,6 +599,22 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	return nodeConfig, nil
 }
 
+// getAPIServers attempts to return a list of apiservers from the server.
+func getAPIServers(ctx context.Context, node *config.Node, proxy proxy.Proxy) ([]string, error) {
+	info, err := clientaccess.ParseAndValidateToken(proxy.SupervisorURL(), node.Token)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := info.Get("/v1-" + version.Program + "/apiservers")
+	if err != nil {
+		return nil, err
+	}
+
+	endpoints := []string{}
+	return endpoints, json.Unmarshal(data, &endpoints)
+}
+
 // getKubeProxyDisabled attempts to return the DisableKubeProxy setting from the server configuration data.
 // It first checks the server readyz endpoint, to ensure that the configuration has stabilized before use.
 func getKubeProxyDisabled(ctx context.Context, node *config.Node, proxy proxy.Proxy) (bool, error) {

pkg/agent/run.go

@@ -208,6 +208,10 @@ func RunStandalone(ctx context.Context, cfg cmds.Agent) error {
 		close(cfg.AgentReady)
 	}
 
+	if err := tunnel.Setup(ctx, nodeConfig, proxy); err != nil {
+		return err
+	}
+
 	<-ctx.Done()
 	return ctx.Err()
 }

pkg/agent/tunnel/tunnel.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/gorilla/websocket"
+	agentconfig "github.com/rancher/k3s/pkg/agent/config"
 	"github.com/rancher/k3s/pkg/agent/proxy"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/util"
@@ -53,14 +54,18 @@ func Setup(ctx context.Context, config *config.Node, proxy proxy.Proxy) error {
 		return err
 	}
 
-	// Do an immediate fill of proxy addresses from the server endpoint list, before going into the
-	// watch loop. This will fail on the first server, as the apiserver won't be started yet - but
-	// that's fine because the local server is already seeded into the proxy address list.
-	endpoint, _ := client.CoreV1().Endpoints("default").Get(ctx, "kubernetes", metav1.GetOptions{})
-	if endpoint != nil {
-		addresses := util.GetAddresses(endpoint)
-		if len(addresses) > 0 {
-			proxy.Update(util.GetAddresses(endpoint))
+	// Try to get a list of apiservers from the server we're connecting to. If that fails, fall back to
+	// querying the endpoints list from Kubernetes. This fallback requires that the server we're joining be
+	// running an apiserver, but is the only safe thing to do if its supervisor is down-level and can't provide us
+	// with an endpoint list.
+	if addresses := agentconfig.APIServers(ctx, config, proxy); len(addresses) > 0 {
+		proxy.SetSupervisorDefault(addresses[0])
+		proxy.Update(addresses)
+	} else {
+		if endpoint, _ := client.CoreV1().Endpoints("default").Get(ctx, "kubernetes", metav1.GetOptions{}); endpoint != nil {
+			if addresses := util.GetAddresses(endpoint); len(addresses) > 0 {
+				proxy.Update(addresses)
+			}
 		}
 	}
 

pkg/server/router.go

@@ -22,9 +22,11 @@ import (
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/nodepassword"
+	"github.com/rancher/k3s/pkg/util"
 	"github.com/rancher/k3s/pkg/version"
 	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/json"
 )
 
@@ -45,6 +47,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 	authed.Path(prefix + "/client-" + version.Program + "-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert, serverConfig.Runtime.ClientK3sControllerKey))
 	authed.Path(prefix + "/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
 	authed.Path(prefix + "/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
+	authed.Path(prefix + "/apiservers").Handler(apiserversHandler(serverConfig))
 	authed.Path(prefix + "/config").Handler(configHandler(serverConfig, cfg))
 	authed.Path(prefix + "/readyz").Handler(readyzHandler(serverConfig))
 
@@ -288,6 +291,29 @@ func fileHandler(fileName ...string) http.Handler {
 	})
 }
 
+func apiserversHandler(server *config.Control) http.Handler {
+	var endpointsClient coreclient.EndpointsClient
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		var endpoints []string
+		if endpointsClient == nil {
+			if server.Runtime.Core != nil {
+				endpointsClient = server.Runtime.Core.Core().V1().Endpoints()
+			}
+		}
+		if endpointsClient != nil {
+			if endpoint, _ := endpointsClient.Get("default", "kubernetes", metav1.GetOptions{}); endpoint != nil {
+				endpoints = util.GetAddresses(endpoint)
+			}
+		}
+
+		resp.Header().Set("content-type", "application/json")
+		if err := json.NewEncoder(resp).Encode(endpoints); err != nil {
+			logrus.Errorf("Failed to encode apiserver endpoints: %v", err)
+			resp.WriteHeader(http.StatusInternalServerError)
+		}
+	})
+}
+
 func configHandler(server *config.Control, cfg *cmds.Server) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		if req.TLS == nil {