Ensure that apiserver ready channel checks re-dial every time

Closing idle connections isn't guaranteed to close out a pooled connection to a
loadbalancer endpoint that has been removed. Instead, ensure that requests used
to wait for the apiserver to become ready aren't reused.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

pkg/daemons/control/server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"math/rand"
 	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -33,6 +34,12 @@ import (
 
 var localhostIP = net.ParseIP("127.0.0.1")
 
+type roundTripFunc func(req *http.Request) (*http.Response, error)
+
+func (w roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return w(req)
+}
+
 func Server(ctx context.Context, cfg *config.Control) error {
 	rand.Seed(time.Now().UTC().UnixNano())
 
@@ -396,6 +403,16 @@ func waitForAPIServerInBackground(ctx context.Context, runtime *config.ControlRu
 		return err
 	}
 
+	// By default, idle connections to the apiserver are returned to a global pool
+	// between requests.  Explicitly flag this client's request for closure so that
+	// we re-dial through the loadbalancer in case the endpoints have changed.
+	restConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
+		return roundTripFunc(func(req *http.Request) (*http.Response, error) {
+			req.Close = true
+			return rt.RoundTrip(req)
+		})
+	})
+
 	k8sClient, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
 		return err

pkg/util/api.go

@@ -18,7 +18,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	coregetter "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 )
 
@@ -57,10 +56,6 @@ func WaitForAPIServerReady(ctx context.Context, client clientset.Interface, time
 
 	err := wait.PollImmediateWithContext(ctx, time.Second, timeout, func(ctx context.Context) (bool, error) {
 		healthStatus := 0
-		// Idle connections to the apiserver are returned to a global pool between requests. Explicitly
-		// close these idle connections so that we re-connect through the loadbalancer in case the endpoints
-		// have changed.
-		restClient.(*rest.RESTClient).Client.CloseIdleConnections()
 		result := restClient.Get().AbsPath("/readyz").Do(ctx).StatusCode(&healthStatus)
 		if rerr := result.Error(); rerr != nil {
 			lastErr = errors.Wrap(rerr, "failed to get apiserver /readyz status")