Use helper to set consistent rest.Config rate limits and timeouts

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit 71918e0d69)
Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

pkg/agent/netpol/netpol.go

@@ -26,12 +26,12 @@ import (
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/metrics"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	v1core "k8s.io/api/core/v1"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 func init() {
@@ -57,7 +57,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 		return nil
 	}
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController)
+	restConfig, err := util.GetRESTConfig(nodeConfig.AgentConfig.KubeConfigK3sController)
 	if err != nil {
 		return err
 	}

pkg/agent/tunnel/tunnel.go

@@ -32,7 +32,6 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/tools/clientcmd"
 	toolswatch "k8s.io/client-go/tools/watch"
 	"k8s.io/kubernetes/pkg/cluster/ports"
 )
@@ -70,7 +69,7 @@ func Setup(ctx context.Context, config *daemonconfig.Node, proxy proxy.Proxy) er
 		return err
 	}
 
-	nodeRestConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigKubelet)
+	nodeRestConfig, err := util.GetRESTConfig(config.AgentConfig.KubeConfigKubelet)
 	if err != nil {
 		return err
 	}

pkg/cli/token/token.go

@@ -24,7 +24,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/util/duration"
-	"k8s.io/client-go/tools/clientcmd"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	bootstraputil "k8s.io/cluster-bootstrap/token/util"
 	"k8s.io/utils/ptr"
@@ -48,7 +47,7 @@ func create(app *cli.Context, cfg *cmds.Token) error {
 		return err
 	}
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", cfg.Kubeconfig)
+	restConfig, err := util.GetRESTConfig(cfg.Kubeconfig)
 	if err != nil {
 		return err
 	}

pkg/daemons/config/types.go

@@ -17,6 +17,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
 	utilsnet "k8s.io/utils/net"
 )
@@ -369,6 +370,7 @@ type ControlRuntime struct {
 	ClientETCDCert           string
 	ClientETCDKey            string
 
+	K8s        kubernetes.Interface
 	K3s        *k3s.Factory
 	Core       *core.Factory
 	Event      record.EventRecorder

pkg/daemons/executor/embed.go

@@ -28,7 +28,6 @@ import (
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/tools/clientcmd"
 	toolswatch "k8s.io/client-go/tools/watch"
 	cloudprovider "k8s.io/cloud-provider"
 	cloudproviderapi "k8s.io/cloud-provider/api"
@@ -269,7 +268,7 @@ func (e *Embedded) Docker(ctx context.Context, cfg *daemonconfig.Node) error {
 // waitForUntaintedNode watches nodes, waiting to find one not tainted as
 // uninitialized by the external cloud provider.
 func waitForUntaintedNode(ctx context.Context, kubeConfig string) error {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfig)
+	restConfig, err := util.GetRESTConfig(kubeConfig)
 	if err != nil {
 		return err
 	}

pkg/secretsencrypt/config.go

@@ -15,7 +15,6 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/prometheus/common/expfmt"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/client-go/tools/clientcmd"
 
 	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
 	"github.com/sirupsen/logrus"
@@ -237,7 +236,7 @@ func GetEncryptionConfigMetrics(runtime *config.ControlRuntime, initialMetrics b
 	var unixUpdateTime int64
 	var reloadSuccessCounter int64
 	var lastFailure string
-	restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigSupervisor)
+	restConfig, err := util.GetRESTConfig(runtime.KubeConfigSupervisor)
 	if err != nil {
 		return 0, 0, err
 	}

pkg/server/context.go

@@ -19,7 +19,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 )
 
@@ -43,7 +42,7 @@ func NewContext(ctx context.Context, config *Config, forServer bool) (*Context,
 	if forServer {
 		cfg = config.ControlConfig.Runtime.KubeConfigSupervisor
 	}
-	restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
+	restConfig, err := util.GetRESTConfig(cfg)
 	if err != nil {
 		return nil, err
 	}

pkg/server/router.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/endpoints/request"
+	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	"k8s.io/kubernetes/pkg/auth/nodeidentifier"
 )
@@ -305,16 +306,17 @@ func fileHandler(fileName ...string) http.Handler {
 }
 
 func apiserversHandler(server *config.Control) http.Handler {
-	var endpointsClient coreclient.EndpointsClient
+	var endpointsClient typedcorev1.EndpointsInterface
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		ctx := req.Context()
 		var endpoints []string
 		if endpointsClient == nil {
 			if server.Runtime.Core != nil {
-				endpointsClient = server.Runtime.Core.Core().V1().Endpoints()
+				endpointsClient = server.Runtime.K8s.CoreV1().Endpoints(metav1.NamespaceDefault)
 			}
 		}
 		if endpointsClient != nil {
-			if endpoint, _ := endpointsClient.Get("default", "kubernetes", metav1.GetOptions{}); endpoint != nil {
+			if endpoint, _ := endpointsClient.Get(ctx, "kubernetes", metav1.GetOptions{}); endpoint != nil {
 				endpoints = util.GetAddresses(endpoint)
 			}
 		}

pkg/server/secrets-encrypt.go

@@ -27,8 +27,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/pager"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/utils/ptr"
@@ -395,18 +393,7 @@ func reencryptAndRemoveKey(ctx context.Context, server *config.Control, skip boo
 }
 
 func updateSecrets(ctx context.Context, server *config.Control, nodeName string) error {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", server.Runtime.KubeConfigSupervisor)
+	k8s := server.Runtime.K8s
-	if err != nil {
-		return err
-	}
-	// For secrets we need a much higher QPS than default
-	restConfig.QPS = secretsencrypt.SecretQPS
-	restConfig.Burst = secretsencrypt.SecretBurst
-	k8s, err := kubernetes.NewForConfig(restConfig)
-	if err != nil {
-		return err
-	}
-
 	nodeRef := &corev1.ObjectReference{
 		Kind:      "Node",
 		Name:      nodeName,

pkg/server/server.go

@@ -36,7 +36,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 func ResolveDataDir(dataDir string) (string, error) {
@@ -114,6 +113,7 @@ func runControllers(ctx context.Context, config *Config) error {
 		controlConfig.Runtime.NodePasswdFile); err != nil {
 		logrus.Warn(errors.Wrap(err, "error migrating node-password file"))
 	}
+	controlConfig.Runtime.K8s = sc.K8s
 	controlConfig.Runtime.K3s = sc.K3s
 	controlConfig.Runtime.Event = sc.Event
 	controlConfig.Runtime.Core = sc.Core
@@ -209,7 +209,7 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
 	}
 
 	if !config.ControlConfig.DisableHelmController {
-		restConfig, err := clientcmd.BuildConfigFromFlags("", config.ControlConfig.Runtime.KubeConfigSupervisor)
+		restConfig, err := util.GetRESTConfig(config.ControlConfig.Runtime.KubeConfigSupervisor)
 		if err != nil {
 			return err
 		}
@@ -290,7 +290,7 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
 		return err
 	}
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigSupervisor)
+	restConfig, err := util.GetRESTConfig(controlConfig.Runtime.KubeConfigSupervisor)
 	if err != nil {
 		return err
 	}

pkg/util/api.go

@@ -23,7 +23,6 @@ import (
 	authorizationv1client "k8s.io/client-go/kubernetes/typed/authorization/v1"
 	coregetter "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 )
 
@@ -58,7 +57,7 @@ func GetAddresses(endpoint *v1.Endpoints) []string {
 // readyz endpoint instead of the deprecated healthz endpoint, and supports context.
 func WaitForAPIServerReady(ctx context.Context, kubeconfigPath string, timeout time.Duration) error {
 	var lastErr error
-	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	restConfig, err := GetRESTConfig(kubeconfigPath)
 	if err != nil {
 		return err
 	}
@@ -112,7 +111,7 @@ type genericAccessReviewRequest func(context.Context) (*authorizationv1.SubjectA
 // the access would be allowed.
 func WaitForRBACReady(ctx context.Context, kubeconfigPath string, timeout time.Duration, ra authorizationv1.ResourceAttributes, user string, groups ...string) error {
 	var lastErr error
-	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	restConfig, err := GetRESTConfig(kubeconfigPath)
 	if err != nil {
 		return err
 	}

pkg/util/client.go

@@ -5,12 +5,15 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/k3s-io/k3s/pkg/datadir"
 	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/rancher/wrangler/pkg/ratelimit"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
@@ -28,7 +31,7 @@ func GetKubeConfigPath(file string) string {
 
 // GetClientSet creates a Kubernetes client from the kubeconfig at the provided path.
 func GetClientSet(file string) (clientset.Interface, error) {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", file)
+	restConfig, err := GetRESTConfig(file)
 	if err != nil {
 		return nil, err
 	}
@@ -36,6 +39,18 @@ func GetClientSet(file string) (clientset.Interface, error) {
 	return clientset.NewForConfig(restConfig)
 }
 
+// GetRESTConfig returns a REST config with default timeouts and ratelimitsi cribbed from wrangler defaults.
+// ref: https://github.com/rancher/wrangler/blob/v3.0.0/pkg/clients/clients.go#L184-L190
+func GetRESTConfig(file string) (*rest.Config, error) {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", file)
+	if err != nil {
+		return nil, err
+	}
+	restConfig.Timeout = 15 * time.Minute
+	restConfig.RateLimiter = ratelimit.None
+	return restConfig, nil
+}
+
 // GetUserAgent builds a complete UserAgent string for a given controller, including the node name if possible.
 func GetUserAgent(controllerName string) string {
 	nodeName := os.Getenv("NODE_NAME")