From fba0f092d4a67280b709db6d7146054bfcba02fb Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Thu, 21 Nov 2024 23:48:49 +0000 Subject: [PATCH] Use helper to set consistent rest.Config rate limits and timeouts Signed-off-by: Brad Davidson (cherry picked from commit 71918e0d69021e19f054e1001d781c57f5047983) Signed-off-by: Brad Davidson --- pkg/agent/netpol/netpol.go | 4 ++-- pkg/agent/tunnel/tunnel.go | 3 +-- pkg/cli/token/token.go | 3 +-- pkg/daemons/config/types.go | 2 ++ pkg/daemons/executor/embed.go | 3 +-- pkg/secretsencrypt/config.go | 3 +-- pkg/server/context.go | 3 +-- pkg/server/router.go | 8 +++++--- pkg/server/secrets-encrypt.go | 15 +-------------- pkg/server/server.go | 6 +++--- pkg/util/api.go | 5 ++--- pkg/util/client.go | 17 ++++++++++++++++- 12 files changed, 36 insertions(+), 36 deletions(-) diff --git a/pkg/agent/netpol/netpol.go b/pkg/agent/netpol/netpol.go index 5c892a668f..a9f7a43f53 100644 --- a/pkg/agent/netpol/netpol.go +++ b/pkg/agent/netpol/netpol.go @@ -26,12 +26,12 @@ import ( "github.com/coreos/go-iptables/iptables" "github.com/k3s-io/k3s/pkg/daemons/config" "github.com/k3s-io/k3s/pkg/metrics" + "github.com/k3s-io/k3s/pkg/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" v1core "k8s.io/api/core/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" ) func init() { @@ -57,7 +57,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error { return nil } - restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController) + restConfig, err := util.GetRESTConfig(nodeConfig.AgentConfig.KubeConfigK3sController) if err != nil { return err } diff --git a/pkg/agent/tunnel/tunnel.go b/pkg/agent/tunnel/tunnel.go index d04f9fdc0b..e1eb566cc6 100644 --- a/pkg/agent/tunnel/tunnel.go +++ b/pkg/agent/tunnel/tunnel.go @@ -32,7 +32,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" toolswatch "k8s.io/client-go/tools/watch" "k8s.io/kubernetes/pkg/cluster/ports" ) @@ -70,7 +69,7 @@ func Setup(ctx context.Context, config *daemonconfig.Node, proxy proxy.Proxy) er return err } - nodeRestConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigKubelet) + nodeRestConfig, err := util.GetRESTConfig(config.AgentConfig.KubeConfigKubelet) if err != nil { return err } diff --git a/pkg/cli/token/token.go b/pkg/cli/token/token.go index e16038fea5..9d514d7b52 100644 --- a/pkg/cli/token/token.go +++ b/pkg/cli/token/token.go @@ -24,7 +24,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/duration" - "k8s.io/client-go/tools/clientcmd" bootstrapapi "k8s.io/cluster-bootstrap/token/api" bootstraputil "k8s.io/cluster-bootstrap/token/util" "k8s.io/utils/ptr" @@ -48,7 +47,7 @@ func create(app *cli.Context, cfg *cmds.Token) error { return err } - restConfig, err := clientcmd.BuildConfigFromFlags("", cfg.Kubeconfig) + restConfig, err := util.GetRESTConfig(cfg.Kubeconfig) if err != nil { return err } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 026dae0225..ee5c5f8131 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -17,6 +17,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" utilsnet "k8s.io/utils/net" ) @@ -369,6 +370,7 @@ type ControlRuntime struct { ClientETCDCert string ClientETCDKey string + K8s kubernetes.Interface K3s *k3s.Factory Core *core.Factory Event record.EventRecorder diff --git a/pkg/daemons/executor/embed.go b/pkg/daemons/executor/embed.go index 0553da84e3..7e69f956e8 100644 --- a/pkg/daemons/executor/embed.go +++ b/pkg/daemons/executor/embed.go @@ -28,7 +28,6 @@ import ( "k8s.io/apiserver/pkg/authentication/authenticator" typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" toolswatch "k8s.io/client-go/tools/watch" cloudprovider "k8s.io/cloud-provider" cloudproviderapi "k8s.io/cloud-provider/api" @@ -269,7 +268,7 @@ func (e *Embedded) Docker(ctx context.Context, cfg *daemonconfig.Node) error { // waitForUntaintedNode watches nodes, waiting to find one not tainted as // uninitialized by the external cloud provider. func waitForUntaintedNode(ctx context.Context, kubeConfig string) error { - restConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfig) + restConfig, err := util.GetRESTConfig(kubeConfig) if err != nil { return err } diff --git a/pkg/secretsencrypt/config.go b/pkg/secretsencrypt/config.go index 5632676e79..4c2a69c82a 100644 --- a/pkg/secretsencrypt/config.go +++ b/pkg/secretsencrypt/config.go @@ -15,7 +15,6 @@ import ( "github.com/k3s-io/k3s/pkg/version" "github.com/prometheus/common/expfmt" corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/tools/clientcmd" "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme" "github.com/sirupsen/logrus" @@ -237,7 +236,7 @@ func GetEncryptionConfigMetrics(runtime *config.ControlRuntime, initialMetrics b var unixUpdateTime int64 var reloadSuccessCounter int64 var lastFailure string - restConfig, err := clientcmd.BuildConfigFromFlags("", runtime.KubeConfigSupervisor) + restConfig, err := util.GetRESTConfig(runtime.KubeConfigSupervisor) if err != nil { return 0, 0, err } diff --git a/pkg/server/context.go b/pkg/server/context.go index 93bee0c6ea..ba32a0c55d 100644 --- a/pkg/server/context.go +++ b/pkg/server/context.go @@ -19,7 +19,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" ) @@ -43,7 +42,7 @@ func NewContext(ctx context.Context, config *Config, forServer bool) (*Context, if forServer { cfg = config.ControlConfig.Runtime.KubeConfigSupervisor } - restConfig, err := clientcmd.BuildConfigFromFlags("", cfg) + restConfig, err := util.GetRESTConfig(cfg) if err != nil { return nil, err } diff --git a/pkg/server/router.go b/pkg/server/router.go index 1edbe6e785..93666f38c3 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" bootstrapapi "k8s.io/cluster-bootstrap/token/api" "k8s.io/kubernetes/pkg/auth/nodeidentifier" ) @@ -305,16 +306,17 @@ func fileHandler(fileName ...string) http.Handler { } func apiserversHandler(server *config.Control) http.Handler { - var endpointsClient coreclient.EndpointsClient + var endpointsClient typedcorev1.EndpointsInterface return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + ctx := req.Context() var endpoints []string if endpointsClient == nil { if server.Runtime.Core != nil { - endpointsClient = server.Runtime.Core.Core().V1().Endpoints() + endpointsClient = server.Runtime.K8s.CoreV1().Endpoints(metav1.NamespaceDefault) } } if endpointsClient != nil { - if endpoint, _ := endpointsClient.Get("default", "kubernetes", metav1.GetOptions{}); endpoint != nil { + if endpoint, _ := endpointsClient.Get(ctx, "kubernetes", metav1.GetOptions{}); endpoint != nil { endpoints = util.GetAddresses(endpoint) } } diff --git a/pkg/server/secrets-encrypt.go b/pkg/server/secrets-encrypt.go index 7e778bc5c2..9f59a109b5 100644 --- a/pkg/server/secrets-encrypt.go +++ b/pkg/server/secrets-encrypt.go @@ -27,8 +27,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/pager" "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" @@ -395,18 +393,7 @@ func reencryptAndRemoveKey(ctx context.Context, server *config.Control, skip boo } func updateSecrets(ctx context.Context, server *config.Control, nodeName string) error { - restConfig, err := clientcmd.BuildConfigFromFlags("", server.Runtime.KubeConfigSupervisor) - if err != nil { - return err - } - // For secrets we need a much higher QPS than default - restConfig.QPS = secretsencrypt.SecretQPS - restConfig.Burst = secretsencrypt.SecretBurst - k8s, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return err - } - + k8s := server.Runtime.K8s nodeRef := &corev1.ObjectReference{ Kind: "Node", Name: nodeName, diff --git a/pkg/server/server.go b/pkg/server/server.go index 81818e6384..d40b27b03b 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -36,7 +36,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" ) func ResolveDataDir(dataDir string) (string, error) { @@ -114,6 +113,7 @@ func runControllers(ctx context.Context, config *Config) error { controlConfig.Runtime.NodePasswdFile); err != nil { logrus.Warn(errors.Wrap(err, "error migrating node-password file")) } + controlConfig.Runtime.K8s = sc.K8s controlConfig.Runtime.K3s = sc.K3s controlConfig.Runtime.Event = sc.Event controlConfig.Runtime.Core = sc.Core @@ -209,7 +209,7 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error { } if !config.ControlConfig.DisableHelmController { - restConfig, err := clientcmd.BuildConfigFromFlags("", config.ControlConfig.Runtime.KubeConfigSupervisor) + restConfig, err := util.GetRESTConfig(config.ControlConfig.Runtime.KubeConfigSupervisor) if err != nil { return err } @@ -290,7 +290,7 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control) return err } - restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigSupervisor) + restConfig, err := util.GetRESTConfig(controlConfig.Runtime.KubeConfigSupervisor) if err != nil { return err } diff --git a/pkg/util/api.go b/pkg/util/api.go index 9000ee998f..42761566c3 100644 --- a/pkg/util/api.go +++ b/pkg/util/api.go @@ -23,7 +23,6 @@ import ( authorizationv1client "k8s.io/client-go/kubernetes/typed/authorization/v1" coregetter "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" ) @@ -58,7 +57,7 @@ func GetAddresses(endpoint *v1.Endpoints) []string { // readyz endpoint instead of the deprecated healthz endpoint, and supports context. func WaitForAPIServerReady(ctx context.Context, kubeconfigPath string, timeout time.Duration) error { var lastErr error - restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + restConfig, err := GetRESTConfig(kubeconfigPath) if err != nil { return err } @@ -112,7 +111,7 @@ type genericAccessReviewRequest func(context.Context) (*authorizationv1.SubjectA // the access would be allowed. func WaitForRBACReady(ctx context.Context, kubeconfigPath string, timeout time.Duration, ra authorizationv1.ResourceAttributes, user string, groups ...string) error { var lastErr error - restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + restConfig, err := GetRESTConfig(kubeconfigPath) if err != nil { return err } diff --git a/pkg/util/client.go b/pkg/util/client.go index 561a5cbc08..c4a6e48499 100644 --- a/pkg/util/client.go +++ b/pkg/util/client.go @@ -5,12 +5,15 @@ import ( "os" "runtime" "strings" + "time" "github.com/k3s-io/k3s/pkg/datadir" "github.com/k3s-io/k3s/pkg/version" + "github.com/rancher/wrangler/pkg/ratelimit" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/validation" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -28,7 +31,7 @@ func GetKubeConfigPath(file string) string { // GetClientSet creates a Kubernetes client from the kubeconfig at the provided path. func GetClientSet(file string) (clientset.Interface, error) { - restConfig, err := clientcmd.BuildConfigFromFlags("", file) + restConfig, err := GetRESTConfig(file) if err != nil { return nil, err } @@ -36,6 +39,18 @@ func GetClientSet(file string) (clientset.Interface, error) { return clientset.NewForConfig(restConfig) } +// GetRESTConfig returns a REST config with default timeouts and ratelimitsi cribbed from wrangler defaults. +// ref: https://github.com/rancher/wrangler/blob/v3.0.0/pkg/clients/clients.go#L184-L190 +func GetRESTConfig(file string) (*rest.Config, error) { + restConfig, err := clientcmd.BuildConfigFromFlags("", file) + if err != nil { + return nil, err + } + restConfig.Timeout = 15 * time.Minute + restConfig.RateLimiter = ratelimit.None + return restConfig, nil +} + // GetUserAgent builds a complete UserAgent string for a given controller, including the node name if possible. func GetUserAgent(controllerName string) string { nodeName := os.Getenv("NODE_NAME")