Use distinct clients for supervisor, deploy, and helm controllers

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
(cherry picked from commit 8748813a61)

pkg/server/context.go

@@ -2,26 +2,20 @@ package server
 
 import (
 	"context"
-	"fmt"
-	"os"
-	"runtime"
 
 	helmcrd "github.com/k3s-io/helm-controller/pkg/crd"
 	"github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io"
 	addoncrd "github.com/k3s-io/k3s/pkg/crd"
-	"github.com/k3s-io/k3s/pkg/deploy"
 	"github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/apply"
 	"github.com/rancher/wrangler/pkg/crd"
 	"github.com/rancher/wrangler/pkg/generated/controllers/apps"
 	"github.com/rancher/wrangler/pkg/generated/controllers/batch"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
 	"github.com/rancher/wrangler/pkg/generated/controllers/rbac"
 	"github.com/rancher/wrangler/pkg/start"
-	"github.com/sirupsen/logrus"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -35,7 +29,6 @@ type Context struct {
 	Auth  *rbac.Factory
 	Core  *core.Factory
 	K8s   kubernetes.Interface
-	Apply apply.Apply
 }
 
 func (c *Context) Start(ctx context.Context) error {
@@ -47,16 +40,7 @@ func NewContext(ctx context.Context, cfg string) (*Context, error) {
 	if err != nil {
 		return nil, err
 	}
-
+	restConfig.UserAgent = util.GetUserAgent(version.Program + "-supervisor")
-	// Construct a custom user-agent string for the apply client used by the deploy controller
-	// so that we can track which node's deploy controller most recently modified a resource.
-	nodeName := os.Getenv("NODE_NAME")
-	managerName := deploy.ControllerName + "@" + nodeName
-	if nodeName == "" || len(managerName) > validation.FieldManagerMaxLength {
-		logrus.Warn("Deploy controller node name is empty or too long, and will not be tracked via server side apply field management")
-		managerName = deploy.ControllerName
-	}
-	restConfig.UserAgent = fmt.Sprintf("%s/%s (%s/%s) %s/%s", managerName, version.Version, runtime.GOOS, runtime.GOARCH, version.Program, version.GitCommit)
 
 	if err := crds(ctx, restConfig); err != nil {
 		return nil, errors.Wrap(err, "failed to register CRDs")
@@ -74,7 +58,6 @@ func NewContext(ctx context.Context, cfg string) (*Context, error) {
 		Apps:  apps.NewFactoryFromConfigOrDie(restConfig),
 		Batch: batch.NewFactoryFromConfigOrDie(restConfig),
 		Core:  core.NewFactoryFromConfigOrDie(restConfig),
-		Apply: apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup(),
 	}, nil
 }
 

pkg/server/server.go

@@ -12,7 +12,7 @@ import (
 	"sync"
 	"time"
 
-	helm "github.com/k3s-io/helm-controller/pkg/controllers/chart"
+	helmchart "github.com/k3s-io/helm-controller/pkg/controllers/chart"
 	helmcommon "github.com/k3s-io/helm-controller/pkg/controllers/common"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
@@ -28,12 +28,15 @@ import (
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
+	"github.com/rancher/wrangler/pkg/apply"
 	v1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/leader"
 	"github.com/rancher/wrangler/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 const (
@@ -205,26 +208,42 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
 
 	// apply SystemDefaultRegistry setting to Helm before starting controllers
 	if config.ControlConfig.SystemDefaultRegistry != "" {
-		helm.DefaultJobImage = config.ControlConfig.SystemDefaultRegistry + "/" + helm.DefaultJobImage
+		helmchart.DefaultJobImage = config.ControlConfig.SystemDefaultRegistry + "/" + helmchart.DefaultJobImage
 	}
 
 	if !config.ControlConfig.DisableHelmController {
-		helm.Register(ctx,
+		restConfig, err := clientcmd.BuildConfigFromFlags("", config.ControlConfig.Runtime.KubeConfigAdmin)
+		if err != nil {
+			return err
+		}
+		restConfig.UserAgent = util.GetUserAgent(helmcommon.Name)
+
+		k8s, err := clientset.NewForConfig(restConfig)
+		if err != nil {
+			return err
+		}
+
+		apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup()
+		helm := sc.Helm.WithAgent(restConfig.UserAgent)
+		batch := sc.Batch.WithAgent(restConfig.UserAgent)
+		auth := sc.Auth.WithAgent(restConfig.UserAgent)
+		core := sc.Core.WithAgent(restConfig.UserAgent)
+		helmchart.Register(ctx,
 			metav1.NamespaceAll,
 			helmcommon.Name,
-			sc.K8s,
+			k8s,
-			sc.Apply,
+			apply,
-			util.BuildControllerEventRecorder(sc.K8s, helmcommon.Name, metav1.NamespaceAll),
+			util.BuildControllerEventRecorder(k8s, helmcommon.Name, metav1.NamespaceAll),
-			sc.Helm.Helm().V1().HelmChart(),
+			helm.V1().HelmChart(),
-			sc.Helm.Helm().V1().HelmChart().Cache(),
+			helm.V1().HelmChart().Cache(),
-			sc.Helm.Helm().V1().HelmChartConfig(),
+			helm.V1().HelmChartConfig(),
-			sc.Helm.Helm().V1().HelmChartConfig().Cache(),
+			helm.V1().HelmChartConfig().Cache(),
-			sc.Batch.Batch().V1().Job(),
+			batch.V1().Job(),
-			sc.Batch.Batch().V1().Job().Cache(),
+			batch.V1().Job().Cache(),
-			sc.Auth.Rbac().V1().ClusterRoleBinding(),
+			auth.V1().ClusterRoleBinding(),
-			sc.Core.Core().V1().ServiceAccount(),
+			core.V1().ServiceAccount(),
-			sc.Core.Core().V1().ConfigMap(),
+			core.V1().ConfigMap(),
-			sc.Core.Core().V1().Secret())
+			core.V1().Secret())
 	}
 
 	if config.ControlConfig.EncryptSecrets {
@@ -274,10 +293,24 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
 		return err
 	}
 
+	restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigAdmin)
+	if err != nil {
+		return err
+	}
+	restConfig.UserAgent = util.GetUserAgent("deploy")
+
+	k8s, err := clientset.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
+	apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup()
+	k3s := sc.K3s.WithAgent(restConfig.UserAgent)
+
 	return deploy.WatchFiles(ctx,
-		sc.K8s,
+		k8s,
-		sc.Apply,
+		apply,
-		sc.K3s.K3s().V1().Addon(),
+		k3s.V1().Addon(),
 		controlConfig.Disables,
 		dataDir)
 }

pkg/util/client.go

@@ -1,9 +1,15 @@
 package util
 
 import (
+	"fmt"
+	"os"
+	"runtime"
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/datadir"
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 )
@@ -30,6 +36,17 @@ func GetClientSet(file string) (clientset.Interface, error) {
 	return clientset.NewForConfig(restConfig)
 }
 
+// GetUserAgent builds a complete UserAgent string for a given controller, including the node name if possible.
+func GetUserAgent(controllerName string) string {
+	nodeName := os.Getenv("NODE_NAME")
+	managerName := controllerName + "@" + nodeName
+	if nodeName == "" || len(managerName) > validation.FieldManagerMaxLength {
+		logrus.Warnf("%s controller node name is empty or too long, and will not be tracked via server side apply field management", controllerName)
+		managerName = controllerName
+	}
+	return fmt.Sprintf("%s/%s (%s/%s) %s/%s", managerName, version.Version, runtime.GOOS, runtime.GOARCH, version.Program, version.GitCommit)
+}
+
 // SplitStringSlice is a helper function to handle StringSliceFlag containing multiple values
 // By default, StringSliceFlag only supports repeated values, not multiple values
 // e.g. --foo="bar,car" --foo=baz will result in []string{"bar", "car". "baz"}