2019-01-09 16:54:15 +00:00
|
|
|
package agent
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2020-11-24 07:59:16 +00:00
|
|
|
"fmt"
|
2022-02-16 22:19:58 +00:00
|
|
|
"net"
|
2019-01-09 16:54:15 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2022-02-16 22:19:58 +00:00
|
|
|
"strconv"
|
2019-02-08 04:13:43 +00:00
|
|
|
"strings"
|
2019-01-09 16:54:15 +00:00
|
|
|
"time"
|
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
systemd "github.com/coreos/go-systemd/daemon"
|
2022-03-02 23:47:27 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/agent/config"
|
|
|
|
"github.com/k3s-io/k3s/pkg/agent/containerd"
|
|
|
|
"github.com/k3s-io/k3s/pkg/agent/flannel"
|
|
|
|
"github.com/k3s-io/k3s/pkg/agent/netpol"
|
|
|
|
"github.com/k3s-io/k3s/pkg/agent/proxy"
|
|
|
|
"github.com/k3s-io/k3s/pkg/agent/syssetup"
|
|
|
|
"github.com/k3s-io/k3s/pkg/agent/tunnel"
|
|
|
|
"github.com/k3s-io/k3s/pkg/cgroups"
|
|
|
|
"github.com/k3s-io/k3s/pkg/cli/cmds"
|
|
|
|
"github.com/k3s-io/k3s/pkg/clientaccess"
|
|
|
|
cp "github.com/k3s-io/k3s/pkg/cloudprovider"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/agent"
|
|
|
|
daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/executor"
|
|
|
|
"github.com/k3s-io/k3s/pkg/nodeconfig"
|
|
|
|
"github.com/k3s-io/k3s/pkg/rootless"
|
|
|
|
"github.com/k3s-io/k3s/pkg/util"
|
2021-04-21 22:56:20 +00:00
|
|
|
"github.com/pkg/errors"
|
2019-01-22 21:14:58 +00:00
|
|
|
"github.com/sirupsen/logrus"
|
2021-10-29 08:59:03 +00:00
|
|
|
corev1 "k8s.io/api/core/v1"
|
2019-10-27 05:53:25 +00:00
|
|
|
"k8s.io/apimachinery/pkg/api/equality"
|
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
2021-10-29 08:59:03 +00:00
|
|
|
"k8s.io/apimachinery/pkg/fields"
|
2020-04-28 22:00:30 +00:00
|
|
|
"k8s.io/apimachinery/pkg/labels"
|
2019-10-27 05:53:25 +00:00
|
|
|
"k8s.io/client-go/kubernetes"
|
2021-10-29 08:59:03 +00:00
|
|
|
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
|
2019-10-15 21:17:26 +00:00
|
|
|
"k8s.io/client-go/tools/clientcmd"
|
2021-05-17 20:30:55 +00:00
|
|
|
app2 "k8s.io/kubernetes/cmd/kube-proxy/app"
|
|
|
|
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
|
2021-04-21 22:56:20 +00:00
|
|
|
utilsnet "k8s.io/utils/net"
|
2021-05-17 20:30:55 +00:00
|
|
|
utilpointer "k8s.io/utils/pointer"
|
2019-01-09 16:54:15 +00:00
|
|
|
)
|
|
|
|
|
2020-06-24 21:16:44 +00:00
|
|
|
func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
|
|
|
|
nodeConfig := config.Get(ctx, cfg, proxy)
|
|
|
|
|
2021-04-21 22:56:20 +00:00
|
|
|
dualCluster, err := utilsnet.IsDualStackCIDRs(nodeConfig.AgentConfig.ClusterCIDRs)
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "failed to validate cluster-cidr")
|
|
|
|
}
|
|
|
|
dualService, err := utilsnet.IsDualStackCIDRs(nodeConfig.AgentConfig.ServiceCIDRs)
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "failed to validate service-cidr")
|
|
|
|
}
|
|
|
|
dualNode, err := utilsnet.IsDualStackIPs(nodeConfig.AgentConfig.NodeIPs)
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "failed to validate node-ip")
|
|
|
|
}
|
2021-11-10 19:23:05 +00:00
|
|
|
serviceIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ServiceCIDR)
|
|
|
|
clusterIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ClusterCIDR)
|
2021-04-21 22:56:20 +00:00
|
|
|
|
2021-11-10 19:23:05 +00:00
|
|
|
enableIPv6 := dualCluster || dualService || dualNode || serviceIPv6 || clusterIPv6
|
2021-05-17 20:30:55 +00:00
|
|
|
conntrackConfig, err := getConntrackConfig(nodeConfig)
|
|
|
|
if err != nil {
|
|
|
|
return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")
|
|
|
|
}
|
|
|
|
syssetup.Configure(enableIPv6, conntrackConfig)
|
2021-11-09 15:44:34 +00:00
|
|
|
nodeConfig.AgentConfig.EnableIPv6 = enableIPv6
|
2021-04-21 22:56:20 +00:00
|
|
|
|
2020-06-24 21:16:44 +00:00
|
|
|
if err := setupCriCtlConfig(cfg, nodeConfig); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2021-05-11 19:50:08 +00:00
|
|
|
if err := executor.Bootstrap(ctx, nodeConfig, cfg); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-01-09 16:54:15 +00:00
|
|
|
if !nodeConfig.NoFlannel {
|
|
|
|
if err := flannel.Prepare(ctx, nodeConfig); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-10 23:16:26 +00:00
|
|
|
if !nodeConfig.Docker && nodeConfig.ContainerRuntimeEndpoint == "" {
|
2019-01-09 16:54:15 +00:00
|
|
|
if err := containerd.Run(ctx, nodeConfig); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2021-06-15 11:20:26 +00:00
|
|
|
|
2021-10-12 06:13:10 +00:00
|
|
|
// the agent runtime is ready to host workloads when containerd is up and the airgap
|
|
|
|
// images have finished loading, as that portion of startup may block for an arbitrary
|
|
|
|
// amount of time depending on how long it takes to import whatever the user has placed
|
|
|
|
// in the images directory.
|
|
|
|
if cfg.AgentReady != nil {
|
|
|
|
close(cfg.AgentReady)
|
|
|
|
}
|
|
|
|
|
2021-06-15 11:20:26 +00:00
|
|
|
notifySocket := os.Getenv("NOTIFY_SOCKET")
|
|
|
|
os.Unsetenv("NOTIFY_SOCKET")
|
|
|
|
|
2021-02-12 15:35:57 +00:00
|
|
|
if err := setupTunnelAndRunAgent(ctx, nodeConfig, cfg, proxy); err != nil {
|
2019-01-09 16:54:15 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
coreClient, err := coreClient(nodeConfig.AgentConfig.KubeConfigKubelet)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2021-02-26 18:37:27 +00:00
|
|
|
|
2021-11-11 21:01:49 +00:00
|
|
|
if err := util.WaitForAPIServerReady(ctx, coreClient, util.DefaultAPIServerReadyTimeout); err != nil {
|
|
|
|
return errors.Wrap(err, "failed to wait for apiserver ready")
|
|
|
|
}
|
2021-02-26 18:37:27 +00:00
|
|
|
|
2021-10-29 08:59:03 +00:00
|
|
|
if err := configureNode(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-01-09 16:54:15 +00:00
|
|
|
if !nodeConfig.NoFlannel {
|
2019-10-27 05:53:25 +00:00
|
|
|
if err := flannel.Run(ctx, nodeConfig, coreClient.CoreV1().Nodes()); err != nil {
|
2019-01-09 16:54:15 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-10-17 21:46:15 +00:00
|
|
|
if !nodeConfig.AgentConfig.DisableNPC {
|
|
|
|
if err := netpol.Run(ctx, nodeConfig); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-15 11:20:26 +00:00
|
|
|
os.Setenv("NOTIFY_SOCKET", notifySocket)
|
|
|
|
systemd.SdNotify(true, "READY=1\n")
|
|
|
|
|
2019-01-09 16:54:15 +00:00
|
|
|
<-ctx.Done()
|
|
|
|
return ctx.Err()
|
|
|
|
}
|
|
|
|
|
2021-05-17 20:30:55 +00:00
|
|
|
// getConntrackConfig uses the kube-proxy code to parse the user-provided kube-proxy-arg values, and
|
|
|
|
// extract the conntrack settings so that K3s can set them itself. This allows us to soft-fail when
|
|
|
|
// running K3s in Docker, where kube-proxy is no longer allowed to set conntrack sysctls on newer kernels.
|
|
|
|
// When running rootless, we do not attempt to set conntrack sysctls - this behavior is copied from kubeadm.
|
|
|
|
func getConntrackConfig(nodeConfig *daemonconfig.Node) (*kubeproxyconfig.KubeProxyConntrackConfiguration, error) {
|
|
|
|
ctConfig := &kubeproxyconfig.KubeProxyConntrackConfiguration{
|
|
|
|
MaxPerCore: utilpointer.Int32Ptr(0),
|
|
|
|
Min: utilpointer.Int32Ptr(0),
|
|
|
|
TCPEstablishedTimeout: &metav1.Duration{},
|
|
|
|
TCPCloseWaitTimeout: &metav1.Duration{},
|
|
|
|
}
|
|
|
|
|
|
|
|
if nodeConfig.AgentConfig.Rootless {
|
|
|
|
return ctConfig, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
cmd := app2.NewProxyCommand()
|
2021-07-28 20:04:19 +00:00
|
|
|
if err := cmd.ParseFlags(daemonconfig.GetArgs(map[string]string{}, nodeConfig.AgentConfig.ExtraKubeProxyArgs)); err != nil {
|
2021-05-17 20:30:55 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
maxPerCore, err := cmd.Flags().GetInt32("conntrack-max-per-core")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
ctConfig.MaxPerCore = &maxPerCore
|
|
|
|
min, err := cmd.Flags().GetInt32("conntrack-min")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
ctConfig.Min = &min
|
|
|
|
establishedTimeout, err := cmd.Flags().GetDuration("conntrack-tcp-timeout-established")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
ctConfig.TCPEstablishedTimeout.Duration = establishedTimeout
|
|
|
|
closeWaitTimeout, err := cmd.Flags().GetDuration("conntrack-tcp-timeout-close-wait")
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
ctConfig.TCPCloseWaitTimeout.Duration = closeWaitTimeout
|
|
|
|
return ctConfig, nil
|
|
|
|
}
|
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
func coreClient(cfg string) (kubernetes.Interface, error) {
|
|
|
|
restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return kubernetes.NewForConfig(restConfig)
|
|
|
|
}
|
|
|
|
|
2022-02-15 00:34:49 +00:00
|
|
|
// RunStandalone bootstraps the executor, but does not run the kubelet or containerd.
|
|
|
|
// This allows other bits of code that expect the executor to be set up properly to function
|
|
|
|
// even when the agent is disabled. It will only return in case of error or context
|
|
|
|
// cancellation.
|
|
|
|
func RunStandalone(ctx context.Context, cfg cmds.Agent) error {
|
|
|
|
proxy, err := createProxyAndValidateToken(ctx, &cfg)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
nodeConfig := config.Get(ctx, cfg, proxy)
|
|
|
|
if err := executor.Bootstrap(ctx, nodeConfig, cfg); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if cfg.AgentReady != nil {
|
|
|
|
close(cfg.AgentReady)
|
|
|
|
}
|
|
|
|
|
|
|
|
<-ctx.Done()
|
|
|
|
return ctx.Err()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Run sets up cgroups, configures the LB proxy, and triggers startup
|
|
|
|
// of containerd and kubelet. It will only return in case of error or context
|
|
|
|
// cancellation.
|
2019-01-09 16:54:15 +00:00
|
|
|
func Run(ctx context.Context, cfg cmds.Agent) error {
|
2021-06-01 19:29:46 +00:00
|
|
|
if err := cgroups.Validate(); err != nil {
|
2019-02-08 04:13:43 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2019-10-19 10:18:51 +00:00
|
|
|
if cfg.Rootless && !cfg.RootlessAlreadyUnshared {
|
2019-03-08 22:47:44 +00:00
|
|
|
if err := rootless.Rootless(cfg.DataDir); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-02-15 00:34:49 +00:00
|
|
|
proxy, err := createProxyAndValidateToken(ctx, &cfg)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return run(ctx, cfg, proxy)
|
|
|
|
}
|
|
|
|
|
|
|
|
func createProxyAndValidateToken(ctx context.Context, cfg *cmds.Agent) (proxy.Proxy, error) {
|
2020-11-03 19:19:26 +00:00
|
|
|
agentDir := filepath.Join(cfg.DataDir, "agent")
|
|
|
|
if err := os.MkdirAll(agentDir, 0700); err != nil {
|
2022-02-15 00:34:49 +00:00
|
|
|
return nil, err
|
2020-04-27 16:42:15 +00:00
|
|
|
}
|
2019-01-09 16:54:15 +00:00
|
|
|
|
2021-03-08 22:10:00 +00:00
|
|
|
proxy, err := proxy.NewSupervisorProxy(ctx, !cfg.DisableLoadBalancer, agentDir, cfg.ServerURL, cfg.LBServerPort)
|
2019-07-24 07:22:31 +00:00
|
|
|
if err != nil {
|
2022-02-15 00:34:49 +00:00
|
|
|
return nil, err
|
2019-07-24 07:22:31 +00:00
|
|
|
}
|
|
|
|
|
2019-01-09 16:54:15 +00:00
|
|
|
for {
|
2020-09-24 06:47:17 +00:00
|
|
|
newToken, err := clientaccess.ParseAndValidateTokenForUser(proxy.SupervisorURL(), cfg.Token, "node")
|
2019-01-09 16:54:15 +00:00
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
2022-02-15 00:34:49 +00:00
|
|
|
return nil, ctx.Err()
|
2019-01-09 16:54:15 +00:00
|
|
|
case <-time.After(2 * time.Second):
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
2020-09-24 06:47:17 +00:00
|
|
|
cfg.Token = newToken.String()
|
2019-01-09 16:54:15 +00:00
|
|
|
break
|
|
|
|
}
|
2022-02-15 00:34:49 +00:00
|
|
|
return proxy, nil
|
2019-01-09 16:54:15 +00:00
|
|
|
}
|
2019-02-08 04:13:43 +00:00
|
|
|
|
2021-10-29 08:59:03 +00:00
|
|
|
func configureNode(ctx context.Context, agentConfig *daemonconfig.Agent, nodes typedcorev1.NodeInterface) error {
|
|
|
|
fieldSelector := fields.Set{metav1.ObjectNameField: agentConfig.NodeName}.String()
|
|
|
|
watch, err := nodes.Watch(ctx, metav1.ListOptions{FieldSelector: fieldSelector})
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer watch.Stop()
|
|
|
|
|
|
|
|
for ev := range watch.ResultChan() {
|
|
|
|
node, ok := ev.Object.(*corev1.Node)
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("could not convert event object to node: %v", ev)
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
2019-10-27 05:53:25 +00:00
|
|
|
|
2021-04-21 22:56:20 +00:00
|
|
|
updateNode := false
|
|
|
|
if labels, changed := updateMutableLabels(agentConfig, node.Labels); changed {
|
|
|
|
node.Labels = labels
|
|
|
|
updateNode = true
|
|
|
|
}
|
2019-12-09 22:54:56 +00:00
|
|
|
|
2021-04-21 22:56:20 +00:00
|
|
|
if !agentConfig.DisableCCM {
|
|
|
|
if annotations, changed := updateAddressAnnotations(agentConfig, node.Annotations); changed {
|
|
|
|
node.Annotations = annotations
|
|
|
|
updateNode = true
|
|
|
|
}
|
|
|
|
if labels, changed := updateLegacyAddressLabels(agentConfig, node.Labels); changed {
|
|
|
|
node.Labels = labels
|
|
|
|
updateNode = true
|
|
|
|
}
|
2019-12-09 22:54:56 +00:00
|
|
|
}
|
|
|
|
|
2020-02-11 23:27:43 +00:00
|
|
|
// inject node config
|
2021-04-21 22:56:20 +00:00
|
|
|
if changed, err := nodeconfig.SetNodeConfigAnnotations(node); err != nil {
|
2020-02-11 23:27:43 +00:00
|
|
|
return err
|
2021-04-21 22:56:20 +00:00
|
|
|
} else if changed {
|
2020-02-11 23:27:43 +00:00
|
|
|
updateNode = true
|
|
|
|
}
|
2021-04-21 22:56:20 +00:00
|
|
|
|
2020-02-11 23:27:43 +00:00
|
|
|
if updateNode {
|
2020-03-26 21:08:47 +00:00
|
|
|
if _, err := nodes.Update(ctx, node, metav1.UpdateOptions{}); err != nil {
|
2019-10-27 05:53:25 +00:00
|
|
|
logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
|
2021-10-29 08:59:03 +00:00
|
|
|
continue
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
2019-12-09 22:54:56 +00:00
|
|
|
logrus.Infof("labels have been set successfully on node: %s", agentConfig.NodeName)
|
2019-10-27 05:53:25 +00:00
|
|
|
} else {
|
2019-12-09 22:54:56 +00:00
|
|
|
logrus.Infof("labels have already set on node: %s", agentConfig.NodeName)
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
break
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-27 05:53:25 +00:00
|
|
|
return nil
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
|
|
|
|
2019-12-09 22:54:56 +00:00
|
|
|
func updateMutableLabels(agentConfig *daemonconfig.Agent, nodeLabels map[string]string) (map[string]string, bool) {
|
2019-10-27 05:53:25 +00:00
|
|
|
result := map[string]string{}
|
2019-12-09 22:54:56 +00:00
|
|
|
|
|
|
|
for _, m := range agentConfig.NodeLabels {
|
|
|
|
var (
|
|
|
|
v string
|
|
|
|
p = strings.SplitN(m, `=`, 2)
|
|
|
|
k = p[0]
|
|
|
|
)
|
|
|
|
if len(p) > 1 {
|
|
|
|
v = p[1]
|
|
|
|
}
|
2019-10-27 05:53:25 +00:00
|
|
|
result[k] = v
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
2019-12-09 22:54:56 +00:00
|
|
|
result = labels.Merge(nodeLabels, result)
|
|
|
|
return result, !equality.Semantic.DeepEqual(nodeLabels, result)
|
|
|
|
}
|
2019-10-27 05:53:25 +00:00
|
|
|
|
2021-04-21 22:56:20 +00:00
|
|
|
func updateLegacyAddressLabels(agentConfig *daemonconfig.Agent, nodeLabels map[string]string) (map[string]string, bool) {
|
|
|
|
ls := labels.Set(nodeLabels)
|
|
|
|
if ls.Has(cp.InternalIPKey) || ls.Has(cp.HostnameKey) {
|
|
|
|
result := map[string]string{
|
|
|
|
cp.InternalIPKey: agentConfig.NodeIP,
|
|
|
|
cp.HostnameKey: agentConfig.NodeName,
|
|
|
|
}
|
|
|
|
|
|
|
|
if agentConfig.NodeExternalIP != "" {
|
|
|
|
result[cp.ExternalIPKey] = agentConfig.NodeExternalIP
|
|
|
|
}
|
|
|
|
|
|
|
|
result = labels.Merge(nodeLabels, result)
|
|
|
|
return result, !equality.Semantic.DeepEqual(nodeLabels, result)
|
|
|
|
}
|
|
|
|
return nil, false
|
|
|
|
}
|
|
|
|
|
|
|
|
func updateAddressAnnotations(agentConfig *daemonconfig.Agent, nodeAnnotations map[string]string) (map[string]string, bool) {
|
2019-12-09 22:54:56 +00:00
|
|
|
result := map[string]string{
|
2021-04-21 22:56:20 +00:00
|
|
|
cp.InternalIPKey: util.JoinIPs(agentConfig.NodeIPs),
|
|
|
|
cp.HostnameKey: agentConfig.NodeName,
|
2019-12-09 22:54:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if agentConfig.NodeExternalIP != "" {
|
2021-04-21 22:56:20 +00:00
|
|
|
result[cp.ExternalIPKey] = util.JoinIPs(agentConfig.NodeExternalIPs)
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
2019-10-27 05:53:25 +00:00
|
|
|
|
2021-04-21 22:56:20 +00:00
|
|
|
result = labels.Merge(nodeAnnotations, result)
|
|
|
|
return result, !equality.Semantic.DeepEqual(nodeAnnotations, result)
|
2019-10-15 21:17:26 +00:00
|
|
|
}
|
2021-02-12 15:35:57 +00:00
|
|
|
|
|
|
|
// setupTunnelAndRunAgent should start the setup tunnel before starting kubelet and kubeproxy
|
|
|
|
// there are special case for etcd agents, it will wait until it can find the apiaddress from
|
|
|
|
// the address channel and update the proxy with the servers addresses, if in rke2 we need to
|
|
|
|
// start the agent before the tunnel is setup to allow kubelet to start first and start the pods
|
|
|
|
func setupTunnelAndRunAgent(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent, proxy proxy.Proxy) error {
|
|
|
|
var agentRan bool
|
2021-07-27 21:56:05 +00:00
|
|
|
// IsAPIServerLBEnabled is used as a shortcut for detecting RKE2, where the kubelet needs to
|
|
|
|
// be run earlier in order to manage static pods. This should probably instead query a
|
|
|
|
// flag on the executor or something.
|
2021-02-12 15:35:57 +00:00
|
|
|
if cfg.ETCDAgent {
|
2021-07-27 21:56:05 +00:00
|
|
|
// ETCDAgent is only set to true on servers that are started with --disable-apiserver.
|
|
|
|
// In this case, we may be running without an apiserver available in the cluster, and need
|
|
|
|
// to wait for one to register and post it's address into APIAddressCh so that we can update
|
|
|
|
// the LB proxy with its address.
|
2021-02-12 15:35:57 +00:00
|
|
|
if proxy.IsAPIServerLBEnabled() {
|
2021-07-27 21:56:05 +00:00
|
|
|
// On RKE2, the agent needs to be started early to run the etcd static pod.
|
|
|
|
if err := agent.Agent(ctx, nodeConfig, proxy); err != nil {
|
2021-02-12 15:35:57 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
agentRan = true
|
|
|
|
}
|
2022-02-16 22:19:58 +00:00
|
|
|
if err := waitForAPIServerAddresses(ctx, nodeConfig, cfg, proxy); err != nil {
|
|
|
|
return err
|
2021-02-12 15:35:57 +00:00
|
|
|
}
|
2021-05-05 15:40:04 +00:00
|
|
|
} else if cfg.ClusterReset && proxy.IsAPIServerLBEnabled() {
|
2021-07-27 21:56:05 +00:00
|
|
|
// If we're doing a cluster-reset on RKE2, the kubelet needs to be started early to clean
|
|
|
|
// up static pods.
|
|
|
|
if err := agent.Agent(ctx, nodeConfig, proxy); err != nil {
|
2021-05-05 15:40:04 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
agentRan = true
|
2021-02-12 15:35:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if err := tunnel.Setup(ctx, nodeConfig, proxy); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !agentRan {
|
2021-07-27 21:56:05 +00:00
|
|
|
return agent.Agent(ctx, nodeConfig, proxy)
|
2021-02-12 15:35:57 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
2022-02-16 22:19:58 +00:00
|
|
|
|
|
|
|
func waitForAPIServerAddresses(ctx context.Context, nodeConfig *daemonconfig.Node, cfg cmds.Agent, proxy proxy.Proxy) error {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-time.After(5 * time.Second):
|
|
|
|
logrus.Info("Waiting for apiserver addresses")
|
|
|
|
case addresses := <-cfg.APIAddressCh:
|
|
|
|
for i, a := range addresses {
|
|
|
|
host, _, err := net.SplitHostPort(a)
|
|
|
|
if err == nil {
|
|
|
|
addresses[i] = net.JoinHostPort(host, strconv.Itoa(nodeConfig.ServerHTTPSPort))
|
|
|
|
if i == 0 {
|
|
|
|
proxy.SetSupervisorDefault(addresses[i])
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
proxy.Update(addresses)
|
|
|
|
return nil
|
|
|
|
case <-ctx.Done():
|
|
|
|
return ctx.Err()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|