2019-01-09 16:54:15 +00:00
package agent
import (
"context"
2019-02-08 04:13:43 +00:00
"errors"
2020-11-24 07:59:16 +00:00
"fmt"
2019-02-08 04:13:43 +00:00
"io/ioutil"
2021-02-12 15:35:57 +00:00
"net/url"
2019-01-09 16:54:15 +00:00
"os"
"path/filepath"
2019-02-08 04:13:43 +00:00
"strings"
2019-01-09 16:54:15 +00:00
"time"
2020-11-24 07:59:16 +00:00
"github.com/containerd/cgroups"
cgroupsv2 "github.com/containerd/cgroups/v2"
2019-10-27 05:53:25 +00:00
systemd "github.com/coreos/go-systemd/daemon"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/agent/config"
"github.com/rancher/k3s/pkg/agent/containerd"
"github.com/rancher/k3s/pkg/agent/flannel"
2019-10-17 21:46:15 +00:00
"github.com/rancher/k3s/pkg/agent/netpol"
2020-04-28 22:00:30 +00:00
"github.com/rancher/k3s/pkg/agent/proxy"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/agent/syssetup"
"github.com/rancher/k3s/pkg/agent/tunnel"
"github.com/rancher/k3s/pkg/cli/cmds"
2019-05-09 22:05:51 +00:00
"github.com/rancher/k3s/pkg/clientaccess"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/daemons/agent"
2019-10-15 21:17:26 +00:00
daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
2020-02-11 23:27:43 +00:00
"github.com/rancher/k3s/pkg/nodeconfig"
2019-03-08 22:47:44 +00:00
"github.com/rancher/k3s/pkg/rootless"
2020-05-05 22:09:04 +00:00
"github.com/rancher/k3s/pkg/version"
2019-01-22 21:14:58 +00:00
"github.com/sirupsen/logrus"
2019-10-27 05:53:25 +00:00
"k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2020-04-28 22:00:30 +00:00
"k8s.io/apimachinery/pkg/labels"
2019-10-27 05:53:25 +00:00
"k8s.io/client-go/kubernetes"
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
2019-10-15 21:17:26 +00:00
"k8s.io/client-go/tools/clientcmd"
2021-02-26 18:37:27 +00:00
"k8s.io/controller-manager/app"
2019-10-15 21:17:26 +00:00
)
2020-05-05 22:09:04 +00:00
var (
InternalIPLabel = version . Program + ".io/internal-ip"
ExternalIPLabel = version . Program + ".io/external-ip"
HostnameLabel = version . Program + ".io/hostname"
2019-01-09 16:54:15 +00:00
)
2020-06-24 21:16:44 +00:00
const (
dockershimSock = "unix:///var/run/dockershim.sock"
containerdSock = "unix:///run/k3s/containerd/containerd.sock"
)
2019-01-09 16:54:15 +00:00
2020-06-24 21:16:44 +00:00
// setupCriCtlConfig creates the crictl config file and populates it
// with the given data from config.
func setupCriCtlConfig ( cfg cmds . Agent , nodeConfig * daemonconfig . Node ) error {
cre := nodeConfig . ContainerRuntimeEndpoint
if cre == "" {
switch {
case cfg . Docker :
cre = dockershimSock
default :
cre = containerdSock
}
}
2020-11-03 19:19:26 +00:00
agentConfDir := filepath . Join ( cfg . DataDir , "agent" , "etc" )
2020-06-24 21:16:44 +00:00
if _ , err := os . Stat ( agentConfDir ) ; os . IsNotExist ( err ) {
2020-11-03 19:19:26 +00:00
if err := os . MkdirAll ( agentConfDir , 0700 ) ; err != nil {
2020-06-24 01:30:24 +00:00
return err
}
}
2020-06-24 21:16:44 +00:00
crp := "runtime-endpoint: " + cre + "\n"
return ioutil . WriteFile ( agentConfDir + "/crictl.yaml" , [ ] byte ( crp ) , 0600 )
}
func run ( ctx context . Context , cfg cmds . Agent , proxy proxy . Proxy ) error {
nodeConfig := config . Get ( ctx , cfg , proxy )
if err := setupCriCtlConfig ( cfg , nodeConfig ) ; err != nil {
return err
}
2019-01-09 16:54:15 +00:00
if ! nodeConfig . NoFlannel {
if err := flannel . Prepare ( ctx , nodeConfig ) ; err != nil {
return err
}
}
2019-12-10 23:16:26 +00:00
if ! nodeConfig . Docker && nodeConfig . ContainerRuntimeEndpoint == "" {
2019-01-09 16:54:15 +00:00
if err := containerd . Run ( ctx , nodeConfig ) ; err != nil {
return err
}
}
2021-02-12 15:35:57 +00:00
if err := setupTunnelAndRunAgent ( ctx , nodeConfig , cfg , proxy ) ; err != nil {
2019-01-09 16:54:15 +00:00
return err
}
2019-10-27 05:53:25 +00:00
coreClient , err := coreClient ( nodeConfig . AgentConfig . KubeConfigKubelet )
if err != nil {
return err
}
2021-02-26 18:37:27 +00:00
app . WaitForAPIServer ( coreClient , 30 * time . Second )
2019-01-09 16:54:15 +00:00
if ! nodeConfig . NoFlannel {
2019-10-27 05:53:25 +00:00
if err := flannel . Run ( ctx , nodeConfig , coreClient . CoreV1 ( ) . Nodes ( ) ) ; err != nil {
2019-01-09 16:54:15 +00:00
return err
}
}
2020-02-11 23:27:43 +00:00
if err := configureNode ( ctx , & nodeConfig . AgentConfig , coreClient . CoreV1 ( ) . Nodes ( ) ) ; err != nil {
2019-12-09 22:54:56 +00:00
return err
2019-10-15 21:17:26 +00:00
}
2019-10-17 21:46:15 +00:00
if ! nodeConfig . AgentConfig . DisableNPC {
if err := netpol . Run ( ctx , nodeConfig ) ; err != nil {
return err
}
}
2019-01-09 16:54:15 +00:00
<- ctx . Done ( )
return ctx . Err ( )
}
2019-10-27 05:53:25 +00:00
func coreClient ( cfg string ) ( kubernetes . Interface , error ) {
restConfig , err := clientcmd . BuildConfigFromFlags ( "" , cfg )
if err != nil {
return nil , err
}
return kubernetes . NewForConfig ( restConfig )
}
2019-01-09 16:54:15 +00:00
func Run ( ctx context . Context , cfg cmds . Agent ) error {
2019-02-08 04:13:43 +00:00
if err := validate ( ) ; err != nil {
return err
}
2019-11-04 18:35:14 +00:00
syssetup . Configure ( )
2019-02-08 04:13:43 +00:00
2019-10-19 10:18:51 +00:00
if cfg . Rootless && ! cfg . RootlessAlreadyUnshared {
2019-03-08 22:47:44 +00:00
if err := rootless . Rootless ( cfg . DataDir ) ; err != nil {
return err
}
}
2020-11-03 19:19:26 +00:00
agentDir := filepath . Join ( cfg . DataDir , "agent" )
if err := os . MkdirAll ( agentDir , 0700 ) ; err != nil {
2020-04-27 16:42:15 +00:00
return err
}
2019-01-09 16:54:15 +00:00
2021-03-08 22:10:00 +00:00
proxy , err := proxy . NewSupervisorProxy ( ctx , ! cfg . DisableLoadBalancer , agentDir , cfg . ServerURL , cfg . LBServerPort )
2019-07-24 07:22:31 +00:00
if err != nil {
return err
}
2019-01-09 16:54:15 +00:00
for {
2020-09-24 06:47:17 +00:00
newToken , err := clientaccess . ParseAndValidateTokenForUser ( proxy . SupervisorURL ( ) , cfg . Token , "node" )
2019-01-09 16:54:15 +00:00
if err != nil {
logrus . Error ( err )
select {
case <- ctx . Done ( ) :
return ctx . Err ( )
case <- time . After ( 2 * time . Second ) :
}
continue
}
2020-09-24 06:47:17 +00:00
cfg . Token = newToken . String ( )
2019-01-09 16:54:15 +00:00
break
}
2019-10-27 05:53:25 +00:00
systemd . SdNotify ( true , "READY=1\n" )
2020-04-28 22:00:30 +00:00
return run ( ctx , cfg , proxy )
2019-01-09 16:54:15 +00:00
}
2019-02-08 04:13:43 +00:00
func validate ( ) error {
2020-11-24 07:59:16 +00:00
if cgroups . Mode ( ) == cgroups . Unified {
return validateCgroupsV2 ( )
}
return validateCgroupsV1 ( )
}
func validateCgroupsV1 ( ) error {
2019-02-08 04:13:43 +00:00
cgroups , err := ioutil . ReadFile ( "/proc/self/cgroup" )
if err != nil {
return err
}
if ! strings . Contains ( string ( cgroups ) , "cpuset" ) {
2020-09-21 16:56:03 +00:00
logrus . Warn ( ` Failed to find cpuset cgroup, you may need to add "cgroup_enable=cpuset" to your linux cmdline (/boot/cmdline.txt on a Raspberry Pi) ` )
2019-02-08 04:13:43 +00:00
}
if ! strings . Contains ( string ( cgroups ) , "memory" ) {
msg := "ailed to find memory cgroup, you may need to add \"cgroup_memory=1 cgroup_enable=memory\" to your linux cmdline (/boot/cmdline.txt on a Raspberry Pi)"
logrus . Error ( "F" + msg )
return errors . New ( "f" + msg )
}
return nil
}
2019-10-15 21:17:26 +00:00
2020-11-24 07:59:16 +00:00
func validateCgroupsV2 ( ) error {
manager , err := cgroupsv2 . LoadManager ( "/sys/fs/cgroup" , "/" )
if err != nil {
return err
}
controllers , err := manager . RootControllers ( )
if err != nil {
return err
}
m := make ( map [ string ] struct { } )
for _ , controller := range controllers {
m [ controller ] = struct { } { }
}
for _ , controller := range [ ] string { "cpu" , "cpuset" , "memory" } {
if _ , ok := m [ controller ] ; ! ok {
2020-12-22 20:35:58 +00:00
return fmt . Errorf ( "failed to find %s cgroup (v2)" , controller )
2020-11-24 07:59:16 +00:00
}
}
return nil
}
2020-02-11 23:27:43 +00:00
func configureNode ( ctx context . Context , agentConfig * daemonconfig . Agent , nodes v1 . NodeInterface ) error {
2020-05-05 21:51:39 +00:00
count := 0
2019-10-15 21:17:26 +00:00
for {
2020-03-26 21:08:47 +00:00
node , err := nodes . Get ( ctx , agentConfig . NodeName , metav1 . GetOptions { } )
2019-10-15 21:17:26 +00:00
if err != nil {
2020-05-05 21:51:39 +00:00
if count % 30 == 0 {
logrus . Infof ( "Waiting for kubelet to be ready on node %s: %v" , agentConfig . NodeName , err )
}
count ++
2019-10-15 21:17:26 +00:00
time . Sleep ( 1 * time . Second )
continue
}
2019-10-27 05:53:25 +00:00
2019-12-09 22:54:56 +00:00
newLabels , updateMutables := updateMutableLabels ( agentConfig , node . Labels )
updateAddresses := ! agentConfig . DisableCCM
if updateAddresses {
newLabels , updateAddresses = updateAddressLabels ( agentConfig , newLabels )
}
2020-02-11 23:27:43 +00:00
// inject node config
updateNode , err := nodeconfig . SetNodeConfigAnnotations ( node )
if err != nil {
return err
}
2019-12-09 22:54:56 +00:00
if updateAddresses || updateMutables {
2019-10-27 05:53:25 +00:00
node . Labels = newLabels
2020-02-11 23:27:43 +00:00
updateNode = true
}
if updateNode {
2020-03-26 21:08:47 +00:00
if _ , err := nodes . Update ( ctx , node , metav1 . UpdateOptions { } ) ; err != nil {
2019-10-27 05:53:25 +00:00
logrus . Infof ( "Failed to update node %s: %v" , agentConfig . NodeName , err )
select {
case <- ctx . Done ( ) :
return ctx . Err ( )
case <- time . After ( time . Second ) :
continue
}
2019-10-15 21:17:26 +00:00
}
2019-12-09 22:54:56 +00:00
logrus . Infof ( "labels have been set successfully on node: %s" , agentConfig . NodeName )
2019-10-27 05:53:25 +00:00
} else {
2019-12-09 22:54:56 +00:00
logrus . Infof ( "labels have already set on node: %s" , agentConfig . NodeName )
2019-10-15 21:17:26 +00:00
}
2019-10-27 05:53:25 +00:00
break
2019-10-15 21:17:26 +00:00
}
2019-10-27 05:53:25 +00:00
return nil
2019-10-15 21:17:26 +00:00
}
2019-12-09 22:54:56 +00:00
func updateMutableLabels ( agentConfig * daemonconfig . Agent , nodeLabels map [ string ] string ) ( map [ string ] string , bool ) {
2019-10-27 05:53:25 +00:00
result := map [ string ] string { }
2019-12-09 22:54:56 +00:00
for _ , m := range agentConfig . NodeLabels {
var (
v string
p = strings . SplitN ( m , ` = ` , 2 )
k = p [ 0 ]
)
if len ( p ) > 1 {
v = p [ 1 ]
}
2019-10-27 05:53:25 +00:00
result [ k ] = v
2019-10-15 21:17:26 +00:00
}
2019-12-09 22:54:56 +00:00
result = labels . Merge ( nodeLabels , result )
return result , ! equality . Semantic . DeepEqual ( nodeLabels , result )
}
2019-10-27 05:53:25 +00:00
2019-12-09 22:54:56 +00:00
func updateAddressLabels ( agentConfig * daemonconfig . Agent , nodeLabels map [ string ] string ) ( map [ string ] string , bool ) {
result := map [ string ] string {
InternalIPLabel : agentConfig . NodeIP ,
HostnameLabel : agentConfig . NodeName ,
}
if agentConfig . NodeExternalIP != "" {
2019-10-27 05:53:25 +00:00
result [ ExternalIPLabel ] = agentConfig . NodeExternalIP
2019-10-15 21:17:26 +00:00
}
2019-10-27 05:53:25 +00:00
2019-12-09 22:54:56 +00:00
result = labels . Merge ( nodeLabels , result )
2019-10-27 05:53:25 +00:00
return result , ! equality . Semantic . DeepEqual ( nodeLabels , result )
2019-10-15 21:17:26 +00:00
}
2021-02-12 15:35:57 +00:00
// setupTunnelAndRunAgent should start the setup tunnel before starting kubelet and kubeproxy
// there are special case for etcd agents, it will wait until it can find the apiaddress from
// the address channel and update the proxy with the servers addresses, if in rke2 we need to
// start the agent before the tunnel is setup to allow kubelet to start first and start the pods
func setupTunnelAndRunAgent ( ctx context . Context , nodeConfig * daemonconfig . Node , cfg cmds . Agent , proxy proxy . Proxy ) error {
var agentRan bool
if cfg . ETCDAgent {
// only in rke2 run the agent before the tunnel setup and check for that later in the function
if proxy . IsAPIServerLBEnabled ( ) {
if err := agent . Agent ( & nodeConfig . AgentConfig ) ; err != nil {
return err
}
agentRan = true
}
select {
case address := <- cfg . APIAddressCh :
cfg . ServerURL = address
u , err := url . Parse ( cfg . ServerURL )
if err != nil {
logrus . Warn ( err )
}
proxy . Update ( [ ] string { fmt . Sprintf ( "%s:%d" , u . Hostname ( ) , nodeConfig . ServerHTTPSPort ) } )
case <- ctx . Done ( ) :
return ctx . Err ( )
}
}
if err := tunnel . Setup ( ctx , nodeConfig , proxy ) ; err != nil {
return err
}
if ! agentRan {
return agent . Agent ( & nodeConfig . AgentConfig )
}
return nil
}