2019-01-09 16:54:15 +00:00
package agent
import (
"context"
2019-02-08 04:13:43 +00:00
"errors"
"io/ioutil"
2019-01-09 16:54:15 +00:00
"os"
"path/filepath"
2019-02-08 04:13:43 +00:00
"strings"
2019-01-09 16:54:15 +00:00
"time"
2019-12-09 22:54:56 +00:00
"k8s.io/apimachinery/pkg/labels"
2019-10-27 05:53:25 +00:00
systemd "github.com/coreos/go-systemd/daemon"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/agent/config"
"github.com/rancher/k3s/pkg/agent/containerd"
"github.com/rancher/k3s/pkg/agent/flannel"
2019-07-24 07:22:31 +00:00
"github.com/rancher/k3s/pkg/agent/loadbalancer"
2019-10-17 21:46:15 +00:00
"github.com/rancher/k3s/pkg/agent/netpol"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/agent/syssetup"
"github.com/rancher/k3s/pkg/agent/tunnel"
"github.com/rancher/k3s/pkg/cli/cmds"
2019-05-09 22:05:51 +00:00
"github.com/rancher/k3s/pkg/clientaccess"
2019-01-09 16:54:15 +00:00
"github.com/rancher/k3s/pkg/daemons/agent"
2019-10-15 21:17:26 +00:00
daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
2020-02-11 23:27:43 +00:00
"github.com/rancher/k3s/pkg/nodeconfig"
2019-03-08 22:47:44 +00:00
"github.com/rancher/k3s/pkg/rootless"
2019-01-22 21:14:58 +00:00
"github.com/sirupsen/logrus"
2019-10-27 05:53:25 +00:00
"k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
2019-10-15 21:17:26 +00:00
"k8s.io/client-go/tools/clientcmd"
)
const (
InternalIPLabel = "k3s.io/internal-ip"
ExternalIPLabel = "k3s.io/external-ip"
HostnameLabel = "k3s.io/hostname"
2019-01-09 16:54:15 +00:00
)
2019-07-24 07:22:31 +00:00
func run ( ctx context . Context , cfg cmds . Agent , lb * loadbalancer . LoadBalancer ) error {
2019-01-09 16:54:15 +00:00
nodeConfig := config . Get ( ctx , cfg )
if ! nodeConfig . NoFlannel {
if err := flannel . Prepare ( ctx , nodeConfig ) ; err != nil {
return err
}
}
2019-12-10 23:16:26 +00:00
if ! nodeConfig . Docker && nodeConfig . ContainerRuntimeEndpoint == "" {
2019-01-09 16:54:15 +00:00
if err := containerd . Run ( ctx , nodeConfig ) ; err != nil {
return err
}
}
2019-07-24 07:22:31 +00:00
if err := tunnel . Setup ( ctx , nodeConfig , lb . Update ) ; err != nil {
2019-01-09 16:54:15 +00:00
return err
}
if err := agent . Agent ( & nodeConfig . AgentConfig ) ; err != nil {
return err
}
2019-10-27 05:53:25 +00:00
coreClient , err := coreClient ( nodeConfig . AgentConfig . KubeConfigKubelet )
if err != nil {
return err
}
2019-01-09 16:54:15 +00:00
if ! nodeConfig . NoFlannel {
2019-10-27 05:53:25 +00:00
if err := flannel . Run ( ctx , nodeConfig , coreClient . CoreV1 ( ) . Nodes ( ) ) ; err != nil {
2019-01-09 16:54:15 +00:00
return err
}
}
2020-02-11 23:27:43 +00:00
if err := configureNode ( ctx , & nodeConfig . AgentConfig , coreClient . CoreV1 ( ) . Nodes ( ) ) ; err != nil {
2019-12-09 22:54:56 +00:00
return err
2019-10-15 21:17:26 +00:00
}
2019-10-17 21:46:15 +00:00
if ! nodeConfig . AgentConfig . DisableNPC {
if err := netpol . Run ( ctx , nodeConfig ) ; err != nil {
return err
}
}
2019-01-09 16:54:15 +00:00
<- ctx . Done ( )
return ctx . Err ( )
}
2019-10-27 05:53:25 +00:00
func coreClient ( cfg string ) ( kubernetes . Interface , error ) {
restConfig , err := clientcmd . BuildConfigFromFlags ( "" , cfg )
if err != nil {
return nil , err
}
return kubernetes . NewForConfig ( restConfig )
}
2019-01-09 16:54:15 +00:00
func Run ( ctx context . Context , cfg cmds . Agent ) error {
2019-02-08 04:13:43 +00:00
if err := validate ( ) ; err != nil {
return err
}
2019-11-04 18:35:14 +00:00
syssetup . Configure ( )
2019-02-08 04:13:43 +00:00
2019-10-19 10:18:51 +00:00
if cfg . Rootless && ! cfg . RootlessAlreadyUnshared {
2019-03-08 22:47:44 +00:00
if err := rootless . Rootless ( cfg . DataDir ) ; err != nil {
return err
}
}
2019-01-09 16:54:15 +00:00
cfg . DataDir = filepath . Join ( cfg . DataDir , "agent" )
2020-04-27 16:42:15 +00:00
if err := os . MkdirAll ( cfg . DataDir , 0700 ) ; err != nil {
return err
}
2019-01-09 16:54:15 +00:00
2019-07-24 07:22:31 +00:00
lb , err := loadbalancer . Setup ( ctx , cfg )
if err != nil {
return err
}
if lb != nil {
cfg . ServerURL = lb . LoadBalancerServerURL ( )
}
2019-01-09 16:54:15 +00:00
for {
2019-10-27 05:53:25 +00:00
newToken , err := clientaccess . NormalizeAndValidateTokenForUser ( cfg . ServerURL , cfg . Token , "node" )
2019-01-09 16:54:15 +00:00
if err != nil {
logrus . Error ( err )
select {
case <- ctx . Done ( ) :
return ctx . Err ( )
case <- time . After ( 2 * time . Second ) :
}
continue
}
2019-10-27 05:53:25 +00:00
cfg . Token = newToken
2019-01-09 16:54:15 +00:00
break
}
2019-10-27 05:53:25 +00:00
systemd . SdNotify ( true , "READY=1\n" )
2019-07-24 07:22:31 +00:00
return run ( ctx , cfg , lb )
2019-01-09 16:54:15 +00:00
}
2019-02-08 04:13:43 +00:00
func validate ( ) error {
cgroups , err := ioutil . ReadFile ( "/proc/self/cgroup" )
if err != nil {
return err
}
if ! strings . Contains ( string ( cgroups ) , "cpuset" ) {
logrus . Warn ( "Failed to find cpuset cgroup, you may need to add \"cgroup_enable=cpuset\" to your linux cmdline (/boot/cmdline.txt on a Raspberry Pi)" )
}
if ! strings . Contains ( string ( cgroups ) , "memory" ) {
msg := "ailed to find memory cgroup, you may need to add \"cgroup_memory=1 cgroup_enable=memory\" to your linux cmdline (/boot/cmdline.txt on a Raspberry Pi)"
logrus . Error ( "F" + msg )
return errors . New ( "f" + msg )
}
return nil
}
2019-10-15 21:17:26 +00:00
2020-02-11 23:27:43 +00:00
func configureNode ( ctx context . Context , agentConfig * daemonconfig . Agent , nodes v1 . NodeInterface ) error {
2020-05-05 21:51:39 +00:00
count := 0
2019-10-15 21:17:26 +00:00
for {
2020-03-26 21:08:47 +00:00
node , err := nodes . Get ( ctx , agentConfig . NodeName , metav1 . GetOptions { } )
2019-10-15 21:17:26 +00:00
if err != nil {
2020-05-05 21:51:39 +00:00
if count % 30 == 0 {
logrus . Infof ( "Waiting for kubelet to be ready on node %s: %v" , agentConfig . NodeName , err )
}
count ++
2019-10-15 21:17:26 +00:00
time . Sleep ( 1 * time . Second )
continue
}
2019-10-27 05:53:25 +00:00
2019-12-09 22:54:56 +00:00
newLabels , updateMutables := updateMutableLabels ( agentConfig , node . Labels )
updateAddresses := ! agentConfig . DisableCCM
if updateAddresses {
newLabels , updateAddresses = updateAddressLabels ( agentConfig , newLabels )
}
2020-02-11 23:27:43 +00:00
// inject node config
updateNode , err := nodeconfig . SetNodeConfigAnnotations ( node )
if err != nil {
return err
}
2019-12-09 22:54:56 +00:00
if updateAddresses || updateMutables {
2019-10-27 05:53:25 +00:00
node . Labels = newLabels
2020-02-11 23:27:43 +00:00
updateNode = true
}
if updateNode {
2020-03-26 21:08:47 +00:00
if _ , err := nodes . Update ( ctx , node , metav1 . UpdateOptions { } ) ; err != nil {
2019-10-27 05:53:25 +00:00
logrus . Infof ( "Failed to update node %s: %v" , agentConfig . NodeName , err )
select {
case <- ctx . Done ( ) :
return ctx . Err ( )
case <- time . After ( time . Second ) :
continue
}
2019-10-15 21:17:26 +00:00
}
2019-12-09 22:54:56 +00:00
logrus . Infof ( "labels have been set successfully on node: %s" , agentConfig . NodeName )
2019-10-27 05:53:25 +00:00
} else {
2019-12-09 22:54:56 +00:00
logrus . Infof ( "labels have already set on node: %s" , agentConfig . NodeName )
2019-10-15 21:17:26 +00:00
}
2019-10-27 05:53:25 +00:00
break
2019-10-15 21:17:26 +00:00
}
2019-10-27 05:53:25 +00:00
return nil
2019-10-15 21:17:26 +00:00
}
2019-12-09 22:54:56 +00:00
func updateMutableLabels ( agentConfig * daemonconfig . Agent , nodeLabels map [ string ] string ) ( map [ string ] string , bool ) {
2019-10-27 05:53:25 +00:00
result := map [ string ] string { }
2019-12-09 22:54:56 +00:00
for _ , m := range agentConfig . NodeLabels {
var (
v string
p = strings . SplitN ( m , ` = ` , 2 )
k = p [ 0 ]
)
if len ( p ) > 1 {
v = p [ 1 ]
}
2019-10-27 05:53:25 +00:00
result [ k ] = v
2019-10-15 21:17:26 +00:00
}
2019-12-09 22:54:56 +00:00
result = labels . Merge ( nodeLabels , result )
return result , ! equality . Semantic . DeepEqual ( nodeLabels , result )
}
2019-10-27 05:53:25 +00:00
2019-12-09 22:54:56 +00:00
func updateAddressLabels ( agentConfig * daemonconfig . Agent , nodeLabels map [ string ] string ) ( map [ string ] string , bool ) {
result := map [ string ] string {
InternalIPLabel : agentConfig . NodeIP ,
HostnameLabel : agentConfig . NodeName ,
}
if agentConfig . NodeExternalIP != "" {
2019-10-27 05:53:25 +00:00
result [ ExternalIPLabel ] = agentConfig . NodeExternalIP
2019-10-15 21:17:26 +00:00
}
2019-10-27 05:53:25 +00:00
2019-12-09 22:54:56 +00:00
result = labels . Merge ( nodeLabels , result )
2019-10-27 05:53:25 +00:00
return result , ! equality . Semantic . DeepEqual ( nodeLabels , result )
2019-10-15 21:17:26 +00:00
}