2019-01-01 08:23:01 +00:00
package config
import (
2019-03-26 22:15:16 +00:00
"bufio"
2019-01-09 16:54:15 +00:00
"context"
2019-04-23 17:53:06 +00:00
cryptorand "crypto/rand"
2019-01-01 08:23:01 +00:00
"crypto/tls"
2019-04-23 17:53:06 +00:00
"encoding/hex"
2019-11-15 23:48:03 +00:00
"encoding/pem"
2019-01-01 08:23:01 +00:00
"fmt"
"io/ioutil"
2021-09-01 06:50:23 +00:00
"net"
2019-04-19 18:20:34 +00:00
"net/http"
2019-01-01 08:23:01 +00:00
"net/url"
"os"
2019-01-09 16:54:15 +00:00
"os/exec"
2019-01-01 08:23:01 +00:00
"path/filepath"
2019-03-26 22:15:16 +00:00
"regexp"
2019-03-05 18:28:26 +00:00
"strings"
2019-01-01 08:23:01 +00:00
"time"
2022-03-02 23:47:27 +00:00
"github.com/k3s-io/k3s/pkg/agent/proxy"
"github.com/k3s-io/k3s/pkg/cli/cmds"
"github.com/k3s-io/k3s/pkg/clientaccess"
"github.com/k3s-io/k3s/pkg/containerd"
"github.com/k3s-io/k3s/pkg/daemons/config"
"github.com/k3s-io/k3s/pkg/daemons/control/deps"
"github.com/k3s-io/k3s/pkg/util"
"github.com/k3s-io/k3s/pkg/version"
2019-01-01 08:23:01 +00:00
"github.com/pkg/errors"
2021-05-10 22:58:41 +00:00
"github.com/rancher/wrangler/pkg/slice"
2019-01-01 08:23:01 +00:00
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/json"
)
2020-04-27 16:41:57 +00:00
const (
DefaultPodManifestPath = "pod-manifests"
)
2021-07-27 21:56:05 +00:00
// Get returns a pointer to a completed Node configuration struct,
// containing a merging of the local CLI configuration with settings from the server.
// A call to this will bock until agent configuration is successfully returned by the
// server.
2020-04-28 22:00:30 +00:00
func Get ( ctx context . Context , agent cmds . Agent , proxy proxy . Proxy ) * config . Node {
2021-07-27 21:56:05 +00:00
ticker := time . NewTicker ( 5 * time . Second )
defer ticker . Stop ( )
RETRY :
2019-01-01 08:23:01 +00:00
for {
2021-03-08 22:10:00 +00:00
agentConfig , err := get ( ctx , & agent , proxy )
2019-01-01 08:23:01 +00:00
if err != nil {
2021-07-29 00:02:01 +00:00
logrus . Infof ( "Waiting to retrieve agent configuration; server is not ready: %v" , err )
2021-07-27 21:56:05 +00:00
for range ticker . C {
continue RETRY
2019-01-09 16:54:15 +00:00
}
2019-01-01 08:23:01 +00:00
}
return agentConfig
}
}
2021-07-27 21:56:05 +00:00
// KubeProxyDisabled returns a bool indicating whether or not kube-proxy has been disabled in the
// server configuration. The server may not have a complete view of cluster configuration until
// after all startup hooks have completed, so a call to this will block until after the server's
// readyz endpoint returns OK.
func KubeProxyDisabled ( ctx context . Context , node * config . Node , proxy proxy . Proxy ) bool {
ticker := time . NewTicker ( 5 * time . Second )
defer ticker . Stop ( )
RETRY :
for {
disabled , err := getKubeProxyDisabled ( ctx , node , proxy )
if err != nil {
2021-07-29 00:02:01 +00:00
logrus . Infof ( "Waiting to retrieve kube-proxy configuration; server is not ready: %v" , err )
2021-07-27 21:56:05 +00:00
for range ticker . C {
continue RETRY
}
}
return disabled
}
}
2019-04-19 18:20:34 +00:00
type HTTPRequester func ( u string , client * http . Client , username , password string ) ( [ ] byte , error )
func Request ( path string , info * clientaccess . Info , requester HTTPRequester ) ( [ ] byte , error ) {
2020-09-24 06:47:17 +00:00
u , err := url . Parse ( info . BaseURL )
2019-01-01 08:23:01 +00:00
if err != nil {
return nil , err
}
2019-04-19 18:20:34 +00:00
u . Path = path
2020-09-24 06:47:17 +00:00
return requester ( u . String ( ) , clientaccess . GetHTTPClient ( info . CACerts ) , info . Username , info . Password )
2019-04-19 18:20:34 +00:00
}
2021-09-01 06:50:23 +00:00
func getNodeNamedCrt ( nodeName string , nodeIPs [ ] net . IP , nodePasswordFile string ) HTTPRequester {
2019-04-19 18:20:34 +00:00
return func ( u string , client * http . Client , username , password string ) ( [ ] byte , error ) {
req , err := http . NewRequest ( http . MethodGet , u , nil )
if err != nil {
return nil , err
}
if username != "" {
req . SetBasicAuth ( username , password )
}
2020-05-05 22:09:04 +00:00
req . Header . Set ( version . Program + "-Node-Name" , nodeName )
2019-04-23 17:53:06 +00:00
nodePassword , err := ensureNodePassword ( nodePasswordFile )
if err != nil {
return nil , err
}
2020-05-05 22:09:04 +00:00
req . Header . Set ( version . Program + "-Node-Password" , nodePassword )
2021-04-21 22:56:20 +00:00
req . Header . Set ( version . Program + "-Node-IP" , util . JoinIPs ( nodeIPs ) )
2019-04-23 17:53:06 +00:00
2019-04-19 18:20:34 +00:00
resp , err := client . Do ( req )
if err != nil {
return nil , err
}
defer resp . Body . Close ( )
2019-05-29 18:53:51 +00:00
if resp . StatusCode == http . StatusForbidden {
2019-11-05 09:45:07 +00:00
return nil , fmt . Errorf ( "Node password rejected, duplicate hostname or contents of '%s' may not match server node-passwd entry, try enabling a unique node name with the --with-node-id flag" , nodePasswordFile )
2019-05-29 18:53:51 +00:00
}
2019-04-19 18:20:34 +00:00
if resp . StatusCode != http . StatusOK {
return nil , fmt . Errorf ( "%s: %s" , u , resp . Status )
}
return ioutil . ReadAll ( resp . Body )
}
}
2019-11-05 09:45:07 +00:00
func ensureNodeID ( nodeIDFile string ) ( string , error ) {
if _ , err := os . Stat ( nodeIDFile ) ; err == nil {
id , err := ioutil . ReadFile ( nodeIDFile )
return strings . TrimSpace ( string ( id ) ) , err
}
id := make ( [ ] byte , 4 , 4 )
_ , err := cryptorand . Read ( id )
if err != nil {
return "" , err
}
nodeID := hex . EncodeToString ( id )
return nodeID , ioutil . WriteFile ( nodeIDFile , [ ] byte ( nodeID + "\n" ) , 0644 )
}
2019-04-25 17:53:21 +00:00
func ensureNodePassword ( nodePasswordFile string ) ( string , error ) {
2019-04-23 17:53:06 +00:00
if _ , err := os . Stat ( nodePasswordFile ) ; err == nil {
2019-04-25 17:53:21 +00:00
password , err := ioutil . ReadFile ( nodePasswordFile )
return strings . TrimSpace ( string ( password ) ) , err
2019-04-23 17:53:06 +00:00
}
password := make ( [ ] byte , 16 , 16 )
_ , err := cryptorand . Read ( password )
if err != nil {
2019-04-25 17:53:21 +00:00
return "" , err
2019-04-23 17:53:06 +00:00
}
2019-04-25 17:53:21 +00:00
nodePassword := hex . EncodeToString ( password )
2019-05-29 18:53:51 +00:00
return nodePassword , ioutil . WriteFile ( nodePasswordFile , [ ] byte ( nodePassword + "\n" ) , 0600 )
2019-04-23 17:53:06 +00:00
}
2019-11-05 09:45:07 +00:00
func upgradeOldNodePasswordPath ( oldNodePasswordFile , newNodePasswordFile string ) {
password , err := ioutil . ReadFile ( oldNodePasswordFile )
if err != nil {
return
}
if err := ioutil . WriteFile ( newNodePasswordFile , password , 0600 ) ; err != nil {
logrus . Warnf ( "Unable to write password file: %v" , err )
return
}
if err := os . Remove ( oldNodePasswordFile ) ; err != nil {
logrus . Warnf ( "Unable to remove old password file: %v" , err )
return
}
}
2021-09-01 06:50:23 +00:00
func getServingCert ( nodeName string , nodeIPs [ ] net . IP , servingCertFile , servingKeyFile , nodePasswordFile string , info * clientaccess . Info ) ( * tls . Certificate , error ) {
2021-04-21 22:56:20 +00:00
servingCert , err := Request ( "/v1-" + version . Program + "/serving-kubelet.crt" , info , getNodeNamedCrt ( nodeName , nodeIPs , nodePasswordFile ) )
2019-01-01 08:23:01 +00:00
if err != nil {
return nil , err
}
2019-11-15 23:48:03 +00:00
servingCert , servingKey := splitCertKeyPEM ( servingCert )
2019-05-29 18:53:51 +00:00
if err := ioutil . WriteFile ( servingCertFile , servingCert , 0600 ) ; err != nil {
2019-04-19 18:20:34 +00:00
return nil , errors . Wrapf ( err , "failed to write node cert" )
}
2019-01-01 08:23:01 +00:00
2019-05-29 18:53:51 +00:00
if err := ioutil . WriteFile ( servingKeyFile , servingKey , 0600 ) ; err != nil {
2019-04-19 18:20:34 +00:00
return nil , errors . Wrapf ( err , "failed to write node key" )
}
2019-01-01 08:23:01 +00:00
2019-05-29 18:53:51 +00:00
cert , err := tls . X509KeyPair ( servingCert , servingKey )
2019-01-01 08:23:01 +00:00
if err != nil {
return nil , err
}
return & cert , nil
}
2019-11-15 23:48:03 +00:00
func getHostFile ( filename , keyFile string , info * clientaccess . Info ) error {
2019-05-29 18:53:51 +00:00
basename := filepath . Base ( filename )
2021-03-06 10:29:57 +00:00
fileBytes , err := info . Get ( "/v1-" + version . Program + "/" + basename )
2019-05-29 18:53:51 +00:00
if err != nil {
return err
}
2019-11-15 23:48:03 +00:00
if keyFile == "" {
if err := ioutil . WriteFile ( filename , fileBytes , 0600 ) ; err != nil {
return errors . Wrapf ( err , "failed to write cert %s" , filename )
}
} else {
fileBytes , keyBytes := splitCertKeyPEM ( fileBytes )
if err := ioutil . WriteFile ( filename , fileBytes , 0600 ) ; err != nil {
return errors . Wrapf ( err , "failed to write cert %s" , filename )
}
if err := ioutil . WriteFile ( keyFile , keyBytes , 0600 ) ; err != nil {
return errors . Wrapf ( err , "failed to write key %s" , filename )
}
2019-01-01 08:23:01 +00:00
}
2019-05-29 18:53:51 +00:00
return nil
}
2019-01-01 08:23:01 +00:00
2019-11-15 23:48:03 +00:00
func splitCertKeyPEM ( bytes [ ] byte ) ( certPem [ ] byte , keyPem [ ] byte ) {
for {
b , rest := pem . Decode ( bytes )
if b == nil {
break
}
bytes = rest
if strings . Contains ( b . Type , "PRIVATE KEY" ) {
keyPem = append ( keyPem , pem . EncodeToMemory ( b ) ... )
} else {
certPem = append ( certPem , pem . EncodeToMemory ( b ) ... )
}
}
return
}
2021-09-01 06:50:23 +00:00
func getNodeNamedHostFile ( filename , keyFile , nodeName string , nodeIPs [ ] net . IP , nodePasswordFile string , info * clientaccess . Info ) error {
2019-05-29 18:53:51 +00:00
basename := filepath . Base ( filename )
2021-04-21 22:56:20 +00:00
fileBytes , err := Request ( "/v1-" + version . Program + "/" + basename , info , getNodeNamedCrt ( nodeName , nodeIPs , nodePasswordFile ) )
2019-05-29 18:53:51 +00:00
if err != nil {
return err
}
2019-11-15 23:48:03 +00:00
fileBytes , keyBytes := splitCertKeyPEM ( fileBytes )
2019-05-29 18:53:51 +00:00
if err := ioutil . WriteFile ( filename , fileBytes , 0600 ) ; err != nil {
return errors . Wrapf ( err , "failed to write cert %s" , filename )
}
2019-11-15 23:48:03 +00:00
if err := ioutil . WriteFile ( keyFile , keyBytes , 0600 ) ; err != nil {
return errors . Wrapf ( err , "failed to write key %s" , filename )
}
2019-05-29 18:53:51 +00:00
return nil
2019-01-01 08:23:01 +00:00
}
2019-03-26 22:15:16 +00:00
func isValidResolvConf ( resolvConfFile string ) bool {
file , err := os . Open ( resolvConfFile )
if err != nil {
return false
}
defer file . Close ( )
nameserver := regexp . MustCompile ( ` ^nameserver\s+([^\s]*) ` )
scanner := bufio . NewScanner ( file )
for scanner . Scan ( ) {
ipMatch := nameserver . FindStringSubmatch ( scanner . Text ( ) )
if len ( ipMatch ) == 2 {
2021-09-01 06:50:23 +00:00
ip := net . ParseIP ( ipMatch [ 1 ] )
2019-03-26 22:15:16 +00:00
if ip == nil || ! ip . IsGlobalUnicast ( ) {
return false
}
}
}
if err := scanner . Err ( ) ; err != nil {
return false
}
return true
}
func locateOrGenerateResolvConf ( envInfo * cmds . Agent ) string {
if envInfo . ResolvConf != "" {
return envInfo . ResolvConf
}
resolvConfs := [ ] string { "/etc/resolv.conf" , "/run/systemd/resolve/resolv.conf" }
for _ , conf := range resolvConfs {
if isValidResolvConf ( conf ) {
return conf
}
}
2020-05-05 22:09:04 +00:00
tmpConf := filepath . Join ( os . TempDir ( ) , version . Program + "-resolv.conf" )
2019-03-26 22:15:16 +00:00
if err := ioutil . WriteFile ( tmpConf , [ ] byte ( "nameserver 8.8.8.8\n" ) , 0444 ) ; err != nil {
2020-09-21 16:56:03 +00:00
logrus . Errorf ( "Failed to write %s: %v" , tmpConf , err )
2019-03-26 22:15:16 +00:00
return ""
}
return tmpConf
}
2021-03-08 22:10:00 +00:00
func get ( ctx context . Context , envInfo * cmds . Agent , proxy proxy . Proxy ) ( * config . Node , error ) {
2019-01-09 16:54:15 +00:00
if envInfo . Debug {
logrus . SetLevel ( logrus . DebugLevel )
2019-01-01 08:23:01 +00:00
}
2020-04-28 22:00:30 +00:00
info , err := clientaccess . ParseAndValidateToken ( proxy . SupervisorURL ( ) , envInfo . Token )
2019-01-01 08:23:01 +00:00
if err != nil {
return nil , err
}
2020-04-28 22:00:30 +00:00
controlConfig , err := getConfig ( info )
2019-01-01 08:23:01 +00:00
if err != nil {
2021-04-21 22:56:20 +00:00
return nil , errors . Wrap ( err , "failed to retrieve configuration from server" )
2019-01-01 08:23:01 +00:00
}
2021-03-06 10:29:57 +00:00
// If the supervisor and externally-facing apiserver are not on the same port, tell the proxy where to find the apiserver.
2020-04-28 22:00:30 +00:00
if controlConfig . SupervisorPort != controlConfig . HTTPSPort {
2021-03-08 22:10:00 +00:00
if err := proxy . SetAPIServerPort ( ctx , controlConfig . HTTPSPort ) ; err != nil {
2020-04-28 22:00:30 +00:00
return nil , errors . Wrapf ( err , "failed to setup access to API Server port %d on at %s" , controlConfig . HTTPSPort , proxy . SupervisorURL ( ) )
}
2019-01-01 08:23:01 +00:00
}
2021-09-01 06:50:23 +00:00
var flannelIface * net . Interface
2019-05-29 18:53:51 +00:00
if ! envInfo . NoFlannel && len ( envInfo . FlannelIface ) > 0 {
2021-09-01 06:50:23 +00:00
flannelIface , err = net . InterfaceByName ( envInfo . FlannelIface )
2019-05-29 18:53:51 +00:00
if err != nil {
return nil , errors . Wrapf ( err , "unable to find interface" )
}
}
2020-11-10 06:15:56 +00:00
clientCAFile := filepath . Join ( envInfo . DataDir , "agent" , "client-ca.crt" )
2019-11-15 23:48:03 +00:00
if err := getHostFile ( clientCAFile , "" , info ) ; err != nil {
2019-01-01 08:23:01 +00:00
return nil , err
}
2020-11-10 06:15:56 +00:00
serverCAFile := filepath . Join ( envInfo . DataDir , "agent" , "server-ca.crt" )
2019-11-15 23:48:03 +00:00
if err := getHostFile ( serverCAFile , "" , info ) ; err != nil {
2019-01-01 08:23:01 +00:00
return nil , err
}
2020-11-10 06:15:56 +00:00
servingKubeletCert := filepath . Join ( envInfo . DataDir , "agent" , "serving-kubelet.crt" )
servingKubeletKey := filepath . Join ( envInfo . DataDir , "agent" , "serving-kubelet.key" )
2019-11-05 09:45:07 +00:00
nodePasswordRoot := "/"
if envInfo . Rootless {
2020-11-10 06:15:56 +00:00
nodePasswordRoot = filepath . Join ( envInfo . DataDir , "agent" )
2019-11-05 09:45:07 +00:00
}
nodeConfigPath := filepath . Join ( nodePasswordRoot , "etc" , "rancher" , "node" )
if err := os . MkdirAll ( nodeConfigPath , 0755 ) ; err != nil {
return nil , err
}
2020-11-10 06:15:56 +00:00
oldNodePasswordFile := filepath . Join ( envInfo . DataDir , "agent" , "node-password.txt" )
2019-11-05 09:45:07 +00:00
newNodePasswordFile := filepath . Join ( nodeConfigPath , "password" )
upgradeOldNodePasswordPath ( oldNodePasswordFile , newNodePasswordFile )
2021-09-01 06:50:23 +00:00
nodeName , nodeIPs , err := util . GetHostnameAndIPs ( envInfo . NodeName , envInfo . NodeIP )
2019-11-05 09:45:07 +00:00
if err != nil {
return nil , err
}
2021-12-07 23:42:40 +00:00
nodeExternalIPs , err := util . ParseStringSliceToIPs ( envInfo . NodeExternalIP )
if err != nil {
return nil , fmt . Errorf ( "invalid node-external-ip: %w" , err )
}
2019-11-05 09:45:07 +00:00
if envInfo . WithNodeID {
nodeID , err := ensureNodeID ( filepath . Join ( nodeConfigPath , "id" ) )
if err != nil {
return nil , err
}
nodeName += "-" + nodeID
}
2020-12-22 22:17:00 +00:00
os . Setenv ( "NODE_NAME" , nodeName )
2021-12-07 23:42:40 +00:00
nodeExternalAndInternalIPs := append ( nodeIPs , nodeExternalIPs ... )
servingCert , err := getServingCert ( nodeName , nodeExternalAndInternalIPs , servingKubeletCert , servingKubeletKey , newNodePasswordFile , info )
2019-01-01 08:23:01 +00:00
if err != nil {
return nil , err
}
2020-11-10 06:15:56 +00:00
clientKubeletCert := filepath . Join ( envInfo . DataDir , "agent" , "client-kubelet.crt" )
clientKubeletKey := filepath . Join ( envInfo . DataDir , "agent" , "client-kubelet.key" )
2021-04-21 22:56:20 +00:00
if err := getNodeNamedHostFile ( clientKubeletCert , clientKubeletKey , nodeName , nodeIPs , newNodePasswordFile , info ) ; err != nil {
2019-05-29 18:53:51 +00:00
return nil , err
}
2020-11-10 06:15:56 +00:00
kubeconfigKubelet := filepath . Join ( envInfo . DataDir , "agent" , "kubelet.kubeconfig" )
2021-03-03 18:14:12 +00:00
if err := deps . KubeConfig ( kubeconfigKubelet , proxy . APIServerURL ( ) , serverCAFile , clientKubeletCert , clientKubeletKey ) ; err != nil {
2019-05-29 18:53:51 +00:00
return nil , err
}
2020-11-10 06:15:56 +00:00
clientKubeProxyCert := filepath . Join ( envInfo . DataDir , "agent" , "client-kube-proxy.crt" )
clientKubeProxyKey := filepath . Join ( envInfo . DataDir , "agent" , "client-kube-proxy.key" )
2019-11-15 23:48:03 +00:00
if err := getHostFile ( clientKubeProxyCert , clientKubeProxyKey , info ) ; err != nil {
2019-05-29 18:53:51 +00:00
return nil , err
}
2020-11-10 06:15:56 +00:00
kubeconfigKubeproxy := filepath . Join ( envInfo . DataDir , "agent" , "kubeproxy.kubeconfig" )
2021-03-03 18:14:12 +00:00
if err := deps . KubeConfig ( kubeconfigKubeproxy , proxy . APIServerURL ( ) , serverCAFile , clientKubeProxyCert , clientKubeProxyKey ) ; err != nil {
2019-05-29 18:53:51 +00:00
return nil , err
2019-03-19 23:28:43 +00:00
}
2020-11-10 06:15:56 +00:00
clientK3sControllerCert := filepath . Join ( envInfo . DataDir , "agent" , "client-" + version . Program + "-controller.crt" )
clientK3sControllerKey := filepath . Join ( envInfo . DataDir , "agent" , "client-" + version . Program + "-controller.key" )
2019-11-15 23:48:03 +00:00
if err := getHostFile ( clientK3sControllerCert , clientK3sControllerKey , info ) ; err != nil {
2019-10-27 05:53:25 +00:00
return nil , err
}
2020-11-10 06:15:56 +00:00
kubeconfigK3sController := filepath . Join ( envInfo . DataDir , "agent" , version . Program + "controller.kubeconfig" )
2021-03-03 18:14:12 +00:00
if err := deps . KubeConfig ( kubeconfigK3sController , proxy . APIServerURL ( ) , serverCAFile , clientK3sControllerCert , clientK3sControllerKey ) ; err != nil {
2019-10-27 05:53:25 +00:00
return nil , err
}
2019-01-09 16:54:15 +00:00
nodeConfig := & config . Node {
2019-03-04 06:29:06 +00:00
Docker : envInfo . Docker ,
2020-08-11 23:17:32 +00:00
SELinux : envInfo . EnableSELinux ,
2019-03-04 06:29:06 +00:00
ContainerRuntimeEndpoint : envInfo . ContainerRuntimeEndpoint ,
2019-09-03 23:41:54 +00:00
FlannelBackend : controlConfig . FlannelBackend ,
2022-01-14 15:54:55 +00:00
FlannelIPv6Masq : controlConfig . FlannelIPv6Masq ,
2021-02-12 15:35:57 +00:00
ServerHTTPSPort : controlConfig . HTTPSPort ,
2021-07-27 21:56:05 +00:00
Token : info . String ( ) ,
2019-01-09 16:54:15 +00:00
}
2019-03-19 23:28:43 +00:00
nodeConfig . FlannelIface = flannelIface
2020-11-10 06:15:56 +00:00
nodeConfig . Images = filepath . Join ( envInfo . DataDir , "agent" , "images" )
2019-01-09 16:54:15 +00:00
nodeConfig . AgentConfig . NodeName = nodeName
2019-11-05 09:45:07 +00:00
nodeConfig . AgentConfig . NodeConfigPath = nodeConfigPath
2019-05-29 18:53:51 +00:00
nodeConfig . AgentConfig . ServingKubeletCert = servingKubeletCert
nodeConfig . AgentConfig . ServingKubeletKey = servingKubeletKey
2019-01-09 16:54:15 +00:00
nodeConfig . AgentConfig . ClusterDNS = controlConfig . ClusterDNS
2019-04-12 06:06:35 +00:00
nodeConfig . AgentConfig . ClusterDomain = controlConfig . ClusterDomain
2019-03-26 22:15:16 +00:00
nodeConfig . AgentConfig . ResolvConf = locateOrGenerateResolvConf ( envInfo )
2019-05-29 18:53:51 +00:00
nodeConfig . AgentConfig . ClientCA = clientCAFile
nodeConfig . AgentConfig . KubeConfigKubelet = kubeconfigKubelet
nodeConfig . AgentConfig . KubeConfigKubeProxy = kubeconfigKubeproxy
2019-10-27 05:53:25 +00:00
nodeConfig . AgentConfig . KubeConfigK3sController = kubeconfigK3sController
2019-10-19 10:18:51 +00:00
if envInfo . Rootless {
2020-11-10 06:15:56 +00:00
nodeConfig . AgentConfig . RootDir = filepath . Join ( envInfo . DataDir , "agent" , "kubelet" )
2019-10-19 10:18:51 +00:00
}
2020-07-17 23:16:23 +00:00
nodeConfig . AgentConfig . Snapshotter = envInfo . Snapshotter
2019-09-03 23:41:54 +00:00
nodeConfig . AgentConfig . IPSECPSK = controlConfig . IPSECPSK
2020-11-10 06:15:56 +00:00
nodeConfig . AgentConfig . StrongSwanDir = filepath . Join ( envInfo . DataDir , "agent" , "strongswan" )
nodeConfig . Containerd . Config = filepath . Join ( envInfo . DataDir , "agent" , "etc" , "containerd" , "config.toml" )
nodeConfig . Containerd . Root = filepath . Join ( envInfo . DataDir , "agent" , "containerd" )
2021-01-11 15:22:14 +00:00
if ! nodeConfig . Docker && nodeConfig . ContainerRuntimeEndpoint == "" {
switch nodeConfig . AgentConfig . Snapshotter {
case "overlayfs" :
2021-06-01 19:29:46 +00:00
if err := containerd . OverlaySupported ( nodeConfig . Containerd . Root ) ; err != nil {
2021-01-11 15:22:14 +00:00
return nil , errors . Wrapf ( err , "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"" ,
nodeConfig . Containerd . Root )
}
case "fuse-overlayfs" :
2021-06-01 19:29:46 +00:00
if err := containerd . FuseoverlayfsSupported ( nodeConfig . Containerd . Root ) ; err != nil {
2021-01-11 15:22:14 +00:00
return nil , errors . Wrapf ( err , "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"" ,
nodeConfig . Containerd . Root )
}
2021-09-01 23:27:42 +00:00
case "stargz" :
if err := containerd . StargzSupported ( nodeConfig . Containerd . Root ) ; err != nil {
return nil , errors . Wrapf ( err , "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"" ,
nodeConfig . Containerd . Root )
}
nodeConfig . AgentConfig . ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
2020-11-25 02:50:24 +00:00
}
}
2020-11-10 06:15:56 +00:00
nodeConfig . Containerd . Opt = filepath . Join ( envInfo . DataDir , "agent" , "containerd" )
2019-02-08 04:12:49 +00:00
if ! envInfo . Debug {
2020-11-10 06:15:56 +00:00
nodeConfig . Containerd . Log = filepath . Join ( envInfo . DataDir , "agent" , "containerd" , "containerd.log" )
2019-02-08 04:12:49 +00:00
}
2021-06-10 19:27:00 +00:00
applyContainerdStateAndAddress ( nodeConfig )
2020-11-10 06:15:56 +00:00
nodeConfig . Containerd . Template = filepath . Join ( envInfo . DataDir , "agent" , "etc" , "containerd" , "config.toml.tmpl" )
2019-05-29 18:53:51 +00:00
nodeConfig . Certificate = servingCert
2019-09-03 23:41:54 +00:00
2021-04-21 22:56:20 +00:00
nodeConfig . AgentConfig . NodeIPs = nodeIPs
2021-11-10 19:23:05 +00:00
nodeIP , listenAddress , _ , err := util . GetFirstIP ( nodeIPs )
2021-04-21 22:56:20 +00:00
if err != nil {
2021-11-10 19:23:05 +00:00
return nil , errors . Wrap ( err , "cannot configure IPv4/IPv6 node-ip" )
2021-04-21 22:56:20 +00:00
}
nodeConfig . AgentConfig . NodeIP = nodeIP . String ( )
2021-11-10 19:23:05 +00:00
nodeConfig . AgentConfig . ListenAddress = listenAddress
2021-12-07 23:42:40 +00:00
nodeConfig . AgentConfig . NodeExternalIPs = nodeExternalIPs
2021-04-21 22:56:20 +00:00
// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
2021-11-10 19:23:05 +00:00
// unless only IPv6 address given
2021-04-21 22:56:20 +00:00
if len ( nodeConfig . AgentConfig . NodeExternalIPs ) > 0 {
2021-11-10 19:23:05 +00:00
nodeExternalIP , _ , _ , err := util . GetFirstIP ( nodeConfig . AgentConfig . NodeExternalIPs )
2021-04-21 22:56:20 +00:00
if err != nil {
2021-11-10 19:23:05 +00:00
return nil , errors . Wrap ( err , "cannot configure IPv4/IPv6 node-external-ip" )
2021-04-21 22:56:20 +00:00
}
nodeConfig . AgentConfig . NodeExternalIP = nodeExternalIP . String ( )
}
2019-09-03 23:41:54 +00:00
if nodeConfig . FlannelBackend == config . FlannelBackendNone {
nodeConfig . NoFlannel = true
} else {
nodeConfig . NoFlannel = envInfo . NoFlannel
}
2019-01-09 16:54:15 +00:00
if ! nodeConfig . NoFlannel {
2020-04-27 16:01:47 +00:00
hostLocal , err := exec . LookPath ( "host-local" )
if err != nil {
return nil , errors . Wrapf ( err , "failed to find host-local" )
}
2019-08-08 05:56:09 +00:00
if envInfo . FlannelConf == "" {
2021-11-20 19:12:03 +00:00
nodeConfig . FlannelConfFile = filepath . Join ( envInfo . DataDir , "agent" , "etc" , "flannel" , "net-conf.json" )
2019-08-08 05:56:09 +00:00
} else {
2021-11-20 19:12:03 +00:00
nodeConfig . FlannelConfFile = envInfo . FlannelConf
2019-08-08 05:56:09 +00:00
nodeConfig . FlannelConfOverride = true
}
2019-01-09 16:54:15 +00:00
nodeConfig . AgentConfig . CNIBinDir = filepath . Dir ( hostLocal )
2020-11-10 06:15:56 +00:00
nodeConfig . AgentConfig . CNIConfDir = filepath . Join ( envInfo . DataDir , "agent" , "etc" , "cni" , "net.d" )
2019-01-09 16:54:15 +00:00
}
2019-12-10 23:16:26 +00:00
2019-03-04 06:29:06 +00:00
if ! nodeConfig . Docker && nodeConfig . ContainerRuntimeEndpoint == "" {
2021-01-21 01:03:22 +00:00
nodeConfig . AgentConfig . RuntimeSocket = nodeConfig . Containerd . Address
2019-03-04 06:29:06 +00:00
} else {
2019-12-10 23:16:26 +00:00
nodeConfig . AgentConfig . RuntimeSocket = nodeConfig . ContainerRuntimeEndpoint
nodeConfig . AgentConfig . CNIPlugin = true
2019-01-09 16:54:15 +00:00
}
2019-12-10 23:16:26 +00:00
2019-01-09 16:54:15 +00:00
if controlConfig . ClusterIPRange != nil {
2021-04-21 22:56:20 +00:00
nodeConfig . AgentConfig . ClusterCIDR = controlConfig . ClusterIPRange
2021-09-01 06:50:23 +00:00
nodeConfig . AgentConfig . ClusterCIDRs = [ ] * net . IPNet { controlConfig . ClusterIPRange }
2021-04-21 22:56:20 +00:00
}
if len ( controlConfig . ClusterIPRanges ) > 0 {
nodeConfig . AgentConfig . ClusterCIDRs = controlConfig . ClusterIPRanges
2019-01-01 08:23:01 +00:00
}
2021-02-01 19:11:17 +00:00
if controlConfig . ServiceIPRange != nil {
2021-04-21 22:56:20 +00:00
nodeConfig . AgentConfig . ServiceCIDR = controlConfig . ServiceIPRange
2021-09-01 06:50:23 +00:00
nodeConfig . AgentConfig . ServiceCIDRs = [ ] * net . IPNet { controlConfig . ServiceIPRange }
2021-04-21 22:56:20 +00:00
}
if len ( controlConfig . ServiceIPRanges ) > 0 {
nodeConfig . AgentConfig . ServiceCIDRs = controlConfig . ServiceIPRanges
2021-02-01 19:11:17 +00:00
}
if controlConfig . ServiceNodePortRange != nil {
nodeConfig . AgentConfig . ServiceNodePortRange = * controlConfig . ServiceNodePortRange
}
2021-04-21 22:56:20 +00:00
if len ( controlConfig . ClusterDNSs ) == 0 {
2021-09-01 06:50:23 +00:00
nodeConfig . AgentConfig . ClusterDNSs = [ ] net . IP { controlConfig . ClusterDNS }
2021-04-21 22:56:20 +00:00
} else {
nodeConfig . AgentConfig . ClusterDNSs = controlConfig . ClusterDNSs
2021-03-01 12:12:25 +00:00
}
2021-05-10 22:58:41 +00:00
nodeConfig . AgentConfig . PauseImage = envInfo . PauseImage
nodeConfig . AgentConfig . AirgapExtraRegistry = envInfo . AirgapExtraRegistry
2021-05-12 23:50:08 +00:00
nodeConfig . AgentConfig . SystemDefaultRegistry = controlConfig . SystemDefaultRegistry
2021-05-10 22:58:41 +00:00
// Apply SystemDefaultRegistry to PauseImage and AirgapExtraRegistry
if controlConfig . SystemDefaultRegistry != "" {
2021-05-12 23:50:08 +00:00
if nodeConfig . AgentConfig . PauseImage != "" && ! strings . HasPrefix ( nodeConfig . AgentConfig . PauseImage , controlConfig . SystemDefaultRegistry ) {
2021-05-10 22:58:41 +00:00
nodeConfig . AgentConfig . PauseImage = controlConfig . SystemDefaultRegistry + "/" + nodeConfig . AgentConfig . PauseImage
}
if ! slice . ContainsString ( nodeConfig . AgentConfig . AirgapExtraRegistry , controlConfig . SystemDefaultRegistry ) {
nodeConfig . AgentConfig . AirgapExtraRegistry = append ( nodeConfig . AgentConfig . AirgapExtraRegistry , controlConfig . SystemDefaultRegistry )
}
}
2019-04-05 00:43:00 +00:00
nodeConfig . AgentConfig . ExtraKubeletArgs = envInfo . ExtraKubeletArgs
nodeConfig . AgentConfig . ExtraKubeProxyArgs = envInfo . ExtraKubeProxyArgs
2019-05-07 23:47:07 +00:00
nodeConfig . AgentConfig . NodeTaints = envInfo . Taints
nodeConfig . AgentConfig . NodeLabels = envInfo . Labels
2021-05-10 22:58:41 +00:00
nodeConfig . AgentConfig . ImageCredProvBinDir = envInfo . ImageCredProvBinDir
nodeConfig . AgentConfig . ImageCredProvConfig = envInfo . ImageCredProvConfig
2019-10-07 23:04:58 +00:00
nodeConfig . AgentConfig . PrivateRegistry = envInfo . PrivateRegistry
2019-10-15 21:17:26 +00:00
nodeConfig . AgentConfig . DisableCCM = controlConfig . DisableCCM
2019-10-17 21:46:15 +00:00
nodeConfig . AgentConfig . DisableNPC = controlConfig . DisableNPC
2019-10-19 10:18:51 +00:00
nodeConfig . AgentConfig . Rootless = envInfo . Rootless
2020-11-10 06:15:56 +00:00
nodeConfig . AgentConfig . PodManifests = filepath . Join ( envInfo . DataDir , "agent" , DefaultPodManifestPath )
2020-07-20 23:31:56 +00:00
nodeConfig . AgentConfig . ProtectKernelDefaults = envInfo . ProtectKernelDefaults
2021-11-09 15:44:34 +00:00
nodeConfig . AgentConfig . DisableServiceLB = envInfo . DisableServiceLB
2019-05-07 23:47:07 +00:00
2021-04-21 22:56:20 +00:00
if err := validateNetworkConfig ( nodeConfig ) ; err != nil {
return nil , err
}
2019-01-01 08:23:01 +00:00
return nodeConfig , nil
}
2021-07-27 21:56:05 +00:00
// getKubeProxyDisabled attempts to return the DisableKubeProxy setting from the server configuration data.
// It first checks the server readyz endpoint, to ensure that the configuration has stabilized before use.
func getKubeProxyDisabled ( ctx context . Context , node * config . Node , proxy proxy . Proxy ) ( bool , error ) {
info , err := clientaccess . ParseAndValidateToken ( proxy . SupervisorURL ( ) , node . Token )
if err != nil {
return false , err
}
// 500 error indicates that the health check has failed; other errors (for example 401 Unauthorized)
// indicate that the server is down-level and doesn't support readyz, so we should just use whatever
// the server has for us.
if err := getReadyz ( info ) ; err != nil && strings . HasSuffix ( err . Error ( ) , "500 Internal Server Error" ) {
return false , err
}
controlConfig , err := getConfig ( info )
if err != nil {
return false , errors . Wrap ( err , "failed to retrieve configuration from server" )
}
return controlConfig . DisableKubeProxy , nil
}
// getConfig returns server configuration data. Note that this may be mutated during system startup; anything that needs
// to ensure stable system state should check the readyz endpoint first. This is required because RKE2 starts up the
// kubelet early, before the apiserver is available.
2019-01-01 08:23:01 +00:00
func getConfig ( info * clientaccess . Info ) ( * config . Control , error ) {
2021-03-06 10:29:57 +00:00
data , err := info . Get ( "/v1-" + version . Program + "/config" )
2019-01-01 08:23:01 +00:00
if err != nil {
return nil , err
}
controlControl := & config . Control { }
return controlControl , json . Unmarshal ( data , controlControl )
}
2021-04-21 22:56:20 +00:00
2021-07-27 21:56:05 +00:00
// getReadyz returns nil if the server is ready, or an error if not.
func getReadyz ( info * clientaccess . Info ) error {
_ , err := info . Get ( "/v1-" + version . Program + "/readyz" )
return err
}
2021-04-21 22:56:20 +00:00
// validateNetworkConfig ensures that the network configuration values provided by the server make sense.
func validateNetworkConfig ( nodeConfig * config . Node ) error {
// Old versions of the server do not send enough information to correctly start the NPC. Users
// need to upgrade the server to at least the same version as the agent, or disable the NPC
// cluster-wide.
if nodeConfig . AgentConfig . DisableNPC == false && ( nodeConfig . AgentConfig . ServiceCIDR == nil || nodeConfig . AgentConfig . ServiceNodePortRange . Size == 0 ) {
return fmt . Errorf ( "incompatible down-level server detected; servers must be upgraded to at least %s, or restarted with --disable-network-policy" , version . Version )
}
return nil
}