Merge pull request #1383 from galal-hussein/node_configuration_injection

Inject node config on startup

pkg/agent/run.go

@@ -23,6 +23,7 @@ import (
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/agent"
 	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/nodeconfig"
 	"github.com/rancher/k3s/pkg/rootless"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -71,7 +72,7 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
 		}
 	}
 
-	if err := syncLabels(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil {
+	if err := configureNode(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil {
 		return err
 	}
 
@@ -155,7 +156,7 @@ func validate() error {
 	return nil
 }
 
-func syncLabels(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.NodeInterface) error {
+func configureNode(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.NodeInterface) error {
 	for {
 		node, err := nodes.Get(agentConfig.NodeName, metav1.GetOptions{})
 		if err != nil {
@@ -171,8 +172,16 @@ func syncLabels(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.N
 			newLabels, updateAddresses = updateAddressLabels(agentConfig, newLabels)
 		}
 
+		// inject node config
+		updateNode, err := nodeconfig.SetNodeConfigAnnotations(node)
+		if err != nil {
+			return err
+		}
 		if updateAddresses || updateMutables {
 			node.Labels = newLabels
+			updateNode = true
+		}
+		if updateNode {
 			if _, err := nodes.Update(node); err != nil {
 				logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
 				select {

pkg/nodeconfig/nodeconfig.go

@@ -0,0 +1,111 @@
+package nodeconfig
+
+import (
+	"crypto/sha256"
+	"encoding/base32"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	NodeArgsAnnotation       = "k3s.io/node-args"
+	NodeEnvAnnotation        = "k3s.io/node-env"
+	NodeConfigHashAnnotation = "k3s.io/node-config-hash"
+)
+
+func getNodeArgs() (string, error) {
+	nodeArgsList := []string{}
+	for _, arg := range os.Args[1:] {
+		if strings.Contains(arg, "=") {
+			parsedArg := strings.SplitN(arg, "=", 2)
+			nodeArgsList = append(nodeArgsList, parsedArg...)
+			continue
+		}
+		nodeArgsList = append(nodeArgsList, arg)
+	}
+	for i, arg := range nodeArgsList {
+		if isSecret(arg) {
+			if i+1 < len(nodeArgsList) {
+				nodeArgsList[i+1] = ""
+			}
+		}
+	}
+	nodeArgs, err := json.Marshal(nodeArgsList)
+	if err != nil {
+		return "", errors.Wrap(err, "Failed to retrieve argument list for node")
+	}
+	return string(nodeArgs), nil
+}
+
+func getNodeEnv() (string, error) {
+	k3sEnv := make(map[string]string)
+	for _, v := range os.Environ() {
+		keyValue := strings.SplitN(v, "=", 2)
+		if strings.HasPrefix(keyValue[0], "K3S_") {
+			k3sEnv[keyValue[0]] = keyValue[1]
+		}
+	}
+	for key := range k3sEnv {
+		if isSecret(key) {
+			k3sEnv[key] = ""
+		}
+	}
+	k3sEnvJSON, err := json.Marshal(k3sEnv)
+	if err != nil {
+		return "", errors.Wrap(err, "Failed to retrieve environment map for node")
+	}
+	return string(k3sEnvJSON), nil
+}
+
+func SetNodeConfigAnnotations(node *corev1.Node) (bool, error) {
+	nodeArgs, err := getNodeArgs()
+	if err != nil {
+		return false, err
+	}
+	nodeEnv, err := getNodeEnv()
+	if err != nil {
+		return false, err
+	}
+	h := sha256.New()
+	_, err = h.Write([]byte(nodeArgs + nodeEnv))
+	if err != nil {
+		return false, fmt.Errorf("Failed to hash the node config: %v", err)
+	}
+	if node.Annotations == nil {
+		node.Annotations = make(map[string]string)
+	}
+	configHash := h.Sum(nil)
+	encoded := base32.StdEncoding.EncodeToString(configHash[:])
+	if node.Annotations[NodeConfigHashAnnotation] == encoded {
+		return false, nil
+	}
+	node.Annotations[NodeEnvAnnotation] = nodeEnv
+	node.Annotations[NodeArgsAnnotation] = nodeArgs
+	node.Annotations[NodeConfigHashAnnotation] = encoded
+	return true, nil
+}
+
+func isSecret(key string) bool {
+	secretData := []string{
+		"K3S_TOKEN",
+		"K3S_DATASTORE_ENDPOINT",
+		"K3S_AGENT_TOKEN",
+		"K3S_CLUSTER_SECRET",
+		"--token",
+		"-t",
+		"--agent-token",
+		"--datastore-endpoint",
+		"--cluster-secret",
+	}
+	for _, secret := range secretData {
+		if key == secret {
+			return true
+		}
+	}
+	return false
+}

pkg/nodeconfig/nodeconfig_test.go

@@ -0,0 +1,86 @@
+package nodeconfig
+
+import (
+	"os"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var FakeNodeWithNoAnnotation = &corev1.Node{
+	TypeMeta: metav1.TypeMeta{
+		Kind:       "Node",
+		APIVersion: "v1",
+	},
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "fakeNode-no-annotation",
+	},
+}
+
+var FakeNodeWithAnnotation = &corev1.Node{
+	TypeMeta: metav1.TypeMeta{
+		Kind:       "Node",
+		APIVersion: "v1",
+	},
+	ObjectMeta: metav1.ObjectMeta{
+		Name: "fakeNode-with-annotation",
+		Annotations: map[string]string{
+			NodeArgsAnnotation:       `["server","--no-flannel"]`,
+			NodeEnvAnnotation:        `{"K3S_NODE_NAME":"fakeNode-with-annotation"}`,
+			NodeConfigHashAnnotation: "LNQOAOIMOQIBRMEMACW7LYHXUNPZADF6RFGOSPIHJCOS47UVUJAA====",
+		},
+	},
+}
+
+func assertEqual(t *testing.T, a interface{}, b interface{}) {
+	if a != b {
+		t.Fatalf("[ %v != %v ]", a, b)
+	}
+}
+
+func TestSetEmptyNodeConfigAnnotations(t *testing.T) {
+	os.Args = []string{"k3s", "server", "--no-flannel"}
+	os.Setenv("K3S_NODE_NAME", "fakeNode-no-annotation")
+	nodeUpdated, err := SetNodeConfigAnnotations(FakeNodeWithNoAnnotation)
+	if err != nil {
+		t.Fatalf("Failed to set node config annotation: %v", err)
+	}
+	assertEqual(t, true, nodeUpdated)
+
+	expectedArgs := `["server","--no-flannel"]`
+	actualArgs := FakeNodeWithNoAnnotation.Annotations[NodeArgsAnnotation]
+	assertEqual(t, expectedArgs, actualArgs)
+
+	expectedEnv := `{"K3S_NODE_NAME":"fakeNode-no-annotation"}`
+	actualEnv := FakeNodeWithNoAnnotation.Annotations[NodeEnvAnnotation]
+	assertEqual(t, expectedEnv, actualEnv)
+
+	expectedHash := "MROOIJGRXUZ53BM74K76TZLRXQOLNNBNJBJOY7JJ22EAEUIBW7YA===="
+	actualHash := FakeNodeWithNoAnnotation.Annotations[NodeConfigHashAnnotation]
+	assertEqual(t, expectedHash, actualHash)
+}
+
+func TestSetExistingNodeConfigAnnotations(t *testing.T) {
+	// adding same config
+	os.Args = []string{"k3s", "server", "--no-flannel"}
+	os.Setenv("K3S_NODE_NAME", "fakeNode-with-annotation")
+	nodeUpdated, err := SetNodeConfigAnnotations(FakeNodeWithAnnotation)
+	if err != nil {
+		t.Fatalf("Failed to set node config annotation: %v", err)
+	}
+	assertEqual(t, false, nodeUpdated)
+}
+
+func TestSetArgsWithEqual(t *testing.T) {
+	os.Args = []string{"k3s", "server", "--no-flannel", "--write-kubeconfig-mode=777"}
+	os.Setenv("K3S_NODE_NAME", "fakeNode-with-no-annotation")
+	nodeUpdated, err := SetNodeConfigAnnotations(FakeNodeWithNoAnnotation)
+	if err != nil {
+		t.Fatalf("Failed to set node config annotation: %v", err)
+	}
+	assertEqual(t, true, nodeUpdated)
+	expectedArgs := `["server","--no-flannel","--write-kubeconfig-mode","777"]`
+	actualArgs := FakeNodeWithNoAnnotation.Annotations[NodeArgsAnnotation]
+	assertEqual(t, expectedArgs, actualArgs)
+}