Merge pull request #4681 from manuelbuil/ha-verify-engine

[k3s-engine] Verify new control plane nodes joining the cluster share the same config as cluster members

pkg/cluster/bootstrap.go

@@ -8,9 +8,11 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"path"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"time"
 
@@ -562,6 +564,10 @@ func (c *Cluster) bootstrap(ctx context.Context) error {
 
 	// bootstrap managed database via HTTPS
 	if c.runtime.HTTPBootstrap {
+		// Assuming we should just compare on managed databases
+		if err := c.compareConfig(); err != nil {
+			return err
+		}
 		return c.httpBootstrap(ctx)
 	}
 
@@ -577,3 +583,37 @@ func (c *Cluster) Snapshot(ctx context.Context, config *config.Control) error {
 	}
 	return c.managedDB.Snapshot(ctx, config)
 }
+
+// compareConfig verifies that the config of the joining control plane node coincides with the cluster's config
+func (c *Cluster) compareConfig() error {
+	agentClientAccessInfo, err := clientaccess.ParseAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "node")
+	if err != nil {
+		return err
+	}
+	serverConfig, err := agentClientAccessInfo.Get("/v1-" + version.Program + "/config")
+	if err != nil {
+		return err
+	}
+	clusterControl := &config.Control{}
+	if err := json.Unmarshal(serverConfig, clusterControl); err != nil {
+		return err
+	}
+
+	// We are saving IPs of ClusterIPRanges and ServiceIPRanges in 4-bytes representation but json decodes in 16-byte
+	ipsTo16Bytes(c.config.CriticalControlArgs.ClusterIPRanges)
+	ipsTo16Bytes(c.config.CriticalControlArgs.ServiceIPRanges)
+
+	if !reflect.DeepEqual(clusterControl.CriticalControlArgs, c.config.CriticalControlArgs) {
+		logrus.Debugf("This is the server CriticalControlArgs: %#v", clusterControl.CriticalControlArgs)
+		logrus.Debugf("This is the local CriticalControlArgs: %#v", c.config.CriticalControlArgs)
+		return errors.New("Unable to join cluster due to critical configuration value mismatch")
+	}
+	return nil
+}
+
+// ipsTo16Bytes makes sure the IPs in the []*net.IPNet slice are represented in 16-byte format
+func ipsTo16Bytes(mySlice []*net.IPNet) {
+	for _, ipNet := range mySlice {
+		ipNet.IP = ipNet.IP.To16()
+	}
+}

pkg/daemons/config/types.go

@@ -101,7 +101,28 @@ type Agent struct {
 	ProtectKernelDefaults   bool
 }
 
+// CriticalControlArgs contains parameters that all control plane nodes in HA must share
+type CriticalControlArgs struct {
+	ClusterDNSs           []net.IP
+	ClusterIPRanges       []*net.IPNet
+	ClusterDNS            net.IP
+	ClusterDomain         string
+	ClusterIPRange        *net.IPNet
+	DisableCCM            bool
+	DisableHelmController bool
+	DisableKubeProxy      bool
+	DisableNPC            bool
+	Disables              map[string]bool
+	DisableServiceLB      bool
+	FlannelBackend        string
+	NoCoreDNS             bool
+	ServiceIPRange        *net.IPNet
+	ServiceIPRanges       []*net.IPNet
+	Skips                 map[string]bool
+}
+
 type Control struct {
+	CriticalControlArgs
 	AdvertisePort int
 	AdvertiseIP   string
 	// The port which kubectl clients can access k8s
@@ -113,21 +134,15 @@ type Control struct {
 	APIServerBindAddress     string
 	AgentToken               string `json:"-"`
 	Token                    string `json:"-"`
-	ClusterIPRange           *net.IPNet
-	ClusterIPRanges          []*net.IPNet
-	ServiceIPRange           *net.IPNet
-	ServiceIPRanges          []*net.IPNet
 	ServiceNodePortRange     *utilnet.PortRange
-	ClusterDNS               net.IP
-	ClusterDNSs              []net.IP
-	ClusterDomain            string
-	NoCoreDNS                bool
 	KubeConfigOutput         string
 	KubeConfigMode           string
 	DataDir                  string
-	Skips                    map[string]bool
-	Disables                 map[string]bool
 	Datastore                endpoint.Config
+	DisableAPIServer         bool
+	DisableControllerManager bool
+	DisableETCD              bool
+	DisableScheduler         bool
 	ExtraAPIArgs             []string
 	ExtraControllerArgs      []string
 	ExtraCloudControllerArgs []string
@@ -135,18 +150,9 @@ type Control struct {
 	ExtraSchedulerAPIArgs    []string
 	NoLeaderElect            bool
 	JoinURL                  string
-	FlannelBackend           string
 	IPSECPSK                 string
 	DefaultLocalStoragePath  string
 	SystemDefaultRegistry    string
-	DisableCCM               bool
-	DisableNPC               bool
-	DisableHelmController    bool
-	DisableKubeProxy         bool
-	DisableAPIServer         bool
-	DisableControllerManager bool
-	DisableScheduler         bool
-	DisableETCD              bool
 	ClusterInit              bool
 	ClusterReset             bool
 	ClusterResetRestorePath  string

pkg/etcd/etcd_test.go

@@ -29,23 +29,26 @@ func mustGetAddress() string {
 func generateTestConfig() *config.Control {
 	agentReady := make(chan struct{})
 	close(agentReady)
+	criticalControlArgs := config.CriticalControlArgs{
+		ClusterDomain:  "cluster.local",
+		ClusterDNS:     net.ParseIP("10.43.0.10"),
+		ClusterIPRange: testutil.ClusterIPNet(),
+		FlannelBackend: "vxlan",
+		ServiceIPRange: testutil.ServiceIPNet(),
+	}
 	return &config.Control{
 		Runtime:               &config.ControlRuntime{AgentReady: agentReady},
 		HTTPSPort:             6443,
 		SupervisorPort:        6443,
 		AdvertisePort:         6443,
-		ClusterDomain:         "cluster.local",
-		ClusterDNS:            net.ParseIP("10.43.0.10"),
-		ClusterIPRange:        testutil.ClusterIPNet(),
 		DataDir:               "/tmp/k3s/", // Different than the default value
-		FlannelBackend:        "vxlan",
 		EtcdSnapshotName:      "etcd-snapshot",
 		EtcdSnapshotCron:      "0 */12 * * *",
 		EtcdSnapshotRetention: 5,
 		EtcdS3Endpoint:        "s3.amazonaws.com",
 		EtcdS3Region:          "us-east-1",
 		SANs:                  []string{"127.0.0.1"},
-		ServiceIPRange:        testutil.ServiceIPNet(),
+		CriticalControlArgs:   criticalControlArgs,
 	}
 }