Add Secrets Encryption to CriticalArgs (#6409)

* Add EncryptSecrets to Critical Control Args
* use deep comparison to extract differences

Signed-off-by: Derek Nola <derek.nola@suse.com>

Signed-off-by: Derek Nola <derek.nola@suse.com>

go.mod

@@ -80,6 +80,7 @@ require (
 	github.com/flannel-io/flannel v0.20.1
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-sql-driver/mysql v1.6.0
+	github.com/go-test/deep v1.0.7
 	github.com/google/cadvisor v0.45.0
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0

package/rpm/install.sh

@@ -8,7 +8,7 @@ set -e
 #
 # Example:
 #   Installing a server without traefik:
-#     curl ... | INSTALL_K3S_EXEC="--no-deploy=traefik" sh -
+#     curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh -
 #   Installing an agent to point at a server:
 #     curl ... | K3S_TOKEN=xxx K3S_URL=https://server-url:6443 sh -
 #
@@ -66,11 +66,11 @@ set -e
 #     of EXEC and script args ($@).
 #
 #     The following commands result in the same behavior:
-#       curl ... | INSTALL_K3S_EXEC="--no-deploy=traefik" sh -s -
-#       curl ... | INSTALL_K3S_EXEC="server --no-deploy=traefik" sh -s -
-#       curl ... | INSTALL_K3S_EXEC="server" sh -s - --no-deploy=traefik
-#       curl ... | sh -s - server --no-deploy=traefik
-#       curl ... | sh -s - --no-deploy=traefik
+#       curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh -s -
+#       curl ... | INSTALL_K3S_EXEC="server --disable=traefik" sh -s -
+#       curl ... | INSTALL_K3S_EXEC="server" sh -s - --disable=traefik
+#       curl ... | sh -s - server --disable=traefik
+#       curl ... | sh -s - --disable=traefik
 #
 #   - INSTALL_K3S_NAME
 #     Name of systemd service to create, will default from the k3s exec command

pkg/cluster/bootstrap.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-test/deep"
 	"github.com/k3s-io/k3s/pkg/bootstrap"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
@@ -475,10 +476,18 @@ func (c *Cluster) compareConfig() error {
 		clusterControl.CriticalControlArgs.EgressSelectorMode = c.config.CriticalControlArgs.EgressSelectorMode
 	}
 
-	if !reflect.DeepEqual(clusterControl.CriticalControlArgs, c.config.CriticalControlArgs) {
-		logrus.Debugf("This is the server CriticalControlArgs: %#v", clusterControl.CriticalControlArgs)
-		logrus.Debugf("This is the local CriticalControlArgs: %#v", c.config.CriticalControlArgs)
-		return errors.New("critical configuration value mismatch")
+	if diff := deep.Equal(c.config.CriticalControlArgs, clusterControl.CriticalControlArgs); diff != nil {
+		rc := reflect.ValueOf(clusterControl.CriticalControlArgs).Type()
+		for _, d := range diff {
+			field := strings.Split(d, ":")[0]
+			v, _ := rc.FieldByName(field)
+			if cliTag, found := v.Tag.Lookup("cli"); found {
+				logrus.Warnf("critical configuration mismatched: %s", cliTag)
+			} else {
+				logrus.Warnf("critical configuration mismatched: %s", field)
+			}
+		}
+		return errors.New("critical configuration value mismatch between servers")
 	}
 	return nil
 }

pkg/daemons/config/types.go

@@ -126,23 +126,24 @@ type Agent struct {
 }
 
 // CriticalControlArgs contains parameters that all control plane nodes in HA must share
+// The cli tag is used to provide better error information to the user on mismatch
 type CriticalControlArgs struct {
-	ClusterDNSs           []net.IP
-	ClusterIPRanges       []*net.IPNet
-	ClusterDNS            net.IP
-	ClusterDomain         string
-	ClusterIPRange        *net.IPNet
-	DisableCCM            bool
-	DisableHelmController bool
-	DisableNPC            bool
-	DisableServiceLB      bool
-	FlannelBackend        string
-	FlannelIPv6Masq       bool
-	FlannelExternalIP     bool
-	EgressSelectorMode    string
-	NoCoreDNS             bool
-	ServiceIPRange        *net.IPNet
-	ServiceIPRanges       []*net.IPNet
+	ClusterDNSs           []net.IP     `cli:"cluster-dns"`
+	ClusterIPRanges       []*net.IPNet `cli:"cluster-cidr"`
+	ClusterDNS            net.IP       `cli:"cluster-dns"`
+	ClusterDomain         string       `cli:"cluster-domain"`
+	ClusterIPRange        *net.IPNet   `cli:"cluster-cidr"`
+	DisableCCM            bool         `cli:"disable-cloud-controller"`
+	DisableHelmController bool         `cli:"disable-helm-controller"`
+	DisableNPC            bool         `cli:"disable-network-policy"`
+	DisableServiceLB      bool         `cli:"disable-service-lb"`
+	EncryptSecrets        bool         `cli:"secrets-encryption"`
+	FlannelBackend        string       `cli:"flannel-backend"`
+	FlannelIPv6Masq       bool         `cli:"flannel-ipv6-masq"`
+	FlannelExternalIP     bool         `cli:"flannel-external-ip"`
+	EgressSelectorMode    string       `cli:"egress-selector-mode"`
+	ServiceIPRange        *net.IPNet   `cli:"service-cidr"`
+	ServiceIPRanges       []*net.IPNet `cli:"service-cidr"`
 }
 
 type Control struct {
@@ -187,7 +188,6 @@ type Control struct {
 	ClusterInit              bool
 	ClusterReset             bool
 	ClusterResetRestorePath  string
-	EncryptSecrets           bool
 	EncryptForce             bool
 	EncryptSkip              bool
 	TLSMinVersion            uint16

tests/perf/server/files/server_userdata.tmpl

@@ -28,11 +28,11 @@ write_files:
     if [ ${db_engine} == "embedded-etcd" ]; then
     curl -sfL https://get.k3s.io | K3S_CLUSTER_SECRET="${k3s_cluster_secret}" \
        INSTALL_K3S_VERSION="${install_k3s_version}" \
-       INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if master_index != 0 } --server https://${lb_address}:6443 %{ else } --cluster-init %{ endif }" sh -
+       INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --disable traefik --disable servicelb --tls-san ${lb_address} %{ if master_index != 0 } --server https://${lb_address}:6443 %{ else } --cluster-init %{ endif }" sh -
     else
       curl -sfL https://get.k3s.io | K3S_CLUSTER_SECRET="${k3s_cluster_secret}" \
          INSTALL_K3S_VERSION="${install_k3s_version}" \
-         INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if use_ha == "true" } --datastore-endpoint=$STORAGE_ENDPOINT %{ endif }" sh -
+         INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --disable traefik --disable servicelb --tls-san ${lb_address} %{ if use_ha == "true" } --datastore-endpoint=$STORAGE_ENDPOINT %{ endif }" sh -
       fi
       if [ $? -eq 0 ]; then
         break