From 13c633da126870ae487b78e050dfa5587c665764 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 4 Nov 2022 10:35:29 -0700 Subject: [PATCH] Add Secrets Encryption to CriticalArgs (#6409) * Add EncryptSecrets to Critical Control Args * use deep comparison to extract differences Signed-off-by: Derek Nola Signed-off-by: Derek Nola --- go.mod | 1 + package/rpm/install.sh | 12 +++---- pkg/cluster/bootstrap.go | 17 +++++++--- pkg/daemons/config/types.go | 34 ++++++++++---------- tests/perf/server/files/server_userdata.tmpl | 4 +-- 5 files changed, 39 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 60fe50840a..aef1f6f48f 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( github.com/flannel-io/flannel v0.20.1 github.com/go-bindata/go-bindata v3.1.2+incompatible github.com/go-sql-driver/mysql v1.6.0 + github.com/go-test/deep v1.0.7 github.com/google/cadvisor v0.45.0 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 diff --git a/package/rpm/install.sh b/package/rpm/install.sh index cc713c5463..26cb304aca 100755 --- a/package/rpm/install.sh +++ b/package/rpm/install.sh @@ -8,7 +8,7 @@ set -e # # Example: # Installing a server without traefik: -# curl ... | INSTALL_K3S_EXEC="--no-deploy=traefik" sh - +# curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh - # Installing an agent to point at a server: # curl ... | K3S_TOKEN=xxx K3S_URL=https://server-url:6443 sh - # @@ -66,11 +66,11 @@ set -e # of EXEC and script args ($@). # # The following commands result in the same behavior: -# curl ... | INSTALL_K3S_EXEC="--no-deploy=traefik" sh -s - -# curl ... | INSTALL_K3S_EXEC="server --no-deploy=traefik" sh -s - -# curl ... | INSTALL_K3S_EXEC="server" sh -s - --no-deploy=traefik -# curl ... | sh -s - server --no-deploy=traefik -# curl ... | sh -s - --no-deploy=traefik +# curl ... | INSTALL_K3S_EXEC="--disable=traefik" sh -s - +# curl ... | INSTALL_K3S_EXEC="server --disable=traefik" sh -s - +# curl ... | INSTALL_K3S_EXEC="server" sh -s - --disable=traefik +# curl ... | sh -s - server --disable=traefik +# curl ... | sh -s - --disable=traefik # # - INSTALL_K3S_NAME # Name of systemd service to create, will default from the k3s exec command diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 86fdc2b7fa..79c436603c 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/go-test/deep" "github.com/k3s-io/k3s/pkg/bootstrap" "github.com/k3s-io/k3s/pkg/clientaccess" "github.com/k3s-io/k3s/pkg/daemons/config" @@ -475,10 +476,18 @@ func (c *Cluster) compareConfig() error { clusterControl.CriticalControlArgs.EgressSelectorMode = c.config.CriticalControlArgs.EgressSelectorMode } - if !reflect.DeepEqual(clusterControl.CriticalControlArgs, c.config.CriticalControlArgs) { - logrus.Debugf("This is the server CriticalControlArgs: %#v", clusterControl.CriticalControlArgs) - logrus.Debugf("This is the local CriticalControlArgs: %#v", c.config.CriticalControlArgs) - return errors.New("critical configuration value mismatch") + if diff := deep.Equal(c.config.CriticalControlArgs, clusterControl.CriticalControlArgs); diff != nil { + rc := reflect.ValueOf(clusterControl.CriticalControlArgs).Type() + for _, d := range diff { + field := strings.Split(d, ":")[0] + v, _ := rc.FieldByName(field) + if cliTag, found := v.Tag.Lookup("cli"); found { + logrus.Warnf("critical configuration mismatched: %s", cliTag) + } else { + logrus.Warnf("critical configuration mismatched: %s", field) + } + } + return errors.New("critical configuration value mismatch between servers") } return nil } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 1a7965acd6..913e444486 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -126,23 +126,24 @@ type Agent struct { } // CriticalControlArgs contains parameters that all control plane nodes in HA must share +// The cli tag is used to provide better error information to the user on mismatch type CriticalControlArgs struct { - ClusterDNSs []net.IP - ClusterIPRanges []*net.IPNet - ClusterDNS net.IP - ClusterDomain string - ClusterIPRange *net.IPNet - DisableCCM bool - DisableHelmController bool - DisableNPC bool - DisableServiceLB bool - FlannelBackend string - FlannelIPv6Masq bool - FlannelExternalIP bool - EgressSelectorMode string - NoCoreDNS bool - ServiceIPRange *net.IPNet - ServiceIPRanges []*net.IPNet + ClusterDNSs []net.IP `cli:"cluster-dns"` + ClusterIPRanges []*net.IPNet `cli:"cluster-cidr"` + ClusterDNS net.IP `cli:"cluster-dns"` + ClusterDomain string `cli:"cluster-domain"` + ClusterIPRange *net.IPNet `cli:"cluster-cidr"` + DisableCCM bool `cli:"disable-cloud-controller"` + DisableHelmController bool `cli:"disable-helm-controller"` + DisableNPC bool `cli:"disable-network-policy"` + DisableServiceLB bool `cli:"disable-service-lb"` + EncryptSecrets bool `cli:"secrets-encryption"` + FlannelBackend string `cli:"flannel-backend"` + FlannelIPv6Masq bool `cli:"flannel-ipv6-masq"` + FlannelExternalIP bool `cli:"flannel-external-ip"` + EgressSelectorMode string `cli:"egress-selector-mode"` + ServiceIPRange *net.IPNet `cli:"service-cidr"` + ServiceIPRanges []*net.IPNet `cli:"service-cidr"` } type Control struct { @@ -187,7 +188,6 @@ type Control struct { ClusterInit bool ClusterReset bool ClusterResetRestorePath string - EncryptSecrets bool EncryptForce bool EncryptSkip bool TLSMinVersion uint16 diff --git a/tests/perf/server/files/server_userdata.tmpl b/tests/perf/server/files/server_userdata.tmpl index d412118b57..3b4e6d5501 100644 --- a/tests/perf/server/files/server_userdata.tmpl +++ b/tests/perf/server/files/server_userdata.tmpl @@ -28,11 +28,11 @@ write_files: if [ ${db_engine} == "embedded-etcd" ]; then curl -sfL https://get.k3s.io | K3S_CLUSTER_SECRET="${k3s_cluster_secret}" \ INSTALL_K3S_VERSION="${install_k3s_version}" \ - INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if master_index != 0 } --server https://${lb_address}:6443 %{ else } --cluster-init %{ endif }" sh - + INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --disable traefik --disable servicelb --tls-san ${lb_address} %{ if master_index != 0 } --server https://${lb_address}:6443 %{ else } --cluster-init %{ endif }" sh - else curl -sfL https://get.k3s.io | K3S_CLUSTER_SECRET="${k3s_cluster_secret}" \ INSTALL_K3S_VERSION="${install_k3s_version}" \ - INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --no-deploy traefik --no-deploy servicelb --tls-san ${lb_address} %{ if use_ha == "true" } --datastore-endpoint=$STORAGE_ENDPOINT %{ endif }" sh - + INSTALL_K3S_EXEC="${k3s_server_args} --cluster-cidr=10.0.0.0/8 --disable traefik --disable servicelb --tls-san ${lb_address} %{ if use_ha == "true" } --datastore-endpoint=$STORAGE_ENDPOINT %{ endif }" sh - fi if [ $? -eq 0 ]; then break