From 50b358048bfb7a62ccc91bccf42d8653210a24f0 Mon Sep 17 00:00:00 2001
From: Brian Downs <brian.downs@gmail.com>
Date: Thu, 9 Dec 2021 17:54:43 -0700
Subject: [PATCH] Resolve restore bootstrap (#4704) (#4717)

---
 pkg/bootstrap/bootstrap.go |  5 ++++-
 pkg/cluster/bootstrap.go   | 32 ++++++++++++++++++++++----------
 pkg/etcd/etcd.go           |  8 +++++---
 3 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go
index db4ac936df..2842d90b2d 100644
--- a/pkg/bootstrap/bootstrap.go
+++ b/pkg/bootstrap/bootstrap.go
@@ -87,9 +87,12 @@ func WriteToDiskFromStorage(r io.Reader, bootstrap *config.ControlRuntimeBootstr
 		if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
 			return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path))
 		}
-		if err := ioutil.WriteFile(path, bsf.Content, 0600); err != nil {
+		if err := os.WriteFile(path, bsf.Content, 0600); err != nil {
 			return errors.Wrapf(err, "failed to write to %s", path)
 		}
+		if err := os.Chtimes(path, bsf.Timestamp, bsf.Timestamp); err != nil {
+			return errors.Wrapf(err, "failed to update modified time on %s", path)
+		}
 	}
 
 	return nil
diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go
index e65c8c91d6..178e60d417 100644
--- a/pkg/cluster/bootstrap.go
+++ b/pkg/cluster/bootstrap.go
@@ -13,11 +13,13 @@ import (
 	"path"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/k3s-io/kine/pkg/client"
 	"github.com/k3s-io/kine/pkg/endpoint"
+	"github.com/otiai10/copy"
 	"github.com/rancher/k3s/pkg/bootstrap"
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/config"
@@ -225,15 +227,6 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
 		}
 	}
 
-	// Check the stamp file to see if we have successfully bootstrapped using this token.
-	// NOTE: The fact that we use a hash of the token to generate the stamp
-	//       means that it is unsafe to use the same token for multiple clusters.
-	// stamp := c.bootstrapStamp()
-	// if _, err := os.Stat(stamp); err == nil {
-	// 	logrus.Info("Cluster bootstrap already complete")
-	// 	return false, nil
-	// }
-
 	// No errors and no bootstrap stamp, need to bootstrap.
 	return true, false, nil
 }
@@ -517,12 +510,31 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 		}
 	}
 
+	if c.config.ClusterReset {
+		serverTLSDir := filepath.Join(c.config.DataDir, "tls")
+		tlsBackupDir := filepath.Join(c.config.DataDir, "tls-"+strconv.Itoa(int(time.Now().Unix())))
+
+		logrus.Infof("Cluster reset: backing up certificates directory to " + tlsBackupDir)
+
+		if _, err := os.Stat(serverTLSDir); err != nil {
+			return err
+		}
+		if err := copy.Copy(serverTLSDir, tlsBackupDir); err != nil {
+			return err
+		}
+	}
+
 	for path, res := range results {
 		switch {
 		case res.disk:
 			updateDisk = true
 			logrus.Warn("datastore newer than " + path)
 		case res.db:
+			if c.config.ClusterReset {
+				logrus.Infof("Cluster reset: replacing file on disk: " + path)
+				updateDisk = true
+				continue
+			}
 			logrus.Fatal(path + " newer than datastore and could cause cluster outage. Remove the file from disk and restart to be recreated from datastore.")
 		case res.conflict:
 			logrus.Warnf("datastore / disk conflict: %s newer than in the datastore", path)
@@ -606,7 +618,7 @@ func (c *Cluster) compareConfig() error {
 	if !reflect.DeepEqual(clusterControl.CriticalControlArgs, c.config.CriticalControlArgs) {
 		logrus.Debugf("This is the server CriticalControlArgs: %#v", clusterControl.CriticalControlArgs)
 		logrus.Debugf("This is the local CriticalControlArgs: %#v", c.config.CriticalControlArgs)
-		return errors.New("Unable to join cluster due to critical configuration value mismatch")
+		return errors.New("unable to join cluster due to critical configuration value mismatch")
 	}
 	return nil
 }
diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go
index bdaef6ad96..bcc126de36 100644
--- a/pkg/etcd/etcd.go
+++ b/pkg/etcd/etcd.go
@@ -198,9 +198,11 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
 					continue
 				}
 
-				// storageBootstrap() - runtime structure has been written with correct certificate data
-				if err := rebootstrap(); err != nil {
-					logrus.Fatal(err)
+				if rebootstrap != nil {
+					// storageBootstrap() - runtime structure has been written with correct certificate data
+					if err := rebootstrap(); err != nil {
+						logrus.Fatal(err)
+					}
 				}
 
 				// call functions to rewrite them from daemons/control/server.go (prepare())