remove etcd data dir when etcd is disabled (#3059)

* remove etcd data dir when etcd is disabled

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* fix comment

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* more fixes

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

* use debug instead of info logs

Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>

pkg/cluster/storage.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"bytes"
 	"context"
+	"strings"
 
 	"github.com/k3s-io/kine/pkg/client"
 	"github.com/rancher/k3s/pkg/bootstrap"
@@ -23,6 +24,7 @@ func (c *Cluster) save(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+
 	storageClient, err := client.New(c.etcdConfig)
 	if err != nil {
 		return err
@@ -32,6 +34,9 @@ func (c *Cluster) save(ctx context.Context) error {
 		if err.Error() == "key exists" {
 			logrus.Warnln("Bootstrap key exists. Please follow documentation updating a node after restore.")
 			return nil
+		} else if strings.Contains(err.Error(), "not supported for learner") {
+			logrus.Debug("Skipping bootstrap data save on learner.")
+			return nil
 		}
 		return err
 	}

pkg/etcd/etcd.go

@@ -1011,5 +1011,13 @@ func (e *ETCD) GetMembersClientURLs(ctx context.Context) ([]string, error) {
 
 // RemoveSelf will remove the member if it exists in the cluster
 func (e *ETCD) RemoveSelf(ctx context.Context) error {
-	return e.removePeer(ctx, e.name, e.address, true)
+	if err := e.removePeer(ctx, e.name, e.address, true); err != nil {
+		return err
+	}
+
+	// backup the data dir to avoid issues when re-enabling etcd
+	oldDataDir := etcdDBDir(e.config) + "-old-" + strconv.Itoa(int(time.Now().Unix()))
+
+	// move the data directory to a temp path
+	return os.Rename(etcdDBDir(e.config), oldDataDir)
 }