Merge pull request #4736 from briandowns/backport_issue-4644-engine-1.21

Resolve Bootstrap Migration Edge Case (#4730)

pkg/bootstrap/bootstrap.go

@@ -67,17 +67,12 @@ type PathsDataformat map[string]File
 
 // WriteToDiskFromStorage writes the contents of the given reader to the paths
 // derived from within the ControlRuntimeBootstrap.
-func WriteToDiskFromStorage(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error {
+func WriteToDiskFromStorage(files PathsDataformat, bootstrap *config.ControlRuntimeBootstrap) error {
 	paths, err := ObjToMap(bootstrap)
 	if err != nil {
 		return err
 	}
 
-	files := make(PathsDataformat)
-	if err := json.NewDecoder(r).Decode(&files); err != nil {
-		return err
-	}
-
 	for pathKey, bsf := range files {
 		path, ok := paths[pathKey]
 		if !ok {

pkg/cli/server/server.go

@@ -174,6 +174,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		serverConfig.ControlConfig.DisableScheduler = true
 		serverConfig.ControlConfig.DisableCCM = true
 
+		close(agentReady)
 		dataDir, err := datadir.LocalHome(cfg.DataDir, false)
 		if err != nil {
 			return err

pkg/cluster/bootstrap.go

@@ -309,17 +309,46 @@ func migrateBootstrapData(ctx context.Context, data io.Reader, files bootstrap.P
 
 const systemTimeSkew = int64(3)
 
+// isMigrated checks to see if the given bootstrap data
+// is in the latest format.
+func isMigrated(buf io.ReadSeeker, files *bootstrap.PathsDataformat) bool {
+	buf.Seek(0, 0)
+	defer buf.Seek(0, 0)
+
+	if err := json.NewDecoder(buf).Decode(files); err != nil {
+		// This will fail if data is being pulled from old an cluster since
+		// older clusters used a map[string][]byte for the data structure.
+		// Therefore, we need to perform a migration to the newer bootstrap
+		// format; bootstrap.BootstrapFile.
+		return false
+	}
+
+	return true
+}
+
 // ReconcileBootstrapData is called before any data is saved to the
 // datastore or locally. It checks to see if the contents of the
 // bootstrap data in the datastore is newer than on disk or different
-// and depending on where the difference is, the newer data is written
-// to disk or if the disk is newer, the process is stopped and a error
-// is issued.
+// and depending on where the difference is. If the datastore is newer,
+// then the data will be written to disk. If the data on disk is newer,
+// k3s will exit with an error.
 func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, crb *config.ControlRuntimeBootstrap, isHTTP bool, ec *endpoint.ETCDConfig) error {
 	logrus.Info("Reconciling bootstrap data between datastore and disk")
 
 	if err := c.certDirsExist(); err != nil {
-		return bootstrap.WriteToDiskFromStorage(buf, crb)
+		// we need to see if the data has been migrated before writing to disk. This
+		// is because the data may have been given to us via the HTTP bootstrap process
+		// from an older version of k3s. That version might not have the new data format
+		// and we should write the correct format.
+		files := make(bootstrap.PathsDataformat)
+		if !isMigrated(buf, &files) {
+			if err := migrateBootstrapData(ctx, buf, files); err != nil {
+				return err
+			}
+			buf.Seek(0, 0)
+		}
+
+		return bootstrap.WriteToDiskFromStorage(files, crb)
 	}
 
 	var dbRawData []byte
@@ -393,18 +422,12 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 	}
 
 	files := make(bootstrap.PathsDataformat)
-	if err := json.NewDecoder(buf).Decode(&files); err != nil {
-		// This will fail if data is being pulled from old an cluster since
-		// older clusters used a map[string][]byte for the data structure.
-		// Therefore, we need to perform a migration to the newer bootstrap
-		// format; bootstrap.BootstrapFile.
-		buf.Seek(0, 0)
-
+	if !isMigrated(buf, &files) {
 		if err := migrateBootstrapData(ctx, buf, files); err != nil {
 			return err
 		}
+		buf.Seek(0, 0)
 	}
-	buf.Seek(0, 0)
 
 	type update struct {
 		db, disk, conflict bool
@@ -543,7 +566,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 
 	if updateDisk {
 		logrus.Warn("updating bootstrap data on disk from datastore")
-		return bootstrap.WriteToDiskFromStorage(buf, crb)
+		return bootstrap.WriteToDiskFromStorage(files, crb)
 	}
 
 	return nil

pkg/etcd/etcd.go

@@ -192,6 +192,11 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
 		t := time.NewTicker(5 * time.Second)
 		defer t.Stop()
 		for range t.C {
+			// resetting the apiaddresses to nil since we are doing a restoration
+			if _, err := e.client.Put(ctx, AddressKey, ""); err != nil {
+				logrus.Warnf("failed to reset api addresses key in etcd: %v", err)
+				continue
+			}
 			if err := e.Test(ctx); err == nil {
 				members, err := e.client.MemberList(ctx)
 				if err != nil {