From bc84ded38cb36d71cee53649845ef9a7e44f74b1 Mon Sep 17 00:00:00 2001 From: Brian Downs Date: Mon, 13 Dec 2021 13:02:30 -0700 Subject: [PATCH 1/2] Resolve Bootstrap Migration Edge Case (#4730) Signed-off-by: Brian Downs --- pkg/bootstrap/bootstrap.go | 7 +----- pkg/cluster/bootstrap.go | 50 ++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go index 2842d90b2d..884fb60dde 100644 --- a/pkg/bootstrap/bootstrap.go +++ b/pkg/bootstrap/bootstrap.go @@ -67,17 +67,12 @@ type PathsDataformat map[string]File // WriteToDiskFromStorage writes the contents of the given reader to the paths // derived from within the ControlRuntimeBootstrap. -func WriteToDiskFromStorage(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error { +func WriteToDiskFromStorage(files PathsDataformat, bootstrap *config.ControlRuntimeBootstrap) error { paths, err := ObjToMap(bootstrap) if err != nil { return err } - files := make(PathsDataformat) - if err := json.NewDecoder(r).Decode(&files); err != nil { - return err - } - for pathKey, bsf := range files { path, ok := paths[pathKey] if !ok { diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 178e60d417..d1ed5bd897 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -309,17 +309,47 @@ func migrateBootstrapData(ctx context.Context, data io.Reader, files bootstrap.P const systemTimeSkew = int64(3) +// isMigrated checks to see if the given bootstrap data +// is in the latest format. +func isMigrated(buf io.ReadSeeker) bool { + buf.Seek(0, 0) + defer buf.Seek(0, 0) + + files := make(bootstrap.PathsDataformat) + if err := json.NewDecoder(buf).Decode(&files); err != nil { + // This will fail if data is being pulled from old an cluster since + // older clusters used a map[string][]byte for the data structure. + // Therefore, we need to perform a migration to the newer bootstrap + // format; bootstrap.BootstrapFile. + return false + } + + return true +} + // ReconcileBootstrapData is called before any data is saved to the // datastore or locally. It checks to see if the contents of the // bootstrap data in the datastore is newer than on disk or different -// and depending on where the difference is, the newer data is written -// to disk or if the disk is newer, the process is stopped and a error -// is issued. +// and depending on where the difference is. If the datastore is newer, +// then the data will be written to disk. If the data on disk is newer, +// k3s will exit with an error. func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, crb *config.ControlRuntimeBootstrap, isHTTP bool, ec *endpoint.ETCDConfig) error { logrus.Info("Reconciling bootstrap data between datastore and disk") if err := c.certDirsExist(); err != nil { - return bootstrap.WriteToDiskFromStorage(buf, crb) + // we need to see if the data has been migrated before writing to disk. This + // is because the data may have been given to us via the HTTP bootstrap process + // from an older version of k3s. That version might not have the new data format + // and we should write the correct format. + files := make(bootstrap.PathsDataformat) + if !isMigrated(buf) { + if err := migrateBootstrapData(ctx, buf, files); err != nil { + return err + } + buf.Seek(0, 0) + } + + return bootstrap.WriteToDiskFromStorage(files, crb) } var dbRawData []byte @@ -393,18 +423,12 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, } files := make(bootstrap.PathsDataformat) - if err := json.NewDecoder(buf).Decode(&files); err != nil { - // This will fail if data is being pulled from old an cluster since - // older clusters used a map[string][]byte for the data structure. - // Therefore, we need to perform a migration to the newer bootstrap - // format; bootstrap.BootstrapFile. - buf.Seek(0, 0) - + if !isMigrated(buf) { if err := migrateBootstrapData(ctx, buf, files); err != nil { return err } + buf.Seek(0, 0) } - buf.Seek(0, 0) type update struct { db, disk, conflict bool @@ -543,7 +567,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, if updateDisk { logrus.Warn("updating bootstrap data on disk from datastore") - return bootstrap.WriteToDiskFromStorage(buf, crb) + return bootstrap.WriteToDiskFromStorage(files, crb) } return nil From 9b676924144319432b8761f05e9096e05618f2bc Mon Sep 17 00:00:00 2001 From: Hussein Galal Date: Tue, 14 Dec 2021 02:04:39 +0200 Subject: [PATCH 2/2] Fix snapshot restoration on fresh nodes (#4737) Signed-off-by: galal-hussein Signed-off-by: Brian Downs --- pkg/cli/server/server.go | 1 + pkg/cluster/bootstrap.go | 9 ++++----- pkg/etcd/etcd.go | 5 +++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 26237eb302..da3e62c770 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -174,6 +174,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.DisableScheduler = true serverConfig.ControlConfig.DisableCCM = true + close(agentReady) dataDir, err := datadir.LocalHome(cfg.DataDir, false) if err != nil { return err diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index d1ed5bd897..b38286e702 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -311,12 +311,11 @@ const systemTimeSkew = int64(3) // isMigrated checks to see if the given bootstrap data // is in the latest format. -func isMigrated(buf io.ReadSeeker) bool { +func isMigrated(buf io.ReadSeeker, files *bootstrap.PathsDataformat) bool { buf.Seek(0, 0) defer buf.Seek(0, 0) - files := make(bootstrap.PathsDataformat) - if err := json.NewDecoder(buf).Decode(&files); err != nil { + if err := json.NewDecoder(buf).Decode(files); err != nil { // This will fail if data is being pulled from old an cluster since // older clusters used a map[string][]byte for the data structure. // Therefore, we need to perform a migration to the newer bootstrap @@ -342,7 +341,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, // from an older version of k3s. That version might not have the new data format // and we should write the correct format. files := make(bootstrap.PathsDataformat) - if !isMigrated(buf) { + if !isMigrated(buf, &files) { if err := migrateBootstrapData(ctx, buf, files); err != nil { return err } @@ -423,7 +422,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker, } files := make(bootstrap.PathsDataformat) - if !isMigrated(buf) { + if !isMigrated(buf, &files) { if err := migrateBootstrapData(ctx, buf, files); err != nil { return err } diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index bcc126de36..1ee8f08d20 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -192,6 +192,11 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error { t := time.NewTicker(5 * time.Second) defer t.Stop() for range t.C { + // resetting the apiaddresses to nil since we are doing a restoration + if _, err := e.client.Put(ctx, AddressKey, ""); err != nil { + logrus.Warnf("failed to reset api addresses key in etcd: %v", err) + continue + } if err := e.Test(ctx); err == nil { members, err := e.client.MemberList(ctx) if err != nil {