diff --git a/go.mod b/go.mod index 7d09cb275d..4913d042b0 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 github.com/k3s-io/helm-controller v0.10.1 - github.com/k3s-io/kine v0.6.0 + github.com/k3s-io/kine v0.6.2 github.com/klauspost/compress v1.12.2 github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000 github.com/lib/pq v1.8.0 diff --git a/go.sum b/go.sum index 0ac3516335..5ac7c95aba 100644 --- a/go.sum +++ b/go.sum @@ -542,8 +542,8 @@ github.com/k3s-io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea h1:7cwby0GoN github.com/k3s-io/etcd v0.5.0-alpha.5.0.20201208200253-50621aee4aea/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= github.com/k3s-io/helm-controller v0.10.1 h1:w98iQsKfA5RnvzdVzU4LTDuLzA3SyoN31ebDUKQUDpM= github.com/k3s-io/helm-controller v0.10.1/go.mod h1:nZP8FH3KZrNNUf5r+SwwiMR63HS6lxdHdpHijgPfF74= -github.com/k3s-io/kine v0.6.0 h1:4l7wjgCxb2oD+7Hyf3xIhkGd/6s1sXpRFdQiyy+7Ki8= -github.com/k3s-io/kine v0.6.0/go.mod h1:rzCs93+rQHZGOiewMd84PDrER92QeZ6eeHbWkfEy4+w= +github.com/k3s-io/kine v0.6.2 h1:1aJTPfB8HG4exqMKFVE5H0z4bepF05tJHtYNXotWXa4= +github.com/k3s-io/kine v0.6.2/go.mod h1:rzCs93+rQHZGOiewMd84PDrER92QeZ6eeHbWkfEy4+w= github.com/k3s-io/kubernetes v1.21.2-k3s1 h1:e5R77BWsEEiZqoZ68t9iKF08KwSVBqASwy4N9Uq4FHw= github.com/k3s-io/kubernetes v1.21.2-k3s1/go.mod h1:HevHCwYnT2nf/6w8I+b2tpz1NvzJmHZ9nOjh9ng7Rwg= github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.21.2-k3s1 h1:CAhRjFSPftQWJXLDtuP9y2bpOhTrGtCvfa+oN2iNY7k= diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index f555cee7a6..4d89edec91 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" "github.com/rancher/k3s/pkg/agent" "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/clientaccess" "github.com/rancher/k3s/pkg/datadir" "github.com/rancher/k3s/pkg/etcd" "github.com/rancher/k3s/pkg/netutil" @@ -396,7 +397,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont } url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.SupervisorPort) - token, err := server.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA) + token, err := clientaccess.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA) if err != nil { return err } diff --git a/pkg/clientaccess/token.go b/pkg/clientaccess/token.go index e87d6f3cb6..062593c926 100644 --- a/pkg/clientaccess/token.go +++ b/pkg/clientaccess/token.go @@ -287,3 +287,20 @@ func get(u string, client *http.Client, username, password string) ([]byte, erro return ioutil.ReadAll(resp.Body) } + +func FormatToken(token string, certFile string) (string, error) { + if len(token) == 0 { + return token, nil + } + + certHash := "" + if len(certFile) > 0 { + bytes, err := ioutil.ReadFile(certFile) + if err != nil { + return "", nil + } + digest := sha256.Sum256(bytes) + certHash = tokenPrefix + hex.EncodeToString(digest[:]) + "::" + } + return certHash + token, nil +} diff --git a/pkg/cluster/bootstrap.go b/pkg/cluster/bootstrap.go index 539fe5fc61..a799dbf2f6 100644 --- a/pkg/cluster/bootstrap.go +++ b/pkg/cluster/bootstrap.go @@ -132,7 +132,7 @@ func (c *Cluster) httpBootstrap() error { func (c *Cluster) bootstrap(ctx context.Context) error { c.joining = true - // bootstrap managed database via HTTP + // bootstrap managed database via HTTPS if c.runtime.HTTPBootstrap { return c.httpBootstrap() } diff --git a/pkg/cluster/https.go b/pkg/cluster/https.go index 29551e3ff4..bc1bad3db6 100644 --- a/pkg/cluster/https.go +++ b/pkg/cluster/https.go @@ -6,6 +6,7 @@ import ( "log" "net" "net/http" + "os" "path/filepath" "github.com/rancher/dynamiclistener" @@ -14,6 +15,7 @@ import ( "github.com/rancher/dynamiclistener/storage/kubernetes" "github.com/rancher/dynamiclistener/storage/memory" "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/etcd" "github.com/rancher/k3s/pkg/version" "github.com/rancher/wrangler-api/pkg/generated/controllers/core" "github.com/sirupsen/logrus" @@ -24,16 +26,21 @@ import ( // dynamiclistener will use the cluster's Server CA to sign the dynamically generate certificate, // and will sync the certs into the Kubernetes datastore, with a local disk cache. func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler, error) { + if c.managedDB != nil { + if _, err := os.Stat(etcd.ResetFile(c.config)); err == nil { + // delete the dynamic listener file if it exists after restoration to fix restoration + // on fresh nodes + os.Remove(filepath.Join(c.config.DataDir, "tls/dynamic-cert.json")) + } + } tcp, err := dynamiclistener.NewTCPListener(c.config.BindAddress, c.config.SupervisorPort) if err != nil { return nil, nil, err } - cert, key, err := factory.LoadCerts(c.runtime.ServerCA, c.runtime.ServerCAKey) if err != nil { return nil, nil, err } - storage := tlsStorage(ctx, c.config.DataDir, c.runtime) return dynamiclistener.NewListener(tcp, storage, cert, key, dynamiclistener.Config{ ExpirationDaysCheck: config.CertificateRenewDays, diff --git a/pkg/cluster/managed.go b/pkg/cluster/managed.go index 4e0441270d..0c5e3519b7 100644 --- a/pkg/cluster/managed.go +++ b/pkg/cluster/managed.go @@ -15,8 +15,10 @@ import ( "github.com/k3s-io/kine/pkg/endpoint" "github.com/rancher/k3s/pkg/cluster/managed" "github.com/rancher/k3s/pkg/etcd" + "github.com/rancher/k3s/pkg/nodepassword" "github.com/rancher/k3s/pkg/version" "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" ) // testClusterDB returns a channel that will be closed when the datastore connection is available. @@ -76,6 +78,10 @@ func (c *Cluster) start(ctx context.Context) error { return fmt.Errorf("cluster-reset was successfully performed, please remove the cluster-reset flag and start %s normally, if you need to perform another cluster reset, you must first manually delete the %s file", version.Program, resetFile) } + if _, err := os.Stat(resetFile); err == nil { + // before removing reset file we need to delete the node passwd secret + go c.deleteNodePasswdSecret(ctx) + } // removing the reset file and ignore error if the file doesn't exist os.Remove(resetFile) @@ -165,3 +171,33 @@ func (c *Cluster) setupEtcdProxy(ctx context.Context, etcdProxy etcd.Proxy) { } }() } + +// deleteNodePasswdSecret wipes out the node password secret after restoration +func (c *Cluster) deleteNodePasswdSecret(ctx context.Context) { + t := time.NewTicker(5 * time.Second) + defer t.Stop() + for range t.C { + nodeName := os.Getenv("NODE_NAME") + if nodeName == "" { + logrus.Infof("waiting for node name to be set") + continue + } + // the core factory may not yet be initialized so we + // want to wait until it is so not to evoke a panic. + if c.runtime.Core == nil { + logrus.Infof("runtime is not yet initialized") + continue + } + secretsClient := c.runtime.Core.Core().V1().Secret() + if err := nodepassword.Delete(secretsClient, nodeName); err != nil { + if apierrors.IsNotFound(err) { + logrus.Debugf("node password secret is not found for node %s", nodeName) + return + } + logrus.Warnf("failed to delete old node password secret: %v", err) + continue + } + return + } + +} diff --git a/pkg/cluster/storage.go b/pkg/cluster/storage.go index 4df5635164..531b698d68 100644 --- a/pkg/cluster/storage.go +++ b/pkg/cluster/storage.go @@ -3,10 +3,15 @@ package cluster import ( "bytes" "context" + "errors" + "io/ioutil" + "os" + "path/filepath" "strings" "github.com/k3s-io/kine/pkg/client" "github.com/rancher/k3s/pkg/bootstrap" + "github.com/rancher/k3s/pkg/clientaccess" "github.com/sirupsen/logrus" ) @@ -19,8 +24,20 @@ func (c *Cluster) save(ctx context.Context) error { if err := bootstrap.Write(buf, &c.runtime.ControlRuntimeBootstrap); err != nil { return err } + token := c.config.Token + if token == "" { + tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir) + if err != nil { + return err + } + token = tokenFromFile + } + normalizedToken, err := normalizeToken(token) + if err != nil { + return err + } - data, err := encrypt(c.config.Token, buf.Bytes()) + data, err := encrypt(normalizedToken, buf.Bytes()) if err != nil { return err } @@ -30,12 +47,17 @@ func (c *Cluster) save(ctx context.Context) error { return err } - if err := storageClient.Create(ctx, storageKey(c.config.Token), data); err != nil { + _, _, err = c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) + if err != nil { + return err + } + + if err := storageClient.Create(ctx, storageKey(normalizedToken), data); err != nil { if err.Error() == "key exists" { - logrus.Warnln("Bootstrap key exists. Please follow documentation updating a node after restore.") + logrus.Warnln("bootstrap key exists; please follow documentation on updating a node after snapshot restore") return nil } else if strings.Contains(err.Error(), "not supported for learner") { - logrus.Debug("Skipping bootstrap data save on learner.") + logrus.Debug("skipping bootstrap data save on learner") return nil } return err @@ -46,6 +68,7 @@ func (c *Cluster) save(ctx context.Context) error { // storageBootstrap loads data from the datastore into the ControlRuntimeBootstrap struct. // The storage key and encryption passphrase are both derived from the join token. +// token is either passed func (c *Cluster) storageBootstrap(ctx context.Context) error { if err := c.startStorage(ctx); err != nil { return err @@ -56,18 +79,104 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error { return err } - value, err := storageClient.Get(ctx, storageKey(c.config.Token)) - if err == client.ErrNotFound { - c.saveBootstrap = true - return nil - } else if err != nil { + token := c.config.Token + if token == "" { + tokenFromFile, err := readTokenFromFile(c.runtime.ServerToken, c.runtime.ServerCA, c.config.DataDir) + if err != nil { + return err + } + if tokenFromFile == "" { + // at this point this is a fresh start in a non managed environment + c.saveBootstrap = true + return nil + } + token = tokenFromFile + } + normalizedToken, err := normalizeToken(token) + if err != nil { return err } - data, err := decrypt(c.config.Token, value.Data) + value, emptyKey, err := c.getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken) + if err != nil { + return err + } + if value == nil { + return nil + } + if emptyKey { + normalizedToken = "" + } + data, err := decrypt(normalizedToken, value.Data) if err != nil { return err } return bootstrap.Read(bytes.NewBuffer(data), &c.runtime.ControlRuntimeBootstrap) } + +// getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is +// hashed with empty string and will check for any key that is hashed by different token than the one +// passed to it, it will return error if it finds a key that is hashed with different token and will return +// value if it finds the key hashed by passed token or empty string +func (c *Cluster) getBootstrapKeyFromStorage(ctx context.Context, storageClient client.Client, token string) (*client.Value, bool, error) { + emptyStringKey := storageKey("") + tokenKey := storageKey(token) + + bootstrapList, err := storageClient.List(ctx, "/bootstrap", 0) + if err != nil { + return nil, false, err + } + if len(bootstrapList) == 0 { + c.saveBootstrap = true + return nil, false, nil + } + if len(bootstrapList) > 1 { + return nil, false, errors.New("found more than one bootstrap keys in storage") + } + bootstrapKV := bootstrapList[0] + // checking for empty string bootstrap key + switch string(bootstrapKV.Key) { + case emptyStringKey: + logrus.Warn("bootstrap data already found and encrypted with empty string, deleting empty key") + c.saveBootstrap = true + if err := storageClient.Delete(ctx, emptyStringKey, bootstrapKV.Modified); err != nil { + return nil, false, err + } + return &bootstrapKV, true, nil + case tokenKey: + return &bootstrapKV, false, nil + } + + return nil, false, errors.New("bootstrap data already found and encrypted with different token") +} + +// readTokenFromFile will attempt to get the token from /token if it the file not found +// in case of fresh installation it will try to use the runtime serverToken saved in memory +// after stripping it from any additional information like the username or cahash, if the file +// found then it will still strip the token from any additional info +func readTokenFromFile(serverToken, certs, dataDir string) (string, error) { + tokenFile := filepath.Join(dataDir, "token") + b, err := ioutil.ReadFile(tokenFile) + if err != nil { + if os.IsNotExist(err) { + token, err := clientaccess.FormatToken(serverToken, certs) + if err != nil { + return token, err + } + return token, nil + } + return "", err + } + // strip the token from any new line if its read from file + return string(bytes.TrimRight(b, "\n")), nil +} + +// normalizeToken will normalize the token read from file or passed as a cli flag +func normalizeToken(token string) (string, error) { + _, password, ok := clientaccess.ParseUsernamePassword(token) + if !ok { + return password, errors.New("failed to normalize token") + } + return password, nil +} diff --git a/pkg/server/server.go b/pkg/server/server.go index c20a1485ab..f574674233 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -2,8 +2,6 @@ package server import ( "context" - "crypto/sha256" - "encoding/hex" "fmt" "io/ioutil" net2 "net" @@ -423,30 +421,12 @@ func printToken(httpsPort int, advertiseIP, prefix, cmd string) { logrus.Infof("%s %s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, version.Program, cmd, ip, httpsPort) } -func FormatToken(token string, certFile string) (string, error) { - if len(token) == 0 { - return token, nil - } - - prefix := "K10" - if len(certFile) > 0 { - bytes, err := ioutil.ReadFile(certFile) - if err != nil { - return "", nil - } - digest := sha256.Sum256(bytes) - prefix = "K10" + hex.EncodeToString(digest[:]) + "::" - } - - return prefix + token, nil -} - func writeToken(token, file, certs string) error { if len(token) == 0 { return nil } - token, err := FormatToken(token, certs) + token, err := clientaccess.FormatToken(token, certs) if err != nil { return err } diff --git a/vendor/github.com/k3s-io/kine/pkg/client/client.go b/vendor/github.com/k3s-io/kine/pkg/client/client.go index db17e8441c..812aaaa842 100644 --- a/vendor/github.com/k3s-io/kine/pkg/client/client.go +++ b/vendor/github.com/k3s-io/kine/pkg/client/client.go @@ -26,6 +26,7 @@ type Client interface { Put(ctx context.Context, key string, value []byte) error Create(ctx context.Context, key string, value []byte) error Update(ctx context.Context, key string, revision int64, value []byte) error + Delete(ctx context.Context, key string, revision int64) error Close() error } @@ -128,6 +129,21 @@ func (c *client) Update(ctx context.Context, key string, revision int64, value [ return nil } +func (c *client) Delete(ctx context.Context, key string, revision int64) error { + resp, err := c.c.Txn(ctx). + If(clientv3.Compare(clientv3.ModRevision(key), "=", revision)). + Then(clientv3.OpDelete(key)). + Else(clientv3.OpGet(key)). + Commit() + if err != nil { + return err + } + if !resp.Succeeded { + return fmt.Errorf("revision %d doesnt match", revision) + } + return nil +} + func (c *client) Close() error { return c.c.Close() } diff --git a/vendor/github.com/k3s-io/kine/pkg/logstructured/sqllog/sql.go b/vendor/github.com/k3s-io/kine/pkg/logstructured/sqllog/sql.go index b7aec6d8fa..5aa115ff28 100644 --- a/vendor/github.com/k3s-io/kine/pkg/logstructured/sqllog/sql.go +++ b/vendor/github.com/k3s-io/kine/pkg/logstructured/sqllog/sql.go @@ -53,9 +53,9 @@ type Dialect interface { BeginTx(ctx context.Context, opts *sql.TxOptions) (*generic.Tx, error) } -func (s *SQLLog) Start(ctx context.Context) (err error) { +func (s *SQLLog) Start(ctx context.Context) error { s.ctx = ctx - return + return s.compactStart(s.ctx) } func (s *SQLLog) compactStart(ctx context.Context) error { @@ -365,10 +365,6 @@ func filter(events interface{}, checkPrefix bool, prefix string) ([]*server.Even } func (s *SQLLog) startWatch() (chan interface{}, error) { - if err := s.compactStart(s.ctx); err != nil { - return nil, err - } - pollStart, err := s.d.GetCompactRevision(s.ctx) if err != nil { return nil, err diff --git a/vendor/modules.txt b/vendor/modules.txt index d9a936aae7..7d9ed5b11e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -707,7 +707,7 @@ github.com/k3s-io/helm-controller/pkg/generated/informers/externalversions/helm. github.com/k3s-io/helm-controller/pkg/generated/informers/externalversions/internalinterfaces github.com/k3s-io/helm-controller/pkg/generated/listers/helm.cattle.io/v1 github.com/k3s-io/helm-controller/pkg/helm -# github.com/k3s-io/kine v0.6.0 +# github.com/k3s-io/kine v0.6.2 ## explicit github.com/k3s-io/kine/pkg/broadcaster github.com/k3s-io/kine/pkg/client