From d6124981d5f1b0a93b8d75642f411c60c211ba4e Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Mon, 1 Mar 2021 23:50:50 +0200 Subject: [PATCH 1/3] remove etcd member if disable etcd is passed Signed-off-by: galal-hussein --- pkg/cli/server/server.go | 9 +++++++-- pkg/cluster/cluster.go | 8 +++++++- pkg/cluster/managed/drivers.go | 1 + pkg/etcd/controller.go | 2 +- pkg/etcd/etcd.go | 24 +++++++++++++++++++----- 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 86558ac321..0e19d34ff9 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -267,8 +267,13 @@ func run(app *cli.Context, cfg *cmds.Server) error { } go func() { - <-serverConfig.ControlConfig.Runtime.APIServerReady - logrus.Info("Kube API server is now running") + if !serverConfig.ControlConfig.DisableAPIServer { + <-serverConfig.ControlConfig.Runtime.APIServerReady + logrus.Info("Kube API server is now running") + } else { + <-serverConfig.ControlConfig.Runtime.ETCDReady + logrus.Info("ETCD server is now running") + } logrus.Info(version.Program + " is up and running") if notifySocket != "" { os.Setenv("NOTIFY_SOCKET", notifySocket) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 1ffec23e62..87060129e8 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -41,7 +41,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { defer close(ready) // try to get /db/info urls first before attempting to use join url - clientURLs, _, err := etcd.ClientURLs(ctx, c.clientAccessInfo) + clientURLs, _, err := etcd.ClientURLs(ctx, c.clientAccessInfo, c.config.PrivateIP) if err != nil { return nil, err } @@ -58,6 +58,12 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { return nil, err } c.setupEtcdProxy(ctx, etcdProxy) + + // remove etcd member if it exists + if err := c.managedDB.RemoveSelf(ctx); err != nil { + return nil, err + } + return ready, nil } diff --git a/pkg/cluster/managed/drivers.go b/pkg/cluster/managed/drivers.go index ac43c9e2d3..0a6e8f38ee 100644 --- a/pkg/cluster/managed/drivers.go +++ b/pkg/cluster/managed/drivers.go @@ -23,6 +23,7 @@ type Driver interface { EndpointName() string Snapshot(ctx context.Context, config *config.Control) error GetMembersClientURLs(ctx context.Context) ([]string, error) + RemoveSelf(ctx context.Context) error } func RegisterDriver(d Driver) { diff --git a/pkg/etcd/controller.go b/pkg/etcd/controller.go index 5da364aafa..81f46a69c8 100644 --- a/pkg/etcd/controller.go +++ b/pkg/etcd/controller.go @@ -87,5 +87,5 @@ func (h *handler) onRemove(key string, node *v1.Node) (*v1.Node, error) { if !ok { return node, nil } - return node, h.etcd.removePeer(h.ctx, id, address) + return node, h.etcd.removePeer(h.ctx, id, address, false) } diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 971d4aa412..67ec4190fe 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -245,7 +245,7 @@ func (e *ETCD) Start(ctx context.Context, clientAccessInfo *clientaccess.Info) e // join attempts to add a member to an existing cluster func (e *ETCD) join(ctx context.Context, clientAccessInfo *clientaccess.Info) error { - clientURLs, memberList, err := ClientURLs(ctx, clientAccessInfo) + clientURLs, memberList, err := ClientURLs(ctx, clientAccessInfo, e.config.PrivateIP) if err != nil { return err } @@ -524,7 +524,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init } // removePeer removes a peer from the cluster. The peer ID and IP address must both match. -func (e *ETCD) removePeer(ctx context.Context, id, address string) error { +func (e *ETCD) removePeer(ctx context.Context, id, address string, removeSelf bool) error { members, err := e.client.MemberList(ctx) if err != nil { return err @@ -540,7 +540,7 @@ func (e *ETCD) removePeer(ctx context.Context, id, address string) error { return err } if u.Hostname() == address { - if e.address == address { + if e.address == address && !removeSelf { return errors.New("node has been deleted from the cluster") } logrus.Infof("Removing name=%s id=%d address=%s from etcd", member.Name, member.ID, address) @@ -687,7 +687,7 @@ func (e *ETCD) setLearnerProgress(ctx context.Context, status *learnerProgress) } // clientURLs returns a list of all non-learner etcd cluster member client access URLs -func ClientURLs(ctx context.Context, clientAccessInfo *clientaccess.Info) ([]string, Members, error) { +func ClientURLs(ctx context.Context, clientAccessInfo *clientaccess.Info, selfIP string) ([]string, Members, error) { var memberList Members resp, err := clientaccess.Get("/db/info", clientAccessInfo) if err != nil { @@ -697,13 +697,22 @@ func ClientURLs(ctx context.Context, clientAccessInfo *clientaccess.Info) ([]str if err := json.Unmarshal(resp, &memberList); err != nil { return nil, memberList, err } - + ip, err := GetAdvertiseAddress(selfIP) + if err != nil { + return nil, memberList, err + } var clientURLs []string +members: for _, member := range memberList.Members { // excluding learner member from the client list if member.IsLearner { continue } + for _, url := range member.ClientURLs { + if strings.Contains(url, ip) { + continue members + } + } clientURLs = append(clientURLs, member.ClientURLs...) } return clientURLs, memberList, nil @@ -941,3 +950,8 @@ func (e *ETCD) GetMembersClientURLs(ctx context.Context) ([]string, error) { } return memberUrls, nil } + +// RemoveSelf will remove the member if it exists in the cluster +func (e *ETCD) RemoveSelf(ctx context.Context) error { + return e.removePeer(ctx, e.name, e.address, true) +} From 885b7391a290ddf054a1ce5cb0219b61b2c1213e Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Mon, 1 Mar 2021 23:51:07 +0200 Subject: [PATCH 2/3] update dynamiclistener Signed-off-by: galal-hussein --- go.mod | 2 +- go.sum | 4 ++-- .../rancher/dynamiclistener/storage/memory/memory.go | 2 +- vendor/modules.txt | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3d7bbf44ac..4391ed1c5b 100644 --- a/go.mod +++ b/go.mod @@ -93,7 +93,7 @@ require ( github.com/opencontainers/selinux v1.6.0 github.com/pierrec/lz4 v2.5.2+incompatible github.com/pkg/errors v0.9.1 - github.com/rancher/dynamiclistener v0.2.1 + github.com/rancher/dynamiclistener v0.2.2 github.com/rancher/remotedialer v0.2.0 github.com/rancher/wrangler v0.6.1 github.com/rancher/wrangler-api v0.6.0 diff --git a/go.sum b/go.sum index 9036ab56d3..aafaf742c6 100644 --- a/go.sum +++ b/go.sum @@ -768,8 +768,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI= github.com/rakelkar/gonetsh v0.0.0-20190930180311-e5c5ffe4bdf0 h1:iXE9kmlAqhusXxzkXictdNgWS7p4ZBnmv9SdyMgTf6E= github.com/rakelkar/gonetsh v0.0.0-20190930180311-e5c5ffe4bdf0/go.mod h1:4XHkfaUj+URzGO9sohoAgt2V9Y8nIW7fugpu0E6gShk= -github.com/rancher/dynamiclistener v0.2.1 h1:QiY1jxs2TOLrKB04G36vE2ehEvPMPGiWp8zEHLKB1nE= -github.com/rancher/dynamiclistener v0.2.1/go.mod h1:9WusTANoiRr8cDWCTtf5txieulezHbpv4vhLADPp0zU= +github.com/rancher/dynamiclistener v0.2.2 h1:70dMwOr1sqb6mQqfU2nDb/fr5cv7HJjH+kFYzoxb8KU= +github.com/rancher/dynamiclistener v0.2.2/go.mod h1:9WusTANoiRr8cDWCTtf5txieulezHbpv4vhLADPp0zU= github.com/rancher/moq v0.0.0-20190404221404-ee5226d43009/go.mod h1:wpITyDPTi/Na5h73XkbuEf2AP9fbgrIGqqxVzFhYD6U= github.com/rancher/remotedialer v0.2.0 h1:xD7t3K6JYwTdAsxmGtTHQMkEkFgKouQ1foLxVW424Dc= github.com/rancher/remotedialer v0.2.0/go.mod h1:tkU8ZvrR5lRgaKWaX71nAy6daeqvPFx/lJEnbW7tXSI= diff --git a/vendor/github.com/rancher/dynamiclistener/storage/memory/memory.go b/vendor/github.com/rancher/dynamiclistener/storage/memory/memory.go index 54f6251780..0edfd1bda3 100644 --- a/vendor/github.com/rancher/dynamiclistener/storage/memory/memory.go +++ b/vendor/github.com/rancher/dynamiclistener/storage/memory/memory.go @@ -32,7 +32,7 @@ func (m *memory) Get() (*v1.Secret, error) { } func (m *memory) Update(secret *v1.Secret) error { - if m.secret == nil || m.secret.ResourceVersion != secret.ResourceVersion { + if m.secret == nil || m.secret.ResourceVersion == "" || m.secret.ResourceVersion != secret.ResourceVersion { if m.storage != nil { if err := m.storage.Update(secret); err != nil { return err diff --git a/vendor/modules.txt b/vendor/modules.txt index b94177b45d..a195a0543a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -842,7 +842,7 @@ github.com/prometheus/procfs/internal/util # github.com/rakelkar/gonetsh v0.0.0-20190930180311-e5c5ffe4bdf0 github.com/rakelkar/gonetsh/netroute github.com/rakelkar/gonetsh/netsh -# github.com/rancher/dynamiclistener v0.2.1 +# github.com/rancher/dynamiclistener v0.2.2 ## explicit github.com/rancher/dynamiclistener github.com/rancher/dynamiclistener/cert From ef999f0b4ff6c533d896bd162d6735c5d8160ef3 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Tue, 2 Mar 2021 00:19:57 +0200 Subject: [PATCH 3/3] change error to warn when removing self from etcd members Signed-off-by: galal-hussein --- pkg/cluster/cluster.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 87060129e8..76f725df33 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -12,6 +12,7 @@ import ( "github.com/rancher/k3s/pkg/cluster/managed" "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/k3s/pkg/etcd" + "github.com/sirupsen/logrus" ) type Cluster struct { @@ -61,7 +62,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) { // remove etcd member if it exists if err := c.managedDB.RemoveSelf(ctx); err != nil { - return nil, err + logrus.Warnf("Failed to remove this node from etcd members") } return ready, nil