mirror of https://github.com/k3s-io/k3s
[master] Add `etcd-member-management` controller to K3s (#4001)
* Initial leader elected etcd member management controller * Bump etcd to v3.5.0-k3s2 Signed-off-by: Chris Kim <oats87g@gmail.com>pull/4012/head
Chris Kim
GitHub
parent
699ea16523
commit
928b8531c3
8
go.mod
8
go.mod
|
|
@ -29,10 +29,10 @@ replace (
|
|||
github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20210316141917-a8c4a9ee0f6b
|
||||
github.com/rancher/k3s/pkg/data => ./pkg/data
|
||||
github.com/rancher/wrangler => github.com/rancher/wrangler v0.8.5
|
||||
go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.0-k3s1
|
||||
go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.0-k3s1
|
||||
go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s1
|
||||
go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.0-k3s1
|
||||
go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.0-k3s2
|
||||
go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.0-k3s2
|
||||
go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s2
|
||||
go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.0-k3s2
|
||||
golang.org/x/crypto => golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
|
||||
golang.org/x/net => golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
|
||||
golang.org/x/sys => golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
|
||||
|
|
|
|||
16
go.sum
16
go.sum
|
|
@ -562,14 +562,14 @@ github.com/k3s-io/continuity v0.0.0-20210309170710-f93269e0d5c1 h1:KEz2rd9IDbrQT
|
|||
github.com/k3s-io/continuity v0.0.0-20210309170710-f93269e0d5c1/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
|
||||
github.com/k3s-io/cri-tools v1.21.0-k3s1 h1:MWQtAsx4HCNXenqU/B4V9eU6HMyafkd1PnW6d4HCfos=
|
||||
github.com/k3s-io/cri-tools v1.21.0-k3s1/go.mod h1:Qsz54zxINPR+WVWX9Kc3CTmuDFB1dNLCNV8jE8lUbtU=
|
||||
github.com/k3s-io/etcd/api/v3 v3.5.0-k3s1 h1:hbVjpbEfKjttv+ZmYnqgR8wK7vtKIDrPxF8mcGVYWSU=
|
||||
github.com/k3s-io/etcd/api/v3 v3.5.0-k3s1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
|
||||
github.com/k3s-io/etcd/client/v3 v3.5.0-k3s1 h1:RAXJQReqdjR6B+wL3HwWuBAdMO31fi7CGft0BdEyI5Q=
|
||||
github.com/k3s-io/etcd/client/v3 v3.5.0-k3s1/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
|
||||
github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s1 h1:G0u+ShI11UDcP4Z8OP3CSl4/Wg63OgRBqYEiKMsUDP4=
|
||||
github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s1/go.mod h1:o98rKMCibbFAG8QS9KmvlYDGDShmmIbmRE8vSofzYNg=
|
||||
github.com/k3s-io/etcd/server/v3 v3.5.0-k3s1 h1:be2d2LS1w+OxyHE/PF6tmGVzF72+7L2FZ6BmxK1Q0ow=
|
||||
github.com/k3s-io/etcd/server/v3 v3.5.0-k3s1/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
|
||||
github.com/k3s-io/etcd/api/v3 v3.5.0-k3s2 h1:56a/c/Mo75uU1k3erdgWk1C7dGUL5rSpNdL2B6zftKo=
|
||||
github.com/k3s-io/etcd/api/v3 v3.5.0-k3s2/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
|
||||
github.com/k3s-io/etcd/client/v3 v3.5.0-k3s2 h1:UUL/eMEf9utdaZNPwjdaupOwIgvq0sJyzvxgYWJQeJc=
|
||||
github.com/k3s-io/etcd/client/v3 v3.5.0-k3s2/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
|
||||
github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s2 h1:Feifl9EStGdmkUnOtouh0VD9n+UbgTxk8DLOsqUdKVo=
|
||||
github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s2/go.mod h1:o98rKMCibbFAG8QS9KmvlYDGDShmmIbmRE8vSofzYNg=
|
||||
github.com/k3s-io/etcd/server/v3 v3.5.0-k3s2 h1:yw8t2/k8Gwsv462XkEVawYGn5N+FI2xG97O3lleSSMI=
|
||||
github.com/k3s-io/etcd/server/v3 v3.5.0-k3s2/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
|
||||
github.com/k3s-io/helm-controller v0.11.3 h1:DSPAOCGHxF5pmF4vzQP5AgPT3tiGRJRZu+08hwWtbGI=
|
||||
github.com/k3s-io/helm-controller v0.11.3/go.mod h1:z0ExsRRIkTO/QC//3/Esn5ItTD6AiQSluwzMaS7RI/4=
|
||||
github.com/k3s-io/kine v0.8.0 h1:k6T9bI9DID7lIbktukXxg1QfeFoAQK4EIvAHoyPAe08=
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ type ControlRuntime struct {
|
|||
APIServerReady <-chan struct{}
|
||||
ETCDReady <-chan struct{}
|
||||
ClusterControllerStart func(ctx context.Context) error
|
||||
LeaderElectedClusterControllerStart func(ctx context.Context) error
|
||||
|
||||
ClientKubeAPICert string
|
||||
ClientKubeAPIKey string
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
package etcd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
NodeID = "etcd.k3s.cattle.io/node-name"
|
||||
NodeAddress = "etcd.k3s.cattle.io/node-address"
|
||||
master = "node-role.kubernetes.io/master"
|
||||
controlPlane = "node-role.kubernetes.io/control-plane"
|
||||
etcdRole = "node-role.kubernetes.io/etcd"
|
||||
)
|
||||
|
||||
type NodeControllerGetter func() controllerv1.NodeController
|
||||
|
||||
func Register(ctx context.Context, etcd *ETCD, nodes controllerv1.NodeController) {
|
||||
h := &handler{
|
||||
etcd: etcd,
|
||||
nodeController: nodes,
|
||||
ctx: ctx,
|
||||
}
|
||||
nodes.OnChange(ctx, "managed-etcd-controller", h.sync)
|
||||
nodes.OnRemove(ctx, "managed-etcd-controller", h.onRemove)
|
||||
}
|
||||
|
||||
type handler struct {
|
||||
etcd *ETCD
|
||||
nodeController controllerv1.NodeController
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (h *handler) sync(key string, node *v1.Node) (*v1.Node, error) {
|
||||
if node == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
nodeName := os.Getenv("NODE_NAME")
|
||||
if nodeName == "" {
|
||||
logrus.Debug("waiting for node to be assigned for etcd controller")
|
||||
h.nodeController.EnqueueAfter(key, 5*time.Second)
|
||||
return node, nil
|
||||
}
|
||||
|
||||
if key == nodeName {
|
||||
return h.handleSelf(node)
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
func (h *handler) handleSelf(node *v1.Node) (*v1.Node, error) {
|
||||
if node.Annotations[NodeID] == h.etcd.name &&
|
||||
node.Annotations[NodeAddress] == h.etcd.address &&
|
||||
node.Labels[etcdRole] == "true" &&
|
||||
node.Labels[controlPlane] == "true" ||
|
||||
h.etcd.config.DisableETCD {
|
||||
return node, nil
|
||||
}
|
||||
|
||||
node = node.DeepCopy()
|
||||
if node.Annotations == nil {
|
||||
node.Annotations = map[string]string{}
|
||||
}
|
||||
node.Annotations[NodeID] = h.etcd.name
|
||||
node.Annotations[NodeAddress] = h.etcd.address
|
||||
node.Labels[etcdRole] = "true"
|
||||
node.Labels[master] = "true"
|
||||
node.Labels[controlPlane] = "true"
|
||||
|
||||
return h.nodeController.Update(node)
|
||||
}
|
||||
|
||||
func (h *handler) onRemove(key string, node *v1.Node) (*v1.Node, error) {
|
||||
if _, ok := node.Labels[etcdRole]; !ok {
|
||||
return node, nil
|
||||
}
|
||||
|
||||
id := node.Annotations[NodeID]
|
||||
address, ok := node.Annotations[NodeAddress]
|
||||
if !ok {
|
||||
return node, nil
|
||||
}
|
||||
return node, h.etcd.removePeer(h.ctx, id, address, false)
|
||||
}
|
||||
|
|
@ -29,6 +29,7 @@ import (
|
|||
"github.com/rancher/k3s/pkg/daemons/control/deps"
|
||||
"github.com/rancher/k3s/pkg/daemons/executor"
|
||||
"github.com/rancher/k3s/pkg/version"
|
||||
controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
|
||||
"github.com/robfig/cron/v3"
|
||||
"github.com/sirupsen/logrus"
|
||||
"go.etcd.io/etcd/api/v3/etcdserverpb"
|
||||
|
|
@ -56,6 +57,10 @@ const (
|
|||
defaultKeepAliveTimeout = 10 * time.Second
|
||||
|
||||
maxBackupRetention = 5
|
||||
|
||||
MasterLabel = "node-role.kubernetes.io/master"
|
||||
ControlPlaneLabel = "node-role.kubernetes.io/control-plane"
|
||||
EtcdRoleLabel = "node-role.kubernetes.io/etcd"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -64,8 +69,13 @@ var (
|
|||
AddressKey = version.Program + "/apiaddresses"
|
||||
|
||||
snapshotConfigMapName = version.Program + "-etcd-snapshots"
|
||||
|
||||
NodeNameAnnotation = "etcd." + version.Program + ".cattle.io/node-name"
|
||||
NodeAddressAnnotation = "etcd." + version.Program + ".cattle.io/node-address"
|
||||
)
|
||||
|
||||
type NodeControllerGetter func() controllerv1.NodeController
|
||||
|
||||
type ETCD struct {
|
||||
client *clientv3.Client
|
||||
config *config.Control
|
||||
|
|
@ -373,14 +383,6 @@ func (e *ETCD) Register(ctx context.Context, config *config.Control, handler htt
|
|||
e.config.Datastore.BackendTLSConfig.CertFile = e.runtime.ClientETCDCert
|
||||
e.config.Datastore.BackendTLSConfig.KeyFile = e.runtime.ClientETCDKey
|
||||
|
||||
if err := e.setName(false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.config.Runtime.ClusterControllerStart = func(ctx context.Context) error {
|
||||
Register(ctx, e, e.config.Runtime.Core.Core().V1().Node())
|
||||
return nil
|
||||
}
|
||||
|
||||
tombstoneFile := filepath.Join(etcdDBDir(e.config), "tombstone")
|
||||
if _, err := os.Stat(tombstoneFile); err == nil {
|
||||
logrus.Infof("tombstone file has been detected, removing data dir to rejoin the cluster")
|
||||
|
|
@ -388,6 +390,20 @@ func (e *ETCD) Register(ctx context.Context, config *config.Control, handler htt
|
|||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := e.setName(false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.config.Runtime.ClusterControllerStart = func(ctx context.Context) error {
|
||||
RegisterMetadataHandlers(ctx, e, e.config.Runtime.Core.Core().V1().Node())
|
||||
return nil
|
||||
}
|
||||
|
||||
e.config.Runtime.LeaderElectedClusterControllerStart = func(ctx context.Context) error {
|
||||
RegisterMemberHandlers(ctx, e, e.config.Runtime.Core.Core().V1().Node())
|
||||
return nil
|
||||
}
|
||||
|
||||
return e.handler(handler), err
|
||||
}
|
||||
|
||||
|
|
@ -449,7 +465,7 @@ func (e *ETCD) infoHandler() http.Handler {
|
|||
})
|
||||
}
|
||||
|
||||
// getClient returns an etcd client connected to the specified endpoints
|
||||
// GetClient returns an etcd client connected to the specified endpoints
|
||||
func GetClient(ctx context.Context, runtime *config.ControlRuntime, endpoints ...string) (*clientv3.Client, error) {
|
||||
cfg, err := getClientConfig(ctx, runtime, endpoints...)
|
||||
if err != nil {
|
||||
|
|
@ -623,8 +639,8 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init
|
|||
})
|
||||
}
|
||||
|
||||
// removePeer removes a peer from the cluster. The peer ID and IP address must both match.
|
||||
func (e *ETCD) removePeer(ctx context.Context, id, address string, removeSelf bool) error {
|
||||
// RemovePeer removes a peer from the cluster. The peer name and IP address must both match.
|
||||
func (e *ETCD) RemovePeer(ctx context.Context, name, address string, allowSelfRemoval bool) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, memberRemovalTimeout)
|
||||
defer cancel()
|
||||
members, err := e.client.MemberList(ctx)
|
||||
|
|
@ -633,7 +649,7 @@ func (e *ETCD) removePeer(ctx context.Context, id, address string, removeSelf bo
|
|||
}
|
||||
|
||||
for _, member := range members.Members {
|
||||
if member.Name != id {
|
||||
if member.Name != name {
|
||||
continue
|
||||
}
|
||||
for _, peerURL := range member.PeerURLs {
|
||||
|
|
@ -642,8 +658,8 @@ func (e *ETCD) removePeer(ctx context.Context, id, address string, removeSelf bo
|
|||
return err
|
||||
}
|
||||
if u.Hostname() == address {
|
||||
if e.address == address && !removeSelf {
|
||||
return errors.New("node has been deleted from the cluster")
|
||||
if e.address == address && !allowSelfRemoval {
|
||||
return errors.New("not removing self from etcd cluster")
|
||||
}
|
||||
logrus.Infof("Removing name=%s id=%d address=%s from etcd", member.Name, member.ID, address)
|
||||
_, err := e.client.MemberRemove(ctx, member.ID)
|
||||
|
|
@ -1398,9 +1414,27 @@ func (e *ETCD) GetMembersClientURLs(ctx context.Context) ([]string, error) {
|
|||
return memberUrls, nil
|
||||
}
|
||||
|
||||
// GetMembersNames will list through the member lists in etcd and return
|
||||
// back a combined list of member names
|
||||
func (e *ETCD) GetMembersNames(ctx context.Context) ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, testTimeout)
|
||||
defer cancel()
|
||||
|
||||
members, err := e.client.MemberList(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var memberNames []string
|
||||
for _, member := range members.Members {
|
||||
memberNames = append(memberNames, member.Name)
|
||||
}
|
||||
return memberNames, nil
|
||||
}
|
||||
|
||||
// RemoveSelf will remove the member if it exists in the cluster
|
||||
func (e *ETCD) RemoveSelf(ctx context.Context) error {
|
||||
if err := e.removePeer(ctx, e.name, e.address, true); err != nil {
|
||||
if err := e.RemovePeer(ctx, e.name, e.address, true); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
package etcd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/rancher/k3s/pkg/version"
|
||||
controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func RegisterMemberHandlers(ctx context.Context, etcd *ETCD, nodes controllerv1.NodeController) {
|
||||
e := &etcdMemberHandler{
|
||||
etcd: etcd,
|
||||
nodeController: nodes,
|
||||
ctx: ctx,
|
||||
}
|
||||
nodes.OnChange(ctx, "managed-etcd-member-controller", e.sync)
|
||||
nodes.OnRemove(ctx, "managed-etcd-member-controller", e.onRemove)
|
||||
}
|
||||
|
||||
var (
|
||||
removalAnnotation = "etcd." + version.Program + ".cattle.io/remove"
|
||||
removedNodeNameAnnotation = "etcd." + version.Program + ".cattle.io/removed-node-name"
|
||||
)
|
||||
|
||||
type etcdMemberHandler struct {
|
||||
etcd *ETCD
|
||||
nodeController controllerv1.NodeController
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (e *etcdMemberHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
|
||||
if node == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if _, ok := node.Labels[EtcdRoleLabel]; !ok {
|
||||
logrus.Debugf("Node %s was not labeled etcd node, skipping sync", key)
|
||||
return node, nil
|
||||
}
|
||||
|
||||
node = node.DeepCopy()
|
||||
|
||||
if removalRequested, ok := node.Annotations[removalAnnotation]; ok {
|
||||
if removed, ok := node.Annotations[removedNodeNameAnnotation]; ok {
|
||||
// check to see if removed is true. if it is, nothing to do.
|
||||
if currentNodeName, ok := node.Annotations[NodeNameAnnotation]; ok {
|
||||
if currentNodeName != removed {
|
||||
// If the current node name is not the same as the removed node name, reset the tainted annotation and removed node name
|
||||
logrus.Infof("Resetting removed node flag as removed node name ( did not match current node name")
|
||||
delete(node.Annotations, removedNodeNameAnnotation)
|
||||
node.Annotations[removalAnnotation] = "false"
|
||||
return e.nodeController.Update(node)
|
||||
}
|
||||
// this is the case where the current node name matches the removed node name. We have already removed the
|
||||
// node, so no need to perform any action. Fallthrough to the non-op below.
|
||||
}
|
||||
// This is the edge case where the removed annotation exists, but there is not a current node name annotation.
|
||||
// This should be a non-op, as we can't remove the node anyway.
|
||||
logrus.Debugf("etcd member %s was already marked via annotations as removed", key)
|
||||
return node, nil
|
||||
}
|
||||
if strings.ToLower(removalRequested) == "true" {
|
||||
// remove the member.
|
||||
name, ok := node.Annotations[NodeNameAnnotation]
|
||||
if !ok {
|
||||
return node, fmt.Errorf("node name annotation for node %s not found", key)
|
||||
}
|
||||
address, ok := node.Annotations[NodeAddressAnnotation]
|
||||
if !ok {
|
||||
return node, fmt.Errorf("node address annotation for node %s not found", key)
|
||||
}
|
||||
|
||||
logrus.Debugf("removing etcd member from cluster name: %s address: %s", name, address)
|
||||
if err := e.etcd.RemovePeer(e.ctx, name, address, true); err != nil {
|
||||
return node, err
|
||||
}
|
||||
logrus.Debugf("etcd member removal successful for name: %s address: %s", name, address)
|
||||
// Set the removed node name annotation and clean up the other etcd node annotations.
|
||||
// These will be set if the tombstone file is then created and the etcd member is re-added, to their new
|
||||
// respective values.
|
||||
node.Annotations[removedNodeNameAnnotation] = name
|
||||
delete(node.Annotations, NodeNameAnnotation)
|
||||
delete(node.Annotations, NodeAddressAnnotation)
|
||||
return e.nodeController.Update(node)
|
||||
}
|
||||
// In the event that we had an unexpected removal value, simply return.
|
||||
// Fallthrough to the non-op below.
|
||||
}
|
||||
// This is a non-op, as we don't have a tainted annotation to worry about.
|
||||
return node, nil
|
||||
}
|
||||
|
||||
func (e *etcdMemberHandler) onRemove(key string, node *v1.Node) (*v1.Node, error) {
|
||||
if _, ok := node.Labels[EtcdRoleLabel]; !ok {
|
||||
logrus.Debugf("Node %s was not labeled etcd node, skipping etcd member removal", key)
|
||||
return node, nil
|
||||
}
|
||||
|
||||
name, ok := node.Annotations[NodeNameAnnotation]
|
||||
if !ok {
|
||||
return node, fmt.Errorf("node name annotation for node %s not found", key)
|
||||
}
|
||||
address, ok := node.Annotations[NodeAddressAnnotation]
|
||||
if !ok {
|
||||
return node, fmt.Errorf("node address annotation for node %s not found", key)
|
||||
}
|
||||
|
||||
return node, e.etcd.RemovePeer(e.ctx, name, address, true)
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
package etcd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
|
||||
"github.com/sirupsen/logrus"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func RegisterMetadataHandlers(ctx context.Context, etcd *ETCD, nodes controllerv1.NodeController) {
|
||||
h := &metadataHandler{
|
||||
etcd: etcd,
|
||||
nodeController: nodes,
|
||||
ctx: ctx,
|
||||
}
|
||||
nodes.OnChange(ctx, "managed-etcd-metadata-controller", h.sync)
|
||||
}
|
||||
|
||||
type metadataHandler struct {
|
||||
etcd *ETCD
|
||||
nodeController controllerv1.NodeController
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (m *metadataHandler) sync(key string, node *v1.Node) (*v1.Node, error) {
|
||||
if node == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
nodeName := os.Getenv("NODE_NAME")
|
||||
if nodeName == "" {
|
||||
logrus.Debug("waiting for node to be assigned for etcd controller")
|
||||
m.nodeController.EnqueueAfter(key, 5*time.Second)
|
||||
return node, nil
|
||||
}
|
||||
|
||||
if key == nodeName {
|
||||
return m.handleSelf(node)
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
func (m *metadataHandler) handleSelf(node *v1.Node) (*v1.Node, error) {
|
||||
if node.Annotations[NodeNameAnnotation] == m.etcd.name &&
|
||||
node.Annotations[NodeAddressAnnotation] == m.etcd.address &&
|
||||
node.Labels[EtcdRoleLabel] == "true" &&
|
||||
node.Labels[ControlPlaneLabel] == "true" ||
|
||||
m.etcd.config.DisableETCD {
|
||||
return node, nil
|
||||
}
|
||||
|
||||
node = node.DeepCopy()
|
||||
if node.Annotations == nil {
|
||||
node.Annotations = map[string]string{}
|
||||
}
|
||||
|
||||
node.Annotations[NodeNameAnnotation] = m.etcd.name
|
||||
node.Annotations[NodeAddressAnnotation] = m.etcd.address
|
||||
node.Labels[EtcdRoleLabel] = "true"
|
||||
node.Labels[MasterLabel] = "true"
|
||||
node.Labels[ControlPlaneLabel] = "true"
|
||||
|
||||
return m.nodeController.Update(node)
|
||||
}
|
||||
|
|
@ -80,14 +80,14 @@ func setETCDLabelsAndAnnotations(ctx context.Context, config *Config) error {
|
|||
continue
|
||||
}
|
||||
etcdNodeName := string(data)
|
||||
node.Annotations[etcd.NodeID] = etcdNodeName
|
||||
node.Annotations[etcd.NodeNameAnnotation] = etcdNodeName
|
||||
|
||||
address, err := etcd.GetAdvertiseAddress(controlConfig.PrivateIP)
|
||||
if err != nil {
|
||||
logrus.Infof("Waiting for etcd node address to be available: %v", err)
|
||||
continue
|
||||
}
|
||||
node.Annotations[etcd.NodeAddress] = address
|
||||
node.Annotations[etcd.NodeAddressAnnotation] = address
|
||||
|
||||
_, err = nodes.Update(node)
|
||||
if err == nil {
|
||||
|
|
|
|||
|
|
@ -154,6 +154,11 @@ func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) err
|
|||
if err := coreControllers(ctx, sc, config); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if controlConfig.Runtime.LeaderElectedClusterControllerStart != nil {
|
||||
if err := controlConfig.Runtime.LeaderElectedClusterControllerStart(ctx); err != nil {
|
||||
panic(errors.Wrap(err, "failed to start leader elected cluster controllers"))
|
||||
}
|
||||
}
|
||||
for _, controller := range config.LeaderControllers {
|
||||
if err := controller(ctx, sc); err != nil {
|
||||
panic(errors.Wrap(err, "leader controller"))
|
||||
|
|
|
|||
|
|
@ -803,11 +803,12 @@ func (c *RaftCluster) IsLocalMemberLearner() bool {
|
|||
defer c.Unlock()
|
||||
localMember, ok := c.members[c.localID]
|
||||
if !ok {
|
||||
c.lg.Panic(
|
||||
c.lg.Error(
|
||||
"failed to find local ID in cluster members",
|
||||
zap.String("cluster-id", c.cid.String()),
|
||||
zap.String("local-member-id", c.localID.String()),
|
||||
)
|
||||
return false
|
||||
}
|
||||
return localMember.IsLearner
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1144,7 +1144,7 @@ github.com/xlab/treeprint
|
|||
go.etcd.io/bbolt
|
||||
# go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738
|
||||
go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes
|
||||
# go.etcd.io/etcd/api/v3 v3.5.0 => github.com/k3s-io/etcd/api/v3 v3.5.0-k3s1
|
||||
# go.etcd.io/etcd/api/v3 v3.5.0 => github.com/k3s-io/etcd/api/v3 v3.5.0-k3s2
|
||||
## explicit
|
||||
go.etcd.io/etcd/api/v3/authpb
|
||||
go.etcd.io/etcd/api/v3/etcdserverpb
|
||||
|
|
@ -1164,7 +1164,7 @@ go.etcd.io/etcd/client/pkg/v3/transport
|
|||
go.etcd.io/etcd/client/pkg/v3/types
|
||||
# go.etcd.io/etcd/client/v2 v2.305.0
|
||||
go.etcd.io/etcd/client/v2
|
||||
# go.etcd.io/etcd/client/v3 v3.5.0 => github.com/k3s-io/etcd/client/v3 v3.5.0-k3s1
|
||||
# go.etcd.io/etcd/client/v3 v3.5.0 => github.com/k3s-io/etcd/client/v3 v3.5.0-k3s2
|
||||
## explicit
|
||||
go.etcd.io/etcd/client/v3
|
||||
go.etcd.io/etcd/client/v3/concurrency
|
||||
|
|
@ -1172,7 +1172,7 @@ go.etcd.io/etcd/client/v3/credentials
|
|||
go.etcd.io/etcd/client/v3/internal/endpoint
|
||||
go.etcd.io/etcd/client/v3/internal/resolver
|
||||
go.etcd.io/etcd/client/v3/snapshot
|
||||
# go.etcd.io/etcd/etcdutl/v3 v3.5.0 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s1
|
||||
# go.etcd.io/etcd/etcdutl/v3 v3.5.0 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s2
|
||||
## explicit
|
||||
go.etcd.io/etcd/etcdutl/v3/snapshot
|
||||
# go.etcd.io/etcd/pkg/v3 v3.5.0
|
||||
|
|
@ -1197,7 +1197,7 @@ go.etcd.io/etcd/raft/v3/confchange
|
|||
go.etcd.io/etcd/raft/v3/quorum
|
||||
go.etcd.io/etcd/raft/v3/raftpb
|
||||
go.etcd.io/etcd/raft/v3/tracker
|
||||
# go.etcd.io/etcd/server/v3 v3.5.0 => github.com/k3s-io/etcd/server/v3 v3.5.0-k3s1
|
||||
# go.etcd.io/etcd/server/v3 v3.5.0 => github.com/k3s-io/etcd/server/v3 v3.5.0-k3s2
|
||||
## explicit
|
||||
go.etcd.io/etcd/server/v3/auth
|
||||
go.etcd.io/etcd/server/v3/config
|
||||
|
|
@ -3408,10 +3408,10 @@ sigs.k8s.io/yaml
|
|||
# github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.3-0.20210316141917-a8c4a9ee0f6b
|
||||
# github.com/rancher/k3s/pkg/data => ./pkg/data
|
||||
# github.com/rancher/wrangler => github.com/rancher/wrangler v0.8.5
|
||||
# go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.0-k3s1
|
||||
# go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.0-k3s1
|
||||
# go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s1
|
||||
# go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.0-k3s1
|
||||
# go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.0-k3s2
|
||||
# go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.0-k3s2
|
||||
# go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.0-k3s2
|
||||
# go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.0-k3s2
|
||||
# golang.org/x/crypto => golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
|
||||
# golang.org/x/net => golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
|
||||
# golang.org/x/sys => golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
|
||||
|
|
|
|||
Loading…
Reference in New Issue