From 75b49bddae18e7469a6fbf4308a96265fee5a627 Mon Sep 17 00:00:00 2001 From: Doflatango Date: Tue, 13 Mar 2018 16:06:43 +0800 Subject: [PATCH] Improve node warning message --- node.go | 5 +++++ noticer.go | 19 ++++++++----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/node.go b/node.go index 6952bc0..fffb8a3 100644 --- a/node.go +++ b/node.go @@ -92,6 +92,11 @@ func GetNodesBy(query interface{}) (nodes []*Node, err error) { return } +func GetNodesByID(id string) (node *Node, err error) { + err = mgoDB.FindId(Coll_Node, id, &node) + return +} + func RemoveNode(query interface{}) error { return mgoDB.WithC(Coll_Node, func(c *mgo.Collection) error { return c.Remove(query) diff --git a/noticer.go b/noticer.go index 53377ea..b360a6e 100644 --- a/noticer.go +++ b/noticer.go @@ -169,28 +169,25 @@ func StartNoticer(n Noticer) { } func monitorNodes(n Noticer) { - var ( - err error - ok bool - id string - ) rch := WatchNode() for wresp := range rch { for _, ev := range wresp.Events { switch { case ev.Type == client.EventTypeDelete: - id = GetIDFromKey(string(ev.Kv.Key)) - ok, err = ISNodeAlive(id) + id := GetIDFromKey(string(ev.Kv.Key)) + log.Errorf("cronnode DELETE event detected, node UUID: %s", id) + + node, err := GetNodesByID(id) if err != nil { - log.Warnf("query node[%s] err: %s", id, err.Error()) + log.Warnf("failed to fetch node[%s] from mongodb: %s", id, err.Error()) continue } - if ok { + if node.Alived { n.Send(&Message{ - Subject: "Node[" + id + "] break away cluster, this happed at " + time.Now().Format(time.RFC3339), - Body: "Node breaked away cluster, this might happed when node crash or network problems.", + Subject: fmt.Sprintf("Node[%s] break away cluster at %s", node.Hostname, time.Now().Format(time.RFC3339)), + Body: fmt.Sprintf("Node breaked away cluster, this might happened when node crash or network problems.\nUUID: %s\nHostname: %s\nIP: %s\n", id, node.Hostname, node.IP), To: conf.Config.Mail.To, }) }