From 8c157e0acd44ddc3e4a6929228d8e826f1df71c9 Mon Sep 17 00:00:00 2001
From: Kyle Havlovitz <kylehav@gmail.com>
Date: Sat, 5 Nov 2016 00:53:22 -0400
Subject: [PATCH] Retry with backoff on session invalidation failure (#2475)

---
 consul/session_ttl.go      | 21 ++++++++++++++++++---
 consul/session_ttl_test.go |  2 ++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/consul/session_ttl.go b/consul/session_ttl.go
index 172ef945e9..c8ce91283c 100644
--- a/consul/session_ttl.go
+++ b/consul/session_ttl.go
@@ -8,6 +8,14 @@ import (
 	"github.com/hashicorp/consul/consul/structs"
 )
 
+const (
+	// maxInvalidateAttempts limits how many invalidate attempts are made
+	maxInvalidateAttempts = 6
+
+	// invalidateRetryBase is a baseline retry time
+	invalidateRetryBase = 10 * time.Second
+)
+
 // initializeSessionTimers is used when a leader is newly elected to create
 // a new map to track session expiration and to reset all the timers from
 // the previously known set of timers.
@@ -110,12 +118,19 @@ func (s *Server) invalidateSession(id string) {
 			ID: id,
 		},
 	}
-	s.logger.Printf("[DEBUG] consul.state: Session %s TTL expired", id)
 
-	// Apply the update to destroy the session
-	if _, err := s.raftApply(structs.SessionRequestType, args); err != nil {
+	// Retry with exponential backoff to invalidate the session
+	for attempt := uint(0); attempt < maxInvalidateAttempts; attempt++ {
+		_, err := s.raftApply(structs.SessionRequestType, args)
+		if err == nil {
+			s.logger.Printf("[DEBUG] consul.state: Session %s TTL expired", id)
+			return
+		}
+
 		s.logger.Printf("[ERR] consul.session: Invalidation failed: %v", err)
+		time.Sleep((1 << attempt) * invalidateRetryBase)
 	}
+	s.logger.Printf("[ERR] consul.session: maximum revoke attempts reached for session: %s", id)
 }
 
 // clearSessionTimer is used to clear the session time for
diff --git a/consul/session_ttl_test.go b/consul/session_ttl_test.go
index 79bd094771..7ef647beb4 100644
--- a/consul/session_ttl_test.go
+++ b/consul/session_ttl_test.go
@@ -143,6 +143,8 @@ func TestResetSessionTimerLocked(t *testing.T) {
 	defer os.RemoveAll(dir1)
 	defer s1.Shutdown()
 
+	testutil.WaitForLeader(t, s1.RPC, "dc1")
+
 	s1.sessionTimersLock.Lock()
 	s1.resetSessionTimerLocked("foo", 5*time.Millisecond)
 	s1.sessionTimersLock.Unlock()