Retry with backoff on session invalidation failure (#2475)

consul/session_ttl.go

@@ -8,6 +8,14 @@ import (
 	"github.com/hashicorp/consul/consul/structs"
 )
 
+const (
+	// maxInvalidateAttempts limits how many invalidate attempts are made
+	maxInvalidateAttempts = 6
+
+	// invalidateRetryBase is a baseline retry time
+	invalidateRetryBase = 10 * time.Second
+)
+
 // initializeSessionTimers is used when a leader is newly elected to create
 // a new map to track session expiration and to reset all the timers from
 // the previously known set of timers.
@@ -110,12 +118,19 @@ func (s *Server) invalidateSession(id string) {
 			ID: id,
 		},
 	}
-	s.logger.Printf("[DEBUG] consul.state: Session %s TTL expired", id)
 
-	// Apply the update to destroy the session
+	// Retry with exponential backoff to invalidate the session
-	if _, err := s.raftApply(structs.SessionRequestType, args); err != nil {
+	for attempt := uint(0); attempt < maxInvalidateAttempts; attempt++ {
+		_, err := s.raftApply(structs.SessionRequestType, args)
+		if err == nil {
+			s.logger.Printf("[DEBUG] consul.state: Session %s TTL expired", id)
+			return
+		}
+
 		s.logger.Printf("[ERR] consul.session: Invalidation failed: %v", err)
+		time.Sleep((1 << attempt) * invalidateRetryBase)
 	}
+	s.logger.Printf("[ERR] consul.session: maximum revoke attempts reached for session: %s", id)
 }
 
 // clearSessionTimer is used to clear the session time for

consul/session_ttl_test.go

@@ -143,6 +143,8 @@ func TestResetSessionTimerLocked(t *testing.T) {
 	defer os.RemoveAll(dir1)
 	defer s1.Shutdown()
 
+	testutil.WaitForLeader(t, s1.RPC, "dc1")
+
 	s1.sessionTimersLock.Lock()
 	s1.resetSessionTimerLocked("foo", 5*time.Millisecond)
 	s1.sessionTimersLock.Unlock()