Fixes panic when timer fires as tombstone GC is being stopped.

consul/state/tombstone_gc.go

@@ -145,12 +145,17 @@ func (t *TombstoneGC) nextExpires() time.Time {
 
 // expireTime is used to expire the entries at the given time.
 func (t *TombstoneGC) expireTime(expires time.Time) {
-	// Get the maximum index and clear the entry
 	t.lock.Lock()
-	exp := t.expires[expires]
+	defer t.lock.Unlock()
-	delete(t.expires, expires)
-	t.lock.Unlock()
 
-	// Notify the expires channel
+	// Get the maximum index and clear the entry. It's possible that the GC
+	// has been shut down while this timer fired and got blocked on the lock,
+	// so if there's nothing in the map for us we just exit out since there
+	// is no work to do.
+	exp, ok := t.expires[expires]
+	if !ok {
+		return
+	}
+	delete(t.expires, expires)
 	t.expireCh <- exp.maxIndex
 }

consul/state/tombstone_gc_test.go

@@ -102,3 +102,18 @@ func TestTombstoneGC_Expire(t *testing.T) {
 	case <-time.After(20 * time.Millisecond):
 	}
 }
+
+func TestTombstoneGC_TimerAfterShutdown(t *testing.T) {
+	ttl := 2 * time.Microsecond
+	gran := 1 * time.Microsecond
+	gc, err := NewTombstoneGC(ttl, gran)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	for i := 0; i < 1000; i++ {
+		gc.SetEnabled(true)
+		gc.Hint(100 + uint64(i))
+		gc.SetEnabled(false)
+	}
+}