mirror of https://github.com/hashicorp/consul
Browse Source
* Implement leader routine manager Switch over the following to use it for go routine management: • Config entry Replication • ACL replication - tokens, policies, roles and legacy tokens • ACL legacy token upgrade • ACL token reaping • Intention Replication • Secondary CA Roots Watching • CA Root Pruning Also added the StopAll call into the Server Shutdown method to ensure all leader routines get killed off when shutting down. This should be mostly unnecessary as `revokeLeadership` should manually stop each one but just in case we really want these to go away (eventually).pull/6581/head
Matt Keeler
5 years ago
committed by
GitHub
8 changed files with 478 additions and 349 deletions
@ -0,0 +1,120 @@
|
||||
package consul |
||||
|
||||
import ( |
||||
"context" |
||||
"log" |
||||
"os" |
||||
"sync" |
||||
) |
||||
|
||||
type LeaderRoutine func(ctx context.Context) error |
||||
|
||||
type leaderRoutine struct { |
||||
running bool |
||||
cancel context.CancelFunc |
||||
} |
||||
|
||||
type LeaderRoutineManager struct { |
||||
lock sync.RWMutex |
||||
logger *log.Logger |
||||
|
||||
routines map[string]*leaderRoutine |
||||
} |
||||
|
||||
func NewLeaderRoutineManager(logger *log.Logger) *LeaderRoutineManager { |
||||
if logger == nil { |
||||
logger = log.New(os.Stderr, "", log.LstdFlags) |
||||
} |
||||
|
||||
return &LeaderRoutineManager{ |
||||
logger: logger, |
||||
routines: make(map[string]*leaderRoutine), |
||||
} |
||||
} |
||||
|
||||
func (m *LeaderRoutineManager) IsRunning(name string) bool { |
||||
m.lock.Lock() |
||||
defer m.lock.Unlock() |
||||
|
||||
if routine, ok := m.routines[name]; ok { |
||||
return routine.running |
||||
} |
||||
|
||||
return false |
||||
} |
||||
|
||||
func (m *LeaderRoutineManager) Start(name string, routine LeaderRoutine) error { |
||||
return m.StartWithContext(nil, name, routine) |
||||
} |
||||
|
||||
func (m *LeaderRoutineManager) StartWithContext(parentCtx context.Context, name string, routine LeaderRoutine) error { |
||||
m.lock.Lock() |
||||
defer m.lock.Unlock() |
||||
|
||||
if instance, ok := m.routines[name]; ok && instance.running { |
||||
return nil |
||||
} |
||||
|
||||
if parentCtx == nil { |
||||
parentCtx = context.Background() |
||||
} |
||||
|
||||
ctx, cancel := context.WithCancel(parentCtx) |
||||
instance := &leaderRoutine{ |
||||
running: true, |
||||
cancel: cancel, |
||||
} |
||||
|
||||
go func() { |
||||
err := routine(ctx) |
||||
if err != nil && err != context.DeadlineExceeded && err != context.Canceled { |
||||
m.logger.Printf("[ERROR] leader: %s routine exited with error: %v", name, err) |
||||
} else { |
||||
m.logger.Printf("[DEBUG] leader: stopped %s routine", name) |
||||
} |
||||
|
||||
m.lock.Lock() |
||||
instance.running = false |
||||
m.lock.Unlock() |
||||
}() |
||||
|
||||
m.routines[name] = instance |
||||
m.logger.Printf("[INFO] leader: started %s routine", name) |
||||
return nil |
||||
} |
||||
|
||||
func (m *LeaderRoutineManager) Stop(name string) error { |
||||
m.lock.Lock() |
||||
defer m.lock.Unlock() |
||||
|
||||
instance, ok := m.routines[name] |
||||
if !ok { |
||||
// no running instance
|
||||
return nil |
||||
} |
||||
|
||||
if !instance.running { |
||||
return nil |
||||
} |
||||
|
||||
m.logger.Printf("[DEBUG] leader: stopping %s routine", name) |
||||
instance.cancel() |
||||
delete(m.routines, name) |
||||
return nil |
||||
} |
||||
|
||||
func (m *LeaderRoutineManager) StopAll() { |
||||
m.lock.Lock() |
||||
defer m.lock.Unlock() |
||||
|
||||
for name, routine := range m.routines { |
||||
if !routine.running { |
||||
continue |
||||
} |
||||
m.logger.Printf("[DEBUG] leader: stopping %s routine", name) |
||||
routine.cancel() |
||||
} |
||||
|
||||
// just whipe out the entire map
|
||||
m.routines = make(map[string]*leaderRoutine) |
||||
} |
||||
@ -0,0 +1,73 @@
|
||||
package consul |
||||
|
||||
import ( |
||||
"context" |
||||
"sync/atomic" |
||||
"testing" |
||||
|
||||
"github.com/hashicorp/consul/sdk/testutil" |
||||
"github.com/hashicorp/consul/sdk/testutil/retry" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestLeaderRoutineManager(t *testing.T) { |
||||
t.Parallel() |
||||
var runs uint32 |
||||
var running uint32 |
||||
// tlog := testutil.NewCancellableTestLogger(t)
|
||||
// defer tlog.Cancel()
|
||||
mgr := NewLeaderRoutineManager(testutil.TestLogger(t)) |
||||
|
||||
run := func(ctx context.Context) error { |
||||
atomic.StoreUint32(&running, 1) |
||||
defer atomic.StoreUint32(&running, 0) |
||||
atomic.AddUint32(&runs, 1) |
||||
<-ctx.Done() |
||||
return nil |
||||
} |
||||
|
||||
// IsRunning on unregistered service should be false
|
||||
require.False(t, mgr.IsRunning("not-found")) |
||||
|
||||
// start
|
||||
require.NoError(t, mgr.Start("run", run)) |
||||
require.True(t, mgr.IsRunning("run")) |
||||
retry.Run(t, func(r *retry.R) { |
||||
require.Equal(r, uint32(1), atomic.LoadUint32(&runs)) |
||||
require.Equal(r, uint32(1), atomic.LoadUint32(&running)) |
||||
}) |
||||
require.NoError(t, mgr.Stop("run")) |
||||
|
||||
// ensure the background go routine was actually cancelled
|
||||
retry.Run(t, func(r *retry.R) { |
||||
require.Equal(r, uint32(1), atomic.LoadUint32(&runs)) |
||||
require.Equal(r, uint32(0), atomic.LoadUint32(&running)) |
||||
}) |
||||
|
||||
// restart and stop
|
||||
require.NoError(t, mgr.Start("run", run)) |
||||
retry.Run(t, func(r *retry.R) { |
||||
require.Equal(r, uint32(2), atomic.LoadUint32(&runs)) |
||||
require.Equal(r, uint32(1), atomic.LoadUint32(&running)) |
||||
}) |
||||
|
||||
require.NoError(t, mgr.Stop("run")) |
||||
retry.Run(t, func(r *retry.R) { |
||||
require.Equal(r, uint32(0), atomic.LoadUint32(&running)) |
||||
}) |
||||
|
||||
// start with a context
|
||||
ctx, cancel := context.WithCancel(context.Background()) |
||||
require.NoError(t, mgr.StartWithContext(ctx, "run", run)) |
||||
cancel() |
||||
|
||||
// The function should exit of its own accord due to the parent
|
||||
// context being canceled
|
||||
retry.Run(t, func(r *retry.R) { |
||||
require.Equal(r, uint32(3), atomic.LoadUint32(&runs)) |
||||
require.Equal(r, uint32(0), atomic.LoadUint32(&running)) |
||||
// the task should automatically set itself to not running if
|
||||
// it exits early
|
||||
require.False(r, mgr.IsRunning("run")) |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue