consul/agent/service_manager.go

package agent

import (
	"fmt"
	"sync"

	"github.com/hashicorp/consul/agent/cache"
	cachetype "github.com/hashicorp/consul/agent/cache-types"
	"github.com/hashicorp/consul/agent/structs"
	"github.com/imdario/mergo"
	"github.com/mitchellh/copystructure"
	"golang.org/x/net/context"
)

// The ServiceManager is a layer for service registration in between the agent
// and the local state. Any services must be registered with the ServiceManager,
// which then maintains a long-running watch of any globally-set service or proxy
// configuration that applies to the service in order to register the final, merged
// service configuration locally in the agent state.
type ServiceManager struct {
	agent *Agent

	// servicesLock guards the services map, but not the watches contained
	// therein
	servicesLock sync.Mutex

	// services tracks all active watches for registered services
	services map[string]*serviceConfigWatch

	// registerCh is a channel for processing service registrations in the
	// background when watches are notified of changes. All sends and receives
	// must also obey the ctx.Done() channel to avoid a deadlock during
	// shutdown.
	registerCh chan *asyncRegisterRequest

	// ctx is the shared context for all goroutines launched
	ctx context.Context

	// cancel can be used to stop all goroutines launched
	cancel context.CancelFunc

	// running keeps track of live goroutines (worker and watcher)
	running sync.WaitGroup
}

func NewServiceManager(agent *Agent) *ServiceManager {
	ctx, cancel := context.WithCancel(context.Background())
	return &ServiceManager{
		agent:      agent,
		services:   make(map[string]*serviceConfigWatch),
		registerCh: make(chan *asyncRegisterRequest), // must be unbuffered
		ctx:        ctx,
		cancel:     cancel,
	}
}

// Stop forces all background goroutines to terminate and blocks until they complete.
//
// NOTE: the caller must NOT hold the Agent.stateLock!
func (s *ServiceManager) Stop() {
	s.cancel()
	s.running.Wait()
}

// Start starts a background worker goroutine that writes back into the Agent
// state. This only exists to keep the need to lock the agent state lock out of
// the main AddService/RemoveService codepaths to avoid deadlocks.
func (s *ServiceManager) Start() {
	s.running.Add(1)

	go func() {
		defer s.running.Done()
		for {
			select {
			case <-s.ctx.Done():
				return
			case req := <-s.registerCh:
				req.Reply <- s.registerOnce(req.Args)
			}
		}
	}()
}

// runOnce will process a single registration request
func (s *ServiceManager) registerOnce(args *addServiceRequest) error {
	s.agent.stateLock.Lock()
	defer s.agent.stateLock.Unlock()

	err := s.agent.addServiceInternal(args)
	if err != nil {
		return fmt.Errorf("error updating service registration: %v", err)
	}
	return nil
}

// AddService will (re)create a serviceConfigWatch on the given service. For
// each call of this function the first registration will happen inline and
// will read the merged global defaults for the service through the agent cache
// (regardless of whether or not the service was already registered).  This
// lets validation or authorization related errors bubble back up to the
// caller's RPC inline with their request. Upon success a goroutine will keep
// this updated in the background.
//
// If waitForCentralConfig=true is used, the initial registration blocks on
// fetching the merged global config through the cache. If false, no such RPC
// occurs and only the previousDefaults are used.
//
// persistServiceConfig controls if the INITIAL registration will result in
// persisting the service config to disk again. All background updates will
// always persist.
//
// service, chkTypes, persist, token, replaceExistingChecks, and source are
// basically pass-through arguments to Agent.addServiceInternal that follow the
// semantics there. The one key difference is that the service provided will be
// merged with the global defaults before registration.
//
// NOTE: the caller must hold the Agent.stateLock!
func (s *ServiceManager) AddService(req *addServiceRequest) error {
	req.fixupForAddServiceLocked()

	// For now only sidecar proxies have anything that can be configured
	// centrally. So bypass the whole manager for regular services.
	if !req.service.IsSidecarProxy() && !req.service.IsMeshGateway() {
		// previousDefaults are ignored here because they are only relevant for central config.
		req.persistService = nil
		req.persistDefaults = nil
		req.persistServiceConfig = false
		return s.agent.addServiceInternal(req)
	}

	var (
		service               = req.service
		chkTypes              = req.chkTypes
		previousDefaults      = req.previousDefaults
		waitForCentralConfig  = req.waitForCentralConfig
		persist               = req.persist
		persistServiceConfig  = req.persistServiceConfig
		token                 = req.token
		replaceExistingChecks = req.replaceExistingChecks
		source                = req.source
	)

	reg := &serviceRegistration{
		service:               service,
		chkTypes:              chkTypes,
		persist:               persist,
		token:                 token,
		replaceExistingChecks: replaceExistingChecks,
		source:                source,
	}

	s.servicesLock.Lock()
	defer s.servicesLock.Unlock()

	// If a service watch already exists, shut it down and replace it.
	oldWatch, updating := s.services[service.ID]
	if updating {
		oldWatch.Stop()
		delete(s.services, service.ID)
	}

	// Get the existing global config and do the initial registration with the
	// merged config.
	watch := &serviceConfigWatch{
		registration: reg,
		updateCh:     make(chan cache.UpdateEvent, 1),
		agent:        s.agent,
		registerCh:   s.registerCh,
	}

	err := watch.RegisterAndStart(
		previousDefaults,
		waitForCentralConfig,
		persistServiceConfig,
		s.ctx,
		&s.running,
	)
	if err != nil {
		return err
	}

	s.services[service.ID] = watch

	if updating {
		s.agent.logger.Printf("[DEBUG] agent.manager: updated local registration for service %q", service.ID)
	} else {
		s.agent.logger.Printf("[DEBUG] agent.manager: added local registration for service %q", service.ID)
	}

	return nil
}

// NOTE: the caller must hold the Agent.stateLock!
func (s *ServiceManager) RemoveService(serviceID string) {
	s.servicesLock.Lock()
	defer s.servicesLock.Unlock()

	if oldWatch, exists := s.services[serviceID]; exists {
		oldWatch.Stop()
		delete(s.services, serviceID)
	}
}

// serviceRegistration represents a locally registered service.
type serviceRegistration struct {
	service               *structs.NodeService
	chkTypes              []*structs.CheckType
	persist               bool
	token                 string
	replaceExistingChecks bool
	source                configSource
}

// serviceConfigWatch is a long running helper for composing the end config
// for a given service from both the local registration and the global
// service/proxy defaults.
type serviceConfigWatch struct {
	registration *serviceRegistration
	defaults     *structs.ServiceConfigResponse

	agent      *Agent
	registerCh chan<- *asyncRegisterRequest

	// cacheKey stores the key of the current request, when registration changes
	// we check to see if a new cache watch is needed.
	cacheKey string

	// updateCh receives changes from cache watchers
	updateCh chan cache.UpdateEvent

	ctx        context.Context
	cancelFunc func()
	running    sync.WaitGroup
}

// NOTE: this is called while holding the Agent.stateLock
func (w *serviceConfigWatch) RegisterAndStart(
	previousDefaults *structs.ServiceConfigResponse,
	waitForCentralConfig bool,
	persistServiceConfig bool,
	ctx context.Context,
	wg *sync.WaitGroup,
) error {
	service := w.registration.service

	// Either we explicitly block waiting for defaults before registering,
	// or we feed it some seed data (or NO data) and bypass the blocking
	// operation. Either way the watcher will end up with something flagged
	// as defaults even if they don't actually reflect actual defaults.
	if waitForCentralConfig {
		if err := w.fetchDefaults(); err != nil {
			return fmt.Errorf("could not retrieve initial service_defaults config for service %q: %v", service.ID, err)
		}
	} else {
		w.defaults = previousDefaults
	}

	// Merge the local registration with the central defaults and update this service
	// in the local state.
	merged, err := w.mergeServiceConfig()
	if err != nil {
		return err
	}

	// The first time we do this interactively, we need to know if it
	// failed for validation reasons which we only get back from the
	// initial underlying add service call.
	err = w.agent.addServiceInternal(&addServiceRequest{
		service:               merged,
		chkTypes:              w.registration.chkTypes,
		persistService:        w.registration.service,
		persistDefaults:       w.defaults,
		persist:               w.registration.persist,
		persistServiceConfig:  persistServiceConfig,
		token:                 w.registration.token,
		replaceExistingChecks: w.registration.replaceExistingChecks,
		source:                w.registration.source,
	})
	if err != nil {
		return fmt.Errorf("error updating service registration: %v", err)
	}

	// Start the config watch, which starts a blocking query for the
	// resolved service config in the background.
	return w.start(ctx, wg)
}

// NOTE: this is called while holding the Agent.stateLock
func (w *serviceConfigWatch) fetchDefaults() error {
	req := makeConfigRequest(w.agent, w.registration)

	raw, _, err := w.agent.cache.Get(cachetype.ResolvedServiceConfigName, req)
	if err != nil {
		return err
	}

	reply, ok := raw.(*structs.ServiceConfigResponse)
	if !ok {
		// This should never happen, but we want to protect against panics
		return fmt.Errorf("internal error: response type not correct")
	}

	w.defaults = reply
	return nil
}

// Start starts the config watch and a goroutine to handle updates over the
// updateCh. This is safe to call more than once assuming you have called Stop
// after each Start.
//
// NOTE: this is called while holding the Agent.stateLock
func (w *serviceConfigWatch) start(ctx context.Context, wg *sync.WaitGroup) error {
	w.ctx, w.cancelFunc = context.WithCancel(ctx)

	// Configure and start a cache.Notify goroutine to run a continuous
	// blocking query on the resolved service config for this service.
	req := makeConfigRequest(w.agent, w.registration)
	w.cacheKey = req.CacheInfo().Key

	// We use the cache key as the correlationID here. Notify in general will not
	// respond on the updateCh after the context is cancelled however there could
	// possible be a race where it has only just got an update and checked the
	// context before we cancel and so might still deliver the old event. Using
	// the cacheKey allows us to ignore updates from the old cache watch and makes
	// even this rare edge case safe.
	err := w.agent.cache.Notify(
		w.ctx,
		cachetype.ResolvedServiceConfigName,
		req,
		w.cacheKey,
		w.updateCh,
	)
	if err != nil {
		w.cancelFunc()
		return err
	}

	w.running.Add(1)
	wg.Add(1)
	go w.runWatch(wg)

	return nil
}

func (w *serviceConfigWatch) Stop() {
	w.cancelFunc()
	w.running.Wait()
}

// runWatch handles any update events from the cache.Notify until the
// config watch is shut down.
//
// NOTE: the caller must NOT hold the Agent.stateLock!
func (w *serviceConfigWatch) runWatch(wg *sync.WaitGroup) {
	defer wg.Done()
	defer w.running.Done()

	for {
		select {
		case <-w.ctx.Done():
			return
		case event := <-w.updateCh:
			if err := w.handleUpdate(event); err != nil {
				w.agent.logger.Printf("[ERR] agent.manager: error handling service update: %v", err)
			}
		}
	}
}

// handleUpdate receives an update event the global config defaults, updates
// the local state and re-registers the service with the newly merged config.
//
// NOTE: the caller must NOT hold the Agent.stateLock!
func (w *serviceConfigWatch) handleUpdate(event cache.UpdateEvent) error {
	// If we got an error, log a warning if this is the first update; otherwise return the error.
	// We want the initial update to cause a service registration no matter what.
	if event.Err != nil {
		return fmt.Errorf("error watching service config: %v", event.Err)
	}

	res, ok := event.Result.(*structs.ServiceConfigResponse)
	if !ok {
		return fmt.Errorf("unknown update event type: %T", event)
	}

	// Sanity check this even came from the currently active watch to ignore
	// rare races when switching cache keys
	if event.CorrelationID != w.cacheKey {
		// It's a no-op. The new watcher will deliver (or may have already
		// delivered) the correct config so just ignore this old message.
		return nil
	}
	w.defaults = res

	// Merge the local registration with the central defaults and update this service
	// in the local state.
	merged, err := w.mergeServiceConfig()
	if err != nil {
		return err
	}

	// While we were waiting on the agent state lock we may have been shutdown.
	// So avoid doing a registration in that case.
	select {
	case <-w.ctx.Done():
		return nil
	default:
	}

	registerReq := &asyncRegisterRequest{
		Args: &addServiceRequest{
			service:               merged,
			chkTypes:              w.registration.chkTypes,
			persistService:        w.registration.service,
			persistDefaults:       w.defaults,
			persist:               w.registration.persist,
			persistServiceConfig:  true,
			token:                 w.registration.token,
			replaceExistingChecks: w.registration.replaceExistingChecks,
			source:                w.registration.source,
		},
		Reply: make(chan error, 1),
	}

	select {
	case <-w.ctx.Done():
		return nil
	case w.registerCh <- registerReq:
	}

	select {
	case <-w.ctx.Done():
		return nil

	case err := <-registerReq.Reply:
		if err != nil {
			return fmt.Errorf("error updating service registration: %v", err)
		}
		return nil
	}
}

type asyncRegisterRequest struct {
	Args  *addServiceRequest
	Reply chan error
}

func makeConfigRequest(agent *Agent, registration *serviceRegistration) *structs.ServiceConfigRequest {
	ns := registration.service
	name := ns.Service
	var upstreams []string

	// Note that only sidecar proxies should even make it here for now although
	// later that will change to add the condition.
	if ns.IsSidecarProxy() {
		// This is a sidecar proxy, ignore the proxy service's config since we are
		// managed by the target service config.
		name = ns.Proxy.DestinationServiceName

		// Also if we have any upstreams defined, add them to the request so we can
		// learn about their configs.
		for _, us := range ns.Proxy.Upstreams {
			if us.DestinationType == "" || us.DestinationType == structs.UpstreamDestTypeService {
				upstreams = append(upstreams, us.DestinationName)
			}
		}
	}

	req := &structs.ServiceConfigRequest{
		Name:         name,
		Datacenter:   agent.config.Datacenter,
		QueryOptions: structs.QueryOptions{Token: agent.tokens.AgentToken()},
		Upstreams:    upstreams,
	}
	if registration.token != "" {
		req.QueryOptions.Token = registration.token
	}
	return req
}

// mergeServiceConfig returns the final effective config for the watched service,
// including the latest known global defaults from the servers.
//
// NOTE: this is called while holding the Agent.stateLock
func (w *serviceConfigWatch) mergeServiceConfig() (*structs.NodeService, error) {
	if w.defaults == nil {
		return w.registration.service, nil
	}

	// We don't want to change s.registration in place since it is our source of
	// truth about what was actually registered before defaults applied. So copy
	// it first.
	nsRaw, err := copystructure.Copy(w.registration.service)
	if err != nil {
		return nil, err
	}

	// Merge proxy defaults
	ns := nsRaw.(*structs.NodeService)

	if err := mergo.Merge(&ns.Proxy.Config, w.defaults.ProxyConfig); err != nil {
		return nil, err
	}

	if err := mergo.Merge(&ns.Proxy.Expose, w.defaults.Expose); err != nil {
		return nil, err
	}

	if ns.Proxy.MeshGateway.Mode == structs.MeshGatewayModeDefault {
		ns.Proxy.MeshGateway.Mode = w.defaults.MeshGateway.Mode
	}

	// Merge upstream defaults if there were any returned
	for i := range ns.Proxy.Upstreams {
		// Get a pointer not a value copy of the upstream struct
		us := &ns.Proxy.Upstreams[i]
		if us.DestinationType != "" && us.DestinationType != structs.UpstreamDestTypeService {
			continue
		}

		// default the upstreams gateway mode if it didn't specify one
		if us.MeshGateway.Mode == structs.MeshGatewayModeDefault {
			us.MeshGateway.Mode = ns.Proxy.MeshGateway.Mode
		}

		usCfg, ok := w.defaults.UpstreamConfigs[us.DestinationName]
		if !ok {
			// No config defaults to merge
			continue
		}
		if err := mergo.Merge(&us.Config, usCfg); err != nil {
			return nil, err
		}
	}
	return ns, err
}