mirror of https://github.com/hashicorp/consul
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1180 lines
34 KiB
1180 lines
34 KiB
package autoconf
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/mock"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/hashicorp/consul/agent/cache"
|
|
cachetype "github.com/hashicorp/consul/agent/cache-types"
|
|
"github.com/hashicorp/consul/agent/config"
|
|
"github.com/hashicorp/consul/agent/connect"
|
|
"github.com/hashicorp/consul/agent/metadata"
|
|
"github.com/hashicorp/consul/agent/structs"
|
|
"github.com/hashicorp/consul/agent/token"
|
|
"github.com/hashicorp/consul/lib/retry"
|
|
"github.com/hashicorp/consul/proto/pbautoconf"
|
|
"github.com/hashicorp/consul/proto/pbconfig"
|
|
"github.com/hashicorp/consul/sdk/testutil"
|
|
testretry "github.com/hashicorp/consul/sdk/testutil/retry"
|
|
)
|
|
|
|
type configLoader struct {
|
|
opts config.BuilderOpts
|
|
}
|
|
|
|
func (c *configLoader) Load(source config.Source) (*config.RuntimeConfig, []string, error) {
|
|
return config.Load(c.opts, source)
|
|
}
|
|
|
|
func (c *configLoader) addConfigHCL(cfg string) {
|
|
c.opts.HCL = append(c.opts.HCL, cfg)
|
|
}
|
|
|
|
func requireChanNotReady(t *testing.T, ch <-chan struct{}) {
|
|
select {
|
|
case <-ch:
|
|
require.Fail(t, "chan is ready when it shouldn't be")
|
|
default:
|
|
return
|
|
}
|
|
}
|
|
|
|
func requireChanReady(t *testing.T, ch <-chan struct{}) {
|
|
select {
|
|
case <-ch:
|
|
return
|
|
default:
|
|
require.Fail(t, "chan is not ready when it should be")
|
|
}
|
|
}
|
|
|
|
func waitForChan(timer *time.Timer, ch <-chan struct{}) bool {
|
|
select {
|
|
case <-timer.C:
|
|
return false
|
|
case <-ch:
|
|
return true
|
|
}
|
|
}
|
|
|
|
func waitForChans(timeout time.Duration, chans ...<-chan struct{}) bool {
|
|
timer := time.NewTimer(timeout)
|
|
defer timer.Stop()
|
|
|
|
for _, ch := range chans {
|
|
if !waitForChan(timer, ch) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func TestNew(t *testing.T) {
|
|
type testCase struct {
|
|
modify func(*Config)
|
|
err string
|
|
validate func(t *testing.T, ac *AutoConfig)
|
|
}
|
|
|
|
cases := map[string]testCase{
|
|
"no-direct-rpc": {
|
|
modify: func(c *Config) {
|
|
c.DirectRPC = nil
|
|
},
|
|
err: "must provide a direct RPC delegate",
|
|
},
|
|
"no-config-loader": {
|
|
modify: func(c *Config) {
|
|
c.Loader = nil
|
|
},
|
|
err: "must provide a config loader",
|
|
},
|
|
"no-cache": {
|
|
modify: func(c *Config) {
|
|
c.Cache = nil
|
|
},
|
|
err: "must provide a cache",
|
|
},
|
|
"no-tls-configurator": {
|
|
modify: func(c *Config) {
|
|
c.TLSConfigurator = nil
|
|
},
|
|
err: "must provide a TLS configurator",
|
|
},
|
|
"no-tokens": {
|
|
modify: func(c *Config) {
|
|
c.Tokens = nil
|
|
},
|
|
err: "must provide a token store",
|
|
},
|
|
"ok": {
|
|
validate: func(t *testing.T, ac *AutoConfig) {
|
|
t.Helper()
|
|
require.NotNil(t, ac.logger)
|
|
require.NotNil(t, ac.acConfig.Waiter)
|
|
require.Equal(t, time.Minute, ac.acConfig.FallbackRetry)
|
|
require.Equal(t, 10*time.Second, ac.acConfig.FallbackLeeway)
|
|
},
|
|
},
|
|
}
|
|
|
|
for name, tcase := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
cfg := Config{
|
|
Loader: func(source config.Source) (cfg *config.RuntimeConfig, warnings []string, err error) {
|
|
return nil, nil, nil
|
|
},
|
|
DirectRPC: newMockDirectRPC(t),
|
|
Tokens: newMockTokenStore(t),
|
|
Cache: newMockCache(t),
|
|
TLSConfigurator: newMockTLSConfigurator(t),
|
|
ServerProvider: newMockServerProvider(t),
|
|
}
|
|
|
|
if tcase.modify != nil {
|
|
tcase.modify(&cfg)
|
|
}
|
|
|
|
ac, err := New(cfg)
|
|
if tcase.err != "" {
|
|
testutil.RequireErrorContains(t, err, tcase.err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.NotNil(t, ac)
|
|
if tcase.validate != nil {
|
|
tcase.validate(t, ac)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReadConfig(t *testing.T) {
|
|
// just testing that some auto config source gets injected
|
|
ac := AutoConfig{
|
|
autoConfigSource: config.LiteralSource{
|
|
Name: autoConfigFileName,
|
|
Config: config.Config{NodeName: stringPointer("hobbiton")},
|
|
},
|
|
logger: testutil.Logger(t),
|
|
acConfig: Config{
|
|
Loader: func(source config.Source) (*config.RuntimeConfig, []string, error) {
|
|
cfg, _, err := source.Parse()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return &config.RuntimeConfig{
|
|
DevMode: true,
|
|
NodeName: *cfg.NodeName,
|
|
}, nil, nil
|
|
},
|
|
},
|
|
}
|
|
|
|
cfg, err := ac.ReadConfig()
|
|
require.NoError(t, err)
|
|
require.NotNil(t, cfg)
|
|
require.Equal(t, "hobbiton", cfg.NodeName)
|
|
require.True(t, cfg.DevMode)
|
|
require.Same(t, ac.config, cfg)
|
|
}
|
|
|
|
func setupRuntimeConfig(t *testing.T) *configLoader {
|
|
t.Helper()
|
|
|
|
dataDir := testutil.TempDir(t, "auto-config")
|
|
|
|
opts := config.BuilderOpts{
|
|
Config: config.Config{
|
|
DataDir: &dataDir,
|
|
Datacenter: stringPointer("dc1"),
|
|
NodeName: stringPointer("autoconf"),
|
|
BindAddr: stringPointer("127.0.0.1"),
|
|
},
|
|
}
|
|
return &configLoader{opts: opts}
|
|
}
|
|
|
|
func TestInitialConfiguration_disabled(t *testing.T) {
|
|
loader := setupRuntimeConfig(t)
|
|
loader.addConfigHCL(`
|
|
primary_datacenter = "primary"
|
|
auto_config = {
|
|
enabled = false
|
|
}
|
|
`)
|
|
|
|
conf := newMockedConfig(t).Config
|
|
conf.Loader = loader.Load
|
|
|
|
ac, err := New(conf)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, ac)
|
|
|
|
cfg, err := ac.InitialConfiguration(context.Background())
|
|
require.NoError(t, err)
|
|
require.NotNil(t, cfg)
|
|
require.Equal(t, "primary", cfg.PrimaryDatacenter)
|
|
require.NoFileExists(t, filepath.Join(*loader.opts.Config.DataDir, autoConfigFileName))
|
|
}
|
|
|
|
func TestInitialConfiguration_cancelled(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
mcfg := newMockedConfig(t)
|
|
|
|
loader := setupRuntimeConfig(t)
|
|
loader.addConfigHCL(`
|
|
primary_datacenter = "primary"
|
|
auto_config = {
|
|
enabled = true
|
|
intro_token = "blarg"
|
|
server_addresses = ["127.0.0.1:8300"]
|
|
}
|
|
verify_outgoing = true
|
|
`)
|
|
mcfg.Config.Loader = loader.Load
|
|
|
|
expectedRequest := pbautoconf.AutoConfigRequest{
|
|
Datacenter: "dc1",
|
|
Node: "autoconf",
|
|
JWT: "blarg",
|
|
}
|
|
|
|
mcfg.directRPC.On("RPC", "dc1", "autoconf", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300}, "AutoConfig.InitialConfiguration", &expectedRequest, mock.Anything).Return(fmt.Errorf("injected error")).Times(0)
|
|
mcfg.serverProvider.On("FindLANServer").Return(nil).Times(0)
|
|
|
|
ac, err := New(mcfg.Config)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, ac)
|
|
|
|
ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(100*time.Millisecond))
|
|
defer cancelFn()
|
|
|
|
cfg, err := ac.InitialConfiguration(ctx)
|
|
testutil.RequireErrorContains(t, err, context.DeadlineExceeded.Error())
|
|
require.Nil(t, cfg)
|
|
}
|
|
|
|
func TestInitialConfiguration_restored(t *testing.T) {
|
|
mcfg := newMockedConfig(t)
|
|
|
|
loader := setupRuntimeConfig(t)
|
|
loader.addConfigHCL(`
|
|
auto_config = {
|
|
enabled = true
|
|
intro_token ="blarg"
|
|
server_addresses = ["127.0.0.1:8300"]
|
|
}
|
|
verify_outgoing = true
|
|
`)
|
|
|
|
mcfg.Config.Loader = loader.Load
|
|
|
|
indexedRoots, cert, extraCACerts := mcfg.setupInitialTLS(t, "autoconf", "dc1", "secret")
|
|
|
|
// persist an auto config response to the data dir where it is expected
|
|
persistedFile := filepath.Join(*loader.opts.Config.DataDir, autoConfigFileName)
|
|
response := &pbautoconf.AutoConfigResponse{
|
|
Config: &pbconfig.Config{
|
|
PrimaryDatacenter: "primary",
|
|
TLS: &pbconfig.TLS{
|
|
VerifyServerHostname: true,
|
|
},
|
|
ACL: &pbconfig.ACL{
|
|
Tokens: &pbconfig.ACLTokens{
|
|
Agent: "secret",
|
|
},
|
|
},
|
|
},
|
|
CARoots: mustTranslateCARootsToProtobuf(t, indexedRoots),
|
|
Certificate: mustTranslateIssuedCertToProtobuf(t, cert),
|
|
ExtraCACertificates: extraCACerts,
|
|
}
|
|
data, err := pbMarshaler.MarshalToString(response)
|
|
require.NoError(t, err)
|
|
require.NoError(t, ioutil.WriteFile(persistedFile, []byte(data), 0600))
|
|
|
|
// recording the initial configuration even when restoring is going to update
|
|
// the agent token in the token store
|
|
mcfg.tokens.On("UpdateAgentToken", "secret", token.TokenSourceConfig).Return(true).Once()
|
|
|
|
// prepopulation is going to grab the token to populate the correct cache key
|
|
mcfg.tokens.On("AgentToken").Return("secret").Times(0)
|
|
|
|
ac, err := New(mcfg.Config)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, ac)
|
|
|
|
cfg, err := ac.InitialConfiguration(context.Background())
|
|
require.NoError(t, err, data)
|
|
require.NotNil(t, cfg)
|
|
require.Equal(t, "primary", cfg.PrimaryDatacenter)
|
|
}
|
|
|
|
func TestInitialConfiguration_success(t *testing.T) {
|
|
mcfg := newMockedConfig(t)
|
|
loader := setupRuntimeConfig(t)
|
|
loader.addConfigHCL(`
|
|
auto_config = {
|
|
enabled = true
|
|
intro_token ="blarg"
|
|
server_addresses = ["127.0.0.1:8300"]
|
|
}
|
|
verify_outgoing = true
|
|
`)
|
|
mcfg.Config.Loader = loader.Load
|
|
|
|
indexedRoots, cert, extraCerts := mcfg.setupInitialTLS(t, "autoconf", "dc1", "secret")
|
|
|
|
// this gets called when InitialConfiguration is invoked to record the token from the
|
|
// auto-config response
|
|
mcfg.tokens.On("UpdateAgentToken", "secret", token.TokenSourceConfig).Return(true).Once()
|
|
|
|
// prepopulation is going to grab the token to populate the correct cache key
|
|
mcfg.tokens.On("AgentToken").Return("secret").Times(0)
|
|
|
|
// no server provider
|
|
mcfg.serverProvider.On("FindLANServer").Return(nil).Times(0)
|
|
|
|
populateResponse := func(args mock.Arguments) {
|
|
resp, ok := args.Get(5).(*pbautoconf.AutoConfigResponse)
|
|
require.True(t, ok)
|
|
resp.Config = &pbconfig.Config{
|
|
PrimaryDatacenter: "primary",
|
|
TLS: &pbconfig.TLS{
|
|
VerifyServerHostname: true,
|
|
},
|
|
ACL: &pbconfig.ACL{
|
|
Tokens: &pbconfig.ACLTokens{
|
|
Agent: "secret",
|
|
},
|
|
},
|
|
}
|
|
|
|
resp.CARoots = mustTranslateCARootsToProtobuf(t, indexedRoots)
|
|
resp.Certificate = mustTranslateIssuedCertToProtobuf(t, cert)
|
|
resp.ExtraCACertificates = extraCerts
|
|
}
|
|
|
|
expectedRequest := pbautoconf.AutoConfigRequest{
|
|
Datacenter: "dc1",
|
|
Node: "autoconf",
|
|
JWT: "blarg",
|
|
}
|
|
|
|
mcfg.directRPC.On(
|
|
"RPC",
|
|
"dc1",
|
|
"autoconf",
|
|
&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8300},
|
|
"AutoConfig.InitialConfiguration",
|
|
&expectedRequest,
|
|
&pbautoconf.AutoConfigResponse{}).Return(nil).Run(populateResponse)
|
|
|
|
ac, err := New(mcfg.Config)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, ac)
|
|
|
|
cfg, err := ac.InitialConfiguration(context.Background())
|
|
require.NoError(t, err)
|
|
require.NotNil(t, cfg)
|
|
require.Equal(t, "primary", cfg.PrimaryDatacenter)
|
|
|
|
// the file was written to.
|
|
persistedFile := filepath.Join(*loader.opts.Config.DataDir, autoConfigFileName)
|
|
require.FileExists(t, persistedFile)
|
|
}
|
|
|
|
func TestInitialConfiguration_retries(t *testing.T) {
|
|
mcfg := newMockedConfig(t)
|
|
loader := setupRuntimeConfig(t)
|
|
loader.addConfigHCL(`
|
|
auto_config = {
|
|
enabled = true
|
|
intro_token ="blarg"
|
|
server_addresses = [
|
|
"198.18.0.1:8300",
|
|
"198.18.0.2:8398",
|
|
"198.18.0.3:8399",
|
|
"127.0.0.1:1234"
|
|
]
|
|
}
|
|
verify_outgoing = true
|
|
`)
|
|
mcfg.Config.Loader = loader.Load
|
|
|
|
// reduce the retry wait times to make this test run faster
|
|
mcfg.Config.Waiter = &retry.Waiter{MinFailures: 2, MaxWait: time.Millisecond}
|
|
|
|
indexedRoots, cert, extraCerts := mcfg.setupInitialTLS(t, "autoconf", "dc1", "secret")
|
|
|
|
// this gets called when InitialConfiguration is invoked to record the token from the
|
|
// auto-config response
|
|
mcfg.tokens.On("UpdateAgentToken", "secret", token.TokenSourceConfig).Return(true).Once()
|
|
|
|
// prepopulation is going to grab the token to populate the correct cache key
|
|
mcfg.tokens.On("AgentToken").Return("secret").Times(0)
|
|
|
|
// no server provider
|
|
mcfg.serverProvider.On("FindLANServer").Return(nil).Times(0)
|
|
|
|
populateResponse := func(args mock.Arguments) {
|
|
resp, ok := args.Get(5).(*pbautoconf.AutoConfigResponse)
|
|
require.True(t, ok)
|
|
resp.Config = &pbconfig.Config{
|
|
PrimaryDatacenter: "primary",
|
|
TLS: &pbconfig.TLS{
|
|
VerifyServerHostname: true,
|
|
},
|
|
ACL: &pbconfig.ACL{
|
|
Tokens: &pbconfig.ACLTokens{
|
|
Agent: "secret",
|
|
},
|
|
},
|
|
}
|
|
|
|
resp.CARoots = mustTranslateCARootsToProtobuf(t, indexedRoots)
|
|
resp.Certificate = mustTranslateIssuedCertToProtobuf(t, cert)
|
|
resp.ExtraCACertificates = extraCerts
|
|
}
|
|
|
|
expectedRequest := pbautoconf.AutoConfigRequest{
|
|
Datacenter: "dc1",
|
|
Node: "autoconf",
|
|
JWT: "blarg",
|
|
}
|
|
|
|
// basically the 198.18.0.* addresses should fail indefinitely. the first time through the
|
|
// outer loop we inject a failure for the DNS resolution of localhost to 127.0.0.1. Then
|
|
// the second time through the outer loop we allow the localhost one to work.
|
|
mcfg.directRPC.On(
|
|
"RPC",
|
|
"dc1",
|
|
"autoconf",
|
|
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
|
|
"AutoConfig.InitialConfiguration",
|
|
&expectedRequest,
|
|
&pbautoconf.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
|
|
mcfg.directRPC.On(
|
|
"RPC",
|
|
"dc1",
|
|
"autoconf",
|
|
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 2), Port: 8398},
|
|
"AutoConfig.InitialConfiguration",
|
|
&expectedRequest,
|
|
&pbautoconf.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
|
|
mcfg.directRPC.On(
|
|
"RPC",
|
|
"dc1",
|
|
"autoconf",
|
|
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 3), Port: 8399},
|
|
"AutoConfig.InitialConfiguration",
|
|
&expectedRequest,
|
|
&pbautoconf.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Times(0)
|
|
mcfg.directRPC.On(
|
|
"RPC",
|
|
"dc1",
|
|
"autoconf",
|
|
&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
|
|
"AutoConfig.InitialConfiguration",
|
|
&expectedRequest,
|
|
&pbautoconf.AutoConfigResponse{}).Return(fmt.Errorf("injected failure")).Once()
|
|
mcfg.directRPC.On(
|
|
"RPC",
|
|
"dc1",
|
|
"autoconf",
|
|
&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 1234},
|
|
"AutoConfig.InitialConfiguration",
|
|
&expectedRequest,
|
|
&pbautoconf.AutoConfigResponse{}).Return(nil).Run(populateResponse).Once()
|
|
|
|
ac, err := New(mcfg.Config)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, ac)
|
|
|
|
cfg, err := ac.InitialConfiguration(context.Background())
|
|
require.NoError(t, err)
|
|
require.NotNil(t, cfg)
|
|
require.Equal(t, "primary", cfg.PrimaryDatacenter)
|
|
|
|
// the file was written to.
|
|
persistedFile := filepath.Join(*loader.opts.Config.DataDir, autoConfigFileName)
|
|
require.FileExists(t, persistedFile)
|
|
}
|
|
|
|
func TestGoRoutineManagement(t *testing.T) {
|
|
mcfg := newMockedConfig(t)
|
|
loader := setupRuntimeConfig(t)
|
|
loader.addConfigHCL(`
|
|
auto_config = {
|
|
enabled = true
|
|
intro_token ="blarg"
|
|
server_addresses = ["127.0.0.1:8300"]
|
|
}
|
|
verify_outgoing = true
|
|
`)
|
|
mcfg.Config.Loader = loader.Load
|
|
|
|
// prepopulation is going to grab the token to populate the correct cache key
|
|
mcfg.tokens.On("AgentToken").Return("secret").Times(0)
|
|
|
|
ac, err := New(mcfg.Config)
|
|
require.NoError(t, err)
|
|
|
|
// priming the config so some other requests will work properly that need to
|
|
// read from the configuration. We are going to avoid doing InitialConfiguration
|
|
// for this test as we only are really concerned with the go routine management
|
|
_, err = ac.ReadConfig()
|
|
require.NoError(t, err)
|
|
|
|
var rootsCtx context.Context
|
|
var leafCtx context.Context
|
|
var ctxLock sync.Mutex
|
|
|
|
rootsReq := ac.caRootsRequest()
|
|
mcfg.cache.On("Notify",
|
|
mock.Anything,
|
|
cachetype.ConnectCARootName,
|
|
&rootsReq,
|
|
rootsWatchID,
|
|
mock.Anything,
|
|
).Return(nil).Times(2).Run(func(args mock.Arguments) {
|
|
ctxLock.Lock()
|
|
rootsCtx = args.Get(0).(context.Context)
|
|
ctxLock.Unlock()
|
|
})
|
|
|
|
leafReq := ac.leafCertRequest()
|
|
mcfg.cache.On("Notify",
|
|
mock.Anything,
|
|
cachetype.ConnectCALeafName,
|
|
&leafReq,
|
|
leafWatchID,
|
|
mock.Anything,
|
|
).Return(nil).Times(2).Run(func(args mock.Arguments) {
|
|
ctxLock.Lock()
|
|
leafCtx = args.Get(0).(context.Context)
|
|
ctxLock.Unlock()
|
|
})
|
|
|
|
// we will start/stop things twice
|
|
mcfg.tokens.On("Notify", token.TokenKindAgent).Return(token.Notifier{}).Times(2)
|
|
mcfg.tokens.On("StopNotify", token.Notifier{}).Times(2)
|
|
|
|
mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(time.Now().Add(10 * time.Minute)).Times(0)
|
|
|
|
// ensure that auto-config isn't running
|
|
require.False(t, ac.IsRunning())
|
|
|
|
// ensure that nothing bad happens and that it reports as stopped
|
|
require.False(t, ac.Stop())
|
|
|
|
// ensure that the Done chan also reports that things are not running
|
|
// in other words the chan is immediately selectable
|
|
requireChanReady(t, ac.Done())
|
|
|
|
// start auto-config
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
require.NoError(t, ac.Start(ctx))
|
|
|
|
waitForContexts := func() bool {
|
|
ctxLock.Lock()
|
|
defer ctxLock.Unlock()
|
|
return !(rootsCtx == nil || leafCtx == nil)
|
|
}
|
|
|
|
// wait for the cache notifications to get started
|
|
require.Eventually(t, waitForContexts, 100*time.Millisecond, 10*time.Millisecond)
|
|
|
|
// hold onto the Done chan to test for the go routine exiting
|
|
done := ac.Done()
|
|
|
|
// ensure we report as running
|
|
require.True(t, ac.IsRunning())
|
|
|
|
// ensure the done chan is not selectable yet
|
|
requireChanNotReady(t, done)
|
|
|
|
// ensure we error if we attempt to start again
|
|
err = ac.Start(ctx)
|
|
testutil.RequireErrorContains(t, err, "AutoConfig is already running")
|
|
|
|
// now stop things - it should return true indicating that it was running
|
|
// when we attempted to stop it.
|
|
require.True(t, ac.Stop())
|
|
|
|
// ensure that the go routine shuts down - it will close the done chan. Also it should cancel
|
|
// the cache watches by cancelling the context it passed into the Notify call.
|
|
require.True(t, waitForChans(100*time.Millisecond, done, leafCtx.Done(), rootsCtx.Done()), "AutoConfig didn't shut down")
|
|
require.False(t, ac.IsRunning())
|
|
|
|
// restart it
|
|
require.NoError(t, ac.Start(ctx))
|
|
|
|
// get the new Done chan
|
|
done = ac.Done()
|
|
|
|
// ensure that context cancellation causes us to stop as well
|
|
cancel()
|
|
require.True(t, waitForChans(100*time.Millisecond, done))
|
|
}
|
|
|
|
type testAutoConfig struct {
|
|
mcfg *mockedConfig
|
|
ac *AutoConfig
|
|
tokenUpdates chan struct{}
|
|
originalToken string
|
|
stop func()
|
|
|
|
initialRoots *structs.IndexedCARoots
|
|
initialCert *structs.IssuedCert
|
|
extraCerts []string
|
|
}
|
|
|
|
func startedAutoConfig(t *testing.T, autoEncrypt bool) testAutoConfig {
|
|
t.Helper()
|
|
mcfg := newMockedConfig(t)
|
|
loader := setupRuntimeConfig(t)
|
|
if !autoEncrypt {
|
|
loader.addConfigHCL(`
|
|
auto_config = {
|
|
enabled = true
|
|
intro_token ="blarg"
|
|
server_addresses = ["127.0.0.1:8300"]
|
|
}
|
|
verify_outgoing = true
|
|
`)
|
|
} else {
|
|
loader.addConfigHCL(`
|
|
auto_encrypt {
|
|
tls = true
|
|
}
|
|
verify_outgoing = true
|
|
`)
|
|
}
|
|
mcfg.Config.Loader = loader.Load
|
|
mcfg.Config.FallbackLeeway = time.Nanosecond
|
|
|
|
originalToken := "a5deaa25-11ca-48bf-a979-4c3a7aa4b9a9"
|
|
|
|
if !autoEncrypt {
|
|
// this gets called when InitialConfiguration is invoked to record the token from the
|
|
// auto-config response
|
|
mcfg.tokens.On("UpdateAgentToken", originalToken, token.TokenSourceConfig).Return(true).Once()
|
|
}
|
|
|
|
// we expect this to be retrieved twice: first during cache prepopulation
|
|
// and then again when setting up the cache watch for the leaf cert.
|
|
// However one of those expectations is setup in the expectInitialTLS
|
|
// method so we only need one more here
|
|
mcfg.tokens.On("AgentToken").Return(originalToken).Once()
|
|
|
|
if autoEncrypt {
|
|
// when using AutoEncrypt we also have to grab the token once more
|
|
// when setting up the initial RPC as the ACL token is what is used
|
|
// to authorize the request.
|
|
mcfg.tokens.On("AgentToken").Return(originalToken).Once()
|
|
}
|
|
|
|
// this is called once during Start to initialze the token watches
|
|
tokenUpdateCh := make(chan struct{})
|
|
tokenNotifier := token.Notifier{
|
|
Ch: tokenUpdateCh,
|
|
}
|
|
mcfg.tokens.On("Notify", token.TokenKindAgent).Once().Return(tokenNotifier)
|
|
mcfg.tokens.On("StopNotify", tokenNotifier).Once()
|
|
|
|
// expect the roots watch on the cache
|
|
mcfg.cache.On("Notify",
|
|
mock.Anything,
|
|
cachetype.ConnectCARootName,
|
|
&structs.DCSpecificRequest{Datacenter: "dc1"},
|
|
rootsWatchID,
|
|
mock.Anything,
|
|
).Return(nil).Once()
|
|
|
|
mcfg.cache.On("Notify",
|
|
mock.Anything,
|
|
cachetype.ConnectCALeafName,
|
|
&cachetype.ConnectCALeafRequest{
|
|
Datacenter: "dc1",
|
|
Agent: "autoconf",
|
|
Token: originalToken,
|
|
DNSSAN: defaultDNSSANs,
|
|
IPSAN: defaultIPSANs,
|
|
},
|
|
leafWatchID,
|
|
mock.Anything,
|
|
).Return(nil).Once()
|
|
|
|
// override the server provider - most of the other tests set it up so that this
|
|
// always returns no server (simulating a state where we haven't joined gossip).
|
|
// this seems like a good place to ensure this other way of finding server information
|
|
// works
|
|
mcfg.serverProvider.On("FindLANServer").Once().Return(&metadata.Server{
|
|
Addr: &net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
|
|
})
|
|
|
|
indexedRoots, cert, extraCerts := mcfg.setupInitialTLS(t, "autoconf", "dc1", originalToken)
|
|
|
|
mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(cert.ValidBefore).Once()
|
|
|
|
populateResponse := func(args mock.Arguments) {
|
|
method := args.String(3)
|
|
|
|
switch method {
|
|
case "AutoConfig.InitialConfiguration":
|
|
resp, ok := args.Get(5).(*pbautoconf.AutoConfigResponse)
|
|
require.True(t, ok)
|
|
resp.Config = &pbconfig.Config{
|
|
PrimaryDatacenter: "primary",
|
|
TLS: &pbconfig.TLS{
|
|
VerifyServerHostname: true,
|
|
},
|
|
ACL: &pbconfig.ACL{
|
|
Tokens: &pbconfig.ACLTokens{
|
|
Agent: originalToken,
|
|
},
|
|
},
|
|
}
|
|
|
|
resp.CARoots = mustTranslateCARootsToProtobuf(t, indexedRoots)
|
|
resp.Certificate = mustTranslateIssuedCertToProtobuf(t, cert)
|
|
resp.ExtraCACertificates = extraCerts
|
|
case "AutoEncrypt.Sign":
|
|
resp, ok := args.Get(5).(*structs.SignedResponse)
|
|
require.True(t, ok)
|
|
*resp = structs.SignedResponse{
|
|
VerifyServerHostname: true,
|
|
ConnectCARoots: *indexedRoots,
|
|
IssuedCert: *cert,
|
|
ManualCARoots: extraCerts,
|
|
}
|
|
}
|
|
}
|
|
|
|
if !autoEncrypt {
|
|
expectedRequest := pbautoconf.AutoConfigRequest{
|
|
Datacenter: "dc1",
|
|
Node: "autoconf",
|
|
JWT: "blarg",
|
|
}
|
|
|
|
mcfg.directRPC.On(
|
|
"RPC",
|
|
"dc1",
|
|
"autoconf",
|
|
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
|
|
"AutoConfig.InitialConfiguration",
|
|
&expectedRequest,
|
|
&pbautoconf.AutoConfigResponse{}).Return(nil).Run(populateResponse).Once()
|
|
} else {
|
|
expectedRequest := structs.CASignRequest{
|
|
WriteRequest: structs.WriteRequest{Token: originalToken},
|
|
Datacenter: "dc1",
|
|
// TODO (autoconf) Maybe in the future we should populate a CSR
|
|
// and do some manual parsing/verification of the contents. The
|
|
// bits not having to do with the signing key such as the requested
|
|
// SANs and CN. For now though the mockDirectRPC type will empty
|
|
// the CSR so we have to pass in an empty string to the expectation.
|
|
CSR: "",
|
|
}
|
|
|
|
mcfg.directRPC.On(
|
|
"RPC",
|
|
"dc1",
|
|
"autoconf", // reusing the same name to prevent needing more configurability
|
|
&net.TCPAddr{IP: net.IPv4(198, 18, 0, 1), Port: 8300},
|
|
"AutoEncrypt.Sign",
|
|
&expectedRequest,
|
|
&structs.SignedResponse{}).Return(nil).Run(populateResponse)
|
|
}
|
|
|
|
ac, err := New(mcfg.Config)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, ac)
|
|
|
|
cfg, err := ac.InitialConfiguration(context.Background())
|
|
require.NoError(t, err)
|
|
require.NotNil(t, cfg)
|
|
if !autoEncrypt {
|
|
// auto-encrypt doesn't modify the config but rather sets the value
|
|
// in the TLS configurator
|
|
require.True(t, cfg.VerifyServerHostname)
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
require.NoError(t, ac.Start(ctx))
|
|
t.Cleanup(func() {
|
|
done := ac.Done()
|
|
cancel()
|
|
timer := time.NewTimer(1 * time.Second)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-done:
|
|
// do nothing
|
|
case <-timer.C:
|
|
t.Fatalf("AutoConfig wasn't stopped within 1 second after test completion")
|
|
}
|
|
})
|
|
|
|
return testAutoConfig{
|
|
mcfg: mcfg,
|
|
ac: ac,
|
|
tokenUpdates: tokenUpdateCh,
|
|
originalToken: originalToken,
|
|
initialRoots: indexedRoots,
|
|
initialCert: cert,
|
|
extraCerts: extraCerts,
|
|
stop: cancel,
|
|
}
|
|
}
|
|
|
|
// this test ensures that the cache watches are restarted with
|
|
// the updated token after receiving a token update
|
|
func TestTokenUpdate(t *testing.T) {
|
|
testAC := startedAutoConfig(t, false)
|
|
|
|
newToken := "1a4cc445-86ed-46b4-a355-bbf5a11dddb0"
|
|
|
|
rootsCtx, rootsCancel := context.WithCancel(context.Background())
|
|
testAC.mcfg.cache.On("Notify",
|
|
mock.Anything,
|
|
cachetype.ConnectCARootName,
|
|
&structs.DCSpecificRequest{Datacenter: testAC.ac.config.Datacenter},
|
|
rootsWatchID,
|
|
mock.Anything,
|
|
).Return(nil).Once().Run(func(args mock.Arguments) {
|
|
rootsCancel()
|
|
})
|
|
|
|
leafCtx, leafCancel := context.WithCancel(context.Background())
|
|
testAC.mcfg.cache.On("Notify",
|
|
mock.Anything,
|
|
cachetype.ConnectCALeafName,
|
|
&cachetype.ConnectCALeafRequest{
|
|
Datacenter: "dc1",
|
|
Agent: "autoconf",
|
|
Token: newToken,
|
|
DNSSAN: defaultDNSSANs,
|
|
IPSAN: defaultIPSANs,
|
|
},
|
|
leafWatchID,
|
|
mock.Anything,
|
|
).Return(nil).Once().Run(func(args mock.Arguments) {
|
|
leafCancel()
|
|
})
|
|
|
|
// this will be retrieved once when resetting the leaf cert watch
|
|
testAC.mcfg.tokens.On("AgentToken").Return(newToken).Once()
|
|
|
|
// send the notification about the token update
|
|
testAC.tokenUpdates <- struct{}{}
|
|
|
|
// wait for the leaf cert watches
|
|
require.True(t, waitForChans(100*time.Millisecond, leafCtx.Done(), rootsCtx.Done()), "New cache watches were not started within 100ms")
|
|
}
|
|
|
|
func TestRootsUpdate(t *testing.T) {
|
|
testAC := startedAutoConfig(t, false)
|
|
|
|
secondCA := connect.TestCA(t, testAC.initialRoots.Roots[0])
|
|
secondRoots := structs.IndexedCARoots{
|
|
ActiveRootID: secondCA.ID,
|
|
TrustDomain: connect.TestClusterID,
|
|
Roots: []*structs.CARoot{
|
|
secondCA,
|
|
testAC.initialRoots.Roots[0],
|
|
},
|
|
QueryMeta: structs.QueryMeta{
|
|
Index: 99,
|
|
},
|
|
}
|
|
|
|
updatedCtx, cancel := context.WithCancel(context.Background())
|
|
testAC.mcfg.tlsCfg.On("UpdateAutoTLS",
|
|
testAC.extraCerts,
|
|
[]string{secondCA.RootCert, testAC.initialRoots.Roots[0].RootCert},
|
|
testAC.initialCert.CertPEM,
|
|
"redacted",
|
|
true,
|
|
).Return(nil).Once().Run(func(args mock.Arguments) {
|
|
cancel()
|
|
})
|
|
|
|
// when a cache event comes in we end up recalculating the fallback timer which requires this call
|
|
testAC.mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(time.Now().Add(10 * time.Minute)).Once()
|
|
|
|
req := structs.DCSpecificRequest{Datacenter: "dc1"}
|
|
require.True(t, testAC.mcfg.cache.sendNotification(context.Background(), req.CacheInfo().Key, cache.UpdateEvent{
|
|
CorrelationID: rootsWatchID,
|
|
Result: &secondRoots,
|
|
Meta: cache.ResultMeta{
|
|
Index: secondRoots.Index,
|
|
},
|
|
}))
|
|
|
|
require.True(t, waitForChans(100*time.Millisecond, updatedCtx.Done()), "TLS certificates were not updated within the alotted time")
|
|
|
|
// persisting these to disk happens right after the chan we are waiting for will have fired above
|
|
// however there is no deterministic way to know once its been written outside of maybe a filesystem
|
|
// event notifier. That seems a little heavy handed just for this and especially to do in any sort
|
|
// of cross platform way.
|
|
testretry.Run(t, func(r *testretry.R) {
|
|
resp, err := testAC.ac.readPersistedAutoConfig()
|
|
require.NoError(r, err)
|
|
require.Equal(r, secondRoots.ActiveRootID, resp.CARoots.GetActiveRootID())
|
|
})
|
|
}
|
|
|
|
func TestCertUpdate(t *testing.T) {
|
|
testAC := startedAutoConfig(t, false)
|
|
secondCert := newLeaf(t, "autoconf", "dc1", testAC.initialRoots.Roots[0], 99, 10*time.Minute)
|
|
|
|
updatedCtx, cancel := context.WithCancel(context.Background())
|
|
testAC.mcfg.tlsCfg.On("UpdateAutoTLS",
|
|
testAC.extraCerts,
|
|
[]string{testAC.initialRoots.Roots[0].RootCert},
|
|
secondCert.CertPEM,
|
|
"redacted",
|
|
true,
|
|
).Return(nil).Once().Run(func(args mock.Arguments) {
|
|
cancel()
|
|
})
|
|
|
|
// when a cache event comes in we end up recalculating the fallback timer which requires this call
|
|
testAC.mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(secondCert.ValidBefore).Once()
|
|
|
|
req := cachetype.ConnectCALeafRequest{
|
|
Datacenter: "dc1",
|
|
Agent: "autoconf",
|
|
Token: testAC.originalToken,
|
|
DNSSAN: defaultDNSSANs,
|
|
IPSAN: defaultIPSANs,
|
|
}
|
|
require.True(t, testAC.mcfg.cache.sendNotification(context.Background(), req.CacheInfo().Key, cache.UpdateEvent{
|
|
CorrelationID: leafWatchID,
|
|
Result: secondCert,
|
|
Meta: cache.ResultMeta{
|
|
Index: secondCert.ModifyIndex,
|
|
},
|
|
}))
|
|
|
|
require.True(t, waitForChans(100*time.Millisecond, updatedCtx.Done()), "TLS certificates were not updated within the alotted time")
|
|
|
|
// persisting these to disk happens after all the things we would wait for in assertCertUpdated
|
|
// will have fired. There is no deterministic way to know once its been written so we wrap
|
|
// this in a retry.
|
|
testretry.Run(t, func(r *testretry.R) {
|
|
resp, err := testAC.ac.readPersistedAutoConfig()
|
|
require.NoError(r, err)
|
|
|
|
// ensure the roots got persisted to disk
|
|
require.Equal(r, secondCert.CertPEM, resp.Certificate.GetCertPEM())
|
|
})
|
|
}
|
|
|
|
func TestFallback(t *testing.T) {
|
|
testAC := startedAutoConfig(t, false)
|
|
|
|
// at this point everything is operating normally and we are just
|
|
// waiting for events. We are going to send a new cert that is basically
|
|
// already expired and then allow the fallback routine to kick in.
|
|
secondCert := newLeaf(t, "autoconf", "dc1", testAC.initialRoots.Roots[0], 100, time.Nanosecond)
|
|
secondCA := connect.TestCA(t, testAC.initialRoots.Roots[0])
|
|
secondRoots := structs.IndexedCARoots{
|
|
ActiveRootID: secondCA.ID,
|
|
TrustDomain: connect.TestClusterID,
|
|
Roots: []*structs.CARoot{
|
|
secondCA,
|
|
testAC.initialRoots.Roots[0],
|
|
},
|
|
QueryMeta: structs.QueryMeta{
|
|
Index: 101,
|
|
},
|
|
}
|
|
thirdCert := newLeaf(t, "autoconf", "dc1", secondCA, 102, 10*time.Minute)
|
|
|
|
// setup the expectation for when the certs got updated initially
|
|
updatedCtx, updateCancel := context.WithCancel(context.Background())
|
|
testAC.mcfg.tlsCfg.On("UpdateAutoTLS",
|
|
testAC.extraCerts,
|
|
[]string{testAC.initialRoots.Roots[0].RootCert},
|
|
secondCert.CertPEM,
|
|
"redacted",
|
|
true,
|
|
).Return(nil).Once().Run(func(args mock.Arguments) {
|
|
updateCancel()
|
|
})
|
|
|
|
// when a cache event comes in we end up recalculating the fallback timer which requires this call
|
|
testAC.mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(secondCert.ValidBefore).Once()
|
|
testAC.mcfg.tlsCfg.On("AutoEncryptCertExpired").Return(true).Once()
|
|
|
|
fallbackCtx, fallbackCancel := context.WithCancel(context.Background())
|
|
|
|
// also testing here that we can change server IPs for ongoing operations
|
|
testAC.mcfg.serverProvider.On("FindLANServer").Once().Return(&metadata.Server{
|
|
Addr: &net.TCPAddr{IP: net.IPv4(198, 18, 23, 2), Port: 8300},
|
|
})
|
|
|
|
// after sending the notification for the cert update another InitialConfiguration RPC
|
|
// will be made to pull down the latest configuration. So we need to set up the response
|
|
// for the second RPC
|
|
populateResponse := func(args mock.Arguments) {
|
|
resp, ok := args.Get(5).(*pbautoconf.AutoConfigResponse)
|
|
require.True(t, ok)
|
|
resp.Config = &pbconfig.Config{
|
|
PrimaryDatacenter: "primary",
|
|
TLS: &pbconfig.TLS{
|
|
VerifyServerHostname: true,
|
|
},
|
|
ACL: &pbconfig.ACL{
|
|
Tokens: &pbconfig.ACLTokens{
|
|
Agent: testAC.originalToken,
|
|
},
|
|
},
|
|
}
|
|
|
|
resp.CARoots = mustTranslateCARootsToProtobuf(t, &secondRoots)
|
|
resp.Certificate = mustTranslateIssuedCertToProtobuf(t, thirdCert)
|
|
resp.ExtraCACertificates = testAC.extraCerts
|
|
|
|
fallbackCancel()
|
|
}
|
|
|
|
expectedRequest := pbautoconf.AutoConfigRequest{
|
|
Datacenter: "dc1",
|
|
Node: "autoconf",
|
|
JWT: "blarg",
|
|
}
|
|
|
|
testAC.mcfg.directRPC.On(
|
|
"RPC",
|
|
"dc1",
|
|
"autoconf",
|
|
&net.TCPAddr{IP: net.IPv4(198, 18, 23, 2), Port: 8300},
|
|
"AutoConfig.InitialConfiguration",
|
|
&expectedRequest,
|
|
&pbautoconf.AutoConfigResponse{}).Return(nil).Run(populateResponse).Once()
|
|
|
|
// this gets called when InitialConfiguration is invoked to record the token from the
|
|
// auto-config response which is how the Fallback for auto-config works
|
|
testAC.mcfg.tokens.On("UpdateAgentToken", testAC.originalToken, token.TokenSourceConfig).Return(true).Once()
|
|
|
|
testAC.mcfg.expectInitialTLS(t, "autoconf", "dc1", testAC.originalToken, secondCA, &secondRoots, thirdCert, testAC.extraCerts)
|
|
|
|
// after the second RPC we now will use the new certs validity period in the next run loop iteration
|
|
testAC.mcfg.tlsCfg.On("AutoEncryptCertNotAfter").Return(time.Now().Add(10 * time.Minute)).Once()
|
|
|
|
// now that all the mocks are set up we can trigger the whole thing by sending the second expired cert
|
|
// as a cache update event.
|
|
req := cachetype.ConnectCALeafRequest{
|
|
Datacenter: "dc1",
|
|
Agent: "autoconf",
|
|
Token: testAC.originalToken,
|
|
DNSSAN: defaultDNSSANs,
|
|
IPSAN: defaultIPSANs,
|
|
}
|
|
require.True(t, testAC.mcfg.cache.sendNotification(context.Background(), req.CacheInfo().Key, cache.UpdateEvent{
|
|
CorrelationID: leafWatchID,
|
|
Result: secondCert,
|
|
Meta: cache.ResultMeta{
|
|
Index: secondCert.ModifyIndex,
|
|
},
|
|
}))
|
|
|
|
// wait for the TLS certificates to get updated
|
|
require.True(t, waitForChans(100*time.Millisecond, updatedCtx.Done()), "TLS certificates were not updated within the alotted time")
|
|
|
|
// now wait for the fallback routine to be invoked
|
|
require.True(t, waitForChans(100*time.Millisecond, fallbackCtx.Done()), "fallback routines did not get invoked within the alotted time")
|
|
|
|
testAC.stop()
|
|
<-testAC.ac.done
|
|
|
|
resp, err := testAC.ac.readPersistedAutoConfig()
|
|
require.NoError(t, err)
|
|
|
|
// ensure the roots got persisted to disk
|
|
require.Equal(t, thirdCert.CertPEM, resp.Certificate.GetCertPEM())
|
|
require.Equal(t, secondRoots.ActiveRootID, resp.CARoots.GetActiveRootID())
|
|
}
|
|
|
|
func TestIntroToken(t *testing.T) {
|
|
tokenFile := testutil.TempFile(t, "intro-token")
|
|
t.Cleanup(func() { os.Remove(tokenFile.Name()) })
|
|
|
|
tokenFileEmpty := testutil.TempFile(t, "intro-token-empty")
|
|
t.Cleanup(func() { os.Remove(tokenFileEmpty.Name()) })
|
|
|
|
tokenFromFile := "8ae34d3a-8adf-446a-b236-69874597cb5b"
|
|
tokenFromConfig := "3ad9b572-ea42-4e47-9cd0-53a398a98abf"
|
|
require.NoError(t, ioutil.WriteFile(tokenFile.Name(), []byte(tokenFromFile), 0600))
|
|
|
|
type testCase struct {
|
|
config *config.RuntimeConfig
|
|
err string
|
|
token string
|
|
}
|
|
|
|
cases := map[string]testCase{
|
|
"config": {
|
|
config: &config.RuntimeConfig{
|
|
AutoConfig: config.AutoConfig{
|
|
IntroToken: tokenFromConfig,
|
|
IntroTokenFile: tokenFile.Name(),
|
|
},
|
|
},
|
|
token: tokenFromConfig,
|
|
},
|
|
"file": {
|
|
config: &config.RuntimeConfig{
|
|
AutoConfig: config.AutoConfig{
|
|
IntroTokenFile: tokenFile.Name(),
|
|
},
|
|
},
|
|
token: tokenFromFile,
|
|
},
|
|
"file-empty": {
|
|
config: &config.RuntimeConfig{
|
|
AutoConfig: config.AutoConfig{
|
|
IntroTokenFile: tokenFileEmpty.Name(),
|
|
},
|
|
},
|
|
err: "intro_token_file did not contain any token",
|
|
},
|
|
}
|
|
|
|
for name, tcase := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
ac := AutoConfig{
|
|
config: tcase.config,
|
|
}
|
|
|
|
token, err := ac.introToken()
|
|
if tcase.err != "" {
|
|
testutil.RequireErrorContains(t, err, tcase.err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.Equal(t, tcase.token, token)
|
|
}
|
|
})
|
|
}
|
|
|
|
}
|