Implement bootstrapping proxy defaults from the config file (#5714)

agent/agent.go

@@ -1136,6 +1136,8 @@ func (a *Agent) consulConfig() (*consul.Config, error) {
 		return nil, fmt.Errorf("Failed to configure keyring: %v", err)
 	}
 
+	base.ConfigEntryBootstrap = a.config.ConfigEntryBootstrap
+
 	return base, nil
 }
 
@@ -3616,6 +3618,12 @@ func (a *Agent) ReloadConfig(newCfg *config.RuntimeConfig) error {
 		}
 	}
 
+	// this only gets used by the consulConfig function and since
+	// that is only ever done during init and reload here then
+	// an in place modification is safe as reloads cannot be
+	// concurrent due to both gaing a full lock on the stateLock
+	a.config.ConfigEntryBootstrap = newCfg.ConfigEntryBootstrap
+
 	// create the config for the rpc server/client
 	consulCfg, err := a.consulConfig()
 	if err != nil {

agent/config/builder.go

@@ -633,6 +633,22 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		verifyOutgoing = true
 	}
 
+	var configEntries []structs.ConfigEntry
+
+	if len(c.ConfigEntries.Bootstrap.ProxyDefaults) > 0 {
+		for name, config := range c.ConfigEntries.Bootstrap.ProxyDefaults {
+			if name != structs.ProxyConfigGlobal {
+				return RuntimeConfig{}, fmt.Errorf("invalid config.proxy_defaults name (%q), only %q is supported", name, structs.ProxyConfigGlobal)
+			}
+
+			configEntries = append(configEntries, &structs.ProxyConfigEntry{
+				Kind:   structs.ProxyDefaults,
+				Name:   structs.ProxyConfigGlobal,
+				Config: config,
+			})
+		}
+	}
+
 	// ----------------------------------------------------------------
 	// build runtime config
 	//
@@ -767,6 +783,7 @@ func (b *Builder) Build() (rt RuntimeConfig, err error) {
 		CheckUpdateInterval:                     b.durationVal("check_update_interval", c.CheckUpdateInterval),
 		Checks:                                  checks,
 		ClientAddrs:                             clientAddrs,
+		ConfigEntryBootstrap:                    configEntries,
 		ConnectEnabled:                          connectEnabled,
 		ConnectCAProvider:                       connectCAProvider,
 		ConnectCAConfig:                         connectCAConfig,

agent/config/config.go

@@ -184,6 +184,7 @@ type Config struct {
 	CheckUpdateInterval              *string                  `json:"check_update_interval,omitempty" hcl:"check_update_interval" mapstructure:"check_update_interval"`
 	Checks                           []CheckDefinition        `json:"checks,omitempty" hcl:"checks" mapstructure:"checks"`
 	ClientAddr                       *string                  `json:"client_addr,omitempty" hcl:"client_addr" mapstructure:"client_addr"`
+	ConfigEntries                    ConfigEntries            `json:"config_entries,omitempty" hcl:"config_entries" mapstructure:"config_entries"`
 	Connect                          Connect                  `json:"connect,omitempty" hcl:"connect" mapstructure:"connect"`
 	DNS                              DNS                      `json:"dns_config,omitempty" hcl:"dns_config" mapstructure:"dns_config"`
 	DNSDomain                        *string                  `json:"domain,omitempty" hcl:"domain" mapstructure:"domain"`
@@ -650,3 +651,11 @@ type Tokens struct {
 	Default     *string `json:"default,omitempty" hcl:"default" mapstructure:"default"`
 	Agent       *string `json:"agent,omitempty" hcl:"agent" mapstructure:"agent"`
 }
+
+type ConfigEntries struct {
+	Bootstrap ConfigEntriesBootstrap `json:"bootstrap,omitempty" hcl:"bootstrap" mapstructure:"bootstrap"`
+}
+
+type ConfigEntriesBootstrap struct {
+	ProxyDefaults map[string]map[string]interface{} `json:"proxy_defaults,omitempty" hcl:"proxy_defaults" mapstructure:"proxy_defaults"`
+}

agent/config/runtime.go

@@ -496,6 +496,10 @@ type RuntimeConfig struct {
 	// flag: -client string
 	ClientAddrs []*net.IPAddr
 
+	// ConfigEntryBootstrap contains a list of ConfigEntries to ensure are created
+	// If entries of the same Kind/Name exist already these will not update them.
+	ConfigEntryBootstrap []structs.ConfigEntry
+
 	// ConnectEnabled opts the agent into connect. It should be set on all clients
 	// and servers in a cluster for correct connect operation.
 	ConnectEnabled bool

agent/config/runtime_test.go

@@ -3007,6 +3007,16 @@ func TestFullConfig(t *testing.T) {
 			],
 			"check_update_interval": "16507s",
 			"client_addr": "93.83.18.19",
+			"config_entries": {
+				"bootstrap": {
+					"proxy_defaults": {
+						"global": {
+							"foo": "bar",
+							"bar": 1.0
+						}
+					}
+				}
+			},
 			"connect": {
 				"ca_provider": "consul",
 				"ca_config": {
@@ -3560,6 +3570,12 @@ func TestFullConfig(t *testing.T) {
 			]
 			check_update_interval = "16507s"
 			client_addr = "93.83.18.19"
+			config_entries {
+				bootstrap proxy_defaults global {
+					foo = "bar"
+					bar = 1.0
+				}
+			}
 			connect {
 				ca_provider = "consul"
 				ca_config {
@@ -4219,6 +4235,17 @@ func TestFullConfig(t *testing.T) {
 		},
 		CheckUpdateInterval: 16507 * time.Second,
 		ClientAddrs:         []*net.IPAddr{ipAddr("93.83.18.19")},
+		ConfigEntryBootstrap: []structs.ConfigEntry{
+			&structs.ProxyConfigEntry{
+				Kind: structs.ProxyDefaults,
+				Name: structs.ProxyConfigGlobal,
+				Config: map[string]interface{}{
+					"foo": "bar",
+					// has to be a float due to being a map[string]interface
+					"bar": float64(1),
+				},
+			},
+		},
 		ConnectEnabled:          true,
 		ConnectProxyBindMinPort: 2000,
 		ConnectProxyBindMaxPort: 3000,
@@ -4996,6 +5023,7 @@ func TestSanitize(t *testing.T) {
 			"Token": "hidden"
 		}],
 		"ClientAddrs": [],
+		"ConfigEntryBootstrap": [],
 		"ConnectCAConfig": {},
 		"ConnectCAProvider": "",
 		"ConnectEnabled": false,

agent/consul/config.go

@@ -391,6 +391,10 @@ type Config struct {
 	// CAConfig is used to apply the initial Connect CA configuration when
 	// bootstrapping.
 	CAConfig *structs.CAConfiguration
+
+	// ConfigEntryBootstrap contains a list of ConfigEntries to ensure are created
+	// If entries of the same Kind/Name exist already these will not update them.
+	ConfigEntryBootstrap []structs.ConfigEntry
 }
 
 func (c *Config) ToTLSUtilConfig() tlsutil.Config {

agent/consul/leader.go

@@ -40,6 +40,10 @@ var (
 	// minAutopilotVersion is the minimum Consul version in which Autopilot features
 	// are supported.
 	minAutopilotVersion = version.Must(version.NewVersion("0.8.0"))
+
+	// minCentralizedConfigVersion is the minimum Consul version in which centralized
+	// config is supported
+	minCentralizedConfigVersion = version.Must(version.NewVersion("1.5.0"))
 )
 
 // monitorLeadership is used to monitor if we acquire or lose our role
@@ -261,6 +265,11 @@ func (s *Server) establishLeadership() error {
 		return err
 	}
 
+	// attempt to bootstrap config entries
+	if err := s.bootstrapConfigEntries(s.config.ConfigEntryBootstrap); err != nil {
+		return err
+	}
+
 	s.getOrCreateAutopilotConfig()
 	s.autopilot.Start()
 
@@ -893,6 +902,44 @@ func (s *Server) getOrCreateAutopilotConfig() *autopilot.Config {
 	return config
 }
 
+func (s *Server) bootstrapConfigEntries(entries []structs.ConfigEntry) error {
+	if s.config.PrimaryDatacenter != "" && s.config.PrimaryDatacenter != s.config.Datacenter {
+		// only bootstrap in the primary datacenter
+		return nil
+	}
+
+	if len(entries) < 1 {
+		// nothing to initialize
+		return nil
+	}
+
+	if !ServersMeetMinimumVersion(s.LANMembers(), minCentralizedConfigVersion) {
+		s.logger.Printf("[WARN] centralized config: can't initialize until all servers >= %s", minCentralizedConfigVersion.String())
+		return nil
+	}
+
+	state := s.fsm.State()
+	for _, entry := range entries {
+		_, existing, err := state.ConfigEntry(nil, entry.GetKind(), entry.GetName())
+		if err != nil {
+			return fmt.Errorf("Failed to determine whether the configuration for %q / %q already exists: %v", entry.GetKind(), entry.GetName(), err)
+		}
+
+		if existing == nil {
+			req := structs.ConfigEntryRequest{
+				Op:         structs.ConfigEntryUpsert,
+				Datacenter: s.config.Datacenter,
+				Entry:      entry,
+			}
+
+			if _, err = s.raftApply(structs.ConfigEntryRequestType, &req); err != nil {
+				return fmt.Errorf("Failed to apply configuration entry %q / %q: %v", entry.GetKind(), entry.GetName(), err)
+			}
+		}
+	}
+	return nil
+}
+
 // initializeCAConfig is used to initialize the CA config if necessary
 // when setting up the CA during establishLeadership
 func (s *Server) initializeCAConfig() (*structs.CAConfiguration, error) {

agent/consul/leader_test.go

@@ -1206,3 +1206,38 @@ func TestLeader_ACLUpgrade(t *testing.T) {
 		require.Equal(t, client.ACL.Rules, token.Rules)
 	})
 }
+
+func TestLeader_ConfigEntryBootstrap(t *testing.T) {
+	t.Parallel()
+	global_entry_init := &structs.ProxyConfigEntry{
+		Kind: structs.ProxyDefaults,
+		Name: structs.ProxyConfigGlobal,
+		Config: map[string]interface{}{
+			// these are made a []uint8 and a int64 to allow the Equals test to pass
+			// otherwise it will fail complaining about data types
+			"foo": []uint8("bar"),
+			"bar": int64(1),
+		},
+	}
+
+	dir1, s1 := testServerWithConfig(t, func(c *Config) {
+		c.Build = "1.5.0"
+		c.ConfigEntryBootstrap = []structs.ConfigEntry{
+			global_entry_init,
+		}
+	})
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+	testrpc.WaitForTestAgent(t, s1.RPC, "dc1")
+
+	retry.Run(t, func(t *retry.R) {
+		_, entry, err := s1.fsm.State().ConfigEntry(nil, structs.ProxyDefaults, structs.ProxyConfigGlobal)
+		require.NoError(t, err)
+		require.NotNil(t, entry)
+		global, ok := entry.(*structs.ProxyConfigEntry)
+		require.True(t, ok)
+		require.Equal(t, global_entry_init.Kind, global.Kind)
+		require.Equal(t, global_entry_init.Name, global.Name)
+		require.Equal(t, global_entry_init.Config, global.Config)
+	})
+}

agent/consul/server.go

@@ -1132,6 +1132,11 @@ func (s *Server) GetLANCoordinate() (lib.CoordinateSet, error) {
 // ReloadConfig is used to have the Server do an online reload of
 // relevant configuration information
 func (s *Server) ReloadConfig(config *Config) error {
+	if s.IsLeader() {
+		// only bootstrap the config entries if we are the leader
+		// this will error if we lose leadership while bootstrapping here.
+		return s.bootstrapConfigEntries(config.ConfigEntryBootstrap)
+	}
 	return nil
 }
 

agent/consul/server_test.go

@@ -970,3 +970,41 @@ func TestServer_RevokeLeadershipIdempotent(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestServer_Reload(t *testing.T) {
+	t.Parallel()
+
+	global_entry_init := &structs.ProxyConfigEntry{
+		Kind: structs.ProxyDefaults,
+		Name: structs.ProxyConfigGlobal,
+		Config: map[string]interface{}{
+			// these are made a []uint8 and a int64 to allow the Equals test to pass
+			// otherwise it will fail complaining about data types
+			"foo": []uint8("bar"),
+			"bar": int64(1),
+		},
+	}
+
+	dir1, s := testServerWithConfig(t, func(c *Config) {
+		c.Build = "1.5.0"
+	})
+	defer os.RemoveAll(dir1)
+	defer s.Shutdown()
+
+	testrpc.WaitForTestAgent(t, s.RPC, "dc1")
+
+	s.config.ConfigEntryBootstrap = []structs.ConfigEntry{
+		global_entry_init,
+	}
+
+	s.ReloadConfig(s.config)
+
+	_, entry, err := s.fsm.State().ConfigEntry(nil, structs.ProxyDefaults, structs.ProxyConfigGlobal)
+	require.NoError(t, err)
+	require.NotNil(t, entry)
+	global, ok := entry.(*structs.ProxyConfigEntry)
+	require.True(t, ok)
+	require.Equal(t, global_entry_init.Kind, global.Kind)
+	require.Equal(t, global_entry_init.Name, global.Name)
+	require.Equal(t, global_entry_init.Config, global.Config)
+}

website/source/docs/agent/options.html.md

@@ -807,6 +807,21 @@ default will automatically work with some tooling.
 * <a name="client_addr"></a><a href="#client_addr">`client_addr`</a> Equivalent to the
   [`-client` command-line flag](#_client).
 
+* <a name="config_entries"></a><a href="#config_entries">`config_entries`</a>
+    This object allows setting options for centralized config entries.
+
+    The following sub-keys are available:
+
+    * <a name="bootstrap"></a><a href="#config_entries_bootstrap">`bootstrap`</a>
+        This object allows configuring centralized config entries to be bootstrapped
+        by the leader. These entries will be reloaded during an agent config reload.
+
+        The following sub-keys are available:
+
+        * <a name="proxy_defaults"></a><a href="#config_entries_bootstrap_proxy_defaults">`proxy_defaults`</a>
+          This object should contain a mapping of config entry names to an opaque proxy configuration mapping.
+          Currently the only supported name is `global`
+
 * <a name="connect"></a><a href="#connect">`connect`</a>
     This object allows setting options for the Connect feature.