Add multiple recursor definition support

command/agent/command.go

@@ -302,7 +302,7 @@ func (c *Command) setupAgent(config *Config, logOutput io.Writer, logWriter *log
 		}
 
 		server, err := NewDNSServer(agent, &config.DNSConfig, logOutput,
-			config.Domain, dnsAddr.String(), config.DNSRecursor)
+			config.Domain, dnsAddr.String(), config.DNSRecursors)
 		if err != nil {
 			agent.Shutdown()
 			c.Ui.Error(fmt.Sprintf("Error starting dns server: %s", err))

command/agent/config.go

@@ -97,9 +97,9 @@ type Config struct {
 	// DataDir is the directory to store our state in
 	DataDir string `mapstructure:"data_dir"`
 
-	// DNSRecursor can be set to allow the DNS server to recursively
+	// DNSRecursors can be set to allow the DNS servers to recursively
 	// resolve non-consul domains
-	DNSRecursor string `mapstructure:"recursor"`
+	DNSRecursors []string `mapstructure:"recursors"`
 
 	// DNS configuration
 	DNSConfig DNSConfig `mapstructure:"dns_config"`
@@ -623,9 +623,12 @@ func MergeConfig(a, b *Config) *Config {
 	if b.DataDir != "" {
 		result.DataDir = b.DataDir
 	}
-	if b.DNSRecursor != "" {
-		result.DNSRecursor = b.DNSRecursor
-	}
+
+	// Copy the dns recursors
+	result.DNSRecursors = make([]string, 0, len(a.DNSRecursors)+len(b.DNSRecursors))
+	result.DNSRecursors = append(result.DNSRecursors, a.DNSRecursors...)
+	result.DNSRecursors = append(result.DNSRecursors, b.DNSRecursors...)
+
 	if b.Domain != "" {
 		result.Domain = b.Domain
 	}

command/agent/config_test.go

@@ -109,7 +109,7 @@ func TestDecodeConfig(t *testing.T) {
 	}
 
 	// DNS setup
-	input = `{"ports": {"dns": 8500}, "recursor": "8.8.8.8", "domain": "foobar"}`
+	input = `{"ports": {"dns": 8500}, "recursor": ["8.8.8.8","8.8.4.4"], "domain": "foobar"}`
 	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -119,7 +119,13 @@ func TestDecodeConfig(t *testing.T) {
 		t.Fatalf("bad: %#v", config)
 	}
 
-	if config.DNSRecursor != "8.8.8.8" {
+	if len(config.DNSRecursors) != 2 {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.DNSRecursors[0] != "8.8.8.8" {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.DNSRecursors[1] != "8.8.4.4" {
 		t.Fatalf("bad: %#v", config)
 	}
 
@@ -791,7 +797,6 @@ func TestMergeConfig(t *testing.T) {
 		BootstrapExpect:        0,
 		Datacenter:             "dc1",
 		DataDir:                "/tmp/foo",
-		DNSRecursor:            "127.0.0.1:1001",
 		Domain:                 "basic",
 		LogLevel:               "debug",
 		NodeName:               "foo",
@@ -811,7 +816,7 @@ func TestMergeConfig(t *testing.T) {
 		BootstrapExpect: 3,
 		Datacenter:      "dc2",
 		DataDir:         "/tmp/bar",
-		DNSRecursor:     "127.0.0.2:1001",
+		DNSRecursors:    []string{"127.0.0.2:1001"},
 		DNSConfig: DNSConfig{
 			NodeTTL: 10 * time.Second,
 			ServiceTTL: map[string]time.Duration{

command/agent/dns.go

@@ -28,12 +28,12 @@ type DNSServer struct {
 	dnsServer    *dns.Server
 	dnsServerTCP *dns.Server
 	domain       string
-	recursor     string
+	recursors    []string
 	logger       *log.Logger
 }
 
 // NewDNSServer starts a new DNS server to provide an agent interface
-func NewDNSServer(agent *Agent, config *DNSConfig, logOutput io.Writer, domain, bind, recursor string) (*DNSServer, error) {
+func NewDNSServer(agent *Agent, config *DNSConfig, logOutput io.Writer, domain string, bind string, recursors []string) (*DNSServer, error) {
 	// Make sure domain is FQDN
 	domain = dns.Fqdn(domain)
 
@@ -61,7 +61,7 @@ func NewDNSServer(agent *Agent, config *DNSConfig, logOutput io.Writer, domain,
 		dnsServer:    server,
 		dnsServerTCP: serverTCP,
 		domain:       domain,
-		recursor:     recursor,
+		recursors:    recursors,
 		logger:       log.New(logOutput, "", log.LstdFlags),
 	}
 
@@ -70,12 +70,19 @@ func NewDNSServer(agent *Agent, config *DNSConfig, logOutput io.Writer, domain,
 	if domain != consulDomain {
 		mux.HandleFunc(consulDomain, srv.handleTest)
 	}
-	if recursor != "" {
+	if len(recursors) > 0 {
+		validatedRecursors := []string{}
+
+		for _, recursor := range recursors {
 			recursor, err := recursorAddr(recursor)
 			if err != nil {
 				return nil, fmt.Errorf("Invalid recursor address: %v", err)
 			}
-		srv.recursor = recursor
+
+			validatedRecursors = append(validatedRecursors, recursor)
+		}
+
+		srv.recursors = validatedRecursors
 		mux.HandleFunc(".", srv.handleRecurse)
 	}
 
@@ -178,7 +185,7 @@ func (d *DNSServer) handleQuery(resp dns.ResponseWriter, req *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetReply(req)
 	m.Authoritative = true
-	m.RecursionAvailable = (d.recursor != "")
+	m.RecursionAvailable = (len(d.recursors) > 0)
 
 	// Only add the SOA if requested
 	if req.Question[0].Qtype == dns.TypeSOA {
@@ -587,10 +594,13 @@ func (d *DNSServer) handleRecurse(resp dns.ResponseWriter, req *dns.Msg) {
 
 	// Recursively resolve
 	c := &dns.Client{Net: network}
-	r, rtt, err := c.Exchange(req, d.recursor)
+	for i,recursor := range d.recursors {
+		r, rtt, err := c.Exchange(req, recursor)
 
-	// On failure, return a SERVFAIL message
-	if err != nil {
+		if i < len(d.recursors) && err != nil {
+			continue
+		} else if err != nil {
+			// On all of failure, return a SERVFAIL message
 			d.logger.Printf("[ERR] dns: recurse failed: %v", err)
 			m := &dns.Msg{}
 			m.SetReply(req)
@@ -606,11 +616,12 @@ func (d *DNSServer) handleRecurse(resp dns.ResponseWriter, req *dns.Msg) {
 			d.logger.Printf("[WARN] dns: failed to respond: %v", err)
 		}
 	}
+}
 
 // resolveCNAME is used to recursively resolve CNAME records
 func (d *DNSServer) resolveCNAME(name string) []dns.RR {
 	// Do nothing if we don't have a recursor
-	if d.recursor == "" {
+	if len(d.recursors) > 0 {
 		return nil
 	}
 
@@ -620,8 +631,12 @@ func (d *DNSServer) resolveCNAME(name string) []dns.RR {
 
 	// Make a DNS lookup request
 	c := &dns.Client{Net: "udp"}
-	r, rtt, err := c.Exchange(m, d.recursor)
-	if err != nil {
+	for i,recursor := range d.recursors {
+		r, rtt, err := c.Exchange(m, recursor)
+
+		if i < len(d.recursors) && err != nil {
+			continue
+		} else if err != nil {
 			d.logger.Printf("[ERR] dns: cname recurse failed: %v", err)
 			return nil
 		}
@@ -630,3 +645,6 @@ func (d *DNSServer) resolveCNAME(name string) []dns.RR {
 		// Return all the answers
 		return r.Answer
 	}
+
+	return nil
+}

command/agent/dns_test.go

@@ -22,7 +22,7 @@ func makeDNSServerConfig(t *testing.T, config *DNSConfig) (string, *DNSServer) {
 	addr, _ := conf.ClientListener(conf.Addresses.DNS, conf.Ports.DNS)
 	dir, agent := makeAgent(t, conf)
 	server, err := NewDNSServer(agent, config, agent.logOutput,
-		conf.Domain, addr.String(), "8.8.8.8:53")
+		conf.Domain, addr.String(), []string{"8.8.8.8:53"})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}

website/source/docs/agent/dns.html.markdown

@@ -20,14 +20,14 @@ provide the redis service, located in the "east-aws" datacenter,
 with no failing health checks. It's that simple!
 
 There are a number of [configuration options](/docs/agent/options.html) that
-are important for the DNS interface. They are `client_addr`, `ports.dns`, `recursor`,
+are important for the DNS interface. They are `client_addr`, `ports.dns`, `recursors`,
 `domain`, and `dns_config`. By default Consul will listen on 127.0.0.1:8600 for DNS queries
 in the "consul." domain, without support for DNS recursion. All queries are case-insensitive, a
 name lookup for `PostgreSQL.node.dc1.consul` will find all nodes named `postgresql`, no matter of case.
 
 There are a few ways to use the DNS interface. One option is to use a custom
 DNS resolver library and point it at Consul. Another option is to set Consul
-as the DNS server for a node, and provide a `recursor` so that non-Consul queries
+as the DNS server for a node, and provide `recursors` so that non-Consul queries
 can also be resolved. The last method is to forward all queries for the "consul."
 domain to a Consul agent from the existing DNS server. To play with the DNS server
 on the command line, dig can be used:

website/source/docs/agent/http.html.markdown

@@ -333,7 +333,7 @@ It returns a JSON body like this:
     "Server": true,
     "Datacenter": "dc1",
     "DataDir": "/tmp/consul",
-    "DNSRecursor": "",
+    "DNSRecursors": [],
     "Domain": "consul.",
     "LogLevel": "INFO",
     "NodeName": "foobar",

website/source/docs/agent/options.html.markdown

@@ -316,10 +316,10 @@ definitions support being updated during a reload.
 
 * `protocol` - Equivalent to the `-protocol` command-line flag.
 
-* `recursor` - This flag provides an address of an upstream DNS server that is used to
+* `recursors` - This flag provides addresses of upstream DNS servers that are used to
   recursively resolve queries if they are not inside the service domain for consul. For example,
   a node can use Consul directly as a DNS server, and if the record is outside of the "consul." domain,
-  the query will be resolved upstream using this server.
+  the query will be resolved upstream using their servers.
 
 * `rejoin_after_leave` - Equivalent to the `-rejoin` command-line flag.