Do not fail Consul discovery on Prometheus startup when Consul is down.

retrieval/discovery/consul/consul.go

@@ -81,17 +81,7 @@ func NewDiscovery(conf *config.ConsulSDConfig) (*Discovery, error) {
 		clientConf:       clientConf,
 		tagSeparator:     conf.TagSeparator,
 		watchedServices:  conf.Services,
-	}
-	// If the datacenter isn't set in the clientConf, let's get it from the local Consul agent
-	// (Consul default is to use local node's datacenter if one isn't given for a query).
-	if clientConf.Datacenter == "" {
-		info, err := client.Agent().Self()
-		if err != nil {
-			return nil, err
-		}
-		cd.clientDatacenter = info["Config"]["Datacenter"].(string)
-	} else {
-		cd.clientDatacenter = clientConf.Datacenter
+		clientDatacenter: clientConf.Datacenter,
 	}
 	return cd, nil
 }
@@ -144,6 +134,18 @@ func (cd *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
 		}
 		lastIndex = meta.LastIndex
 
+		// If the datacenter was not set from clientConf, let's get it from the local Consul agent
+		// (Consul default is to use local node's datacenter if one isn't given for a query).
+		if cd.clientDatacenter == "" {
+			info, err := cd.client.Agent().Self()
+			if err != nil {
+				log.Errorf("Error retrieving datacenter name: %s", err)
+				time.Sleep(retryInterval)
+				continue
+			}
+			cd.clientDatacenter = info["Config"]["Datacenter"].(string)
+		}
+
 		// Check for new services.
 		for name := range srvs {
 			if !cd.shouldWatch(name) {