github
/
consul
mirror of https://github.com/hashicorp/consul
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Only pass one hostname via EDS and prefer healthy ones (#8084) 

Co-authored-by: Matt Keeler <mkeeler@users.noreply.github.com>

Currently when passing hostname clusters to Envoy, we set each service instance registered with Consul as an LbEndpoint for the cluster.

However, Envoy can only handle one per cluster:
[2020-06-04 18:32:34.094][1][warning][config] [source/common/config/grpc_subscription_impl.cc:87] gRPC config for type.googleapis.com/envoy.api.v2.Cluster rejected: Error adding/updating cluster(s) dc2.internal.ddd90499-9b47-91c5-4616-c0cbf0fc358a.consul: LOGICAL_DNS clusters must have a single locality_lb_endpoint and a single lb_endpoint, server.dc2.consul: LOGICAL_DNS clusters must have a single locality_lb_endpoint and a single lb_endpoint

Envoy is currently handling this gracefully by only picking one of the endpoints. However, we should avoid passing multiple to avoid these warning logs.

This PR:

* Ensures we only pass one endpoint, which is tied to one service instance.
* We prefer sending an endpoint which is marked as Healthy by Consul.
* If no endpoints are healthy we emit a warning and skip the cluster.
* If multiple unique hostnames are spread across service instances we emit a warning and let the user know which will be resolved.
pull/8103/head
Freddy 5 years ago committed by GitHub
parent
54d69bbb8f
commit
166a8b2a58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 48 additions and 116 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      agent/proxycfg/state.go
  2. 54
      agent/xds/clusters.go
  3. 12
      agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden
  4. 12
      agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden
  5. 12
      agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden
  6. 12
      agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden
  7. 12
      agent/xds/testdata/clusters/mesh-gateway.golden
  8. 12
      agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden
  9. 12
      agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden
  10. 12
      agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden
  11. 12
      agent/xds/testdata/clusters/terminating-gateway.golden

2
agent/proxycfg/state.go
Unescape View File

@ -1568,7 +1568,7 @@ func (s *state) hostnameEndpoints(loggerName string, localDC string, nodes struc
sid := nodes[0].Service.CompoundServiceName()
s.logger.Named(loggerName).
Warn("service contains instances with mix of hostnames and IP addresses; only hostnames will be passed to Envoy.",
Warn("service contains instances with mix of hostnames and IP addresses; only hostnames will be passed to Envoy",
"dc", dc, "service", sid.String())
}
return resp

54
agent/xds/clusters.go
Unescape View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/hashicorp/consul/logging"
"time"
envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2"
@ -582,6 +583,7 @@ type gatewayClusterOpts struct {
hostnameEndpoints structs.CheckServiceNodes
}
// makeGatewayCluster creates an Envoy cluster for a mesh or terminating gateway
func (s *Server) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, opts gatewayClusterOpts) *envoy.Cluster {
cfg, err := ParseGatewayConfig(snap.Proxy.Config)
if err != nil {
@ -631,11 +633,52 @@ func (s *Server) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, opts gatewayC
}
cluster.ClusterDiscoveryType = &discoveryType
endpoints := make([]envoyendpoint.LbEndpoint, 0, len(opts.hostnameEndpoints))
endpoints := make([]envoyendpoint.LbEndpoint, 0, 1)
uniqueHostnames := make(map[string]bool)
for _, e := range opts.hostnameEndpoints {
endpoints = append(endpoints, makeLbEndpoint(e, opts.isRemote, opts.onlyPassing))
var (
hostname string
idx int
)
for i, e := range opts.hostnameEndpoints {
addr, port := e.BestAddress(opts.isRemote)
uniqueHostnames[addr] = true
health, weight := calculateEndpointHealthAndWeight(e, opts.onlyPassing)
if health == envoycore.HealthStatus_UNHEALTHY {
continue
}
if len(endpoints) == 0 {
endpoints = append(endpoints, makeLbEndpoint(addr, port, health, weight))
hostname = addr
idx = i
break
}
}
dc := opts.hostnameEndpoints[idx].Node.Datacenter
service := opts.hostnameEndpoints[idx].Service.CompoundServiceName()
loggerName := logging.TerminatingGateway
if snap.Kind == structs.ServiceKindMeshGateway {
loggerName = logging.MeshGateway
}
if len(endpoints) == 0 {
s.Logger.Named(loggerName).
Warn("service does not contain any healthy instances, skipping Envoy cluster creation",
"dc", dc, "service", service.String())
return nil
}
if len(uniqueHostnames) > 1 {
s.Logger.Named(loggerName).
Warn(fmt.Sprintf("service contains instances with more than one unique hostname; only %q be resolved by Envoy", hostname),
"dc", dc, "service", service.String())
}
cluster.LoadAssignment = &envoy.ClusterLoadAssignment{
ClusterName: cluster.Name,
Endpoints: []envoyendpoint.LocalityLbEndpoints{
@ -683,10 +726,7 @@ func makeThresholdsIfNeeded(limits UpstreamLimits) []*envoycluster.CircuitBreake
return []*envoycluster.CircuitBreakers_Thresholds{threshold}
}
func makeLbEndpoint(csn structs.CheckServiceNode, isRemote, onlyPassing bool) envoyendpoint.LbEndpoint {
health, weight := calculateEndpointHealthAndWeight(csn, onlyPassing)
addr, port := csn.BestAddress(isRemote)
func makeLbEndpoint(addr string, port int, health envoycore.HealthStatus, weight int) envoyendpoint.LbEndpoint {
return envoyendpoint.LbEndpoint{
HostIdentifier: &envoyendpoint.LbEndpoint_Endpoint{
Endpoint: &envoyendpoint.Endpoint{

12
agent/xds/testdata/clusters/mesh-gateway-ignore-extra-resolvers.golden vendored
Unescape View File

@ -54,18 +54,6 @@
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}

12
agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden vendored
Unescape View File

@ -54,18 +54,6 @@
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}

12
agent/xds/testdata/clusters/mesh-gateway-service-timeouts.golden vendored
Unescape View File

@ -54,18 +54,6 @@
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}

12
agent/xds/testdata/clusters/mesh-gateway-using-federation-states.golden vendored
Unescape View File

@ -54,18 +54,6 @@
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}

12
agent/xds/testdata/clusters/mesh-gateway.golden vendored
Unescape View File

@ -54,18 +54,6 @@
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {
"socketAddress": {
"address": "456.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}

12
agent/xds/testdata/clusters/terminating-gateway-hostname-service-subsets.golden vendored
Unescape View File

@ -65,18 +65,6 @@
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {

12
agent/xds/testdata/clusters/terminating-gateway-ignore-extra-resolvers.golden vendored
Unescape View File

@ -11,18 +11,6 @@
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {

12
agent/xds/testdata/clusters/terminating-gateway-service-subsets.golden vendored
Unescape View File

@ -11,18 +11,6 @@
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {

12
agent/xds/testdata/clusters/terminating-gateway.golden vendored
Unescape View File

@ -11,18 +11,6 @@
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
},
{
"endpoint": {
"address": {

Write Preview
Loading…
Cancel
Save