Implement mesh gateway management of service subsets

Fixup some error handling

agent/proxycfg/snapshot.go

@@ -19,6 +19,7 @@ type configSnapshotMeshGateway struct {
 	WatchedServices    map[string]context.CancelFunc
 	WatchedDatacenters map[string]context.CancelFunc
 	ServiceGroups      map[string]structs.CheckServiceNodes
+	ServiceResolvers   map[string]*structs.ServiceResolverConfigEntry
 	GatewayGroups      map[string]structs.CheckServiceNodes
 }
 

agent/proxycfg/state.go

@@ -22,6 +22,7 @@ const (
 	intentionsWatchID                = "intentions"
 	serviceListWatchID               = "service-list"
 	datacentersWatchID               = "datacenters"
+	serviceResolversWatchID          = "service-resolvers"
 	serviceIDPrefix                  = string(structs.UpstreamDestTypeService) + ":"
 	preparedQueryIDPrefix            = string(structs.UpstreamDestTypePreparedQuery) + ":"
 	defaultPreparedQueryPollInterval = 30 * time.Second
@@ -347,9 +348,10 @@ func (s *state) run() {
 	case structs.ServiceKindMeshGateway:
 		snap.MeshGateway.WatchedServices = make(map[string]context.CancelFunc)
 		snap.MeshGateway.WatchedDatacenters = make(map[string]context.CancelFunc)
-		// TODO (mesh-gateway) - maybe reuse UpstreamEndpoints?
 		snap.MeshGateway.ServiceGroups = make(map[string]structs.CheckServiceNodes)
 		snap.MeshGateway.GatewayGroups = make(map[string]structs.CheckServiceNodes)
+		// there is no need to initialize the map of service resolvers as we
+		// fully rebuild it every time we get updates
 	}
 
 	// This turns out to be really fiddly/painful by just using time.Timer.C
@@ -675,7 +677,17 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
 					return err
 				}
 
-				// TODO (mesh-gateway) also should watch the resolver config for the service here
+				err = s.cache.Notify(ctx, cachetype.ConfigEntriesName, &structs.ConfigEntryQuery{
+					Datacenter:   s.source.Datacenter,
+					QueryOptions: structs.QueryOptions{Token: s.token},
+					Kind:         structs.ServiceResolver,
+				}, serviceResolversWatchID, s.ch)
+
+				if err != nil {
+					s.logger.Printf("[ERR] mesh-gateway: failed to register watch for service-resolver config entries")
+					cancel()
+					return err
+				}
 				snap.MeshGateway.WatchedServices[svcName] = cancel
 			}
 		}
@@ -735,6 +747,19 @@ func (s *state) handleUpdateMeshGateway(u cache.UpdateEvent, snap *ConfigSnapsho
 				cancelFn()
 			}
 		}
+	case serviceResolversWatchID:
+		configEntries, ok := u.Result.(*structs.IndexedConfigEntries)
+		if !ok {
+			return fmt.Errorf("invalid type for services response: %T", u.Result)
+		}
+
+		resolvers := make(map[string]*structs.ServiceResolverConfigEntry)
+		for _, entry := range configEntries.Entries {
+			if resolver, ok := entry.(*structs.ServiceResolverConfigEntry); ok {
+				resolvers[resolver.Name] = resolver
+			}
+		}
+		snap.MeshGateway.ServiceResolvers = resolvers
 	default:
 		switch {
 		case strings.HasPrefix(u.CorrelationID, "connect-service:"):

agent/proxycfg/testing.go

@@ -183,7 +183,75 @@ func TestGatewayNodesDC2(t testing.T) structs.CheckServiceNodes {
 	}
 }
 
-func TestGatewayServicesDC1(t testing.T) structs.CheckServiceNodes {
+func TestGatewayServiceGroupBarDC1(t testing.T) structs.CheckServiceNodes {
+	return structs.CheckServiceNodes{
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "bar-node-1",
+				Node:       "bar-node-1",
+				Address:    "10.1.1.4",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "bar-sidecar-proxy",
+				Address: "172.16.1.6",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "1",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "bar",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "bar-node-2",
+				Node:       "bar-node-2",
+				Address:    "10.1.1.5",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "bar-sidecar-proxy",
+				Address: "172.16.1.7",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "1",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "bar",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "bar-node-3",
+				Node:       "bar-node-3",
+				Address:    "10.1.1.6",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "bar-sidecar-proxy",
+				Address: "172.16.1.8",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "2",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "bar",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+	}
+}
+
+func TestGatewayServiceGroupFooDC1(t testing.T) structs.CheckServiceNodes {
 	return structs.CheckServiceNodes{
 		structs.CheckServiceNode{
 			Node: &structs.Node{
@@ -192,7 +260,19 @@ func TestGatewayServicesDC1(t testing.T) structs.CheckServiceNodes {
 				Address:    "10.1.1.1",
 				Datacenter: "dc1",
 			},
-			Service: structs.TestNodeServiceProxy(t),
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "foo-sidecar-proxy",
+				Address: "172.16.1.3",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "1",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "foo",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
 		},
 		structs.CheckServiceNode{
 			Node: &structs.Node{
@@ -201,7 +281,69 @@ func TestGatewayServicesDC1(t testing.T) structs.CheckServiceNodes {
 				Address:    "10.1.1.2",
 				Datacenter: "dc1",
 			},
-			Service: structs.TestNodeServiceProxy(t),
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "foo-sidecar-proxy",
+				Address: "172.16.1.4",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "1",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "foo",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "foo-node-3",
+				Node:       "foo-node-3",
+				Address:    "10.1.1.3",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "foo-sidecar-proxy",
+				Address: "172.16.1.5",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "2",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "foo",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+		},
+		structs.CheckServiceNode{
+			Node: &structs.Node{
+				ID:         "foo-node-4",
+				Node:       "foo-node-4",
+				Address:    "10.1.1.7",
+				Datacenter: "dc1",
+			},
+			Service: &structs.NodeService{
+				Kind:    structs.ServiceKindConnectProxy,
+				Service: "foo-sidecar-proxy",
+				Address: "172.16.1.9",
+				Port:    2222,
+				Meta: map[string]string{
+					"version": "2",
+				},
+				Proxy: structs.ConnectProxyConfig{
+					DestinationServiceName: "foo",
+					Upstreams:              structs.TestUpstreams(t),
+				},
+			},
+			Checks: structs.HealthChecks{
+				&structs.HealthCheck{
+					Node:        "foo-node-4",
+					ServiceName: "foo-sidecar-proxy",
+					Name:        "proxy-alive",
+					Status:      "warning",
+				},
+			},
 		},
 	}
 }
@@ -268,7 +410,8 @@ func TestConfigSnapshotMeshGateway(t testing.T) *ConfigSnapshot {
 				"dc2": nil,
 			},
 			ServiceGroups: map[string]structs.CheckServiceNodes{
-				"foo": TestGatewayServicesDC1(t),
+				"foo": TestGatewayServiceGroupFooDC1(t),
+				"bar": TestGatewayServiceGroupBarDC1(t),
 			},
 			GatewayGroups: map[string]structs.CheckServiceNodes{
 				"dc2": TestGatewayNodesDC2(t),

agent/xds/clusters.go

@@ -84,33 +84,42 @@ func (s *Server) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapsh
 // for a mesh gateway. This will include 1 cluster per remote datacenter as well as
 // 1 cluster for each service subset.
 func (s *Server) clustersFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) {
-	// TODO (mesh-gateway) will need to generate 1 cluster for each service subset as well.
+	// 1 cluster per remote dc + 1 cluster per local service (this is a lower bound - all subset specific clusters will be appended)
+	clusters := make([]proto.Message, 0, len(cfgSnap.MeshGateway.GatewayGroups)+len(cfgSnap.MeshGateway.ServiceGroups))
 
-	// 1 cluster per remote dc + 1 cluster per local service
-	clusters := make([]proto.Message, len(cfgSnap.MeshGateway.GatewayGroups)+len(cfgSnap.MeshGateway.ServiceGroups))
-
-	var err error
-	idx := 0
 	// generate the remote dc clusters
 	for dc, _ := range cfgSnap.MeshGateway.GatewayGroups {
 		clusterName := DatacenterSNI(dc, cfgSnap)
 
-		clusters[idx], err = s.makeMeshGatewayCluster(clusterName, cfgSnap)
+		cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap)
 		if err != nil {
 			return nil, err
 		}
-		idx += 1
+		clusters = append(clusters, cluster)
 	}
 
 	// generate the per-service clusters
 	for svc, _ := range cfgSnap.MeshGateway.ServiceGroups {
-		clusterName := ServiceSNI(svc, "default", cfgSnap.Datacenter, cfgSnap)
+		clusterName := ServiceSNI(svc, "", "default", cfgSnap.Datacenter, cfgSnap)
 
-		clusters[idx], err = s.makeMeshGatewayCluster(clusterName, cfgSnap)
+		cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap)
 		if err != nil {
 			return nil, err
 		}
-		idx += 1
+		clusters = append(clusters, cluster)
+	}
+
+	// generate the service subset clusters
+	for svc, resolver := range cfgSnap.MeshGateway.ServiceResolvers {
+		for subsetName, _ := range resolver.Subsets {
+			clusterName := ServiceSNI(svc, subsetName, "default", cfgSnap.Datacenter, cfgSnap)
+
+			cluster, err := s.makeMeshGatewayCluster(clusterName, cfgSnap)
+			if err != nil {
+				return nil, err
+			}
+			clusters = append(clusters, cluster)
+		}
 	}
 
 	return clusters, nil
@@ -174,7 +183,7 @@ func (s *Server) makeUpstreamCluster(upstream structs.Upstream, cfgSnap *proxycf
 	if upstream.Datacenter != "" {
 		dc = upstream.Datacenter
 	}
-	sni := ServiceSNI(upstream.DestinationName, ns, dc, cfgSnap)
+	sni := ServiceSNI(upstream.DestinationName, "", ns, dc, cfgSnap)
 
 	cfg, err := ParseUpstreamConfig(upstream.Config)
 	if err != nil {

agent/xds/clusters_test.go

@@ -9,6 +9,7 @@ import (
 	"text/template"
 
 	"github.com/hashicorp/consul/agent/proxycfg"
+	"github.com/hashicorp/consul/agent/structs"
 	testinf "github.com/mitchellh/go-testing-interface"
 	"github.com/stretchr/testify/require"
 )
@@ -102,6 +103,27 @@ func TestClustersFromSnapshot(t *testing.T) {
 			create: proxycfg.TestConfigSnapshotMeshGateway,
 			setup:  nil,
 		},
+		{
+			name:   "mesh-gateway-service-subsets",
+			create: proxycfg.TestConfigSnapshotMeshGateway,
+			setup: func(snap *proxycfg.ConfigSnapshot) {
+				snap.MeshGateway.ServiceResolvers = map[string]*structs.ServiceResolverConfigEntry{
+					"bar": &structs.ServiceResolverConfigEntry{
+						Kind: structs.ServiceResolver,
+						Name: "bar",
+						Subsets: map[string]structs.ServiceResolverSubset{
+							"v1": structs.ServiceResolverSubset{
+								Filter: "Service.Meta.Version == 1",
+							},
+							"v2": structs.ServiceResolverSubset{
+								Filter:      "Service.Meta.Version == 2",
+								OnlyPassing: true,
+							},
+						},
+					},
+				}
+			},
+		},
 	}
 
 	for _, tt := range tests {

agent/xds/endpoints.go

@@ -12,6 +12,8 @@ import (
 	"github.com/hashicorp/consul/agent/proxycfg"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/api"
+
+	bexpr "github.com/hashicorp/go-bexpr"
 )
 
 // endpointsFromSnapshot returns the xDS API representation of the "endpoints"
@@ -143,7 +145,7 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh
 
 	// generate the endpoints for the local service groups
 	for svc, endpoints := range cfgSnap.MeshGateway.ServiceGroups {
-		clusterName := ServiceSNI(svc, "default", cfgSnap.Datacenter, cfgSnap)
+		clusterName := ServiceSNI(svc, "", "default", cfgSnap.Datacenter, cfgSnap)
 		la := makeLoadAssignment(
 			clusterName,
 			0,
@@ -155,6 +157,51 @@ func (s *Server) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.ConfigSnapsh
 		resources = append(resources, la)
 	}
 
+	// generate the endpoints for the service subsets
+	for svc, resolver := range cfgSnap.MeshGateway.ServiceResolvers {
+		for subsetName, subset := range resolver.Subsets {
+			clusterName := ServiceSNI(svc, subsetName, "default", cfgSnap.Datacenter, cfgSnap)
+
+			endpoints := cfgSnap.MeshGateway.ServiceGroups[svc]
+
+			// locally execute the subsets filter
+			filterExp := subset.Filter
+			if subset.OnlyPassing {
+				// we could do another filter pass without bexpr but this simplifies things a bit
+				if filterExp != "" {
+					// TODO (filtering) - Update to "and all Checks as chk { chk.Status == passing }"
+					//                    once the syntax is supported
+					filterExp = fmt.Sprintf("(%s) and not Checks.Status != passing", filterExp)
+				} else {
+					filterExp = "not Checks.Status != passing"
+				}
+			}
+
+			if filterExp != "" {
+				filter, err := bexpr.CreateFilter(filterExp, nil, endpoints)
+				if err != nil {
+					return nil, err
+				}
+
+				raw, err := filter.Execute(endpoints)
+				if err != nil {
+					return nil, err
+				}
+				endpoints = raw.(structs.CheckServiceNodes)
+			}
+
+			la := makeLoadAssignment(
+				clusterName,
+				0,
+				[]structs.CheckServiceNodes{
+					endpoints,
+				},
+				cfgSnap.Datacenter,
+			)
+			resources = append(resources, la)
+		}
+	}
+
 	return resources, nil
 }
 

agent/xds/endpoints_test.go

@@ -233,6 +233,40 @@ func Test_endpointsFromSnapshot(t *testing.T) {
 			create: proxycfg.TestConfigSnapshotMeshGateway,
 			setup:  nil,
 		},
+		{
+			name:   "mesh-gateway-service-subsets",
+			create: proxycfg.TestConfigSnapshotMeshGateway,
+			setup: func(snap *proxycfg.ConfigSnapshot) {
+				snap.MeshGateway.ServiceResolvers = map[string]*structs.ServiceResolverConfigEntry{
+					"bar": &structs.ServiceResolverConfigEntry{
+						Kind: structs.ServiceResolver,
+						Name: "bar",
+						Subsets: map[string]structs.ServiceResolverSubset{
+							"v1": structs.ServiceResolverSubset{
+								Filter: "Service.Meta.version == 1",
+							},
+							"v2": structs.ServiceResolverSubset{
+								Filter:      "Service.Meta.version == 2",
+								OnlyPassing: true,
+							},
+						},
+					},
+					"foo": &structs.ServiceResolverConfigEntry{
+						Kind: structs.ServiceResolver,
+						Name: "foo",
+						Subsets: map[string]structs.ServiceResolverSubset{
+							"v1": structs.ServiceResolverSubset{
+								Filter: "Service.Meta.version == 1",
+							},
+							"v2": structs.ServiceResolverSubset{
+								Filter:      "Service.Meta.version == 2",
+								OnlyPassing: true,
+							},
+						},
+					},
+				}
+			},
+		},
 	}
 
 	for _, tt := range tests {

agent/xds/sni.go

@@ -10,7 +10,10 @@ func DatacenterSNI(dc string, cfgSnap *proxycfg.ConfigSnapshot) string {
 	return fmt.Sprintf("%s.internal.%s", dc, cfgSnap.Roots.TrustDomain)
 }
 
-func ServiceSNI(service string, namespace string, datacenter string, cfgSnap *proxycfg.ConfigSnapshot) string {
+func ServiceSNI(service string, subset string, namespace string, datacenter string, cfgSnap *proxycfg.ConfigSnapshot) string {
-	// TODO (mesh-gateway) - support service subsets here too
+	if subset == "" {
-	return fmt.Sprintf("%s.%s.%s.internal.%s", service, namespace, datacenter, cfgSnap.Roots.TrustDomain)
+		return fmt.Sprintf("%s.%s.%s.internal.%s", service, namespace, datacenter, cfgSnap.Roots.TrustDomain)
+	} else {
+		return fmt.Sprintf("%s.%s.%s.%s.internal.%s", subset, service, namespace, datacenter, cfgSnap.Roots.TrustDomain)
+	}
 }

agent/xds/testdata/clusters/mesh-gateway-service-subsets.golden

@@ -0,0 +1,87 @@
+{
+  "versionInfo": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "dc2.internal.11111111-2222-3333-4444-555555555555",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "v1.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "v2.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
+      }
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.api.v2.Cluster",
+  "nonce": "00000001"
+}

agent/xds/testdata/clusters/mesh-gateway.golden

@@ -31,6 +31,22 @@
       "connectTimeout": "5s",
       "outlierDetection": {
 
+      }
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.Cluster",
+      "name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "type": "EDS",
+      "edsClusterConfig": {
+        "edsConfig": {
+          "ads": {
+
+          }
+        }
+      },
+      "connectTimeout": "5s",
+      "outlierDetection": {
+
       }
     }
   ],

agent/xds/testdata/endpoints/mesh-gateway-service-subsets.golden

@@ -0,0 +1,257 @@
+{
+  "versionInfo": "00000001",
+  "resources": [
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "dc2.internal.11111111-2222-3333-4444-555555555555",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.18.1.1",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "198.18.1.2",
+                    "portValue": 443
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.6",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.7",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.8",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.3",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.4",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.5",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.9",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "v1.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.6",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.7",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "v2.bar.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.8",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "v1.foo.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.3",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.4",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "v2.foo.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.5",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "typeUrl": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+  "nonce": "00000001"
+}

agent/xds/testdata/endpoints/mesh-gateway.golden

@@ -45,7 +45,7 @@
               "endpoint": {
                 "address": {
                   "socketAddress": {
-                    "address": "127.0.0.2",
+                    "address": "172.16.1.3",
                     "portValue": 2222
                   }
                 }
@@ -57,7 +57,77 @@
               "endpoint": {
                 "address": {
                   "socketAddress": {
-                    "address": "127.0.0.2",
+                    "address": "172.16.1.4",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.5",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.9",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "@type": "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment",
+      "clusterName": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555",
+      "endpoints": [
+        {
+          "lbEndpoints": [
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.6",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.7",
+                    "portValue": 2222
+                  }
+                }
+              },
+              "healthStatus": "HEALTHY",
+              "loadBalancingWeight": 1
+            },
+            {
+              "endpoint": {
+                "address": {
+                  "socketAddress": {
+                    "address": "172.16.1.8",
                     "portValue": 2222
                   }
                 }