v2: various fixes to make K8s tproxy multiport acceptance tests and manual explicit upstreams (single port) tests pass (#18874)

Adding coauthors who mobbed/paired at various points throughout last week.
Co-authored-by: Dan Stough <dan.stough@hashicorp.com>
Co-authored-by: Iryna Shustava <iryna@hashicorp.com>
Co-authored-by: John Murret <john.murret@hashicorp.com>
Co-authored-by: Michael Zalimeni <michael.zalimeni@hashicorp.com>
Co-authored-by: Ashwin Venkatesh <ashwin@hashicorp.com>
Co-authored-by: Michael Wilkerson <mwilkerson@hashicorp.com>
pull/18917/head
Nitya Dhanushkodi 2023-09-19 17:02:01 -07:00 committed by GitHub
parent 1a3081ab32
commit 3a2e62053a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 187 additions and 54 deletions

View File

@ -59,7 +59,6 @@ func (s *Server) GetEnvoyBootstrapParams(ctx context.Context, req *pbdataplane.G
Tenancy: &pbresource.Tenancy{
Namespace: req.Namespace,
Partition: req.Partition,
PeerName: "local",
},
Type: catalog.WorkloadType,
}
@ -69,6 +68,7 @@ func (s *Server) GetEnvoyBootstrapParams(ctx context.Context, req *pbdataplane.G
if err != nil {
// This error should already include the gRPC status code and so we don't need to wrap it
// in status.Error.
logger.Error("Error looking up workload", "error", err)
return nil, err
}
var workload pbcatalog.Workload
@ -93,6 +93,7 @@ func (s *Server) GetEnvoyBootstrapParams(ctx context.Context, req *pbdataplane.G
Type: mesh.ProxyConfigurationType,
})
if err != nil {
logger.Error("Error looking up proxyConfiguration", "error", err)
return nil, err
}

View File

@ -116,7 +116,13 @@ func getEnvoyConfiguration(proxySnapshot proxysnapshot.ProxySnapshot, logger hcl
)
c := proxySnapshot.(*proxytracker.ProxyState)
logger.Trace("ProxyState", c)
return generator.AllResourcesFromIR(c)
resources, err := generator.AllResourcesFromIR(c)
if err != nil {
logger.Error("error generating resources from proxy state template", "err", err)
return nil, err
}
logger.Trace("generated resources from proxy state template", "resources", resources)
return resources, nil
default:
return nil, errors.New("proxysnapshot must be of type ProxyState or ConfigSnapshot")
}
@ -428,9 +434,8 @@ func newResourceIDFromEnvoyNode(node *envoy_config_core_v3.Node) *pbresource.ID
Tenancy: &pbresource.Tenancy{
Namespace: entMeta.NamespaceOrDefault(),
Partition: entMeta.PartitionOrDefault(),
PeerName: "local",
},
Type: mesh.ProxyStateTemplateV1AlphaType,
Type: mesh.ProxyStateTemplateType,
}
}

View File

@ -376,6 +376,8 @@ func addEnvoyLBToCluster(dynamicConfig *pbproxystate.DynamicEndpointGroupConfig,
}
// TODO(proxystate): In a future PR this will create clusters and add it to ProxyResources.proxyState
// Currently, we do not traverse the listener -> endpoint paths and instead just generate each resource by iterating
// through its top level map. In the future we want to traverse these paths to ensure each listener has a cluster, etc.
func (pr *ProxyResources) makeEnvoyClusterFromL4Destination(l4 *pbproxystate.L4Destination) error {
return nil
}

View File

@ -540,14 +540,13 @@ func (pr *ProxyResources) makeEnvoyTLSParameters(defaultParams *pbproxystate.TLS
}
func (pr *ProxyResources) makeEnvoyTransportSocket(ts *pbproxystate.TransportSocket) (*envoy_core_v3.TransportSocket, error) {
// TODO(JM): did this just make tests pass. Figure out whether proxyState.Tls will always be available.
if pr.proxyState.Tls == nil {
return nil, nil
}
if ts == nil {
return nil, nil
}
commonTLSContext := &envoy_tls_v3.CommonTlsContext{}
if ts.AlpnProtocols != nil {
commonTLSContext.AlpnProtocols = ts.AlpnProtocols
}
// Create connection TLS. Listeners should only look at inbound TLS.
switch ts.ConnectionTls.(type) {
@ -555,16 +554,16 @@ func (pr *ProxyResources) makeEnvoyTransportSocket(ts *pbproxystate.TransportSoc
downstreamContext := &envoy_tls_v3.DownstreamTlsContext{}
downstreamContext.CommonTlsContext = commonTLSContext
// Set TLS Parameters.
tlsParams := pr.makeEnvoyTLSParameters(pr.proxyState.Tls.InboundTlsParameters, ts.TlsParameters)
commonTLSContext.TlsParams = tlsParams
if pr.proxyState.Tls != nil {
tlsParams := pr.makeEnvoyTLSParameters(pr.proxyState.Tls.InboundTlsParameters, ts.TlsParameters)
commonTLSContext.TlsParams = tlsParams
} else {
commonTLSContext.TlsParams = &envoy_tls_v3.TlsParameters{}
}
// Set the certificate config on the tls context.
// For inbound mesh, we need to add the identity certificate
// and the validation context for the mesh depending on the provided trust bundle names.
if pr.proxyState.Tls == nil {
// if tls is nil but connection tls is provided, then the proxy state is misconfigured
return nil, fmt.Errorf("proxyState.Tls is required to generate router's transport socket")
}
im := ts.ConnectionTls.(*pbproxystate.TransportSocket_InboundMesh).InboundMesh
leaf, ok := pr.proxyState.LeafCertificates[im.IdentityKey]
if !ok {
@ -640,9 +639,13 @@ func (pr *ProxyResources) makeEnvoyTransportSocket(ts *pbproxystate.TransportSoc
case *pbproxystate.TransportSocket_InboundNonMesh:
downstreamContext := &envoy_tls_v3.DownstreamTlsContext{}
downstreamContext.CommonTlsContext = commonTLSContext
// Set TLS Parameters
tlsParams := pr.makeEnvoyTLSParameters(pr.proxyState.Tls.InboundTlsParameters, ts.TlsParameters)
commonTLSContext.TlsParams = tlsParams
// Set TLS Parameters.
if pr.proxyState.Tls != nil {
tlsParams := pr.makeEnvoyTLSParameters(pr.proxyState.Tls.InboundTlsParameters, ts.TlsParameters)
commonTLSContext.TlsParams = tlsParams
} else {
commonTLSContext.TlsParams = &envoy_tls_v3.TlsParameters{}
}
// For non-mesh, we don't care about validation context as currently we don't support mTLS for non-mesh connections.
nonMeshTLS := ts.ConnectionTls.(*pbproxystate.TransportSocket_InboundNonMesh).InboundNonMesh
err := pr.addNonMeshCertConfig(commonTLSContext, nonMeshTLS)
@ -657,15 +660,15 @@ func (pr *ProxyResources) makeEnvoyTransportSocket(ts *pbproxystate.TransportSoc
case *pbproxystate.TransportSocket_OutboundMesh:
upstreamContext := &envoy_tls_v3.UpstreamTlsContext{}
upstreamContext.CommonTlsContext = commonTLSContext
// Set TLS Parameters
tlsParams := pr.makeEnvoyTLSParameters(pr.proxyState.Tls.OutboundTlsParameters, ts.TlsParameters)
commonTLSContext.TlsParams = tlsParams
// Set TLS Parameters.
if pr.proxyState.Tls != nil {
tlsParams := pr.makeEnvoyTLSParameters(pr.proxyState.Tls.OutboundTlsParameters, ts.TlsParameters)
commonTLSContext.TlsParams = tlsParams
} else {
commonTLSContext.TlsParams = &envoy_tls_v3.TlsParameters{}
}
// For outbound mesh, we need to insert the mesh identity certificate
// and the validation context for the mesh depending on the provided trust bundle names.
if pr.proxyState.Tls == nil {
// if tls is nil but connection tls is provided, then the proxy state is misconfigured
return nil, fmt.Errorf("proxyState.Tls is required to generate router's transport socket")
}
om := ts.GetOutboundMesh()
leaf, ok := pr.proxyState.LeafCertificates[om.IdentityKey]
if !ok {

View File

@ -4,17 +4,19 @@
package xdsv2
import (
"os"
"path/filepath"
"sort"
"testing"
envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
envoy_listener_v3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
"github.com/hashicorp/consul/agent/xds/response"
"github.com/hashicorp/consul/envoyextensions/xdscommon"
proxytracker "github.com/hashicorp/consul/internal/mesh/proxy-tracker"
meshv1alpha1 "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1"
"github.com/hashicorp/consul/sdk/testutil"
"os"
"path/filepath"
"sort"
"testing"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"

View File

@ -53,7 +53,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
]
],
"trustBundlePeerNameKey": "local"
},
"sni": "api-1.default.dc1.internal.foo.consul"
},
@ -64,8 +65,20 @@
}
}
}
},
"leafCertificates": {
"test-identity": {
"cert": "cert1",
"key": "key1"
}
},
"trustBundles": {
"local": {
"trustDomain": "foo.consul",
"roots": ["root1"]
}
}
},
"requiredEndpoints": {
"api-1.default.dc1.internal.foo.consul": {
"id": {

View File

@ -10,7 +10,42 @@
"ads": {},
"resourceApiVersion": "V3"
}
}
},
"name": "tcp.api-1.default.dc1.internal.foo.consul",
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"alpnProtocols": [
"consul~tcp"
],
"tlsCertificates": [
{
"certificateChain": {
"inlineString": "cert1\n"
},
"privateKey": {
"inlineString": "key1\n"
}
}
],
"tlsParams": {},
"validationContext": {
"matchSubjectAltNames": [
{
"exact": "spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
}
],
"trustedCa": {
"inlineString": "root1\n"
}
}
},
"sni": "api-1.default.dc1.internal.foo.consul"
}
},
"type": "EDS"
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",

View File

@ -508,7 +508,8 @@ func (b *Builder) addCluster(clusterName, sni, portName string, destinationIdent
OutboundMesh: &pbproxystate.OutboundMeshMTLS{
IdentityKey: b.proxyStateTemplate.ProxyState.Identity.Name,
ValidationContext: &pbproxystate.MeshOutboundValidationContext{
SpiffeIds: spiffeIDs,
SpiffeIds: spiffeIDs,
TrustBundlePeerNameKey: b.id.Tenancy.PeerName,
},
Sni: sni,
},

View File

@ -243,6 +243,9 @@ func (b *Builder) addInboundListener(name string, workload *pbcatalog.Workload)
},
}
// Add TLS inspection capability to be able to parse ALPN and/or SNI information from inbound connections.
listener.Capabilities = append(listener.Capabilities, pbproxystate.Capability_CAPABILITY_L4_TLS_INSPECTION)
return b.NewListenerBuilder(listener)
}

View File

@ -18,7 +18,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -43,7 +44,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api2-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}

View File

@ -28,7 +28,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -53,7 +54,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api2-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}

View File

@ -18,7 +18,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -43,7 +44,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api2-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}

View File

@ -28,7 +28,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -50,7 +51,9 @@
"outboundMesh": {
"identityKey": "test-identity",
"sni": "api-2.default.dc1.internal.foo.consul",
"validationContext": {}
"validationContext": {
"trustBundlePeerNameKey": "local"
}
}
}
}

View File

@ -18,7 +18,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api2-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}

View File

@ -18,7 +18,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}

View File

@ -18,7 +18,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -40,7 +41,9 @@
"outboundMesh": {
"identityKey": "test-identity",
"sni": "api-2.default.dc1.internal.foo.consul",
"validationContext": {}
"validationContext": {
"trustBundlePeerNameKey": "local"
}
}
}
}
@ -74,7 +77,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -99,7 +103,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api2-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}

View File

@ -18,7 +18,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api-app-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -43,7 +44,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api-app2-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -68,7 +70,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api-app-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -93,7 +96,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api-app2-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}

View File

@ -18,7 +18,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api-app-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -43,7 +44,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api-app-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}

View File

@ -18,7 +18,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api-app-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}
@ -43,7 +44,8 @@
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api-app-identity"
]
],
"trustBundlePeerNameKey": "local"
}
}
}

View File

@ -29,6 +29,9 @@
},
"listeners": [
{
"capabilities": [
"CAPABILITY_L4_TLS_INSPECTION"
],
"direction": "DIRECTION_INBOUND",
"hostPort": {
"host": "10.0.0.2",

View File

@ -29,6 +29,9 @@
},
"listeners": [
{
"capabilities": [
"CAPABILITY_L4_TLS_INSPECTION"
],
"direction": "DIRECTION_INBOUND",
"hostPort": {
"host": "10.0.0.1",

View File

@ -29,6 +29,9 @@
},
"listeners": [
{
"capabilities": [
"CAPABILITY_L4_TLS_INSPECTION"
],
"direction": "DIRECTION_INBOUND",
"hostPort": {
"host": "10.0.0.1",

View File

@ -44,6 +44,9 @@
},
"listeners": [
{
"capabilities": [
"CAPABILITY_L4_TLS_INSPECTION"
],
"direction": "DIRECTION_INBOUND",
"hostPort": {
"host": "10.0.0.3",

View File

@ -44,6 +44,9 @@
},
"listeners": [
{
"capabilities": [
"CAPABILITY_L4_TLS_INSPECTION"
],
"direction": "DIRECTION_INBOUND",
"hostPort": {
"host": "10.0.0.1",

View File

@ -44,6 +44,9 @@
},
"listeners": [
{
"capabilities": [
"CAPABILITY_L4_TLS_INSPECTION"
],
"direction": "DIRECTION_INBOUND",
"hostPort": {
"host": "10.0.0.1",

View File

@ -17,6 +17,9 @@
},
"listeners": [
{
"capabilities": [
"CAPABILITY_L4_TLS_INSPECTION"
],
"direction": "DIRECTION_INBOUND",
"hostPort": {
"host": "10.0.0.1",

View File

@ -202,6 +202,11 @@ func (r *reconciler) Reconcile(ctx context.Context, rt controller.Runtime, req c
// Get all destinationsData.
destinationsRefs := r.destinationsCache.DestinationsBySourceProxy(req.ID)
if len(destinationsRefs) > 0 {
rt.Logger.Trace("found destinations for this proxy", "id", req.ID, "destination_refs", destinationsRefs)
} else {
rt.Logger.Trace("did not find any destinations for this proxy", "id", req.ID)
}
destinationsData, statuses, err := dataFetcher.FetchExplicitDestinationsData(ctx, destinationsRefs)
if err != nil {
rt.Logger.Error("error fetching explicit destinations for this proxy", "error", err)

View File

@ -352,7 +352,12 @@ func (f *Fetcher) FetchImplicitDestinationsData(
if err != nil {
return nil, err
}
endpointsMap[seRK] = se
// We only add the endpoint to the map if it's not nil. If it's missing on lookup now, the
// controller should get triggered when the endpoint exists again since it watches service
// endpoints.
if se != nil {
endpointsMap[seRK] = se
}
}
}
}
@ -439,6 +444,12 @@ func (f *Fetcher) FetchAndMergeProxyConfigurations(ctx context.Context, id *pbre
proto.Merge(result.DynamicConfig, proxyCfg.DynamicConfig)
}
// Default the outbound listener port. If we don't do the nil check here, then BuildDestinations will panic creating
// the outbound listener.
if result.DynamicConfig.TransparentProxy == nil {
result.DynamicConfig.TransparentProxy = &pbmesh.TransparentProxy{OutboundListenerPort: 15001}
}
return result, nil
}

View File

@ -1029,6 +1029,9 @@ func (suite *dataFetcherSuite) TestFetcher_FetchAndMergeProxyConfigurations() {
DynamicConfig: &pbmesh.DynamicConfig{
Mode: pbmesh.ProxyMode_PROXY_MODE_TRANSPARENT,
MutualTlsMode: pbmesh.MutualTLSMode_MUTUAL_TLS_MODE_DEFAULT,
TransparentProxy: &pbmesh.TransparentProxy{
OutboundListenerPort: 15001,
},
},
}

View File

@ -5,6 +5,7 @@ package sidecarproxymapper
import (
"context"
"fmt"
"github.com/hashicorp/consul/internal/catalog"
"github.com/hashicorp/consul/internal/controller"
@ -54,6 +55,9 @@ func mapSelectorToProxyStateTemplates(ctx context.Context,
if err != nil {
return nil, err
}
if len(resp.Resources) == 0 {
return nil, fmt.Errorf("no workloads found")
}
for _, r := range resp.Resources {
id := resource.ReplaceType(types.ProxyStateTemplateType, r.Id)
result = append(result, controller.Request{