From 68e7f27fd2be44ac4463ad1d261ebb609620755e Mon Sep 17 00:00:00 2001 From: Kumar Kavish <108507473+kkavish@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:32:36 +0530 Subject: [PATCH] [NET-6438] Add tenancy to xDS Tests (#19551) * [NET-6438] Add tenancy to xDS Tests * [NET-6438] Add tenancy to xDS Tests - Fixing imports * [NET-6438] Add tenancy to xDS Tests - Added cleanup post test run * [NET-6356] Add tenancy to xDS Tests - Added cleanup post test run * [NET-6438] Add tenancy to xDS Tests - using t.Cleanup instead of defer delete * [NET-6438] Add tenancy to xDS Tests - rebased * [NET-6438] Add tenancy to xDS Tests - rebased --- .../services/resource/testing/testing.go | 2 +- .../controllers/xds/controller_test.go | 1234 +++++++++-------- internal/resource/resourcetest/tenancy.go | 2 +- 3 files changed, 684 insertions(+), 554 deletions(-) diff --git a/agent/grpc-external/services/resource/testing/testing.go b/agent/grpc-external/services/resource/testing/testing.go index 6184d9241b..f38d745321 100644 --- a/agent/grpc-external/services/resource/testing/testing.go +++ b/agent/grpc-external/services/resource/testing/testing.go @@ -5,7 +5,6 @@ package testing import ( "context" - rtest "github.com/hashicorp/consul/internal/resource/resourcetest" "testing" "github.com/hashicorp/go-uuid" @@ -21,6 +20,7 @@ import ( internal "github.com/hashicorp/consul/agent/grpc-internal" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/internal/resource" + rtest "github.com/hashicorp/consul/internal/resource/resourcetest" "github.com/hashicorp/consul/internal/storage/inmem" "github.com/hashicorp/consul/internal/tenancy" "github.com/hashicorp/consul/proto-public/pbresource" diff --git a/internal/mesh/internal/controllers/xds/controller_test.go b/internal/mesh/internal/controllers/xds/controller_test.go index 15c4536e55..f0aca598f0 100644 --- a/internal/mesh/internal/controllers/xds/controller_test.go +++ b/internal/mesh/internal/controllers/xds/controller_test.go @@ -11,7 +11,6 @@ import ( "strings" "testing" - "github.com/hashicorp/consul/internal/testing/golden" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "google.golang.org/protobuf/encoding/protojson" @@ -26,6 +25,7 @@ import ( "github.com/hashicorp/consul/internal/resource" "github.com/hashicorp/consul/internal/resource/mappers/bimapper" "github.com/hashicorp/consul/internal/resource/resourcetest" + "github.com/hashicorp/consul/internal/testing/golden" pbcatalog "github.com/hashicorp/consul/proto-public/pbcatalog/v2beta1" pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1" "github.com/hashicorp/consul/proto-public/pbmesh/v2beta1/pbproxystate" @@ -51,6 +51,7 @@ type xdsControllerTestSuite struct { leafCancels *LeafCancels leafCertEvents chan controller.Event signer *leafcert.TestSigner + tenancies []*pbresource.Tenancy fooProxyStateTemplate *pbresource.Resource barProxyStateTemplate *pbresource.Resource @@ -69,7 +70,7 @@ type xdsControllerTestSuite struct { func (suite *xdsControllerTestSuite) SetupTest() { suite.ctx = testutil.TestContext(suite.T()) - resourceClient := svctest.RunResourceService(suite.T(), types.Register, catalog.RegisterTypes) + resourceClient := svctest.RunResourceServiceWithTenancies(suite.T(), types.Register, catalog.RegisterTypes) suite.runtime = controller.Runtime{Client: resourceClient, Logger: testutil.Logger(suite.T())} suite.client = resourcetest.NewClient(resourceClient) suite.fetcher = mockFetcher @@ -99,6 +100,8 @@ func (suite *xdsControllerTestSuite) SetupTest() { leafCertEvents: suite.leafCertEvents, datacenter: "dc1", } + + suite.tenancies = resourcetest.TestTenancies() } func mockFetcher() (*pbproxystate.TrustBundle, error) { @@ -112,205 +115,233 @@ func mockFetcher() (*pbproxystate.TrustBundle, error) { // This test ensures when a ProxyState is deleted, it is no longer tracked in the mappers. func (suite *xdsControllerTestSuite) TestReconcile_NoProxyStateTemplate() { - // Track the id of a non-existent ProxyStateTemplate. - proxyStateTemplateId := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "not-found").ID() - suite.mapper.TrackItem(proxyStateTemplateId, []resource.ReferenceOrID{}) - suite.leafMapper.TrackItem(proxyStateTemplateId, []resource.ReferenceOrID{}) - require.False(suite.T(), suite.mapper.IsEmpty()) - require.False(suite.T(), suite.leafMapper.IsEmpty()) - - // Run the reconcile, and since no ProxyStateTemplate is stored, this simulates a deletion. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: proxyStateTemplateId, - }) - require.NoError(suite.T(), err) + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Track the id of a non-existent ProxyStateTemplate. + proxyStateTemplateId := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "not-found").WithTenancy(tenancy).ID() + + suite.T().Cleanup(suite.deleteResourceFunc(proxyStateTemplateId)) + + suite.mapper.TrackItem(proxyStateTemplateId, []resource.ReferenceOrID{}) + suite.leafMapper.TrackItem(proxyStateTemplateId, []resource.ReferenceOrID{}) + require.False(suite.T(), suite.mapper.IsEmpty()) + require.False(suite.T(), suite.leafMapper.IsEmpty()) - // Assert that nothing is tracked in the endpoints mapper. - require.True(suite.T(), suite.mapper.IsEmpty()) - require.True(suite.T(), suite.leafMapper.IsEmpty()) + // Run the reconcile, and since no ProxyStateTemplate is stored, this simulates a deletion. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: proxyStateTemplateId, + }) + require.NoError(suite.T(), err) + + // Assert that nothing is tracked in the endpoints mapper. + require.True(suite.T(), suite.mapper.IsEmpty()) + require.True(suite.T(), suite.leafMapper.IsEmpty()) + }) } // This test ensures if the controller was previously tracking a ProxyStateTemplate, and now that proxy has // disconnected from this server, it's ignored and removed from the mapper. func (suite *xdsControllerTestSuite) TestReconcile_RemoveTrackingProxiesNotConnectedToServer() { - // Store the initial ProxyStateTemplate and track it in the mapper. - proxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "test"). - WithData(suite.T(), &pbmesh.ProxyStateTemplate{}). - Write(suite.T(), suite.client) + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Store the initial ProxyStateTemplate and track it in the mapper. + proxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "test"). + WithData(suite.T(), &pbmesh.ProxyStateTemplate{}). + WithTenancy(tenancy). + Write(suite.T(), suite.client) - suite.mapper.TrackItem(proxyStateTemplate.Id, []resource.ReferenceOrID{}) + suite.T().Cleanup(suite.deleteResourceFunc(proxyStateTemplate.Id)) - // Simulate the proxy disconnecting from this server. The resource still exists, but this proxy might be connected - // to a different server now, so we no longer need to track it. - suite.updater.notConnected = true + suite.mapper.TrackItem(proxyStateTemplate.Id, []resource.ReferenceOrID{}) - // Run the reconcile. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: proxyStateTemplate.Id, - }) - require.NoError(suite.T(), err) + // Simulate the proxy disconnecting from this server. The resource still exists, but this proxy might be connected + // to a different server now, so we no longer need to track it. + suite.updater.notConnected = true + + // Run the reconcile. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: proxyStateTemplate.Id, + }) + require.NoError(suite.T(), err) - // Assert that nothing is tracked in the mapper. - require.True(suite.T(), suite.mapper.IsEmpty()) + // Assert that nothing is tracked in the mapper. + require.True(suite.T(), suite.mapper.IsEmpty()) + }) } // This test sets up the updater to return an error when calling PushChange, and ensures the status is set // correctly. func (suite *xdsControllerTestSuite) TestReconcile_PushChangeError() { - // Have the mock simulate an error from the PushChange call. - suite.updater.pushChangeError = true + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Have the mock simulate an error from the PushChange call. + suite.updater.pushChangeError = true - // Setup a happy path scenario. - suite.setupFooProxyStateTemplateWithReferences() + // Setup a happy path scenario. + suite.setupFooProxyStateTemplateWithReferences(tenancy) - // Run the reconcile. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: suite.fooProxyStateTemplate.Id, - }) - require.Error(suite.T(), err) + // Run the reconcile. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: suite.fooProxyStateTemplate.Id, + }) + require.Error(suite.T(), err) - // Assert on the status reflecting endpoint not found. - suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionRejectedPushChangeFailed(status.KeyFromID(suite.fooProxyStateTemplate.Id))) + // Assert on the status reflecting endpoint not found. + suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionRejectedPushChangeFailed(status.KeyFromID(suite.fooProxyStateTemplate.Id))) + }) } // This test sets up a ProxyStateTemplate that references a ServiceEndpoints that doesn't exist, and ensures the // status is correct. func (suite *xdsControllerTestSuite) TestReconcile_MissingEndpoint() { - // Set fooProxyStateTemplate with a reference to fooEndpoints, without storing fooEndpoints so the controller should - // notice it's missing. - fooEndpointsId := resourcetest.Resource(pbcatalog.ServiceEndpointsType, "foo-service").WithTenancy(resource.DefaultNamespacedTenancy()).ID() - fooRequiredEndpoints := make(map[string]*pbproxystate.EndpointRef) - fooRequiredEndpoints["test-cluster-1"] = &pbproxystate.EndpointRef{ - Id: fooEndpointsId, - Port: "mesh", - } + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Set fooProxyStateTemplate with a reference to fooEndpoints, without storing fooEndpoints so the controller should + // notice it's missing. + fooEndpointsId := resourcetest.Resource(pbcatalog.ServiceEndpointsType, "foo-service").WithTenancy(tenancy).ID() + fooRequiredEndpoints := make(map[string]*pbproxystate.EndpointRef) + fooRequiredEndpoints["test-cluster-1"] = &pbproxystate.EndpointRef{ + Id: fooEndpointsId, + Port: "mesh", + } - fooProxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "foo-pst"). - WithData(suite.T(), &pbmesh.ProxyStateTemplate{ - RequiredEndpoints: fooRequiredEndpoints, - ProxyState: &pbmesh.ProxyState{}, - }). - Write(suite.T(), suite.client) + fooProxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "foo-pst"). + WithData(suite.T(), &pbmesh.ProxyStateTemplate{ + RequiredEndpoints: fooRequiredEndpoints, + ProxyState: &pbmesh.ProxyState{}, + }). + WithTenancy(tenancy). + Write(suite.T(), suite.client) - retry.Run(suite.T(), func(r *retry.R) { - suite.client.RequireResourceExists(r, fooProxyStateTemplate.Id) - }) + suite.T().Cleanup(suite.deleteResourceFunc(fooProxyStateTemplate.Id)) - // Run the reconcile. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: fooProxyStateTemplate.Id, - }) - require.Error(suite.T(), err) + retry.Run(suite.T(), func(r *retry.R) { + suite.client.RequireResourceExists(r, fooProxyStateTemplate.Id) + }) - // Assert on the status reflecting endpoint not found. - suite.client.RequireStatusCondition(suite.T(), fooProxyStateTemplate.Id, ControllerName, status.ConditionRejectedErrorReadingEndpoints(status.KeyFromID(fooEndpointsId), "rpc error: code = NotFound desc = resource not found")) + // Run the reconcile. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: fooProxyStateTemplate.Id, + }) + require.Error(suite.T(), err) + + // Assert on the status reflecting endpoint not found. + suite.client.RequireStatusCondition(suite.T(), fooProxyStateTemplate.Id, ControllerName, status.ConditionRejectedErrorReadingEndpoints(status.KeyFromID(fooEndpointsId), "rpc error: code = NotFound desc = resource not found")) + }) } // This test sets up a ProxyStateTemplate that references a ServiceEndpoints that can't be read correctly, and // checks the status is correct. func (suite *xdsControllerTestSuite) TestReconcile_ReadEndpointError() { - badID := &pbresource.ID{ - Type: &pbresource.Type{ - Group: "not", - Kind: "found", - GroupVersion: "vfake", - }, - Tenancy: &pbresource.Tenancy{Namespace: "default", Partition: "default", PeerName: "local"}, - } - fooRequiredEndpoints := make(map[string]*pbproxystate.EndpointRef) - fooRequiredEndpoints["test-cluster-1"] = &pbproxystate.EndpointRef{ - Id: badID, - Port: "mesh", - } + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + badID := &pbresource.ID{ + Type: &pbresource.Type{ + Group: "not", + Kind: "found", + GroupVersion: "vfake", + }, + Tenancy: tenancy, + } + fooRequiredEndpoints := make(map[string]*pbproxystate.EndpointRef) + fooRequiredEndpoints["test-cluster-1"] = &pbproxystate.EndpointRef{ + Id: badID, + Port: "mesh", + } - fooProxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "foo-pst"). - WithData(suite.T(), &pbmesh.ProxyStateTemplate{ - RequiredEndpoints: fooRequiredEndpoints, - ProxyState: &pbmesh.ProxyState{}, - }). - Write(suite.T(), suite.client) + fooProxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "foo-pst"). + WithData(suite.T(), &pbmesh.ProxyStateTemplate{ + RequiredEndpoints: fooRequiredEndpoints, + ProxyState: &pbmesh.ProxyState{}, + }). + WithTenancy(tenancy). + Write(suite.T(), suite.client) - retry.Run(suite.T(), func(r *retry.R) { - suite.client.RequireResourceExists(r, fooProxyStateTemplate.Id) - }) + suite.T().Cleanup(suite.deleteResourceFunc(fooProxyStateTemplate.Id)) - // Run the reconcile. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: fooProxyStateTemplate.Id, - }) - require.Error(suite.T(), err) + retry.Run(suite.T(), func(r *retry.R) { + suite.client.RequireResourceExists(r, fooProxyStateTemplate.Id) + }) - // Assert on the status reflecting endpoint couldn't be read. - suite.client.RequireStatusCondition(suite.T(), fooProxyStateTemplate.Id, ControllerName, status.ConditionRejectedErrorReadingEndpoints( - status.KeyFromID(badID), - "rpc error: code = InvalidArgument desc = id.name invalid: a resource name must consist of lower case alphanumeric characters or '-', must start and end with an alphanumeric character and be less than 64 characters, got: \"\"", - )) + // Run the reconcile. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: fooProxyStateTemplate.Id, + }) + require.Error(suite.T(), err) + + // Assert on the status reflecting endpoint couldn't be read. + suite.client.RequireStatusCondition(suite.T(), fooProxyStateTemplate.Id, ControllerName, status.ConditionRejectedErrorReadingEndpoints( + status.KeyFromID(badID), + "rpc error: code = InvalidArgument desc = id.name invalid: a resource name must consist of lower case alphanumeric characters or '-', must start and end with an alphanumeric character and be less than 64 characters, got: \"\"", + )) + }) } // This test is a happy path creation test to make sure pbproxystate.Endpoints are created in the computed // pbmesh.ProxyState from the RequiredEndpoints references. More specific translations between endpoint references // and pbproxystate.Endpoints are unit tested in endpoint_builder.go. func (suite *xdsControllerTestSuite) TestReconcile_ProxyStateTemplateComputesEndpoints() { - // Set up fooEndpoints and fooProxyStateTemplate with a reference to fooEndpoints and store them in the state store. - // This setup saves expected values in the suite so it can be asserted against later. - suite.setupFooProxyStateTemplateWithReferences() - - // Run the reconcile. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: suite.fooProxyStateTemplate.Id, - }) - require.NoError(suite.T(), err) + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Set up fooEndpoints and fooProxyStateTemplate with a reference to fooEndpoints and store them in the state store. + // This setup saves expected values in the suite so it can be asserted against later. + suite.setupFooProxyStateTemplateWithReferences(tenancy) + + // Run the reconcile. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: suite.fooProxyStateTemplate.Id, + }) + require.NoError(suite.T(), err) - // Assert on the status. - suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert on the status. + suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) - prototest.AssertDeepEqual(suite.T(), suite.expectedFooProxyStateEndpoints, actualEndpoints) + // Assert that the endpoints computed in the controller matches the expected endpoints. + actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) + prototest.AssertDeepEqual(suite.T(), suite.expectedFooProxyStateEndpoints, actualEndpoints) + }) } func (suite *xdsControllerTestSuite) TestReconcile_ProxyStateTemplateComputesLeafCerts() { - // Set up fooEndpoints and fooProxyStateTemplate with a reference to fooEndpoints and store them in the state store. - // This setup saves expected values in the suite so it can be asserted against later. - suite.setupFooProxyStateTemplateWithReferences() - - // Run the reconcile. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: suite.fooProxyStateTemplate.Id, - }) - require.NoError(suite.T(), err) + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Set up fooEndpoints and fooProxyStateTemplate with a reference to fooEndpoints and store them in the state store. + // This setup saves expected values in the suite so it can be asserted against later. + suite.setupFooProxyStateTemplateWithReferences(tenancy) + + // Run the reconcile. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: suite.fooProxyStateTemplate.Id, + }) + require.NoError(suite.T(), err) - // Assert on the status. - suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert on the status. + suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the actual leaf certs computed are match the expected leaf cert spiffes. - actualLeafs := suite.updater.GetLeafs(suite.fooProxyStateTemplate.Id.Name) + // Assert that the actual leaf certs computed are match the expected leaf cert spiffes. + actualLeafs := suite.updater.GetLeafs(suite.fooProxyStateTemplate.Id.Name) - for k, l := range actualLeafs { - pem, _ := pem.Decode([]byte(l.Cert)) - cert, err := x509.ParseCertificate(pem.Bytes) - require.NoError(suite.T(), err) - require.Equal(suite.T(), cert.URIs[0].String(), suite.expectedFooProxyStateSpiffes[k]) - } + for k, l := range actualLeafs { + pemDecode, _ := pem.Decode([]byte(l.Cert)) + cert, err := x509.ParseCertificate(pemDecode.Bytes) + require.NoError(suite.T(), err) + require.Equal(suite.T(), cert.URIs[0].String(), suite.expectedFooProxyStateSpiffes[k]) + } + }) } // This test is a happy path creation test to make sure pbproxystate.Template.TrustBundles are created in the computed // pbmesh.ProxyState from the TrustBundleFetcher. func (suite *xdsControllerTestSuite) TestReconcile_ProxyStateTemplateSetsTrustBundles() { - suite.setupFooProxyStateTemplateWithReferences() + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + suite.setupFooProxyStateTemplateWithReferences(tenancy) - // Run the reconcile. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: suite.fooProxyStateTemplate.Id, - }) - require.NoError(suite.T(), err) + // Run the reconcile. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: suite.fooProxyStateTemplate.Id, + }) + require.NoError(suite.T(), err) - // Assert on the status. - suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert on the status. + suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - actualTrustBundle := suite.updater.GetTrustBundle(suite.fooProxyStateTemplate.Id.Name) - prototest.AssertDeepEqual(suite.T(), suite.expectedTrustBundle, actualTrustBundle) + // Assert that the endpoints computed in the controller matches the expected endpoints. + actualTrustBundle := suite.updater.GetTrustBundle(suite.fooProxyStateTemplate.Id.Name) + prototest.AssertDeepEqual(suite.T(), suite.expectedTrustBundle, actualTrustBundle) + }) } // This test is a happy path creation test that calls reconcile multiple times with a more complex setup. This @@ -318,342 +349,368 @@ func (suite *xdsControllerTestSuite) TestReconcile_ProxyStateTemplateSetsTrustBu // stored in the state store. So this test ensures that between multiple reconciles the correct ProxyStates are // computed for each ProxyStateTemplate. func (suite *xdsControllerTestSuite) TestReconcile_MultipleProxyStateTemplatesComputesMultipleEndpoints() { - // Set up fooProxyStateTemplate and barProxyStateTemplate and their associated resources and store them. Resources - // and expected results are stored in the suite to assert against. - suite.setupFooBarProxyStateTemplateAndEndpoints() - - // Reconcile the fooProxyStateTemplate. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: suite.fooProxyStateTemplate.Id, - }) - require.NoError(suite.T(), err) + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Set up fooProxyStateTemplate and barProxyStateTemplate and their associated resources and store them. Resources + // and expected results are stored in the suite to assert against. + suite.setupFooBarProxyStateTemplateAndEndpoints(tenancy) + + // Reconcile the fooProxyStateTemplate. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: suite.fooProxyStateTemplate.Id, + }) + require.NoError(suite.T(), err) - // Assert on the status. - suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert on the status. + suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) - prototest.AssertDeepEqual(suite.T(), suite.expectedFooProxyStateEndpoints, actualEndpoints) + // Assert that the endpoints computed in the controller matches the expected endpoints. + actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) + prototest.AssertDeepEqual(suite.T(), suite.expectedFooProxyStateEndpoints, actualEndpoints) - // Reconcile the barProxyStateTemplate. - err = suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: suite.barProxyStateTemplate.Id, - }) - require.NoError(suite.T(), err) + // Reconcile the barProxyStateTemplate. + err = suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: suite.barProxyStateTemplate.Id, + }) + require.NoError(suite.T(), err) - // Assert on the status. - suite.client.RequireStatusCondition(suite.T(), suite.barProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert on the status. + suite.client.RequireStatusCondition(suite.T(), suite.barProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - actualBarEndpoints := suite.updater.GetEndpoints(suite.barProxyStateTemplate.Id.Name) - prototest.AssertDeepEqual(suite.T(), suite.expectedBarProxyStateEndpoints, actualBarEndpoints) + // Assert that the endpoints computed in the controller matches the expected endpoints. + actualBarEndpoints := suite.updater.GetEndpoints(suite.barProxyStateTemplate.Id.Name) + prototest.AssertDeepEqual(suite.T(), suite.expectedBarProxyStateEndpoints, actualBarEndpoints) + }) } // Sets up a full controller, and tests that reconciles are getting triggered for the events it should. func (suite *xdsControllerTestSuite) TestController_ComputeAddUpdateEndpointReferences() { - // Run the controller manager. - mgr := controller.NewManager(suite.client, suite.runtime.Logger) - mgr.Register(Controller(suite.mapper, suite.updater, suite.fetcher, suite.leafCertManager, suite.leafMapper, suite.leafCancels, "dc1")) - mgr.SetRaftLeader(true) - go mgr.Run(suite.ctx) - - suite.setupFooProxyStateTemplateWithReferences() - - // Assert that the expected ProxyState matches the actual ProxyState that PushChange was called with. This needs to - // be in a retry block unlike the Reconcile tests because the controller triggers asynchronously. - retry.Run(suite.T(), func(r *retry.R) { - actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) - // Assert on the status. - suite.client.RequireStatusCondition(r, suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) - }) + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Run the controller manager. + mgr := controller.NewManager(suite.client, suite.runtime.Logger) + mgr.Register(Controller(suite.mapper, suite.updater, suite.fetcher, suite.leafCertManager, suite.leafMapper, suite.leafCancels, "dc1")) + mgr.SetRaftLeader(true) + go mgr.Run(suite.ctx) + + suite.setupFooProxyStateTemplateWithReferences(tenancy) + + // Assert that the expected ProxyState matches the actual ProxyState that PushChange was called with. This needs to + // be in a retry block unlike the Reconcile tests because the controller triggers asynchronously. + retry.Run(suite.T(), func(r *retry.R) { + actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) + // Assert on the status. + suite.client.RequireStatusCondition(r, suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert that the endpoints computed in the controller matches the expected endpoints. + prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) + }) - // Now, update the endpoint to be unhealthy. This will ensure the controller is getting triggered on changes to this - // endpoint that it should be tracking, even when the ProxyStateTemplate does not change. - resourcetest.Resource(pbcatalog.ServiceEndpointsType, "foo-service"). - WithData(suite.T(), &pbcatalog.ServiceEndpoints{Endpoints: []*pbcatalog.Endpoint{ - { - Ports: map[string]*pbcatalog.WorkloadPort{ - "mesh": { - Port: 20000, - Protocol: pbcatalog.Protocol_PROTOCOL_MESH, - }, - }, - Addresses: []*pbcatalog.WorkloadAddress{ - { - Host: "10.1.1.1", - Ports: []string{"mesh"}, + // Now, update the endpoint to be unhealthy. This will ensure the controller is getting triggered on changes to this + // endpoint that it should be tracking, even when the ProxyStateTemplate does not change. + svc := resourcetest.Resource(pbcatalog.ServiceEndpointsType, "foo-service"). + WithData(suite.T(), &pbcatalog.ServiceEndpoints{Endpoints: []*pbcatalog.Endpoint{ + { + Ports: map[string]*pbcatalog.WorkloadPort{ + "mesh": { + Port: 20000, + Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + }, }, - { - Host: "10.2.2.2", - Ports: []string{"mesh"}, + Addresses: []*pbcatalog.WorkloadAddress{ + { + Host: "10.1.1.1", + Ports: []string{"mesh"}, + }, + { + Host: "10.2.2.2", + Ports: []string{"mesh"}, + }, }, + HealthStatus: pbcatalog.Health_HEALTH_CRITICAL, }, - HealthStatus: pbcatalog.Health_HEALTH_CRITICAL, - }, - }}). - WithOwner(suite.fooService.Id). - Write(suite.T(), suite.client) + }}). + WithOwner(suite.fooService.Id). + WithTenancy(tenancy). + Write(suite.T(), suite.client) - // Wait for the endpoint to be written. - retry.Run(suite.T(), func(r *retry.R) { - suite.client.RequireVersionChanged(suite.T(), suite.fooEndpoints.Id, suite.fooEndpoints.Version) - }) + suite.T().Cleanup(suite.deleteResourceFunc(svc.Id)) - // Update the expected endpoints to also have unhealthy status. - suite.expectedFooProxyStateEndpoints["test-cluster-1"].Endpoints[0].HealthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY - suite.expectedFooProxyStateEndpoints["test-cluster-1"].Endpoints[1].HealthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + // Wait for the endpoint to be written. + retry.Run(suite.T(), func(r *retry.R) { + suite.client.RequireVersionChanged(suite.T(), suite.fooEndpoints.Id, suite.fooEndpoints.Version) + }) - retry.Run(suite.T(), func(r *retry.R) { - actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) - // Assert on the status. - suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) - }) + // Update the expected endpoints to also have unhealthy status. + suite.expectedFooProxyStateEndpoints["test-cluster-1"].Endpoints[0].HealthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + suite.expectedFooProxyStateEndpoints["test-cluster-1"].Endpoints[1].HealthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY - // Now add a new endpoint reference and endpoint to the fooProxyStateTemplate. This will ensure that the controller - // now tracks the newly added endpoint. - secondService := resourcetest.Resource(pbcatalog.ServiceType, "second-service"). - WithData(suite.T(), &pbcatalog.Service{}). - Write(suite.T(), suite.client) + retry.Run(suite.T(), func(r *retry.R) { + actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) + // Assert on the status. + suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert that the endpoints computed in the controller matches the expected endpoints. + prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) + }) - secondEndpoints := resourcetest.Resource(pbcatalog.ServiceEndpointsType, "second-service"). - WithData(suite.T(), &pbcatalog.ServiceEndpoints{Endpoints: []*pbcatalog.Endpoint{ - { - Ports: map[string]*pbcatalog.WorkloadPort{ - "mesh": { - Port: 20000, - Protocol: pbcatalog.Protocol_PROTOCOL_MESH, - }, - }, - Addresses: []*pbcatalog.WorkloadAddress{ - { - Host: "10.5.5.5", - Ports: []string{"mesh"}, + // Now add a new endpoint reference and endpoint to the fooProxyStateTemplate. This will ensure that the controller + // now tracks the newly added endpoint. + secondService := resourcetest.Resource(pbcatalog.ServiceType, "second-service"). + WithData(suite.T(), &pbcatalog.Service{}). + WithTenancy(tenancy). + Write(suite.T(), suite.client) + + suite.T().Cleanup(suite.deleteResourceFunc(secondService.Id)) + + secondEndpoints := resourcetest.Resource(pbcatalog.ServiceEndpointsType, "second-service"). + WithData(suite.T(), &pbcatalog.ServiceEndpoints{Endpoints: []*pbcatalog.Endpoint{ + { + Ports: map[string]*pbcatalog.WorkloadPort{ + "mesh": { + Port: 20000, + Protocol: pbcatalog.Protocol_PROTOCOL_MESH, + }, }, - { - Host: "10.6.6.6", - Ports: []string{"mesh"}, + Addresses: []*pbcatalog.WorkloadAddress{ + { + Host: "10.5.5.5", + Ports: []string{"mesh"}, + }, + { + Host: "10.6.6.6", + Ports: []string{"mesh"}, + }, }, }, - }, - }}). - WithOwner(secondService.Id). - Write(suite.T(), suite.client) + }}). + WithOwner(secondService.Id). + WithTenancy(tenancy). + Write(suite.T(), suite.client) - // Update the endpoint references on the fooProxyStateTemplate. - suite.fooEndpointRefs["test-cluster-2"] = &pbproxystate.EndpointRef{ - Id: secondEndpoints.Id, - Port: "mesh", - } + suite.T().Cleanup(suite.deleteResourceFunc(secondEndpoints.Id)) - oldVersion := suite.fooProxyStateTemplate.Version - fooProxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "foo-pst"). - WithData(suite.T(), &pbmesh.ProxyStateTemplate{ - RequiredEndpoints: suite.fooEndpointRefs, - ProxyState: &pbmesh.ProxyState{}, - RequiredLeafCertificates: suite.fooLeafRefs, - }). - Write(suite.T(), suite.client) + // Update the endpoint references on the fooProxyStateTemplate. + suite.fooEndpointRefs["test-cluster-2"] = &pbproxystate.EndpointRef{ + Id: secondEndpoints.Id, + Port: "mesh", + } - retry.Run(suite.T(), func(r *retry.R) { - suite.client.RequireVersionChanged(r, fooProxyStateTemplate.Id, oldVersion) - }) + oldVersion := suite.fooProxyStateTemplate.Version + fooProxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "foo-pst"). + WithData(suite.T(), &pbmesh.ProxyStateTemplate{ + RequiredEndpoints: suite.fooEndpointRefs, + ProxyState: &pbmesh.ProxyState{}, + RequiredLeafCertificates: suite.fooLeafRefs, + }). + WithTenancy(tenancy). + Write(suite.T(), suite.client) - // Update the expected endpoints with this new endpoints. - suite.expectedFooProxyStateEndpoints["test-cluster-2"] = &pbproxystate.Endpoints{ - Endpoints: []*pbproxystate.Endpoint{ - { - Address: &pbproxystate.Endpoint_HostPort{ - HostPort: &pbproxystate.HostPortAddress{ - Host: "10.5.5.5", - Port: 20000, + suite.T().Cleanup(suite.deleteResourceFunc(fooProxyStateTemplate.Id)) + + retry.Run(suite.T(), func(r *retry.R) { + suite.client.RequireVersionChanged(r, fooProxyStateTemplate.Id, oldVersion) + }) + + // Update the expected endpoints with this new endpoints. + suite.expectedFooProxyStateEndpoints["test-cluster-2"] = &pbproxystate.Endpoints{ + Endpoints: []*pbproxystate.Endpoint{ + { + Address: &pbproxystate.Endpoint_HostPort{ + HostPort: &pbproxystate.HostPortAddress{ + Host: "10.5.5.5", + Port: 20000, + }, }, + HealthStatus: pbproxystate.HealthStatus_HEALTH_STATUS_HEALTHY, }, - HealthStatus: pbproxystate.HealthStatus_HEALTH_STATUS_HEALTHY, - }, - { - Address: &pbproxystate.Endpoint_HostPort{ - HostPort: &pbproxystate.HostPortAddress{ - Host: "10.6.6.6", - Port: 20000, + { + Address: &pbproxystate.Endpoint_HostPort{ + HostPort: &pbproxystate.HostPortAddress{ + Host: "10.6.6.6", + Port: 20000, + }, }, + HealthStatus: pbproxystate.HealthStatus_HEALTH_STATUS_HEALTHY, }, - HealthStatus: pbproxystate.HealthStatus_HEALTH_STATUS_HEALTHY, }, - }, - } + } - retry.Run(suite.T(), func(r *retry.R) { - actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) - // Assert on the status. - suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) - }) + retry.Run(suite.T(), func(r *retry.R) { + actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) + // Assert on the status. + suite.client.RequireStatusCondition(suite.T(), suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert that the endpoints computed in the controller matches the expected endpoints. + prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) + }) + }) } // Sets up a full controller, and tests that reconciles are getting triggered for the leaf cert events it should. // This test ensures when a CA is updated, the controller is triggered to update the leaf cert when it changes. func (suite *xdsControllerTestSuite) TestController_ComputeAddUpdateDeleteLeafReferences() { - // Run the controller manager. - mgr := controller.NewManager(suite.client, suite.runtime.Logger) - mgr.Register(Controller(suite.mapper, suite.updater, suite.fetcher, suite.leafCertManager, suite.leafMapper, suite.leafCancels, "dc1")) - mgr.SetRaftLeader(true) - go mgr.Run(suite.ctx) - - suite.setupFooProxyStateTemplateWithReferences() - leafCertRef := suite.fooLeafRefs["foo-workload-identity"] - fooLeafResRef := leafResourceRef(leafCertRef.Name, leafCertRef.Namespace, leafCertRef.Partition) - - // oldLeaf will store the original leaf from before we trigger a CA update. - var oldLeaf *x509.Certificate + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Run the controller manager. + mgr := controller.NewManager(suite.client, suite.runtime.Logger) + mgr.Register(Controller(suite.mapper, suite.updater, suite.fetcher, suite.leafCertManager, suite.leafMapper, suite.leafCancels, "dc1")) + mgr.SetRaftLeader(true) + go mgr.Run(suite.ctx) + + suite.setupFooProxyStateTemplateWithReferences(tenancy) + leafCertRef := suite.fooLeafRefs["foo-workload-identity"] + fooLeafResRef := leafResourceRef(leafCertRef.Name, leafCertRef.Namespace, leafCertRef.Partition) + + // oldLeaf will store the original leaf from before we trigger a CA update. + var oldLeaf *x509.Certificate + + // Assert that the expected ProxyState matches the actual ProxyState that PushChange was called with. This needs to + // be in a retry block unlike the Reconcile tests because the controller triggers asynchronously. + retry.Run(suite.T(), func(r *retry.R) { + actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) + actualLeafs := suite.updater.GetLeafs(suite.fooProxyStateTemplate.Id.Name) + // Assert on the status. + suite.client.RequireStatusCondition(r, suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert that the endpoints computed in the controller matches the expected endpoints. + prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) + // Assert that the leafs computed in the controller matches the expected leafs. + require.Len(r, actualLeafs, 1) + for k, l := range actualLeafs { + pemDecode, _ := pem.Decode([]byte(l.Cert)) + cert, err := x509.ParseCertificate(pemDecode.Bytes) + oldLeaf = cert + require.NoError(r, err) + require.Equal(r, cert.URIs[0].String(), suite.expectedFooProxyStateSpiffes[k]) + // Check the state of the cancel functions map. + _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) + require.True(r, ok) + } + }) - // Assert that the expected ProxyState matches the actual ProxyState that PushChange was called with. This needs to - // be in a retry block unlike the Reconcile tests because the controller triggers asynchronously. - retry.Run(suite.T(), func(r *retry.R) { - actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) - actualLeafs := suite.updater.GetLeafs(suite.fooProxyStateTemplate.Id.Name) - // Assert on the status. - suite.client.RequireStatusCondition(r, suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) - // Assert that the leafs computed in the controller matches the expected leafs. - require.Len(r, actualLeafs, 1) - for k, l := range actualLeafs { - pem, _ := pem.Decode([]byte(l.Cert)) - cert, err := x509.ParseCertificate(pem.Bytes) - oldLeaf = cert - require.NoError(r, err) - require.Equal(r, cert.URIs[0].String(), suite.expectedFooProxyStateSpiffes[k]) - // Check the state of the cancel functions map. - _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) - require.True(r, ok) - } - }) + // Update the CA, and ensure the leaf cert is different from the leaf certificate from the step above. + suite.signer.UpdateCA(suite.T(), nil) + retry.Run(suite.T(), func(r *retry.R) { + actualLeafs := suite.updater.GetLeafs(suite.fooProxyStateTemplate.Id.Name) + require.Len(r, actualLeafs, 1) + for k, l := range actualLeafs { + pemDecode, _ := pem.Decode([]byte(l.Cert)) + cert, err := x509.ParseCertificate(pemDecode.Bytes) + // Ensure the leaf was actually updated by checking that the leaf we just got is different from the old leaf. + require.NotEqual(r, oldLeaf.Raw, cert.Raw) + require.NoError(r, err) + require.Equal(r, suite.expectedFooProxyStateSpiffes[k], cert.URIs[0].String()) + // Check the state of the cancel functions map. Even though we've updated the leaf cert, we should still + // have a watch going for it. + _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) + require.True(r, ok) + } + }) - // Update the CA, and ensure the leaf cert is different from the leaf certificate from the step above. - suite.signer.UpdateCA(suite.T(), nil) - retry.Run(suite.T(), func(r *retry.R) { - actualLeafs := suite.updater.GetLeafs(suite.fooProxyStateTemplate.Id.Name) - require.Len(r, actualLeafs, 1) - for k, l := range actualLeafs { - pem, _ := pem.Decode([]byte(l.Cert)) - cert, err := x509.ParseCertificate(pem.Bytes) - // Ensure the leaf was actually updated by checking that the leaf we just got is different from the old leaf. - require.NotEqual(r, oldLeaf.Raw, cert.Raw) - require.NoError(r, err) - require.Equal(r, suite.expectedFooProxyStateSpiffes[k], cert.URIs[0].String()) - // Check the state of the cancel functions map. Even though we've updated the leaf cert, we should still - // have a watch going for it. - _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) - require.True(r, ok) - } - }) + // Delete the leaf references on the fooProxyStateTemplate + delete(suite.fooLeafRefs, "foo-workload-identity") + oldVersion := suite.fooProxyStateTemplate.Version + fooProxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "foo-pst"). + WithData(suite.T(), &pbmesh.ProxyStateTemplate{ + RequiredEndpoints: suite.fooEndpointRefs, + ProxyState: &pbmesh.ProxyState{}, + RequiredLeafCertificates: suite.fooLeafRefs, + }). + WithTenancy(tenancy). + Write(suite.T(), suite.client) - // Delete the leaf references on the fooProxyStateTemplate - delete(suite.fooLeafRefs, "foo-workload-identity") - oldVersion := suite.fooProxyStateTemplate.Version - fooProxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "foo-pst"). - WithData(suite.T(), &pbmesh.ProxyStateTemplate{ - RequiredEndpoints: suite.fooEndpointRefs, - ProxyState: &pbmesh.ProxyState{}, - RequiredLeafCertificates: suite.fooLeafRefs, - }). - Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(fooProxyStateTemplate.Id)) - retry.Run(suite.T(), func(r *retry.R) { - suite.client.RequireVersionChanged(r, fooProxyStateTemplate.Id, oldVersion) - }) + retry.Run(suite.T(), func(r *retry.R) { + suite.client.RequireVersionChanged(r, fooProxyStateTemplate.Id, oldVersion) + }) - // Ensure the leaf certificate watches were cancelled since we deleted the leaf reference. - retry.Run(suite.T(), func(r *retry.R) { - _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) - require.False(r, ok) + // Ensure the leaf certificate watches were cancelled since we deleted the leaf reference. + retry.Run(suite.T(), func(r *retry.R) { + _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) + require.False(r, ok) + }) }) } // Sets up a full controller, and tests that reconciles are getting triggered for the leaf cert events it should. // This test ensures that when a ProxyStateTemplate is deleted, the leaf watches are cancelled. func (suite *xdsControllerTestSuite) TestController_ComputeLeafReferencesDeletePST() { - // Run the controller manager. - mgr := controller.NewManager(suite.client, suite.runtime.Logger) - mgr.Register(Controller(suite.mapper, suite.updater, suite.fetcher, suite.leafCertManager, suite.leafMapper, suite.leafCancels, "dc1")) - mgr.SetRaftLeader(true) - go mgr.Run(suite.ctx) - - suite.setupFooProxyStateTemplateWithReferences() - leafCertRef := suite.fooLeafRefs["foo-workload-identity"] - fooLeafResRef := leafResourceRef(leafCertRef.Name, leafCertRef.Namespace, leafCertRef.Partition) - - // Assert that the expected ProxyState matches the actual ProxyState that PushChange was called with. This needs to - // be in a retry block unlike the Reconcile tests because the controller triggers asynchronously. - retry.Run(suite.T(), func(r *retry.R) { - actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) - actualLeafs := suite.updater.GetLeafs(suite.fooProxyStateTemplate.Id.Name) - // Assert on the status. - suite.client.RequireStatusCondition(r, suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) - // Assert that the leafs computed in the controller matches the expected leafs. - require.Len(r, actualLeafs, 1) - for k, l := range actualLeafs { - pem, _ := pem.Decode([]byte(l.Cert)) - cert, err := x509.ParseCertificate(pem.Bytes) - require.NoError(r, err) - require.Equal(r, cert.URIs[0].String(), suite.expectedFooProxyStateSpiffes[k]) - // Check the state of the cancel functions map. - _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) - require.True(r, ok) - } - }) + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Run the controller manager. + mgr := controller.NewManager(suite.client, suite.runtime.Logger) + mgr.Register(Controller(suite.mapper, suite.updater, suite.fetcher, suite.leafCertManager, suite.leafMapper, suite.leafCancels, "dc1")) + mgr.SetRaftLeader(true) + go mgr.Run(suite.ctx) + + suite.setupFooProxyStateTemplateWithReferences(tenancy) + leafCertRef := suite.fooLeafRefs["foo-workload-identity"] + fooLeafResRef := leafResourceRef(leafCertRef.Name, leafCertRef.Namespace, leafCertRef.Partition) + + // Assert that the expected ProxyState matches the actual ProxyState that PushChange was called with. This needs to + // be in a retry block unlike the Reconcile tests because the controller triggers asynchronously. + retry.Run(suite.T(), func(r *retry.R) { + actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) + actualLeafs := suite.updater.GetLeafs(suite.fooProxyStateTemplate.Id.Name) + // Assert on the status. + suite.client.RequireStatusCondition(r, suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert that the endpoints computed in the controller matches the expected endpoints. + prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) + // Assert that the leafs computed in the controller matches the expected leafs. + require.Len(r, actualLeafs, 1) + for k, l := range actualLeafs { + pemDecode, _ := pem.Decode([]byte(l.Cert)) + cert, err := x509.ParseCertificate(pemDecode.Bytes) + require.NoError(r, err) + require.Equal(r, cert.URIs[0].String(), suite.expectedFooProxyStateSpiffes[k]) + // Check the state of the cancel functions map. + _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) + require.True(r, ok) + } + }) - // Delete the fooProxyStateTemplate + // Delete the fooProxyStateTemplate - req := &pbresource.DeleteRequest{ - Id: suite.fooProxyStateTemplate.Id, - } - suite.client.Delete(suite.ctx, req) + req := &pbresource.DeleteRequest{ + Id: suite.fooProxyStateTemplate.Id, + } + _, err := suite.client.Delete(suite.ctx, req) + require.NoError(suite.T(), err) - // Ensure the leaf certificate watches were cancelled since we deleted the leaf reference. - retry.Run(suite.T(), func(r *retry.R) { - _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) - require.False(r, ok) + // Ensure the leaf certificate watches were cancelled since we deleted the leaf reference. + retry.Run(suite.T(), func(r *retry.R) { + _, ok := suite.leafCancels.Get(keyFromReference(fooLeafResRef)) + require.False(r, ok) + }) }) } // Sets up a full controller, and tests that reconciles are getting triggered for the events it should. func (suite *xdsControllerTestSuite) TestController_ComputeEndpointForProxyConnections() { - // Run the controller manager. - mgr := controller.NewManager(suite.client, suite.runtime.Logger) - - mgr.Register(Controller(suite.mapper, suite.updater, suite.fetcher, suite.leafCertManager, suite.leafMapper, suite.leafCancels, "dc1")) - mgr.SetRaftLeader(true) - go mgr.Run(suite.ctx) + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + // Run the controller manager. + mgr := controller.NewManager(suite.client, suite.runtime.Logger) + + mgr.Register(Controller(suite.mapper, suite.updater, suite.fetcher, suite.leafCertManager, suite.leafMapper, suite.leafCancels, "dc1")) + mgr.SetRaftLeader(true) + go mgr.Run(suite.ctx) + + // Set up fooEndpoints and fooProxyStateTemplate with a reference to fooEndpoints. These need to be stored + // because the controller reconcile looks them up. + suite.setupFooProxyStateTemplateWithReferences(tenancy) + + // Assert that the expected ProxyState matches the actual ProxyState that PushChange was called with. This needs to + // be in a retry block unlike the Reconcile tests because the controller triggers asynchronously. + retry.Run(suite.T(), func(r *retry.R) { + actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) + // Assert on the status. + suite.client.RequireStatusCondition(r, suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) + // Assert that the endpoints computed in the controller matches the expected endpoints. + prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) + }) - // Set up fooEndpoints and fooProxyStateTemplate with a reference to fooEndpoints. These need to be stored - // because the controller reconcile looks them up. - suite.setupFooProxyStateTemplateWithReferences() + eventChannel := suite.updater.EventChannel() + eventChannel <- controller.Event{Obj: &proxytracker.ProxyConnection{ProxyID: suite.fooProxyStateTemplate.Id}} - // Assert that the expected ProxyState matches the actual ProxyState that PushChange was called with. This needs to - // be in a retry block unlike the Reconcile tests because the controller triggers asynchronously. - retry.Run(suite.T(), func(r *retry.R) { - actualEndpoints := suite.updater.GetEndpoints(suite.fooProxyStateTemplate.Id.Name) - // Assert on the status. - suite.client.RequireStatusCondition(r, suite.fooProxyStateTemplate.Id, ControllerName, status.ConditionAccepted()) - // Assert that the endpoints computed in the controller matches the expected endpoints. - prototest.AssertDeepEqual(r, suite.expectedFooProxyStateEndpoints, actualEndpoints) + // Wait for the proxy state template to be re-evaluated. + proxyStateTemp := suite.client.WaitForNewVersion(suite.T(), suite.fooProxyStateTemplate.Id, suite.fooProxyStateTemplate.Version) + require.NotNil(suite.T(), proxyStateTemp) }) - - eventChannel := suite.updater.EventChannel() - eventChannel <- controller.Event{Obj: &proxytracker.ProxyConnection{ProxyID: suite.fooProxyStateTemplate.Id}} - - // Wait for the proxy state template to be re-evaluated. - proxyStateTemp := suite.client.WaitForNewVersion(suite.T(), suite.fooProxyStateTemplate.Id, suite.fooProxyStateTemplate.Version) - require.NotNil(suite.T(), proxyStateTemp) } // Setup: fooProxyStateTemplate with: @@ -661,11 +718,14 @@ func (suite *xdsControllerTestSuite) TestController_ComputeEndpointForProxyConne // - a LeafCertificateRef to "foo-workload-identity" // // Saves all related resources to the suite so they can be looked up by the controller or modified if needed. -func (suite *xdsControllerTestSuite) setupFooProxyStateTemplateWithReferences() { +func (suite *xdsControllerTestSuite) setupFooProxyStateTemplateWithReferences(tenancy *pbresource.Tenancy) { fooService := resourcetest.Resource(pbcatalog.ServiceType, "foo-service"). WithData(suite.T(), &pbcatalog.Service{}). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(fooService.Id)) + fooEndpoints := resourcetest.Resource(pbcatalog.ServiceEndpointsType, "foo-service"). WithData(suite.T(), &pbcatalog.ServiceEndpoints{Endpoints: []*pbcatalog.Endpoint{ { @@ -687,9 +747,12 @@ func (suite *xdsControllerTestSuite) setupFooProxyStateTemplateWithReferences() }, }, }}). + WithTenancy(tenancy). WithOwner(fooService.Id). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(fooEndpoints.Id)) + fooRequiredEndpoints := make(map[string]*pbproxystate.EndpointRef) fooRequiredEndpoints["test-cluster-1"] = &pbproxystate.EndpointRef{ Id: fooEndpoints.Id, @@ -707,8 +770,11 @@ func (suite *xdsControllerTestSuite) setupFooProxyStateTemplateWithReferences() RequiredLeafCertificates: fooRequiredLeafs, ProxyState: &pbmesh.ProxyState{}, }). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(fooProxyStateTemplate.Id)) + retry.Run(suite.T(), func(r *retry.R) { suite.client.RequireResourceExists(r, fooProxyStateTemplate.Id) }) @@ -754,6 +820,14 @@ func (suite *xdsControllerTestSuite) setupFooProxyStateTemplateWithReferences() suite.expectedFooProxyStateEndpoints = expectedFooProxyStateEndpoints suite.expectedTrustBundle = expectedTrustBundle suite.expectedFooProxyStateSpiffes = expectedFooLeafSpiffes + + suite.T().Cleanup(func() { + suite.fooEndpointRefs = make(map[string]*pbproxystate.EndpointRef) + suite.fooLeafRefs = make(map[string]*pbproxystate.LeafCertificateRef) + suite.expectedFooProxyStateEndpoints = make(map[string]*pbproxystate.Endpoints) + suite.expectedTrustBundle = make(map[string]*pbproxystate.TrustBundle) + suite.expectedFooProxyStateSpiffes = make(map[string]string) + }) } // Setup: @@ -761,11 +835,14 @@ func (suite *xdsControllerTestSuite) setupFooProxyStateTemplateWithReferences() // - barProxyStateTemplate with an EndpointsRef to fooBarEndpoints. // // Saves all related resources to the suite so they can be modified if needed. -func (suite *xdsControllerTestSuite) setupFooBarProxyStateTemplateAndEndpoints() { +func (suite *xdsControllerTestSuite) setupFooBarProxyStateTemplateAndEndpoints(tenancy *pbresource.Tenancy) { fooService := resourcetest.Resource(pbcatalog.ServiceType, "foo-service"). WithData(suite.T(), &pbcatalog.Service{}). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(fooService.Id)) + fooEndpoints := resourcetest.Resource(pbcatalog.ServiceEndpointsType, "foo-service"). WithData(suite.T(), &pbcatalog.ServiceEndpoints{Endpoints: []*pbcatalog.Endpoint{ { @@ -788,12 +865,18 @@ func (suite *xdsControllerTestSuite) setupFooBarProxyStateTemplateAndEndpoints() }, }}). WithOwner(fooService.Id). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(fooEndpoints.Id)) + fooBarService := resourcetest.Resource(pbcatalog.ServiceType, "foo-bar-service"). WithData(suite.T(), &pbcatalog.Service{}). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(fooBarService.Id)) + fooBarEndpoints := resourcetest.Resource(pbcatalog.ServiceEndpointsType, "foo-bar-service"). WithData(suite.T(), &pbcatalog.ServiceEndpoints{Endpoints: []*pbcatalog.Endpoint{ { @@ -816,8 +899,11 @@ func (suite *xdsControllerTestSuite) setupFooBarProxyStateTemplateAndEndpoints() }, }}). WithOwner(fooBarService.Id). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(fooBarEndpoints.Id)) + fooRequiredEndpoints := make(map[string]*pbproxystate.EndpointRef) fooRequiredEndpoints["test-cluster-1"] = &pbproxystate.EndpointRef{ Id: fooEndpoints.Id, @@ -841,8 +927,11 @@ func (suite *xdsControllerTestSuite) setupFooBarProxyStateTemplateAndEndpoints() RequiredEndpoints: fooRequiredEndpoints, ProxyState: &pbmesh.ProxyState{}, }). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(fooProxyStateTemplate.Id)) + retry.Run(suite.T(), func(r *retry.R) { suite.client.RequireResourceExists(r, fooProxyStateTemplate.Id) }) @@ -853,8 +942,11 @@ func (suite *xdsControllerTestSuite) setupFooBarProxyStateTemplateAndEndpoints() RequiredEndpoints: barRequiredEndpoints, ProxyState: &pbmesh.ProxyState{}, }). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(barProxyStateTemplate.Id)) + retry.Run(suite.T(), func(r *retry.R) { suite.client.RequireResourceExists(r, barProxyStateTemplate.Id) }) @@ -935,68 +1027,77 @@ func (suite *xdsControllerTestSuite) setupFooBarProxyStateTemplateAndEndpoints() suite.fooBarService = fooBarService suite.expectedFooProxyStateEndpoints = expectedFooProxyStateEndpoints suite.expectedBarProxyStateEndpoints = expectedBarProxyStateEndpoints + + suite.T().Cleanup(func() { + suite.barEndpointRefs = make(map[string]*pbproxystate.EndpointRef) + suite.fooEndpointRefs = make(map[string]*pbproxystate.EndpointRef) + suite.expectedFooProxyStateEndpoints = make(map[string]*pbproxystate.Endpoints) + suite.expectedBarProxyStateEndpoints = make(map[string]*pbproxystate.Endpoints) + }) } func (suite *xdsControllerTestSuite) TestReconcile_prevWatchesToCancel() { - makeRef := func(names ...string) []*pbresource.Reference { - out := make([]*pbresource.Reference, len(names)) - for i, name := range names { - out[i] = &pbresource.Reference{ - Name: name, - Type: &pbresource.Type{ - Group: "g", - GroupVersion: "v", - Kind: "k", - }, - Tenancy: &pbresource.Tenancy{}, + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + makeRef := func(names ...string) []*pbresource.Reference { + out := make([]*pbresource.Reference, len(names)) + for i, name := range names { + out[i] = &pbresource.Reference{ + Name: name, + Type: &pbresource.Type{ + Group: "g", + GroupVersion: "v", + Kind: "k", + }, + Tenancy: tenancy, + } } + return out } - return out - } - convert := func(input []*pbresource.Reference) []resource.ReferenceOrID { - asInterface := make([]resource.ReferenceOrID, len(input)) - for i := range input { - asInterface[i] = input[i] + convert := func(input []*pbresource.Reference) []resource.ReferenceOrID { + asInterface := make([]resource.ReferenceOrID, len(input)) + for i := range input { + asInterface[i] = input[i] + } + return asInterface } - return asInterface - } - cases := []struct { - old []*pbresource.Reference - new []*pbresource.Reference - expect []*pbresource.Reference - }{ - { - old: makeRef("a", "b", "c"), - new: makeRef("a", "c"), - expect: makeRef("b"), - }, - { - old: makeRef("a", "b", "c"), - new: makeRef("a", "b", "c"), - expect: makeRef(), - }, - { - old: makeRef(), - new: makeRef("a", "b"), - expect: makeRef(), - }, - { - old: makeRef("a", "b"), - new: makeRef(), - expect: makeRef("a", "b"), - }, - { - old: makeRef(), - new: makeRef(), - expect: makeRef(), - }, - } + cases := []struct { + old []*pbresource.Reference + new []*pbresource.Reference + expect []*pbresource.Reference + }{ + { + old: makeRef("a", "b", "c"), + new: makeRef("a", "c"), + expect: makeRef("b"), + }, + { + old: makeRef("a", "b", "c"), + new: makeRef("a", "b", "c"), + expect: makeRef(), + }, + { + old: makeRef(), + new: makeRef("a", "b"), + expect: makeRef(), + }, + { + old: makeRef("a", "b"), + new: makeRef(), + expect: makeRef("a", "b"), + }, + { + old: makeRef(), + new: makeRef(), + expect: makeRef(), + }, + } - for _, tc := range cases { - toCancel := prevWatchesToCancel(tc.old, convert(tc.new)) - require.ElementsMatch(suite.T(), toCancel, tc.expect) - } + for _, tc := range cases { + toCancel := prevWatchesToCancel(tc.old, convert(tc.new)) + require.ElementsMatch(suite.T(), toCancel, tc.expect) + } + }) } func TestXdsController(t *testing.T) { @@ -1017,87 +1118,92 @@ func TestXdsController(t *testing.T) { // rather than just endpoints, leaf certs, and trust bundles, the test also ensures // side effects or change in scope to XDS controller are not introduce mistakenly. func (suite *xdsControllerTestSuite) TestReconcile_SidecarProxyGoldenFileInputs() { - path := "../sidecarproxy/builder/testdata" - cases := []string{ - // destinations - please add in alphabetical order - "destination/l4-single-destination-ip-port-bind-address", - "destination/l4-single-destination-unix-socket-bind-address", - "destination/l4-single-implicit-destination-tproxy", - "destination/l4-multi-destination", - "destination/l4-multiple-implicit-destinations-tproxy", - "destination/l4-implicit-and-explicit-destinations-tproxy", - "destination/mixed-multi-destination", - "destination/multiport-l4-and-l7-multiple-implicit-destinations-tproxy", - "destination/multiport-l4-and-l7-single-implicit-destination-tproxy", - "destination/multiport-l4-and-l7-single-implicit-destination-with-multiple-workloads-tproxy", - - //sources - please add in alphabetical order - "source/l7-expose-paths", - "source/local-and-inbound-connections", - "source/multiple-workload-addresses-with-specific-ports", - "source/multiple-workload-addresses-without-ports", - "source/multiport-l4-multiple-workload-addresses-with-specific-ports", - "source/multiport-l4-multiple-workload-addresses-without-ports", - "source/multiport-l4-workload-with-only-mesh-port", - "source/multiport-l7-multiple-workload-addresses-with-specific-ports", - "source/multiport-l7-multiple-workload-addresses-without-ports", - "source/multiport-l7-multiple-workload-addresses-without-ports", - "source/single-workload-address-without-ports", - } - - for _, name := range cases { - suite.Run(name, func() { - // Create ProxyStateTemplate from the golden file. - pst := JSONToProxyTemplate(suite.T(), - golden.GetBytesAtFilePath(suite.T(), fmt.Sprintf("%s/%s.golden", path, name))) - - // Destinations will need endpoint refs set up. - if strings.Split(name, "/")[0] == "destination" && len(pst.ProxyState.Endpoints) == 0 { - suite.addRequiredEndpointsAndRefs(pst) - } - - // Store the initial ProxyStateTemplate. - proxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "test"). - WithData(suite.T(), pst). - Write(suite.T(), suite.client) - - // Check with resource service that it exists. - retry.Run(suite.T(), func(r *retry.R) { - suite.client.RequireResourceExists(r, proxyStateTemplate.Id) - }) + suite.runTestCaseWithTenancies(func(tenancy *pbresource.Tenancy) { + path := "../sidecarproxy/builder/testdata" + cases := []string{ + // destinations - please add in alphabetical order + "destination/l4-single-destination-ip-port-bind-address", + "destination/l4-single-destination-unix-socket-bind-address", + "destination/l4-single-implicit-destination-tproxy", + "destination/l4-multi-destination", + "destination/l4-multiple-implicit-destinations-tproxy", + "destination/l4-implicit-and-explicit-destinations-tproxy", + "destination/mixed-multi-destination", + "destination/multiport-l4-and-l7-multiple-implicit-destinations-tproxy", + "destination/multiport-l4-and-l7-single-implicit-destination-tproxy", + "destination/multiport-l4-and-l7-single-implicit-destination-with-multiple-workloads-tproxy", + + //sources - please add in alphabetical order + "source/l7-expose-paths", + "source/local-and-inbound-connections", + "source/multiple-workload-addresses-with-specific-ports", + "source/multiple-workload-addresses-without-ports", + "source/multiport-l4-multiple-workload-addresses-with-specific-ports", + "source/multiport-l4-multiple-workload-addresses-without-ports", + "source/multiport-l4-workload-with-only-mesh-port", + "source/multiport-l7-multiple-workload-addresses-with-specific-ports", + "source/multiport-l7-multiple-workload-addresses-without-ports", + "source/multiport-l7-multiple-workload-addresses-without-ports", + "source/single-workload-address-without-ports", + } - // Track it in the mapper. - suite.mapper.TrackItem(proxyStateTemplate.Id, []resource.ReferenceOrID{}) + for _, name := range cases { + suite.Run(name, func() { + // Create ProxyStateTemplate from the golden file. + pst := JSONToProxyTemplate(suite.T(), + golden.GetBytesAtFilePath(suite.T(), fmt.Sprintf("%s/%s.golden", path, name))) + + // Destinations will need endpoint refs set up. + if strings.Split(name, "/")[0] == "destination" && len(pst.ProxyState.Endpoints) == 0 { + suite.addRequiredEndpointsAndRefs(pst, tenancy) + } + + // Store the initial ProxyStateTemplate. + proxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "test"). + WithData(suite.T(), pst). + WithTenancy(tenancy). + Write(suite.T(), suite.client) + + suite.T().Cleanup(suite.deleteResourceFunc(proxyStateTemplate.Id)) + + // Check with resource service that it exists. + retry.Run(suite.T(), func(r *retry.R) { + suite.client.RequireResourceExists(r, proxyStateTemplate.Id) + }) + + // Track it in the mapper. + suite.mapper.TrackItem(proxyStateTemplate.Id, []resource.ReferenceOrID{}) + + // Run the reconcile, and since no ProxyStateTemplate is stored, this simulates a deletion. + err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ + ID: proxyStateTemplate.Id, + }) + require.NoError(suite.T(), err) + require.NotNil(suite.T(), proxyStateTemplate) + + // Get the reconciled proxyStateTemplate to check the reconcile results. + reconciledPS := suite.updater.Get(proxyStateTemplate.Id.Name) + + // Verify leaf cert contents then hard code them for comparison + // and downstream tests since they change from test run to test run. + require.NotEmpty(suite.T(), reconciledPS.LeafCertificates) + reconciledPS.LeafCertificates = map[string]*pbproxystate.LeafCertificate{ + "test-identity": { + Cert: "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n", + Key: "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n", + }, + } - // Run the reconcile, and since no ProxyStateTemplate is stored, this simulates a deletion. - err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{ - ID: proxyStateTemplate.Id, + // Compare actual vs expected. + actual := prototest.ProtoToJSON(suite.T(), reconciledPS) + expected := golden.Get(suite.T(), actual, name+".golden") + require.JSONEq(suite.T(), expected, actual) }) - require.NoError(suite.T(), err) - require.NotNil(suite.T(), proxyStateTemplate) - - // Get the reconciled proxyStateTemplate to check the reconcile results. - reconciledPS := suite.updater.Get(proxyStateTemplate.Id.Name) - - // Verify leaf cert contents then hard code them for comparison - // and downstream tests since they change from test run to test run. - require.NotEmpty(suite.T(), reconciledPS.LeafCertificates) - reconciledPS.LeafCertificates = map[string]*pbproxystate.LeafCertificate{ - "test-identity": { - Cert: "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n", - Key: "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n", - }, - } - - // Compare actual vs expected. - actual := prototest.ProtoToJSON(suite.T(), reconciledPS) - expected := golden.Get(suite.T(), actual, name+".golden") - require.JSONEq(suite.T(), expected, actual) - }) - } + } + }) } -func (suite *xdsControllerTestSuite) addRequiredEndpointsAndRefs(pst *pbmesh.ProxyStateTemplate) { +func (suite *xdsControllerTestSuite) addRequiredEndpointsAndRefs(pst *pbmesh.ProxyStateTemplate, tenancy *pbresource.Tenancy) { //get service data serviceData := &pbcatalog.Service{} var vp uint32 = 7000 @@ -1125,8 +1231,11 @@ func (suite *xdsControllerTestSuite) addRequiredEndpointsAndRefs(pst *pbmesh.Pro // create service. svc := resourcetest.Resource(pbcatalog.ServiceType, svcName). WithData(suite.T(), &pbcatalog.Service{}). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(svc.Id)) + // create endpoints with svc as owner. eps := resourcetest.Resource(pbcatalog.ServiceEndpointsType, svcName). WithData(suite.T(), &pbcatalog.ServiceEndpoints{Endpoints: []*pbcatalog.Endpoint{ @@ -1146,8 +1255,11 @@ func (suite *xdsControllerTestSuite) addRequiredEndpointsAndRefs(pst *pbmesh.Pro }, }}). WithOwner(svc.Id). + WithTenancy(tenancy). Write(suite.T(), suite.client) + suite.T().Cleanup(suite.deleteResourceFunc(eps.Id)) + // add to working list of required endpoints. requiredEps[clusterName] = &pbproxystate.EndpointRef{ Id: eps.Id, @@ -1167,3 +1279,21 @@ func JSONToProxyTemplate(t *testing.T, json []byte) *pbmesh.ProxyStateTemplate { require.NoError(t, err) return proxyTemplate } + +func (suite *xdsControllerTestSuite) runTestCaseWithTenancies(testCase func(tenancy *pbresource.Tenancy)) { + for _, tenancy := range suite.tenancies { + suite.Run(suite.appendTenancyInfo(tenancy), func() { + testCase(tenancy) + }) + } +} + +func (suite *xdsControllerTestSuite) appendTenancyInfo(tenancy *pbresource.Tenancy) string { + return fmt.Sprintf("%s_Namespace_%s_Partition", tenancy.Namespace, tenancy.Partition) +} + +func (suite *xdsControllerTestSuite) deleteResourceFunc(id *pbresource.ID) func() { + return func() { + suite.client.MustDelete(suite.T(), id) + } +} diff --git a/internal/resource/resourcetest/tenancy.go b/internal/resource/resourcetest/tenancy.go index 826b20d17d..d77c9b42ac 100644 --- a/internal/resource/resourcetest/tenancy.go +++ b/internal/resource/resourcetest/tenancy.go @@ -4,10 +4,10 @@ package resourcetest import ( - "github.com/hashicorp/consul/agent/structs" "strings" "testing" + "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/internal/resource" "github.com/hashicorp/consul/proto-public/pbresource" )