From 3cc01a3088a32cea2e1adedeb17d85fc07ec7d37 Mon Sep 17 00:00:00 2001 From: Bora Tunca Date: Sat, 11 Nov 2017 21:39:08 -0500 Subject: [PATCH] Add more discovery tests for updating target groups (#3426) * Adds a test covering the case where a target providers sends updated versions of the same target groups and the system should reconcile to the latest version of each of the target groups * Refactors how input data is represented in the tests. It used to be literal declarations of necessary structs, now it is parsing yaml. Yaml declarations are half as long as the former. And these can be put in a fixture file * Adds a tiny bit of refactoring on test timeouts --- discovery/discovery_test.go | 953 ++++++++++++++++++++++++++++++++---- 1 file changed, 858 insertions(+), 95 deletions(-) diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 02fc2f981..7cf9200d2 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -15,87 +15,469 @@ package discovery import ( "context" + "reflect" "sync" "sync/atomic" "testing" "time" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" yaml "gopkg.in/yaml.v2" ) -func TestSingleTargetSetWithSingleProviderOnlySendsNewTargetGroups(t *testing.T) { - - testCases := [][]update{ - []update{}, // No updates. - - []update{{[]string{}, 0}}, // Empty initials. - - []update{{[]string{}, 6000}}, // Empty initials with a delay. - - []update{{[]string{"initial1", "initial2"}, 0}}, // Initials only. - - []update{{[]string{"initial1", "initial2"}, 6000}}, // Initials only but after a delay. - - []update{ // Initials and new groups. - {[]string{"initial1", "initial2"}, 0}, - {[]string{"update1", "update2"}, 0}, +func TestTargetSetThrottlesTheSyncCalls(t *testing.T) { + testCases := []struct { + title string + updates map[string][]update + expectedSyncCalls [][]string + }{ + { + title: "Single TP no updates", + updates: map[string][]update{ + "tp1": {}, + }, + expectedSyncCalls: [][]string{ + {}, + }, }, - - []update{ // Initials and new groups after a delay. - {[]string{"initial1", "initial2"}, 6000}, - {[]string{"update1", "update2"}, 500}, + { + title: "Multips TPs no updates", + updates: map[string][]update{ + "tp1": {}, + "tp2": {}, + "tp3": {}, + }, + expectedSyncCalls: [][]string{ + {}, + }, }, - - []update{ - {[]string{"initial1", "initial2"}, 100}, - {[]string{"update1", "update2", "update3", "update4", "update5", "update6", "update7", "update8", "update9", "update10", "update11"}, 100}, + { + title: "Single TP empty initials", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{}, + interval: 5, + }, + }, + }, + expectedSyncCalls: [][]string{ + {}, + }, }, - - []update{ - {[]string{"initial1"}, 10}, - {[]string{"update1"}, 45}, - {[]string{"update2", "update3", "update4"}, 0}, - {[]string{"update5"}, 10}, - {[]string{"update6", "update7", "update8", "update9"}, 70}, + { + title: "Multiple TPs empty initials", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{}, + interval: 5, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{}, + interval: 500, + }, + }, + "tp3": { + { + targetGroups: []config.TargetGroup{}, + interval: 100, + }, + }, + }, + expectedSyncCalls: [][]string{ + {}, + }, }, - - []update{ - {[]string{"initial1", "initial2"}, 5}, - {[]string{}, 100}, - {[]string{"update1", "update2"}, 100}, - {[]string{"update3", "update4", "update5"}, 70}, + { + title: "Multiple TPs empty initials with a delay", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{}, + interval: 6000, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{}, + interval: 6500, + }, + }, + }, + expectedSyncCalls: [][]string{ + {}, + }, + }, + { + title: "Single TP initials only", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "initial1"}, {Source: "initial2"}}, + interval: 0, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"initial1", "initial2"}, + }, + }, + { + title: "Multiple TPs initials only", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "tp1-initial1"}, {Source: "tp1-initial2"}}, + interval: 0, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{{Source: "tp2-initial1"}}, + interval: 0, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"tp1-initial1", "tp1-initial2", "tp2-initial1"}, + }, + }, + { + title: "Single TP delayed initials only", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "initial1"}, {Source: "initial2"}}, + interval: 6000, + }, + }, + }, + expectedSyncCalls: [][]string{ + {}, + {"initial1", "initial2"}, + }, + }, + { + title: "Multiple TPs with some delayed initials", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "tp1-initial1"}, {Source: "tp1-initial2"}}, + interval: 100, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{{Source: "tp2-initial1"}, {Source: "tp2-initial2"}, {Source: "tp2-initial3"}}, + interval: 6000, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"tp1-initial1", "tp1-initial2"}, + {"tp1-initial1", "tp1-initial2", "tp2-initial1", "tp2-initial2", "tp2-initial3"}, + }, + }, + { + title: "Single TP initials followed by empty updates", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "initial1"}, {Source: "initial2"}}, + interval: 0, + }, + { + targetGroups: []config.TargetGroup{}, + interval: 10, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"initial1", "initial2"}, + }, + }, + { + title: "Single TP initials and new groups", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "initial1"}, {Source: "initial2"}}, + interval: 0, + }, + { + targetGroups: []config.TargetGroup{{Source: "update1"}, {Source: "update2"}}, + interval: 10, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"initial1", "initial2"}, + {"initial1", "initial2", "update1", "update2"}, + }, + }, + { + title: "Multiple TPs initials and new groups", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "tp1-initial1"}, {Source: "tp1-initial2"}}, + interval: 10, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp1-update1"}, {Source: "tp1-update2"}}, + interval: 1500, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{{Source: "tp2-initial1"}, {Source: "tp2-initial2"}}, + interval: 100, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp2-update1"}, {Source: "tp2-update2"}}, + interval: 10, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"tp1-initial1", "tp1-initial2", "tp2-initial1", "tp2-initial2"}, + {"tp1-initial1", "tp1-initial2", "tp2-initial1", "tp2-initial2", "tp1-update1", "tp1-update2", "tp2-update1", "tp2-update2"}, + }, + }, + { + title: "One tp initials arrive after other tp updates but still within 5 seconds", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "tp1-initial1"}, {Source: "tp1-initial2"}}, + interval: 10, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp1-update1"}, {Source: "tp1-update2"}}, + interval: 1500, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{{Source: "tp2-initial1"}, {Source: "tp2-initial2"}}, + interval: 2000, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp2-update1"}, {Source: "tp2-update2"}}, + interval: 1000, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"tp1-initial1", "tp1-initial2", "tp1-update1", "tp1-update2", "tp2-initial1", "tp2-initial2"}, + {"tp1-initial1", "tp1-initial2", "tp1-update1", "tp1-update2", "tp2-initial1", "tp2-initial2", "tp2-update1", "tp2-update2"}, + }, + }, + { + title: "One tp initials arrive after other tp updates and after 5 seconds", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "tp1-initial1"}, {Source: "tp1-initial2"}}, + interval: 10, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp1-update1"}, {Source: "tp1-update2"}}, + interval: 1500, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp1-update3"}}, + interval: 5000, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{{Source: "tp2-initial1"}, {Source: "tp2-initial2"}}, + interval: 6000, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"tp1-initial1", "tp1-initial2", "tp1-update1", "tp1-update2"}, + {"tp1-initial1", "tp1-initial2", "tp1-update1", "tp1-update2", "tp2-initial1", "tp2-initial2", "tp1-update3"}, + }, + }, + { + title: "Single TP initials and new groups after a delay", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "initial1"}, {Source: "initial2"}}, + interval: 6000, + }, + { + targetGroups: []config.TargetGroup{{Source: "update1"}, {Source: "update2"}}, + interval: 10, + }, + }, + }, + expectedSyncCalls: [][]string{ + {}, + {"initial1", "initial2", "update1", "update2"}, + }, + }, + { + title: "Single TP initial and successive updates", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "initial1"}}, + interval: 100, + }, + { + targetGroups: []config.TargetGroup{{Source: "update1"}}, + interval: 100, + }, + { + targetGroups: []config.TargetGroup{{Source: "update2"}}, + interval: 10, + }, + { + targetGroups: []config.TargetGroup{{Source: "update3"}}, + interval: 10, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"initial1"}, + {"initial1", "update1", "update2", "update3"}, + }, + }, + { + title: "Multiple TPs initials and successive updates", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "tp1-initial1"}}, + interval: 1000, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp1-update1"}}, + interval: 1000, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp1-update2"}}, + interval: 2000, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp1-update3"}}, + interval: 2000, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{{Source: "tp2-initial1"}}, + interval: 3000, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp2-update1"}}, + interval: 1000, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp2-update2"}}, + interval: 3000, + }, + { + targetGroups: []config.TargetGroup{{Source: "tp2-update3"}}, + interval: 2000, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"tp1-initial1", "tp1-update1", "tp2-initial1"}, + {"tp1-initial1", "tp1-update1", "tp2-initial1", "tp1-update2", "tp1-update3", "tp2-update1", "tp2-update2"}, + {"tp1-initial1", "tp1-update1", "tp2-initial1", "tp1-update2", "tp1-update3", "tp2-update1", "tp2-update2", "tp2-update3"}, + }, + }, + { + title: "Single TP Multiple updates 5 second window", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "initial1"}}, + interval: 10, + }, + { + targetGroups: []config.TargetGroup{{Source: "update1"}}, + interval: 25, + }, + { + targetGroups: []config.TargetGroup{{Source: "update2"}, {Source: "update3"}, {Source: "update4"}}, + interval: 10, + }, + { + targetGroups: []config.TargetGroup{{Source: "update5"}}, + interval: 0, + }, + { + targetGroups: []config.TargetGroup{{Source: "update6"}, {Source: "update7"}, {Source: "update8"}}, + interval: 70, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"initial1"}, + {"initial1", "update1", "update2", "update3", "update4", "update5", "update6", "update7", "update8"}, + }, + }, + { + title: "Single TP Single provider empty update in between", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{{Source: "initial1"}, {Source: "initial2"}}, + interval: 30, + }, + { + targetGroups: []config.TargetGroup{{Source: "update1"}, {Source: "update2"}}, + interval: 300, + }, + { + targetGroups: []config.TargetGroup{}, + interval: 10, + }, + { + targetGroups: []config.TargetGroup{{Source: "update3"}, {Source: "update4"}, {Source: "update5"}, {Source: "update6"}}, + interval: 6000, + }, + }, + }, + expectedSyncCalls: [][]string{ + {"initial1", "initial2"}, + {"initial1", "initial2", "update1", "update2"}, + {"initial1", "initial2", "update1", "update2", "update3", "update4", "update5", "update6"}, + }, }, } - for i, updates := range testCases { - - expectedGroups := make(map[string]struct{}) - for _, update := range updates { - for _, target := range update.targets { - expectedGroups[target] = struct{}{} - } - } - - var wg sync.WaitGroup - wg.Add(1) + for i, testCase := range testCases { + finalize := make(chan bool) - isFirstSyncCall := true - var initialGroups []*config.TargetGroup - var syncedGroups []*config.TargetGroup + syncCallCount := 0 + syncedGroups := make([][]string, 0) targetSet := NewTargetSet(&mockSyncer{ sync: func(tgs []*config.TargetGroup) { - syncedGroups = tgs - if isFirstSyncCall { - isFirstSyncCall = false - initialGroups = tgs + currentCallGroup := make([]string, len(tgs)) + for i, tg := range tgs { + currentCallGroup[i] = tg.Source } + syncedGroups = append(syncedGroups, currentCallGroup) - if len(tgs) == len(expectedGroups) { + syncCallCount++ + if syncCallCount == len(testCase.expectedSyncCalls) { // All the groups are sent, we can start asserting. - wg.Done() + close(finalize) } }, }) @@ -103,54 +485,418 @@ func TestSingleTargetSetWithSingleProviderOnlySendsNewTargetGroups(t *testing.T) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tp := newMockTargetProvider(updates) targetProviders := map[string]TargetProvider{} - targetProviders["testProvider"] = tp + for tpName, tpUpdates := range testCase.updates { + tp := newMockTargetProvider(tpUpdates) + targetProviders[tpName] = tp + } go targetSet.Run(ctx) targetSet.UpdateProviders(targetProviders) - finalize := make(chan struct{}) - go func() { - defer close(finalize) - wg.Wait() - }() - select { - case <-time.After(20000 * time.Millisecond): - t.Errorf("In test case %v: Test timed out after 20000 millisecond. All targets should be sent within the timeout", i) + case <-time.After(20 * time.Second): + t.Errorf("%d. %q: Test timed out after 20 seconds. All targets should be sent within the timeout", i, testCase.title) case <-finalize: + for name, tp := range targetProviders { + runCallCount := *tp.(mockTargetProvider).callCount + if runCallCount != 1 { + t.Errorf("%d. %q: TargetProvider Run should be called once for each target provider. For %q was called %d times", i, testCase.title, name, runCallCount) + } + } - if *tp.callCount != 1 { - t.Errorf("In test case %v: TargetProvider Run should be called once only, was called %v times", i, *tp.callCount) + if len(syncedGroups) != len(testCase.expectedSyncCalls) { + t.Errorf("%d. %q: received sync calls: \n %v \n do not match expected calls: \n %v \n", i, testCase.title, syncedGroups, testCase.expectedSyncCalls) } - if len(updates) > 0 && updates[0].interval > 5000 { - // If the initial set of targets never arrive or arrive after 5 seconds. - // The first sync call should receive empty set of targets. - if len(initialGroups) != 0 { - t.Errorf("In test case %v: Expecting 0 initial target groups, received %v", i, len(initialGroups)) + for j := range syncedGroups { + if len(syncedGroups[j]) != len(testCase.expectedSyncCalls[j]) { + t.Errorf("%d. %q: received sync calls in call [%v]: \n %v \n do not match expected calls: \n %v \n", i, testCase.title, j, syncedGroups[j], testCase.expectedSyncCalls[j]) } + + expectedGroupsMap := make(map[string]struct{}) + for _, expectedGroup := range testCase.expectedSyncCalls[j] { + expectedGroupsMap[expectedGroup] = struct{}{} + } + + for _, syncedGroup := range syncedGroups[j] { + if _, ok := expectedGroupsMap[syncedGroup]; !ok { + t.Errorf("%d. %q: '%s' does not exist in expected target groups: %s", i, testCase.title, syncedGroup, testCase.expectedSyncCalls[j]) + } else { + delete(expectedGroupsMap, syncedGroup) // Remove used targets from the map. + } + } + } + } + } +} + +func TestTargetSetConsolidatesToTheLatestState(t *testing.T) { + testCases := []struct { + title string + updates map[string][]update + }{ + { + title: "Single TP update same initial group multiple times", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{ + { + Source: "initial1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.11:6003"}}, + }, + }, + interval: 100, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "initial1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.11:6003"}, {"__instance__": "10.11.122.12:6003"}}, + }, + }, + interval: 250, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "initial1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.12:6003"}}, + }, + }, + interval: 250, + }, + }, + }, + }, + { + title: "Multiple TPs update", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-initial1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.11:6003"}}, + }, + }, + interval: 3, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-update1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.12:6003"}}, + }, + }, + interval: 10, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-update1", + Targets: []model.LabelSet{ + {"__instance__": "10.11.122.12:6003"}, + {"__instance__": "10.11.122.13:6003"}, + {"__instance__": "10.11.122.14:6003"}, + }, + }, + }, + interval: 10, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{ + { + Source: "tp2-initial1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.11:6003"}}, + }, + }, + interval: 3, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp2-initial1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.12:6003"}}, + }, + }, + interval: 10, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp2-update1", + Targets: []model.LabelSet{ + {"__instance__": "10.11.122.12:6003"}, + {"__instance__": "10.11.122.13:6003"}, + {"__instance__": "10.11.122.14:6003"}, + }, + }, + }, + interval: 10, + }, + }, + }, + }, + { + title: "Multiple TPs update II", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-initial1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.11:6003"}}, + }, + { + Source: "tp1-initial2", + Targets: []model.LabelSet{{"__instance__": "10.11.122.12:6003"}}, + }, + }, + interval: 100, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-update1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.13:6003"}}, + }, + { + Source: "tp1-initial2", + Targets: []model.LabelSet{ + {"__instance__": "10.11.122.12:6003"}, + {"__instance__": "10.11.122.14:6003"}, + }, + }, + }, + interval: 100, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-update2", + Targets: []model.LabelSet{{"__instance__": "10.11.122.15:6003"}}, + }, + { + Source: "tp1-initial1", + Targets: []model.LabelSet{}, + }, + }, + interval: 100, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-update1", + Targets: []model.LabelSet{ + {"__instance__": "10.11.122.16:6003"}, + {"__instance__": "10.11.122.17:6003"}, + {"__instance__": "10.11.122.18:6003"}, + }, + }, + }, + interval: 100, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{}, + interval: 100, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp2-update1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.13:6003"}}, + }, + }, + interval: 100, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp2-update2", + Targets: []model.LabelSet{{"__instance__": "10.11.122.15:6003"}}, + }, + { + Source: "tp2-update1", + Targets: []model.LabelSet{}, + }, + }, + interval: 300, + }, + }, + }, + }, + { + title: "Three rounds of sync call", + updates: map[string][]update{ + "tp1": { + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-initial1", + Targets: []model.LabelSet{ + {"__instance__": "10.11.122.11:6003"}, + {"__instance__": "10.11.122.12:6003"}, + {"__instance__": "10.11.122.13:6003"}, + }, + }, + { + Source: "tp1-initial2", + Targets: []model.LabelSet{{"__instance__": "10.11.122.14:6003"}}, + }, + }, + interval: 1000, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-initial1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.12:6003"}}, + }, + { + Source: "tp1-update1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.15:6003"}}, + }, + }, + interval: 3000, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp1-initial1", + Targets: []model.LabelSet{}, + }, + { + Source: "tp1-update1", + Targets: []model.LabelSet{ + {"__instance__": "10.11.122.15:6003"}, + {"__instance__": "10.11.122.16:6003"}, + }, + }, + }, + interval: 3000, + }, + }, + "tp2": { + { + targetGroups: []config.TargetGroup{ + { + Source: "tp2-initial1", + Targets: []model.LabelSet{ + {"__instance__": "10.11.122.11:6003"}, + }, + }, + { + Source: "tp2-initial2", + Targets: []model.LabelSet{{"__instance__": "10.11.122.14:6003"}}, + }, + { + Source: "tp2-initial3", + Targets: []model.LabelSet{{"__instance__": "10.11.122.15:6003"}}, + }, + }, + interval: 6000, + }, + { + targetGroups: []config.TargetGroup{ + { + Source: "tp2-initial1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.11:6003"}, {"__instance__": "10.11.122.12:6003"}}, + }, + { + Source: "tp2-update1", + Targets: []model.LabelSet{{"__instance__": "10.11.122.15:6003"}}, + }, + }, + interval: 3000, + }, + }, + }, + }, + } + + // Function to determine if the sync call received the latest state of + // all the target groups that came out of the target provider. + endStateAchieved := func(groupsSentToSyc []*config.TargetGroup, endState map[string]config.TargetGroup) bool { + + if len(groupsSentToSyc) != len(endState) { + return false + } + + for _, tg := range groupsSentToSyc { + if _, ok := endState[tg.Source]; ok == false { + // The target group does not exist in the end state. + return false } - if len(syncedGroups) != len(expectedGroups) { - t.Errorf("In test case %v: Expecting %v target groups in total, received %v", i, len(expectedGroups), len(syncedGroups)) + if reflect.DeepEqual(endState[tg.Source], *tg) == false { + // The target group has not reached its final state yet. + return false } - for _, tg := range syncedGroups { - if _, ok := expectedGroups[tg.Source]; ok == false { - t.Errorf("In test case %v: '%s' does not exist in expected target groups: %s", i, tg.Source, expectedGroups) - } else { - delete(expectedGroups, tg.Source) // Remove used targets from the map. + delete(endState, tg.Source) // Remove used target groups. + } + + return true + } + + for i, testCase := range testCases { + expectedGroups := make(map[string]config.TargetGroup) + for _, tpUpdates := range testCase.updates { + for _, update := range tpUpdates { + for _, targetGroup := range update.targetGroups { + expectedGroups[targetGroup.Source] = targetGroup } } } + + finalize := make(chan bool) + + targetSet := NewTargetSet(&mockSyncer{ + sync: func(tgs []*config.TargetGroup) { + + endState := make(map[string]config.TargetGroup) + for k, v := range expectedGroups { + endState[k] = v + } + + if endStateAchieved(tgs, endState) == false { + return + } + + close(finalize) + }, + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + targetProviders := map[string]TargetProvider{} + for tpName, tpUpdates := range testCase.updates { + tp := newMockTargetProvider(tpUpdates) + targetProviders[tpName] = tp + } + + go targetSet.Run(ctx) + targetSet.UpdateProviders(targetProviders) + + select { + case <-time.After(20 * time.Second): + t.Errorf("%d. %q: Test timed out after 20 seconds. All targets should be sent within the timeout", i, testCase.title) + + case <-finalize: + // System successfully reached to the end state. + } } } func TestTargetSetRecreatesTargetGroupsEveryRun(t *testing.T) { - verifyPresence := func(tgroups map[string]*config.TargetGroup, name string, present bool) { if _, ok := tgroups[name]; ok != present { msg := "" @@ -203,7 +949,6 @@ static_configs: } func TestTargetSetRunsSameTargetProviderMultipleTimes(t *testing.T) { - var wg sync.WaitGroup wg.Add(2) @@ -219,7 +964,14 @@ func TestTargetSetRunsSameTargetProviderMultipleTimes(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tp := newMockTargetProvider([]update{{[]string{"initial1", "initial2"}, 10}}) + updates := []update{ + { + targetGroups: []config.TargetGroup{{Source: "initial1"}, {Source: "initial2"}}, + interval: 10, + }, + } + + tp := newMockTargetProvider(updates) targetProviders := map[string]TargetProvider{} targetProviders["testProvider"] = tp @@ -228,10 +980,21 @@ func TestTargetSetRunsSameTargetProviderMultipleTimes(t *testing.T) { ts1.UpdateProviders(targetProviders) ts2.UpdateProviders(targetProviders) - wg.Wait() - if *tp.callCount != 2 { - t.Errorf("Was expecting 2 calls received %v", tp.callCount) + finalize := make(chan struct{}) + go func() { + defer close(finalize) + wg.Wait() + }() + + select { + case <-time.After(20 * time.Second): + t.Error("Test timed out after 20 seconds. All targets should be sent within the timeout") + + case <-finalize: + if *tp.callCount != 2 { + t.Errorf("Was expecting 2 calls, received %d", tp.callCount) + } } } @@ -245,17 +1008,17 @@ func (s *mockSyncer) Sync(tgs []*config.TargetGroup) { } } +type update struct { + targetGroups []config.TargetGroup + interval time.Duration +} + type mockTargetProvider struct { callCount *uint32 updates []update up chan<- []*config.TargetGroup } -type update struct { - targets []string - interval time.Duration -} - func newMockTargetProvider(updates []update) mockTargetProvider { var callCount uint32 @@ -278,9 +1041,9 @@ func (tp mockTargetProvider) sendUpdates() { time.Sleep(update.interval * time.Millisecond) - tgs := make([]*config.TargetGroup, len(update.targets)) - for i, tg := range update.targets { - tgs[i] = &config.TargetGroup{Source: tg} + tgs := make([]*config.TargetGroup, len(update.targetGroups)) + for i := range update.targetGroups { + tgs[i] = &update.targetGroups[i] } tp.up <- tgs