marathon-sd - change port gathering strategy, support for container networking (#4499)

* marathon-sd - change port gathering strategy, add support for container networking

- removed unnecessary error check on HTTPClientConfig.Validate()
- renamed PortDefinitions and PortMappings to PortDefinition and PortMapping respectively
- extended data model for extra parsed fields from Marathon json
- support container networking on Marathon 1.5+ (target Task.IPAddresses.x.Address)
- expanded test suite to cover all new cases
- test: cancel context when reading from doneCh before returning from function
- test: split test suite into Ports/PortMappings/PortDefinitions

Signed-off-by: Timo Beckers <timo@incline.eu>

discovery/marathon/marathon.go

@@ -107,11 +107,7 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if (len(c.HTTPClientConfig.BearerToken) > 0 || len(c.HTTPClientConfig.BearerTokenFile) > 0) && (len(c.AuthToken) > 0 || len(c.AuthTokenFile) > 0) {
 		return fmt.Errorf("marathon_sd: at most one of bearer_token, bearer_token_file, auth_token & auth_token_file must be configured")
 	}
-	if err := c.HTTPClientConfig.Validate(); err != nil {
+	return c.HTTPClientConfig.Validate()
-		return err
-	}
-
-	return nil
 }
 
 func init() {
@@ -278,31 +274,47 @@ func (d *Discovery) fetchTargetGroups() (map[string]*targetgroup.Group, error) {
 
 // Task describes one instance of a service running on Marathon.
 type Task struct {
-	ID    string   `json:"id"`
+	ID          string      `json:"id"`
-	Host  string   `json:"host"`
+	Host        string      `json:"host"`
-	Ports []uint32 `json:"ports"`
+	Ports       []uint32    `json:"ports"`
+	IPAddresses []IPAddress `json:"ipAddresses"`
 }
 
-// PortMappings describes in which port the process are binding inside the docker container.
+// IPAddress describes the address and protocol the container's network interface is bound to.
-type PortMappings struct {
+type IPAddress struct {
-	Labels map[string]string `json:"labels"`
+	Address string `json:"ipAddress"`
+	Proto   string `json:"protocol"`
+}
+
+// PortMapping describes in which port the process are binding inside the docker container.
+type PortMapping struct {
+	Labels        map[string]string `json:"labels"`
+	ContainerPort uint32            `json:"containerPort"`
+	ServicePort   uint32            `json:"servicePort"`
 }
 
 // DockerContainer describes a container which uses the docker runtime.
 type DockerContainer struct {
-	Image        string         `json:"image"`
+	Image        string        `json:"image"`
-	PortMappings []PortMappings `json:"portMappings"`
+	PortMappings []PortMapping `json:"portMappings"`
 }
 
 // Container describes the runtime an app in running in.
 type Container struct {
 	Docker       DockerContainer `json:"docker"`
-	PortMappings []PortMappings  `json:"portMappings"`
+	PortMappings []PortMapping   `json:"portMappings"`
 }
 
-// PortDefinitions describes which load balancer port you should access to access the service.
+// PortDefinition describes which load balancer port you should access to access the service.
-type PortDefinitions struct {
+type PortDefinition struct {
 	Labels map[string]string `json:"labels"`
+	Port   uint32            `json:"port"`
+}
+
+// Network describes the name and type of network the container is attached to.
+type Network struct {
+	Name string `json:"name"`
+	Mode string `json:"mode"`
 }
 
 // App describes a service running on Marathon.
@@ -312,7 +324,13 @@ type App struct {
 	RunningTasks    int               `json:"tasksRunning"`
 	Labels          map[string]string `json:"labels"`
 	Container       Container         `json:"container"`
-	PortDefinitions []PortDefinitions `json:"portDefinitions"`
+	PortDefinitions []PortDefinition  `json:"portDefinitions"`
+	Networks        []Network         `json:"networks"`
+}
+
+// isContainerNet checks if the app's first network is set to mode 'container'.
+func (app App) isContainerNet() bool {
+	return len(app.Networks) > 0 && app.Networks[0].Mode == "container"
 }
 
 // AppList is a list of Marathon apps.
@@ -403,46 +421,109 @@ func createTargetGroup(app *App) *targetgroup.Group {
 
 func targetsForApp(app *App) []model.LabelSet {
 	targets := make([]model.LabelSet, 0, len(app.Tasks))
-	for _, t := range app.Tasks {
+
-		if len(t.Ports) == 0 {
+	var ports []uint32
-			continue
+	var labels []map[string]string
+	var prefix string
+
+	if len(app.Container.PortMappings) != 0 {
+		// In Marathon 1.5.x the "container.docker.portMappings" object was moved
+		// to "container.portMappings".
+		ports, labels = extractPortMapping(app.Container.PortMappings, app.isContainerNet())
+		prefix = portMappingLabelPrefix
+
+	} else if len(app.Container.Docker.PortMappings) != 0 {
+		// Prior to Marathon 1.5 the port mappings could be found at the path
+		// "container.docker.portMappings".
+		ports, labels = extractPortMapping(app.Container.Docker.PortMappings, app.isContainerNet())
+		prefix = portMappingLabelPrefix
+
+	} else if len(app.PortDefinitions) != 0 {
+		// PortDefinitions deprecates the "ports" array and can be used to specify
+		// a list of ports with metadata in case a mapping is not required.
+		ports = make([]uint32, len(app.PortDefinitions))
+		labels = make([]map[string]string, len(app.PortDefinitions))
+
+		for i := 0; i < len(app.PortDefinitions); i++ {
+			labels[i] = app.PortDefinitions[i].Labels
+			ports[i] = app.PortDefinitions[i].Port
 		}
-		for i := 0; i < len(t.Ports); i++ {
+
-			targetAddress := targetForTask(&t, i)
+		prefix = portDefinitionLabelPrefix
+	}
+
+	// Gather info about the app's 'tasks'. Each instance (container) is considered a task
+	// and can be reachable at one or more host:port endpoints.
+	for _, t := range app.Tasks {
+
+		// There are no labels to gather if only Ports is defined. (eg. with host networking)
+		// Ports can only be gathered from the Task (not from the app) and are guaranteed
+		// to be the same across all tasks. If we haven't gathered any ports by now,
+		// use the task's ports as the port list.
+		if len(ports) == 0 && len(t.Ports) != 0 {
+			ports = t.Ports
+		}
+
+		// Iterate over the ports we gathered using one of the methods above.
+		for i, port := range ports {
+
+			// Each port represents a possible Prometheus target.
+			targetAddress := targetEndpoint(&t, port, app.isContainerNet())
 			target := model.LabelSet{
 				model.AddressLabel: model.LabelValue(targetAddress),
 				taskLabel:          model.LabelValue(t.ID),
 				portIndexLabel:     model.LabelValue(strconv.Itoa(i)),
 			}
-			if i < len(app.PortDefinitions) {
+
-				for ln, lv := range app.PortDefinitions[i].Labels {
+			// Gather all port labels and set them on the current target, skip if the port has no Marathon labels.
-					ln = portDefinitionLabelPrefix + strutil.SanitizeLabelName(ln)
+			// This will happen in the host networking case with only `ports` defined, where
-					target[model.LabelName(ln)] = model.LabelValue(lv)
+			// it is inefficient to allocate a list of possibly hundreds of empty label maps per host port.
-				}
+			if len(labels) > 0 {
-			}
+				for ln, lv := range labels[i] {
-			// Prior to Marathon 1.5 the port mappings could be found at the path
+					ln = prefix + strutil.SanitizeLabelName(ln)
-			// "container.docker.portMappings".  When support for Marathon 1.4
-			// is dropped then this section of code can be removed.
-			if i < len(app.Container.Docker.PortMappings) {
-				for ln, lv := range app.Container.Docker.PortMappings[i].Labels {
-					ln = portMappingLabelPrefix + strutil.SanitizeLabelName(ln)
-					target[model.LabelName(ln)] = model.LabelValue(lv)
-				}
-			}
-			// In Marathon 1.5.x the container.docker.portMappings object was moved
-			// to container.portMappings.
-			if i < len(app.Container.PortMappings) {
-				for ln, lv := range app.Container.PortMappings[i].Labels {
-					ln = portMappingLabelPrefix + strutil.SanitizeLabelName(ln)
 					target[model.LabelName(ln)] = model.LabelValue(lv)
 				}
 			}
+
 			targets = append(targets, target)
 		}
 	}
 	return targets
 }
 
-func targetForTask(task *Task, index int) string {
+// Generate a target endpoint string in host:port format.
-	return net.JoinHostPort(task.Host, fmt.Sprintf("%d", task.Ports[index]))
+func targetEndpoint(task *Task, port uint32, containerNet bool) string {
+
+	var host string
+
+	// Use the task's ipAddress field when it's in a container network
+	if containerNet && len(task.IPAddresses) > 0 {
+		host = task.IPAddresses[0].Address
+	} else {
+		host = task.Host
+	}
+
+	return net.JoinHostPort(host, fmt.Sprintf("%d", port))
+}
+
+// Get a list of ports and a list of labels from a PortMapping.
+func extractPortMapping(portMappings []PortMapping, containerNet bool) ([]uint32, []map[string]string) {
+
+	ports := make([]uint32, len(portMappings))
+	labels := make([]map[string]string, len(portMappings))
+
+	for i := 0; i < len(portMappings); i++ {
+
+		labels[i] = portMappings[i].Labels
+
+		if containerNet {
+			// If the app is in a container network, connect directly to the container port.
+			ports[i] = portMappings[i].ContainerPort
+		} else {
+			// Otherwise, connect to the randomly-generated service port.
+			ports[i] = portMappings[i].ServicePort
+		}
+	}
+
+	return ports, labels
 }

discovery/marathon/marathon_test.go

@@ -77,26 +77,22 @@ func TestMarathonSDEmptyList(t *testing.T) {
 func marathonTestAppList(labels map[string]string, runningTasks int) *AppList {
 	var (
 		task = Task{
-			ID:    "test-task-1",
+			ID:   "test-task-1",
-			Host:  "mesos-slave1",
+			Host: "mesos-slave1",
-			Ports: []uint32{31000},
 		}
 		docker = DockerContainer{
 			Image: "repo/image:tag",
-			PortMappings: []PortMappings{
-				{Labels: labels},
-			},
 		}
-		container = Container{Docker: docker}
+		portMappings = []PortMapping{
+			{Labels: labels, ServicePort: 31000},
+		}
+		container = Container{Docker: docker, PortMappings: portMappings}
 		app       = App{
 			ID:           "test-service",
 			Tasks:        []Task{task},
 			RunningTasks: runningTasks,
 			Labels:       labels,
 			Container:    container,
-			PortDefinitions: []PortDefinitions{
-				{Labels: make(map[string]string)},
-			},
 		}
 	)
 	return &AppList{
@@ -131,9 +127,6 @@ func TestMarathonSDSendGroup(t *testing.T) {
 		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
 			t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
 		}
-		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-			t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-		}
 	default:
 		t.Fatal("Did not get a target group.")
 	}
@@ -197,6 +190,7 @@ func TestMarathonSDRunAndStop(t *testing.T) {
 		case <-ch:
 			cancel()
 		case <-doneCh:
+			cancel()
 			return
 		case <-timeout:
 			t.Fatalf("Update took too long.")
@@ -204,31 +198,26 @@ func TestMarathonSDRunAndStop(t *testing.T) {
 	}
 }
 
-func marathonTestAppListWithMutiplePorts(labels map[string]string, runningTasks int) *AppList {
+func marathonTestAppListWithMultiplePorts(labels map[string]string, runningTasks int) *AppList {
 	var (
 		task = Task{
-			ID:    "test-task-1",
+			ID:   "test-task-1",
-			Host:  "mesos-slave1",
+			Host: "mesos-slave1",
-			Ports: []uint32{31000, 32000},
 		}
 		docker = DockerContainer{
 			Image: "repo/image:tag",
-			PortMappings: []PortMappings{
-				{Labels: labels},
-				{Labels: make(map[string]string)},
-			},
 		}
-		container = Container{Docker: docker}
+		portMappings = []PortMapping{
+			{Labels: labels, ServicePort: 31000},
+			{Labels: make(map[string]string), ServicePort: 32000},
+		}
+		container = Container{Docker: docker, PortMappings: portMappings}
 		app       = App{
 			ID:           "test-service",
 			Tasks:        []Task{task},
 			RunningTasks: runningTasks,
 			Labels:       labels,
 			Container:    container,
-			PortDefinitions: []PortDefinitions{
-				{Labels: make(map[string]string)},
-				{Labels: labels},
-			},
 		}
 	)
 	return &AppList{
@@ -236,11 +225,11 @@ func marathonTestAppListWithMutiplePorts(labels map[string]string, runningTasks
 	}
 }
 
-func TestMarathonSDSendGroupWithMutiplePort(t *testing.T) {
+func TestMarathonSDSendGroupWithMultiplePort(t *testing.T) {
 	var (
 		ch     = make(chan []*targetgroup.Group, 1)
 		client = func(client *http.Client, url string) (*AppList, error) {
-			return marathonTestAppListWithMutiplePorts(marathonValidLabel, 1), nil
+			return marathonTestAppListWithMultiplePorts(marathonValidLabel, 1), nil
 		}
 	)
 	if err := testUpdateServices(client, ch); err != nil {
@@ -263,9 +252,6 @@ func TestMarathonSDSendGroupWithMutiplePort(t *testing.T) {
 		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
 			t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
 		}
-		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
-			t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
-		}
 		tgt = tg.Targets[1]
 		if tgt[model.AddressLabel] != "mesos-slave1:32000" {
 			t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
@@ -273,9 +259,6 @@ func TestMarathonSDSendGroupWithMutiplePort(t *testing.T) {
 		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
 			t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
 		}
-		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
-			t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
-		}
 	default:
 		t.Fatal("Did not get a target group.")
 	}
@@ -355,12 +338,11 @@ func Test500ErrorHttpResponseWithValidJSONBody(t *testing.T) {
 	}
 }
 
-func marathonTestAppListWithoutPortMappings(labels map[string]string, runningTasks int) *AppList {
+func marathonTestAppListWithPortDefinitions(labels map[string]string, runningTasks int) *AppList {
 	var (
 		task = Task{
-			ID:    "test-task-1",
+			ID:   "test-task-1",
-			Host:  "mesos-slave1",
+			Host: "mesos-slave1",
-			Ports: []uint32{31000, 32000},
 		}
 		docker = DockerContainer{
 			Image: "repo/image:tag",
@@ -372,9 +354,9 @@ func marathonTestAppListWithoutPortMappings(labels map[string]string, runningTas
 			RunningTasks: runningTasks,
 			Labels:       labels,
 			Container:    container,
-			PortDefinitions: []PortDefinitions{
+			PortDefinitions: []PortDefinition{
-				{Labels: make(map[string]string)},
+				{Labels: make(map[string]string), Port: 31000},
-				{Labels: labels},
+				{Labels: labels, Port: 32000},
 			},
 		}
 	)
@@ -383,11 +365,11 @@ func marathonTestAppListWithoutPortMappings(labels map[string]string, runningTas
 	}
 }
 
-func TestMarathonSDSendGroupWithoutPortMappings(t *testing.T) {
+func TestMarathonSDSendGroupWithPortDefinitions(t *testing.T) {
 	var (
 		ch     = make(chan []*targetgroup.Group, 1)
 		client = func(client *http.Client, url string) (*AppList, error) {
-			return marathonTestAppListWithoutPortMappings(marathonValidLabel, 1), nil
+			return marathonTestAppListWithPortDefinitions(marathonValidLabel, 1), nil
 		}
 	)
 	if err := testUpdateServices(client, ch); err != nil {
@@ -428,7 +410,7 @@ func TestMarathonSDSendGroupWithoutPortMappings(t *testing.T) {
 	}
 }
 
-func marathonTestAppListWithoutPortDefinitions(labels map[string]string, runningTasks int) *AppList {
+func marathonTestAppListWithPorts(labels map[string]string, runningTasks int) *AppList {
 	var (
 		task = Task{
 			ID:    "test-task-1",
@@ -437,10 +419,6 @@ func marathonTestAppListWithoutPortDefinitions(labels map[string]string, running
 		}
 		docker = DockerContainer{
 			Image: "repo/image:tag",
-			PortMappings: []PortMappings{
-				{Labels: labels},
-				{Labels: make(map[string]string)},
-			},
 		}
 		container = Container{Docker: docker}
 		app       = App{
@@ -456,11 +434,11 @@ func marathonTestAppListWithoutPortDefinitions(labels map[string]string, running
 	}
 }
 
-func TestMarathonSDSendGroupWithoutPortDefinitions(t *testing.T) {
+func TestMarathonSDSendGroupWithPorts(t *testing.T) {
 	var (
 		ch     = make(chan []*targetgroup.Group, 1)
 		client = func(client *http.Client, url string) (*AppList, error) {
-			return marathonTestAppListWithoutPortDefinitions(marathonValidLabel, 1), nil
+			return marathonTestAppListWithPorts(marathonValidLabel, 1), nil
 		}
 	)
 	if err := testUpdateServices(client, ch); err != nil {
@@ -480,7 +458,7 @@ func TestMarathonSDSendGroupWithoutPortDefinitions(t *testing.T) {
 		if tgt[model.AddressLabel] != "mesos-slave1:31000" {
 			t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
 		}
-		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
 			t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
 		}
 		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
@@ -504,18 +482,17 @@ func TestMarathonSDSendGroupWithoutPortDefinitions(t *testing.T) {
 func marathonTestAppListWithContainerPortMappings(labels map[string]string, runningTasks int) *AppList {
 	var (
 		task = Task{
-			ID:    "test-task-1",
+			ID:   "test-task-1",
-			Host:  "mesos-slave1",
+			Host: "mesos-slave1",
-			Ports: []uint32{31000, 32000},
 		}
 		docker = DockerContainer{
 			Image: "repo/image:tag",
 		}
 		container = Container{
 			Docker: docker,
-			PortMappings: []PortMappings{
+			PortMappings: []PortMapping{
-				{Labels: labels},
+				{Labels: labels, ServicePort: 31000},
-				{Labels: make(map[string]string)},
+				{Labels: make(map[string]string), ServicePort: 32000},
 			},
 		}
 		app = App{
@@ -575,3 +552,241 @@ func TestMarathonSDSendGroupWithContainerPortMappings(t *testing.T) {
 		t.Fatal("Did not get a target group.")
 	}
 }
+
+func marathonTestAppListWithDockerContainerPortMappings(labels map[string]string, runningTasks int) *AppList {
+	var (
+		task = Task{
+			ID:   "test-task-1",
+			Host: "mesos-slave1",
+		}
+		docker = DockerContainer{
+			Image: "repo/image:tag",
+			PortMappings: []PortMapping{
+				{Labels: labels, ServicePort: 31000},
+				{Labels: make(map[string]string), ServicePort: 32000},
+			},
+		}
+		container = Container{
+			Docker: docker,
+		}
+		app = App{
+			ID:           "test-service",
+			Tasks:        []Task{task},
+			RunningTasks: runningTasks,
+			Labels:       labels,
+			Container:    container,
+		}
+	)
+	return &AppList{
+		Apps: []App{app},
+	}
+}
+
+func TestMarathonSDSendGroupWithDockerContainerPortMappings(t *testing.T) {
+	var (
+		ch     = make(chan []*targetgroup.Group, 1)
+		client = func(client *http.Client, url string) (*AppList, error) {
+			return marathonTestAppListWithDockerContainerPortMappings(marathonValidLabel, 1), nil
+		}
+	)
+	if err := testUpdateServices(client, ch); err != nil {
+		t.Fatalf("Got error: %s", err)
+	}
+	select {
+	case tgs := <-ch:
+		tg := tgs[0]
+
+		if tg.Source != "test-service" {
+			t.Fatalf("Wrong target group name: %s", tg.Source)
+		}
+		if len(tg.Targets) != 2 {
+			t.Fatalf("Wrong number of targets: %v", tg.Targets)
+		}
+		tgt := tg.Targets[0]
+		if tgt[model.AddressLabel] != "mesos-slave1:31000" {
+			t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+			t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
+			t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
+		}
+		tgt = tg.Targets[1]
+		if tgt[model.AddressLabel] != "mesos-slave1:32000" {
+			t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+			t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
+			t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
+		}
+	default:
+		t.Fatal("Did not get a target group.")
+	}
+}
+
+func marathonTestAppListWithContainerNetworkAndPortMappings(labels map[string]string, runningTasks int) *AppList {
+	var (
+		task = Task{
+			ID:   "test-task-1",
+			Host: "mesos-slave1",
+			IPAddresses: []IPAddress{
+				{Address: "1.2.3.4"},
+			},
+		}
+		docker = DockerContainer{
+			Image: "repo/image:tag",
+		}
+		portMappings = []PortMapping{
+			{Labels: labels, ContainerPort: 8080, ServicePort: 31000},
+			{Labels: make(map[string]string), ContainerPort: 1234, ServicePort: 32000},
+		}
+		container = Container{
+			Docker:       docker,
+			PortMappings: portMappings,
+		}
+		networks = []Network{
+			{Mode: "container", Name: "test-network"},
+		}
+		app = App{
+			ID:           "test-service",
+			Tasks:        []Task{task},
+			RunningTasks: runningTasks,
+			Labels:       labels,
+			Container:    container,
+			Networks:     networks,
+		}
+	)
+	return &AppList{
+		Apps: []App{app},
+	}
+}
+
+func TestMarathonSDSendGroupWithContainerNetworkAndPortMapping(t *testing.T) {
+	var (
+		ch     = make(chan []*targetgroup.Group, 1)
+		client = func(client *http.Client, url string) (*AppList, error) {
+			return marathonTestAppListWithContainerNetworkAndPortMappings(marathonValidLabel, 1), nil
+		}
+	)
+	if err := testUpdateServices(client, ch); err != nil {
+		t.Fatalf("Got error: %s", err)
+	}
+	select {
+	case tgs := <-ch:
+		tg := tgs[0]
+
+		if tg.Source != "test-service" {
+			t.Fatalf("Wrong target group name: %s", tg.Source)
+		}
+		if len(tg.Targets) != 2 {
+			t.Fatalf("Wrong number of targets: %v", tg.Targets)
+		}
+		tgt := tg.Targets[0]
+		if tgt[model.AddressLabel] != "1.2.3.4:8080" {
+			t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" {
+			t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
+			t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
+		}
+		tgt = tg.Targets[1]
+		if tgt[model.AddressLabel] != "1.2.3.4:1234" {
+			t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+			t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
+			t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
+		}
+	default:
+		t.Fatal("Did not get a target group.")
+	}
+}
+
+func marathonTestAppListWithContainerNetworkAndPortDefinition(labels map[string]string, runningTasks int) *AppList {
+	var (
+		task = Task{
+			ID:   "test-task-1",
+			Host: "mesos-slave1",
+			IPAddresses: []IPAddress{
+				{Address: "1.2.3.4"},
+			},
+		}
+		docker = DockerContainer{
+			Image: "repo/image:tag",
+		}
+		portDefinitions = []PortDefinition{
+			{Labels: labels, Port: 8080},
+			{Labels: make(map[string]string), Port: 1234},
+		}
+		container = Container{
+			Docker: docker,
+		}
+		networks = []Network{
+			{Mode: "container", Name: "test-network"},
+		}
+		app = App{
+			ID:              "test-service",
+			Tasks:           []Task{task},
+			RunningTasks:    runningTasks,
+			Labels:          labels,
+			Container:       container,
+			Networks:        networks,
+			PortDefinitions: portDefinitions,
+		}
+	)
+	return &AppList{
+		Apps: []App{app},
+	}
+}
+
+func TestMarathonSDSendGroupWithContainerNetworkAndPortDefinition(t *testing.T) {
+	var (
+		ch     = make(chan []*targetgroup.Group, 1)
+		client = func(client *http.Client, url string) (*AppList, error) {
+			return marathonTestAppListWithContainerNetworkAndPortDefinition(marathonValidLabel, 1), nil
+		}
+	)
+	if err := testUpdateServices(client, ch); err != nil {
+		t.Fatalf("Got error: %s", err)
+	}
+	select {
+	case tgs := <-ch:
+		tg := tgs[0]
+
+		if tg.Source != "test-service" {
+			t.Fatalf("Wrong target group name: %s", tg.Source)
+		}
+		if len(tg.Targets) != 2 {
+			t.Fatalf("Wrong number of targets: %v", tg.Targets)
+		}
+		tgt := tg.Targets[0]
+		if tgt[model.AddressLabel] != "1.2.3.4:8080" {
+			t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+			t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" {
+			t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel])
+		}
+		tgt = tg.Targets[1]
+		if tgt[model.AddressLabel] != "1.2.3.4:1234" {
+			t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" {
+			t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel])
+		}
+		if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" {
+			t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel])
+		}
+	default:
+		t.Fatal("Did not get a target group.")
+	}
+}