Merge pull request #22893 from Q-Lee/nanny-heapster

Support addon Deployments, make heapster a deployment with a nanny.

cluster/addons/cluster-monitoring/google/heapster-controller.yaml

@@ -1,15 +1,17 @@
 {% set metrics_memory = "200Mi" -%}
 {% set eventer_memory = "200Mi" -%}
+{% set metrics_memory_per_node = 4 -%}
+{% set eventer_memory_per_node = 500 -%}
 {% set num_nodes = pillar.get('num_nodes', -1) -%}
 {% if num_nodes >= 0 -%}
-  {% set metrics_memory = (200 + num_nodes * 4)|string + "Mi" -%}
+  {% set metrics_memory = (200 + num_nodes * metrics_memory_per_node)|string + "Mi" -%}
-  {% set eventer_memory = (200 * 1024 + num_nodes * 500)|string + "Ki" -%}
+  {% set eventer_memory = (200 * 1024 + num_nodes * eventer_memory_per_node)|string + "Ki" -%}
 {% endif -%}
 
-apiVersion: v1
+apiVersion: extensions/v1beta1
-kind: ReplicationController
+kind: Deployment
 metadata:
-  name: heapster-v1.0.0
+  name: heapster
   namespace: kube-system
   labels:
     k8s-app: heapster
@@ -17,7 +19,8 @@ metadata:
 spec:
   replicas: 1
   selector:
-    k8s-app: heapster
+    matchLabels:
+      k8s-app: heapster
   template:
     metadata:
       labels:
@@ -62,6 +65,62 @@ spec:
             - name: ssl-certs
               mountPath: /etc/ssl/certs
               readOnly: true
+        - image: gcr.io/google_containers/addon-resizer:1.0
+          name: heapster-nanny
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+            requests:
+              cpu: 50m
+              memory: 100Mi
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /pod_nanny
+            - --cpu=100m
+            - --extra-cpu=0m
+            - --memory={{ metrics_memory }}
+            - --extra-memory={{metrics_memory_per_node}}Mi
+            - --threshold=5
+            - --deployment=heapster
+            - --container=heapster
+            - --poll-period=300000
+        - image: gcr.io/google_containers/addon-resizer:1.0
+          name: eventer-nanny
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+            requests:
+              cpu: 50m
+              memory: 100Mi
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /pod_nanny
+            - --cpu=100m
+            - --extra-cpu=0m
+            - --memory={{eventer_memory}}
+            - --extra-memory={{eventer_memory_per_node}}Ki
+            - --threshold=5
+            - --deployment=heapster
+            - --container=eventer
+            - --poll-period=300000
       volumes:
         - name: ssl-certs
           hostPath:

cluster/addons/cluster-monitoring/googleinfluxdb/heapster-controller-combined.yaml

@@ -1,15 +1,17 @@
 {% set metrics_memory = "200Mi" -%}
 {% set eventer_memory = "200Mi" -%}
+{% set metrics_memory_per_node = 4 -%}
+{% set eventer_memory_per_node = 500 -%}
 {% set num_nodes = pillar.get('num_nodes', -1) -%}
 {% if num_nodes >= 0 -%}
-  {% set metrics_memory = (200 + num_nodes * 4)|string + "Mi" -%}
+  {% set metrics_memory = (200 + num_nodes * metrics_memory_per_node)|string + "Mi" -%}
-  {% set eventer_memory = (200 * 1024 + num_nodes * 500)|string + "Ki" -%}
+  {% set eventer_memory = (200 * 1024 + num_nodes * eventer_memory_per_node)|string + "Ki" -%}
 {% endif -%}
 
-apiVersion: v1
+apiVersion: extensions/v1beta1
-kind: ReplicationController
+kind: Deployment
 metadata:
-  name: heapster-v1.0.0
+  name: heapster
   namespace: kube-system
   labels:
     k8s-app: heapster
@@ -17,7 +19,8 @@ metadata:
 spec:
   replicas: 1
   selector:
-    k8s-app: heapster
+    matchLabels:
+      k8s-app: heapster
   template:
     metadata:
       labels:
@@ -63,6 +66,62 @@ spec:
             - name: ssl-certs
               mountPath: /etc/ssl/certs
               readOnly: true
+        - image: gcr.io/google_containers/addon-resizer:1.0
+          name: heapster-nanny
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+            requests:
+              cpu: 50m
+              memory: 100Mi
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /pod_nanny
+            - --cpu=100m
+            - --extra-cpu=0m
+            - --memory={{ metrics_memory }}
+            - --extra-memory={{ metrics_memory_per_node }}Mi
+            - --threshold=5
+            - --deployment=heapster
+            - --container=heapster
+            - --poll-period=300000
+        - image: gcr.io/google_containers/addon-resizer:1.0
+          name: eventer-nanny
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+            requests:
+              cpu: 50m
+              memory: 100Mi
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /pod_nanny
+            - --cpu=100m
+            - --extra-cpu=0m
+            - --memory={{ eventer_memory }}
+            - --extra-memory={{ eventer_memory_per_node }}Ki
+            - --threshold=5
+            - --deployment=heapster
+            - --container=eventer
+            - --poll-period=300000
       volumes:
         - name: ssl-certs
           hostPath:

cluster/addons/cluster-monitoring/influxdb/heapster-controller.yaml

@@ -1,15 +1,17 @@
 {% set metrics_memory = "200Mi" -%}
 {% set eventer_memory = "200Mi" -%}
+{% set metrics_memory_per_node = 4 -%}
+{% set eventer_memory_per_node = 500 -%}
 {% set num_nodes = pillar.get('num_nodes', -1) -%}
 {% if num_nodes >= 0 -%}
-  {% set metrics_memory = (200 + num_nodes * 4)|string + "Mi" -%}
+  {% set metrics_memory = (200 + num_nodes * metrics_memory_per_node)|string + "Mi" -%}
-  {% set eventer_memory = (200 * 1024 + num_nodes * 500)|string + "Ki" -%}
+  {% set eventer_memory = (200 * 1024 + num_nodes * eventer_memory_per_node)|string + "Ki" -%}
 {% endif -%}
 
-apiVersion: v1
+apiVersion: extensions/v1beta1
-kind: ReplicationController
+kind: Deployment
 metadata:
-  name: heapster-v1.0.0
+  name: heapster
   namespace: kube-system
   labels:
     k8s-app: heapster
@@ -17,7 +19,8 @@ metadata:
 spec:
   replicas: 1
   selector:
-    k8s-app: heapster
+    matchLabels:
+      k8s-app: heapster
   template:
     metadata:
       labels:
@@ -54,3 +57,60 @@ spec:
             - /eventer
             - --source=kubernetes:''
             - --sink=influxdb:http://monitoring-influxdb:8086
+        - image: gcr.io/google_containers/addon-resizer:1.0
+          name: heapster-nanny
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+            requests:
+              cpu: 50m
+              memory: 100Mi
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /pod_nanny
+            - --cpu=100m
+            - --extra-cpu=0m
+            - --memory={{ metrics_memory }}
+            - --extra-memory={{ metrics_memory_per_node }}Mi
+            - --threshold=5
+            - --deployment=heapster
+            - --container=heapster
+            - --poll-period=300000
+        - image: gcr.io/google_containers/addon-resizer:1.0
+          name: eventer-nanny
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+            requests:
+              cpu: 50m
+              memory: 100Mi
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /pod_nanny
+            - --cpu=100m
+            - --extra-cpu=0m
+            - --memory={{ eventer_memory }}
+            - --extra-memory={{ eventer_memory_per_node }}Ki
+            - --threshold=5
+            - --deployment=heapster
+            - --container=eventer
+            - --poll-period=300000
+

cluster/addons/cluster-monitoring/standalone/heapster-controller.yaml

@@ -1,13 +1,14 @@
 {% set metrics_memory = "200Mi" -%}
+{% set metrics_memory_per_node = 4 -%}
 {% set num_nodes = pillar.get('num_nodes', -1) -%}
 {% if num_nodes >= 0 -%}
-  {% set metrics_memory = (200 + num_nodes * 4)|string + "Mi" -%}
+  {% set metrics_memory = (200 + num_nodes * metrics_memory_per_node)|string + "Mi" -%}
 {% endif -%}
 
-apiVersion: v1
+apiVersion: extensions/v1beta1
-kind: ReplicationController
+kind: Deployment
 metadata:
-  name: heapster-v1.0.0
+  name: heapster
   namespace: kube-system
   labels:
     k8s-app: heapster
@@ -15,7 +16,8 @@ metadata:
 spec:
   replicas: 1
   selector:
-    k8s-app: heapster
+    matchLabels:
+      k8s-app: heapster
   template:
     metadata:
       labels:
@@ -37,3 +39,31 @@ spec:
             - /heapster
             - --source=kubernetes.summary_api:''
             - --metric_resolution=60s
+        - image: gcr.io/google_containers/addon-resizer:1.0
+          name: heapster-nanny
+          resources:
+            limits:
+              cpu: 50m
+              memory: 100Mi
+            requests:
+              cpu: 50m
+              memory: 100Mi
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: MY_POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          command:
+            - /pod_nanny
+            - --cpu=100m
+            - --extra-cpu=0m
+            - --memory={{ metrics_memory }}
+            - --extra-memory={{ metrics_memory_per_node }}Mi
+            - --threshold=5
+            - --deployment=heapster
+            - --container=heapster
+            - --poll-period=300000

cluster/saltbase/salt/kube-addons/kube-addon-update.sh

@@ -481,6 +481,7 @@ function update-addons() {
     # That's why we pass an empty string as the version separator.
     # If the description differs on disk, the object should be recreated.
     # This is not implemented in this version.
+    reconcile-objects ${addon_path} Deployment "" &
     reconcile-objects ${addon_path} Service "" &
     reconcile-objects ${addon_path} PersistentVolume "" &
     reconcile-objects ${addon_path} PersistentVolumeClaim "" &

test/e2e/monitoring.go

@@ -102,14 +102,19 @@ func verifyExpectedRcsExistAndGetExpectedPods(c *client.Client) ([]string, error
 	for _, rcLabel := range rcLabels {
 		selector := labels.Set{"k8s-app": rcLabel}.AsSelector()
 		options := api.ListOptions{LabelSelector: selector}
+		deploymentList, err := c.Deployments(api.NamespaceSystem).List(options)
+		if err != nil {
+			return nil, err
+		}
 		rcList, err := c.ReplicationControllers(api.NamespaceSystem).List(options)
 		if err != nil {
 			return nil, err
 		}
-		if len(rcList.Items) != 1 {
+		if (len(rcList.Items) + len(deploymentList.Items)) != 1 {
-			return nil, fmt.Errorf("expected to find one replica for RC with label %s but got %d",
+			return nil, fmt.Errorf("expected to find one replica for RC or deployment with label %s but got %d",
 				rcLabel, len(rcList.Items))
 		}
+		// Check all the replication controllers.
 		for _, rc := range rcList.Items {
 			selector := labels.Set(rc.Spec.Selector).AsSelector()
 			options := api.ListOptions{LabelSelector: selector}
@@ -124,6 +129,21 @@ func verifyExpectedRcsExistAndGetExpectedPods(c *client.Client) ([]string, error
 				expectedPods = append(expectedPods, string(pod.UID))
 			}
 		}
+		// Do the same for all deployments.
+		for _, rc := range deploymentList.Items {
+			selector := labels.Set(rc.Spec.Selector.MatchLabels).AsSelector()
+			options := api.ListOptions{LabelSelector: selector}
+			podList, err := c.Pods(api.NamespaceSystem).List(options)
+			if err != nil {
+				return nil, err
+			}
+			for _, pod := range podList.Items {
+				if pod.DeletionTimestamp != nil {
+					continue
+				}
+				expectedPods = append(expectedPods, string(pod.UID))
+			}
+		}
 	}
 	return expectedPods, nil
 }