Merge pull request #4792 from vmarmol/mon-containers-per-pod

Add Kubelet metrics for pod and container counts.

pkg/kubelet/kubelet.go

@@ -135,6 +135,8 @@ func NewMainKubelet(
 	}
 	klet.dockerCache = dockerCache
 
+	metrics.Register(dockerCache)
+
 	if err = klet.setupDataDirs(); err != nil {
 		return nil, err
 	}

pkg/kubelet/metrics/metrics.go

@@ -17,6 +17,11 @@ limitations under the License.
 package metrics
 
 import (
+	"sync"
+
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/types"
+	"github.com/golang/glog"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -30,30 +35,71 @@ var (
 			Help:      "Image pull latency in microseconds.",
 		},
 	)
-	// TODO(vmarmol): Implement.
-	// TODO(vmarmol): Split by source?
-	PodCount = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Subsystem: kubeletSubsystem,
-			Name:      "pod_count",
-			Help:      "Number of pods currently running.",
-		},
-	)
-	// TODO(vmarmol): Implement.
-	// TODO(vmarmol): Split by source?
-	ContainerCount = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Subsystem: kubeletSubsystem,
-			Name:      "container_count",
-			Help:      "Number of containers currently running.",
-		},
-	)
 	// TODO(vmarmol): Containers per pod
 	// TODO(vmarmol): Latency of pod startup
 	// TODO(vmarmol): Latency of SyncPods
 )
 
-func init() {
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register(containerCache dockertools.DockerCache) {
 	// Register the metrics.
-	prometheus.MustRegister(ImagePullLatency)
+	registerMetrics.Do(func() {
+		prometheus.MustRegister(ImagePullLatency)
+		prometheus.MustRegister(newPodAndContainerCollector(containerCache))
+	})
+}
+
+func newPodAndContainerCollector(containerCache dockertools.DockerCache) *podAndContainerCollector {
+	return &podAndContainerCollector{
+		containerCache: containerCache,
+	}
+}
+
+// Custom collector for current pod and container counts.
+type podAndContainerCollector struct {
+	// Cache for accessing information about running containers.
+	containerCache dockertools.DockerCache
+}
+
+// TODO(vmarmol): Split by source?
+var (
+	runningPodCountDesc = prometheus.NewDesc(
+		prometheus.BuildFQName("", kubeletSubsystem, "running_pod_count"),
+		"Number of pods currently running",
+		nil, nil)
+	runningContainerCountDesc = prometheus.NewDesc(
+		prometheus.BuildFQName("", kubeletSubsystem, "running_container_count"),
+		"Number of containers currently running",
+		nil, nil)
+)
+
+func (self *podAndContainerCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- runningPodCountDesc
+	ch <- runningContainerCountDesc
+}
+
+func (self *podAndContainerCollector) Collect(ch chan<- prometheus.Metric) {
+	runningContainers, err := self.containerCache.RunningContainers()
+	if err != nil {
+		glog.Warning("Failed to get running container information while collecting metrics: %v", err)
+		return
+	}
+
+	// Get a mapping of pod to number of containers in that pod.
+	podToContainerCount := make(map[types.UID]struct{})
+	for _, cont := range runningContainers {
+		_, uid, _, _ := dockertools.ParseDockerName(cont.Names[0])
+		podToContainerCount[uid] = struct{}{}
+	}
+
+	ch <- prometheus.MustNewConstMetric(
+		runningPodCountDesc,
+		prometheus.GaugeValue,
+		float64(len(podToContainerCount)))
+	ch <- prometheus.MustNewConstMetric(
+		runningContainerCountDesc,
+		prometheus.GaugeValue,
+		float64(len(runningContainers)))
 }