Merge pull request #19944 from mwielgus/simple-cm

Auto commit by PR queue bot

cmd/kubelet/app/options/options.go

@@ -77,6 +77,7 @@ func NewKubeletServer() *KubeletServer {
 			DockerExecHandlerName:       "native",
 			EventBurst:                  10,
 			EventRecordQPS:              5.0,
+			EnableCustomMetrics:         false,
 			EnableDebuggingHandlers:     true,
 			EnableServer:                true,
 			FileCheckFrequency:          unversioned.Duration{20 * time.Second},
@@ -214,4 +215,5 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.ExperimentalFlannelOverlay, "experimental-flannel-overlay", s.ExperimentalFlannelOverlay, "Experimental support for starting the kubelet with the default overlay network (flannel). Assumes flanneld is already running in client mode. [default=false]")
 	fs.DurationVar(&s.OutOfDiskTransitionFrequency.Duration, "outofdisk-transition-frequency", s.OutOfDiskTransitionFrequency.Duration, "Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status. Default: 5m0s")
 	fs.StringVar(&s.NodeIP, "node-ip", s.NodeIP, "IP address of the node. If set, kubelet will use this IP address for the node")
+	fs.BoolVar(&s.EnableCustomMetrics, "enable-custom-metrics", s.EnableCustomMetrics, "Support for gathering custom metrics.")
 }

cmd/kubelet/app/server.go

@@ -193,6 +193,7 @@ func UnsecuredKubeletConfig(s *options.KubeletServer) (*KubeletConfig, error) {
 		DockerClient:              dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
 		DockerDaemonContainer:     s.DockerDaemonContainer,
 		DockerExecHandler:         dockerExecHandler,
+		EnableCustomMetrics:       s.EnableCustomMetrics,
 		EnableDebuggingHandlers:   s.EnableDebuggingHandlers,
 		EnableServer:              s.EnableServer,
 		EventBurst:                s.EventBurst,
@@ -491,6 +492,7 @@ func SimpleKubelet(client *clientset.Clientset,
 		DockerClient:              dockerClient,
 		DockerDaemonContainer:     "/docker-daemon",
 		DockerExecHandler:         &dockertools.NativeExecHandler{},
+		EnableCustomMetrics:       false,
 		EnableDebuggingHandlers:   true,
 		EnableServer:              true,
 		FileCheckFrequency:        fileCheckFrequency,
@@ -665,6 +667,7 @@ type KubeletConfig struct {
 	DockerClient                   dockertools.DockerInterface
 	DockerDaemonContainer          string
 	DockerExecHandler              dockertools.ExecHandler
+	EnableCustomMetrics            bool
 	EnableDebuggingHandlers        bool
 	EnableServer                   bool
 	EventClient                    *clientset.Clientset
@@ -811,6 +814,7 @@ func CreateAndInitKubelet(kc *KubeletConfig) (k KubeletBootstrap, pc *config.Pod
 		kc.ExperimentalFlannelOverlay,
 		kc.NodeIP,
 		kc.Reservation,
+		kc.EnableCustomMetrics,
 	)
 
 	if err != nil {

docs/admin/kubelet.md

@@ -82,6 +82,7 @@ kubelet
       --cpu-cfs-quota[=true]: Enable CPU CFS quota enforcement for containers that specify CPU limits
       --docker-endpoint="": If non-empty, use this for the docker endpoint to communicate with
       --docker-exec-handler="native": Handler to use when executing a command in a container. Valid values are 'native' and 'nsenter'. Defaults to 'native'.
+      --enable-custom-metrics[=false]: Support for gathering custom metrics.
       --enable-debugging-handlers[=true]: Enables server endpoints for log collection and local running of containers and commands
       --enable-server[=true]: Enable the Kubelet's server
       --event-burst=10: Maximum size of a bursty event records, temporarily allows event records to burst to this number, while still not exceeding event-qps. Only used if --event-qps > 0
@@ -146,7 +147,7 @@ kubelet
       --volume-plugin-dir="/usr/libexec/kubernetes/kubelet-plugins/volume/exec/": <Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins
 ```
 
-###### Auto generated by spf13/cobra on 29-Jan-2016
+###### Auto generated by spf13/cobra on 3-Feb-2016
 
 
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

hack/verify-flags/known-flags.txt

@@ -86,6 +86,7 @@ dry-run
 duration-sec
 e2e-output-dir
 e2e-verify-service-account
+enable-custom-metrics
 enable-debugging-handlers
 enable-hostpath-provisioner
 enable-server

pkg/apis/componentconfig/types.go

@@ -292,6 +292,8 @@ type KubeletConfiguration struct {
 	NodeLabels map[string]string `json:"nodeLabels"`
 	// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
 	NonMasqueradeCIDR string `json:"nonMasqueradeCIDR"`
+	// enable gathering custom metrics.
+	EnableCustomMetrics bool `json:"enableCustomMetrics"`
 }
 
 type KubeSchedulerConfiguration struct {

pkg/kubelet/custommetrics/custom_metrics.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package custommetrics contains support for instrumenting cAdvisor to gather custom metrics from pods.
+package custommetrics
+
+import (
+	"path"
+
+	"k8s.io/kubernetes/pkg/api"
+)
+
+const (
+	CustomMetricsDefinitionContainerFile = "definition.json"
+
+	CustomMetricsDefinitionDir = "/etc/custom-metrics"
+)
+
+// Alpha implementation.
+// Returns a path to a cAdvisor-specific custom metrics configuration.
+func GetCAdvisorCustomMetricsDefinitionPath(container *api.Container) (*string, error) {
+	// Assuemes that the container has Custom Metrics enabled if it has "/etc/custom-metrics" directory
+	// mounted as a volume. Custom Metrics definition is expected to be in "definition.json".
+	if container.VolumeMounts != nil {
+		for _, volumeMount := range container.VolumeMounts {
+			if path.Clean(volumeMount.MountPath) == path.Clean(CustomMetricsDefinitionDir) {
+				// TODO: add defintion file validation.
+				definitionPath := path.Clean(path.Join(volumeMount.MountPath, CustomMetricsDefinitionContainerFile))
+				return &definitionPath, nil
+			}
+		}
+	}
+	// No Custom Metrics definition available.
+	return nil, nil
+}

pkg/kubelet/custommetrics/custom_metrics_test.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package custommetrics
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/kubernetes/pkg/api"
+)
+
+func TestGetCAdvisorCustomMetricsDefinitionPath(t *testing.T) {
+
+	regularContainer := &api.Container{
+		Name: "test_container",
+	}
+
+	cmContainer := &api.Container{
+		Name: "test_container",
+		VolumeMounts: []api.VolumeMount{
+			{
+				Name:      "cm",
+				MountPath: CustomMetricsDefinitionDir,
+			},
+		},
+	}
+	path, err := GetCAdvisorCustomMetricsDefinitionPath(regularContainer)
+	assert.Nil(t, path)
+	assert.NoError(t, err)
+
+	path, err = GetCAdvisorCustomMetricsDefinitionPath(cmContainer)
+	assert.NotEmpty(t, *path)
+	assert.NoError(t, err)
+}

pkg/kubelet/dockertools/fake_manager.go

@@ -47,7 +47,7 @@ func NewFakeDockerManager(
 	fakeProcFs := procfs.NewFakeProcFS()
 	dm := NewDockerManager(client, recorder, livenessManager, containerRefManager, machineInfo, podInfraContainerImage, qps,
 		burst, containerLogsDir, osInterface, networkPlugin, runtimeHelper, httpClient, &NativeExecHandler{},
-		fakeOOMAdjuster, fakeProcFs, false, imageBackOff, true)
+		fakeOOMAdjuster, fakeProcFs, false, imageBackOff, true, false)
 	dm.dockerPuller = &FakeDockerPuller{}
 	return dm
 }

pkg/kubelet/dockertools/labels.go

@@ -23,6 +23,7 @@ import (
 	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/api"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/custommetrics"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	"k8s.io/kubernetes/pkg/runtime"
 	"k8s.io/kubernetes/pkg/types"
@@ -47,6 +48,8 @@ const (
 
 	// TODO(random-liu): Keep this for old containers, remove this when we drop support for v1.1.
 	kubernetesPodLabel = "io.kubernetes.pod.data"
+
+	cadvisorPrometheusMetricsLabel = "io.cadvisor.metric.prometheus"
 )
 
 // Container information which has been labelled on each docker container
@@ -80,7 +83,7 @@ func GetPodNamespace(labels map[string]string) string {
 	return labels[kubernetesPodNamespaceLabel]
 }
 
-func newLabels(container *api.Container, pod *api.Pod, restartCount int) map[string]string {
+func newLabels(container *api.Container, pod *api.Pod, restartCount int, enableCustomMetrics bool) map[string]string {
 	labels := map[string]string{}
 	labels[kubernetesPodNameLabel] = pod.Name
 	labels[kubernetesPodNamespaceLabel] = pod.Namespace
@@ -106,6 +109,13 @@ func newLabels(container *api.Container, pod *api.Pod, restartCount int) map[str
 		}
 	}
 
+	if enableCustomMetrics {
+		path, err := custommetrics.GetCAdvisorCustomMetricsDefinitionPath(container)
+		if path != nil && err == nil {
+			labels[cadvisorPrometheusMetricsLabel] = *path
+		}
+	}
+
 	return labels
 }
 

pkg/kubelet/dockertools/labels_test.go

@@ -81,7 +81,7 @@ func TestLabels(t *testing.T) {
 	}
 
 	// Test whether we can get right information from label
-	labels := newLabels(container, pod, restartCount)
+	labels := newLabels(container, pod, restartCount, false)
 	containerInfo := getContainerInfoFromLabel(labels)
 	if !reflect.DeepEqual(containerInfo, expected) {
 		t.Errorf("expected %v, got %v", expected, containerInfo)
@@ -97,7 +97,7 @@ func TestLabels(t *testing.T) {
 	expected.PreStopHandler = nil
 	// Because container is changed, the Hash should be updated
 	expected.Hash = strconv.FormatUint(kubecontainer.HashContainer(container), 16)
-	labels = newLabels(container, pod, restartCount)
+	labels = newLabels(container, pod, restartCount, false)
 	containerInfo = getContainerInfoFromLabel(labels)
 	if !reflect.DeepEqual(containerInfo, expected) {
 		t.Errorf("expected %v, got %v", expected, containerInfo)

pkg/kubelet/dockertools/manager.go

@@ -149,6 +149,9 @@ type DockerManager struct {
 
 	// Container GC manager
 	containerGC *containerGC
+
+	// Support for gathering custom metrics.
+	enableCustomMetrics bool
 }
 
 func NewDockerManager(
@@ -170,7 +173,8 @@ func NewDockerManager(
 	procFs procfs.ProcFSInterface,
 	cpuCFSQuota bool,
 	imageBackOff *util.Backoff,
-	serializeImagePulls bool) *DockerManager {
+	serializeImagePulls bool,
+	enableCustomMetrics bool) *DockerManager {
 
 	// Work out the location of the Docker runtime, defaulting to /var/lib/docker
 	// if there are any problems.
@@ -204,6 +208,7 @@ func NewDockerManager(
 		oomAdjuster:            oomAdjuster,
 		procFs:                 procFs,
 		cpuCFSQuota:            cpuCFSQuota,
+		enableCustomMetrics:    enableCustomMetrics,
 	}
 	dm.runner = lifecycle.NewHandlerRunner(httpClient, dm, dm)
 	if serializeImagePulls {
@@ -638,7 +643,7 @@ func (dm *DockerManager) runContainer(
 	// Pod information is recorded on the container as labels to preserve it in the event the pod is deleted
 	// while the Kubelet is down and there is no information available to recover the pod.
 	// TODO: keep these labels up to date if the pod changes
-	labels := newLabels(container, pod, restartCount)
+	labels := newLabels(container, pod, restartCount, dm.enableCustomMetrics)
 
 	// TODO(random-liu): Remove this when we start to use new labels for KillContainerInPod
 	if container.Lifecycle != nil && container.Lifecycle.PreStop != nil {

pkg/kubelet/kubelet.go

@@ -202,6 +202,7 @@ func NewMainKubelet(
 	flannelExperimentalOverlay bool,
 	nodeIP net.IP,
 	reservation kubetypes.Reservation,
+	enableCustomMetrics bool,
 ) (*Kubelet, error) {
 	if rootDirectory == "" {
 		return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
@@ -326,6 +327,7 @@ func NewMainKubelet(
 		clock:                          util.RealClock{},
 		outOfDiskTransitionFrequency: outOfDiskTransitionFrequency,
 		reservation:                  reservation,
+		enableCustomMetrics:          enableCustomMetrics,
 	}
 	if klet.flannelExperimentalOverlay {
 		glog.Infof("Flannel is in charge of podCIDR and overlay networking.")
@@ -378,6 +380,7 @@ func NewMainKubelet(
 			klet.cpuCFSQuota,
 			imageBackOff,
 			serializeImagePulls,
+			enableCustomMetrics,
 		)
 	case "rkt":
 		conf := &rkt.Config{
@@ -684,6 +687,9 @@ type Kubelet struct {
 	// reservation specifies resources which are reserved for non-pod usage, including kubernetes and
 	// non-kubernetes system processes.
 	reservation kubetypes.Reservation
+
+	// support gathering custom metrics.
+	enableCustomMetrics bool
 }
 
 // Validate given node IP belongs to the current host