From 1fc92bef53fdd1bc70f623c0693736c763cff45f Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Wed, 8 Oct 2014 17:05:04 -0700 Subject: [PATCH] Load cadvisor connection in background. --- cmd/kubelet/kubelet.go | 19 +++++++++++++------ pkg/kubelet/kubelet.go | 43 ++++++++++++++++++++++++++++++++---------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go index 17c1239674..f25b167765 100644 --- a/cmd/kubelet/kubelet.go +++ b/cmd/kubelet/kubelet.go @@ -131,11 +131,6 @@ func main() { glog.Fatal("Couldn't connect to docker.") } - cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194") - if err != nil { - glog.Errorf("Error on creating cadvisor client: %v", err) - } - hostname := getHostname() if *rootDirectory == "" { @@ -182,7 +177,6 @@ func main() { k := kubelet.NewMainKubelet( getHostname(), dockerClient, - cadvisorClient, etcdClient, *rootDirectory, *networkContainerImage, @@ -190,6 +184,19 @@ func main() { float32(*registryPullQPS), *registryBurst) + go func() { + defer util.HandleCrash() + // TODO: Monitor this connection, reconnect if needed? + glog.V(1).Infof("Trying to create cadvisor client.") + cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194") + if err != nil { + glog.Errorf("Error on creating cadvisor client: %v", err) + return + } + glog.V(1).Infof("Successfully created cadvisor client.") + k.SetCadvisorClient(cadvisorClient) + }() + // TODO: These should probably become more plugin-ish: register a factory func // in each checker's init(), iterate those here. health.AddHealthChecker(health.NewExecHealthChecker(k)) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 0491fa73db..76feb49f22 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -64,7 +64,6 @@ type volumeMap map[string]volume.Interface func NewMainKubelet( hn string, dc dockertools.DockerInterface, - cc CadvisorInterface, ec tools.EtcdClient, rd string, ni string, @@ -74,7 +73,6 @@ func NewMainKubelet( return &Kubelet{ hostname: hn, dockerClient: dc, - cadvisorClient: cc, etcdClient: ec, rootDirectory: rd, resyncInterval: ri, @@ -117,8 +115,6 @@ type Kubelet struct { // Optional, no events will be sent without it etcdClient tools.EtcdClient - // Optional, no statistics will be available if omitted - cadvisorClient CadvisorInterface // Optional, defaults to simple implementaiton healthChecker health.HealthChecker // Optional, defaults to simple Docker implementation @@ -133,6 +129,24 @@ type Kubelet struct { pullQPS float32 // Optional, maximum burst QPS from the docker registry, must be positive if QPS is > 0.0 pullBurst int + + // Optional, no statistics will be available if omitted + cadvisorClient CadvisorInterface + cadvisorLock sync.RWMutex +} + +// SetCadvisorClient sets the cadvisor client in a thread-safe way. +func (kl *Kubelet) SetCadvisorClient(c CadvisorInterface) { + kl.cadvisorLock.Lock() + defer kl.cadvisorLock.Unlock() + kl.cadvisorClient = c +} + +// GetCadvisorClient gets the cadvisor client. +func (kl *Kubelet) GetCadvisorClient() CadvisorInterface { + kl.cadvisorLock.RLock() + defer kl.cadvisorLock.RUnlock() + return kl.cadvisorClient } // Run starts the kubelet reacting to config updates @@ -742,8 +756,8 @@ func getCadvisorContainerInfoRequest(req *info.ContainerInfoRequest) *info.Conta // container. The container's absolute path refers to its hierarchy in the // cgroup file system. e.g. The root container, which represents the whole // machine, has path "/"; all docker containers have path "/docker/" -func (kl *Kubelet) statsFromContainerPath(containerPath string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { - cinfo, err := kl.cadvisorClient.ContainerInfo(containerPath, getCadvisorContainerInfoRequest(req)) +func (kl *Kubelet) statsFromContainerPath(cc CadvisorInterface, containerPath string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { + cinfo, err := cc.ContainerInfo(containerPath, getCadvisorContainerInfoRequest(req)) if err != nil { return nil, err } @@ -782,7 +796,8 @@ func (kl *Kubelet) GetPodInfo(podFullName, uuid string) (api.PodInfo, error) { // GetContainerInfo returns stats (from Cadvisor) for a container. func (kl *Kubelet) GetContainerInfo(podFullName, uuid, containerName string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { - if kl.cadvisorClient == nil { + cc := kl.GetCadvisorClient() + if cc == nil { return nil, nil } dockerContainers, err := dockertools.GetKubeletDockerContainers(kl.dockerClient, false) @@ -793,16 +808,24 @@ func (kl *Kubelet) GetContainerInfo(podFullName, uuid, containerName string, req if !found { return nil, errors.New("couldn't find container") } - return kl.statsFromContainerPath(fmt.Sprintf("/docker/%s", dockerContainer.ID), req) + return kl.statsFromContainerPath(cc, fmt.Sprintf("/docker/%s", dockerContainer.ID), req) } // GetRootInfo returns stats (from Cadvisor) of current machine (root container). func (kl *Kubelet) GetRootInfo(req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { - return kl.statsFromContainerPath("/", req) + cc := kl.GetCadvisorClient() + if cc == nil { + return nil, fmt.Errorf("no cadvisor connection") + } + return kl.statsFromContainerPath(cc, "/", req) } func (kl *Kubelet) GetMachineInfo() (*info.MachineInfo, error) { - return kl.cadvisorClient.MachineInfo() + cc := kl.GetCadvisorClient() + if cc == nil { + return nil, fmt.Errorf("no cadvisor connection") + } + return cc.MachineInfo() } func (kl *Kubelet) healthy(podFullName string, currentState api.PodState, container api.Container, dockerContainer *docker.APIContainers) (health.Status, error) {