Merge pull request #25596 from derekparker/inotify

kubelet: Optionally, have kubelet exit if lock file contention is observed, using --exit-on-lock-contention flag

cmd/kubelet/app/options/options.go

@@ -116,6 +116,7 @@ func NewKubeletServer() *KubeletServer {
 			NodeLabels:                  make(map[string]string),
 			OOMScoreAdj:                 int32(qos.KubeletOOMScoreAdj),
 			LockFilePath:                "",
+			ExitOnLockContention:        false,
 			PodInfraContainerImage:      GetDefaultPodInfraContainerImage(),
 			Port:                             ports.KubeletPort,
 			ReadOnlyPort:                     ports.KubeletReadOnlyPort,
@@ -220,6 +221,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.CgroupRoot, "cgroup-root", s.CgroupRoot, "Optional root cgroup to use for pods. This is handled by the container runtime on a best effort basis. Default: '', which means use the container runtime default.")
 	fs.StringVar(&s.ContainerRuntime, "container-runtime", s.ContainerRuntime, "The container runtime to use. Possible values: 'docker', 'rkt'. Default: 'docker'.")
 	fs.StringVar(&s.LockFilePath, "lock-file", s.LockFilePath, "<Warning: Alpha feature> The path to file for kubelet to use as a lock file.")
+	fs.BoolVar(&s.ExitOnLockContention, "exit-on-lock-contention", s.ExitOnLockContention, "Whether kubelet should exit upon lock-file contention.")
 	fs.StringVar(&s.RktPath, "rkt-path", s.RktPath, "Path of rkt binary. Leave empty to use the first rkt in $PATH.  Only used if --container-runtime='rkt'.")
 	fs.StringVar(&s.RktAPIEndpoint, "rkt-api-endpoint", s.RktAPIEndpoint, "The endpoint of the rkt API service to communicate with. Only used if --container-runtime='rkt'.")
 	fs.StringVar(&s.RktStage1Image, "rkt-stage1-image", s.RktStage1Image, "image to use as stage1. Local paths and http/https URLs are supported. If empty, the 'stage1.aci' in the same directory as '--rkt-path' will be used.")

cmd/kubelet/app/server.go

@@ -19,6 +19,7 @@ package app
 
 import (
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"math/rand"
 	"net"
@@ -289,11 +290,22 @@ func Run(s *options.KubeletServer, kcfg *KubeletConfig) error {
 }
 
 func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) {
+	if s.ExitOnLockContention && s.LockFilePath == "" {
+		return errors.New("cannot exit on lock file contention: no lock file specified")
+	}
+
+	done := make(chan struct{})
 	if s.LockFilePath != "" {
 		glog.Infof("aquiring lock on %q", s.LockFilePath)
 		if err := flock.Acquire(s.LockFilePath); err != nil {
 			return fmt.Errorf("unable to aquire file lock on %q: %v", s.LockFilePath, err)
 		}
+		if s.ExitOnLockContention {
+			glog.Infof("watching for inotify events for: %v", s.LockFilePath)
+			if err := watchForLockfileContention(s.LockFilePath, done); err != nil {
+				return err
+			}
+		}
 	}
 	if c, err := configz.New("componentconfig"); err == nil {
 		c.Set(s.KubeletConfiguration)
@@ -383,8 +395,8 @@ func run(s *options.KubeletServer, kcfg *KubeletConfig) (err error) {
 		return nil
 	}
 
-	// run forever
-	select {}
+	<-done
+	return nil
 }
 
 // InitializeTLS checks for a configured TLSCertFile and TLSPrivateKeyFile: if unspecified a new self-signed

cmd/kubelet/app/server_linux.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"github.com/golang/glog"
+	"golang.org/x/exp/inotify"
+)
+
+func watchForLockfileContention(path string, done chan struct{}) error {
+	watcher, err := inotify.NewWatcher()
+	if err != nil {
+		glog.Errorf("unable to create watcher for lockfile: %v", err)
+		return err
+	}
+	if err = watcher.AddWatch(path, inotify.IN_OPEN|inotify.IN_DELETE_SELF); err != nil {
+		glog.Errorf("unable to watch lockfile: %v", err)
+		return err
+	}
+	go func() {
+		select {
+		case ev := <-watcher.Event:
+			glog.Infof("inotify event: %v", ev)
+		case err = <-watcher.Error:
+			glog.Errorf("inotify watcher error: %v", err)
+		}
+		close(done)
+	}()
+	return nil
+}

cmd/kubelet/app/server_test.go

@@ -17,8 +17,9 @@ limitations under the License.
 package app
 
 import (
-	"k8s.io/kubernetes/pkg/util/config"
 	"testing"
+
+	"k8s.io/kubernetes/pkg/util/config"
 )
 
 func TestValueOfAllocatableResources(t *testing.T) {

cmd/kubelet/app/server_unsupported.go

@@ -0,0 +1,25 @@
+// +build !linux
+
+/*
+Copyright 2015 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import "errors"
+
+func watchForLockfileContention(path string, done chan struct{}) error {
+	return errors.New("kubelet unsupported in this build")
+}

docs/admin/kubelet.md

@@ -92,6 +92,7 @@ kubelet
       --eviction-pressure-transition-period=5m0s: Duration for which the kubelet has to wait before transitioning out of an eviction pressure condition.
       --eviction-soft="": A set of eviction thresholds (e.g. memory.available<1.5Gi) that if met over a corresponding grace period would trigger a pod eviction.
       --eviction-soft-grace-period="": A set of eviction grace periods (e.g. memory.available=1m30s) that correspond to how long a soft eviction threshold must hold before triggering a pod eviction.
+      --exit-on-lock-contention[=false]: Whether kubelet should exit upon lock-file contention.
       --experimental-flannel-overlay[=false]: Experimental support for starting the kubelet with the default overlay network (flannel). Assumes flanneld is already running in client mode. [default=false]
       --experimental-nvidia-gpus=0: Number of NVIDIA GPU devices on this node. Only 0 (default) and 1 are currently supported.
       --file-check-frequency=20s: Duration between checking config files for new data
@@ -159,7 +160,7 @@ kubelet
       --volume-stats-agg-period=1m0s: Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes.  To disable volume calculations, set to 0.  Default: '1m'
 ```
 
-###### Auto generated by spf13/cobra on 13-May-2016
+###### Auto generated by spf13/cobra on 18-May-2016
 
 
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->

hack/verify-flags/known-flags.txt

@@ -134,6 +134,7 @@ experimental-nvidia-gpus
 experimental-prefix
 external-hostname
 external-ip
+exit-on-lock-contention
 failover-timeout
 failure-domains
 fake-clientset

pkg/apis/componentconfig/deep_copy_generated.go

@@ -274,6 +274,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in KubeletConfiguration, out
 	out.RktAPIEndpoint = in.RktAPIEndpoint
 	out.RktStage1Image = in.RktStage1Image
 	out.LockFilePath = in.LockFilePath
+	out.ExitOnLockContention = in.ExitOnLockContention
 	out.ConfigureCBR0 = in.ConfigureCBR0
 	out.HairpinMode = in.HairpinMode
 	out.BabysitDaemons = in.BabysitDaemons

pkg/apis/componentconfig/types.go

@@ -273,6 +273,11 @@ type KubeletConfiguration struct {
 	// It uses this file as a lock to synchronize with other kubelet processes
 	// that may be running.
 	LockFilePath string `json:"lockFilePath"`
+	// ExitOnLockContention is a flag that signifies to the kubelet that it is running
+	// in "bootstrap" mode. This requires that 'LockFilePath' has been set.
+	// This will cause the kubelet to listen to inotify events on the lock file,
+	// releasing it and exiting when another process tries to open that file.
+	ExitOnLockContention bool `json:"exitOnLockContention"`
 	// configureCBR0 enables the kublet to configure cbr0 based on
 	// Node.Spec.PodCIDR.
 	ConfigureCBR0 bool `json:"configureCbr0"`

pkg/kubelet/container/runtime.go

@@ -69,7 +69,7 @@ type Runtime interface {
 	APIVersion() (Version, error)
 	// Status returns error if the runtime is unhealthy; nil otherwise.
 	Status() error
-	// GetPods returns a list containers group by pods. The boolean parameter
+	// GetPods returns a list of containers grouped by pods. The boolean parameter
 	// specifies whether the runtime returns all containers including those already
 	// exited and dead containers (used for garbage collection).
 	GetPods(all bool) ([]*Pod, error)