Add support for chaos to Kubelet and hack/local-up-cluster.sh

cmd/kubelet/app/server.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/chaosclient"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
 	"github.com/GoogleCloudPlatform/kubernetes/pkg/credentialprovider"
@@ -84,7 +85,6 @@ type KubeletServer struct {
 	ClusterDomain                  string
 	MasterServiceNamespace         string
 	ClusterDNS                     util.IP
-	ReallyCrashForTesting          bool
 	StreamingConnectionIdleTimeout time.Duration
 	ImageGCHighThresholdPercent    int
 	ImageGCLowThresholdPercent     int
@@ -95,6 +95,13 @@ type KubeletServer struct {
 	TLSPrivateKeyFile              string
 	CertDirectory                  string
 	NodeStatusUpdateFrequency      time.Duration
+
+	// Flags intended for testing
+
+	// Crash immediately, rather than eating panics.
+	ReallyCrashForTesting bool
+	// Insert a probability of random errors during calls to the master.
+	ChaosChance float64
 }
 
 // bootstrapping interface for kubelet, targets the initialization protocol
@@ -181,7 +188,6 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.ClusterDomain, "cluster_domain", s.ClusterDomain, "Domain for this cluster.  If set, kubelet will configure all containers to search this domain in addition to the host's search domains")
 	fs.StringVar(&s.MasterServiceNamespace, "master_service_namespace", s.MasterServiceNamespace, "The namespace from which the kubernetes master services should be injected into pods")
 	fs.Var(&s.ClusterDNS, "cluster_dns", "IP address for a cluster DNS server.  If set, kubelet will configure all containers to use this for DNS resolution in addition to the host's DNS servers")
-	fs.BoolVar(&s.ReallyCrashForTesting, "really_crash_for_testing", s.ReallyCrashForTesting, "If true, crash with panics more often.")
 	fs.DurationVar(&s.StreamingConnectionIdleTimeout, "streaming_connection_idle_timeout", 0, "Maximum time a streaming connection can be idle before the connection is automatically closed.  Example: '5m'")
 	fs.DurationVar(&s.NodeStatusUpdateFrequency, "node_status_update_frequency", s.NodeStatusUpdateFrequency, "Specifies how often kubelet posts node status to master. Note: be cautious when changing the constant, it must work with nodeMonitorGracePeriod in nodecontroller. Default: 10s")
 	fs.IntVar(&s.ImageGCHighThresholdPercent, "image_gc_high_threshold", s.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run. Default: 90%%")
@@ -189,6 +195,10 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.NetworkPluginName, "network_plugin", s.NetworkPluginName, "<Warning: Alpha feature> The name of the network plugin to be invoked for various events in kubelet/pod lifecycle")
 	fs.StringVar(&s.CloudProvider, "cloud_provider", s.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
 	fs.StringVar(&s.CloudConfigFile, "cloud_config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
+
+	// Flags intended for testing, not recommended used in production environments.
+	fs.BoolVar(&s.ReallyCrashForTesting, "really_crash_for_testing", s.ReallyCrashForTesting, "If true, when panics occur crash. Intended for testing.")
+	fs.Float64Var(&s.ChaosChance, "chaos_chance", s.ChaosChance, "If > 0.0, introduce random client errors and latency. Intended for testing. [default=0.0]")
 }
 
 // Run runs the specified KubeletServer.  This should never exit.
@@ -337,6 +347,9 @@ func (s *KubeletServer) createAPIServerClient() (*client.Client, error) {
 		glog.Infof("Multiple api servers specified.  Picking first one")
 	}
 	clientConfig.Host = s.APIServerList[0]
+
+	s.addChaosToClientConfig(&clientConfig)
+
 	c, err := client.New(&clientConfig)
 	if err != nil {
 		return nil, err
@@ -344,6 +357,18 @@ func (s *KubeletServer) createAPIServerClient() (*client.Client, error) {
 	return c, nil
 }
 
+// addChaosToClientConfig injects random errors into client connections if configured.
+func (s *KubeletServer) addChaosToClientConfig(config *client.Config) {
+	if s.ChaosChance != 0.0 {
+		config.WrapTransport = func(rt http.RoundTripper) http.RoundTripper {
+			seed := chaosclient.NewSeed(1)
+			// TODO: introduce a standard chaos package with more tunables - this is just a proof of concept
+			// TODO: introduce random latency and stalls
+			return chaosclient.NewChaosRoundTripper(rt, chaosclient.LogChaos, seed.P(s.ChaosChance, chaosclient.ErrSimulatedConnectionResetByPeer))
+		}
+	}
+}
+
 // SimpleRunKubelet is a simple way to start a Kubelet talking to dockerEndpoint, using an API Client.
 // Under the hood it calls RunKubelet (below)
 func SimpleKubelet(client *client.Client,

hack/local-up-cluster.sh

@@ -45,6 +45,7 @@ API_HOST=${API_HOST:-127.0.0.1}
 API_CORS_ALLOWED_ORIGINS=${API_CORS_ALLOWED_ORIGINS:-"/127.0.0.1(:[0-9]+)?$,/localhost(:[0-9]+)?$"}
 KUBELET_PORT=${KUBELET_PORT:-10250}
 LOG_LEVEL=${LOG_LEVEL:-3}
+CHAOS_CHANCE=${CHAOS_CHANCE:-0.0}
 
 # For the common local scenario, fail fast if server is already running.
 # this can happen if you run local-up-cluster.sh twice and kill etcd in between.
@@ -139,6 +140,7 @@ CTLRMGR_PID=$!
 KUBELET_LOG=/tmp/kubelet.log
 sudo -E "${GO_OUT}/kubelet" \
   --v=${LOG_LEVEL} \
+  --chaos_chance="${CHAOS_CHANCE}" \
   --hostname_override="127.0.0.1" \
   --address="127.0.0.1" \
   --api_servers="${API_HOST}:${API_PORT}" \

pkg/kubectl/cmd/update.go

@@ -114,7 +114,6 @@ func RunUpdate(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []str
 		fmt.Fprintf(out, "%s/%s\n", info.Mapping.Resource, info.Name)
 		return nil
 	})
-
 }
 
 func updateWithPatch(cmd *cobra.Command, args []string, f *cmdutil.Factory, patch string) (string, error) {