Add liveness/readiness probe parameters

- PeriodSeconds - How often to probe
- SuccessThreshold - Number of successful probes to go from failure to success state
- FailureThreshold - Number of failing probes to go from success to failure state

This commit includes to changes in behavior:

1. InitialDelaySeconds now defaults to 10 seconds, rather than the
kubelet sync interval (although that also defaults to 10 seconds).
2. Prober only retries on probe error, not failure. To compensate, the
default FailureThreshold is set to the maxRetries, 3.

api/swagger-spec/v1.json

@@ -13524,7 +13524,7 @@
    },
    "v1.Probe": {
     "id": "v1.Probe",
-    "description": "Probe describes a liveness probe to be examined to the container.",
+    "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.",
     "properties": {
      "exec": {
       "$ref": "v1.ExecAction",
@@ -13546,7 +13546,22 @@
      "timeoutSeconds": {
       "type": "integer",
       "format": "int64",
-      "description": "Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes"
+      "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes"
+     },
+     "periodSeconds": {
+      "type": "integer",
+      "format": "int64",
+      "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1."
+     },
+     "successThreshold": {
+      "type": "integer",
+      "format": "int32",
+      "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1."
+     },
+     "failureThreshold": {
+      "type": "integer",
+      "format": "int32",
+      "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1."
      }
     }
    },

api/swagger-spec/v1beta1.json

@@ -3768,7 +3768,7 @@
    },
    "v1.Probe": {
     "id": "v1.Probe",
-    "description": "Probe describes a liveness probe to be examined to the container.",
+    "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.",
     "properties": {
      "exec": {
       "$ref": "v1.ExecAction",
@@ -3790,7 +3790,22 @@
      "timeoutSeconds": {
       "type": "integer",
       "format": "int64",
-      "description": "Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes"
+      "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes"
+     },
+     "periodSeconds": {
+      "type": "integer",
+      "format": "int64",
+      "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1."
+     },
+     "successThreshold": {
+      "type": "integer",
+      "format": "int32",
+      "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1."
+     },
+     "failureThreshold": {
+      "type": "integer",
+      "format": "int32",
+      "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1."
      }
     }
    },

docs/api-reference/extensions/v1beta1/definitions.html

@@ -3257,7 +3257,7 @@ Populated by the system when a graceful deletion is requested. Read-only. More i
 <div class="sect2">
 <h3 id="_v1_probe">v1.Probe</h3>
 <div class="paragraph">
-<p>Probe describes a liveness probe to be examined to the container.</p>
+<p>Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.</p>
 </div>
 <table class="tableblock frame-all grid-all" style="width:100%; ">
 <colgroup>
@@ -3307,11 +3307,32 @@ Populated by the system when a graceful deletion is requested. Read-only. More i
 </tr>
 <tr>
 <td class="tableblock halign-left valign-top"><p class="tableblock">timeoutSeconds</p></td>
-<td class="tableblock halign-left valign-top"><p class="tableblock">Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: <a href="http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes">http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes</a></p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: <a href="http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes">http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes</a></p></td>
 <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
 <td class="tableblock halign-left valign-top"><p class="tableblock">integer (int64)</p></td>
 <td class="tableblock halign-left valign-top"></td>
 </tr>
+<tr>
+<td class="tableblock halign-left valign-top"><p class="tableblock">periodSeconds</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int64)</p></td>
+<td class="tableblock halign-left valign-top"></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top"><p class="tableblock">successThreshold</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int32)</p></td>
+<td class="tableblock halign-left valign-top"></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top"><p class="tableblock">failureThreshold</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int32)</p></td>
+<td class="tableblock halign-left valign-top"></td>
+</tr>
 </tbody>
 </table>
 
@@ -4240,7 +4261,7 @@ Populated by the system when a graceful deletion is requested. Read-only. More i
 </div>
 <div id="footer">
 <div id="footer-text">
-Last updated 2015-11-04 22:53:25 UTC
+Last updated 2015-11-06 18:46:07 UTC
 </div>
 </div>
 </body>

docs/api-reference/extensions/v1beta1/operations.html

@@ -5095,7 +5095,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
 </div>
 <div id="footer">
 <div id="footer-text">
-Last updated 2015-11-04 22:53:25 UTC
+Last updated 2015-11-06 18:46:07 UTC
 </div>
 </div>
 </body>

docs/api-reference/v1/definitions.html

@@ -3023,7 +3023,7 @@ The resulting set of endpoints can be viewed as:<br>
 <div class="sect2">
 <h3 id="_v1_probe">v1.Probe</h3>
 <div class="paragraph">
-<p>Probe describes a liveness probe to be examined to the container.</p>
+<p>Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.</p>
 </div>
 <table class="tableblock frame-all grid-all" style="width:100%; ">
 <colgroup>
@@ -3073,11 +3073,32 @@ The resulting set of endpoints can be viewed as:<br>
 </tr>
 <tr>
 <td class="tableblock halign-left valign-top"><p class="tableblock">timeoutSeconds</p></td>
-<td class="tableblock halign-left valign-top"><p class="tableblock">Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: <a href="http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes">http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes</a></p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: <a href="http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes">http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes</a></p></td>
 <td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
 <td class="tableblock halign-left valign-top"><p class="tableblock">integer (int64)</p></td>
 <td class="tableblock halign-left valign-top"></td>
 </tr>
+<tr>
+<td class="tableblock halign-left valign-top"><p class="tableblock">periodSeconds</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int64)</p></td>
+<td class="tableblock halign-left valign-top"></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top"><p class="tableblock">successThreshold</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int32)</p></td>
+<td class="tableblock halign-left valign-top"></td>
+</tr>
+<tr>
+<td class="tableblock halign-left valign-top"><p class="tableblock">failureThreshold</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
+<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int32)</p></td>
+<td class="tableblock halign-left valign-top"></td>
+</tr>
 </tbody>
 </table>
 
@@ -6866,7 +6887,7 @@ The resulting set of endpoints can be viewed as:<br>
 </div>
 <div id="footer">
 <div id="footer-text">
-Last updated 2015-11-04 22:53:19 UTC
+Last updated 2015-11-06 18:46:00 UTC
 </div>
 </div>
 </body>

docs/api-reference/v1/operations.html

@@ -23664,7 +23664,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
 </div>
 <div id="footer">
 <div id="footer-text">
-Last updated 2015-11-04 22:53:19 UTC
+Last updated 2015-11-06 18:46:00 UTC
 </div>
 </div>
 </body>

pkg/api/deep_copy_generated.go

@@ -1694,6 +1694,9 @@ func deepCopy_api_Probe(in Probe, out *Probe, c *conversion.Cloner) error {
 	}
 	out.InitialDelaySeconds = in.InitialDelaySeconds
 	out.TimeoutSeconds = in.TimeoutSeconds
+	out.PeriodSeconds = in.PeriodSeconds
+	out.SuccessThreshold = in.SuccessThreshold
+	out.FailureThreshold = in.FailureThreshold
 	return nil
 }
 

pkg/api/testing/fuzzer.go

@@ -283,6 +283,18 @@ func FuzzerFor(t *testing.T, version string, src rand.Source) *fuzz.Fuzzer {
 			c.FuzzNoCustom(ct)                                          // fuzz self without calling this function again
 			ct.TerminationMessagePath = "/" + ct.TerminationMessagePath // Must be non-empty
 		},
+		func(p *api.Probe, c fuzz.Continue) {
+			c.FuzzNoCustom(p)
+			// These fields have default values.
+			intFieldsWithDefaults := [...]string{"TimeoutSeconds", "PeriodSeconds", "SuccessThreshold", "FailureThreshold"}
+			v := reflect.ValueOf(p).Elem()
+			for _, field := range intFieldsWithDefaults {
+				f := v.FieldByName(field)
+				if f.Int() == 0 {
+					f.SetInt(1)
+				}
+			}
+		},
 		func(ev *api.EnvVar, c fuzz.Continue) {
 			ev.Name = c.RandString()
 			if c.RandBool() {

pkg/api/types.go

@@ -713,7 +713,8 @@ type ExecAction struct {
 	Command []string `json:"command,omitempty"`
 }
 
-// Probe describes a liveness probe to be examined to the container.
+// Probe describes a health check to be performed against a container to determine whether it is
+// alive or ready to recieve traffic.
 type Probe struct {
 	// The action taken to determine the health of a container
 	Handler `json:",inline"`
@@ -721,6 +722,13 @@ type Probe struct {
 	InitialDelaySeconds int64 `json:"initialDelaySeconds,omitempty"`
 	// Length of time before health checking times out.  In seconds.
 	TimeoutSeconds int64 `json:"timeoutSeconds,omitempty"`
+	// How often (in seconds) to perform the probe.
+	PeriodSeconds int64 `json:"periodSeconds,omitempty"`
+	// Minimum consecutive successes for the probe to be considered successful after having failed.
+	// Must be 1 for liveness.
+	SuccessThreshold int `json:"successThreshold,omitempty"`
+	// Minimum consecutive failures for the probe to be considered failed after having succeeded.
+	FailureThreshold int `json:"failureThreshold,omitempty"`
 }
 
 // PullPolicy describes a policy for if/when to pull a container image

pkg/api/v1/conversion_generated.go

@@ -2243,6 +2243,9 @@ func autoconvert_api_Probe_To_v1_Probe(in *api.Probe, out *Probe, s conversion.S
 	}
 	out.InitialDelaySeconds = in.InitialDelaySeconds
 	out.TimeoutSeconds = in.TimeoutSeconds
+	out.PeriodSeconds = in.PeriodSeconds
+	out.SuccessThreshold = in.SuccessThreshold
+	out.FailureThreshold = in.FailureThreshold
 	return nil
 }
 
@@ -5268,6 +5271,9 @@ func autoconvert_v1_Probe_To_api_Probe(in *Probe, out *api.Probe, s conversion.S
 	}
 	out.InitialDelaySeconds = in.InitialDelaySeconds
 	out.TimeoutSeconds = in.TimeoutSeconds
+	out.PeriodSeconds = in.PeriodSeconds
+	out.SuccessThreshold = in.SuccessThreshold
+	out.FailureThreshold = in.FailureThreshold
 	return nil
 }
 

pkg/api/v1/deep_copy_generated.go

@@ -1713,6 +1713,9 @@ func deepCopy_v1_Probe(in Probe, out *Probe, c *conversion.Cloner) error {
 	}
 	out.InitialDelaySeconds = in.InitialDelaySeconds
 	out.TimeoutSeconds = in.TimeoutSeconds
+	out.PeriodSeconds = in.PeriodSeconds
+	out.SuccessThreshold = in.SuccessThreshold
+	out.FailureThreshold = in.FailureThreshold
 	return nil
 }
 

pkg/api/v1/defaults.go

@@ -128,6 +128,15 @@ func addDefaultingFuncs() {
 			if obj.TimeoutSeconds == 0 {
 				obj.TimeoutSeconds = 1
 			}
+			if obj.PeriodSeconds == 0 {
+				obj.PeriodSeconds = 10
+			}
+			if obj.SuccessThreshold == 0 {
+				obj.SuccessThreshold = 1
+			}
+			if obj.FailureThreshold == 0 {
+				obj.FailureThreshold = 3
+			}
 		},
 		func(obj *Secret) {
 			if obj.Type == "" {

pkg/api/v1/defaults_test.go

@@ -545,3 +545,26 @@ func TestSetDefaultLimitRangeItem(t *testing.T) {
 		t.Errorf("Expected request memory: %s, got: %s", "100Mi", requestMinValue.String())
 	}
 }
+
+func TestSetDefaultProbe(t *testing.T) {
+	originalProbe := versioned.Probe{}
+	expectedProbe := versioned.Probe{
+		InitialDelaySeconds: 0,
+		TimeoutSeconds:      1,
+		PeriodSeconds:       10,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+
+	pod := &versioned.Pod{
+		Spec: versioned.PodSpec{
+			Containers: []versioned.Container{{LivenessProbe: &originalProbe}},
+		},
+	}
+
+	output := roundTrip(t, runtime.Object(pod)).(*versioned.Pod)
+	actualProbe := *output.Spec.Containers[0].LivenessProbe
+	if actualProbe != expectedProbe {
+		t.Errorf("Expected probe: %+v\ngot: %+v\n", expectedProbe, actualProbe)
+	}
+}

pkg/api/v1/types.go

@@ -846,17 +846,27 @@ type ExecAction struct {
 	Command []string `json:"command,omitempty"`
 }
 
-// Probe describes a liveness probe to be examined to the container.
+// Probe describes a health check to be performed against a container to determine whether it is
+// alive or ready to recieve traffic.
 type Probe struct {
 	// The action taken to determine the health of a container
 	Handler `json:",inline"`
 	// Number of seconds after the container has started before liveness probes are initiated.
 	// More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes
 	InitialDelaySeconds int64 `json:"initialDelaySeconds,omitempty"`
-	// Number of seconds after which liveness probes timeout.
+	// Number of seconds after which the probe times out.
-	// Defaults to 1 second.
+	// Defaults to 1 second. Minimum value is 1.
 	// More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes
 	TimeoutSeconds int64 `json:"timeoutSeconds,omitempty"`
+	// How often (in seconds) to perform the probe.
+	// Default to 10 seconds. Minimum value is 1.
+	PeriodSeconds int64 `json:"periodSeconds,omitempty"`
+	// Minimum consecutive successes for the probe to be considered successful after having failed.
+	// Defaults to 1. Must be 1 for liveness. Minimum value is 1.
+	SuccessThreshold int `json:"successThreshold,omitempty"`
+	// Minimum consecutive failures for the probe to be considered failed after having succeeded.
+	// Defaults to 3. Minimum value is 1.
+	FailureThreshold int `json:"failureThreshold,omitempty"`
 }
 
 // PullPolicy describes a policy for if/when to pull a container image

pkg/api/v1/types_swagger_doc_generated.go

@@ -1068,9 +1068,12 @@ func (PodTemplateSpec) SwaggerDoc() map[string]string {
 }
 
 var map_Probe = map[string]string{
-	"": "Probe describes a liveness probe to be examined to the container.",
+	"": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.",
 	"initialDelaySeconds": "Number of seconds after the container has started before liveness probes are initiated. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes",
-	"timeoutSeconds":      "Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes",
+	"timeoutSeconds":      "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes",
+	"periodSeconds":       "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.",
+	"successThreshold":    "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.",
+	"failureThreshold":    "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.",
 }
 
 func (Probe) SwaggerDoc() map[string]string {

pkg/api/validation/validation.go

@@ -881,12 +881,11 @@ func validateProbe(probe *api.Probe) errs.ValidationErrorList {
 		return allErrs
 	}
 	allErrs = append(allErrs, validateHandler(&probe.Handler)...)
-	if probe.InitialDelaySeconds < 0 {
+	allErrs = append(allErrs, ValidatePositiveField(probe.InitialDelaySeconds, "initialDelaySeconds")...)
-		allErrs = append(allErrs, errs.NewFieldInvalid("initialDelay", probe.InitialDelaySeconds, "may not be less than zero"))
+	allErrs = append(allErrs, ValidatePositiveField(probe.TimeoutSeconds, "timeoutSeconds")...)
-	}
+	allErrs = append(allErrs, ValidatePositiveField(int64(probe.PeriodSeconds), "periodSeconds")...)
-	if probe.TimeoutSeconds < 0 {
+	allErrs = append(allErrs, ValidatePositiveField(int64(probe.SuccessThreshold), "successThreshold")...)
-		allErrs = append(allErrs, errs.NewFieldInvalid("timeout", probe.TimeoutSeconds, "may not be less than zero"))
+	allErrs = append(allErrs, ValidatePositiveField(int64(probe.FailureThreshold), "failureThreshold")...)
-	}
 	return allErrs
 }
 
@@ -1030,6 +1029,11 @@ func validateContainers(containers []api.Container, volumes sets.String) errs.Va
 			cErrs = append(cErrs, validateLifecycle(ctr.Lifecycle).Prefix("lifecycle")...)
 		}
 		cErrs = append(cErrs, validateProbe(ctr.LivenessProbe).Prefix("livenessProbe")...)
+		// Liveness-specific validation
+		if ctr.LivenessProbe != nil && ctr.LivenessProbe.SuccessThreshold != 1 {
+			allErrs = append(allErrs, errs.NewFieldForbidden("livenessProbe.successThreshold", "must be 1"))
+		}
+
 		cErrs = append(cErrs, validateProbe(ctr.ReadinessProbe).Prefix("readinessProbe")...)
 		cErrs = append(cErrs, validatePorts(ctr.Ports).Prefix("ports")...)
 		cErrs = append(cErrs, validateEnv(ctr.Env).Prefix("env")...)

pkg/api/validation/validation_test.go

@@ -18,6 +18,7 @@ package validation
 
 import (
 	"math/rand"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -807,22 +808,26 @@ func TestValidateVolumeMounts(t *testing.T) {
 
 func TestValidateProbe(t *testing.T) {
 	handler := api.Handler{Exec: &api.ExecAction{Command: []string{"echo"}}}
-	successCases := []*api.Probe{
+	// These fields must be positive.
-		nil,
+	positiveFields := [...]string{"InitialDelaySeconds", "TimeoutSeconds", "PeriodSeconds", "SuccessThreshold", "FailureThreshold"}
-		{TimeoutSeconds: 10, InitialDelaySeconds: 0, Handler: handler},
+	successCases := []*api.Probe{nil}
-		{TimeoutSeconds: 0, InitialDelaySeconds: 10, Handler: handler},
+	for _, field := range positiveFields {
+		probe := &api.Probe{Handler: handler}
+		reflect.ValueOf(probe).Elem().FieldByName(field).SetInt(10)
+		successCases = append(successCases, probe)
 	}
+
 	for _, p := range successCases {
 		if errs := validateProbe(p); len(errs) != 0 {
 			t.Errorf("expected success: %v", errs)
 		}
 	}
 
-	errorCases := []*api.Probe{
+	errorCases := []*api.Probe{{TimeoutSeconds: 10, InitialDelaySeconds: 10}}
-		{TimeoutSeconds: 10, InitialDelaySeconds: 10},
+	for _, field := range positiveFields {
-		{TimeoutSeconds: 10, InitialDelaySeconds: -10, Handler: handler},
+		probe := &api.Probe{Handler: handler}
-		{TimeoutSeconds: -10, InitialDelaySeconds: 10, Handler: handler},
+		reflect.ValueOf(probe).Elem().FieldByName(field).SetInt(-10)
-		{TimeoutSeconds: -10, InitialDelaySeconds: -10, Handler: handler},
+		errorCases = append(errorCases, probe)
 	}
 	for _, p := range errorCases {
 		if errs := validateProbe(p); len(errs) == 0 {

pkg/apis/extensions/deep_copy_generated.go

@@ -583,6 +583,9 @@ func deepCopy_api_Probe(in api.Probe, out *api.Probe, c *conversion.Cloner) erro
 	}
 	out.InitialDelaySeconds = in.InitialDelaySeconds
 	out.TimeoutSeconds = in.TimeoutSeconds
+	out.PeriodSeconds = in.PeriodSeconds
+	out.SuccessThreshold = in.SuccessThreshold
+	out.FailureThreshold = in.FailureThreshold
 	return nil
 }
 

pkg/apis/extensions/v1beta1/conversion_generated.go

@@ -755,6 +755,9 @@ func autoconvert_api_Probe_To_v1_Probe(in *api.Probe, out *v1.Probe, s conversio
 	}
 	out.InitialDelaySeconds = in.InitialDelaySeconds
 	out.TimeoutSeconds = in.TimeoutSeconds
+	out.PeriodSeconds = in.PeriodSeconds
+	out.SuccessThreshold = in.SuccessThreshold
+	out.FailureThreshold = in.FailureThreshold
 	return nil
 }
 
@@ -1815,6 +1818,9 @@ func autoconvert_v1_Probe_To_api_Probe(in *v1.Probe, out *api.Probe, s conversio
 	}
 	out.InitialDelaySeconds = in.InitialDelaySeconds
 	out.TimeoutSeconds = in.TimeoutSeconds
+	out.PeriodSeconds = in.PeriodSeconds
+	out.SuccessThreshold = in.SuccessThreshold
+	out.FailureThreshold = in.FailureThreshold
 	return nil
 }
 

pkg/apis/extensions/v1beta1/deep_copy_generated.go

@@ -620,6 +620,9 @@ func deepCopy_v1_Probe(in v1.Probe, out *v1.Probe, c *conversion.Cloner) error {
 	}
 	out.InitialDelaySeconds = in.InitialDelaySeconds
 	out.TimeoutSeconds = in.TimeoutSeconds
+	out.PeriodSeconds = in.PeriodSeconds
+	out.SuccessThreshold = in.SuccessThreshold
+	out.FailureThreshold = in.FailureThreshold
 	return nil
 }
 

pkg/kubelet/kubelet.go

@@ -418,7 +418,6 @@ func NewMainKubelet(
 	klet.statusManager = status.NewManager(kubeClient, klet.podManager)
 
 	klet.probeManager = prober.NewManager(
-		klet.resyncInterval,
 		klet.statusManager,
 		readinessManager,
 		klet.livenessManager,

pkg/kubelet/prober/manager.go

@@ -18,7 +18,6 @@ package prober
 
 import (
 	"sync"
-	"time"
 
 	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/api"
@@ -71,13 +70,9 @@ type manager struct {
 
 	// prober executes the probe actions.
 	prober *prober
-
-	// Default period for workers to execute a probe.
-	defaultProbePeriod time.Duration
 }
 
 func NewManager(
-	defaultProbePeriod time.Duration,
 	statusManager status.Manager,
 	readinessManager results.Manager,
 	livenessManager results.Manager,
@@ -86,12 +81,11 @@ func NewManager(
 	recorder record.EventRecorder) Manager {
 	prober := newProber(runner, refManager, recorder)
 	return &manager{
-		defaultProbePeriod: defaultProbePeriod,
+		statusManager:    statusManager,
-		statusManager:      statusManager,
+		prober:           prober,
-		prober:             prober,
+		readinessManager: readinessManager,
-		readinessManager:   readinessManager,
+		livenessManager:  livenessManager,
-		livenessManager:    livenessManager,
+		workers:          make(map[probeKey]*worker),
-		workers:            make(map[probeKey]*worker),
 	}
 }
 

pkg/kubelet/prober/manager_test.go

@@ -34,6 +34,16 @@ import (
 	"k8s.io/kubernetes/pkg/util/wait"
 )
 
+var defaultProbe *api.Probe = &api.Probe{
+	Handler: api.Handler{
+		Exec: &api.ExecAction{},
+	},
+	TimeoutSeconds:   1,
+	PeriodSeconds:    1,
+	SuccessThreshold: 1,
+	FailureThreshold: 3,
+}
+
 func TestAddRemovePods(t *testing.T) {
 	noProbePod := api.Pod{
 		ObjectMeta: api.ObjectMeta{
@@ -57,12 +67,12 @@ func TestAddRemovePods(t *testing.T) {
 				Name: "no_probe1",
 			}, {
 				Name:           "readiness",
-				ReadinessProbe: &api.Probe{},
+				ReadinessProbe: defaultProbe,
 			}, {
 				Name: "no_probe2",
 			}, {
 				Name:          "liveness",
-				LivenessProbe: &api.Probe{},
+				LivenessProbe: defaultProbe,
 			}},
 		},
 	}
@@ -119,10 +129,10 @@ func TestCleanupPods(t *testing.T) {
 		Spec: api.PodSpec{
 			Containers: []api.Container{{
 				Name:           "prober1",
-				ReadinessProbe: &api.Probe{},
+				ReadinessProbe: defaultProbe,
 			}, {
 				Name:          "prober2",
-				LivenessProbe: &api.Probe{},
+				LivenessProbe: defaultProbe,
 			}},
 		},
 	}
@@ -133,10 +143,10 @@ func TestCleanupPods(t *testing.T) {
 		Spec: api.PodSpec{
 			Containers: []api.Container{{
 				Name:           "prober1",
-				ReadinessProbe: &api.Probe{},
+				ReadinessProbe: defaultProbe,
 			}, {
 				Name:          "prober2",
-				LivenessProbe: &api.Probe{},
+				LivenessProbe: defaultProbe,
 			}},
 		},
 	}
@@ -266,9 +276,7 @@ outer:
 }
 
 func newTestManager() *manager {
-	const probePeriod = 1
 	m := NewManager(
-		probePeriod,
 		status.NewManager(&testclient.Fake{}, kubepod.NewBasicPodManager(nil)),
 		results.NewManager(),
 		results.NewManager(),

pkg/kubelet/prober/prober.go

@@ -118,8 +118,8 @@ func (pb *prober) runProbeWithRetries(p *api.Probe, pod *api.Pod, status api.Pod
 	var output string
 	for i := 0; i < retries; i++ {
 		result, output, err = pb.runProbe(p, pod, status, container, containerID)
-		if result == probe.Success {
+		if err == nil {
-			return probe.Success, output, nil
+			return result, output, nil
 		}
 	}
 	return result, output, err

pkg/kubelet/prober/worker.go

@@ -56,6 +56,10 @@ type worker struct {
 
 	// The last known container ID for this worker.
 	containerID kubecontainer.ContainerID
+	// The last probe result for this worker.
+	lastResult results.Result
+	// How many times in a row the probe has returned the same result.
+	resultRun int
 }
 
 // Creates and starts a new probe worker.
@@ -89,7 +93,7 @@ func newWorker(
 
 // run periodically probes the container.
 func (w *worker) run() {
-	probeTicker := time.NewTicker(w.probeManager.defaultProbePeriod)
+	probeTicker := time.NewTicker(time.Duration(w.spec.PeriodSeconds) * time.Second)
 
 	defer func() {
 		// Clean up.
@@ -145,6 +149,7 @@ func (w *worker) doProbe() (keepGoing bool) {
 			w.resultsManager.Remove(w.containerID)
 		}
 		w.containerID = kubecontainer.ParseContainerID(c.ContainerID)
+		w.resultsManager.Set(w.containerID, w.initialValue, w.pod)
 	}
 
 	if c.State.Running == nil {
@@ -159,14 +164,29 @@ func (w *worker) doProbe() (keepGoing bool) {
 	}
 
 	if int64(time.Since(c.State.Running.StartedAt.Time).Seconds()) < w.spec.InitialDelaySeconds {
-		w.resultsManager.Set(w.containerID, w.initialValue, w.pod)
 		return true
 	}
 
 	result, err := w.probeManager.prober.probe(w.probeType, w.pod, status, w.container, w.containerID)
-	if err == nil {
+	if err != nil {
-		w.resultsManager.Set(w.containerID, result, w.pod)
+		// Prober error, throw away the result.
+		return true
 	}
 
+	if w.lastResult == result {
+		w.resultRun++
+	} else {
+		w.lastResult = result
+		w.resultRun = 1
+	}
+
+	if (result == results.Failure && w.resultRun < w.spec.FailureThreshold) ||
+		(result == results.Success && w.resultRun < w.spec.SuccessThreshold) {
+		// Success or failure is below threshold - leave the probe state unchanged.
+		return true
+	}
+
+	w.resultsManager.Set(w.containerID, result, w.pod)
+
 	return true
 }

pkg/kubelet/prober/worker_test.go

@@ -18,6 +18,7 @@ package prober
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 	"time"
 
@@ -135,18 +136,8 @@ func TestInitialDelay(t *testing.T) {
 		})
 		m.statusManager.SetPodStatus(w.pod, getRunningStatus())
 
-		if !w.doProbe() {
+		expectContinue(t, w, w.doProbe(), "during initial delay")
-			t.Errorf("[%s] Expected to continue, but did not", probeType)
+		expectResult(t, w, results.Result(probeType == liveness), "during initial delay")
-		}
-
-		expectedResult := results.Result(probeType == liveness)
-		result, ok := resultsManager(m, probeType).Get(containerID)
-		if !ok {
-			t.Errorf("[%s] Expected result to be set during initial delay, but was not set", probeType)
-		} else if result != expectedResult {
-			t.Errorf("[%s] Expected result to be %v during initial delay, but was %v",
-				probeType, expectedResult, result)
-		}
 
 		// 100 seconds later...
 		laterStatus := getRunningStatus()
@@ -155,16 +146,76 @@ func TestInitialDelay(t *testing.T) {
 		m.statusManager.SetPodStatus(w.pod, laterStatus)
 
 		// Second call should succeed (already waited).
-		if !w.doProbe() {
+		expectContinue(t, w, w.doProbe(), "after initial delay")
-			t.Errorf("[%s] Expected to continue, but did not", probeType)
+		expectResult(t, w, results.Success, "after initial delay")
+	}
+}
+
+func TestFailureThreshold(t *testing.T) {
+	m := newTestManager()
+	w := newTestWorker(m, readiness, api.Probe{})
+	m.statusManager.SetPodStatus(w.pod, getRunningStatus())
+
+	for i := 0; i < 2; i++ {
+		// First probe should succeed.
+		m.prober.exec = fakeExecProber{probe.Success, nil}
+
+		for j := 0; j < 3; j++ {
+			msg := fmt.Sprintf("%d success (%d)", j+1, i)
+			expectContinue(t, w, w.doProbe(), msg)
+			expectResult(t, w, results.Success, msg)
 		}
 
-		result, ok = resultsManager(m, probeType).Get(containerID)
+		// Prober starts failing :(
-		if !ok {
+		m.prober.exec = fakeExecProber{probe.Failure, nil}
-			t.Errorf("[%s] Expected result to be true, but was not set", probeType)
+
-		} else if !result {
+		// Next 2 probes should still be "success".
-			t.Errorf("[%s] Expected result to be true, but was false", probeType)
+		for j := 0; j < 2; j++ {
+			msg := fmt.Sprintf("%d failure (%d)", j+1, i)
+			expectContinue(t, w, w.doProbe(), msg)
+			expectResult(t, w, results.Success, msg)
 		}
+
+		// Third & following fail.
+		for j := 0; j < 3; j++ {
+			msg := fmt.Sprintf("%d failure (%d)", j+3, i)
+			expectContinue(t, w, w.doProbe(), msg)
+			expectResult(t, w, results.Failure, msg)
+		}
+	}
+}
+
+func TestSuccessThreshold(t *testing.T) {
+	m := newTestManager()
+	w := newTestWorker(m, readiness, api.Probe{SuccessThreshold: 3, FailureThreshold: 1})
+	m.statusManager.SetPodStatus(w.pod, getRunningStatus())
+
+	// Start out failure.
+	w.resultsManager.Set(containerID, results.Failure, nil)
+
+	for i := 0; i < 2; i++ {
+		// Probe defaults to Failure.
+		for j := 0; j < 2; j++ {
+			msg := fmt.Sprintf("%d success (%d)", j+1, i)
+			expectContinue(t, w, w.doProbe(), msg)
+			expectResult(t, w, results.Failure, msg)
+		}
+
+		// Continuing success!
+		for j := 0; j < 3; j++ {
+			msg := fmt.Sprintf("%d success (%d)", j+3, i)
+			expectContinue(t, w, w.doProbe(), msg)
+			expectResult(t, w, results.Success, msg)
+		}
+
+		// Prober flakes :(
+		m.prober.exec = fakeExecProber{probe.Failure, nil}
+		msg := fmt.Sprintf("1 failure (%d)", i)
+		expectContinue(t, w, w.doProbe(), msg)
+		expectResult(t, w, results.Failure, msg)
+
+		// Back to success.
+		m.prober.exec = fakeExecProber{probe.Success, nil}
 	}
 }
 
@@ -205,22 +256,22 @@ func TestCleanUp(t *testing.T) {
 
 func TestHandleCrash(t *testing.T) {
 	m := newTestManager()
+	w := newTestWorker(m, readiness, api.Probe{})
+	m.statusManager.SetPodStatus(w.pod, getRunningStatus())
+
+	expectContinue(t, w, w.doProbe(), "Initial successful probe.")
+	expectResult(t, w, results.Success, "Initial successful probe.")
+
+	// Prober starts crashing.
 	m.prober = &prober{
 		refManager: kubecontainer.NewRefManager(),
 		recorder:   &record.FakeRecorder{},
 		exec:       crashingExecProber{},
 	}
 
-	w := newTestWorker(m, readiness, api.Probe{})
-	m.statusManager.SetPodStatus(w.pod, getRunningStatus())
-
 	// doProbe should recover from the crash, and keep going.
-	if !w.doProbe() {
+	expectContinue(t, w, w.doProbe(), "Crashing probe.")
-		t.Error("Expected to keep going, but terminated.")
+	expectResult(t, w, results.Success, "Crashing probe unchanged.")
-	}
-	if _, ok := m.readinessManager.Get(containerID); ok {
-		t.Error("Expected readiness to be unchanged from crash.")
-	}
 }
 
 func newTestWorker(m *manager, probeType probeType, probeSpec api.Probe) *worker {
@@ -228,6 +279,19 @@ func newTestWorker(m *manager, probeType probeType, probeSpec api.Probe) *worker
 	probeSpec.Handler = api.Handler{
 		Exec: &api.ExecAction{},
 	}
+	// Apply default values.
+	defaults := map[string]int64{
+		"TimeoutSeconds":   1,
+		"PeriodSeconds":    10,
+		"SuccessThreshold": 1,
+		"FailureThreshold": 3,
+	}
+	for field, value := range defaults {
+		f := reflect.ValueOf(&probeSpec).Elem().FieldByName(field)
+		if f.Int() == 0 {
+			f.SetInt(value)
+		}
+	}
 
 	pod := getTestPod(probeType, probeSpec)
 	return newWorker(m, probeType, &pod, pod.Spec.Containers[0])
@@ -266,6 +330,22 @@ func getTestPod(probeType probeType, probeSpec api.Probe) api.Pod {
 	return pod
 }
 
+func expectResult(t *testing.T, w *worker, expectedResult results.Result, msg string) {
+	result, ok := resultsManager(w.probeManager, w.probeType).Get(containerID)
+	if !ok {
+		t.Errorf("[%s - %s] Expected result to be set, but was not set", w.probeType, msg)
+	} else if result != expectedResult {
+		t.Errorf("[%s - %s] Expected result to be %v, but was %v",
+			w.probeType, msg, expectedResult, result)
+	}
+}
+
+func expectContinue(t *testing.T, w *worker, c bool, msg string) {
+	if !c {
+		t.Errorf("[%s - %s] Expected to continue, but did not", w.probeType, msg)
+	}
+}
+
 func resultsManager(m *manager, probeType probeType) results.Manager {
 	switch probeType {
 	case readiness:

test/e2e/pods.go

@@ -560,6 +560,7 @@ var _ = Describe("Pods", func() {
 								},
 							},
 							InitialDelaySeconds: 15,
+							FailureThreshold:    1,
 						},
 					},
 				},
@@ -586,6 +587,7 @@ var _ = Describe("Pods", func() {
 								},
 							},
 							InitialDelaySeconds: 15,
+							FailureThreshold:    1,
 						},
 					},
 				},
@@ -613,6 +615,7 @@ var _ = Describe("Pods", func() {
 								},
 							},
 							InitialDelaySeconds: 15,
+							FailureThreshold:    1,
 						},
 					},
 				},
@@ -640,6 +643,7 @@ var _ = Describe("Pods", func() {
 								},
 							},
 							InitialDelaySeconds: 5,
+							FailureThreshold:    1,
 						},
 					},
 				},
@@ -673,6 +677,7 @@ var _ = Describe("Pods", func() {
 								},
 							},
 							InitialDelaySeconds: 15,
+							FailureThreshold:    1,
 						},
 					},
 				},