github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add liveness/readiness probe parameters 

- PeriodSeconds - How often to probe
- SuccessThreshold - Number of successful probes to go from failure to success state
- FailureThreshold - Number of failing probes to go from success to failure state

This commit includes to changes in behavior:

1. InitialDelaySeconds now defaults to 10 seconds, rather than the
kubelet sync interval (although that also defaults to 10 seconds).
2. Prober only retries on probe error, not failure. To compensate, the
default FailureThreshold is set to the maxRetries, 3.
pull/6/head
Tim St. Clair 2015-11-05 15:38:46 -08:00
parent fbee1b59d0
commit 1e88a682da
29 changed files with 24536 additions and 23996 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
api/swagger-spec/v1.json
View File

@ -13524,7 +13524,7 @@
},
"v1.Probe": {
"id": "v1.Probe",
"description": "Probe describes a liveness probe to be examined to the container.",
"description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.",
"properties": {
"exec": {
"$ref": "v1.ExecAction",
@ -13546,7 +13546,22 @@
"timeoutSeconds": {
"type": "integer",
"format": "int64",
"description": "Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes"
"description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes"
},
"periodSeconds": {
"type": "integer",
"format": "int64",
"description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1."
},
"successThreshold": {
"type": "integer",
"format": "int32",
"description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1."
},
"failureThreshold": {
"type": "integer",
"format": "int32",
"description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1."
}
}
},

19
api/swagger-spec/v1beta1.json
View File

@ -3768,7 +3768,7 @@
},
"v1.Probe": {
"id": "v1.Probe",
"description": "Probe describes a liveness probe to be examined to the container.",
"description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.",
"properties": {
"exec": {
"$ref": "v1.ExecAction",
@ -3790,7 +3790,22 @@
"timeoutSeconds": {
"type": "integer",
"format": "int64",
"description": "Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes"
"description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes"
},
"periodSeconds": {
"type": "integer",
"format": "int64",
"description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1."
},
"successThreshold": {
"type": "integer",
"format": "int32",
"description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1."
},
"failureThreshold": {
"type": "integer",
"format": "int32",
"description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1."
}
}
},

27
docs/api-reference/extensions/v1beta1/definitions.html
View File

@ -3257,7 +3257,7 @@ Populated by the system when a graceful deletion is requested. Read-only. More i
<div class="sect2">
<h3 id="_v1_probe">v1.Probe</h3>
<div class="paragraph">
<p>Probe describes a liveness probe to be examined to the container.</p>
<p>Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.</p>
</div>
<table class="tableblock frame-all grid-all" style="width:100%; ">
<colgroup>
@ -3307,11 +3307,32 @@ Populated by the system when a graceful deletion is requested. Read-only. More i
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">timeoutSeconds</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: <a href="http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes">http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes</a></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: <a href="http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes">http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes</a></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int64)</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">periodSeconds</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int64)</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">successThreshold</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int32)</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">failureThreshold</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int32)</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
</tbody>
</table>
@ -4240,7 +4261,7 @@ Populated by the system when a graceful deletion is requested. Read-only. More i
</div>
<div id="footer">
<div id="footer-text">
Last updated 2015-11-04 22:53:25 UTC
Last updated 2015-11-06 18:46:07 UTC
</div>
</div>
</body>

2
docs/api-reference/extensions/v1beta1/operations.html
View File

@ -5095,7 +5095,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
</div>
<div id="footer">
<div id="footer-text">
Last updated 2015-11-04 22:53:25 UTC
Last updated 2015-11-06 18:46:07 UTC
</div>
</div>
</body>

27
docs/api-reference/v1/definitions.html
View File

@ -3023,7 +3023,7 @@ The resulting set of endpoints can be viewed as:<br>
<div class="sect2">
<h3 id="_v1_probe">v1.Probe</h3>
<div class="paragraph">
<p>Probe describes a liveness probe to be examined to the container.</p>
<p>Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.</p>
</div>
<table class="tableblock frame-all grid-all" style="width:100%; ">
<colgroup>
@ -3073,11 +3073,32 @@ The resulting set of endpoints can be viewed as:<br>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">timeoutSeconds</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: <a href="http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes">http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes</a></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: <a href="http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes">http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes</a></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int64)</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">periodSeconds</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int64)</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">successThreshold</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int32)</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">failureThreshold</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">integer (int32)</p></td>
<td class="tableblock halign-left valign-top"></td>
</tr>
</tbody>
</table>
@ -6866,7 +6887,7 @@ The resulting set of endpoints can be viewed as:<br>
</div>
<div id="footer">
<div id="footer-text">
Last updated 2015-11-04 22:53:19 UTC
Last updated 2015-11-06 18:46:00 UTC
</div>
</div>
</body>

2
docs/api-reference/v1/operations.html
View File

@ -23664,7 +23664,7 @@ span.icon > [class^="icon-"], span.icon > [class*=" icon-"] { cursor: default; }
</div>
<div id="footer">
<div id="footer-text">
Last updated 2015-11-04 22:53:19 UTC
Last updated 2015-11-06 18:46:00 UTC
</div>
</div>
</body>

3
pkg/api/deep_copy_generated.go
View File

@ -1694,6 +1694,9 @@ func deepCopy_api_Probe(in Probe, out *Probe, c *conversion.Cloner) error {
}
out.InitialDelaySeconds = in.InitialDelaySeconds
out.TimeoutSeconds = in.TimeoutSeconds
out.PeriodSeconds = in.PeriodSeconds
out.SuccessThreshold = in.SuccessThreshold
out.FailureThreshold = in.FailureThreshold
return nil
}

12
pkg/api/testing/fuzzer.go
View File

@ -283,6 +283,18 @@ func FuzzerFor(t *testing.T, version string, src rand.Source) *fuzz.Fuzzer {
c.FuzzNoCustom(ct) // fuzz self without calling this function again
ct.TerminationMessagePath = "/" + ct.TerminationMessagePath // Must be non-empty
},
func(p *api.Probe, c fuzz.Continue) {
c.FuzzNoCustom(p)
// These fields have default values.
intFieldsWithDefaults := [...]string{"TimeoutSeconds", "PeriodSeconds", "SuccessThreshold", "FailureThreshold"}
v := reflect.ValueOf(p).Elem()
for _, field := range intFieldsWithDefaults {
f := v.FieldByName(field)
if f.Int() == 0 {
f.SetInt(1)
}
}
},
func(ev *api.EnvVar, c fuzz.Continue) {
ev.Name = c.RandString()
if c.RandBool() {

23604
pkg/api/types.generated.go
View File

File diff suppressed because it is too large Load Diff

10
pkg/api/types.go
View File

@ -713,7 +713,8 @@ type ExecAction struct {
Command []string `json:"command,omitempty"`
}
// Probe describes a liveness probe to be examined to the container.
// Probe describes a health check to be performed against a container to determine whether it is
// alive or ready to recieve traffic.
type Probe struct {
// The action taken to determine the health of a container
Handler `json:",inline"`
@ -721,6 +722,13 @@ type Probe struct {
InitialDelaySeconds int64 `json:"initialDelaySeconds,omitempty"`
// Length of time before health checking times out. In seconds.
TimeoutSeconds int64 `json:"timeoutSeconds,omitempty"`
// How often (in seconds) to perform the probe.
PeriodSeconds int64 `json:"periodSeconds,omitempty"`
// Minimum consecutive successes for the probe to be considered successful after having failed.
// Must be 1 for liveness.
SuccessThreshold int `json:"successThreshold,omitempty"`
// Minimum consecutive failures for the probe to be considered failed after having succeeded.
FailureThreshold int `json:"failureThreshold,omitempty"`
}
// PullPolicy describes a policy for if/when to pull a container image

6
pkg/api/v1/conversion_generated.go
View File

@ -2243,6 +2243,9 @@ func autoconvert_api_Probe_To_v1_Probe(in *api.Probe, out *Probe, s conversion.S
}
out.InitialDelaySeconds = in.InitialDelaySeconds
out.TimeoutSeconds = in.TimeoutSeconds
out.PeriodSeconds = in.PeriodSeconds
out.SuccessThreshold = in.SuccessThreshold
out.FailureThreshold = in.FailureThreshold
return nil
}
@ -5268,6 +5271,9 @@ func autoconvert_v1_Probe_To_api_Probe(in *Probe, out *api.Probe, s conversion.S
}
out.InitialDelaySeconds = in.InitialDelaySeconds
out.TimeoutSeconds = in.TimeoutSeconds
out.PeriodSeconds = in.PeriodSeconds
out.SuccessThreshold = in.SuccessThreshold
out.FailureThreshold = in.FailureThreshold
return nil
}

3
pkg/api/v1/deep_copy_generated.go
View File

@ -1713,6 +1713,9 @@ func deepCopy_v1_Probe(in Probe, out *Probe, c *conversion.Cloner) error {
}
out.InitialDelaySeconds = in.InitialDelaySeconds
out.TimeoutSeconds = in.TimeoutSeconds
out.PeriodSeconds = in.PeriodSeconds
out.SuccessThreshold = in.SuccessThreshold
out.FailureThreshold = in.FailureThreshold
return nil
}

9
pkg/api/v1/defaults.go
View File

@ -128,6 +128,15 @@ func addDefaultingFuncs() {
if obj.TimeoutSeconds == 0 {
obj.TimeoutSeconds = 1
}
if obj.PeriodSeconds == 0 {
obj.PeriodSeconds = 10
}
if obj.SuccessThreshold == 0 {
obj.SuccessThreshold = 1
}
if obj.FailureThreshold == 0 {
obj.FailureThreshold = 3
}
},
func(obj *Secret) {
if obj.Type == "" {

23
pkg/api/v1/defaults_test.go
View File

@ -545,3 +545,26 @@ func TestSetDefaultLimitRangeItem(t *testing.T) {
t.Errorf("Expected request memory: %s, got: %s", "100Mi", requestMinValue.String())
}
}
func TestSetDefaultProbe(t *testing.T) {
originalProbe := versioned.Probe{}
expectedProbe := versioned.Probe{
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
pod := &versioned.Pod{
Spec: versioned.PodSpec{
Containers: []versioned.Container{{LivenessProbe: &originalProbe}},
},
}
output := roundTrip(t, runtime.Object(pod)).(*versioned.Pod)
actualProbe := *output.Spec.Containers[0].LivenessProbe
if actualProbe != expectedProbe {
t.Errorf("Expected probe: %+v\ngot: %+v\n", expectedProbe, actualProbe)
}
}

24478
pkg/api/v1/types.generated.go
View File

File diff suppressed because it is too large Load Diff

16
pkg/api/v1/types.go
View File

@ -846,17 +846,27 @@ type ExecAction struct {
Command []string `json:"command,omitempty"`
}
// Probe describes a liveness probe to be examined to the container.
// Probe describes a health check to be performed against a container to determine whether it is
// alive or ready to recieve traffic.
type Probe struct {
// The action taken to determine the health of a container
Handler `json:",inline"`
// Number of seconds after the container has started before liveness probes are initiated.
// More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes
InitialDelaySeconds int64 `json:"initialDelaySeconds,omitempty"`
// Number of seconds after which liveness probes timeout.
// Defaults to 1 second.
// Number of seconds after which the probe times out.
// Defaults to 1 second. Minimum value is 1.
// More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes
TimeoutSeconds int64 `json:"timeoutSeconds,omitempty"`
// How often (in seconds) to perform the probe.
// Default to 10 seconds. Minimum value is 1.
PeriodSeconds int64 `json:"periodSeconds,omitempty"`
// Minimum consecutive successes for the probe to be considered successful after having failed.
// Defaults to 1. Must be 1 for liveness. Minimum value is 1.
SuccessThreshold int `json:"successThreshold,omitempty"`
// Minimum consecutive failures for the probe to be considered failed after having succeeded.
// Defaults to 3. Minimum value is 1.
FailureThreshold int `json:"failureThreshold,omitempty"`
}
// PullPolicy describes a policy for if/when to pull a container image

7
pkg/api/v1/types_swagger_doc_generated.go
View File

@ -1068,9 +1068,12 @@ func (PodTemplateSpec) SwaggerDoc() map[string]string {
}
var map_Probe = map[string]string{
"": "Probe describes a liveness probe to be examined to the container.",
"": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to recieve traffic.",
"initialDelaySeconds": "Number of seconds after the container has started before liveness probes are initiated. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes",
"timeoutSeconds": "Number of seconds after which liveness probes timeout. Defaults to 1 second. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes",
"timeoutSeconds": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes",
"periodSeconds": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.",
"successThreshold": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness. Minimum value is 1.",
"failureThreshold": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.",
}
func (Probe) SwaggerDoc() map[string]string {

16
pkg/api/validation/validation.go
View File

@ -881,12 +881,11 @@ func validateProbe(probe *api.Probe) errs.ValidationErrorList {
return allErrs
}
allErrs = append(allErrs, validateHandler(&probe.Handler)...)
if probe.InitialDelaySeconds < 0 {
allErrs = append(allErrs, errs.NewFieldInvalid("initialDelay", probe.InitialDelaySeconds, "may not be less than zero"))
}
if probe.TimeoutSeconds < 0 {
allErrs = append(allErrs, errs.NewFieldInvalid("timeout", probe.TimeoutSeconds, "may not be less than zero"))
}
allErrs = append(allErrs, ValidatePositiveField(probe.InitialDelaySeconds, "initialDelaySeconds")...)
allErrs = append(allErrs, ValidatePositiveField(probe.TimeoutSeconds, "timeoutSeconds")...)
allErrs = append(allErrs, ValidatePositiveField(int64(probe.PeriodSeconds), "periodSeconds")...)
allErrs = append(allErrs, ValidatePositiveField(int64(probe.SuccessThreshold), "successThreshold")...)
allErrs = append(allErrs, ValidatePositiveField(int64(probe.FailureThreshold), "failureThreshold")...)
return allErrs
}
@ -1030,6 +1029,11 @@ func validateContainers(containers []api.Container, volumes sets.String) errs.Va
cErrs = append(cErrs, validateLifecycle(ctr.Lifecycle).Prefix("lifecycle")...)
}
cErrs = append(cErrs, validateProbe(ctr.LivenessProbe).Prefix("livenessProbe")...)
// Liveness-specific validation
if ctr.LivenessProbe != nil && ctr.LivenessProbe.SuccessThreshold != 1 {
allErrs = append(allErrs, errs.NewFieldForbidden("livenessProbe.successThreshold", "must be 1"))
}
cErrs = append(cErrs, validateProbe(ctr.ReadinessProbe).Prefix("readinessProbe")...)
cErrs = append(cErrs, validatePorts(ctr.Ports).Prefix("ports")...)
cErrs = append(cErrs, validateEnv(ctr.Env).Prefix("env")...)

23
pkg/api/validation/validation_test.go
View File

@ -18,6 +18,7 @@ package validation
import (
"math/rand"
"reflect"
"strings"
"testing"
"time"
@ -807,22 +808,26 @@ func TestValidateVolumeMounts(t *testing.T) {
func TestValidateProbe(t *testing.T) {
handler := api.Handler{Exec: &api.ExecAction{Command: []string{"echo"}}}
successCases := []*api.Probe{
nil,
{TimeoutSeconds: 10, InitialDelaySeconds: 0, Handler: handler},
{TimeoutSeconds: 0, InitialDelaySeconds: 10, Handler: handler},
// These fields must be positive.
positiveFields := [...]string{"InitialDelaySeconds", "TimeoutSeconds", "PeriodSeconds", "SuccessThreshold", "FailureThreshold"}
successCases := []*api.Probe{nil}
for _, field := range positiveFields {
probe := &api.Probe{Handler: handler}
reflect.ValueOf(probe).Elem().FieldByName(field).SetInt(10)
successCases = append(successCases, probe)
}
for _, p := range successCases {
if errs := validateProbe(p); len(errs) != 0 {
t.Errorf("expected success: %v", errs)
}
}
errorCases := []*api.Probe{
{TimeoutSeconds: 10, InitialDelaySeconds: 10},
{TimeoutSeconds: 10, InitialDelaySeconds: -10, Handler: handler},
{TimeoutSeconds: -10, InitialDelaySeconds: 10, Handler: handler},
{TimeoutSeconds: -10, InitialDelaySeconds: -10, Handler: handler},
errorCases := []*api.Probe{{TimeoutSeconds: 10, InitialDelaySeconds: 10}}
for _, field := range positiveFields {
probe := &api.Probe{Handler: handler}
reflect.ValueOf(probe).Elem().FieldByName(field).SetInt(-10)
errorCases = append(errorCases, probe)
}
for _, p := range errorCases {
if errs := validateProbe(p); len(errs) == 0 {

3
pkg/apis/extensions/deep_copy_generated.go
View File

@ -583,6 +583,9 @@ func deepCopy_api_Probe(in api.Probe, out *api.Probe, c *conversion.Cloner) erro
}
out.InitialDelaySeconds = in.InitialDelaySeconds
out.TimeoutSeconds = in.TimeoutSeconds
out.PeriodSeconds = in.PeriodSeconds
out.SuccessThreshold = in.SuccessThreshold
out.FailureThreshold = in.FailureThreshold
return nil
}

6
pkg/apis/extensions/v1beta1/conversion_generated.go
View File

@ -755,6 +755,9 @@ func autoconvert_api_Probe_To_v1_Probe(in *api.Probe, out *v1.Probe, s conversio
}
out.InitialDelaySeconds = in.InitialDelaySeconds
out.TimeoutSeconds = in.TimeoutSeconds
out.PeriodSeconds = in.PeriodSeconds
out.SuccessThreshold = in.SuccessThreshold
out.FailureThreshold = in.FailureThreshold
return nil
}
@ -1815,6 +1818,9 @@ func autoconvert_v1_Probe_To_api_Probe(in *v1.Probe, out *api.Probe, s conversio
}
out.InitialDelaySeconds = in.InitialDelaySeconds
out.TimeoutSeconds = in.TimeoutSeconds
out.PeriodSeconds = in.PeriodSeconds
out.SuccessThreshold = in.SuccessThreshold
out.FailureThreshold = in.FailureThreshold
return nil
}

3
pkg/apis/extensions/v1beta1/deep_copy_generated.go
View File

@ -620,6 +620,9 @@ func deepCopy_v1_Probe(in v1.Probe, out *v1.Probe, c *conversion.Cloner) error {
}
out.InitialDelaySeconds = in.InitialDelaySeconds
out.TimeoutSeconds = in.TimeoutSeconds
out.PeriodSeconds = in.PeriodSeconds
out.SuccessThreshold = in.SuccessThreshold
out.FailureThreshold = in.FailureThreshold
return nil
}

1
pkg/kubelet/kubelet.go
View File

@ -418,7 +418,6 @@ func NewMainKubelet(
klet.statusManager = status.NewManager(kubeClient, klet.podManager)
klet.probeManager = prober.NewManager(
klet.resyncInterval,
klet.statusManager,
readinessManager,
klet.livenessManager,

16
pkg/kubelet/prober/manager.go
View File

@ -18,7 +18,6 @@ package prober
import (
"sync"
"time"
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
@ -71,13 +70,9 @@ type manager struct {
// prober executes the probe actions.
prober *prober
// Default period for workers to execute a probe.
defaultProbePeriod time.Duration
}
func NewManager(
defaultProbePeriod time.Duration,
statusManager status.Manager,
readinessManager results.Manager,
livenessManager results.Manager,
@ -86,12 +81,11 @@ func NewManager(
recorder record.EventRecorder) Manager {
prober := newProber(runner, refManager, recorder)
return &manager{
defaultProbePeriod: defaultProbePeriod,
statusManager: statusManager,
prober: prober,
readinessManager: readinessManager,
livenessManager: livenessManager,
workers: make(map[probeKey]*worker),
statusManager: statusManager,
prober: prober,
readinessManager: readinessManager,
livenessManager: livenessManager,
workers: make(map[probeKey]*worker),
}
}

24
pkg/kubelet/prober/manager_test.go
View File

@ -34,6 +34,16 @@ import (
"k8s.io/kubernetes/pkg/util/wait"
)
var defaultProbe *api.Probe = &api.Probe{
Handler: api.Handler{
Exec: &api.ExecAction{},
},
TimeoutSeconds: 1,
PeriodSeconds: 1,
SuccessThreshold: 1,
FailureThreshold: 3,
}
func TestAddRemovePods(t *testing.T) {
noProbePod := api.Pod{
ObjectMeta: api.ObjectMeta{
@ -57,12 +67,12 @@ func TestAddRemovePods(t *testing.T) {
Name: "no_probe1",
}, {
Name: "readiness",
ReadinessProbe: &api.Probe{},
ReadinessProbe: defaultProbe,
}, {
Name: "no_probe2",
}, {
Name: "liveness",
LivenessProbe: &api.Probe{},
LivenessProbe: defaultProbe,
}},
},
}
@ -119,10 +129,10 @@ func TestCleanupPods(t *testing.T) {
Spec: api.PodSpec{
Containers: []api.Container{{
Name: "prober1",
ReadinessProbe: &api.Probe{},
ReadinessProbe: defaultProbe,
}, {
Name: "prober2",
LivenessProbe: &api.Probe{},
LivenessProbe: defaultProbe,
}},
},
}
@ -133,10 +143,10 @@ func TestCleanupPods(t *testing.T) {
Spec: api.PodSpec{
Containers: []api.Container{{
Name: "prober1",
ReadinessProbe: &api.Probe{},
ReadinessProbe: defaultProbe,
}, {
Name: "prober2",
LivenessProbe: &api.Probe{},
LivenessProbe: defaultProbe,
}},
},
}
@ -266,9 +276,7 @@ outer:
}
func newTestManager() *manager {
const probePeriod = 1
m := NewManager(
probePeriod,
status.NewManager(&testclient.Fake{}, kubepod.NewBasicPodManager(nil)),
results.NewManager(),
results.NewManager(),

4
pkg/kubelet/prober/prober.go
View File

@ -118,8 +118,8 @@ func (pb *prober) runProbeWithRetries(p *api.Probe, pod *api.Pod, status api.Pod
var output string
for i := 0; i < retries; i++ {
result, output, err = pb.runProbe(p, pod, status, container, containerID)
if result == probe.Success {
return probe.Success, output, nil
if err == nil {
return result, output, nil
}
}
return result, output, err

28
pkg/kubelet/prober/worker.go
View File

@ -56,6 +56,10 @@ type worker struct {
// The last known container ID for this worker.
containerID kubecontainer.ContainerID
// The last probe result for this worker.
lastResult results.Result
// How many times in a row the probe has returned the same result.
resultRun int
}
// Creates and starts a new probe worker.
@ -89,7 +93,7 @@ func newWorker(
// run periodically probes the container.
func (w *worker) run() {
probeTicker := time.NewTicker(w.probeManager.defaultProbePeriod)
probeTicker := time.NewTicker(time.Duration(w.spec.PeriodSeconds) * time.Second)
defer func() {
// Clean up.
@ -145,6 +149,7 @@ func (w *worker) doProbe() (keepGoing bool) {
w.resultsManager.Remove(w.containerID)
}
w.containerID = kubecontainer.ParseContainerID(c.ContainerID)
w.resultsManager.Set(w.containerID, w.initialValue, w.pod)
}
if c.State.Running == nil {
@ -159,14 +164,29 @@ func (w *worker) doProbe() (keepGoing bool) {
}
if int64(time.Since(c.State.Running.StartedAt.Time).Seconds()) < w.spec.InitialDelaySeconds {
w.resultsManager.Set(w.containerID, w.initialValue, w.pod)
return true
}
result, err := w.probeManager.prober.probe(w.probeType, w.pod, status, w.container, w.containerID)
if err == nil {
w.resultsManager.Set(w.containerID, result, w.pod)
if err != nil {
// Prober error, throw away the result.
return true
}
if w.lastResult == result {
w.resultRun++
} else {
w.lastResult = result
w.resultRun = 1
}
if (result == results.Failure && w.resultRun < w.spec.FailureThreshold) ||
(result == results.Success && w.resultRun < w.spec.SuccessThreshold) {
// Success or failure is below threshold - leave the probe state unchanged.
return true
}
w.resultsManager.Set(w.containerID, result, w.pod)
return true
}

136
pkg/kubelet/prober/worker_test.go
View File

@ -18,6 +18,7 @@ package prober
import (
"fmt"
"reflect"
"testing"
"time"
@ -135,18 +136,8 @@ func TestInitialDelay(t *testing.T) {
})
m.statusManager.SetPodStatus(w.pod, getRunningStatus())
if !w.doProbe() {
t.Errorf("[%s] Expected to continue, but did not", probeType)
}
expectedResult := results.Result(probeType == liveness)
result, ok := resultsManager(m, probeType).Get(containerID)
if !ok {
t.Errorf("[%s] Expected result to be set during initial delay, but was not set", probeType)
} else if result != expectedResult {
t.Errorf("[%s] Expected result to be %v during initial delay, but was %v",
probeType, expectedResult, result)
}
expectContinue(t, w, w.doProbe(), "during initial delay")
expectResult(t, w, results.Result(probeType == liveness), "during initial delay")
// 100 seconds later...
laterStatus := getRunningStatus()
@ -155,16 +146,76 @@ func TestInitialDelay(t *testing.T) {
m.statusManager.SetPodStatus(w.pod, laterStatus)
// Second call should succeed (already waited).
if !w.doProbe() {
t.Errorf("[%s] Expected to continue, but did not", probeType)
expectContinue(t, w, w.doProbe(), "after initial delay")
expectResult(t, w, results.Success, "after initial delay")
}
}
func TestFailureThreshold(t *testing.T) {
m := newTestManager()
w := newTestWorker(m, readiness, api.Probe{})
m.statusManager.SetPodStatus(w.pod, getRunningStatus())
for i := 0; i < 2; i++ {
// First probe should succeed.
m.prober.exec = fakeExecProber{probe.Success, nil}
for j := 0; j < 3; j++ {
msg := fmt.Sprintf("%d success (%d)", j+1, i)
expectContinue(t, w, w.doProbe(), msg)
expectResult(t, w, results.Success, msg)
}
result, ok = resultsManager(m, probeType).Get(containerID)
if !ok {
t.Errorf("[%s] Expected result to be true, but was not set", probeType)
} else if !result {
t.Errorf("[%s] Expected result to be true, but was false", probeType)
// Prober starts failing :(
m.prober.exec = fakeExecProber{probe.Failure, nil}
// Next 2 probes should still be "success".
for j := 0; j < 2; j++ {
msg := fmt.Sprintf("%d failure (%d)", j+1, i)
expectContinue(t, w, w.doProbe(), msg)
expectResult(t, w, results.Success, msg)
}
// Third & following fail.
for j := 0; j < 3; j++ {
msg := fmt.Sprintf("%d failure (%d)", j+3, i)
expectContinue(t, w, w.doProbe(), msg)
expectResult(t, w, results.Failure, msg)
}
}
}
func TestSuccessThreshold(t *testing.T) {
m := newTestManager()
w := newTestWorker(m, readiness, api.Probe{SuccessThreshold: 3, FailureThreshold: 1})
m.statusManager.SetPodStatus(w.pod, getRunningStatus())
// Start out failure.
w.resultsManager.Set(containerID, results.Failure, nil)
for i := 0; i < 2; i++ {
// Probe defaults to Failure.
for j := 0; j < 2; j++ {
msg := fmt.Sprintf("%d success (%d)", j+1, i)
expectContinue(t, w, w.doProbe(), msg)
expectResult(t, w, results.Failure, msg)
}
// Continuing success!
for j := 0; j < 3; j++ {
msg := fmt.Sprintf("%d success (%d)", j+3, i)
expectContinue(t, w, w.doProbe(), msg)
expectResult(t, w, results.Success, msg)
}
// Prober flakes :(
m.prober.exec = fakeExecProber{probe.Failure, nil}
msg := fmt.Sprintf("1 failure (%d)", i)
expectContinue(t, w, w.doProbe(), msg)
expectResult(t, w, results.Failure, msg)
// Back to success.
m.prober.exec = fakeExecProber{probe.Success, nil}
}
}
@ -205,22 +256,22 @@ func TestCleanUp(t *testing.T) {
func TestHandleCrash(t *testing.T) {
m := newTestManager()
w := newTestWorker(m, readiness, api.Probe{})
m.statusManager.SetPodStatus(w.pod, getRunningStatus())
expectContinue(t, w, w.doProbe(), "Initial successful probe.")
expectResult(t, w, results.Success, "Initial successful probe.")
// Prober starts crashing.
m.prober = &prober{
refManager: kubecontainer.NewRefManager(),
recorder: &record.FakeRecorder{},
exec: crashingExecProber{},
}
w := newTestWorker(m, readiness, api.Probe{})
m.statusManager.SetPodStatus(w.pod, getRunningStatus())
// doProbe should recover from the crash, and keep going.
if !w.doProbe() {
t.Error("Expected to keep going, but terminated.")
}
if _, ok := m.readinessManager.Get(containerID); ok {
t.Error("Expected readiness to be unchanged from crash.")
}
expectContinue(t, w, w.doProbe(), "Crashing probe.")
expectResult(t, w, results.Success, "Crashing probe unchanged.")
}
func newTestWorker(m *manager, probeType probeType, probeSpec api.Probe) *worker {
@ -228,6 +279,19 @@ func newTestWorker(m *manager, probeType probeType, probeSpec api.Probe) *worker
probeSpec.Handler = api.Handler{
Exec: &api.ExecAction{},
}
// Apply default values.
defaults := map[string]int64{
"TimeoutSeconds": 1,
"PeriodSeconds": 10,
"SuccessThreshold": 1,
"FailureThreshold": 3,
}
for field, value := range defaults {
f := reflect.ValueOf(&probeSpec).Elem().FieldByName(field)
if f.Int() == 0 {
f.SetInt(value)
}
}
pod := getTestPod(probeType, probeSpec)
return newWorker(m, probeType, &pod, pod.Spec.Containers[0])
@ -266,6 +330,22 @@ func getTestPod(probeType probeType, probeSpec api.Probe) api.Pod {
return pod
}
func expectResult(t *testing.T, w *worker, expectedResult results.Result, msg string) {
result, ok := resultsManager(w.probeManager, w.probeType).Get(containerID)
if !ok {
t.Errorf("[%s - %s] Expected result to be set, but was not set", w.probeType, msg)
} else if result != expectedResult {
t.Errorf("[%s - %s] Expected result to be %v, but was %v",
w.probeType, msg, expectedResult, result)
}
}
func expectContinue(t *testing.T, w *worker, c bool, msg string) {
if !c {
t.Errorf("[%s - %s] Expected to continue, but did not", w.probeType, msg)
}
}
func resultsManager(m *manager, probeType probeType) results.Manager {
switch probeType {
case readiness:

5
test/e2e/pods.go
View File

@ -560,6 +560,7 @@ var _ = Describe("Pods", func() {
},
},
InitialDelaySeconds: 15,
FailureThreshold: 1,
},
},
},
@ -586,6 +587,7 @@ var _ = Describe("Pods", func() {
},
},
InitialDelaySeconds: 15,
FailureThreshold: 1,
},
},
},
@ -613,6 +615,7 @@ var _ = Describe("Pods", func() {
},
},
InitialDelaySeconds: 15,
FailureThreshold: 1,
},
},
},
@ -640,6 +643,7 @@ var _ = Describe("Pods", func() {
},
},
InitialDelaySeconds: 5,
FailureThreshold: 1,
},
},
},
@ -673,6 +677,7 @@ var _ = Describe("Pods", func() {
},
},
InitialDelaySeconds: 15,
FailureThreshold: 1,
},
},
},