2019-01-12 04:58:27 +00:00
|
|
|
/*
|
|
|
|
Copyright 2014 The Kubernetes Authors.
|
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package healthz
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"fmt"
|
|
|
|
"net/http"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"sync/atomic"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
2020-03-26 21:07:15 +00:00
|
|
|
"k8s.io/apiserver/pkg/endpoints/metrics"
|
2019-09-27 21:51:53 +00:00
|
|
|
"k8s.io/apiserver/pkg/server/httplog"
|
|
|
|
"k8s.io/klog"
|
2019-01-12 04:58:27 +00:00
|
|
|
)
|
|
|
|
|
2019-09-27 21:51:53 +00:00
|
|
|
// HealthChecker is a named healthz checker.
|
|
|
|
type HealthChecker interface {
|
2019-01-12 04:58:27 +00:00
|
|
|
Name() string
|
|
|
|
Check(req *http.Request) error
|
|
|
|
}
|
|
|
|
|
|
|
|
// PingHealthz returns true automatically when checked
|
2019-09-27 21:51:53 +00:00
|
|
|
var PingHealthz HealthChecker = ping{}
|
2019-01-12 04:58:27 +00:00
|
|
|
|
|
|
|
// ping implements the simplest possible healthz checker.
|
|
|
|
type ping struct{}
|
|
|
|
|
|
|
|
func (ping) Name() string {
|
|
|
|
return "ping"
|
|
|
|
}
|
|
|
|
|
|
|
|
// PingHealthz is a health check that returns true.
|
|
|
|
func (ping) Check(_ *http.Request) error {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// LogHealthz returns true if logging is not blocked
|
2019-09-27 21:51:53 +00:00
|
|
|
var LogHealthz HealthChecker = &log{}
|
2019-01-12 04:58:27 +00:00
|
|
|
|
|
|
|
type log struct {
|
|
|
|
startOnce sync.Once
|
|
|
|
lastVerified atomic.Value
|
|
|
|
}
|
|
|
|
|
|
|
|
func (l *log) Name() string {
|
|
|
|
return "log"
|
|
|
|
}
|
|
|
|
|
|
|
|
func (l *log) Check(_ *http.Request) error {
|
|
|
|
l.startOnce.Do(func() {
|
|
|
|
l.lastVerified.Store(time.Now())
|
|
|
|
go wait.Forever(func() {
|
|
|
|
klog.Flush()
|
|
|
|
l.lastVerified.Store(time.Now())
|
|
|
|
}, time.Minute)
|
|
|
|
})
|
|
|
|
|
|
|
|
lastVerified := l.lastVerified.Load().(time.Time)
|
|
|
|
if time.Since(lastVerified) < (2 * time.Minute) {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
return fmt.Errorf("logging blocked")
|
|
|
|
}
|
|
|
|
|
|
|
|
// NamedCheck returns a healthz checker for the given name and function.
|
2019-09-27 21:51:53 +00:00
|
|
|
func NamedCheck(name string, check func(r *http.Request) error) HealthChecker {
|
2019-01-12 04:58:27 +00:00
|
|
|
return &healthzCheck{name, check}
|
|
|
|
}
|
|
|
|
|
|
|
|
// InstallHandler registers handlers for health checking on the path
|
|
|
|
// "/healthz" to mux. *All handlers* for mux must be specified in
|
|
|
|
// exactly one call to InstallHandler. Calling InstallHandler more
|
|
|
|
// than once for the same mux will result in a panic.
|
2019-09-27 21:51:53 +00:00
|
|
|
func InstallHandler(mux mux, checks ...HealthChecker) {
|
2019-01-12 04:58:27 +00:00
|
|
|
InstallPathHandler(mux, "/healthz", checks...)
|
|
|
|
}
|
|
|
|
|
2019-09-27 21:51:53 +00:00
|
|
|
// InstallReadyzHandler registers handlers for health checking on the path
|
|
|
|
// "/readyz" to mux. *All handlers* for mux must be specified in
|
|
|
|
// exactly one call to InstallHandler. Calling InstallHandler more
|
|
|
|
// than once for the same mux will result in a panic.
|
|
|
|
func InstallReadyzHandler(mux mux, checks ...HealthChecker) {
|
|
|
|
InstallPathHandler(mux, "/readyz", checks...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// InstallLivezHandler registers handlers for liveness checking on the path
|
|
|
|
// "/livez" to mux. *All handlers* for mux must be specified in
|
|
|
|
// exactly one call to InstallHandler. Calling InstallHandler more
|
|
|
|
// than once for the same mux will result in a panic.
|
|
|
|
func InstallLivezHandler(mux mux, checks ...HealthChecker) {
|
|
|
|
InstallPathHandler(mux, "/livez", checks...)
|
|
|
|
}
|
|
|
|
|
2019-01-12 04:58:27 +00:00
|
|
|
// InstallPathHandler registers handlers for health checking on
|
|
|
|
// a specific path to mux. *All handlers* for the path must be
|
|
|
|
// specified in exactly one call to InstallPathHandler. Calling
|
|
|
|
// InstallPathHandler more than once for the same path and mux will
|
|
|
|
// result in a panic.
|
2019-09-27 21:51:53 +00:00
|
|
|
func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
|
2019-01-12 04:58:27 +00:00
|
|
|
if len(checks) == 0 {
|
|
|
|
klog.V(5).Info("No default health checks specified. Installing the ping handler.")
|
2019-09-27 21:51:53 +00:00
|
|
|
checks = []HealthChecker{PingHealthz}
|
2019-01-12 04:58:27 +00:00
|
|
|
}
|
|
|
|
|
2019-09-27 21:51:53 +00:00
|
|
|
klog.V(5).Infof("Installing health checkers for (%v): %v", path, formatQuoted(checkerNames(checks...)...))
|
2019-01-12 04:58:27 +00:00
|
|
|
|
2020-03-26 21:07:15 +00:00
|
|
|
mux.Handle(path,
|
|
|
|
metrics.InstrumentHandlerFunc("GET",
|
|
|
|
/* group = */ "",
|
|
|
|
/* version = */ "",
|
|
|
|
/* resource = */ "",
|
|
|
|
/* subresource = */ path,
|
|
|
|
/* scope = */ "",
|
|
|
|
/* component = */ "",
|
|
|
|
handleRootHealthz(checks...)))
|
2019-01-12 04:58:27 +00:00
|
|
|
for _, check := range checks {
|
|
|
|
mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// mux is an interface describing the methods InstallHandler requires.
|
|
|
|
type mux interface {
|
|
|
|
Handle(pattern string, handler http.Handler)
|
|
|
|
}
|
|
|
|
|
2019-09-27 21:51:53 +00:00
|
|
|
// healthzCheck implements HealthChecker on an arbitrary name and check function.
|
2019-01-12 04:58:27 +00:00
|
|
|
type healthzCheck struct {
|
|
|
|
name string
|
|
|
|
check func(r *http.Request) error
|
|
|
|
}
|
|
|
|
|
2019-09-27 21:51:53 +00:00
|
|
|
var _ HealthChecker = &healthzCheck{}
|
2019-01-12 04:58:27 +00:00
|
|
|
|
|
|
|
func (c *healthzCheck) Name() string {
|
|
|
|
return c.name
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *healthzCheck) Check(r *http.Request) error {
|
|
|
|
return c.check(r)
|
|
|
|
}
|
|
|
|
|
|
|
|
// getExcludedChecks extracts the health check names to be excluded from the query param
|
|
|
|
func getExcludedChecks(r *http.Request) sets.String {
|
|
|
|
checks, found := r.URL.Query()["exclude"]
|
|
|
|
if found {
|
|
|
|
return sets.NewString(checks...)
|
|
|
|
}
|
|
|
|
return sets.NewString()
|
|
|
|
}
|
|
|
|
|
|
|
|
// handleRootHealthz returns an http.HandlerFunc that serves the provided checks.
|
2019-09-27 21:51:53 +00:00
|
|
|
func handleRootHealthz(checks ...HealthChecker) http.HandlerFunc {
|
2019-01-12 04:58:27 +00:00
|
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
|
|
failed := false
|
|
|
|
excluded := getExcludedChecks(r)
|
|
|
|
var verboseOut bytes.Buffer
|
|
|
|
for _, check := range checks {
|
|
|
|
// no-op the check if we've specified we want to exclude the check
|
|
|
|
if excluded.Has(check.Name()) {
|
|
|
|
excluded.Delete(check.Name())
|
|
|
|
fmt.Fprintf(&verboseOut, "[+]%v excluded: ok\n", check.Name())
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if err := check.Check(r); err != nil {
|
|
|
|
// don't include the error since this endpoint is public. If someone wants more detail
|
|
|
|
// they should have explicit permission to the detailed checks.
|
2019-04-07 17:07:55 +00:00
|
|
|
klog.V(4).Infof("healthz check %v failed: %v", check.Name(), err)
|
2019-01-12 04:58:27 +00:00
|
|
|
fmt.Fprintf(&verboseOut, "[-]%v failed: reason withheld\n", check.Name())
|
|
|
|
failed = true
|
|
|
|
} else {
|
|
|
|
fmt.Fprintf(&verboseOut, "[+]%v ok\n", check.Name())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if excluded.Len() > 0 {
|
|
|
|
fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %v\n", formatQuoted(excluded.List()...))
|
|
|
|
klog.Warningf("cannot exclude some health checks, no health checks are installed matching %v",
|
|
|
|
formatQuoted(excluded.List()...))
|
|
|
|
}
|
|
|
|
// always be verbose on failure
|
|
|
|
if failed {
|
2019-08-30 18:33:25 +00:00
|
|
|
klog.V(2).Infof("%vhealthz check failed", verboseOut.String())
|
2019-09-27 21:51:53 +00:00
|
|
|
http.Error(httplog.Unlogged(r, w), fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError)
|
2019-01-12 04:58:27 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2019-04-07 17:07:55 +00:00
|
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
2019-01-12 04:58:27 +00:00
|
|
|
if _, found := r.URL.Query()["verbose"]; !found {
|
|
|
|
fmt.Fprint(w, "ok")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
verboseOut.WriteTo(w)
|
|
|
|
fmt.Fprint(w, "healthz check passed\n")
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
// adaptCheckToHandler returns an http.HandlerFunc that serves the provided checks.
|
|
|
|
func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc {
|
|
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
|
|
err := c(r)
|
|
|
|
if err != nil {
|
|
|
|
http.Error(w, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
|
|
|
|
} else {
|
|
|
|
fmt.Fprint(w, "ok")
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
// checkerNames returns the names of the checks in the same order as passed in.
|
2019-09-27 21:51:53 +00:00
|
|
|
func checkerNames(checks ...HealthChecker) []string {
|
2019-01-12 04:58:27 +00:00
|
|
|
// accumulate the names of checks for printing them out.
|
|
|
|
checkerNames := make([]string, 0, len(checks))
|
|
|
|
for _, check := range checks {
|
|
|
|
checkerNames = append(checkerNames, check.Name())
|
|
|
|
}
|
|
|
|
return checkerNames
|
|
|
|
}
|
|
|
|
|
|
|
|
// formatQuoted returns a formatted string of the health check names,
|
|
|
|
// preserving the order passed in.
|
|
|
|
func formatQuoted(names ...string) string {
|
|
|
|
quoted := make([]string, 0, len(names))
|
|
|
|
for _, name := range names {
|
|
|
|
quoted = append(quoted, fmt.Sprintf("%q", name))
|
|
|
|
}
|
|
|
|
return strings.Join(quoted, ",")
|
|
|
|
}
|