Merge pull request #70676 from logicalhan/exclude-checks

add ability to disable health checks on kube-apiserver for healthz using query-params

cluster/gce/manifests/kube-apiserver.manifest

@@ -34,11 +34,19 @@
       "httpGet": {
         "host": "127.0.0.1",
         "port": 8080,
-        "path": "/healthz"
+        "path": "/healthz?exclude=etcd"
       },
       "initialDelaySeconds": {{liveness_probe_initial_delay}},
       "timeoutSeconds": 15
     },
+    "readinessProbe": {
+      "httpGet": {
+        "host": "127.0.0.1",
+        "port": 8080,
+        "path": "/healthz"
+      },
+      "timeoutSeconds": 15
+    },
     "ports":[
       { "name": "https",
         "containerPort": {{secure_port}},

staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD

@@ -10,6 +10,9 @@ go_test(
     name = "go_default_test",
     srcs = ["healthz_test.go"],
     embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
 )
 
 go_library(
@@ -21,6 +24,7 @@ go_library(
     importmap = "k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/healthz",
     importpath = "k8s.io/apiserver/pkg/server/healthz",
     deps = [
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/klog:go_default_library",
     ],

staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go

@@ -27,6 +27,7 @@ import (
 
 	"k8s.io/klog"
 
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
@@ -141,12 +142,28 @@ func (c *healthzCheck) Check(r *http.Request) error {
 	return c.check(r)
 }
 
+// getExcludedChecks extracts the health check names to be excluded from the query param
+func getExcludedChecks(r *http.Request) sets.String {
+	checks, found := r.URL.Query()["exclude"]
+	if found {
+		return sets.NewString(checks...)
+	}
+	return sets.NewString()
+}
+
 // handleRootHealthz returns an http.HandlerFunc that serves the provided checks.
 func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		failed := false
+		excluded := getExcludedChecks(r)
 		var verboseOut bytes.Buffer
 		for _, check := range checks {
+			// no-op the check if we've specified we want to exclude the check
+			if excluded.Has(check.Name()) {
+				excluded.Delete(check.Name())
+				fmt.Fprintf(&verboseOut, "[+]%v excluded: ok\n", check.Name())
+				continue
+			}
 			if err := check.Check(r); err != nil {
 				// don't include the error since this endpoint is public.  If someone wants more detail
 				// they should have explicit permission to the detailed checks.
@@ -157,6 +174,11 @@ func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc {
 				fmt.Fprintf(&verboseOut, "[+]%v ok\n", check.Name())
 			}
 		}
+		if excluded.Len() > 0 {
+			fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %v\n", formatQuoted(excluded.List()...))
+			klog.Warningf("cannot exclude some health checks, no health checks are installed matching %v",
+				formatQuoted(excluded.List()...))
+		}
 		// always be verbose on failure
 		if failed {
 			http.Error(w, fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError)

staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go

@@ -21,8 +21,11 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"reflect"
 	"testing"
+
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 func TestInstallHandler(t *testing.T) {
@@ -82,6 +85,10 @@ func testMultipleChecks(path string, t *testing.T) {
 		addBadCheck      bool
 	}{
 		{"?verbose", "[+]ping ok\nhealthz check passed\n", http.StatusOK, false},
+		{"?exclude=dontexist", "ok", http.StatusOK, false},
+		{"?exclude=bad", "ok", http.StatusOK, true},
+		{"?verbose=true&exclude=bad", "[+]ping ok\n[+]bad excluded: ok\nhealthz check passed\n", http.StatusOK, true},
+		{"?verbose=true&exclude=dontexist", "[+]ping ok\nwarn: some health checks cannot be excluded: no matches for \"dontexist\"\nhealthz check passed\n", http.StatusOK, false},
 		{"/ping", "ok", http.StatusOK, false},
 		{"", "ok", http.StatusOK, false},
 		{"?verbose", "[+]ping ok\n[-]bad failed: reason withheld\nhealthz check failed\n", http.StatusInternalServerError, true},
@@ -177,3 +184,43 @@ func TestFormatQuoted(t *testing.T) {
 		})
 	}
 }
+
+func TestGetExcludedChecks(t *testing.T) {
+	type args struct {
+		r *http.Request
+	}
+	tests := []struct {
+		name string
+		r    *http.Request
+		want sets.String
+	}{
+		{"Should have no excluded health checks",
+			createGetRequestWithUrl("/healthz?verbose=true"),
+			sets.NewString(),
+		},
+		{"Should extract out the ping health check",
+			createGetRequestWithUrl("/healthz?exclude=ping"),
+			sets.NewString("ping"),
+		},
+		{"Should extract out ping and log health check",
+			createGetRequestWithUrl("/healthz?exclude=ping&exclude=log"),
+			sets.NewString("ping", "log"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getExcludedChecks(tt.r); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getExcludedChecks() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func createGetRequestWithUrl(rawUrlString string) *http.Request {
+	url, _ := url.Parse(rawUrlString)
+	return &http.Request{
+		Method: http.MethodGet,
+		Proto:  "HTTP/1.1",
+		URL:    url,
+	}
+}