Merge pull request #60054 from MikeSpreitzer/issue-60042-field

Automatic merge from submit-queue (batch tested with PRs 60054, 60202, 60219, 58090, 60275). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Fixes for HTTP/2 max streams per connection setting

**What this PR does / why we need it**:
This PR makes two changes.  One is to introduce a parameter
for the HTTP/2 setting that an api-server sends to its clients
telling them how many streams they may have concurrently open in
an HTTP/2 connection.  If left at its default value of zero,
this means to use the default in golang's HTTP/2 code (which
is currently 250; see https://github.com/golang/net/blob/master/http2/server.go).

The other change is to make the recommended options for an aggregated
api-server set this limit to 1000.  The limit of 250 is annoyingly low
for the use case of many controllers watching objects of Kinds served
by an aggregated api-server reached through the main api-server (in
its mode as a proxy for the aggregated api-server, in which it uses a
single HTTP/2 connection for all calls proxied to that aggregated
api-server).

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #60042

**Special notes for your reviewer**:

**Release note**:

```release-note
Introduced `--http2-max-streams-per-connection` command line flag on api-servers and set default to 1000 for aggregated API servers.
```

cmd/cloud-controller-manager/app/options/options_test.go

@@ -46,6 +46,7 @@ func TestAddFlags(t *testing.T) {
 		"--configure-cloud-routes=false",
 		"--contention-profiling=true",
 		"--controller-start-interval=2m",
+		"--http2-max-streams-per-connection=47",
 		"--min-resync-period=5m",
 		"--kube-api-burst=100",
 		"--kube-api-content-type=application/vnd.kubernetes.protobuf",
@@ -150,6 +151,7 @@ func TestAddFlags(t *testing.T) {
 					CertDirectory: "/a/b/c",
 					PairName:      "cloud-controller-manager",
 				},
+				HTTP2MaxStreamsPerConnection: 47,
 			},
 			InsecureServing: &cmoptions.InsecureServingOptions{
 				BindAddress: net.ParseIP("192.168.4.10"),

cmd/kube-apiserver/app/options/options_test.go

@@ -84,6 +84,7 @@ func TestAddFlags(t *testing.T) {
 		"--etcd-keyfile=/var/run/kubernetes/etcd.key",
 		"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
 		"--etcd-cafile=/var/run/kubernetes/etcdca.crt",
+		"--http2-max-streams-per-connection=42",
 		"--kubelet-https=true",
 		"--kubelet-read-only-port=10255",
 		"--kubelet-timeout=5s",
@@ -146,6 +147,7 @@ func TestAddFlags(t *testing.T) {
 				CertDirectory: "/var/run/kubernetes",
 				PairName:      "apiserver",
 			},
+			HTTP2MaxStreamsPerConnection: 42,
 		}),
 		InsecureServing: &kubeoptions.InsecureServingOptions{
 			BindAddress: net.ParseIP("127.0.0.1"),

cmd/kube-controller-manager/app/options/options_test.go

@@ -72,6 +72,7 @@ func TestAddFlags(t *testing.T) {
 		"--horizontal-pod-autoscaler-downscale-delay=2m",
 		"--horizontal-pod-autoscaler-sync-period=45s",
 		"--horizontal-pod-autoscaler-upscale-delay=1m",
+		"--http2-max-streams-per-connection=47",
 		"--kube-api-burst=100",
 		"--kube-api-content-type=application/json",
 		"--kube-api-qps=50.0",
@@ -216,6 +217,7 @@ func TestAddFlags(t *testing.T) {
 					CertDirectory: "/a/b/c",
 					PairName:      "kube-controller-manager",
 				},
+				HTTP2MaxStreamsPerConnection: 47,
 			},
 			InsecureServing: &cmoptions.InsecureServingOptions{
 				BindAddress: net.ParseIP("192.168.4.10"),

staging/src/k8s.io/apiserver/pkg/server/BUILD

@@ -97,6 +97,7 @@ go_library(
         "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/pborman/uuid:go_default_library",
+        "//vendor/golang.org/x/net/http2:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library",

staging/src/k8s.io/apiserver/pkg/server/config.go

@@ -227,6 +227,10 @@ type SecureServingInfo struct {
 	// CipherSuites optionally overrides the list of allowed cipher suites for the server.
 	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
 	CipherSuites []uint16
+
+	// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
+	// A value of zero means to use the default provided by golang's HTTP/2 support.
+	HTTP2MaxStreamsPerConnection int
 }
 
 type AuthenticationInfo struct {

staging/src/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -44,9 +44,17 @@ type RecommendedOptions struct {
 }
 
 func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions {
+	sso := NewSecureServingOptions()
+
+	// We are composing recommended options for an aggregated api-server,
+	// whose client is typically a proxy multiplexing many operations ---
+	// notably including long-running ones --- into one HTTP/2 connection
+	// into this server.  So allow many concurrent operations.
+	sso.HTTP2MaxStreamsPerConnection = 1000
+
 	return &RecommendedOptions{
 		Etcd:                       NewEtcdOptions(storagebackend.NewDefaultConfig(prefix, codec)),
-		SecureServing:              WithLoopback(NewSecureServingOptions()),
+		SecureServing:              WithLoopback(sso),
 		Authentication:             NewDelegatingAuthenticationOptions(),
 		Authorization:              NewDelegatingAuthorizationOptions(),
 		Audit:                      NewAuditOptions(),

staging/src/k8s.io/apiserver/pkg/server/options/serving.go

@@ -55,6 +55,10 @@ type SecureServingOptions struct {
 	// MinTLSVersion is the minimum TLS version supported.
 	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
 	MinTLSVersion string
+
+	// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
+	// A value of zero means to use the default provided by golang's HTTP/2 support.
+	HTTP2MaxStreamsPerConnection int
 }
 
 type CertKey struct {
@@ -148,6 +152,11 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 		"--tls-sni-cert-key multiple times. "+
 		"Examples: \"example.crt,example.key\" or \"foo.crt,foo.key:*.foo.com,foo.com\".")
 
+	fs.IntVar(&s.HTTP2MaxStreamsPerConnection, "http2-max-streams-per-connection", s.HTTP2MaxStreamsPerConnection, ""+
+		"The limit that the server gives to clients for "+
+		"the maximum number of streams in an HTTP/2 connection. "+
+		"Zero means to use golang's default.")
+
 	// TODO remove this flag in 1.11.  The flag had no effect before this will prevent scripts from immediately failing on upgrade.
 	fs.String("tls-ca-file", "", "This flag has no effect.")
 	fs.MarkDeprecated("tls-ca-file", "This flag has no effect.")
@@ -179,7 +188,8 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	}
 
 	*config = &server.SecureServingInfo{
-		Listener: s.Listener,
+		Listener:                     s.Listener,
+		HTTP2MaxStreamsPerConnection: s.HTTP2MaxStreamsPerConnection,
 	}
 	c := *config
 

staging/src/k8s.io/apiserver/pkg/server/serve.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/golang/glog"
+	"golang.org/x/net/http2"
 
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/validation"
@@ -86,6 +87,12 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur
 		secureServer.TLSConfig.ClientCAs = s.ClientCA
 	}
 
+	if s.HTTP2MaxStreamsPerConnection > 0 {
+		http2.ConfigureServer(secureServer, &http2.Server{
+			MaxConcurrentStreams: uint32(s.HTTP2MaxStreamsPerConnection),
+		})
+	}
+
 	glog.Infof("Serving securely on %s", secureServer.Addr)
 	return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh)
 }