Merge pull request #63383 from liggitt/lease-reconciler

Automatic merge from submit-queue (batch tested with PRs 63315, 63383, 63318, 63439). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Re-enable lease reconciler, fix shutdown race

Fixes #63378
Fixes #57617

* Fixes the openapi script to wait for the apiserver on shutdown (like all the other scripts do)
* Fixes the apiserver shutdown to not hang forever if the kubernetes service reconciler cannot persist to etcd
* Readds https://github.com/kubernetes/kubernetes/pull/58474 to make the default the lease reconciler

```release-note
kube-apiserver: the default `--endpoint-reconciler-type` is now `lease`. The `master-count` endpoint reconciler type is deprecated and will be removed in 1.13.
```

cmd/kube-apiserver/app/options/options.go

@@ -93,7 +93,7 @@ func NewServerRunOptions() *ServerRunOptions {
 		EnableLogsHandler:      true,
 		EventTTL:               1 * time.Hour,
 		MasterCount:            1,
-		EndpointReconcilerType: string(reconcilers.MasterCountReconcilerType),
+		EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
 		KubeletConfig: kubeletclient.KubeletClientConfig{
 			Port:         ports.KubeletPort,
 			ReadOnlyPort: ports.KubeletReadOnlyPort,
@@ -166,7 +166,7 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
 		"Currently only applies to long-running requests.")
 
 	fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
-		"The number of apiservers running in the cluster, must be a positive number.")
+		"The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)")
 
 	fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", string(s.EndpointReconcilerType),
 		"Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+")")

cmd/kube-apiserver/app/options/options_test.go

@@ -96,7 +96,7 @@ func TestAddFlags(t *testing.T) {
 		"--enable-aggregator-routing=true",
 		"--enable-logs-handler=false",
 		"--enable-swagger-ui=true",
-		"--endpoint-reconciler-type=" + string(reconcilers.MasterCountReconcilerType),
+		"--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType),
 		"--etcd-quorum-read=false",
 		"--etcd-keyfile=/var/run/kubernetes/etcd.key",
 		"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
@@ -120,7 +120,7 @@ func TestAddFlags(t *testing.T) {
 		ServiceNodePortRange:   kubeoptions.DefaultServiceNodePortRange,
 		ServiceClusterIPRange:  kubeoptions.DefaultServiceIPCIDR,
 		MasterCount:            5,
-		EndpointReconcilerType: string(reconcilers.MasterCountReconcilerType),
+		EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
 		AllowPrivileged:        false,
 		GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
 			AdvertiseAddress:            net.ParseIP("192.168.10.10"),

hack/update-openapi-spec.sh

@@ -31,7 +31,11 @@ make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver
 
 function cleanup()
 {
-    [[ -n ${APISERVER_PID-} ]] && kill ${APISERVER_PID} 1>&2 2>/dev/null
+    if [[ -n ${APISERVER_PID-} ]]; then
+      kill ${APISERVER_PID} 1>&2 2>/dev/null
+      wait ${APISERVER_PID} || true
+    fi
+    unset APISERVER_PID
 
     kube::etcd::cleanup
 

pkg/master/controller.go

@@ -152,7 +152,22 @@ func (c *Controller) Stop() {
 		c.runner.Stop()
 	}
 	endpointPorts := createEndpointPortSpec(c.PublicServicePort, "https", c.ExtraEndpointPorts)
-	c.EndpointReconciler.StopReconciling("kubernetes", c.PublicIP, endpointPorts)
+	finishedReconciling := make(chan struct{})
+	go func() {
+		defer close(finishedReconciling)
+		glog.Infof("Shutting down kubernetes service endpoint reconciler")
+		if err := c.EndpointReconciler.StopReconciling("kubernetes", c.PublicIP, endpointPorts); err != nil {
+			glog.Error(err)
+		}
+	}()
+
+	select {
+	case <-finishedReconciling:
+		// done
+	case <-time.After(2 * c.EndpointInterval):
+		// don't block server shutdown forever if we can't reach etcd to remove ourselves
+		glog.Warning("StopReconciling() timed out")
+	}
 }
 
 // RunKubernetesNamespaces periodically makes sure that all internal namespaces exist