fix apiserver startup check

pkg/genericapiserver/genericapiserver.go

@@ -227,7 +227,6 @@ func (s *GenericAPIServer) Run() {
 		s.InstallOpenAPI()
 	}
 
-	secureStartedCh := make(chan struct{})
 	if s.SecureServingInfo != nil {
 		secureServer := &http.Server{
 			Addr:           s.SecureServingInfo.BindAddress,
@@ -274,23 +273,15 @@ func (s *GenericAPIServer) Run() {
 		go func() {
 			defer utilruntime.HandleCrash()
 
-			notifyStarted := sync.Once{}
 			for {
 				if err := secureServer.ListenAndServeTLS(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile); err != nil {
 					glog.Errorf("Unable to listen for secure (%v); will try again.", err)
-				} else {
-					notifyStarted.Do(func() {
-						close(secureStartedCh)
-					})
 				}
 				time.Sleep(15 * time.Second)
 			}
 		}()
-	} else {
-		close(secureStartedCh)
 	}
 
-	insecureStartedCh := make(chan struct{})
 	if s.InsecureServingInfo != nil {
 		insecureServer := &http.Server{
 			Addr:           s.InsecureServingInfo.BindAddress,
@@ -301,24 +292,27 @@ func (s *GenericAPIServer) Run() {
 		go func() {
 			defer utilruntime.HandleCrash()
 
-			notifyStarted := sync.Once{}
 			for {
 				if err := insecureServer.ListenAndServe(); err != nil {
 					glog.Errorf("Unable to listen for insecure (%v); will try again.", err)
-				} else {
-					notifyStarted.Do(func() {
-						close(insecureStartedCh)
-					})
 				}
 				time.Sleep(15 * time.Second)
 			}
 		}()
-	} else {
-		close(insecureStartedCh)
 	}
 
-	<-secureStartedCh
-	<-insecureStartedCh
+	// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) per port
+	if s.SecureServingInfo != nil {
+		if err := waitForSuccessfulDial(true, "tcp", s.SecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
+			glog.Fatalf("Secure server never started: %v", err)
+		}
+	}
+	if s.InsecureServingInfo != nil {
+		if err := waitForSuccessfulDial(false, "tcp", s.InsecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
+			glog.Fatalf("Insecure server never started: %v", err)
+		}
+	}
+
 	s.RunPostStartHooks()
 
 	// err == systemd.SdNotifyNoSocket when not running on a systemd system
@@ -575,3 +569,27 @@ func NewDefaultAPIGroupInfo(group string) APIGroupInfo {
 		NegotiatedSerializer:         api.Codecs,
 	}
 }
+
+// waitForSuccessfulDial attempts to connect to the given address, closing and returning nil on the first successful connection.
+func waitForSuccessfulDial(https bool, network, address string, timeout, interval time.Duration, retries int) error {
+	var (
+		conn net.Conn
+		err  error
+	)
+	for i := 0; i <= retries; i++ {
+		dialer := net.Dialer{Timeout: timeout}
+		if https {
+			conn, err = tls.DialWithDialer(&dialer, network, address, &tls.Config{InsecureSkipVerify: true})
+		} else {
+			conn, err = dialer.Dial(network, address)
+		}
+		if err != nil {
+			glog.V(5).Infof("Got error %#v, trying again: %#v\n", err, address)
+			time.Sleep(interval)
+			continue
+		}
+		conn.Close()
+		return nil
+	}
+	return err
+}