diff --git a/command/agent/agent.go b/command/agent/agent.go
index ab48c28692..b7c0b8cf8d 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -371,9 +371,6 @@ func (a *Agent) listenHTTP(addrs []ProtoAddr) ([]net.Listener, error) {
 
 		case p.Net == "tcp" && p.Proto == "http":
 			l, err = net.Listen("tcp", p.Addr)
-			if err != nil {
-				l = &tcpKeepAliveListener{l.(*net.TCPListener)}
-			}
 
 		case p.Net == "tcp" && p.Proto == "https":
 			var tlscfg *tls.Config
@@ -382,9 +379,6 @@ func (a *Agent) listenHTTP(addrs []ProtoAddr) ([]net.Listener, error) {
 				break
 			}
 			l, err = tls.Listen("tcp", p.Addr, tlscfg)
-			if err != nil {
-				l = &tcpKeepAliveListener{l.(*net.TCPListener)}
-			}
 
 		default:
 			return nil, fmt.Errorf("%s:%s listener not supported", p.Net, p.Proto)
@@ -397,6 +391,10 @@ func (a *Agent) listenHTTP(addrs []ProtoAddr) ([]net.Listener, error) {
 			return nil, err
 		}
 
+		if tcpl, ok := l.(*net.TCPListener); ok {
+			l = &tcpKeepAliveListener{tcpl}
+		}
+
 		ln = append(ln, l)
 	}
 	return ln, nil