Add --cleanup-iptables flag to kube-proxy

Adds a flag to cleanup iptables rules created by kube-proxy per
https://github.com/mesosphere/kubernetes-mesos/issues/353#issuecomment-1
27382832

cmd/kube-proxy/app/server.go

@@ -19,6 +19,7 @@ limitations under the License.
 package app
 
 import (
+	"errors"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
@@ -62,6 +63,7 @@ type ProxyServer struct {
 	SyncPeriod          time.Duration
 	nodeRef             *api.ObjectReference // Reference to this node.
 	MasqueradeAll       bool
+	CleanupAndExit      bool
 }
 
 // NewProxyServer creates a new ProxyServer object with default parameters
@@ -90,10 +92,28 @@ func (s *ProxyServer) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.ForceUserspaceProxy, "legacy-userspace-proxy", true, "Use the legacy userspace proxy (instead of the pure iptables proxy).")
 	fs.DurationVar(&s.SyncPeriod, "iptables-sync-period", 5*time.Second, "How often iptables rules are refreshed (e.g. '5s', '1m', '2h22m').  Must be greater than 0.")
 	fs.BoolVar(&s.MasqueradeAll, "masquerade-all", false, "If using the pure iptables proxy, SNAT everything")
+	fs.BoolVar(&s.CleanupAndExit, "cleanup-iptables", false, "If true cleanup iptables rules and exit.")
 }
 
-// Run runs the specified ProxyServer.  This should never exit.
+// Run runs the specified ProxyServer.  This should never exit (unless CleanupAndExit is set).
 func (s *ProxyServer) Run(_ []string) error {
+	protocol := utiliptables.ProtocolIpv4
+	if s.BindAddress.To4() == nil {
+		protocol = utiliptables.ProtocolIpv6
+	}
+
+	// remove iptables rules and exit
+	if s.CleanupAndExit {
+		execer := exec.New()
+		ipt := utiliptables.New(execer, protocol)
+		encounteredError := userspace.CleanupLeftovers(ipt)
+		encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError
+		if encounteredError {
+			return errors.New("Encountered an error while tearing down rules.")
+		}
+		return nil
+	}
+
 	// TODO(vmarmol): Use container config for this.
 	oomAdjuster := oom.NewOomAdjuster()
 	if err := oomAdjuster.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
@@ -145,11 +165,6 @@ func (s *ProxyServer) Run(_ []string) error {
 	serviceConfig := config.NewServiceConfig()
 	endpointsConfig := config.NewEndpointsConfig()
 
-	protocol := utiliptables.ProtocolIpv4
-	if s.BindAddress.To4() == nil {
-		protocol = utiliptables.ProtocolIpv6
-	}
-
 	var proxier proxy.ProxyProvider
 	var endpointsHandler config.EndpointsConfigHandler
 
@@ -162,12 +177,17 @@ func (s *ProxyServer) Run(_ []string) error {
 		glog.V(2).Info("Using iptables Proxier.")
 
 		execer := exec.New()
-		proxierIptables, err := iptables.NewProxier(utiliptables.New(execer, protocol), execer, s.SyncPeriod, s.MasqueradeAll)
+		ipt := utiliptables.New(execer, protocol)
+		proxierIptables, err := iptables.NewProxier(ipt, execer, s.SyncPeriod, s.MasqueradeAll)
 		if err != nil {
 			glog.Fatalf("Unable to create proxier: %v", err)
 		}
 		proxier = proxierIptables
 		endpointsHandler = proxierIptables
+		// No turning back. Remove artifacts that might still exist from the userspace Proxier.
+		glog.V(2).Info("Tearing down userspace rules. Errors here are acceptable.")
+		userspace.CleanupLeftovers(ipt)
+
 	} else {
 		glog.V(2).Info("Using userspace Proxier.")
 		// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
@@ -176,11 +196,16 @@ func (s *ProxyServer) Run(_ []string) error {
 		// set EndpointsConfigHandler to our loadBalancer
 		endpointsHandler = loadBalancer
 
-		proxierUserspace, err := userspace.NewProxier(loadBalancer, s.BindAddress, utiliptables.New(exec.New(), protocol), s.PortRange, s.SyncPeriod)
+		execer := exec.New()
+		ipt := utiliptables.New(execer, protocol)
+		proxierUserspace, err := userspace.NewProxier(loadBalancer, s.BindAddress, ipt, s.PortRange, s.SyncPeriod)
 		if err != nil {
 			glog.Fatalf("Unable to create proxer: %v", err)
 		}
 		proxier = proxierUserspace
+		// Remove artifacts from the pure-iptables Proxier.
+		glog.V(2).Info("Tearing down pure-iptables proxy rules. Errors here are acceptable.")
+		iptables.CleanupLeftovers(ipt)
 	}
 
 	// Wire proxier to handle changes to services

hack/verify-flags/known-flags.txt

@@ -31,6 +31,7 @@ certificate-authority
 cgroup-prefix
 cgroup-root
 chaos-chance
+cleanup-iptables
 client-ca-file
 client-certificate
 client-key

pkg/proxy/iptables/proxier.go

@@ -176,10 +176,6 @@ func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod
 		return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlBridgeCallIptables, err)
 	}
 
-	// No turning back. Remove artifacts that might still exist from the userspace Proxier.
-	glog.V(2).Info("Tearing down userspace rules. Errors here are acceptable.")
-	tearDownUserspaceIptables(ipt)
-
 	return &Proxier{
 		serviceMap:    make(map[proxy.ServicePortName]*serviceInfo),
 		syncPeriod:    syncPeriod,
@@ -188,47 +184,20 @@ func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod
 	}, nil
 }
 
-// Chains from the userspace proxy
+// CleanupLeftovers removes all iptables rules and chains created by the Proxier
-// TODO: Remove these Chains and tearDownUserspaceIptables once the userspace Proxier has been removed.
+// It returns true if an error was encountered. Errors are logged.
-var iptablesContainerPortalChain utiliptables.Chain = "KUBE-PORTALS-CONTAINER"
+func CleanupLeftovers(ipt utiliptables.Interface) (encounteredError bool) {
-var iptablesHostPortalChain utiliptables.Chain = "KUBE-PORTALS-HOST"
+	//TODO: actually tear down all rules and chains.
-var iptablesContainerNodePortChain utiliptables.Chain = "KUBE-NODEPORT-CONTAINER"
+	args := []string{"-j", "KUBE-SERVICES"}
-var iptablesHostNodePortChain utiliptables.Chain = "KUBE-NODEPORT-HOST"
+	if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainOutput, args...); err != nil {
-
+		glog.Errorf("Error removing pure-iptables proxy rule: %v", err)
-// tearDownUserspaceIptables removes all iptables rules and chains created by the userspace Proxier
+		encounteredError = true
-func tearDownUserspaceIptables(ipt utiliptables.Interface) {
-	// NOTE: Warning, this needs to be kept in sync with the userspace Proxier,
-	// we want to ensure we remove all of the iptables rules it creates.
-	// Currently they are all in iptablesInit()
-	// Delete Rules first, then Flush and Delete Chains
-	args := []string{"-m", "comment", "--comment", "handle ClusterIPs; NOTE: this must be before the NodePort rules"}
-	if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainOutput, append(args, "-j", string(iptablesHostPortalChain))...); err != nil {
-		glog.Errorf("Error removing userspace rule: %v", err)
 	}
-	if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPrerouting, append(args, "-j", string(iptablesContainerPortalChain))...); err != nil {
+	if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPrerouting, args...); err != nil {
-		glog.Errorf("Error removing userspace rule: %v", err)
+		glog.Errorf("Error removing pure-iptables proxy rule: %v", err)
-	}
+		encounteredError = true
-	args = []string{"-m", "addrtype", "--dst-type", "LOCAL"}
-	args = append(args, "-m", "comment", "--comment", "handle service NodePorts; NOTE: this must be the last rule in the chain")
-	if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainOutput, append(args, "-j", string(iptablesHostNodePortChain))...); err != nil {
-		glog.Errorf("Error removing userspace rule: %v", err)
-	}
-	if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPrerouting, append(args, "-j", string(iptablesContainerNodePortChain))...); err != nil {
-		glog.Errorf("Error removing userspace rule: %v", err)
-	}
-
-	// flush and delete chains.
-	chains := []utiliptables.Chain{iptablesContainerPortalChain, iptablesHostPortalChain, iptablesHostNodePortChain, iptablesContainerNodePortChain}
-	for _, c := range chains {
-		// flush chain, then if sucessful delete, delete will fail if flush fails.
-		if err := ipt.FlushChain(utiliptables.TableNAT, c); err != nil {
-			glog.Errorf("Error flushing userspace chain: %v", err)
-		} else {
-			if err = ipt.DeleteChain(utiliptables.TableNAT, c); err != nil {
-				glog.Errorf("Error flushing userspace chain: %v", err)
-			}
-		}
 	}
+	return encounteredError
 }
 
 func (proxier *Proxier) sameConfig(info *serviceInfo, service *api.Service, port *api.ServicePort) bool {

pkg/proxy/userspace/proxier.go

@@ -134,8 +134,6 @@ func createProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables
 	if proxyPorts == nil {
 		proxyPorts = newPortAllocator(util.PortRange{})
 	}
-	glog.V(2).Info("Tearing down pure-iptables proxy rules. Errors here are acceptable.")
-	tearDownIptablesProxierRules(iptables)
 	// Set up the iptables foundations we need.
 	if err := iptablesInit(iptables); err != nil {
 		return nil, fmt.Errorf("failed to initialize iptables: %v", err)
@@ -157,17 +155,48 @@ func createProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables
 	}, nil
 }
 
-// remove the iptables rules from the pure iptables Proxier
+// CleanupLeftovers removes all iptables rules and chains created by the Proxier
-func tearDownIptablesProxierRules(ipt iptables.Interface) {
+// It returns true if an error was encountered. Errors are logged.
-	//TODO: actually tear down all rules and chains.
+func CleanupLeftovers(ipt iptables.Interface) (encounteredError bool) {
-	//NOTE: this needs to be kept in sync with the proxy/iptables Proxier's rules.
+	// NOTE: Warning, this needs to be kept in sync with the userspace Proxier,
-	args := []string{"-j", "KUBE-SERVICES"}
+	// we want to ensure we remove all of the iptables rules it creates.
-	if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, args...); err != nil {
+	// Currently they are all in iptablesInit()
-		glog.Errorf("Error removing pure-iptables proxy rule: %v", err)
+	// Delete Rules first, then Flush and Delete Chains
+	args := []string{"-m", "comment", "--comment", "handle ClusterIPs; NOTE: this must be before the NodePort rules"}
+	if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, append(args, "-j", string(iptablesHostPortalChain))...); err != nil {
+		glog.Errorf("Error removing userspace rule: %v", err)
+		encounteredError = true
 	}
-	if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, args...); err != nil {
+	if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, append(args, "-j", string(iptablesContainerPortalChain))...); err != nil {
-		glog.Errorf("Error removing pure-iptables proxy rule: %v", err)
+		glog.Errorf("Error removing userspace rule: %v", err)
+		encounteredError = true
 	}
+	args = []string{"-m", "addrtype", "--dst-type", "LOCAL"}
+	args = append(args, "-m", "comment", "--comment", "handle service NodePorts; NOTE: this must be the last rule in the chain")
+	if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, append(args, "-j", string(iptablesHostNodePortChain))...); err != nil {
+		glog.Errorf("Error removing userspace rule: %v", err)
+		encounteredError = true
+	}
+	if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, append(args, "-j", string(iptablesContainerNodePortChain))...); err != nil {
+		glog.Errorf("Error removing userspace rule: %v", err)
+		encounteredError = true
+	}
+
+	// flush and delete chains.
+	chains := []iptables.Chain{iptablesContainerPortalChain, iptablesHostPortalChain, iptablesHostNodePortChain, iptablesContainerNodePortChain}
+	for _, c := range chains {
+		// flush chain, then if sucessful delete, delete will fail if flush fails.
+		if err := ipt.FlushChain(iptables.TableNAT, c); err != nil {
+			glog.Errorf("Error flushing userspace chain: %v", err)
+			encounteredError = true
+		} else {
+			if err = ipt.DeleteChain(iptables.TableNAT, c); err != nil {
+				glog.Errorf("Error flushing userspace chain: %v", err)
+				encounteredError = true
+			}
+		}
+	}
+	return encounteredError
 }
 
 // SyncLoop runs periodic work.  This is expected to run as a goroutine or as the main loop of the app.  It does not return.