diff --git a/cmd/kube-proxy/app/server.go b/cmd/kube-proxy/app/server.go index 47f5933d10..5ef9169121 100644 --- a/cmd/kube-proxy/app/server.go +++ b/cmd/kube-proxy/app/server.go @@ -19,6 +19,7 @@ limitations under the License. package app import ( + "errors" "net" "net/http" _ "net/http/pprof" @@ -62,6 +63,7 @@ type ProxyServer struct { SyncPeriod time.Duration nodeRef *api.ObjectReference // Reference to this node. MasqueradeAll bool + CleanupAndExit bool } // NewProxyServer creates a new ProxyServer object with default parameters @@ -90,10 +92,28 @@ func (s *ProxyServer) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.ForceUserspaceProxy, "legacy-userspace-proxy", true, "Use the legacy userspace proxy (instead of the pure iptables proxy).") fs.DurationVar(&s.SyncPeriod, "iptables-sync-period", 5*time.Second, "How often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.") fs.BoolVar(&s.MasqueradeAll, "masquerade-all", false, "If using the pure iptables proxy, SNAT everything") + fs.BoolVar(&s.CleanupAndExit, "cleanup-iptables", false, "If true cleanup iptables rules and exit.") } -// Run runs the specified ProxyServer. This should never exit. +// Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set). func (s *ProxyServer) Run(_ []string) error { + protocol := utiliptables.ProtocolIpv4 + if s.BindAddress.To4() == nil { + protocol = utiliptables.ProtocolIpv6 + } + + // remove iptables rules and exit + if s.CleanupAndExit { + execer := exec.New() + ipt := utiliptables.New(execer, protocol) + encounteredError := userspace.CleanupLeftovers(ipt) + encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError + if encounteredError { + return errors.New("Encountered an error while tearing down rules.") + } + return nil + } + // TODO(vmarmol): Use container config for this. oomAdjuster := oom.NewOomAdjuster() if err := oomAdjuster.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil { @@ -145,11 +165,6 @@ func (s *ProxyServer) Run(_ []string) error { serviceConfig := config.NewServiceConfig() endpointsConfig := config.NewEndpointsConfig() - protocol := utiliptables.ProtocolIpv4 - if s.BindAddress.To4() == nil { - protocol = utiliptables.ProtocolIpv6 - } - var proxier proxy.ProxyProvider var endpointsHandler config.EndpointsConfigHandler @@ -162,12 +177,17 @@ func (s *ProxyServer) Run(_ []string) error { glog.V(2).Info("Using iptables Proxier.") execer := exec.New() - proxierIptables, err := iptables.NewProxier(utiliptables.New(execer, protocol), execer, s.SyncPeriod, s.MasqueradeAll) + ipt := utiliptables.New(execer, protocol) + proxierIptables, err := iptables.NewProxier(ipt, execer, s.SyncPeriod, s.MasqueradeAll) if err != nil { glog.Fatalf("Unable to create proxier: %v", err) } proxier = proxierIptables endpointsHandler = proxierIptables + // No turning back. Remove artifacts that might still exist from the userspace Proxier. + glog.V(2).Info("Tearing down userspace rules. Errors here are acceptable.") + userspace.CleanupLeftovers(ipt) + } else { glog.V(2).Info("Using userspace Proxier.") // This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for @@ -176,11 +196,16 @@ func (s *ProxyServer) Run(_ []string) error { // set EndpointsConfigHandler to our loadBalancer endpointsHandler = loadBalancer - proxierUserspace, err := userspace.NewProxier(loadBalancer, s.BindAddress, utiliptables.New(exec.New(), protocol), s.PortRange, s.SyncPeriod) + execer := exec.New() + ipt := utiliptables.New(execer, protocol) + proxierUserspace, err := userspace.NewProxier(loadBalancer, s.BindAddress, ipt, s.PortRange, s.SyncPeriod) if err != nil { glog.Fatalf("Unable to create proxer: %v", err) } proxier = proxierUserspace + // Remove artifacts from the pure-iptables Proxier. + glog.V(2).Info("Tearing down pure-iptables proxy rules. Errors here are acceptable.") + iptables.CleanupLeftovers(ipt) } // Wire proxier to handle changes to services diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index bd51451c28..46c3fcbdae 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -31,6 +31,7 @@ certificate-authority cgroup-prefix cgroup-root chaos-chance +cleanup-iptables client-ca-file client-certificate client-key diff --git a/pkg/proxy/iptables/proxier.go b/pkg/proxy/iptables/proxier.go index 971e71e203..dd513c8052 100644 --- a/pkg/proxy/iptables/proxier.go +++ b/pkg/proxy/iptables/proxier.go @@ -176,10 +176,6 @@ func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlBridgeCallIptables, err) } - // No turning back. Remove artifacts that might still exist from the userspace Proxier. - glog.V(2).Info("Tearing down userspace rules. Errors here are acceptable.") - tearDownUserspaceIptables(ipt) - return &Proxier{ serviceMap: make(map[proxy.ServicePortName]*serviceInfo), syncPeriod: syncPeriod, @@ -188,47 +184,20 @@ func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod }, nil } -// Chains from the userspace proxy -// TODO: Remove these Chains and tearDownUserspaceIptables once the userspace Proxier has been removed. -var iptablesContainerPortalChain utiliptables.Chain = "KUBE-PORTALS-CONTAINER" -var iptablesHostPortalChain utiliptables.Chain = "KUBE-PORTALS-HOST" -var iptablesContainerNodePortChain utiliptables.Chain = "KUBE-NODEPORT-CONTAINER" -var iptablesHostNodePortChain utiliptables.Chain = "KUBE-NODEPORT-HOST" - -// tearDownUserspaceIptables removes all iptables rules and chains created by the userspace Proxier -func tearDownUserspaceIptables(ipt utiliptables.Interface) { - // NOTE: Warning, this needs to be kept in sync with the userspace Proxier, - // we want to ensure we remove all of the iptables rules it creates. - // Currently they are all in iptablesInit() - // Delete Rules first, then Flush and Delete Chains - args := []string{"-m", "comment", "--comment", "handle ClusterIPs; NOTE: this must be before the NodePort rules"} - if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainOutput, append(args, "-j", string(iptablesHostPortalChain))...); err != nil { - glog.Errorf("Error removing userspace rule: %v", err) +// CleanupLeftovers removes all iptables rules and chains created by the Proxier +// It returns true if an error was encountered. Errors are logged. +func CleanupLeftovers(ipt utiliptables.Interface) (encounteredError bool) { + //TODO: actually tear down all rules and chains. + args := []string{"-j", "KUBE-SERVICES"} + if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainOutput, args...); err != nil { + glog.Errorf("Error removing pure-iptables proxy rule: %v", err) + encounteredError = true } - if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPrerouting, append(args, "-j", string(iptablesContainerPortalChain))...); err != nil { - glog.Errorf("Error removing userspace rule: %v", err) - } - args = []string{"-m", "addrtype", "--dst-type", "LOCAL"} - args = append(args, "-m", "comment", "--comment", "handle service NodePorts; NOTE: this must be the last rule in the chain") - if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainOutput, append(args, "-j", string(iptablesHostNodePortChain))...); err != nil { - glog.Errorf("Error removing userspace rule: %v", err) - } - if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPrerouting, append(args, "-j", string(iptablesContainerNodePortChain))...); err != nil { - glog.Errorf("Error removing userspace rule: %v", err) - } - - // flush and delete chains. - chains := []utiliptables.Chain{iptablesContainerPortalChain, iptablesHostPortalChain, iptablesHostNodePortChain, iptablesContainerNodePortChain} - for _, c := range chains { - // flush chain, then if sucessful delete, delete will fail if flush fails. - if err := ipt.FlushChain(utiliptables.TableNAT, c); err != nil { - glog.Errorf("Error flushing userspace chain: %v", err) - } else { - if err = ipt.DeleteChain(utiliptables.TableNAT, c); err != nil { - glog.Errorf("Error flushing userspace chain: %v", err) - } - } + if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPrerouting, args...); err != nil { + glog.Errorf("Error removing pure-iptables proxy rule: %v", err) + encounteredError = true } + return encounteredError } func (proxier *Proxier) sameConfig(info *serviceInfo, service *api.Service, port *api.ServicePort) bool { diff --git a/pkg/proxy/userspace/proxier.go b/pkg/proxy/userspace/proxier.go index 8084a01c16..255442071b 100644 --- a/pkg/proxy/userspace/proxier.go +++ b/pkg/proxy/userspace/proxier.go @@ -134,8 +134,6 @@ func createProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables if proxyPorts == nil { proxyPorts = newPortAllocator(util.PortRange{}) } - glog.V(2).Info("Tearing down pure-iptables proxy rules. Errors here are acceptable.") - tearDownIptablesProxierRules(iptables) // Set up the iptables foundations we need. if err := iptablesInit(iptables); err != nil { return nil, fmt.Errorf("failed to initialize iptables: %v", err) @@ -157,17 +155,48 @@ func createProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables }, nil } -// remove the iptables rules from the pure iptables Proxier -func tearDownIptablesProxierRules(ipt iptables.Interface) { - //TODO: actually tear down all rules and chains. - //NOTE: this needs to be kept in sync with the proxy/iptables Proxier's rules. - args := []string{"-j", "KUBE-SERVICES"} - if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, args...); err != nil { - glog.Errorf("Error removing pure-iptables proxy rule: %v", err) +// CleanupLeftovers removes all iptables rules and chains created by the Proxier +// It returns true if an error was encountered. Errors are logged. +func CleanupLeftovers(ipt iptables.Interface) (encounteredError bool) { + // NOTE: Warning, this needs to be kept in sync with the userspace Proxier, + // we want to ensure we remove all of the iptables rules it creates. + // Currently they are all in iptablesInit() + // Delete Rules first, then Flush and Delete Chains + args := []string{"-m", "comment", "--comment", "handle ClusterIPs; NOTE: this must be before the NodePort rules"} + if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, append(args, "-j", string(iptablesHostPortalChain))...); err != nil { + glog.Errorf("Error removing userspace rule: %v", err) + encounteredError = true } - if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, args...); err != nil { - glog.Errorf("Error removing pure-iptables proxy rule: %v", err) + if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, append(args, "-j", string(iptablesContainerPortalChain))...); err != nil { + glog.Errorf("Error removing userspace rule: %v", err) + encounteredError = true } + args = []string{"-m", "addrtype", "--dst-type", "LOCAL"} + args = append(args, "-m", "comment", "--comment", "handle service NodePorts; NOTE: this must be the last rule in the chain") + if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, append(args, "-j", string(iptablesHostNodePortChain))...); err != nil { + glog.Errorf("Error removing userspace rule: %v", err) + encounteredError = true + } + if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, append(args, "-j", string(iptablesContainerNodePortChain))...); err != nil { + glog.Errorf("Error removing userspace rule: %v", err) + encounteredError = true + } + + // flush and delete chains. + chains := []iptables.Chain{iptablesContainerPortalChain, iptablesHostPortalChain, iptablesHostNodePortChain, iptablesContainerNodePortChain} + for _, c := range chains { + // flush chain, then if sucessful delete, delete will fail if flush fails. + if err := ipt.FlushChain(iptables.TableNAT, c); err != nil { + glog.Errorf("Error flushing userspace chain: %v", err) + encounteredError = true + } else { + if err = ipt.DeleteChain(iptables.TableNAT, c); err != nil { + glog.Errorf("Error flushing userspace chain: %v", err) + encounteredError = true + } + } + } + return encounteredError } // SyncLoop runs periodic work. This is expected to run as a goroutine or as the main loop of the app. It does not return.