github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add --cleanup-iptables flag to kube-proxy 

Adds a flag to cleanup iptables rules created by kube-proxy per
https://github.com/mesosphere/kubernetes-mesos/issues/353#issuecomment-1
27382832
pull/6/head
BenTheElder 2015-08-20 13:01:37 -04:00
parent 65f4ebd927
commit 81ab51709a
4 changed files with 86 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
cmd/kube-proxy/app/server.go
View File

@ -19,6 +19,7 @@ limitations under the License.
package app
import (
"errors"
"net"
"net/http"
_ "net/http/pprof"
@ -62,6 +63,7 @@ type ProxyServer struct {
SyncPeriod time.Duration
nodeRef *api.ObjectReference // Reference to this node.
MasqueradeAll bool
CleanupAndExit bool
}
// NewProxyServer creates a new ProxyServer object with default parameters
@ -90,10 +92,28 @@ func (s *ProxyServer) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&s.ForceUserspaceProxy, "legacy-userspace-proxy", true, "Use the legacy userspace proxy (instead of the pure iptables proxy).")
fs.DurationVar(&s.SyncPeriod, "iptables-sync-period", 5*time.Second, "How often iptables rules are refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0.")
fs.BoolVar(&s.MasqueradeAll, "masquerade-all", false, "If using the pure iptables proxy, SNAT everything")
fs.BoolVar(&s.CleanupAndExit, "cleanup-iptables", false, "If true cleanup iptables rules and exit.")
}
// Run runs the specified ProxyServer. This should never exit.
// Run runs the specified ProxyServer. This should never exit (unless CleanupAndExit is set).
func (s *ProxyServer) Run(_ []string) error {
protocol := utiliptables.ProtocolIpv4
if s.BindAddress.To4() == nil {
protocol = utiliptables.ProtocolIpv6
}
// remove iptables rules and exit
if s.CleanupAndExit {
execer := exec.New()
ipt := utiliptables.New(execer, protocol)
encounteredError := userspace.CleanupLeftovers(ipt)
encounteredError = iptables.CleanupLeftovers(ipt) || encounteredError
if encounteredError {
return errors.New("Encountered an error while tearing down rules.")
}
return nil
}
// TODO(vmarmol): Use container config for this.
oomAdjuster := oom.NewOomAdjuster()
if err := oomAdjuster.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
@ -145,11 +165,6 @@ func (s *ProxyServer) Run(_ []string) error {
serviceConfig := config.NewServiceConfig()
endpointsConfig := config.NewEndpointsConfig()
protocol := utiliptables.ProtocolIpv4
if s.BindAddress.To4() == nil {
protocol = utiliptables.ProtocolIpv6
}
var proxier proxy.ProxyProvider
var endpointsHandler config.EndpointsConfigHandler
@ -162,12 +177,17 @@ func (s *ProxyServer) Run(_ []string) error {
glog.V(2).Info("Using iptables Proxier.")
execer := exec.New()
proxierIptables, err := iptables.NewProxier(utiliptables.New(execer, protocol), execer, s.SyncPeriod, s.MasqueradeAll)
ipt := utiliptables.New(execer, protocol)
proxierIptables, err := iptables.NewProxier(ipt, execer, s.SyncPeriod, s.MasqueradeAll)
if err != nil {
glog.Fatalf("Unable to create proxier: %v", err)
}
proxier = proxierIptables
endpointsHandler = proxierIptables
// No turning back. Remove artifacts that might still exist from the userspace Proxier.
glog.V(2).Info("Tearing down userspace rules. Errors here are acceptable.")
userspace.CleanupLeftovers(ipt)
} else {
glog.V(2).Info("Using userspace Proxier.")
// This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for
@ -176,11 +196,16 @@ func (s *ProxyServer) Run(_ []string) error {
// set EndpointsConfigHandler to our loadBalancer
endpointsHandler = loadBalancer
proxierUserspace, err := userspace.NewProxier(loadBalancer, s.BindAddress, utiliptables.New(exec.New(), protocol), s.PortRange, s.SyncPeriod)
execer := exec.New()
ipt := utiliptables.New(execer, protocol)
proxierUserspace, err := userspace.NewProxier(loadBalancer, s.BindAddress, ipt, s.PortRange, s.SyncPeriod)
if err != nil {
glog.Fatalf("Unable to create proxer: %v", err)
}
proxier = proxierUserspace
// Remove artifacts from the pure-iptables Proxier.
glog.V(2).Info("Tearing down pure-iptables proxy rules. Errors here are acceptable.")
iptables.CleanupLeftovers(ipt)
}
// Wire proxier to handle changes to services

1
hack/verify-flags/known-flags.txt
View File

@ -31,6 +31,7 @@ certificate-authority
cgroup-prefix
cgroup-root
chaos-chance
cleanup-iptables
client-ca-file
client-certificate
client-key

55
pkg/proxy/iptables/proxier.go
View File

@ -176,10 +176,6 @@ func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod
return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlBridgeCallIptables, err)
}
// No turning back. Remove artifacts that might still exist from the userspace Proxier.
glog.V(2).Info("Tearing down userspace rules. Errors here are acceptable.")
tearDownUserspaceIptables(ipt)
return &Proxier{
serviceMap: make(map[proxy.ServicePortName]*serviceInfo),
syncPeriod: syncPeriod,
@ -188,47 +184,20 @@ func NewProxier(ipt utiliptables.Interface, exec utilexec.Interface, syncPeriod
}, nil
}
// Chains from the userspace proxy
// TODO: Remove these Chains and tearDownUserspaceIptables once the userspace Proxier has been removed.
var iptablesContainerPortalChain utiliptables.Chain = "KUBE-PORTALS-CONTAINER"
var iptablesHostPortalChain utiliptables.Chain = "KUBE-PORTALS-HOST"
var iptablesContainerNodePortChain utiliptables.Chain = "KUBE-NODEPORT-CONTAINER"
var iptablesHostNodePortChain utiliptables.Chain = "KUBE-NODEPORT-HOST"
// tearDownUserspaceIptables removes all iptables rules and chains created by the userspace Proxier
func tearDownUserspaceIptables(ipt utiliptables.Interface) {
// NOTE: Warning, this needs to be kept in sync with the userspace Proxier,
// we want to ensure we remove all of the iptables rules it creates.
// Currently they are all in iptablesInit()
// Delete Rules first, then Flush and Delete Chains
args := []string{"-m", "comment", "--comment", "handle ClusterIPs; NOTE: this must be before the NodePort rules"}
if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainOutput, append(args, "-j", string(iptablesHostPortalChain))...); err != nil {
glog.Errorf("Error removing userspace rule: %v", err)
}
if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPrerouting, append(args, "-j", string(iptablesContainerPortalChain))...); err != nil {
glog.Errorf("Error removing userspace rule: %v", err)
}
args = []string{"-m", "addrtype", "--dst-type", "LOCAL"}
args = append(args, "-m", "comment", "--comment", "handle service NodePorts; NOTE: this must be the last rule in the chain")
if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainOutput, append(args, "-j", string(iptablesHostNodePortChain))...); err != nil {
glog.Errorf("Error removing userspace rule: %v", err)
}
if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPrerouting, append(args, "-j", string(iptablesContainerNodePortChain))...); err != nil {
glog.Errorf("Error removing userspace rule: %v", err)
}
// flush and delete chains.
chains := []utiliptables.Chain{iptablesContainerPortalChain, iptablesHostPortalChain, iptablesHostNodePortChain, iptablesContainerNodePortChain}
for _, c := range chains {
// flush chain, then if sucessful delete, delete will fail if flush fails.
if err := ipt.FlushChain(utiliptables.TableNAT, c); err != nil {
glog.Errorf("Error flushing userspace chain: %v", err)
} else {
if err = ipt.DeleteChain(utiliptables.TableNAT, c); err != nil {
glog.Errorf("Error flushing userspace chain: %v", err)
}
// CleanupLeftovers removes all iptables rules and chains created by the Proxier
// It returns true if an error was encountered. Errors are logged.
func CleanupLeftovers(ipt utiliptables.Interface) (encounteredError bool) {
//TODO: actually tear down all rules and chains.
args := []string{"-j", "KUBE-SERVICES"}
if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainOutput, args...); err != nil {
glog.Errorf("Error removing pure-iptables proxy rule: %v", err)
encounteredError = true
}
if err := ipt.DeleteRule(utiliptables.TableNAT, utiliptables.ChainPrerouting, args...); err != nil {
glog.Errorf("Error removing pure-iptables proxy rule: %v", err)
encounteredError = true
}
return encounteredError
}
func (proxier *Proxier) sameConfig(info *serviceInfo, service *api.Service, port *api.ServicePort) bool {

51
pkg/proxy/userspace/proxier.go
View File

@ -134,8 +134,6 @@ func createProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables
if proxyPorts == nil {
proxyPorts = newPortAllocator(util.PortRange{})
}
glog.V(2).Info("Tearing down pure-iptables proxy rules. Errors here are acceptable.")
tearDownIptablesProxierRules(iptables)
// Set up the iptables foundations we need.
if err := iptablesInit(iptables); err != nil {
return nil, fmt.Errorf("failed to initialize iptables: %v", err)
@ -157,17 +155,48 @@ func createProxier(loadBalancer LoadBalancer, listenIP net.IP, iptables iptables
}, nil
}
// remove the iptables rules from the pure iptables Proxier
func tearDownIptablesProxierRules(ipt iptables.Interface) {
//TODO: actually tear down all rules and chains.
//NOTE: this needs to be kept in sync with the proxy/iptables Proxier's rules.
args := []string{"-j", "KUBE-SERVICES"}
if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, args...); err != nil {
glog.Errorf("Error removing pure-iptables proxy rule: %v", err)
// CleanupLeftovers removes all iptables rules and chains created by the Proxier
// It returns true if an error was encountered. Errors are logged.
func CleanupLeftovers(ipt iptables.Interface) (encounteredError bool) {
// NOTE: Warning, this needs to be kept in sync with the userspace Proxier,
// we want to ensure we remove all of the iptables rules it creates.
// Currently they are all in iptablesInit()
// Delete Rules first, then Flush and Delete Chains
args := []string{"-m", "comment", "--comment", "handle ClusterIPs; NOTE: this must be before the NodePort rules"}
if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, append(args, "-j", string(iptablesHostPortalChain))...); err != nil {
glog.Errorf("Error removing userspace rule: %v", err)
encounteredError = true
}
if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, args...); err != nil {
glog.Errorf("Error removing pure-iptables proxy rule: %v", err)
if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, append(args, "-j", string(iptablesContainerPortalChain))...); err != nil {
glog.Errorf("Error removing userspace rule: %v", err)
encounteredError = true
}
args = []string{"-m", "addrtype", "--dst-type", "LOCAL"}
args = append(args, "-m", "comment", "--comment", "handle service NodePorts; NOTE: this must be the last rule in the chain")
if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainOutput, append(args, "-j", string(iptablesHostNodePortChain))...); err != nil {
glog.Errorf("Error removing userspace rule: %v", err)
encounteredError = true
}
if err := ipt.DeleteRule(iptables.TableNAT, iptables.ChainPrerouting, append(args, "-j", string(iptablesContainerNodePortChain))...); err != nil {
glog.Errorf("Error removing userspace rule: %v", err)
encounteredError = true
}
// flush and delete chains.
chains := []iptables.Chain{iptablesContainerPortalChain, iptablesHostPortalChain, iptablesHostNodePortChain, iptablesContainerNodePortChain}
for _, c := range chains {
// flush chain, then if sucessful delete, delete will fail if flush fails.
if err := ipt.FlushChain(iptables.TableNAT, c); err != nil {
glog.Errorf("Error flushing userspace chain: %v", err)
encounteredError = true
} else {
if err = ipt.DeleteChain(iptables.TableNAT, c); err != nil {
glog.Errorf("Error flushing userspace chain: %v", err)
encounteredError = true
}
}
}
return encounteredError
}
// SyncLoop runs periodic work. This is expected to run as a goroutine or as the main loop of the app. It does not return.