From 10664ee8c6c76ea22b265d1cdddee71f3941803c Mon Sep 17 00:00:00 2001 From: Weibin Lin Date: Tue, 8 May 2018 20:25:32 +0800 Subject: [PATCH] Add comments for ipset entries in kube-proxy --- pkg/proxy/ipvs/ipset.go | 51 +++++++++++++++--------------- pkg/proxy/ipvs/ipset_test.go | 2 +- pkg/proxy/ipvs/proxier.go | 42 +++++++++++++++--------- pkg/proxy/ipvs/proxier_test.go | 58 +++++++++++++++++----------------- pkg/util/ipset/ipset.go | 8 +++-- 5 files changed, 88 insertions(+), 73 deletions(-) diff --git a/pkg/proxy/ipvs/ipset.go b/pkg/proxy/ipvs/ipset.go index 54634e342d..7f361d6403 100644 --- a/pkg/proxy/ipvs/ipset.go +++ b/pkg/proxy/ipvs/ipset.go @@ -28,41 +28,41 @@ const ( // MinIPSetCheckVersion is the min ipset version we need. IPv6 is supported in ipset 6.x MinIPSetCheckVersion = "6.0" - // KubeLoopBackIPSet is used to store endpoints dst ip:port, source ip for solving hairpin purpose. - KubeLoopBackIPSet = "KUBE-LOOP-BACK" + kubeLoopBackIPSetComment = "Kubernetes endpoints dst ip:port, source ip for solving hairpin purpose" + kubeLoopBackIPSet = "KUBE-LOOP-BACK" - // KubeClusterIPSet is used to store service cluster ip + port for masquerade purpose. - KubeClusterIPSet = "KUBE-CLUSTER-IP" + kubeClusterIPSetComment = "Kubernetes service cluster ip + port for masquerade purpose" + kubeClusterIPSet = "KUBE-CLUSTER-IP" - // KubeExternalIPSet is used to store service external ip + port for masquerade and filter purpose. - KubeExternalIPSet = "KUBE-EXTERNAL-IP" + kubeExternalIPSetComment = "Kubernetes service external ip + port for masquerade and filter purpose" + kubeExternalIPSet = "KUBE-EXTERNAL-IP" - // KubeLoadBalancerSet is used to store service load balancer ingress ip + port, it is the service lb portal. - KubeLoadBalancerSet = "KUBE-LOAD-BALANCER" + kubeLoadBalancerSetComment = "Kubernetes service lb portal" + kubeLoadBalancerSet = "KUBE-LOAD-BALANCER" - // KubeLoadBalancerLocalSet is used to store service load balancer ingress ip + port with externalTrafficPolicy=local. - KubeLoadBalancerLocalSet = "KUBE-LOAD-BALANCER-LOCAL" + kubeLoadBalancerLocalSetComment = "Kubernetes service load balancer ip + port with externalTrafficPolicy=local" + kubeLoadBalancerLocalSet = "KUBE-LOAD-BALANCER-LOCAL" - // KubeLoadbalancerFWSet is used to store service load balancer ingress ip + port for load balancer with sourceRange. - KubeLoadbalancerFWSet = "KUBE-LOAD-BALANCER-FW" + kubeLoadbalancerFWSetComment = "Kubernetes service load balancer ip + port for load balancer with sourceRange" + kubeLoadbalancerFWSet = "KUBE-LOAD-BALANCER-FW" - // KubeLoadBalancerSourceIPSet is used to store service load balancer ingress ip + port + source IP for packet filter purpose. - KubeLoadBalancerSourceIPSet = "KUBE-LOAD-BALANCER-SOURCE-IP" + kubeLoadBalancerSourceIPSetComment = "Kubernetes service load balancer ip + port + source IP for packet filter purpose" + kubeLoadBalancerSourceIPSet = "KUBE-LOAD-BALANCER-SOURCE-IP" - // KubeLoadBalancerSourceCIDRSet is used to store service load balancer ingress ip + port + source cidr for packet filter purpose. - KubeLoadBalancerSourceCIDRSet = "KUBE-LOAD-BALANCER-SOURCE-CIDR" + kubeLoadBalancerSourceCIDRSetComment = "Kubernetes service load balancer ip + port + source cidr for packet filter purpose" + kubeLoadBalancerSourceCIDRSet = "KUBE-LOAD-BALANCER-SOURCE-CIDR" - // KubeNodePortSetTCP is used to store the nodeport TCP port for masquerade purpose. - KubeNodePortSetTCP = "KUBE-NODE-PORT-TCP" + kubeNodePortSetTCPComment = "Kubernetes nodeport TCP port for masquerade purpose" + kubeNodePortSetTCP = "KUBE-NODE-PORT-TCP" - // KubeNodePortLocalSetTCP is used to store the nodeport TCP port with externalTrafficPolicy=local. - KubeNodePortLocalSetTCP = "KUBE-NODE-PORT-LOCAL-TCP" + kubeNodePortLocalSetTCPComment = "Kubernetes nodeport TCP port with externalTrafficPolicy=local" + kubeNodePortLocalSetTCP = "KUBE-NODE-PORT-LOCAL-TCP" - // KubeNodePortSetUDP is used to store the nodeport UDP port for masquerade purpose. - KubeNodePortSetUDP = "KUBE-NODE-PORT-UDP" + kubeNodePortSetUDPComment = "Kubernetes nodeport UDP port for masquerade purpose" + kubeNodePortSetUDP = "KUBE-NODE-PORT-UDP" - // KubeNodePortLocalSetUDP is used to store the nodeport UDP port with externalTrafficPolicy=local. - KubeNodePortLocalSetUDP = "KUBE-NODE-PORT-LOCAL-UDP" + kubeNodePortLocalSetUDPComment = "Kubernetes nodeport UDP port with externalTrafficPolicy=local" + kubeNodePortLocalSetUDP = "KUBE-NODE-PORT-LOCAL-UDP" ) // IPSetVersioner can query the current ipset version. @@ -81,7 +81,7 @@ type IPSet struct { } // NewIPSet initialize a new IPSet struct -func NewIPSet(handle utilipset.Interface, name string, setType utilipset.Type, isIPv6 bool) *IPSet { +func NewIPSet(handle utilipset.Interface, name string, setType utilipset.Type, isIPv6 bool, comment string) *IPSet { hashFamily := utilipset.ProtocolFamilyIPV4 if isIPv6 { hashFamily = utilipset.ProtocolFamilyIPV6 @@ -91,6 +91,7 @@ func NewIPSet(handle utilipset.Interface, name string, setType utilipset.Type, i Name: name, SetType: setType, HashFamily: hashFamily, + Comment: comment, }, activeEntries: sets.NewString(), handle: handle, diff --git a/pkg/proxy/ipvs/ipset_test.go b/pkg/proxy/ipvs/ipset_test.go index 4610aceb9e..2e6f667de5 100644 --- a/pkg/proxy/ipvs/ipset_test.go +++ b/pkg/proxy/ipvs/ipset_test.go @@ -182,7 +182,7 @@ func TestSyncIPSetEntries(t *testing.T) { } for i := range testCases { - set := NewIPSet(fakeipset.NewFake(testIPSetVersion), testCases[i].set.Name, testCases[i].setType, testCases[i].ipv6) + set := NewIPSet(fakeipset.NewFake(testIPSetVersion), testCases[i].set.Name, testCases[i].setType, testCases[i].ipv6, "comment-"+testCases[i].set.Name) if err := set.handle.CreateSet(&set.IPSet, true); err != nil { t.Errorf("Unexpected error: %v", err) diff --git a/pkg/proxy/ipvs/proxier.go b/pkg/proxy/ipvs/proxier.go index 38df2bb4d2..b2d89ff863 100644 --- a/pkg/proxy/ipvs/proxier.go +++ b/pkg/proxy/ipvs/proxier.go @@ -353,18 +353,18 @@ func NewProxier(ipt utiliptables.Interface, filterRules: bytes.NewBuffer(nil), netlinkHandle: NewNetLinkHandle(), ipset: ipset, - loopbackSet: NewIPSet(ipset, KubeLoopBackIPSet, utilipset.HashIPPortIP, isIPv6), - clusterIPSet: NewIPSet(ipset, KubeClusterIPSet, utilipset.HashIPPort, isIPv6), - externalIPSet: NewIPSet(ipset, KubeExternalIPSet, utilipset.HashIPPort, isIPv6), - lbSet: NewIPSet(ipset, KubeLoadBalancerSet, utilipset.HashIPPort, isIPv6), - lbFWSet: NewIPSet(ipset, KubeLoadbalancerFWSet, utilipset.HashIPPort, isIPv6), - lbLocalSet: NewIPSet(ipset, KubeLoadBalancerLocalSet, utilipset.HashIPPort, isIPv6), - lbWhiteListIPSet: NewIPSet(ipset, KubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, isIPv6), - lbWhiteListCIDRSet: NewIPSet(ipset, KubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, isIPv6), - nodePortSetTCP: NewIPSet(ipset, KubeNodePortSetTCP, utilipset.BitmapPort, false), - nodePortLocalSetTCP: NewIPSet(ipset, KubeNodePortLocalSetTCP, utilipset.BitmapPort, false), - nodePortSetUDP: NewIPSet(ipset, KubeNodePortSetUDP, utilipset.BitmapPort, false), - nodePortLocalSetUDP: NewIPSet(ipset, KubeNodePortLocalSetUDP, utilipset.BitmapPort, false), + loopbackSet: NewIPSet(ipset, kubeLoopBackIPSet, utilipset.HashIPPortIP, isIPv6, kubeLoopBackIPSetComment), + clusterIPSet: NewIPSet(ipset, kubeClusterIPSet, utilipset.HashIPPort, isIPv6, kubeClusterIPSetComment), + externalIPSet: NewIPSet(ipset, kubeExternalIPSet, utilipset.HashIPPort, isIPv6, kubeExternalIPSetComment), + lbSet: NewIPSet(ipset, kubeLoadBalancerSet, utilipset.HashIPPort, isIPv6, kubeLoadBalancerSetComment), + lbFWSet: NewIPSet(ipset, kubeLoadbalancerFWSet, utilipset.HashIPPort, isIPv6, kubeLoadbalancerFWSetComment), + lbLocalSet: NewIPSet(ipset, kubeLoadBalancerLocalSet, utilipset.HashIPPort, isIPv6, kubeLoadBalancerLocalSetComment), + lbWhiteListIPSet: NewIPSet(ipset, kubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, isIPv6, kubeLoadBalancerSourceIPSetComment), + lbWhiteListCIDRSet: NewIPSet(ipset, kubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, isIPv6, kubeLoadBalancerSourceCIDRSetComment), + nodePortSetTCP: NewIPSet(ipset, kubeNodePortSetTCP, utilipset.BitmapPort, false, kubeNodePortSetTCPComment), + nodePortLocalSetTCP: NewIPSet(ipset, kubeNodePortLocalSetTCP, utilipset.BitmapPort, false, kubeNodePortLocalSetTCPComment), + nodePortSetUDP: NewIPSet(ipset, kubeNodePortSetUDP, utilipset.BitmapPort, false, kubeNodePortSetUDPComment), + nodePortLocalSetUDP: NewIPSet(ipset, kubeNodePortLocalSetUDP, utilipset.BitmapPort, false, kubeNodePortLocalSetUDPComment), nodePortAddresses: nodePortAddresses, networkInterfacer: utilproxy.RealNetwork{}, } @@ -558,9 +558,9 @@ func CleanupLeftovers(ipvs utilipvs.Interface, ipt utiliptables.Interface, ipset encounteredError = cleanupIptablesLeftovers(ipt) || encounteredError // Destroy ip sets created by ipvs Proxier. We should call it after cleaning up // iptables since we can NOT delete ip set which is still referenced by iptables. - ipSetsToDestroy := []string{KubeLoopBackIPSet, KubeClusterIPSet, KubeLoadBalancerSet, KubeNodePortSetTCP, KubeNodePortSetUDP, - KubeExternalIPSet, KubeLoadbalancerFWSet, KubeLoadBalancerSourceIPSet, KubeLoadBalancerSourceCIDRSet, - KubeLoadBalancerLocalSet, KubeNodePortLocalSetUDP, KubeNodePortLocalSetTCP} + ipSetsToDestroy := []string{kubeClusterIPSet, kubeClusterIPSet, kubeLoadBalancerSet, kubeNodePortSetTCP, kubeNodePortSetUDP, + kubeExternalIPSet, kubeLoadbalancerFWSet, kubeLoadBalancerSourceIPSet, kubeLoadBalancerSourceCIDRSet, + kubeLoadBalancerLocalSet, kubeNodePortLocalSetUDP, kubeNodePortLocalSetTCP} for _, set := range ipSetsToDestroy { err = ipset.DestroySet(set) if err != nil { @@ -1191,6 +1191,7 @@ func (proxier *Proxier) syncProxyRules() { if !proxier.loopbackSet.isEmpty() { args = append(args[:0], "-A", string(kubePostroutingChain), + "-m", "comment", "--comment", proxier.loopbackSet.Comment, "-m", "set", "--match-set", proxier.loopbackSet.Name, "dst,dst,src", ) @@ -1199,6 +1200,7 @@ func (proxier *Proxier) syncProxyRules() { if !proxier.clusterIPSet.isEmpty() { args = append(args[:0], "-A", string(kubeServicesChain), + "-m", "comment", "--comment", proxier.clusterIPSet.Comment, "-m", "set", "--match-set", proxier.clusterIPSet.Name, "dst,dst", ) @@ -1217,6 +1219,7 @@ func (proxier *Proxier) syncProxyRules() { // Build masquerade rules for packets to external IPs. args = append(args[:0], "-A", string(kubeServicesChain), + "-m", "comment", "--comment", proxier.externalIPSet.Comment, "-m", "set", "--match-set", proxier.externalIPSet.Name, "dst,dst", ) @@ -1238,6 +1241,7 @@ func (proxier *Proxier) syncProxyRules() { // Build masquerade rules for packets which cross node visit load balancer ingress IPs. args = append(args[:0], "-A", string(kubeServicesChain), + "-m", "comment", "--comment", proxier.lbSet.Comment, "-m", "set", "--match-set", proxier.lbSet.Name, "dst,dst", ) @@ -1247,6 +1251,7 @@ func (proxier *Proxier) syncProxyRules() { if !proxier.lbFWSet.isEmpty() { args = append(args[:0], "-A", string(KubeLoadBalancerChain), + "-m", "comment", "--comment", proxier.lbFWSet.Comment, "-m", "set", "--match-set", proxier.lbFWSet.Name, "dst,dst", ) @@ -1255,6 +1260,7 @@ func (proxier *Proxier) syncProxyRules() { if !proxier.lbWhiteListCIDRSet.isEmpty() { args = append(args[:0], "-A", string(KubeFireWallChain), + "-m", "comment", "--comment", proxier.lbWhiteListCIDRSet.Comment, "-m", "set", "--match-set", proxier.lbWhiteListCIDRSet.Name, "dst,dst,src", ) @@ -1263,6 +1269,7 @@ func (proxier *Proxier) syncProxyRules() { if !proxier.lbWhiteListIPSet.isEmpty() { args = append(args[:0], "-A", string(KubeFireWallChain), + "-m", "comment", "--comment", proxier.lbWhiteListIPSet.Comment, "-m", "set", "--match-set", proxier.lbWhiteListIPSet.Name, "dst,dst,src", ) @@ -1279,6 +1286,7 @@ func (proxier *Proxier) syncProxyRules() { if !proxier.lbLocalSet.isEmpty() { args = append(args[:0], "-A", string(KubeLoadBalancerChain), + "-m", "comment", "--comment", proxier.lbLocalSet.Comment, "-m", "set", "--match-set", proxier.lbLocalSet.Name, "dst,dst", ) @@ -1297,6 +1305,7 @@ func (proxier *Proxier) syncProxyRules() { args = append(args[:0], "-A", string(kubeServicesChain), "-m", "tcp", "-p", "tcp", + "-m", "comment", "--comment", proxier.nodePortSetTCP.Comment, "-m", "set", "--match-set", proxier.nodePortSetTCP.Name, "dst", ) @@ -1305,6 +1314,7 @@ func (proxier *Proxier) syncProxyRules() { if !proxier.nodePortLocalSetTCP.isEmpty() { args = append(args[:0], "-A", string(KubeNodePortChain), + "-m", "comment", "--comment", proxier.nodePortLocalSetTCP.Comment, "-m", "set", "--match-set", proxier.nodePortLocalSetTCP.Name, "dst", ) @@ -1323,6 +1333,7 @@ func (proxier *Proxier) syncProxyRules() { args = append(args[:0], "-A", string(kubeServicesChain), "-m", "udp", "-p", "udp", + "-m", "comment", "--comment", proxier.nodePortSetUDP.Comment, "-m", "set", "--match-set", proxier.nodePortSetUDP.Name, "dst", ) @@ -1330,6 +1341,7 @@ func (proxier *Proxier) syncProxyRules() { if !proxier.nodePortLocalSetUDP.isEmpty() { args = append(args[:0], "-A", string(KubeNodePortChain), + "-m", "comment", "--comment", proxier.nodePortLocalSetUDP.Comment, "-m", "set", "--match-set", proxier.nodePortLocalSetUDP.Name, "dst", ) diff --git a/pkg/proxy/ipvs/proxier_test.go b/pkg/proxy/ipvs/proxier_test.go index bcc506ddb2..611d30ae00 100644 --- a/pkg/proxy/ipvs/proxier_test.go +++ b/pkg/proxy/ipvs/proxier_test.go @@ -142,18 +142,18 @@ func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset u filterChains: bytes.NewBuffer(nil), filterRules: bytes.NewBuffer(nil), netlinkHandle: netlinktest.NewFakeNetlinkHandle(), - loopbackSet: NewIPSet(ipset, KubeLoopBackIPSet, utilipset.HashIPPortIP, false), - clusterIPSet: NewIPSet(ipset, KubeClusterIPSet, utilipset.HashIPPort, false), - externalIPSet: NewIPSet(ipset, KubeExternalIPSet, utilipset.HashIPPort, false), - lbSet: NewIPSet(ipset, KubeLoadBalancerSet, utilipset.HashIPPort, false), - lbFWSet: NewIPSet(ipset, KubeLoadbalancerFWSet, utilipset.HashIPPort, false), - lbLocalSet: NewIPSet(ipset, KubeLoadBalancerLocalSet, utilipset.HashIPPort, false), - lbWhiteListIPSet: NewIPSet(ipset, KubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, false), - lbWhiteListCIDRSet: NewIPSet(ipset, KubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, false), - nodePortSetTCP: NewIPSet(ipset, KubeNodePortSetTCP, utilipset.BitmapPort, false), - nodePortLocalSetTCP: NewIPSet(ipset, KubeNodePortLocalSetTCP, utilipset.BitmapPort, false), - nodePortLocalSetUDP: NewIPSet(ipset, KubeNodePortLocalSetUDP, utilipset.BitmapPort, false), - nodePortSetUDP: NewIPSet(ipset, KubeNodePortSetUDP, utilipset.BitmapPort, false), + loopbackSet: NewIPSet(ipset, kubeLoopBackIPSet, utilipset.HashIPPortIP, false, kubeLoopBackIPSetComment), + clusterIPSet: NewIPSet(ipset, kubeClusterIPSet, utilipset.HashIPPort, false, kubeClusterIPSetComment), + externalIPSet: NewIPSet(ipset, kubeExternalIPSet, utilipset.HashIPPort, false, kubeExternalIPSetComment), + lbSet: NewIPSet(ipset, kubeLoadBalancerSet, utilipset.HashIPPort, false, kubeLoadBalancerSetComment), + lbFWSet: NewIPSet(ipset, kubeLoadbalancerFWSet, utilipset.HashIPPort, false, kubeLoadbalancerFWSetComment), + lbLocalSet: NewIPSet(ipset, kubeLoadBalancerLocalSet, utilipset.HashIPPort, false, kubeLoadBalancerLocalSetComment), + lbWhiteListIPSet: NewIPSet(ipset, kubeLoadBalancerSourceIPSet, utilipset.HashIPPortIP, false, kubeLoadBalancerSourceIPSetComment), + lbWhiteListCIDRSet: NewIPSet(ipset, kubeLoadBalancerSourceCIDRSet, utilipset.HashIPPortNet, false, kubeLoadBalancerSourceCIDRSetComment), + nodePortSetTCP: NewIPSet(ipset, kubeNodePortSetTCP, utilipset.BitmapPort, false, kubeNodePortSetTCPComment), + nodePortLocalSetTCP: NewIPSet(ipset, kubeNodePortLocalSetTCP, utilipset.BitmapPort, false, kubeNodePortSetTCPComment), + nodePortLocalSetUDP: NewIPSet(ipset, kubeNodePortLocalSetUDP, utilipset.BitmapPort, false, kubeNodePortLocalSetUDPComment), + nodePortSetUDP: NewIPSet(ipset, kubeNodePortSetUDP, utilipset.BitmapPort, false, kubeNodePortSetUDPComment), nodePortAddresses: make([]string, 0), networkInterfacer: proxyutiltest.NewFakeNetwork(), } @@ -876,18 +876,18 @@ func TestOnlyLocalNodePorts(t *testing.T) { SetType: utilipset.BitmapPort, } epIPSet := netlinktest.ExpectedIPSet{ - KubeNodePortSetTCP: {epEntry}, - KubeNodePortLocalSetTCP: {epEntry}, + kubeNodePortSetTCP: {epEntry}, + kubeNodePortLocalSetTCP: {epEntry}, } checkIPSet(t, fp, epIPSet) // Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ - JumpChain: string(KubeNodePortChain), MatchSet: KubeNodePortSetTCP, + JumpChain: string(KubeNodePortChain), MatchSet: kubeNodePortSetTCP, }}, string(KubeNodePortChain): {{ - JumpChain: "ACCEPT", MatchSet: KubeNodePortLocalSetTCP, + JumpChain: "ACCEPT", MatchSet: kubeNodePortLocalSetTCP, }, { JumpChain: string(KubeMarkMasqChain), MatchSet: "", }}, @@ -952,19 +952,19 @@ func TestLoadBalanceSourceRanges(t *testing.T) { // Check ipset entry epIPSet := netlinktest.ExpectedIPSet{ - KubeLoadBalancerSet: {{ + kubeLoadBalancerSet: {{ IP: svcLBIP, Port: svcPort, Protocol: strings.ToLower(string(api.ProtocolTCP)), SetType: utilipset.HashIPPort, }}, - KubeLoadbalancerFWSet: {{ + kubeLoadbalancerFWSet: {{ IP: svcLBIP, Port: svcPort, Protocol: strings.ToLower(string(api.ProtocolTCP)), SetType: utilipset.HashIPPort, }}, - KubeLoadBalancerSourceCIDRSet: {{ + kubeLoadBalancerSourceCIDRSet: {{ IP: svcLBIP, Port: svcPort, Protocol: strings.ToLower(string(api.ProtocolTCP)), @@ -977,15 +977,15 @@ func TestLoadBalanceSourceRanges(t *testing.T) { // Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ - JumpChain: string(KubeLoadBalancerChain), MatchSet: KubeLoadBalancerSet, + JumpChain: string(KubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet, }}, string(KubeLoadBalancerChain): {{ - JumpChain: string(KubeFireWallChain), MatchSet: KubeLoadbalancerFWSet, + JumpChain: string(KubeFireWallChain), MatchSet: kubeLoadbalancerFWSet, }, { JumpChain: string(KubeMarkMasqChain), MatchSet: "", }}, string(KubeFireWallChain): {{ - JumpChain: "RETURN", MatchSet: KubeLoadBalancerSourceCIDRSet, + JumpChain: "RETURN", MatchSet: kubeLoadBalancerSourceCIDRSet, }, { JumpChain: string(KubeMarkDropChain), MatchSet: "", }}, @@ -1050,9 +1050,9 @@ func TestAcceptIPVSTraffic(t *testing.T) { // Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): { - {JumpChain: "ACCEPT", MatchSet: KubeClusterIPSet}, - {JumpChain: "ACCEPT", MatchSet: KubeLoadBalancerSet}, - {JumpChain: "ACCEPT", MatchSet: KubeExternalIPSet}, + {JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet}, + {JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet}, + {JumpChain: "ACCEPT", MatchSet: kubeExternalIPSet}, }, } checkIptables(t, ipt, epIpt) @@ -1115,13 +1115,13 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { // check ipSet rules epIPSet := netlinktest.ExpectedIPSet{ - KubeLoadBalancerSet: {{ + kubeLoadBalancerSet: {{ IP: svcLBIP, Port: svcPort, Protocol: strings.ToLower(string(api.ProtocolTCP)), SetType: utilipset.HashIPPort, }}, - KubeLoadBalancerLocalSet: {{ + kubeLoadBalancerLocalSet: {{ IP: svcLBIP, Port: svcPort, Protocol: strings.ToLower(string(api.ProtocolTCP)), @@ -1133,10 +1133,10 @@ func TestOnlyLocalLoadBalancing(t *testing.T) { // Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ - JumpChain: string(KubeLoadBalancerChain), MatchSet: KubeLoadBalancerSet, + JumpChain: string(KubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet, }}, string(KubeLoadBalancerChain): {{ - JumpChain: "RETURN", MatchSet: KubeLoadBalancerLocalSet, + JumpChain: "RETURN", MatchSet: kubeLoadBalancerLocalSet, }, { JumpChain: string(KubeMarkMasqChain), MatchSet: "", }}, diff --git a/pkg/util/ipset/ipset.go b/pkg/util/ipset/ipset.go index 3615caefec..6ec5ca221d 100644 --- a/pkg/util/ipset/ipset.go +++ b/pkg/util/ipset/ipset.go @@ -87,7 +87,8 @@ type IPSet struct { MaxElem int // PortRange specifies the port range of bitmap:port type ipset. PortRange string - // TODO: add comment message for ipset + // comment message for ipset + Comment string } // Validate checks if a given ipset is valid or not. @@ -288,7 +289,7 @@ func (runner *runner) CreateSet(set *IPSet, ignoreExistErr bool) error { // If ignoreExistErr is set to true, then the -exist option of ipset will be specified, ipset ignores the error // otherwise raised when the same set (setname and create parameters are identical) already exists. func (runner *runner) createSet(set *IPSet, ignoreExistErr bool) error { - args := []string{"create", set.Name, string(set.SetType)} + args := []string{"create", set.Name, string(set.SetType), "comment"} if set.SetType == HashIPPortIP || set.SetType == HashIPPort { args = append(args, "family", set.HashFamily, @@ -312,7 +313,7 @@ func (runner *runner) createSet(set *IPSet, ignoreExistErr bool) error { // If the -exist option is specified, ipset ignores the error otherwise raised when // the same set (setname and create parameters are identical) already exists. func (runner *runner) AddEntry(entry string, set *IPSet, ignoreExistErr bool) error { - args := []string{"add", set.Name, entry} + args := []string{"add", set.Name, entry, "comment", set.Comment} if ignoreExistErr { args = append(args, "-exist") } @@ -324,6 +325,7 @@ func (runner *runner) AddEntry(entry string, set *IPSet, ignoreExistErr bool) er // DelEntry is used to delete the specified entry from the set. func (runner *runner) DelEntry(entry string, set string) error { + entry = strings.Split(entry, " comment")[0] if _, err := runner.exec.Command(IPSetCmd, "del", set, entry).CombinedOutput(); err != nil { return fmt.Errorf("error deleting entry %s: from set: %s, error: %v", entry, set, err) }