Merge pull request #67948 from wojtek-t/use_buffers_in_kube_proxy

Automatic merge from submit-queue (batch tested with PRs 66577, 67948, 68001, 67982). If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

Reduce amount of allocations in kube-proxy

Follow up from https://github.com/kubernetes/kubernetes/pull/65902

pkg/proxy/iptables/proxier.go

@@ -242,11 +242,12 @@ type Proxier struct {
 
 	// The following buffers are used to reuse memory and avoid allocations
 	// that are significantly impacting performance.
-	iptablesData *bytes.Buffer
-	filterChains *bytes.Buffer
-	filterRules  *bytes.Buffer
-	natChains    *bytes.Buffer
-	natRules     *bytes.Buffer
+	iptablesData             *bytes.Buffer
+	existingFilterChainsData *bytes.Buffer
+	filterChains             *bytes.Buffer
+	filterRules              *bytes.Buffer
+	natChains                *bytes.Buffer
+	natRules                 *bytes.Buffer
 
 	// endpointChainsNumber is the total amount of endpointChains across all
 	// services that we will generate (it is computed at the beginning of
@@ -340,6 +341,7 @@ func NewProxier(ipt utiliptables.Interface,
 		healthzServer:            healthzServer,
 		precomputedProbabilities: make([]string, 0, 1001),
 		iptablesData:             bytes.NewBuffer(nil),
+		existingFilterChainsData: bytes.NewBuffer(nil),
 		filterChains:             bytes.NewBuffer(nil),
 		filterRules:              bytes.NewBuffer(nil),
 		natChains:                bytes.NewBuffer(nil),
@@ -682,14 +684,12 @@ func (proxier *Proxier) syncProxyRules() {
 	// Get iptables-save output so we can check for existing chains and rules.
 	// This will be a map of chain name to chain with rules as stored in iptables-save/iptables-restore
 	existingFilterChains := make(map[utiliptables.Chain][]byte)
-	// TODO: Filter table is small so we're not reusing this buffer over rounds.
-	// However, to optimize it further, we should do that.
-	existingFilterChainsData := bytes.NewBuffer(nil)
-	err := proxier.iptables.SaveInto(utiliptables.TableFilter, existingFilterChainsData)
+	proxier.existingFilterChainsData.Reset()
+	err := proxier.iptables.SaveInto(utiliptables.TableFilter, proxier.existingFilterChainsData)
 	if err != nil { // if we failed to get any rules
 		glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err)
 	} else { // otherwise parse the output
-		existingFilterChains = utiliptables.GetChainLines(utiliptables.TableFilter, existingFilterChainsData.Bytes())
+		existingFilterChains = utiliptables.GetChainLines(utiliptables.TableFilter, proxier.existingFilterChainsData.Bytes())
 	}
 
 	// IMPORTANT: existingNATChains may share memory with proxier.iptablesData.

pkg/proxy/iptables/proxier_test.go

@@ -388,6 +388,7 @@ func NewFakeProxier(ipt utiliptables.Interface) *Proxier {
 		healthChecker:            newFakeHealthChecker(),
 		precomputedProbabilities: make([]string, 0, 1001),
 		iptablesData:             bytes.NewBuffer(nil),
+		existingFilterChainsData: bytes.NewBuffer(nil),
 		filterChains:             bytes.NewBuffer(nil),
 		filterRules:              bytes.NewBuffer(nil),
 		natChains:                bytes.NewBuffer(nil),

pkg/proxy/ipvs/proxier.go

@@ -217,11 +217,12 @@ type Proxier struct {
 	ipGetter IPGetter
 	// The following buffers are used to reuse memory and avoid allocations
 	// that are significantly impacting performance.
-	iptablesData *bytes.Buffer
-	natChains    *bytes.Buffer
-	filterChains *bytes.Buffer
-	natRules     *bytes.Buffer
-	filterRules  *bytes.Buffer
+	iptablesData     *bytes.Buffer
+	filterChainsData *bytes.Buffer
+	natChains        *bytes.Buffer
+	filterChains     *bytes.Buffer
+	natRules         *bytes.Buffer
+	filterRules      *bytes.Buffer
 	// Added as a member to the struct to allow injection for testing.
 	netlinkHandle NetLinkHandle
 	// ipsetList is the list of ipsets that ipvs proxier used.
@@ -369,6 +370,7 @@ func NewProxier(ipt utiliptables.Interface,
 		ipvsScheduler:     scheduler,
 		ipGetter:          &realIPGetter{nl: NewNetLinkHandle()},
 		iptablesData:      bytes.NewBuffer(nil),
+		filterChainsData:  bytes.NewBuffer(nil),
 		natChains:         bytes.NewBuffer(nil),
 		natRules:          bytes.NewBuffer(nil),
 		filterChains:      bytes.NewBuffer(nil),
@@ -1357,10 +1359,7 @@ func (proxier *Proxier) acceptIPVSTraffic() {
 
 // createAndLinkeKubeChain create all kube chains that ipvs proxier need and write basic link.
 func (proxier *Proxier) createAndLinkeKubeChain() {
-	// TODO: Filter table is small so we're not reusing this buffer over rounds.
-	// However, to optimize it further, we should do that.
-	filterBuffer := bytes.NewBuffer(nil)
-	existingFilterChains := proxier.getExistingChains(filterBuffer, utiliptables.TableFilter)
+	existingFilterChains := proxier.getExistingChains(proxier.filterChainsData, utiliptables.TableFilter)
 	existingNATChains := proxier.getExistingChains(proxier.iptablesData, utiliptables.TableNAT)
 
 	// Make sure we keep stats for the top-level chains

pkg/proxy/ipvs/proxier_test.go

@@ -163,6 +163,7 @@ func NewFakeProxier(ipt utiliptables.Interface, ipvs utilipvs.Interface, ipset u
 		ipvsScheduler:     DefaultScheduler,
 		ipGetter:          &fakeIPGetter{nodeIPs: nodeIPs},
 		iptablesData:      bytes.NewBuffer(nil),
+		filterChainsData:  bytes.NewBuffer(nil),
 		natChains:         bytes.NewBuffer(nil),
 		natRules:          bytes.NewBuffer(nil),
 		filterChains:      bytes.NewBuffer(nil),