Merge pull request #64696 from dixudx/kubeadm_proxy_arch

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubeadm: only run kube-proxy on architecture consistent nodes

**What this PR does / why we need it**:
Currently `gcr.io/google_containers/kube-proxy-{.Arch}` is not multi-arch image. We should use `affinity` or `nodeSelector` to only run the pods on the nodes with the matching architecture, like what we've done in `kube-dns`. 

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
partial Fixes #64647

**Special notes for your reviewer**:
/cc luxas timothysc 
**Release note**:

```release-note
kubeadm: only run kube-proxy on architecture consistent nodes
```

cmd/kubeadm/app/phases/addons/dns/manifests.go

@@ -173,16 +173,8 @@ spec:
         operator: Exists
       - key: {{ .MasterTaintKey }}
         effect: NoSchedule
-      # TODO: Remove this affinity field as soon as we are using manifest lists
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-            - matchExpressions:
-              - key: beta.kubernetes.io/arch
-                operator: In
-                values:
-                - {{ .Arch }}
+      nodeSelector:
+        beta.kubernetes.io/arch: {{ .Arch }}
 `
 
 	// KubeDNSService is the kube-dns Service manifest

cmd/kubeadm/app/phases/addons/proxy/BUILD

@@ -34,6 +34,7 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons/proxy",
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
+        "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig/scheme:go_default_library",

cmd/kubeadm/app/phases/addons/proxy/manifests.go

@@ -90,8 +90,6 @@ spec:
           readOnly: true
       hostNetwork: true
       serviceAccountName: kube-proxy
-      tolerations:
-      - operator: Exists
       volumes:
       - name: kube-proxy
         configMap:
@@ -103,5 +101,12 @@ spec:
       - name: lib-modules
         hostPath:
           path: /lib/modules
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: {{ .MasterTaintKey }}
+        effect: NoSchedule
+      nodeSelector:
+        beta.kubernetes.io/arch: {{ .Arch }}
 `
 )

cmd/kubeadm/app/phases/addons/proxy/proxy.go

@@ -29,6 +29,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	kubeproxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
@@ -75,11 +76,12 @@ func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte
 	if err != nil {
 		return fmt.Errorf("error when parsing kube-proxy configmap template: %v", err)
 	}
-	proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ ImageRepository, Arch, Version, ImageOverride string }{
+	proxyDaemonSetBytes, err = kubeadmutil.ParseTemplate(KubeProxyDaemonSet19, struct{ ImageRepository, Arch, Version, ImageOverride, MasterTaintKey string }{
 		ImageRepository: cfg.GetControlPlaneImageRepository(),
 		Arch:            runtime.GOARCH,
 		Version:         kubeadmutil.KubernetesVersionToImageTag(cfg.KubernetesVersion),
 		ImageOverride:   cfg.UnifiedControlPlaneImage,
+		MasterTaintKey:  kubeadmconstants.LabelNodeRoleMaster,
 	})
 	if err != nil {
 		return fmt.Errorf("error when parsing kube-proxy daemonset template: %v", err)