github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #54838 from Random-Liu/fix-calico-network-policy 

Automatic merge from submit-queue (batch tested with PRs 54488, 54838, 54964). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Fix calico network policy for opensource.

For https://github.com/kubernetes/features/issues/286

This PR:
1) Add `NON_MASTER_NODE_LABELS` env, and only apply calico node label on non-master nodes.
2) Set ip masq rules in cloud init, so that we don't need the ip masq agent. @dchen1107 @dnardo As we discussed.
3) Let master use `${NETWORK_PROVIDER}` instead of fixed cni, because we won't run calico node agent on master. The master network should be configured separately (kubenet by default).

With this PR, I could bring up a cluster with `NETWORK_POLICY_PROVIDER=calico` on GCE now.
```console
$ cluster/kubectl.sh get pods --all-namespaces
NAMESPACE     NAME                                                  READY     STATUS    RESTARTS   AGE
kube-system   calico-node-9bxbv                                     2/2       Running   0          13m
kube-system   calico-node-kjxtw                                     2/2       Running   0          13m
kube-system   calico-node-vertical-autoscaler-67fb4f45bd-hcjmw      1/1       Running   0          16m
kube-system   calico-node-xs2s2                                     2/2       Running   0          13m
kube-system   calico-typha-7c4d876ddf-d4dtx                         1/1       Running   0          15m
kube-system   calico-typha-horizontal-autoscaler-5f477cdc66-qwwph   1/1       Running   0          16m
kube-system   calico-typha-vertical-autoscaler-58f7d686f7-pn72s     1/1       Running   0          16m
kube-system   etcd-empty-dir-cleanup-e2e-test-lantaol-master        1/1       Running   0          16m
kube-system   etcd-server-e2e-test-lantaol-master                   1/1       Running   0          16m
kube-system   etcd-server-events-e2e-test-lantaol-master            1/1       Running   0          16m
kube-system   event-exporter-v0.1.7-9d4dbb69c-m76v5                 2/2       Running   0          16m
kube-system   fluentd-gcp-v2.0.10-25dmf                             2/2       Running   0          16m
kube-system   fluentd-gcp-v2.0.10-kgxsk                             2/2       Running   0          16m
kube-system   fluentd-gcp-v2.0.10-p75xg                             2/2       Running   0          16m
kube-system   fluentd-gcp-v2.0.10-xzh77                             2/2       Running   0          16m
kube-system   heapster-v1.5.0-beta.0-5cf4d9dff7-dmvm7               4/4       Running   0          13m
kube-system   kube-addon-manager-e2e-test-lantaol-master            1/1       Running   0          15m
kube-system   kube-apiserver-e2e-test-lantaol-master                1/1       Running   0          16m
kube-system   kube-controller-manager-e2e-test-lantaol-master       1/1       Running   0          16m
kube-system   kube-dns-79bdcb6c9f-2bpc8                             3/3       Running   0          15m
kube-system   kube-dns-79bdcb6c9f-gr686                             3/3       Running   0          16m
kube-system   kube-dns-autoscaler-996dcfc9d-pfs4s                   1/1       Running   0          16m
kube-system   kube-proxy-e2e-test-lantaol-minion-group-3khw         1/1       Running   0          16m
kube-system   kube-proxy-e2e-test-lantaol-minion-group-6878         1/1       Running   0          16m
kube-system   kube-proxy-e2e-test-lantaol-minion-group-j9rq         1/1       Running   0          16m
kube-system   kube-scheduler-e2e-test-lantaol-master                1/1       Running   0          16m
kube-system   kubernetes-dashboard-765c6f47bd-lsw5r                 1/1       Running   0          16m
kube-system   l7-default-backend-6d477bf555-x54zf                   1/1       Running   0          16m
kube-system   l7-lb-controller-v0.9.7-e2e-test-lantaol-master       1/1       Running   0          16m
kube-system   metrics-server-v0.2.0-9c4f8c48d-gkl79                 2/2       Running   0          13m
kube-system   monitoring-influxdb-grafana-v4-54df94856c-krkvb       2/2       Running   0          16m
kube-system   rescheduler-v0.3.1-e2e-test-lantaol-master            1/1       Running   0          16m
```

**Note that with this PR, master node will be using kubenet by default. And network policy will not apply on master node.**

**We need this to unblock `cri-containerd` integration with `kube-up.sh`.**
/cc @dchen1107 @dnardo Please take a look.
@kubernetes/sig-network-misc @kubernetes/sig-cluster-lifecycle-misc 

Signed-off-by: Lantao Liu <lantaol@google.com>

```release-note
None
```
pull/6/head
Kubernetes Submit Queue 2017-11-02 18:01:16 -07:00 committed by GitHub
parent 2084f7f4f3 8279916c65
commit 52ec6f425b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 46 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cluster/common.sh
View File

@ -899,12 +899,17 @@ EOF
cat >>$file <<EOF
NODE_LABELS: $(yaml-quote ${NODE_LABELS})
EOF
fi
fi
if [ -n "${NON_MASTER_NODE_LABELS:-}" ]; then
cat >>$file <<EOF
NON_MASTER_NODE_LABELS: $(yaml-quote ${NON_MASTER_NODE_LABELS})
EOF
fi
if [ -n "${EVICTION_HARD:-}" ]; then
cat >>$file <<EOF
EVICTION_HARD: $(yaml-quote ${EVICTION_HARD})
EOF
fi
fi
if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "container-linux" ]] || \
[[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "container-linux" ]]; then
# Container-Linux-only env vars. TODO(yifan): Make them available on other distros.

5
cluster/gce/config-default.sh
View File

@ -144,10 +144,13 @@ HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
# TODO(piosz): remove this in 1.8
NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
# NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes.
NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}"
# To avoid running Calico on a node that is not configured appropriately,
# label each Node so that the DaemonSet can run the Pods only on ready Nodes.
if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
NODE_LABELS="${NODE_LABELS},projectcalico.org/ds-ready=true"
NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true"
fi
# Enable metadata concealment by firewalling pod traffic to the metadata server

5
cluster/gce/config-test.sh
View File

@ -199,10 +199,13 @@ KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"
# TODO(piosz): remove this in 1.8
NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
# NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes.
NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}"
# To avoid running Calico on a node that is not configured appropriately,
# label each Node so that the DaemonSet can run the Pods only on ready Nodes.
if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
NODE_LABELS="$NODE_LABELS,projectcalico.org/ds-ready=true"
NON_MASTER_NODE_LABELS="${NON_MASTER_NODE_LABELS:+${NON_MASTER_NODE_LABELS},}projectcalico.org/ds-ready=true"
fi
# Enable metadata concealment by firewalling pod traffic to the metadata server

5
cluster/gce/configure-vm.sh
View File

@ -584,6 +584,11 @@ EOF
if [ -n "${NODE_LABELS:-}" ]; then
cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
node_labels: '$(echo "${NODE_LABELS}" | sed -e "s/'/''/g")'
EOF
fi
if [ -n "${NON_MASTER_NODE_LABELS:-}" ]; then
cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
non_master_node_labels: '$(echo "${NON_MASTER_NODE_LABELS}" | sed -e "s/'/''/g")'
EOF
fi
if [ -n "${NODE_TAINTS:-}" ]; then

3
cluster/gce/container-linux/configure-helper.sh
View File

@ -598,6 +598,9 @@ function start-kubelet {
if [[ -n "${NODE_LABELS:-}" ]]; then
node_labels="${node_labels:+${node_labels},}${NODE_LABELS}"
fi
if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${KUBERNETES_MASTER:-}" != "true" ]]; then
node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}"
fi
if [[ -n "${node_labels:-}" ]]; then
flags+=" --node-labels=${node_labels}"
fi

24
cluster/gce/gci/configure-helper.sh
View File

@ -60,6 +60,20 @@ function config-ip-firewall {
if [[ "${ENABLE_METADATA_CONCEALMENT:-}" == "true" ]]; then
iptables -A KUBE-METADATA-SERVER -j DROP
fi
# Flush iptables nat table
iptables -t nat -F || true
if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" && "${KUBERNETES_MASTER:-}" == false ]]; then
echo "Add rules for ip masquerade"
iptables -t nat -N IP-MASQ
iptables -t nat -A POSTROUTING -m comment --comment "ip-masq: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom IP-MASQ chain" -m addrtype ! --dst-type LOCAL -j IP-MASQ
iptables -t nat -A IP-MASQ -d 169.254.0.0/16 -m comment --comment "ip-masq: local traffic is not subject to MASQUERADE" -j RETURN
iptables -t nat -A IP-MASQ -d 10.0.0.0/8 -m comment --comment "ip-masq: local traffic is not subject to MASQUERADE" -j RETURN
iptables -t nat -A IP-MASQ -d 172.16.0.0/12 -m comment --comment "ip-masq: local traffic is not subject to MASQUERADE" -j RETURN
iptables -t nat -A IP-MASQ -d 192.168.0.0/16 -m comment --comment "ip-masq: local traffic is not subject to MASQUERADE" -j RETURN
iptables -t nat -A IP-MASQ -m comment --comment "ip-masq: outbound traffic is subject to MASQUERADE (must be last in chain)" -j MASQUERADE
fi
}
function create-dirs {
@ -938,7 +952,9 @@ function start-kubelet {
flags+=" --cni-bin-dir=/home/kubernetes/bin"
if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
# Calico uses CNI always.
if [[ "${KUBERNETES_PRIVATE_MASTER:-}" == "true" ]]; then
# Keep KUBERNETES_PRIVATE_MASTER for backward compatibility.
# Note that network policy won't work for master node.
if [[ "${KUBERNETES_PRIVATE_MASTER:-}" == "true" || "${KUBERNETES_MASTER:-}" == "true" ]]; then
flags+=" --network-plugin=${NETWORK_PROVIDER}"
else
flags+=" --network-plugin=cni"
@ -971,6 +987,9 @@ function start-kubelet {
if [[ -n "${NODE_LABELS:-}" ]]; then
node_labels="${node_labels:+${node_labels},}${NODE_LABELS}"
fi
if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${KUBERNETES_MASTER:-}" != "true" ]]; then
node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}"
fi
if [[ -n "${node_labels:-}" ]]; then
flags+=" --node-labels=${node_labels}"
fi
@ -1007,9 +1026,6 @@ ExecStart=${kubelet_bin} \$KUBELET_OPTS
WantedBy=multi-user.target
EOF
# Flush iptables nat table
iptables -t nat -F || true
systemctl start kubelet.service
}

3
cluster/saltbase/salt/kubelet/default
View File

@ -171,6 +171,9 @@
{% set kube_proxy_ds_label = "beta.kubernetes.io/kube-proxy-ds-ready=true," %}
{% endif %}
{% set node_labels = kube_proxy_ds_label + pillar['node_labels'] %}
{% if grains['roles'][0] != 'kubernetes-master' and pillar['non_master_node_labels'] is defined -%}
{% set node_labels = pillar['non_master_node_labels'] + "," + node_labels %}
{% endif %}
{% if node_labels != "" %}
{% set node_labels="--node-labels=" + node_labels %}
{% endif %}