Update Calico add-on

cluster/addons/calico-policy-controller/README.md

@@ -1,11 +1,17 @@
 # Calico Policy Controller
 ==============
 
-Calico Policy Controller is an implementation of the Kubernetes network policy API.
+Calico is an implementation of the Kubernetes network policy API.  The provided manifest installs a DaemonSet which runs Calico on each node in the cluster.
 
-Learn more at:
+### Templating
-- https://github.com/projectcalico/k8s-policy
-- http://kubernetes.io/docs/user-guide/networkpolicies/
 
+The provided `calico-node.yaml` manifest includes the following placeholders which are populated
+via templating.
+
+- `__CLUSTER_CIDR__`: The IP range from which Pod IP addresses are assigned.
+
+### Learn More
+
+Learn more about Calico at http://docs.projectcalico.org
 
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cluster/addons/calico-policy-controller/README.md?pixel)]()

cluster/addons/calico-policy-controller/calico-etcd-service.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    k8s-app: calico-etcd
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
-  name: calico-etcd
-  namespace: kube-system
-spec:
-  clusterIP: 10.0.0.17
-  ports:
-    - port: 6666
-  selector:
-    k8s-app: calico-etcd

cluster/addons/calico-policy-controller/calico-etcd-statefulset.yaml

@@ -1,42 +0,0 @@
-apiVersion: "apps/v1beta1"
-kind: StatefulSet
-metadata:
-  name: calico-etcd
-  namespace: kube-system
-  labels:
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
-    k8s-app: calico-etcd
-spec:
-  serviceName: calico-etcd
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        kubernetes.io/cluster-service: "true"
-        k8s-app: calico-etcd
-    spec:
-      hostNetwork: true
-      containers:
-        - name: calico-etcd
-          image: gcr.io/google_containers/etcd:2.2.1
-          env:
-            - name: CALICO_ETCD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-          command: ["/bin/sh","-c"]
-          args: ["/usr/local/bin/etcd --name=calico --data-dir=/var/etcd/calico-data --advertise-client-urls=http://$CALICO_ETCD_IP:6666 --listen-client-urls=http://0.0.0.0:6666 --listen-peer-urls=http://0.0.0.0:6667"]
-          volumeMounts:
-            - name: var-etcd
-              mountPath: /var/etcd
-  volumeClaimTemplates:
-    - metadata:
-        name: var-etcd
-        annotations:
-          volume.alpha.kubernetes.io/storage-class: anything
-      spec:
-        accessModes: [ "ReadWriteOnce" ]
-        resources:
-          requests:
-            storage: 1Gi

cluster/addons/calico-policy-controller/calico-node.yaml

@@ -0,0 +1,136 @@
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: calico-node
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: calico-node
+spec:
+  selector:
+    matchLabels:
+      k8s-app: calico-node
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-node
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: |
+          [{"key": "dedicated", "value": "master", "effect": "NoSchedule" },
+           {"key":"CriticalAddonsOnly", "operator":"Exists"}]
+    spec:
+      nodeSelector:
+        projectcalico.org/ds-ready: "true"
+      hostNetwork: true
+      containers:
+        # Runs calico/node container on each Kubernetes node.  This
+        # container programs network policy and routes on each
+        # host.
+        - name: calico-node
+          image: calico/node:v1.2.1
+          env:
+            - name: CALICO_DISABLE_FILE_LOGGING
+              value: "true"
+            - name: CALICO_NETWORKING_BACKEND
+              value: "none"
+            - name: CALICO_IPV4POOL_CIDR
+              value: "__CLUSTER_CIDR__"
+            - name: CALICO_IPV4POOL_IPIP
+              value: "off"
+            - name: DATASTORE_TYPE
+              value: "kubernetes"
+            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
+              value: "ACCEPT"
+            - name: FELIX_IPV6SUPPORT
+              value: "false"
+            - name: FELIX_LOGSEVERITYSYS
+              value: "none"
+            - name: FELIX_PROMETHEUSMETRICSENABLED
+              value: "true"
+            - name: IP
+              value: ""
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: WAIT_FOR_DATASTORE
+              value: "true"
+          securityContext:
+            privileged: true
+          resources:
+            requests:
+              cpu: 250m
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - mountPath: /etc/calico
+              name: etc-calico
+              readOnly: true
+        # This container installs the Calico CNI binaries
+        # and CNI network config file on each node.
+        - name: install-cni
+          image: calico/cni:v1.8.3-hostport
+          command: ["/install-cni.sh"]
+          env:
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+            - name: CNI_NETWORK_CONFIG
+              value: |-
+                {
+                  "name": "k8s-pod-network",
+                  "cniVersion": "0.3.0",
+                  "plugins": [
+                    {
+                      "type": "calico",
+                      "log_level": "debug",
+                      "datastore_type": "kubernetes",
+                      "nodename": "__KUBERNETES_NODE_NAME__",
+                      "ipam": {
+                        "type": "host-local",
+                        "subnet": "usePodCidr"
+                      },
+                      "policy": {
+                        "type": "k8s",
+                        "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
+                      },
+                      "kubernetes": {
+                        "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
+                        "kubeconfig": "__KUBECONFIG_FILEPATH__"
+                      }
+                    },
+                    {
+                      "type": "portmap",
+                      "capabilities": {"portMappings": true},
+                      "noSnat": true
+                    }
+                  ]
+                }
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+      volumes:
+        # Used to ensure proper kmods are installed.
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        # Mount in the Felix config file from the host.
+        - name: etc-calico
+          hostPath:
+            path: /etc/calico
+        # Used to install CNI binaries.
+        - name: cni-bin-dir
+          hostPath:
+            path: /home/kubernetes/bin
+        # Used to install CNI network config.
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d

cluster/addons/calico-policy-controller/calico-policy-controller.yaml

@@ -1,32 +0,0 @@
-apiVersion: v1
-kind: ReplicationController
-metadata:
-  name: calico-policy-controller
-  namespace: kube-system
-  labels:
-    k8s-app: calico-policy
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
-spec:
-  replicas: 1
-  selector:
-    k8s-app: calico-policy
-  template:
-    metadata:
-      name: calico-policy-controller
-      namespace: kube-system
-      labels:
-        kubernetes.io/cluster-service: "true"
-        k8s-app: calico-policy
-    spec:
-      hostNetwork: true
-      containers:
-        - name: calico-policy-controller
-          image: calico/kube-policy-controller:v0.2.0
-          env:
-            - name: ETCD_ENDPOINTS
-              value: "http://10.0.0.17:6666"
-            - name: K8S_API
-              value: "https://kubernetes.default:443"
-            - name: CONFIGURE_ETC_HOSTS
-              value: "true"

cluster/gce/config-default.sh

@@ -124,6 +124,12 @@ ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}"
 # TODO(piosz): remove this in 1.8
 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
 
+# To avoid running Calico on a node that is not configured appropriately, 
+# label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
+	NODE_LABELS="$NODE_LABELS,projectcalico.org/ds-ready=true"
+fi
+
 # Optional: Enable node logging.
 ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-true}"
 LOGGING_DESTINATION="${KUBE_LOGGING_DESTINATION:-gcp}" # options: elasticsearch, gcp

cluster/gce/config-test.sh

@@ -161,6 +161,12 @@ KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}"
 # TODO(piosz): remove this in 1.8
 NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}"
 
+# To avoid running Calico on a node that is not configured appropriately, 
+# label each Node so that the DaemonSet can run the Pods only on ready Nodes.
+if [[ ${NETWORK_POLICY_PROVIDER:-} == "calico" ]]; then
+	NODE_LABELS="$NODE_LABELS,projectcalico.org/ds-ready=true"
+fi
+
 # Optional: Enable node logging.
 ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-true}"
 LOGGING_DESTINATION="${KUBE_LOGGING_DESTINATION:-gcp}" # options: elasticsearch, gcp

cluster/gce/container-linux/configure-helper.sh

@@ -1201,6 +1201,10 @@ function start-kube-addons {
   fi
   if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
     setup-addon-manifests "addons" "calico-policy-controller"
+
+    # Replace the cluster cidr.
+    local -r calico_file="${dst_dir}/calico-policy-controller/calico-node.yaml"
+    sed -i -e "s@__CLUSTER_CIDR__@${CLUSTER_IP_RANGE}@g" "${calico_file}"
   fi
   if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
     setup-addon-manifests "addons" "storage-class/gce"

cluster/gce/gci/configure-helper.sh

@@ -701,13 +701,19 @@ function start-kubelet {
     flags+=" --anonymous-auth=false --authorization-mode=Webhook --client-ca-file=${CA_CERT_BUNDLE_PATH}"
   fi
   # Network plugin
-  if [[ -n "${NETWORK_PROVIDER:-}" ]]; then
+  if [[ -n "${NETWORK_PROVIDER:-}" || -n "${NETWORK_POLICY_PROVIDER:-}" ]]; then
-    if [[ "${NETWORK_PROVIDER:-}" == "cni" ]]; then
+    if [[ "${NETWORK_PROVIDER:-}" == "cni" || "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
       flags+=" --cni-bin-dir=/home/kubernetes/bin"
     else
       flags+=" --network-plugin-dir=/home/kubernetes/bin"
     fi
-    flags+=" --network-plugin=${NETWORK_PROVIDER}"
+    if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
+      # Calico uses CNI always.
+      flags+=" --network-plugin=cni"
+    else
+      # Otherwise use the configured value.
+      flags+=" --network-plugin=${NETWORK_PROVIDER}"
+    fi
   fi
   if [[ -n "${NON_MASQUERADE_CIDR:-}" ]]; then
     flags+=" --non-masquerade-cidr=${NON_MASQUERADE_CIDR}"
@@ -1419,6 +1425,10 @@ function start-kube-addons {
   fi
   if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
     setup-addon-manifests "addons" "calico-policy-controller"
+
+    # Replace the cluster cidr.
+    local -r calico_file="${dst_dir}/calico-policy-controller/calico-node.yaml"
+    sed -i -e "s@__CLUSTER_CIDR__@${CLUSTER_IP_RANGE}@g" "${calico_file}"
   fi
   if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
     setup-addon-manifests "addons" "storage-class/gce"

cluster/saltbase/salt/calico/10-calico.conf

@@ -1,16 +0,0 @@
-{
-    "name": "k8s-pod-network",
-    "type": "calico",
-    "etcd_authority": "10.0.0.17:6666",
-    "log_level": "info",
-    "ipam": {
-        "type": "host-local",
-        "subnet": "usePodCidr"
-    },
-    "policy": {
-        "type": "k8s"
-    },
-    "kubernetes": {
-        "kubeconfig": "/var/lib/kubelet/kubeconfig"
-    }
-}

cluster/saltbase/salt/calico/calico-node.manifest

@@ -1,40 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: calico-node
-  namespace: kube-system
-  labels:
-    kubernetes.io/cluster-service: "true"
-    k8s-app: calico-node
-spec:
-  hostNetwork: true
-  containers:
-    - name: calico-node
-      image: quay.io/calico/node:v0.20.0
-      env:
-        - name: ETCD_ENDPOINTS
-          value: "http://10.0.0.17:6666"
-        - name: CALICO_NETWORKING
-          value: "false"
-      securityContext:
-        privileged: true
-      volumeMounts:
-        - mountPath: /lib/modules
-          name: lib-modules
-          readOnly: true
-        - mountPath: /var/log/calico
-          name: var-log-calico
-          readOnly: false
-        - mountPath: /var/run/calico
-          name: var-run-calico
-          readOnly: false
-  volumes:
-    - name: lib-modules
-      hostPath:
-        path: /lib/modules
-    - name: var-run-calico
-      hostPath:
-        path: /var/run/calico
-    - name: var-log-calico
-      hostPath:
-        path: /var/log/calico

cluster/saltbase/salt/calico/master.sls

@@ -1,19 +0,0 @@
-{% if pillar.get('network_policy_provider', '').lower() == 'calico' %}
-
-calico-policy-controller:
-  file.managed:
-    - name: /etc/kubernetes/manifests/calico-policy-controller.manifest
-    - source: salt://calico/calico-policy-controller.manifest
-    - template: jinja
-    - user: root
-    - group: root
-    - mode: 644
-    - makedirs: true
-    - dir_mode: 755
-    - context:
-        cpurequest: '20m'
-    - require:
-      - service: docker
-      - service: kubelet
-
-{% endif -%}

cluster/saltbase/salt/calico/node.sls

@@ -1,37 +1,5 @@
 {% if pillar.get('network_policy_provider', '').lower() == 'calico' %}
 
-calico-node:
-  file.managed:
-    - name: /etc/kubernetes/manifests/calico-node.manifest
-    - source: salt://calico/calico-node.manifest
-    - template: jinja
-    - user: root
-    - group: root
-    - mode: 644
-    - makedirs: true
-    - dir_mode: 755
-    - require:
-      - kmod: ip6_tables
-      - kmod: xt_set
-      - service: docker
-      - service: kubelet
-
-calico-cni:
-  file.managed:
-    - name: /opt/cni/bin/calico
-    - source: https://github.com/projectcalico/calico-cni/releases/download/v1.3.1/calico
-    - source_hash: sha256=ac05cb9254b5aaa5822cf10325983431bd25489147f2edf9dec7e43d99c43e77
-    - makedirs: True
-    - mode: 744
-
-calico-cni-config:
-  file.managed:
-    - name: /etc/cni/net.d/10-calico.conf
-    - source: salt://calico/10-calico.conf
-    - makedirs: True
-    - mode: 644
-    - template: jinja
-
 ip6_tables:
   kmod.present
 

cluster/saltbase/salt/kubelet/default

@@ -140,7 +140,7 @@
 {% elif pillar.get('network_provider', '').lower() == 'cni' %}
   {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/" %}
 {%elif pillar.get('network_policy_provider', '').lower() == 'calico' and grains['roles'][0] != 'kubernetes-master' -%}
-  {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/" %}
+  {% set network_plugin = "--network-plugin=cni --network-plugin-dir=/etc/cni/net.d/ --cni-bin-dir=/home/kubernetes/bin/" %}
 {% elif pillar.get('network_provider', '').lower() == 'kubenet' %}
   {% set network_plugin = "--network-plugin=kubenet" -%}
 {% endif -%}

cluster/saltbase/salt/top.sls

@@ -84,6 +84,3 @@ base:
 {% if pillar.get('enable_rescheduler', '').lower() == 'true' %}
     - rescheduler
 {% endif %}
-{% if pillar.get('network_policy_provider', '').lower() == 'calico' %}
-    - calico.master
-{% endif %}