diff --git a/cluster/addons/fluentd-elasticsearch/README.md b/cluster/addons/fluentd-elasticsearch/README.md index 7066dd1769..7e58dd3d7e 100644 --- a/cluster/addons/fluentd-elasticsearch/README.md +++ b/cluster/addons/fluentd-elasticsearch/README.md @@ -52,10 +52,6 @@ Fluentd is deployed as a [DaemonSet][daemonSet] which spawns a pod on each node that reads logs, generated by kubelet, container runtime and containers and sends them to Elasticsearch. -**Note:** in order for Fluentd to work, every Kubernetes node must be labeled -with `beta.kubernetes.io/fluentd-ds-ready=true`, as otherwise the Fluentd -DaemonSet will ignore them. - Learn more in the [official Kubernetes documentation][k8sElasticsearchDocs]. ## Building diff --git a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml index c3df2ac253..e0b97e3006 100644 --- a/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml +++ b/cluster/addons/fluentd-elasticsearch/fluentd-es-ds.yaml @@ -95,8 +95,6 @@ spec: readOnly: true - name: config-volume mountPath: /etc/fluent/config.d - nodeSelector: - beta.kubernetes.io/fluentd-ds-ready: "true" terminationGracePeriodSeconds: 30 volumes: - name: varlog diff --git a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml index 5c195fb086..b41fadafab 100644 --- a/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml +++ b/cluster/addons/fluentd-gcp/fluentd-gcp-ds.yaml @@ -103,7 +103,6 @@ spec: fieldPath: metadata.namespace # END_PROMETHEUS_TO_SD nodeSelector: - beta.kubernetes.io/fluentd-ds-ready: "true" beta.kubernetes.io/os: linux terminationGracePeriodSeconds: 60 tolerations: diff --git a/cluster/addons/ip-masq-agent/ip-masq-agent.yaml b/cluster/addons/ip-masq-agent/ip-masq-agent.yaml index b0ee2e7dea..39f709f798 100644 --- a/cluster/addons/ip-masq-agent/ip-masq-agent.yaml +++ b/cluster/addons/ip-masq-agent/ip-masq-agent.yaml @@ -45,6 +45,7 @@ spec: - name: config mountPath: /etc/config nodeSelector: + # TODO(liggitt): switch to node.kubernetes.io/masq-agent-ds-ready in 1.16 beta.kubernetes.io/masq-agent-ds-ready: "true" volumes: - name: config diff --git a/cluster/addons/kube-proxy/kube-proxy-ds.yaml b/cluster/addons/kube-proxy/kube-proxy-ds.yaml index e082ca4d16..fb28b02a4c 100644 --- a/cluster/addons/kube-proxy/kube-proxy-ds.yaml +++ b/cluster/addons/kube-proxy/kube-proxy-ds.yaml @@ -27,6 +27,7 @@ spec: priorityClassName: system-node-critical hostNetwork: true nodeSelector: + # TODO(liggitt): switch to node.kubernetes.io/kube-proxy-ds-ready in 1.16 beta.kubernetes.io/kube-proxy-ds-ready: "true" tolerations: - operator: "Exists" diff --git a/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml b/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml index 4c4c29f2d5..92f28103f8 100644 --- a/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml +++ b/cluster/addons/metadata-proxy/gce/metadata-proxy.yaml @@ -88,6 +88,7 @@ spec: fieldPath: metadata.namespace # END_PROMETHEUS_TO_SD nodeSelector: + # TODO(liggitt): switch to cloud.google.com/metadata-proxy-ready=true in v1.16 beta.kubernetes.io/metadata-proxy-ready: "true" beta.kubernetes.io/os: linux terminationGracePeriodSeconds: 30 diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index ac22b90e38..91ad0be7b3 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -38,13 +38,7 @@ MASTER_ROOT_DISK_SIZE=${MASTER_ROOT_DISK_SIZE:-$(get-master-root-disk-size)} NODE_DISK_TYPE=${NODE_DISK_TYPE:-pd-standard} NODE_DISK_SIZE=${NODE_DISK_SIZE:-100GB} NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0} - -# Historically fluentd was a manifest pod and then was migrated to DaemonSet. -# To avoid situation during cluster upgrade when there are two instances -# of fluentd running on a node, kubelet need to mark node on which -# fluentd is not running as a manifest pod with appropriate label. -# TODO(piosz): remove this in 1.8 -NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}" +NODE_LABELS="${KUBE_NODE_LABELS:-}" WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}" # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices @@ -217,6 +211,8 @@ ENABLE_METADATA_CONCEALMENT="${ENABLE_METADATA_CONCEALMENT:-false}" # true, fals METADATA_CONCEALMENT_NO_FIREWALL="${METADATA_CONCEALMENT_NO_FIREWALL:-false}" # true, false if [[ ${ENABLE_METADATA_CONCEALMENT:-} == "true" ]]; then # Put the necessary label on the node so the daemonset gets scheduled. + NODE_LABELS="${NODE_LABELS},cloud.google.com/metadata-proxy-ready=true" + # TODO(liggitt): remove this in v1.16 NODE_LABELS="${NODE_LABELS},beta.kubernetes.io/metadata-proxy-ready=true" # Add to the provider custom variables. PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_METADATA_CONCEALMENT METADATA_CONCEALMENT_NO_FIREWALL" diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 1ef1548e81..7ff43a1b55 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -38,13 +38,7 @@ MASTER_ROOT_DISK_SIZE=${MASTER_ROOT_DISK_SIZE:-$(get-master-root-disk-size)} NODE_DISK_TYPE=${NODE_DISK_TYPE:-pd-standard} NODE_DISK_SIZE=${NODE_DISK_SIZE:-100GB} NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0} - -# Historically fluentd was a manifest pod and then was migrated to DaemonSet. -# To avoid situation during cluster upgrade when there are two instances -# of fluentd running on a node, kubelet need to mark node on which -# fluentd is not running as a manifest pod with appropriate label. -# TODO(piosz): remove this in 1.8 -NODE_LABELS="${KUBE_NODE_LABELS:-beta.kubernetes.io/fluentd-ds-ready=true}" +NODE_LABELS="${KUBE_NODE_LABELS:-}" WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}" # An extension to local SSDs allowing users to specify block/fs and SCSI/NVMe devices @@ -257,6 +251,8 @@ ENABLE_METADATA_CONCEALMENT="${ENABLE_METADATA_CONCEALMENT:-true}" # true, false METADATA_CONCEALMENT_NO_FIREWALL="${METADATA_CONCEALMENT_NO_FIREWALL:-false}" # true, false if [[ ${ENABLE_METADATA_CONCEALMENT:-} == "true" ]]; then # Put the necessary label on the node so the daemonset gets scheduled. + NODE_LABELS="${NODE_LABELS},cloud.google.com/metadata-proxy-ready=true" + # TODO(liggitt): remove this in v1.16 NODE_LABELS="${NODE_LABELS},beta.kubernetes.io/metadata-proxy-ready=true" # Add to the provider custom variables. PROVIDER_VARS="${PROVIDER_VARS:-} ENABLE_METADATA_CONCEALMENT METADATA_CONCEALMENT_NO_FIREWALL" diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index a310783a41..256f5b90f8 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1931,6 +1931,35 @@ function setup-etcd-encryption { fi } +# Updates node labels used by addons. +function update-legacy-addon-node-labels() { + # need kube-apiserver to be ready + until kubectl get nodes; do + sleep 5 + done + update-node-label "beta.kubernetes.io/metadata-proxy-ready=true,cloud.google.com/metadata-proxy-ready!=true" "cloud.google.com/metadata-proxy-ready=true" + update-node-label "beta.kubernetes.io/kube-proxy-ds-ready=true,node.kubernetes.io/kube-proxy-ds-ready!=true" "node.kubernetes.io/kube-proxy-ds-ready=true" + update-node-label "beta.kubernetes.io/masq-agent-ds-ready=true,node.kubernetes.io/masq-agent-ds-ready!=true" "node.kubernetes.io/masq-agent-ds-ready=true" +} + +# A helper function for labeling all nodes matching a given selector. +# Runs: kubectl label --overwrite nodes -l "${1}" "${2}" +# Retries on failure +# +# $1: label selector of nodes +# $2: label to apply +function update-node-label() { + local selector="$1" + local label="$2" + local retries=5 + until (( retries == 0 )); do + if kubectl label --overwrite nodes -l "${selector}" "${label}"; then + break + fi + (( retries-- )) + sleep 3 + done +} # Applies encryption provider config. # This function may be triggered in two scenarios: @@ -2903,6 +2932,7 @@ function main() { start-kube-addons start-cluster-autoscaler start-lb-controller + update-legacy-addon-node-labels & apply-encryption-config & else if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 3074cc0ffb..dbe053a89e 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -610,7 +610,8 @@ function build-linux-node-labels { if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${master}" != "true" ]]; then # Add kube-proxy daemonset label to node to avoid situation during cluster # upgrade/downgrade when there are two instances of kube-proxy running on a node. - node_labels="beta.kubernetes.io/kube-proxy-ds-ready=true" + # TODO(liggitt): drop beta.kubernetes.io/kube-proxy-ds-ready in 1.16 + node_labels="node.kubernetes.io/kube-proxy-ds-ready=true,beta.kubernetes.io/kube-proxy-ds-ready=true" fi if [[ -n "${NODE_LABELS:-}" ]]; then node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go index 9c62bef285..213999036a 100644 --- a/cmd/kubelet/app/options/options.go +++ b/cmd/kubelet/app/options/options.go @@ -263,9 +263,9 @@ func ValidateKubeletFlags(f *KubeletFlags) error { } } if len(unknownLabels) > 0 { - // TODO(liggitt): in 1.15, return an error + // TODO(liggitt): in 1.16, return an error klog.Warningf("unknown 'kubernetes.io' or 'k8s.io' labels specified with --node-labels: %v", unknownLabels.List()) - klog.Warningf("in 1.15, --node-labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", ")) + klog.Warningf("in 1.16, --node-labels in the 'kubernetes.io' namespace must begin with an allowed prefix (%s) or be in the specifically allowed set (%s)", strings.Join(kubeletapis.KubeletLabelNamespaces(), ", "), strings.Join(kubeletapis.KubeletLabels(), ", ")) } return nil