Merge pull request #61198 from jpbetz/etcd-3.2-upgrade-reattempt

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Update default etcd server to 3.2 for kubernetes 1.11

Repply #59836 but with latest etcd 3.2 patch version (3.2.18 which includes mvcc fix and leader election timeout fix) and default `--snapshot-count` to 10k to resolve performance regression in previous etcd 3.2 server upgrade attempt (https://github.com/kubernetes/kubernetes/issues/60589#issuecomment-372458578).

See https://github.com/kubernetes/kubernetes/issues/60589#issuecomment-372458578 for details on the root cause of the performance regression and scalability test results of setting `--snapshot-count` to 10k.

```release-note
Upgrade the default etcd server version to 3.2.18
```
@gyuho @shyamjvs @jdumars @timothysc

build/build-image/cross/Dockerfile

@@ -71,7 +71,7 @@ RUN go get golang.org/x/tools/cmd/cover \
             golang.org/x/tools/cmd/goimports
 
 # Download and symlink etcd. We need this for our integration tests.
-RUN export ETCD_VERSION=v3.1.12; \
+RUN export ETCD_VERSION=v3.2.18; \
   mkdir -p /usr/local/src/etcd \
   && cd /usr/local/src/etcd \
   && curl -fsSL https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | tar -xz \

build/root/WORKSPACE

@@ -18,12 +18,12 @@ http_archive(
     urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"],
 )
 
-ETCD_VERSION = "3.1.12"
+ETCD_VERSION = "3.2.18"
 
 new_http_archive(
     name = "com_coreos_etcd",
     build_file = "third_party/etcd.BUILD",
-    sha256 = "4b22184bef1bba8b4908b14bae6af4a6d33ec2b91e4f7a240780e07fa43f2111",
+    sha256 = "b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097",
     strip_prefix = "etcd-v%s-linux-amd64" % ETCD_VERSION,
     urls = ["https://github.com/coreos/etcd/releases/download/v%s/etcd-v%s-linux-amd64.tar.gz" % (ETCD_VERSION, ETCD_VERSION)],
 )

cluster/addons/etcd-empty-dir-cleanup/etcd-empty-dir-cleanup.yaml

@@ -24,4 +24,4 @@ spec:
   dnsPolicy: Default
   containers:
   - name: etcd-empty-dir-cleanup
-    image: k8s.gcr.io/etcd-empty-dir-cleanup:3.1.12.0
+    image: k8s.gcr.io/etcd-empty-dir-cleanup:3.2.18.0

cluster/gce/config-test.sh

@@ -162,7 +162,7 @@ METADATA_AGENT_VERSION="${KUBE_METADATA_AGENT_VERSION:-0.2-0.0.19-1}"
 # Useful for scheduling heapster in large clusters with nodes of small size.
 HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
 
-# Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. 3.1.12-1) if you need
+# Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. 3.2.18-0) if you need
 # non-default version.
 ETCD_IMAGE="${TEST_ETCD_IMAGE:-}"
 ETCD_DOCKER_REPOSITORY="${TEST_ETCD_DOCKER_REPOSITORY:-}"

cluster/gce/manifests/etcd.manifest

@@ -13,7 +13,7 @@
 "containers":[
     {
     "name": "etcd-container",
-    "image": "{{ pillar.get('etcd_docker_repository', 'k8s.gcr.io/etcd') }}:{{ pillar.get('etcd_docker_tag', '3.1.12-1') }}",
+    "image": "{{ pillar.get('etcd_docker_repository', 'k8s.gcr.io/etcd') }}:{{ pillar.get('etcd_docker_tag', '3.2.18-0') }}",
     "resources": {
       "requests": {
         "cpu": {{ cpulimit }}
@@ -29,7 +29,7 @@
         "value": "{{ pillar.get('storage_backend', 'etcd3') }}"
       },
       { "name": "TARGET_VERSION",
-        "value": "{{ pillar.get('etcd_version', '3.1.12') }}"
+        "value": "{{ pillar.get('etcd_version', '3.2.18') }}"
       },
       { "name": "DATA_DIRECTORY",
         "value": "/var/etcd/data{{ suffix }}"
@@ -45,6 +45,9 @@
       },
       { "name": "ETCD_CREDS",
         "value": "{{ etcd_creds }}"
+      },
+      { "name": "ETCD_SNAPSHOT_COUNT",
+        "value": "10000"
       }
         ],
     "livenessProbe": {

cluster/gce/upgrade-aliases.sh

@@ -161,8 +161,8 @@ export KUBE_GCE_ENABLE_IP_ALIASES=true
 export SECONDARY_RANGE_NAME="pods-default"
 export STORAGE_BACKEND="etcd3"
 export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf"
-export ETCD_IMAGE=3.1.12-1
-export ETCD_VERSION=3.1.12
+export ETCD_IMAGE=3.2.18-0
+export ETCD_VERSION=3.2.18
 
 # Upgrade master with updated kube envs
 ${KUBE_ROOT}/cluster/gce/upgrade.sh -M -l

cluster/images/etcd-empty-dir-cleanup/Makefile

@@ -14,13 +14,13 @@
 
 .PHONY:	build push
 
-ETCD_VERSION = 3.1.12
+ETCD_VERSION = 3.2.18
 # Image should be pulled from k8s.gcr.io, which will auto-detect
 # region (us, eu, asia, ...) and pull from the closest.
 REGISTRY = k8s.gcr.io
 # Images should be pushed to staging-k8s.gcr.io.
 PUSH_REGISTRY = staging-k8s.gcr.io
-TAG = 3.1.12.0
+TAG = 3.2.18.0
 
 clean:
 	rm -rf etcdctl etcd-v$(ETCD_VERSION)-linux-amd64 etcd-v$(ETCD_VERSION)-linux-amd64.tar.gz

cluster/images/etcd/Makefile

@@ -15,8 +15,8 @@
 # Build the etcd image
 #
 # Usage:
-# 	[BUNDLED_ETCD_VERSIONS=2.2.1 2.3.7 3.0.17 3.1.12] [REGISTRY=k8s.gcr.io] [ARCH=amd64] [BASEIMAGE=busybox] make (build|push)
-
+# 	[BUNDLED_ETCD_VERSIONS=2.2.1 2.3.7 3.0.17 3.1.12 3.2.18] [REGISTRY=k8s.gcr.io] [ARCH=amd64] [BASEIMAGE=busybox] make (build|push)
+#
 # The image contains different etcd versions to simplify
 # upgrades. Thus be careful when removing any versions from here.
 #
@@ -26,15 +26,15 @@
 # Except from etcd-$(version) and etcdctl-$(version) binaries, we also
 # need etcd and etcdctl binaries for backward compatibility reasons.
 # That binary will be set to the last version from $(BUNDLED_ETCD_VERSIONS).
-BUNDLED_ETCD_VERSIONS?=2.2.1 2.3.7 3.0.17 3.1.12
+BUNDLED_ETCD_VERSIONS?=2.2.1 2.3.7 3.0.17 3.1.12 3.2.18
 
 # LATEST_ETCD_VERSION identifies the most recent etcd version available.
-LATEST_ETCD_VERSION?=3.1.12
+LATEST_ETCD_VERSION?=3.2.18
 
 # REVISION provides a version number fo this image and all it's bundled
 # artifacts. It should start at zero for each LATEST_ETCD_VERSION and increment
 # for each revision of this image at that etcd version.
-REVISION?=1
+REVISION?=0
 
 # IMAGE_TAG Uniquely identifies k8s.gcr.io/etcd docker image with a tag of the form "<etcd-version>-<revision>".
 IMAGE_TAG=$(LATEST_ETCD_VERSION)-$(REVISION)

cluster/images/etcd/migrate-if-needed.sh

@@ -18,7 +18,7 @@
 # This script performs etcd upgrade based on the following environmental
 # variables:
 # TARGET_STORAGE - API of etcd to be used (supported: 'etcd2', 'etcd3')
-# TARGET_VERSION - etcd release to be used (supported: '2.2.1', '2.3.7', '3.0.17', '3.1.12')
+# TARGET_VERSION - etcd release to be used (supported: '2.2.1', '2.3.7', '3.0.17', '3.1.12', '3.2.18')
 # DATA_DIRECTORY - directory with etcd data
 #
 # The current etcd version and storage format is detected based on the
@@ -29,6 +29,7 @@
 # - 2.2.1/etcd2 -> 2.3.7/etcd2
 # - 2.3.7/etcd2 -> 3.0.17/etcd2
 # - 3.0.17/etcd3 -> 3.1.12/etcd3
+# - 3.1.12/etcd3 -> 3.2.18/etcd3
 #
 # NOTE: The releases supported in this script has to match release binaries
 # present in the etcd image (to make this script work correctly).
@@ -41,7 +42,7 @@ set -o nounset
 
 # NOTE: BUNDLED_VERSION has to match release binaries present in the
 # etcd image (to make this script work correctly).
-BUNDLED_VERSIONS="2.2.1, 2.3.7, 3.0.17, 3.1.12"
+BUNDLED_VERSIONS="2.2.1, 2.3.7, 3.0.17, 3.1.12 3.2.18"
 
 ETCD_NAME="${ETCD_NAME:-etcd-$(hostname)}"
 if [ -z "${DATA_DIRECTORY:-}" ]; then

cmd/kubeadm/app/constants/constants.go

@@ -208,10 +208,10 @@ const (
 	KubeletBaseConfigurationFile = "kubelet"
 
 	// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
-	MinExternalEtcdVersion = "3.1.12"
+	MinExternalEtcdVersion = "3.2.17"
 
 	// DefaultEtcdVersion indicates the default etcd version that kubeadm uses
-	DefaultEtcdVersion = "3.1.12"
+	DefaultEtcdVersion = "3.2.18"
 
 	// Etcd defines variable used internally when referring to etcd component
 	Etcd = "etcd"
@@ -298,7 +298,7 @@ var (
 	SupportedEtcdVersion = map[uint8]string{
 		9:  "3.1.12",
 		10: "3.1.12",
-		11: "3.1.12",
+		11: "3.2.18",
 	}
 )
 

cmd/kubeadm/app/constants/constants_test.go

@@ -18,9 +18,10 @@ package constants
 
 import (
 	"fmt"
-	"k8s.io/kubernetes/pkg/util/version"
 	"strings"
 	"testing"
+
+	"k8s.io/kubernetes/pkg/util/version"
 )
 
 func TestGetStaticPodDirectory(t *testing.T) {

cmd/kubeadm/app/phases/etcd/local.go

@@ -88,6 +88,7 @@ func getEtcdCommand(cfg *kubeadmapi.MasterConfiguration) []string {
 		"peer-key-file":         filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName),
 		"peer-trusted-ca-file":  filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
 		"peer-client-cert-auth": "true",
+		"snapshot-count":        "10000",
 	}
 
 	command := []string{"etcd"}

cmd/kubeadm/app/phases/etcd/local_test.go

@@ -89,6 +89,7 @@ func TestGetEtcdCommand(t *testing.T) {
 				"--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName,
 				"--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName,
 				"--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
+				"--snapshot-count=10000",
 				"--peer-client-cert-auth=true",
 			},
 		},
@@ -114,6 +115,7 @@ func TestGetEtcdCommand(t *testing.T) {
 				"--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName,
 				"--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName,
 				"--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
+				"--snapshot-count=10000",
 				"--peer-client-cert-auth=true",
 			},
 		},
@@ -133,6 +135,7 @@ func TestGetEtcdCommand(t *testing.T) {
 				"--peer-cert-file=" + kubeadmconstants.EtcdPeerCertName,
 				"--peer-key-file=" + kubeadmconstants.EtcdPeerKeyName,
 				"--peer-trusted-ca-file=" + kubeadmconstants.EtcdCACertName,
+				"--snapshot-count=10000",
 				"--peer-client-cert-auth=true",
 			},
 		},

cmd/kubeadm/app/phases/upgrade/compute_test.go

@@ -17,10 +17,11 @@ limitations under the License.
 package upgrade
 
 import (
-	"github.com/coreos/etcd/clientv3"
-	versionutil "k8s.io/kubernetes/pkg/util/version"
 	"reflect"
 	"testing"
+
+	"github.com/coreos/etcd/clientv3"
+	versionutil "k8s.io/kubernetes/pkg/util/version"
 )
 
 type fakeVersionGetter struct {
@@ -430,7 +431,7 @@ func TestGetAvailableUpgrades(t *testing.T) {
 						KubeVersion:    "v1.11.0-alpha.2",
 						KubeadmVersion: "v1.11.0-alpha.2",
 						DNSVersion:     "1.14.9",
-						EtcdVersion:    "3.1.12",
+						EtcdVersion:    "3.2.18",
 					},
 				},
 			},

hack/lib/etcd.sh

@@ -16,7 +16,7 @@
 
 # A set of helpers for starting/running etcd for tests
 
-ETCD_VERSION=${ETCD_VERSION:-3.1.12}
+ETCD_VERSION=${ETCD_VERSION:-3.2.18}
 ETCD_HOST=${ETCD_HOST:-127.0.0.1}
 ETCD_PORT=${ETCD_PORT:-2379}
 

staging/src/k8s.io/apiextensions-apiserver/artifacts/example/rc.yaml

@@ -23,4 +23,4 @@ spec:
         - "--etcd-servers=http://localhost:2379"
         - "--audit-log-path=-"
       - name: etcd
-        image: quay.io/coreos/etcd:v3.1.12
+        image: quay.io/coreos/etcd:v3.2.18

staging/src/k8s.io/sample-apiserver/artifacts/example/rc.yaml

@@ -21,4 +21,4 @@ spec:
         imagePullPolicy: Never
         command: [ "/kube-sample-apiserver", "--etcd-servers=http://localhost:2379" ]
       - name: etcd
-        image: quay.io/coreos/etcd:v3.1.12
+        image: quay.io/coreos/etcd:v3.2.18

test/e2e/apimachinery/aggregator.go

@@ -133,7 +133,7 @@ func TestSampleAPIServer(f *framework.Framework, image string) {
 
 	// kubectl create -f deploy.yaml
 	deploymentName := "sample-apiserver-deployment"
-	etcdImage := "quay.io/coreos/etcd:v3.1.12"
+	etcdImage := "quay.io/coreos/etcd:v3.2.18"
 	podLabels := map[string]string{"app": "sample-apiserver", "apiserver": "true"}
 	replicas := int32(1)
 	zero := int64(0)

test/e2e/framework/nodes_util.go

@@ -63,7 +63,7 @@ func etcdUpgradeGCE(target_storage, target_version string) error {
 		os.Environ(),
 		"TEST_ETCD_VERSION="+target_version,
 		"STORAGE_BACKEND="+target_storage,
-		"TEST_ETCD_IMAGE=3.1.12-1")
+		"TEST_ETCD_IMAGE=3.2.18-0")
 
 	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-l", "-M")
 	return err
@@ -103,7 +103,7 @@ func masterUpgradeGCE(rawV string, enableKubeProxyDaemonSet bool) error {
 		env = append(env,
 			"TEST_ETCD_VERSION="+TestContext.EtcdUpgradeVersion,
 			"STORAGE_BACKEND="+TestContext.EtcdUpgradeStorage,
-			"TEST_ETCD_IMAGE=3.1.12-1")
+			"TEST_ETCD_IMAGE=3.2.18-0")
 	} else {
 		// In e2e tests, we skip the confirmation prompt about
 		// implicit etcd upgrades to simulate the user entering "y".

test/kubemark/start-kubemark.sh

@@ -59,7 +59,7 @@ SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-}"
 EVENT_PD="${EVENT_PD:-}"
 
 # Etcd related variables.
-ETCD_IMAGE="${ETCD_IMAGE:-3.1.12-1}"
+ETCD_IMAGE="${ETCD_IMAGE:-3.2.18-0}"
 ETCD_VERSION="${ETCD_VERSION:-}"
 
 # Controller-manager related variables.