diff --git a/cluster/aws/config-default.sh b/cluster/aws/config-default.sh index 4ff88d6393..d9709919b6 100644 --- a/cluster/aws/config-default.sh +++ b/cluster/aws/config-default.sh @@ -143,6 +143,8 @@ ENABLE_NODE_PUBLIC_IP=${KUBE_ENABLE_NODE_PUBLIC_IP:-true} # OS options for minions KUBE_OS_DISTRIBUTION="${KUBE_OS_DISTRIBUTION:-jessie}" +KUBE_MASTER_OS_DISTRIBUTION="${KUBE_OS_DISTRIBUTION}" +KUBE_NODE_OS_DISTRIBUTION="${KUBE_OS_DISTRIBUTION}" KUBE_NODE_IMAGE="${KUBE_NODE_IMAGE:-}" COREOS_CHANNEL="${COREOS_CHANNEL:-alpha}" CONTAINER_RUNTIME="${KUBE_CONTAINER_RUNTIME:-docker}" diff --git a/cluster/aws/config-test.sh b/cluster/aws/config-test.sh index df073cf996..598f055535 100755 --- a/cluster/aws/config-test.sh +++ b/cluster/aws/config-test.sh @@ -129,6 +129,8 @@ ENABLE_NODE_PUBLIC_IP=${KUBE_ENABLE_NODE_PUBLIC_IP:-true} # OS options for minions KUBE_OS_DISTRIBUTION="${KUBE_OS_DISTRIBUTION:-jessie}" +KUBE_MASTER_OS_DISTRIBUTION="${KUBE_OS_DISTRIBUTION}" +KUBE_NODE_OS_DISTRIBUTION="${KUBE_OS_DISTRIBUTION}" KUBE_NODE_IMAGE="${KUBE_NODE_IMAGE:-}" COREOS_CHANNEL="${COREOS_CHANNEL:-alpha}" CONTAINER_RUNTIME="${KUBE_CONTAINER_RUNTIME:-docker}" diff --git a/cluster/common.sh b/cluster/common.sh index abc6701b0b..820ad69fed 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -394,7 +394,8 @@ function find-release-tars() { # This tarball is used by GCI, Ubuntu Trusty, and CoreOS. KUBE_MANIFESTS_TAR= - if [[ "${OS_DISTRIBUTION:-}" == "trusty" || "${OS_DISTRIBUTION:-}" == "gci" || "${OS_DISTRIBUTION:-}" == "coreos" ]]; then + if [[ "${MASTER_OS_DISTRIBUTION:-}" == "trusty" || "${MASTER_OS_DISTRIBUTION:-}" == "gci" || "${MASTER_OS_DISTRIBUTION:-}" == "coreos" ]] || \ + [[ "${NODE_OS_DISTRIBUTION:-}" == "trusty" || "${NODE_OS_DISTRIBUTION:-}" == "gci" || "${NODE_OS_DISTRIBUTION:-}" == "coreos" ]] ; then KUBE_MANIFESTS_TAR="${KUBE_ROOT}/server/kubernetes-manifests.tar.gz" if [[ ! -f "${KUBE_MANIFESTS_TAR}" ]]; then KUBE_MANIFESTS_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-manifests.tar.gz" @@ -514,6 +515,17 @@ function build-kube-env { local master=$1 local file=$2 + local server_binary_tar_url=$SERVER_BINARY_TAR_URL + local salt_tar_url=$SALT_TAR_URL + local kube_manifests_tar_url=$KUBE_MANIFESTS_TAR_URL + if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "coreos" ]] || \ + [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "coreos" ]] ; then + # TODO: Support fallback .tar.gz settings on CoreOS + server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}") + salt_tar_url=$(split_csv "${SALT_TAR_URL}") + kube_manifests_tar_url=$(split_csv "${KUBE_MANIFESTS_TAR_URL}") + fi + build-runtime-config gen-uid @@ -523,9 +535,9 @@ ENV_TIMESTAMP: $(yaml-quote $(date -u +%Y-%m-%dT%T%z)) INSTANCE_PREFIX: $(yaml-quote ${INSTANCE_PREFIX}) NODE_INSTANCE_PREFIX: $(yaml-quote ${NODE_INSTANCE_PREFIX}) CLUSTER_IP_RANGE: $(yaml-quote ${CLUSTER_IP_RANGE:-10.244.0.0/16}) -SERVER_BINARY_TAR_URL: $(yaml-quote ${SERVER_BINARY_TAR_URL}) +SERVER_BINARY_TAR_URL: $(yaml-quote ${server_binary_tar_url}) SERVER_BINARY_TAR_HASH: $(yaml-quote ${SERVER_BINARY_TAR_HASH}) -SALT_TAR_URL: $(yaml-quote ${SALT_TAR_URL}) +SALT_TAR_URL: $(yaml-quote ${salt_tar_url}) SALT_TAR_HASH: $(yaml-quote ${SALT_TAR_HASH}) SERVICE_CLUSTER_IP_RANGE: $(yaml-quote ${SERVICE_CLUSTER_IP_RANGE}) KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME}) @@ -583,9 +595,10 @@ EOF TERMINATED_POD_GC_THRESHOLD: $(yaml-quote ${TERMINATED_POD_GC_THRESHOLD}) EOF fi - if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]]; then + if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "coreos") ]] || \ + [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "coreos") ]] ; then cat >>$file <>$file <&2 } diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index f81c33cc72..9ae556cd07 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -23,10 +23,17 @@ source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" source "${KUBE_ROOT}/cluster/lib/util.sh" -if [[ "${OS_DISTRIBUTION}" == "debian" || "${OS_DISTRIBUTION}" == "coreos" || "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]]; then - source "${KUBE_ROOT}/cluster/gce/${OS_DISTRIBUTION}/helper.sh" +if [[ "${NODE_OS_DISTRIBUTION}" == "debian" || "${NODE_OS_DISTRIBUTION}" == "coreos" || "${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then + source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh" else - echo "Cannot operate on cluster using os distro: ${OS_DISTRIBUTION}" >&2 + echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2 + exit 1 +fi + +if [[ "${MASTER_OS_DISTRIBUTION}" == "debian" || "${MASTER_OS_DISTRIBUTION}" == "coreos" || "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then + source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh" +else + echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2 exit 1 fi @@ -49,14 +56,17 @@ function get_latest_gci_image() { echo "${gci_images[0]}" } -if [[ "${OS_DISTRIBUTION}" == "gci" ]]; then - # If the master image is not set, we use the pinned GCI image. +if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then + # If the master image is not set, we use the latest GCI image. # Otherwise, we respect whatever is set by the user. MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-"$(get_latest_gci_image)"} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} - # The default node image when using GCI is still the Debian based ContainerVM - # until GCI gets validated for node usage. - NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}} +fi + +if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then + # If the node image is not set, we use the latest GCI image. + # Otherwise, we respect whatever is set by the user. + NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-"$(get_latest_gci_image)"} NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers} fi @@ -85,6 +95,11 @@ function join_csv { local IFS=','; echo "$*"; } +# This function returns the first string before the comma +function split_csv { + echo "$*" | cut -d',' -f1 +} + # Verify prereqs function verify-prereqs { local cmd @@ -252,9 +267,7 @@ function upload-server-tars() { SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}") SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}") - if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" || "${OS_DISTRIBUTION}" == "coreos" ]]; then - KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}") - fi + KUBE_MANIFESTS_TAR_HASH=$(sha1sum-file "${KUBE_MANIFESTS_TAR}") local server_binary_tar_urls=() local salt_tar_urls=() @@ -285,26 +298,15 @@ function upload-server-tars() { server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}") salt_tar_urls+=("${salt_gs_url/gs:\/\//https://storage.googleapis.com/}") - if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" || "${OS_DISTRIBUTION}" == "coreos" ]]; then - local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}" - copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}" - # Convert from gs:// URL to an https:// URL - kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}") - fi + local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}" + copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}" + # Convert from gs:// URL to an https:// URL + kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}") done - if [[ "${OS_DISTRIBUTION}" == "coreos" ]]; then - # TODO: Support fallback .tar.gz settings on CoreOS - SERVER_BINARY_TAR_URL="${server_binary_tar_urls[0]}" - SALT_TAR_URL="${salt_tar_urls[0]}" - KUBE_MANIFESTS_TAR_URL="${kube_manifests_tar_urls[0]}" - else - SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}") - SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}") - if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]]; then - KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}") - fi - fi + SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}") + SALT_TAR_URL=$(join_csv "${salt_tar_urls[@]}") + KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}") } # Detect minions created in the minion group @@ -755,11 +757,6 @@ function create-nodes-template() { local template_name="${NODE_INSTANCE_PREFIX}-template" - # For master on GCI or trusty, we support the hybrid mode with nodes on ContainerVM. - if [[ "${OS_DISTRIBUTION}" == "trusty" || "${OS_DISTRIBUTION}" == "gci" ]] && \ - [[ "${NODE_IMAGE}" == container* ]]; then - source "${KUBE_ROOT}/cluster/gce/debian/helper.sh" - fi create-node-instance-template $template_name } @@ -800,7 +797,7 @@ function create-nodes() { # Spread the remaining number of nodes evenly this_mig_size=$((${instances_left} / (${NUM_MIGS}-${i}+1))) instances_left=$((instances_left-${this_mig_size})) - + gcloud compute instance-groups managed \ create "${group_name}" \ --project "${PROJECT}" \ @@ -827,20 +824,20 @@ function create-nodes() { function create-cluster-autoscaler-mig-config() { # Each MIG must have at least one node, so the min number of nodes - # must be greater or equal to the number of migs. + # must be greater or equal to the number of migs. if [[ ${AUTOSCALER_MIN_NODES} < ${NUM_MIGS} ]]; then echo "AUTOSCALER_MIN_NODES must be greater or equal ${NUM_MIGS}" exit 2 fi # Each MIG must have at least one node, so the min number of nodes - # must be greater or equal to the number of migs. + # must be greater or equal to the number of migs. if [[ ${AUTOSCALER_MAX_NODES} < ${NUM_MIGS} ]]; then echo "AUTOSCALER_MAX_NODES must be greater or equal ${NUM_MIGS}" exit 2 fi - # The code assumes that the migs were created with create-nodes + # The code assumes that the migs were created with create-nodes # function which tries to evenly spread nodes across the migs. AUTOSCALER_MIG_CONFIG="" @@ -1196,9 +1193,14 @@ function check-resources { # Prepare to push new binaries to kubernetes cluster # $1 - whether prepare push to node function prepare-push() { + local node="${1-}" #TODO(dawnchen): figure out how to upgrade coreos node - if [[ "${OS_DISTRIBUTION}" != "debian" ]]; then - echo "Updating a kubernetes cluster with ${OS_DISTRIBUTION} is not supported yet." >&2 + if [[ "${node}" == "true" && "${NODE_OS_DISTRIBUTION}" != "debian" ]]; then + echo "Updating nodes in a kubernetes cluster with ${NODE_OS_DISTRIBUTION} is not supported yet." >&2 + exit 1 + fi + if [[ "${node}" != "true" && "${MASTER_OS_DISTRIBUTION}" != "debian" ]]; then + echo "Updating the master in a kubernetes cluster with ${MASTER_OS_DISTRIBUTION} is not supported yet." >&2 exit 1 fi @@ -1216,7 +1218,7 @@ function prepare-push() { tars_from_version # Prepare node env vars and update MIG template - if [[ "${1-}" == "true" ]]; then + if [[ "${node}" == "true" ]]; then write-node-env # TODO(zmerlynn): Refactor setting scope flags. diff --git a/cluster/kubemark/config-default.sh b/cluster/kubemark/config-default.sh index 7affd528a7..8000d39c82 100644 --- a/cluster/kubemark/config-default.sh +++ b/cluster/kubemark/config-default.sh @@ -30,7 +30,8 @@ MASTER_DISK_SIZE=${MASTER_DISK_SIZE:-20GB} REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false} PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} -OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION:-debian} +MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-debian} +NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-debian} MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-container-vm-v20160321} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} diff --git a/hack/ginkgo-e2e.sh b/hack/ginkgo-e2e.sh index fadc3fd67c..cbb781d2ce 100755 --- a/hack/ginkgo-e2e.sh +++ b/hack/ginkgo-e2e.sh @@ -119,8 +119,9 @@ export PATH=$(dirname "${e2e_test}"):"${PATH}" --repo-root="${KUBE_ROOT}" \ --node-instance-group="${NODE_INSTANCE_GROUP:-}" \ --prefix="${KUBE_GCE_INSTANCE_PREFIX:-e2e}" \ - ${OS_DISTRIBUTION:+"--os-distro=${OS_DISTRIBUTION}"} \ ${KUBE_CONTAINER_RUNTIME:+"--container-runtime=${KUBE_CONTAINER_RUNTIME}"} \ + ${MASTER_OS_DISTRIBUTION:+"--master-os-distro=${MASTER_OS_DISTRIBUTION}"} \ + ${NODE_OS_DISTRIBUTION:+"--node-os-distro=${NODE_OS_DISTRIBUTION}"} \ ${NUM_NODES:+"--num-nodes=${NUM_NODES}"} \ ${E2E_CLEAN_START:+"--clean-start=true"} \ ${E2E_MIN_STARTUP_PODS:+"--minStartupPods=${E2E_MIN_STARTUP_PODS}"} \ diff --git a/hack/jenkins/e2e-runner.sh b/hack/jenkins/e2e-runner.sh index 238ad36782..3a53d1f4a2 100755 --- a/hack/jenkins/e2e-runner.sh +++ b/hack/jenkins/e2e-runner.sh @@ -215,7 +215,7 @@ if [[ -n "${JENKINS_GCI_IMAGE_FAMILY:-}" ]]; then GCI_STAGING_PROJECT=container-vm-image-staging export KUBE_GCE_MASTER_PROJECT="${GCI_STAGING_PROJECT}" export KUBE_GCE_MASTER_IMAGE="$(get_latest_gci_image "${GCI_STAGING_PROJECT}" "${JENKINS_GCI_IMAGE_FAMILY}")" - export KUBE_OS_DISTRIBUTION="gci" + export KUBE_MASTER_OS_DISTRIBUTION="gci" if [[ "${JENKINS_GCI_IMAGE_FAMILY}" == "gci-canary-test" ]]; then # The family "gci-canary-test" is reserved for a special type of GCI images # that are used to continuously validate Docker releases. diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 58b90e2d77..6902e6a6df 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -13,7 +13,7 @@ cluster/gce/configure-vm.sh: api_servers: '${KUBERNETES_MASTER_NAME}' cluster/gce/configure-vm.sh: cloud_config: ${CLOUD_CONFIG} cluster/gce/configure-vm.sh: env-to-grains "runtime_config" cluster/gce/configure-vm.sh: kubelet_api_servers: '${KUBELET_APISERVER}' -cluster/gce/coreos/helper.sh:# cloud_config yaml file should be passed +cluster/gce/coreos/node-helper.sh:# cloud_config yaml file should be passed cluster/gce/gci/configure-helper.sh: reconcile_cidr="false" cluster/gce/gci/configure-helper.sh: local api_servers="--master=https://${KUBERNETES_MASTER_NAME}" cluster/gce/gci/configure-helper.sh: local reconcile_cidr="true" diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index aaa532b55e..89b7cb25d4 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -280,6 +280,7 @@ make-symlinks manifest-url manifest-url-header masquerade-all +master-os-distro master-service-namespace max-concurrency max-connection-bytes-per-sec @@ -327,6 +328,7 @@ node-labels node-monitor-grace-period node-monitor-period node-name +node-os-distro node-startup-grace-period node-status-update-frequency node-sync-period @@ -339,7 +341,6 @@ oidc-issuer-url oidc-username-claim only-idl oom-score-adj -os-distro out-version outofdisk-transition-frequency output-base diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 2eeb188fb1..c1e4d1dfdf 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -46,8 +46,9 @@ type TestContextType struct { SystemPodsStartupTimeout time.Duration UpgradeTarget string PrometheusPushGateway string - OSDistro string ContainerRuntime string + MasterOSDistro string + NodeOSDistro string VerifyServiceAccount bool DeleteNamespace bool CleanStart bool @@ -124,8 +125,9 @@ func RegisterClusterFlags() { flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.") flag.StringVar(&TestContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.") flag.StringVar(&TestContext.Prefix, "prefix", "e2e", "A prefix to be added to cloud resources created during testing.") - flag.StringVar(&TestContext.OSDistro, "os-distro", "debian", "The OS distribution of cluster VM instances (debian, trusty, or coreos).") flag.StringVar(&TestContext.ContainerRuntime, "container-runtime", "docker", "The container runtime of cluster VM instances (docker or rkt).") + flag.StringVar(&TestContext.MasterOSDistro, "master-os-distro", "debian", "The OS distribution of cluster master (debian, trusty, or coreos).") + flag.StringVar(&TestContext.NodeOSDistro, "node-os-distro", "debian", "The OS distribution of cluster VM instances (debian, trusty, or coreos).") // TODO: Flags per provider? Rename gce-project/gce-zone? cloudConfig := &TestContext.CloudConfig