diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index c0a18753c0..6aae9232eb 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -44,6 +44,8 @@ MINION_TAG="${INSTANCE_PREFIX}-minion" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/16}" MINION_SCOPES="${MINION_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}" +RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" +ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}" # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 @@ -87,7 +89,6 @@ CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" # Optional: Create autoscaler for cluster's nodes. -# NOT WORKING YET! ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}" if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" @@ -95,6 +96,13 @@ if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" fi +# Optional: Enable feature for autoscaling number of pods +# Experimental feature, not ready for production use. +ENABLE_HORIZONTAL_POD_AUTOSCALER="${KUBE_ENABLE_HORIZONTAL_POD_AUTOSCALER:-false}" +if [[ "${ENABLE_HORIZONTAL_POD_AUTOSCALER}" == "true" ]]; then + ENABLE_EXPERIMENTAL_API=true +fi + # Admission Controllers to invoke prior to persisting objects in cluster ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index a45477a3e7..ef6ffa6f44 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -45,6 +45,9 @@ MINION_TAG="${INSTANCE_PREFIX}-minion" CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.245.0.0/16}" MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" MINION_SCOPES="${MINION_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}" +RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}" +ENABLE_EXPERIMENTAL_API="${KUBE_ENABLE_EXPERIMENTAL_API:-false}" + # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 SERVICE_CLUSTER_IP_RANGE="10.0.0.0/16" # formerly PORTAL_NET @@ -92,7 +95,6 @@ CLUSTER_REGISTRY_DISK_TYPE_GCE="${CLUSTER_REGISTRY_DISK_TYPE_GCE:-pd-standard}" ENABLE_CLUSTER_UI="${KUBE_ENABLE_CLUSTER_UI:-true}" # Optional: Create autoscaler for cluster's nodes. -# NOT WORKING YET! ENABLE_NODE_AUTOSCALER="${KUBE_ENABLE_NODE_AUTOSCALER:-false}" if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" @@ -100,6 +102,13 @@ if [[ "${ENABLE_NODE_AUTOSCALER}" == "true" ]]; then TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" fi +# Optional: Enable feature for autoscaling number of pods +# Experimental feature, not ready for production use. +ENABLE_HORIZONTAL_POD_AUTOSCALER="${KUBE_ENABLE_HORIZONTAL_POD_AUTOSCALER:-false}" +if [[ "${ENABLE_HORIZONTAL_POD_AUTOSCALER}" == "true" ]]; then + ENABLE_EXPERIMENTAL_API=true +fi + ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota # Optional: if set to true kube-up will automatically check for existing resources and clean them up. diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index c199a6643c..eaf65b89db 100644 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -310,6 +310,11 @@ EOF cluster_registry_disk_type: gce cluster_registry_disk_size: $(convert-bytes-gce-kube ${CLUSTER_REGISTRY_DISK_SIZE}) cluster_registry_disk_name: ${CLUSTER_REGISTRY_DISK} +EOF + fi + if [ -n "${ENABLE_HORIZONTAL_POD_AUTOSCALER:-}" ]; then + cat <>/srv/salt-overlay/pillar/cluster-params.sls +enable_horizontal_pod_autoscaler: '$(echo "$ENABLE_HORIZONTAL_POD_AUTOSCALER" | sed -e "s/'/''/g")' EOF fi } @@ -568,6 +573,11 @@ EOF # CIDR range. cat <>/etc/salt/minion.d/grains.conf cbr-cidr: ${MASTER_IP_RANGE} +EOF + fi + if [[ ! -z "${RUNTIME_CONFIG:-}" ]]; then + cat <>/etc/salt/minion.d/grains.conf + runtime_config: '$(echo "$RUNTIME_CONFIG" | sed -e "s/'/''/g")' EOF fi } diff --git a/cluster/gce/coreos/helper.sh b/cluster/gce/coreos/helper.sh index 5ef057acf4..846bb2591b 100755 --- a/cluster/gce/coreos/helper.sh +++ b/cluster/gce/coreos/helper.sh @@ -54,6 +54,8 @@ KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) +ENABLE_HORIZONTAL_POD_AUTOSCALER: $(yaml-quote ${ENABLE_HORIZONTAL_POD_AUTOSCALER}) +RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG}) KUBERNETES_MASTER_NAME: $(yaml-quote ${MASTER_NAME}) KUBERNETES_CONTAINER_RUNTIME: $(yaml-quote ${CONTAINER_RUNTIME}) RKT_VERSION: $(yaml-quote ${RKT_VERSION}) diff --git a/cluster/gce/debian/helper.sh b/cluster/gce/debian/helper.sh index 6b929c2d9e..694f29ec85 100755 --- a/cluster/gce/debian/helper.sh +++ b/cluster/gce/debian/helper.sh @@ -51,6 +51,8 @@ KUBELET_TOKEN: $(yaml-quote ${KUBELET_TOKEN:-}) KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) +ENABLE_HORIZONTAL_POD_AUTOSCALER: $(yaml-quote ${ENABLE_HORIZONTAL_POD_AUTOSCALER}) +RUNTIME_CONFIG: $(yaml-quote ${RUNTIME_CONFIG}) CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-}) KUBELET_CERT: $(yaml-quote ${KUBELET_CERT_BASE64:-}) KUBELET_KEY: $(yaml-quote ${KUBELET_KEY_BASE64:-}) diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index c884c9c219..604c855f23 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -53,6 +53,18 @@ function join_csv { # Verify prereqs function verify-prereqs { + if [[ "${ENABLE_EXPERIMENTAL_API}" == "true" ]]; then + if [[ -z "${RUNTIME_CONFIG}" ]]; then + RUNTIME_CONFIG="experimental/v1=true" + else + # TODO: add checking if RUNTIME_CONFIG contains "experimental/v1=false" and appending "experimental/v1=true" if not. + if echo "${RUNTIME_CONFIG}" | grep -q -v "experimental/v1=true"; then + echo "Experimental API should be turned on, but is not turned on in RUNTIME_CONFIG!" + exit 1 + fi + fi + fi + local cmd for cmd in gcloud gsutil; do if ! which "${cmd}" >/dev/null; then @@ -465,6 +477,7 @@ function write-master-env { if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then KUBELET_APISERVER="${MASTER_NAME}" fi + build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" } diff --git a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest index 7c0e214213..a4f82888a0 100644 --- a/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest +++ b/cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest @@ -1,6 +1,7 @@ {% set cluster_name = "" -%} {% set cluster_cidr = "" -%} {% set allocate_node_cidrs = "" -%} +{% set enable_horizontal_pod_autoscaler = "" -%} {% if pillar['instance_prefix'] is defined -%} {% set cluster_name = "--cluster-name=" + pillar['instance_prefix'] -%} @@ -11,6 +12,9 @@ {% if pillar['allocate_node_cidrs'] is defined -%} {% set allocate_node_cidrs = "--allocate-node-cidrs=" + pillar['allocate_node_cidrs'] -%} {% endif -%} +{% if pillar['enable_horizontal_pod_autoscaler'] is defined -%} + {% set enable_horizontal_pod_autoscaler = "--enable-horizontal-pod-autoscaler=" + pillar['enable_horizontal_pod_autoscaler'] -%} +{% endif -%} {% set cloud_provider = "" -%} {% set cloud_config = "" -%} @@ -34,7 +38,7 @@ {% set root_ca_file = "--root-ca-file=/srv/kubernetes/ca.crt" -%} {% endif -%} -{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} +{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + enable_horizontal_pod_autoscaler + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration {% if pillar['controller_manager_test_args'] is defined -%} diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index a8816eff7e..c541cc2118 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -1,4 +1,3 @@ -cluster/addons/cluster-monitoring/README.md:Heapster enables monitoring of Kubernetes Clusters using [cAdvisor](https://github.com/google/cadvisor). The kubelet will communicate with an instance of cAdvisor running on localhost and proxy container stats to Heapster. Kubelet will attempt to connect to cAdvisor on port 4194 by default but this port can be configured with kubelet's `--cadvisor-port` run flag. Detailed information about heapster can be found [here](https://github.com/GoogleCloudPlatform/heapster). cluster/addons/registry/images/Dockerfile:ADD run_proxy.sh /usr/bin/run_proxy cluster/addons/registry/images/Dockerfile:CMD ["/usr/bin/run_proxy"] cluster/aws/templates/salt-minion.sh:# We set the hostname_override to the full EC2 private dns name @@ -39,7 +38,7 @@ cluster/saltbase/salt/kube-addons/kube-addons.sh:# Create admission_control obje cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + etcd_servers + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + admission_control + " " + service_cluster_ip_range + " " + client_ca_file + " " + basic_auth_file + " " + min_request_timeout -%} cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = params + " " + cluster_name + " " + cert_file + " " + key_file + " --secure-port=" + secure_port + " " + token_auth_file + " " + bind_address + " " + pillar['log_level'] + " " + advertise_address + " " + proxy_ssh_options -%} -cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} +cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + enable_horizontal_pod_autoscaler + " " + cloud_provider + " " + cloud_config + service_account_key + pillar['log_level'] + " " + root_ca_file -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers + ":6443" -%} cluster/saltbase/salt/kube-proxy/default: {% set api_servers_with_port = api_servers + ":7080" -%}