diff --git a/cluster/common.sh b/cluster/common.sh index 3c9d1e4b01..5638838077 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -734,6 +734,7 @@ MANIFEST_URL: $(yaml-quote ${MANIFEST_URL:-}) MANIFEST_URL_HEADER: $(yaml-quote ${MANIFEST_URL_HEADER:-}) NUM_NODES: $(yaml-quote ${NUM_NODES}) STORAGE_BACKEND: $(yaml-quote ${STORAGE_BACKEND:-etcd3}) +STORAGE_MEDIA_TYPE: $(yaml-quote ${STORAGE_MEDIA_TYPE:-}) ENABLE_GARBAGE_COLLECTOR: $(yaml-quote ${ENABLE_GARBAGE_COLLECTOR:-}) MASTER_ADVERTISE_ADDRESS: $(yaml-quote ${MASTER_ADVERTISE_ADDRESS:-}) ETCD_CA_KEY: $(yaml-quote ${ETCD_CA_KEY_BASE64:-}) diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 953448c308..c23313e393 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -130,11 +130,9 @@ TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m # ContentType used by all components to communicate with apiserver. TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}" -# ContentType used to store objects in underlying database. -TEST_CLUSTER_STORAGE_CONTENT_TYPE="${TEST_CLUSTER_STORAGE_CONTENT_TYPE:-}" KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false --outofdisk-transition-frequency=0 ${TEST_CLUSTER_API_CONTENT_TYPE}" -APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_STORAGE_CONTENT_TYPE}" +APISERVER_TEST_ARGS="${APISERVER_TEST_ARGS:-} --runtime-config=extensions/v1beta1 ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT}" CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_ARGS:-} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE}" SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}" KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_ARGS:-} ${TEST_CLUSTER_API_CONTENT_TYPE}" @@ -195,8 +193,10 @@ KUBE_UP_AUTOMATIC_CLEANUP=${KUBE_UP_AUTOMATIC_CLEANUP:-false} # is only supported in trusty or GCI. TEST_CLUSTER="${TEST_CLUSTER:-true}" -# Storage backend. 'etcd2' supported, 'etcd3' experimental. +# Storage backend. 'etcd2' and 'etcd3' are supported. STORAGE_BACKEND=${STORAGE_BACKEND:-} +# Storage media type: application/json and application/vnd.kubernetes.protobuf are supported. +STORAGE_MEDIA_TYPE=${STORAGE_MEDIA_TYPE:-} # OpenContrail networking plugin specific settings NETWORK_PROVIDER="${NETWORK_PROVIDER:-kubenet}" # none, opencontrail, kubenet diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh index 03509bcf43..04c076f3eb 100755 --- a/cluster/gce/configure-vm.sh +++ b/cluster/gce/configure-vm.sh @@ -483,6 +483,11 @@ EOF if [ -n "${STORAGE_BACKEND:-}" ]; then cat <>/srv/salt-overlay/pillar/cluster-params.sls storage_backend: '$(echo "$STORAGE_BACKEND" | sed -e "s/'/''/g")' +EOF + fi + if [ -n "${STORAGE_MEDIA_TYPE:-}" ]; then + cat <>/srv/salt-overlay/pillar/cluster-params.sls +storage_media_type: '$(echo "$STORAGE_MEDIA_TYPE" | sed -e "s/'/''/g")' EOF fi if [ -n "${ADMISSION_CONTROL:-}" ] && [ ${ADMISSION_CONTROL} == *"ImagePolicyWebhook"* ]; then diff --git a/cluster/gce/container-linux/configure-helper.sh b/cluster/gce/container-linux/configure-helper.sh index edb0a1bd86..f8cf437327 100755 --- a/cluster/gce/container-linux/configure-helper.sh +++ b/cluster/gce/container-linux/configure-helper.sh @@ -807,6 +807,9 @@ function start-kube-apiserver { if [[ -n "${STORAGE_BACKEND:-}" ]]; then params+=" --storage-backend=${STORAGE_BACKEND}" fi + if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then + params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" + fi if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" fi diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index 0d81eb4d6a..d8e82a1bac 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -870,6 +870,9 @@ function start-kube-apiserver { if [[ -n "${STORAGE_BACKEND:-}" ]]; then params+=" --storage-backend=${STORAGE_BACKEND}" fi + if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then + params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" + fi if [[ -n "${ENABLE_GARBAGE_COLLECTOR:-}" ]]; then params+=" --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" fi diff --git a/cluster/gce/trusty/configure-helper.sh b/cluster/gce/trusty/configure-helper.sh index 4aa8329c20..8dbe0efacc 100644 --- a/cluster/gce/trusty/configure-helper.sh +++ b/cluster/gce/trusty/configure-helper.sh @@ -598,6 +598,9 @@ start_kube_apiserver() { if [[ -n "${STORAGE_BACKEND:-}" ]]; then params="${params} --storage-backend=${STORAGE_BACKEND}" fi + if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then + params="${params} --storage-media-type=${STORAGE_MEDIA_TYPE}" + fi if [ -n "${NUM_NODES:-}" ]; then # If the cluster is large, increase max-requests-inflight limit in apiserver. if [[ "${NUM_NODES}" -ge 1000 ]]; then diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh index d08b9d3bbe..055f91ca6c 100644 --- a/cluster/kubemark/gce/config-default.sh +++ b/cluster/kubemark/gce/config-default.sh @@ -73,12 +73,15 @@ KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS="${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS:- # ContentType used by all components to communicate with apiserver. TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}" # ContentType used to store objects in underlying database. -TEST_CLUSTER_STORAGE_CONTENT_TYPE="${TEST_CLUSTER_STORAGE_CONTENT_TYPE:-}" +TEST_CLUSTER_STORAGE_MEDIA_TYPE="" +if [ -n "${STORAGE_MEDIA_TYPE:-}" ]; then + TEST_CLUSTER_STORAGE_MEDIA_TYPE="--storage-media-type=${STORAGE_MEDIA_TYPE}" +fi ENABLE_GARBAGE_COLLECTOR=${ENABLE_GARBAGE_COLLECTOR:-true} KUBELET_TEST_ARGS="--max-pods=100 $TEST_CLUSTER_LOG_LEVEL ${TEST_CLUSTER_API_CONTENT_TYPE}" -APISERVER_TEST_ARGS="--runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_STORAGE_CONTENT_TYPE} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" +APISERVER_TEST_ARGS="--runtime-config=extensions/v1beta1 ${API_SERVER_TEST_LOG_LEVEL} ${TEST_CLUSTER_STORAGE_MEDIA_TYPE} ${TEST_CLUSTER_MAX_REQUESTS_INFLIGHT} ${TEST_CLUSTER_DELETE_COLLECTION_WORKERS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" CONTROLLER_MANAGER_TEST_ARGS="${CONTROLLER_MANAGER_TEST_LOG_LEVEL} ${TEST_CLUSTER_RESYNC_PERIOD} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS} --enable-garbage-collector=${ENABLE_GARBAGE_COLLECTOR}" SCHEDULER_TEST_ARGS="${SCHEDULER_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE} ${KUBEMARK_MASTER_COMPONENTS_QPS_LIMITS}" KUBEPROXY_TEST_ARGS="${KUBEPROXY_TEST_LOG_LEVEL} ${TEST_CLUSTER_API_CONTENT_TYPE}" diff --git a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest index 58c1c78cdf..bfde14d4b1 100644 --- a/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest +++ b/cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest @@ -61,6 +61,11 @@ {% set etcd_servers = "--etcd-servers=http://127.0.0.1:2379" -%} {% set etcd_servers_overrides = "--etcd-servers-overrides=/events#http://127.0.0.1:4002" -%} +{% set storage_media_type = "" -%} +{% if pillar['storage_media_type'] is defined -%} + {% set storage_media_type = "--storage-media-type=" + pillar['storage_media_type'] -%} +{% endif -%} + {% set max_requests_inflight = "" -%} {% set target_ram_mb = "" -%} {% if pillar['num_nodes'] is defined -%} @@ -175,7 +180,7 @@ {% set audit_log = "--audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%} {% endif -%} -{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read + " " + audit_log -%} +{% set params = address + " " + storage_backend + " " + storage_media_type + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read + " " + audit_log -%} {% set params = params + " " + cert_file + " " + key_file + " " + kubelet_cert_file + " " + kubelet_key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config + image_review_config -%} # test_args has to be kept at the end, so they'll overwrite any prior configuration diff --git a/hack/verify-flags/exceptions.txt b/hack/verify-flags/exceptions.txt index 07d8d593b7..a77365d442 100644 --- a/hack/verify-flags/exceptions.txt +++ b/hack/verify-flags/exceptions.txt @@ -63,7 +63,7 @@ cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest:{% set para cluster/saltbase/salt/etcd/etcd.manifest: "value": "{{ pillar.get('storage_backend', 'etcd3') }}" cluster/saltbase/salt/etcd/etcd.manifest:{% if pillar.get('storage_backend', 'etcd3') == 'etcd3' -%} cluster/saltbase/salt/kube-admission-controls/init.sls:{% if 'LimitRanger' in pillar.get('admission_control', '') %} -cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + storage_backend + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read + " " + audit_log -%} +cluster/saltbase/salt/kube-apiserver/kube-apiserver.manifest:{% set params = address + " " + storage_backend + " " + storage_media_type + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read + " " + audit_log -%} cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% if pillar.get('enable_hostpath_provisioner', '').lower() == 'true' -%} cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = "--master=127.0.0.1:8080" + " " + cluster_name + " " + cluster_cidr + " " + allocate_node_cidrs + " " + service_cluster_ip_range + " " + terminated_pod_gc + " " + enable_garbage_collector + " " + cloud_provider + " " + cloud_config + " " + service_account_key + " " + log_level + " " + root_ca_file -%} cluster/saltbase/salt/kube-controller-manager/kube-controller-manager.manifest:{% set params = params + " " + feature_gates -%}