mirror of https://github.com/k3s-io/k3s
342 lines
15 KiB
Plaintext
342 lines
15 KiB
Plaintext
{% set daemon_args = "$DAEMON_ARGS" -%}
|
|
{% if grains['os_family'] == 'RedHat' -%}
|
|
{% set daemon_args = "" -%}
|
|
{% endif -%}
|
|
|
|
{% set cloud_provider = "" -%}
|
|
{% set cloud_config = "" -%}
|
|
{% set cloud_config_mount = "" -%}
|
|
{% set cloud_config_volume = "" -%}
|
|
{% set additional_cloud_config_mount = "{\"name\": \"usrsharessl\",\"mountPath\": \"/usr/share/ssl\", \"readOnly\": true}, {\"name\": \"usrssl\",\"mountPath\": \"/usr/ssl\", \"readOnly\": true}, {\"name\": \"usrlibssl\",\"mountPath\": \"/usr/lib/ssl\", \"readOnly\": true}, {\"name\": \"usrlocalopenssl\",\"mountPath\": \"/usr/local/openssl\", \"readOnly\": true}," -%}
|
|
{% set additional_cloud_config_volume = "{\"name\": \"usrsharessl\",\"hostPath\": {\"path\": \"/usr/share/ssl\"}}, {\"name\": \"usrssl\",\"hostPath\": {\"path\": \"/usr/ssl\"}}, {\"name\": \"usrlibssl\",\"hostPath\": {\"path\": \"/usr/lib/ssl\"}}, {\"name\": \"usrlocalopenssl\",\"hostPath\": {\"path\": \"/usr/local/openssl\"}}," -%}
|
|
|
|
{% set srv_kube_path = "/srv/kubernetes" -%}
|
|
{% set srv_sshproxy_path = "/srv/sshproxy" -%}
|
|
|
|
{% if grains.cloud is defined -%}
|
|
{% set cloud_provider = "--cloud-provider=" + grains.cloud -%}
|
|
|
|
{% if grains.cloud == 'gce' and grains.cloud_config is defined -%}
|
|
{% set cloud_config = "--cloud-config=" + grains.cloud_config -%}
|
|
{% set cloud_config_mount = "{\"name\": \"cloudconfigmount\",\"mountPath\": \"" + grains.cloud_config + "\", \"readOnly\": true}," -%}
|
|
{% set cloud_config_volume = "{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"" + grains.cloud_config + "\", \"type\": \"FileOrCreate\"}}," -%}
|
|
{% endif -%}
|
|
|
|
{% endif -%}
|
|
|
|
{% set advertise_address = "" -%}
|
|
{% if grains.advertise_address is defined -%}
|
|
{% set advertise_address = "--advertise-address=" + grains.advertise_address -%}
|
|
{% endif -%}
|
|
|
|
{% set proxy_ssh_options = "" -%}
|
|
{% if grains.proxy_ssh_user is defined -%}
|
|
{% set proxy_ssh_options = "--ssh-user=" + grains.proxy_ssh_user + " --ssh-keyfile=/srv/sshproxy/.sshkeyfile" -%}
|
|
{# Append 40 characters onto command to work around #9822. #}
|
|
{# If mount list changes, this may also need to change. #}
|
|
{% set proxy_ssh_options = proxy_ssh_options + " " -%}
|
|
{% endif -%}
|
|
|
|
{% set address = "--address=127.0.0.1" -%}
|
|
|
|
{% set bind_address = "" -%}
|
|
{% if grains.publicAddressOverride is defined -%}
|
|
{% set bind_address = "--bind-address=" + grains.publicAddressOverride -%}
|
|
{% endif -%}
|
|
|
|
{% set storage_backend = "" -%}
|
|
{% if pillar['storage_backend'] is defined -%}
|
|
{% set storage_backend = "--storage-backend=" + pillar['storage_backend'] -%}
|
|
{% endif -%}
|
|
{% set etcd_servers = "--etcd-servers=http://127.0.0.1:2379" -%}
|
|
{% set etcd_servers_overrides = "--etcd-servers-overrides=/events#http://127.0.0.1:4002" -%}
|
|
|
|
{% set storage_media_type = "" -%}
|
|
{% if pillar['storage_media_type'] is defined -%}
|
|
{% set storage_media_type = "--storage-media-type=" + pillar['storage_media_type'] -%}
|
|
{% endif -%}
|
|
|
|
{% set liveness_probe_initial_delay = pillar.get('kube_apiserver_liveness_probe_initial_delay', 15) -%}
|
|
|
|
{% set request_timeout = "" -%}
|
|
{% if pillar['kube_apiserver_request_timeout_sec'] is defined -%}
|
|
{% set request_timeout = "--request-timeout=" + pillar['kube_apiserver_request_timeout_sec'] + "s" -%}
|
|
{% endif -%}
|
|
|
|
{% set max_requests_inflight = "" -%}
|
|
{% set target_ram_mb = "" -%}
|
|
{% if pillar['num_nodes'] is defined -%}
|
|
# If the cluster is large, increase max-requests-inflight limit in apiserver.
|
|
{% if pillar['num_nodes']|int >= 1000 -%}
|
|
{% set max_requests_inflight = "--max-requests-inflight=1500 --max-mutating-requests-inflight=500" -%}
|
|
{% endif -%}
|
|
# Set amount of memory available for apiserver based on number of nodes.
|
|
# TODO: Once we start setting proper requests and limits for apiserver
|
|
# we should reuse the same logic here instead of current heuristic.
|
|
{% set tmp_ram_mb = pillar['num_nodes']|int * 60 %}
|
|
{% set target_ram_mb = "--target-ram-mb=" + tmp_ram_mb|string -%}
|
|
{% endif -%}
|
|
|
|
{% set service_cluster_ip_range = "" -%}
|
|
{% if pillar['service_cluster_ip_range'] is defined -%}
|
|
{% set service_cluster_ip_range = "--service-cluster-ip-range=" + pillar['service_cluster_ip_range'] -%}
|
|
{% endif -%}
|
|
|
|
{% set cert_file = "--tls-cert-file=/srv/kubernetes/server.cert" -%}
|
|
{% set key_file = "--tls-private-key-file=/srv/kubernetes/server.key" -%}
|
|
{% set kubelet_cert_file = "--kubelet-client-certificate=/srv/kubernetes/kubeapiserver.cert" -%}
|
|
{% set kubelet_key_file = "--kubelet-client-key=/srv/kubernetes/kubeapiserver.key" -%}
|
|
{% set client_ca_file = "" -%}
|
|
|
|
{% set secure_port = "6443" -%}
|
|
{% if grains['cloud'] is defined and grains.cloud == 'gce' %}
|
|
{% set secure_port = "443" -%}
|
|
{% set client_ca_file = "--client-ca-file=/srv/kubernetes/ca.crt" -%}
|
|
{% endif -%}
|
|
|
|
{% set min_request_timeout = "" -%}
|
|
{% if grains.minRequestTimeout is defined -%}
|
|
{% set min_request_timeout = "--min-request-timeout=" + grains.minRequestTimeout -%}
|
|
{% endif -%}
|
|
|
|
{% set token_auth_file = " --token-auth-file=/dev/null" -%}
|
|
{% set basic_auth_file = "" -%}
|
|
{% set authz_mode = "" -%}
|
|
{% set abac_policy_file = "" -%}
|
|
{% if grains['cloud'] is defined and grains.cloud == 'gce' %}
|
|
{% set token_auth_file = " --token-auth-file=/srv/kubernetes/known_tokens.csv" -%}
|
|
{% set basic_auth_file = " --basic-auth-file=/srv/kubernetes/basic_auth.csv" -%}
|
|
{% set authz_mode = " --authorization-mode=ABAC" -%}
|
|
{% set abac_policy_file = " --authorization-policy-file=/srv/kubernetes/abac-authz-policy.jsonl" -%}
|
|
{% endif -%}
|
|
|
|
{% set webhook_authentication_config = "" -%}
|
|
{% set webhook_authn_config_mount = "" -%}
|
|
{% set webhook_authn_config_volume = "" -%}
|
|
{% if grains.webhook_authentication_config is defined -%}
|
|
{% set webhook_authentication_config = " --authentication-token-webhook-config-file=" + grains.webhook_authentication_config -%}
|
|
{% set webhook_authn_config_mount = "{\"name\": \"webhookauthnconfigmount\",\"mountPath\": \"" + grains.webhook_authentication_config + "\", \"readOnly\": false}," -%}
|
|
{% set webhook_authn_config_volume = "{\"name\": \"webhookauthnconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authentication_config + "\", \"type\": \"FileOrCreate\"}}," -%}
|
|
{% endif -%}
|
|
|
|
{% set webhook_authorization_config = "" -%}
|
|
{% set webhook_config_mount = "" -%}
|
|
{% set webhook_config_volume = "" -%}
|
|
{% if grains.webhook_authorization_config is defined -%}
|
|
{% set webhook_authorization_config = " --authorization-webhook-config-file=" + grains.webhook_authorization_config -%}
|
|
{% set webhook_config_mount = "{\"name\": \"webhookconfigmount\",\"mountPath\": \"" + grains.webhook_authorization_config + "\", \"readOnly\": false}," -%}
|
|
{% set webhook_config_volume = "{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"" + grains.webhook_authorization_config + "\", \"type\": \"FileOrCreate\"}}," -%}
|
|
{% set authz_mode = authz_mode + ",Webhook" -%}
|
|
{% endif -%}
|
|
|
|
{% set image_review_config = "" -%}
|
|
{% set admission_controller_config_mount = "" -%}
|
|
{% set admission_controller_config_volume = "" -%}
|
|
{% set image_policy_webhook_config_mount = "" -%}
|
|
{% set image_policy_webhook_config_volume = "" -%}
|
|
{% if grains.image_review_config is defined -%}
|
|
{% set image_review_config = " --admission-control-config-file=" + grains.image_review_config -%}
|
|
{% set admission_controller_config_mount = "{\"name\": \"admissioncontrollerconfigmount\",\"mountPath\": \"" + grains.image_review_config + "\", \"readOnly\": false}," -%}
|
|
{% set admission_controller_config_volume = "{\"name\": \"admissioncontrollerconfigmount\",\"hostPath\": {\"path\": \"" + grains.image_review_config + "\", \"type\": \"FileOrCreate\"}}," -%}
|
|
{% set image_policy_webhook_config_mount = "{\"name\": \"imagepolicywebhookconfigmount\",\"mountPath\": \"/etc/gcp_image_review.config\", \"readOnly\": false}," -%}
|
|
{% set image_policy_webhook_config_volume = "{\"name\": \"imagepolicywebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_image_review.config\", \"type\": \"FileOrCreate\"}}," -%}
|
|
{% endif -%}
|
|
|
|
{% set admission_control = "" -%}
|
|
{% if pillar['admission_control'] is defined -%}
|
|
{% set admission_control = "--admission-control=" + pillar['admission_control'] -%}
|
|
{% endif -%}
|
|
|
|
{% set runtime_config = "" -%}
|
|
{% if grains.runtime_config is defined -%}
|
|
{% set runtime_config = "--runtime-config=" + grains.runtime_config -%}
|
|
{% endif -%}
|
|
|
|
{% set feature_gates = "" -%}
|
|
{% if grains.feature_gates is defined -%}
|
|
{% set feature_gates = "--feature-gates=" + grains.feature_gates -%}
|
|
{% endif -%}
|
|
|
|
{% set log_level = pillar['log_level'] -%}
|
|
{% if pillar['api_server_test_log_level'] is defined -%}
|
|
{% set log_level = pillar['api_server_test_log_level'] -%}
|
|
{% endif -%}
|
|
|
|
{% set enable_garbage_collector = "" -%}
|
|
{% if pillar['enable_garbage_collector'] is defined -%}
|
|
{% set enable_garbage_collector = "--enable-garbage-collector=" + pillar['enable_garbage_collector'] -%}
|
|
{% endif -%}
|
|
|
|
{% set etcd_quorum_read = "" %}
|
|
{% if pillar['etcd_quorum_read'] is defined -%}
|
|
{% set etcd_quorum_read = "--etcd_quorum_read=" + pillar['etcd_quorum_read'] -%}
|
|
{% endif -%}
|
|
|
|
{% set audit_log = "" -%}
|
|
{% set audit_policy_config_mount = "" -%}
|
|
{% set audit_policy_config_volume = "" -%}
|
|
{% set audit_webhook_config_mount = "" -%}
|
|
{% set audit_webhook_config_volume = "" -%}
|
|
{% if pillar['enable_apiserver_basic_audit'] is defined and pillar['enable_apiserver_basic_audit'] in ['true'] -%}
|
|
{% set audit_log = "--audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%}
|
|
{% elif pillar['enable_apiserver_advanced_audit'] is defined and pillar['enable_apiserver_advanced_audit'] in ['true'] -%}
|
|
{% set audit_log = "--audit-policy-file=/etc/audit_policy.config" -%}
|
|
{% set audit_policy_config_mount = "{\"name\": \"auditpolicyconfigmount\",\"mountPath\": \"/etc/audit_policy.config\", \"readOnly\": true}," -%}
|
|
{% set audit_policy_config_volume = "{\"name\": \"auditpolicyconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_policy.config\", \"type\": \"FileOrCreate\"}}," -%}
|
|
{% if pillar['advanced_audit_backend'] is defined and 'log' in pillar['advanced_audit_backend'] -%}
|
|
{% set audit_log = audit_log + " --audit-log-path=/var/log/kube-apiserver-audit.log --audit-log-maxage=0 --audit-log-maxbackup=0 --audit-log-maxsize=2000000000" -%}
|
|
{% endif %}
|
|
{% if pillar['advanced_audit_backend'] is defined and 'webhook' in pillar['advanced_audit_backend'] -%}
|
|
{% set audit_log = audit_log + " --audit-webhook-mode=batch" -%}
|
|
{% set audit_webhook_config_mount = "{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"/etc/audit_webhook.config\", \"readOnly\": true}," -%}
|
|
{% set audit_webhook_config_volume = "{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"/etc/audit_webhook.config\", \"type\": \"FileOrCreate\"}}," -%}
|
|
{% endif %}
|
|
{% endif -%}
|
|
|
|
{% set params = address + " " + storage_backend + " " + storage_media_type + " " + etcd_servers + " " + etcd_servers_overrides + " " + cloud_provider + " " + cloud_config + " " + runtime_config + " " + feature_gates + " " + admission_control + " " + max_requests_inflight + " " + target_ram_mb + " " + service_cluster_ip_range + " " + client_ca_file + basic_auth_file + " " + min_request_timeout + " " + enable_garbage_collector + " " + etcd_quorum_read + " " + audit_log + " " + request_timeout -%}
|
|
{% set params = params + " " + cert_file + " " + key_file + " " + kubelet_cert_file + " " + kubelet_key_file + " --secure-port=" + secure_port + token_auth_file + " " + bind_address + " " + log_level + " " + advertise_address + " " + proxy_ssh_options + authz_mode + abac_policy_file + webhook_authentication_config + webhook_authorization_config + image_review_config -%}
|
|
|
|
# test_args has to be kept at the end, so they'll overwrite any prior configuration
|
|
{% if pillar['apiserver_test_args'] is defined -%}
|
|
{% set params = params + " " + pillar['apiserver_test_args'] -%}
|
|
{% endif -%}
|
|
|
|
{% set container_env = "" -%}
|
|
|
|
{
|
|
"apiVersion": "v1",
|
|
"kind": "Pod",
|
|
"metadata": {
|
|
"name":"kube-apiserver",
|
|
"namespace": "kube-system",
|
|
"annotations": {
|
|
"scheduler.alpha.kubernetes.io/critical-pod": ""
|
|
},
|
|
"labels": {
|
|
"tier": "control-plane",
|
|
"component": "kube-apiserver"
|
|
}
|
|
},
|
|
"spec":{
|
|
"hostNetwork": true,
|
|
"containers":[
|
|
{
|
|
"name": "kube-apiserver",
|
|
"image": "{{pillar['kube_docker_registry']}}/kube-apiserver:{{pillar['kube-apiserver_docker_tag']}}",
|
|
"resources": {
|
|
"requests": {
|
|
"cpu": "250m"
|
|
}
|
|
},
|
|
"command": [
|
|
"/bin/sh",
|
|
"-c",
|
|
"exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1"
|
|
],
|
|
{{container_env}}
|
|
"livenessProbe": {
|
|
"httpGet": {
|
|
"host": "127.0.0.1",
|
|
"port": 8080,
|
|
"path": "/healthz"
|
|
},
|
|
"initialDelaySeconds": {{liveness_probe_initial_delay}},
|
|
"timeoutSeconds": 15
|
|
},
|
|
"ports":[
|
|
{ "name": "https",
|
|
"containerPort": {{secure_port}},
|
|
"hostPort": {{secure_port}}},{
|
|
"name": "local",
|
|
"containerPort": 8080,
|
|
"hostPort": 8080}
|
|
],
|
|
"volumeMounts": [
|
|
{{cloud_config_mount}}
|
|
{{additional_cloud_config_mount}}
|
|
{{webhook_config_mount}}
|
|
{{webhook_authn_config_mount}}
|
|
{{audit_policy_config_mount}}
|
|
{{audit_webhook_config_mount}}
|
|
{{admission_controller_config_mount}}
|
|
{{image_policy_webhook_config_mount}}
|
|
{ "name": "srvkube",
|
|
"mountPath": "{{srv_kube_path}}",
|
|
"readOnly": true},
|
|
{ "name": "logfile",
|
|
"mountPath": "/var/log/kube-apiserver.log",
|
|
"readOnly": false},
|
|
{ "name": "auditlogfile",
|
|
"mountPath": "/var/log/kube-apiserver-audit.log",
|
|
"readOnly": false},
|
|
{ "name": "etcssl",
|
|
"mountPath": "/etc/ssl",
|
|
"readOnly": true},
|
|
{ "name": "usrsharecacerts",
|
|
"mountPath": "/usr/share/ca-certificates",
|
|
"readOnly": true},
|
|
{ "name": "varssl",
|
|
"mountPath": "/var/ssl",
|
|
"readOnly": true},
|
|
{ "name": "etcopenssl",
|
|
"mountPath": "/etc/openssl",
|
|
"readOnly": true},
|
|
{ "name": "etcpki",
|
|
"mountPath": "/etc/srv/pki",
|
|
"readOnly": true},
|
|
{ "name": "srvsshproxy",
|
|
"mountPath": "{{srv_sshproxy_path}}",
|
|
"readOnly": false}
|
|
]
|
|
}
|
|
],
|
|
"volumes":[
|
|
{{cloud_config_volume}}
|
|
{{additional_cloud_config_volume}}
|
|
{{webhook_config_volume}}
|
|
{{webhook_authn_config_volume}}
|
|
{{audit_policy_config_volume}}
|
|
{{audit_webhook_config_volume}}
|
|
{{admission_controller_config_volume}}
|
|
{{image_policy_webhook_config_volume}}
|
|
{ "name": "srvkube",
|
|
"hostPath": {
|
|
"path": "{{srv_kube_path}}"}
|
|
},
|
|
{ "name": "logfile",
|
|
"hostPath": {
|
|
"path": "/var/log/kube-apiserver.log",
|
|
"type": "FileOrCreate"}
|
|
},
|
|
{ "name": "auditlogfile",
|
|
"hostPath": {
|
|
"path": "/var/log/kube-apiserver-audit.log",
|
|
"type": "FileOrCreate"}
|
|
},
|
|
{ "name": "etcssl",
|
|
"hostPath": {
|
|
"path": "/etc/ssl"}
|
|
},
|
|
{ "name": "usrsharecacerts",
|
|
"hostPath": {
|
|
"path": "/usr/share/ca-certificates"}
|
|
},
|
|
{ "name": "varssl",
|
|
"hostPath": {
|
|
"path": "/var/ssl"}
|
|
},
|
|
{ "name": "etcopenssl",
|
|
"hostPath": {
|
|
"path": "/etc/openssl"}
|
|
},
|
|
{ "name": "etcpki",
|
|
"hostPath": {
|
|
"path": "/etc/srv/pki"}
|
|
},
|
|
{ "name": "srvsshproxy",
|
|
"hostPath": {
|
|
"path": "{{srv_sshproxy_path}}"}
|
|
}
|
|
]
|
|
}}
|