diff --git a/cluster/kubemark/config-default.sh b/cluster/kubemark/config-default.sh index 385cf90ce1..60306151ef 100644 --- a/cluster/kubemark/config-default.sh +++ b/cluster/kubemark/config-default.sh @@ -34,9 +34,9 @@ MASTER_ROOT_DISK_SIZE=${KUBEMARK_MASTER_ROOT_DISK_SIZE:-10GB} REGISTER_MASTER_KUBELET=${REGISTER_MASTER:-false} PREEMPTIBLE_NODE=${PREEMPTIBLE_NODE:-false} -MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-debian} +MASTER_OS_DISTRIBUTION=${KUBE_MASTER_OS_DISTRIBUTION:-gci} NODE_OS_DISTRIBUTION=${KUBE_NODE_OS_DISTRIBUTION:-debian} -MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-container-vm-v20161208} +MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-gci-dev-56-8977-0-0} MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} NETWORK=${KUBE_GCE_NETWORK:-default} diff --git a/test/kubemark/resources/manifests/kube-apiserver.yaml b/test/kubemark/resources/manifests/kube-apiserver.yaml index 9e7ae840b2..f1475e23cd 100644 --- a/test/kubemark/resources/manifests/kube-apiserver.yaml +++ b/test/kubemark/resources/manifests/kube-apiserver.yaml @@ -37,7 +37,7 @@ spec: protocol: TCP volumeMounts: - name: srvkube - mountPath: /srv/kubernetes + mountPath: /etc/srv/kubernetes readOnly: true - name: logfile mountPath: /var/log/kube-apiserver.log @@ -48,11 +48,11 @@ spec: mountPath: /usr/share/ca-certificates readOnly: true - name: srvsshproxy - mountPath: /srv/sshproxy + mountPath: /etc/srv/sshproxy volumes: - name: srvkube hostPath: - path: /srv/kubernetes + path: /etc/srv/kubernetes - name: logfile hostPath: path: /var/log/kube-apiserver.log @@ -64,4 +64,4 @@ spec: path: /usr/share/ca-certificates - name: srvsshproxy hostPath: - path: /srv/sshproxy + path: /etc/srv/sshproxy diff --git a/test/kubemark/resources/manifests/kube-controller-manager.yaml b/test/kubemark/resources/manifests/kube-controller-manager.yaml index 5f600b9ac9..a3f1b491c0 100644 --- a/test/kubemark/resources/manifests/kube-controller-manager.yaml +++ b/test/kubemark/resources/manifests/kube-controller-manager.yaml @@ -28,7 +28,7 @@ spec: timeoutSeconds: 15 volumeMounts: - name: srvkube - mountPath: /srv/kubernetes + mountPath: /etc/srv/kubernetes readOnly: true - name: logfile mountPath: /var/log/kube-controller-manager.log @@ -41,7 +41,7 @@ spec: volumes: - name: srvkube hostPath: - path: /srv/kubernetes + path: /etc/srv/kubernetes - name: logfile hostPath: path: /var/log/kube-controller-manager.log diff --git a/test/kubemark/resources/start-kubemark-master.sh b/test/kubemark/resources/start-kubemark-master.sh index 0f975e56d7..123951fd5a 100644 --- a/test/kubemark/resources/start-kubemark-master.sh +++ b/test/kubemark/resources/start-kubemark-master.sh @@ -21,6 +21,24 @@ KUBE_ROOT="/home/kubernetes" KUBE_BINDIR="${KUBE_ROOT}/kubernetes/server/bin" +function config-ip-firewall { + echo "Configuring IP firewall rules" + # The GCI image has host firewall which drop most inbound/forwarded packets. + # We need to add rules to accept all TCP/UDP/ICMP packets. + if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then + echo "Add rules to accept all inbound TCP/UDP/ICMP packets" + iptables -A INPUT -w -p TCP -j ACCEPT + iptables -A INPUT -w -p UDP -j ACCEPT + iptables -A INPUT -w -p ICMP -j ACCEPT + fi + if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then + echo "Add rules to accept all forwarded TCP/UDP/ICMP packets" + iptables -A FORWARD -w -p TCP -j ACCEPT + iptables -A FORWARD -w -p UDP -j ACCEPT + iptables -A FORWARD -w -p ICMP -j ACCEPT + fi +} + function create-dirs { echo "Creating required directories" mkdir -p /var/lib/kubelet @@ -102,8 +120,8 @@ function mount-pd() { return fi - echo "Mounting PD '${pd_path}' at '${mount_point}'" local -r pd_path="/dev/disk/by-id/${pd_name}" + echo "Mounting PD '${pd_path}' at '${mount_point}'" # Format and mount the disk, create directories on it for all of the master's # persistent data, and link them to where they're used. mkdir -p "${mount_point}" @@ -123,8 +141,7 @@ function assemble-docker-flags { # TODO(shyamjvs): Incorporate network plugin options, etc later. echo "DOCKER_OPTS=\"${docker_opts}\"" > /etc/default/docker echo "DOCKER_NOFILE=65536" >> /etc/default/docker # For setting ulimit -n - service docker restart - # TODO(shyamjvs): Make docker run through systemd/supervisord. + systemd restart docker } # A helper function for loading a docker image. It keeps trying up to 5 times. @@ -176,39 +193,40 @@ function compute-kubelet-params { echo "${params}" } -# Creates the supervisord config file for kubelet from the exec_command ($1). +# Creates the systemd config file for kubelet.service. function create-kubelet-conf() { - local -r name="kubelet" - local exec_command="$1 " - exec_command+=$(compute-kubelet-params) + local -r kubelet_bin="$1" + local -r kubelet_env_file="/etc/default/kubelet" + local -r flags=$(compute-kubelet-params) + echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}" - cat >>"/etc/supervisor/conf.d/${name}.conf" </etc/systemd/system/kubelet.service +[Unit] +Description=Kubermark kubelet +Requires=network-online.target +After=network-online.target + +[Service] +Restart=always +RestartSec=10 +EnvironmentFile=${kubelet_env_file} +ExecStart=${kubelet_bin} \$KUBELET_OPTS + +[Install] +WantedBy=multi-user.target EOF } -# This function assembles the kubelet supervisord config file and starts it using -# supervisorctl, on the kubemark master. +# This function assembles the kubelet systemd service file and starts it using +# systemctl, on the kubemark master. function start-kubelet { - # Kill any pre-existing kubelet process(es). - pkill kubelet - # Replace the builtin kubelet (if any) with the correct binary. - local -r builtin_kubelet="$(which kubelet)" - if [[ -n "${builtin_kubelet}" ]]; then - cp "${KUBE_BINDIR}/kubelet" "$(dirname "$builtin_kubelet")" - fi + # Create systemd config. + local -r kubelet_bin="/usr/bin/kubelet" + create-kubelet-conf "${kubelet_bin}" - # Create supervisord config for kubelet. - create-kubelet-conf "${KUBE_BINDIR}/kubelet" - - # Update supervisord to make it run kubelet. - supervisorctl reread - supervisorctl update + # Start the kubelet service. + systemctl start kubelet.service } # Create the log file and set its properties. @@ -248,12 +266,12 @@ function compute-kube-apiserver-params { params+=" --insecure-bind-address=0.0.0.0" params+=" --etcd-servers=http://127.0.0.1:2379" params+=" --etcd-servers-overrides=/events#${EVENT_STORE_URL}" - params+=" --tls-cert-file=/srv/kubernetes/server.cert" - params+=" --tls-private-key-file=/srv/kubernetes/server.key" - params+=" --client-ca-file=/srv/kubernetes/ca.crt" - params+=" --token-auth-file=/srv/kubernetes/known_tokens.csv" + params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert" + params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key" + params+=" --client-ca-file=/etc/srv/kubernetes/ca.crt" + params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" params+=" --secure-port=443" - params+=" --basic-auth-file=/srv/kubernetes/basic_auth.csv" + params+=" --basic-auth-file=/etc/srv/kubernetes/basic_auth.csv" params+=" --target-ram-mb=$((${NUM_NODES} * 60))" params+=" --storage-backend=${STORAGE_BACKEND}" params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" @@ -265,8 +283,8 @@ function compute-kube-apiserver-params { function compute-kube-controller-manager-params { local params="${CONTROLLER_MANAGER_TEST_ARGS:-}" params+=" --master=127.0.0.1:8080" - params+=" --service-account-private-key-file=/srv/kubernetes/server.key" - params+=" --root-ca-file=/srv/kubernetes/ca.crt" + params+=" --service-account-private-key-file=/etc/srv/kubernetes/server.key" + params+=" --root-ca-file=/etc/srv/kubernetes/ca.crt" params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" params+=" --cluster-cidr=${CLUSTER_IP_RANGE}" params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" @@ -326,7 +344,8 @@ cd "${KUBE_ROOT}" tar xzf kubernetes-server-linux-amd64.tar.gz source "${KUBE_ROOT}/kubemark-master-env.sh" -# Setup required directory structure and etcd variables. +# Setup IP firewall rules, required directory structure and etcd variables. +config-ip-firewall create-dirs setup-kubelet-dir delete-default-etcd-configs diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index 37fa59c251..c57bfd7c9c 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -145,16 +145,16 @@ done password=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))') gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \ - --command="sudo mkdir /home/kubernetes -p && sudo mkdir /srv/kubernetes -p && \ - sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /srv/kubernetes/server.cert\" && \ - sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /srv/kubernetes/server.key\" && \ - sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /srv/kubernetes/ca.crt\" && \ - sudo bash -c \"echo ${KUBECFG_CERT_BASE64} | base64 --decode > /srv/kubernetes/kubecfg.crt\" && \ - sudo bash -c \"echo ${KUBECFG_KEY_BASE64} | base64 --decode > /srv/kubernetes/kubecfg.key\" && \ - sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo \"${KUBELET_TOKEN},kubelet,kubelet\" >> /srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy\" >> /srv/kubernetes/known_tokens.csv\" && \ - sudo bash -c \"echo ${password},admin,admin > /srv/kubernetes/basic_auth.csv\"" + --command="sudo mkdir /home/kubernetes -p && sudo mkdir /etc/srv/kubernetes -p && \ + sudo bash -c \"echo ${MASTER_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/server.cert\" && \ + sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /etc/srv/kubernetes/server.key\" && \ + sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/ca.crt\" && \ + sudo bash -c \"echo ${KUBECFG_CERT_BASE64} | base64 --decode > /etc/srv/kubernetes/kubecfg.crt\" && \ + sudo bash -c \"echo ${KUBECFG_KEY_BASE64} | base64 --decode > /etc/srv/kubernetes/kubecfg.key\" && \ + sudo bash -c \"echo \"${KUBE_BEARER_TOKEN},admin,admin\" > /etc/srv/kubernetes/known_tokens.csv\" && \ + sudo bash -c \"echo \"${KUBELET_TOKEN},kubelet,kubelet\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ + sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy\" >> /etc/srv/kubernetes/known_tokens.csv\" && \ + sudo bash -c \"echo ${password},admin,admin > /etc/srv/kubernetes/basic_auth.csv\"" gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \ @@ -167,12 +167,12 @@ gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \ "${RESOURCE_DIRECTORY}/manifests/kube-apiserver.yaml" \ "${RESOURCE_DIRECTORY}/manifests/kube-scheduler.yaml" \ "${RESOURCE_DIRECTORY}/manifests/kube-controller-manager.yaml" \ - "root@${MASTER_NAME}":/home/kubernetes/ + "kubernetes@${MASTER_NAME}":/home/kubernetes/ gcloud compute ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" \ --command="sudo chmod a+x /home/kubernetes/configure-kubectl.sh && \ sudo chmod a+x /home/kubernetes/start-kubemark-master.sh && \ - sudo /home/kubernetes/start-kubemark-master.sh" + sudo bash /home/kubernetes/start-kubemark-master.sh" # create kubeconfig for Kubelet: KUBECONFIG_CONTENTS=$(echo "apiVersion: v1