diff --git a/test/kubemark/resources/heapster_template.json b/test/kubemark/resources/heapster_template.json index 7ddb879837..d9ec508bc6 100644 --- a/test/kubemark/resources/heapster_template.json +++ b/test/kubemark/resources/heapster_template.json @@ -37,14 +37,14 @@ "resources": { "requests": { "cpu": "100m", - "memory": "##METRICS_MEM##Mi" + "memory": "{{METRICS_MEM}}Mi" } }, "command": [ "/heapster" ], "args": [ - "--source=kubernetes:https://##MASTER_IP##:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/kubeconfig" + "--source=kubernetes:https://{{MASTER_IP}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/kubeconfig" ], "volumeMounts": [ { @@ -59,7 +59,7 @@ "resources": { "requests": { "cpu": "100m", - "memory": "##EVENTER_MEM##Ki" + "memory": "{{EVENTER_MEM}}Ki" } }, "command": [ diff --git a/test/kubemark/resources/hollow-node_template.json b/test/kubemark/resources/hollow-node_template.json index 5d0d16c561..0493274032 100644 --- a/test/kubemark/resources/hollow-node_template.json +++ b/test/kubemark/resources/hollow-node_template.json @@ -8,7 +8,7 @@ } }, "spec": { - "replicas": ##numreplicas##, + "replicas": {{numreplicas}}, "selector": { "name": "hollow-node" }, @@ -36,7 +36,7 @@ "containers": [ { "name": "hollow-kubelet", - "image": "gcr.io/##project##/kubemark:latest", + "image": "gcr.io/{{project}}/kubemark:latest", "ports": [ {"containerPort": 4194}, {"containerPort": 10250}, @@ -89,7 +89,7 @@ }, { "name": "hollow-proxy", - "image": "gcr.io/##project##/kubemark:latest", + "image": "gcr.io/{{project}}/kubemark:latest", "env": [ { "name": "CONTENT_TYPE", diff --git a/test/kubemark/resources/manifests/etcd-events.yaml b/test/kubemark/resources/manifests/etcd-events.yaml new file mode 100644 index 0000000000..59fb70dc4a --- /dev/null +++ b/test/kubemark/resources/manifests/etcd-events.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: Pod +metadata: + name: etcd-server-events + namespace: kube-system +spec: + hostNetwork: true + nodeName: {{instance_prefix}}-master + containers: + - name: etcd-container + image: {{kube_docker_registry}}/etcd:{{etcd_image}} + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + command: + - /bin/sh + - -c + - /usr/local/bin/etcd + {{params}} + 1>>/var/log/etcd-events.log 2>&1 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /health + port: 4002 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 15 + ports: + - name: serverport + containerPort: 2381 + hostPort: 2381 + protocol: TCP + - name: clientport + containerPort: 4002 + hostPort: 4002 + protocol: TCP + volumeMounts: + - name: varetcd + mountPath: /var/etcd + - name: varlogetcd + mountPath: /var/log/etcd-events.log + volumes: + - name: varetcd + hostPath: + path: /var/etcd/events + - name: varlogetcd + hostPath: + path: /var/log/etcd-events.log diff --git a/test/kubemark/resources/manifests/etcd.yaml b/test/kubemark/resources/manifests/etcd.yaml new file mode 100644 index 0000000000..6c1ab1dfca --- /dev/null +++ b/test/kubemark/resources/manifests/etcd.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Pod +metadata: + name: etcd-server + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: etcd-container + image: {{kube_docker_registry}}/etcd:{{etcd_image}} + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 200m + command: + - /bin/sh + - -c + - /usr/local/bin/etcd + {{params}} + 1>>/var/log/etcd.log 2>&1 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /health + port: 2379 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 15 + ports: + - name: serverport + containerPort: 2380 + hostPort: 2380 + protocol: TCP + - name: clientport + containerPort: 2379 + hostPort: 2379 + protocol: TCP + volumeMounts: + - name: varetcd + mountPath: /var/etcd + - name: varlogetcd + mountPath: /var/log/etcd.log + volumes: + - name: varetcd + hostPath: + path: /var/etcd + - name: varlogetcd + hostPath: + path: /var/log/etcd.log diff --git a/test/kubemark/resources/manifests/kube-apiserver.yaml b/test/kubemark/resources/manifests/kube-apiserver.yaml new file mode 100644 index 0000000000..9e7ae840b2 --- /dev/null +++ b/test/kubemark/resources/manifests/kube-apiserver.yaml @@ -0,0 +1,67 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-apiserver + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: kube-apiserver + image: {{kube_docker_registry}}/kube-apiserver:{{kube-apiserver_docker_tag}} + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 250m + command: + - /bin/sh + - -c + - /usr/local/bin/kube-apiserver + {{params}} + 1>>/var/log/kube-apiserver.log 2>&1 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 15 + ports: + - name: https + containerPort: 443 + hostPort: 443 + protocol: TCP + - name: local + containerPort: 8080 + hostPort: 8080 + protocol: TCP + volumeMounts: + - name: srvkube + mountPath: /srv/kubernetes + readOnly: true + - name: logfile + mountPath: /var/log/kube-apiserver.log + - name: etcssl + mountPath: /etc/ssl + readOnly: true + - name: usrsharecacerts + mountPath: /usr/share/ca-certificates + readOnly: true + - name: srvsshproxy + mountPath: /srv/sshproxy + volumes: + - name: srvkube + hostPath: + path: /srv/kubernetes + - name: logfile + hostPath: + path: /var/log/kube-apiserver.log + - name: etcssl + hostPath: + path: /etc/ssl + - name: usrsharecacerts + hostPath: + path: /usr/share/ca-certificates + - name: srvsshproxy + hostPath: + path: /srv/sshproxy diff --git a/test/kubemark/resources/manifests/kube-controller-manager.yaml b/test/kubemark/resources/manifests/kube-controller-manager.yaml new file mode 100644 index 0000000000..5f600b9ac9 --- /dev/null +++ b/test/kubemark/resources/manifests/kube-controller-manager.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-controller-manager + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: kube-controller-manager + image: {{kube_docker_registry}}/kube-controller-manager:{{kube-controller-manager_docker_tag}} + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 200m + command: + - /bin/sh + - -c + - /usr/local/bin/kube-controller-manager + {{params}} + 1>>/var/log/kube-controller-manager.log 2>&1 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10252 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 15 + volumeMounts: + - name: srvkube + mountPath: /srv/kubernetes + readOnly: true + - name: logfile + mountPath: /var/log/kube-controller-manager.log + - name: etcssl + mountPath: /etc/ssl + readOnly: true + - name: usrsharecacerts + mountPath: /usr/share/ca-certificates + readOnly: true + volumes: + - name: srvkube + hostPath: + path: /srv/kubernetes + - name: logfile + hostPath: + path: /var/log/kube-controller-manager.log + - name: etcssl + hostPath: + path: /etc/ssl + - name: usrsharecacerts + hostPath: + path: /usr/share/ca-certificates diff --git a/test/kubemark/resources/manifests/kube-scheduler.yaml b/test/kubemark/resources/manifests/kube-scheduler.yaml new file mode 100644 index 0000000000..3cf01ac950 --- /dev/null +++ b/test/kubemark/resources/manifests/kube-scheduler.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-scheduler + namespace: kube-system +spec: + hostNetwork: true + nodeName: {{instance_prefix}}-master + containers: + - name: kube-scheduler + image: {{kube_docker_registry}}/kube-scheduler:{{kube-scheduler_docker_tag}} + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + command: + - /bin/sh + - -c + - /usr/local/bin/kube-scheduler + {{params}} + 1>>/var/log/kube-scheduler.log 2>&1 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10251 + scheme: HTTP + initialDelaySeconds: 15 + timeoutSeconds: 15 + volumeMounts: + - name: logfile + mountPath: /var/log/kube-scheduler.log + volumes: + - name: logfile + hostPath: + path: /var/log/kube-scheduler.log diff --git a/test/kubemark/resources/start-kubemark-master.sh b/test/kubemark/resources/start-kubemark-master.sh index e75e66543f..0f975e56d7 100644 --- a/test/kubemark/resources/start-kubemark-master.sh +++ b/test/kubemark/resources/start-kubemark-master.sh @@ -14,143 +14,372 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: figure out how to get etcd tag from some real configuration and put it here. +# Script that starts kubelet on kubemark-master as a supervisord process +# and then runs the master components as pods using kubelet. + +# Define key path variables. +KUBE_ROOT="/home/kubernetes" +KUBE_BINDIR="${KUBE_ROOT}/kubernetes/server/bin" + +function create-dirs { + echo "Creating required directories" + mkdir -p /var/lib/kubelet + mkdir -p /etc/kubernetes/manifests +} + +# Setup working directory for kubelet. +function setup-kubelet-dir { + echo "Making /var/lib/kubelet executable for kubelet" + mount -B /var/lib/kubelet /var/lib/kubelet/ + mount -B -o remount,exec,suid,dev /var/lib/kubelet +} + +# Remove any default etcd config dirs/files. +function delete-default-etcd-configs { + if [[ -d /etc/etcd ]]; then + rm -rf /etc/etcd + fi + if [[ -e /etc/default/etcd ]]; then + rm -f /etc/default/etcd + fi + if [[ -e /etc/systemd/system/etcd.service ]]; then + rm -f /etc/systemd/system/etcd.service + fi + if [[ -e /etc/init.d/etcd ]]; then + rm -f /etc/init.d/etcd + fi +} + +# Compute etcd related variables. +function compute-etcd-variables { + ETCD_IMAGE="${ETCD_IMAGE:-}" + ETCD_QUOTA_BYTES="" + if [ "${ETCD_VERSION:0:2}" == "3." ]; then + # TODO: Set larger quota to see if that helps with + # 'mvcc: database space exceeded' errors. If so, pipe + # though our setup scripts. + ETCD_QUOTA_BYTES=" --quota-backend-bytes=4294967296 " + fi +} + +# Formats the given device ($1) if needed and mounts it at given mount point +# ($2). +function safe-format-and-mount() { + device=$1 + mountpoint=$2 + + # Format only if the disk is not already formatted. + if ! tune2fs -l "${device}" ; then + echo "Formatting '${device}'" + mkfs.ext4 -F -E lazy_itable_init=0,lazy_journal_init=0,discard "${device}" + fi + + mkdir -p "${mountpoint}" + echo "Mounting '${device}' at '${mountpoint}'" + mount -o discard,defaults "${device}" "${mountpoint}" +} + +# Finds a PD device with name '$1' attached to the master. +function find-attached-pd() { + local -r pd_name=$1 + if [[ ! -e /dev/disk/by-id/${pd_name} ]]; then + echo "" + fi + device_info=$(ls -l /dev/disk/by-id/${pd_name}) + relative_path=${device_info##* } + echo "/dev/disk/by-id/${relative_path}" +} + +# Mounts a persistent disk (formatting if needed) to store the persistent data +# on the master. safe-format-and-mount only formats an unformatted disk, and +# mkdir -p will leave a directory be if it already exists. +function mount-pd() { + local -r pd_name=$1 + local -r mount_point=$2 + + if [[ -z "${find-attached-pd ${pd_name}}" ]]; then + echo "Can't find ${pd_name}. Skipping mount." + return + fi + + echo "Mounting PD '${pd_path}' at '${mount_point}'" + local -r pd_path="/dev/disk/by-id/${pd_name}" + # Format and mount the disk, create directories on it for all of the master's + # persistent data, and link them to where they're used. + mkdir -p "${mount_point}" + safe-format-and-mount "${pd_path}" "${mount_point}" + echo "Mounted PD '${pd_path}' at '${mount_point}'" + + # NOTE: These locations on the PD store persistent data, so to maintain + # upgradeability, these locations should not change. If they do, take care + # to maintain a migration path from these locations to whatever new + # locations. +} + +function assemble-docker-flags { + echo "Assemble docker command line flags" + local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" + docker_opts+=" --log-level=debug" # Since it's a test cluster + # TODO(shyamjvs): Incorporate network plugin options, etc later. + echo "DOCKER_OPTS=\"${docker_opts}\"" > /etc/default/docker + echo "DOCKER_NOFILE=65536" >> /etc/default/docker # For setting ulimit -n + service docker restart + # TODO(shyamjvs): Make docker run through systemd/supervisord. +} + +# A helper function for loading a docker image. It keeps trying up to 5 times. +# +# $1: Full path of the docker image +function try-load-docker-image { + local -r img=$1 + echo "Try to load docker image file ${img}" + # Temporarily turn off errexit, because we don't want to exit on first failure. + set +e + local -r max_attempts=5 + local -i attempt_num=1 + until timeout 30 docker load -i "${img}"; do + if [[ "${attempt_num}" == "${max_attempts}" ]]; then + echo "Fail to load docker image file ${img} after ${max_attempts} retries. Exit!!" + exit 1 + else + attempt_num=$((attempt_num+1)) + sleep 5 + fi + done + # Re-enable errexit. + set -e +} + +# Loads kube-system docker images. It is better to do it before starting kubelet, +# as kubelet will restart docker daemon, which may interfere with loading images. +function load-docker-images { + echo "Start loading kube-system docker images" + local -r img_dir="${KUBE_BINDIR}" + try-load-docker-image "${img_dir}/kube-apiserver.tar" + try-load-docker-image "${img_dir}/kube-controller-manager.tar" + try-load-docker-image "${img_dir}/kube-scheduler.tar" +} + +# Computes command line arguments to be passed to kubelet. +function compute-kubelet-params { + local params="${KUBELET_TEST_ARGS:-}" + params+=" --allow-privileged=true" + params+=" --babysit-daemons=true" + params+=" --cgroup-root=/" + params+=" --cloud-provider=gce" + params+=" --config=/etc/kubernetes/manifests" + if [[ -n "${KUBELET_PORT:-}" ]]; then + params+=" --port=${KUBELET_PORT}" + fi + params+=" --enable-debugging-handlers=false" + params+=" --hairpin-mode=none" + echo "${params}" +} + +# Creates the supervisord config file for kubelet from the exec_command ($1). +function create-kubelet-conf() { + local -r name="kubelet" + local exec_command="$1 " + exec_command+=$(compute-kubelet-params) -function write_supervisor_conf() { - local name=$1 - local exec_command=$2 cat >>"/etc/supervisor/conf.d/${name}.conf" <> /var/log/etcd-events.log 2>&1" -fi +# Computes command line arguments to be passed to apiserver. +function compute-kube-apiserver-params { + local params="${APISERVER_TEST_ARGS:-}" + params+=" --insecure-bind-address=0.0.0.0" + params+=" --etcd-servers=http://127.0.0.1:2379" + params+=" --etcd-servers-overrides=/events#${EVENT_STORE_URL}" + params+=" --tls-cert-file=/srv/kubernetes/server.cert" + params+=" --tls-private-key-file=/srv/kubernetes/server.key" + params+=" --client-ca-file=/srv/kubernetes/ca.crt" + params+=" --token-auth-file=/srv/kubernetes/known_tokens.csv" + params+=" --secure-port=443" + params+=" --basic-auth-file=/srv/kubernetes/basic_auth.csv" + params+=" --target-ram-mb=$((${NUM_NODES} * 60))" + params+=" --storage-backend=${STORAGE_BACKEND}" + params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" + params+=" --admission-control=${CUSTOM_ADMISSION_PLUGINS}" + echo "${params}" +} -# Retry starting etcd to avoid pulling image errors. -retry sudo docker run --net=host \ - -v /var/etcd/data:/var/etcd/data -v /var/log:/var/log -d \ - gcr.io/google_containers/etcd:${KUBEMARK_ETCD_IMAGE} /bin/sh -c "/usr/local/bin/etcd \ - --listen-peer-urls http://127.0.0.1:2380 \ - --advertise-client-urls=http://127.0.0.1:2379 \ - --listen-client-urls=http://0.0.0.0:2379 \ - --data-dir=/var/etcd/data ${ETCD_QUOTA_BYTES} 1>> /var/log/etcd.log 2>&1" +# Computes command line arguments to be passed to controller-manager. +function compute-kube-controller-manager-params { + local params="${CONTROLLER_MANAGER_TEST_ARGS:-}" + params+=" --master=127.0.0.1:8080" + params+=" --service-account-private-key-file=/srv/kubernetes/server.key" + params+=" --root-ca-file=/srv/kubernetes/ca.crt" + params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" + params+=" --cluster-cidr=${CLUSTER_IP_RANGE}" + params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" + params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" + echo "${params}" +} -ulimit_command='bash -c "ulimit -n 65536;' +# Computes command line arguments to be passed to scheduler. +function compute-kube-scheduler-params { + local params="${SCHEDULER_TEST_ARGS:-}" + params+=" --master=127.0.0.1:8080" + echo "${params}" +} -cd / +# Start a kubernetes master component '$1' which can be any of the following: +# 1. etcd +# 2. etcd-events +# 3. kube-apiserver +# 4. kube-controller-manager +# 5. kube-scheduler +# +# It prepares the log file, loads the docker tag, calculates variables, sets them +# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. +# +# Assumed vars: +# DOCKER_REGISTRY +function start-kubemaster-component() { + local -r component=$1 + local component_is_etcd=false + if [ "${component:0:4}" == "etcd" ]; then + component_is_etcd=true + fi + + echo "Start master component ${component}" + prepare-log-file /var/log/"${component}".log + local -r src_file="${KUBE_ROOT}/${component}.yaml" + local -r params=$(compute-${component}-params) + + # Evaluate variables. + sed -i -e "s@{{params}}@${params}@g" "${src_file}" + sed -i -e "s@{{kube_docker_registry}}@${DOCKER_REGISTRY}@g" "${src_file}" + sed -i -e "s@{{instance_prefix}}@${INSTANCE_PREFIX}@g" "${src_file}" + if [ "${component_is_etcd}" == "true" ]; then + sed -i -e "s@{{etcd_image}}@${ETCD_IMAGE}@g" "${src_file}" + else + local -r component_docker_tag=$(cat ${KUBE_BINDIR}/${component}.docker_tag) + sed -i -e "s@{{${component}_docker_tag}}@${component_docker_tag}@g" "${src_file}" + fi + cp "${src_file}" /etc/kubernetes/manifests +} + +############################### Main Function ######################################## +echo "Start to configure master instance for kubemark" + +# Extract files from the server tar and setup master env variables. +cd "${KUBE_ROOT}" tar xzf kubernetes-server-linux-amd64.tar.gz +source "${KUBE_ROOT}/kubemark-master-env.sh" -write_supervisor_conf "kube-scheduler" "${ulimit_command} /kubernetes/server/bin/kube-scheduler --master=127.0.0.1:8080 $(cat /scheduler_flags | tr '\n' ' ')\"" -write_supervisor_conf "kube-apiserver" "${ulimit_command} /kubernetes/server/bin/kube-apiserver --insecure-bind-address=0.0.0.0 \ - --etcd-servers=http://127.0.0.1:2379 \ - --etcd-servers-overrides=/events#${EVENT_STORE_URL} \ - --tls-cert-file=/srv/kubernetes/server.cert \ - --tls-private-key-file=/srv/kubernetes/server.key \ - --client-ca-file=/srv/kubernetes/ca.crt \ - --token-auth-file=/srv/kubernetes/known_tokens.csv \ - --secure-port=443 \ - --basic-auth-file=/srv/kubernetes/basic_auth.csv \ - --target-ram-mb=$((${NUM_NODES} * 60)) \ - $(cat /apiserver_flags | tr '\n' ' ')\"" -write_supervisor_conf "kube-controller-manager" "${ulimit_command} /kubernetes/server/bin/kube-controller-manager \ - --master=127.0.0.1:8080 \ - --service-account-private-key-file=/srv/kubernetes/server.key \ - --root-ca-file=/srv/kubernetes/ca.crt \ - $(cat /controllers_flags | tr '\n' ' ')\"" +# Setup required directory structure and etcd variables. +create-dirs +setup-kubelet-dir +delete-default-etcd-configs +compute-etcd-variables -supervisorctl reread -supervisorctl update +# Mount master PD for etcd and create symbolic links to it. +{ + main_etcd_mount_point="/mnt/disks/master-pd" + mount-pd "google-master-pd" "${main_etcd_mount_point}" + # Contains all the data stored in etcd. + mkdir -m 700 -p "${main_etcd_mount_point}/var/etcd" + ln -s -f "${main_etcd_mount_point}/var/etcd" /var/etcd + mkdir -p /etc/srv + # Contains the dynamically generated apiserver auth certs and keys. + mkdir -p "${main_etcd_mount_point}/srv/kubernetes" + ln -s -f "${main_etcd_mount_point}/srv/kubernetes" /etc/srv/kubernetes + # Directory for kube-apiserver to store SSH key (if necessary). + mkdir -p "${main_etcd_mount_point}/srv/sshproxy" + ln -s -f "${main_etcd_mount_point}/srv/sshproxy" /etc/srv/sshproxy +} +# Mount master PD for event-etcd (if required) and create symbolic links to it. +{ + EVENT_STORE_IP="${EVENT_STORE_IP:-127.0.0.1}" + EVENT_STORE_URL="${EVENT_STORE_URL:-http://${EVENT_STORE_IP}:4002}" + EVENT_PD="${EVENT_PD:-false}" + if [ "${EVENT_PD:-false}" == "true" ]; then + event_etcd_mount_point="/mnt/disks/master-event-pd" + mount-pd "google-master-event-pd" "${event_etcd_mount_point}" + # Contains all the data stored in event etcd. + mkdir -m 700 -p "${event_etcd_mount_point}/var/etcd/events" + ln -s -f "${event_etcd_mount_point}/var/etcd/events" /var/etcd/events + fi +} + +# Setup docker flags and load images of the master components. +assemble-docker-flags +DOCKER_REGISTRY="gcr.io/google_containers" +load-docker-images + +# Start kubelet as a supervisord process and master components as pods. +start-kubelet +start-kubemaster-component "etcd" +if [ "${EVENT_STORE_IP:-}" == "127.0.0.1" ]; then + start-kubemaster-component "etcd-events" +fi +start-kubemaster-component "kube-apiserver" +start-kubemaster-component "kube-controller-manager" +start-kubemaster-component "kube-scheduler" + +# Wait till apiserver is working fine. until [ "$(curl 127.0.0.1:8080/healthz 2> /dev/null)" == "ok" ]; do sleep 1 done + +echo "Done for the configuration for kubermark master" diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh index b4b87a818b..58aa0be574 100755 --- a/test/kubemark/start-kubemark.sh +++ b/test/kubemark/start-kubemark.sh @@ -22,36 +22,38 @@ KUBE_ROOT=$(readlink -e ${TMP_ROOT} 2> /dev/null || perl -MCwd -e 'print Cwd::ab source "${KUBE_ROOT}/test/kubemark/common.sh" -function writeEnvironmentFiles() { - cat > "${RESOURCE_DIRECTORY}/apiserver_flags" < "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" <> "${RESOURCE_DIRECTORY}/apiserver_flags" <> "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" <> "${RESOURCE_DIRECTORY}/apiserver_flags" <> "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" < "${RESOURCE_DIRECTORY}/scheduler_flags" < "${RESOURCE_DIRECTORY}/controllers_flags" < /srv/kubernetes/server.cert\" && \ sudo bash -c \"echo ${MASTER_KEY_BASE64} | base64 --decode > /srv/kubernetes/server.key\" && \ sudo bash -c \"echo ${CA_CERT_BASE64} | base64 --decode > /srv/kubernetes/ca.crt\" && \ @@ -159,21 +161,24 @@ gcloud compute ssh --zone="${ZONE}" --project="${PROJECT}" "${MASTER_NAME}" \ sudo bash -c \"echo \"${KUBE_PROXY_TOKEN},kube_proxy,kube_proxy\" >> /srv/kubernetes/known_tokens.csv\" && \ sudo bash -c \"echo ${password},admin,admin > /srv/kubernetes/basic_auth.csv\"" -writeEnvironmentFiles +writeEnvironmentFile gcloud compute copy-files --zone="${ZONE}" --project="${PROJECT}" \ "${SERVER_BINARY_TAR}" \ + "${RESOURCE_DIRECTORY}/kubemark-master-env.sh" \ "${RESOURCE_DIRECTORY}/start-kubemark-master.sh" \ "${KUBEMARK_DIRECTORY}/configure-kubectl.sh" \ - "${RESOURCE_DIRECTORY}/apiserver_flags" \ - "${RESOURCE_DIRECTORY}/scheduler_flags" \ - "${RESOURCE_DIRECTORY}/controllers_flags" \ - "root@${MASTER_NAME}":/ - + "${RESOURCE_DIRECTORY}/manifests/etcd.yaml" \ + "${RESOURCE_DIRECTORY}/manifests/etcd-events.yaml" \ + "${RESOURCE_DIRECTORY}/manifests/kube-apiserver.yaml" \ + "${RESOURCE_DIRECTORY}/manifests/kube-scheduler.yaml" \ + "${RESOURCE_DIRECTORY}/manifests/kube-controller-manager.yaml" \ + "root@${MASTER_NAME}":/home/kubernetes/ gcloud compute ssh "${MASTER_NAME}" --zone="${ZONE}" --project="${PROJECT}" \ - --command="sudo chmod a+x /configure-kubectl.sh && sudo chmod a+x /start-kubemark-master.sh && \ - sudo /start-kubemark-master.sh ${EVENT_STORE_IP:-127.0.0.1} ${NUM_NODES:-0} ${EVENT_PD:-false} ${ETCD_IMAGE:-}" + --command="sudo chmod a+x /home/kubernetes/configure-kubectl.sh && \ + sudo chmod a+x /home/kubernetes/start-kubemark-master.sh && \ + sudo /home/kubernetes/start-kubemark-master.sh" # create kubeconfig for Kubelet: KUBECONFIG_CONTENTS=$(echo "apiVersion: v1 @@ -248,18 +253,18 @@ contexts: current-context: kubemark-context EOF -sed "s/##numreplicas##/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json" -sed -i'' -e "s/##project##/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" +sed "s/{{numreplicas}}/${NUM_NODES:-10}/g" "${RESOURCE_DIRECTORY}/hollow-node_template.json" > "${RESOURCE_DIRECTORY}/hollow-node.json" +sed -i'' -e "s/{{project}}/${PROJECT}/g" "${RESOURCE_DIRECTORY}/hollow-node.json" mkdir "${RESOURCE_DIRECTORY}/addons" || true -sed "s/##MASTER_IP##/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json" +sed "s/{{MASTER_IP}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json" metrics_mem_per_node=4 metrics_mem=$((200 + ${metrics_mem_per_node}*${NUM_NODES:-10})) -sed -i'' -e "s/##METRICS_MEM##/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" +sed -i'' -e "s/{{METRICS_MEM}}/${metrics_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" eventer_mem_per_node=500 eventer_mem=$((200 * 1024 + ${eventer_mem_per_node}*${NUM_NODES:-10})) -sed -i'' -e "s/##EVENTER_MEM##/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" +sed -i'' -e "s/{{EVENTER_MEM}}/${eventer_mem}/g" "${RESOURCE_DIRECTORY}/addons/heapster.json" "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" "${KUBECTL}" create -f "${KUBECONFIG_SECRET}" --namespace="kubemark"