diff --git a/cluster/aws/config-default.sh b/cluster/aws/config-default.sh index f8e5430516..0ff0987226 100644 --- a/cluster/aws/config-default.sh +++ b/cluster/aws/config-default.sh @@ -77,3 +77,11 @@ ADMISSION_CONTROL=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,Security # Optional: Enable/disable public IP assignment for minions. # Important Note: disable only if you have setup a NAT instance for internet access and configured appropriate routes! ENABLE_MINION_PUBLIC_IP=${KUBE_ENABLE_MINION_PUBLIC_IP:-true} + +# OS options for minions +KUBE_OS_DISTRIBUTION="${KUBE_OS_DISTRIBUTION:-ubuntu}" +KUBE_MINION_IMAGE="${KUBE_MINION_IMAGE:-}" +COREOS_CHANNEL="${COREOS_CHANNEL:-alpha}" +CONTAINER_RUNTIME="${KUBE_CONTAINER_RUNTIME:-docker}" +RKT_VERSION="${KUBE_RKT_VERSION:-0.5.5}" + diff --git a/cluster/aws/config-test.sh b/cluster/aws/config-test.sh index ea55c15f5e..a779f70a83 100755 --- a/cluster/aws/config-test.sh +++ b/cluster/aws/config-test.sh @@ -73,3 +73,10 @@ ADMISSION_CONTROL=NamespaceLifecycle,NamespaceAutoProvision,LimitRanger,Security # Optional: Enable/disable public IP assignment for minions. # Important Note: disable only if you have setup a NAT instance for internet access and configured appropriate routes! ENABLE_MINION_PUBLIC_IP=${KUBE_ENABLE_MINION_PUBLIC_IP:-true} + +# OS options for minions +KUBE_OS_DISTRIBUTION="${KUBE_OS_DISTRIBUTION:-ubuntu}" +KUBE_MINION_IMAGE="${KUBE_MINION_IMAGE:-}" +COREOS_CHANNEL="${COREOS_CHANNEL:-alpha}" +CONTAINER_RUNTIME="${KUBE_CONTAINER_RUNTIME:-docker}" +RKT_VERSION="${KUBE_RKT_VERSION:-0.5.5}" diff --git a/cluster/aws/coreos/node.yaml b/cluster/aws/coreos/node.yaml new file mode 100644 index 0000000000..a5d5486605 --- /dev/null +++ b/cluster/aws/coreos/node.yaml @@ -0,0 +1,180 @@ +#cloud-config + +coreos: + units: + - name: kubernetes-install-rkt.service + command: start + content: | + [Unit] + Description=Fetch Rocket + Documentation=http://github.com/coreos/rkt + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/rkt + ExecStartPre=/usr/bin/wget \ + -O /opt/rkt/rkt-v${RKT_VERSION}.tar.gz \ + https://github.com/coreos/rkt/releases/download/v${RKT_VERSION}/rkt-v${RKT_VERSION}.tar.gz + ExecStart=/usr/bin/tar xzvf /opt/rkt/rkt-v${RKT_VERSION}.tar.gz -C /opt --overwrite + + - name: hostname-override.service + command: start + content: | + [Unit] + Description=Kubelet Hostname Override + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStart=/run/setup-hostname-override.sh + + - name: kubernetes-install-minion.service + command: start + content: | + [Unit] + Description=Install Kubernetes Server + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=yes + EnvironmentFile=/etc/kube-env + ExecStartPre=/usr/bin/mkdir -p /opt/kubernetes/pkg + ExecStartPre=/usr/bin/curl --location --create-dirs --output /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz ${SERVER_BINARY_TAR_URL} + ExecStart=/usr/bin/tar xf /opt/kubernetes/pkg/kubernetes-server-linux-amd64.tar.gz -C /opt --overwrite + + - name: kubelet.service + command: start + content: | + [Unit] + Description=Run Kubelet service + Requires=kubernetes-install-minion.service + After=kubernetes-install-minion.service + Requires=hostname-override.service + After=hostname-override.service + + [Service] + EnvironmentFile=/etc/kube-env + EnvironmentFile=/etc/hostname-override + ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests + ExecStartPre=/run/setup-auth.sh + ExecStart=/opt/kubernetes/server/bin/kubelet \ + --api_servers=https://${MASTER_IP} \ + --config=/etc/kubernetes/manifests \ + --allow_privileged=False \ + --v=2 \ + --cluster_dns=10.0.0.10 \ + --cluster_domain=kubernetes.local \ + --logtostderr=true \ + --hostname-override=${HOSTNAME_OVERRIDE} \ + --container-runtime=${KUBERNETES_CONTAINER_RUNTIME} + Restart=always + RestartSec=10 + + - name: kube-proxy.service + command: start + content: | + [Unit] + Description=Start Kube-proxy service as Daemon + Requires=kubernetes-install-minion.service + After=kubernetes-install-minion.service + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStartPre=/run/config-kube-proxy.sh + ExecStart=/opt/kubernetes/server/bin/kube-proxy \ + --master=https://${MASTER_IP} \ + --kubeconfig=/var/lib/kube-proxy/kubeconfig \ + --v=2 \ + --logtostderr=true + Restart=always + RestartSec=10 + + - name: rkt-metadata.socket + command: start + content: | + [Unit] + Description=rkt metadata service socket + PartOf=rkt-metadata.service + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + [Socket] + ListenStream=/run/rkt/metadata-svc.sock + SocketMode=0660 + SocketUser=root + SocketGroup=root + RemoveOnStop=true + + - name: rkt-metadata.service + command: start + content: | + [Unit] + Description=rkt metadata service + Documentation=http://github.com/coreos/rkt + Requires=rkt-metadata.socket + After=network.target rkt-metadata.socket + Requires=kubernetes-install-rkt.service + After=kubernetes-install-rkt.service + [Service] + EnvironmentFile=/etc/kube-env + ExecStart=/opt/rkt-${RKT_VERSION}/rkt metadata-service +write_files: + - path: /run/setup-hostname-override.sh + permissions: "0755" + content: | + #!/bin/bash + set -x + source /etc/kube-env + if [[ -z "${HOSTNAME_OVERRIDE}" ]]; then + HOSTNAME_OVERRIDE=`curl --silent http://169.254.169.254/2007-01-19/meta-data/local-hostname` + fi + if [[ -z "${HOSTNAME_OVERRIDE}" ]]; then + HOSTNAME_OVERRIDE=`hostname -f` + fi + echo "HOSTNAME_OVERRIDE=${HOSTNAME_OVERRIDE}" > /etc/hostname-override + - path: /run/setup-auth.sh + permissions: "0755" + content: | + #!/bin/bash -e + set -x + source /etc/kube-env + + /usr/bin/mkdir -p /var/lib/kubelet + printf '{"BearerToken": "%s", "Insecure": true }' ${KUBE_BEARER_TOKEN} > /var/lib/kubelet/kubernetes_auth + - path: /run/config-kube-proxy.sh + permissions: "0755" + content: | + #!/bin/bash -e + set -x + source /etc/kube-env + + /usr/bin/mkdir -p /var/lib/kube-proxy + cat > /var/lib/kube-proxy/kubeconfig << EOF + apiVersion: v1 + kind: Config + users: + - name: kube-proxy + user: + token: $KUBE_PROXY_TOKEN + clusters: + - name: local + cluster: + insecure-skip-tls-verify: true + contexts: + - context: + cluster: local + user: kube-proxy + name: service-account-context + current-context: service-account-context + EOF + - path: /etc/kube-env + permissions: 0644 + owner: root + content: | diff --git a/cluster/aws/coreos/util.sh b/cluster/aws/coreos/util.sh new file mode 100644 index 0000000000..f1f98227a7 --- /dev/null +++ b/cluster/aws/coreos/util.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions for CoreOS. + +function detect-minion-image (){ + if [[ -z "${KUBE_MINION_IMAGE-}" ]]; then + KUBE_MINION_IMAGE=$(curl -s -L http://${COREOS_CHANNEL}.release.core-os.net/amd64-usr/current/coreos_production_ami_all.json | python -c "import json,sys;obj=json.load(sys.stdin);print filter(lambda t: t['name']=='${AWS_REGION}', obj['amis'])[0]['hvm']") + fi + if [[ -z "${KUBE_MINION_IMAGE-}" ]]; then + echo "unable to determine KUBE_MINION_IMAGE" + exit 2 + fi +} + +function generate-minion-user-data() { + i=$1 + MINION_PRIVATE_IP=$INTERNAL_IP_BASE.1${i} + MINION_IP_RANGE=${MINION_IP_RANGES[$i]} + + # this is a bit of a hack. We make all of our "variables" in + # our cloud config controlled by env vars from this script + cat ${KUBE_ROOT}/cluster/aws/coreos/node.yaml + cat < /dev/null) -kube_proxy_token=$(cat /dev/urandom | base64 | tr -d "=+/" | dd bs=32 count=1 2> /dev/null) +kubelet_token=$KUBELET_TOKEN +kube_proxy_token=$KUBE_PROXY_TOKEN # Make a list of tokens and usernames to be pushed to the apiserver mkdir -p /srv/salt-overlay/salt/kube-apiserver diff --git a/cluster/aws/ubuntu/util.sh b/cluster/aws/ubuntu/util.sh new file mode 100644 index 0000000000..35b923274f --- /dev/null +++ b/cluster/aws/ubuntu/util.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Copyright 2015 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# A library of helper functions for Ubuntu. + +function detect-minion-image() { + if [[ -z "${KUBE_MINION_IMAGE=-}" ]]; then + detect-image + KUBE_MINION_IMAGE=$AWS_IMAGE + fi +} + +function generate-minion-user-data { + i=$1 + # We pipe this to the ami as a startup script in the user-data field. Requires a compatible ami + echo "#! /bin/bash" + echo "SALT_MASTER='${MASTER_INTERNAL_IP}'" + echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'" + echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS:-}'" + grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh" + grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh" + grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/salt-minion.sh" +} + +function check-minion() { + local minion_name=$1 + local minion_ip=$2 + + local output=$(ssh -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ubuntu@$minion_ip sudo docker ps -a 2>/dev/null) + if [[ -z "${output}" ]]; then + ssh -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ubuntu@$minion_ip sudo service docker start > $LOG 2>&1 + echo "not working yet" + else + echo "working" + fi +} diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 2629d3af27..7aa9cac60b 100644 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -22,6 +22,17 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/cluster/aws/${KUBE_CONFIG_FILE-"config-default.sh"}" source "${KUBE_ROOT}/cluster/common.sh" +case "${KUBE_OS_DISTRIBUTION}" in + ubuntu|coreos) + echo "Starting cluster using os distro: ${KUBE_OS_DISTRIBUTION}" >&2 + source "${KUBE_ROOT}/cluster/aws/${KUBE_OS_DISTRIBUTION}/util.sh" + ;; + *) + echo "Cannot start cluster using os distro: ${KUBE_OS_DISTRIBUTION}" >&2 + exit 2 + ;; +esac + # This removes the final character in bash (somehow) AWS_REGION=${ZONE%?} @@ -366,7 +377,13 @@ function wait-for-instance-running { } function kube-up { + get-tokens + + detect-image + detect-minion-image + find-release-tars + upload-server-tars ensure-temp-dir @@ -379,8 +396,6 @@ function kube-up { ssh-keygen -f "$AWS_SSH_KEY" -N '' fi - detect-image - $AWS_CMD import-key-pair --key-name kubernetes --public-key-material "file://$AWS_SSH_KEY.pub" > $LOG 2>&1 || true VPC_ID=$(get_vpc_id) @@ -439,7 +454,6 @@ function kube-up { SEC_GROUP_ID=$($AWS_CMD create-security-group --group-name kubernetes-sec-group --description kubernetes-sec-group --vpc-id $VPC_ID | json_val '["GroupId"]') $AWS_CMD authorize-security-group-ingress --group-id $SEC_GROUP_ID --protocol -1 --port all --cidr 0.0.0.0/0 > $LOG fi - ( # We pipe this to the ami as a startup script in the user-data field. Requires a compatible ami echo "#! /bin/bash" @@ -466,6 +480,8 @@ function kube-up { echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'" echo "readonly ADMISSION_CONTROL='${ADMISSION_CONTROL:-}'" echo "readonly MASTER_IP_RANGE='${MASTER_IP_RANGE:-}'" + echo "readonly KUBELET_TOKEN='${KUBELET_TOKEN}'" + echo "readonly KUBE_PROXY_TOKEN='${KUBE_PROXY_TOKEN}'" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/create-dynamic-salt-files.sh" @@ -546,16 +562,7 @@ function kube-up { MINION_IDS=() for (( i=0; i<${#MINION_NAMES[@]}; i++)); do echo "Starting Minion (${MINION_NAMES[$i]})" - ( - # We pipe this to the ami as a startup script in the user-data field. Requires a compatible ami - echo "#! /bin/bash" - echo "SALT_MASTER='${MASTER_INTERNAL_IP}'" - echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'" - echo "DOCKER_OPTS='${EXTRA_DOCKER_OPTS:-}'" - grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/common.sh" - grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/format-disks.sh" - grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/salt-minion.sh" - ) > "${KUBE_TEMP}/minion-start-${i}.sh" + generate-minion-user-data $i > "${KUBE_TEMP}/minion-user-data-${i}" local public_ip_option if [[ "${ENABLE_MINION_PUBLIC_IP}" == "true" ]]; then @@ -565,7 +572,7 @@ function kube-up { fi minion_id=$($AWS_CMD run-instances \ - --image-id $AWS_IMAGE \ + --image-id $KUBE_MINION_IMAGE \ --iam-instance-profile Name=$IAM_PROFILE_MINION \ --instance-type $MINION_SIZE \ --subnet-id $SUBNET_ID \ @@ -573,7 +580,7 @@ function kube-up { --key-name kubernetes \ --security-group-ids $SEC_GROUP_ID \ ${public_ip_option} \ - --user-data file://${KUBE_TEMP}/minion-start-${i}.sh | json_val '["Instances"][0]["InstanceId"]') + --user-data "file://${KUBE_TEMP}/minion-user-data-${i}" | json_val '["Instances"][0]["InstanceId"]') add-tag $minion_id Name ${MINION_NAMES[$i]} add-tag $minion_id Role $MINION_TAG @@ -669,31 +676,19 @@ function kube-up { local minion_name=${MINION_NAMES[$i]} local minion_ip=${KUBE_MINION_IP_ADDRESSES[$i]} echo -n Attempt "$(($attempt+1))" to check Docker on node "${minion_name} @ ${minion_ip}" ... - local output=$(ssh -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ubuntu@$minion_ip sudo docker ps -a 2>/dev/null) - if [[ -z "${output}" ]]; then + local output=`check-minion ${minion_name} ${minion_ip}` + echo $output + if [[ "${output}" != "working" ]]; then if (( attempt > 9 )); then echo - echo -e "${color_red}Docker failed to install on node ${minion_name}. Your cluster is unlikely" >&2 - echo "to work correctly. Please run ./cluster/kube-down.sh and re-create the" >&2 - echo -e "cluster. (sorry!)${color_norm}" >&2 + echo -e "Your cluster is unlikely to work correctly." >&2 + echo "Please run ./cluster/kube-down.sh and re-create the" >&2 + echo -e "cluster. (sorry!)" >&2 exit 1 fi - # TODO: Reintroduce this (where does this container come from?) -# elif [[ "${output}" != *"kubernetes/pause"* ]]; then -# if (( attempt > 9 )); then -# echo -# echo -e "${color_red}Failed to observe kubernetes/pause on node ${minion_name}. Your cluster is unlikely" >&2 -# echo "to work correctly. Please run ./cluster/kube-down.sh and re-create the" >&2 -# echo -e "cluster. (sorry!)${color_norm}" >&2 -# exit 1 -# fi else - echo -e " ${color_green}[working]${color_norm}" break fi - echo -e " ${color_yellow}[not working yet]${color_norm}" - # Start Docker, in case it failed to start. - ssh -oStrictHostKeyChecking=no -i "${AWS_SSH_KEY}" ubuntu@$minion_ip sudo service docker start > $LOG 2>&1 attempt=$(($attempt+1)) sleep 30 done @@ -893,3 +888,8 @@ function prepare-e2e() { # Note: we can't print anything here, or else the test tools will break with the extra output return } + +function get-tokens() { + KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) +}