mirror of https://github.com/k3s-io/k3s
Merge pull request #5119 from zmerlynn/salt_drones
Change GCE to use standalone Saltstack configpull/6/head
Brendan Burns
commit
00f3779c6f
|
|
@ -786,8 +786,8 @@ function kube::release::gcs::copy_release_artifacts() {
|
|||
# Having the "template" scripts from the GCE cluster deploy hosted with the
|
||||
# release is useful for GKE. Copy everything from that directory up also.
|
||||
gsutil -m "${gcs_options[@]+${gcs_options[@]}}" cp \
|
||||
"${RELEASE_STAGE}/full/kubernetes/cluster/gce/templates/*.sh" \
|
||||
"${gcs_destination}extra/gce-templates/"
|
||||
"${RELEASE_STAGE}/full/kubernetes/cluster/gce/configure-vm.sh" \
|
||||
"${gcs_destination}extra/gce/"
|
||||
|
||||
# Upload the "naked" binaries to GCS. This is useful for install scripts that
|
||||
# download the binaries directly and don't need tars.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,359 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2015 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# If we have any arguments at all, this is a push and not just setup.
|
||||
is_push=$@
|
||||
|
||||
function ensure-install-dir() {
|
||||
INSTALL_DIR="/var/cache/kubernetes-install"
|
||||
mkdir -p ${INSTALL_DIR}
|
||||
cd ${INSTALL_DIR}
|
||||
}
|
||||
|
||||
function set-broken-motd() {
|
||||
echo -e '\nBroken (or in progress) GCE Kubernetes node setup! Suggested first step:\n tail /var/log/startupscript.log\n' > /etc/motd
|
||||
}
|
||||
|
||||
function set-good-motd() {
|
||||
echo -e '\n=== GCE Kubernetes node setup complete ===\n' > /etc/motd
|
||||
}
|
||||
|
||||
function curl-metadata() {
|
||||
curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/attributes/${1}"
|
||||
}
|
||||
|
||||
function set-kube-env() {
|
||||
local kube_env_yaml="${INSTALL_DIR}/kube_env.yaml"
|
||||
|
||||
until curl-metadata kube-env > "${kube_env_yaml}"; do
|
||||
echo 'Waiting for kube-env...'
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# kube-env has all the environment variables we care about, in a flat yaml format
|
||||
eval $(python -c '''
|
||||
import pipes,sys,yaml
|
||||
|
||||
for k,v in yaml.load(sys.stdin).iteritems():
|
||||
print "readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))
|
||||
''' < "${kube_env_yaml}")
|
||||
|
||||
# We bake the KUBELET_TOKEN in separately to avoid auth information
|
||||
# having to be re-communicated on kube-push. (Otherwise the client
|
||||
# has to keep the bearer token around to handle generating a valid
|
||||
# kube-env.)
|
||||
if [[ -z "${KUBELET_TOKEN:-}" ]]; then
|
||||
until KUBELET_TOKEN=$(curl-metadata kube-token); do
|
||||
echo 'Waiting for metadata KUBELET_TOKEN...'
|
||||
sleep 3
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
|
||||
# TODO(zmerlynn): This block of code should disappear once #4561 & #4562 are done
|
||||
if [[ -z "${KUBERNETES_NODE_NAMES:-}" ]]; then
|
||||
until KUBERNETES_NODE_NAMES=$(curl-metadata kube-node-names); do
|
||||
echo 'Waiting for metadata KUBERNETES_NODE_NAMES...'
|
||||
sleep 3
|
||||
done
|
||||
fi
|
||||
else
|
||||
# And this should go away once the master can allocate CIDRs
|
||||
if [[ -z "${MINION_IP_RANGE:-}" ]]; then
|
||||
until MINION_IP_RANGE=$(curl-metadata node-ip-range); do
|
||||
echo 'Waiting for metadata MINION_IP_RANGE...'
|
||||
sleep 3
|
||||
done
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function remove-docker-artifacts() {
|
||||
# Remove docker artifacts on minion nodes, if present
|
||||
iptables -t nat -F || true
|
||||
ifconfig docker0 down || true
|
||||
brctl delbr docker0 || true
|
||||
}
|
||||
|
||||
# Retry a download until we get it.
|
||||
#
|
||||
# $1 is the URL to download
|
||||
download-or-bust() {
|
||||
local -r url="$1"
|
||||
local -r file="${url##*/}"
|
||||
rm -f "$file"
|
||||
until [[ -e "${1##*/}" ]]; do
|
||||
echo "Downloading file ($1)"
|
||||
curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$1"
|
||||
done
|
||||
}
|
||||
|
||||
# Install salt from GCS. See README.md for instructions on how to update these
|
||||
# debs.
|
||||
install-salt() {
|
||||
apt-get update
|
||||
|
||||
mkdir -p /var/cache/salt-install
|
||||
cd /var/cache/salt-install
|
||||
|
||||
TARS=(
|
||||
libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
|
||||
python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
|
||||
salt-common_2014.1.13+ds-1~bpo70+1_all.deb
|
||||
salt-minion_2014.1.13+ds-1~bpo70+1_all.deb
|
||||
)
|
||||
URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
|
||||
|
||||
for tar in "${TARS[@]}"; do
|
||||
download-or-bust "${URL_BASE}/${tar}"
|
||||
dpkg -i "${tar}" || true
|
||||
done
|
||||
|
||||
# This will install any of the unmet dependencies from above.
|
||||
apt-get install -f -y
|
||||
}
|
||||
|
||||
# Ensure salt-minion *isn't* running
|
||||
stop-salt-minion() {
|
||||
# This ensures it on next reboot
|
||||
echo manual > /etc/init/salt-minion.override
|
||||
|
||||
service salt-minion stop
|
||||
while service salt-minion status >/dev/null; do
|
||||
service salt-minion stop # No, really.
|
||||
echo "Waiting for salt-minion to shut down"
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# Mounts a persistent disk (formatting if needed) to store the persistent data
|
||||
# on the master -- etcd's data, a few settings, and security certs/keys/tokens.
|
||||
#
|
||||
# This function can be reused to mount an existing PD because all of its
|
||||
# operations modifying the disk are idempotent -- safe_format_and_mount only
|
||||
# formats an unformatted disk, and mkdir -p will leave a directory be if it
|
||||
# already exists.
|
||||
mount-master-pd() {
|
||||
device_info=$(ls -l /dev/disk/by-id/google-master-pd)
|
||||
relative_path=${device_info##* }
|
||||
device_path="/dev/disk/by-id/${relative_path}"
|
||||
|
||||
# Format and mount the disk, create directories on it for all of the master's
|
||||
# persistent data, and link them to where they're used.
|
||||
mkdir -p /mnt/master-pd
|
||||
/usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd
|
||||
# Contains all the data stored in etcd
|
||||
mkdir -m 700 -p /mnt/master-pd/var/etcd
|
||||
# Contains the dynamically generated apiserver auth certs and keys
|
||||
mkdir -p /mnt/master-pd/srv/kubernetes
|
||||
# Contains the cluster's initial config parameters and auth tokens
|
||||
mkdir -p /mnt/master-pd/srv/salt-overlay
|
||||
ln -s /mnt/master-pd/var/etcd /var/etcd
|
||||
ln -s /mnt/master-pd/srv/kubernetes /srv/kubernetes
|
||||
ln -s /mnt/master-pd/srv/salt-overlay /srv/salt-overlay
|
||||
|
||||
# This is a bit of a hack to get around the fact that salt has to run after the
|
||||
# PD and mounted directory are already set up. We can't give ownership of the
|
||||
# directory to etcd until the etcd user and group exist, but they don't exist
|
||||
# until salt runs if we don't create them here. We could alternatively make the
|
||||
# permissions on the directory more permissive, but this seems less bad.
|
||||
useradd -s /sbin/nologin -d /var/etcd etcd
|
||||
chown etcd /mnt/master-pd/var/etcd
|
||||
chgrp etcd /mnt/master-pd/var/etcd
|
||||
}
|
||||
|
||||
# Create the overlay files for the salt tree. We create these in a separate
|
||||
# place so that we can blow away the rest of the salt configs on a kube-push and
|
||||
# re-apply these.
|
||||
function create-salt-pillar() {
|
||||
# Always overwrite the cluster-params.sls (even on a push, we have
|
||||
# these variables)
|
||||
mkdir -p /srv/salt-overlay/pillar
|
||||
cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
|
||||
instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")'
|
||||
node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")'
|
||||
portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")'
|
||||
enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")'
|
||||
enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")'
|
||||
enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")'
|
||||
enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")'
|
||||
logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")'
|
||||
elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")'
|
||||
enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")'
|
||||
dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")'
|
||||
dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")'
|
||||
dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
|
||||
if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
|
||||
cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
|
||||
gce_node_names: '$(echo "$KUBERNETES_NODE_NAMES" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# This should only happen on cluster initialization
|
||||
function create-salt-auth() {
|
||||
mkdir -p /srv/salt-overlay/salt/nginx
|
||||
echo "${MASTER_HTPASSWD}" > /srv/salt-overlay/salt/nginx/htpasswd
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/kube-apiserver
|
||||
known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
|
||||
(umask 077;
|
||||
echo "${KUBELET_TOKEN},kubelet,kubelet" > "${known_tokens_file}")
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/kubelet
|
||||
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
|
||||
(umask 077;
|
||||
echo "{\"BearerToken\": \"${KUBELET_TOKEN}\", \"Insecure\": true }" > "${kubelet_auth_file}")
|
||||
}
|
||||
|
||||
function download-release() {
|
||||
echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
|
||||
download-or-bust "$SERVER_BINARY_TAR_URL"
|
||||
|
||||
echo "Downloading binary release tar ($SALT_TAR_URL)"
|
||||
download-or-bust "$SALT_TAR_URL"
|
||||
|
||||
echo "Unpacking Salt tree"
|
||||
rm -rf kubernetes
|
||||
tar xzf "${SALT_TAR_URL##*/}"
|
||||
|
||||
echo "Running release install script"
|
||||
sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"
|
||||
}
|
||||
|
||||
function fix-apt-sources() {
|
||||
sed -i -e "\|^deb.*http://http.debian.net/debian| s/^/#/" /etc/apt/sources.list
|
||||
sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d/backports.list
|
||||
}
|
||||
|
||||
function salt-run-local() {
|
||||
cat <<EOF >/etc/salt/minion.d/local.conf
|
||||
file_client: local
|
||||
file_roots:
|
||||
base:
|
||||
- /srv/salt
|
||||
EOF
|
||||
}
|
||||
|
||||
function salt-debug-log() {
|
||||
cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
|
||||
log_level: debug
|
||||
log_level_logfile: debug
|
||||
EOF
|
||||
}
|
||||
|
||||
function salt-master-role() {
|
||||
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||
grains:
|
||||
roles:
|
||||
- kubernetes-master
|
||||
cloud: gce
|
||||
EOF
|
||||
}
|
||||
|
||||
function salt-node-role() {
|
||||
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||
grains:
|
||||
roles:
|
||||
- kubernetes-pool
|
||||
cbr-cidr: '$(echo "$MINION_IP_RANGE" | sed -e "s/'/''/g")'
|
||||
cloud: gce
|
||||
EOF
|
||||
}
|
||||
|
||||
function salt-docker-opts() {
|
||||
DOCKER_OPTS=""
|
||||
|
||||
if [[ -n "${EXTRA_DOCKER_OPTS-}" ]]; then
|
||||
DOCKER_OPTS="${EXTRA_DOCKER_OPTS}"
|
||||
fi
|
||||
|
||||
# Decide whether to enable the cache
|
||||
if [[ "${ENABLE_DOCKER_REGISTRY_CACHE}" == "true" ]]; then
|
||||
REGION=$(echo "${ZONE}" | cut -f 1,2 -d -)
|
||||
echo "Enable docker registry cache at region: " $REGION
|
||||
DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror='https://${REGION}.docker-cache.clustermaster.net'"
|
||||
fi
|
||||
|
||||
if [[ -n "{DOCKER_OPTS}" ]]; then
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
function salt-set-apiserver() {
|
||||
local kube_master_ip
|
||||
until kube_master_ip=$(getent hosts ${KUBERNETES_MASTER_NAME} | cut -f1 -d\ ); do
|
||||
echo 'Waiting for DNS resolution of ${KUBERNETES_MASTER_NAME}...'
|
||||
sleep 3
|
||||
done
|
||||
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
api_servers: '${kube_master_ip}'
|
||||
apiservers: '${kube_master_ip}'
|
||||
EOF
|
||||
}
|
||||
|
||||
function configure-salt() {
|
||||
fix-apt-sources
|
||||
mkdir -p /etc/salt/minion.d
|
||||
salt-run-local
|
||||
if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
|
||||
salt-master-role
|
||||
else
|
||||
salt-node-role
|
||||
salt-docker-opts
|
||||
salt-set-apiserver
|
||||
fi
|
||||
install-salt
|
||||
stop-salt-minion
|
||||
}
|
||||
|
||||
function run-salt() {
|
||||
salt-call --local state.highstate || true
|
||||
}
|
||||
|
||||
####################################################################################
|
||||
|
||||
if [[ -z "${is_push}" ]]; then
|
||||
echo "== kube-up node config starting =="
|
||||
set-broken-motd
|
||||
ensure-install-dir
|
||||
set-kube-env
|
||||
[[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd
|
||||
create-salt-pillar
|
||||
create-salt-auth
|
||||
download-release
|
||||
configure-salt
|
||||
remove-docker-artifacts
|
||||
run-salt
|
||||
set-good-motd
|
||||
echo "== kube-up node config done =="
|
||||
else
|
||||
echo "== kube-push node config starting =="
|
||||
ensure-install-dir
|
||||
set-kube-env
|
||||
create-salt-pillar
|
||||
run-salt
|
||||
echo "== kube-push node config done =="
|
||||
fi
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# Copyright 2015 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
def mutate_env(path, var, value):
|
||||
# Load the existing arguments
|
||||
if os.path.exists(path):
|
||||
args = yaml.load(open(path))
|
||||
else:
|
||||
args = {}
|
||||
args[var] = value
|
||||
yaml.dump(args, stream=open(path, 'w'), default_flow_style=False)
|
||||
|
||||
if __name__ == "__main__":
|
||||
mutate_env(sys.argv[1], sys.argv[2], sys.argv[3])
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
# Updating Salt debs
|
||||
|
||||
We are caching all of the salt debs in GCS for speed and reliability.
|
||||
|
||||
To update them, follow this simple N step process:
|
||||
|
||||
1. Start up a new base image without salt installed. SSH into this image.
|
||||
2. Install salt via their recommended method: `curl -L https://bootstrap.saltstack.com | sudo sh -s -- -M -X`
|
||||
3. Find and download the debs that originated at the saltstack.com repo: `aptitude search --disable-columns -F "%p %V" "?installed?origin(saltstack.com)" | xargs aptitude download`
|
||||
4. Upload these to GCS: `gsutil cp *.deb gs://kubernetes-release/salt/`
|
||||
5. Make sure that everything is publicly readable: `gsutil acl ch -R -g all:R gs://kubernetes-release/salt/`
|
||||
6. Test things well :)
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Retry a download until we get it.
|
||||
#
|
||||
# $1 is the URL to download
|
||||
download-or-bust() {
|
||||
local -r url="$1"
|
||||
local -r file="${url##*/}"
|
||||
rm -f "$file"
|
||||
until [[ -e "${1##*/}" ]]; do
|
||||
echo "Downloading file ($1)"
|
||||
curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$1"
|
||||
md5sum "$file"
|
||||
done
|
||||
}
|
||||
|
||||
# Install salt from GCS. See README.md for instructions on how to update these
|
||||
# debs.
|
||||
#
|
||||
# $1 If set to --master, also install the master
|
||||
install-salt() {
|
||||
apt-get update
|
||||
|
||||
mkdir -p /var/cache/salt-install
|
||||
cd /var/cache/salt-install
|
||||
|
||||
TARS=(
|
||||
libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
|
||||
python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
|
||||
salt-common_2014.1.13+ds-1~bpo70+1_all.deb
|
||||
salt-minion_2014.1.13+ds-1~bpo70+1_all.deb
|
||||
)
|
||||
if [[ ${1-} == '--master' ]]; then
|
||||
TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb)
|
||||
fi
|
||||
URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
|
||||
|
||||
for tar in "${TARS[@]}"; do
|
||||
download-or-bust "${URL_BASE}/${tar}"
|
||||
dpkg -i "${tar}"
|
||||
done
|
||||
|
||||
# This will install any of the unmet dependencies from above.
|
||||
apt-get install -f -y
|
||||
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Create the overlay files for the salt tree. We create these in a separate
|
||||
# place so that we can blow away the rest of the salt configs on a kube-push and
|
||||
# re-apply these.
|
||||
|
||||
mkdir -p /srv/salt-overlay/pillar
|
||||
cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
|
||||
instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")'
|
||||
node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")'
|
||||
portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")'
|
||||
enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")'
|
||||
enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")'
|
||||
enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")'
|
||||
enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")'
|
||||
logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")'
|
||||
elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")'
|
||||
enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")'
|
||||
dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")'
|
||||
dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")'
|
||||
dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/nginx
|
||||
echo $MASTER_HTPASSWD > /srv/salt-overlay/salt/nginx/htpasswd
|
||||
|
||||
# Generate and distribute a shared secret (bearer token) to
|
||||
# apiserver and kubelet so that kubelet can authenticate to
|
||||
# apiserver to send events.
|
||||
# This works on CoreOS, so it should work on a lot of distros.
|
||||
kubelet_token=$(cat /dev/urandom | base64 | tr -d "=+/" | dd bs=32 count=1 2> /dev/null)
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/kube-apiserver
|
||||
known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
|
||||
(umask u=rw,go= ; echo "$kubelet_token,kubelet,kubelet" > $known_tokens_file)
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/kubelet
|
||||
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
|
||||
(umask u=rw,go= ; echo "{\"BearerToken\": \"$kubelet_token\", \"Insecure\": true }" > $kubelet_auth_file)
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Download and install release
|
||||
|
||||
# This script assumes that the environment variable MASTER_RELEASE_TAR contains
|
||||
# the release tar to download and unpack. It is meant to be pushed to the
|
||||
# master and run.
|
||||
|
||||
|
||||
echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
|
||||
download-or-bust "$SERVER_BINARY_TAR_URL"
|
||||
|
||||
echo "Downloading binary release tar ($SALT_TAR_URL)"
|
||||
download-or-bust "$SALT_TAR_URL"
|
||||
|
||||
echo "Unpacking Salt tree"
|
||||
rm -rf kubernetes
|
||||
tar xzf "${SALT_TAR_URL##*/}"
|
||||
|
||||
echo "Running release install script"
|
||||
sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Mounts a persistent disk (formatting if needed) to store the persistent data
|
||||
# on the master -- etcd's data, a few settings, and security certs/keys/tokens.
|
||||
#
|
||||
# This script can be reused to mount an existing PD because all of its
|
||||
# operations modifying the disk are idempotent -- safe_format_and_mount only
|
||||
# formats an unformatted disk, and mkdir -p will leave a directory be if it
|
||||
# already exists.
|
||||
|
||||
device_info=$(ls -l /dev/disk/by-id/google-master-pd)
|
||||
relative_path=${device_info##* }
|
||||
device_path="/dev/disk/by-id/${relative_path}"
|
||||
|
||||
# Format and mount the disk, create directories on it for all of the master's
|
||||
# persistent data, and link them to where they're used.
|
||||
mkdir -p /mnt/master-pd
|
||||
/usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd
|
||||
# Contains all the data stored in etcd
|
||||
mkdir -m 700 -p /mnt/master-pd/var/etcd
|
||||
# Contains the dynamically generated apiserver auth certs and keys
|
||||
mkdir -p /mnt/master-pd/srv/kubernetes
|
||||
# Contains the cluster's initial config parameters and auth tokens
|
||||
mkdir -p /mnt/master-pd/srv/salt-overlay
|
||||
# Contains salt's dynamically generated RSA keys
|
||||
mkdir -m 770 -p /mnt/master-pd/etc/salt/pki
|
||||
ln -s /mnt/master-pd/var/etcd /var/etcd
|
||||
ln -s /mnt/master-pd/srv/kubernetes /srv/kubernetes
|
||||
ln -s /mnt/master-pd/srv/salt-overlay /srv/salt-overlay
|
||||
ln -s /mnt/master-pd/etc/salt/pki /etc/salt/pki
|
||||
|
||||
# This is a bit of a hack to get around the fact that salt has to run after the
|
||||
# PD and mounted directory are already set up. We can't give ownership of the
|
||||
# directory to etcd until the etcd user and group exist, but they don't exist
|
||||
# until salt runs if we don't create them here. We could alternatively make the
|
||||
# permissions on the directory more permissive, but this seems less bad.
|
||||
useradd -s /sbin/nologin -d /var/etcd etcd
|
||||
chown etcd /mnt/master-pd/var/etcd
|
||||
chgrp etcd /mnt/master-pd/var/etcd
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
sed -i -e "\|^deb.*http://http.debian.net/debian| s/^/#/" /etc/apt/sources.list
|
||||
sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d/backports.list
|
||||
|
||||
# Prepopulate the name of the Master
|
||||
mkdir -p /etc/salt/minion.d
|
||||
cat <<EOF >/etc/salt/minion.d/master.conf
|
||||
master: '$(echo "$MASTER_NAME" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
|
||||
log_level: debug
|
||||
log_level_logfile: debug
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||
grains:
|
||||
roles:
|
||||
- kubernetes-master
|
||||
cloud: gce
|
||||
EOF
|
||||
|
||||
# Auto accept all keys from minions that try to join
|
||||
mkdir -p /etc/salt/master.d
|
||||
cat <<EOF >/etc/salt/master.d/auto-accept.conf
|
||||
auto_accept: True
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/master.d/reactor.conf
|
||||
# React to new minions starting by running highstate on them.
|
||||
reactor:
|
||||
- 'salt/minion/*/start':
|
||||
- /srv/reactor/highstate-new.sls
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/master.d/log-level-debug.d
|
||||
log_level: debug
|
||||
log_level_logfile: debug
|
||||
EOF
|
||||
|
||||
install-salt --master
|
||||
|
||||
# Wait a few minutes and trigger another Salt run to better recover from
|
||||
# any transient errors.
|
||||
echo "Sleeping 180"
|
||||
sleep 180
|
||||
salt-call state.highstate || true
|
||||
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# The repositories are really slow and there are GCE mirrors
|
||||
sed -i -e "\|^deb.*http://http.debian.net/debian| s/^/#/" /etc/apt/sources.list
|
||||
sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d/backports.list
|
||||
|
||||
# Prepopulate the name of the Master
|
||||
mkdir -p /etc/salt/minion.d
|
||||
cat <<EOF >/etc/salt/minion.d/master.conf
|
||||
master: '$(echo "$MASTER_NAME" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
|
||||
log_level: debug
|
||||
log_level_logfile: debug
|
||||
EOF
|
||||
|
||||
# Our minions will have a pool role to distinguish them from the master.
|
||||
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||
grains:
|
||||
roles:
|
||||
- kubernetes-pool
|
||||
cbr-cidr: '$(echo "$MINION_IP_RANGE" | sed -e "s/'/''/g")'
|
||||
cloud: gce
|
||||
EOF
|
||||
|
||||
DOCKER_OPTS=""
|
||||
|
||||
if [[ -n "${EXTRA_DOCKER_OPTS-}" ]]; then
|
||||
DOCKER_OPTS="${EXTRA_DOCKER_OPTS}"
|
||||
fi
|
||||
|
||||
# Decide if enable the cache
|
||||
if [[ "${ENABLE_DOCKER_REGISTRY_CACHE}" == "true" ]]; then
|
||||
REGION=$(echo "${ZONE}" | cut -f 1,2 -d -)
|
||||
echo "Enable docker registry cache at region: " $REGION
|
||||
DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror='https://${REGION}.docker-cache.clustermaster.net'"
|
||||
fi
|
||||
|
||||
if [[ -n "{DOCKER_OPTS}" ]]; then
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
|
||||
install-salt
|
||||
|
||||
# Wait a few minutes and trigger another Salt run to better recover from
|
||||
# any transient errors.
|
||||
echo "Sleeping 180"
|
||||
sleep 180
|
||||
salt-call state.highstate || true
|
||||
|
|
@ -208,6 +208,7 @@ function detect-minions () {
|
|||
# Vars set:
|
||||
# KUBE_MASTER
|
||||
# KUBE_MASTER_IP
|
||||
# KUBE_MASTER_IP_INTERNAL
|
||||
function detect-master () {
|
||||
detect-project
|
||||
KUBE_MASTER=${MASTER_NAME}
|
||||
|
|
@ -246,6 +247,14 @@ function get-password {
|
|||
KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))')
|
||||
}
|
||||
|
||||
# Set MASTER_HTPASSWD
|
||||
function set-master-htpasswd {
|
||||
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
|
||||
-b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
|
||||
local htpasswd
|
||||
MASTER_HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd")
|
||||
}
|
||||
|
||||
# Generate authentication token for admin user. Will
|
||||
# read from $HOME/.kubernetes_auth if available.
|
||||
#
|
||||
|
|
@ -332,6 +341,8 @@ function create-route {
|
|||
# $1: The name of the instance template.
|
||||
# $2: The scopes flag.
|
||||
# $3: The minion start script metadata from file.
|
||||
# $4: The kube-env metadata.
|
||||
# $5: Raw metadata
|
||||
function create-node-template {
|
||||
detect-project
|
||||
local attempt=0
|
||||
|
|
@ -347,7 +358,8 @@ function create-node-template {
|
|||
--network "${NETWORK}" \
|
||||
$2 \
|
||||
--can-ip-forward \
|
||||
--metadata-from-file "$3"; then
|
||||
--metadata-from-file "$3" "$4" \
|
||||
--metadata "$5"; then
|
||||
if (( attempt > 5 )); then
|
||||
echo -e "${color_red}Failed to create instance template $1 ${color_norm}"
|
||||
exit 2
|
||||
|
|
@ -383,26 +395,94 @@ function add-instance-metadata {
|
|||
done
|
||||
}
|
||||
|
||||
# Robustly try to add metadata on an instance, from a file.
|
||||
# $1: The name of the instace.
|
||||
# $2: The metadata key=file pair to add.
|
||||
function add-instance-metadata-from-file {
|
||||
detect-project
|
||||
local attempt=0
|
||||
while true; do
|
||||
if ! gcloud compute instances add-metadata "$1" \
|
||||
--project "${PROJECT}" \
|
||||
--zone "${ZONE}" \
|
||||
--metadata-from-file "$2"; then
|
||||
if (( attempt > 5 )); then
|
||||
echo -e "${color_red}Failed to add instance metadata in $1 ${color_norm}"
|
||||
exit 2
|
||||
fi
|
||||
echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in $1. Retrying.${color_norm}"
|
||||
attempt=$(($attempt+1))
|
||||
else
|
||||
break
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Given a yaml file, add or mutate the given env variable
|
||||
#
|
||||
# TODO(zmerlynn): Yes, this is an O(n^2) build-up right now. If we end
|
||||
# up with so many environment variables feeding into Salt that this
|
||||
# matters, there's probably an issue...
|
||||
function add-to-env {
|
||||
${KUBE_ROOT}/cluster/gce/kube-env.py "$1" "$2" "$3"
|
||||
}
|
||||
|
||||
# $1: if 'true', we're building a master yaml, else a node
|
||||
function build-kube-env {
|
||||
local master=$1
|
||||
local file=$2
|
||||
|
||||
rm -f ${file}
|
||||
add-to-env ${file} ENV_TIMESTAMP "$(date -uIs)" # Just to track it
|
||||
add-to-env ${file} KUBERNETES_MASTER "${master}"
|
||||
add-to-env ${file} INSTANCE_PREFIX "${INSTANCE_PREFIX}"
|
||||
add-to-env ${file} NODE_INSTANCE_PREFIX "${NODE_INSTANCE_PREFIX}"
|
||||
add-to-env ${file} SERVER_BINARY_TAR_URL "${SERVER_BINARY_TAR_URL}"
|
||||
add-to-env ${file} SALT_TAR_URL "${SALT_TAR_URL}"
|
||||
add-to-env ${file} PORTAL_NET "${PORTAL_NET}"
|
||||
add-to-env ${file} ENABLE_CLUSTER_MONITORING "${ENABLE_CLUSTER_MONITORING:-false}"
|
||||
add-to-env ${file} ENABLE_NODE_MONITORING "${ENABLE_NODE_MONITORING:-false}"
|
||||
add-to-env ${file} ENABLE_CLUSTER_LOGGING "${ENABLE_CLUSTER_LOGGING:-false}"
|
||||
add-to-env ${file} ENABLE_NODE_LOGGING "${ENABLE_NODE_LOGGING:-false}"
|
||||
add-to-env ${file} LOGGING_DESTINATION "${LOGGING_DESTINATION:-}"
|
||||
add-to-env ${file} ELASTICSEARCH_LOGGING_REPLICAS "${ELASTICSEARCH_LOGGING_REPLICAS:-}"
|
||||
add-to-env ${file} ENABLE_CLUSTER_DNS "${ENABLE_CLUSTER_DNS:-false}"
|
||||
add-to-env ${file} DNS_REPLICAS "${DNS_REPLICAS:-}"
|
||||
add-to-env ${file} DNS_SERVER_IP "${DNS_SERVER_IP:-}"
|
||||
add-to-env ${file} DNS_DOMAIN "${DNS_DOMAIN:-}"
|
||||
add-to-env ${file} MASTER_HTPASSWD "${MASTER_HTPASSWD}"
|
||||
if [[ "${master}" != "true" ]]; then
|
||||
add-to-env ${file} KUBERNETES_MASTER_NAME "${MASTER_NAME}"
|
||||
add-to-env ${file} ZONE "${ZONE}"
|
||||
add-to-env ${file} EXTRA_DOCKER_OPTS "${EXTRA_DOCKER_OPTS}"
|
||||
add-to-env ${file} ENABLE_DOCKER_REGISTRY_CACHE "${ENABLE_DOCKER_REGISTRY_CACHE:-false}"
|
||||
fi
|
||||
}
|
||||
|
||||
function write-master-env {
|
||||
build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml"
|
||||
}
|
||||
|
||||
function write-node-env {
|
||||
build-kube-env false "${KUBE_TEMP}/node-kube-env.yaml"
|
||||
}
|
||||
|
||||
# Instantiate a kubernetes cluster
|
||||
#
|
||||
# Assumed vars
|
||||
# KUBE_ROOT
|
||||
# <Various vars set in config file>
|
||||
function kube-up {
|
||||
ensure-temp-dir
|
||||
detect-project
|
||||
|
||||
get-password
|
||||
set-master-htpasswd
|
||||
|
||||
# Make sure we have the tar files staged on Google Storage
|
||||
find-release-tars
|
||||
upload-server-tars
|
||||
|
||||
ensure-temp-dir
|
||||
|
||||
get-password
|
||||
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
|
||||
-b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
|
||||
local htpasswd
|
||||
htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
|
||||
|
||||
if ! gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" &>/dev/null; then
|
||||
echo "Creating new network: ${NETWORK}"
|
||||
# The network needs to be created synchronously or we have a race. The
|
||||
|
|
@ -426,7 +506,7 @@ function kube-up {
|
|||
--allow "tcp:22" &
|
||||
fi
|
||||
|
||||
echo "Starting VMs and configuring firewalls"
|
||||
echo "Starting master and configuring firewalls"
|
||||
gcloud compute firewall-rules create "${MASTER_NAME}-https" \
|
||||
--project "${PROJECT}" \
|
||||
--network "${NETWORK}" \
|
||||
|
|
@ -440,34 +520,13 @@ function kube-up {
|
|||
--zone "${ZONE}" \
|
||||
--size "10GB"
|
||||
|
||||
(
|
||||
echo "#! /bin/bash"
|
||||
echo "mkdir -p /var/cache/kubernetes-install"
|
||||
echo "cd /var/cache/kubernetes-install"
|
||||
echo "readonly MASTER_NAME='${MASTER_NAME}'"
|
||||
echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
|
||||
echo "readonly NODE_INSTANCE_PREFIX='${NODE_INSTANCE_PREFIX}'"
|
||||
echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
|
||||
echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
|
||||
echo "readonly MASTER_HTPASSWD='${htpasswd}'"
|
||||
echo "readonly PORTAL_NET='${PORTAL_NET}'"
|
||||
echo "readonly ENABLE_CLUSTER_MONITORING='${ENABLE_CLUSTER_MONITORING:-false}'"
|
||||
echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'"
|
||||
echo "readonly ENABLE_CLUSTER_LOGGING='${ENABLE_CLUSTER_LOGGING:-false}'"
|
||||
echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'"
|
||||
echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'"
|
||||
echo "readonly ELASTICSEARCH_LOGGING_REPLICAS='${ELASTICSEARCH_LOGGING_REPLICAS:-}'"
|
||||
echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'"
|
||||
echo "readonly DNS_REPLICAS='${DNS_REPLICAS:-}'"
|
||||
echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'"
|
||||
echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/mount-pd.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh"
|
||||
) > "${KUBE_TEMP}/master-start.sh"
|
||||
# Generate a bearer token for this cluster. We push this separately
|
||||
# from the other cluster variables so that the client (this
|
||||
# computer) can forget it later. This should disappear with
|
||||
# https://github.com/GoogleCloudPlatform/kubernetes/issues/3168
|
||||
KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
|
||||
write-master-env
|
||||
gcloud compute instances create "${MASTER_NAME}" \
|
||||
--project "${PROJECT}" \
|
||||
--zone "${ZONE}" \
|
||||
|
|
@ -477,7 +536,9 @@ function kube-up {
|
|||
--tags "${MASTER_TAG}" \
|
||||
--network "${NETWORK}" \
|
||||
--scopes "storage-ro" "compute-rw" \
|
||||
--metadata-from-file "startup-script=${KUBE_TEMP}/master-start.sh" \
|
||||
--metadata-from-file \
|
||||
"startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" \
|
||||
"kube-env=${KUBE_TEMP}/master-kube-env.yaml" \
|
||||
--disk name="${MASTER_NAME}-pd" device-name=master-pd mode=rw boot=no auto-delete=no &
|
||||
|
||||
# Create a single firewall rule for all minions.
|
||||
|
|
@ -492,8 +553,11 @@ function kube-up {
|
|||
fi
|
||||
fi
|
||||
|
||||
# Wait for last batch of jobs.
|
||||
# Wait for last batch of jobs
|
||||
wait-for-jobs
|
||||
add-instance-metadata "${MASTER_NAME}" "kube-token=${KUBELET_TOKEN}"
|
||||
|
||||
echo "Creating minions."
|
||||
|
||||
local -a scope_flags=()
|
||||
if (( "${#MINION_SCOPES[@]}" > 0 )); then
|
||||
|
|
@ -502,29 +566,11 @@ function kube-up {
|
|||
scope_flags=("--no-scopes")
|
||||
fi
|
||||
|
||||
(
|
||||
echo "#! /bin/bash"
|
||||
echo "ZONE='${ZONE}'"
|
||||
echo "MASTER_NAME='${MASTER_NAME}'"
|
||||
echo "until MINION_IP_RANGE=\$(curl --fail --silent -H 'Metadata-Flavor: Google'\\"
|
||||
echo " http://metadata/computeMetadata/v1/instance/attributes/node-ip-range); do"
|
||||
echo " echo 'Waiting for metadata MINION_IP_RANGE...'"
|
||||
echo " sleep 3"
|
||||
echo "done"
|
||||
echo ""
|
||||
echo "# Remove docker artifacts on minion nodes"
|
||||
echo "iptables -t nat -F"
|
||||
echo "ifconfig docker0 down"
|
||||
echo "brctl delbr docker0"
|
||||
echo ""
|
||||
echo "EXTRA_DOCKER_OPTS='${EXTRA_DOCKER_OPTS}'"
|
||||
echo "ENABLE_DOCKER_REGISTRY_CACHE='${ENABLE_DOCKER_REGISTRY_CACHE:-false}'"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-minion.sh"
|
||||
) > "${KUBE_TEMP}/minion-start.sh"
|
||||
|
||||
write-node-env
|
||||
create-node-template "${NODE_INSTANCE_PREFIX}-template" "${scope_flags[*]}" \
|
||||
"startup-script=${KUBE_TEMP}/minion-start.sh"
|
||||
"startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" \
|
||||
"kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
|
||||
"kube-token=${KUBELET_TOKEN}"
|
||||
|
||||
gcloud preview managed-instance-groups --zone "${ZONE}" \
|
||||
create "${NODE_INSTANCE_PREFIX}-group" \
|
||||
|
|
@ -537,7 +583,17 @@ function kube-up {
|
|||
# to gcloud's deficiency.
|
||||
wait-for-minions-to-run
|
||||
|
||||
# Give the master an initial node list (it's waiting in
|
||||
# startup). This resolves a bit of a chicken-egg issue: The minions
|
||||
# need to know the master's ip, so we boot the master first. The
|
||||
# master still needs to know the initial minion list (until all the
|
||||
# pieces #156 are complete), so we have it wait on the minion
|
||||
# boot. (The minions further wait until the loop below, where CIDRs
|
||||
# get filled in.)
|
||||
detect-minion-names
|
||||
local kube_node_names
|
||||
kube_node_names=$(IFS=,; echo "${MINION_NAMES[*]}")
|
||||
add-instance-metadata "${MASTER_NAME}" "kube-node-names=${kube_node_names}"
|
||||
|
||||
# Create the routes and set IP ranges to instance metadata, 5 instances at a time.
|
||||
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||
|
|
@ -550,6 +606,7 @@ function kube-up {
|
|||
fi
|
||||
|
||||
done
|
||||
|
||||
# Wait for last batch of jobs.
|
||||
wait-for-jobs
|
||||
|
||||
|
|
@ -701,6 +758,14 @@ function kube-down {
|
|||
--delete-disks all \
|
||||
--zone "${ZONE}" \
|
||||
"${MASTER_NAME}" || true
|
||||
|
||||
# Delete the master pd (possibly leaked by kube-up if master create failed)
|
||||
gcloud compute disks delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
--zone "${ZONE}" \
|
||||
"${MASTER_NAME}"-pd || true
|
||||
|
||||
# Find out what minions are running.
|
||||
local -a minions
|
||||
minions=( $(gcloud compute instances list \
|
||||
|
|
@ -756,27 +821,49 @@ function kube-down {
|
|||
|
||||
# Update a kubernetes cluster with latest source
|
||||
function kube-push {
|
||||
OUTPUT=${KUBE_ROOT}/_output/logs
|
||||
mkdir -p ${OUTPUT}
|
||||
|
||||
ensure-temp-dir
|
||||
detect-project
|
||||
detect-master
|
||||
detect-minion-names
|
||||
get-password
|
||||
set-master-htpasswd
|
||||
|
||||
# Make sure we have the tar files staged on Google Storage
|
||||
find-release-tars
|
||||
upload-server-tars
|
||||
|
||||
(
|
||||
echo "#! /bin/bash"
|
||||
echo "mkdir -p /var/cache/kubernetes-install"
|
||||
echo "cd /var/cache/kubernetes-install"
|
||||
echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
|
||||
echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
|
||||
echo "echo Executing configuration"
|
||||
echo "sudo salt '*' mine.update"
|
||||
echo "sudo salt --force-color '*' state.highstate"
|
||||
) | gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "$KUBE_MASTER" --command "sudo bash"
|
||||
write-master-env
|
||||
add-instance-metadata-from-file "${KUBE_MASTER}" "kube-env=${KUBE_TEMP}/master-kube-env.yaml"
|
||||
echo "Pushing to master (log at ${OUTPUT}/kube-push-${KUBE_MASTER}.log) ..."
|
||||
cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${KUBE_MASTER}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-push-"${KUBE_MASTER}".log
|
||||
|
||||
get-password
|
||||
echo "Pushing metadata to minions... "
|
||||
write-node-env
|
||||
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||
add-instance-metadata-from-file "${MINION_NAMES[$i]}" "kube-env=${KUBE_TEMP}/node-kube-env.yaml" &
|
||||
done
|
||||
wait-for-jobs
|
||||
echo "Done"
|
||||
|
||||
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||
echo "Starting push to node (log at ${OUTPUT}/kube-push-${MINION_NAMES[$i]}.log) ..."
|
||||
cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${MINION_NAMES[$i]}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-push-"${MINION_NAMES[$i]}".log &
|
||||
done
|
||||
|
||||
echo -n "Waiting for node pushes... "
|
||||
wait-for-jobs
|
||||
echo "Done"
|
||||
|
||||
# TODO(zmerlynn): Re-create instance-template with the new
|
||||
# node-kube-env. This isn't important until the node-ip-range issue
|
||||
# is solved (because that's blocking automatic dynamic nodes from
|
||||
# working). The node-kube-env has to be composed with the kube-token
|
||||
# metadata. Ideally we would have
|
||||
# https://github.com/GoogleCloudPlatform/kubernetes/issues/3168
|
||||
# implemented before then, though, so avoiding this mess until then.
|
||||
|
||||
echo
|
||||
echo "Kubernetes cluster is running. The master is running at:"
|
||||
|
|
@ -785,7 +872,6 @@ function kube-push {
|
|||
echo
|
||||
echo "The user name and password to use is located in ~/.kubernetes_auth."
|
||||
echo
|
||||
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
# On GCE, there is no Salt mine. We run standalone.
|
||||
{% if grains.cloud != 'gce' -%}
|
||||
|
||||
# Allow everyone to see cached values of who sits at what IP
|
||||
{% set networkInterfaceName = "eth0" %}
|
||||
{% if grains.networkInterfaceName is defined %}
|
||||
|
|
@ -6,3 +9,5 @@
|
|||
mine_functions:
|
||||
network.ip_addrs: [{{networkInterfaceName}}]
|
||||
grains.items: []
|
||||
|
||||
{% endif -%}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@
|
|||
|
||||
{% if grains.etcd_servers is defined -%}
|
||||
{% set etcd_servers = "--etcd_servers=http://" + grains.etcd_servers + ":4001" -%}
|
||||
{% elif grains.cloud == 'gce' -%}
|
||||
{# TODO(zmerlynn): I can't see why this isn't generally applicable, but making this change surgical for now. #}
|
||||
{% set etcd_servers = "--etcd_servers=http://127.0.0.1:4001" -%}
|
||||
{% else -%}
|
||||
{% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
|
||||
{% set etcd_servers = "--etcd_servers=http://" + ips[0][0] + ":4001" -%}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
{% if grains.cloud is defined -%}
|
||||
{% if grains.cloud == 'gce' -%}
|
||||
{% set cloud_provider = "--cloud_provider=gce" -%}
|
||||
{% set machines = "--machines=" + ','.join(salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', expr_form='grain').keys()) -%}
|
||||
{% set machines = "--machines=" + pillar['gce_node_names'] -%}
|
||||
{% endif -%}
|
||||
{% if grains.cloud == 'aws' -%}
|
||||
{% set cloud_provider = "--cloud_provider=aws" -%}
|
||||
|
|
|
|||
Loading…
Reference in New Issue