mirror of https://github.com/k3s-io/k3s
Change GCE to use standalone Saltstack config:
Change provisioning to pass all variables to both master and node. Run Salt in a masterless setup on all nodes ala http://docs.saltstack.com/en/latest/topics/tutorials/quickstart.html, which involves ensuring Salt daemon is NOT running after install. Kill Salt master install. And fix push to actually work in this new flow. As part of this, the GCE Salt config no longer has access to the Salt mine, which is primarily obnoxious for two reasons: - The minions can't use Salt to see the master: this is easily fixed by static config. - The master can't see the list of all the minions: this is fixed temporarily by static config in util.sh, but later, by other means (see https://github.com/GoogleCloudPlatform/kubernetes/issues/156, which should eventually remove this direction). As part of it, flatten all of cluster/gce/templates/* into configure-vm.sh, using a single, separate piece of YAML to drive the environment variables, rather than constantly rewriting the startup script.pull/6/head
Zach Loafman
parent
6ba53b112f
commit
120dba474e
|
|
@ -786,8 +786,8 @@ function kube::release::gcs::copy_release_artifacts() {
|
|||
# Having the "template" scripts from the GCE cluster deploy hosted with the
|
||||
# release is useful for GKE. Copy everything from that directory up also.
|
||||
gsutil -m "${gcs_options[@]+${gcs_options[@]}}" cp \
|
||||
"${RELEASE_STAGE}/full/kubernetes/cluster/gce/templates/*.sh" \
|
||||
"${gcs_destination}extra/gce-templates/"
|
||||
"${RELEASE_STAGE}/full/kubernetes/cluster/gce/configure-vm.sh" \
|
||||
"${gcs_destination}extra/gce/"
|
||||
|
||||
# Upload the "naked" binaries to GCS. This is useful for install scripts that
|
||||
# download the binaries directly and don't need tars.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,353 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2015 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
# If we have any arguments at all, this is a push and not just setup.
|
||||
is_push=$@
|
||||
|
||||
function ensure-install-dir() {
|
||||
INSTALL_DIR="/var/cache/kubernetes-install"
|
||||
mkdir -p ${INSTALL_DIR}
|
||||
cd ${INSTALL_DIR}
|
||||
}
|
||||
|
||||
function set-broken-motd() {
|
||||
echo -e '\nBroken (or in progress) GCE Kubernetes node setup! Suggested first step:\n tail /var/log/startupscript.log\n' > /etc/motd
|
||||
}
|
||||
|
||||
function set-good-motd() {
|
||||
echo -e '\n=== GCE Kubernetes node setup complete ===\n' > /etc/motd
|
||||
}
|
||||
|
||||
function curl-metadata() {
|
||||
curl --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/attributes/${1}"
|
||||
}
|
||||
|
||||
function set-kube-env() {
|
||||
local kube_env_yaml="${INSTALL_DIR}/kube_env.yaml"
|
||||
|
||||
until curl-metadata kube-env > "${kube_env_yaml}"; do
|
||||
echo 'Waiting for kube-env...'
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# kube-env has all the environment variables we care about, in a flat yaml format
|
||||
eval $(python -c '''
|
||||
import pipes,sys,yaml
|
||||
|
||||
for k,v in yaml.load(sys.stdin).iteritems():
|
||||
print "readonly {var}={value}".format(var = k, value = pipes.quote(str(v)))
|
||||
''' < "${kube_env_yaml}")
|
||||
|
||||
# We bake the KUBELET_TOKEN in separately to avoid auth information
|
||||
# having to be re-communicated on kube-push. (Otherwise the client
|
||||
# has to keep the bearer token around to handle generating a valid
|
||||
# kube-env.)
|
||||
if [[ -z "${KUBELET_TOKEN:-}" ]]; then
|
||||
until KUBELET_TOKEN=$(curl-metadata kube-token); do
|
||||
echo 'Waiting for metadata KUBELET_TOKEN...'
|
||||
sleep 3
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
|
||||
# TODO(zmerlynn): This block of code should disappear once #4561 & #4562 are done
|
||||
if [[ -z "${KUBERNETES_NODE_NAMES:-}" ]]; then
|
||||
until KUBERNETES_NODE_NAMES=$(curl-metadata kube-node-names); do
|
||||
echo 'Waiting for metadata KUBERNETES_NODE_NAMES...'
|
||||
sleep 3
|
||||
done
|
||||
fi
|
||||
else
|
||||
# And this should go away once the master can allocate CIDRs
|
||||
if [[ -z "${MINION_IP_RANGE:-}" ]]; then
|
||||
until MINION_IP_RANGE=$(curl-metadata node-ip-range); do
|
||||
echo 'Waiting for metadata MINION_IP_RANGE...'
|
||||
sleep 3
|
||||
done
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function remove-docker-artifacts() {
|
||||
# Remove docker artifacts on minion nodes, if present
|
||||
iptables -t nat -F || true
|
||||
ifconfig docker0 down || true
|
||||
brctl delbr docker0 || true
|
||||
}
|
||||
|
||||
# Retry a download until we get it.
|
||||
#
|
||||
# $1 is the URL to download
|
||||
download-or-bust() {
|
||||
local -r url="$1"
|
||||
local -r file="${url##*/}"
|
||||
rm -f "$file"
|
||||
until [[ -e "${1##*/}" ]]; do
|
||||
echo "Downloading file ($1)"
|
||||
curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$1"
|
||||
done
|
||||
}
|
||||
|
||||
# Install salt from GCS. See README.md for instructions on how to update these
|
||||
# debs.
|
||||
install-salt() {
|
||||
apt-get update
|
||||
|
||||
mkdir -p /var/cache/salt-install
|
||||
cd /var/cache/salt-install
|
||||
|
||||
TARS=(
|
||||
libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
|
||||
python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
|
||||
salt-common_2014.1.13+ds-1~bpo70+1_all.deb
|
||||
salt-minion_2014.1.13+ds-1~bpo70+1_all.deb
|
||||
)
|
||||
URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
|
||||
|
||||
for tar in "${TARS[@]}"; do
|
||||
download-or-bust "${URL_BASE}/${tar}"
|
||||
dpkg -i "${tar}" || true
|
||||
done
|
||||
|
||||
# This will install any of the unmet dependencies from above.
|
||||
apt-get install -f -y
|
||||
}
|
||||
|
||||
# Ensure salt-minion *isn't* running
|
||||
stop-salt-minion() {
|
||||
# This ensures it on next reboot
|
||||
echo manual > /etc/init/salt-minion.override
|
||||
|
||||
service salt-minion stop
|
||||
while service salt-minion status >/dev/null; do
|
||||
service salt-minion stop # No, really.
|
||||
echo "Waiting for salt-minion to shut down"
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# Mounts a persistent disk (formatting if needed) to store the persistent data
|
||||
# on the master -- etcd's data, a few settings, and security certs/keys/tokens.
|
||||
#
|
||||
# This function can be reused to mount an existing PD because all of its
|
||||
# operations modifying the disk are idempotent -- safe_format_and_mount only
|
||||
# formats an unformatted disk, and mkdir -p will leave a directory be if it
|
||||
# already exists.
|
||||
mount-master-pd() {
|
||||
device_info=$(ls -l /dev/disk/by-id/google-master-pd)
|
||||
relative_path=${device_info##* }
|
||||
device_path="/dev/disk/by-id/${relative_path}"
|
||||
|
||||
# Format and mount the disk, create directories on it for all of the master's
|
||||
# persistent data, and link them to where they're used.
|
||||
mkdir -p /mnt/master-pd
|
||||
/usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd
|
||||
# Contains all the data stored in etcd
|
||||
mkdir -m 700 -p /mnt/master-pd/var/etcd
|
||||
# Contains the dynamically generated apiserver auth certs and keys
|
||||
mkdir -p /mnt/master-pd/srv/kubernetes
|
||||
# Contains the cluster's initial config parameters and auth tokens
|
||||
mkdir -p /mnt/master-pd/srv/salt-overlay
|
||||
ln -s /mnt/master-pd/var/etcd /var/etcd
|
||||
ln -s /mnt/master-pd/srv/kubernetes /srv/kubernetes
|
||||
ln -s /mnt/master-pd/srv/salt-overlay /srv/salt-overlay
|
||||
|
||||
# This is a bit of a hack to get around the fact that salt has to run after the
|
||||
# PD and mounted directory are already set up. We can't give ownership of the
|
||||
# directory to etcd until the etcd user and group exist, but they don't exist
|
||||
# until salt runs if we don't create them here. We could alternatively make the
|
||||
# permissions on the directory more permissive, but this seems less bad.
|
||||
useradd -s /sbin/nologin -d /var/etcd etcd
|
||||
chown etcd /mnt/master-pd/var/etcd
|
||||
chgrp etcd /mnt/master-pd/var/etcd
|
||||
}
|
||||
|
||||
# Create the overlay files for the salt tree. We create these in a separate
|
||||
# place so that we can blow away the rest of the salt configs on a kube-push and
|
||||
# re-apply these.
|
||||
function create-salt-pillar() {
|
||||
# Always overwrite the cluster-params.sls (even on a push, we have
|
||||
# these variables)
|
||||
mkdir -p /srv/salt-overlay/pillar
|
||||
cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
|
||||
instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")'
|
||||
node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")'
|
||||
portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")'
|
||||
enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")'
|
||||
enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")'
|
||||
enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")'
|
||||
enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")'
|
||||
logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")'
|
||||
elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")'
|
||||
enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")'
|
||||
dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")'
|
||||
dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")'
|
||||
dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
|
||||
if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
|
||||
cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
|
||||
gce_node_names: '$(echo "$KUBERNETES_NODE_NAMES" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# This should only happen on cluster initialization
|
||||
function create-salt-auth() {
|
||||
mkdir -p /srv/salt-overlay/salt/nginx
|
||||
echo "${MASTER_HTPASSWD}" > /srv/salt-overlay/salt/nginx/htpasswd
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/kube-apiserver
|
||||
known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
|
||||
(umask 077;
|
||||
echo "${KUBELET_TOKEN},kubelet,kubelet" > "${known_tokens_file}")
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/kubelet
|
||||
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
|
||||
(umask 077;
|
||||
echo "{\"BearerToken\": \"${KUBELET_TOKEN}\", \"Insecure\": true }" > "${kubelet_auth_file}")
|
||||
}
|
||||
|
||||
function download-release() {
|
||||
echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
|
||||
download-or-bust "$SERVER_BINARY_TAR_URL"
|
||||
|
||||
echo "Downloading binary release tar ($SALT_TAR_URL)"
|
||||
download-or-bust "$SALT_TAR_URL"
|
||||
|
||||
echo "Unpacking Salt tree"
|
||||
rm -rf kubernetes
|
||||
tar xzf "${SALT_TAR_URL##*/}"
|
||||
|
||||
echo "Running release install script"
|
||||
sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"
|
||||
}
|
||||
|
||||
function fix-apt-sources() {
|
||||
sed -i -e "\|^deb.*http://http.debian.net/debian| s/^/#/" /etc/apt/sources.list
|
||||
sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d/backports.list
|
||||
}
|
||||
|
||||
function salt-run-local() {
|
||||
cat <<EOF >/etc/salt/minion.d/local.conf
|
||||
file_client: local
|
||||
file_roots:
|
||||
base:
|
||||
- /srv/salt
|
||||
EOF
|
||||
}
|
||||
|
||||
function salt-debug-log() {
|
||||
cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
|
||||
log_level: debug
|
||||
log_level_logfile: debug
|
||||
EOF
|
||||
}
|
||||
|
||||
function salt-master-role() {
|
||||
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||
grains:
|
||||
roles:
|
||||
- kubernetes-master
|
||||
cloud: gce
|
||||
EOF
|
||||
}
|
||||
|
||||
function salt-node-role() {
|
||||
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||
grains:
|
||||
roles:
|
||||
- kubernetes-pool
|
||||
cbr-cidr: '$(echo "$MINION_IP_RANGE" | sed -e "s/'/''/g")'
|
||||
cloud: gce
|
||||
EOF
|
||||
}
|
||||
|
||||
function salt-docker-opts() {
|
||||
DOCKER_OPTS=""
|
||||
|
||||
if [[ -n "${EXTRA_DOCKER_OPTS-}" ]]; then
|
||||
DOCKER_OPTS="${EXTRA_DOCKER_OPTS}"
|
||||
fi
|
||||
|
||||
# Decide whether to enable the cache
|
||||
if [[ "${ENABLE_DOCKER_REGISTRY_CACHE}" == "true" ]]; then
|
||||
REGION=$(echo "${ZONE}" | cut -f 1,2 -d -)
|
||||
echo "Enable docker registry cache at region: " $REGION
|
||||
DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror='https://${REGION}.docker-cache.clustermaster.net'"
|
||||
fi
|
||||
|
||||
if [[ -n "{DOCKER_OPTS}" ]]; then
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
function salt-set-apiserver() {
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
api_servers: '${KUBERNETES_MASTER_IP}'
|
||||
apiservers: '${KUBERNETES_MASTER_IP}'
|
||||
EOF
|
||||
}
|
||||
|
||||
function configure-salt() {
|
||||
fix-apt-sources
|
||||
mkdir -p /etc/salt/minion.d
|
||||
salt-run-local
|
||||
if [[ "${KUBERNETES_MASTER}" == "true" ]]; then
|
||||
salt-master-role
|
||||
else
|
||||
salt-node-role
|
||||
salt-docker-opts
|
||||
salt-set-apiserver
|
||||
fi
|
||||
install-salt
|
||||
stop-salt-minion
|
||||
}
|
||||
|
||||
function run-salt() {
|
||||
salt-call --local state.highstate || true
|
||||
}
|
||||
|
||||
####################################################################################
|
||||
|
||||
if [[ -z "${is_push}" ]]; then
|
||||
echo "== kube-up node config starting =="
|
||||
set-broken-motd
|
||||
ensure-install-dir
|
||||
set-kube-env
|
||||
[[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd
|
||||
create-salt-pillar
|
||||
create-salt-auth
|
||||
download-release
|
||||
configure-salt
|
||||
remove-docker-artifacts
|
||||
run-salt
|
||||
set-good-motd
|
||||
echo "== kube-up node config done =="
|
||||
else
|
||||
echo "== kube-push node config starting =="
|
||||
ensure-install-dir
|
||||
set-kube-env
|
||||
create-salt-pillar
|
||||
run-salt
|
||||
echo "== kube-push node config done =="
|
||||
fi
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# Copyright 2015 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
def mutate_env(path, var, value):
|
||||
# Load the existing arguments
|
||||
if os.path.exists(path):
|
||||
args = yaml.load(open(path))
|
||||
else:
|
||||
args = {}
|
||||
args[var] = value
|
||||
yaml.dump(args, stream=open(path, 'w'), default_flow_style=False)
|
||||
|
||||
if __name__ == "__main__":
|
||||
mutate_env(sys.argv[1], sys.argv[2], sys.argv[3])
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
# Updating Salt debs
|
||||
|
||||
We are caching all of the salt debs in GCS for speed and reliability.
|
||||
|
||||
To update them, follow this simple N step process:
|
||||
|
||||
1. Start up a new base image without salt installed. SSH into this image.
|
||||
2. Install salt via their recommended method: `curl -L https://bootstrap.saltstack.com | sudo sh -s -- -M -X`
|
||||
3. Find and download the debs that originated at the saltstack.com repo: `aptitude search --disable-columns -F "%p %V" "?installed?origin(saltstack.com)" | xargs aptitude download`
|
||||
4. Upload these to GCS: `gsutil cp *.deb gs://kubernetes-release/salt/`
|
||||
5. Make sure that everything is publicly readable: `gsutil acl ch -R -g all:R gs://kubernetes-release/salt/`
|
||||
6. Test things well :)
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Retry a download until we get it.
|
||||
#
|
||||
# $1 is the URL to download
|
||||
download-or-bust() {
|
||||
local -r url="$1"
|
||||
local -r file="${url##*/}"
|
||||
rm -f "$file"
|
||||
until [[ -e "${1##*/}" ]]; do
|
||||
echo "Downloading file ($1)"
|
||||
curl --ipv4 -Lo "$file" --connect-timeout 20 --retry 6 --retry-delay 10 "$1"
|
||||
md5sum "$file"
|
||||
done
|
||||
}
|
||||
|
||||
# Install salt from GCS. See README.md for instructions on how to update these
|
||||
# debs.
|
||||
#
|
||||
# $1 If set to --master, also install the master
|
||||
install-salt() {
|
||||
apt-get update
|
||||
|
||||
mkdir -p /var/cache/salt-install
|
||||
cd /var/cache/salt-install
|
||||
|
||||
TARS=(
|
||||
libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
|
||||
python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
|
||||
salt-common_2014.1.13+ds-1~bpo70+1_all.deb
|
||||
salt-minion_2014.1.13+ds-1~bpo70+1_all.deb
|
||||
)
|
||||
if [[ ${1-} == '--master' ]]; then
|
||||
TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb)
|
||||
fi
|
||||
URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
|
||||
|
||||
for tar in "${TARS[@]}"; do
|
||||
download-or-bust "${URL_BASE}/${tar}"
|
||||
dpkg -i "${tar}"
|
||||
done
|
||||
|
||||
# This will install any of the unmet dependencies from above.
|
||||
apt-get install -f -y
|
||||
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Create the overlay files for the salt tree. We create these in a separate
|
||||
# place so that we can blow away the rest of the salt configs on a kube-push and
|
||||
# re-apply these.
|
||||
|
||||
mkdir -p /srv/salt-overlay/pillar
|
||||
cat <<EOF >/srv/salt-overlay/pillar/cluster-params.sls
|
||||
instance_prefix: '$(echo "$INSTANCE_PREFIX" | sed -e "s/'/''/g")'
|
||||
node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")'
|
||||
portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")'
|
||||
enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")'
|
||||
enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")'
|
||||
enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")'
|
||||
enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")'
|
||||
logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")'
|
||||
elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")'
|
||||
enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")'
|
||||
dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")'
|
||||
dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")'
|
||||
dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/nginx
|
||||
echo $MASTER_HTPASSWD > /srv/salt-overlay/salt/nginx/htpasswd
|
||||
|
||||
# Generate and distribute a shared secret (bearer token) to
|
||||
# apiserver and kubelet so that kubelet can authenticate to
|
||||
# apiserver to send events.
|
||||
# This works on CoreOS, so it should work on a lot of distros.
|
||||
kubelet_token=$(cat /dev/urandom | base64 | tr -d "=+/" | dd bs=32 count=1 2> /dev/null)
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/kube-apiserver
|
||||
known_tokens_file="/srv/salt-overlay/salt/kube-apiserver/known_tokens.csv"
|
||||
(umask u=rw,go= ; echo "$kubelet_token,kubelet,kubelet" > $known_tokens_file)
|
||||
|
||||
mkdir -p /srv/salt-overlay/salt/kubelet
|
||||
kubelet_auth_file="/srv/salt-overlay/salt/kubelet/kubernetes_auth"
|
||||
(umask u=rw,go= ; echo "{\"BearerToken\": \"$kubelet_token\", \"Insecure\": true }" > $kubelet_auth_file)
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Download and install release
|
||||
|
||||
# This script assumes that the environment variable MASTER_RELEASE_TAR contains
|
||||
# the release tar to download and unpack. It is meant to be pushed to the
|
||||
# master and run.
|
||||
|
||||
|
||||
echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
|
||||
download-or-bust "$SERVER_BINARY_TAR_URL"
|
||||
|
||||
echo "Downloading binary release tar ($SALT_TAR_URL)"
|
||||
download-or-bust "$SALT_TAR_URL"
|
||||
|
||||
echo "Unpacking Salt tree"
|
||||
rm -rf kubernetes
|
||||
tar xzf "${SALT_TAR_URL##*/}"
|
||||
|
||||
echo "Running release install script"
|
||||
sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR_URL##*/}"
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Mounts a persistent disk (formatting if needed) to store the persistent data
|
||||
# on the master -- etcd's data, a few settings, and security certs/keys/tokens.
|
||||
#
|
||||
# This script can be reused to mount an existing PD because all of its
|
||||
# operations modifying the disk are idempotent -- safe_format_and_mount only
|
||||
# formats an unformatted disk, and mkdir -p will leave a directory be if it
|
||||
# already exists.
|
||||
|
||||
device_info=$(ls -l /dev/disk/by-id/google-master-pd)
|
||||
relative_path=${device_info##* }
|
||||
device_path="/dev/disk/by-id/${relative_path}"
|
||||
|
||||
# Format and mount the disk, create directories on it for all of the master's
|
||||
# persistent data, and link them to where they're used.
|
||||
mkdir -p /mnt/master-pd
|
||||
/usr/share/google/safe_format_and_mount -m "mkfs.ext4 -F" "${device_path}" /mnt/master-pd
|
||||
# Contains all the data stored in etcd
|
||||
mkdir -m 700 -p /mnt/master-pd/var/etcd
|
||||
# Contains the dynamically generated apiserver auth certs and keys
|
||||
mkdir -p /mnt/master-pd/srv/kubernetes
|
||||
# Contains the cluster's initial config parameters and auth tokens
|
||||
mkdir -p /mnt/master-pd/srv/salt-overlay
|
||||
# Contains salt's dynamically generated RSA keys
|
||||
mkdir -m 770 -p /mnt/master-pd/etc/salt/pki
|
||||
ln -s /mnt/master-pd/var/etcd /var/etcd
|
||||
ln -s /mnt/master-pd/srv/kubernetes /srv/kubernetes
|
||||
ln -s /mnt/master-pd/srv/salt-overlay /srv/salt-overlay
|
||||
ln -s /mnt/master-pd/etc/salt/pki /etc/salt/pki
|
||||
|
||||
# This is a bit of a hack to get around the fact that salt has to run after the
|
||||
# PD and mounted directory are already set up. We can't give ownership of the
|
||||
# directory to etcd until the etcd user and group exist, but they don't exist
|
||||
# until salt runs if we don't create them here. We could alternatively make the
|
||||
# permissions on the directory more permissive, but this seems less bad.
|
||||
useradd -s /sbin/nologin -d /var/etcd etcd
|
||||
chown etcd /mnt/master-pd/var/etcd
|
||||
chgrp etcd /mnt/master-pd/var/etcd
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
sed -i -e "\|^deb.*http://http.debian.net/debian| s/^/#/" /etc/apt/sources.list
|
||||
sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d/backports.list
|
||||
|
||||
# Prepopulate the name of the Master
|
||||
mkdir -p /etc/salt/minion.d
|
||||
cat <<EOF >/etc/salt/minion.d/master.conf
|
||||
master: '$(echo "$MASTER_NAME" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
|
||||
log_level: debug
|
||||
log_level_logfile: debug
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||
grains:
|
||||
roles:
|
||||
- kubernetes-master
|
||||
cloud: gce
|
||||
EOF
|
||||
|
||||
# Auto accept all keys from minions that try to join
|
||||
mkdir -p /etc/salt/master.d
|
||||
cat <<EOF >/etc/salt/master.d/auto-accept.conf
|
||||
auto_accept: True
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/master.d/reactor.conf
|
||||
# React to new minions starting by running highstate on them.
|
||||
reactor:
|
||||
- 'salt/minion/*/start':
|
||||
- /srv/reactor/highstate-new.sls
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/master.d/log-level-debug.d
|
||||
log_level: debug
|
||||
log_level_logfile: debug
|
||||
EOF
|
||||
|
||||
install-salt --master
|
||||
|
||||
# Wait a few minutes and trigger another Salt run to better recover from
|
||||
# any transient errors.
|
||||
echo "Sleeping 180"
|
||||
sleep 180
|
||||
salt-call state.highstate || true
|
||||
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# The repositories are really slow and there are GCE mirrors
|
||||
sed -i -e "\|^deb.*http://http.debian.net/debian| s/^/#/" /etc/apt/sources.list
|
||||
sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d/backports.list
|
||||
|
||||
# Prepopulate the name of the Master
|
||||
mkdir -p /etc/salt/minion.d
|
||||
cat <<EOF >/etc/salt/minion.d/master.conf
|
||||
master: '$(echo "$MASTER_NAME" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
|
||||
cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
|
||||
log_level: debug
|
||||
log_level_logfile: debug
|
||||
EOF
|
||||
|
||||
# Our minions will have a pool role to distinguish them from the master.
|
||||
cat <<EOF >/etc/salt/minion.d/grains.conf
|
||||
grains:
|
||||
roles:
|
||||
- kubernetes-pool
|
||||
cbr-cidr: '$(echo "$MINION_IP_RANGE" | sed -e "s/'/''/g")'
|
||||
cloud: gce
|
||||
EOF
|
||||
|
||||
DOCKER_OPTS=""
|
||||
|
||||
if [[ -n "${EXTRA_DOCKER_OPTS-}" ]]; then
|
||||
DOCKER_OPTS="${EXTRA_DOCKER_OPTS}"
|
||||
fi
|
||||
|
||||
# Decide if enable the cache
|
||||
if [[ "${ENABLE_DOCKER_REGISTRY_CACHE}" == "true" ]]; then
|
||||
REGION=$(echo "${ZONE}" | cut -f 1,2 -d -)
|
||||
echo "Enable docker registry cache at region: " $REGION
|
||||
DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror='https://${REGION}.docker-cache.clustermaster.net'"
|
||||
fi
|
||||
|
||||
if [[ -n "{DOCKER_OPTS}" ]]; then
|
||||
cat <<EOF >>/etc/salt/minion.d/grains.conf
|
||||
docker_opts: '$(echo "$DOCKER_OPTS" | sed -e "s/'/''/g")'
|
||||
EOF
|
||||
fi
|
||||
|
||||
install-salt
|
||||
|
||||
# Wait a few minutes and trigger another Salt run to better recover from
|
||||
# any transient errors.
|
||||
echo "Sleeping 180"
|
||||
sleep 180
|
||||
salt-call state.highstate || true
|
||||
|
|
@ -208,6 +208,7 @@ function detect-minions () {
|
|||
# Vars set:
|
||||
# KUBE_MASTER
|
||||
# KUBE_MASTER_IP
|
||||
# KUBE_MASTER_IP_INTERNAL
|
||||
function detect-master () {
|
||||
detect-project
|
||||
KUBE_MASTER=${MASTER_NAME}
|
||||
|
|
@ -215,6 +216,9 @@ function detect-master () {
|
|||
KUBE_MASTER_IP=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
|
||||
"${MASTER_NAME}" --fields networkInterfaces[0].accessConfigs[0].natIP \
|
||||
--format=text | awk '{ print $2 }')
|
||||
KUBE_MASTER_IP_INTERNAL=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \
|
||||
"${MASTER_NAME}" --fields networkInterfaces[0].networkIP \
|
||||
--format=text | awk '{ print $2 }')
|
||||
fi
|
||||
if [[ -z "${KUBE_MASTER_IP-}" ]]; then
|
||||
echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2
|
||||
|
|
@ -246,6 +250,14 @@ function get-password {
|
|||
KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))')
|
||||
}
|
||||
|
||||
# Set MASTER_HTPASSWD
|
||||
function set-master-htpasswd {
|
||||
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
|
||||
-b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
|
||||
local htpasswd
|
||||
MASTER_HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd")
|
||||
}
|
||||
|
||||
# Generate authentication token for admin user. Will
|
||||
# read from $HOME/.kubernetes_auth if available.
|
||||
#
|
||||
|
|
@ -332,6 +344,8 @@ function create-route {
|
|||
# $1: The name of the instance template.
|
||||
# $2: The scopes flag.
|
||||
# $3: The minion start script metadata from file.
|
||||
# $4: The kube-env metadata.
|
||||
# $5: Raw metadata
|
||||
function create-node-template {
|
||||
detect-project
|
||||
local attempt=0
|
||||
|
|
@ -347,7 +361,8 @@ function create-node-template {
|
|||
--network "${NETWORK}" \
|
||||
$2 \
|
||||
--can-ip-forward \
|
||||
--metadata-from-file "$3"; then
|
||||
--metadata-from-file "$3" "$4" \
|
||||
--metadata "$5"; then
|
||||
if (( attempt > 5 )); then
|
||||
echo -e "${color_red}Failed to create instance template $1 ${color_norm}"
|
||||
exit 2
|
||||
|
|
@ -383,26 +398,94 @@ function add-instance-metadata {
|
|||
done
|
||||
}
|
||||
|
||||
# Robustly try to add metadata on an instance, from a file.
|
||||
# $1: The name of the instace.
|
||||
# $2: The metadata key=file pair to add.
|
||||
function add-instance-metadata-from-file {
|
||||
detect-project
|
||||
local attempt=0
|
||||
while true; do
|
||||
if ! gcloud compute instances add-metadata "$1" \
|
||||
--project "${PROJECT}" \
|
||||
--zone "${ZONE}" \
|
||||
--metadata-from-file "$2"; then
|
||||
if (( attempt > 5 )); then
|
||||
echo -e "${color_red}Failed to add instance metadata in $1 ${color_norm}"
|
||||
exit 2
|
||||
fi
|
||||
echo -e "${color_yellow}Attempt $(($attempt+1)) failed to add metadata in $1. Retrying.${color_norm}"
|
||||
attempt=$(($attempt+1))
|
||||
else
|
||||
break
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Given a yaml file, add or mutate the given env variable
|
||||
#
|
||||
# TODO(zmerlynn): Yes, this is an O(n^2) build-up right now. If we end
|
||||
# up with so many environment variables feeding into Salt that this
|
||||
# matters, there's probably an issue...
|
||||
function add-to-env {
|
||||
${KUBE_ROOT}/cluster/gce/kube-env.py "$1" "$2" "$3"
|
||||
}
|
||||
|
||||
# $1: if 'true', we're building a master yaml, else a node
|
||||
function build-kube-env {
|
||||
local master=$1
|
||||
local file=$2
|
||||
|
||||
rm -f ${file}
|
||||
add-to-env ${file} ENV_TIMESTAMP "$(date -uIs)" # Just to track it
|
||||
add-to-env ${file} KUBERNETES_MASTER "${master}"
|
||||
add-to-env ${file} INSTANCE_PREFIX "${INSTANCE_PREFIX}"
|
||||
add-to-env ${file} NODE_INSTANCE_PREFIX "${NODE_INSTANCE_PREFIX}"
|
||||
add-to-env ${file} SERVER_BINARY_TAR_URL "${SERVER_BINARY_TAR_URL}"
|
||||
add-to-env ${file} SALT_TAR_URL "${SALT_TAR_URL}"
|
||||
add-to-env ${file} PORTAL_NET "${PORTAL_NET}"
|
||||
add-to-env ${file} ENABLE_CLUSTER_MONITORING "${ENABLE_CLUSTER_MONITORING:-false}"
|
||||
add-to-env ${file} ENABLE_NODE_MONITORING "${ENABLE_NODE_MONITORING:-false}"
|
||||
add-to-env ${file} ENABLE_CLUSTER_LOGGING "${ENABLE_CLUSTER_LOGGING:-false}"
|
||||
add-to-env ${file} ENABLE_NODE_LOGGING "${ENABLE_NODE_LOGGING:-false}"
|
||||
add-to-env ${file} LOGGING_DESTINATION "${LOGGING_DESTINATION:-}"
|
||||
add-to-env ${file} ELASTICSEARCH_LOGGING_REPLICAS "${ELASTICSEARCH_LOGGING_REPLICAS:-}"
|
||||
add-to-env ${file} ENABLE_CLUSTER_DNS "${ENABLE_CLUSTER_DNS:-false}"
|
||||
add-to-env ${file} DNS_REPLICAS "${DNS_REPLICAS:-}"
|
||||
add-to-env ${file} DNS_SERVER_IP "${DNS_SERVER_IP:-}"
|
||||
add-to-env ${file} DNS_DOMAIN "${DNS_DOMAIN:-}"
|
||||
add-to-env ${file} MASTER_HTPASSWD "${MASTER_HTPASSWD}"
|
||||
if [[ "${master}" != "true" ]]; then
|
||||
add-to-env ${file} KUBERNETES_MASTER_IP "${KUBE_MASTER_IP_INTERNAL}"
|
||||
add-to-env ${file} ZONE "${ZONE}"
|
||||
add-to-env ${file} EXTRA_DOCKER_OPTS "${EXTRA_DOCKER_OPTS}"
|
||||
add-to-env ${file} ENABLE_DOCKER_REGISTRY_CACHE "${ENABLE_DOCKER_REGISTRY_CACHE:-false}"
|
||||
fi
|
||||
}
|
||||
|
||||
function write-master-env {
|
||||
build-kube-env true "${KUBE_TEMP}/master-kube-env.yaml"
|
||||
}
|
||||
|
||||
function write-node-env {
|
||||
build-kube-env false "${KUBE_TEMP}/node-kube-env.yaml"
|
||||
}
|
||||
|
||||
# Instantiate a kubernetes cluster
|
||||
#
|
||||
# Assumed vars
|
||||
# KUBE_ROOT
|
||||
# <Various vars set in config file>
|
||||
function kube-up {
|
||||
ensure-temp-dir
|
||||
detect-project
|
||||
|
||||
get-password
|
||||
set-master-htpasswd
|
||||
|
||||
# Make sure we have the tar files staged on Google Storage
|
||||
find-release-tars
|
||||
upload-server-tars
|
||||
|
||||
ensure-temp-dir
|
||||
|
||||
get-password
|
||||
python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \
|
||||
-b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD"
|
||||
local htpasswd
|
||||
htpasswd=$(cat "${KUBE_TEMP}/htpasswd")
|
||||
|
||||
if ! gcloud compute networks --project "${PROJECT}" describe "${NETWORK}" &>/dev/null; then
|
||||
echo "Creating new network: ${NETWORK}"
|
||||
# The network needs to be created synchronously or we have a race. The
|
||||
|
|
@ -426,7 +509,7 @@ function kube-up {
|
|||
--allow "tcp:22" &
|
||||
fi
|
||||
|
||||
echo "Starting VMs and configuring firewalls"
|
||||
echo "Starting master and configuring firewalls"
|
||||
gcloud compute firewall-rules create "${MASTER_NAME}-https" \
|
||||
--project "${PROJECT}" \
|
||||
--network "${NETWORK}" \
|
||||
|
|
@ -440,34 +523,13 @@ function kube-up {
|
|||
--zone "${ZONE}" \
|
||||
--size "10GB"
|
||||
|
||||
(
|
||||
echo "#! /bin/bash"
|
||||
echo "mkdir -p /var/cache/kubernetes-install"
|
||||
echo "cd /var/cache/kubernetes-install"
|
||||
echo "readonly MASTER_NAME='${MASTER_NAME}'"
|
||||
echo "readonly INSTANCE_PREFIX='${INSTANCE_PREFIX}'"
|
||||
echo "readonly NODE_INSTANCE_PREFIX='${NODE_INSTANCE_PREFIX}'"
|
||||
echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
|
||||
echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
|
||||
echo "readonly MASTER_HTPASSWD='${htpasswd}'"
|
||||
echo "readonly PORTAL_NET='${PORTAL_NET}'"
|
||||
echo "readonly ENABLE_CLUSTER_MONITORING='${ENABLE_CLUSTER_MONITORING:-false}'"
|
||||
echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'"
|
||||
echo "readonly ENABLE_CLUSTER_LOGGING='${ENABLE_CLUSTER_LOGGING:-false}'"
|
||||
echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'"
|
||||
echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'"
|
||||
echo "readonly ELASTICSEARCH_LOGGING_REPLICAS='${ELASTICSEARCH_LOGGING_REPLICAS:-}'"
|
||||
echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'"
|
||||
echo "readonly DNS_REPLICAS='${DNS_REPLICAS:-}'"
|
||||
echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'"
|
||||
echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/mount-pd.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh"
|
||||
) > "${KUBE_TEMP}/master-start.sh"
|
||||
# Generate a bearer token for this cluster. We push this separately
|
||||
# from the other cluster variables so that the client (this
|
||||
# computer) can forget it later. This should disappear with
|
||||
# https://github.com/GoogleCloudPlatform/kubernetes/issues/3168
|
||||
KUBELET_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
|
||||
write-master-env
|
||||
gcloud compute instances create "${MASTER_NAME}" \
|
||||
--project "${PROJECT}" \
|
||||
--zone "${ZONE}" \
|
||||
|
|
@ -477,7 +539,9 @@ function kube-up {
|
|||
--tags "${MASTER_TAG}" \
|
||||
--network "${NETWORK}" \
|
||||
--scopes "storage-ro" "compute-rw" \
|
||||
--metadata-from-file "startup-script=${KUBE_TEMP}/master-start.sh" \
|
||||
--metadata-from-file \
|
||||
"startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" \
|
||||
"kube-env=${KUBE_TEMP}/master-kube-env.yaml" \
|
||||
--disk name="${MASTER_NAME}-pd" device-name=master-pd mode=rw boot=no auto-delete=no &
|
||||
|
||||
# Create a single firewall rule for all minions.
|
||||
|
|
@ -492,8 +556,12 @@ function kube-up {
|
|||
fi
|
||||
fi
|
||||
|
||||
# Wait for last batch of jobs.
|
||||
# Wait for last batch of jobs
|
||||
wait-for-jobs
|
||||
detect-master # We need the KUBE_MASTER_IP_INTERNAL for the node startup script
|
||||
add-instance-metadata "${MASTER_NAME}" "kube-token=${KUBELET_TOKEN}"
|
||||
|
||||
echo "Creating minions."
|
||||
|
||||
local -a scope_flags=()
|
||||
if (( "${#MINION_SCOPES[@]}" > 0 )); then
|
||||
|
|
@ -502,29 +570,11 @@ function kube-up {
|
|||
scope_flags=("--no-scopes")
|
||||
fi
|
||||
|
||||
(
|
||||
echo "#! /bin/bash"
|
||||
echo "ZONE='${ZONE}'"
|
||||
echo "MASTER_NAME='${MASTER_NAME}'"
|
||||
echo "until MINION_IP_RANGE=\$(curl --fail --silent -H 'Metadata-Flavor: Google'\\"
|
||||
echo " http://metadata/computeMetadata/v1/instance/attributes/node-ip-range); do"
|
||||
echo " echo 'Waiting for metadata MINION_IP_RANGE...'"
|
||||
echo " sleep 3"
|
||||
echo "done"
|
||||
echo ""
|
||||
echo "# Remove docker artifacts on minion nodes"
|
||||
echo "iptables -t nat -F"
|
||||
echo "ifconfig docker0 down"
|
||||
echo "brctl delbr docker0"
|
||||
echo ""
|
||||
echo "EXTRA_DOCKER_OPTS='${EXTRA_DOCKER_OPTS}'"
|
||||
echo "ENABLE_DOCKER_REGISTRY_CACHE='${ENABLE_DOCKER_REGISTRY_CACHE:-false}'"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-minion.sh"
|
||||
) > "${KUBE_TEMP}/minion-start.sh"
|
||||
|
||||
write-node-env
|
||||
create-node-template "${NODE_INSTANCE_PREFIX}-template" "${scope_flags[*]}" \
|
||||
"startup-script=${KUBE_TEMP}/minion-start.sh"
|
||||
"startup-script=${KUBE_ROOT}/cluster/gce/configure-vm.sh" \
|
||||
"kube-env=${KUBE_TEMP}/node-kube-env.yaml" \
|
||||
"kube-token=${KUBELET_TOKEN}"
|
||||
|
||||
gcloud preview managed-instance-groups --zone "${ZONE}" \
|
||||
create "${NODE_INSTANCE_PREFIX}-group" \
|
||||
|
|
@ -537,7 +587,17 @@ function kube-up {
|
|||
# to gcloud's deficiency.
|
||||
wait-for-minions-to-run
|
||||
|
||||
# Give the master an initial node list (it's waiting in
|
||||
# startup). This resolves a bit of a chicken-egg issue: The minions
|
||||
# need to know the master's ip, so we boot the master first. The
|
||||
# master still needs to know the initial minion list (until all the
|
||||
# pieces #156 are complete), so we have it wait on the minion
|
||||
# boot. (The minions further wait until the loop below, where CIDRs
|
||||
# get filled in.)
|
||||
detect-minion-names
|
||||
local kube_node_names
|
||||
kube_node_names=$(IFS=,; echo "${MINION_NAMES[*]}")
|
||||
add-instance-metadata "${MASTER_NAME}" "kube-node-names=${kube_node_names}"
|
||||
|
||||
# Create the routes and set IP ranges to instance metadata, 5 instances at a time.
|
||||
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||
|
|
@ -550,11 +610,10 @@ function kube-up {
|
|||
fi
|
||||
|
||||
done
|
||||
|
||||
# Wait for last batch of jobs.
|
||||
wait-for-jobs
|
||||
|
||||
detect-master
|
||||
|
||||
# Reserve the master's IP so that it can later be transferred to another VM
|
||||
# without disrupting the kubelets. IPs are associated with regions, not zones,
|
||||
# so extract the region name, which is the same as the zone but with the final
|
||||
|
|
@ -701,6 +760,14 @@ function kube-down {
|
|||
--delete-disks all \
|
||||
--zone "${ZONE}" \
|
||||
"${MASTER_NAME}" || true
|
||||
|
||||
# Delete the master pd (possibly leaked by kube-up if master create failed)
|
||||
gcloud compute disks delete \
|
||||
--project "${PROJECT}" \
|
||||
--quiet \
|
||||
--zone "${ZONE}" \
|
||||
"${MASTER_NAME}"-pd || true
|
||||
|
||||
# Find out what minions are running.
|
||||
local -a minions
|
||||
minions=( $(gcloud compute instances list \
|
||||
|
|
@ -756,27 +823,49 @@ function kube-down {
|
|||
|
||||
# Update a kubernetes cluster with latest source
|
||||
function kube-push {
|
||||
OUTPUT=${KUBE_ROOT}/_output/logs
|
||||
mkdir -p ${OUTPUT}
|
||||
|
||||
ensure-temp-dir
|
||||
detect-project
|
||||
detect-master
|
||||
detect-minion-names
|
||||
get-password
|
||||
set-master-htpasswd
|
||||
|
||||
# Make sure we have the tar files staged on Google Storage
|
||||
find-release-tars
|
||||
upload-server-tars
|
||||
|
||||
(
|
||||
echo "#! /bin/bash"
|
||||
echo "mkdir -p /var/cache/kubernetes-install"
|
||||
echo "cd /var/cache/kubernetes-install"
|
||||
echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'"
|
||||
echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh"
|
||||
grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh"
|
||||
echo "echo Executing configuration"
|
||||
echo "sudo salt '*' mine.update"
|
||||
echo "sudo salt --force-color '*' state.highstate"
|
||||
) | gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "$KUBE_MASTER" --command "sudo bash"
|
||||
write-master-env
|
||||
add-instance-metadata-from-file "${KUBE_MASTER}" "kube-env=${KUBE_TEMP}/master-kube-env.yaml"
|
||||
echo "Pushing to master (log at ${OUTPUT}/kube-push-${KUBE_MASTER}.log) ..."
|
||||
cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${KUBE_MASTER}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-push-"${KUBE_MASTER}".log
|
||||
|
||||
get-password
|
||||
echo "Pushing metadata to minions... "
|
||||
write-node-env
|
||||
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||
add-instance-metadata-from-file "${MINION_NAMES[$i]}" "kube-env=${KUBE_TEMP}/node-kube-env.yaml" &
|
||||
done
|
||||
wait-for-jobs
|
||||
echo "Done"
|
||||
|
||||
for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
|
||||
echo "Starting push to node (log at ${OUTPUT}/kube-push-${MINION_NAMES[$i]}.log) ..."
|
||||
cat ${KUBE_ROOT}/cluster/gce/configure-vm.sh | gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --project "${PROJECT}" --zone "${ZONE}" "${MINION_NAMES[$i]}" --command "sudo bash -s -- --push" &> ${OUTPUT}/kube-push-"${MINION_NAMES[$i]}".log &
|
||||
done
|
||||
|
||||
echo -n "Waiting for node pushes... "
|
||||
wait-for-jobs
|
||||
echo "Done"
|
||||
|
||||
# TODO(zmerlynn): Re-create instance-template with the new
|
||||
# node-kube-env. This isn't important until the node-ip-range issue
|
||||
# is solved (because that's blocking automatic dynamic nodes from
|
||||
# working). The node-kube-env has to be composed with the kube-token
|
||||
# metadata. Ideally we would have
|
||||
# https://github.com/GoogleCloudPlatform/kubernetes/issues/3168
|
||||
# implemented before then, though, so avoiding this mess until then.
|
||||
|
||||
echo
|
||||
echo "Kubernetes cluster is running. The master is running at:"
|
||||
|
|
@ -785,7 +874,6 @@ function kube-push {
|
|||
echo
|
||||
echo "The user name and password to use is located in ~/.kubernetes_auth."
|
||||
echo
|
||||
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
# On GCE, there is no Salt mine. We run standalone.
|
||||
{% if grains.cloud != 'gce' -%}
|
||||
|
||||
# Allow everyone to see cached values of who sits at what IP
|
||||
{% set networkInterfaceName = "eth0" %}
|
||||
{% if grains.networkInterfaceName is defined %}
|
||||
|
|
@ -6,3 +9,5 @@
|
|||
mine_functions:
|
||||
network.ip_addrs: [{{networkInterfaceName}}]
|
||||
grains.items: []
|
||||
|
||||
{% endif -%}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@
|
|||
|
||||
{% if grains.etcd_servers is defined -%}
|
||||
{% set etcd_servers = "--etcd_servers=http://" + grains.etcd_servers + ":4001" -%}
|
||||
{% elif grains.cloud == 'gce' -%}
|
||||
{# TODO(zmerlynn): I can't see why this isn't generally applicable, but making this change surgical for now. #}
|
||||
{% set etcd_servers = "--etcd_servers=http://127.0.0.1:4001" -%}
|
||||
{% else -%}
|
||||
{% set ips = salt['mine.get']('roles:kubernetes-master', 'network.ip_addrs', 'grain').values() -%}
|
||||
{% set etcd_servers = "--etcd_servers=http://" + ips[0][0] + ":4001" -%}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
{% if grains.cloud is defined -%}
|
||||
{% if grains.cloud == 'gce' -%}
|
||||
{% set cloud_provider = "--cloud_provider=gce" -%}
|
||||
{% set machines = "--machines=" + ','.join(salt['mine.get']('roles:kubernetes-pool', 'network.ip_addrs', expr_form='grain').keys()) -%}
|
||||
{% set machines = "--machines=" + pillar['gce_node_names'] -%}
|
||||
{% endif -%}
|
||||
{% if grains.cloud == 'aws' -%}
|
||||
{% set cloud_provider = "--cloud_provider=aws" -%}
|
||||
|
|
|
|||
Loading…
Reference in New Issue