#!/bin/bash # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # exit on any error set -e KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/cluster/vagrant/provision-config.sh" MINION_IP=$4 MINION_ID=$5 DOCKER_BRIDGE=kbr0 OVS_SWITCH=obr0 GRE_TUNNEL_BASE=gre BRIDGE_BASE=10.244 BRIDGE_ADDRESS=${BRIDGE_BASE}.${MINION_ID}.1 BRIDGE_NETWORK=${BRIDGE_ADDRESS}/24 BRIDGE_NETMASK=255.255.255.0 NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/ POST_NETWORK_SCRIPT=/vagrant/network_closure.sh # add docker bridge ifcfg file cat < ${NETWORK_CONF_PATH}ifcfg-${DOCKER_BRIDGE} # Generated by yours truly DEVICE=${DOCKER_BRIDGE} ONBOOT=yes TYPE=Bridge BOOTPROTO=static IPADDR=${BRIDGE_ADDRESS} NETMASK=${BRIDGE_NETMASK} STP=yes EOF # add the ovs bridge ifcfg file cat < ${NETWORK_CONF_PATH}ifcfg-${OVS_SWITCH} DEVICE=${OVS_SWITCH} ONBOOT=yes DEVICETYPE=ovs TYPE=OVSBridge BOOTPROTO=static HOTPLUG=no BRIDGE=${DOCKER_BRIDGE} EOF # now loop through all other minions and create persistent gre tunnels MINION_IPS=$3 MINION_IP_ARRAY=(`echo ${MINION_IPS} | tr "," "\n"`) GRE_NUM=0 for remote_ip in "${MINION_IP_ARRAY[@]}" do if [ "${remote_ip}" == "${MINION_IP}" ]; then continue fi ((GRE_NUM++)) || echo GRE_TUNNEL=${GRE_TUNNEL_BASE}${GRE_NUM} # ovs-vsctl add-port ${OVS_SWITCH} ${GRE_TUNNEL} -- set interface ${GRE_TUNNEL} type=gre options:remote_ip=${remote_ip} cat < ${NETWORK_CONF_PATH}ifcfg-${GRE_TUNNEL} DEVICE=${GRE_TUNNEL} ONBOOT=yes DEVICETYPE=ovs TYPE=OVSTunnel OVS_BRIDGE=${OVS_SWITCH} OVS_TUNNEL_TYPE=gre OVS_TUNNEL_OPTIONS="options:remote_ip=${remote_ip}" EOF done # add ip route rules such that all pod traffic flows through docker bridge and consequently to the gre tunnels cat < /${NETWORK_CONF_PATH}route-${DOCKER_BRIDGE} ${BRIDGE_BASE}.0.0/16 dev ${DOCKER_BRIDGE} scope link src ${BRIDGE_ADDRESS} EOF # generate the post-configure script to be called by salt as cmd.wait cat < ${POST_NETWORK_SCRIPT} #!/bin/bash set -e # NAT interface fails to revive on network restart, so OR-gate to true systemctl restart network.service || true # set docker bridge up, and set stp on the ovs bridge ip link set dev ${DOCKER_BRIDGE} up ovs-vsctl set Bridge ${OVS_SWITCH} stp_enable=true # modify the docker service file such that it uses the kube docker bridge and not its own sed -ie "s/ExecStart=\/usr\/bin\/docker -d/ExecStart=\/usr\/bin\/docker -d -b=${DOCKER_BRIDGE} --iptables=false/g" /usr/lib/systemd/system/docker.service systemctl daemon-reload systemctl restart docker.service # setup iptables masquerade rules so the pods can reach the internet iptables -t nat -A POSTROUTING -s ${BRIDGE_BASE}.0.0/16 ! -d ${BRIDGE_BASE}.0.0/16 -j MASQUERADE # persist please iptables-save >& /etc/sysconfig/iptables # self-destruct after doing the job #rm -f ${POST_NETWORK_SCRIPT} EOF chmod +x ${POST_NETWORK_SCRIPT}