mirror of https://github.com/k3s-io/k3s
Merge pull request #58023 from mikedanese/rm-libvirt-coreos
Automatic merge from submit-queue (batch tested with PRs 57919, 58021, 58020, 58023). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. removed deprecated libvirt-coreos kube-up/ from cluster This kube-up hasn't been maintained for over a year and has been deprecated for > 6 months. see #49213 @kubernetes/sig-cluster-lifecycle-pr-reviews ```release-note Remove deprecated and unmaintained libvirt-coreos kube-up.sh. ```pull/6/head
Kubernetes Submit Queue
GitHub
commit
78c8249cb3
|
|
@ -1,2 +0,0 @@
|
|||
/libvirt_storage_pool/
|
||||
/coreos_production_qemu_image.img.bz2
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
# Kubernetes CoreOS cluster
|
||||
|
||||
With this tutorial one creates a Kubernetes CoreOS cluster containing of one
|
||||
master and three nodes (workers) running on `192.168.10.1`-`192.168.10.4`.
|
||||
|
||||
For working correctly you need to create the directory addressed as `POOL_PATH` in
|
||||
`util.sh`:
|
||||
```
|
||||
$ sudo mkdir /var/lib/libvirt/images/kubernetes
|
||||
$ sudo chown -R $USER:$USER /var/lib/libvirt/images/kubernetes/
|
||||
```
|
||||
|
||||
Then we follow the instructions in the main `kubernetes` directory.
|
||||
|
||||
For debugging set `export UTIL_SH_DEBUG=1`.
|
||||
```
|
||||
$ export KUBERNETES_PROVIDER=libvirt-coreos
|
||||
$ make release-skip-tests
|
||||
$ ./cluster/kube-up.sh
|
||||
```
|
||||
|
||||
To bring the cluster down again, execute:
|
||||
```
|
||||
$ ./cluster/kube-down.sh
|
||||
```
|
||||
|
||||
Have fun!
|
||||
|
||||
|
||||
|
||||
[]()
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## Contains configuration values for interacting with the libvirt CoreOS cluster
|
||||
|
||||
# Number of minions in the cluster
|
||||
NUM_NODES=${NUM_NODES:-3}
|
||||
export NUM_NODES
|
||||
|
||||
# The IP of the master
|
||||
export MASTER_IP="192.168.10.1"
|
||||
|
||||
export INSTANCE_PREFIX=kubernetes
|
||||
export MASTER_NAME="${INSTANCE_PREFIX}-master"
|
||||
|
||||
# Map out the IPs, names and container subnets of each node
|
||||
export NODE_IP_BASE="192.168.10."
|
||||
NODE_CONTAINER_SUBNET_BASE="10.10"
|
||||
MASTER_CONTAINER_NETMASK="255.255.255.0"
|
||||
MASTER_CONTAINER_ADDR="${NODE_CONTAINER_SUBNET_BASE}.0.1"
|
||||
MASTER_CONTAINER_SUBNET="${NODE_CONTAINER_SUBNET_BASE}.0.1/24"
|
||||
CONTAINER_SUBNET="${NODE_CONTAINER_SUBNET_BASE}.0.0/16"
|
||||
if [[ "$NUM_NODES" -gt 253 ]]; then
|
||||
echo "ERROR: Because of how IPs are allocated in ${BASH_SOURCE}, you cannot create more than 253 nodes"
|
||||
exit 1
|
||||
fi
|
||||
for ((i=0; i < NUM_NODES; i++)) do
|
||||
NODE_IPS[$i]="${NODE_IP_BASE}$((i+2))"
|
||||
NODE_NAMES[$i]="${INSTANCE_PREFIX}-node-$((i+1))"
|
||||
NODE_CONTAINER_SUBNETS[$i]="${NODE_CONTAINER_SUBNET_BASE}.$((i+1)).1/24"
|
||||
NODE_CONTAINER_ADDRS[$i]="${NODE_CONTAINER_SUBNET_BASE}.$((i+1)).1"
|
||||
NODE_CONTAINER_NETMASKS[$i]="255.255.255.0"
|
||||
done
|
||||
NODE_CONTAINER_SUBNETS[$NUM_NODES]=$MASTER_CONTAINER_SUBNET
|
||||
|
||||
SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-10.11.0.0/16}" # formerly PORTAL_NET
|
||||
|
||||
|
||||
# Optional: Enable node logging.
|
||||
ENABLE_NODE_LOGGING=false
|
||||
LOGGING_DESTINATION=elasticsearch
|
||||
|
||||
# Optional: Install cluster DNS.
|
||||
ENABLE_CLUSTER_DNS="${KUBE_ENABLE_CLUSTER_DNS:-true}"
|
||||
DNS_SERVER_IP="${SERVICE_CLUSTER_IP_RANGE%.*}.254"
|
||||
DNS_DOMAIN="cluster.local"
|
||||
|
||||
# Optional: Install cluster registry
|
||||
ENABLE_CLUSTER_REGISTRY="${KUBE_ENABLE_CLUSTER_REGISTRY:-true}"
|
||||
|
||||
# Optional: Enable DNS horizontal autoscaler
|
||||
ENABLE_DNS_HORIZONTAL_AUTOSCALER="${KUBE_ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}"
|
||||
|
||||
#Generate dns files
|
||||
sed -f "${KUBE_ROOT}/cluster/addons/dns/transforms2sed.sed" < "${KUBE_ROOT}/cluster/addons/dns/kube-dns.yaml.base" | sed -f "${KUBE_ROOT}/cluster/libvirt-coreos/forShellEval.sed" > "${KUBE_ROOT}/cluster/libvirt-coreos/kube-dns.yaml"
|
||||
|
||||
|
||||
#Generate registry files
|
||||
sed -f "${KUBE_ROOT}/cluster/libvirt-coreos/forEmptyDirRegistry.sed" < "${KUBE_ROOT}/cluster/addons/registry/registry-rc.yaml" > "${KUBE_ROOT}/cluster/libvirt-coreos/registry-rc.yaml"
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## Contains configuration values for interacting with the Vagrant cluster in test mode
|
||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
||||
source "${KUBE_ROOT}/cluster/libvirt-coreos/config-default.sh"
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
<domain type='kvm'>
|
||||
<name>${name}</name>
|
||||
<memory unit='MiB'>512</memory>
|
||||
<currentMemory unit='MiB'>512</currentMemory>
|
||||
<vcpu placement='static'>2</vcpu>
|
||||
<os>
|
||||
<type arch='x86_64' machine='pc'>hvm</type>
|
||||
<boot dev='hd'/>
|
||||
</os>
|
||||
<features>
|
||||
<acpi/>
|
||||
<apic/>
|
||||
<pae/>
|
||||
</features>
|
||||
<clock offset='utc'/>
|
||||
<on_poweroff>destroy</on_poweroff>
|
||||
<on_reboot>restart</on_reboot>
|
||||
<on_crash>restart</on_crash>
|
||||
<devices>
|
||||
<emulator>$(which qemu-system-$(uname -m))</emulator>
|
||||
<disk type='file' device='disk'>
|
||||
<driver name='qemu' type='qcow2'/>
|
||||
<source file='${POOL_PATH}/${image}'/>
|
||||
<target dev='vda' bus='virtio'/>
|
||||
</disk>
|
||||
<controller type='usb' index='0'>
|
||||
</controller>
|
||||
<filesystem type='mount' accessmode='squash'>
|
||||
<source dir='${POOL_PATH}/${config}'/>
|
||||
<target dir='config-2'/>
|
||||
<readonly/>
|
||||
</filesystem>
|
||||
<filesystem type='mount' accessmode='squash'>
|
||||
<source dir='${kubernetes_dir}'/>
|
||||
<target dir='kubernetes'/>
|
||||
<readonly/>
|
||||
</filesystem>
|
||||
<interface type='network'>
|
||||
<mac address='52:54:00:00:00:${i}'/>
|
||||
<source network='kubernetes_global'/>
|
||||
<model type='virtio'/>
|
||||
</interface>
|
||||
<interface type='network'>
|
||||
<mac address='52:54:00:00:01:${i}'/>
|
||||
<source network='kubernetes_pods'/>
|
||||
<model type='virtio'/>
|
||||
</interface>
|
||||
<serial type='pty'>
|
||||
<target port='0'/>
|
||||
</serial>
|
||||
<console type='pty'>
|
||||
<target type='serial' port='0'/>
|
||||
</console>
|
||||
<channel type='spicevmc'>
|
||||
<target type='virtio' name='com.redhat.spice.0'/>
|
||||
</channel>
|
||||
<input type='tablet' bus='usb'/>
|
||||
<input type='mouse' bus='ps2'/>
|
||||
<input type='keyboard' bus='ps2'/>
|
||||
<graphics type='spice' autoport='yes'/>
|
||||
<sound model='ich6'>
|
||||
</sound>
|
||||
<video>
|
||||
<model type='qxl' vram='9216' heads='1'/>
|
||||
</video>
|
||||
<redirdev bus='usb' type='spicevmc'>
|
||||
</redirdev>
|
||||
<memballoon model='virtio'>
|
||||
</memballoon>
|
||||
</devices>
|
||||
</domain>
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
s/persistentVolumeClaim:/emptyDir: {}/g
|
||||
s/claimName: kube-registry-pvc//g
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
s/\"/\\"/g
|
||||
s/DNS_SERVER_IP/{DNS_SERVER_IP}/g
|
||||
s/DNS_DOMAIN/{DNS_DOMAIN}/g
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: kube-system
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
<network>
|
||||
<name>kubernetes_global</name>
|
||||
<forward mode='nat'>
|
||||
<nat>
|
||||
<port start='1024' end='65535'/>
|
||||
</nat>
|
||||
</forward>
|
||||
<bridge name='virbr_kub_gl' stp='off' delay='0'/>
|
||||
<ip address='192.168.10.254' netmask='255.255.255.0'>
|
||||
</ip>
|
||||
</network>
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
<network>
|
||||
<name>kubernetes_pods</name>
|
||||
<bridge name='virbr_kub_pods' stp='off' delay='0'/>
|
||||
<ip address='10.10.0.100' netmask='255.255.0.0'>
|
||||
</ip>
|
||||
</network>
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
IP.1 = $ENV::WORKER_IP
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
[req]
|
||||
req_extensions = v3_req
|
||||
distinguished_name = req_distinguished_name
|
||||
[req_distinguished_name]
|
||||
[ v3_req ]
|
||||
basicConstraints = CA:FALSE
|
||||
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
|
||||
subjectAltName = @alt_names
|
||||
[alt_names]
|
||||
DNS.1 = kubernetes
|
||||
DNS.2 = kubernetes.default
|
||||
DNS.3 = kubernetes.default.svc
|
||||
DNS.4 = kubernetes.default.svc.cluster.local
|
||||
IP.1 = $ENV::KUBERNETES_SVC
|
||||
IP.2 = $ENV::MASTER_IP
|
||||
|
|
@ -1,116 +0,0 @@
|
|||
#cloud-config
|
||||
|
||||
hostname: ${name}
|
||||
|
||||
ssh_authorized_keys:
|
||||
${ssh_keys}
|
||||
|
||||
write_files:
|
||||
- path: /etc/systemd/journald.conf
|
||||
permissions: 0644
|
||||
content: |
|
||||
[Journal]
|
||||
SystemMaxUse=50M
|
||||
RuntimeMaxUse=50M
|
||||
|
||||
coreos:
|
||||
etcd2:
|
||||
advertise-client-urls: http://${public_ip}:2379
|
||||
initial-advertise-peer-urls: http://${public_ip}:2380
|
||||
listen-client-urls: http://0.0.0.0:2379
|
||||
listen-peer-urls: http://${public_ip}:2380
|
||||
initial-cluster-state: new
|
||||
initial-cluster: ${etcd2_initial_cluster}
|
||||
units:
|
||||
- name: static.network
|
||||
command: start
|
||||
content: |
|
||||
[Match]
|
||||
# Name=eth0
|
||||
MACAddress=52:54:00:00:00:${i}
|
||||
|
||||
[Network]
|
||||
Address=${public_ip}/24
|
||||
DNS=192.168.10.254
|
||||
Gateway=192.168.10.254
|
||||
- name: cbr0.netdev
|
||||
command: start
|
||||
content: |
|
||||
[NetDev]
|
||||
Kind=bridge
|
||||
Name=cbr0
|
||||
- name: cbr0.network
|
||||
command: start
|
||||
content: |
|
||||
[Match]
|
||||
Name=cbr0
|
||||
|
||||
[Network]
|
||||
Address=${NODE_CONTAINER_SUBNETS[$i]}
|
||||
|
||||
[Route]
|
||||
Destination=${CONTAINER_SUBNET}
|
||||
- name: cbr0-interface.network
|
||||
command: start
|
||||
content: |
|
||||
[Match]
|
||||
# Name=eth1
|
||||
MACAddress=52:54:00:00:01:${i}
|
||||
|
||||
[Network]
|
||||
Bridge=cbr0
|
||||
- name: nat.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
Description=NAT non container traffic
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/sbin/iptables -w -t nat -A POSTROUTING -o eth0 -j MASQUERADE ! -d ${CONTAINER_SUBNET}
|
||||
RemainAfterExit=yes
|
||||
Type=oneshot
|
||||
- name: etcd2.service
|
||||
command: start
|
||||
drop-ins:
|
||||
- name: 10-override-name.conf
|
||||
content: |
|
||||
[Service]
|
||||
Environment=ETCD_NAME=%H
|
||||
- name: docker.service
|
||||
command: start
|
||||
drop-ins:
|
||||
- name: 50-opts.conf
|
||||
content: |
|
||||
[Service]
|
||||
Environment='DOCKER_OPTS=--bridge=cbr0 --iptables=false'
|
||||
- name: docker-tcp.socket
|
||||
command: start
|
||||
enable: yes
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Docker Socket for the API
|
||||
|
||||
[Socket]
|
||||
ListenStream=2375
|
||||
BindIPv6Only=both
|
||||
Service=docker.service
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
- name: opt-kubernetes.mount
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
ConditionVirtualization=|vm
|
||||
|
||||
[Mount]
|
||||
What=kubernetes
|
||||
Where=/opt/kubernetes
|
||||
Options=ro,trans=virtio,version=9p2000.L
|
||||
Type=9p
|
||||
update:
|
||||
group: ${COREOS_CHANNEL:-alpha}
|
||||
reboot-strategy: off
|
||||
|
||||
$( [[ ${type} =~ "master" ]] && render-template "$ROOT/user_data_master.yml" )
|
||||
$( [[ ${type} =~ "node" ]] && render-template "$ROOT/user_data_minion.yml" )
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
#cloud-config
|
||||
|
||||
coreos:
|
||||
units:
|
||||
- name: kube-apiserver.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
After=opt-kubernetes.mount etcd2.service
|
||||
ConditionFileIsExecutable=/opt/kubernetes/bin/kube-apiserver
|
||||
Description=Kubernetes API Server
|
||||
Documentation=https://github.com/kubernetes/kubernetes
|
||||
Requires=opt-kubernetes.mount etcd2.service
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/kubernetes/bin/kube-apiserver \
|
||||
--tls-cert-file=./opt/kubernetes/certs/apiserver.pem \
|
||||
--tls-private-key-file=/opt/kubernetes/certs/apiserver-key.pem \
|
||||
--client-ca-file=/opt/kubernetes/certs/ca.pem \
|
||||
--service-account-key-file=/opt/kubernetes/certs/apiserver-key.pem \
|
||||
--service-account-lookup=${SERVICE_ACCOUNT_LOOKUP} \
|
||||
--admission-control=${ADMISSION_CONTROL} \
|
||||
--insecure-bind-address=0.0.0.0 \
|
||||
--insecure-port=8080 \
|
||||
--etcd-servers=http://127.0.0.1:2379 \
|
||||
--kubelet-port=10250 \
|
||||
--v=4 \
|
||||
--service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- name: kube-controller-manager.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
After=opt-kubernetes.mount kube-apiserver.service
|
||||
ConditionFileIsExecutable=/opt/kubernetes/bin/kube-controller-manager
|
||||
Description=Kubernetes Controller Manager
|
||||
Documentation=https://github.com/kubernetes/kubernetes
|
||||
Requires=opt-kubernetes.mount kube-apiserver.service
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/kubernetes/bin/kube-controller-manager \
|
||||
--master=127.0.0.1:8080 \
|
||||
--service-account-private-key-file=/opt/kubernetes/certs/apiserver-key.pem \
|
||||
--root-ca-file=/opt/kubernetes/certs/ca.pem \
|
||||
--v=4
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- name: kube-scheduler.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
After=opt-kubernetes.mount kube-apiserver.service
|
||||
ConditionFileIsExecutable=/opt/kubernetes/bin/kube-scheduler
|
||||
Description=Kubernetes Scheduler
|
||||
Documentation=https://github.com/kubernetes/kubernetes
|
||||
Requires=opt-kubernetes.mount kube-apiserver.service
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/kubernetes/bin/kube-scheduler \
|
||||
--master=127.0.0.1:8080
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- name: kube-addons.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
After=opt-kubernetes.mount kube-apiserver.service
|
||||
ConditionPathIsDirectory=/opt/kubernetes/addons
|
||||
Description=Kubernetes addons
|
||||
Documentation=https://github.com/kubernetes/kubernetes
|
||||
Requires=opt-kubernetes.mount
|
||||
Wants=kube-apiserver.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStartPre=/bin/bash -c 'while [[ \"\$(curl -s http://127.0.0.1:8080/healthz)\" != \"ok\" ]]; do sleep 1; done'
|
||||
ExecStartPre=/bin/sleep 10
|
||||
ExecStart=/opt/kubernetes/bin/kubectl create -f /opt/kubernetes/addons
|
||||
ExecStop=/opt/kubernetes/bin/kubectl delete -f /opt/kubernetes/addons
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
#cloud-config
|
||||
|
||||
coreos:
|
||||
units:
|
||||
- name: kubelet.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
After=opt-kubernetes.mount docker.socket
|
||||
ConditionFileIsExecutable=/opt/kubernetes/bin/kubelet
|
||||
Description=Kubernetes Kubelet
|
||||
Documentation=https://github.com/kubernetes/kubernetes
|
||||
Requires=opt-kubernetes.mount docker.socket
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/kubernetes/bin/kubelet \
|
||||
--address=0.0.0.0 \
|
||||
--hostname-override=${NODE_IPS[$i]} \
|
||||
--cluster-domain=cluster.local \
|
||||
--kubeconfig=/opt/kubernetes/kubeconfig/kubelet.kubeconfig \
|
||||
--tls-cert-file=/opt/kubernetes/certs/${NODE_NAMES[$i]}-node.pem \ \
|
||||
--tls-private-key-file=/opt/kubernetes/certs/${NODE_NAMES[$i]}-node-key.pem \
|
||||
$( [[ "$ENABLE_CLUSTER_DNS" == "true" ]] && echo "--cluster-dns=${DNS_SERVER_IP}" ) \
|
||||
$( [[ "$ENABLE_CLUSTER_DNS" == "true" ]] && echo "--cluster-domain=${DNS_DOMAIN}" ) \
|
||||
--pod-manifest-path=/opt/kubernetes/manifests
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
- name: kube-proxy.service
|
||||
command: start
|
||||
content: |
|
||||
[Unit]
|
||||
After=opt-kubernetes.mount
|
||||
ConditionFileIsExecutable=/opt/kubernetes/bin/kube-proxy
|
||||
Description=Kubernetes Proxy
|
||||
Documentation=https://github.com/kubernetes/kubernetes
|
||||
Requires=opt-kubernetes.mount
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/kubernetes/bin/kube-proxy \
|
||||
--master=http://${MASTER_IP}:8080 \
|
||||
--hostname-override=${NODE_IPS[$i]}
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,459 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2014 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# A library of helper functions that each provider hosting Kubernetes must implement to use cluster/kube-*.sh scripts.
|
||||
|
||||
[ ! -z ${UTIL_SH_DEBUG+x} ] && set -x
|
||||
|
||||
command -v kubectl >/dev/null 2>&1 || { echo >&2 "kubectl not found in path. Aborting."; exit 1; }
|
||||
|
||||
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
|
||||
readonly ROOT=$(dirname "${BASH_SOURCE}")
|
||||
source "$ROOT/${KUBE_CONFIG_FILE:-"config-default.sh"}"
|
||||
source "$KUBE_ROOT/cluster/common.sh"
|
||||
|
||||
export LIBVIRT_DEFAULT_URI=qemu:///system
|
||||
export SERVICE_ACCOUNT_LOOKUP=${SERVICE_ACCOUNT_LOOKUP:-true}
|
||||
export ADMISSION_CONTROL=${ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,PVCProtection,ResourceQuota}
|
||||
readonly POOL=kubernetes
|
||||
readonly POOL_PATH=/var/lib/libvirt/images/kubernetes
|
||||
|
||||
[ ! -d "${POOL_PATH}" ] && (echo "$POOL_PATH" does not exist ; exit 1 )
|
||||
|
||||
# Creates a kubeconfig file for the kubelet.
|
||||
# Args: address (e.g. "http://localhost:8080"), destination file path
|
||||
function create-kubelet-kubeconfig() {
|
||||
local apiserver_address="${1}"
|
||||
local destination="${2}"
|
||||
if [[ -z "${apiserver_address}" ]]; then
|
||||
echo "Must provide API server address to create Kubelet kubeconfig file!"
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "${destination}" ]]; then
|
||||
echo "Must provide destination path to create Kubelet kubeconfig file!"
|
||||
exit 1
|
||||
fi
|
||||
echo "Creating Kubelet kubeconfig file"
|
||||
local dest_dir="$(dirname "${destination}")"
|
||||
mkdir -p "${dest_dir}" &>/dev/null || sudo mkdir -p "${dest_dir}"
|
||||
sudo=$(test -w "${dest_dir}" || echo "sudo -E")
|
||||
cat <<EOF | ${sudo} tee "${destination}" > /dev/null
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
clusters:
|
||||
- cluster:
|
||||
server: ${apiserver_address}
|
||||
name: local
|
||||
contexts:
|
||||
- context:
|
||||
cluster: local
|
||||
name: local
|
||||
current-context: local
|
||||
EOF
|
||||
}
|
||||
|
||||
# join <delim> <list...>
|
||||
# Concatenates the list elements with the delimiter passed as first parameter
|
||||
#
|
||||
# Ex: join , a b c
|
||||
# -> a,b,c
|
||||
function join {
|
||||
local IFS="$1"
|
||||
shift
|
||||
echo "$*"
|
||||
}
|
||||
|
||||
# Must ensure that the following ENV vars are set
|
||||
function detect-master {
|
||||
KUBE_MASTER_IP=$MASTER_IP
|
||||
KUBE_MASTER=$MASTER_NAME
|
||||
export KUBERNETES_MASTER=http://$KUBE_MASTER_IP:8080
|
||||
echo "KUBE_MASTER_IP: $KUBE_MASTER_IP"
|
||||
echo "KUBE_MASTER: $KUBE_MASTER"
|
||||
}
|
||||
|
||||
# Get node IP addresses and store in KUBE_NODE_IP_ADDRESSES[]
|
||||
function detect-nodes {
|
||||
KUBE_NODE_IP_ADDRESSES=("${NODE_IPS[@]}")
|
||||
}
|
||||
|
||||
function generate_certs {
|
||||
node_names=("${@}")
|
||||
#Root-CA
|
||||
tempdir=$(mktemp -d)
|
||||
CA_KEY=${CA_KEY:-"$tempdir/ca-key.pem"}
|
||||
CA_CERT=${CA_CERT:-"$tempdir/ca.pem"}
|
||||
openssl genrsa -out "${CA_KEY}" 2048 2>/dev/null
|
||||
openssl req -x509 -new -nodes -key "${CA_KEY}" -days 10000 -out "${CA_CERT}" -subj "/CN=kube-ca" 2>/dev/null
|
||||
|
||||
#API server key pair
|
||||
KUBE_KEY=${KUBE_KEY:-"$tempdir/apiserver-key.pem"}
|
||||
API_SERVER_CERT_REQ=${API_SERVER_CERT_REQ:-"$tempdir/apiserver.csr"}
|
||||
openssl genrsa -out "${KUBE_KEY}" 2048 2>/dev/null
|
||||
KUBERNETES_SVC=${SERVICE_CLUSTER_IP_RANGE%.*}.1 openssl req -new -key "${KUBE_KEY}" -out "${API_SERVER_CERT_REQ}" -subj "/CN=kube-apiserver" -config cluster/libvirt-coreos/openssl.cnf 2>/dev/null
|
||||
KUBE_CERT=${KUBE_CERT:-"$tempdir/apiserver.pem"}
|
||||
KUBERNETES_SVC=${SERVICE_CLUSTER_IP_RANGE%.*}.1 openssl x509 -req -in "${API_SERVER_CERT_REQ}" -CA "${CA_CERT}" -CAkey "${CA_KEY}" -CAcreateserial -out "${KUBE_CERT}" -days 365 -extensions v3_req -extfile cluster/libvirt-coreos/openssl.cnf 2>/dev/null
|
||||
|
||||
#Copy apiserver and controller tsl assets
|
||||
mkdir -p "$POOL_PATH/kubernetes/certs"
|
||||
cp "${KUBE_CERT}" "$POOL_PATH/kubernetes/certs"
|
||||
cp "${KUBE_KEY}" "$POOL_PATH/kubernetes/certs"
|
||||
cp "${CA_CERT}" "$POOL_PATH/kubernetes/certs"
|
||||
|
||||
#Generate nodes certificate
|
||||
for (( i = 0 ; i < $NUM_NODES ; i++ )); do
|
||||
openssl genrsa -out $tempdir/${node_names[$i]}-node-key.pem 2048 2>/dev/null
|
||||
cp "$tempdir/${node_names[$i]}-node-key.pem" "$POOL_PATH/kubernetes/certs"
|
||||
WORKER_IP=${NODE_IPS[$i]} openssl req -new -key $tempdir/${node_names[$i]}-node-key.pem -out $tempdir/${node_names[$i]}-node.csr -subj "/CN=${node_names[$i]}" -config cluster/libvirt-coreos/node-openssl.cnf 2>/dev/null
|
||||
WORKER_IP=${NODE_IPS[$i]} openssl x509 -req -in $tempdir/${node_names[$i]}-node.csr -CA "${CA_CERT}" -CAkey "${CA_KEY}" -CAcreateserial -out $tempdir/${node_names[$i]}-node.pem -days 365 -extensions v3_req -extfile cluster/libvirt-coreos/node-openssl.cnf 2>/dev/null
|
||||
cp "$tempdir/${node_names[$i]}-node.pem" "$POOL_PATH/kubernetes/certs"
|
||||
done
|
||||
echo "TLS assets generated..."
|
||||
}
|
||||
|
||||
#Setup registry proxy
|
||||
function setup_registry_proxy {
|
||||
if [[ "$ENABLE_CLUSTER_REGISTRY" == "true" ]]; then
|
||||
cp "./cluster/saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml" "$POOL_PATH/kubernetes/manifests"
|
||||
fi
|
||||
}
|
||||
|
||||
# Verify prereqs on host machine
|
||||
function verify-prereqs {
|
||||
if ! which virsh >/dev/null; then
|
||||
echo "Can't find virsh in PATH, please fix and retry." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! virsh nodeinfo >/dev/null; then
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$(</sys/kernel/mm/ksm/run)" -ne "1" ]]; then
|
||||
echo "KSM is not enabled" >&2
|
||||
echo "Enabling it would reduce the memory footprint of large clusters" >&2
|
||||
if [[ -t 0 ]]; then
|
||||
read -t 5 -n 1 -p "Do you want to enable KSM (requires root password) (y/n)? " answer
|
||||
echo ""
|
||||
if [[ "$answer" == 'y' ]]; then
|
||||
su -c 'echo 1 > /sys/kernel/mm/ksm/run'
|
||||
fi
|
||||
else
|
||||
echo "You can enable it with (as root):" >&2
|
||||
echo "" >&2
|
||||
echo " echo 1 > /sys/kernel/mm/ksm/run" >&2
|
||||
echo "" >&2
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Destroy the libvirt storage pool and all the images inside
|
||||
#
|
||||
# If 'keep_base_image' is passed as first parameter,
|
||||
# the base image is kept, as well as the storage pool.
|
||||
# All the other images are deleted.
|
||||
function destroy-pool {
|
||||
virsh pool-info $POOL >/dev/null 2>&1 || return
|
||||
|
||||
rm -rf "$POOL_PATH"/kubernetes/*
|
||||
rm -rf "$POOL_PATH"/kubernetes_config*/*
|
||||
local vol
|
||||
virsh vol-list $POOL | awk 'NR>2 && !/^$/ && $1 ~ /^kubernetes/ {print $1}' | \
|
||||
while read vol; do
|
||||
virsh vol-delete $vol --pool $POOL
|
||||
done
|
||||
|
||||
[[ "$1" == 'keep_base_image' ]] && return
|
||||
|
||||
set +e
|
||||
virsh vol-delete coreos_base.img --pool $POOL
|
||||
virsh pool-destroy $POOL
|
||||
rmdir "$POOL_PATH"
|
||||
set -e
|
||||
}
|
||||
|
||||
# Creates the libvirt storage pool and populate it with
|
||||
# - the CoreOS base image
|
||||
# - the kubernetes binaries
|
||||
function initialize-pool {
|
||||
mkdir -p "$POOL_PATH"
|
||||
if ! virsh pool-info $POOL >/dev/null 2>&1; then
|
||||
virsh pool-create-as $POOL dir --target "$POOL_PATH"
|
||||
fi
|
||||
|
||||
wget -N -P "$ROOT" https://${COREOS_CHANNEL:-alpha}.release.core-os.net/amd64-usr/current/coreos_production_qemu_image.img.bz2
|
||||
if [[ "$ROOT/coreos_production_qemu_image.img.bz2" -nt "$POOL_PATH/coreos_base.img" ]]; then
|
||||
bunzip2 -f -k "$ROOT/coreos_production_qemu_image.img.bz2"
|
||||
virsh vol-delete coreos_base.img --pool $POOL 2> /dev/null || true
|
||||
fi
|
||||
if ! virsh vol-list $POOL | grep -q coreos_base.img; then
|
||||
virsh vol-create-as $POOL coreos_base.img 10G --format qcow2
|
||||
virsh vol-upload coreos_base.img "$ROOT/coreos_production_qemu_image.img" --pool $POOL
|
||||
fi
|
||||
|
||||
mkdir -p "$POOL_PATH/kubernetes"
|
||||
kube-push-internal
|
||||
|
||||
mkdir -p "$POOL_PATH/kubernetes/manifests"
|
||||
if [[ "$ENABLE_NODE_LOGGING" == "true" ]]; then
|
||||
if [[ "$LOGGING_DESTINATION" == "elasticsearch" ]]; then
|
||||
cp "$KUBE_ROOT/cluster/saltbase/salt/fluentd-es/fluentd-es.manifest" "$POOL_PATH/kubernetes/manifests"
|
||||
elif [[ "$LOGGING_DESTINATION" == "gcp" ]]; then
|
||||
cp "$KUBE_ROOT/cluster/saltbase/salt/fluentd-gcp/fluentd-gcp.manifest" "$POOL_PATH/kubernetes/manifests"
|
||||
fi
|
||||
fi
|
||||
|
||||
mkdir -p "$POOL_PATH/kubernetes/addons"
|
||||
if [[ "$ENABLE_CLUSTER_DNS" == "true" ]]; then
|
||||
render-template "$ROOT/namespace.yaml" > "$POOL_PATH/kubernetes/addons/namespace.yaml"
|
||||
render-template "$ROOT/kube-dns.yaml" > "$POOL_PATH/kubernetes/addons/kube-dns.yaml"
|
||||
fi
|
||||
|
||||
virsh pool-refresh $POOL
|
||||
}
|
||||
|
||||
function destroy-network {
|
||||
set +e
|
||||
virsh net-destroy kubernetes_global
|
||||
virsh net-destroy kubernetes_pods
|
||||
set -e
|
||||
}
|
||||
|
||||
function initialize-network {
|
||||
virsh net-create "$ROOT/network_kubernetes_global.xml"
|
||||
virsh net-create "$ROOT/network_kubernetes_pods.xml"
|
||||
}
|
||||
|
||||
function render-template {
|
||||
eval "echo \"$(cat $1)\""
|
||||
}
|
||||
|
||||
function wait-cluster-readiness {
|
||||
echo "Wait for cluster readiness"
|
||||
|
||||
local timeout=120
|
||||
while [[ $timeout -ne 0 ]]; do
|
||||
nb_ready_nodes=$(kubectl get nodes -o go-template="{{range.items}}{{range.status.conditions}}{{.type}}{{end}}:{{end}}" 2>/dev/null | tr ':' '\n' | grep -c Ready || true)
|
||||
echo "Nb ready nodes: $nb_ready_nodes / $NUM_NODES"
|
||||
if [[ "$nb_ready_nodes" -eq "$NUM_NODES" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
timeout=$(($timeout-1))
|
||||
sleep .5
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# Instantiate a kubernetes cluster
|
||||
function kube-up {
|
||||
detect-master
|
||||
detect-nodes
|
||||
initialize-pool keep_base_image
|
||||
generate_certs "${NODE_NAMES[@]}"
|
||||
setup_registry_proxy
|
||||
initialize-network
|
||||
|
||||
readonly ssh_keys="$(cat ~/.ssh/*.pub | sed 's/^/ - /')"
|
||||
readonly kubernetes_dir="$POOL_PATH/kubernetes"
|
||||
|
||||
local i
|
||||
for (( i = 0 ; i <= $NUM_NODES ; i++ )); do
|
||||
if [[ $i -eq $NUM_NODES ]]; then
|
||||
etcd2_initial_cluster[$i]="${MASTER_NAME}=http://${MASTER_IP}:2380"
|
||||
else
|
||||
etcd2_initial_cluster[$i]="${NODE_NAMES[$i]}=http://${NODE_IPS[$i]}:2380"
|
||||
fi
|
||||
done
|
||||
etcd2_initial_cluster=$(join , "${etcd2_initial_cluster[@]}")
|
||||
readonly machines=$(join , "${KUBE_NODE_IP_ADDRESSES[@]}")
|
||||
|
||||
for (( i = 0 ; i <= $NUM_NODES ; i++ )); do
|
||||
if [[ $i -eq $NUM_NODES ]]; then
|
||||
type=master
|
||||
name=$MASTER_NAME
|
||||
public_ip=$MASTER_IP
|
||||
else
|
||||
type=node-$(printf "%02d" $i)
|
||||
name=${NODE_NAMES[$i]}
|
||||
public_ip=${NODE_IPS[$i]}
|
||||
fi
|
||||
image=$name.img
|
||||
config=kubernetes_config_$type
|
||||
|
||||
virsh vol-create-as $POOL $image 10G --format qcow2 --backing-vol coreos_base.img --backing-vol-format qcow2
|
||||
|
||||
mkdir -p "$POOL_PATH/$config/openstack/latest"
|
||||
render-template "$ROOT/user_data.yml" > "$POOL_PATH/$config/openstack/latest/user_data"
|
||||
virsh pool-refresh $POOL
|
||||
|
||||
domain_xml=$(mktemp)
|
||||
render-template $ROOT/coreos.xml > $domain_xml
|
||||
virsh create $domain_xml
|
||||
rm $domain_xml
|
||||
done
|
||||
|
||||
export KUBE_SERVER="http://192.168.10.1:8080"
|
||||
export CONTEXT="libvirt-coreos"
|
||||
create-kubeconfig
|
||||
create-kubelet-kubeconfig "http://${MASTER_IP}:8080" "${POOL_PATH}/kubernetes/kubeconfig/kubelet.kubeconfig"
|
||||
|
||||
wait-cluster-readiness
|
||||
|
||||
echo "Kubernetes cluster is running. The master is running at:"
|
||||
echo
|
||||
echo " http://${KUBE_MASTER_IP}:8080"
|
||||
echo
|
||||
echo "You can control the Kubernetes cluster with: 'kubectl'"
|
||||
echo "You can connect on the master with: 'ssh core@${KUBE_MASTER_IP}'"
|
||||
|
||||
wait-registry-readiness
|
||||
|
||||
}
|
||||
|
||||
function create_registry_rc() {
|
||||
echo " Create registry replication controller"
|
||||
kubectl create -f $ROOT/registry-rc.yaml
|
||||
local timeout=120
|
||||
while [[ $timeout -ne 0 ]]; do
|
||||
phase=$(kubectl get pods -n kube-system -lk8s-app=kube-registry --output='jsonpath={.items..status.phase}')
|
||||
if [ "$phase" = "Running" ]; then
|
||||
return 0
|
||||
fi
|
||||
timeout=$(($timeout-1))
|
||||
sleep .5
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
function create_registry_svc() {
|
||||
echo " Create registry service"
|
||||
kubectl create -f "${KUBE_ROOT}/cluster/addons/registry/registry-svc.yaml"
|
||||
}
|
||||
|
||||
function wait-registry-readiness() {
|
||||
if [[ "$ENABLE_CLUSTER_REGISTRY" != "true" ]]; then
|
||||
return 0
|
||||
fi
|
||||
echo "Wait for registry readiness..."
|
||||
local timeout=120
|
||||
while [[ $timeout -ne 0 ]]; do
|
||||
phase=$(kubectl get namespaces --output=jsonpath='{.items[?(@.metadata.name=="kube-system")].status.phase}')
|
||||
if [ "$phase" = "Active" ]; then
|
||||
create_registry_rc
|
||||
create_registry_svc
|
||||
return 0
|
||||
fi
|
||||
echo "waiting for namespace kube-system"
|
||||
timeout=$(($timeout-1))
|
||||
sleep .5
|
||||
done
|
||||
}
|
||||
|
||||
# Delete a kubernetes cluster
|
||||
function kube-down {
|
||||
virsh list | awk 'NR>2 && !/^$/ && $2 ~ /^kubernetes/ {print $2}' | \
|
||||
while read dom; do
|
||||
virsh destroy $dom
|
||||
done
|
||||
destroy-pool keep_base_image
|
||||
destroy-network
|
||||
}
|
||||
|
||||
# The kubernetes binaries are pushed to a host directory which is exposed to the VM
|
||||
function upload-server-tars {
|
||||
tar -x -C "$POOL_PATH/kubernetes" -f "$SERVER_BINARY_TAR" kubernetes
|
||||
rm -rf "$POOL_PATH/kubernetes/bin"
|
||||
mv "$POOL_PATH/kubernetes/kubernetes/server/bin" "$POOL_PATH/kubernetes/bin"
|
||||
chmod -R 755 "$POOL_PATH/kubernetes/bin"
|
||||
rm -fr "$POOL_PATH/kubernetes/kubernetes"
|
||||
}
|
||||
|
||||
# Update a kubernetes cluster with latest source
|
||||
function kube-push {
|
||||
kube-push-internal
|
||||
ssh-to-node "$MASTER_NAME" "sudo systemctl restart kube-apiserver kube-controller-manager kube-scheduler"
|
||||
for ((i=0; i < NUM_NODES; i++)); do
|
||||
ssh-to-node "${NODE_NAMES[$i]}" "sudo systemctl restart kubelet kube-proxy"
|
||||
done
|
||||
wait-cluster-readiness
|
||||
}
|
||||
|
||||
function kube-push-internal {
|
||||
case "${KUBE_PUSH:-release}" in
|
||||
release)
|
||||
kube-push-release;;
|
||||
local)
|
||||
kube-push-local;;
|
||||
*)
|
||||
echo "The only known push methods are \"release\" to use the release tarball or \"local\" to use the binaries built by make. KUBE_PUSH is set \"$KUBE_PUSH\"" >&2
|
||||
return 1;;
|
||||
esac
|
||||
}
|
||||
|
||||
function kube-push-release {
|
||||
find-release-tars
|
||||
upload-server-tars
|
||||
}
|
||||
|
||||
function kube-push-local {
|
||||
rm -rf "$POOL_PATH/kubernetes/bin/*"
|
||||
mkdir -p "$POOL_PATH/kubernetes/bin"
|
||||
cp "${KUBE_ROOT}/_output/local/go/bin"/* "$POOL_PATH/kubernetes/bin"
|
||||
}
|
||||
|
||||
# Execute prior to running tests to build a release if required for env
|
||||
function test-build-release {
|
||||
echo "TODO"
|
||||
}
|
||||
|
||||
# Execute prior to running tests to initialize required structure
|
||||
function test-setup {
|
||||
"${KUBE_ROOT}/cluster/kube-up.sh"
|
||||
}
|
||||
|
||||
# Execute after running tests to perform any required clean-up
|
||||
function test-teardown {
|
||||
kube-down
|
||||
}
|
||||
|
||||
# SSH to a node by name or IP ($1) and run a command ($2).
|
||||
function ssh-to-node {
|
||||
local node="$1"
|
||||
local cmd="$2"
|
||||
local machine
|
||||
|
||||
if [[ "$node" == "$MASTER_IP" ]] || [[ "$node" =~ ^"$NODE_IP_BASE" ]]; then
|
||||
machine="$node"
|
||||
elif [[ "$node" == "$MASTER_NAME" ]]; then
|
||||
machine="$MASTER_IP"
|
||||
else
|
||||
for ((i=0; i < NUM_NODES; i++)); do
|
||||
if [[ "$node" == "${NODE_NAMES[$i]}" ]]; then
|
||||
machine="${NODE_IPS[$i]}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [[ -z "$machine" ]]; then
|
||||
echo "$node is an unknown machine to ssh to" >&2
|
||||
fi
|
||||
ssh -o ConnectTimeout=30 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ControlMaster=no "core@$machine" "$cmd"
|
||||
}
|
||||
|
||||
# Perform preparations required to run e2e tests
|
||||
function prepare-e2e() {
|
||||
echo "libvirt-coreos doesn't need special preparations for e2e tests" 1>&2
|
||||
}
|
||||
Loading…
Reference in New Issue