github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #44394 from rthallisey/pre-existing-provider 

Automatic merge from submit-queue

Launch kubemark with an existing Kubemark master

In order to expand the use of kubemark, allow developers to use kubemark with a pre-existing Kubernetes cluster.

Ref issue  #44393
pull/6/head
Kubernetes Submit Queue 2017-07-06 04:41:53 -07:00 committed by GitHub
parent 40825b26d1 f9354bf6b7
commit 20e629b1c6
6 changed files with 230 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
cluster/kubemark/pre-existing/config-default.sh Normal file
View File

@ -0,0 +1,39 @@
#!/bin/bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Configuration for landing a Kubemark cluster on a pre-existing Kubernetes
# cluster.
# Pre-existing provider expects a MASTER_IP.
# If you need to specify a port that's not the default (443), add it to MASTER_IP.
#
# Example: Connect to the Master on the secure port 6443
# MASTER_IP=192.168.122.5:6443
#
MASTER_IP="${MASTER_IP:-}"
# The container registry and project given to the kubemark container:
# $CONTAINER_REGISTRY/$PROJECT/kubemark
#
CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-}"
PROJECT="${PROJECT:-}"
NUM_NODES="${NUM_NODES:-1}"
TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-}"
KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-}"
MASTER_NAME="${MASTER_NAME:-}"
USE_REAL_PROXIER="${USE_REAL_PROXIER:-true}"

62
cluster/pre-existing/util.sh Normal file
View File

@ -0,0 +1,62 @@
#!/bin/bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A library of helper functions for landing kubemark containers on a
# pre-existing Kubernetes master. See test/kubemark/pre-existing/README.md
# for me details on using a pre-existing provider.
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/common.sh"
source "${KUBE_ROOT}/hack/lib/util.sh"
function detect-project() {
if [[ -z "${MASTER_IP:-}" ]]; then
echo "Set 'MASTER_IP' to the instance assigned to be the Kubernetes master" 1>&2
exit 1
fi
if [[ -z "${PROJECT:-}" ]]; then
echo "Set 'PROJECT' to the name of the container project: $CONTAINER_REGISTRY/$PROJECT/kubemark" >&2
exit 1
fi
if [[ -z "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
cluster_range=$(echo "${MASTER_IP}" | awk -F '.' '{printf("%d.%d.%d.0", $1, $2, $3)}')
SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-$cluster_range/16}"
fi
}
function create-certs {
rm /tmp/kubeconfig
execute-cmd-on-pre-existing-master-with-retries "sudo cat /etc/kubernetes/admin.conf" > /tmp/kubeconfig
CA_CERT_BASE64=$(cat /tmp/kubeconfig | grep certificate-authority | awk '{print $2}' | head -n 1)
KUBELET_CERT_BASE64=$(cat /tmp/kubeconfig | grep client-certificate-data | awk '{print $2}' | head -n 1)
KUBELET_KEY_BASE64=$(cat /tmp/kubeconfig | grep client-key-data | awk '{print $2}' | head -n 1)
# Local kubeconfig.kubemark vars
KUBECFG_CERT_BASE64="${KUBELET_CERT_BASE64}"
KUBECFG_KEY_BASE64="${KUBELET_KEY_BASE64}"
# The pre-existing Kubernetes master already has these setup
# Set these vars but don't use them
CA_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
MASTER_CERT_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
MASTER_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
KUBEAPISERVER_CERT_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
KUBEAPISERVER_KEY_BASE64=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
}

43
test/kubemark/common/util.sh Normal file
View File

@ -0,0 +1,43 @@
#!/bin/bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Running cmd $RETRIES times in case of failures.
function run-cmd-with-retries {
RETRIES="${RETRIES:-3}"
for attempt in $(seq 1 ${RETRIES}); do
exec 5>&1 # Duplicate &1 to &5 for use below.
# We don't use 'local' to declare result as then ret_val always gets value 0.
# We use tee to output to &5 (redirected to stdout) while also storing it in the variable.
result=$("$@" 2>&1 | tee >(cat - >&5)) || local ret_val="$?"
if [[ "${ret_val:-0}" -ne "0" ]]; then
if [[ $(echo "${result}" | grep -c "already exists") -gt 0 ]]; then
if [[ "${attempt}" == 1 ]]; then
echo -e "${color_red}Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2
exit 1
fi
echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}"
return 0
fi
echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2
sleep $(($attempt * 5))
else
echo -e "${color_green}Succeeded to $1 $2 $3.${color_norm}"
return 0
fi
done
echo -e "${color_red}Failed to $1 $2 $3.${color_norm}" >& 2
exit 1
}

29
test/kubemark/gce/util.sh
View File

@ -14,34 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../..
source "${KUBE_ROOT}/test/kubemark/common/util.sh"
# Wrapper for gcloud compute, running it $RETRIES times in case of failures.
# Args:
# $@: all stuff that goes after 'gcloud compute'
function run-gcloud-compute-with-retries {
RETRIES="${RETRIES:-3}"
for attempt in $(seq 1 ${RETRIES}); do
exec 5>&1 # Duplicate &1 to &5 for use below.
# We don't use 'local' to declare gcloud_result as then ret_val always gets value 0.
# We use tee to output to &5 (redirected to stdout) while also storing it in the variable.
gcloud_result=$(gcloud compute "$@" 2>&1 | tee >(cat - >&5)) || local ret_val="$?"
if [[ "${ret_val:-0}" -ne "0" ]]; then
if [[ $(echo "${gcloud_result}" | grep -c "already exists") -gt 0 ]]; then
if [[ "${attempt}" == 1 ]]; then
echo -e "${color_red}Failed to $1 $2 $3 as the resource hasn't been deleted from a previous run.${color_norm}" >& 2
exit 1
fi
echo -e "${color_yellow}Succeeded to $1 $2 $3 in the previous attempt, but status response wasn't received.${color_norm}"
return 0
fi
echo -e "${color_yellow}Attempt $attempt failed to $1 $2 $3. Retrying.${color_norm}" >& 2
sleep $(($attempt * 5))
else
echo -e "${color_green}Succeeded to gcloud compute $1 $2 $3.${color_norm}"
return 0
fi
done
echo -e "${color_red}Failed to $1 $2 $3.${color_norm}" >& 2
exit 1
run-cmd-with-retries gcloud compute "$@"
}
function create-master-instance-with-resources {

54
test/kubemark/pre-existing/README.md Normal file
View File

@ -0,0 +1,54 @@
# Kubemark Pre-existing Provider Guide
**Kubemark Master**
- A set of Kubernetes control plane components running in a VM
**Kubernetes Cluster**
- A real Kubernetes Cluster that has master and minions. The hollow-node pods
are run in this cluster, but appear as nodes to the Kubemark Master
## Introduction
Every running Kubemark setup looks like the following:
1) A running Kubernetes cluster pointed to by the local kubeconfig
2) A separate VM where the kubemark master is running
3) Some hollow-nodes that run on the Kubernetes Cluster from #1
4) The hollow-nodes are configured to talk with the kubemark master at #2
When using the pre-existing provider, the developer is responsible for creating
#1 and #2. Therefore, the kubemark scripts will not create any infrastructure
or start a kubemark master like in other providers. Instead, the existing
resources provided by the VM at $MASTER_IP will serve as the kubemark master.
## Use Case
The goal of the pre-existing provider is to use the kubemark tools with an
existing kubermark master. It's meant to provide the developer with
additional flexibility to customize the cluster infrastructure and still use
the kubemark setup tools. The pre-existing provider is an **advanced** use
case that requires the developer to have knowledge of setting up a kubemark
master.
## Requirements
To use the pre-existing provider, the expectation is that there's a kubemark
master that is rechable at $MASTER_IP. The machine that the kubemark master is
on has to be ssh able from the host that's executing the kubemark scripts. And
the user on that machine has to be 'kubernetes'.
Requirement checklist:
- Set MASTER_IP to ip address to the kubemark master
- The host where you execute the kubemark scripts must be able to ssh to
kubernetes@$MASTER_IP
## Example Configuration
_test/kubemark/cloud-provider-config.sh_
```
CLOUD_PROVIDER="pre-existing"
KUBEMARK_IMAGE_MAKE_TARGET="push"
CONTAINER_REGISTRY=docker.io
PROJECT="rthallisey"
MASTER_IP="192.168.121.29:6443"
```

27
test/kubemark/pre-existing/util.sh Normal file
View File

@ -0,0 +1,27 @@
#!/bin/bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../../..
source "${KUBE_ROOT}/test/kubemark/common/util.sh"
# Leave the skeleton definition of execute-cmd-on-master-with-retries
# so only the pre-existing provider functions will target this.
function execute-cmd-on-pre-existing-master-with-retries() {
IP_WITHOUT_PORT=$(echo "${MASTER_IP}" | cut -f 1 -d ':') || "${MASTER_IP}"
RETRIES="${2:-1}" run-cmd-with-retries ssh kubernetes@"${IP_WITHOUT_PORT}" $1
}