mirror of https://github.com/k3s-io/k3s
Merge pull request #62669 from immutableT/deploy_helper_test
Automatic merge from submit-queue (batch tested with PRs 63007, 62919, 62669, 62860). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Add unit test for configure-helper.sh. **What this PR does / why we need it**: Add a framework for unit-testing configure-helper.sh. configure-helper.sh plays a critical role in initializing clusters both on GCE and GKE. It is currently, over 2K lines of code, yet it has no unit test coverage. This PR proposes a framework/approach on how to provide test coverage for this component. Notes: 1. Changes to configure-helper.sh itself were necessary to enable sourcing of this script for the purposes of testing. 2. As POC api_manifest_test.go covers the logic related to the initialization of apiserver when integration with KMS was requested. The hope is that the same approach could be extended to the rest of the script. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # **Special notes for your reviewer**: **Release note**: ```release-note NONE ```pull/8/head
Kubernetes Submit Queue
GitHub
commit
eea406c108
|
|
@ -28,13 +28,13 @@ gcs_upload(
|
|||
data = [
|
||||
":_binary-artifacts-and-hashes",
|
||||
"//build/release-tars:release-tars-and-hashes",
|
||||
"//cluster/gce:gcs-release-artifacts-and-hashes",
|
||||
"//cluster/gce/gci:gcs-release-artifacts-and-hashes",
|
||||
],
|
||||
tags = ["manual"],
|
||||
upload_paths = {
|
||||
"//:_binary-artifacts-and-hashes": "bin/linux/amd64",
|
||||
"//build/release-tars:release-tars-and-hashes": "",
|
||||
"//cluster/gce:gcs-release-artifacts-and-hashes": "extra/gce",
|
||||
"//cluster/gce/gci:gcs-release-artifacts-and-hashes": "extra/gce",
|
||||
},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,9 +29,9 @@ pkg_tar(
|
|||
package_dir = "kubernetes/gci-trusty",
|
||||
deps = [
|
||||
"//cluster/addons",
|
||||
"//cluster/gce:gce-master-manifests",
|
||||
"//cluster/gce:gci-trusty-manifests",
|
||||
"//cluster/gce/addons",
|
||||
"//cluster/gce/gci:gci-trusty-manifests",
|
||||
"//cluster/gce/manifests:gce-master-manifests",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,17 +3,6 @@ package(default_visibility = ["//visibility:public"])
|
|||
load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
|
||||
load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
|
||||
|
||||
pkg_tar(
|
||||
name = "gci-trusty-manifests",
|
||||
files = {
|
||||
"//cluster/gce/gci/mounter": "gci-mounter",
|
||||
"gci/configure-helper.sh": "gci-configure-helper.sh",
|
||||
"gci/health-monitor.sh": "health-monitor.sh",
|
||||
},
|
||||
mode = "0755",
|
||||
strip_prefix = ".",
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
|
|
@ -26,38 +15,8 @@ filegroup(
|
|||
srcs = [
|
||||
":package-srcs",
|
||||
"//cluster/gce/addons:all-srcs",
|
||||
"//cluster/gce/gci/mounter:all-srcs",
|
||||
"//cluster/gce/gci:all-srcs",
|
||||
"//cluster/gce/manifests:all-srcs",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
)
|
||||
|
||||
# Having the COS code from the GCE cluster deploy hosted with the release is
|
||||
# useful for GKE. This list should match the list in
|
||||
# kubernetes/release/lib/releaselib.sh.
|
||||
release_filegroup(
|
||||
name = "gcs-release-artifacts",
|
||||
srcs = [
|
||||
"gci/configure.sh",
|
||||
"gci/master.yaml",
|
||||
"gci/node.yaml",
|
||||
],
|
||||
)
|
||||
|
||||
pkg_tar(
|
||||
name = "gce-master-manifests",
|
||||
srcs = [
|
||||
"manifests/abac-authz-policy.jsonl",
|
||||
"manifests/cluster-autoscaler.manifest",
|
||||
"manifests/e2e-image-puller.manifest",
|
||||
"manifests/etcd.manifest",
|
||||
"manifests/glbc.manifest",
|
||||
"manifests/kms-plugin-container.manifest",
|
||||
"manifests/kube-addon-manager.yaml",
|
||||
"manifests/kube-apiserver.manifest",
|
||||
"manifests/kube-controller-manager.manifest",
|
||||
"manifests/kube-proxy.manifest",
|
||||
"manifests/kube-scheduler.manifest",
|
||||
"manifests/rescheduler.manifest",
|
||||
],
|
||||
mode = "0644",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_test")
|
||||
load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
|
||||
load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"apiserver_manifest_test.go",
|
||||
"configure_helper_test.go",
|
||||
],
|
||||
data = [
|
||||
":scripts-test-data",
|
||||
"//cluster/gce/manifests:manifests-test-data",
|
||||
],
|
||||
deps = [
|
||||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
# Having the COS code from the GCE cluster deploy hosted with the release is
|
||||
# useful for GKE. This list should match the list in
|
||||
# kubernetes/release/lib/releaselib.sh.
|
||||
release_filegroup(
|
||||
name = "gcs-release-artifacts",
|
||||
srcs = [
|
||||
"configure.sh",
|
||||
"master.yaml",
|
||||
"node.yaml",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
pkg_tar(
|
||||
name = "gci-trusty-manifests",
|
||||
files = {
|
||||
"//cluster/gce/gci/mounter": "gci-mounter",
|
||||
"configure-helper.sh": "gci-configure-helper.sh",
|
||||
"health-monitor.sh": "health-monitor.sh",
|
||||
},
|
||||
mode = "0755",
|
||||
strip_prefix = ".",
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [
|
||||
":package-srcs",
|
||||
"//cluster/gce/gci/mounter:all-srcs",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "scripts-test-data",
|
||||
srcs = [
|
||||
"configure-helper.sh",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package gci
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
/*
|
||||
Template for defining the environment state of configure-helper.sh
|
||||
The environment of configure-helper.sh is initially configured via kube-env file. However, as deploy-helper
|
||||
executes new variables are created. ManifestTestCase does not care where a variable came from. However, future
|
||||
test scenarios, may require such a distinction.
|
||||
The list of variables is, by no means, complete - this is what is required to run currently defined tests.
|
||||
*/
|
||||
deployHelperEnv = `
|
||||
readonly KUBE_HOME={{.KubeHome}}
|
||||
readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log
|
||||
readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log
|
||||
readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf
|
||||
readonly CA_CERT_BUNDLE_PATH=/foo/bar
|
||||
readonly APISERVER_SERVER_CERT_PATH=/foo/bar
|
||||
readonly APISERVER_SERVER_KEY_PATH=/foo/bar
|
||||
readonly APISERVER_CLIENT_CERT_PATH=/foo/bar
|
||||
readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
|
||||
readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
|
||||
readonly DOCKER_REGISTRY="k8s.gcr.io"
|
||||
readonly ENABLE_LEGACY_ABAC=false
|
||||
readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
|
||||
readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
|
||||
readonly LOG_OWNER_USER=$(whoami)
|
||||
readonly LOG_OWNER_GROUP=$(id -gn)
|
||||
ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
|
||||
ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
|
||||
readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}}
|
||||
`
|
||||
kubeAPIServerManifestFileName = "kube-apiserver.manifest"
|
||||
kmsPluginManifestFileName = "kms-plugin-container.manifest"
|
||||
kubeAPIServerStartFuncName = "start-kube-apiserver"
|
||||
|
||||
// Position of containers within a pod manifest
|
||||
kmsPluginContainerIndex = 0
|
||||
apiServerContainerIndexNoKMS = 0
|
||||
apiServerContainerIndexWithKMS = 1
|
||||
|
||||
// command": [
|
||||
// "/bin/sh", - Index 0
|
||||
// "-c", - Index 1
|
||||
// "exec /usr/local/bin/kube-apiserver " - Index 2
|
||||
execArgsIndex = 2
|
||||
|
||||
socketVolumeMountIndexKMSPlugin = 1
|
||||
socketVolumeMountIndexAPIServer = 0
|
||||
)
|
||||
|
||||
type kubeAPIServerEnv struct {
|
||||
KubeHome string
|
||||
EncryptionProviderConfig string
|
||||
EncryptionProviderConfigPath string
|
||||
ETCDKMSKeyID string
|
||||
}
|
||||
|
||||
type kubeAPIServerManifestTestCase struct {
|
||||
*ManifestTestCase
|
||||
apiServerContainer v1.Container
|
||||
kmsPluginContainer v1.Container
|
||||
}
|
||||
|
||||
func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
|
||||
return &kubeAPIServerManifestTestCase{
|
||||
ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubeAPIServerManifestTestCase) mustLoadContainers() {
|
||||
c.mustLoadPodFromManifest()
|
||||
|
||||
switch len(c.pod.Spec.Containers) {
|
||||
case 1:
|
||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS]
|
||||
case 2:
|
||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS]
|
||||
c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex]
|
||||
default:
|
||||
c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) {
|
||||
c.mustInvokeFunc(deployHelperEnv, e)
|
||||
c.mustLoadContainers()
|
||||
}
|
||||
|
||||
func getEncryptionProviderConfigFlag(path string) string {
|
||||
return fmt.Sprintf("--experimental-encryption-provider-config=%s", path)
|
||||
}
|
||||
|
||||
func TestEncryptionProviderFlag(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
||||
if !strings.Contains(execArgs, expectedFlag) {
|
||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptionProviderConfig(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
||||
EncryptionProviderConfigPath: p,
|
||||
}
|
||||
|
||||
c.mustInvokeFunc(deployHelperEnv, e)
|
||||
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce
|
||||
// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag.
|
||||
func TestKMSEncryptionProviderConfig(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
ETCDKMSKeyID: "FOO",
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
||||
if !strings.Contains(execArgs, expectedFlag) {
|
||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
||||
}
|
||||
|
||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
||||
}
|
||||
|
||||
d, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to read encryption provider config %s", p)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(d), "name: grpc-kms-provider") {
|
||||
c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d))
|
||||
}
|
||||
}
|
||||
|
||||
func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
var e = kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
ETCDKMSKeyID: "FOO",
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath
|
||||
a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath
|
||||
|
||||
if k != a {
|
||||
t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s",
|
||||
k, a, k, a)
|
||||
}
|
||||
}
|
||||
|
|
@ -1179,7 +1179,7 @@ EOF
|
|||
function prepare-log-file {
|
||||
touch $1
|
||||
chmod 644 $1
|
||||
chown root:root $1
|
||||
chown "${LOG_OWNER_USER:-root}":"${LOG_OWNER_GROUP:-root}" $1
|
||||
}
|
||||
|
||||
# Prepares parameters for kube-proxy manifest.
|
||||
|
|
@ -1404,8 +1404,8 @@ function prepare-mounter-rootfs {
|
|||
# DOCKER_REGISTRY
|
||||
function start-kube-apiserver {
|
||||
echo "Start kubernetes api-server"
|
||||
prepare-log-file /var/log/kube-apiserver.log
|
||||
prepare-log-file /var/log/kube-apiserver-audit.log
|
||||
prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}"
|
||||
prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}"
|
||||
|
||||
# Calculate variables and assemble the command line.
|
||||
local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
|
||||
|
|
@ -1702,14 +1702,14 @@ EOM
|
|||
fi
|
||||
|
||||
if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
|
||||
local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml"
|
||||
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
|
||||
params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
|
||||
ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}"
|
||||
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
||||
params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
||||
fi
|
||||
|
||||
src_file="${src_dir}/kube-apiserver.manifest"
|
||||
# Evaluate variables.
|
||||
local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
|
||||
local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
|
||||
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
||||
sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
|
||||
sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
||||
|
|
@ -1756,8 +1756,8 @@ EOM
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${encryption_provider_config_path}" ]]; then
|
||||
echo "Error: KMS Integration was requested, but "${encryption_provider_config_path}" is missing."
|
||||
if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then
|
||||
echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -1771,8 +1771,8 @@ EOM
|
|||
local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}"
|
||||
local kms_path_to_socket="${kms_socket_dir}/socket.sock"
|
||||
|
||||
local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}"
|
||||
local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}"
|
||||
local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}"
|
||||
local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}"
|
||||
|
||||
# TODO these are used in other places, convert to global.
|
||||
local gce_conf_path="/etc/gce.conf"
|
||||
|
|
@ -1797,7 +1797,7 @@ EOM
|
|||
} " "${src_file}"
|
||||
fi
|
||||
|
||||
cp "${src_file}" /etc/kubernetes/manifests
|
||||
cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
|
||||
}
|
||||
|
||||
# Starts kubernetes controller manager.
|
||||
|
|
@ -2488,95 +2488,104 @@ EOF
|
|||
}
|
||||
|
||||
########### Main Function ###########
|
||||
echo "Start to configure instance for kubernetes"
|
||||
function main() {
|
||||
echo "Start to configure instance for kubernetes"
|
||||
|
||||
KUBE_HOME="/home/kubernetes"
|
||||
CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
|
||||
PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
|
||||
KUBE_HOME="/home/kubernetes"
|
||||
CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
|
||||
PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
|
||||
|
||||
if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
|
||||
echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source "${KUBE_HOME}/kube-env"
|
||||
|
||||
|
||||
if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
|
||||
echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
|
||||
KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
|
||||
fi
|
||||
|
||||
if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
|
||||
source "${KUBE_HOME}/kube-master-certs"
|
||||
fi
|
||||
|
||||
if [[ -n "${KUBE_USER:-}" ]]; then
|
||||
if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
|
||||
echo "Bad KUBE_USER format."
|
||||
if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
|
||||
echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# generate the controller manager and scheduler tokens here since they are only used on the master.
|
||||
KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
source "${KUBE_HOME}/kube-env"
|
||||
|
||||
setup-os-params
|
||||
config-ip-firewall
|
||||
create-dirs
|
||||
setup-kubelet-dir
|
||||
ensure-local-ssds
|
||||
setup-logrotate
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
mount-master-pd
|
||||
create-node-pki
|
||||
create-master-pki
|
||||
create-master-auth
|
||||
create-master-kubelet-auth
|
||||
create-master-etcd-auth
|
||||
override-pv-recycler
|
||||
|
||||
if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
|
||||
echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
|
||||
KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
|
||||
fi
|
||||
|
||||
if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
|
||||
source "${KUBE_HOME}/kube-master-certs"
|
||||
fi
|
||||
|
||||
if [[ -n "${KUBE_USER:-}" ]]; then
|
||||
if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
|
||||
echo "Bad KUBE_USER format."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# generate the controller manager and scheduler tokens here since they are only used on the master.
|
||||
KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
|
||||
setup-os-params
|
||||
config-ip-firewall
|
||||
create-dirs
|
||||
setup-kubelet-dir
|
||||
ensure-local-ssds
|
||||
setup-logrotate
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
mount-master-pd
|
||||
create-node-pki
|
||||
create-master-pki
|
||||
create-master-auth
|
||||
create-master-kubelet-auth
|
||||
create-master-etcd-auth
|
||||
override-pv-recycler
|
||||
else
|
||||
create-node-pki
|
||||
create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
create-kubeproxy-user-kubeconfig
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
create-node-problem-detector-kubeconfig
|
||||
fi
|
||||
fi
|
||||
|
||||
override-kubectl
|
||||
# Run the containerized mounter once to pre-cache the container image.
|
||||
if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
|
||||
assemble-docker-flags
|
||||
fi
|
||||
start-kubelet
|
||||
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
compute-master-manifest-variables
|
||||
start-etcd-servers
|
||||
start-etcd-empty-dir-cleanup-pod
|
||||
start-kube-apiserver
|
||||
start-kube-controller-manager
|
||||
start-kube-scheduler
|
||||
start-kube-addons
|
||||
start-cluster-autoscaler
|
||||
start-lb-controller
|
||||
start-rescheduler
|
||||
else
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
start-kube-proxy
|
||||
fi
|
||||
if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then
|
||||
start-image-puller
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
start-node-problem-detector
|
||||
fi
|
||||
fi
|
||||
reset-motd
|
||||
prepare-mounter-rootfs
|
||||
modprobe configs
|
||||
echo "Done for the configuration for kubernetes"
|
||||
}
|
||||
|
||||
# use --source-only to test functions defined in this script.
|
||||
if [[ "$#" -eq 1 && "${1}" == "--source-only" ]]; then
|
||||
:
|
||||
else
|
||||
create-node-pki
|
||||
create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
create-kubeproxy-user-kubeconfig
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
create-node-problem-detector-kubeconfig
|
||||
fi
|
||||
main "${@}"
|
||||
fi
|
||||
|
||||
override-kubectl
|
||||
# Run the containerized mounter once to pre-cache the container image.
|
||||
if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
|
||||
assemble-docker-flags
|
||||
fi
|
||||
start-kubelet
|
||||
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
compute-master-manifest-variables
|
||||
start-etcd-servers
|
||||
start-etcd-empty-dir-cleanup-pod
|
||||
start-kube-apiserver
|
||||
start-kube-controller-manager
|
||||
start-kube-scheduler
|
||||
start-kube-addons
|
||||
start-cluster-autoscaler
|
||||
start-lb-controller
|
||||
start-rescheduler
|
||||
else
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
start-kube-proxy
|
||||
fi
|
||||
if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then
|
||||
start-image-puller
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
start-node-problem-detector
|
||||
fi
|
||||
fi
|
||||
reset-motd
|
||||
prepare-mounter-rootfs
|
||||
modprobe configs
|
||||
echo "Done for the configuration for kubernetes"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,172 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package gci
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
)
|
||||
|
||||
const (
|
||||
envScriptFileName = "kube-env"
|
||||
configureHelperScriptName = "configure-helper.sh"
|
||||
)
|
||||
|
||||
type ManifestTestCase struct {
|
||||
pod v1.Pod
|
||||
envScriptPath string
|
||||
manifest string
|
||||
auxManifests []string
|
||||
kubeHome string
|
||||
manifestSources string
|
||||
manifestDestination string
|
||||
manifestTemplateDir string
|
||||
manifestTemplate string
|
||||
manifestFuncName string
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func newManifestTestCase(t *testing.T, manifest, funcName string, auxManifests []string) *ManifestTestCase {
|
||||
c := &ManifestTestCase{
|
||||
t: t,
|
||||
manifest: manifest,
|
||||
auxManifests: auxManifests,
|
||||
manifestFuncName: funcName,
|
||||
}
|
||||
|
||||
d, err := ioutil.TempDir("", "configure-helper-test")
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to create temp directory: %v", err)
|
||||
}
|
||||
|
||||
c.kubeHome = d
|
||||
c.envScriptPath = filepath.Join(c.kubeHome, envScriptFileName)
|
||||
c.manifestSources = filepath.Join(c.kubeHome, "kube-manifests", "kubernetes", "gci-trusty")
|
||||
|
||||
currentPath, err := os.Getwd()
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to get current directory: %v", err)
|
||||
}
|
||||
gceDir := filepath.Dir(currentPath)
|
||||
c.manifestTemplateDir = filepath.Join(gceDir, "manifests")
|
||||
c.manifestTemplate = filepath.Join(c.manifestTemplateDir, c.manifest)
|
||||
c.manifestDestination = filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests", c.manifest)
|
||||
|
||||
c.mustCopyFromTemplate()
|
||||
c.mustCopyAuxFromTemplate()
|
||||
c.mustCreateManifestDstDir()
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCopyFromTemplate() {
|
||||
if err := os.MkdirAll(c.manifestSources, os.ModePerm); err != nil {
|
||||
c.t.Fatalf("Failed to create source directory: %v", err)
|
||||
}
|
||||
|
||||
if err := copyFile(c.manifestTemplate, filepath.Join(c.manifestSources, c.manifest)); err != nil {
|
||||
c.t.Fatalf("Failed to copy source manifest to KUBE_HOME: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCopyAuxFromTemplate() {
|
||||
for _, m := range c.auxManifests {
|
||||
err := copyFile(filepath.Join(c.manifestTemplateDir, m), filepath.Join(c.manifestSources, m))
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to copy source manifest %s to KUBE_HOME: %v", m, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCreateManifestDstDir() {
|
||||
p := filepath.Join(filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests"))
|
||||
if err := os.MkdirAll(p, os.ModePerm); err != nil {
|
||||
c.t.Fatalf("Failed to create designation folder for kube-apiserver.manifest: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) {
|
||||
f, err := os.Create(filepath.Join(c.kubeHome, envScriptFileName))
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to create envScript: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
t := template.Must(template.New("env").Parse(envTemplate))
|
||||
|
||||
if err = t.Execute(f, env); err != nil {
|
||||
c.t.Fatalf("Failed to execute template: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustInvokeFunc(envTemplate string, env interface{}) {
|
||||
c.mustCreateEnv(envTemplate, env)
|
||||
args := fmt.Sprintf("source %s ; source %s --source-only ; %s", c.envScriptPath, configureHelperScriptName, c.manifestFuncName)
|
||||
cmd := exec.Command("bash", "-c", args)
|
||||
|
||||
bs, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
c.t.Logf("%s", bs)
|
||||
c.t.Fatalf("Failed to run configure-helper.sh: %v", err)
|
||||
}
|
||||
c.t.Logf("%s", string(bs))
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustLoadPodFromManifest() {
|
||||
json, err := ioutil.ReadFile(c.manifestDestination)
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to read manifest: %s, %v", c.manifestDestination, err)
|
||||
}
|
||||
|
||||
if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
|
||||
c.t.Fatalf("Failed to decode manifest: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) tearDown() {
|
||||
os.RemoveAll(c.kubeHome)
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) (err error) {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
cerr := out.Close()
|
||||
if cerr == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
_, err = io.Copy(out, in)
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
|
||||
load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
|
||||
|
||||
pkg_tar(
|
||||
name = "gce-master-manifests",
|
||||
srcs = [
|
||||
"abac-authz-policy.jsonl",
|
||||
"cluster-autoscaler.manifest",
|
||||
"e2e-image-puller.manifest",
|
||||
"etcd.manifest",
|
||||
"glbc.manifest",
|
||||
"kms-plugin-container.manifest",
|
||||
"kube-addon-manager.yaml",
|
||||
"kube-apiserver.manifest",
|
||||
"kube-controller-manager.manifest",
|
||||
"kube-proxy.manifest",
|
||||
"kube-scheduler.manifest",
|
||||
"rescheduler.manifest",
|
||||
],
|
||||
mode = "0644",
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "manifests-test-data",
|
||||
srcs = [
|
||||
"abac-authz-policy.jsonl",
|
||||
"cluster-autoscaler.manifest",
|
||||
"e2e-image-puller.manifest",
|
||||
"etcd.manifest",
|
||||
"glbc.manifest",
|
||||
"kms-plugin-container.manifest",
|
||||
"kube-addon-manager.yaml",
|
||||
"kube-apiserver.manifest",
|
||||
"kube-controller-manager.manifest",
|
||||
"kube-proxy.manifest",
|
||||
"kube-scheduler.manifest",
|
||||
"rescheduler.manifest",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
Loading…
Reference in New Issue