mirror of https://github.com/k3s-io/k3s
Add unit test for configure-helper.
immutablet
parent
89b56c25f8
commit
dc78d72f04
|
|
@ -28,13 +28,13 @@ gcs_upload(
|
|||
data = [
|
||||
":_binary-artifacts-and-hashes",
|
||||
"//build/release-tars:release-tars-and-hashes",
|
||||
"//cluster/gce:gcs-release-artifacts-and-hashes",
|
||||
"//cluster/gce/gci:gcs-release-artifacts-and-hashes",
|
||||
],
|
||||
tags = ["manual"],
|
||||
upload_paths = {
|
||||
"//:_binary-artifacts-and-hashes": "bin/linux/amd64",
|
||||
"//build/release-tars:release-tars-and-hashes": "",
|
||||
"//cluster/gce:gcs-release-artifacts-and-hashes": "extra/gce",
|
||||
"//cluster/gce/gci:gcs-release-artifacts-and-hashes": "extra/gce",
|
||||
},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,9 +29,9 @@ pkg_tar(
|
|||
package_dir = "kubernetes/gci-trusty",
|
||||
deps = [
|
||||
"//cluster/addons",
|
||||
"//cluster/gce:gce-master-manifests",
|
||||
"//cluster/gce:gci-trusty-manifests",
|
||||
"//cluster/gce/addons",
|
||||
"//cluster/gce/gci:gci-trusty-manifests",
|
||||
"//cluster/gce/manifests:gce-master-manifests",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,17 +3,6 @@ package(default_visibility = ["//visibility:public"])
|
|||
load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
|
||||
load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
|
||||
|
||||
pkg_tar(
|
||||
name = "gci-trusty-manifests",
|
||||
files = {
|
||||
"//cluster/gce/gci/mounter": "gci-mounter",
|
||||
"gci/configure-helper.sh": "gci-configure-helper.sh",
|
||||
"gci/health-monitor.sh": "health-monitor.sh",
|
||||
},
|
||||
mode = "0755",
|
||||
strip_prefix = ".",
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
|
|
@ -26,38 +15,8 @@ filegroup(
|
|||
srcs = [
|
||||
":package-srcs",
|
||||
"//cluster/gce/addons:all-srcs",
|
||||
"//cluster/gce/gci/mounter:all-srcs",
|
||||
"//cluster/gce/gci:all-srcs",
|
||||
"//cluster/gce/manifests:all-srcs",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
)
|
||||
|
||||
# Having the COS code from the GCE cluster deploy hosted with the release is
|
||||
# useful for GKE. This list should match the list in
|
||||
# kubernetes/release/lib/releaselib.sh.
|
||||
release_filegroup(
|
||||
name = "gcs-release-artifacts",
|
||||
srcs = [
|
||||
"gci/configure.sh",
|
||||
"gci/master.yaml",
|
||||
"gci/node.yaml",
|
||||
],
|
||||
)
|
||||
|
||||
pkg_tar(
|
||||
name = "gce-master-manifests",
|
||||
srcs = [
|
||||
"manifests/abac-authz-policy.jsonl",
|
||||
"manifests/cluster-autoscaler.manifest",
|
||||
"manifests/e2e-image-puller.manifest",
|
||||
"manifests/etcd.manifest",
|
||||
"manifests/glbc.manifest",
|
||||
"manifests/kms-plugin-container.manifest",
|
||||
"manifests/kube-addon-manager.yaml",
|
||||
"manifests/kube-apiserver.manifest",
|
||||
"manifests/kube-controller-manager.manifest",
|
||||
"manifests/kube-proxy.manifest",
|
||||
"manifests/kube-scheduler.manifest",
|
||||
"manifests/rescheduler.manifest",
|
||||
],
|
||||
mode = "0644",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_test")
|
||||
load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
|
||||
load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"apiserver_manifest_test.go",
|
||||
"configure_helper_test.go",
|
||||
],
|
||||
data = [
|
||||
":scripts-test-data",
|
||||
"//cluster/gce/manifests:manifests-test-data",
|
||||
],
|
||||
deps = [
|
||||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
# Having the COS code from the GCE cluster deploy hosted with the release is
|
||||
# useful for GKE. This list should match the list in
|
||||
# kubernetes/release/lib/releaselib.sh.
|
||||
release_filegroup(
|
||||
name = "gcs-release-artifacts",
|
||||
srcs = [
|
||||
"configure.sh",
|
||||
"master.yaml",
|
||||
"node.yaml",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
pkg_tar(
|
||||
name = "gci-trusty-manifests",
|
||||
files = {
|
||||
"//cluster/gce/gci/mounter": "gci-mounter",
|
||||
"configure-helper.sh": "gci-configure-helper.sh",
|
||||
"health-monitor.sh": "health-monitor.sh",
|
||||
},
|
||||
mode = "0755",
|
||||
strip_prefix = ".",
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [
|
||||
":package-srcs",
|
||||
"//cluster/gce/gci/mounter:all-srcs",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "scripts-test-data",
|
||||
srcs = [
|
||||
"configure-helper.sh",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package gci
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
/*
|
||||
Template for defining the environment state of configure-helper.sh
|
||||
The environment of configure-helper.sh is initially configured via kube-env file. However, as deploy-helper
|
||||
executes new variables are created. ManifestTestCase does not care where a variable came from. However, future
|
||||
test scenarios, may require such a distinction.
|
||||
The list of variables is, by no means, complete - this is what is required to run currently defined tests.
|
||||
*/
|
||||
deployHelperEnv = `
|
||||
readonly KUBE_HOME={{.KubeHome}}
|
||||
readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log
|
||||
readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log
|
||||
readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf
|
||||
readonly CA_CERT_BUNDLE_PATH=/foo/bar
|
||||
readonly APISERVER_SERVER_CERT_PATH=/foo/bar
|
||||
readonly APISERVER_SERVER_KEY_PATH=/foo/bar
|
||||
readonly APISERVER_CLIENT_CERT_PATH=/foo/bar
|
||||
readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
|
||||
readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
|
||||
readonly DOCKER_REGISTRY="k8s.gcr.io"
|
||||
readonly ENABLE_LEGACY_ABAC=false
|
||||
readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
|
||||
readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
|
||||
readonly LOG_OWNER_USER=$(whoami)
|
||||
readonly LOG_OWNER_GROUP=$(id -gn)
|
||||
ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
|
||||
ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
|
||||
readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}}
|
||||
`
|
||||
kubeAPIServerManifestFileName = "kube-apiserver.manifest"
|
||||
kmsPluginManifestFileName = "kms-plugin-container.manifest"
|
||||
kubeAPIServerStartFuncName = "start-kube-apiserver"
|
||||
|
||||
// Position of containers within a pod manifest
|
||||
kmsPluginContainerIndex = 0
|
||||
apiServerContainerIndexNoKMS = 0
|
||||
apiServerContainerIndexWithKMS = 1
|
||||
|
||||
// command": [
|
||||
// "/bin/sh", - Index 0
|
||||
// "-c", - Index 1
|
||||
// "exec /usr/local/bin/kube-apiserver " - Index 2
|
||||
execArgsIndex = 2
|
||||
|
||||
socketVolumeMountIndexKMSPlugin = 1
|
||||
socketVolumeMountIndexAPIServer = 0
|
||||
)
|
||||
|
||||
type kubeAPIServerEnv struct {
|
||||
KubeHome string
|
||||
EncryptionProviderConfig string
|
||||
EncryptionProviderConfigPath string
|
||||
ETCDKMSKeyID string
|
||||
}
|
||||
|
||||
type kubeAPIServerManifestTestCase struct {
|
||||
*ManifestTestCase
|
||||
apiServerContainer v1.Container
|
||||
kmsPluginContainer v1.Container
|
||||
}
|
||||
|
||||
func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
|
||||
return &kubeAPIServerManifestTestCase{
|
||||
ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubeAPIServerManifestTestCase) mustLoadContainers() {
|
||||
c.mustLoadPodFromManifest()
|
||||
|
||||
switch len(c.pod.Spec.Containers) {
|
||||
case 1:
|
||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS]
|
||||
case 2:
|
||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS]
|
||||
c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex]
|
||||
default:
|
||||
c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) {
|
||||
c.mustInvokeFunc(deployHelperEnv, e)
|
||||
c.mustLoadContainers()
|
||||
}
|
||||
|
||||
func getEncryptionProviderConfigFlag(path string) string {
|
||||
return fmt.Sprintf("--experimental-encryption-provider-config=%s", path)
|
||||
}
|
||||
|
||||
func TestEncryptionProviderFlag(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
||||
if !strings.Contains(execArgs, expectedFlag) {
|
||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptionProviderConfig(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
||||
EncryptionProviderConfigPath: p,
|
||||
}
|
||||
|
||||
c.mustInvokeFunc(deployHelperEnv, e)
|
||||
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce
|
||||
// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag.
|
||||
func TestKMSEncryptionProviderConfig(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
ETCDKMSKeyID: "FOO",
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
||||
if !strings.Contains(execArgs, expectedFlag) {
|
||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
||||
}
|
||||
|
||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
||||
}
|
||||
|
||||
d, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to read encryption provider config %s", p)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(d), "name: grpc-kms-provider") {
|
||||
c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d))
|
||||
}
|
||||
}
|
||||
|
||||
func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
var e = kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
ETCDKMSKeyID: "FOO",
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath
|
||||
a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath
|
||||
|
||||
if k != a {
|
||||
t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s",
|
||||
k, a, k, a)
|
||||
}
|
||||
}
|
||||
|
|
@ -1177,7 +1177,7 @@ EOF
|
|||
function prepare-log-file {
|
||||
touch $1
|
||||
chmod 644 $1
|
||||
chown root:root $1
|
||||
chown "${LOG_OWNER_USER:-root}":"${LOG_OWNER_GROUP:-root}" $1
|
||||
}
|
||||
|
||||
# Prepares parameters for kube-proxy manifest.
|
||||
|
|
@ -1402,8 +1402,8 @@ function prepare-mounter-rootfs {
|
|||
# DOCKER_REGISTRY
|
||||
function start-kube-apiserver {
|
||||
echo "Start kubernetes api-server"
|
||||
prepare-log-file /var/log/kube-apiserver.log
|
||||
prepare-log-file /var/log/kube-apiserver-audit.log
|
||||
prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}"
|
||||
prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}"
|
||||
|
||||
# Calculate variables and assemble the command line.
|
||||
local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
|
||||
|
|
@ -1700,14 +1700,14 @@ EOM
|
|||
fi
|
||||
|
||||
if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
|
||||
local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml"
|
||||
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
|
||||
params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
|
||||
ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}"
|
||||
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
||||
params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
||||
fi
|
||||
|
||||
src_file="${src_dir}/kube-apiserver.manifest"
|
||||
# Evaluate variables.
|
||||
local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
|
||||
local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
|
||||
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
||||
sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
|
||||
sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
||||
|
|
@ -1754,8 +1754,8 @@ EOM
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${encryption_provider_config_path}" ]]; then
|
||||
echo "Error: KMS Integration was requested, but "${encryption_provider_config_path}" is missing."
|
||||
if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then
|
||||
echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -1769,8 +1769,8 @@ EOM
|
|||
local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}"
|
||||
local kms_path_to_socket="${kms_socket_dir}/socket.sock"
|
||||
|
||||
local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}"
|
||||
local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}"
|
||||
local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}"
|
||||
local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}"
|
||||
|
||||
# TODO these are used in other places, convert to global.
|
||||
local gce_conf_path="/etc/gce.conf"
|
||||
|
|
@ -1795,7 +1795,7 @@ EOM
|
|||
} " "${src_file}"
|
||||
fi
|
||||
|
||||
cp "${src_file}" /etc/kubernetes/manifests
|
||||
cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
|
||||
}
|
||||
|
||||
# Starts kubernetes controller manager.
|
||||
|
|
@ -2473,95 +2473,104 @@ EOF
|
|||
}
|
||||
|
||||
########### Main Function ###########
|
||||
echo "Start to configure instance for kubernetes"
|
||||
function main() {
|
||||
echo "Start to configure instance for kubernetes"
|
||||
|
||||
KUBE_HOME="/home/kubernetes"
|
||||
CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
|
||||
PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
|
||||
KUBE_HOME="/home/kubernetes"
|
||||
CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
|
||||
PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
|
||||
|
||||
if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
|
||||
echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source "${KUBE_HOME}/kube-env"
|
||||
|
||||
|
||||
if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
|
||||
echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
|
||||
KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
|
||||
fi
|
||||
|
||||
if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
|
||||
source "${KUBE_HOME}/kube-master-certs"
|
||||
fi
|
||||
|
||||
if [[ -n "${KUBE_USER:-}" ]]; then
|
||||
if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
|
||||
echo "Bad KUBE_USER format."
|
||||
if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
|
||||
echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# generate the controller manager and scheduler tokens here since they are only used on the master.
|
||||
KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
source "${KUBE_HOME}/kube-env"
|
||||
|
||||
setup-os-params
|
||||
config-ip-firewall
|
||||
create-dirs
|
||||
setup-kubelet-dir
|
||||
ensure-local-ssds
|
||||
setup-logrotate
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
mount-master-pd
|
||||
create-node-pki
|
||||
create-master-pki
|
||||
create-master-auth
|
||||
create-master-kubelet-auth
|
||||
create-master-etcd-auth
|
||||
override-pv-recycler
|
||||
|
||||
if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
|
||||
echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
|
||||
KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
|
||||
fi
|
||||
|
||||
if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
|
||||
source "${KUBE_HOME}/kube-master-certs"
|
||||
fi
|
||||
|
||||
if [[ -n "${KUBE_USER:-}" ]]; then
|
||||
if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
|
||||
echo "Bad KUBE_USER format."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# generate the controller manager and scheduler tokens here since they are only used on the master.
|
||||
KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
|
||||
setup-os-params
|
||||
config-ip-firewall
|
||||
create-dirs
|
||||
setup-kubelet-dir
|
||||
ensure-local-ssds
|
||||
setup-logrotate
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
mount-master-pd
|
||||
create-node-pki
|
||||
create-master-pki
|
||||
create-master-auth
|
||||
create-master-kubelet-auth
|
||||
create-master-etcd-auth
|
||||
override-pv-recycler
|
||||
else
|
||||
create-node-pki
|
||||
create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
create-kubeproxy-user-kubeconfig
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
create-node-problem-detector-kubeconfig
|
||||
fi
|
||||
fi
|
||||
|
||||
override-kubectl
|
||||
# Run the containerized mounter once to pre-cache the container image.
|
||||
if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
|
||||
assemble-docker-flags
|
||||
fi
|
||||
start-kubelet
|
||||
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
compute-master-manifest-variables
|
||||
start-etcd-servers
|
||||
start-etcd-empty-dir-cleanup-pod
|
||||
start-kube-apiserver
|
||||
start-kube-controller-manager
|
||||
start-kube-scheduler
|
||||
start-kube-addons
|
||||
start-cluster-autoscaler
|
||||
start-lb-controller
|
||||
start-rescheduler
|
||||
else
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
start-kube-proxy
|
||||
fi
|
||||
if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then
|
||||
start-image-puller
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
start-node-problem-detector
|
||||
fi
|
||||
fi
|
||||
reset-motd
|
||||
prepare-mounter-rootfs
|
||||
modprobe configs
|
||||
echo "Done for the configuration for kubernetes"
|
||||
}
|
||||
|
||||
# use --source-only to test functions defined in this script.
|
||||
if [[ "$#" -eq 1 && "${1}" == "--source-only" ]]; then
|
||||
:
|
||||
else
|
||||
create-node-pki
|
||||
create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
create-kubeproxy-user-kubeconfig
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
create-node-problem-detector-kubeconfig
|
||||
fi
|
||||
fi
|
||||
|
||||
override-kubectl
|
||||
# Run the containerized mounter once to pre-cache the container image.
|
||||
if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
|
||||
assemble-docker-flags
|
||||
fi
|
||||
start-kubelet
|
||||
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
compute-master-manifest-variables
|
||||
start-etcd-servers
|
||||
start-etcd-empty-dir-cleanup-pod
|
||||
start-kube-apiserver
|
||||
start-kube-controller-manager
|
||||
start-kube-scheduler
|
||||
start-kube-addons
|
||||
start-cluster-autoscaler
|
||||
start-lb-controller
|
||||
start-rescheduler
|
||||
else
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
start-kube-proxy
|
||||
fi
|
||||
if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then
|
||||
start-image-puller
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
start-node-problem-detector
|
||||
fi
|
||||
fi
|
||||
reset-motd
|
||||
prepare-mounter-rootfs
|
||||
modprobe configs
|
||||
echo "Done for the configuration for kubernetes"
|
||||
main "${@}"
|
||||
fi
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package gci
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
)
|
||||
|
||||
const (
|
||||
envScriptFileName = "kube-env"
|
||||
configureHelperScriptName = "configure-helper.sh"
|
||||
)
|
||||
|
||||
type ManifestTestCase struct {
|
||||
pod v1.Pod
|
||||
envScriptPath string
|
||||
manifest string
|
||||
auxManifests []string
|
||||
kubeHome string
|
||||
manifestSources string
|
||||
manifestDestination string
|
||||
manifestTemplateDir string
|
||||
manifestTemplate string
|
||||
manifestFuncName string
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func newManifestTestCase(t *testing.T, manifest, funcName string, auxManifests []string) *ManifestTestCase {
|
||||
c := &ManifestTestCase{
|
||||
t: t,
|
||||
manifest: manifest,
|
||||
auxManifests: auxManifests,
|
||||
manifestFuncName: funcName,
|
||||
}
|
||||
|
||||
d, err := ioutil.TempDir("", "configure-helper-test")
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to create temp directory: %v", err)
|
||||
}
|
||||
|
||||
c.kubeHome = d
|
||||
c.envScriptPath = filepath.Join(c.kubeHome, envScriptFileName)
|
||||
c.manifestSources = filepath.Join(c.kubeHome, "kube-manifests", "kubernetes", "gci-trusty")
|
||||
|
||||
currentPath, err := os.Getwd()
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to get current directory: %v", err)
|
||||
}
|
||||
gceDir := filepath.Dir(currentPath)
|
||||
c.manifestTemplateDir = filepath.Join(gceDir, "manifests")
|
||||
c.manifestTemplate = filepath.Join(c.manifestTemplateDir, c.manifest)
|
||||
c.manifestDestination = filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests", c.manifest)
|
||||
|
||||
c.mustCopyFromTemplate()
|
||||
c.mustCopyAuxFromTemplate()
|
||||
c.mustCreateManifestDstDir()
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCopyFromTemplate() {
|
||||
if err := os.MkdirAll(c.manifestSources, os.ModePerm); err != nil {
|
||||
c.t.Fatalf("Failed to create source directory: %v", err)
|
||||
}
|
||||
|
||||
if err := copyFile(c.manifestTemplate, filepath.Join(c.manifestSources, c.manifest)); err != nil {
|
||||
c.t.Fatalf("Failed to copy source manifest to KUBE_HOME: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCopyAuxFromTemplate() {
|
||||
for _, m := range c.auxManifests {
|
||||
err := copyFile(filepath.Join(c.manifestTemplateDir, m), filepath.Join(c.manifestSources, m))
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to copy source manifest %s to KUBE_HOME: %v", m, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCreateManifestDstDir() {
|
||||
p := filepath.Join(filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests"))
|
||||
if err := os.MkdirAll(p, os.ModePerm); err != nil {
|
||||
c.t.Fatalf("Failed to create designation folder for kube-apiserver.manifest: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) {
|
||||
f, err := os.Create(filepath.Join(c.kubeHome, envScriptFileName))
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to create envScript: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
t := template.Must(template.New("env").Parse(envTemplate))
|
||||
|
||||
if err = t.Execute(f, env); err != nil {
|
||||
c.t.Fatalf("Failed to execute template: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustInvokeFunc(envTemplate string, env interface{}) {
|
||||
c.mustCreateEnv(envTemplate, env)
|
||||
args := fmt.Sprintf("source %s ; source %s --source-only ; %s", c.envScriptPath, configureHelperScriptName, c.manifestFuncName)
|
||||
cmd := exec.Command("bash", "-c", args)
|
||||
|
||||
bs, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
c.t.Logf("%s", bs)
|
||||
c.t.Fatalf("Failed to run configure-helper.sh: %v", err)
|
||||
}
|
||||
c.t.Logf("%s", string(bs))
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustLoadPodFromManifest() {
|
||||
json, err := ioutil.ReadFile(c.manifestDestination)
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to read manifest: %s, %v", c.manifestDestination, err)
|
||||
}
|
||||
|
||||
if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
|
||||
c.t.Fatalf("Failed to decode manifest: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) tearDown() {
|
||||
os.RemoveAll(c.kubeHome)
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) (err error) {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
cerr := out.Close()
|
||||
if cerr == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
_, err = io.Copy(out, in)
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
|
||||
load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
|
||||
|
||||
pkg_tar(
|
||||
name = "gce-master-manifests",
|
||||
srcs = [
|
||||
"abac-authz-policy.jsonl",
|
||||
"cluster-autoscaler.manifest",
|
||||
"e2e-image-puller.manifest",
|
||||
"etcd.manifest",
|
||||
"glbc.manifest",
|
||||
"kms-plugin-container.manifest",
|
||||
"kube-addon-manager.yaml",
|
||||
"kube-apiserver.manifest",
|
||||
"kube-controller-manager.manifest",
|
||||
"kube-proxy.manifest",
|
||||
"kube-scheduler.manifest",
|
||||
"rescheduler.manifest",
|
||||
],
|
||||
mode = "0644",
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "manifests-test-data",
|
||||
srcs = [
|
||||
"abac-authz-policy.jsonl",
|
||||
"cluster-autoscaler.manifest",
|
||||
"e2e-image-puller.manifest",
|
||||
"etcd.manifest",
|
||||
"glbc.manifest",
|
||||
"kms-plugin-container.manifest",
|
||||
"kube-addon-manager.yaml",
|
||||
"kube-apiserver.manifest",
|
||||
"kube-controller-manager.manifest",
|
||||
"kube-proxy.manifest",
|
||||
"kube-scheduler.manifest",
|
||||
"rescheduler.manifest",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
Loading…
Reference in New Issue