From dc78d72f04b2273d187d6181e010afd04491b4b1 Mon Sep 17 00:00:00 2001 From: immutablet Date: Thu, 12 Apr 2018 11:14:22 -0700 Subject: [PATCH] Add unit test for configure-helper. --- build/root/BUILD.root | 4 +- cluster/BUILD | 4 +- cluster/gce/BUILD | 45 +---- cluster/gce/gci/BUILD | 69 +++++++ cluster/gce/gci/apiserver_manifest_test.go | 212 +++++++++++++++++++++ cluster/gce/gci/configure-helper.sh | 203 ++++++++++---------- cluster/gce/gci/configure_helper_test.go | 172 +++++++++++++++++ cluster/gce/manifests/BUILD | 55 ++++++ 8 files changed, 620 insertions(+), 144 deletions(-) create mode 100644 cluster/gce/gci/BUILD create mode 100644 cluster/gce/gci/apiserver_manifest_test.go create mode 100644 cluster/gce/gci/configure_helper_test.go create mode 100644 cluster/gce/manifests/BUILD diff --git a/build/root/BUILD.root b/build/root/BUILD.root index 0eb9b61248..1e7cc4853d 100644 --- a/build/root/BUILD.root +++ b/build/root/BUILD.root @@ -28,13 +28,13 @@ gcs_upload( data = [ ":_binary-artifacts-and-hashes", "//build/release-tars:release-tars-and-hashes", - "//cluster/gce:gcs-release-artifacts-and-hashes", + "//cluster/gce/gci:gcs-release-artifacts-and-hashes", ], tags = ["manual"], upload_paths = { "//:_binary-artifacts-and-hashes": "bin/linux/amd64", "//build/release-tars:release-tars-and-hashes": "", - "//cluster/gce:gcs-release-artifacts-and-hashes": "extra/gce", + "//cluster/gce/gci:gcs-release-artifacts-and-hashes": "extra/gce", }, ) diff --git a/cluster/BUILD b/cluster/BUILD index 0f2c264867..0654bb05ee 100644 --- a/cluster/BUILD +++ b/cluster/BUILD @@ -29,9 +29,9 @@ pkg_tar( package_dir = "kubernetes/gci-trusty", deps = [ "//cluster/addons", - "//cluster/gce:gce-master-manifests", - "//cluster/gce:gci-trusty-manifests", "//cluster/gce/addons", + "//cluster/gce/gci:gci-trusty-manifests", + "//cluster/gce/manifests:gce-master-manifests", ], ) diff --git a/cluster/gce/BUILD b/cluster/gce/BUILD index f14a7382b1..37b3f97e0f 100644 --- a/cluster/gce/BUILD +++ b/cluster/gce/BUILD @@ -3,17 +3,6 @@ package(default_visibility = ["//visibility:public"]) load("@io_kubernetes_build//defs:build.bzl", "release_filegroup") load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar") -pkg_tar( - name = "gci-trusty-manifests", - files = { - "//cluster/gce/gci/mounter": "gci-mounter", - "gci/configure-helper.sh": "gci-configure-helper.sh", - "gci/health-monitor.sh": "health-monitor.sh", - }, - mode = "0755", - strip_prefix = ".", -) - filegroup( name = "package-srcs", srcs = glob(["**"]), @@ -26,38 +15,8 @@ filegroup( srcs = [ ":package-srcs", "//cluster/gce/addons:all-srcs", - "//cluster/gce/gci/mounter:all-srcs", + "//cluster/gce/gci:all-srcs", + "//cluster/gce/manifests:all-srcs", ], tags = ["automanaged"], ) - -# Having the COS code from the GCE cluster deploy hosted with the release is -# useful for GKE. This list should match the list in -# kubernetes/release/lib/releaselib.sh. -release_filegroup( - name = "gcs-release-artifacts", - srcs = [ - "gci/configure.sh", - "gci/master.yaml", - "gci/node.yaml", - ], -) - -pkg_tar( - name = "gce-master-manifests", - srcs = [ - "manifests/abac-authz-policy.jsonl", - "manifests/cluster-autoscaler.manifest", - "manifests/e2e-image-puller.manifest", - "manifests/etcd.manifest", - "manifests/glbc.manifest", - "manifests/kms-plugin-container.manifest", - "manifests/kube-addon-manager.yaml", - "manifests/kube-apiserver.manifest", - "manifests/kube-controller-manager.manifest", - "manifests/kube-proxy.manifest", - "manifests/kube-scheduler.manifest", - "manifests/rescheduler.manifest", - ], - mode = "0644", -) diff --git a/cluster/gce/gci/BUILD b/cluster/gce/gci/BUILD new file mode 100644 index 0000000000..a9c43c2108 --- /dev/null +++ b/cluster/gce/gci/BUILD @@ -0,0 +1,69 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_test") +load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar") +load("@io_kubernetes_build//defs:build.bzl", "release_filegroup") + +go_test( + name = "go_default_test", + srcs = [ + "apiserver_manifest_test.go", + "configure_helper_test.go", + ], + data = [ + ":scripts-test-data", + "//cluster/gce/manifests:manifests-test-data", + ], + deps = [ + "//pkg/api/legacyscheme:go_default_library", + "//vendor/k8s.io/api/core/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + ], +) + +# Having the COS code from the GCE cluster deploy hosted with the release is +# useful for GKE. This list should match the list in +# kubernetes/release/lib/releaselib.sh. +release_filegroup( + name = "gcs-release-artifacts", + srcs = [ + "configure.sh", + "master.yaml", + "node.yaml", + ], + visibility = ["//visibility:public"], +) + +pkg_tar( + name = "gci-trusty-manifests", + files = { + "//cluster/gce/gci/mounter": "gci-mounter", + "configure-helper.sh": "gci-configure-helper.sh", + "health-monitor.sh": "health-monitor.sh", + }, + mode = "0755", + strip_prefix = ".", + visibility = ["//visibility:public"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [ + ":package-srcs", + "//cluster/gce/gci/mounter:all-srcs", + ], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "scripts-test-data", + srcs = [ + "configure-helper.sh", + ], +) diff --git a/cluster/gce/gci/apiserver_manifest_test.go b/cluster/gce/gci/apiserver_manifest_test.go new file mode 100644 index 0000000000..5b2d570573 --- /dev/null +++ b/cluster/gce/gci/apiserver_manifest_test.go @@ -0,0 +1,212 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gci + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/api/core/v1" +) + +const ( + /* + Template for defining the environment state of configure-helper.sh + The environment of configure-helper.sh is initially configured via kube-env file. However, as deploy-helper + executes new variables are created. ManifestTestCase does not care where a variable came from. However, future + test scenarios, may require such a distinction. + The list of variables is, by no means, complete - this is what is required to run currently defined tests. + */ + deployHelperEnv = ` +readonly KUBE_HOME={{.KubeHome}} +readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log +readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log +readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf +readonly CA_CERT_BUNDLE_PATH=/foo/bar +readonly APISERVER_SERVER_CERT_PATH=/foo/bar +readonly APISERVER_SERVER_KEY_PATH=/foo/bar +readonly APISERVER_CLIENT_CERT_PATH=/foo/bar +readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}," +readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}}," +readonly DOCKER_REGISTRY="k8s.gcr.io" +readonly ENABLE_LEGACY_ABAC=false +readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests +readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty +readonly LOG_OWNER_USER=$(whoami) +readonly LOG_OWNER_GROUP=$(id -gn) +ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}} +ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}} +readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}} +` + kubeAPIServerManifestFileName = "kube-apiserver.manifest" + kmsPluginManifestFileName = "kms-plugin-container.manifest" + kubeAPIServerStartFuncName = "start-kube-apiserver" + + // Position of containers within a pod manifest + kmsPluginContainerIndex = 0 + apiServerContainerIndexNoKMS = 0 + apiServerContainerIndexWithKMS = 1 + + // command": [ + // "/bin/sh", - Index 0 + // "-c", - Index 1 + // "exec /usr/local/bin/kube-apiserver " - Index 2 + execArgsIndex = 2 + + socketVolumeMountIndexKMSPlugin = 1 + socketVolumeMountIndexAPIServer = 0 +) + +type kubeAPIServerEnv struct { + KubeHome string + EncryptionProviderConfig string + EncryptionProviderConfigPath string + ETCDKMSKeyID string +} + +type kubeAPIServerManifestTestCase struct { + *ManifestTestCase + apiServerContainer v1.Container + kmsPluginContainer v1.Container +} + +func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase { + return &kubeAPIServerManifestTestCase{ + ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}), + } +} + +func (c *kubeAPIServerManifestTestCase) mustLoadContainers() { + c.mustLoadPodFromManifest() + + switch len(c.pod.Spec.Containers) { + case 1: + c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS] + case 2: + c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS] + c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex] + default: + c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers)) + } +} + +func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) { + c.mustInvokeFunc(deployHelperEnv, e) + c.mustLoadContainers() +} + +func getEncryptionProviderConfigFlag(path string) string { + return fmt.Sprintf("--experimental-encryption-provider-config=%s", path) +} + +func TestEncryptionProviderFlag(t *testing.T) { + c := newKubeAPIServerManifestTestCase(t) + defer c.tearDown() + + e := kubeAPIServerEnv{ + KubeHome: c.kubeHome, + EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")), + EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"), + } + + c.invokeTest(e) + + expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath) + execArgs := c.apiServerContainer.Command[execArgsIndex] + if !strings.Contains(execArgs, expectedFlag) { + c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag) + } +} + +func TestEncryptionProviderConfig(t *testing.T) { + c := newKubeAPIServerManifestTestCase(t) + defer c.tearDown() + + p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml") + e := kubeAPIServerEnv{ + KubeHome: c.kubeHome, + EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")), + EncryptionProviderConfigPath: p, + } + + c.mustInvokeFunc(deployHelperEnv, e) + + if _, err := os.Stat(p); err != nil { + c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err) + } +} + +// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce +// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag. +func TestKMSEncryptionProviderConfig(t *testing.T) { + c := newKubeAPIServerManifestTestCase(t) + defer c.tearDown() + + e := kubeAPIServerEnv{ + KubeHome: c.kubeHome, + EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"), + ETCDKMSKeyID: "FOO", + } + + c.invokeTest(e) + + expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath) + execArgs := c.apiServerContainer.Command[execArgsIndex] + if !strings.Contains(execArgs, expectedFlag) { + c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag) + } + + p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml") + if _, err := os.Stat(p); err != nil { + c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err) + } + + d, err := ioutil.ReadFile(p) + if err != nil { + c.t.Fatalf("Failed to read encryption provider config %s", p) + } + + if !strings.Contains(string(d), "name: grpc-kms-provider") { + c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d)) + } +} + +func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) { + c := newKubeAPIServerManifestTestCase(t) + defer c.tearDown() + + var e = kubeAPIServerEnv{ + KubeHome: c.kubeHome, + EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"), + ETCDKMSKeyID: "FOO", + } + + c.invokeTest(e) + + k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath + a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath + + if k != a { + t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s", + k, a, k, a) + } +} diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index b118dd431f..4ad2f122a9 100644 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1177,7 +1177,7 @@ EOF function prepare-log-file { touch $1 chmod 644 $1 - chown root:root $1 + chown "${LOG_OWNER_USER:-root}":"${LOG_OWNER_GROUP:-root}" $1 } # Prepares parameters for kube-proxy manifest. @@ -1402,8 +1402,8 @@ function prepare-mounter-rootfs { # DOCKER_REGISTRY function start-kube-apiserver { echo "Start kubernetes api-server" - prepare-log-file /var/log/kube-apiserver.log - prepare-log-file /var/log/kube-apiserver-audit.log + prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}" + prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}" # Calculate variables and assemble the command line. local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}" @@ -1700,14 +1700,14 @@ EOM fi if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then - local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml" - echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}" - params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}" + ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}" + echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}" + params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}" fi src_file="${src_dir}/kube-apiserver.manifest" # Evaluate variables. - local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag) + local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}" sed -i -e "s@{{params}}@${params}@g" "${src_file}" sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file} sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}" @@ -1754,8 +1754,8 @@ EOM exit 1 fi - if [[ ! -f "${encryption_provider_config_path}" ]]; then - echo "Error: KMS Integration was requested, but "${encryption_provider_config_path}" is missing." + if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then + echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing." exit 1 fi @@ -1769,8 +1769,8 @@ EOM local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}" local kms_path_to_socket="${kms_socket_dir}/socket.sock" - local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${encryption_provider_config_path}\", \"readOnly\": true}" - local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${encryption_provider_config_path}\", \"type\": \"File\"}}" + local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}" + local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}" # TODO these are used in other places, convert to global. local gce_conf_path="/etc/gce.conf" @@ -1795,7 +1795,7 @@ EOM } " "${src_file}" fi - cp "${src_file}" /etc/kubernetes/manifests + cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}" } # Starts kubernetes controller manager. @@ -2473,95 +2473,104 @@ EOF } ########### Main Function ########### -echo "Start to configure instance for kubernetes" +function main() { + echo "Start to configure instance for kubernetes" -KUBE_HOME="/home/kubernetes" -CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter" -PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml" + KUBE_HOME="/home/kubernetes" + CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter" + PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml" -if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then - echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization." - exit 1 -fi - -source "${KUBE_HOME}/kube-env" - - -if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then - echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml" - KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml" -fi - -if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then - source "${KUBE_HOME}/kube-master-certs" -fi - -if [[ -n "${KUBE_USER:-}" ]]; then - if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then - echo "Bad KUBE_USER format." + if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then + echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization." exit 1 fi -fi -# generate the controller manager and scheduler tokens here since they are only used on the master. -KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) -KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + source "${KUBE_HOME}/kube-env" -setup-os-params -config-ip-firewall -create-dirs -setup-kubelet-dir -ensure-local-ssds -setup-logrotate -if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then - mount-master-pd - create-node-pki - create-master-pki - create-master-auth - create-master-kubelet-auth - create-master-etcd-auth - override-pv-recycler + + if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then + echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml" + KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml" + fi + + if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then + source "${KUBE_HOME}/kube-master-certs" + fi + + if [[ -n "${KUBE_USER:-}" ]]; then + if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then + echo "Bad KUBE_USER format." + exit 1 + fi + fi + + # generate the controller manager and scheduler tokens here since they are only used on the master. + KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) + + setup-os-params + config-ip-firewall + create-dirs + setup-kubelet-dir + ensure-local-ssds + setup-logrotate + if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then + mount-master-pd + create-node-pki + create-master-pki + create-master-auth + create-master-kubelet-auth + create-master-etcd-auth + override-pv-recycler + else + create-node-pki + create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME} + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + create-kubeproxy-user-kubeconfig + fi + if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then + create-node-problem-detector-kubeconfig + fi + fi + + override-kubectl + # Run the containerized mounter once to pre-cache the container image. + if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then + assemble-docker-flags + fi + start-kubelet + + if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then + compute-master-manifest-variables + start-etcd-servers + start-etcd-empty-dir-cleanup-pod + start-kube-apiserver + start-kube-controller-manager + start-kube-scheduler + start-kube-addons + start-cluster-autoscaler + start-lb-controller + start-rescheduler + else + if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then + start-kube-proxy + fi + if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then + start-image-puller + fi + if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then + start-node-problem-detector + fi + fi + reset-motd + prepare-mounter-rootfs + modprobe configs + echo "Done for the configuration for kubernetes" +} + +# use --source-only to test functions defined in this script. +if [[ "$#" -eq 1 && "${1}" == "--source-only" ]]; then + : else - create-node-pki - create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME} - if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then - create-kubeproxy-user-kubeconfig - fi - if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then - create-node-problem-detector-kubeconfig - fi -fi - -override-kubectl -# Run the containerized mounter once to pre-cache the container image. -if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then - assemble-docker-flags -fi -start-kubelet - -if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then - compute-master-manifest-variables - start-etcd-servers - start-etcd-empty-dir-cleanup-pod - start-kube-apiserver - start-kube-controller-manager - start-kube-scheduler - start-kube-addons - start-cluster-autoscaler - start-lb-controller - start-rescheduler -else - if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then - start-kube-proxy - fi - if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then - start-image-puller - fi - if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then - start-node-problem-detector - fi -fi -reset-motd -prepare-mounter-rootfs -modprobe configs -echo "Done for the configuration for kubernetes" + main "${@}" +fi \ No newline at end of file diff --git a/cluster/gce/gci/configure_helper_test.go b/cluster/gce/gci/configure_helper_test.go new file mode 100644 index 0000000000..0297733acf --- /dev/null +++ b/cluster/gce/gci/configure_helper_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gci + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + "text/template" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kubernetes/pkg/api/legacyscheme" +) + +const ( + envScriptFileName = "kube-env" + configureHelperScriptName = "configure-helper.sh" +) + +type ManifestTestCase struct { + pod v1.Pod + envScriptPath string + manifest string + auxManifests []string + kubeHome string + manifestSources string + manifestDestination string + manifestTemplateDir string + manifestTemplate string + manifestFuncName string + t *testing.T +} + +func newManifestTestCase(t *testing.T, manifest, funcName string, auxManifests []string) *ManifestTestCase { + c := &ManifestTestCase{ + t: t, + manifest: manifest, + auxManifests: auxManifests, + manifestFuncName: funcName, + } + + d, err := ioutil.TempDir("", "configure-helper-test") + if err != nil { + c.t.Fatalf("Failed to create temp directory: %v", err) + } + + c.kubeHome = d + c.envScriptPath = filepath.Join(c.kubeHome, envScriptFileName) + c.manifestSources = filepath.Join(c.kubeHome, "kube-manifests", "kubernetes", "gci-trusty") + + currentPath, err := os.Getwd() + if err != nil { + c.t.Fatalf("Failed to get current directory: %v", err) + } + gceDir := filepath.Dir(currentPath) + c.manifestTemplateDir = filepath.Join(gceDir, "manifests") + c.manifestTemplate = filepath.Join(c.manifestTemplateDir, c.manifest) + c.manifestDestination = filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests", c.manifest) + + c.mustCopyFromTemplate() + c.mustCopyAuxFromTemplate() + c.mustCreateManifestDstDir() + + return c +} + +func (c *ManifestTestCase) mustCopyFromTemplate() { + if err := os.MkdirAll(c.manifestSources, os.ModePerm); err != nil { + c.t.Fatalf("Failed to create source directory: %v", err) + } + + if err := copyFile(c.manifestTemplate, filepath.Join(c.manifestSources, c.manifest)); err != nil { + c.t.Fatalf("Failed to copy source manifest to KUBE_HOME: %v", err) + } +} + +func (c *ManifestTestCase) mustCopyAuxFromTemplate() { + for _, m := range c.auxManifests { + err := copyFile(filepath.Join(c.manifestTemplateDir, m), filepath.Join(c.manifestSources, m)) + if err != nil { + c.t.Fatalf("Failed to copy source manifest %s to KUBE_HOME: %v", m, err) + } + } +} + +func (c *ManifestTestCase) mustCreateManifestDstDir() { + p := filepath.Join(filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests")) + if err := os.MkdirAll(p, os.ModePerm); err != nil { + c.t.Fatalf("Failed to create designation folder for kube-apiserver.manifest: %v", err) + } +} + +func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) { + f, err := os.Create(filepath.Join(c.kubeHome, envScriptFileName)) + if err != nil { + c.t.Fatalf("Failed to create envScript: %v", err) + } + defer f.Close() + + t := template.Must(template.New("env").Parse(envTemplate)) + + if err = t.Execute(f, env); err != nil { + c.t.Fatalf("Failed to execute template: %v", err) + } +} + +func (c *ManifestTestCase) mustInvokeFunc(envTemplate string, env interface{}) { + c.mustCreateEnv(envTemplate, env) + args := fmt.Sprintf("source %s ; source %s --source-only ; %s", c.envScriptPath, configureHelperScriptName, c.manifestFuncName) + cmd := exec.Command("bash", "-c", args) + + bs, err := cmd.CombinedOutput() + if err != nil { + c.t.Logf("%s", bs) + c.t.Fatalf("Failed to run configure-helper.sh: %v", err) + } + c.t.Logf("%s", string(bs)) +} + +func (c *ManifestTestCase) mustLoadPodFromManifest() { + json, err := ioutil.ReadFile(c.manifestDestination) + if err != nil { + c.t.Fatalf("Failed to read manifest: %s, %v", c.manifestDestination, err) + } + + if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil { + c.t.Fatalf("Failed to decode manifest: %v", err) + } +} + +func (c *ManifestTestCase) tearDown() { + os.RemoveAll(c.kubeHome) +} + +func copyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + defer func() { + cerr := out.Close() + if cerr == nil { + err = cerr + } + }() + _, err = io.Copy(out, in) + return err +} diff --git a/cluster/gce/manifests/BUILD b/cluster/gce/manifests/BUILD new file mode 100644 index 0000000000..2f352fcaec --- /dev/null +++ b/cluster/gce/manifests/BUILD @@ -0,0 +1,55 @@ +package(default_visibility = ["//visibility:public"]) + +load("@io_kubernetes_build//defs:build.bzl", "release_filegroup") +load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar") + +pkg_tar( + name = "gce-master-manifests", + srcs = [ + "abac-authz-policy.jsonl", + "cluster-autoscaler.manifest", + "e2e-image-puller.manifest", + "etcd.manifest", + "glbc.manifest", + "kms-plugin-container.manifest", + "kube-addon-manager.yaml", + "kube-apiserver.manifest", + "kube-controller-manager.manifest", + "kube-proxy.manifest", + "kube-scheduler.manifest", + "rescheduler.manifest", + ], + mode = "0644", +) + +filegroup( + name = "manifests-test-data", + srcs = [ + "abac-authz-policy.jsonl", + "cluster-autoscaler.manifest", + "e2e-image-puller.manifest", + "etcd.manifest", + "glbc.manifest", + "kms-plugin-container.manifest", + "kube-addon-manager.yaml", + "kube-apiserver.manifest", + "kube-controller-manager.manifest", + "kube-proxy.manifest", + "kube-scheduler.manifest", + "rescheduler.manifest", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +)