From 7358084fd5bb9c4eaab537f745a55912b5352412 Mon Sep 17 00:00:00 2001 From: Jeff Lowdermilk Date: Fri, 6 Mar 2015 14:34:38 -0800 Subject: [PATCH 1/2] Utilities to create and clear kubeconfig for use by kube-up scripts --- cluster/common.sh | 71 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 cluster/common.sh diff --git a/cluster/common.sh b/cluster/common.sh new file mode 100644 index 0000000000..3ffcb288f3 --- /dev/null +++ b/cluster/common.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Common utilites for kube-up/kube-down + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +# Generate kubeconfig data for the created cluster. +# Assumed vars: +# KUBE_USER +# KUBE_PASSWORD +# KUBE_MASTER_IP +# KUBECONFIG +# +# KUBE_CERT +# KUBE_KEY +# CA_CERT +# CONTEXT +function create-kubeconfig() { + local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" + + # We expect KUBECONFIG to be defined, which determines the file we write to. + "${kubectl}" config set-cluster "${CONTEXT}" --server="https://${KUBE_MASTER_IP}" \ + --certificate-authority="${CA_CERT}" \ + --embed-certs=true + "${kubectl}" config set-credentials "${CONTEXT}" --username="${KUBE_USER}" \ + --password="${KUBE_PASSWORD}" \ + --client-certificate="${KUBE_CERT}" \ + --client-key="${KUBE_KEY}" \ + --embed-certs=true + "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="${CONTEXT}" + "${kubectl}" config use-context "${CONTEXT}" --cluster="${CONTEXT}" + + echo "Wrote config for ${CONTEXT} to ${KUBE_ROOT}/.kubeconfig" +} + +# Clear kubeconfig data for a context +# Assumed vars: +# KUBECONFIG +# CONTEXT +function clear-kubeconfig() { + local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" + "${kubectl}" config unset "clusters.${CONTEXT}" + "${kubectl}" config unset "users.${CONTEXT}" + "${kubectl}" config unset "contexts.${CONTEXT}" + + local current + current=$("${kubectl}" config view -o template --template='{{ index . "current-context" }}') + if [[ "${current}" == "${CONTEXT}" ]]; then + "${kubectl}" config unset current-context + fi + + echo "Cleared config for ${CONTEXT} from ${KUBE_ROOT}/.kubeconfig" +} From 7db006ab1aa09d9a134147cbdb377e0c2fe25f0e Mon Sep 17 00:00:00 2001 From: Jeff Lowdermilk Date: Fri, 6 Mar 2015 14:49:25 -0800 Subject: [PATCH 2/2] Generate standalone kubeconfig on kube-up, clear on kube-down. Also tweaked the ginkgo tests to pull auth directly from a kubeconfig file instead of the legacy kubernetes_auth file. --- cluster/common.sh | 10 +++-- cluster/gce/util.sh | 65 ++++++++++------------------ cmd/e2e/e2e.go | 6 ++- hack/ginkgo-e2e.sh | 2 +- pkg/client/clientcmd/auth_loaders.go | 2 +- test/e2e/driver.go | 4 +- test/e2e/kubectl.go | 18 +++++--- test/e2e/util.go | 49 +++++++++++++-------- 8 files changed, 81 insertions(+), 75 deletions(-) diff --git a/cluster/common.sh b/cluster/common.sh index 3ffcb288f3..e778e13fef 100644 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -36,7 +36,11 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. function create-kubeconfig() { local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - # We expect KUBECONFIG to be defined, which determines the file we write to. + # KUBECONFIG determines the file we write to, but it may not exist yet + if [[ ! -e "${KUBECONFIG}" ]]; then + mkdir -p $(dirname "${KUBECONFIG}") + touch "${KUBECONFIG}" + fi "${kubectl}" config set-cluster "${CONTEXT}" --server="https://${KUBE_MASTER_IP}" \ --certificate-authority="${CA_CERT}" \ --embed-certs=true @@ -48,7 +52,7 @@ function create-kubeconfig() { "${kubectl}" config set-context "${CONTEXT}" --cluster="${CONTEXT}" --user="${CONTEXT}" "${kubectl}" config use-context "${CONTEXT}" --cluster="${CONTEXT}" - echo "Wrote config for ${CONTEXT} to ${KUBE_ROOT}/.kubeconfig" + echo "Wrote config for ${CONTEXT} to ${KUBECONFIG}" } # Clear kubeconfig data for a context @@ -67,5 +71,5 @@ function clear-kubeconfig() { "${kubectl}" config unset current-context fi - echo "Cleared config for ${CONTEXT} from ${KUBE_ROOT}/.kubeconfig" + echo "Cleared config for ${CONTEXT} from ${KUBECONFIG}" } diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 77f668be46..ca83d06561 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -20,6 +20,7 @@ # config-default.sh. KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}" +source "${KUBE_ROOT}/cluster/common.sh" NODE_INSTANCE_PREFIX="${INSTANCE_PREFIX}-minion" @@ -234,17 +235,16 @@ function detect-master () { # KUBE_USER # KUBE_PASSWORD function get-password { - # go template to extract the auth-path of the current-context user + # templates to extract the username,password for the current-context user # Note: we save dot ('.') to $dot because the 'with' action overrides dot - local template='{{$dot := .}}{{with $ctx := index $dot "current-context"}}{{$user := index $dot "contexts" $ctx "user"}}{{index $dot "users" $user "auth-path"}}{{end}}' - local file=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${template}") - if [[ ! -z "$file" && -r "$file" ]]; then - KUBE_USER=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["User"]') - KUBE_PASSWORD=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["Password"]') - return + local username='{{$dot := .}}{{with $ctx := index $dot "current-context"}}{{$user := index $dot "contexts" $ctx "user"}}{{index $dot "users" $user "username"}}{{end}}' + local password='{{$dot := .}}{{with $ctx := index $dot "current-context"}}{{$user := index $dot "contexts" $ctx "user"}}{{index $dot "users" $user "password"}}{{end}}' + KUBE_USER=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${username}") + KUBE_PASSWORD=$("${KUBE_ROOT}/cluster/kubectl.sh" config view -o template --template="${password}") + if [[ -z "${KUBE_USER}" || -z "${KUBE_PASSWORD}" ]]; then + KUBE_USER=admin + KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') fi - KUBE_USER=admin - KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') } # Set MASTER_HTPASSWD @@ -637,44 +637,22 @@ function kube-up { echo "Kubernetes cluster created." - local kube_cert="kubecfg.crt" - local kube_key="kubecfg.key" - local ca_cert="kubernetes.ca.crt" - # TODO use token instead of kube_auth - local kube_auth="kubernetes_auth" - - local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - local context="${PROJECT}_${INSTANCE_PREFIX}" - local user="${context}-admin" - local config_dir="${HOME}/.kube/${context}" + # TODO use token instead of basic auth + export KUBECONFIG="${HOME}/.kube/.kubeconfig" + export KUBE_CERT="/tmp/kubecfg.crt" + export KUBE_KEY="/tmp/kubecfg.key" + export CA_CERT="/tmp/kubernetes.ca.crt" + export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}" # TODO: generate ADMIN (and KUBELET) tokens and put those in the master's # config file. Distribute the same way the htpasswd is done. ( - mkdir -p "${config_dir}" umask 077 - gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/kubecfg.crt" >"${config_dir}/${kube_cert}" 2>/dev/null - gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/kubecfg.key" >"${config_dir}/${kube_key}" 2>/dev/null - gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/ca.crt" >"${config_dir}/${ca_cert}" 2>/dev/null + gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/kubecfg.crt" >"${KUBE_CERT}" 2>/dev/null + gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/kubecfg.key" >"${KUBE_KEY}" 2>/dev/null + gcloud compute ssh --project "${PROJECT}" --zone "$ZONE" "${MASTER_NAME}" --command "sudo cat /srv/kubernetes/ca.crt" >"${CA_CERT}" 2>/dev/null - "${kubectl}" config set-cluster "${context}" --server="https://${KUBE_MASTER_IP}" --certificate-authority="${config_dir}/${ca_cert}" --global - "${kubectl}" config set-credentials "${user}" --auth-path="${config_dir}/${kube_auth}" --global - "${kubectl}" config set-context "${context}" --cluster="${context}" --user="${user}" --global - "${kubectl}" config use-context "${context}" --global - - cat << EOF > "${config_dir}/${kube_auth}" -{ - "User": "$KUBE_USER", - "Password": "$KUBE_PASSWORD", - "CAFile": "${config_dir}/${ca_cert}", - "CertFile": "${config_dir}/${kube_cert}", - "KeyFile": "${config_dir}/${kube_key}" -} -EOF - - chmod 0600 "${config_dir}/${kube_auth}" "${config_dir}/$kube_cert" \ - "${config_dir}/${kube_key}" "${config_dir}/${ca_cert}" - echo "Wrote ${config_dir}/${kube_auth}" + create-kubeconfig ) echo "Sanity checking cluster..." @@ -722,7 +700,7 @@ EOF echo echo -e "${color_yellow} https://${KUBE_MASTER_IP}" echo - echo -e "${color_green}The user name and password to use is located in ${config_dir}/${kube_auth}.${color_norm}" + echo -e "${color_green}The user name and password to use is located in ${KUBECONFIG}.${color_norm}" echo } @@ -817,6 +795,9 @@ function kube-down { --quiet \ "${MASTER_NAME}-ip" || true + export KUBECONFIG="${HOME}/.kube/.kubeconfig" + export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}" + clear-kubeconfig } # Update a kubernetes cluster with latest source diff --git a/cmd/e2e/e2e.go b/cmd/e2e/e2e.go index 544be42357..c5e2edd0cd 100644 --- a/cmd/e2e/e2e.go +++ b/cmd/e2e/e2e.go @@ -20,6 +20,7 @@ import ( "os" goruntime "runtime" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" "github.com/GoogleCloudPlatform/kubernetes/test/e2e" "github.com/golang/glog" @@ -27,7 +28,8 @@ import ( ) var ( - authConfig = flag.String("auth_config", os.Getenv("HOME")+"/.kubernetes_auth", "Path to the auth info file.") + kubeConfig = flag.String(clientcmd.RecommendedConfigPathFlag, "", "Path to kubeconfig containing embeded authinfo. Will use cluster/user info from 'current-context'") + authConfig = flag.String("auth_config", "", "Path to the auth info file.") certDir = flag.String("cert_dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") gceProject = flag.String("gce_project", "", "The GCE project being used, if applicable") gceZone = flag.String("gce_zone", "", "GCE zone being used, if applicable") @@ -61,5 +63,5 @@ func main() { Zone: *gceZone, MasterName: *masterName, } - e2e.RunE2ETests(*authConfig, *certDir, *host, *repoRoot, *provider, gceConfig, *orderseed, *times, *reportDir, testList) + e2e.RunE2ETests(*kubeConfig, *authConfig, *certDir, *host, *repoRoot, *provider, gceConfig, *orderseed, *times, *reportDir, testList) } diff --git a/hack/ginkgo-e2e.sh b/hack/ginkgo-e2e.sh index e2c0f2f8f6..53e29d6f0e 100755 --- a/hack/ginkgo-e2e.sh +++ b/hack/ginkgo-e2e.sh @@ -93,7 +93,7 @@ elif [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then ) elif [[ "${KUBERNETES_PROVIDER}" == "gce" ]]; then auth_config=( - "--auth_config=${HOME}/.kube/${PROJECT}_${INSTANCE_PREFIX}/kubernetes_auth" + "--kubeconfig=${HOME}/.kube/.kubeconfig" ) elif [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then auth_config=( diff --git a/pkg/client/clientcmd/auth_loaders.go b/pkg/client/clientcmd/auth_loaders.go index e3c63962e8..cd18d93bbe 100644 --- a/pkg/client/clientcmd/auth_loaders.go +++ b/pkg/client/clientcmd/auth_loaders.go @@ -80,7 +80,7 @@ func promptForString(field string, r io.Reader) string { return result } -// NewDefaultAuthLoader is an AuthLoader that parses an AuthInfo object from a file path. It prompts user and creates file if it doesn't exist. +// NewPromptingAuthLoader is an AuthLoader that parses an AuthInfo object from a file path. It prompts user and creates file if it doesn't exist. func NewPromptingAuthLoader(reader io.Reader) *PromptingAuthLoader { return &PromptingAuthLoader{reader} } diff --git a/test/e2e/driver.go b/test/e2e/driver.go index ac2056aa5d..1249fbf5d9 100644 --- a/test/e2e/driver.go +++ b/test/e2e/driver.go @@ -53,8 +53,8 @@ func (t *testResult) Fail() { *t = false } // Run each Go end-to-end-test. This function assumes the // creation of a test cluster. -func RunE2ETests(authConfig, certDir, host, repoRoot, provider string, gceConfig *GCEConfig, orderseed int64, times int, reportDir string, testList []string) { - testContext = testContextType{authConfig, certDir, host, repoRoot, provider, *gceConfig} +func RunE2ETests(kubeConfig, authConfig, certDir, host, repoRoot, provider string, gceConfig *GCEConfig, orderseed int64, times int, reportDir string, testList []string) { + testContext = testContextType{kubeConfig, authConfig, certDir, host, repoRoot, provider, *gceConfig} util.ReallyCrash = true util.InitLogs() defer util.FlushLogs() diff --git a/test/e2e/kubectl.go b/test/e2e/kubectl.go index 97bd06da28..5679b6618c 100644 --- a/test/e2e/kubectl.go +++ b/test/e2e/kubectl.go @@ -26,6 +26,7 @@ import ( "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" . "github.com/onsi/ginkgo" ) @@ -182,12 +183,17 @@ func getData(c *client.Client, podID string) (*updateDemoData, error) { } func kubectlCmd(args ...string) *exec.Cmd { - defaultArgs := []string{"--auth-path=" + testContext.authConfig} - if testContext.certDir != "" { - defaultArgs = append(defaultArgs, - fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")), - fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")), - fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key"))) + defaultArgs := []string{} + if testContext.kubeConfig != "" { + defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+testContext.kubeConfig) + } else { + defaultArgs = append(defaultArgs, "--"+clientcmd.FlagAuthPath+"="+testContext.authConfig) + if testContext.certDir != "" { + defaultArgs = append(defaultArgs, + fmt.Sprintf("--certificate-authority=%s", filepath.Join(testContext.certDir, "ca.crt")), + fmt.Sprintf("--client-certificate=%s", filepath.Join(testContext.certDir, "kubecfg.crt")), + fmt.Sprintf("--client-key=%s", filepath.Join(testContext.certDir, "kubecfg.key"))) + } } kubectlArgs := append(defaultArgs, args...) // TODO: Remove this once gcloud writes a proper entry in the kubeconfig file. diff --git a/test/e2e/util.go b/test/e2e/util.go index 8331bd5148..747b48aeb3 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -25,8 +25,8 @@ import ( "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" + "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd" "github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -38,6 +38,7 @@ const ( ) type testContextType struct { + kubeConfig string authConfig string certDir string host string @@ -117,32 +118,44 @@ func waitForPodSuccess(c *client.Client, podName string, contName string) error } func loadConfig() (*client.Config, error) { - config := &client.Config{ - Host: testContext.host, + switch { + case testContext.kubeConfig != "": + fmt.Printf(">>> testContext.kubeConfig: %s\n", testContext.kubeConfig) + c, err := clientcmd.LoadFromFile(testContext.kubeConfig) + if err != nil { + return nil, fmt.Errorf("error loading kubeConfig: %v", err.Error()) + } + return clientcmd.NewDefaultClientConfig(*c, &clientcmd.ConfigOverrides{}).ClientConfig() + case testContext.authConfig != "": + config := &client.Config{ + Host: testContext.host, + } + info, err := clientauth.LoadFromFile(testContext.authConfig) + if err != nil { + return nil, fmt.Errorf("error loading authConfig: %v", err.Error()) + } + // If the certificate directory is provided, set the cert paths to be there. + if testContext.certDir != "" { + Logf("Expecting certs in %v.", testContext.certDir) + info.CAFile = filepath.Join(testContext.certDir, "ca.crt") + info.CertFile = filepath.Join(testContext.certDir, "kubecfg.crt") + info.KeyFile = filepath.Join(testContext.certDir, "kubecfg.key") + } + mergedConfig, err := info.MergeWithConfig(*config) + return &mergedConfig, err + default: + return nil, fmt.Errorf("either kubeConfig or authConfig must be specified to load client config") } - info, err := clientauth.LoadFromFile(testContext.authConfig) - if err != nil { - return nil, fmt.Errorf("Error loading auth: %v", err.Error()) - } - // If the certificate directory is provided, set the cert paths to be there. - if testContext.certDir != "" { - Logf("Expecting certs in %v.", testContext.certDir) - info.CAFile = filepath.Join(testContext.certDir, "ca.crt") - info.CertFile = filepath.Join(testContext.certDir, "kubecfg.crt") - info.KeyFile = filepath.Join(testContext.certDir, "kubecfg.key") - } - mergedConfig, err := info.MergeWithConfig(*config) - return &mergedConfig, err } func loadClient() (*client.Client, error) { config, err := loadConfig() if err != nil { - return nil, fmt.Errorf("Error creating client: %v", err.Error()) + return nil, fmt.Errorf("error creating client: %v", err.Error()) } c, err := client.New(config) if err != nil { - return nil, fmt.Errorf("Error creating client: %v", err.Error()) + return nil, fmt.Errorf("error creating client: %v", err.Error()) } return c, nil }