From 0c9e922fcda5c2457b3b97ab0cf0f2c0f0a3f03b Mon Sep 17 00:00:00 2001 From: Yago Nobre Date: Fri, 19 Oct 2018 02:45:41 -0300 Subject: [PATCH 1/2] Get etcd data dir from kubeadm config or use etcd manifest as fallback on kubeadm reset --- cmd/kubeadm/app/cmd/reset.go | 67 +++++++++++++++++----- cmd/kubeadm/app/cmd/reset_test.go | 95 +++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 15 deletions(-) diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index 0b2ac7a891..963b5d37e8 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -30,13 +30,17 @@ import ( "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/sets" + clientset "k8s.io/client-go/kubernetes" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" "k8s.io/kubernetes/cmd/kubeadm/app/cmd/options" + cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" + utilstaticpod "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" "k8s.io/kubernetes/pkg/util/initsystem" utilsexec "k8s.io/utils/exec" ) @@ -47,6 +51,7 @@ func NewCmdReset(in io.Reader, out io.Writer) *cobra.Command { var criSocketPath string var ignorePreflightErrors []string var forceReset bool + kubeConfigFile := kubeadmconstants.GetAdminKubeConfigPath() cmd := &cobra.Command{ Use: "reset", @@ -55,13 +60,18 @@ func NewCmdReset(in io.Reader, out io.Writer) *cobra.Command { ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(ignorePreflightErrors) kubeadmutil.CheckErr(err) + kubeConfigFile = cmdutil.FindExistingKubeConfig(kubeConfigFile) + client, err := getClientset(kubeConfigFile, false) + kubeadmutil.CheckErr(err) + r, err := NewReset(in, ignorePreflightErrorsSet, forceReset, certsDir, criSocketPath) kubeadmutil.CheckErr(err) - kubeadmutil.CheckErr(r.Run(out)) + kubeadmutil.CheckErr(r.Run(out, client)) }, } options.AddIgnorePreflightErrorsFlag(cmd.PersistentFlags(), &ignorePreflightErrors) + options.AddKubeConfigFlag(cmd.PersistentFlags(), &kubeConfigFile) cmd.PersistentFlags().StringVar( &certsDir, "cert-dir", kubeadmapiv1beta1.DefaultCertificatesDir, @@ -114,7 +124,19 @@ func NewReset(in io.Reader, ignorePreflightErrors sets.String, forceReset bool, } // Run reverts any changes made to this host by "kubeadm init" or "kubeadm join". -func (r *Reset) Run(out io.Writer) error { +func (r *Reset) Run(out io.Writer, client clientset.Interface) error { + var dirsToClean []string + // Only clear etcd data when using local etcd. + etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml") + + glog.V(1).Infof("[reset] checking for etcd config") + etcdDataDir, err := getEtcdDataDir(etcdManifestPath, client) + if err == nil { + dirsToClean = append(dirsToClean, etcdDataDir) + } else { + fmt.Println("[reset] no etcd config found. Assuming external etcd") + fmt.Println("[reset] please manually reset etcd to prevent further issues") + } // Try to stop the kubelet service glog.V(1).Infof("[reset] getting init system") @@ -144,19 +166,7 @@ func (r *Reset) Run(out io.Writer) error { if err := removeContainers(utilsexec.New(), r.criSocketPath); err != nil { glog.Errorf("[reset] failed to remove containers: %+v", err) } - dirsToClean := []string{kubeadmconstants.KubeletRunDirectory, "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"} - - // Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user - // provided external etcd endpoints. In that case, it is their own responsibility to reset etcd - etcdManifestPath := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName, "etcd.yaml") - glog.V(1).Infof("[reset] checking for etcd manifest") - if _, err := os.Stat(etcdManifestPath); err == nil { - glog.V(1).Infof("Found one at %s", etcdManifestPath) - dirsToClean = append(dirsToClean, "/var/lib/etcd") - } else { - fmt.Printf("[reset] no etcd manifest found in %q. Assuming external etcd\n", etcdManifestPath) - fmt.Println("[reset] please manually reset etcd to prevent further issues") - } + dirsToClean = append(dirsToClean, []string{kubeadmconstants.KubeletRunDirectory, "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"}...) // Then clean contents from the stateful kubelet, etcd and cni directories fmt.Printf("[reset] deleting contents of stateful directories: %v\n", dirsToClean) @@ -175,6 +185,33 @@ func (r *Reset) Run(out io.Writer) error { return nil } +func getEtcdDataDir(manifestPath string, client clientset.Interface) (string, error) { + const etcdVolumeName = "etcd-data" + var dataDir string + + cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "reset", "", false) + if err == nil { + return cfg.Etcd.Local.DataDir, nil + } + glog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap, using etcd pod spec as fallback: %v", err) + + etcdPod, err := utilstaticpod.ReadStaticPodFromDisk(manifestPath) + if err != nil { + return "", err + } + + for _, volumeMount := range etcdPod.Spec.Volumes { + if volumeMount.Name == etcdVolumeName { + dataDir = volumeMount.HostPath.Path + break + } + } + if dataDir == "" { + return dataDir, fmt.Errorf("invalid etcd pod manifest") + } + return dataDir, nil +} + func removeContainers(execer utilsexec.Interface, criSocketPath string) error { containerRuntime, err := utilruntime.NewContainerRuntime(execer, criSocketPath) if err != nil { diff --git a/cmd/kubeadm/app/cmd/reset_test.go b/cmd/kubeadm/app/cmd/reset_test.go index be35b56cb4..f15dec5039 100644 --- a/cmd/kubeadm/app/cmd/reset_test.go +++ b/cmd/kubeadm/app/cmd/reset_test.go @@ -23,14 +23,46 @@ import ( "path/filepath" "testing" + "github.com/renstrom/dedent" + + clientsetfake "k8s.io/client-go/kubernetes/fake" kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/kubernetes/cmd/kubeadm/app/preflight" + testutil "k8s.io/kubernetes/cmd/kubeadm/test" "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" ) +const ( + etcdPod = `apiVersion: v1 +kind: Pod +metadata: +spec: + volumes: + - hostPath: + path: /path/to/etcd + type: DirectoryOrCreate + name: etcd-data + - hostPath: + path: /etc/kubernetes/pki/etcd + type: DirectoryOrCreate + name: etcd-certs` + + etcdPodWithoutDataVolume = `apiVersion: v1 +kind: Pod +metadata: +spec: + volumes: + - hostPath: + path: /etc/kubernetes/pki/etcd + type: DirectoryOrCreate + name: etcd-certs` + + etcdPodInvalid = `invalid pod` +) + func assertExists(t *testing.T, path string) { if _, err := os.Stat(path); os.IsNotExist(err) { t.Errorf("file/directory does not exist; error: %s", err) @@ -234,3 +266,66 @@ func TestRemoveContainers(t *testing.T) { removeContainers(&fexec, "unix:///var/run/crio/crio.sock") } + +func TestGetEtcdDataDir(t *testing.T) { + tests := map[string]struct { + dataDir string + podYaml string + expectErr bool + writeManifest bool + }{ + "non-existent file returns error": { + dataDir: "", + podYaml: "", + expectErr: true, + writeManifest: false, + }, + "return etcd data dir": { + dataDir: "/path/to/etcd", + podYaml: etcdPod, + expectErr: false, + writeManifest: true, + }, + "invalid etcd pod": { + dataDir: "", + podYaml: etcdPodInvalid, + expectErr: true, + writeManifest: true, + }, + "etcd pod spec without data volume": { + dataDir: "", + podYaml: etcdPodWithoutDataVolume, + expectErr: true, + writeManifest: true, + }, + } + + for name, test := range tests { + tmpdir := testutil.SetupTempDir(t) + defer os.RemoveAll(tmpdir) + + manifestPath := filepath.Join(tmpdir, "etcd.yaml") + if test.writeManifest { + err := ioutil.WriteFile(manifestPath, []byte(test.podYaml), 0644) + if err != nil { + t.Fatalf(dedent.Dedent("failed to write pod manifest\n%s\n\tfatal error: %v"), name, err) + } + } + + client := clientsetfake.NewSimpleClientset() + dataDir, err := getEtcdDataDir(manifestPath, client) + if (err != nil) != test.expectErr { + t.Fatalf(dedent.Dedent( + "getEtcdDataDir failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v"), + name, + test.expectErr, + (err != nil), + err, + ) + } + + if dataDir != test.dataDir { + t.Fatalf(dedent.Dedent("getEtcdDataDir failed\n%s\n\texpected: %s\ngot: %s"), name, test.dataDir, dataDir) + } + } +} From b35e22abe173302d266ef5a5ce2388bb17f5bf4c Mon Sep 17 00:00:00 2001 From: Yago Nobre Date: Fri, 26 Oct 2018 11:45:44 -0300 Subject: [PATCH 2/2] Update bazel --- cmd/kubeadm/app/cmd/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/kubeadm/app/cmd/BUILD b/cmd/kubeadm/app/cmd/BUILD index 1900bc9280..68f1958bc0 100644 --- a/cmd/kubeadm/app/cmd/BUILD +++ b/cmd/kubeadm/app/cmd/BUILD @@ -56,6 +56,7 @@ go_library( "//cmd/kubeadm/app/util/dryrun:go_default_library", "//cmd/kubeadm/app/util/kubeconfig:go_default_library", "//cmd/kubeadm/app/util/runtime:go_default_library", + "//cmd/kubeadm/app/util/staticpod:go_default_library", "//pkg/util/initsystem:go_default_library", "//pkg/version:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -100,6 +101,7 @@ go_test( "//cmd/kubeadm/app/preflight:go_default_library", "//cmd/kubeadm/app/util/config:go_default_library", "//cmd/kubeadm/app/util/runtime:go_default_library", + "//cmd/kubeadm/test:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",