kubeadm: Empty directories during reset, but do not delete them.

This will allow packages to maintain ownership of config and data
directories, which may carry selinux or other attributes that should be
preserved, but we do not wish to manage within kubeadm itself.

cmd/kubeadm/app/cmd/reset.go

@@ -67,18 +67,40 @@ func NewReset(skipPreFlight bool) (*Reset, error) {
 	return &Reset{}, nil
 }
 
+// cleanDir removes everything in a directory, but not the directory itself:
+func cleanDir(path string) {
+	// If the directory doesn't even exist there's nothing to do, and we do
+	// not consider this an error:
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return
+	}
+
+	d, err := os.Open(path)
+	if err != nil {
+		fmt.Printf("failed to remove directory: [%v]\n", err)
+	}
+	defer d.Close()
+	names, err := d.Readdirnames(-1)
+	if err != nil {
+		fmt.Printf("failed to remove directory: [%v]\n", err)
+	}
+	for _, name := range names {
+		err = os.RemoveAll(filepath.Join(path, name))
+		if err != nil {
+			fmt.Printf("failed to remove directory: [%v]\n", err)
+		}
+	}
+}
+
 // resetConfigDir is used to cleanup the files kubeadm writes in /etc/kubernetes/.
 func resetConfigDir(configDirPath string) {
 	dirsToClean := []string{
 		filepath.Join(configDirPath, "manifests"),
 		filepath.Join(configDirPath, "pki"),
 	}
-	fmt.Printf("Deleting config directories: %v\n", dirsToClean)
+	fmt.Printf("Deleting contents of config directories: %v\n", dirsToClean)
 	for _, dir := range dirsToClean {
-		err := os.RemoveAll(dir)
-		if err != nil {
-			fmt.Printf("failed to remove directory: [%v]\n", err)
-		}
+		cleanDir(dir)
 	}
 
 	filesToClean := []string{
@@ -112,12 +134,9 @@ func (r *Reset) Run(out io.Writer) error {
 	resetConfigDir("/etc/kubernetes/")
 
 	dirsToClean := []string{"/var/lib/kubelet", "/var/lib/etcd"}
-	fmt.Printf("Deleting stateful directories: %v\n", dirsToClean)
+	fmt.Printf("Deleting contents of stateful directories: %v\n", dirsToClean)
 	for _, dir := range dirsToClean {
-		err := os.RemoveAll(dir)
-		if err != nil {
-			fmt.Printf("failed to remove directory: [%v]\n", err)
-		}
+		cleanDir(dir)
 	}
 
 	dockerCheck := preflight.ServiceCheck{Service: "docker"}

cmd/kubeadm/app/cmd/reset_test.go

@@ -66,6 +66,10 @@ func TestConfigDirCleaner(t *testing.T) {
 				"admin.conf",
 				"kubelet.conf",
 			},
+			verifyExists: []string{
+				"manifests",
+				"pki",
+			},
 		},
 		"partial reset": {
 			setupDirs: []string{
@@ -75,6 +79,12 @@ func TestConfigDirCleaner(t *testing.T) {
 				"pki/ca.pem",
 				"kubelet.conf",
 			},
+			verifyExists: []string{
+				"pki",
+			},
+			verifyNotExists: []string{
+				"manifests",
+			},
 		},
 		"preserve cloud-config.json": {
 			setupDirs: []string{
@@ -90,6 +100,8 @@ func TestConfigDirCleaner(t *testing.T) {
 				"cloud-config.json",
 			},
 			verifyExists: []string{
+				"manifests",
+				"pki",
 				"cloud-config.json",
 			},
 		},
@@ -109,6 +121,8 @@ func TestConfigDirCleaner(t *testing.T) {
 				".mydir/.myfile",
 			},
 			verifyExists: []string{
+				"manifests",
+				"pki",
 				".cloud-config.json",
 				".mydir",
 				".mydir/.myfile",
@@ -154,8 +168,8 @@ func TestConfigDirCleaner(t *testing.T) {
 		assertExists(t, tmpDir)
 		assertNotExists(t, filepath.Join(tmpDir, "admin.conf"))
 		assertNotExists(t, filepath.Join(tmpDir, "kubelet.conf"))
-		assertNotExists(t, filepath.Join(tmpDir, "manifests"))
-		assertNotExists(t, filepath.Join(tmpDir, "pki"))
+		assertDirEmpty(t, filepath.Join(tmpDir, "manifests"))
+		assertDirEmpty(t, filepath.Join(tmpDir, "pki"))
 
 		// Verify the files as requested by the test:
 		for _, path := range test.verifyExists {