diff --git a/pkg/kubelet/BUILD b/pkg/kubelet/BUILD index d412f34fcb..a64dd007a0 100644 --- a/pkg/kubelet/BUILD +++ b/pkg/kubelet/BUILD @@ -97,6 +97,7 @@ go_library( "//pkg/util/node:go_default_library", "//pkg/util/oom:go_default_library", "//pkg/util/procfs:go_default_library", + "//pkg/util/removeall:go_default_library", "//pkg/util/term:go_default_library", "//pkg/version:go_default_library", "//pkg/volume:go_default_library", diff --git a/pkg/kubelet/kubelet_volumes.go b/pkg/kubelet/kubelet_volumes.go index 5b8c3146a7..10bcee9f0b 100644 --- a/pkg/kubelet/kubelet_volumes.go +++ b/pkg/kubelet/kubelet_volumes.go @@ -18,7 +18,6 @@ package kubelet import ( "fmt" - "os" "github.com/golang/glog" "k8s.io/apimachinery/pkg/types" @@ -26,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/api/v1" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" + "k8s.io/kubernetes/pkg/util/removeall" "k8s.io/kubernetes/pkg/volume" volumetypes "k8s.io/kubernetes/pkg/volume/util/types" ) @@ -115,7 +115,7 @@ func (kl *Kubelet) cleanupOrphanedPodDirs( continue } glog.V(3).Infof("Orphaned pod %q found, removing", uid) - if err := os.RemoveAll(kl.getPodDir(uid)); err != nil { + if err := removeall.RemoveAllOneFilesystem(kl.mounter, kl.getPodDir(uid)); err != nil { glog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err) errlist = append(errlist, err) } diff --git a/pkg/util/BUILD b/pkg/util/BUILD index 179c9614e3..21f18a3c7e 100644 --- a/pkg/util/BUILD +++ b/pkg/util/BUILD @@ -81,6 +81,7 @@ filegroup( "//pkg/util/parsers:all-srcs", "//pkg/util/procfs:all-srcs", "//pkg/util/rand:all-srcs", + "//pkg/util/removeall:all-srcs", "//pkg/util/resourcecontainer:all-srcs", "//pkg/util/rlimit:all-srcs", "//pkg/util/runtime:all-srcs", diff --git a/pkg/util/removeall/BUILD b/pkg/util/removeall/BUILD new file mode 100644 index 0000000000..a980eb9209 --- /dev/null +++ b/pkg/util/removeall/BUILD @@ -0,0 +1,40 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +load( + "@io_bazel_rules_go//go:def.bzl", + "go_library", + "go_test", +) + +go_test( + name = "go_default_test", + srcs = ["removeall_test.go"], + library = ":go_default_library", + tags = ["automanaged"], + deps = [ + "//pkg/util/mount:go_default_library", + "//vendor:k8s.io/client-go/util/testing", + ], +) + +go_library( + name = "go_default_library", + srcs = ["removeall.go"], + tags = ["automanaged"], + deps = ["//pkg/util/mount:go_default_library"], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], +) diff --git a/pkg/util/removeall/removeall.go b/pkg/util/removeall/removeall.go new file mode 100644 index 0000000000..fa15ac1bdd --- /dev/null +++ b/pkg/util/removeall/removeall.go @@ -0,0 +1,108 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package removeall + +import ( + "fmt" + "io" + "os" + "syscall" + + "k8s.io/kubernetes/pkg/util/mount" +) + +// RemoveAllOneFilesystem removes path and any children it contains. +// It removes everything it can but returns the first error +// it encounters. If the path does not exist, RemoveAll +// returns nil (no error). +// It makes sure it does not cross mount boundary, i.e. it does *not* remove +// files from another filesystems. Like 'rm -rf --one-file-system'. +// It is copied from RemoveAll() sources, with IsLikelyNotMountPoint +func RemoveAllOneFilesystem(mounter mount.Interface, path string) error { + // Simple case: if Remove works, we're done. + err := os.Remove(path) + if err == nil || os.IsNotExist(err) { + return nil + } + + // Otherwise, is this a directory we need to recurse into? + dir, serr := os.Lstat(path) + if serr != nil { + if serr, ok := serr.(*os.PathError); ok && (os.IsNotExist(serr.Err) || serr.Err == syscall.ENOTDIR) { + return nil + } + return serr + } + if !dir.IsDir() { + // Not a directory; return the error from Remove. + return err + } + + // Directory. + isNotMount, err := mounter.IsLikelyNotMountPoint(path) + if err != nil { + return err + } + if !isNotMount { + return fmt.Errorf("cannot delete directory %s: it is a mount point", path) + } + + fd, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + // Race. It was deleted between the Lstat and Open. + // Return nil per RemoveAll's docs. + return nil + } + return err + } + + // Remove contents & return first error. + err = nil + for { + names, err1 := fd.Readdirnames(100) + for _, name := range names { + err1 := RemoveAllOneFilesystem(mounter, path+string(os.PathSeparator)+name) + if err == nil { + err = err1 + } + } + if err1 == io.EOF { + break + } + // If Readdirnames returned an error, use it. + if err == nil { + err = err1 + } + if len(names) == 0 { + break + } + } + + // Close directory, because windows won't remove opened directory. + fd.Close() + + // Remove directory. + err1 := os.Remove(path) + if err1 == nil || os.IsNotExist(err1) { + return nil + } + if err == nil { + err = err1 + } + return err +} diff --git a/pkg/util/removeall/removeall_test.go b/pkg/util/removeall/removeall_test.go new file mode 100644 index 0000000000..938ef08e35 --- /dev/null +++ b/pkg/util/removeall/removeall_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package removeall + +import ( + "errors" + "os" + "path" + "strings" + "testing" + + utiltesting "k8s.io/client-go/util/testing" + "k8s.io/kubernetes/pkg/util/mount" +) + +type fakeMounter struct{} + +var _ mount.Interface = &fakeMounter{} + +func (mounter *fakeMounter) Mount(source string, target string, fstype string, options []string) error { + return errors.New("not implemented") +} +func (mounter *fakeMounter) Unmount(target string) error { + return errors.New("not implemented") +} +func (mounter *fakeMounter) List() ([]mount.MountPoint, error) { + return nil, errors.New("not implemented") +} +func (mounter fakeMounter) DeviceOpened(pathname string) (bool, error) { + return false, errors.New("not implemented") +} +func (mounter *fakeMounter) PathIsDevice(pathname string) (bool, error) { + return false, errors.New("not implemented") +} +func (mounter *fakeMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) { + return "", errors.New("not implemented") +} +func (mounter *fakeMounter) IsLikelyNotMountPoint(file string) (bool, error) { + name := path.Base(file) + if strings.HasPrefix(name, "mount") { + return false, nil + } + if strings.HasPrefix(name, "err") { + return false, errors.New("mock error") + } + return true, nil +} + +func TestRemoveAllOneFilesystem(t *testing.T) { + tests := []struct { + name string + // Items of the test directory. Directories end with "/". + // Directories starting with "mount" are considered to be mount points. + // Directories starting with "err" will cause an error in + // IsLikelyNotMountPoint. + items []string + expectError bool + }{ + { + "empty dir", + []string{}, + false, + }, + { + "non-mount", + []string{ + "dir/", + "dir/file", + "dir2/", + "file2", + }, + false, + }, + { + "mount", + []string{ + "dir/", + "dir/file", + "dir2/", + "file2", + "mount/", + "mount/file3", + }, + true, + }, + { + "innermount", + []string{ + "dir/", + "dir/file", + "dir/dir2/", + "dir/dir2/file2", + "dir/dir2/mount/", + "dir/dir2/mount/file3", + }, + true, + }, + { + "error", + []string{ + "dir/", + "dir/file", + "dir2/", + "file2", + "err/", + "err/file3", + }, + true, + }, + } + + for _, test := range tests { + tmpDir, err := utiltesting.MkTmpdir("removeall-" + test.name + "-") + if err != nil { + t.Fatalf("Can't make a tmp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + // Create the directory structure + for _, item := range test.items { + if strings.HasSuffix(item, "/") { + item = strings.TrimRight(item, "/") + if err = os.Mkdir(path.Join(tmpDir, item), 0777); err != nil { + t.Fatalf("error creating %s: %v", item, err) + } + } else { + f, err := os.Create(path.Join(tmpDir, item)) + if err != nil { + t.Fatalf("error creating %s: %v", item, err) + } + f.Close() + } + } + + mounter := &fakeMounter{} + err = RemoveAllOneFilesystem(mounter, tmpDir) + if err == nil && test.expectError { + t.Errorf("test %q failed: expected error and got none", test.name) + } + if err != nil && !test.expectError { + t.Errorf("test %q failed: unexpected error: %v", test.name, err) + } + } +}