github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #61044 from liggitt/subpath-master 

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

subpath fixes

fixes #60813 for master / 1.10

```release-note
Fixes CVE-2017-1002101 - See https://issue.k8s.io/60813 for details
```
pull/6/head
Kubernetes Submit Queue 2018-03-12 11:51:59 -07:00 committed by GitHub
parent e88a0cebfb f6d97b5d2b
commit 3d1331f297
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 3662 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/apis/core/validation/validation.go
View File

@ -2264,7 +2264,11 @@ func ValidateVolumeMounts(mounts []core.VolumeMount, voldevices map[string]strin
}
if len(mnt.SubPath) > 0 {
allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPath, fldPath.Child("subPath"))...)
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
allErrs = append(allErrs, field.Forbidden(fldPath.Child("subPath"), "subPath is disabled by feature-gate"))
} else {
allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPath, fldPath.Child("subPath"))...)
}
}
if mnt.MountPropagation != nil {

62
pkg/apis/core/validation/validation_test.go
View File

@ -4766,6 +4766,68 @@ func TestValidateVolumeMounts(t *testing.T) {
}
}
func TestValidateDisabledSubpath(t *testing.T) {
utilfeature.DefaultFeatureGate.Set("VolumeSubpath=false")
defer utilfeature.DefaultFeatureGate.Set("VolumeSubpath=true")
volumes := []core.Volume{
{Name: "abc", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim1"}}},
{Name: "abc-123", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim2"}}},
{Name: "123", VolumeSource: core.VolumeSource{HostPath: &core.HostPathVolumeSource{Path: "/foo/baz", Type: newHostPathType(string(core.HostPathUnset))}}},
}
vols, v1err := ValidateVolumes(volumes, field.NewPath("field"))
if len(v1err) > 0 {
t.Errorf("Invalid test volume - expected success %v", v1err)
return
}
container := core.Container{
SecurityContext: nil,
}
goodVolumeDevices := []core.VolumeDevice{
{Name: "xyz", DevicePath: "/foofoo"},
{Name: "uvw", DevicePath: "/foofoo/share/test"},
}
cases := map[string]struct {
mounts []core.VolumeMount
expectError bool
}{
"subpath not specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
},
},
false,
},
"subpath specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
SubPath: "baz",
},
},
true,
},
}
for name, test := range cases {
errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
if len(errs) != 0 && !test.expectError {
t.Errorf("test %v failed: %+v", name, errs)
}
if len(errs) == 0 && test.expectError {
t.Errorf("test %v failed, expected error", name)
}
}
}
func TestValidateMountPropagation(t *testing.T) {
bTrue := true
bFalse := false

8
pkg/features/kube_features.go
View File

@ -267,6 +267,13 @@ const (
//
// Enables control over the primary group ID of containers' init processes.
RunAsGroup utilfeature.Feature = "RunAsGroup"
// owner: @saad-ali
// ga
//
// Allow mounting a subpath of a volume in a container
// Do not remove this feature gate even though it's GA
VolumeSubpath utilfeature.Feature = "VolumeSubpath"
)
func init() {
@ -313,6 +320,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
CRIContainerLogRotation: {Default: false, PreRelease: utilfeature.Alpha},
GCERegionalPersistentDisk: {Default: true, PreRelease: utilfeature.Beta},
RunAsGroup: {Default: false, PreRelease: utilfeature.Alpha},
VolumeSubpath: {Default: true, PreRelease: utilfeature.GA},
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
// unintentionally on either side:

12
pkg/kubelet/cm/container_manager_linux_test.go
View File

@ -95,6 +95,18 @@ func (mi *fakeMountInterface) ExistsPath(pathname string) bool {
return true
}
func (mi *fakeMountInterface) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) {
return "", nil, nil
}
func (mi *fakeMountInterface) CleanSubPaths(_, _ string) error {
return nil
}
func (mi *fakeMountInterface) SafeMakeDir(_, _ string, _ os.FileMode) error {
return nil
}
func fakeContainerMgrMountInt() mount.Interface {
return &fakeMountInterface{
[]mount.MountPoint{

2
pkg/kubelet/container/helpers.go
View File

@ -46,7 +46,7 @@ type HandlerRunner interface {
// RuntimeHelper wraps kubelet to make container runtime
// able to get necessary informations like the RunContainerOptions, DNS settings, Host IP.
type RuntimeHelper interface {
GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (contOpts *RunContainerOptions, err error)
GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (contOpts *RunContainerOptions, cleanupAction func(), err error)
GetPodDNS(pod *v1.Pod) (dnsConfig *runtimeapi.DNSConfig, err error)
// GetPodCgroupParent returns the CgroupName identifier, and its literal cgroupfs form on the host
// of a pod.

4
pkg/kubelet/container/testing/fake_runtime_helper.go
View File

@ -34,12 +34,12 @@ type FakeRuntimeHelper struct {
Err error
}
func (f *FakeRuntimeHelper) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, error) {
func (f *FakeRuntimeHelper) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, func(), error) {
var opts kubecontainer.RunContainerOptions
if len(container.TerminationMessagePath) != 0 {
opts.PodContainerDir = f.PodContainerDir
}
return &opts, nil
return &opts, nil, nil
}
func (f *FakeRuntimeHelper) GetPodCgroupParent(pod *v1.Pod) string {

76
pkg/kubelet/kubelet_pods.go
View File

@ -61,6 +61,7 @@ import (
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
"k8s.io/kubernetes/pkg/kubelet/util/format"
utilfile "k8s.io/kubernetes/pkg/util/file"
mountutil "k8s.io/kubernetes/pkg/util/mount"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
volumevalidation "k8s.io/kubernetes/pkg/volume/validation"
@ -160,7 +161,7 @@ func (kl *Kubelet) makeBlockVolumes(pod *v1.Pod, container *v1.Container, podVol
}
// makeMounts determines the mount points for the given container.
func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap) ([]kubecontainer.Mount, error) {
func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, hostDomain, podIP string, podVolumes kubecontainer.VolumeMap, mounter mountutil.Interface) ([]kubecontainer.Mount, func(), error) {
// Kubernetes only mounts on /etc/hosts if:
// - container is not an infrastructure (pause) container
// - container is not already mounting on /etc/hosts
@ -170,13 +171,14 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
mountEtcHostsFile := len(podIP) > 0 && runtime.GOOS != "windows"
glog.V(3).Infof("container: %v/%v/%v podIP: %q creating hosts mount: %v", pod.Namespace, pod.Name, container.Name, podIP, mountEtcHostsFile)
mounts := []kubecontainer.Mount{}
for _, mount := range container.VolumeMounts {
var cleanupAction func() = nil
for i, mount := range container.VolumeMounts {
// do not mount /etc/hosts if container is already mounting on the path
mountEtcHostsFile = mountEtcHostsFile && (mount.MountPath != etcHostsPath)
vol, ok := podVolumes[mount.Name]
if !ok || vol.Mounter == nil {
glog.Errorf("Mount cannot be satisfied for container %q, because the volume is missing or the volume mounter is nil: %+v", container.Name, mount)
return nil, fmt.Errorf("cannot find volume %q to mount into container %q", mount.Name, container.Name)
return nil, cleanupAction, fmt.Errorf("cannot find volume %q to mount into container %q", mount.Name, container.Name)
}
relabelVolume := false
@ -189,25 +191,33 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
}
hostPath, err := volumeutil.GetPath(vol.Mounter)
if err != nil {
return nil, err
return nil, cleanupAction, err
}
if mount.SubPath != "" {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
}
if filepath.IsAbs(mount.SubPath) {
return nil, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath)
return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath)
}
err = volumevalidation.ValidatePathNoBacksteps(mount.SubPath)
if err != nil {
return nil, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
}
fileinfo, err := os.Lstat(hostPath)
if err != nil {
return nil, err
return nil, cleanupAction, err
}
perm := fileinfo.Mode()
hostPath = filepath.Join(hostPath, mount.SubPath)
volumePath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return nil, cleanupAction, err
}
hostPath = filepath.Join(volumePath, mount.SubPath)
if subPathExists, err := utilfile.FileOrSymlinkExists(hostPath); err != nil {
glog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath)
@ -216,17 +226,25 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
// incorrect ownership and mode. For example, the sub path directory must have at least g+rwx
// when the pod specifies an fsGroup, and if the directory is not created here, Docker will
// later auto-create it with the incorrect mode 0750
if err := os.MkdirAll(hostPath, perm); err != nil {
glog.Errorf("failed to mkdir:%s", hostPath)
return nil, err
}
// chmod the sub path because umask may have prevented us from making the sub path with the same
// permissions as the mounter path
if err := os.Chmod(hostPath, perm); err != nil {
return nil, err
// Make extra care not to escape the volume!
if err := mounter.SafeMakeDir(hostPath, volumePath, perm); err != nil {
glog.Errorf("failed to mkdir %q: %v", hostPath, err)
return nil, cleanupAction, err
}
}
hostPath, cleanupAction, err = mounter.PrepareSafeSubpath(mountutil.Subpath{
VolumeMountIndex: i,
Path: hostPath,
VolumeName: mount.Name,
VolumePath: volumePath,
PodDir: podDir,
ContainerName: container.Name,
})
if err != nil {
// Don't pass detailed error back to the user because it could give information about host filesystem
glog.Errorf("failed to prepare subPath for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
return nil, cleanupAction, fmt.Errorf("failed to prepare subPath for volumeMount %q of container %q", mount.Name, container.Name)
}
}
// Docker Volume Mounts fail on Windows if it is not of the form C:/
@ -242,7 +260,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
propagation, err := translateMountPropagation(mount.MountPropagation)
if err != nil {
return nil, err
return nil, cleanupAction, err
}
glog.V(5).Infof("Pod %q container %q mount %q has propagation %q", format.Pod(pod), container.Name, mount.Name, propagation)
@ -261,11 +279,11 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
hostAliases := pod.Spec.HostAliases
hostsMount, err := makeHostsMount(podDir, podIP, hostName, hostDomain, hostAliases, pod.Spec.HostNetwork)
if err != nil {
return nil, err
return nil, cleanupAction, err
}
mounts = append(mounts, *hostsMount)
}
return mounts, nil
return mounts, cleanupAction, nil
}
// translateMountPropagation transforms v1.MountPropagationMode to
@ -437,17 +455,17 @@ func (kl *Kubelet) GetPodCgroupParent(pod *v1.Pod) string {
// GenerateRunContainerOptions generates the RunContainerOptions, which can be used by
// the container runtime to set parameters for launching a container.
func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, error) {
func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Container, podIP string) (*kubecontainer.RunContainerOptions, func(), error) {
opts, err := kl.containerManager.GetResources(pod, container)
if err != nil {
return nil, err
return nil, nil, err
}
cgroupParent := kl.GetPodCgroupParent(pod)
opts.CgroupParent = cgroupParent
hostname, hostDomainName, err := kl.GeneratePodHostNameAndDomain(pod)
if err != nil {
return nil, err
return nil, nil, err
}
opts.Hostname = hostname
podName := volumeutil.GetUniquePodName(pod)
@ -457,7 +475,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
// TODO(random-liu): Move following convert functions into pkg/kubelet/container
devices, err := kl.makeGPUDevices(pod, container)
if err != nil {
return nil, err
return nil, nil, err
}
opts.Devices = append(opts.Devices, devices...)
@ -466,20 +484,20 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
blkutil := volumepathhandler.NewBlockVolumePathHandler()
blkVolumes, err := kl.makeBlockVolumes(pod, container, volumes, blkutil)
if err != nil {
return nil, err
return nil, nil, err
}
opts.Devices = append(opts.Devices, blkVolumes...)
}
mounts, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes)
mounts, cleanupAction, err := makeMounts(pod, kl.getPodDir(pod.UID), container, hostname, hostDomainName, podIP, volumes, kl.mounter)
if err != nil {
return nil, err
return nil, cleanupAction, err
}
opts.Mounts = append(opts.Mounts, mounts...)
envs, err := kl.makeEnvironmentVariables(pod, container, podIP)
if err != nil {
return nil, err
return nil, cleanupAction, err
}
opts.Envs = append(opts.Envs, envs...)
@ -499,7 +517,7 @@ func (kl *Kubelet) GenerateRunContainerOptions(pod *v1.Pod, container *v1.Contai
opts.EnableHostUserNamespace = kl.enableHostUserNamespace(pod)
}
return opts, nil
return opts, cleanupAction, nil
}
var masterServices = sets.NewString("kubernetes")

62
pkg/kubelet/kubelet_pods_test.go
View File

@ -47,6 +47,7 @@ import (
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
"k8s.io/kubernetes/pkg/kubelet/server/portforward"
"k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
"k8s.io/kubernetes/pkg/util/mount"
volumetest "k8s.io/kubernetes/pkg/volume/testing"
)
@ -303,6 +304,7 @@ func TestMakeMounts(t *testing.T) {
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
fm := &mount.FakeMounter{}
pod := v1.Pod{
Spec: v1.PodSpec{
HostNetwork: true,
@ -315,7 +317,7 @@ func TestMakeMounts(t *testing.T) {
return
}
mounts, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes)
mounts, _, err := makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm)
// validate only the error if we expect an error
if tc.expectErr {
@ -338,7 +340,7 @@ func TestMakeMounts(t *testing.T) {
t.Errorf("Failed to enable feature gate for MountPropagation: %v", err)
return
}
mounts, err = makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes)
mounts, _, err = makeMounts(&pod, "/pod", &tc.container, "fakepodname", "", "", tc.podVolumes, fm)
if !tc.expectErr {
expectedPrivateMounts := []kubecontainer.Mount{}
for _, mount := range tc.expectedMounts {
@ -353,6 +355,62 @@ func TestMakeMounts(t *testing.T) {
}
}
func TestDisabledSubpath(t *testing.T) {
fm := &mount.FakeMounter{}
pod := v1.Pod{
Spec: v1.PodSpec{
HostNetwork: true,
},
}
podVolumes := kubecontainer.VolumeMap{
"disk": kubecontainer.VolumeInfo{Mounter: &stubVolume{path: "/mnt/disk"}},
}
cases := map[string]struct {
container v1.Container
expectError bool
}{
"subpath not specified": {
v1.Container{
VolumeMounts: []v1.VolumeMount{
{
MountPath: "/mnt/path3",
Name: "disk",
ReadOnly: true,
},
},
},
false,
},
"subpath specified": {
v1.Container{
VolumeMounts: []v1.VolumeMount{
{
MountPath: "/mnt/path3",
SubPath: "/must/not/be/absolute",
Name: "disk",
ReadOnly: true,
},
},
},
true,
},
}
utilfeature.DefaultFeatureGate.Set("VolumeSubpath=false")
defer utilfeature.DefaultFeatureGate.Set("VolumeSubpath=true")
for name, test := range cases {
_, _, err := makeMounts(&pod, "/pod", &test.container, "fakepodname", "", "", podVolumes, fm)
if err != nil && !test.expectError {
t.Errorf("test %v failed: %v", name, err)
}
if err == nil && test.expectError {
t.Errorf("test %v failed: expected error", name)
}
}
}
func TestMakeBlockVolumes(t *testing.T) {
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
defer testKubelet.Cleanup()

4
pkg/kubelet/kubelet_pods_windows_test.go
View File

@ -24,6 +24,7 @@ import (
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/util/mount"
)
func TestMakeMountsWindows(t *testing.T) {
@ -64,7 +65,8 @@ func TestMakeMountsWindows(t *testing.T) {
},
}
mounts, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes)
fm := &mount.FakeMounter{}
mounts, _, _ := makeMounts(&pod, "/pod", &container, "fakepodname", "", "", podVolumes, fm)
expectedMounts := []kubecontainer.Mount{
{

22
pkg/kubelet/kuberuntime/kuberuntime_container.go
View File

@ -106,11 +106,15 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
restartCount = containerStatus.RestartCount + 1
}
containerConfig, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef, containerType)
containerConfig, cleanupAction, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef, containerType)
if cleanupAction != nil {
defer cleanupAction()
}
if err != nil {
m.recordContainerEvent(pod, container, "", v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err))
return grpc.ErrorDesc(err), ErrCreateContainerConfig
}
containerID, err := m.runtimeService.CreateContainer(podSandboxID, containerConfig, podSandboxConfig)
if err != nil {
m.recordContainerEvent(pod, container, containerID, v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", grpc.ErrorDesc(err))
@ -172,27 +176,27 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
}
// generateContainerConfig generates container config for kubelet runtime v1.
func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string, containerType kubecontainer.ContainerType) (*runtimeapi.ContainerConfig, error) {
opts, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string, containerType kubecontainer.ContainerType) (*runtimeapi.ContainerConfig, func(), error) {
opts, cleanupAction, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
if err != nil {
return nil, err
return nil, nil, err
}
uid, username, err := m.getImageUser(container.Image)
if err != nil {
return nil, err
return nil, cleanupAction, err
}
// Verify RunAsNonRoot. Non-root verification only supports numeric user.
if err := verifyRunAsNonRoot(pod, container, uid, username); err != nil {
return nil, err
return nil, cleanupAction, err
}
command, args := kubecontainer.ExpandContainerCommandAndArgs(container, opts.Envs)
logDir := BuildContainerLogsDirectory(kubetypes.UID(pod.UID), container.Name)
err = m.osInterface.MkdirAll(logDir, 0755)
if err != nil {
return nil, fmt.Errorf("create container log directory for container %s failed: %v", container.Name, err)
return nil, cleanupAction, fmt.Errorf("create container log directory for container %s failed: %v", container.Name, err)
}
containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
restartCountUint32 := uint32(restartCount)
@ -217,7 +221,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai
// set platform specific configurations.
if err := m.applyPlatformSpecificContainerConfig(config, container, pod, uid, username); err != nil {
return nil, err
return nil, cleanupAction, err
}
// set environment variables
@ -231,7 +235,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai
}
config.Envs = envs
return config, nil
return config, cleanupAction, nil
}
// makeDevices generates container devices for kubelet runtime v1.

8
pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go
View File

@ -32,7 +32,7 @@ func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerInde
container := &pod.Spec.Containers[containerIndex]
podIP := ""
restartCount := 0
opts, _ := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
opts, _, _ := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP)
containerLogsPath := buildContainerLogsPath(container.Name, restartCount)
restartCountUint32 := uint32(restartCount)
envs := make([]*runtimeapi.KeyValue, len(opts.Envs))
@ -84,7 +84,7 @@ func TestGenerateContainerConfig(t *testing.T) {
}
expectedConfig := makeExpectedConfig(m, pod, 0)
containerConfig, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
containerConfig, _, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
assert.NoError(t, err)
assert.Equal(t, expectedConfig, containerConfig, "generate container config for kubelet runtime v1.")
@ -113,7 +113,7 @@ func TestGenerateContainerConfig(t *testing.T) {
},
}
_, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
_, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
assert.Error(t, err)
imageId, _ := imageService.PullImage(&runtimeapi.ImageSpec{Image: "busybox"}, nil)
@ -125,6 +125,6 @@ func TestGenerateContainerConfig(t *testing.T) {
podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsUser = nil
podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsNonRoot = &runAsNonRootTrue
_, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
_, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, kubecontainer.ContainerTypeRegular)
assert.Error(t, err, "RunAsNonRoot should fail for non-numeric username")
}

2
pkg/kubelet/kuberuntime/kuberuntime_manager_test.go
View File

@ -143,7 +143,7 @@ func makeFakeContainer(t *testing.T, m *kubeGenericRuntimeManager, template cont
sandboxConfig, err := m.generatePodSandboxConfig(template.pod, template.sandboxAttempt)
assert.NoError(t, err, "generatePodSandboxConfig for container template %+v", template)
containerConfig, err := m.generateContainerConfig(template.container, template.pod, template.attempt, "", template.container.Image, template.containerType)
containerConfig, _, err := m.generateContainerConfig(template.container, template.pod, template.attempt, "", template.container.Image, template.containerType)
assert.NoError(t, err, "generateContainerConfig for container template %+v", template)
podSandboxID := apitest.BuildSandboxName(sandboxConfig.Metadata)

2
pkg/kubelet/rkt/rkt.go
View File

@ -842,7 +842,7 @@ func (r *Runtime) newAppcRuntimeApp(pod *v1.Pod, podIP string, c v1.Container, r
}
// TODO: determine how this should be handled for rkt
opts, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP)
opts, _, err := r.runtimeHelper.GenerateRunContainerOptions(pod, &c, podIP)
if err != nil {
return err
}

8
pkg/kubelet/util/BUILD
View File

@ -8,9 +8,13 @@ load(
go_test(
name = "go_default_test",
srcs = ["util_test.go"],
srcs = [
"util_test.go",
],
embed = [":go_default_library"],
deps = ["//vendor/github.com/stretchr/testify/assert:go_default_library"],
deps = [
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
)
go_library(

13
pkg/kubelet/util/util.go
View File

@ -19,6 +19,8 @@ package util
import (
"fmt"
"net/url"
"path/filepath"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@ -45,3 +47,14 @@ func parseEndpoint(endpoint string) (string, string, error) {
return u.Scheme, "", fmt.Errorf("protocol %q not supported", u.Scheme)
}
}
func pathWithinBase(fullPath, basePath string) bool {
rel, err := filepath.Rel(basePath, fullPath)
if err != nil {
return false
}
if strings.HasPrefix(rel, "..") {
return false
}
return true
}

9
pkg/kubelet/util/util_unsupported.go
View File

@ -31,3 +31,12 @@ func CreateListener(endpoint string) (net.Listener, error) {
func GetAddressAndDialer(endpoint string) (string, func(addr string, timeout time.Duration) (net.Conn, error), error) {
return "", nil, fmt.Errorf("GetAddressAndDialer is unsupported in this build")
}
// LockAndCheckSubPath empty implementation
func LockAndCheckSubPath(volumePath, subPath string) ([]uintptr, error) {
return []uintptr{}, nil
}
// UnlockPath empty implementation
func UnlockPath(fileHandles []uintptr) {
}

4
pkg/kubelet/volumemanager/reconciler/reconciler.go
View File

@ -169,7 +169,7 @@ func (rc *reconciler) reconcile() {
// Volume is mounted, unmount it
glog.V(5).Infof(mountedVolume.GenerateMsgDetailed("Starting operationExecutor.UnmountVolume", ""))
err := rc.operationExecutor.UnmountVolume(
mountedVolume.MountedVolume, rc.actualStateOfWorld)
mountedVolume.MountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
if err != nil &&
!nestedpendingoperations.IsAlreadyExists(err) &&
!exponentialbackoff.IsExponentialBackoff(err) {
@ -401,7 +401,7 @@ func (rc *reconciler) cleanupMounts(volume podVolume) {
}
// TODO: Currently cleanupMounts only includes UnmountVolume operation. In the next PR, we will add
// to unmount both volume and device in the same routine.
err := rc.operationExecutor.UnmountVolume(mountedVolume, rc.actualStateOfWorld)
err := rc.operationExecutor.UnmountVolume(mountedVolume, rc.actualStateOfWorld, rc.kubeletPodsDir)
if err != nil {
glog.Errorf(mountedVolume.GenerateErrorDetailed(fmt.Sprintf("volumeHandler.UnmountVolumeHandler for UnmountVolume failed"), err).Error())
return

2
pkg/kubelet/volumemanager/reconciler/reconciler_test.go
View File

@ -908,7 +908,7 @@ func Test_GenerateUnmapVolumeFunc_Plugin_Not_Found(t *testing.T) {
volumeToUnmount := operationexecutor.MountedVolume{
PluginName: "fake-file-plugin",
VolumeSpec: tmpSpec}
err := oex.UnmountVolume(volumeToUnmount, asw)
err := oex.UnmountVolume(volumeToUnmount, asw, "" /* podsDir */)
// Assert
if assert.Error(t, err) {
assert.Contains(t, err.Error(), tc.expectedErrMsg)

20
pkg/util/mount/BUILD
View File

@ -1,10 +1,4 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
@ -72,6 +66,7 @@ go_library(
"//conditions:default": [],
}),
importpath = "k8s.io/kubernetes/pkg/util/mount",
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
@ -104,7 +99,15 @@ go_test(
embed = [":go_default_library"],
deps = [
"//vendor/k8s.io/utils/exec/testing:go_default_library",
],
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/github.com/golang/glog:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/github.com/stretchr/testify/assert:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
@ -118,4 +121,5 @@ filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

13
pkg/util/mount/exec_mount.go
View File

@ -20,6 +20,7 @@ package mount
import (
"fmt"
"os"
"github.com/golang/glog"
)
@ -138,3 +139,15 @@ func (m *execMounter) MakeDir(pathname string) error {
func (m *execMounter) ExistsPath(pathname string) bool {
return m.wrappedMounter.ExistsPath(pathname)
}
func (m *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return m.wrappedMounter.PrepareSafeSubpath(subPath)
}
func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return m.wrappedMounter.CleanSubPaths(podDir, volumeName)
}
func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return m.wrappedMounter.SafeMakeDir(pathname, base, perm)
}

12
pkg/util/mount/exec_mount_test.go
View File

@ -20,6 +20,7 @@ package mount
import (
"fmt"
"os"
"reflect"
"strings"
"testing"
@ -151,3 +152,14 @@ func (fm *fakeMounter) ExistsPath(pathname string) bool {
func (fm *fakeMounter) GetFileType(pathname string) (FileType, error) {
return FileTypeFile, nil
}
func (fm *fakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (fm *fakeMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (fm *fakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}

13
pkg/util/mount/exec_mount_unsupported.go
View File

@ -20,6 +20,7 @@ package mount
import (
"errors"
"os"
)
type execMounter struct{}
@ -85,3 +86,15 @@ func (mounter *execMounter) MakeFile(pathname string) error {
func (mounter *execMounter) ExistsPath(pathname string) bool {
return true
}
func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}

11
pkg/util/mount/fake.go
View File

@ -197,3 +197,14 @@ func (f *FakeMounter) MakeFile(pathname string) error {
func (f *FakeMounter) ExistsPath(pathname string) bool {
return false
}
func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}

59
pkg/util/mount/mount.go
View File

@ -22,6 +22,7 @@ import (
"os"
"path/filepath"
"strings"
"syscall"
)
type FileType string
@ -83,9 +84,45 @@ type Interface interface {
// MakeDir creates a new directory.
// Will operate in the host mount namespace if kubelet is running in a container
MakeDir(pathname string) error
// SafeMakeDir makes sure that the created directory does not escape given
// base directory mis-using symlinks. The directory is created in the same
// mount namespace as where kubelet is running. Note that the function makes
// sure that it creates the directory somewhere under the base, nothing
// else. E.g. if the directory already exists, it may exists outside of the
// base due to symlinks.
SafeMakeDir(pathname string, base string, perm os.FileMode) error
// ExistsPath checks whether the path exists.
// Will operate in the host mount namespace if kubelet is running in a container
ExistsPath(pathname string) bool
// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
// pod volume directory.
CleanSubPaths(podDir string, volumeName string) error
// PrepareSafeSubpath does everything that's necessary to prepare a subPath
// that's 1) inside given volumePath and 2) immutable after this call.
//
// newHostPath - location of prepared subPath. It should be used instead of
// hostName when running the container.
// cleanupAction - action to run when the container is running or it failed to start.
//
// CleanupAction must be called immediately after the container with given
// subpath starts. On the other hand, Interface.CleanSubPaths must be called
// when the pod finishes.
PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
}
type Subpath struct {
// index of the VolumeMount for this container
VolumeMountIndex int
// Full path to the subpath directory on the host
Path string
// name of the volume that is a valid directory name.
VolumeName string
// Full path to the volume path
VolumePath string
// Path to the pod's directory, including pod UID
PodDir string
// Name of the container
ContainerName string
}
// Exec executes command where mount utilities are. This can be either the host,
@ -195,6 +232,13 @@ func GetDeviceNameFromMount(mounter Interface, mountPath string) (string, int, e
return device, refCount, nil
}
func isNotDirErr(err error) bool {
if e, ok := err.(*os.PathError); ok && e.Err == syscall.ENOTDIR {
return true
}
return false
}
// IsNotMountPoint determines if a directory is a mountpoint.
// It should return ErrNotExist when the directory does not exist.
// This method uses the List() of all mountpoints
@ -210,7 +254,7 @@ func IsNotMountPoint(mounter Interface, file string) (bool, error) {
notMnt = true
notMntErr = nil
}
if notMntErr != nil {
if notMntErr != nil && isNotDirErr(notMntErr) {
return notMnt, notMntErr
}
// identified as mountpoint, so return this fact
@ -274,3 +318,16 @@ func HasMountRefs(mountPath string, mountRefs []string) bool {
}
return count > 0
}
// pathWithinBase checks if give path is within given base directory.
func pathWithinBase(fullPath, basePath string) bool {
rel, err := filepath.Rel(basePath, fullPath)
if err != nil {
return false
}
if strings.HasPrefix(rel, "..") {
// Needed to escape the base path
return false
}
return true
}

445
pkg/util/mount/mount_linux.go
View File

@ -21,6 +21,7 @@ package mount
import (
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
@ -49,6 +50,11 @@ const (
fsckErrorsCorrected = 1
// 'fsck' found errors but exited without correcting them
fsckErrorsUncorrected = 4
// place for subpath mounts
containerSubPathDirectoryName = "volume-subpaths"
// syscall.Openat flags used to traverse directories not following symlinks
nofollowFlags = syscall.O_RDONLY | syscall.O_NOFOLLOW
)
// Mounter provides the default implementation of mount.Interface
@ -636,6 +642,7 @@ func isShared(path string, filename string) (bool, error) {
}
type mountInfo struct {
// Path of the mount point
mountPoint string
// list of "optional parameters", mount propagation is one of them
optional []string
@ -655,6 +662,7 @@ func parseMountInfo(filename string) ([]mountInfo, error) {
// the last split() item is empty string following the last \n
continue
}
// See `man proc` for authoritative description of format of the file.
fields := strings.Fields(line)
if len(fields) < 7 {
return nil, fmt.Errorf("wrong number of fields in (expected %d, got %d): %s", 8, len(fields), line)
@ -663,6 +671,7 @@ func parseMountInfo(filename string) ([]mountInfo, error) {
mountPoint: fields[4],
optional: []string{},
}
// All fields until "-" are "optional fields".
for i := 6; i < len(fields) && fields[i] != "-"; i++ {
info.optional = append(info.optional, fields[i])
}
@ -698,3 +707,439 @@ func doMakeRShared(path string, mountInfoFilename string) error {
return nil
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
newHostPath, err = doBindSubPath(mounter, subPath, os.Getpid())
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
// This implementation is shared between Linux and NsEnterMounter
// kubeletPid is PID of kubelet in the PID namespace where bind-mount is done,
// i.e. pid on the *host* if kubelet runs in a container.
func doBindSubPath(mounter Interface, subpath Subpath, kubeletPid int) (hostPath string, err error) {
// Check early for symlink. This is just a pre-check to avoid bind-mount
// before the final check.
evalSubPath, err := filepath.EvalSymlinks(subpath.Path)
if err != nil {
return "", fmt.Errorf("evalSymlinks %q failed: %v", subpath.Path, err)
}
glog.V(5).Infof("doBindSubPath %q, full subpath %q for volumepath %q", subpath.Path, evalSubPath, subpath.VolumePath)
evalSubPath = filepath.Clean(evalSubPath)
if !pathWithinBase(evalSubPath, subpath.VolumePath) {
return "", fmt.Errorf("subpath %q not within volume path %q", evalSubPath, subpath.VolumePath)
}
// Prepare directory for bind mounts
// containerName is DNS label, i.e. safe as a directory name.
bindDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
err = os.MkdirAll(bindDir, 0750)
if err != nil && !os.IsExist(err) {
return "", fmt.Errorf("error creating directory %s: %s", bindDir, err)
}
bindPathTarget := filepath.Join(bindDir, strconv.Itoa(subpath.VolumeMountIndex))
success := false
defer func() {
// Cleanup subpath on error
if !success {
glog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
glog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
// Check it's not already bind-mounted
notMount, err := IsNotMountPoint(mounter, bindPathTarget)
if err != nil {
if !os.IsNotExist(err) {
return "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
}
// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
notMount = true
}
if !notMount {
// It's already mounted
glog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
success = true
return bindPathTarget, nil
}
// Create target of the bind mount. A directory for directories, empty file
// for everything else.
t, err := os.Lstat(subpath.Path)
if err != nil {
return "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
}
if t.Mode()&os.ModeDir > 0 {
if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
return "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
}
} else {
// "/bin/touch <bindDir>".
// A file is enough for all possible targets (symlink, device, pipe,
// socket, ...), bind-mounting them into a file correctly changes type
// of the target file.
if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
return "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
}
}
// Safe open subpath and get the fd
fd, err := doSafeOpen(evalSubPath, subpath.VolumePath)
if err != nil {
return "", fmt.Errorf("error opening subpath %v: %v", evalSubPath, err)
}
defer syscall.Close(fd)
mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
// Do the bind mount
glog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
}
success = true
glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
// This implementation is shared between Linux and NsEnterMounter
func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error {
glog.V(4).Infof("Cleaning up subpath mounts for %s", podDir)
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
containerDirs, err := ioutil.ReadDir(subPathDir)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return fmt.Errorf("error reading %s: %s", subPathDir, err)
}
for _, containerDir := range containerDirs {
if !containerDir.IsDir() {
glog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
continue
}
glog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
subPaths, err := ioutil.ReadDir(fullContainerDirPath)
if err != nil {
return fmt.Errorf("error reading %s: %s", fullContainerDirPath, err)
}
for _, subPath := range subPaths {
if err = doCleanSubPath(mounter, fullContainerDirPath, subPath.Name()); err != nil {
return err
}
}
// Whole container has been processed, remove its directory.
if err := os.Remove(fullContainerDirPath); err != nil {
return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
}
glog.V(5).Infof("Removed %s", fullContainerDirPath)
}
// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
if err := os.Remove(subPathDir); err != nil {
return fmt.Errorf("error deleting %s: %s", subPathDir, err)
}
glog.V(5).Infof("Removed %s", subPathDir)
// Remove entire subpath directory if it's the last one
podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
}
glog.V(5).Infof("Removed %s", podSubPathDir)
return nil
}
// doCleanSubPath tears down the single subpath bind mount
func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error {
// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
glog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
notMnt, err := IsNotMountPoint(mounter, fullSubPath)
if err != nil {
return fmt.Errorf("error checking %s for mount: %s", fullSubPath, err)
}
// Unmount it
if !notMnt {
if err = mounter.Unmount(fullSubPath); err != nil {
return fmt.Errorf("error unmounting %s: %s", fullSubPath, err)
}
glog.V(5).Infof("Unmounted %s", fullSubPath)
}
// Remove it *non*-recursively, just in case there were some hiccups.
if err = os.Remove(fullSubPath); err != nil {
return fmt.Errorf("error deleting %s: %s", fullSubPath, err)
}
glog.V(5).Infof("Removed %s", fullSubPath)
return nil
}
// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
func cleanSubPath(mounter Interface, subpath Subpath) error {
containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
// Clean subdir bindmount
if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
return err
}
// Recusively remove directories if empty
if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
return err
}
return nil
}
// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
// if it is empty. It stops once it encounters a directory that has content
func removeEmptyDirs(baseDir, endDir string) error {
if !pathWithinBase(endDir, baseDir) {
return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
}
for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
s, err := os.Stat(curDir)
if err != nil {
if os.IsNotExist(err) {
glog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
continue
}
return fmt.Errorf("error stat %q: %v", curDir, err)
}
if !s.IsDir() {
return fmt.Errorf("path %q not a directory", curDir)
}
err = os.Remove(curDir)
if os.IsExist(err) {
glog.V(5).Infof("Directory %q not empty, not removing", curDir)
break
} else if err != nil {
return fmt.Errorf("error removing directory %q: %v", curDir, err)
}
glog.V(5).Infof("Removed directory %q", curDir)
}
return nil
}
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return doSafeMakeDir(pathname, base, perm)
}
// This implementation is shared between Linux and NsEnterMounter
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
glog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !pathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
glog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", existingPath, err)
}
if !pathWithinBase(fullExistingPath, base) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
glog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
parentFD, err := doSafeOpen(fullExistingPath, base)
if err != nil {
return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
}
childFD := -1
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
}
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink.
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
glog.V(4).Infof("Creating %s", dir)
err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
if err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
// Dive into the created directory
childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
if err != nil {
return fmt.Errorf("cannot open %s: %s", currentPath, err)
}
// We can be sure that childFD is safe to use. It could be changed
// by user after Mkdirat() and before Openat(), however:
// - it could not be changed to symlink - we use nofollowFlags
// - it could be changed to a file (or device, pipe, socket, ...)
// but either subsequent Mkdirat() fails or we mount this file
// to user's container. Security is no violated in both cases
// and user either gets error or the file that it can already access.
if err = syscall.Close(parentFD); err != nil {
glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
}
parentFD = childFD
childFD = -1
}
// Everything was created. mkdirat(..., perm) above was affected by current
// umask and we must apply the right permissions to the last directory
// (that's the one that will be available to the container as subpath)
// so user can read/write it. This is the behavior of previous code.
// TODO: chmod all created directories, not just the last one.
// parentFD is the last created directory.
if err = syscall.Fchmod(parentFD, uint32(perm)&uint32(os.ModePerm)); err != nil {
return fmt.Errorf("chmod %q failed: %s", currentPath, err)
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
dirs := strings.Split(rel, string(filepath.Separator))
// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
// This should be faster than looping through all dirs and calling os.Stat()
// on each of them, as the symlinks are resolved only once with OpenAt().
currentPath := base
fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
if err != nil {
return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
}
defer func() {
if err = syscall.Close(fd); err != nil {
glog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
}()
for i, dir := range dirs {
childFD, err := syscall.Openat(fd, dir, syscall.O_RDONLY, 0)
if err != nil {
if os.IsNotExist(err) {
return currentPath, dirs[i:], nil
}
return base, nil, err
}
if err = syscall.Close(fd); err != nil {
glog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
}
fd = childFD
currentPath = filepath.Join(currentPath, dir)
}
return pathname, []string{}, nil
}
// This implementation is shared between Linux and NsEnterMounter
// Open path and return its fd.
// Symlinks are disallowed (pathname must already resolve symlinks),
// and the path must be within the base directory.
func doSafeOpen(pathname string, base string) (int, error) {
// Calculate segments to follow
subpath, err := filepath.Rel(base, pathname)
if err != nil {
return -1, err
}
segments := strings.Split(subpath, string(filepath.Separator))
// Assumption: base is the only directory that we have under control.
// Base dir is not allowed to be a symlink.
parentFD, err := syscall.Open(base, nofollowFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
}
defer func() {
if parentFD != -1 {
if err = syscall.Close(parentFD); err != nil {
glog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
}
}
}()
childFD := -1
defer func() {
if childFD != -1 {
if err = syscall.Close(childFD); err != nil {
glog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
}
}
}()
currentPath := base
// Follow the segments one by one using openat() to make
// sure the user cannot change already existing directories into symlinks.
for _, seg := range segments {
currentPath = filepath.Join(currentPath, seg)
if !pathWithinBase(currentPath, base) {
return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
}
glog.V(5).Infof("Opening path %s", currentPath)
childFD, err = syscall.Openat(parentFD, seg, nofollowFlags, 0)
if err != nil {
return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
}
// Close parentFD
if err = syscall.Close(parentFD); err != nil {
return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
}
// Set child to new parent
parentFD = childFD
childFD = -1
}
// We made it to the end, return this fd, don't close it
finalFD := parentFD
parentFD = -1
return finalFD, nil
}

1206
pkg/util/mount/mount_linux_test.go
View File

File diff suppressed because it is too large Load Diff

13
pkg/util/mount/mount_unsupported.go
View File

@ -20,6 +20,7 @@ package mount
import (
"errors"
"os"
)
type Mounter struct {
@ -108,3 +109,15 @@ func (mounter *Mounter) MakeFile(pathname string) error {
func (mounter *Mounter) ExistsPath(pathname string) bool {
return true
}
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}

234
pkg/util/mount/mount_windows.go
View File

@ -260,6 +260,123 @@ func (mounter *Mounter) ExistsPath(pathname string) bool {
return true
}
// check whether hostPath is within volume path
// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(hostPath) == 0 {
return []uintptr{}, nil
}
finalSubPath, err := filepath.EvalSymlinks(hostPath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
}
finalVolumePath, err := filepath.EvalSymlinks(volumePath)
if err != nil {
return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
}
return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
}
// lock all intermediate subPath directories and check they are all within volumePath
// volumePath & subPath should not contain any symlink, otherwise it will return error
func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
if len(volumePath) == 0 || len(subPath) == 0 {
return []uintptr{}, nil
}
// get relative path to volumePath
relSubPath, err := filepath.Rel(volumePath, subPath)
if err != nil {
return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
}
if strings.HasPrefix(relSubPath, "..") {
return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
}
if relSubPath == "." {
// volumePath and subPath are equal
return []uintptr{}, nil
}
fileHandles := []uintptr{}
var errorResult error
currentFullPath := volumePath
dirs := strings.Split(relSubPath, string(os.PathSeparator))
for _, dir := range dirs {
// lock intermediate subPath directory first
currentFullPath = filepath.Join(currentFullPath, dir)
handle, err := lockPath(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
break
}
fileHandles = append(fileHandles, handle)
// make sure intermediate subPath directory does not contain symlink any more
stat, err := os.Lstat(currentFullPath)
if err != nil {
errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
break
}
if stat.Mode()&os.ModeSymlink != 0 {
errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
break
}
if !pathWithinBase(currentFullPath, volumePath) {
errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
break
}
}
return fileHandles, errorResult
}
// unlockPath unlock directories
func unlockPath(fileHandles []uintptr) {
if fileHandles != nil {
for _, handle := range fileHandles {
syscall.CloseHandle(syscall.Handle(handle))
}
}
}
// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
func lockPath(path string) (uintptr, error) {
if len(path) == 0 {
return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
}
pathp, err := syscall.UTF16PtrFromString(path)
if err != nil {
return uintptr(syscall.InvalidHandle), err
}
access := uint32(syscall.GENERIC_READ)
sharemode := uint32(syscall.FILE_SHARE_READ)
createmode := uint32(syscall.OPEN_EXISTING)
flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
return uintptr(fd), err
}
// Lock all directories in subPath and check they're not symlinks.
func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
// Unlock the directories when the container starts
cleanupAction = func() {
unlockPath(handles)
}
return subPath.Path, cleanupAction, err
}
// No bind-mounts for subpaths are necessary on Windows
func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}
func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
// Try to mount the disk
glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
@ -344,3 +461,120 @@ func getAllParentLinks(path string) ([]string, error) {
return links, nil
}
// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return doSafeMakeDir(pathname, base, perm)
}
func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
glog.V(4).Infof("Creating directory %q within base %q", pathname, base)
if !pathWithinBase(pathname, base) {
return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
}
// Quick check if the directory already exists
s, err := os.Stat(pathname)
if err == nil {
// Path exists
if s.IsDir() {
// The directory already exists. It can be outside of the parent,
// but there is no race-proof check.
glog.V(4).Infof("Directory %s already exists", pathname)
return nil
}
return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
}
// Find all existing directories
existingPath, toCreate, err := findExistingPrefix(base, pathname)
if err != nil {
return fmt.Errorf("error opening directory %s: %s", pathname, err)
}
if len(toCreate) == 0 {
return nil
}
// Ensure the existing directory is inside allowed base
fullExistingPath, err := filepath.EvalSymlinks(existingPath)
if err != nil {
return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
}
fullBasePath, err := filepath.EvalSymlinks(base)
if err != nil {
return fmt.Errorf("cannot read link %s: %s", base, err)
}
if !pathWithinBase(fullExistingPath, fullBasePath) {
return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
}
// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
defer unlockPath(fileHandles)
if err != nil {
return err
}
glog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
currentPath := fullExistingPath
// create the directories one by one, making sure nobody can change
// created directory into symlink by lock that directory immediately
for _, dir := range toCreate {
currentPath = filepath.Join(currentPath, dir)
glog.V(4).Infof("Creating %s", dir)
if err := os.Mkdir(currentPath, perm); err != nil {
return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
}
handle, err := lockPath(currentPath)
if err != nil {
return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
}
defer syscall.CloseHandle(syscall.Handle(handle))
// make sure newly created directory does not contain symlink after lock
stat, err := os.Lstat(currentPath)
if err != nil {
return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
}
if stat.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
}
}
return nil
}
// findExistingPrefix finds prefix of pathname that exists. In addition, it
// returns list of remaining directories that don't exist yet.
func findExistingPrefix(base, pathname string) (string, []string, error) {
rel, err := filepath.Rel(base, pathname)
if err != nil {
return base, nil, err
}
if strings.HasPrefix(rel, "..") {
return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
}
if rel == "." {
// base and pathname are equal
return pathname, []string{}, nil
}
dirs := strings.Split(rel, string(filepath.Separator))
parent := base
currentPath := base
for i, dir := range dirs {
parent = currentPath
currentPath = filepath.Join(parent, dir)
if _, err := os.Lstat(currentPath); err != nil {
if os.IsNotExist(err) {
return parent, dirs[i:], nil
}
return base, nil, err
}
}
return pathname, []string{}, nil
}

414
pkg/util/mount/mount_windows_test.go
View File

@ -20,8 +20,12 @@ package mount
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestNormalizeWindowsPath(t *testing.T) {
@ -132,3 +136,413 @@ func TestGetMountRefs(t *testing.T) {
}
}
}
func TestDoSafeMakeDir(t *testing.T) {
const testingVolumePath = `c:\tmp\DoSafeMakeDirTest`
os.MkdirAll(testingVolumePath, 0755)
defer os.RemoveAll(testingVolumePath)
tests := []struct {
volumePath string
subPath string
expectError bool
symlinkTarget string
}{
{
volumePath: testingVolumePath,
subPath: ``,
expectError: true,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `x`),
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d`),
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink`),
expectError: false,
symlinkTarget: `c:\tmp`,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink\c\d`),
expectError: true,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink\y926`),
expectError: true,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\symlink`),
expectError: false,
symlinkTarget: `c:\tmp`,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\x\symlink`),
expectError: false,
symlinkTarget: filepath.Join(testingVolumePath, `a`),
},
}
for _, test := range tests {
if len(test.volumePath) > 0 && len(test.subPath) > 0 && len(test.symlinkTarget) > 0 {
// make all parent sub directories
if parent := filepath.Dir(test.subPath); parent != "." {
os.MkdirAll(parent, 0755)
}
// make last element as symlink
linkPath := test.subPath
if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
if err := makeLink(linkPath, test.symlinkTarget); err != nil {
t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
}
}
}
err := doSafeMakeDir(test.subPath, test.volumePath, os.FileMode(0755))
if test.expectError {
assert.NotNil(t, err, "Expect error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
continue
}
assert.Nil(t, err, "Expect no error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
if _, err := os.Stat(test.subPath); os.IsNotExist(err) {
t.Errorf("subPath should exists after doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
}
}
}
func TestLockAndCheckSubPath(t *testing.T) {
const testingVolumePath = `c:\tmp\LockAndCheckSubPathTest`
tests := []struct {
volumePath string
subPath string
expectedHandleCount int
expectError bool
symlinkTarget string
}{
{
volumePath: `c:\`,
subPath: ``,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: ``,
subPath: `a`,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a`),
expectedHandleCount: 1,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d`),
expectedHandleCount: 4,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink`),
expectedHandleCount: 0,
expectError: true,
symlinkTarget: `c:\tmp`,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`),
expectedHandleCount: 0,
expectError: true,
symlinkTarget: `c:\tmp`,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
expectedHandleCount: 2,
expectError: false,
symlinkTarget: filepath.Join(testingVolumePath, `a\b`),
},
}
for _, test := range tests {
if len(test.volumePath) > 0 && len(test.subPath) > 0 {
os.MkdirAll(test.volumePath, 0755)
if len(test.symlinkTarget) == 0 {
// make all intermediate sub directories
os.MkdirAll(test.subPath, 0755)
} else {
// make all parent sub directories
if parent := filepath.Dir(test.subPath); parent != "." {
os.MkdirAll(parent, 0755)
}
// make last element as symlink
linkPath := test.subPath
if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
if err := makeLink(linkPath, test.symlinkTarget); err != nil {
t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
}
}
}
}
fileHandles, err := lockAndCheckSubPath(test.volumePath, test.subPath)
unlockPath(fileHandles)
assert.Equal(t, test.expectedHandleCount, len(fileHandles))
if test.expectError {
assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
continue
}
assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
}
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestLockAndCheckSubPathWithoutSymlink(t *testing.T) {
const testingVolumePath = `c:\tmp\LockAndCheckSubPathWithoutSymlinkTest`
tests := []struct {
volumePath string
subPath string
expectedHandleCount int
expectError bool
symlinkTarget string
}{
{
volumePath: `c:\`,
subPath: ``,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: ``,
subPath: `a`,
expectedHandleCount: 0,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a`),
expectedHandleCount: 1,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d`),
expectedHandleCount: 4,
expectError: false,
symlinkTarget: "",
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `symlink`),
expectedHandleCount: 1,
expectError: true,
symlinkTarget: `c:\tmp`,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\symlink`),
expectedHandleCount: 4,
expectError: true,
symlinkTarget: `c:\tmp`,
},
{
volumePath: testingVolumePath,
subPath: filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
expectedHandleCount: 5,
expectError: true,
symlinkTarget: filepath.Join(testingVolumePath, `a\b`),
},
}
for _, test := range tests {
if len(test.volumePath) > 0 && len(test.subPath) > 0 {
os.MkdirAll(test.volumePath, 0755)
if len(test.symlinkTarget) == 0 {
// make all intermediate sub directories
os.MkdirAll(test.subPath, 0755)
} else {
// make all parent sub directories
if parent := filepath.Dir(test.subPath); parent != "." {
os.MkdirAll(parent, 0755)
}
// make last element as symlink
linkPath := test.subPath
if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
if err := makeLink(linkPath, test.symlinkTarget); err != nil {
t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
}
}
}
}
fileHandles, err := lockAndCheckSubPathWithoutSymlink(test.volumePath, test.subPath)
unlockPath(fileHandles)
assert.Equal(t, test.expectedHandleCount, len(fileHandles))
if test.expectError {
assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
continue
}
assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
}
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestFindExistingPrefix(t *testing.T) {
const testingVolumePath = `c:\tmp\FindExistingPrefixTest`
tests := []struct {
base string
pathname string
expectError bool
expectedExistingPath string
expectedToCreateDirs []string
createSubPathBeforeTest bool
}{
{
base: `c:\tmp\a`,
pathname: `c:\tmp\b`,
expectError: true,
expectedExistingPath: "",
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: ``,
pathname: `c:\tmp\b`,
expectError: true,
expectedExistingPath: "",
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: `c:\tmp\a`,
pathname: `d:\tmp\b`,
expectError: true,
expectedExistingPath: "",
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: testingVolumePath,
pathname: testingVolumePath,
expectError: false,
expectedExistingPath: testingVolumePath,
expectedToCreateDirs: []string{},
createSubPathBeforeTest: false,
},
{
base: testingVolumePath,
pathname: filepath.Join(testingVolumePath, `a\b`),
expectError: false,
expectedExistingPath: filepath.Join(testingVolumePath, `a\b`),
expectedToCreateDirs: []string{},
createSubPathBeforeTest: true,
},
{
base: testingVolumePath,
pathname: filepath.Join(testingVolumePath, `a\b\c\`),
expectError: false,
expectedExistingPath: filepath.Join(testingVolumePath, `a\b`),
expectedToCreateDirs: []string{`c`},
createSubPathBeforeTest: false,
},
{
base: testingVolumePath,
pathname: filepath.Join(testingVolumePath, `a\b\c\d`),
expectError: false,
expectedExistingPath: filepath.Join(testingVolumePath, `a\b`),
expectedToCreateDirs: []string{`c`, `d`},
createSubPathBeforeTest: false,
},
}
for _, test := range tests {
if test.createSubPathBeforeTest {
os.MkdirAll(test.pathname, 0755)
}
existingPath, toCreate, err := findExistingPrefix(test.base, test.pathname)
if test.expectError {
assert.NotNil(t, err, "Expect error during findExistingPrefix(%s, %s)", test.base, test.pathname)
continue
}
assert.Nil(t, err, "Expect no error during findExistingPrefix(%s, %s)", test.base, test.pathname)
assert.Equal(t, test.expectedExistingPath, existingPath, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
test.base, test.pathname, existingPath, test.expectedExistingPath)
assert.Equal(t, test.expectedToCreateDirs, toCreate, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
test.base, test.pathname, toCreate, test.expectedToCreateDirs)
}
// remove dir will happen after closing all file handles
assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
}
func TestPathWithinBase(t *testing.T) {
tests := []struct {
fullPath string
basePath string
expectedResult bool
}{
{
fullPath: `c:\tmp\a\b\c`,
basePath: `c:\tmp`,
expectedResult: true,
},
{
fullPath: `c:\tmp1`,
basePath: `c:\tmp2`,
expectedResult: false,
},
{
fullPath: `c:\tmp`,
basePath: `c:\tmp`,
expectedResult: true,
},
{
fullPath: `c:\tmp`,
basePath: `c:\tmp\a\b\c`,
expectedResult: false,
},
}
for _, test := range tests {
result := pathWithinBase(test.fullPath, test.basePath)
assert.Equal(t, result, test.expectedResult, "Expect result not equal with pathWithinBase(%s, %s) return: %q, expected: %q",
test.fullPath, test.basePath, result, test.expectedResult)
}
}

48
pkg/util/mount/nsenter_mount.go
View File

@ -22,9 +22,12 @@ import (
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"github.com/golang/glog"
utilio "k8s.io/kubernetes/pkg/util/io"
"k8s.io/kubernetes/pkg/util/nsenter"
)
@ -33,6 +36,13 @@ const (
hostProcMountsPath = "/rootfs/proc/1/mounts"
// hostProcMountinfoPath is the default mount info path for rootfs
hostProcMountinfoPath = "/rootfs/proc/1/mountinfo"
// hostProcSelfStatusPath is the default path to /proc/self/status on the host
hostProcSelfStatusPath = "/rootfs/proc/self/status"
)
var (
// pidRegExp matches "Pid: <pid>" in /proc/self/status
pidRegExp = regexp.MustCompile(`\nPid:\t([0-9]*)\n`)
)
// Currently, all docker containers receive their own mount namespaces.
@ -270,3 +280,41 @@ func (mounter *NsenterMounter) ExistsPath(pathname string) bool {
}
return false
}
func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(mounter, podDir, volumeName)
}
// getPidOnHost returns kubelet's pid in the host pid namespace
func (mounter *NsenterMounter) getPidOnHost(procStatusPath string) (int, error) {
// Get the PID from /rootfs/proc/self/status
statusBytes, err := utilio.ConsistentRead(procStatusPath, maxListTries)
if err != nil {
return 0, fmt.Errorf("error reading %s: %s", procStatusPath, err)
}
matches := pidRegExp.FindSubmatch(statusBytes)
if len(matches) < 2 {
return 0, fmt.Errorf("cannot parse %s: no Pid:", procStatusPath)
}
return strconv.Atoi(string(matches[1]))
}
func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
hostPid, err := mounter.getPidOnHost(hostProcSelfStatusPath)
if err != nil {
return "", nil, err
}
glog.V(4).Infof("Kubelet's PID on the host is %d", hostPid)
// Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = doBindSubPath(mounter, subPath, hostPid)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
func (mounter *NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return doSafeMakeDir(pathname, base, perm)
}

126
pkg/util/mount/nsenter_mount_test.go
View File

@ -18,7 +18,13 @@ limitations under the License.
package mount
import "testing"
import (
"io/ioutil"
"os"
"path"
"strconv"
"testing"
)
func TestParseFindMnt(t *testing.T) {
tests := []struct {
@ -65,3 +71,121 @@ func TestParseFindMnt(t *testing.T) {
}
}
}
func TestGetPidOnHost(t *testing.T) {
tempDir, err := ioutil.TempDir("", "get_pid_on_host_tests")
if err != nil {
t.Fatalf(err.Error())
}
defer os.RemoveAll(tempDir)
tests := []struct {
name string
procFile string
expectedPid int
expectError bool
}{
{
name: "valid status file",
procFile: `Name: cat
Umask: 0002
State: R (running)
Tgid: 15041
Ngid: 0
Pid: 15041
PPid: 22699
TracerPid: 0
Uid: 1000 1000 1000 1000
Gid: 1000 1000 1000 1000
FDSize: 256
Groups: 10 135 156 157 158 973 984 1000 1001
NStgid: 15041
NSpid: 15041
NSpgid: 15041
NSsid: 22699
VmPeak: 115016 kB
VmSize: 115016 kB
VmLck: 0 kB
VmPin: 0 kB
VmHWM: 816 kB
VmRSS: 816 kB
RssAnon: 64 kB
RssFile: 752 kB
RssShmem: 0 kB
VmData: 312 kB
VmStk: 136 kB
VmExe: 32 kB
VmLib: 2060 kB
VmPTE: 44 kB
VmPMD: 12 kB
VmSwap: 0 kB
HugetlbPages: 0 kB
Threads: 1
SigQ: 2/60752
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 0000000000000000
SigIgn: 0000000000000000
SigCgt: 0000000000000000
CapInh: 0000000000000000
CapPrm: 0000000000000000
CapEff: 0000000000000000
CapBnd: 0000003fffffffff
CapAmb: 0000000000000000
NoNewPrivs: 0
Seccomp: 0
Cpus_allowed: ff
Cpus_allowed_list: 0-7
Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000001
Mems_allowed_list: 0
voluntary_ctxt_switches: 0
nonvoluntary_ctxt_switches: 0
`,
expectedPid: 15041,
},
{
name: "no Pid:",
procFile: `Name: cat
Umask: 0002
State: R (running)
Tgid: 15041
Ngid: 0
PPid: 22699
`,
expectedPid: 0,
expectError: true,
},
{
name: "invalid Pid:",
procFile: `Name: cat
Umask: 0002
State: R (running)
Tgid: 15041
Ngid: 0
Pid: invalid
PPid: 22699
`,
expectedPid: 0,
expectError: true,
},
}
for i, test := range tests {
filename := path.Join(tempDir, strconv.Itoa(i))
err := ioutil.WriteFile(filename, []byte(test.procFile), 0666)
if err != nil {
t.Fatalf(err.Error())
}
mounter := NsenterMounter{}
pid, err := mounter.getPidOnHost(filename)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if err == nil && test.expectError {
t.Errorf("Test %q: expected error, got none", test.name)
}
if pid != test.expectedPid {
t.Errorf("Test %q: expected pid %d, got %d", test.name, test.expectedPid, pid)
}
}
}

13
pkg/util/mount/nsenter_mount_unsupported.go
View File

@ -20,6 +20,7 @@ package mount
import (
"errors"
"os"
)
type NsenterMounter struct{}
@ -85,3 +86,15 @@ func (*NsenterMounter) MakeFile(pathname string) error {
func (*NsenterMounter) ExistsPath(pathname string) bool {
return true
}
func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
return nil
}
func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
return subPath.Path, nil, nil
}
func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
return nil
}

12
pkg/util/removeall/removeall_test.go
View File

@ -79,6 +79,18 @@ func (mounter *fakeMounter) ExistsPath(pathname string) bool {
return true
}
func (mounter *fakeMounter) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) {
return "", nil, nil
}
func (mounter *fakeMounter) CleanSubPaths(_, _ string) error {
return nil
}
func (mounter *fakeMounter) SafeMakeDir(_, _ string, _ os.FileMode) error {
return nil
}
func (mounter *fakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
name := path.Base(file)
if strings.HasPrefix(name, "mount") {

12
pkg/volume/host_path/host_path_test.go
View File

@ -376,6 +376,18 @@ func (fftc *fakeFileTypeChecker) GetFileType(_ string) (utilmount.FileType, erro
return utilmount.FileType(fftc.desiredType), nil
}
func (fftc *fakeFileTypeChecker) PrepareSafeSubpath(subPath utilmount.Subpath) (newHostPath string, cleanupAction func(), err error) {
return "", nil, nil
}
func (fftc *fakeFileTypeChecker) CleanSubPaths(_, _ string) error {
return nil
}
func (fftc *fakeFileTypeChecker) SafeMakeDir(_, _ string, _ os.FileMode) error {
return nil
}
func setUp() error {
err := os.MkdirAll("/tmp/ExistingFolder", os.FileMode(0755))
if err != nil {

7
pkg/volume/util/operationexecutor/operation_executor.go
View File

@ -112,7 +112,7 @@ type OperationExecutor interface {
// For 'Block' volumeMode, this method unmaps symbolic link to the volume
// from both the pod device map path in volumeToUnmount and global map path.
// And then, updates the actual state of the world to reflect that.
UnmountVolume(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) error
UnmountVolume(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) error
// If a volume has 'Filesystem' volumeMode, UnmountDevice unmounts the
// volumes global mount path from the device (for attachable volumes only,
@ -746,7 +746,8 @@ func (oe *operationExecutor) MountVolume(
func (oe *operationExecutor) UnmountVolume(
volumeToUnmount MountedVolume,
actualStateOfWorld ActualStateOfWorldMounterUpdater) error {
actualStateOfWorld ActualStateOfWorldMounterUpdater,
podsDir string) error {
fsVolume, err := util.CheckVolumeModeFilesystem(volumeToUnmount.VolumeSpec)
if err != nil {
return err
@ -756,7 +757,7 @@ func (oe *operationExecutor) UnmountVolume(
// Filesystem volume case
// Unmount a volume if a volume is mounted
generatedOperations, err = oe.operationGenerator.GenerateUnmountVolumeFunc(
volumeToUnmount, actualStateOfWorld)
volumeToUnmount, actualStateOfWorld, podsDir)
} else {
// Block volume case
// Unmap a volume if a volume is mapped

6
pkg/volume/util/operationexecutor/operation_executor_test.go
View File

@ -124,7 +124,7 @@ func TestOperationExecutor_UnmountVolume_ConcurrentUnmountForAllPlugins(t *testi
PodUID: pod.UID,
}
}
oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */)
oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */, "" /*podsDir*/)
}
// Assert
@ -318,7 +318,7 @@ func TestOperationExecutor_UnmountVolume_ConcurrentUnmountForAllPlugins_VolumeMo
VolumeSpec: tmpSpec,
}
}
oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */)
oe.UnmountVolume(volumesToUnmount[i], nil /* actualStateOfWorldMounterUpdater */, "" /* podsDir */)
}
// Assert
@ -372,7 +372,7 @@ func (fopg *fakeOperationGenerator) GenerateMountVolumeFunc(waitForAttachTimeout
OperationFunc: opFunc,
}, nil
}
func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (volumetypes.GeneratedOperations, error) {
func (fopg *fakeOperationGenerator) GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (volumetypes.GeneratedOperations, error) {
opFunc := func() (error, error) {
startOperationAndBlock(fopg.ch, fopg.quit)
return nil, nil

14
pkg/volume/util/operationexecutor/operation_generator.go
View File

@ -18,6 +18,7 @@ package operationexecutor
import (
"fmt"
"path"
"strings"
"time"
@ -85,7 +86,7 @@ type OperationGenerator interface {
GenerateMountVolumeFunc(waitForAttachTimeout time.Duration, volumeToMount VolumeToMount, actualStateOfWorldMounterUpdater ActualStateOfWorldMounterUpdater, isRemount bool) (volumetypes.GeneratedOperations, error)
// Generates the UnmountVolume function needed to perform the unmount of a volume plugin
GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater) (volumetypes.GeneratedOperations, error)
GenerateUnmountVolumeFunc(volumeToUnmount MountedVolume, actualStateOfWorld ActualStateOfWorldMounterUpdater, podsDir string) (volumetypes.GeneratedOperations, error)
// Generates the AttachVolume function needed to perform attach of a volume plugin
GenerateAttachVolumeFunc(volumeToAttach VolumeToAttach, actualStateOfWorld ActualStateOfWorldAttacherUpdater) (volumetypes.GeneratedOperations, error)
@ -651,7 +652,8 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi
func (og *operationGenerator) GenerateUnmountVolumeFunc(
volumeToUnmount MountedVolume,
actualStateOfWorld ActualStateOfWorldMounterUpdater) (volumetypes.GeneratedOperations, error) {
actualStateOfWorld ActualStateOfWorldMounterUpdater,
podsDir string) (volumetypes.GeneratedOperations, error) {
// Get mountable plugin
volumePlugin, err :=
og.volumePluginMgr.FindPluginByName(volumeToUnmount.PluginName)
@ -666,6 +668,14 @@ func (og *operationGenerator) GenerateUnmountVolumeFunc(
}
unmountVolumeFunc := func() (error, error) {
mounter := og.volumePluginMgr.Host.GetMounter(volumeToUnmount.PluginName)
// Remove all bind-mounts for subPaths
podDir := path.Join(podsDir, string(volumeToUnmount.PodUID))
if err := mounter.CleanSubPaths(podDir, volumeToUnmount.OuterVolumeSpecName); err != nil {
return volumeToUnmount.GenerateError("error cleaning subPath mounts", err)
}
// Execute unmount
unmountErr := volumeUnmounter.TearDown()
if unmountErr != nil {

8
test/e2e/framework/util.go
View File

@ -1368,23 +1368,23 @@ func WaitForPodRunningInNamespace(c clientset.Interface, pod *v1.Pod) error {
if pod.Status.Phase == v1.PodRunning {
return nil
}
return waitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace, PodStartTimeout)
return WaitTimeoutForPodRunningInNamespace(c, pod.Name, pod.Namespace, PodStartTimeout)
}
// Waits default amount of time (PodStartTimeout) for the specified pod to become running.
// Returns an error if timeout occurs first, or pod goes in to failed state.
func WaitForPodNameRunningInNamespace(c clientset.Interface, podName, namespace string) error {
return waitTimeoutForPodRunningInNamespace(c, podName, namespace, PodStartTimeout)
return WaitTimeoutForPodRunningInNamespace(c, podName, namespace, PodStartTimeout)
}
// Waits an extended amount of time (slowPodStartTimeout) for the specified pod to become running.
// The resourceVersion is used when Watching object changes, it tells since when we care
// about changes to the pod. Returns an error if timeout occurs first, or pod goes in to failed state.
func waitForPodRunningInNamespaceSlow(c clientset.Interface, podName, namespace string) error {
return waitTimeoutForPodRunningInNamespace(c, podName, namespace, slowPodStartTimeout)
return WaitTimeoutForPodRunningInNamespace(c, podName, namespace, slowPodStartTimeout)
}
func waitTimeoutForPodRunningInNamespace(c clientset.Interface, podName, namespace string, timeout time.Duration) error {
func WaitTimeoutForPodRunningInNamespace(c clientset.Interface, podName, namespace string, timeout time.Duration) error {
return wait.PollImmediate(Poll, timeout, podRunning(c, podName, namespace))
}

11
test/e2e/storage/BUILD
View File

@ -1,9 +1,4 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
@ -21,6 +16,7 @@ go_library(
"pv_protection.go",
"pvc_protection.go",
"regional_pd.go",
"subpath.go",
"volume_expand.go",
"volume_io.go",
"volume_metrics.go",
@ -28,6 +24,7 @@ go_library(
"volumes.go",
],
importpath = "k8s.io/kubernetes/test/e2e/storage",
visibility = ["//visibility:public"],
deps = [
"//pkg/api/testapi:go_default_library",
"//pkg/api/v1/pod:go_default_library",
@ -70,6 +67,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/rand:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
@ -95,4 +93,5 @@ filegroup(
"//test/e2e/storage/vsphere:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

743
test/e2e/storage/subpath.go Normal file
View File

@ -0,0 +1,743 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package storage
import (
"fmt"
"path/filepath"
"strings"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/utils"
imageutils "k8s.io/kubernetes/test/utils/image"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var (
volumePath = "/test-volume"
volumeName = "test-volume"
fileName = "test-file"
retryDuration = 180
mountImage = imageutils.GetE2EImage(imageutils.Mounttest)
)
type volInfo struct {
source *v1.VolumeSource
node string
}
type volSource interface {
createVolume(f *framework.Framework) volInfo
cleanupVolume(f *framework.Framework)
}
var initVolSources = map[string]func() volSource{
"hostPath": initHostpath,
"hostPathSymlink": initHostpathSymlink,
"emptyDir": initEmptydir,
"gcePD": initGCEPD,
"gcePDPartitioned": initGCEPDPartition,
"nfs": initNFS,
"nfsPVC": initNFSPVC,
"gluster": initGluster,
}
var _ = utils.SIGDescribe("Subpath", func() {
var (
subPath string
subPathDir string
filePathInSubpath string
filePathInVolume string
pod *v1.Pod
vol volSource
)
f := framework.NewDefaultFramework("subpath")
for volType, volInit := range initVolSources {
curVolType := volType
curVolInit := volInit
Context(fmt.Sprintf("[Volume type: %v]", curVolType), func() {
BeforeEach(func() {
By(fmt.Sprintf("Initializing %s volume", curVolType))
vol = curVolInit()
subPath = f.Namespace.Name
subPathDir = filepath.Join(volumePath, subPath)
filePathInSubpath = filepath.Join(volumePath, fileName)
filePathInVolume = filepath.Join(subPathDir, fileName)
volInfo := vol.createVolume(f)
pod = testPodSubpath(subPath, curVolType, volInfo.source)
pod.Namespace = f.Namespace.Name
pod.Spec.NodeName = volInfo.node
})
AfterEach(func() {
By("Deleting pod")
err := framework.DeletePodWithWait(f, f.ClientSet, pod)
Expect(err).ToNot(HaveOccurred())
By("Cleaning up volume")
vol.cleanupVolume(f)
})
It("should support non-existent path", func() {
// Write the file in the subPath from container 0
setWriteCommand(filePathInSubpath, &pod.Spec.Containers[0])
// Read it from outside the subPath from container 1
testReadFile(f, filePathInVolume, pod, 1)
})
It("should support existing directory", func() {
// Create the directory
setInitCommand(pod, fmt.Sprintf("mkdir -p %s", subPathDir))
// Write the file in the subPath from container 0
setWriteCommand(filePathInSubpath, &pod.Spec.Containers[0])
// Read it from outside the subPath from container 1
testReadFile(f, filePathInVolume, pod, 1)
})
It("should support existing single file", func() {
// Create the file in the init container
setInitCommand(pod, fmt.Sprintf("mkdir -p %s; echo \"mount-tester new file\" > %s", subPathDir, filePathInVolume))
// Read it from inside the subPath from container 0
testReadFile(f, filePathInSubpath, pod, 0)
})
It("should fail if subpath directory is outside the volume [Slow]", func() {
// Create the subpath outside the volume
setInitCommand(pod, fmt.Sprintf("ln -s /bin %s", subPathDir))
// Pod should fail
testPodFailSupath(f, pod)
})
It("should fail if subpath file is outside the volume [Slow]", func() {
// Create the subpath outside the volume
setInitCommand(pod, fmt.Sprintf("ln -s /bin/sh %s", subPathDir))
// Pod should fail
testPodFailSupath(f, pod)
})
It("should fail if non-existent subpath is outside the volume [Slow]", func() {
// Create the subpath outside the volume
setInitCommand(pod, fmt.Sprintf("ln -s /bin/notanexistingpath %s", subPathDir))
// Pod should fail
testPodFailSupath(f, pod)
})
It("should fail if subpath with backstepping is outside the volume [Slow]", func() {
// Create the subpath outside the volume
setInitCommand(pod, fmt.Sprintf("ln -s ../ %s", subPathDir))
// Pod should fail
testPodFailSupath(f, pod)
})
It("should support creating multiple subpath from same volumes [Slow]", func() {
subpathDir1 := filepath.Join(volumePath, "subpath1")
subpathDir2 := filepath.Join(volumePath, "subpath2")
filepath1 := filepath.Join("/test-subpath1", fileName)
filepath2 := filepath.Join("/test-subpath2", fileName)
setInitCommand(pod, fmt.Sprintf("mkdir -p %s; mkdir -p %s", subpathDir1, subpathDir2))
addSubpathVolumeContainer(&pod.Spec.Containers[0], v1.VolumeMount{
Name: volumeName,
MountPath: "/test-subpath1",
SubPath: "subpath1",
})
addSubpathVolumeContainer(&pod.Spec.Containers[0], v1.VolumeMount{
Name: volumeName,
MountPath: "/test-subpath2",
SubPath: "subpath2",
})
addMultipleWrites(&pod.Spec.Containers[0], filepath1, filepath2)
testMultipleReads(f, pod, 0, filepath1, filepath2)
})
It("should support restarting containers [Slow]", func() {
// Create the directory
setInitCommand(pod, fmt.Sprintf("mkdir -p %v", subPathDir))
testPodContainerRestart(f, pod, filePathInVolume, filePathInSubpath)
})
})
}
// TODO: add a test case for the same disk with two partitions
})
func testPodSubpath(subpath, volumeType string, source *v1.VolumeSource) *v1.Pod {
suffix := strings.ToLower(fmt.Sprintf("%s-%s", volumeType, rand.String(4)))
privileged := true
gracePeriod := int64(1)
return &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("pod-subpath-test-%s", suffix),
},
Spec: v1.PodSpec{
InitContainers: []v1.Container{
{
Name: fmt.Sprintf("init-volume-%s", suffix),
Image: "busybox",
VolumeMounts: []v1.VolumeMount{
{
Name: volumeName,
MountPath: volumePath,
},
},
SecurityContext: &v1.SecurityContext{
Privileged: &privileged,
},
},
},
Containers: []v1.Container{
{
Name: fmt.Sprintf("test-container-subpath-%s", suffix),
Image: mountImage,
VolumeMounts: []v1.VolumeMount{
{
Name: volumeName,
MountPath: volumePath,
SubPath: subpath,
},
},
SecurityContext: &v1.SecurityContext{
Privileged: &privileged,
},
},
{
Name: fmt.Sprintf("test-container-volume-%s", suffix),
Image: mountImage,
VolumeMounts: []v1.VolumeMount{
{
Name: volumeName,
MountPath: volumePath,
},
},
SecurityContext: &v1.SecurityContext{
Privileged: &privileged,
},
},
},
RestartPolicy: v1.RestartPolicyNever,
TerminationGracePeriodSeconds: &gracePeriod,
Volumes: []v1.Volume{
{
Name: volumeName,
VolumeSource: *source,
},
},
},
}
}
func setInitCommand(pod *v1.Pod, command string) {
pod.Spec.InitContainers[0].Command = []string{"/bin/sh", "-ec", command}
}
func setWriteCommand(file string, container *v1.Container) {
container.Args = []string{
fmt.Sprintf("--new_file_0644=%v", file),
fmt.Sprintf("--file_mode=%v", file),
}
}
func addSubpathVolumeContainer(container *v1.Container, volumeMount v1.VolumeMount) {
existingMounts := container.VolumeMounts
container.VolumeMounts = append(existingMounts, volumeMount)
}
func addMultipleWrites(container *v1.Container, file1 string, file2 string) {
container.Args = []string{
fmt.Sprintf("--new_file_0644=%v", file1),
fmt.Sprintf("--new_file_0666=%v", file2),
}
}
func testMultipleReads(f *framework.Framework, pod *v1.Pod, containerIndex int, file1 string, file2 string) {
By(fmt.Sprintf("Creating pod %s", pod.Name))
f.TestContainerOutput("multi_subpath", pod, containerIndex, []string{
"content of file \"" + file1 + "\": mount-tester new file",
"content of file \"" + file2 + "\": mount-tester new file",
})
}
func setReadCommand(file string, container *v1.Container) {
container.Args = []string{
fmt.Sprintf("--file_content_in_loop=%v", file),
fmt.Sprintf("--retry_time=%d", retryDuration),
}
}
func testReadFile(f *framework.Framework, file string, pod *v1.Pod, containerIndex int) {
setReadCommand(file, &pod.Spec.Containers[containerIndex])
By(fmt.Sprintf("Creating pod %s", pod.Name))
f.TestContainerOutput("subpath", pod, containerIndex, []string{
"content of file \"" + file + "\": mount-tester new file",
})
}
func testPodFailSupath(f *framework.Framework, pod *v1.Pod) {
By(fmt.Sprintf("Creating pod %s", pod.Name))
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
Expect(err).ToNot(HaveOccurred())
defer func() {
framework.DeletePodWithWait(f, f.ClientSet, pod)
}()
err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, time.Minute)
Expect(err).To(HaveOccurred())
By("Checking for subpath error event")
selector := fields.Set{
"involvedObject.kind": "Pod",
"involvedObject.name": pod.Name,
"involvedObject.namespace": f.Namespace.Name,
"reason": "Failed",
}.AsSelector().String()
options := metav1.ListOptions{FieldSelector: selector}
events, err := f.ClientSet.CoreV1().Events(f.Namespace.Name).List(options)
Expect(err).NotTo(HaveOccurred())
Expect(len(events.Items)).NotTo(Equal(0))
Expect(events.Items[0].Message).To(ContainSubstring("subPath"))
}
func testPodContainerRestart(f *framework.Framework, pod *v1.Pod, fileInVolume, fileInSubpath string) {
pod.Spec.RestartPolicy = v1.RestartPolicyOnFailure
// Add liveness probe to subpath container
pod.Spec.Containers[0].Image = "busybox"
pod.Spec.Containers[0].Command = []string{"/bin/sh", "-ec", "sleep 100000"}
pod.Spec.Containers[0].LivenessProbe = &v1.Probe{
Handler: v1.Handler{
Exec: &v1.ExecAction{
Command: []string{"cat", fileInSubpath},
},
},
InitialDelaySeconds: 15,
FailureThreshold: 1,
PeriodSeconds: 2,
}
// Set volume container to write file
writeCmd := fmt.Sprintf("echo test > %v", fileInVolume)
pod.Spec.Containers[1].Image = "busybox"
pod.Spec.Containers[1].Command = []string{"/bin/sh", "-ec", fmt.Sprintf("%v; sleep 100000", writeCmd)}
// Start pod
By(fmt.Sprintf("Creating pod %s", pod.Name))
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
Expect(err).ToNot(HaveOccurred())
err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, time.Minute)
Expect(err).ToNot(HaveOccurred())
By("Failing liveness probe")
out, err := podContainerExec(pod, 1, fmt.Sprintf("rm %v", fileInVolume))
framework.Logf("Pod exec output: %v", out)
Expect(err).ToNot(HaveOccurred())
// Check that container has restarted
By("Waiting for container to restart")
restarts := int32(0)
err = wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) {
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, status := range pod.Status.ContainerStatuses {
if status.Name == pod.Spec.Containers[0].Name {
framework.Logf("Container %v, restarts: %v", status.Name, status.RestartCount)
restarts = status.RestartCount
if restarts > 0 {
framework.Logf("Container has restart count: %v", restarts)
return true, nil
}
}
}
return false, nil
})
Expect(err).ToNot(HaveOccurred())
// Fix liveness probe
By("Rewriting the file")
writeCmd = fmt.Sprintf("echo test-after > %v", fileInVolume)
out, err = podContainerExec(pod, 1, writeCmd)
framework.Logf("Pod exec output: %v", out)
Expect(err).ToNot(HaveOccurred())
// Wait for container restarts to stabilize
By("Waiting for container to stop restarting")
stableCount := int(0)
stableThreshold := int(time.Minute / framework.Poll)
err = wait.PollImmediate(framework.Poll, 2*time.Minute, func() (bool, error) {
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, status := range pod.Status.ContainerStatuses {
if status.Name == pod.Spec.Containers[0].Name {
if status.RestartCount == restarts {
stableCount++
if stableCount > stableThreshold {
framework.Logf("Container restart has stabilized")
return true, nil
}
} else {
restarts = status.RestartCount
stableCount = 0
framework.Logf("Container has restart count: %v", restarts)
}
break
}
}
return false, nil
})
Expect(err).ToNot(HaveOccurred())
// Verify content of file in subpath
out, err = podContainerExec(pod, 0, fmt.Sprintf("cat %v", fileInSubpath))
framework.Logf("Pod exec output: %v", out)
Expect(err).ToNot(HaveOccurred())
Expect(strings.TrimSpace(out)).To(BeEquivalentTo("test-after"))
}
func podContainerExec(pod *v1.Pod, containerIndex int, bashExec string) (string, error) {
return framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--container", pod.Spec.Containers[containerIndex].Name, "--", "/bin/sh", "-c", bashExec)
}
type hostpathSource struct {
}
func initHostpath() volSource {
return &hostpathSource{}
}
func (s *hostpathSource) createVolume(f *framework.Framework) volInfo {
return volInfo{
source: &v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/tmp",
},
},
}
}
func (s *hostpathSource) cleanupVolume(f *framework.Framework) {
}
type hostpathSymlinkSource struct {
}
func initHostpathSymlink() volSource {
return &hostpathSymlinkSource{}
}
func (s *hostpathSymlinkSource) createVolume(f *framework.Framework) volInfo {
nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
Expect(len(nodes.Items)).NotTo(BeZero(), "No available nodes for scheduling")
node0 := &nodes.Items[0]
sourcePath := fmt.Sprintf("/tmp/%v", f.Namespace.Name)
targetPath := fmt.Sprintf("/tmp/%v-link", f.Namespace.Name)
cmd := fmt.Sprintf("mkdir %v -m 777 && ln -s %v %v", sourcePath, sourcePath, targetPath)
privileged := true
// Launch pod to initialize hostpath directory and symlink
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("hostpath-symlink-prep-%s", f.Namespace.Name),
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: fmt.Sprintf("init-volume-%s", f.Namespace.Name),
Image: "busybox",
Command: []string{"/bin/sh", "-ec", cmd},
VolumeMounts: []v1.VolumeMount{
{
Name: volumeName,
MountPath: "/tmp",
},
},
SecurityContext: &v1.SecurityContext{
Privileged: &privileged,
},
},
},
RestartPolicy: v1.RestartPolicyNever,
Volumes: []v1.Volume{
{
Name: volumeName,
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: "/tmp",
},
},
},
},
NodeName: node0.Name,
},
}
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
Expect(err).ToNot(HaveOccurred())
err = framework.WaitForPodSuccessInNamespace(f.ClientSet, pod.Name, pod.Namespace)
Expect(err).ToNot(HaveOccurred())
err = framework.DeletePodWithWait(f, f.ClientSet, pod)
Expect(err).ToNot(HaveOccurred())
return volInfo{
source: &v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{
Path: targetPath,
},
},
node: node0.Name,
}
}
func (s *hostpathSymlinkSource) cleanupVolume(f *framework.Framework) {
}
type emptydirSource struct {
}
func initEmptydir() volSource {
return &emptydirSource{}
}
func (s *emptydirSource) createVolume(f *framework.Framework) volInfo {
return volInfo{
source: &v1.VolumeSource{
EmptyDir: &v1.EmptyDirVolumeSource{},
},
}
}
func (s *emptydirSource) cleanupVolume(f *framework.Framework) {
}
type gcepdSource struct {
diskName string
}
func initGCEPD() volSource {
framework.SkipUnlessProviderIs("gce", "gke")
return &gcepdSource{}
}
func (s *gcepdSource) createVolume(f *framework.Framework) volInfo {
var err error
framework.Logf("Creating GCE PD volume")
s.diskName, err = framework.CreatePDWithRetry()
framework.ExpectNoError(err, "Error creating PD")
return volInfo{
source: &v1.VolumeSource{
GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{PDName: s.diskName},
},
}
}
func (s *gcepdSource) cleanupVolume(f *framework.Framework) {
if s.diskName != "" {
err := framework.DeletePDWithRetry(s.diskName)
framework.ExpectNoError(err, "Error deleting PD")
}
}
type gcepdPartitionSource struct {
diskName string
}
func initGCEPDPartition() volSource {
// Need to manually create, attach, partition, detach the GCE PD
// with disk name "subpath-partitioned-disk" before running this test
manual := true
if manual {
framework.Skipf("Skipping manual GCE PD partition test")
}
framework.SkipUnlessProviderIs("gce", "gke")
return &gcepdPartitionSource{diskName: "subpath-partitioned-disk"}
}
func (s *gcepdPartitionSource) createVolume(f *framework.Framework) volInfo {
// TODO: automate partitioned of GCE PD once it supports raw block volumes
// framework.Logf("Creating GCE PD volume")
// s.diskName, err = framework.CreatePDWithRetry()
// framework.ExpectNoError(err, "Error creating PD")
return volInfo{
source: &v1.VolumeSource{
GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
PDName: s.diskName,
Partition: 1,
},
},
}
}
func (s *gcepdPartitionSource) cleanupVolume(f *framework.Framework) {
if s.diskName != "" {
// err := framework.DeletePDWithRetry(s.diskName)
// framework.ExpectNoError(err, "Error deleting PD")
}
}
type nfsSource struct {
serverPod *v1.Pod
}
func initNFS() volSource {
return &nfsSource{}
}
func (s *nfsSource) createVolume(f *framework.Framework) volInfo {
var serverIP string
framework.Logf("Creating NFS server")
_, s.serverPod, serverIP = framework.NewNFSServer(f.ClientSet, f.Namespace.Name, []string{"-G", "777", "/exports"})
return volInfo{
source: &v1.VolumeSource{
NFS: &v1.NFSVolumeSource{
Server: serverIP,
Path: "/exports",
},
},
}
}
func (s *nfsSource) cleanupVolume(f *framework.Framework) {
if s.serverPod != nil {
framework.DeletePodWithWait(f, f.ClientSet, s.serverPod)
}
}
type glusterSource struct {
serverPod *v1.Pod
}
func initGluster() volSource {
framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu")
return &glusterSource{}
}
func (s *glusterSource) createVolume(f *framework.Framework) volInfo {
framework.Logf("Creating GlusterFS server")
_, s.serverPod, _ = framework.NewGlusterfsServer(f.ClientSet, f.Namespace.Name)
return volInfo{
source: &v1.VolumeSource{
Glusterfs: &v1.GlusterfsVolumeSource{
EndpointsName: "gluster-server",
Path: "test_vol",
},
},
}
}
func (s *glusterSource) cleanupVolume(f *framework.Framework) {
if s.serverPod != nil {
framework.DeletePodWithWait(f, f.ClientSet, s.serverPod)
err := f.ClientSet.CoreV1().Endpoints(f.Namespace.Name).Delete("gluster-server", nil)
Expect(err).NotTo(HaveOccurred(), "Gluster delete endpoints failed")
}
}
// TODO: need a better way to wrap PVC. A generic framework should support both static and dynamic PV.
// For static PV, can reuse createVolume methods for inline volumes
type nfsPVCSource struct {
serverPod *v1.Pod
pvc *v1.PersistentVolumeClaim
pv *v1.PersistentVolume
}
func initNFSPVC() volSource {
return &nfsPVCSource{}
}
func (s *nfsPVCSource) createVolume(f *framework.Framework) volInfo {
var serverIP string
framework.Logf("Creating NFS server")
_, s.serverPod, serverIP = framework.NewNFSServer(f.ClientSet, f.Namespace.Name, []string{"-G", "777", "/exports"})
pvConfig := framework.PersistentVolumeConfig{
NamePrefix: "nfs-",
StorageClassName: f.Namespace.Name,
PVSource: v1.PersistentVolumeSource{
NFS: &v1.NFSVolumeSource{
Server: serverIP,
Path: "/exports",
},
},
}
pvcConfig := framework.PersistentVolumeClaimConfig{
StorageClassName: &f.Namespace.Name,
}
framework.Logf("Creating PVC and PV")
pv, pvc, err := framework.CreatePVCPV(f.ClientSet, pvConfig, pvcConfig, f.Namespace.Name, false)
Expect(err).NotTo(HaveOccurred(), "PVC, PV creation failed")
err = framework.WaitOnPVandPVC(f.ClientSet, f.Namespace.Name, pv, pvc)
Expect(err).NotTo(HaveOccurred(), "PVC, PV failed to bind")
s.pvc = pvc
s.pv = pv
return volInfo{
source: &v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: pvc.Name,
},
},
}
}
func (s *nfsPVCSource) cleanupVolume(f *framework.Framework) {
if s.pvc != nil || s.pv != nil {
if errs := framework.PVPVCCleanup(f.ClientSet, f.Namespace.Name, s.pv, s.pvc); len(errs) != 0 {
framework.Failf("Failed to delete PVC or PV: %v", utilerrors.NewAggregate(errs))
}
}
if s.serverPod != nil {
framework.DeletePodWithWait(f, f.ClientSet, s.serverPod)
}
}