github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #71351 from HotelsDotCom/kep/VolumeSubpathEnvExpansion 

kep/VolumeSubpathEnvExpansion
pull/564/head
Kubernetes Prow Robot 2019-02-20 14:05:20 -08:00 committed by GitHub
parent a2a5bd03fd a64b854137
commit 5bfea15e7b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1762 additions and 830 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/openapi-spec/swagger.json generated
View File

@ -11058,6 +11058,10 @@
"subPath": {
"description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).",
"type": "string"
},
"subPathExpr": {
"description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is alpha in 1.14.",
"type": "string"
}
},
"required": [

36
pkg/api/pod/util.go
View File

@ -350,6 +350,20 @@ func dropDisabledFields(
}
}
if (!utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) || !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion)) && !subpathExprInUse(oldPodSpec) {
// drop subpath env expansion from the pod if either of the subpath features is disabled and the old spec did not specify subpath env expansion
for i := range podSpec.Containers {
for j := range podSpec.Containers[i].VolumeMounts {
podSpec.Containers[i].VolumeMounts[j].SubPathExpr = ""
}
}
for i := range podSpec.InitContainers {
for j := range podSpec.InitContainers[i].VolumeMounts {
podSpec.InitContainers[i].VolumeMounts[j].SubPathExpr = ""
}
}
}
dropDisabledVolumeDevicesFields(podSpec, oldPodSpec)
dropDisabledRunAsGroupField(podSpec, oldPodSpec)
@ -595,3 +609,25 @@ func runAsGroupInUse(podSpec *api.PodSpec) bool {
}
return false
}
// subpathExprInUse returns true if the pod spec is non-nil and has a volume mount that makes use of the subPathExpr feature
func subpathExprInUse(podSpec *api.PodSpec) bool {
if podSpec == nil {
return false
}
for i := range podSpec.Containers {
for j := range podSpec.Containers[i].VolumeMounts {
if len(podSpec.Containers[i].VolumeMounts[j].SubPathExpr) > 0 {
return true
}
}
}
for i := range podSpec.InitContainers {
for j := range podSpec.InitContainers[i].VolumeMounts {
if len(podSpec.InitContainers[i].VolumeMounts[j].SubPathExpr) > 0 {
return true
}
}
}
return false
}

94
pkg/api/pod/util_test.go
View File

@ -1549,3 +1549,97 @@ func TestDropPodSysctls(t *testing.T) {
}
}
}
func TestDropSubPathExpr(t *testing.T) {
podWithSubpaths := func() *api.Pod {
return &api.Pod{
Spec: api.PodSpec{
RestartPolicy: api.RestartPolicyNever,
Containers: []api.Container{{Name: "container1", Image: "testimage", VolumeMounts: []api.VolumeMount{{Name: "a", SubPathExpr: "foo"}, {Name: "a", SubPathExpr: "foo2"}, {Name: "a", SubPathExpr: "foo3"}}}},
InitContainers: []api.Container{{Name: "container1", Image: "testimage", VolumeMounts: []api.VolumeMount{{Name: "a", SubPathExpr: "foo"}, {Name: "a", SubPathExpr: "foo2"}}}},
Volumes: []api.Volume{{Name: "a", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/dev/xvdc"}}}},
},
}
}
podWithoutSubpaths := func() *api.Pod {
return &api.Pod{
Spec: api.PodSpec{
RestartPolicy: api.RestartPolicyNever,
Containers: []api.Container{{Name: "container1", Image: "testimage", VolumeMounts: []api.VolumeMount{{Name: "a", SubPathExpr: ""}, {Name: "a", SubPathExpr: ""}, {Name: "a", SubPathExpr: ""}}}},
InitContainers: []api.Container{{Name: "container1", Image: "testimage", VolumeMounts: []api.VolumeMount{{Name: "a", SubPathExpr: ""}, {Name: "a", SubPathExpr: ""}}}},
Volumes: []api.Volume{{Name: "a", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/dev/xvdc"}}}},
},
}
}
podInfo := []struct {
description string
hasSubpaths bool
pod func() *api.Pod
}{
{
description: "has subpaths",
hasSubpaths: true,
pod: podWithSubpaths,
},
{
description: "does not have subpaths",
hasSubpaths: false,
pod: podWithoutSubpaths,
},
{
description: "is nil",
hasSubpaths: false,
pod: func() *api.Pod { return nil },
},
}
for _, enabled := range []bool{true, false} {
for _, oldPodInfo := range podInfo {
for _, newPodInfo := range podInfo {
oldPodHasSubpaths, oldPod := oldPodInfo.hasSubpaths, oldPodInfo.pod()
newPodHasSubpaths, newPod := newPodInfo.hasSubpaths, newPodInfo.pod()
if newPod == nil {
continue
}
t.Run(fmt.Sprintf("feature enabled=%v, old pod %v, new pod %v", enabled, oldPodInfo.description, newPodInfo.description), func(t *testing.T) {
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpathEnvExpansion, enabled)()
var oldPodSpec *api.PodSpec
if oldPod != nil {
oldPodSpec = &oldPod.Spec
}
dropDisabledFields(&newPod.Spec, nil, oldPodSpec, nil)
// old pod should never be changed
if !reflect.DeepEqual(oldPod, oldPodInfo.pod()) {
t.Errorf("old pod changed: %v", diff.ObjectReflectDiff(oldPod, oldPodInfo.pod()))
}
switch {
case enabled || oldPodHasSubpaths:
// new pod should not be changed if the feature is enabled, or if the old pod had subpaths
if !reflect.DeepEqual(newPod, newPodInfo.pod()) {
t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodInfo.pod()))
}
case newPodHasSubpaths:
// new pod should be changed
if reflect.DeepEqual(newPod, newPodInfo.pod()) {
t.Errorf("new pod was not changed")
}
// new pod should not have subpaths
if !reflect.DeepEqual(newPod, podWithoutSubpaths()) {
t.Errorf("new pod had subpaths: %v", diff.ObjectReflectDiff(newPod, podWithoutSubpaths()))
}
default:
// new pod should not need to be changed
if !reflect.DeepEqual(newPod, newPodInfo.pod()) {
t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodInfo.pod()))
}
}
})
}
}
}
}

7
pkg/apis/core/types.go
View File

@ -1644,6 +1644,13 @@ type VolumeMount struct {
// This field is beta in 1.10.
// +optional
MountPropagation *MountPropagationMode
// Expanded path within the volume from which the container's volume should be mounted.
// Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment.
// Defaults to "" (volume's root).
// SubPathExpr and SubPath are mutually exclusive.
// This field is alpha in 1.14.
// +optional
SubPathExpr string
}
// MountPropagationMode describes mount propagation.

2
pkg/apis/core/v1/zz_generated.conversion.go generated
View File

@ -7372,6 +7372,7 @@ func autoConvert_v1_VolumeMount_To_core_VolumeMount(in *v1.VolumeMount, out *cor
out.MountPath = in.MountPath
out.SubPath = in.SubPath
out.MountPropagation = (*core.MountPropagationMode)(unsafe.Pointer(in.MountPropagation))
out.SubPathExpr = in.SubPathExpr
return nil
}
@ -7386,6 +7387,7 @@ func autoConvert_core_VolumeMount_To_v1_VolumeMount(in *core.VolumeMount, out *v
out.MountPath = in.MountPath
out.SubPath = in.SubPath
out.MountPropagation = (*v1.MountPropagationMode)(unsafe.Pointer(in.MountPropagation))
out.SubPathExpr = in.SubPathExpr
return nil
}

8
pkg/apis/core/validation/validation.go
View File

@ -2264,6 +2264,14 @@ func ValidateVolumeMounts(mounts []core.VolumeMount, voldevices map[string]strin
allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPath, fldPath.Child("subPath"))...)
}
if len(mnt.SubPathExpr) > 0 {
if len(mnt.SubPath) > 0 {
allErrs = append(allErrs, field.Invalid(idxPath.Child("subPathExpr"), mnt.SubPathExpr, "subPathExpr and subPath are mutually exclusive"))
}
allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPathExpr, fldPath.Child("subPathExpr"))...)
}
if mnt.MountPropagation != nil {
allErrs = append(allErrs, validateMountPropagation(mnt.MountPropagation, container, fldPath.Child("mountPropagation"))...)
}

224
pkg/apis/core/validation/validation_test.go
View File

@ -4816,6 +4816,230 @@ func TestValidateDisabledSubpath(t *testing.T) {
}
}
func TestValidateSubpathMutuallyExclusive(t *testing.T) {
// Enable feature VolumeSubpathEnvExpansion and VolumeSubpath
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpathEnvExpansion, true)()
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpath, true)()
volumes := []core.Volume{
{Name: "abc", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim1"}}},
{Name: "abc-123", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim2"}}},
{Name: "123", VolumeSource: core.VolumeSource{HostPath: &core.HostPathVolumeSource{Path: "/foo/baz", Type: newHostPathType(string(core.HostPathUnset))}}},
}
vols, v1err := ValidateVolumes(volumes, field.NewPath("field"))
if len(v1err) > 0 {
t.Errorf("Invalid test volume - expected success %v", v1err)
return
}
container := core.Container{
SecurityContext: nil,
}
goodVolumeDevices := []core.VolumeDevice{
{Name: "xyz", DevicePath: "/foofoo"},
{Name: "uvw", DevicePath: "/foofoo/share/test"},
}
cases := map[string]struct {
mounts []core.VolumeMount
expectError bool
}{
"subpath and subpathexpr not specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
},
},
false,
},
"subpath expr specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
SubPathExpr: "$(POD_NAME)",
},
},
false,
},
"subpath specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
SubPath: "baz",
},
},
false,
},
"subpath and subpathexpr specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
SubPath: "baz",
SubPathExpr: "$(POD_NAME)",
},
},
true,
},
}
for name, test := range cases {
errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
if len(errs) != 0 && !test.expectError {
t.Errorf("test %v failed: %+v", name, errs)
}
if len(errs) == 0 && test.expectError {
t.Errorf("test %v failed, expected error", name)
}
}
}
func TestValidateDisabledSubpathExpr(t *testing.T) {
// Enable feature VolumeSubpathEnvExpansion
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpathEnvExpansion, true)()
volumes := []core.Volume{
{Name: "abc", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim1"}}},
{Name: "abc-123", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim2"}}},
{Name: "123", VolumeSource: core.VolumeSource{HostPath: &core.HostPathVolumeSource{Path: "/foo/baz", Type: newHostPathType(string(core.HostPathUnset))}}},
}
vols, v1err := ValidateVolumes(volumes, field.NewPath("field"))
if len(v1err) > 0 {
t.Errorf("Invalid test volume - expected success %v", v1err)
return
}
container := core.Container{
SecurityContext: nil,
}
goodVolumeDevices := []core.VolumeDevice{
{Name: "xyz", DevicePath: "/foofoo"},
{Name: "uvw", DevicePath: "/foofoo/share/test"},
}
cases := map[string]struct {
mounts []core.VolumeMount
expectError bool
}{
"subpath expr not specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
},
},
false,
},
"subpath expr specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
SubPathExpr: "$(POD_NAME)",
},
},
false,
},
}
for name, test := range cases {
errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
if len(errs) != 0 && !test.expectError {
t.Errorf("test %v failed: %+v", name, errs)
}
if len(errs) == 0 && test.expectError {
t.Errorf("test %v failed, expected error", name)
}
}
// Repeat with feature gate off
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpathEnvExpansion, false)()
cases = map[string]struct {
mounts []core.VolumeMount
expectError bool
}{
"subpath expr not specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
},
},
false,
},
"subpath expr specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
SubPathExpr: "$(POD_NAME)",
},
},
false, // validation should not fail, dropping the field is handled in PrepareForCreate/PrepareForUpdate
},
}
for name, test := range cases {
errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
if len(errs) != 0 && !test.expectError {
t.Errorf("test %v failed: %+v", name, errs)
}
if len(errs) == 0 && test.expectError {
t.Errorf("test %v failed, expected error", name)
}
}
// Repeat with subpath feature gate off
defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpath, false)()
cases = map[string]struct {
mounts []core.VolumeMount
expectError bool
}{
"subpath expr not specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
},
},
false,
},
"subpath expr specified": {
[]core.VolumeMount{
{
Name: "abc-123",
MountPath: "/bab",
SubPathExpr: "$(POD_NAME)",
},
},
false, // validation should not fail, dropping the field is handled in PrepareForCreate/PrepareForUpdate
},
}
for name, test := range cases {
errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
if len(errs) != 0 && !test.expectError {
t.Errorf("test %v failed: %+v", name, errs)
}
if len(errs) == 0 && test.expectError {
t.Errorf("test %v failed, expected error", name)
}
}
}
func TestValidateMountPropagation(t *testing.T) {
bTrue := true
bFalse := false

1
pkg/kubelet/container/BUILD
View File

@ -29,6 +29,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
"//staging/src/k8s.io/client-go/tools/reference:go_default_library",
"//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library",

20
pkg/kubelet/container/helpers.go
View File

@ -27,6 +27,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/record"
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
"k8s.io/kubernetes/pkg/kubelet/util/format"
@ -130,9 +131,22 @@ func ExpandContainerCommandOnlyStatic(containerCommand []string, envs []v1.EnvVa
return command
}
func ExpandContainerVolumeMounts(mount v1.VolumeMount, envs []EnvVar) (expandedSubpath string) {
mapping := expansion.MappingFuncFor(EnvVarsToMap(envs))
return expansion.Expand(mount.SubPath, mapping)
func ExpandContainerVolumeMounts(mount v1.VolumeMount, envs []EnvVar) (string, error) {
envmap := EnvVarsToMap(envs)
missingKeys := sets.NewString()
expanded := expansion.Expand(mount.SubPathExpr, func(key string) string {
value, ok := envmap[key]
if !ok || len(value) == 0 {
missingKeys.Insert(key)
}
return value
})
if len(missingKeys) > 0 {
return "", fmt.Errorf("missing value for %s", strings.Join(missingKeys.List(), ", "))
}
return expanded, nil
}
func ExpandContainerCommandAndArgs(container *v1.Container, envs []EnvVar) (command []string, args []string) {

88
pkg/kubelet/container/helpers_test.go
View File

@ -145,19 +145,21 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
envs []EnvVar
expectedSubPath string
expectedMountPath string
expectedOk bool
}{
{
name: "subpath with no expansion",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{SubPath: "foo"}},
VolumeMounts: []v1.VolumeMount{{SubPathExpr: "foo"}},
},
expectedSubPath: "foo",
expectedMountPath: "",
expectedOk: true,
},
{
name: "volumes with expanded subpath",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{SubPath: "foo/$(POD_NAME)"}},
VolumeMounts: []v1.VolumeMount{{SubPathExpr: "foo/$(POD_NAME)"}},
},
envs: []EnvVar{
{
@ -167,11 +169,12 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
},
expectedSubPath: "foo/bar",
expectedMountPath: "",
expectedOk: true,
},
{
name: "volumes expanded with empty subpath",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{SubPath: ""}},
VolumeMounts: []v1.VolumeMount{{SubPathExpr: ""}},
},
envs: []EnvVar{
{
@ -181,19 +184,21 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
},
expectedSubPath: "",
expectedMountPath: "",
expectedOk: true,
},
{
name: "volumes expanded with no envs subpath",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{SubPath: "/foo/$(POD_NAME)"}},
VolumeMounts: []v1.VolumeMount{{SubPathExpr: "/foo/$(POD_NAME)"}},
},
expectedSubPath: "/foo/$(POD_NAME)",
expectedMountPath: "",
expectedOk: false,
},
{
name: "volumes expanded with leading environment variable",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{SubPath: "$(POD_NAME)/bar"}},
VolumeMounts: []v1.VolumeMount{{SubPathExpr: "$(POD_NAME)/bar"}},
},
envs: []EnvVar{
{
@ -203,11 +208,12 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
},
expectedSubPath: "foo/bar",
expectedMountPath: "",
expectedOk: true,
},
{
name: "volumes with volume and subpath",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{MountPath: "/foo", SubPath: "$(POD_NAME)/bar"}},
VolumeMounts: []v1.VolumeMount{{MountPath: "/foo", SubPathExpr: "$(POD_NAME)/bar"}},
},
envs: []EnvVar{
{
@ -217,6 +223,7 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
},
expectedSubPath: "foo/bar",
expectedMountPath: "/foo",
expectedOk: true,
},
{
name: "volumes with volume and no subpath",
@ -231,11 +238,78 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
},
expectedSubPath: "",
expectedMountPath: "/foo",
expectedOk: true,
},
{
name: "subpaths with empty environment variable",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{SubPathExpr: "foo/$(POD_NAME)/$(ANNOTATION)"}},
},
envs: []EnvVar{
{
Name: "ANNOTATION",
Value: "",
},
},
expectedSubPath: "foo/$(POD_NAME)/$(ANNOTATION)",
expectedMountPath: "",
expectedOk: false,
},
{
name: "subpaths with missing env variables",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{SubPathExpr: "foo/$(ODD_NAME)/$(POD_NAME)"}},
},
envs: []EnvVar{
{
Name: "ODD_NAME",
Value: "bar",
},
},
expectedSubPath: "foo/$(ODD_NAME)/$(POD_NAME)",
expectedMountPath: "",
expectedOk: false,
},
{
name: "subpaths with empty expansion",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{SubPathExpr: "$()"}},
},
expectedSubPath: "$()",
expectedMountPath: "",
expectedOk: false,
},
{
name: "subpaths with nested expandable envs",
container: &v1.Container{
VolumeMounts: []v1.VolumeMount{{SubPathExpr: "$(POD_NAME$(ANNOTATION))"}},
},
envs: []EnvVar{
{
Name: "POD_NAME",
Value: "foo",
},
{
Name: "ANNOTATION",
Value: "bar",
},
},
expectedSubPath: "$(POD_NAME$(ANNOTATION))",
expectedMountPath: "",
expectedOk: false,
},
}
for _, tc := range cases {
actualSubPath := ExpandContainerVolumeMounts(tc.container.VolumeMounts[0], tc.envs)
actualSubPath, err := ExpandContainerVolumeMounts(tc.container.VolumeMounts[0], tc.envs)
ok := err == nil
if e, a := tc.expectedOk, ok; !reflect.DeepEqual(e, a) {
t.Errorf("%v: unexpected validation failure of subpath; expected %v, got %v", tc.name, e, a)
}
if !ok {
// if ExpandContainerVolumeMounts returns an error, we don't care what the actualSubPath value is
continue
}
if e, a := tc.expectedSubPath, actualSubPath; !reflect.DeepEqual(e, a) {
t.Errorf("%v: unexpected subpath; expected %v, got %v", tc.name, e, a)
}

35
pkg/kubelet/kubelet_pods.go
View File

@ -159,27 +159,40 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
if err != nil {
return nil, cleanupAction, err
}
if mount.SubPath != "" {
subPath := mount.SubPath
if mount.SubPathExpr != "" {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
}
// Expand subpath variables
if utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) {
mount.SubPath = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs)
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) {
return nil, cleanupAction, fmt.Errorf("volume subpath expansion is disabled")
}
if filepath.IsAbs(mount.SubPath) {
return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath)
}
subPath, err = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs)
err = volumevalidation.ValidatePathNoBacksteps(mount.SubPath)
if err != nil {
return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
return nil, cleanupAction, err
}
}
if subPath != "" {
if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
}
if filepath.IsAbs(subPath) {
return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", subPath)
}
err = volumevalidation.ValidatePathNoBacksteps(subPath)
if err != nil {
return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", subPath, err)
}
volumePath := hostPath
hostPath = filepath.Join(volumePath, mount.SubPath)
hostPath = filepath.Join(volumePath, subPath)
if subPathExists, err := mounter.ExistsPath(hostPath); err != nil {
klog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath)
@ -193,7 +206,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
if err != nil {
return nil, cleanupAction, err
}
if err := mounter.SafeMakeDir(mount.SubPath, volumePath, perm); err != nil {
if err := mounter.SafeMakeDir(subPath, volumePath, perm); err != nil {
// Don't pass detailed error back to the user because it could give information about host filesystem
klog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name)

1637
staging/src/k8s.io/api/core/v1/generated.pb.go generated
View File

File diff suppressed because it is too large Load Diff

8
staging/src/k8s.io/api/core/v1/generated.proto
View File

@ -4604,6 +4604,14 @@ message VolumeMount {
// This field is beta in 1.10.
// +optional
optional string mountPropagation = 5;
// Expanded path within the volume from which the container's volume should be mounted.
// Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment.
// Defaults to "" (volume's root).
// SubPathExpr and SubPath are mutually exclusive.
// This field is alpha in 1.14.
// +optional
optional string subPathExpr = 6;
}
// VolumeNodeAffinity defines constraints that limit what nodes this volume can be accessed from.

7
staging/src/k8s.io/api/core/v1/types.go
View File

@ -1737,6 +1737,13 @@ type VolumeMount struct {
// This field is beta in 1.10.
// +optional
MountPropagation *MountPropagationMode `json:"mountPropagation,omitempty" protobuf:"bytes,5,opt,name=mountPropagation,casttype=MountPropagationMode"`
// Expanded path within the volume from which the container's volume should be mounted.
// Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment.
// Defaults to "" (volume's root).
// SubPathExpr and SubPath are mutually exclusive.
// This field is alpha in 1.14.
// +optional
SubPathExpr string `json:"subPathExpr,omitempty" protobuf:"bytes,6,opt,name=subPathExpr"`
}
// MountPropagationMode describes mount propagation.

1
staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go generated
View File

@ -2260,6 +2260,7 @@ var map_VolumeMount = map[string]string{
"mountPath": "Path within the container at which the volume should be mounted. Must not contain ':'.",
"subPath": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).",
"mountPropagation": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.",
"subPathExpr": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is alpha in 1.14.",
}
func (VolumeMount) SwaggerDoc() map[string]string {

420
test/e2e/common/expansion.go
View File

@ -17,11 +17,14 @@ limitations under the License.
package common
import (
"fmt"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/test/e2e/framework"
imageutils "k8s.io/kubernetes/test/utils/image"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -175,9 +178,9 @@ var _ = framework.KubeDescribe("Variable Expansion", func() {
},
VolumeMounts: []v1.VolumeMount{
{
Name: "workdir1",
MountPath: "/logscontainer",
SubPath: "$(POD_NAME)",
Name: "workdir1",
MountPath: "/logscontainer",
SubPathExpr: "$(POD_NAME)",
},
{
Name: "workdir2",
@ -235,9 +238,9 @@ var _ = framework.KubeDescribe("Variable Expansion", func() {
},
VolumeMounts: []v1.VolumeMount{
{
Name: "workdir1",
MountPath: "/logscontainer",
SubPath: "$(POD_NAME)",
Name: "workdir1",
MountPath: "/logscontainer",
SubPathExpr: "$(POD_NAME)",
},
},
},
@ -284,9 +287,9 @@ var _ = framework.KubeDescribe("Variable Expansion", func() {
},
VolumeMounts: []v1.VolumeMount{
{
Name: "workdir1",
MountPath: "/logscontainer",
SubPath: "$(POD_NAME)",
Name: "workdir1",
MountPath: "/logscontainer",
SubPathExpr: "$(POD_NAME)",
},
},
},
@ -306,6 +309,338 @@ var _ = framework.KubeDescribe("Variable Expansion", func() {
// Pod should fail
testPodFailSubpath(f, pod)
})
/*
Testname: var-expansion-subpath-ready-from-failed-state
Description: Verify that a failing subpath expansion can be modified during the lifecycle of a container.
*/
It("should verify that a failing subpath expansion can be modified during the lifecycle of a container [Feature:VolumeSubpathEnvExpansion][NodeAlphaFeature:VolumeSubpathEnvExpansion][Slow]", func() {
podName := "var-expansion-" + string(uuid.NewUUID())
containerName := "dapi-container"
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Labels: map[string]string{"name": podName},
Annotations: map[string]string{"notmysubpath": "mypath"},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: containerName,
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"sh", "-c", "tail -f /dev/null"},
Env: []v1.EnvVar{
{
Name: "POD_NAME",
Value: "foo",
},
{
Name: "ANNOTATION",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
APIVersion: "v1",
FieldPath: "metadata.annotations['mysubpath']",
},
},
},
},
VolumeMounts: []v1.VolumeMount{
{
Name: "workdir1",
MountPath: "/subpath_mount",
SubPathExpr: "$(ANNOTATION)/$(POD_NAME)",
},
{
Name: "workdir2",
MountPath: "/volume_mount",
},
},
},
},
Volumes: []v1.Volume{
{
Name: "workdir1",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
},
},
{
Name: "workdir2",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
},
},
},
},
}
By("creating the pod with failed condition")
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
Expect(err).ToNot(HaveOccurred(), "while creating pod")
err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
Expect(err).To(HaveOccurred(), "while waiting for pod to be running")
var podClient *framework.PodClient
podClient = f.PodClient()
By("updating the pod")
podClient.Update(podName, func(pod *v1.Pod) {
pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "mypath"}
})
By("waiting for pod running")
err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
Expect(err).NotTo(HaveOccurred(), "while waiting for pod to be running")
By("deleting the pod gracefully")
err = framework.DeletePodWithWait(f, f.ClientSet, pod)
Expect(err).NotTo(HaveOccurred(), "failed to delete pod")
})
/*
Testname: var-expansion-subpath-test-writes
Description: Verify that a subpath expansion can be used to write files into subpaths.
1. valid subpathexpr starts a container running
2. test for valid subpath writes
3. successful expansion of the subpathexpr isn't required for volume cleanup
*/
It("should succeed in writing subpaths in container [Feature:VolumeSubpathEnvExpansion][NodeAlphaFeature:VolumeSubpathEnvExpansion][Slow]", func() {
podName := "var-expansion-" + string(uuid.NewUUID())
containerName := "dapi-container"
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Labels: map[string]string{"name": podName},
Annotations: map[string]string{"mysubpath": "mypath"},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: containerName,
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"sh", "-c", "tail -f /dev/null"},
Env: []v1.EnvVar{
{
Name: "POD_NAME",
Value: "foo",
},
{
Name: "ANNOTATION",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
APIVersion: "v1",
FieldPath: "metadata.annotations['mysubpath']",
},
},
},
},
VolumeMounts: []v1.VolumeMount{
{
Name: "workdir1",
MountPath: "/subpath_mount",
SubPathExpr: "$(ANNOTATION)/$(POD_NAME)",
},
{
Name: "workdir2",
MountPath: "/volume_mount",
},
},
},
},
RestartPolicy: v1.RestartPolicyNever,
Volumes: []v1.Volume{
{
Name: "workdir1",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
},
},
{
Name: "workdir2",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
},
},
},
},
}
By("creating the pod")
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
By("waiting for pod running")
err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
Expect(err).NotTo(HaveOccurred(), "while waiting for pod to be running")
By("creating a file in subpath")
cmd := "touch /volume_mount/mypath/foo/test.log"
_, err = framework.RunHostCmd(pod.Namespace, pod.Name, cmd)
if err != nil {
framework.Failf("expected to be able to write to subpath")
}
By("test for file in mounted path")
cmd = "test -f /subpath_mount/test.log"
_, err = framework.RunHostCmd(pod.Namespace, pod.Name, cmd)
if err != nil {
framework.Failf("expected to be able to verify file")
}
By("updating the annotation value")
var podClient *framework.PodClient
podClient = f.PodClient()
podClient.Update(podName, func(pod *v1.Pod) {
pod.ObjectMeta.Annotations["mysubpath"] = "mynewpath"
})
By("waiting for annotated pod running")
err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
Expect(err).NotTo(HaveOccurred(), "while waiting for annotated pod to be running")
By("deleting the pod gracefully")
err = framework.DeletePodWithWait(f, f.ClientSet, pod)
Expect(err).NotTo(HaveOccurred(), "failed to delete pod")
})
/*
Testname: var-expansion-subpath-lifecycle
Description: Verify should not change the subpath mount on a container restart if the environment variable changes
1. valid subpathexpr starts a container running
2. test for valid subpath writes
3. container restarts
4. delete cleanly
*/
It("should not change the subpath mount on a container restart if the environment variable changes [Feature:VolumeSubpathEnvExpansion][NodeAlphaFeature:VolumeSubpathEnvExpansion][Slow]", func() {
suffix := string(uuid.NewUUID())
podName := fmt.Sprintf("var-expansion-%s", suffix)
containerName := "dapi-container"
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Labels: map[string]string{"name": podName},
Annotations: map[string]string{"mysubpath": "foo"},
},
Spec: v1.PodSpec{
InitContainers: []v1.Container{
{
Name: fmt.Sprintf("init-volume-%s", suffix),
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"sh", "-c", "mkdir -p /volume_mount/foo; touch /volume_mount/foo/test.log"},
VolumeMounts: []v1.VolumeMount{
{
Name: "workdir1",
MountPath: "/subpath_mount",
},
{
Name: "workdir2",
MountPath: "/volume_mount",
},
},
},
},
Containers: []v1.Container{
{
Name: containerName,
Image: imageutils.GetE2EImage(imageutils.BusyBox),
Command: []string{"/bin/sh", "-ec", "sleep 100000"},
Env: []v1.EnvVar{
{
Name: "POD_NAME",
ValueFrom: &v1.EnvVarSource{
FieldRef: &v1.ObjectFieldSelector{
APIVersion: "v1",
FieldPath: "metadata.annotations['mysubpath']",
},
},
},
},
VolumeMounts: []v1.VolumeMount{
{
Name: "workdir1",
MountPath: "/subpath_mount",
SubPathExpr: "$(POD_NAME)",
},
{
Name: "workdir2",
MountPath: "/volume_mount",
},
},
},
},
RestartPolicy: v1.RestartPolicyOnFailure,
Volumes: []v1.Volume{
{
Name: "workdir1",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
},
},
{
Name: "workdir2",
VolumeSource: v1.VolumeSource{
HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
},
},
},
},
}
// Add liveness probe to subpath container
pod.Spec.Containers[0].LivenessProbe = &v1.Probe{
Handler: v1.Handler{
Exec: &v1.ExecAction{
Command: []string{"cat", "/subpath_mount/test.log"},
},
},
InitialDelaySeconds: 1,
FailureThreshold: 1,
PeriodSeconds: 2,
}
// Start pod
By(fmt.Sprintf("Creating pod %s", pod.Name))
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
Expect(err).ToNot(HaveOccurred(), "while creating pod")
defer func() {
framework.DeletePodWithWait(f, f.ClientSet, pod)
}()
err = framework.WaitForPodRunningInNamespace(f.ClientSet, pod)
Expect(err).ToNot(HaveOccurred(), "while waiting for pod to be running")
var podClient *framework.PodClient
podClient = f.PodClient()
By("updating the pod")
podClient.Update(podName, func(pod *v1.Pod) {
pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "newsubpath"}
})
By("waiting for pod and container restart")
waitForPodContainerRestart(f, pod, "/volume_mount/foo/test.log")
By("test for subpath mounted with old value")
cmd := "test -f /volume_mount/foo/test.log"
_, err = framework.RunHostCmd(pod.Namespace, pod.Name, cmd)
if err != nil {
framework.Failf("expected to be able to verify old file exists")
}
cmd = "test ! -f /volume_mount/newsubpath/test.log"
_, err = framework.RunHostCmd(pod.Namespace, pod.Name, cmd)
if err != nil {
framework.Failf("expected to be able to verify new file does not exist")
}
})
})
func testPodFailSubpath(f *framework.Framework, pod *v1.Pod) {
@ -320,3 +655,70 @@ func testPodFailSubpath(f *framework.Framework, pod *v1.Pod) {
err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
Expect(err).To(HaveOccurred(), "while waiting for pod to be running")
}
// Tests that the existing subpath mount is detected when a container restarts
func waitForPodContainerRestart(f *framework.Framework, pod *v1.Pod, volumeMount string) {
By("Failing liveness probe")
out, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--container", pod.Spec.Containers[0].Name, "--", "/bin/sh", "-c", fmt.Sprintf("rm %v", volumeMount))
framework.Logf("Pod exec output: %v", out)
Expect(err).ToNot(HaveOccurred(), "while failing liveness probe")
// Check that container has restarted
By("Waiting for container to restart")
restarts := int32(0)
err = wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) {
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, status := range pod.Status.ContainerStatuses {
if status.Name == pod.Spec.Containers[0].Name {
framework.Logf("Container %v, restarts: %v", status.Name, status.RestartCount)
restarts = status.RestartCount
if restarts > 0 {
framework.Logf("Container has restart count: %v", restarts)
return true, nil
}
}
}
return false, nil
})
Expect(err).ToNot(HaveOccurred(), "while waiting for container to restart")
// Fix liveness probe
By("Rewriting the file")
out, err = framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--container", pod.Spec.Containers[0].Name, "--", "/bin/sh", "-c", fmt.Sprintf("echo test-after > %v", volumeMount))
framework.Logf("Pod exec output: %v", out)
Expect(err).ToNot(HaveOccurred(), "while rewriting the probe file")
// Wait for container restarts to stabilize
By("Waiting for container to stop restarting")
stableCount := int(0)
stableThreshold := int(time.Minute / framework.Poll)
err = wait.PollImmediate(framework.Poll, 2*time.Minute, func() (bool, error) {
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, status := range pod.Status.ContainerStatuses {
if status.Name == pod.Spec.Containers[0].Name {
if status.RestartCount == restarts {
stableCount++
if stableCount > stableThreshold {
framework.Logf("Container restart has stabilized")
return true, nil
}
} else {
restarts = status.RestartCount
stableCount = 0
framework.Logf("Container has restart count: %v", restarts)
}
break
}
}
return false, nil
})
Expect(err).ToNot(HaveOccurred(), "while waiting for container to stabilize")
}