Merge pull request #71351 from HotelsDotCom/kep/VolumeSubpathEnvExpansion

kep/VolumeSubpathEnvExpansion

api/openapi-spec/swagger.json

@@ -11058,6 +11058,10 @@
         "subPath": {
           "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).",
           "type": "string"
+        },
+        "subPathExpr": {
+          "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is alpha in 1.14.",
+          "type": "string"
         }
       },
       "required": [

pkg/api/pod/util.go

@@ -350,6 +350,20 @@ func dropDisabledFields(
 		}
 	}
 
+	if (!utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) || !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion)) && !subpathExprInUse(oldPodSpec) {
+		// drop subpath env expansion from the pod if either of the subpath features is disabled and the old spec did not specify subpath env expansion
+		for i := range podSpec.Containers {
+			for j := range podSpec.Containers[i].VolumeMounts {
+				podSpec.Containers[i].VolumeMounts[j].SubPathExpr = ""
+			}
+		}
+		for i := range podSpec.InitContainers {
+			for j := range podSpec.InitContainers[i].VolumeMounts {
+				podSpec.InitContainers[i].VolumeMounts[j].SubPathExpr = ""
+			}
+		}
+	}
+
 	dropDisabledVolumeDevicesFields(podSpec, oldPodSpec)
 
 	dropDisabledRunAsGroupField(podSpec, oldPodSpec)
@@ -595,3 +609,25 @@ func runAsGroupInUse(podSpec *api.PodSpec) bool {
 	}
 	return false
 }
+
+// subpathExprInUse returns true if the pod spec is non-nil and has a volume mount that makes use of the subPathExpr feature
+func subpathExprInUse(podSpec *api.PodSpec) bool {
+	if podSpec == nil {
+		return false
+	}
+	for i := range podSpec.Containers {
+		for j := range podSpec.Containers[i].VolumeMounts {
+			if len(podSpec.Containers[i].VolumeMounts[j].SubPathExpr) > 0 {
+				return true
+			}
+		}
+	}
+	for i := range podSpec.InitContainers {
+		for j := range podSpec.InitContainers[i].VolumeMounts {
+			if len(podSpec.InitContainers[i].VolumeMounts[j].SubPathExpr) > 0 {
+				return true
+			}
+		}
+	}
+	return false
+}

pkg/api/pod/util_test.go

@@ -1549,3 +1549,97 @@ func TestDropPodSysctls(t *testing.T) {
 		}
 	}
 }
+
+func TestDropSubPathExpr(t *testing.T) {
+	podWithSubpaths := func() *api.Pod {
+		return &api.Pod{
+			Spec: api.PodSpec{
+				RestartPolicy:  api.RestartPolicyNever,
+				Containers:     []api.Container{{Name: "container1", Image: "testimage", VolumeMounts: []api.VolumeMount{{Name: "a", SubPathExpr: "foo"}, {Name: "a", SubPathExpr: "foo2"}, {Name: "a", SubPathExpr: "foo3"}}}},
+				InitContainers: []api.Container{{Name: "container1", Image: "testimage", VolumeMounts: []api.VolumeMount{{Name: "a", SubPathExpr: "foo"}, {Name: "a", SubPathExpr: "foo2"}}}},
+				Volumes:        []api.Volume{{Name: "a", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/dev/xvdc"}}}},
+			},
+		}
+	}
+	podWithoutSubpaths := func() *api.Pod {
+		return &api.Pod{
+			Spec: api.PodSpec{
+				RestartPolicy:  api.RestartPolicyNever,
+				Containers:     []api.Container{{Name: "container1", Image: "testimage", VolumeMounts: []api.VolumeMount{{Name: "a", SubPathExpr: ""}, {Name: "a", SubPathExpr: ""}, {Name: "a", SubPathExpr: ""}}}},
+				InitContainers: []api.Container{{Name: "container1", Image: "testimage", VolumeMounts: []api.VolumeMount{{Name: "a", SubPathExpr: ""}, {Name: "a", SubPathExpr: ""}}}},
+				Volumes:        []api.Volume{{Name: "a", VolumeSource: api.VolumeSource{HostPath: &api.HostPathVolumeSource{Path: "/dev/xvdc"}}}},
+			},
+		}
+	}
+
+	podInfo := []struct {
+		description string
+		hasSubpaths bool
+		pod         func() *api.Pod
+	}{
+		{
+			description: "has subpaths",
+			hasSubpaths: true,
+			pod:         podWithSubpaths,
+		},
+		{
+			description: "does not have subpaths",
+			hasSubpaths: false,
+			pod:         podWithoutSubpaths,
+		},
+		{
+			description: "is nil",
+			hasSubpaths: false,
+			pod:         func() *api.Pod { return nil },
+		},
+	}
+
+	for _, enabled := range []bool{true, false} {
+		for _, oldPodInfo := range podInfo {
+			for _, newPodInfo := range podInfo {
+				oldPodHasSubpaths, oldPod := oldPodInfo.hasSubpaths, oldPodInfo.pod()
+				newPodHasSubpaths, newPod := newPodInfo.hasSubpaths, newPodInfo.pod()
+				if newPod == nil {
+					continue
+				}
+
+				t.Run(fmt.Sprintf("feature enabled=%v, old pod %v, new pod %v", enabled, oldPodInfo.description, newPodInfo.description), func(t *testing.T) {
+					defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpathEnvExpansion, enabled)()
+
+					var oldPodSpec *api.PodSpec
+					if oldPod != nil {
+						oldPodSpec = &oldPod.Spec
+					}
+					dropDisabledFields(&newPod.Spec, nil, oldPodSpec, nil)
+
+					// old pod should never be changed
+					if !reflect.DeepEqual(oldPod, oldPodInfo.pod()) {
+						t.Errorf("old pod changed: %v", diff.ObjectReflectDiff(oldPod, oldPodInfo.pod()))
+					}
+
+					switch {
+					case enabled || oldPodHasSubpaths:
+						// new pod should not be changed if the feature is enabled, or if the old pod had subpaths
+						if !reflect.DeepEqual(newPod, newPodInfo.pod()) {
+							t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodInfo.pod()))
+						}
+					case newPodHasSubpaths:
+						// new pod should be changed
+						if reflect.DeepEqual(newPod, newPodInfo.pod()) {
+							t.Errorf("new pod was not changed")
+						}
+						// new pod should not have subpaths
+						if !reflect.DeepEqual(newPod, podWithoutSubpaths()) {
+							t.Errorf("new pod had subpaths: %v", diff.ObjectReflectDiff(newPod, podWithoutSubpaths()))
+						}
+					default:
+						// new pod should not need to be changed
+						if !reflect.DeepEqual(newPod, newPodInfo.pod()) {
+							t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodInfo.pod()))
+						}
+					}
+				})
+			}
+		}
+	}
+}

pkg/apis/core/types.go

@@ -1644,6 +1644,13 @@ type VolumeMount struct {
 	// This field is beta in 1.10.
 	// +optional
 	MountPropagation *MountPropagationMode
+	// Expanded path within the volume from which the container's volume should be mounted.
+	// Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment.
+	// Defaults to "" (volume's root).
+	// SubPathExpr and SubPath are mutually exclusive.
+	// This field is alpha in 1.14.
+	// +optional
+	SubPathExpr string
 }
 
 // MountPropagationMode describes mount propagation.

pkg/apis/core/v1/zz_generated.conversion.go

@@ -7372,6 +7372,7 @@ func autoConvert_v1_VolumeMount_To_core_VolumeMount(in *v1.VolumeMount, out *cor
 	out.MountPath = in.MountPath
 	out.SubPath = in.SubPath
 	out.MountPropagation = (*core.MountPropagationMode)(unsafe.Pointer(in.MountPropagation))
+	out.SubPathExpr = in.SubPathExpr
 	return nil
 }
 
@@ -7386,6 +7387,7 @@ func autoConvert_core_VolumeMount_To_v1_VolumeMount(in *core.VolumeMount, out *v
 	out.MountPath = in.MountPath
 	out.SubPath = in.SubPath
 	out.MountPropagation = (*v1.MountPropagationMode)(unsafe.Pointer(in.MountPropagation))
+	out.SubPathExpr = in.SubPathExpr
 	return nil
 }
 

pkg/apis/core/validation/validation.go

@@ -2264,6 +2264,14 @@ func ValidateVolumeMounts(mounts []core.VolumeMount, voldevices map[string]strin
 			allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPath, fldPath.Child("subPath"))...)
 		}
 
+		if len(mnt.SubPathExpr) > 0 {
+			if len(mnt.SubPath) > 0 {
+				allErrs = append(allErrs, field.Invalid(idxPath.Child("subPathExpr"), mnt.SubPathExpr, "subPathExpr and subPath are mutually exclusive"))
+			}
+
+			allErrs = append(allErrs, validateLocalDescendingPath(mnt.SubPathExpr, fldPath.Child("subPathExpr"))...)
+		}
+
 		if mnt.MountPropagation != nil {
 			allErrs = append(allErrs, validateMountPropagation(mnt.MountPropagation, container, fldPath.Child("mountPropagation"))...)
 		}

pkg/apis/core/validation/validation_test.go

@@ -4816,6 +4816,230 @@ func TestValidateDisabledSubpath(t *testing.T) {
 	}
 }
 
+func TestValidateSubpathMutuallyExclusive(t *testing.T) {
+	// Enable feature VolumeSubpathEnvExpansion and VolumeSubpath
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpathEnvExpansion, true)()
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpath, true)()
+
+	volumes := []core.Volume{
+		{Name: "abc", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim1"}}},
+		{Name: "abc-123", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim2"}}},
+		{Name: "123", VolumeSource: core.VolumeSource{HostPath: &core.HostPathVolumeSource{Path: "/foo/baz", Type: newHostPathType(string(core.HostPathUnset))}}},
+	}
+	vols, v1err := ValidateVolumes(volumes, field.NewPath("field"))
+	if len(v1err) > 0 {
+		t.Errorf("Invalid test volume - expected success %v", v1err)
+		return
+	}
+
+	container := core.Container{
+		SecurityContext: nil,
+	}
+
+	goodVolumeDevices := []core.VolumeDevice{
+		{Name: "xyz", DevicePath: "/foofoo"},
+		{Name: "uvw", DevicePath: "/foofoo/share/test"},
+	}
+
+	cases := map[string]struct {
+		mounts      []core.VolumeMount
+		expectError bool
+	}{
+		"subpath and subpathexpr not specified": {
+			[]core.VolumeMount{
+				{
+					Name:      "abc-123",
+					MountPath: "/bab",
+				},
+			},
+			false,
+		},
+		"subpath expr specified": {
+			[]core.VolumeMount{
+				{
+					Name:        "abc-123",
+					MountPath:   "/bab",
+					SubPathExpr: "$(POD_NAME)",
+				},
+			},
+			false,
+		},
+		"subpath specified": {
+			[]core.VolumeMount{
+				{
+					Name:      "abc-123",
+					MountPath: "/bab",
+					SubPath:   "baz",
+				},
+			},
+			false,
+		},
+		"subpath and subpathexpr specified": {
+			[]core.VolumeMount{
+				{
+					Name:        "abc-123",
+					MountPath:   "/bab",
+					SubPath:     "baz",
+					SubPathExpr: "$(POD_NAME)",
+				},
+			},
+			true,
+		},
+	}
+
+	for name, test := range cases {
+		errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
+
+		if len(errs) != 0 && !test.expectError {
+			t.Errorf("test %v failed: %+v", name, errs)
+		}
+
+		if len(errs) == 0 && test.expectError {
+			t.Errorf("test %v failed, expected error", name)
+		}
+	}
+}
+
+func TestValidateDisabledSubpathExpr(t *testing.T) {
+	// Enable feature VolumeSubpathEnvExpansion
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpathEnvExpansion, true)()
+
+	volumes := []core.Volume{
+		{Name: "abc", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim1"}}},
+		{Name: "abc-123", VolumeSource: core.VolumeSource{PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ClaimName: "testclaim2"}}},
+		{Name: "123", VolumeSource: core.VolumeSource{HostPath: &core.HostPathVolumeSource{Path: "/foo/baz", Type: newHostPathType(string(core.HostPathUnset))}}},
+	}
+	vols, v1err := ValidateVolumes(volumes, field.NewPath("field"))
+	if len(v1err) > 0 {
+		t.Errorf("Invalid test volume - expected success %v", v1err)
+		return
+	}
+
+	container := core.Container{
+		SecurityContext: nil,
+	}
+
+	goodVolumeDevices := []core.VolumeDevice{
+		{Name: "xyz", DevicePath: "/foofoo"},
+		{Name: "uvw", DevicePath: "/foofoo/share/test"},
+	}
+
+	cases := map[string]struct {
+		mounts      []core.VolumeMount
+		expectError bool
+	}{
+		"subpath expr not specified": {
+			[]core.VolumeMount{
+				{
+					Name:      "abc-123",
+					MountPath: "/bab",
+				},
+			},
+			false,
+		},
+		"subpath expr specified": {
+			[]core.VolumeMount{
+				{
+					Name:        "abc-123",
+					MountPath:   "/bab",
+					SubPathExpr: "$(POD_NAME)",
+				},
+			},
+			false,
+		},
+	}
+
+	for name, test := range cases {
+		errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
+
+		if len(errs) != 0 && !test.expectError {
+			t.Errorf("test %v failed: %+v", name, errs)
+		}
+
+		if len(errs) == 0 && test.expectError {
+			t.Errorf("test %v failed, expected error", name)
+		}
+	}
+
+	// Repeat with feature gate off
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpathEnvExpansion, false)()
+	cases = map[string]struct {
+		mounts      []core.VolumeMount
+		expectError bool
+	}{
+		"subpath expr not specified": {
+			[]core.VolumeMount{
+				{
+					Name:      "abc-123",
+					MountPath: "/bab",
+				},
+			},
+			false,
+		},
+		"subpath expr specified": {
+			[]core.VolumeMount{
+				{
+					Name:        "abc-123",
+					MountPath:   "/bab",
+					SubPathExpr: "$(POD_NAME)",
+				},
+			},
+			false, // validation should not fail, dropping the field is handled in PrepareForCreate/PrepareForUpdate
+		},
+	}
+
+	for name, test := range cases {
+		errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
+
+		if len(errs) != 0 && !test.expectError {
+			t.Errorf("test %v failed: %+v", name, errs)
+		}
+
+		if len(errs) == 0 && test.expectError {
+			t.Errorf("test %v failed, expected error", name)
+		}
+	}
+
+	// Repeat with subpath feature gate off
+	defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeSubpath, false)()
+	cases = map[string]struct {
+		mounts      []core.VolumeMount
+		expectError bool
+	}{
+		"subpath expr not specified": {
+			[]core.VolumeMount{
+				{
+					Name:      "abc-123",
+					MountPath: "/bab",
+				},
+			},
+			false,
+		},
+		"subpath expr specified": {
+			[]core.VolumeMount{
+				{
+					Name:        "abc-123",
+					MountPath:   "/bab",
+					SubPathExpr: "$(POD_NAME)",
+				},
+			},
+			false, // validation should not fail, dropping the field is handled in PrepareForCreate/PrepareForUpdate
+		},
+	}
+
+	for name, test := range cases {
+		errs := ValidateVolumeMounts(test.mounts, GetVolumeDeviceMap(goodVolumeDevices), vols, &container, field.NewPath("field"))
+
+		if len(errs) != 0 && !test.expectError {
+			t.Errorf("test %v failed: %+v", name, errs)
+		}
+
+		if len(errs) == 0 && test.expectError {
+			t.Errorf("test %v failed, expected error", name)
+		}
+	}
+}
+
 func TestValidateMountPropagation(t *testing.T) {
 	bTrue := true
 	bFalse := false

pkg/kubelet/container/BUILD

@@ -29,6 +29,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/client-go/tools/record:go_default_library",
         "//staging/src/k8s.io/client-go/tools/reference:go_default_library",
         "//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library",

pkg/kubelet/container/helpers.go

@@ -27,6 +27,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/tools/record"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
@@ -130,9 +131,22 @@ func ExpandContainerCommandOnlyStatic(containerCommand []string, envs []v1.EnvVa
 	return command
 }
 
-func ExpandContainerVolumeMounts(mount v1.VolumeMount, envs []EnvVar) (expandedSubpath string) {
+func ExpandContainerVolumeMounts(mount v1.VolumeMount, envs []EnvVar) (string, error) {
-	mapping := expansion.MappingFuncFor(EnvVarsToMap(envs))
+
-	return expansion.Expand(mount.SubPath, mapping)
+	envmap := EnvVarsToMap(envs)
+	missingKeys := sets.NewString()
+	expanded := expansion.Expand(mount.SubPathExpr, func(key string) string {
+		value, ok := envmap[key]
+		if !ok || len(value) == 0 {
+			missingKeys.Insert(key)
+		}
+		return value
+	})
+
+	if len(missingKeys) > 0 {
+		return "", fmt.Errorf("missing value for %s", strings.Join(missingKeys.List(), ", "))
+	}
+	return expanded, nil
 }
 
 func ExpandContainerCommandAndArgs(container *v1.Container, envs []EnvVar) (command []string, args []string) {

pkg/kubelet/container/helpers_test.go

@@ -145,19 +145,21 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
 		envs              []EnvVar
 		expectedSubPath   string
 		expectedMountPath string
+		expectedOk        bool
 	}{
 		{
 			name: "subpath with no expansion",
 			container: &v1.Container{
-				VolumeMounts: []v1.VolumeMount{{SubPath: "foo"}},
+				VolumeMounts: []v1.VolumeMount{{SubPathExpr: "foo"}},
 			},
 			expectedSubPath:   "foo",
 			expectedMountPath: "",
+			expectedOk:        true,
 		},
 		{
 			name: "volumes with expanded subpath",
 			container: &v1.Container{
-				VolumeMounts: []v1.VolumeMount{{SubPath: "foo/$(POD_NAME)"}},
+				VolumeMounts: []v1.VolumeMount{{SubPathExpr: "foo/$(POD_NAME)"}},
 			},
 			envs: []EnvVar{
 				{
@@ -167,11 +169,12 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
 			},
 			expectedSubPath:   "foo/bar",
 			expectedMountPath: "",
+			expectedOk:        true,
 		},
 		{
 			name: "volumes expanded with empty subpath",
 			container: &v1.Container{
-				VolumeMounts: []v1.VolumeMount{{SubPath: ""}},
+				VolumeMounts: []v1.VolumeMount{{SubPathExpr: ""}},
 			},
 			envs: []EnvVar{
 				{
@@ -181,19 +184,21 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
 			},
 			expectedSubPath:   "",
 			expectedMountPath: "",
+			expectedOk:        true,
 		},
 		{
 			name: "volumes expanded with no envs subpath",
 			container: &v1.Container{
-				VolumeMounts: []v1.VolumeMount{{SubPath: "/foo/$(POD_NAME)"}},
+				VolumeMounts: []v1.VolumeMount{{SubPathExpr: "/foo/$(POD_NAME)"}},
 			},
 			expectedSubPath:   "/foo/$(POD_NAME)",
 			expectedMountPath: "",
+			expectedOk:        false,
 		},
 		{
 			name: "volumes expanded with leading environment variable",
 			container: &v1.Container{
-				VolumeMounts: []v1.VolumeMount{{SubPath: "$(POD_NAME)/bar"}},
+				VolumeMounts: []v1.VolumeMount{{SubPathExpr: "$(POD_NAME)/bar"}},
 			},
 			envs: []EnvVar{
 				{
@@ -203,11 +208,12 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
 			},
 			expectedSubPath:   "foo/bar",
 			expectedMountPath: "",
+			expectedOk:        true,
 		},
 		{
 			name: "volumes with volume and subpath",
 			container: &v1.Container{
-				VolumeMounts: []v1.VolumeMount{{MountPath: "/foo", SubPath: "$(POD_NAME)/bar"}},
+				VolumeMounts: []v1.VolumeMount{{MountPath: "/foo", SubPathExpr: "$(POD_NAME)/bar"}},
 			},
 			envs: []EnvVar{
 				{
@@ -217,6 +223,7 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
 			},
 			expectedSubPath:   "foo/bar",
 			expectedMountPath: "/foo",
+			expectedOk:        true,
 		},
 		{
 			name: "volumes with volume and no subpath",
@@ -231,11 +238,78 @@ func TestExpandVolumeMountsWithSubpath(t *testing.T) {
 			},
 			expectedSubPath:   "",
 			expectedMountPath: "/foo",
+			expectedOk:        true,
+		},
+		{
+			name: "subpaths with empty environment variable",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{SubPathExpr: "foo/$(POD_NAME)/$(ANNOTATION)"}},
+			},
+			envs: []EnvVar{
+				{
+					Name:  "ANNOTATION",
+					Value: "",
+				},
+			},
+			expectedSubPath:   "foo/$(POD_NAME)/$(ANNOTATION)",
+			expectedMountPath: "",
+			expectedOk:        false,
+		},
+		{
+			name: "subpaths with missing env variables",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{SubPathExpr: "foo/$(ODD_NAME)/$(POD_NAME)"}},
+			},
+			envs: []EnvVar{
+				{
+					Name:  "ODD_NAME",
+					Value: "bar",
+				},
+			},
+			expectedSubPath:   "foo/$(ODD_NAME)/$(POD_NAME)",
+			expectedMountPath: "",
+			expectedOk:        false,
+		},
+		{
+			name: "subpaths with empty expansion",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{SubPathExpr: "$()"}},
+			},
+			expectedSubPath:   "$()",
+			expectedMountPath: "",
+			expectedOk:        false,
+		},
+		{
+			name: "subpaths with nested expandable envs",
+			container: &v1.Container{
+				VolumeMounts: []v1.VolumeMount{{SubPathExpr: "$(POD_NAME$(ANNOTATION))"}},
+			},
+			envs: []EnvVar{
+				{
+					Name:  "POD_NAME",
+					Value: "foo",
+				},
+				{
+					Name:  "ANNOTATION",
+					Value: "bar",
+				},
+			},
+			expectedSubPath:   "$(POD_NAME$(ANNOTATION))",
+			expectedMountPath: "",
+			expectedOk:        false,
 		},
 	}
 
 	for _, tc := range cases {
-		actualSubPath := ExpandContainerVolumeMounts(tc.container.VolumeMounts[0], tc.envs)
+		actualSubPath, err := ExpandContainerVolumeMounts(tc.container.VolumeMounts[0], tc.envs)
+		ok := err == nil
+		if e, a := tc.expectedOk, ok; !reflect.DeepEqual(e, a) {
+			t.Errorf("%v: unexpected validation failure of subpath; expected %v, got %v", tc.name, e, a)
+		}
+		if !ok {
+			// if ExpandContainerVolumeMounts returns an error, we don't care what the actualSubPath value is
+			continue
+		}
 		if e, a := tc.expectedSubPath, actualSubPath; !reflect.DeepEqual(e, a) {
 			t.Errorf("%v: unexpected subpath; expected %v, got %v", tc.name, e, a)
 		}

pkg/kubelet/kubelet_pods.go

@@ -159,27 +159,40 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 		if err != nil {
 			return nil, cleanupAction, err
 		}
-		if mount.SubPath != "" {
+
+		subPath := mount.SubPath
+		if mount.SubPathExpr != "" {
 			if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
 				return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
 			}
 
-			// Expand subpath variables
+			if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) {
-			if utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) {
+				return nil, cleanupAction, fmt.Errorf("volume subpath expansion is disabled")
-				mount.SubPath = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs)
 			}
 
-			if filepath.IsAbs(mount.SubPath) {
+			subPath, err = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs)
-				return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", mount.SubPath)
-			}
 
-			err = volumevalidation.ValidatePathNoBacksteps(mount.SubPath)
 			if err != nil {
-				return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", mount.SubPath, err)
+				return nil, cleanupAction, err
+			}
+		}
+
+		if subPath != "" {
+			if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpath) {
+				return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled")
+			}
+
+			if filepath.IsAbs(subPath) {
+				return nil, cleanupAction, fmt.Errorf("error SubPath `%s` must not be an absolute path", subPath)
+			}
+
+			err = volumevalidation.ValidatePathNoBacksteps(subPath)
+			if err != nil {
+				return nil, cleanupAction, fmt.Errorf("unable to provision SubPath `%s`: %v", subPath, err)
 			}
 
 			volumePath := hostPath
-			hostPath = filepath.Join(volumePath, mount.SubPath)
+			hostPath = filepath.Join(volumePath, subPath)
 
 			if subPathExists, err := mounter.ExistsPath(hostPath); err != nil {
 				klog.Errorf("Could not determine if subPath %s exists; will not attempt to change its permissions", hostPath)
@@ -193,7 +206,7 @@ func makeMounts(pod *v1.Pod, podDir string, container *v1.Container, hostName, h
 				if err != nil {
 					return nil, cleanupAction, err
 				}
-				if err := mounter.SafeMakeDir(mount.SubPath, volumePath, perm); err != nil {
+				if err := mounter.SafeMakeDir(subPath, volumePath, perm); err != nil {
 					// Don't pass detailed error back to the user because it could give information about host filesystem
 					klog.Errorf("failed to create subPath directory for volumeMount %q of container %q: %v", mount.Name, container.Name, err)
 					return nil, cleanupAction, fmt.Errorf("failed to create subPath directory for volumeMount %q of container %q", mount.Name, container.Name)

staging/src/k8s.io/api/core/v1/generated.proto

@@ -4604,6 +4604,14 @@ message VolumeMount {
   // This field is beta in 1.10.
   // +optional
   optional string mountPropagation = 5;
+
+  // Expanded path within the volume from which the container's volume should be mounted.
+  // Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment.
+  // Defaults to "" (volume's root).
+  // SubPathExpr and SubPath are mutually exclusive.
+  // This field is alpha in 1.14.
+  // +optional
+  optional string subPathExpr = 6;
 }
 
 // VolumeNodeAffinity defines constraints that limit what nodes this volume can be accessed from.

staging/src/k8s.io/api/core/v1/types.go

@@ -1737,6 +1737,13 @@ type VolumeMount struct {
 	// This field is beta in 1.10.
 	// +optional
 	MountPropagation *MountPropagationMode `json:"mountPropagation,omitempty" protobuf:"bytes,5,opt,name=mountPropagation,casttype=MountPropagationMode"`
+	// Expanded path within the volume from which the container's volume should be mounted.
+	// Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment.
+	// Defaults to "" (volume's root).
+	// SubPathExpr and SubPath are mutually exclusive.
+	// This field is alpha in 1.14.
+	// +optional
+	SubPathExpr string `json:"subPathExpr,omitempty" protobuf:"bytes,6,opt,name=subPathExpr"`
 }
 
 // MountPropagationMode describes mount propagation.

staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go

@@ -2260,6 +2260,7 @@ var map_VolumeMount = map[string]string{
 	"mountPath":        "Path within the container at which the volume should be mounted.  Must not contain ':'.",
 	"subPath":          "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).",
 	"mountPropagation": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10.",
+	"subPathExpr":      "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive. This field is alpha in 1.14.",
 }
 
 func (VolumeMount) SwaggerDoc() map[string]string {

test/e2e/common/expansion.go

@@ -17,11 +17,14 @@ limitations under the License.
 package common
 
 import (
+	"fmt"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
 	imageutils "k8s.io/kubernetes/test/utils/image"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -175,9 +178,9 @@ var _ = framework.KubeDescribe("Variable Expansion", func() {
 						},
 						VolumeMounts: []v1.VolumeMount{
 							{
-								Name:      "workdir1",
+								Name:        "workdir1",
-								MountPath: "/logscontainer",
+								MountPath:   "/logscontainer",
-								SubPath:   "$(POD_NAME)",
+								SubPathExpr: "$(POD_NAME)",
 							},
 							{
 								Name:      "workdir2",
@@ -235,9 +238,9 @@ var _ = framework.KubeDescribe("Variable Expansion", func() {
 						},
 						VolumeMounts: []v1.VolumeMount{
 							{
-								Name:      "workdir1",
+								Name:        "workdir1",
-								MountPath: "/logscontainer",
+								MountPath:   "/logscontainer",
-								SubPath:   "$(POD_NAME)",
+								SubPathExpr: "$(POD_NAME)",
 							},
 						},
 					},
@@ -284,9 +287,9 @@ var _ = framework.KubeDescribe("Variable Expansion", func() {
 						},
 						VolumeMounts: []v1.VolumeMount{
 							{
-								Name:      "workdir1",
+								Name:        "workdir1",
-								MountPath: "/logscontainer",
+								MountPath:   "/logscontainer",
-								SubPath:   "$(POD_NAME)",
+								SubPathExpr: "$(POD_NAME)",
 							},
 						},
 					},
@@ -306,6 +309,338 @@ var _ = framework.KubeDescribe("Variable Expansion", func() {
 		// Pod should fail
 		testPodFailSubpath(f, pod)
 	})
+
+	/*
+	   Testname: var-expansion-subpath-ready-from-failed-state
+	   Description: Verify that a failing subpath expansion can be modified during the lifecycle of a container.
+	*/
+	It("should verify that a failing subpath expansion can be modified during the lifecycle of a container [Feature:VolumeSubpathEnvExpansion][NodeAlphaFeature:VolumeSubpathEnvExpansion][Slow]", func() {
+
+		podName := "var-expansion-" + string(uuid.NewUUID())
+		containerName := "dapi-container"
+		pod := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        podName,
+				Labels:      map[string]string{"name": podName},
+				Annotations: map[string]string{"notmysubpath": "mypath"},
+			},
+			Spec: v1.PodSpec{
+				Containers: []v1.Container{
+					{
+						Name:    containerName,
+						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+						Command: []string{"sh", "-c", "tail -f /dev/null"},
+						Env: []v1.EnvVar{
+							{
+								Name:  "POD_NAME",
+								Value: "foo",
+							},
+							{
+								Name: "ANNOTATION",
+								ValueFrom: &v1.EnvVarSource{
+									FieldRef: &v1.ObjectFieldSelector{
+										APIVersion: "v1",
+										FieldPath:  "metadata.annotations['mysubpath']",
+									},
+								},
+							},
+						},
+						VolumeMounts: []v1.VolumeMount{
+							{
+								Name:        "workdir1",
+								MountPath:   "/subpath_mount",
+								SubPathExpr: "$(ANNOTATION)/$(POD_NAME)",
+							},
+							{
+								Name:      "workdir2",
+								MountPath: "/volume_mount",
+							},
+						},
+					},
+				},
+				Volumes: []v1.Volume{
+					{
+						Name: "workdir1",
+						VolumeSource: v1.VolumeSource{
+							HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
+						},
+					},
+					{
+						Name: "workdir2",
+						VolumeSource: v1.VolumeSource{
+							HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
+						},
+					},
+				},
+			},
+		}
+
+		By("creating the pod with failed condition")
+		pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
+		Expect(err).ToNot(HaveOccurred(), "while creating pod")
+
+		err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
+		Expect(err).To(HaveOccurred(), "while waiting for pod to be running")
+
+		var podClient *framework.PodClient
+		podClient = f.PodClient()
+
+		By("updating the pod")
+		podClient.Update(podName, func(pod *v1.Pod) {
+			pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "mypath"}
+		})
+
+		By("waiting for pod running")
+		err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
+		Expect(err).NotTo(HaveOccurred(), "while waiting for pod to be running")
+
+		By("deleting the pod gracefully")
+		err = framework.DeletePodWithWait(f, f.ClientSet, pod)
+		Expect(err).NotTo(HaveOccurred(), "failed to delete pod")
+	})
+
+	/*
+		    Testname: var-expansion-subpath-test-writes
+		    Description: Verify that a subpath expansion can be used to write files into subpaths.
+			1.	valid subpathexpr starts a container running
+			2.	test for valid subpath writes
+			3.	successful expansion of the subpathexpr isn't required for volume cleanup
+
+	*/
+	It("should succeed in writing subpaths in container [Feature:VolumeSubpathEnvExpansion][NodeAlphaFeature:VolumeSubpathEnvExpansion][Slow]", func() {
+
+		podName := "var-expansion-" + string(uuid.NewUUID())
+		containerName := "dapi-container"
+		pod := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        podName,
+				Labels:      map[string]string{"name": podName},
+				Annotations: map[string]string{"mysubpath": "mypath"},
+			},
+			Spec: v1.PodSpec{
+				Containers: []v1.Container{
+					{
+						Name:    containerName,
+						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+						Command: []string{"sh", "-c", "tail -f /dev/null"},
+						Env: []v1.EnvVar{
+							{
+								Name:  "POD_NAME",
+								Value: "foo",
+							},
+							{
+								Name: "ANNOTATION",
+								ValueFrom: &v1.EnvVarSource{
+									FieldRef: &v1.ObjectFieldSelector{
+										APIVersion: "v1",
+										FieldPath:  "metadata.annotations['mysubpath']",
+									},
+								},
+							},
+						},
+						VolumeMounts: []v1.VolumeMount{
+							{
+								Name:        "workdir1",
+								MountPath:   "/subpath_mount",
+								SubPathExpr: "$(ANNOTATION)/$(POD_NAME)",
+							},
+							{
+								Name:      "workdir2",
+								MountPath: "/volume_mount",
+							},
+						},
+					},
+				},
+				RestartPolicy: v1.RestartPolicyNever,
+				Volumes: []v1.Volume{
+					{
+						Name: "workdir1",
+						VolumeSource: v1.VolumeSource{
+							HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
+						},
+					},
+					{
+						Name: "workdir2",
+						VolumeSource: v1.VolumeSource{
+							HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
+						},
+					},
+				},
+			},
+		}
+
+		By("creating the pod")
+		pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
+
+		By("waiting for pod running")
+		err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
+		Expect(err).NotTo(HaveOccurred(), "while waiting for pod to be running")
+
+		By("creating a file in subpath")
+		cmd := "touch /volume_mount/mypath/foo/test.log"
+		_, err = framework.RunHostCmd(pod.Namespace, pod.Name, cmd)
+		if err != nil {
+			framework.Failf("expected to be able to write to subpath")
+		}
+
+		By("test for file in mounted path")
+		cmd = "test -f /subpath_mount/test.log"
+		_, err = framework.RunHostCmd(pod.Namespace, pod.Name, cmd)
+		if err != nil {
+			framework.Failf("expected to be able to verify file")
+		}
+
+		By("updating the annotation value")
+		var podClient *framework.PodClient
+		podClient = f.PodClient()
+
+		podClient.Update(podName, func(pod *v1.Pod) {
+			pod.ObjectMeta.Annotations["mysubpath"] = "mynewpath"
+		})
+
+		By("waiting for annotated pod running")
+		err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
+		Expect(err).NotTo(HaveOccurred(), "while waiting for annotated pod to be running")
+
+		By("deleting the pod gracefully")
+		err = framework.DeletePodWithWait(f, f.ClientSet, pod)
+		Expect(err).NotTo(HaveOccurred(), "failed to delete pod")
+	})
+
+	/*
+		    Testname: var-expansion-subpath-lifecycle
+		    Description: Verify should not change the subpath mount on a container restart if the environment variable changes
+			1.	valid subpathexpr starts a container running
+			2.	test for valid subpath writes
+			3.	container restarts
+			4.	delete cleanly
+
+	*/
+
+	It("should not change the subpath mount on a container restart if the environment variable changes [Feature:VolumeSubpathEnvExpansion][NodeAlphaFeature:VolumeSubpathEnvExpansion][Slow]", func() {
+
+		suffix := string(uuid.NewUUID())
+		podName := fmt.Sprintf("var-expansion-%s", suffix)
+		containerName := "dapi-container"
+		pod := &v1.Pod{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:        podName,
+				Labels:      map[string]string{"name": podName},
+				Annotations: map[string]string{"mysubpath": "foo"},
+			},
+			Spec: v1.PodSpec{
+				InitContainers: []v1.Container{
+					{
+						Name:    fmt.Sprintf("init-volume-%s", suffix),
+						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+						Command: []string{"sh", "-c", "mkdir -p /volume_mount/foo; touch /volume_mount/foo/test.log"},
+						VolumeMounts: []v1.VolumeMount{
+							{
+								Name:      "workdir1",
+								MountPath: "/subpath_mount",
+							},
+							{
+								Name:      "workdir2",
+								MountPath: "/volume_mount",
+							},
+						},
+					},
+				},
+
+				Containers: []v1.Container{
+					{
+						Name:    containerName,
+						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+						Command: []string{"/bin/sh", "-ec", "sleep 100000"},
+						Env: []v1.EnvVar{
+							{
+								Name: "POD_NAME",
+								ValueFrom: &v1.EnvVarSource{
+									FieldRef: &v1.ObjectFieldSelector{
+										APIVersion: "v1",
+										FieldPath:  "metadata.annotations['mysubpath']",
+									},
+								},
+							},
+						},
+						VolumeMounts: []v1.VolumeMount{
+							{
+								Name:        "workdir1",
+								MountPath:   "/subpath_mount",
+								SubPathExpr: "$(POD_NAME)",
+							},
+							{
+								Name:      "workdir2",
+								MountPath: "/volume_mount",
+							},
+						},
+					},
+				},
+				RestartPolicy: v1.RestartPolicyOnFailure,
+				Volumes: []v1.Volume{
+					{
+						Name: "workdir1",
+						VolumeSource: v1.VolumeSource{
+							HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
+						},
+					},
+					{
+						Name: "workdir2",
+						VolumeSource: v1.VolumeSource{
+							HostPath: &v1.HostPathVolumeSource{Path: "/tmp"},
+						},
+					},
+				},
+			},
+		}
+
+		// Add liveness probe to subpath container
+		pod.Spec.Containers[0].LivenessProbe = &v1.Probe{
+			Handler: v1.Handler{
+				Exec: &v1.ExecAction{
+
+					Command: []string{"cat", "/subpath_mount/test.log"},
+				},
+			},
+			InitialDelaySeconds: 1,
+			FailureThreshold:    1,
+			PeriodSeconds:       2,
+		}
+
+		// Start pod
+		By(fmt.Sprintf("Creating pod %s", pod.Name))
+		pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
+		Expect(err).ToNot(HaveOccurred(), "while creating pod")
+		defer func() {
+			framework.DeletePodWithWait(f, f.ClientSet, pod)
+		}()
+		err = framework.WaitForPodRunningInNamespace(f.ClientSet, pod)
+		Expect(err).ToNot(HaveOccurred(), "while waiting for pod to be running")
+
+		var podClient *framework.PodClient
+		podClient = f.PodClient()
+
+		By("updating the pod")
+		podClient.Update(podName, func(pod *v1.Pod) {
+			pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "newsubpath"}
+		})
+
+		By("waiting for pod and container restart")
+		waitForPodContainerRestart(f, pod, "/volume_mount/foo/test.log")
+
+		By("test for subpath mounted with old value")
+		cmd := "test -f /volume_mount/foo/test.log"
+		_, err = framework.RunHostCmd(pod.Namespace, pod.Name, cmd)
+		if err != nil {
+			framework.Failf("expected to be able to verify old file exists")
+		}
+
+		cmd = "test ! -f /volume_mount/newsubpath/test.log"
+		_, err = framework.RunHostCmd(pod.Namespace, pod.Name, cmd)
+		if err != nil {
+			framework.Failf("expected to be able to verify new file does not exist")
+		}
+	})
 })
 
 func testPodFailSubpath(f *framework.Framework, pod *v1.Pod) {
@@ -320,3 +655,70 @@ func testPodFailSubpath(f *framework.Framework, pod *v1.Pod) {
 	err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout)
 	Expect(err).To(HaveOccurred(), "while waiting for pod to be running")
 }
+
+// Tests that the existing subpath mount is detected when a container restarts
+func waitForPodContainerRestart(f *framework.Framework, pod *v1.Pod, volumeMount string) {
+
+	By("Failing liveness probe")
+	out, err := framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--container", pod.Spec.Containers[0].Name, "--", "/bin/sh", "-c", fmt.Sprintf("rm %v", volumeMount))
+
+	framework.Logf("Pod exec output: %v", out)
+	Expect(err).ToNot(HaveOccurred(), "while failing liveness probe")
+
+	// Check that container has restarted
+	By("Waiting for container to restart")
+	restarts := int32(0)
+	err = wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) {
+		pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, status := range pod.Status.ContainerStatuses {
+			if status.Name == pod.Spec.Containers[0].Name {
+				framework.Logf("Container %v, restarts: %v", status.Name, status.RestartCount)
+				restarts = status.RestartCount
+				if restarts > 0 {
+					framework.Logf("Container has restart count: %v", restarts)
+					return true, nil
+				}
+			}
+		}
+		return false, nil
+	})
+	Expect(err).ToNot(HaveOccurred(), "while waiting for container to restart")
+
+	// Fix liveness probe
+	By("Rewriting the file")
+	out, err = framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--container", pod.Spec.Containers[0].Name, "--", "/bin/sh", "-c", fmt.Sprintf("echo test-after > %v", volumeMount))
+	framework.Logf("Pod exec output: %v", out)
+	Expect(err).ToNot(HaveOccurred(), "while rewriting the probe file")
+
+	// Wait for container restarts to stabilize
+	By("Waiting for container to stop restarting")
+	stableCount := int(0)
+	stableThreshold := int(time.Minute / framework.Poll)
+	err = wait.PollImmediate(framework.Poll, 2*time.Minute, func() (bool, error) {
+		pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(pod.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, status := range pod.Status.ContainerStatuses {
+			if status.Name == pod.Spec.Containers[0].Name {
+				if status.RestartCount == restarts {
+					stableCount++
+					if stableCount > stableThreshold {
+						framework.Logf("Container restart has stabilized")
+						return true, nil
+					}
+				} else {
+					restarts = status.RestartCount
+					stableCount = 0
+					framework.Logf("Container has restart count: %v", restarts)
+				}
+				break
+			}
+		}
+		return false, nil
+	})
+	Expect(err).ToNot(HaveOccurred(), "while waiting for container to stabilize")
+}