Implement necessary API changes

Introduce feature gate for expanding PVs
Add a field to SC
Add new Conditions and feature tag pvc update
Add tests for size update via feature gate
register the resize admission plugin
Update golint failures

cmd/kube-apiserver/app/options/BUILD

@@ -37,6 +37,7 @@ go_library(
         "//plugin/pkg/admission/namespace/exists:go_default_library",
         "//plugin/pkg/admission/noderestriction:go_default_library",
         "//plugin/pkg/admission/persistentvolume/label:go_default_library",
+        "//plugin/pkg/admission/persistentvolume/resize:go_default_library",
         "//plugin/pkg/admission/podnodeselector:go_default_library",
         "//plugin/pkg/admission/podpreset:go_default_library",
         "//plugin/pkg/admission/podtolerationrestriction:go_default_library",

cmd/kube-apiserver/app/options/plugins.go

@@ -41,6 +41,7 @@ import (
 	"k8s.io/kubernetes/plugin/pkg/admission/namespace/exists"
 	"k8s.io/kubernetes/plugin/pkg/admission/noderestriction"
 	"k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label"
+	"k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/resize"
 	"k8s.io/kubernetes/plugin/pkg/admission/podnodeselector"
 	"k8s.io/kubernetes/plugin/pkg/admission/podpreset"
 	"k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction"
@@ -81,4 +82,5 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
 	serviceaccount.Register(plugins)
 	setdefault.Register(plugins)
 	webhook.Register(plugins)
+	resize.Register(plugins)
 }

hack/.golint_failures

@@ -190,6 +190,7 @@ pkg/controller/volume/attachdetach
 pkg/controller/volume/attachdetach/statusupdater
 pkg/controller/volume/attachdetach/testing
 pkg/controller/volume/events
+pkg/controller/volume/expand
 pkg/controller/volume/persistentvolume
 pkg/controller/volume/persistentvolume/options
 pkg/credentialprovider

pkg/api/types.go

@@ -547,6 +547,27 @@ type PersistentVolumeClaimSpec struct {
 	StorageClassName *string
 }
 
+type PersistentVolumeClaimConditionType string
+
+// These are valid conditions of Pvc
+const (
+	// An user trigger resize of pvc has been started
+	PersistentVolumeClaimResizing PersistentVolumeClaimConditionType = "Resizing"
+)
+
+type PersistentVolumeClaimCondition struct {
+	Type   PersistentVolumeClaimConditionType
+	Status ConditionStatus
+	// +optional
+	LastProbeTime metav1.Time
+	// +optional
+	LastTransitionTime metav1.Time
+	// +optional
+	Reason string
+	// +optional
+	Message string
+}
+
 type PersistentVolumeClaimStatus struct {
 	// Phase represents the current phase of PersistentVolumeClaim
 	// +optional
@@ -557,6 +578,8 @@ type PersistentVolumeClaimStatus struct {
 	// Represents the actual resources of the underlying volume
 	// +optional
 	Capacity ResourceList
+	// +optional
+	Conditions []PersistentVolumeClaimCondition
 }
 
 type PersistentVolumeAccessMode string

pkg/api/validation/validation.go

@@ -1588,10 +1588,31 @@ func ValidatePersistentVolumeClaimUpdate(newPvc, oldPvc *api.PersistentVolumeCla
 		oldPvc.Spec.VolumeName = newPvc.Spec.VolumeName
 		defer func() { oldPvc.Spec.VolumeName = "" }()
 	}
-	// changes to Spec are not allowed, but updates to label/and some annotations are OK.
+
-	// no-op updates pass validation.
+	if utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
-	if !apiequality.Semantic.DeepEqual(newPvc.Spec, oldPvc.Spec) {
+		newPVCSpecCopy := newPvc.Spec.DeepCopy()
-		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "field is immutable after creation"))
+
+		// lets make sure storage values are same.
+		if newPvc.Status.Phase == api.ClaimBound && newPVCSpecCopy.Resources.Requests != nil {
+			newPVCSpecCopy.Resources.Requests["storage"] = oldPvc.Spec.Resources.Requests["storage"]
+		}
+
+		oldSize := oldPvc.Spec.Resources.Requests["storage"]
+		newSize := newPvc.Spec.Resources.Requests["storage"]
+
+		if !apiequality.Semantic.DeepEqual(*newPVCSpecCopy, oldPvc.Spec) {
+			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "is immutable after creation except resources.requests for bound claims"))
+		}
+		if newSize.Cmp(oldSize) < 0 {
+			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "resources", "requests", "storage"), "field can not be less than previous value"))
+		}
+
+	} else {
+		// changes to Spec are not allowed, but updates to label/and some annotations are OK.
+		// no-op updates pass validation.
+		if !apiequality.Semantic.DeepEqual(newPvc.Spec, oldPvc.Spec) {
+			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), "field is immutable after creation"))
+		}
 	}
 
 	// storageclass annotation should be immutable after creation
@@ -1611,6 +1632,10 @@ func ValidatePersistentVolumeClaimStatusUpdate(newPvc, oldPvc *api.PersistentVol
 	if len(newPvc.Spec.AccessModes) == 0 {
 		allErrs = append(allErrs, field.Required(field.NewPath("Spec", "accessModes"), ""))
 	}
+	if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) && len(newPvc.Status.Conditions) > 0 {
+		conditionPath := field.NewPath("status", "conditions")
+		allErrs = append(allErrs, field.Forbidden(conditionPath, "invalid field"))
+	}
 	capPath := field.NewPath("status", "capacity")
 	for r, qty := range newPvc.Status.Capacity {
 		allErrs = append(allErrs, validateBasicResource(qty, capPath.Key(string(r)))...)

pkg/api/validation/validation_test.go

@@ -530,6 +530,17 @@ func testVolumeClaim(name string, namespace string, spec api.PersistentVolumeCla
 	}
 }
 
+func testVolumeClaimWithStatus(
+	name, namespace string,
+	spec api.PersistentVolumeClaimSpec,
+	status api.PersistentVolumeClaimStatus) *api.PersistentVolumeClaim {
+	return &api.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
+		Spec:       spec,
+		Status:     status,
+	}
+}
+
 func testVolumeClaimStorageClass(name string, namespace string, annval string, spec api.PersistentVolumeClaimSpec) *api.PersistentVolumeClaim {
 	annotations := map[string]string{
 		v1.BetaStorageClassAnnotation: annval,
@@ -728,7 +739,7 @@ func TestValidatePersistentVolumeClaim(t *testing.T) {
 }
 
 func TestValidatePersistentVolumeClaimUpdate(t *testing.T) {
-	validClaim := testVolumeClaim("foo", "ns", api.PersistentVolumeClaimSpec{
+	validClaim := testVolumeClaimWithStatus("foo", "ns", api.PersistentVolumeClaimSpec{
 		AccessModes: []api.PersistentVolumeAccessMode{
 			api.ReadWriteOnce,
 			api.ReadOnlyMany,
@@ -738,7 +749,10 @@ func TestValidatePersistentVolumeClaimUpdate(t *testing.T) {
 				api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
 			},
 		},
+	}, api.PersistentVolumeClaimStatus{
+		Phase: api.ClaimBound,
 	})
+
 	validClaimStorageClass := testVolumeClaimStorageClass("foo", "ns", "fast", api.PersistentVolumeClaimSpec{
 		AccessModes: []api.PersistentVolumeAccessMode{
 			api.ReadOnlyMany,
@@ -828,50 +842,125 @@ func TestValidatePersistentVolumeClaimUpdate(t *testing.T) {
 		},
 		VolumeName: "volume",
 	})
+	validSizeUpdate := testVolumeClaimWithStatus("foo", "ns", api.PersistentVolumeClaimSpec{
+		AccessModes: []api.PersistentVolumeAccessMode{
+			api.ReadWriteOnce,
+			api.ReadOnlyMany,
+		},
+		Resources: api.ResourceRequirements{
+			Requests: api.ResourceList{
+				api.ResourceName(api.ResourceStorage): resource.MustParse("15G"),
+			},
+		},
+	}, api.PersistentVolumeClaimStatus{
+		Phase: api.ClaimBound,
+	})
+
+	invalidSizeUpdate := testVolumeClaimWithStatus("foo", "ns", api.PersistentVolumeClaimSpec{
+		AccessModes: []api.PersistentVolumeAccessMode{
+			api.ReadWriteOnce,
+			api.ReadOnlyMany,
+		},
+		Resources: api.ResourceRequirements{
+			Requests: api.ResourceList{
+				api.ResourceName(api.ResourceStorage): resource.MustParse("5G"),
+			},
+		},
+	}, api.PersistentVolumeClaimStatus{
+		Phase: api.ClaimBound,
+	})
+
+	unboundSizeUpdate := testVolumeClaimWithStatus("foo", "ns", api.PersistentVolumeClaimSpec{
+		AccessModes: []api.PersistentVolumeAccessMode{
+			api.ReadWriteOnce,
+			api.ReadOnlyMany,
+		},
+		Resources: api.ResourceRequirements{
+			Requests: api.ResourceList{
+				api.ResourceName(api.ResourceStorage): resource.MustParse("12G"),
+			},
+		},
+	}, api.PersistentVolumeClaimStatus{
+		Phase: api.ClaimPending,
+	})
+
 	scenarios := map[string]struct {
 		isExpectedFailure bool
 		oldClaim          *api.PersistentVolumeClaim
 		newClaim          *api.PersistentVolumeClaim
+		enableResize      bool
 	}{
 		"valid-update-volumeName-only": {
 			isExpectedFailure: false,
 			oldClaim:          validClaim,
 			newClaim:          validUpdateClaim,
+			enableResize:      false,
 		},
 		"valid-no-op-update": {
 			isExpectedFailure: false,
 			oldClaim:          validUpdateClaim,
 			newClaim:          validUpdateClaim,
+			enableResize:      false,
 		},
 		"invalid-update-change-resources-on-bound-claim": {
 			isExpectedFailure: true,
 			oldClaim:          validUpdateClaim,
 			newClaim:          invalidUpdateClaimResources,
+			enableResize:      false,
 		},
 		"invalid-update-change-access-modes-on-bound-claim": {
 			isExpectedFailure: true,
 			oldClaim:          validUpdateClaim,
 			newClaim:          invalidUpdateClaimAccessModes,
+			enableResize:      false,
 		},
 		"invalid-update-change-storage-class-annotation-after-creation": {
 			isExpectedFailure: true,
 			oldClaim:          validClaimStorageClass,
 			newClaim:          invalidUpdateClaimStorageClass,
+			enableResize:      false,
 		},
 		"valid-update-mutable-annotation": {
 			isExpectedFailure: false,
 			oldClaim:          validClaimAnnotation,
 			newClaim:          validUpdateClaimMutableAnnotation,
+			enableResize:      false,
 		},
 		"valid-update-add-annotation": {
 			isExpectedFailure: false,
 			oldClaim:          validClaim,
 			newClaim:          validAddClaimAnnotation,
+			enableResize:      false,
+		},
+		"valid-size-update-resize-disabled": {
+			isExpectedFailure: true,
+			oldClaim:          validClaim,
+			newClaim:          validSizeUpdate,
+			enableResize:      false,
+		},
+		"valid-size-update-resize-enabled": {
+			isExpectedFailure: false,
+			oldClaim:          validClaim,
+			newClaim:          validSizeUpdate,
+			enableResize:      true,
+		},
+		"invalid-size-update-resize-enabled": {
+			isExpectedFailure: true,
+			oldClaim:          validClaim,
+			newClaim:          invalidSizeUpdate,
+			enableResize:      true,
+		},
+		"unbound-size-update-resize-enabled": {
+			isExpectedFailure: true,
+			oldClaim:          validClaim,
+			newClaim:          unboundSizeUpdate,
+			enableResize:      true,
 		},
 	}
 
 	for name, scenario := range scenarios {
 		// ensure we have a resource version specified for updates
+		togglePVExpandFeature(scenario.enableResize, t)
 		scenario.oldClaim.ResourceVersion = "1"
 		scenario.newClaim.ResourceVersion = "1"
 		errs := ValidatePersistentVolumeClaimUpdate(scenario.newClaim, scenario.oldClaim)
@@ -884,6 +973,23 @@ func TestValidatePersistentVolumeClaimUpdate(t *testing.T) {
 	}
 }
 
+func togglePVExpandFeature(toggleFlag bool, t *testing.T) {
+	if toggleFlag {
+		// Enable alpha feature LocalStorageCapacityIsolation
+		err := utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=true")
+		if err != nil {
+			t.Errorf("Failed to enable feature gate for ExpandPersistentVolumes: %v", err)
+			return
+		}
+	} else {
+		err := utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=false")
+		if err != nil {
+			t.Errorf("Failed to disable feature gate for ExpandPersistentVolumes: %v", err)
+			return
+		}
+	}
+}
+
 func TestValidateKeyToPath(t *testing.T) {
 	testCases := []struct {
 		kp      api.KeyToPath
@@ -9232,6 +9338,68 @@ func TestValidateLimitRange(t *testing.T) {
 
 }
 
+func TestValidatePersistentVolumeClaimStatusUpdate(t *testing.T) {
+	validClaim := testVolumeClaim("foo", "ns", api.PersistentVolumeClaimSpec{
+		AccessModes: []api.PersistentVolumeAccessMode{
+			api.ReadWriteOnce,
+			api.ReadOnlyMany,
+		},
+		Resources: api.ResourceRequirements{
+			Requests: api.ResourceList{
+				api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
+			},
+		},
+	})
+	validConditionUpdate := testVolumeClaimWithStatus("foo", "ns", api.PersistentVolumeClaimSpec{
+		AccessModes: []api.PersistentVolumeAccessMode{
+			api.ReadWriteOnce,
+			api.ReadOnlyMany,
+		},
+		Resources: api.ResourceRequirements{
+			Requests: api.ResourceList{
+				api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
+			},
+		},
+	}, api.PersistentVolumeClaimStatus{
+		Phase: api.ClaimPending,
+		Conditions: []api.PersistentVolumeClaimCondition{
+			{Type: api.PersistentVolumeClaimResizing, Status: api.ConditionTrue},
+		},
+	})
+	scenarios := map[string]struct {
+		isExpectedFailure bool
+		oldClaim          *api.PersistentVolumeClaim
+		newClaim          *api.PersistentVolumeClaim
+		enableResize      bool
+	}{
+		"condition-update-with-disabled-feature-gate": {
+			isExpectedFailure: true,
+			oldClaim:          validClaim,
+			newClaim:          validConditionUpdate,
+			enableResize:      false,
+		},
+		"condition-update-with-enabled-feature-gate": {
+			isExpectedFailure: false,
+			oldClaim:          validClaim,
+			newClaim:          validConditionUpdate,
+			enableResize:      true,
+		},
+	}
+	for name, scenario := range scenarios {
+		// ensure we have a resource version specified for updates
+		togglePVExpandFeature(scenario.enableResize, t)
+		scenario.oldClaim.ResourceVersion = "1"
+		scenario.newClaim.ResourceVersion = "1"
+		errs := ValidatePersistentVolumeClaimStatusUpdate(scenario.newClaim, scenario.oldClaim)
+		if len(errs) == 0 && scenario.isExpectedFailure {
+			t.Errorf("Unexpected success for scenario: %s", name)
+		}
+		if len(errs) > 0 && !scenario.isExpectedFailure {
+			t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs)
+		}
+	}
+}
+
 func TestValidateResourceQuota(t *testing.T) {
 	spec := api.ResourceQuotaSpec{
 		Hard: api.ResourceList{

pkg/apis/storage/types.go

@@ -59,6 +59,12 @@ type StorageClass struct {
 	// PersistentVolumes of this storage class are created with
 	// +optional
 	MountOptions []string
+
+	// AllowVolumeExpansion shows whether the storage class allow volume expand
+	// If the field is nil or not set, it would amount to expansion disabled
+	// for all PVs created from this storageclass.
+	// +optional
+	AllowVolumeExpansion *bool
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/storage/validation/BUILD

@@ -13,9 +13,11 @@ go_library(
         "//pkg/api:go_default_library",
         "//pkg/api/validation:go_default_library",
         "//pkg/apis/storage:go_default_library",
+        "//pkg/features:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 
@@ -27,6 +29,7 @@ go_test(
         "//pkg/api:go_default_library",
         "//pkg/apis/storage:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
 

pkg/apis/storage/validation/validation.go

@@ -23,9 +23,11 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
 	apivalidation "k8s.io/kubernetes/pkg/api/validation"
 	"k8s.io/kubernetes/pkg/apis/storage"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 // ValidateStorageClass validates a StorageClass.
@@ -34,6 +36,7 @@ func ValidateStorageClass(storageClass *storage.StorageClass) field.ErrorList {
 	allErrs = append(allErrs, validateProvisioner(storageClass.Provisioner, field.NewPath("provisioner"))...)
 	allErrs = append(allErrs, validateParameters(storageClass.Parameters, field.NewPath("parameters"))...)
 	allErrs = append(allErrs, validateReclaimPolicy(storageClass.ReclaimPolicy, field.NewPath("reclaimPolicy"))...)
+	allErrs = append(allErrs, validateAllowVolumeExpansion(storageClass.AllowVolumeExpansion, field.NewPath("allowVolumeExpansion"))...)
 
 	return allErrs
 }
@@ -108,3 +111,13 @@ func validateReclaimPolicy(reclaimPolicy *api.PersistentVolumeReclaimPolicy, fld
 	}
 	return allErrs
 }
+
+// validateAllowVolumeExpansion tests that if ExpandPersistentVolumes feature gate is disabled, whether the AllowVolumeExpansion filed
+// of storage class is set
+func validateAllowVolumeExpansion(allowExpand *bool, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if allowExpand != nil && !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
+		allErrs = append(allErrs, field.Forbidden(fldPath, "field is disabled by feature-gate ExpandPersistentVolumes"))
+	}
+	return allErrs
+}

pkg/apis/storage/validation/validation_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/storage"
 )
@@ -123,3 +124,36 @@ func TestValidateStorageClass(t *testing.T) {
 		}
 	}
 }
+
+func TestAlphaExpandPersistentVolumesFeatureValidation(t *testing.T) {
+	deleteReclaimPolicy := api.PersistentVolumeReclaimPolicy("Delete")
+	falseVar := false
+	testSC := &storage.StorageClass{
+		// empty parameters
+		ObjectMeta:           metav1.ObjectMeta{Name: "foo"},
+		Provisioner:          "kubernetes.io/foo-provisioner",
+		Parameters:           map[string]string{},
+		ReclaimPolicy:        &deleteReclaimPolicy,
+		AllowVolumeExpansion: &falseVar,
+	}
+
+	// Enable alpha feature ExpandPersistentVolumes
+	err := utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=true")
+	if err != nil {
+		t.Errorf("Failed to enable feature gate for ExpandPersistentVolumes: %v", err)
+		return
+	}
+	if errs := ValidateStorageClass(testSC); len(errs) != 0 {
+		t.Errorf("expected success: %v", errs)
+	}
+	// Disable alpha feature ExpandPersistentVolumes
+	err = utilfeature.DefaultFeatureGate.Set("ExpandPersistentVolumes=false")
+	if err != nil {
+		t.Errorf("Failed to disable feature gate for ExpandPersistentVolumes: %v", err)
+		return
+	}
+	if errs := ValidateStorageClass(testSC); len(errs) == 0 {
+		t.Errorf("expected failure, but got no error")
+	}
+
+}

pkg/features/kube_features.go

@@ -114,6 +114,11 @@ const (
 	// New local storage types to support local storage capacity isolation
 	LocalStorageCapacityIsolation utilfeature.Feature = "LocalStorageCapacityIsolation"
 
+	// owner: @gnufied
+	// alpha: v1.8
+	// Ability to Expand persistent volumes
+	ExpandPersistentVolumes utilfeature.Feature = "ExpandPersistentVolumes"
+
 	// owner: @verb
 	// alpha: v1.8
 	//
@@ -179,6 +184,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	EnableEquivalenceClassCache:                 {Default: false, PreRelease: utilfeature.Alpha},
 	TaintNodesByCondition:                       {Default: false, PreRelease: utilfeature.Alpha},
 	MountPropagation:                            {Default: false, PreRelease: utilfeature.Alpha},
+	ExpandPersistentVolumes:                     {Default: false, PreRelease: utilfeature.Alpha},
 
 	// inherited features from generic apiserver, relisted here to get a conflict if it is changed
 	// unintentionally on either side:

pkg/kubelet/events/event.go

@@ -45,6 +45,7 @@ const (
 	FailedAttachVolume                   = "FailedAttachVolume"
 	FailedDetachVolume                   = "FailedDetachVolume"
 	FailedMountVolume                    = "FailedMount"
+	VolumeResizeFailed                   = "VolumeResizeFailed"
 	FailedUnMountVolume                  = "FailedUnMount"
 	WarnAlreadyMountedVolume             = "AlreadyMountedVolume"
 	SuccessfulDetachVolume               = "SuccessfulDetachVolume"

pkg/quota/evaluator/core/BUILD

@@ -25,6 +25,7 @@ go_library(
         "//pkg/api/helper/qos:go_default_library",
         "//pkg/api/v1:go_default_library",
         "//pkg/api/validation:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/kubeapiserver/admission/util:go_default_library",
         "//pkg/quota:go_default_library",
         "//pkg/quota/generic:go_default_library",
@@ -36,6 +37,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
     ],

pkg/quota/evaluator/core/persistent_volume_claims.go

@@ -27,11 +27,13 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/admission"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/helper"
 	k8s_api_v1 "k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/kubeapiserver/admission/util"
 	"k8s.io/kubernetes/pkg/quota"
 	"k8s.io/kubernetes/pkg/quota/generic"
@@ -147,6 +149,10 @@ func (p *pvcEvaluator) Handles(a admission.Attributes) bool {
 	if op == admission.Create {
 		return true
 	}
+	if op == admission.Update && utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
+		return true
+	}
+
 	updateUninitialized, err := util.IsUpdatingUninitializedObject(a)
 	if err != nil {
 		// fail closed, will try to give an evaluation.

pkg/registry/storage/storageclass/strategy.go

@@ -21,9 +21,11 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/storage/names"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/storage"
 	"k8s.io/kubernetes/pkg/apis/storage/validation"
+	"k8s.io/kubernetes/pkg/features"
 )
 
 // storageClassStrategy implements behavior for StorageClass objects
@@ -42,7 +44,11 @@ func (storageClassStrategy) NamespaceScoped() bool {
 
 // ResetBeforeCreate clears the Status field which is not allowed to be set by end users on creation.
 func (storageClassStrategy) PrepareForCreate(ctx genericapirequest.Context, obj runtime.Object) {
-	_ = obj.(*storage.StorageClass)
+	class := obj.(*storage.StorageClass)
+
+	if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
+		class.AllowVolumeExpansion = nil
+	}
 }
 
 func (storageClassStrategy) Validate(ctx genericapirequest.Context, obj runtime.Object) field.ErrorList {
@@ -60,8 +66,13 @@ func (storageClassStrategy) AllowCreateOnUpdate() bool {
 
 // PrepareForUpdate sets the Status fields which is not allowed to be set by an end user updating a PV
 func (storageClassStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old runtime.Object) {
-	_ = obj.(*storage.StorageClass)
+	newClass := obj.(*storage.StorageClass)
-	_ = old.(*storage.StorageClass)
+	oldClass := old.(*storage.StorageClass)
+
+	if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
+		newClass.AllowVolumeExpansion = nil
+		oldClass.AllowVolumeExpansion = nil
+	}
 }
 
 func (storageClassStrategy) ValidateUpdate(ctx genericapirequest.Context, obj, old runtime.Object) field.ErrorList {

plugin/pkg/admission/persistentvolume/resize/BUILD

@@ -0,0 +1,55 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["admission_test.go"],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/apis/storage:go_default_library",
+        "//pkg/client/informers/informers_generated/internalversion:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["admission.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/api/helper:go_default_library",
+        "//pkg/client/informers/informers_generated/internalversion:go_default_library",
+        "//pkg/client/listers/core/internalversion:go_default_library",
+        "//pkg/client/listers/storage/internalversion:go_default_library",
+        "//pkg/kubeapiserver/admission:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

plugin/pkg/admission/persistentvolume/resize/admission.go

@@ -0,0 +1,146 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resize
+
+import (
+	"fmt"
+	"io"
+
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/kubernetes/pkg/api"
+	apihelper "k8s.io/kubernetes/pkg/api/helper"
+	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
+	pvlister "k8s.io/kubernetes/pkg/client/listers/core/internalversion"
+	storagelisters "k8s.io/kubernetes/pkg/client/listers/storage/internalversion"
+	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
+)
+
+const (
+	// PluginName is the name of pvc resize admission plugin
+	PluginName = "PersistentVolumeClaimResize"
+)
+
+// Register registers a plugin
+func Register(plugins *admission.Plugins) {
+	plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
+		plugin := newPlugin()
+		return plugin, nil
+	})
+}
+
+var _ admission.Interface = &persistentVolumeClaimResize{}
+var _ = kubeapiserveradmission.WantsInternalKubeInformerFactory(&persistentVolumeClaimResize{})
+
+type persistentVolumeClaimResize struct {
+	*admission.Handler
+
+	pvLister pvlister.PersistentVolumeLister
+	scLister storagelisters.StorageClassLister
+}
+
+func newPlugin() *persistentVolumeClaimResize {
+	return &persistentVolumeClaimResize{
+		Handler: admission.NewHandler(admission.Update),
+	}
+}
+
+func (pvcr *persistentVolumeClaimResize) SetInternalKubeInformerFactory(f informers.SharedInformerFactory) {
+	pvcInformer := f.Core().InternalVersion().PersistentVolumes()
+	pvcr.pvLister = pvcInformer.Lister()
+	scInformer := f.Storage().InternalVersion().StorageClasses()
+	pvcr.scLister = scInformer.Lister()
+	pvcr.SetReadyFunc(func() bool {
+		return pvcInformer.Informer().HasSynced() && scInformer.Informer().HasSynced()
+	})
+}
+
+// Validate ensures lister is set.
+func (pvcr *persistentVolumeClaimResize) Validate() error {
+	if pvcr.pvLister == nil {
+		return fmt.Errorf("missing persistent volume lister")
+	}
+	if pvcr.scLister == nil {
+		return fmt.Errorf("missing storageclass lister")
+	}
+	return nil
+}
+
+func (pvcr *persistentVolumeClaimResize) Admit(a admission.Attributes) error {
+	if a.GetResource().GroupResource() != api.Resource("persistentvolumeclaims") {
+		return nil
+	}
+
+	if len(a.GetSubresource()) != 0 {
+		return nil
+	}
+
+	pvc, ok := a.GetObject().(*api.PersistentVolumeClaim)
+	// if we can't convert then we don't handle this object so just return
+	if !ok {
+		return nil
+	}
+	oldPvc, ok := a.GetOldObject().(*api.PersistentVolumeClaim)
+	if !ok {
+		return nil
+	}
+
+	// Growing Persistent volumes is only allowed for PVCs for which their StorageClass
+	// explicitly allows it
+	if !pvcr.allowResize(pvc, oldPvc) {
+		return admission.NewForbidden(a, fmt.Errorf("only dynamically provisioned pvc can be resized and "+
+			"the storageclass that provisions the pvc must support resize"))
+	}
+
+	// volume plugin must support resize
+	pv, err := pvcr.pvLister.Get(pvc.Spec.VolumeName)
+	if err != nil {
+		return nil
+	}
+
+	if !pvcr.checkVolumePlugin(pv) {
+		return admission.NewForbidden(a, fmt.Errorf("volume plugin does not support resize"))
+	}
+
+	return nil
+}
+
+// Growing Persistent volumes is only allowed for PVCs for which their StorageClass
+// explicitly allows it.
+func (pvcr *persistentVolumeClaimResize) allowResize(pvc, oldPvc *api.PersistentVolumeClaim) bool {
+	pvcStorageClass := apihelper.GetPersistentVolumeClaimClass(pvc)
+	oldPvcStorageClass := apihelper.GetPersistentVolumeClaimClass(oldPvc)
+	if pvcStorageClass == "" || oldPvcStorageClass == "" || pvcStorageClass != oldPvcStorageClass {
+		return false
+	}
+	sc, err := pvcr.scLister.Get(pvcStorageClass)
+	if err != nil {
+		return false
+	}
+	if sc.AllowVolumeExpansion != nil {
+		return *sc.AllowVolumeExpansion
+	}
+	return false
+}
+
+// checkVolumePlugin checks whether the volume plugin supports resize
+func (pvcr *persistentVolumeClaimResize) checkVolumePlugin(pv *api.PersistentVolume) bool {
+	if pv.Spec.Glusterfs != nil {
+		return true
+	}
+	return false
+
+}

plugin/pkg/admission/persistentvolume/resize/admission_test.go

@@ -0,0 +1,265 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resize
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/storage"
+	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
+	"k8s.io/kubernetes/pkg/controller"
+)
+
+func getResourceList(storage string) api.ResourceList {
+	res := api.ResourceList{}
+	if storage != "" {
+		res[api.ResourceStorage] = resource.MustParse(storage)
+	}
+	return res
+}
+
+func TestPVCResizeAdmission(t *testing.T) {
+	goldClassName := "gold"
+	trueVal := true
+	falseVar := false
+	goldClass := &storage.StorageClass{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "StorageClass",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: goldClassName,
+		},
+		Provisioner:          "kubernetes.io/glusterfs",
+		AllowVolumeExpansion: &trueVal,
+	}
+	silverClassName := "silver"
+	silverClass := &storage.StorageClass{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "StorageClass",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: silverClassName,
+		},
+		Provisioner:          "kubernetes.io/glusterfs",
+		AllowVolumeExpansion: &falseVar,
+	}
+	expectNoError := func(err error) bool {
+		return err == nil
+	}
+	expectDynamicallyProvisionedError := func(err error) bool {
+		return strings.Contains(err.Error(), "only dynamically provisioned pvc can be resized and "+
+			"the storageclass that provisions the pvc must support resize")
+	}
+	expectVolumePluginError := func(err error) bool {
+		return strings.Contains(err.Error(), "volume plugin does not support resize")
+	}
+	tests := []struct {
+		name        string
+		resource    schema.GroupVersionResource
+		subresource string
+		oldObj      runtime.Object
+		newObj      runtime.Object
+
+		checkError func(error) bool
+	}{
+		{
+			name:     "pvc-resize, update, no error",
+			resource: api.SchemeGroupVersion.WithResource("persistentvolumeclaims"),
+			oldObj: &api.PersistentVolumeClaim{
+				Spec: api.PersistentVolumeClaimSpec{
+					VolumeName: "volume1",
+					Resources: api.ResourceRequirements{
+						Requests: getResourceList("1Gi"),
+					},
+					StorageClassName: &goldClassName,
+				},
+				Status: api.PersistentVolumeClaimStatus{
+					Capacity: getResourceList("1Gi"),
+				},
+			},
+			newObj: &api.PersistentVolumeClaim{
+				Spec: api.PersistentVolumeClaimSpec{
+					VolumeName: "volume1",
+					Resources: api.ResourceRequirements{
+						Requests: getResourceList("2Gi"),
+					},
+					StorageClassName: &goldClassName,
+				},
+				Status: api.PersistentVolumeClaimStatus{
+					Capacity: getResourceList("2Gi"),
+				},
+			},
+			checkError: expectNoError,
+		},
+		{
+			name:     "pvc-resize, update, volume plugin error",
+			resource: api.SchemeGroupVersion.WithResource("persistentvolumeclaims"),
+			oldObj: &api.PersistentVolumeClaim{
+				Spec: api.PersistentVolumeClaimSpec{
+					VolumeName: "volume2",
+					Resources: api.ResourceRequirements{
+						Requests: getResourceList("1Gi"),
+					},
+					StorageClassName: &goldClassName,
+				},
+				Status: api.PersistentVolumeClaimStatus{
+					Capacity: getResourceList("1Gi"),
+				},
+			},
+			newObj: &api.PersistentVolumeClaim{
+				Spec: api.PersistentVolumeClaimSpec{
+					VolumeName: "volume2",
+					Resources: api.ResourceRequirements{
+						Requests: getResourceList("2Gi"),
+					},
+					StorageClassName: &goldClassName,
+				},
+				Status: api.PersistentVolumeClaimStatus{
+					Capacity: getResourceList("2Gi"),
+				},
+			},
+			checkError: expectVolumePluginError,
+		},
+		{
+			name:     "pvc-resize, update, dynamically provisioned error",
+			resource: api.SchemeGroupVersion.WithResource("persistentvolumeclaims"),
+			oldObj: &api.PersistentVolumeClaim{
+				Spec: api.PersistentVolumeClaimSpec{
+					VolumeName: "volume3",
+					Resources: api.ResourceRequirements{
+						Requests: getResourceList("1Gi"),
+					},
+				},
+				Status: api.PersistentVolumeClaimStatus{
+					Capacity: getResourceList("1Gi"),
+				},
+			},
+			newObj: &api.PersistentVolumeClaim{
+				Spec: api.PersistentVolumeClaimSpec{
+					VolumeName: "volume3",
+					Resources: api.ResourceRequirements{
+						Requests: getResourceList("2Gi"),
+					},
+				},
+				Status: api.PersistentVolumeClaimStatus{
+					Capacity: getResourceList("2Gi"),
+				},
+			},
+			checkError: expectDynamicallyProvisionedError,
+		},
+		{
+			name:     "pvc-resize, update, dynamically provisioned error",
+			resource: api.SchemeGroupVersion.WithResource("persistentvolumeclaims"),
+			oldObj: &api.PersistentVolumeClaim{
+				Spec: api.PersistentVolumeClaimSpec{
+					VolumeName: "volume4",
+					Resources: api.ResourceRequirements{
+						Requests: getResourceList("1Gi"),
+					},
+					StorageClassName: &silverClassName,
+				},
+				Status: api.PersistentVolumeClaimStatus{
+					Capacity: getResourceList("1Gi"),
+				},
+			},
+			newObj: &api.PersistentVolumeClaim{
+				Spec: api.PersistentVolumeClaimSpec{
+					VolumeName: "volume4",
+					Resources: api.ResourceRequirements{
+						Requests: getResourceList("2Gi"),
+					},
+					StorageClassName: &silverClassName,
+				},
+				Status: api.PersistentVolumeClaimStatus{
+					Capacity: getResourceList("2Gi"),
+				},
+			},
+			checkError: expectDynamicallyProvisionedError,
+		},
+	}
+
+	ctrl := newPlugin()
+	informerFactory := informers.NewSharedInformerFactory(nil, controller.NoResyncPeriodFunc())
+	ctrl.SetInternalKubeInformerFactory(informerFactory)
+	err := ctrl.Validate()
+	if err != nil {
+		t.Fatalf("neither pv lister nor storageclass lister can be nil")
+	}
+
+	pv1 := &api.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{Name: "volume1"},
+		Spec: api.PersistentVolumeSpec{
+			PersistentVolumeSource: api.PersistentVolumeSource{
+				Glusterfs: &api.GlusterfsVolumeSource{
+					EndpointsName: "http://localhost:8080/",
+					Path:          "/heketi",
+					ReadOnly:      false,
+				},
+			},
+			StorageClassName: goldClassName,
+		},
+	}
+	pv2 := &api.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{Name: "volume2"},
+		Spec: api.PersistentVolumeSpec{
+			PersistentVolumeSource: api.PersistentVolumeSource{
+				HostPath: &api.HostPathVolumeSource{},
+			},
+			StorageClassName: goldClassName,
+		},
+	}
+
+	pvs := []*api.PersistentVolume{}
+	pvs = append(pvs, pv1, pv2)
+
+	for _, pv := range pvs {
+		err := informerFactory.Core().InternalVersion().PersistentVolumes().Informer().GetStore().Add(pv)
+		if err != nil {
+			fmt.Println("add pv error: ", err)
+		}
+	}
+
+	scs := []*storage.StorageClass{}
+	scs = append(scs, goldClass, silverClass)
+	for _, sc := range scs {
+		err := informerFactory.Storage().InternalVersion().StorageClasses().Informer().GetStore().Add(sc)
+		if err != nil {
+			fmt.Println("add storageclass error: ", err)
+		}
+	}
+
+	for _, tc := range tests {
+		operation := admission.Update
+		attributes := admission.NewAttributesRecord(tc.newObj, tc.oldObj, schema.GroupVersionKind{}, metav1.NamespaceDefault, "foo", tc.resource, tc.subresource, operation, nil)
+
+		err := ctrl.Admit(attributes)
+		fmt.Println(tc.name)
+		fmt.Println(err)
+		if !tc.checkError(err) {
+			t.Errorf("%v: unexpected err: %v", tc.name, err)
+		}
+	}
+
+}

staging/src/k8s.io/api/core/v1/types.go

@@ -630,6 +630,34 @@ type PersistentVolumeClaimSpec struct {
 	StorageClassName *string `json:"storageClassName,omitempty" protobuf:"bytes,5,opt,name=storageClassName"`
 }
 
+// PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type
+type PersistentVolumeClaimConditionType string
+
+const (
+	// PersistentVolumeClaimResizing - a user trigger resize of pvc has been started
+	PersistentVolumeClaimResizing PersistentVolumeClaimConditionType = "Resizing"
+)
+
+// PersistentVolumeClaimCondition contails details about state of pvc
+type PersistentVolumeClaimCondition struct {
+	Type   PersistentVolumeClaimConditionType `json:"type" protobuf:"bytes,1,opt,name=type,casttype=PersistentVolumeClaimConditionType"`
+	Status ConditionStatus                    `json:"status" protobuf:"bytes,2,opt,name=status,casttype=ConditionStatus"`
+	// Last time we probed the condition.
+	// +optional
+	LastProbeTime metav1.Time `json:"lastProbeTime,omitempty" protobuf:"bytes,3,opt,name=lastProbeTime"`
+	// Last time the condition transitioned from one status to another.
+	// +optional
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,4,opt,name=lastTransitionTime"`
+	// Unique, this should be a short, machine understandable string that gives the reason
+	// for condition's last transition. If it reports "ResizeStarted" that means the underlying
+	// persistent volume is being resized.
+	// +optional
+	Reason string `json:"reason,omitempty" protobuf:"bytes,5,opt,name=reason"`
+	// Human-readable message indicating details about last transition.
+	// +optional
+	Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"`
+}
+
 // PersistentVolumeClaimStatus is the current status of a persistent volume claim.
 type PersistentVolumeClaimStatus struct {
 	// Phase represents the current phase of PersistentVolumeClaim.
@@ -642,6 +670,12 @@ type PersistentVolumeClaimStatus struct {
 	// Represents the actual resources of the underlying volume.
 	// +optional
 	Capacity ResourceList `json:"capacity,omitempty" protobuf:"bytes,3,rep,name=capacity,casttype=ResourceList,castkey=ResourceName"`
+	// Current Condition of persistent volume claim. If underlying persistent volume is being
+	// resized then the Condition will be set to 'ResizeStarted'.
+	// +optional
+	// +patchMergeKey=type
+	// +patchStrategy=merge
+	Conditions []PersistentVolumeClaimCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,4,rep,name=conditions"`
 }
 
 type PersistentVolumeAccessMode string

staging/src/k8s.io/api/storage/v1/types.go

@@ -55,6 +55,10 @@ type StorageClass struct {
 	// mount of the PVs will simply fail if one is invalid.
 	// +optional
 	MountOptions []string `json:"mountOptions,omitempty" protobuf:"bytes,5,opt,name=mountOptions"`
+
+	// AllowVolumeExpansion shows whether the storage class allow volume expand
+	// +optional
+	AllowVolumeExpansion *bool `json:"allowVolumeExpansion,omitempty" protobuf:"varint,6,opt,name=allowVolumeExpansion"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

staging/src/k8s.io/api/storage/v1beta1/types.go

@@ -55,6 +55,10 @@ type StorageClass struct {
 	// mount of the PVs will simply fail if one is invalid.
 	// +optional
 	MountOptions []string `json:"mountOptions,omitempty" protobuf:"bytes,5,opt,name=mountOptions"`
+
+	// AllowVolumeExpansion shows whether the storage class allow volume expand
+	// +optional
+	AllowVolumeExpansion *bool `json:"allowVolumeExpansion,omitempty" protobuf:"varint,6,opt,name=allowVolumeExpansion"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object