diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go index 2881c956eb..39659a8b3e 100644 --- a/cmd/kube-controller-manager/app/plugins.go +++ b/cmd/kube-controller-manager/app/plugins.go @@ -106,10 +106,6 @@ func ProbeControllerVolumePlugins(config kubectrlmgrconfig.VolumeConfiguration) allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(nfsConfig)...) allPlugins = append(allPlugins, local.ProbeVolumePlugins()...) - if utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { - allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) - } - return allPlugins } diff --git a/pkg/api/pod/util.go b/pkg/api/pod/util.go index 555ad95345..233cf77ea7 100644 --- a/pkg/api/pod/util.go +++ b/pkg/api/pod/util.go @@ -443,7 +443,7 @@ func dropDisabledVolumeDevicesFields(podSpec, oldPodSpec *api.PodSpec) { // dropDisabledCSIVolumeSourceAlphaFields removes disabled alpha fields from []CSIVolumeSource. // This should be called from PrepareForCreate/PrepareForUpdate for all pod specs resources containing a CSIVolumeSource func dropDisabledCSIVolumeSourceAlphaFields(podSpec, oldPodSpec *api.PodSpec) { - if !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) && !csiInUse(oldPodSpec) { + if !csiInUse(oldPodSpec) { for i := range podSpec.Volumes { podSpec.Volumes[i].CSI = nil } diff --git a/pkg/api/podsecuritypolicy/util.go b/pkg/api/podsecuritypolicy/util.go index c8ec3493e4..cd8d6cf0c6 100644 --- a/pkg/api/podsecuritypolicy/util.go +++ b/pkg/api/podsecuritypolicy/util.go @@ -35,9 +35,7 @@ func DropDisabledFields(pspSpec, oldPSPSpec *policy.PodSecurityPolicySpec) { pspSpec.AllowedUnsafeSysctls = nil pspSpec.ForbiddenSysctls = nil } - if !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { - pspSpec.AllowedCSIDrivers = nil - } + pspSpec.AllowedCSIDrivers = nil } func allowedProcMountTypesInUse(oldPSPSpec *policy.PodSecurityPolicySpec) bool { diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index 34defb2eb3..35ac6d1c0e 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -139,12 +139,6 @@ func NewAttachDetachController( pvcQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pvcs"), } - if utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && - utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) { - adc.csiNodeLister = csiNodeInformer.Lister() - adc.csiNodeSynced = csiNodeInformer.Informer().HasSynced - } - if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { adc.csiDriverLister = csiDriverInformer.Lister() adc.csiDriversSynced = csiDriverInformer.Informer().HasSynced diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 5d29d406d7..4511bf2306 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -94,11 +94,6 @@ const ( // Ability to Expand persistent volumes ExpandPersistentVolumes utilfeature.Feature = "ExpandPersistentVolumes" - // owner: @gnufied - // alpha: v1.14 - // Ability to expand CSI volumes - ExpandCSIVolumes utilfeature.Feature = "ExpandCSIVolumes" - // owner: @verb // beta: v1.12 // @@ -267,12 +262,6 @@ const ( // Enables CSI to use raw block storage volumes CSIBlockVolume utilfeature.Feature = "CSIBlockVolume" - // owner: @vladimirvivien - // alpha: v1.14 - // - // Enables CSI Inline volumes support for pods - CSIInlineVolume utilfeature.Feature = "CSIInlineVolume" - // owner: @tallclair // alpha: v1.12 // beta: v1.14 @@ -286,30 +275,6 @@ const ( // Kubelet uses the new Lease API to report node heartbeats, // (Kube) Node Lifecycle Controller uses these heartbeats as a node health signal. NodeLease utilfeature.Feature = "NodeLease" - - // owner: @davidz627 - // alpha: v1.14 - // - // Enables the in-tree storage to CSI Plugin migration feature. - CSIMigration utilfeature.Feature = "CSIMigration" - - // owner: @davidz627 - // alpha: v1.14 - // - // Enables the GCE PD in-tree driver to GCE CSI Driver migration feature. - CSIMigrationGCE utilfeature.Feature = "CSIMigrationGCE" - - // owner: @leakingtapan - // alpha: v1.14 - // - // Enables the AWS EBS in-tree driver to AWS EBS CSI Driver migration feature. - CSIMigrationAWS utilfeature.Feature = "CSIMigrationAWS" - - // owner: @adisky - // alpha: v1.14 - // - // Enables the OpenStack Cinder in-tree driver to OpenStack Cinder CSI Driver migration feature. - CSIMigrationOpenStack utilfeature.Feature = "CSIMigrationOpenStack" ) func init() { @@ -335,7 +300,6 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS PodPriority: {Default: true, PreRelease: utilfeature.GA}, TaintNodesByCondition: {Default: true, PreRelease: utilfeature.Beta}, ExpandPersistentVolumes: {Default: true, PreRelease: utilfeature.Beta}, - ExpandCSIVolumes: {Default: false, PreRelease: utilfeature.Alpha}, AttachVolumeLimit: {Default: true, PreRelease: utilfeature.Beta}, CPUManager: {Default: true, PreRelease: utilfeature.Beta}, VolumeScheduling: {Default: true, PreRelease: utilfeature.GA, LockToDefault: true}, // remove in 1.16 @@ -351,17 +315,12 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS TokenRequest: {Default: true, PreRelease: utilfeature.Beta}, TokenRequestProjection: {Default: true, PreRelease: utilfeature.Beta}, CRIContainerLogRotation: {Default: true, PreRelease: utilfeature.Beta}, - CSIMigration: {Default: false, PreRelease: utilfeature.Alpha}, - CSIMigrationGCE: {Default: false, PreRelease: utilfeature.Alpha}, - CSIMigrationAWS: {Default: false, PreRelease: utilfeature.Alpha}, RunAsGroup: {Default: true, PreRelease: utilfeature.Beta}, - CSIMigrationOpenStack: {Default: false, PreRelease: utilfeature.Alpha}, VolumeSubpath: {Default: true, PreRelease: utilfeature.GA}, PodReadinessGates: {Default: true, PreRelease: utilfeature.GA, LockToDefault: true}, // remove in 1.16 KubeletPluginsWatcher: {Default: true, PreRelease: utilfeature.GA, LockToDefault: true}, // remove in 1.16 ResourceQuotaScopeSelectors: {Default: true, PreRelease: utilfeature.Beta}, CSIBlockVolume: {Default: true, PreRelease: utilfeature.Beta}, - CSIInlineVolume: {Default: false, PreRelease: utilfeature.Alpha}, RuntimeClass: {Default: true, PreRelease: utilfeature.Beta}, NodeLease: {Default: true, PreRelease: utilfeature.Beta}, diff --git a/pkg/volume/csi/csi_mounter.go b/pkg/volume/csi/csi_mounter.go index c64c91baa1..5bd021bbc5 100644 --- a/pkg/volume/csi/csi_mounter.go +++ b/pkg/volume/csi/csi_mounter.go @@ -141,23 +141,7 @@ func (c *csiMountMgr) SetUpAt(dir string, fsGroup *int64) error { switch { case volSrc != nil: - if !utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { - return fmt.Errorf("CSIInlineVolume feature required") - } - if c.driverMode != ephemeralDriverMode { - return fmt.Errorf("unexpected driver mode: %s", c.driverMode) - } - if volSrc.FSType != nil { - fsType = *volSrc.FSType - } - - volAttribs = volSrc.VolumeAttributes - - if volSrc.NodePublishSecretRef != nil { - secretName := volSrc.NodePublishSecretRef.Name - ns := c.pod.Namespace - secretRef = &api.SecretReference{Name: secretName, Namespace: ns} - } + return fmt.Errorf("CSIInlineVolume feature required") case pvSrc != nil: if c.driverMode != persistentDriverMode { return fmt.Errorf("unexpected driver mode: %s", c.driverMode) diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go index 0b21d81e53..310aee5e79 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go @@ -33,13 +33,10 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilversion "k8s.io/apimachinery/pkg/util/version" - "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" storagelisters "k8s.io/client-go/listers/storage/v1beta1" - csitranslationplugins "k8s.io/csi-translation-lib/plugins" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csi/nodeinfomanager" @@ -232,81 +229,9 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } } - var migratedPlugins = map[string](func() bool){ - csitranslationplugins.GCEPDInTreePluginName: func() bool { - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationGCE) - }, - csitranslationplugins.AWSEBSInTreePluginName: func() bool { - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationAWS) - }, - csitranslationplugins.CinderInTreePluginName: func() bool { - return utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigrationOpenStack) - }, - } - // Initializing the label management channels - nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, migratedPlugins) + nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host, nil) - if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) && - utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { - // This function prevents Kubelet from posting Ready status until CSINodeInfo - // is both installed and initialized - if err := initializeCSINode(host); err != nil { - return fmt.Errorf("failed to initialize CSINodeInfo: %v", err) - } - } - - return nil -} - -func initializeCSINode(host volume.VolumeHost) error { - kvh, ok := host.(volume.KubeletVolumeHost) - if !ok { - klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINodeInfo initialization, not running on kubelet") - return nil - } - kubeClient := host.GetKubeClient() - if kubeClient == nil { - // Kubelet running in standalone mode. Skip CSINodeInfo initialization - klog.Warning("Skipping CSINodeInfo initialization, kubelet running in standalone mode") - return nil - } - - kvh.SetKubeletError(errors.New("CSINodeInfo is not yet initialized")) - - go func() { - defer utilruntime.HandleCrash() - - // Backoff parameters tuned to retry over 140 seconds. Will fail and restart the Kubelet - // after max retry steps. - initBackoff := wait.Backoff{ - Steps: 6, - Duration: 15 * time.Millisecond, - Factor: 6.0, - Jitter: 0.1, - } - err := wait.ExponentialBackoff(initBackoff, func() (bool, error) { - klog.V(4).Infof("Initializing migrated drivers on CSINodeInfo") - err := nim.InitializeCSINodeWithAnnotation() - if err != nil { - kvh.SetKubeletError(fmt.Errorf("Failed to initialize CSINodeInfo: %v", err)) - klog.Errorf("Failed to initialize CSINodeInfo: %v", err) - return false, nil - } - - // Successfully initialized drivers, allow Kubelet to post Ready - kvh.SetKubeletError(nil) - return true, nil - }) - if err != nil { - // 2 releases after CSIMigration and all CSIMigrationX (where X is a volume plugin) - // are permanently enabled the apiserver/controllers can assume that the kubelet is - // using CSI for all Migrated volume plugins. Then all the CSINode initialization - // code can be dropped from Kubelet. - // Kill the Kubelet process and allow it to restart to retry initialization - klog.Fatalf("Failed to initialize CSINodeInfo after retrying") - } - }() return nil } @@ -333,10 +258,6 @@ func (p *csiPlugin) CanSupport(spec *volume.Spec) bool { if spec == nil { return false } - if utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { - return (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CSI != nil) || - (spec.Volume != nil && spec.Volume.CSI != nil) - } return spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CSI != nil } @@ -354,7 +275,7 @@ func (p *csiPlugin) NewMounter( pod *api.Pod, _ volume.VolumeOptions) (volume.Mounter, error) { - volSrc, pvSrc, err := getSourceFromSpec(spec) + _, pvSrc, err := getSourceFromSpec(spec) if err != nil { return nil, err } @@ -366,12 +287,6 @@ func (p *csiPlugin) NewMounter( ) switch { - case volSrc != nil && utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume): - volumeHandle = makeVolumeHandle(string(pod.UID), spec.Name()) - driverName = volSrc.Driver - if volSrc.ReadOnly != nil { - readOnly = *volSrc.ReadOnly - } case pvSrc != nil: driverName = pvSrc.Driver volumeHandle = pvSrc.VolumeHandle @@ -478,22 +393,7 @@ func (p *csiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.S klog.V(4).Info(log("plugin.ConstructVolumeSpec extracted [%#v]", volData)) var spec *volume.Spec - inlineEnabled := utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) - - if inlineEnabled { - mode := driverMode(volData[volDataKey.driverMode]) - switch { - case mode == ephemeralDriverMode: - spec = p.constructVolSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName]) - - case mode == persistentDriverMode: - fallthrough - default: - spec = p.constructPVSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName], volData[volDataKey.volHandle]) - } - } else { - spec = p.constructPVSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName], volData[volDataKey.volHandle]) - } + spec = p.constructPVSourceSpec(volData[volDataKey.specVolID], volData[volDataKey.driverName], volData[volDataKey.volHandle]) return spec, nil } @@ -787,18 +687,6 @@ func (p *csiPlugin) skipAttach(driver string) (bool, error) { // 2) If Mode cannot be resolved to either {persistent | ephemeral}, an error is returned // See https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md func (p *csiPlugin) getDriverMode(spec *volume.Spec) (driverMode, error) { - // TODO (vladimirvivien) ultimately, mode will be retrieved from CSIDriver.Spec.Mode. - // However, in alpha version, mode is determined by the volume source: - // 1) if volume.Spec.Volume.CSI != nil -> mode is ephemeral - // 2) if volume.Spec.PersistentVolume.Spec.CSI != nil -> persistent - volSrc, _, err := getSourceFromSpec(spec) - if err != nil { - return "", err - } - - if volSrc != nil && utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { - return ephemeralDriverMode, nil - } return persistentDriverMode, nil } diff --git a/pkg/volume/csi/csi_util.go b/pkg/volume/csi/csi_util.go index 393a2cd527..dbb51c5eb5 100644 --- a/pkg/volume/csi/csi_util.go +++ b/pkg/volume/csi/csi_util.go @@ -25,10 +25,8 @@ import ( api "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" "k8s.io/klog" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" utilstrings "k8s.io/utils/strings" ) @@ -146,9 +144,6 @@ func getSourceFromSpec(spec *volume.Spec) (*api.CSIVolumeSource, *api.CSIPersist if spec.Volume != nil && spec.PersistentVolume != nil { return nil, nil, fmt.Errorf("volume.Spec has both volume and persistent volume sources") } - if spec.Volume != nil && spec.Volume.CSI != nil && utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) { - return spec.Volume.CSI, nil, nil - } if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CSI != nil { return nil, spec.PersistentVolume.Spec.CSI, nil diff --git a/pkg/volume/csi/expander.go b/pkg/volume/csi/expander.go index 265d5e5453..993793d52b 100644 --- a/pkg/volume/csi/expander.go +++ b/pkg/volume/csi/expander.go @@ -21,23 +21,15 @@ import ( "fmt" api "k8s.io/api/core/v1" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" ) var _ volume.NodeExpandableVolumePlugin = &csiPlugin{} func (c *csiPlugin) RequiresFSResize() bool { - // We could check plugin's node capability but we instead are going to rely on - // NodeExpand to do the right thing and return early if plugin does not have - // node expansion capability. - if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandCSIVolumes) { - klog.V(4).Infof("Resizing is not enabled for CSI volume") - return false - } - return true + klog.V(4).Infof("Resizing is not enabled for CSI volume") + return false } func (c *csiPlugin) NodeExpand(resizeOptions volume.NodeResizeOptions) (bool, error) { diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 91e4825329..9c9c76920a 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -450,25 +450,9 @@ func (og *operationGenerator) GenerateDetachVolumeFunc( return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.SplitUniqueName failed", err) } - // TODO(dyzz): This case can't distinguish between PV and In-line which is necessary because - // if it was PV it may have been migrated, but the same plugin with in-line may not have been. - // Suggestions welcome... - if csilib.IsMigratableIntreePluginByName(pluginName) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { - // The volume represented by this spec is CSI and thus should be migrated - attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(csi.CSIPluginName) - if err != nil || attachableVolumePlugin == nil { - return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("AttachVolume.FindAttachablePluginBySpec failed", err) - } - // volumeToDetach.VolumeName here is always the in-tree volume name - // therefore a workaround is required. volumeToDetach.DevicePath - // is the attachID which happens to be what volumeName is needed for in Detach. - // Therefore we set volumeName to the attachID. And CSI Detach can detect and use that. - volumeName = volumeToDetach.DevicePath - } else { - attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(pluginName) - if err != nil || attachableVolumePlugin == nil { - return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginByName failed", err) - } + attachableVolumePlugin, err = og.volumePluginMgr.FindAttachablePluginByName(pluginName) + if err != nil || attachableVolumePlugin == nil { + return volumetypes.GeneratedOperations{}, volumeToDetach.GenerateErrorDetailed("DetachVolume.FindAttachablePluginByName failed", err) } } @@ -1624,15 +1608,6 @@ func useCSIPlugin(vpm *volume.VolumePluginMgr, spec *volume.Spec) bool { // we can throw a better error when the driver is not installed. // The error should be of the approximate form: // fmt.Errorf("in-tree plugin %s is migrated on node %s but driver %s is not installed", pluginName, string(nodeName), driverName) - if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { - return false - } - if csilib.IsPVMigratable(spec.PersistentVolume) || csilib.IsInlineMigratable(spec.Volume) { - migratable, err := vpm.IsPluginMigratableBySpec(spec) - if err == nil && migratable { - return true - } - } return false } @@ -1641,8 +1616,7 @@ func nodeUsingCSIPlugin(og *operationGenerator, spec *volume.Spec, nodeName type if err != nil { return false, err } - if !utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) || - !utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) || + if !utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) || !migratable { return false, nil } diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go index c8aea053ad..a7f0f1850a 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/controller_policy.go @@ -76,9 +76,6 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding) if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) { role.Rules = append(role.Rules, rbacv1helpers.NewRule("get", "watch", "list").Groups("storage.k8s.io").Resources("csidrivers").RuleOrDie()) } - if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { - role.Rules = append(role.Rules, rbacv1helpers.NewRule("get", "watch", "list").Groups("storage.k8s.io").Resources("csinodes").RuleOrDie()) - } } return role