2019-01-12 04:58:27 +00:00
/ *
Copyright 2017 The Kubernetes Authors .
Licensed under the Apache License , Version 2.0 ( the "License" ) ;
you may not use this file except in compliance with the License .
You may obtain a copy of the License at
http : //www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing , software
distributed under the License is distributed on an "AS IS" BASIS ,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND , either express or implied .
See the License for the specific language governing permissions and
limitations under the License .
* /
package csi
import (
2020-05-26 22:59:35 +00:00
"context"
2019-01-12 04:58:27 +00:00
"errors"
"fmt"
"os"
2020-03-26 21:07:15 +00:00
"path/filepath"
2019-01-12 04:58:27 +00:00
"strings"
"time"
"k8s.io/klog"
api "k8s.io/api/core/v1"
2020-03-26 21:07:15 +00:00
storage "k8s.io/api/storage/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
2019-01-12 04:58:27 +00:00
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
2019-08-30 18:33:25 +00:00
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
2019-01-12 04:58:27 +00:00
utilversion "k8s.io/apimachinery/pkg/util/version"
2019-08-30 18:33:25 +00:00
"k8s.io/apimachinery/pkg/util/wait"
2019-04-07 17:07:55 +00:00
utilfeature "k8s.io/apiserver/pkg/util/feature"
2019-01-12 04:58:27 +00:00
clientset "k8s.io/client-go/kubernetes"
2020-03-26 21:07:15 +00:00
storagelisters "k8s.io/client-go/listers/storage/v1"
2019-08-30 18:33:25 +00:00
csitranslationplugins "k8s.io/csi-translation-lib/plugins"
2019-04-07 17:07:55 +00:00
"k8s.io/kubernetes/pkg/features"
2019-01-12 04:58:27 +00:00
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/csi/nodeinfomanager"
)
const (
2019-04-07 17:07:55 +00:00
// CSIPluginName is the name of the in-tree CSI Plugin
CSIPluginName = "kubernetes.io/csi"
2019-01-12 04:58:27 +00:00
2019-08-19 16:30:36 +00:00
csiTimeout = 2 * time . Minute
2019-01-12 04:58:27 +00:00
volNameSep = "^"
volDataFileName = "vol_data.json"
fsTypeBlockName = "block"
2019-04-07 17:07:55 +00:00
2019-09-27 21:51:53 +00:00
// CsiResyncPeriod is default resync period duration
2019-04-07 17:07:55 +00:00
// TODO: increase to something useful
2019-09-27 21:51:53 +00:00
CsiResyncPeriod = time . Minute
2019-01-12 04:58:27 +00:00
)
type csiPlugin struct {
2019-07-14 07:58:54 +00:00
host volume . VolumeHost
blockEnabled bool
csiDriverLister storagelisters . CSIDriverLister
2019-01-12 04:58:27 +00:00
}
// ProbeVolumePlugins returns implemented plugins
func ProbeVolumePlugins ( ) [ ] volume . VolumePlugin {
2019-09-01 05:39:24 +00:00
p := & csiPlugin {
2019-04-07 17:07:55 +00:00
host : nil ,
blockEnabled : utilfeature . DefaultFeatureGate . Enabled ( features . CSIBlockVolume ) ,
2019-01-12 04:58:27 +00:00
}
2019-09-01 05:39:24 +00:00
return [ ] volume . VolumePlugin { p }
2019-01-12 04:58:27 +00:00
}
// volume.VolumePlugin methods
var _ volume . VolumePlugin = & csiPlugin { }
// RegistrationHandler is the handler which is fed to the pluginwatcher API.
type RegistrationHandler struct {
}
// TODO (verult) consider using a struct instead of global variables
// csiDrivers map keep track of all registered CSI drivers on the node and their
// corresponding sockets
2019-04-07 17:07:55 +00:00
var csiDrivers = & DriversStore { }
2019-01-12 04:58:27 +00:00
var nim nodeinfomanager . Interface
// PluginHandler is the plugin registration handler interface passed to the
// pluginwatcher module in kubelet
var PluginHandler = & RegistrationHandler { }
// ValidatePlugin is called by kubelet's plugin watcher upon detection
// of a new registration socket opened by CSI Driver registrar side car.
2019-12-12 01:27:03 +00:00
func ( h * RegistrationHandler ) ValidatePlugin ( pluginName string , endpoint string , versions [ ] string ) error {
klog . Infof ( log ( "Trying to validate a new CSI Driver with name: %s endpoint: %s versions: %s" ,
pluginName , endpoint , strings . Join ( versions , "," ) ) )
2019-01-12 04:58:27 +00:00
_ , err := h . validateVersions ( "ValidatePlugin" , pluginName , endpoint , versions )
2019-12-12 01:27:03 +00:00
if err != nil {
return fmt . Errorf ( "validation failed for CSI Driver %s at endpoint %s: %v" , pluginName , endpoint , err )
}
2019-01-12 04:58:27 +00:00
return err
}
// RegisterPlugin is called when a plugin can be registered
func ( h * RegistrationHandler ) RegisterPlugin ( pluginName string , endpoint string , versions [ ] string ) error {
klog . Infof ( log ( "Register new plugin with name: %s at endpoint: %s" , pluginName , endpoint ) )
highestSupportedVersion , err := h . validateVersions ( "RegisterPlugin" , pluginName , endpoint , versions )
if err != nil {
return err
}
2019-04-07 17:07:55 +00:00
// Storing endpoint of newly registered CSI driver into the map, where CSI driver name will be the key
// all other CSI components will be able to get the actual socket of CSI drivers by its name.
csiDrivers . Set ( pluginName , Driver {
endpoint : endpoint ,
highestSupportedVersion : highestSupportedVersion ,
} )
2019-01-12 04:58:27 +00:00
// Get node info from the driver.
csi , err := newCsiDriverClient ( csiDriverName ( pluginName ) )
if err != nil {
return err
}
ctx , cancel := context . WithTimeout ( context . Background ( ) , csiTimeout )
defer cancel ( )
driverNodeID , maxVolumePerNode , accessibleTopology , err := csi . NodeGetInfo ( ctx )
if err != nil {
if unregErr := unregisterDriver ( pluginName ) ; unregErr != nil {
2019-04-07 17:07:55 +00:00
klog . Error ( log ( "registrationHandler.RegisterPlugin failed to unregister plugin due to previous error: %v" , unregErr ) )
2019-01-12 04:58:27 +00:00
}
return err
}
err = nim . InstallCSIDriver ( pluginName , driverNodeID , maxVolumePerNode , accessibleTopology )
if err != nil {
if unregErr := unregisterDriver ( pluginName ) ; unregErr != nil {
klog . Error ( log ( "registrationHandler.RegisterPlugin failed to unregister plugin due to previous error: %v" , unregErr ) )
}
return err
}
return nil
}
func ( h * RegistrationHandler ) validateVersions ( callerName , pluginName string , endpoint string , versions [ ] string ) ( * utilversion . Version , error ) {
if len ( versions ) == 0 {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "%s for CSI driver %q failed. Plugin returned an empty list for supported versions" , callerName , pluginName ) )
2019-01-12 04:58:27 +00:00
}
// Validate version
newDriverHighestVersion , err := highestSupportedVersion ( versions )
if err != nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "%s for CSI driver %q failed. None of the versions specified %q are supported. err=%v" , callerName , pluginName , versions , err ) )
2019-01-12 04:58:27 +00:00
}
2019-04-07 17:07:55 +00:00
existingDriver , driverExists := csiDrivers . Get ( pluginName )
2019-01-12 04:58:27 +00:00
if driverExists {
if ! existingDriver . highestSupportedVersion . LessThan ( newDriverHighestVersion ) {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "%s for CSI driver %q failed. Another driver with the same name is already registered with a higher supported version: %q" , callerName , pluginName , existingDriver . highestSupportedVersion ) )
2019-01-12 04:58:27 +00:00
}
}
return newDriverHighestVersion , nil
}
// DeRegisterPlugin is called when a plugin removed its socket, signaling
// it is no longer available
func ( h * RegistrationHandler ) DeRegisterPlugin ( pluginName string ) {
2019-09-27 21:51:53 +00:00
klog . Info ( log ( "registrationHandler.DeRegisterPlugin request for plugin %s" , pluginName ) )
2019-01-12 04:58:27 +00:00
if err := unregisterDriver ( pluginName ) ; err != nil {
klog . Error ( log ( "registrationHandler.DeRegisterPlugin failed: %v" , err ) )
}
}
func ( p * csiPlugin ) Init ( host volume . VolumeHost ) error {
p . host = host
2020-03-26 21:07:15 +00:00
csiClient := host . GetKubeClient ( )
if csiClient == nil {
klog . Warning ( log ( "kubeclient not set, assuming standalone kubelet" ) )
} else {
// set CSIDriverLister
adcHost , ok := host . ( volume . AttachDetachVolumeHost )
if ok {
p . csiDriverLister = adcHost . CSIDriverLister ( )
if p . csiDriverLister == nil {
klog . Error ( log ( "CSIDriverLister not found on AttachDetachVolumeHost" ) )
2019-07-14 07:58:54 +00:00
}
2020-03-26 21:07:15 +00:00
}
kletHost , ok := host . ( volume . KubeletVolumeHost )
if ok {
p . csiDriverLister = kletHost . CSIDriverLister ( )
if p . csiDriverLister == nil {
klog . Error ( log ( "CSIDriverLister not found on KubeletVolumeHost" ) )
2019-07-14 07:58:54 +00:00
}
2019-04-07 17:07:55 +00:00
}
}
2019-01-12 04:58:27 +00:00
2019-08-30 18:33:25 +00:00
var migratedPlugins = map [ string ] ( func ( ) bool ) {
csitranslationplugins . GCEPDInTreePluginName : func ( ) bool {
return utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigration ) && utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigrationGCE )
} ,
csitranslationplugins . AWSEBSInTreePluginName : func ( ) bool {
return utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigration ) && utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigrationAWS )
} ,
csitranslationplugins . CinderInTreePluginName : func ( ) bool {
return utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigration ) && utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigrationOpenStack )
} ,
2020-03-13 19:24:50 +00:00
csitranslationplugins . AzureDiskInTreePluginName : func ( ) bool {
return utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigration ) && utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigrationAzureDisk )
} ,
csitranslationplugins . AzureFileInTreePluginName : func ( ) bool {
return utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigration ) && utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigrationAzureFile )
} ,
2019-08-30 18:33:25 +00:00
}
2019-04-07 17:07:55 +00:00
// Initializing the label management channels
2019-09-01 05:39:24 +00:00
localNim := nodeinfomanager . NewNodeInfoManager ( host . GetNodeName ( ) , host , migratedPlugins )
2019-08-30 18:33:25 +00:00
if utilfeature . DefaultFeatureGate . Enabled ( features . CSINodeInfo ) &&
utilfeature . DefaultFeatureGate . Enabled ( features . CSIMigration ) {
2020-05-26 22:59:35 +00:00
// This function prevents Kubelet from posting Ready status until CSINode
2019-08-30 18:33:25 +00:00
// is both installed and initialized
2019-09-01 05:39:24 +00:00
if err := initializeCSINode ( host , localNim ) ; err != nil {
2019-09-27 21:51:53 +00:00
return errors . New ( log ( "failed to initialize CSINodeInfo: %v" , err ) )
2019-08-30 18:33:25 +00:00
}
}
2019-01-12 04:58:27 +00:00
2019-09-01 05:39:24 +00:00
if _ , ok := host . ( volume . KubeletVolumeHost ) ; ok {
nim = localNim
}
2019-01-12 04:58:27 +00:00
return nil
}
2019-09-01 05:39:24 +00:00
func initializeCSINode ( host volume . VolumeHost , nim nodeinfomanager . Interface ) error {
2019-08-30 18:33:25 +00:00
kvh , ok := host . ( volume . KubeletVolumeHost )
if ! ok {
2020-05-26 22:59:35 +00:00
klog . V ( 4 ) . Info ( "Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet" )
2019-08-30 18:33:25 +00:00
return nil
}
kubeClient := host . GetKubeClient ( )
if kubeClient == nil {
2020-05-26 22:59:35 +00:00
// Kubelet running in standalone mode. Skip CSINode initialization
klog . Warning ( "Skipping CSINode initialization, kubelet running in standalone mode" )
2019-08-30 18:33:25 +00:00
return nil
}
2020-05-26 22:59:35 +00:00
kvh . SetKubeletError ( errors . New ( "CSINode is not yet initialized" ) )
2019-08-30 18:33:25 +00:00
go func ( ) {
defer utilruntime . HandleCrash ( )
2020-05-26 22:59:35 +00:00
// First wait indefinitely to talk to Kube APIServer
nodeName := host . GetNodeName ( )
err := waitForAPIServerForever ( kubeClient , nodeName )
if err != nil {
klog . Fatalf ( "Failed to initialize CSINode while waiting for API server to report ok: %v" , err )
}
2019-08-30 18:33:25 +00:00
// Backoff parameters tuned to retry over 140 seconds. Will fail and restart the Kubelet
// after max retry steps.
initBackoff := wait . Backoff {
Steps : 6 ,
Duration : 15 * time . Millisecond ,
Factor : 6.0 ,
Jitter : 0.1 ,
}
2020-05-26 22:59:35 +00:00
err = wait . ExponentialBackoff ( initBackoff , func ( ) ( bool , error ) {
klog . V ( 4 ) . Infof ( "Initializing migrated drivers on CSINode" )
2019-08-30 18:33:25 +00:00
err := nim . InitializeCSINodeWithAnnotation ( )
if err != nil {
2020-05-26 22:59:35 +00:00
kvh . SetKubeletError ( fmt . Errorf ( "Failed to initialize CSINode: %v" , err ) )
klog . Errorf ( "Failed to initialize CSINode: %v" , err )
2019-08-30 18:33:25 +00:00
return false , nil
}
// Successfully initialized drivers, allow Kubelet to post Ready
kvh . SetKubeletError ( nil )
return true , nil
} )
if err != nil {
// 2 releases after CSIMigration and all CSIMigrationX (where X is a volume plugin)
// are permanently enabled the apiserver/controllers can assume that the kubelet is
// using CSI for all Migrated volume plugins. Then all the CSINode initialization
// code can be dropped from Kubelet.
// Kill the Kubelet process and allow it to restart to retry initialization
2020-05-26 22:59:35 +00:00
klog . Fatalf ( "Failed to initialize CSINode after retrying: %v" , err )
2019-08-30 18:33:25 +00:00
}
} ( )
return nil
}
2019-01-12 04:58:27 +00:00
func ( p * csiPlugin ) GetPluginName ( ) string {
2019-04-07 17:07:55 +00:00
return CSIPluginName
2019-01-12 04:58:27 +00:00
}
// GetvolumeName returns a concatenated string of CSIVolumeSource.Driver<volNameSe>CSIVolumeSource.VolumeHandle
// That string value is used in Detach() to extract driver name and volumeName.
func ( p * csiPlugin ) GetVolumeName ( spec * volume . Spec ) ( string , error ) {
2019-04-07 17:07:55 +00:00
csi , err := getPVSourceFromSpec ( spec )
2019-01-12 04:58:27 +00:00
if err != nil {
2019-09-27 21:51:53 +00:00
return "" , errors . New ( log ( "plugin.GetVolumeName failed to extract volume source from spec: %v" , err ) )
2019-01-12 04:58:27 +00:00
}
// return driverName<separator>volumeHandle
return fmt . Sprintf ( "%s%s%s" , csi . Driver , volNameSep , csi . VolumeHandle ) , nil
}
func ( p * csiPlugin ) CanSupport ( spec * volume . Spec ) bool {
// TODO (vladimirvivien) CanSupport should also take into account
// the availability/registration of specified Driver in the volume source
2019-04-07 17:07:55 +00:00
if spec == nil {
return false
}
2019-08-30 18:33:25 +00:00
if utilfeature . DefaultFeatureGate . Enabled ( features . CSIInlineVolume ) {
return ( spec . PersistentVolume != nil && spec . PersistentVolume . Spec . CSI != nil ) ||
( spec . Volume != nil && spec . Volume . CSI != nil )
}
2019-04-07 17:07:55 +00:00
2019-01-12 04:58:27 +00:00
return spec . PersistentVolume != nil && spec . PersistentVolume . Spec . CSI != nil
}
func ( p * csiPlugin ) RequiresRemount ( ) bool {
return false
}
func ( p * csiPlugin ) NewMounter (
spec * volume . Spec ,
pod * api . Pod ,
_ volume . VolumeOptions ) ( volume . Mounter , error ) {
2019-04-07 17:07:55 +00:00
2019-08-30 18:33:25 +00:00
volSrc , pvSrc , err := getSourceFromSpec ( spec )
2019-01-12 04:58:27 +00:00
if err != nil {
return nil , err
}
2019-04-07 17:07:55 +00:00
var (
driverName string
volumeHandle string
readOnly bool
)
switch {
2019-08-30 18:33:25 +00:00
case volSrc != nil && utilfeature . DefaultFeatureGate . Enabled ( features . CSIInlineVolume ) :
volumeHandle = makeVolumeHandle ( string ( pod . UID ) , spec . Name ( ) )
driverName = volSrc . Driver
if volSrc . ReadOnly != nil {
readOnly = * volSrc . ReadOnly
}
2019-04-07 17:07:55 +00:00
case pvSrc != nil :
driverName = pvSrc . Driver
volumeHandle = pvSrc . VolumeHandle
readOnly = spec . ReadOnly
default :
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "volume source not found in volume.Spec" ) )
2019-04-07 17:07:55 +00:00
}
2019-09-27 21:51:53 +00:00
volumeLifecycleMode , err := p . getVolumeLifecycleMode ( spec )
2019-01-12 04:58:27 +00:00
if err != nil {
return nil , err
}
2019-09-27 21:51:53 +00:00
// Check CSIDriver.Spec.Mode to ensure that the CSI driver
// supports the current volumeLifecycleMode.
if err := p . supportsVolumeLifecycleMode ( driverName , volumeLifecycleMode ) ; err != nil {
return nil , err
}
2019-01-12 04:58:27 +00:00
k8s := p . host . GetKubeClient ( )
if k8s == nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "failed to get a kubernetes client" ) )
}
kvh , ok := p . host . ( volume . KubeletVolumeHost )
if ! ok {
return nil , errors . New ( log ( "cast from VolumeHost to KubeletVolumeHost failed" ) )
2019-01-12 04:58:27 +00:00
}
mounter := & csiMountMgr {
2019-09-27 21:51:53 +00:00
plugin : p ,
k8s : k8s ,
spec : spec ,
pod : pod ,
podUID : pod . UID ,
driverName : csiDriverName ( driverName ) ,
volumeLifecycleMode : volumeLifecycleMode ,
volumeID : volumeHandle ,
specVolumeID : spec . Name ( ) ,
readOnly : readOnly ,
kubeVolHost : kvh ,
2019-01-12 04:58:27 +00:00
}
2019-04-07 17:07:55 +00:00
mounter . csiClientGetter . driverName = csiDriverName ( driverName )
2019-01-12 04:58:27 +00:00
// Save volume info in pod dir
dir := mounter . GetPath ( )
2020-03-26 21:07:15 +00:00
dataDir := filepath . Dir ( dir ) // dropoff /mount at end
2019-01-12 04:58:27 +00:00
if err := os . MkdirAll ( dataDir , 0750 ) ; err != nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "failed to create dir %#v: %v" , dataDir , err ) )
2019-01-12 04:58:27 +00:00
}
klog . V ( 4 ) . Info ( log ( "created path successfully [%s]" , dataDir ) )
2019-08-30 18:33:25 +00:00
mounter . MetricsProvider = NewMetricsCsi ( volumeHandle , dir , csiDriverName ( driverName ) )
2019-01-12 04:58:27 +00:00
// persist volume info data for teardown
node := string ( p . host . GetNodeName ( ) )
volData := map [ string ] string {
2019-09-27 21:51:53 +00:00
volDataKey . specVolID : spec . Name ( ) ,
volDataKey . volHandle : volumeHandle ,
volDataKey . driverName : driverName ,
volDataKey . nodeName : node ,
volDataKey . volumeLifecycleMode : string ( volumeLifecycleMode ) ,
2019-01-12 04:58:27 +00:00
}
2019-04-07 17:07:55 +00:00
attachID := getAttachmentName ( volumeHandle , driverName , node )
volData [ volDataKey . attachmentID ] = attachID
2019-01-12 04:58:27 +00:00
if err := saveVolumeData ( dataDir , volDataFileName , volData ) ; err != nil {
2019-09-27 21:51:53 +00:00
if removeErr := os . RemoveAll ( dataDir ) ; removeErr != nil {
klog . Error ( log ( "failed to remove dir after error [%s]: %v" , dataDir , removeErr ) )
2019-01-12 04:58:27 +00:00
}
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "failed to save volume info data: %v" , err ) )
2019-01-12 04:58:27 +00:00
}
klog . V ( 4 ) . Info ( log ( "mounter created successfully" ) )
return mounter , nil
}
func ( p * csiPlugin ) NewUnmounter ( specName string , podUID types . UID ) ( volume . Unmounter , error ) {
klog . V ( 4 ) . Infof ( log ( "setting up unmounter for [name=%v, podUID=%v]" , specName , podUID ) )
2019-09-27 21:51:53 +00:00
kvh , ok := p . host . ( volume . KubeletVolumeHost )
if ! ok {
return nil , errors . New ( log ( "cast from VolumeHost to KubeletVolumeHost failed" ) )
}
2019-01-12 04:58:27 +00:00
unmounter := & csiMountMgr {
plugin : p ,
podUID : podUID ,
specVolumeID : specName ,
2019-09-27 21:51:53 +00:00
kubeVolHost : kvh ,
2019-01-12 04:58:27 +00:00
}
// load volume info from file
dir := unmounter . GetPath ( )
2020-03-26 21:07:15 +00:00
dataDir := filepath . Dir ( dir ) // dropoff /mount at end
2019-01-12 04:58:27 +00:00
data , err := loadVolumeData ( dataDir , volDataFileName )
if err != nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "unmounter failed to load volume data file [%s]: %v" , dir , err ) )
2019-01-12 04:58:27 +00:00
}
unmounter . driverName = csiDriverName ( data [ volDataKey . driverName ] )
unmounter . volumeID = data [ volDataKey . volHandle ]
2019-03-29 00:03:05 +00:00
unmounter . csiClientGetter . driverName = unmounter . driverName
2019-01-12 04:58:27 +00:00
return unmounter , nil
}
func ( p * csiPlugin ) ConstructVolumeSpec ( volumeName , mountPath string ) ( * volume . Spec , error ) {
klog . V ( 4 ) . Info ( log ( "plugin.ConstructVolumeSpec [pv.Name=%v, path=%v]" , volumeName , mountPath ) )
volData , err := loadVolumeData ( mountPath , volDataFileName )
if err != nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "plugin.ConstructVolumeSpec failed loading volume data using [%s]: %v" , mountPath , err ) )
2019-01-12 04:58:27 +00:00
}
klog . V ( 4 ) . Info ( log ( "plugin.ConstructVolumeSpec extracted [%#v]" , volData ) )
2019-04-07 17:07:55 +00:00
var spec * volume . Spec
2019-08-30 18:33:25 +00:00
inlineEnabled := utilfeature . DefaultFeatureGate . Enabled ( features . CSIInlineVolume )
2019-09-27 21:51:53 +00:00
// If inlineEnabled is true and mode is VolumeLifecycleEphemeral,
// use constructVolSourceSpec to construct volume source spec.
// If inlineEnabled is false or mode is VolumeLifecyclePersistent,
// use constructPVSourceSpec to construct volume construct pv source spec.
if inlineEnabled && storage . VolumeLifecycleMode ( volData [ volDataKey . volumeLifecycleMode ] ) == storage . VolumeLifecycleEphemeral {
spec = p . constructVolSourceSpec ( volData [ volDataKey . specVolID ] , volData [ volDataKey . driverName ] )
return spec , nil
2019-08-30 18:33:25 +00:00
}
2019-09-27 21:51:53 +00:00
spec = p . constructPVSourceSpec ( volData [ volDataKey . specVolID ] , volData [ volDataKey . driverName ] , volData [ volDataKey . volHandle ] )
2019-04-07 17:07:55 +00:00
return spec , nil
}
// constructVolSourceSpec constructs volume.Spec with CSIVolumeSource
func ( p * csiPlugin ) constructVolSourceSpec ( volSpecName , driverName string ) * volume . Spec {
vol := & api . Volume {
Name : volSpecName ,
VolumeSource : api . VolumeSource {
CSI : & api . CSIVolumeSource {
Driver : driverName ,
} ,
} ,
}
return volume . NewSpecFromVolume ( vol )
}
//constructPVSourceSpec constructs volume.Spec with CSIPersistentVolumeSource
func ( p * csiPlugin ) constructPVSourceSpec ( volSpecName , driverName , volumeHandle string ) * volume . Spec {
2019-01-12 04:58:27 +00:00
fsMode := api . PersistentVolumeFilesystem
pv := & api . PersistentVolume {
ObjectMeta : meta . ObjectMeta {
2019-04-07 17:07:55 +00:00
Name : volSpecName ,
2019-01-12 04:58:27 +00:00
} ,
Spec : api . PersistentVolumeSpec {
PersistentVolumeSource : api . PersistentVolumeSource {
CSI : & api . CSIPersistentVolumeSource {
2019-04-07 17:07:55 +00:00
Driver : driverName ,
VolumeHandle : volumeHandle ,
2019-01-12 04:58:27 +00:00
} ,
} ,
VolumeMode : & fsMode ,
} ,
}
2019-04-07 17:07:55 +00:00
return volume . NewSpecFromPersistentVolume ( pv , false )
2019-01-12 04:58:27 +00:00
}
func ( p * csiPlugin ) SupportsMountOption ( ) bool {
// TODO (vladimirvivien) use CSI VolumeCapability.MountVolume.mount_flags
// to probe for the result for this method
// (bswartz) Until the CSI spec supports probing, our only option is to
// make plugins register their support for mount options or lack thereof
// directly with kubernetes.
return true
}
func ( p * csiPlugin ) SupportsBulkVolumeVerification ( ) bool {
return false
}
// volume.AttachableVolumePlugin methods
var _ volume . AttachableVolumePlugin = & csiPlugin { }
var _ volume . DeviceMountableVolumePlugin = & csiPlugin { }
func ( p * csiPlugin ) NewAttacher ( ) ( volume . Attacher , error ) {
2019-08-30 18:33:25 +00:00
return p . newAttacherDetacher ( )
2019-01-12 04:58:27 +00:00
}
func ( p * csiPlugin ) NewDeviceMounter ( ) ( volume . DeviceMounter , error ) {
return p . NewAttacher ( )
}
func ( p * csiPlugin ) NewDetacher ( ) ( volume . Detacher , error ) {
2019-08-30 18:33:25 +00:00
return p . newAttacherDetacher ( )
2019-01-12 04:58:27 +00:00
}
2019-08-30 18:33:25 +00:00
func ( p * csiPlugin ) CanAttach ( spec * volume . Spec ) ( bool , error ) {
2019-09-27 21:51:53 +00:00
inlineEnabled := utilfeature . DefaultFeatureGate . Enabled ( features . CSIInlineVolume )
if inlineEnabled {
volumeLifecycleMode , err := p . getVolumeLifecycleMode ( spec )
if err != nil {
return false , err
}
2019-04-07 17:07:55 +00:00
2019-09-27 21:51:53 +00:00
if volumeLifecycleMode == storage . VolumeLifecycleEphemeral {
klog . V ( 5 ) . Info ( log ( "plugin.CanAttach = false, ephemeral mode detected for spec %v" , spec . Name ( ) ) )
return false , nil
}
2019-04-07 17:07:55 +00:00
}
pvSrc , err := getCSISourceFromSpec ( spec )
if err != nil {
2019-08-30 18:33:25 +00:00
return false , err
2019-04-07 17:07:55 +00:00
}
driverName := pvSrc . Driver
skipAttach , err := p . skipAttach ( driverName )
if err != nil {
2019-08-30 18:33:25 +00:00
return false , err
2019-04-07 17:07:55 +00:00
}
2019-08-30 18:33:25 +00:00
return ! skipAttach , nil
}
// CanDeviceMount returns true if the spec supports device mount
func ( p * csiPlugin ) CanDeviceMount ( spec * volume . Spec ) ( bool , error ) {
2019-09-27 21:51:53 +00:00
inlineEnabled := utilfeature . DefaultFeatureGate . Enabled ( features . CSIInlineVolume )
if ! inlineEnabled {
// No need to check anything, we assume it is a persistent volume.
return true , nil
}
volumeLifecycleMode , err := p . getVolumeLifecycleMode ( spec )
2019-08-30 18:33:25 +00:00
if err != nil {
return false , err
}
2019-09-27 21:51:53 +00:00
if volumeLifecycleMode == storage . VolumeLifecycleEphemeral {
2019-08-30 18:33:25 +00:00
klog . V ( 5 ) . Info ( log ( "plugin.CanDeviceMount skipped ephemeral mode detected for spec %v" , spec . Name ( ) ) )
return false , nil
}
2019-09-27 21:51:53 +00:00
// Persistent volumes support device mount.
2019-08-30 18:33:25 +00:00
return true , nil
2019-04-07 17:07:55 +00:00
}
2019-01-12 04:58:27 +00:00
func ( p * csiPlugin ) NewDeviceUnmounter ( ) ( volume . DeviceUnmounter , error ) {
return p . NewDetacher ( )
}
func ( p * csiPlugin ) GetDeviceMountRefs ( deviceMountPath string ) ( [ ] string , error ) {
m := p . host . GetMounter ( p . GetPluginName ( ) )
return m . GetMountRefs ( deviceMountPath )
}
// BlockVolumePlugin methods
var _ volume . BlockVolumePlugin = & csiPlugin { }
func ( p * csiPlugin ) NewBlockVolumeMapper ( spec * volume . Spec , podRef * api . Pod , opts volume . VolumeOptions ) ( volume . BlockVolumeMapper , error ) {
2019-04-07 17:07:55 +00:00
if ! p . blockEnabled {
return nil , errors . New ( "CSIBlockVolume feature not enabled" )
}
pvSource , err := getCSISourceFromSpec ( spec )
if err != nil {
return nil , err
}
readOnly , err := getReadOnlyFromSpec ( spec )
if err != nil {
return nil , err
}
klog . V ( 4 ) . Info ( log ( "setting up block mapper for [volume=%v,driver=%v]" , pvSource . VolumeHandle , pvSource . Driver ) )
k8s := p . host . GetKubeClient ( )
if k8s == nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "failed to get a kubernetes client" ) )
2019-04-07 17:07:55 +00:00
}
mapper := & csiBlockMapper {
k8s : k8s ,
plugin : p ,
volumeID : pvSource . VolumeHandle ,
driverName : csiDriverName ( pvSource . Driver ) ,
readOnly : readOnly ,
spec : spec ,
specName : spec . Name ( ) ,
podUID : podRef . UID ,
}
mapper . csiClientGetter . driverName = csiDriverName ( pvSource . Driver )
// Save volume info in pod dir
dataDir := getVolumeDeviceDataDir ( spec . Name ( ) , p . host )
if err := os . MkdirAll ( dataDir , 0750 ) ; err != nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "failed to create data dir %s: %v" , dataDir , err ) )
2019-04-07 17:07:55 +00:00
}
klog . V ( 4 ) . Info ( log ( "created path successfully [%s]" , dataDir ) )
// persist volume info data for teardown
node := string ( p . host . GetNodeName ( ) )
attachID := getAttachmentName ( pvSource . VolumeHandle , pvSource . Driver , node )
volData := map [ string ] string {
volDataKey . specVolID : spec . Name ( ) ,
volDataKey . volHandle : pvSource . VolumeHandle ,
volDataKey . driverName : pvSource . Driver ,
volDataKey . nodeName : node ,
volDataKey . attachmentID : attachID ,
}
if err := saveVolumeData ( dataDir , volDataFileName , volData ) ; err != nil {
2019-09-27 21:51:53 +00:00
if removeErr := os . RemoveAll ( dataDir ) ; removeErr != nil {
klog . Error ( log ( "failed to remove dir after error [%s]: %v" , dataDir , removeErr ) )
2019-04-07 17:07:55 +00:00
}
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "failed to save volume info data: %v" , err ) )
2019-04-07 17:07:55 +00:00
}
return mapper , nil
2019-01-12 04:58:27 +00:00
}
func ( p * csiPlugin ) NewBlockVolumeUnmapper ( volName string , podUID types . UID ) ( volume . BlockVolumeUnmapper , error ) {
2019-04-07 17:07:55 +00:00
if ! p . blockEnabled {
return nil , errors . New ( "CSIBlockVolume feature not enabled" )
}
klog . V ( 4 ) . Infof ( log ( "setting up block unmapper for [Spec=%v, podUID=%v]" , volName , podUID ) )
unmapper := & csiBlockMapper {
plugin : p ,
podUID : podUID ,
specName : volName ,
}
// load volume info from file
dataDir := getVolumeDeviceDataDir ( unmapper . specName , p . host )
data , err := loadVolumeData ( dataDir , volDataFileName )
if err != nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "unmapper failed to load volume data file [%s]: %v" , dataDir , err ) )
2019-04-07 17:07:55 +00:00
}
unmapper . driverName = csiDriverName ( data [ volDataKey . driverName ] )
unmapper . volumeID = data [ volDataKey . volHandle ]
unmapper . csiClientGetter . driverName = unmapper . driverName
return unmapper , nil
2019-01-12 04:58:27 +00:00
}
func ( p * csiPlugin ) ConstructBlockVolumeSpec ( podUID types . UID , specVolName , mapPath string ) ( * volume . Spec , error ) {
2019-04-07 17:07:55 +00:00
if ! p . blockEnabled {
return nil , errors . New ( "CSIBlockVolume feature not enabled" )
}
klog . V ( 4 ) . Infof ( "plugin.ConstructBlockVolumeSpec [podUID=%s, specVolName=%s, path=%s]" , string ( podUID ) , specVolName , mapPath )
dataDir := getVolumeDeviceDataDir ( specVolName , p . host )
volData , err := loadVolumeData ( dataDir , volDataFileName )
if err != nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "plugin.ConstructBlockVolumeSpec failed loading volume data using [%s]: %v" , mapPath , err ) )
2019-04-07 17:07:55 +00:00
}
klog . V ( 4 ) . Info ( log ( "plugin.ConstructBlockVolumeSpec extracted [%#v]" , volData ) )
blockMode := api . PersistentVolumeBlock
pv := & api . PersistentVolume {
ObjectMeta : meta . ObjectMeta {
Name : volData [ volDataKey . specVolID ] ,
} ,
Spec : api . PersistentVolumeSpec {
PersistentVolumeSource : api . PersistentVolumeSource {
CSI : & api . CSIPersistentVolumeSource {
Driver : volData [ volDataKey . driverName ] ,
VolumeHandle : volData [ volDataKey . volHandle ] ,
} ,
} ,
VolumeMode : & blockMode ,
} ,
}
return volume . NewSpecFromPersistentVolume ( pv , false ) , nil
2019-01-12 04:58:27 +00:00
}
2019-04-07 17:07:55 +00:00
// skipAttach looks up CSIDriver object associated with driver name
2019-08-30 18:33:25 +00:00
// to determine if driver requires attachment volume operation
2019-01-12 04:58:27 +00:00
func ( p * csiPlugin ) skipAttach ( driver string ) ( bool , error ) {
2019-07-14 07:58:54 +00:00
kletHost , ok := p . host . ( volume . KubeletVolumeHost )
if ok {
2019-12-12 01:27:03 +00:00
if err := kletHost . WaitForCacheSync ( ) ; err != nil {
return false , err
}
2019-07-14 07:58:54 +00:00
}
2019-04-07 17:07:55 +00:00
if p . csiDriverLister == nil {
return false , errors . New ( "CSIDriver lister does not exist" )
}
csiDriver , err := p . csiDriverLister . Get ( driver )
if err != nil {
2020-03-26 21:07:15 +00:00
if apierrors . IsNotFound ( err ) {
2019-04-07 17:07:55 +00:00
// Don't skip attach if CSIDriver does not exist
return false , nil
}
return false , err
}
if csiDriver . Spec . AttachRequired != nil && * csiDriver . Spec . AttachRequired == false {
return true , nil
}
2019-01-12 04:58:27 +00:00
return false , nil
}
2019-09-27 21:51:53 +00:00
// supportsVolumeMode checks whether the CSI driver supports a volume in the given mode.
// An error indicates that it isn't supported and explains why.
func ( p * csiPlugin ) supportsVolumeLifecycleMode ( driver string , volumeMode storage . VolumeLifecycleMode ) error {
if ! utilfeature . DefaultFeatureGate . Enabled ( features . CSIInlineVolume ) {
// Feature disabled, therefore only "persistent" volumes are supported.
if volumeMode != storage . VolumeLifecyclePersistent {
return fmt . Errorf ( "CSIInlineVolume feature not enabled, %q volumes not supported" , volumeMode )
}
return nil
}
// Retrieve CSIDriver. It's not an error if that isn't
// possible (we don't have the lister if CSIDriverRegistry is
// disabled) or the driver isn't found (CSIDriver is
// optional), but then only persistent volumes are supported.
var csiDriver * storage . CSIDriver
if p . csiDriverLister != nil {
kletHost , ok := p . host . ( volume . KubeletVolumeHost )
if ok {
2019-12-12 01:27:03 +00:00
if err := kletHost . WaitForCacheSync ( ) ; err != nil {
return err
}
2019-09-27 21:51:53 +00:00
}
c , err := p . csiDriverLister . Get ( driver )
2020-03-26 21:07:15 +00:00
if err != nil && ! apierrors . IsNotFound ( err ) {
2019-09-27 21:51:53 +00:00
// Some internal error.
return err
}
csiDriver = c
}
// The right response depends on whether we have information
// about the driver and the volume mode.
switch {
case csiDriver == nil && volumeMode == storage . VolumeLifecyclePersistent :
// No information, but that's okay for persistent volumes (and only those).
return nil
case csiDriver == nil :
return fmt . Errorf ( "volume mode %q not supported by driver %s (no CSIDriver object)" , volumeMode , driver )
case containsVolumeMode ( csiDriver . Spec . VolumeLifecycleModes , volumeMode ) :
// Explicitly listed.
return nil
default :
return fmt . Errorf ( "volume mode %q not supported by driver %s (only supports %q)" , volumeMode , driver , csiDriver . Spec . VolumeLifecycleModes )
}
}
// containsVolumeMode checks whether the given volume mode is listed.
func containsVolumeMode ( modes [ ] storage . VolumeLifecycleMode , mode storage . VolumeLifecycleMode ) bool {
for _ , m := range modes {
if m == mode {
return true
}
}
return false
}
// getVolumeLifecycleMode returns the mode for the specified spec: {persistent|ephemeral}.
2019-04-07 17:07:55 +00:00
// 1) If mode cannot be determined, it will default to "persistent".
// 2) If Mode cannot be resolved to either {persistent | ephemeral}, an error is returned
// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/20190122-csi-inline-volumes.md
2019-09-27 21:51:53 +00:00
func ( p * csiPlugin ) getVolumeLifecycleMode ( spec * volume . Spec ) ( storage . VolumeLifecycleMode , error ) {
2019-08-30 18:33:25 +00:00
// 1) if volume.Spec.Volume.CSI != nil -> mode is ephemeral
// 2) if volume.Spec.PersistentVolume.Spec.CSI != nil -> persistent
volSrc , _ , err := getSourceFromSpec ( spec )
if err != nil {
return "" , err
}
if volSrc != nil && utilfeature . DefaultFeatureGate . Enabled ( features . CSIInlineVolume ) {
2019-09-27 21:51:53 +00:00
return storage . VolumeLifecycleEphemeral , nil
2019-08-30 18:33:25 +00:00
}
2019-09-27 21:51:53 +00:00
return storage . VolumeLifecyclePersistent , nil
2019-04-07 17:07:55 +00:00
}
2019-01-12 04:58:27 +00:00
func ( p * csiPlugin ) getPublishContext ( client clientset . Interface , handle , driver , nodeName string ) ( map [ string ] string , error ) {
skip , err := p . skipAttach ( driver )
if err != nil {
return nil , err
}
if skip {
return nil , nil
}
attachID := getAttachmentName ( handle , driver , nodeName )
// search for attachment by VolumeAttachment.Spec.Source.PersistentVolumeName
2020-03-26 21:07:15 +00:00
attachment , err := client . StorageV1 ( ) . VolumeAttachments ( ) . Get ( context . TODO ( ) , attachID , meta . GetOptions { } )
2019-01-12 04:58:27 +00:00
if err != nil {
return nil , err // This err already has enough context ("VolumeAttachment xyz not found")
}
if attachment == nil {
err = errors . New ( "no existing VolumeAttachment found" )
return nil , err
}
return attachment . Status . AttachmentMetadata , nil
}
2019-08-30 18:33:25 +00:00
func ( p * csiPlugin ) newAttacherDetacher ( ) ( * csiAttacher , error ) {
k8s := p . host . GetKubeClient ( )
if k8s == nil {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "unable to get kubernetes client from host" ) )
2019-08-30 18:33:25 +00:00
}
return & csiAttacher {
plugin : p ,
k8s : k8s ,
waitSleepTime : 1 * time . Second ,
} , nil
}
2019-01-12 04:58:27 +00:00
func unregisterDriver ( driverName string ) error {
2019-04-07 17:07:55 +00:00
csiDrivers . Delete ( driverName )
2019-01-12 04:58:27 +00:00
if err := nim . UninstallCSIDriver ( driverName ) ; err != nil {
2019-09-27 21:51:53 +00:00
return errors . New ( log ( "Error uninstalling CSI driver: %v" , err ) )
2019-01-12 04:58:27 +00:00
}
return nil
}
// Return the highest supported version
func highestSupportedVersion ( versions [ ] string ) ( * utilversion . Version , error ) {
if len ( versions ) == 0 {
2019-09-27 21:51:53 +00:00
return nil , errors . New ( log ( "CSI driver reporting empty array for supported versions" ) )
2019-01-12 04:58:27 +00:00
}
2019-12-12 01:27:03 +00:00
var highestSupportedVersion * utilversion . Version
var theErr error
2019-01-12 04:58:27 +00:00
for i := len ( versions ) - 1 ; i >= 0 ; i -- {
2019-12-12 01:27:03 +00:00
currentHighestVer , err := utilversion . ParseGeneric ( versions [ i ] )
2019-01-12 04:58:27 +00:00
if err != nil {
2019-12-12 01:27:03 +00:00
theErr = err
continue
2019-01-12 04:58:27 +00:00
}
2019-12-12 01:27:03 +00:00
if currentHighestVer . Major ( ) > 1 {
// CSI currently only has version 0.x and 1.x (see https://github.com/container-storage-interface/spec/releases).
// Therefore any driver claiming version 2.x+ is ignored as an unsupported versions.
// Future 1.x versions of CSI are supposed to be backwards compatible so this version of Kubernetes will work with any 1.x driver
// (or 0.x), but it may not work with 2.x drivers (because 2.x does not have to be backwards compatible with 1.x).
continue
}
if highestSupportedVersion == nil || highestSupportedVersion . LessThan ( currentHighestVer ) {
highestSupportedVersion = currentHighestVer
2019-01-12 04:58:27 +00:00
}
}
2019-12-12 01:27:03 +00:00
if highestSupportedVersion == nil {
return nil , fmt . Errorf ( "could not find a highest supported version from versions (%v) reported by this driver: %v" , versions , theErr )
2019-01-12 04:58:27 +00:00
}
2019-12-12 01:27:03 +00:00
if highestSupportedVersion . Major ( ) != 1 {
// CSI v0.x is no longer supported as of Kubernetes v1.17 in
// accordance with deprecation policy set out in Kubernetes v1.13
return nil , fmt . Errorf ( "highest supported version reported by driver is %v, must be v1.x" , highestSupportedVersion )
}
return highestSupportedVersion , nil
2019-01-12 04:58:27 +00:00
}
2020-05-26 22:59:35 +00:00
// waitForAPIServerForever waits forever to get a CSINode instance as a proxy
// for a healthy APIServer
func waitForAPIServerForever ( client clientset . Interface , nodeName types . NodeName ) error {
var lastErr error
err := wait . PollImmediateInfinite ( time . Second , func ( ) ( bool , error ) {
// Get a CSINode from API server to make sure 1) kubelet can reach API server
// and 2) it has enough permissions. Kubelet may have restricted permissions
// when it's bootstrapping TLS.
// https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/
_ , lastErr = client . StorageV1 ( ) . CSINodes ( ) . Get ( context . TODO ( ) , string ( nodeName ) , meta . GetOptions { } )
if lastErr == nil || apierrors . IsNotFound ( lastErr ) {
// API server contacted
return true , nil
}
klog . V ( 2 ) . Infof ( "Failed to contact API server when waiting for CSINode publishing: %s" , lastErr )
return false , nil
} )
if err != nil {
// In theory this is unreachable, but just in case:
return fmt . Errorf ( "%v: %v" , err , lastErr )
}
return nil
}