Block volumes Support: iSCSI plugin update

This patch adds block volume support to iSCSI volume plugin.
pull/6/head
Mitsuhiro Tanino 2017-10-28 15:28:52 -04:00 committed by mtanino
parent df2428c6dd
commit a6d979dd88
5 changed files with 658 additions and 201 deletions

View File

@ -19,16 +19,17 @@ package iscsi
import (
"fmt"
"os"
"strconv"
"time"
"github.com/golang/glog"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/volumehelper"
)
type iscsiAttacher struct {
@ -66,7 +67,7 @@ func (attacher *iscsiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName
}
func (attacher *iscsiAttacher) WaitForAttach(spec *volume.Spec, devicePath string, pod *v1.Pod, timeout time.Duration) (string, error) {
mounter, err := attacher.volumeSpecToMounter(spec, attacher.host, pod)
mounter, err := volumeSpecToMounter(spec, attacher.host, pod)
if err != nil {
glog.Warningf("failed to get iscsi mounter: %v", err)
return "", err
@ -76,7 +77,7 @@ func (attacher *iscsiAttacher) WaitForAttach(spec *volume.Spec, devicePath strin
func (attacher *iscsiAttacher) GetDeviceMountPath(
spec *volume.Spec) (string, error) {
mounter, err := attacher.volumeSpecToMounter(spec, attacher.host, nil)
mounter, err := volumeSpecToMounter(spec, attacher.host, nil)
if err != nil {
glog.Warningf("failed to get iscsi mounter: %v", err)
return "", err
@ -143,7 +144,7 @@ func (detacher *iscsiDetacher) Detach(volumeName string, nodeName types.NodeName
}
func (detacher *iscsiDetacher) UnmountDevice(deviceMountPath string) error {
unMounter := detacher.volumeSpecToUnmounter(detacher.mounter)
unMounter := volumeSpecToUnmounter(detacher.mounter, detacher.host)
err := detacher.manager.DetachDisk(*unMounter, deviceMountPath)
if err != nil {
return fmt.Errorf("iscsi: failed to detach disk: %s\nError: %v", deviceMountPath, err)
@ -157,94 +158,49 @@ func (detacher *iscsiDetacher) UnmountDevice(deviceMountPath string) error {
return nil
}
func (attacher *iscsiAttacher) volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, pod *v1.Pod) (*iscsiDiskMounter, error) {
func volumeSpecToMounter(spec *volume.Spec, host volume.VolumeHost, pod *v1.Pod) (*iscsiDiskMounter, error) {
var secret map[string]string
var bkportal []string
readOnly, fsType, err := getISCSIVolumeInfo(spec)
if err != nil {
return nil, err
}
var podUID types.UID
if pod != nil {
chapDiscovery, err := getISCSIDiscoveryCHAPInfo(spec)
secret, err = createSecretMap(spec, &iscsiPlugin{host: host}, pod.Namespace)
if err != nil {
return nil, err
}
chapSession, err := getISCSISessionCHAPInfo(spec)
podUID = pod.UID
}
iscsiDisk, err := createISCSIDisk(spec,
podUID,
&iscsiPlugin{host: host},
&ISCSIUtil{},
secret,
)
if err != nil {
return nil, err
}
exec := host.GetExec(iscsiPluginName)
// TODO: remove feature gate check after no longer needed
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
volumeMode, err := volumehelper.GetVolumeMode(spec)
if err != nil {
return nil, err
}
if chapDiscovery || chapSession {
secretName, secretNamespace, err := getISCSISecretNameAndNamespace(spec, pod.Namespace)
if err != nil {
return nil, err
}
if len(secretNamespace) == 0 || len(secretName) == 0 {
return nil, fmt.Errorf("CHAP enabled but secret name or namespace is empty")
}
// if secret is provided, retrieve it
kubeClient := host.GetKubeClient()
if kubeClient == nil {
return nil, fmt.Errorf("Cannot get kube client")
}
secretObj, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(secretName, metav1.GetOptions{})
if err != nil {
err = fmt.Errorf("Couldn't get secret %v/%v error: %v", secretNamespace, secretName, err)
return nil, err
}
secret = make(map[string]string)
for name, data := range secretObj.Data {
glog.V(6).Infof("retrieving CHAP secret name: %s", name)
secret[name] = string(data)
}
}
glog.V(5).Infof("iscsi: VolumeSpecToMounter volumeMode %s", volumeMode)
return &iscsiDiskMounter{
iscsiDisk: iscsiDisk,
fsType: fsType,
volumeMode: volumeMode,
readOnly: readOnly,
mounter: &mount.SafeFormatAndMount{Interface: host.GetMounter(iscsiPluginName), Exec: exec},
exec: exec,
deviceUtil: volumeutil.NewDeviceHandler(volumeutil.NewIOHandler()),
}, nil
}
tp, portals, iqn, lunStr, err := getISCSITargetInfo(spec)
if err != nil {
return nil, err
}
lun := strconv.Itoa(int(lunStr))
portal := portalMounter(tp)
bkportal = append(bkportal, portal)
for _, p := range portals {
bkportal = append(bkportal, portalMounter(string(p)))
}
iface, initiatorNamePtr, err := getISCSIInitiatorInfo(spec)
if err != nil {
return nil, err
}
var initiatorName string
if initiatorNamePtr != nil {
initiatorName = *initiatorNamePtr
}
chapDiscovery, err := getISCSIDiscoveryCHAPInfo(spec)
if err != nil {
return nil, err
}
chapSession, err := getISCSISessionCHAPInfo(spec)
if err != nil {
return nil, err
}
exec := attacher.host.GetExec(iscsiPluginName)
return &iscsiDiskMounter{
iscsiDisk: &iscsiDisk{
plugin: &iscsiPlugin{
host: host,
},
VolName: spec.Name(),
Portals: bkportal,
Iqn: iqn,
lun: lun,
Iface: iface,
chap_discovery: chapDiscovery,
chap_session: chapSession,
secret: secret,
InitiatorName: initiatorName,
manager: &ISCSIUtil{}},
iscsiDisk: iscsiDisk,
fsType: fsType,
readOnly: readOnly,
mounter: &mount.SafeFormatAndMount{Interface: host.GetMounter(iscsiPluginName), Exec: exec},
@ -253,8 +209,8 @@ func (attacher *iscsiAttacher) volumeSpecToMounter(spec *volume.Spec, host volum
}, nil
}
func (detacher *iscsiDetacher) volumeSpecToUnmounter(mounter mount.Interface) *iscsiDiskUnmounter {
exec := detacher.host.GetExec(iscsiPluginName)
func volumeSpecToUnmounter(mounter mount.Interface, host volume.VolumeHost) *iscsiDiskUnmounter {
exec := host.GetExec(iscsiPluginName)
return &iscsiDiskUnmounter{
iscsiDisk: &iscsiDisk{
plugin: &iscsiPlugin{},

View File

@ -27,15 +27,19 @@ import (
// Abstract interface to disk operations.
type diskManager interface {
MakeGlobalPDName(disk iscsiDisk) string
MakeGlobalVDPDName(disk iscsiDisk) string
// Attaches the disk to the kubelet's host machine.
AttachDisk(b iscsiDiskMounter) (string, error)
// Detaches the disk from the kubelet's host machine.
DetachDisk(disk iscsiDiskUnmounter, mntPath string) error
// Detaches the block disk from the kubelet's host machine.
DetachBlockISCSIDisk(disk iscsiDiskUnmapper, mntPath string) error
}
// utility to mount a disk based filesystem
// globalPDPath: global mount path like, /var/lib/kubelet/plugins/kubernetes.io/iscsi/{ifaceName}/{portal-some_iqn-lun-lun_id}
// volPath: pod volume dir path like, /var/lib/kubelet/pods/{podUID}/volumes/kubernetes.io~iscsi/{volumeName}
func diskSetUp(manager diskManager, b iscsiDiskMounter, volPath string, mounter mount.Interface, fsGroup *int64) error {
// TODO: handle failed mounts here.
notMnt, err := mounter.IsLikelyNotMountPoint(volPath)
if err != nil && !os.IsNotExist(err) {
glog.Errorf("cannot validate mountpoint: %s", volPath)

View File

@ -18,6 +18,8 @@ package iscsi
import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
@ -42,6 +44,7 @@ type iscsiPlugin struct {
var _ volume.VolumePlugin = &iscsiPlugin{}
var _ volume.PersistentVolumePlugin = &iscsiPlugin{}
var _ volume.BlockVolumePlugin = &iscsiPlugin{}
const (
iscsiPluginName = "kubernetes.io/iscsi"
@ -93,98 +96,27 @@ func (plugin *iscsiPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
}
func (plugin *iscsiPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
// Inject real implementations here, test through the internal function.
var secret map[string]string
if pod == nil {
return nil, fmt.Errorf("nil pod")
}
chapDiscover, err := getISCSIDiscoveryCHAPInfo(spec)
secret, err := createSecretMap(spec, plugin, pod.Namespace)
if err != nil {
return nil, err
}
chapSession, err := getISCSISessionCHAPInfo(spec)
if err != nil {
return nil, err
}
if chapDiscover || chapSession {
secretName, secretNamespace, err := getISCSISecretNameAndNamespace(spec, pod.Namespace)
if err != nil {
return nil, err
}
if len(secretName) > 0 && len(secretNamespace) > 0 {
// if secret is provideded, retrieve it
kubeClient := plugin.host.GetKubeClient()
if kubeClient == nil {
return nil, fmt.Errorf("Cannot get kube client")
}
secretObj, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(secretName, metav1.GetOptions{})
if err != nil {
err = fmt.Errorf("Couldn't get secret %v/%v error: %v", secretNamespace, secretName, err)
return nil, err
}
secret = make(map[string]string)
for name, data := range secretObj.Data {
glog.V(4).Infof("retrieving CHAP secret name: %s", name)
secret[name] = string(data)
}
}
}
return plugin.newMounterInternal(spec, pod.UID, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret)
}
func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec, secret map[string]string) (volume.Mounter, error) {
// iscsi volumes used directly in a pod have a ReadOnly flag set by the pod author.
// iscsi volumes used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
readOnly, fsType, err := getISCSIVolumeInfo(spec)
if err != nil {
return nil, err
}
tp, portals, iqn, lunStr, err := getISCSITargetInfo(spec)
iscsiDisk, err := createISCSIDisk(spec, podUID, plugin, manager, secret)
if err != nil {
return nil, err
}
lun := strconv.Itoa(int(lunStr))
portal := portalMounter(tp)
var bkportal []string
bkportal = append(bkportal, portal)
for _, p := range portals {
bkportal = append(bkportal, portalMounter(string(p)))
}
iface, initiatorNamePtr, err := getISCSIInitiatorInfo(spec)
if err != nil {
return nil, err
}
var initiatorName string
if initiatorNamePtr != nil {
initiatorName = *initiatorNamePtr
}
chapDiscovery, err := getISCSIDiscoveryCHAPInfo(spec)
if err != nil {
return nil, err
}
chapSession, err := getISCSISessionCHAPInfo(spec)
if err != nil {
return nil, err
}
return &iscsiDiskMounter{
iscsiDisk: &iscsiDisk{
podUID: podUID,
VolName: spec.Name(),
Portals: bkportal,
Iqn: iqn,
lun: lun,
Iface: iface,
chap_discovery: chapDiscovery,
chap_session: chapSession,
secret: secret,
InitiatorName: initiatorName,
manager: manager,
plugin: plugin},
iscsiDisk: iscsiDisk,
fsType: fsType,
readOnly: readOnly,
mounter: &mount.SafeFormatAndMount{Interface: mounter, Exec: exec},
@ -194,8 +126,41 @@ func (plugin *iscsiPlugin) newMounterInternal(spec *volume.Spec, podUID types.UI
}, nil
}
// NewBlockVolumeMapper creates a new volume.BlockVolumeMapper from an API specification.
func (plugin *iscsiPlugin) NewBlockVolumeMapper(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.BlockVolumeMapper, error) {
// If this is called via GenerateUnmapDeviceFunc(), pod is nil.
// Pass empty string as dummy uid since uid isn't used in the case.
var uid types.UID
var secret map[string]string
var err error
if pod != nil {
uid = pod.UID
secret, err = createSecretMap(spec, plugin, pod.Namespace)
if err != nil {
return nil, err
}
}
return plugin.newBlockVolumeMapperInternal(spec, uid, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()), secret)
}
func (plugin *iscsiPlugin) newBlockVolumeMapperInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, exec mount.Exec, secret map[string]string) (volume.BlockVolumeMapper, error) {
readOnly, _, err := getISCSIVolumeInfo(spec)
if err != nil {
return nil, err
}
iscsiDisk, err := createISCSIDisk(spec, podUID, plugin, manager, secret)
if err != nil {
return nil, err
}
return &iscsiDiskMapper{
iscsiDisk: iscsiDisk,
readOnly: readOnly,
exec: exec,
deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
}, nil
}
func (plugin *iscsiPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
// Inject real implementations here, test through the internal function.
return plugin.newUnmounterInternal(volName, podUID, &ISCSIUtil{}, plugin.host.GetMounter(plugin.GetPluginName()), plugin.host.GetExec(plugin.GetPluginName()))
}
@ -212,25 +177,88 @@ func (plugin *iscsiPlugin) newUnmounterInternal(volName string, podUID types.UID
}, nil
}
// NewBlockVolumeUnmapper creates a new volume.BlockVolumeUnmapper from recoverable state.
func (plugin *iscsiPlugin) NewBlockVolumeUnmapper(volName string, podUID types.UID) (volume.BlockVolumeUnmapper, error) {
return plugin.newUnmapperInternal(volName, podUID, &ISCSIUtil{}, plugin.host.GetExec(plugin.GetPluginName()))
}
func (plugin *iscsiPlugin) newUnmapperInternal(volName string, podUID types.UID, manager diskManager, exec mount.Exec) (volume.BlockVolumeUnmapper, error) {
return &iscsiDiskUnmapper{
iscsiDisk: &iscsiDisk{
podUID: podUID,
VolName: volName,
manager: manager,
plugin: plugin,
},
exec: exec,
deviceUtil: ioutil.NewDeviceHandler(ioutil.NewIOHandler()),
}, nil
}
func (plugin *iscsiPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
// Find globalPDPath from pod volume directory(mountPath)
var globalPDPath string
mounter := plugin.host.GetMounter(plugin.GetPluginName())
paths, err := mount.GetMountRefs(mounter, mountPath)
if err != nil {
return nil, err
}
for _, path := range paths {
if strings.Contains(path, plugin.host.GetPluginDir(iscsiPluginName)) {
globalPDPath = path
break
}
}
// Couldn't fetch globalPDPath
if len(globalPDPath) == 0 {
return nil, fmt.Errorf("couldn't fetch globalPDPath. failed to obtain volume spec")
}
// Obtain iscsi disk configurations from globalPDPath
device, _, err := extractDeviceAndPrefix(globalPDPath)
if err != nil {
return nil, err
}
bkpPortal, iqn, err := extractPortalAndIqn(device)
if err != nil {
return nil, err
}
iface, _ := extractIface(globalPDPath)
iscsiVolume := &v1.Volume{
Name: volumeName,
VolumeSource: v1.VolumeSource{
ISCSI: &v1.ISCSIVolumeSource{
TargetPortal: volumeName,
IQN: volumeName,
TargetPortal: bkpPortal,
IQN: iqn,
ISCSIInterface: iface,
},
},
}
return volume.NewSpecFromVolume(iscsiVolume), nil
}
func (plugin *iscsiPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) {
pluginDir := plugin.host.GetVolumeDevicePluginDir(iscsiPluginName)
blkutil := ioutil.NewBlockVolumePathHandler()
globalMapPathUUID, err := blkutil.FindGlobalMapPathUUIDFromPod(pluginDir, mapPath, podUID)
if err != nil {
return nil, err
}
glog.V(5).Infof("globalMapPathUUID: %v, err: %v", globalMapPathUUID, err)
// Retreive volume information from globalMapPathUUID
// globalMapPathUUID example:
// plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid}
// plugins/kubernetes.io/iscsi/volumeDevices/iface-default/192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0/{pod uuid}
globalMapPath := filepath.Dir(globalMapPathUUID)
return getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath)
}
type iscsiDisk struct {
VolName string
podUID types.UID
Portals []string
Iqn string
lun string
Lun string
Iface string
chap_discovery bool
chap_session bool
@ -248,10 +276,25 @@ func (iscsi *iscsiDisk) GetPath() string {
return iscsi.plugin.host.GetPodVolumeDir(iscsi.podUID, utilstrings.EscapeQualifiedNameForDisk(name), iscsi.VolName)
}
func (iscsi *iscsiDisk) iscsiGlobalMapPath(spec *volume.Spec) (string, error) {
mounter, err := volumeSpecToMounter(spec, iscsi.plugin.host, nil /* pod */)
if err != nil {
glog.Warningf("failed to get iscsi mounter: %v", err)
return "", err
}
return iscsi.manager.MakeGlobalVDPDName(*mounter.iscsiDisk), nil
}
func (iscsi *iscsiDisk) iscsiPodDeviceMapPath() (string, string) {
name := iscsiPluginName
return iscsi.plugin.host.GetPodVolumeDeviceDir(iscsi.podUID, utilstrings.EscapeQualifiedNameForDisk(name)), iscsi.VolName
}
type iscsiDiskMounter struct {
*iscsiDisk
readOnly bool
fsType string
volumeMode v1.PersistentVolumeMode
mounter *mount.SafeFormatAndMount
exec mount.Exec
deviceUtil ioutil.DeviceUtil
@ -306,6 +349,58 @@ func (c *iscsiDiskUnmounter) TearDownAt(dir string) error {
return ioutil.UnmountPath(dir, c.mounter)
}
// Block Volumes Support
type iscsiDiskMapper struct {
*iscsiDisk
readOnly bool
exec mount.Exec
deviceUtil ioutil.DeviceUtil
}
var _ volume.BlockVolumeMapper = &iscsiDiskMapper{}
func (b *iscsiDiskMapper) SetUpDevice() (string, error) {
return "", nil
}
type iscsiDiskUnmapper struct {
*iscsiDisk
exec mount.Exec
deviceUtil ioutil.DeviceUtil
}
var _ volume.BlockVolumeUnmapper = &iscsiDiskUnmapper{}
// Even though iSCSI plugin has attacher/detacher implementation, iSCSI plugin
// needs volume detach operation during TearDownDevice(). This method is only
// chance that operations are done on kubelet node during volume teardown sequences.
func (c *iscsiDiskUnmapper) TearDownDevice(mapPath, _ string) error {
err := c.manager.DetachBlockISCSIDisk(*c, mapPath)
if err != nil {
return fmt.Errorf("iscsi: failed to detach disk: %s\nError: %v", mapPath, err)
}
glog.V(4).Infof("iscsi: %q is unmounted, deleting the directory", mapPath)
err = os.RemoveAll(mapPath)
if err != nil {
return fmt.Errorf("iscsi: failed to delete the directory: %s\nError: %v", mapPath, err)
}
glog.V(4).Infof("iscsi: successfully detached disk: %s", mapPath)
return nil
}
// GetGlobalMapPath returns global map path and error
// path: plugins/kubernetes.io/{PluginName}/volumeDevices/{ifaceName}/{portal-some_iqn-lun-lun_id}
func (iscsi *iscsiDisk) GetGlobalMapPath(spec *volume.Spec) (string, error) {
return iscsi.iscsiGlobalMapPath(spec)
}
// GetPodDeviceMapPath returns pod device map path and volume name
// path: pods/{podUid}/volumeDevices/kubernetes.io~iscsi
// volumeName: pv0001
func (iscsi *iscsiDisk) GetPodDeviceMapPath() (string, string) {
return iscsi.iscsiPodDeviceMapPath()
}
func portalMounter(portal string) string {
if !strings.Contains(portal, ":") {
portal = portal + ":3260"
@ -316,7 +411,7 @@ func portalMounter(portal string) string {
// get iSCSI volume info: readOnly and fstype
func getISCSIVolumeInfo(spec *volume.Spec) (bool, string, error) {
// for volume source, readonly is in volume spec
// for PV, readonly is in PV spec
// for PV, readonly is in PV spec. PV gets the ReadOnly flag indirectly through the PVC source
if spec.Volume != nil && spec.Volume.ISCSI != nil {
return spec.Volume.ISCSI.ReadOnly, spec.Volume.ISCSI.FSType, nil
} else if spec.PersistentVolume != nil &&
@ -397,3 +492,155 @@ func getISCSISecretNameAndNamespace(spec *volume.Spec, defaultSecretNamespace st
return "", "", fmt.Errorf("Spec does not reference an ISCSI volume type")
}
func createISCSIDisk(spec *volume.Spec, podUID types.UID, plugin *iscsiPlugin, manager diskManager, secret map[string]string) (*iscsiDisk, error) {
tp, portals, iqn, lunStr, err := getISCSITargetInfo(spec)
if err != nil {
return nil, err
}
lun := strconv.Itoa(int(lunStr))
portal := portalMounter(tp)
var bkportal []string
bkportal = append(bkportal, portal)
for _, p := range portals {
bkportal = append(bkportal, portalMounter(string(p)))
}
iface, initiatorNamePtr, err := getISCSIInitiatorInfo(spec)
if err != nil {
return nil, err
}
var initiatorName string
if initiatorNamePtr != nil {
initiatorName = *initiatorNamePtr
}
chapDiscovery, err := getISCSIDiscoveryCHAPInfo(spec)
if err != nil {
return nil, err
}
chapSession, err := getISCSISessionCHAPInfo(spec)
if err != nil {
return nil, err
}
return &iscsiDisk{
podUID: podUID,
VolName: spec.Name(),
Portals: bkportal,
Iqn: iqn,
Lun: lun,
Iface: iface,
chap_discovery: chapDiscovery,
chap_session: chapSession,
secret: secret,
InitiatorName: initiatorName,
manager: manager,
plugin: plugin}, nil
}
func createSecretMap(spec *volume.Spec, plugin *iscsiPlugin, namespace string) (map[string]string, error) {
var secret map[string]string
chapDiscover, err := getISCSIDiscoveryCHAPInfo(spec)
if err != nil {
return nil, err
}
chapSession, err := getISCSISessionCHAPInfo(spec)
if err != nil {
return nil, err
}
if chapDiscover || chapSession {
secretName, secretNamespace, err := getISCSISecretNameAndNamespace(spec, namespace)
if err != nil {
return nil, err
}
if len(secretName) > 0 && len(secretNamespace) > 0 {
// if secret is provideded, retrieve it
kubeClient := plugin.host.GetKubeClient()
if kubeClient == nil {
return nil, fmt.Errorf("Cannot get kube client")
}
secretObj, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(secretName, metav1.GetOptions{})
if err != nil {
err = fmt.Errorf("Couldn't get secret %v/%v error: %v", secretNamespace, secretName, err)
return nil, err
}
secret = make(map[string]string)
for name, data := range secretObj.Data {
glog.V(4).Infof("retrieving CHAP secret name: %s", name)
secret[name] = string(data)
}
}
}
return secret, err
}
func createVolumeFromISCSIVolumeSource(volumeName string, iscsi v1.ISCSIVolumeSource) *v1.Volume {
return &v1.Volume{
Name: volumeName,
VolumeSource: v1.VolumeSource{
ISCSI: &iscsi,
},
}
}
func createPersistentVolumeFromISCSIPVSource(volumeName string, iscsi v1.ISCSIPersistentVolumeSource) *v1.PersistentVolume {
block := v1.PersistentVolumeBlock
return &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: volumeName,
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{
ISCSI: &iscsi,
},
VolumeMode: &block,
},
}
}
func getVolumeSpecFromGlobalMapPath(volumeName, globalMapPath string) (*volume.Spec, error) {
// Retreive volume spec information from globalMapPath
// globalMapPath example:
// plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}
// plugins/kubernetes.io/iscsi/volumeDevices/iface-default/192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0
// device: 192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0
device, _, err := extractDeviceAndPrefix(globalMapPath)
if err != nil {
return nil, err
}
bkpPortal, iqn, err := extractPortalAndIqn(device)
if err != nil {
return nil, err
}
arr := strings.Split(device, "-lun-")
if len(arr) < 2 {
return nil, fmt.Errorf("failed to retreive lun from globalMapPath: %v", globalMapPath)
}
lun, err := strconv.Atoi(arr[1])
if err != nil {
return nil, err
}
iface, found := extractIface(globalMapPath)
if !found {
return nil, fmt.Errorf("failed to retreive iface from globalMapPath: %v", globalMapPath)
}
iscsiPV := createPersistentVolumeFromISCSIPVSource(volumeName,
v1.ISCSIPersistentVolumeSource{
TargetPortal: bkpPortal,
IQN: iqn,
Lun: int32(lun),
ISCSIInterface: iface,
},
)
glog.V(5).Infof("ConstructBlockVolumeSpec: TargetPortal: %v, IQN: %v, Lun: %v, ISCSIInterface: %v",
iscsiPV.Spec.PersistentVolumeSource.ISCSI.TargetPortal,
iscsiPV.Spec.PersistentVolumeSource.ISCSI.IQN,
iscsiPV.Spec.PersistentVolumeSource.ISCSI.Lun,
iscsiPV.Spec.PersistentVolumeSource.ISCSI.ISCSIInterface,
)
return volume.NewSpecFromPersistentVolume(iscsiPV, false), nil
}

View File

@ -19,6 +19,7 @@ package iscsi
import (
"fmt"
"os"
"strings"
"testing"
"k8s.io/api/core/v1"
@ -80,7 +81,7 @@ type fakeDiskManager struct {
func NewFakeDiskManager() *fakeDiskManager {
return &fakeDiskManager{
tmpDir: utiltesting.MkTmpdirOrDie("fc_test"),
tmpDir: utiltesting.MkTmpdirOrDie("iscsi_test"),
}
}
@ -91,6 +92,11 @@ func (fake *fakeDiskManager) Cleanup() {
func (fake *fakeDiskManager) MakeGlobalPDName(disk iscsiDisk) string {
return fake.tmpDir
}
func (fake *fakeDiskManager) MakeGlobalVDPDName(disk iscsiDisk) string {
return fake.tmpDir
}
func (fake *fakeDiskManager) AttachDisk(b iscsiDiskMounter) (string, error) {
globalPath := b.manager.MakeGlobalPDName(*b.iscsiDisk)
err := os.MkdirAll(globalPath, 0750)
@ -113,6 +119,15 @@ func (fake *fakeDiskManager) DetachDisk(c iscsiDiskUnmounter, mntPath string) er
return nil
}
func (fake *fakeDiskManager) DetachBlockISCSIDisk(c iscsiDiskUnmapper, mntPath string) error {
globalPath := c.manager.MakeGlobalVDPDName(*c.iscsiDisk)
err := os.RemoveAll(globalPath)
if err != nil {
return err
}
return nil
}
func doTestPlugin(t *testing.T, spec *volume.Spec) {
tmpDir, err := utiltesting.MkTmpdir("iscsi_test")
if err != nil {
@ -289,10 +304,12 @@ type testcase struct {
defaultNs string
spec *volume.Spec
// Expected return of the test
expectedName string
expectedNs string
expectedIface string
expectedError error
expectedName string
expectedNs string
expectedIface string
expectedError error
expectedDiscoveryCHAP bool
expectedSessionCHAP bool
}
func TestGetSecretNameAndNamespaceForPV(t *testing.T) {
@ -424,5 +441,105 @@ func TestGetISCSIInitiatorInfo(t *testing.T) {
err, resultIface)
}
}
}
func TestGetISCSICHAP(t *testing.T) {
tests := []testcase{
{
name: "persistent volume source",
spec: &volume.Spec{
PersistentVolume: &v1.PersistentVolume{
Spec: v1.PersistentVolumeSpec{
PersistentVolumeSource: v1.PersistentVolumeSource{
ISCSI: &v1.ISCSIPersistentVolumeSource{
DiscoveryCHAPAuth: true,
SessionCHAPAuth: true,
},
},
},
},
},
expectedDiscoveryCHAP: true,
expectedSessionCHAP: true,
expectedError: nil,
},
{
name: "pod volume source",
spec: &volume.Spec{
Volume: &v1.Volume{
VolumeSource: v1.VolumeSource{
ISCSI: &v1.ISCSIVolumeSource{
DiscoveryCHAPAuth: true,
SessionCHAPAuth: true,
},
},
},
},
expectedDiscoveryCHAP: true,
expectedSessionCHAP: true,
expectedError: nil,
},
{
name: "no volume",
spec: &volume.Spec{},
expectedDiscoveryCHAP: false,
expectedSessionCHAP: false,
expectedError: fmt.Errorf("Spec does not reference an ISCSI volume type"),
},
}
for _, testcase := range tests {
resultDiscoveryCHAP, err := getISCSIDiscoveryCHAPInfo(testcase.spec)
resultSessionCHAP, err := getISCSISessionCHAPInfo(testcase.spec)
switch testcase.name {
case "no volume":
if err.Error() != testcase.expectedError.Error() || resultDiscoveryCHAP != testcase.expectedDiscoveryCHAP || resultSessionCHAP != testcase.expectedSessionCHAP {
t.Errorf("%s failed: expected err=%v DiscoveryCHAP=%v SessionCHAP=%v, got %v/%v/%v",
testcase.name, testcase.expectedError, testcase.expectedDiscoveryCHAP, testcase.expectedSessionCHAP,
err, resultDiscoveryCHAP, resultSessionCHAP)
}
default:
if err != testcase.expectedError || resultDiscoveryCHAP != testcase.expectedDiscoveryCHAP || resultSessionCHAP != testcase.expectedSessionCHAP {
t.Errorf("%s failed: expected err=%v DiscoveryCHAP=%v SessionCHAP=%v, got %v/%v/%v", testcase.name, testcase.expectedError, testcase.expectedDiscoveryCHAP, testcase.expectedSessionCHAP,
err, resultDiscoveryCHAP, resultSessionCHAP)
}
}
}
}
func TestGetVolumeSpec(t *testing.T) {
path := "plugins/kubernetes.io/iscsi/volumeDevices/iface-default/127.0.0.1:3260-iqn.2014-12.server:storage.target01-lun-0"
spec, _ := getVolumeSpecFromGlobalMapPath("test", path)
portal := spec.PersistentVolume.Spec.PersistentVolumeSource.ISCSI.TargetPortal
if portal != "127.0.0.1:3260" {
t.Errorf("wrong portal: %v", portal)
}
iqn := spec.PersistentVolume.Spec.PersistentVolumeSource.ISCSI.IQN
if iqn != "iqn.2014-12.server:storage.target01" {
t.Errorf("wrong iqn: %v", iqn)
}
lun := spec.PersistentVolume.Spec.PersistentVolumeSource.ISCSI.Lun
if lun != 0 {
t.Errorf("wrong lun: %v", lun)
}
iface := spec.PersistentVolume.Spec.PersistentVolumeSource.ISCSI.ISCSIInterface
if iface != "default" {
t.Errorf("wrong ISCSIInterface: %v", iface)
}
}
func TestGetVolumeSpec_no_lun(t *testing.T) {
path := "plugins/kubernetes.io/iscsi/volumeDevices/iface-default/127.0.0.1:3260-iqn.2014-12.server:storage.target01"
_, err := getVolumeSpecFromGlobalMapPath("test", path)
if !strings.Contains(err.Error(), "malformatted mnt path") {
t.Errorf("should get error: malformatted mnt path")
}
}
func TestGetVolumeSpec_no_iface(t *testing.T) {
path := "plugins/kubernetes.io/iscsi/volumeDevices/default/127.0.0.1:3260-iqn.2014-12.server:storage.target01-lun-0"
_, err := getVolumeSpecFromGlobalMapPath("test", path)
if !strings.Contains(err.Error(), "failed to retreive iface") {
t.Errorf("should get error: failed to retreive iface")
}
}

View File

@ -27,6 +27,9 @@ import (
"time"
"github.com/golang/glog"
"k8s.io/api/core/v1"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/util/mount"
"k8s.io/kubernetes/pkg/volume"
volumeutil "k8s.io/kubernetes/pkg/volume/util"
@ -163,10 +166,21 @@ func makePDNameInternal(host volume.VolumeHost, portal string, iqn string, lun s
return path.Join(host.GetPluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun)
}
// make a directory like /var/lib/kubelet/plugins/kubernetes.io/iscsi/volumeDevices/iface_name/portal-some_iqn-lun-lun_id
func makeVDPDNameInternal(host volume.VolumeHost, portal string, iqn string, lun string, iface string) string {
return path.Join(host.GetVolumeDevicePluginDir(iscsiPluginName), "iface-"+iface, portal+"-"+iqn+"-lun-"+lun)
}
type ISCSIUtil struct{}
// MakeGlobalPDName returns path of global plugin dir
func (util *ISCSIUtil) MakeGlobalPDName(iscsi iscsiDisk) string {
return makePDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.lun, iscsi.Iface)
return makePDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.Lun, iscsi.Iface)
}
// MakeGlobalVDPDName returns path of global volume device plugin dir
func (util *ISCSIUtil) MakeGlobalVDPDName(iscsi iscsiDisk) string {
return makeVDPDNameInternal(iscsi.plugin.host, iscsi.Portals[0], iscsi.Iqn, iscsi.Lun, iscsi.Iface)
}
func (util *ISCSIUtil) persistISCSI(conf iscsiDisk, mnt string) error {
@ -184,7 +198,6 @@ func (util *ISCSIUtil) persistISCSI(conf iscsiDisk, mnt string) error {
}
func (util *ISCSIUtil) loadISCSI(conf *iscsiDisk, mnt string) error {
// NOTE: The iscsi config json is not deleted after logging out from target portals.
file := path.Join(mnt, "iscsi.json")
fp, err := os.Open(file)
if err != nil {
@ -198,6 +211,7 @@ func (util *ISCSIUtil) loadISCSI(conf *iscsiDisk, mnt string) error {
return nil
}
// AttachDisk returns devicePath of volume if attach succeeded otherwise returns error
func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) {
var devicePath string
var devicePaths []string
@ -240,9 +254,9 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) {
return "", fmt.Errorf("Could not parse iface file for %s", b.Iface)
}
if iscsiTransport == "tcp" {
devicePath = strings.Join([]string{"/dev/disk/by-path/ip", tp, "iscsi", b.Iqn, "lun", b.lun}, "-")
devicePath = strings.Join([]string{"/dev/disk/by-path/ip", tp, "iscsi", b.Iqn, "lun", b.Lun}, "-")
} else {
devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", tp, "iscsi", b.Iqn, "lun", b.lun}, "-")
devicePath = strings.Join([]string{"/dev/disk/by-path/pci", "*", "ip", tp, "iscsi", b.Iqn, "lun", b.Lun}, "-")
}
if exist := waitForPathToExist(&devicePath, 1, iscsiTransport); exist {
@ -307,26 +321,6 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) {
//Make sure we use a valid devicepath to find mpio device.
devicePath = devicePaths[0]
// mount it
globalPDPath := b.manager.MakeGlobalPDName(*b.iscsiDisk)
notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
if err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("Heuristic determination of mount point failed:%v", err)
}
if !notMnt {
glog.Infof("iscsi: %s already mounted", globalPDPath)
return "", nil
}
if err := os.MkdirAll(globalPDPath, 0750); err != nil {
glog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath)
return "", err
}
// Persist iscsi disk config to json file for DetachDisk path
util.persistISCSI(*(b.iscsiDisk), globalPDPath)
for _, path := range devicePaths {
// There shouldnt be any empty device paths. However adding this check
// for safer side to avoid the possibility of an empty entry.
@ -339,14 +333,67 @@ func (util *ISCSIUtil) AttachDisk(b iscsiDiskMounter) (string, error) {
break
}
}
err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil)
if err != nil {
glog.Errorf("iscsi: failed to mount iscsi volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
}
return devicePath, err
glog.V(5).Infof("iscsi: AttachDisk devicePath: %s", devicePath)
// run global mount path related operations based on volumeMode
return globalPDPathOperation(b)(b, devicePath, util)
}
// globalPDPathOperation returns global mount path related operations based on volumeMode.
// If the volumeMode is 'Filesystem' or not defined, plugin needs to create a dir, persist
// iscsi configrations, and then format/mount the volume.
// If the volumeMode is 'Block', plugin creates a dir and persists iscsi configrations.
// Since volume type is block, plugin doesn't need to format/mount the volume.
func globalPDPathOperation(b iscsiDiskMounter) func(iscsiDiskMounter, string, *ISCSIUtil) (string, error) {
// TODO: remove feature gate check after no longer needed
if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
glog.V(5).Infof("iscsi: AttachDisk volumeMode: %s", b.volumeMode)
if b.volumeMode == v1.PersistentVolumeBlock {
// If the volumeMode is 'Block', plugin don't need to format the volume.
return func(b iscsiDiskMounter, devicePath string, util *ISCSIUtil) (string, error) {
globalPDPath := b.manager.MakeGlobalVDPDName(*b.iscsiDisk)
// Create dir like /var/lib/kubelet/plugins/kubernetes.io/iscsi/volumeDevices/{ifaceName}/{portal-some_iqn-lun-lun_id}
if err := os.MkdirAll(globalPDPath, 0750); err != nil {
glog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath)
return "", err
}
// Persist iscsi disk config to json file for DetachDisk path
util.persistISCSI(*(b.iscsiDisk), globalPDPath)
return devicePath, nil
}
}
}
// If the volumeMode is 'Filesystem', plugin needs to format the volume
// and mount it to globalPDPath.
return func(b iscsiDiskMounter, devicePath string, util *ISCSIUtil) (string, error) {
globalPDPath := b.manager.MakeGlobalPDName(*b.iscsiDisk)
notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
if err != nil && !os.IsNotExist(err) {
return "", fmt.Errorf("Heuristic determination of mount point failed:%v", err)
}
// Return confirmed devicePath to caller
if !notMnt {
glog.Infof("iscsi: %s already mounted", globalPDPath)
return devicePath, nil
}
// Create dir like /var/lib/kubelet/plugins/kubernetes.io/iscsi/{ifaceName}/{portal-some_iqn-lun-lun_id}
if err := os.MkdirAll(globalPDPath, 0750); err != nil {
glog.Errorf("iscsi: failed to mkdir %s, error", globalPDPath)
return "", err
}
// Persist iscsi disk config to json file for DetachDisk path
util.persistISCSI(*(b.iscsiDisk), globalPDPath)
err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil)
if err != nil {
glog.Errorf("iscsi: failed to mount iscsi volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
}
return devicePath, nil
}
}
// DetachDisk unmounts and detaches a volume from node
func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
_, cnt, err := mount.GetDeviceNameFromMount(c.mounter, mntPath)
if err != nil {
@ -401,9 +448,91 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
}
portals := removeDuplicate(bkpPortal)
if len(portals) == 0 {
return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations.")
return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations")
}
err = util.detachISCSIDisk(c.exec, portals, iqn, iface, volName, initiatorName, found)
if err != nil {
return fmt.Errorf("failed to finish detachISCSIDisk, err: %v", err)
}
return nil
}
// DetachBlockISCSIDisk removes loopback device for a volume and detaches a volume from node
func (util *ISCSIUtil) DetachBlockISCSIDisk(c iscsiDiskUnmapper, mapPath string) error {
if pathExists, pathErr := volumeutil.PathExists(mapPath); pathErr != nil {
return fmt.Errorf("Error checking if path exists: %v", pathErr)
} else if !pathExists {
glog.Warningf("Warning: Unmap skipped because path does not exist: %v", mapPath)
return nil
}
// If we arrive here, device is no longer used, see if need to logout the target
// device: 192.168.0.10:3260-iqn.2017-05.com.example:test-lun-0
device, _, err := extractDeviceAndPrefix(mapPath)
if err != nil {
return err
}
var bkpPortal []string
var volName, iqn, lun, iface, initiatorName string
found := true
// load iscsi disk config from json file
if err := util.loadISCSI(c.iscsiDisk, mapPath); err == nil {
bkpPortal, iqn, lun, iface, volName = c.iscsiDisk.Portals, c.iscsiDisk.Iqn, c.iscsiDisk.Lun, c.iscsiDisk.Iface, c.iscsiDisk.VolName
initiatorName = c.iscsiDisk.InitiatorName
} else {
// If the iscsi disk config is not found, fall back to the original behavior.
// This portal/iqn/iface is no longer referenced, log out.
// Extract the portal and iqn from device path.
bkpPortal = make([]string, 1)
bkpPortal[0], iqn, err = extractPortalAndIqn(device)
if err != nil {
return err
}
arr := strings.Split(device, "-lun-")
if len(arr) < 2 {
return fmt.Errorf("failed to retreive lun from mapPath: %v", mapPath)
}
lun = arr[1]
// Extract the iface from the mountPath and use it to log out. If the iface
// is not found, maintain the previous behavior to facilitate kubelet upgrade.
// Logout may fail as no session may exist for the portal/IQN on the specified interface.
iface, found = extractIface(mapPath)
}
portals := removeDuplicate(bkpPortal)
if len(portals) == 0 {
return fmt.Errorf("iscsi detach disk: failed to detach iscsi disk. Couldn't get connected portals from configurations")
}
devicePath := getDevByPath(portals[0], iqn, lun)
glog.V(5).Infof("iscsi: devicePath: %s", devicePath)
if _, err = os.Stat(devicePath); err != nil {
return fmt.Errorf("failed to validate devicePath: %s", devicePath)
}
// check if the dev is using mpio and if so mount it via the dm-XX device
if mappedDevicePath := c.deviceUtil.FindMultipathDeviceForDevice(devicePath); mappedDevicePath != "" {
devicePath = mappedDevicePath
}
// Get loopback device which takes fd lock for devicePath before
// detaching a volume from node.
blkUtil := volumeutil.NewBlockVolumePathHandler()
loop, err := volumeutil.BlockVolumePathHandler.GetLoopDevice(blkUtil, devicePath)
if err != nil {
return fmt.Errorf("failed to get loopback for device: %v, err: %v", devicePath, err)
}
// Detach a volume from kubelet node
err = util.detachISCSIDisk(c.exec, portals, iqn, iface, volName, initiatorName, found)
if err != nil {
return fmt.Errorf("failed to finish detachISCSIDisk, err: %v", err)
}
// The volume was successfully detached from node. We can safely remove the loopback.
err = volumeutil.BlockVolumePathHandler.RemoveLoopDevice(blkUtil, loop)
if err != nil {
return fmt.Errorf("failed to remove loopback :%v, err: %v", loop, err)
}
return nil
}
func (util *ISCSIUtil) detachISCSIDisk(exec mount.Exec, portals []string, iqn, iface, volName, initiatorName string, found bool) error {
for _, portal := range portals {
logoutArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "--logout"}
deleteArgs := []string{"-m", "node", "-p", portal, "-T", iqn, "-o", "delete"}
@ -412,13 +541,13 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
deleteArgs = append(deleteArgs, []string{"-I", iface}...)
}
glog.Infof("iscsi: log out target %s iqn %s iface %s", portal, iqn, iface)
out, err := c.exec.Run("iscsiadm", logoutArgs...)
out, err := exec.Run("iscsiadm", logoutArgs...)
if err != nil {
glog.Errorf("iscsi: failed to detach disk Error: %s", string(out))
}
// Delete the node record
glog.Infof("iscsi: delete node record target %s iqn %s", portal, iqn)
out, err = c.exec.Run("iscsiadm", deleteArgs...)
out, err = exec.Run("iscsiadm", deleteArgs...)
if err != nil {
glog.Errorf("iscsi: failed to delete node record Error: %s", string(out))
}
@ -427,7 +556,7 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
// If the iface is not created via iscsi plugin, skip to delete
if initiatorName != "" && found && iface == (portals[0]+":"+volName) {
deleteArgs := []string{"-m", "iface", "-I", iface, "-o", "delete"}
out, err := c.exec.Run("iscsiadm", deleteArgs...)
out, err := exec.Run("iscsiadm", deleteArgs...)
if err != nil {
glog.Errorf("iscsi: failed to delete iface Error: %s", string(out))
}
@ -436,6 +565,10 @@ func (util *ISCSIUtil) DetachDisk(c iscsiDiskUnmounter, mntPath string) error {
return nil
}
func getDevByPath(portal, iqn, lun string) string {
return "/dev/disk/by-path/ip-" + portal + "-iscsi-" + iqn + "-lun-" + lun
}
func extractTransportname(ifaceOutput string) (iscsiTransport string) {
rexOutput := ifaceTransportNameRe.FindStringSubmatch(ifaceOutput)
if rexOutput == nil {