diff --git a/test/e2e/framework/volume_util.go b/test/e2e/framework/volume_util.go index a929e87858..c5cdb317b6 100644 --- a/test/e2e/framework/volume_util.go +++ b/test/e2e/framework/volume_util.go @@ -81,6 +81,9 @@ const ( // PodCleanupTimeout is a waiting period for pod to be cleaned up and unmount its volumes so we // don't tear down containers with NFS/Ceph/Gluster server too early. PodCleanupTimeout = 20 * time.Second + + // Template for iSCSI IQN. + iSCSIIQNTemplate = "iqn.2003-01.io.k8s:e2e.%s" ) // VolumeTestConfig is a struct for configuration of one tests. The test consist of: @@ -104,6 +107,8 @@ type VolumeTestConfig struct { ServerVolumes map[string]string // Message to wait for before starting clients ServerReadyMessage string + // Use HostNetwork for the server + ServerHostNetwork bool // Wait for the pod to terminate successfully // False indicates that the pod is long running WaitForCompletion bool @@ -183,20 +188,30 @@ func NewGlusterfsServer(cs clientset.Interface, namespace string) (config Volume } // NewISCSIServer is an iSCSI-specific wrapper for CreateStorageServer. -func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip string) { +func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip, iqn string) { + // Generate cluster-wide unique IQN + iqn = fmt.Sprintf(iSCSIIQNTemplate, namespace) + config = VolumeTestConfig{ Namespace: namespace, Prefix: "iscsi", ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer), - ServerPorts: []int{3260}, + ServerArgs: []string{iqn}, ServerVolumes: map[string]string{ // iSCSI container needs to insert modules from the host "/lib/modules": "/lib/modules", + // iSCSI container needs to configure kernel + "/sys/kernel": "/sys/kernel", + // iSCSI source "block devices" must be available on the host + "/srv/iscsi": "/srv/iscsi", }, - ServerReadyMessage: "Configuration restored from /etc/target/saveconfig.json", + ServerReadyMessage: "iscsi target started", + ServerHostNetwork: true, } pod, ip = CreateStorageServer(cs, config) - return config, pod, ip + // Make sure the client runs on the same node as server so we don't need to open any firewalls. + config.ClientNodeName = pod.Spec.NodeName + return config, pod, ip, iqn } // NewRBDServer is a CephRBD-specific wrapper for CreateStorageServer. @@ -312,6 +327,7 @@ func StartVolumeServer(client clientset.Interface, config VolumeTestConfig) *v1. }, Spec: v1.PodSpec{ + HostNetwork: config.ServerHostNetwork, Containers: []v1.Container{ { Name: serverPodName, diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go index 2debd039b6..485241e74f 100644 --- a/test/e2e/storage/drivers/in_tree.go +++ b/test/e2e/storage/drivers/in_tree.go @@ -330,6 +330,7 @@ type iSCSIVolume struct { serverPod *v1.Pod serverIP string f *framework.Framework + iqn string } var _ testsuites.TestDriver = &iSCSIDriver{} @@ -374,11 +375,10 @@ func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, volume tests volSource := v1.VolumeSource{ ISCSI: &v1.ISCSIVolumeSource{ - TargetPortal: iv.serverIP + ":3260", - // from test/images/volume/iscsi/initiatorname.iscsi - IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c", - Lun: 0, - ReadOnly: readOnly, + TargetPortal: "127.0.0.1:3260", + IQN: iv.iqn, + Lun: 0, + ReadOnly: readOnly, }, } if fsType != "" { @@ -393,8 +393,8 @@ func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, vo pvSource := v1.PersistentVolumeSource{ ISCSI: &v1.ISCSIPersistentVolumeSource{ - TargetPortal: iv.serverIP + ":3260", - IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c", + TargetPortal: "127.0.0.1:3260", + IQN: iv.iqn, Lun: 0, ReadOnly: readOnly, }, @@ -418,11 +418,13 @@ func (i *iSCSIDriver) CreateVolume(config *testsuites.PerTestConfig, volType tes cs := f.ClientSet ns := f.Namespace - c, serverPod, serverIP := framework.NewISCSIServer(cs, ns.Name) + c, serverPod, serverIP, iqn := framework.NewISCSIServer(cs, ns.Name) config.ServerConfig = &c + config.ClientNodeName = c.ClientNodeName return &iSCSIVolume{ serverPod: serverPod, serverIP: serverIP, + iqn: iqn, f: f, } } diff --git a/test/e2e/storage/testsuites/volumes.go b/test/e2e/storage/testsuites/volumes.go index d09addc12d..9c3c4a3e0b 100644 --- a/test/e2e/storage/testsuites/volumes.go +++ b/test/e2e/storage/testsuites/volumes.go @@ -168,7 +168,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T init() defer cleanup() - testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config.ClientNodeSelector) + testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config) }) } @@ -176,7 +176,7 @@ func testScriptInPod( f *framework.Framework, volumeType string, source *v1.VolumeSource, - nodeSelector map[string]string) { + config *PerTestConfig) { const ( volPath = "/vol1" @@ -217,7 +217,8 @@ func testScriptInPod( }, }, RestartPolicy: v1.RestartPolicyNever, - NodeSelector: nodeSelector, + NodeSelector: config.ClientNodeSelector, + NodeName: config.ClientNodeName, }, } By(fmt.Sprintf("Creating pod %s", pod.Name)) diff --git a/test/images/volume/iscsi/Dockerfile b/test/images/volume/iscsi/Dockerfile index cba82664c4..bff98b575b 100644 --- a/test/images/volume/iscsi/Dockerfile +++ b/test/images/volume/iscsi/Dockerfile @@ -16,20 +16,8 @@ FROM BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ -RUN yum install -y iscsi-initiator-utils targetcli net-tools strace procps-ng psmisc && yum clean all -ADD run_iscsid.sh /usr/local/bin/ -ADD initiatorname.iscsi /etc/iscsi/ +RUN yum install -y targetcli && yum clean all +ADD run_iscsi_target.sh /usr/local/bin/ ADD block.tar.gz / -# This JSON file was generated by targetcli with these commands: -# /backstores/fileio create block /block -# /iscsi create -# # Enable demo mode (no authentication!): -# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1 -# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1/luns create /backstores/fileio/block -# saveconfig -ADD saveconfig.json /etc/target/ - -EXPOSE 3260/tcp - -ENTRYPOINT ["/usr/local/bin/run_iscsid.sh"] +ENTRYPOINT ["/usr/local/bin/run_iscsi_target.sh"] diff --git a/test/images/volume/iscsi/VERSION b/test/images/volume/iscsi/VERSION index d3827e75a5..cd5ac039d6 100644 --- a/test/images/volume/iscsi/VERSION +++ b/test/images/volume/iscsi/VERSION @@ -1 +1 @@ -1.0 +2.0 diff --git a/test/images/volume/iscsi/block.tar.gz b/test/images/volume/iscsi/block.tar.gz index 8840f54386..fcb0738927 100644 Binary files a/test/images/volume/iscsi/block.tar.gz and b/test/images/volume/iscsi/block.tar.gz differ diff --git a/test/images/volume/iscsi/create_block.sh b/test/images/volume/iscsi/create_block.sh index 9421568690..eb975001db 100755 --- a/test/images/volume/iscsi/create_block.sh +++ b/test/images/volume/iscsi/create_block.sh @@ -40,7 +40,7 @@ mkfs.ext2 block # Add index.html to it mount -o loop block $MNTDIR -echo "Hello from iSCSI" > $MNTDIR/index.html +echo "Hello from iscsi" > $MNTDIR/index.html umount $MNTDIR rm block.tar.gz 2>/dev/null || : diff --git a/test/images/volume/iscsi/initiatorname.iscsi b/test/images/volume/iscsi/initiatorname.iscsi deleted file mode 100644 index 66d3f723c0..0000000000 --- a/test/images/volume/iscsi/initiatorname.iscsi +++ /dev/null @@ -1 +0,0 @@ -InitiatorName=iqn.1994-05.com.redhat:eb59fbe2c4c5 diff --git a/test/images/volume/iscsi/run_iscsi_target.sh b/test/images/volume/iscsi/run_iscsi_target.sh new file mode 100755 index 0000000000..034beb4427 --- /dev/null +++ b/test/images/volume/iscsi/run_iscsi_target.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script does not run any daemon, it only configures iSCSI target (=server) +# in kernel. It is possible to run this script multiple times on a single +# node, each run will create its own IQN and LUN. + +# Kubernetes must provide unique name. +IQN=$1 + +# targetcli synchronizes over dbus, however it does not work in +# containers. Use flock instead +LOCK=/srv/iscsi/targetcli.lock + +function start() +{ + # targetcli need dbus. It may not run on the host, so start a private one + mkdir /run/dbus + dbus-daemon --system + + # Create new IQN (iSCSI Qualified Name) + flock $LOCK targetcli /iscsi create "$IQN" + # Run it in demo mode, i.e. no authentication + flock $LOCK targetcli /iscsi/"$IQN"/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1 + + # Create unique "block volume" (i.e. flat file) on the *host*. + # Having it in the container confuses kernel from some reason + # and it's not able to server multiple LUNs from different + # containers. + # /srv/iscsi must be bind-mount from the host. + cp /block /srv/iscsi/"$IQN" + + # Make the block volume available through our IQN as LUN 0 + flock $LOCK targetcli /backstores/fileio create block-"$IQN" /srv/iscsi/"$IQN" + flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns create /backstores/fileio/block-"$IQN" + + echo "iscsi target started" +} + +function stop() +{ + echo "stopping iscsi target" + # Remove IQN + flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns/ delete 0 + flock $LOCK targetcli /iscsi delete "$IQN" + # Remove block device mapping + flock $LOCK targetcli /backstores/fileio delete block-"$IQN" + /bin/rm -f /srv/iscsi/"$IQN" + echo "iscsi target stopped" + exit 0 +} + + +trap stop TERM +start + +while true; do + sleep 1 +done diff --git a/test/images/volume/iscsi/run_iscsid.sh b/test/images/volume/iscsi/run_iscsid.sh deleted file mode 100755 index a5bded46cd..0000000000 --- a/test/images/volume/iscsi/run_iscsid.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -function start() -{ - # targetcli need dbus - mkdir /run/dbus - dbus-daemon --system - - # clear any previous configuration - targetcli clearconfig confirm=True - - # restore configuration from saveconfig.json - targetcli restoreconfig - - # maximum log level - iscsid -f -d 8 - - echo "iscsid started" -} - -function stop() -{ - echo "Stopping iscsid" - killall iscsid - targetcli clearconfig confirm=True - - echo "iscsid stopped" - exit 0 -} - - -trap stop TERM -start - -while true; do - sleep 5 -done diff --git a/test/images/volume/iscsi/saveconfig.json b/test/images/volume/iscsi/saveconfig.json deleted file mode 100644 index 4efc14c8a2..0000000000 --- a/test/images/volume/iscsi/saveconfig.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "fabric_modules": [], - "storage_objects": [ - { - "attributes": { - "block_size": 512, - "emulate_3pc": 1, - "emulate_caw": 1, - "emulate_dpo": 0, - "emulate_fua_read": 0, - "emulate_fua_write": 1, - "emulate_model_alias": 1, - "emulate_rest_reord": 0, - "emulate_tas": 1, - "emulate_tpu": 0, - "emulate_tpws": 0, - "emulate_ua_intlck_ctrl": 0, - "emulate_write_cache": 1, - "enforce_pr_isids": 1, - "force_pr_aptpl": 0, - "is_nonrot": 0, - "max_unmap_block_desc_count": 1, - "max_unmap_lba_count": 8192, - "max_write_same_len": 4096, - "optimal_sectors": 16384, - "pi_prot_format": 0, - "pi_prot_type": 0, - "queue_depth": 128, - "unmap_granularity": 1, - "unmap_granularity_alignment": 0 - }, - "dev": "block", - "name": "block", - "plugin": "fileio", - "size": 126877696, - "write_back": true, - "wwn": "521c57aa-9d9b-4e5d-ab1a-527487f92a33" - } - ], - "targets": [ - { - "fabric": "iscsi", - "tpgs": [ - { - "attributes": { - "authentication": 0, - "cache_dynamic_acls": 1, - "default_cmdsn_depth": 64, - "default_erl": 0, - "demo_mode_discovery": 1, - "demo_mode_write_protect": 0, - "generate_node_acls": 1, - "login_timeout": 15, - "netif_timeout": 2, - "prod_mode_write_protect": 0, - "t10_pi": 0 - }, - "enable": true, - "luns": [ - { - "index": 0, - "storage_object": "/backstores/fileio/block" - } - ], - "node_acls": [], - "parameters": { - "AuthMethod": "CHAP,None", - "DataDigest": "CRC32C,None", - "DataPDUInOrder": "Yes", - "DataSequenceInOrder": "Yes", - "DefaultTime2Retain": "20", - "DefaultTime2Wait": "2", - "ErrorRecoveryLevel": "0", - "FirstBurstLength": "65536", - "HeaderDigest": "CRC32C,None", - "IFMarkInt": "2048~65535", - "IFMarker": "No", - "ImmediateData": "Yes", - "InitialR2T": "Yes", - "MaxBurstLength": "262144", - "MaxConnections": "1", - "MaxOutstandingR2T": "1", - "MaxRecvDataSegmentLength": "8192", - "MaxXmitDataSegmentLength": "262144", - "OFMarkInt": "2048~65535", - "OFMarker": "No", - "TargetAlias": "LIO Target" - }, - "portals": [ - { - "ip_address": "0.0.0.0", - "iser": false, - "port": 3260 - } - ], - "tag": 1 - } - ], - "wwn": "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c" - } - ] -} diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 4c5df340e1..b37860e1c0 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -238,9 +238,9 @@ func initImageConfigs() map[int]Config { configs[ServeHostname] = Config{e2eRegistry, "serve-hostname", "1.1"} configs[TestWebserver] = Config{e2eRegistry, "test-webserver", "1.0"} configs[VolumeNFSServer] = Config{e2eRegistry, "volume/nfs", "1.0"} - configs[VolumeISCSIServer] = Config{e2eRegistry, "volume/iscsi", "1.0"} + configs[VolumeISCSIServer] = Config{"quay.io", "jsafrane/iscsi-test", "2.0"} configs[VolumeGlusterServer] = Config{e2eRegistry, "volume/gluster", "1.0"} - configs[VolumeRBDServer] = Config{e2eRegistry, "volume/rbd", "1.0.1"} + configs[VolumeRBDServer] = Config{"quay.io", "jsafrane/rbd-test", "2.0"} return configs }