mirror of https://github.com/k3s-io/k3s
Parallelize iSCSI tests
iSCSI target (=the server) is implemented in Linux kernel. The "iSCSI server" pod is not a real server, it just configures the kernel on the host. In order to run iSCSI tests in parallel, we need to be able to run multiple such pods on a single node, serving different LUNs to different tests. The "server pod" must run with HostNetwork=true to achieve that. Each pod then creates its own IQN with namespace name, so it can't collide with other server pods running in another namespaces on the same node.k3s-v1.15.3
Jan Safranek
parent
a93f803f8e
commit
b18dba3794
|
|
@ -81,6 +81,9 @@ const (
|
|||
// PodCleanupTimeout is a waiting period for pod to be cleaned up and unmount its volumes so we
|
||||
// don't tear down containers with NFS/Ceph/Gluster server too early.
|
||||
PodCleanupTimeout = 20 * time.Second
|
||||
|
||||
// Template for iSCSI IQN.
|
||||
iSCSIIQNTemplate = "iqn.2003-01.io.k8s:e2e.%s"
|
||||
)
|
||||
|
||||
// VolumeTestConfig is a struct for configuration of one tests. The test consist of:
|
||||
|
|
@ -104,6 +107,8 @@ type VolumeTestConfig struct {
|
|||
ServerVolumes map[string]string
|
||||
// Message to wait for before starting clients
|
||||
ServerReadyMessage string
|
||||
// Use HostNetwork for the server
|
||||
ServerHostNetwork bool
|
||||
// Wait for the pod to terminate successfully
|
||||
// False indicates that the pod is long running
|
||||
WaitForCompletion bool
|
||||
|
|
@ -183,20 +188,30 @@ func NewGlusterfsServer(cs clientset.Interface, namespace string) (config Volume
|
|||
}
|
||||
|
||||
// NewISCSIServer is an iSCSI-specific wrapper for CreateStorageServer.
|
||||
func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip string) {
|
||||
func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip, iqn string) {
|
||||
// Generate cluster-wide unique IQN
|
||||
iqn = fmt.Sprintf(iSCSIIQNTemplate, namespace)
|
||||
|
||||
config = VolumeTestConfig{
|
||||
Namespace: namespace,
|
||||
Prefix: "iscsi",
|
||||
ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer),
|
||||
ServerPorts: []int{3260},
|
||||
ServerArgs: []string{iqn},
|
||||
ServerVolumes: map[string]string{
|
||||
// iSCSI container needs to insert modules from the host
|
||||
"/lib/modules": "/lib/modules",
|
||||
// iSCSI container needs to configure kernel
|
||||
"/sys/kernel": "/sys/kernel",
|
||||
// iSCSI source "block devices" must be available on the host
|
||||
"/srv/iscsi": "/srv/iscsi",
|
||||
},
|
||||
ServerReadyMessage: "Configuration restored from /etc/target/saveconfig.json",
|
||||
ServerReadyMessage: "iscsi target started",
|
||||
ServerHostNetwork: true,
|
||||
}
|
||||
pod, ip = CreateStorageServer(cs, config)
|
||||
return config, pod, ip
|
||||
// Make sure the client runs on the same node as server so we don't need to open any firewalls.
|
||||
config.ClientNodeName = pod.Spec.NodeName
|
||||
return config, pod, ip, iqn
|
||||
}
|
||||
|
||||
// NewRBDServer is a CephRBD-specific wrapper for CreateStorageServer.
|
||||
|
|
@ -312,6 +327,7 @@ func StartVolumeServer(client clientset.Interface, config VolumeTestConfig) *v1.
|
|||
},
|
||||
|
||||
Spec: v1.PodSpec{
|
||||
HostNetwork: config.ServerHostNetwork,
|
||||
Containers: []v1.Container{
|
||||
{
|
||||
Name: serverPodName,
|
||||
|
|
|
|||
|
|
@ -330,6 +330,7 @@ type iSCSIVolume struct {
|
|||
serverPod *v1.Pod
|
||||
serverIP string
|
||||
f *framework.Framework
|
||||
iqn string
|
||||
}
|
||||
|
||||
var _ testsuites.TestDriver = &iSCSIDriver{}
|
||||
|
|
@ -374,9 +375,8 @@ func (i *iSCSIDriver) GetVolumeSource(readOnly bool, fsType string, volume tests
|
|||
|
||||
volSource := v1.VolumeSource{
|
||||
ISCSI: &v1.ISCSIVolumeSource{
|
||||
TargetPortal: iv.serverIP + ":3260",
|
||||
// from test/images/volume/iscsi/initiatorname.iscsi
|
||||
IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c",
|
||||
TargetPortal: "127.0.0.1:3260",
|
||||
IQN: iv.iqn,
|
||||
Lun: 0,
|
||||
ReadOnly: readOnly,
|
||||
},
|
||||
|
|
@ -393,8 +393,8 @@ func (i *iSCSIDriver) GetPersistentVolumeSource(readOnly bool, fsType string, vo
|
|||
|
||||
pvSource := v1.PersistentVolumeSource{
|
||||
ISCSI: &v1.ISCSIPersistentVolumeSource{
|
||||
TargetPortal: iv.serverIP + ":3260",
|
||||
IQN: "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c",
|
||||
TargetPortal: "127.0.0.1:3260",
|
||||
IQN: iv.iqn,
|
||||
Lun: 0,
|
||||
ReadOnly: readOnly,
|
||||
},
|
||||
|
|
@ -418,11 +418,13 @@ func (i *iSCSIDriver) CreateVolume(config *testsuites.PerTestConfig, volType tes
|
|||
cs := f.ClientSet
|
||||
ns := f.Namespace
|
||||
|
||||
c, serverPod, serverIP := framework.NewISCSIServer(cs, ns.Name)
|
||||
c, serverPod, serverIP, iqn := framework.NewISCSIServer(cs, ns.Name)
|
||||
config.ServerConfig = &c
|
||||
config.ClientNodeName = c.ClientNodeName
|
||||
return &iSCSIVolume{
|
||||
serverPod: serverPod,
|
||||
serverIP: serverIP,
|
||||
iqn: iqn,
|
||||
f: f,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ func (t *volumesTestSuite) defineTests(driver TestDriver, pattern testpatterns.T
|
|||
init()
|
||||
defer cleanup()
|
||||
|
||||
testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config.ClientNodeSelector)
|
||||
testScriptInPod(f, l.resource.volType, l.resource.volSource, l.config)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -176,7 +176,7 @@ func testScriptInPod(
|
|||
f *framework.Framework,
|
||||
volumeType string,
|
||||
source *v1.VolumeSource,
|
||||
nodeSelector map[string]string) {
|
||||
config *PerTestConfig) {
|
||||
|
||||
const (
|
||||
volPath = "/vol1"
|
||||
|
|
@ -217,7 +217,8 @@ func testScriptInPod(
|
|||
},
|
||||
},
|
||||
RestartPolicy: v1.RestartPolicyNever,
|
||||
NodeSelector: nodeSelector,
|
||||
NodeSelector: config.ClientNodeSelector,
|
||||
NodeName: config.ClientNodeName,
|
||||
},
|
||||
}
|
||||
By(fmt.Sprintf("Creating pod %s", pod.Name))
|
||||
|
|
|
|||
|
|
@ -16,20 +16,8 @@ FROM BASEIMAGE
|
|||
|
||||
CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
|
||||
|
||||
RUN yum install -y iscsi-initiator-utils targetcli net-tools strace procps-ng psmisc && yum clean all
|
||||
ADD run_iscsid.sh /usr/local/bin/
|
||||
ADD initiatorname.iscsi /etc/iscsi/
|
||||
RUN yum install -y targetcli && yum clean all
|
||||
ADD run_iscsi_target.sh /usr/local/bin/
|
||||
ADD block.tar.gz /
|
||||
|
||||
# This JSON file was generated by targetcli with these commands:
|
||||
# /backstores/fileio create block /block
|
||||
# /iscsi create
|
||||
# # Enable demo mode (no authentication!):
|
||||
# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
|
||||
# /iscsi/iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c/tpg1/luns create /backstores/fileio/block
|
||||
# saveconfig
|
||||
ADD saveconfig.json /etc/target/
|
||||
|
||||
EXPOSE 3260/tcp
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/run_iscsid.sh"]
|
||||
ENTRYPOINT ["/usr/local/bin/run_iscsi_target.sh"]
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
1.0
|
||||
2.0
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -40,7 +40,7 @@ mkfs.ext2 block
|
|||
|
||||
# Add index.html to it
|
||||
mount -o loop block $MNTDIR
|
||||
echo "Hello from iSCSI" > $MNTDIR/index.html
|
||||
echo "Hello from iscsi" > $MNTDIR/index.html
|
||||
umount $MNTDIR
|
||||
|
||||
rm block.tar.gz 2>/dev/null || :
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
InitiatorName=iqn.1994-05.com.redhat:eb59fbe2c4c5
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2018 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This script does not run any daemon, it only configures iSCSI target (=server)
|
||||
# in kernel. It is possible to run this script multiple times on a single
|
||||
# node, each run will create its own IQN and LUN.
|
||||
|
||||
# Kubernetes must provide unique name.
|
||||
IQN=$1
|
||||
|
||||
# targetcli synchronizes over dbus, however it does not work in
|
||||
# containers. Use flock instead
|
||||
LOCK=/srv/iscsi/targetcli.lock
|
||||
|
||||
function start()
|
||||
{
|
||||
# targetcli need dbus. It may not run on the host, so start a private one
|
||||
mkdir /run/dbus
|
||||
dbus-daemon --system
|
||||
|
||||
# Create new IQN (iSCSI Qualified Name)
|
||||
flock $LOCK targetcli /iscsi create "$IQN"
|
||||
# Run it in demo mode, i.e. no authentication
|
||||
flock $LOCK targetcli /iscsi/"$IQN"/tpg1 set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1
|
||||
|
||||
# Create unique "block volume" (i.e. flat file) on the *host*.
|
||||
# Having it in the container confuses kernel from some reason
|
||||
# and it's not able to server multiple LUNs from different
|
||||
# containers.
|
||||
# /srv/iscsi must be bind-mount from the host.
|
||||
cp /block /srv/iscsi/"$IQN"
|
||||
|
||||
# Make the block volume available through our IQN as LUN 0
|
||||
flock $LOCK targetcli /backstores/fileio create block-"$IQN" /srv/iscsi/"$IQN"
|
||||
flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns create /backstores/fileio/block-"$IQN"
|
||||
|
||||
echo "iscsi target started"
|
||||
}
|
||||
|
||||
function stop()
|
||||
{
|
||||
echo "stopping iscsi target"
|
||||
# Remove IQN
|
||||
flock $LOCK targetcli /iscsi/"$IQN"/tpg1/luns/ delete 0
|
||||
flock $LOCK targetcli /iscsi delete "$IQN"
|
||||
# Remove block device mapping
|
||||
flock $LOCK targetcli /backstores/fileio delete block-"$IQN"
|
||||
/bin/rm -f /srv/iscsi/"$IQN"
|
||||
echo "iscsi target stopped"
|
||||
exit 0
|
||||
}
|
||||
|
||||
|
||||
trap stop TERM
|
||||
start
|
||||
|
||||
while true; do
|
||||
sleep 1
|
||||
done
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2015 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
function start()
|
||||
{
|
||||
# targetcli need dbus
|
||||
mkdir /run/dbus
|
||||
dbus-daemon --system
|
||||
|
||||
# clear any previous configuration
|
||||
targetcli clearconfig confirm=True
|
||||
|
||||
# restore configuration from saveconfig.json
|
||||
targetcli restoreconfig
|
||||
|
||||
# maximum log level
|
||||
iscsid -f -d 8
|
||||
|
||||
echo "iscsid started"
|
||||
}
|
||||
|
||||
function stop()
|
||||
{
|
||||
echo "Stopping iscsid"
|
||||
killall iscsid
|
||||
targetcli clearconfig confirm=True
|
||||
|
||||
echo "iscsid stopped"
|
||||
exit 0
|
||||
}
|
||||
|
||||
|
||||
trap stop TERM
|
||||
start
|
||||
|
||||
while true; do
|
||||
sleep 5
|
||||
done
|
||||
|
|
@ -1,102 +0,0 @@
|
|||
{
|
||||
"fabric_modules": [],
|
||||
"storage_objects": [
|
||||
{
|
||||
"attributes": {
|
||||
"block_size": 512,
|
||||
"emulate_3pc": 1,
|
||||
"emulate_caw": 1,
|
||||
"emulate_dpo": 0,
|
||||
"emulate_fua_read": 0,
|
||||
"emulate_fua_write": 1,
|
||||
"emulate_model_alias": 1,
|
||||
"emulate_rest_reord": 0,
|
||||
"emulate_tas": 1,
|
||||
"emulate_tpu": 0,
|
||||
"emulate_tpws": 0,
|
||||
"emulate_ua_intlck_ctrl": 0,
|
||||
"emulate_write_cache": 1,
|
||||
"enforce_pr_isids": 1,
|
||||
"force_pr_aptpl": 0,
|
||||
"is_nonrot": 0,
|
||||
"max_unmap_block_desc_count": 1,
|
||||
"max_unmap_lba_count": 8192,
|
||||
"max_write_same_len": 4096,
|
||||
"optimal_sectors": 16384,
|
||||
"pi_prot_format": 0,
|
||||
"pi_prot_type": 0,
|
||||
"queue_depth": 128,
|
||||
"unmap_granularity": 1,
|
||||
"unmap_granularity_alignment": 0
|
||||
},
|
||||
"dev": "block",
|
||||
"name": "block",
|
||||
"plugin": "fileio",
|
||||
"size": 126877696,
|
||||
"write_back": true,
|
||||
"wwn": "521c57aa-9d9b-4e5d-ab1a-527487f92a33"
|
||||
}
|
||||
],
|
||||
"targets": [
|
||||
{
|
||||
"fabric": "iscsi",
|
||||
"tpgs": [
|
||||
{
|
||||
"attributes": {
|
||||
"authentication": 0,
|
||||
"cache_dynamic_acls": 1,
|
||||
"default_cmdsn_depth": 64,
|
||||
"default_erl": 0,
|
||||
"demo_mode_discovery": 1,
|
||||
"demo_mode_write_protect": 0,
|
||||
"generate_node_acls": 1,
|
||||
"login_timeout": 15,
|
||||
"netif_timeout": 2,
|
||||
"prod_mode_write_protect": 0,
|
||||
"t10_pi": 0
|
||||
},
|
||||
"enable": true,
|
||||
"luns": [
|
||||
{
|
||||
"index": 0,
|
||||
"storage_object": "/backstores/fileio/block"
|
||||
}
|
||||
],
|
||||
"node_acls": [],
|
||||
"parameters": {
|
||||
"AuthMethod": "CHAP,None",
|
||||
"DataDigest": "CRC32C,None",
|
||||
"DataPDUInOrder": "Yes",
|
||||
"DataSequenceInOrder": "Yes",
|
||||
"DefaultTime2Retain": "20",
|
||||
"DefaultTime2Wait": "2",
|
||||
"ErrorRecoveryLevel": "0",
|
||||
"FirstBurstLength": "65536",
|
||||
"HeaderDigest": "CRC32C,None",
|
||||
"IFMarkInt": "2048~65535",
|
||||
"IFMarker": "No",
|
||||
"ImmediateData": "Yes",
|
||||
"InitialR2T": "Yes",
|
||||
"MaxBurstLength": "262144",
|
||||
"MaxConnections": "1",
|
||||
"MaxOutstandingR2T": "1",
|
||||
"MaxRecvDataSegmentLength": "8192",
|
||||
"MaxXmitDataSegmentLength": "262144",
|
||||
"OFMarkInt": "2048~65535",
|
||||
"OFMarker": "No",
|
||||
"TargetAlias": "LIO Target"
|
||||
},
|
||||
"portals": [
|
||||
{
|
||||
"ip_address": "0.0.0.0",
|
||||
"iser": false,
|
||||
"port": 3260
|
||||
}
|
||||
],
|
||||
"tag": 1
|
||||
}
|
||||
],
|
||||
"wwn": "iqn.2003-01.org.linux-iscsi.f21.x8664:sn.4b0aae584f7c"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -238,7 +238,7 @@ func initImageConfigs() map[int]Config {
|
|||
configs[ServeHostname] = Config{e2eRegistry, "serve-hostname", "1.1"}
|
||||
configs[TestWebserver] = Config{e2eRegistry, "test-webserver", "1.0"}
|
||||
configs[VolumeNFSServer] = Config{e2eRegistry, "volume/nfs", "1.0"}
|
||||
configs[VolumeISCSIServer] = Config{e2eRegistry, "volume/iscsi", "1.0"}
|
||||
configs[VolumeISCSIServer] = Config{e2eRegistry, "volume/iscsi", "2.0"}
|
||||
configs[VolumeGlusterServer] = Config{e2eRegistry, "volume/gluster", "1.0"}
|
||||
configs[VolumeRBDServer] = Config{e2eRegistry, "volume/rbd", "1.0.1"}
|
||||
return configs
|
||||
|
|
|
|||
Loading…
Reference in New Issue