[sheepdog] [PATCH] tests/functional: add a new test for iSCSI multipath
Hitoshi Mitake
mitake.hitoshi at lab.ntt.co.jp
Tue Aug 19 08:59:55 CEST 2014
At Mon, 18 Aug 2014 15:58:01 +0900,
Hitoshi Mitake wrote:
>
> The new test checks whether the iSCSI multipath functionality of
> sheepdog works collect or not. I excluded it from auto group because
> it heavily depends on distribution specific stuff (e.g. configuration
> file of multipathd and open-iscsi).
>
> Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
> ---
> tests/functional/.gitignore | 6 +
> tests/functional/097 | 157 +++++++++++++++++++++
> tests/functional/097.out | 13 ++
> tests/functional/common.config | 7 +
> tests/functional/group | 1 +
> tests/functional/iscsid.conf | 304 +++++++++++++++++++++++++++++++++++++++++
> 6 files changed, 488 insertions(+)
> create mode 100755 tests/functional/097
> create mode 100644 tests/functional/097.out
> create mode 100644 tests/functional/iscsid.conf
Applied.
Thanks,
Hitoshi
>
> diff --git a/tests/functional/.gitignore b/tests/functional/.gitignore
> index 41bf6d0..1fb73c0 100644
> --- a/tests/functional/.gitignore
> +++ b/tests/functional/.gitignore
> @@ -3,3 +3,9 @@
> *.full
> check.log
> check.time
> +
> +# 097 related files
> +097_orig_devfiles
> +097_login_devfiles
> +097_diff_devfiles
> +multipath.conf
> diff --git a/tests/functional/097 b/tests/functional/097
> new file mode 100755
> index 0000000..ec7e285
> --- /dev/null
> +++ b/tests/functional/097
> @@ -0,0 +1,157 @@
> +#! /bin/bash
> +
> +# Test iSCSI multipath functionality
> +# heavly depends on Debian or Ubuntu conventions
> +
> +. ./common
> +
> +_need_to_be_root
> +
> +which $TGTD > /dev/null || _notrun "Require tgtd but it's not installed"
> +which $TGTADM > /dev/null || _notrun "Require tgtadm but it's not installed"
> +which $ISCSID > /dev/null || _notrun "Require iscsid but it's not installed"
> +which $ISCSIADM > /dev/null || _notrun "Require iscsiadm but it's not installed"
> +which $MULTIPATHD > /dev/null || _notrun "Require multipathd but it's not installed"
> +which $SCSI_ID > /dev/null || _notrun "Require scsi_id but it's not installed"
> +
> +$ISCSIADM -m node --logout &> /dev/null
> +pkill -9 $ISCSID > /dev/null
> +pkill -9 $ISCSIADM > /dev/null
> +pkill -9 $MULTIPATHD > /dev/null
> +pkill -9 $TGTD > /dev/null
> +pkill -9 $TGTADM > /dev/null
> +
> +ORIG_DEVFILES=097_orig_devfiles
> +LOGIN_DEVFILES=097_login_devfiles
> +DIFF_DEVFILES=097_diff_devfiles
> +
> +/bin/ls /dev/sd* > $ORIG_DEVFILES
> +
> +for i in `seq 0 2`; do
> + _start_sheep $i
> +done
> +
> +_wait_for_sheep 3
> +_cluster_format
> +
> +$DOG vdi create test 128M
> +
> +_setup_tgtd()
> +# $1: control port
> +# $2: iscsi portal
> +# $3: VDI name for backing store
> +{
> + $TGTD -t 1 -C $1 --iscsi portal=$2
> + $TGTADM -C $1 --lld iscsi --mode target --op new --tid 1 --targetname iqn.2014-06.org.sheepdog-project
> + $TGTADM -C $1 --mode logicalunit --op new --tid 1 --lun 1 --bstype sheepdog --backing-store $3
> + $TGTADM -C $1 --mode target --op bind --tid 1 --initiator-address ALL
> +}
> +PORTAL1=127.0.0.1:3260
> +PORTAL2=127.0.0.1:3261
> +
> +_setup_tgtd 0 $PORTAL1 unix:$STORE/0/sock:test
> +_setup_tgtd 1 $PORTAL2 unix:$STORE/1/sock:test
> +
> +$DOG vdi lock list
> +
> +$ISCSID -c `pwd`/iscsid.conf
> +
> +$ISCSIADM -m discovery -t sendtargets -p $PORTAL1
> +$ISCSIADM -m discovery -t sendtargets -p $PORTAL2
> +
> +$ISCSIADM -m node --login &> /dev/null
> +sleep 15
> +
> +/bin/ls /dev/sd* > $LOGIN_DEVFILES
> +
> +comm -3 $LOGIN_DEVFILES $ORIG_DEVFILES > $DIFF_DEVFILES
> +
> +if [[ "2 $DIFF_DEVFILES" != `wc -l $DIFF_DEVFILES` ]]
> +then
> + exit 2
> + _notrun "Device files were not created correctly"
> +fi
> +
> +declare -a SCSI_IDS
> +IDX=0
> +for line in `cat $DIFF_DEVFILES`; do
> + SCSI_IDS[$IDX]=`$SCSI_ID --whitelisted --device=$line`
> + IDX=`expr $IDX + 1`
> +done
> +
> +if [ ${SCSI_IDS[0]} != ${SCSI_IDS[1]} ]
> +then
> + _notrun "SCSI IDs are different: ${SCSI_IDS[0]}, ${SCSI_IDS[1]}"
> +fi
> +
> +SCSI_ID=${SCSI_IDS[0]}
> +
> +MULTIPATH_CONF=`pwd`/multipath.conf
> +
> +cat <<EOF > $MULTIPATH_CONF
> +multipaths {
> + multipath {
> + wwid $SCSI_ID
> + alias sheepdog-iscsi
> + path_grouping_policy multibus
> + path_checker readsector0
> + path_selector "round-robin 0"
> + failback manual
> + rr_weight priorities
> + no_path_retry 5
> + }
> +}
> +EOF
> +
> +SAVED_MULTIPATH_CONF=`pwd`/saved_multipath.conf
> +ORIG_MULTIPATH_CONF=/etc/multipath.conf
> +
> +cp $ORIG_MULTIPATH_CONF $SAVED_MULTIPATH_CONF
> +cp $MULTIPATH_CONF $ORIG_MULTIPATH_CONF
> +
> +/etc/init.d/multipath-tools restart
> +sleep 3
> +
> +if [ ! -b /dev/mapper/sheepdog-iscsi ]
> +then
> + _notrun "device file doesn't exist"
> +fi
> +
> +file /dev/mapper/sheepdog-iscsi
> +
> +MNTPOINT=`pwd`/mnt
> +if [ ! -d $MNTPOINT ]
> +then
> + mkdir $MNTPOINT
> +else
> + umount $MNTPOINT 2> /dev/null
> +fi
> +
> +mkfs.ext4 /dev/mapper/sheepdog-iscsi &> /dev/null
> +mount /dev/mapper/sheepdog-iscsi $MNTPOINT
> +
> +TESTFILE=$MNTPOINT/rand
> +dd if=/dev/urandom of=$TESTFILE bs=1M count=64 oflag=direct iflag=fullblock &> /dev/null
> +
> +BEFORE=`md5sum $TESTFILE`
> +
> +sync
> +echo 1 > /proc/sys/vm/drop_caches # drop page caches
> +
> +# kill the default tgtd with control port 0
> +for pid in `pgrep -f "tgtd -C 0"`; do
> + kill -9 $pid 2> /dev/null
> +done
> +sleep 20 # wait fallback
> +AFTER=`md5sum $TESTFILE`
> +
> +if [[ $BEFORE = $AFTER ]]
> +then
> + echo "checksum matched, success"
> +else
> + echo "checksum not matched, fail"
> +fi
> +
> +umount mnt
> +
> +cp $SAVED_MULTIPATH_CONF $ORIG_MULTIPATH_CONF
> diff --git a/tests/functional/097.out b/tests/functional/097.out
> new file mode 100644
> index 0000000..e42dda8
> --- /dev/null
> +++ b/tests/functional/097.out
> @@ -0,0 +1,13 @@
> +QA output created by 097
> +using backend plain store
> +VDI | owner node
> +test | IPv4 ip:127.0.0.1 port:7000(shared) IPv4 ip:127.0.0.1 port:7001(shared)
> +Try `iscsiadm --help' for more information.
> +127.0.0.1:3260,1 iqn.2014-06.org.sheepdog-project
> +127.0.0.1:3261,1 iqn.2014-06.org.sheepdog-project
> + * Stopping multipath daemon multipathd
> + ...done.
> + * Starting multipath daemon multipathd
> + ...done.
> +/dev/mapper/sheepdog-iscsi: block special
> +checksum matched, success
> diff --git a/tests/functional/common.config b/tests/functional/common.config
> index 87c5583..5fd9534 100644
> --- a/tests/functional/common.config
> +++ b/tests/functional/common.config
> @@ -88,5 +88,12 @@ export QEMU_IMG=${QEMU_IMG_PROG:-qemu-img}
> export SHEEPFS=${SHEEPFS:-../../sheepfs/sheepfs}
> export SOURCE=${SOURCE:-../..}
>
> +export TGTD=${TGTD_PROG:-tgtd}
> +export TGTADM=${TGTADM_PROG:-tgtadm}
> +export ISCSID=${ISCSID_PROG:-iscsiadm}
> +export ISCSIADM=${ISCSIADM_PROG:-iscsiadm}
> +export SCSI_ID=${SCSI_ID_PROG:-/lib/udev/scsi_id}
> +export MULTIPATHD=${MULTIPATHD_PROG:-multipathd}
> +
> # make sure this script returns success
> /bin/true
> diff --git a/tests/functional/group b/tests/functional/group
> index 5cd02f6..3c49815 100644
> --- a/tests/functional/group
> +++ b/tests/functional/group
> @@ -111,3 +111,4 @@
> 094 auto quick store md
> 095 auto quick store md
> 096 auto quick cluster
> +097 vdi cluster
> diff --git a/tests/functional/iscsid.conf b/tests/functional/iscsid.conf
> new file mode 100644
> index 0000000..f0d38b4
> --- /dev/null
> +++ b/tests/functional/iscsid.conf
> @@ -0,0 +1,304 @@
> +#
> +# Open-iSCSI default configuration.
> +# Could be located at /etc/iscsi/iscsid.conf or ~/.iscsid.conf
> +#
> +# Note: To set any of these values for a specific node/session run
> +# the iscsiadm --mode node --op command for the value. See the README
> +# and man page for iscsiadm for details on the --op command.
> +#
> +
> +######################
> +# iscsid daemon config
> +######################
> +# If you want iscsid to start the first time a iscsi tool
> +# needs to access it, instead of starting it when the init
> +# scripts run, set the iscsid startup command here. This
> +# should normally only need to be done by distro package
> +# maintainers.
> +#
> +# Default for Fedora and RHEL. (uncomment to activate).
> +# iscsid.startup = /etc/rc.d/init.d/iscsid force-start
> +#
> +# Default for upstream open-iscsi scripts (uncomment to activate).
> +iscsid.startup = /usr/sbin/iscsid
> +
> +
> +#############################
> +# NIC/HBA and driver settings
> +#############################
> +# open-iscsi can create a session and bind it to a NIC/HBA.
> +# To set this up see the example iface config file.
> +
> +#*****************
> +# Startup settings
> +#*****************
> +
> +# To request that the iscsi initd scripts startup a session set to "automatic".
> +# node.startup = automatic
> +#
> +# To manually startup the session set to "manual". The default is manual.
> +node.startup = manual
> +
> +# For "automatic" startup nodes, setting this to "Yes" will try logins on each
> +# available iface until one succeeds, and then stop. The default "No" will try
> +# logins on all availble ifaces simultaneously.
> +node.leading_login = No
> +
> +# *************
> +# CHAP Settings
> +# *************
> +
> +# To enable CHAP authentication set node.session.auth.authmethod
> +# to CHAP. The default is None.
> +#node.session.auth.authmethod = CHAP
> +
> +# To set a CHAP username and password for initiator
> +# authentication by the target(s), uncomment the following lines:
> +#node.session.auth.username = username
> +#node.session.auth.password = password
> +
> +# To set a CHAP username and password for target(s)
> +# authentication by the initiator, uncomment the following lines:
> +#node.session.auth.username_in = username_in
> +#node.session.auth.password_in = password_in
> +
> +# To enable CHAP authentication for a discovery session to the target
> +# set discovery.sendtargets.auth.authmethod to CHAP. The default is None.
> +#discovery.sendtargets.auth.authmethod = CHAP
> +
> +# To set a discovery session CHAP username and password for the initiator
> +# authentication by the target(s), uncomment the following lines:
> +#discovery.sendtargets.auth.username = username
> +#discovery.sendtargets.auth.password = password
> +
> +# To set a discovery session CHAP username and password for target(s)
> +# authentication by the initiator, uncomment the following lines:
> +#discovery.sendtargets.auth.username_in = username_in
> +#discovery.sendtargets.auth.password_in = password_in
> +
> +# ********
> +# Timeouts
> +# ********
> +#
> +# See the iSCSI REAME's Advanced Configuration section for tips
> +# on setting timeouts when using multipath or doing root over iSCSI.
> +#
> +# To specify the length of time to wait for session re-establishment
> +# before failing SCSI commands back to the application when running
> +# the Linux SCSI Layer error handler, edit the line.
> +# The value is in seconds and the default is 120 seconds.
> +# Special values:
> +# - If the value is 0, IO will be failed immediately.
> +# - If the value is less than 0, IO will remain queued until the session
> +# is logged back in, or until the user runs the logout command.
> +node.session.timeo.replacement_timeout = 10
> +
> +# To specify the time to wait for login to complete, edit the line.
> +# The value is in seconds and the default is 15 seconds.
> +node.conn[0].timeo.login_timeout = 15
> +
> +# To specify the time to wait for logout to complete, edit the line.
> +# The value is in seconds and the default is 15 seconds.
> +node.conn[0].timeo.logout_timeout = 15
> +
> +# Time interval to wait for on connection before sending a ping.
> +node.conn[0].timeo.noop_out_interval = 5
> +
> +# To specify the time to wait for a Nop-out response before failing
> +# the connection, edit this line. Failing the connection will
> +# cause IO to be failed back to the SCSI layer. If using dm-multipath
> +# this will cause the IO to be failed to the multipath layer.
> +node.conn[0].timeo.noop_out_timeout = 5
> +
> +# To specify the time to wait for abort response before
> +# failing the operation and trying a logical unit reset edit the line.
> +# The value is in seconds and the default is 15 seconds.
> +node.session.err_timeo.abort_timeout = 10
> +
> +# To specify the time to wait for a logical unit response
> +# before failing the operation and trying session re-establishment
> +# edit the line.
> +# The value is in seconds and the default is 30 seconds.
> +node.session.err_timeo.lu_reset_timeout = 10
> +
> +# To specify the time to wait for a target response
> +# before failing the operation and trying session re-establishment
> +# edit the line.
> +# The value is in seconds and the default is 30 seconds.
> +node.session.err_timeo.tgt_reset_timeout = 10
> +
> +
> +#******
> +# Retry
> +#******
> +
> +# To specify the number of times iscsid should retry a login
> +# if the login attempt fails due to the node.conn[0].timeo.login_timeout
> +# expiring modify the following line. Note that if the login fails
> +# quickly (before node.conn[0].timeo.login_timeout fires) because the network
> +# layer or the target returns an error, iscsid may retry the login more than
> +# node.session.initial_login_retry_max times.
> +#
> +# This retry count along with node.conn[0].timeo.login_timeout
> +# determines the maximum amount of time iscsid will try to
> +# establish the initial login. node.session.initial_login_retry_max is
> +# multiplied by the node.conn[0].timeo.login_timeout to determine the
> +# maximum amount.
> +#
> +# The default node.session.initial_login_retry_max is 8 and
> +# node.conn[0].timeo.login_timeout is 15 so we have:
> +#
> +# node.conn[0].timeo.login_timeout * node.session.initial_login_retry_max =
> +# 120 seconds
> +#
> +# Valid values are any integer value. This only
> +# affects the initial login. Setting it to a high value can slow
> +# down the iscsi service startup. Setting it to a low value can
> +# cause a session to not get logged into, if there are distuptions
> +# during startup or if the network is not ready at that time.
> +node.session.initial_login_retry_max = 8
> +
> +################################
> +# session and device queue depth
> +################################
> +
> +# To control how many commands the session will queue set
> +# node.session.cmds_max to an integer between 2 and 2048 that is also
> +# a power of 2. The default is 128.
> +node.session.cmds_max = 128
> +
> +# To control the device's queue depth set node.session.queue_depth
> +# to a value between 1 and 1024. The default is 32.
> +node.session.queue_depth = 32
> +
> +##################################
> +# MISC SYSTEM PERFORMANCE SETTINGS
> +##################################
> +
> +# For software iscsi (iscsi_tcp) and iser (ib_iser) each session
> +# has a thread used to transmit or queue data to the hardware. For
> +# cxgb3i you will get a thread per host.
> +#
> +# Setting the thread's priority to a lower value can lead to higher throughput
> +# and lower latencies. The lowest value is -20. Setting the priority to
> +# a higher value, can lead to reduced IO performance, but if you are seeing
> +# the iscsi or scsi threads dominate the use of the CPU then you may want
> +# to set this value higher.
> +#
> +# Note: For cxgb3i you must set all sessions to the same value, or the
> +# behavior is not defined.
> +#
> +# The default value is -20. The setting must be between -20 and 20.
> +node.session.xmit_thread_priority = -20
> +
> +
> +#***************
> +# iSCSI settings
> +#***************
> +
> +# To enable R2T flow control (i.e., the initiator must wait for an R2T
> +# command before sending any data), uncomment the following line:
> +#
> +#node.session.iscsi.InitialR2T = Yes
> +#
> +# To disable R2T flow control (i.e., the initiator has an implied
> +# initial R2T of "FirstBurstLength" at offset 0), uncomment the following line:
> +#
> +# The defaults is No.
> +node.session.iscsi.InitialR2T = No
> +
> +#
> +# To disable immediate data (i.e., the initiator does not send
> +# unsolicited data with the iSCSI command PDU), uncomment the following line:
> +#
> +#node.session.iscsi.ImmediateData = No
> +#
> +# To enable immediate data (i.e., the initiator sends unsolicited data
> +# with the iSCSI command packet), uncomment the following line:
> +#
> +# The default is Yes
> +node.session.iscsi.ImmediateData = Yes
> +
> +# To specify the maximum number of unsolicited data bytes the initiator
> +# can send in an iSCSI PDU to a target, edit the following line.
> +#
> +# The value is the number of bytes in the range of 512 to (2^24-1) and
> +# the default is 262144
> +node.session.iscsi.FirstBurstLength = 262144
> +
> +# To specify the maximum SCSI payload that the initiator will negotiate
> +# with the target for, edit the following line.
> +#
> +# The value is the number of bytes in the range of 512 to (2^24-1) and
> +# the defauls it 16776192
> +node.session.iscsi.MaxBurstLength = 16776192
> +
> +# To specify the maximum number of data bytes the initiator can receive
> +# in an iSCSI PDU from a target, edit the following line.
> +#
> +# The value is the number of bytes in the range of 512 to (2^24-1) and
> +# the default is 262144
> +node.conn[0].iscsi.MaxRecvDataSegmentLength = 262144
> +
> +# To specify the maximum number of data bytes the initiator will send
> +# in an iSCSI PDU to the target, edit the following line.
> +#
> +# The value is the number of bytes in the range of 512 to (2^24-1).
> +# Zero is a special case. If set to zero, the initiator will use
> +# the target's MaxRecvDataSegmentLength for the MaxXmitDataSegmentLength.
> +# The default is 0.
> +node.conn[0].iscsi.MaxXmitDataSegmentLength = 0
> +
> +# To specify the maximum number of data bytes the initiator can receive
> +# in an iSCSI PDU from a target during a discovery session, edit the
> +# following line.
> +#
> +# The value is the number of bytes in the range of 512 to (2^24-1) and
> +# the default is 32768
> +#
> +discovery.sendtargets.iscsi.MaxRecvDataSegmentLength = 32768
> +
> +# To allow the targets to control the setting of the digest checking,
> +# with the initiator requesting a preference of enabling the checking, uncomment# one or both of the following lines:
> +#node.conn[0].iscsi.HeaderDigest = CRC32C,None
> +#node.conn[0].iscsi.DataDigest = CRC32C,None
> +#
> +# To allow the targets to control the setting of the digest checking,
> +# with the initiator requesting a preference of disabling the checking,
> +# uncomment one or both of the following lines:
> +#node.conn[0].iscsi.HeaderDigest = None,CRC32C
> +#node.conn[0].iscsi.DataDigest = None,CRC32C
> +#
> +# To enable CRC32C digest checking for the header and/or data part of
> +# iSCSI PDUs, uncomment one or both of the following lines:
> +#node.conn[0].iscsi.HeaderDigest = CRC32C
> +#node.conn[0].iscsi.DataDigest = CRC32C
> +#
> +# To disable digest checking for the header and/or data part of
> +# iSCSI PDUs, uncomment one or both of the following lines:
> +#node.conn[0].iscsi.HeaderDigest = None
> +#node.conn[0].iscsi.DataDigest = None
> +#
> +# The default is to never use DataDigests or HeaderDigests.
> +#
> +
> +# For multipath configurations, you may want more than one session to be
> +# created on each iface record. If node.session.nr_sessions is greater
> +# than 1, performing a 'login' for that node will ensure that the
> +# appropriate number of sessions is created.
> +node.session.nr_sessions = 1
> +
> +#************
> +# Workarounds
> +#************
> +
> +# Some targets like IET prefer after an initiator has sent a task
> +# management function like an ABORT TASK or LOGICAL UNIT RESET, that
> +# it does not respond to PDUs like R2Ts. To enable this behavior uncomment
> +# the following line (The default behavior is Yes):
> +node.session.iscsi.FastAbort = Yes
> +
> +# Some targets like Equalogic prefer that after an initiator has sent
> +# a task management function like an ABORT TASK or LOGICAL UNIT RESET, that
> +# it continue to respond to R2Ts. To enable this uncomment this line
> +# node.session.iscsi.FastAbort = No
> --
> 1.8.3.2
>
More information about the sheepdog
mailing list