[sheepdog] [PATCH 05/14] tests: wait for sheep to be stopped

[MORITA Kazutaka]

This patch ensures that the next sheep can start successfully by
checking that its port is not in LISTEN state.  We can use this, e.g.,
after 'collie cluster shutdown'.

Signed-off-by: MORITA Kazutaka <morita.kazutaka at lab.ntt.co.jp>
---
 tests/001       |  3 +++
 tests/002       |  3 +++
 tests/003       |  3 +++
 tests/004       |  5 +++++
 tests/005       |  6 +++++-
 tests/007       |  2 +-
 tests/031       |  2 +-
 tests/049       |  2 +-
 tests/052       |  2 +-
 tests/053       |  2 +-
 tests/common.rc | 31 +++++++++++++++++++++++++------
 11 files changed, 49 insertions(+), 12 deletions(-)

diff --git a/tests/001 b/tests/001
index 40ef849..38d0295 100755
--- a/tests/001
+++ b/tests/001
@@ -44,6 +44,9 @@ for i in 0 1 2; do
     sleep 1
 done
 
+_wait_for_sheep_stop 0
+_wait_for_sheep_stop 1
+
 _wait_for_sheep 1 2
 echo check whether Sheepdog is running with only one node
 $COLLIE cluster info -p 7002 | _filter_cluster_info
diff --git a/tests/002 b/tests/002
index c8ab2b6..ac9f16e 100755
--- a/tests/002
+++ b/tests/002
@@ -37,6 +37,9 @@ for i in 1 0 2; do
     sleep 1
 done
 
+_wait_for_sheep_stop 0
+_wait_for_sheep_stop 1
+
 echo check whether Sheepdog is running with only one node
 $COLLIE cluster info -p 7002 | _filter_cluster_info
 
diff --git a/tests/003 b/tests/003
index b45e2ed..38ede2a 100755
--- a/tests/003
+++ b/tests/003
@@ -36,6 +36,9 @@ for i in 0 1 2; do
     sleep 1
 done
 
+_wait_for_sheep_stop 0
+_wait_for_sheep_stop 1
+
 echo check whether Sheepdog is running with only one node
 $COLLIE cluster info -p 7002 | _filter_cluster_info
 
diff --git a/tests/004 b/tests/004
index 6c1c95b..6a11a56 100755
--- a/tests/004
+++ b/tests/004
@@ -43,6 +43,11 @@ for i in 0 1 2 3 4; do
     sleep 1
 done
 
+_wait_for_sheep_stop 0
+_wait_for_sheep_stop 1
+_wait_for_sheep_stop 2
+_wait_for_sheep_stop 3
+
 echo check whether Sheepdog is running with only one node
 $COLLIE cluster info -p 7004 | _filter_cluster_info
 
diff --git a/tests/005 b/tests/005
index 9a6980f..3ed013d 100755
--- a/tests/005
+++ b/tests/005
@@ -34,13 +34,17 @@ done
 _wait_for_sheep 2 4
 
 $COLLIE cluster shutdown -p 7004
-sleep 2
+_wait_for_sheep_stop
 
 for i in 0 1 2 3 4; do
     _start_sheep $i
     sleep 1
 done
 
+_wait_for_sheep_stop 0
+_wait_for_sheep_stop 1
+_wait_for_sheep_stop 2
+
 _wait_for_sheep 2 4
 
 echo check whether Sheepdog is working with two nodes
diff --git a/tests/007 b/tests/007
index 8253d20..2af9254 100755
--- a/tests/007
+++ b/tests/007
@@ -21,7 +21,7 @@ _start_sheep 1
 _wait_for_sheep 1 1
 $COLLIE cluster format -p 7001 -c 1
 $COLLIE cluster shutdown -p 7001
-sleep 2
+_wait_for_sheep_stop
 
 # start Sheepdog with one node
 _start_sheep 0
diff --git a/tests/031 b/tests/031
index 0f3f550..61fde5b 100755
--- a/tests/031
+++ b/tests/031
@@ -27,7 +27,7 @@ done
 $COLLIE vdi list | _filter_short_date
 
 $COLLIE cluster shutdown
-sleep 2
+_wait_for_sheep_stop
 
 _start_sheep 0
 _wait_for_sheep 1
diff --git a/tests/049 b/tests/049
index 184b3a6..1336b5f 100755
--- a/tests/049
+++ b/tests/049
@@ -28,7 +28,7 @@ $COLLIE vdi create test 20M
 dd if=/dev/urandom | $COLLIE vdi write -w test
 
 $COLLIE cluster shutdown
-sleep 2
+_wait_for_sheep_stop
 
 #trigger an object reclaim at startup
 for i in `seq 0 2`; do
diff --git a/tests/052 b/tests/052
index 7e41ca6..77c2b09 100755
--- a/tests/052
+++ b/tests/052
@@ -25,7 +25,7 @@ $COLLIE vdi create test 20M -P
 $COLLIE cluster info | _filter_cluster_info
 
 $COLLIE cluster shutdown
-sleep 1
+_wait_for_sheep_stop
 
 for i in 0 1 2 4; do
     _start_sheep $i
diff --git a/tests/053 b/tests/053
index 128170f..d9c1d71 100755
--- a/tests/053
+++ b/tests/053
@@ -29,7 +29,7 @@ for i in 0 1 2 3; do
     $COLLIE cluster info -p 700$i | _filter_cluster_info
 done
 $COLLIE cluster shutdown
-sleep 1
+_wait_for_sheep_stop
 
 _start_sheep 0 -g
 _start_sheep 1 -g
diff --git a/tests/common.rc b/tests/common.rc
index 7d34bdb..e8adf5d 100644
--- a/tests/common.rc
+++ b/tests/common.rc
@@ -154,6 +154,29 @@ _count_sheep_processes()
     pgrep -f "$SHEEP_PROG $STORE/" -l | awk '{ $1=""; print }' | sort | uniq | wc -l
 }
 
+# Wait for the specified sheep to be stopped.  If no argument is
+# passed, this waits for all the sheeps to be stopped
+_wait_for_sheep_stop()
+{
+    local cnt
+    for cnt in `seq 60`; do # wait at most 60 seconds
+	# make sure that the sheep port is available
+	if [ "$1" == "" ]; then
+	    netstat -an | grep tcp | grep :70[0-9][0-9] | \
+		grep LISTEN > /dev/null 2>&1
+	else
+	    netstat -an | grep tcp | grep :$((7000 + $1)) | \
+		grep LISTEN > /dev/null 2>&1
+	fi
+	if [ $? != 0 ]; then
+	    return
+	fi
+	sleep 1
+    done
+
+    _die "sheep $1 is expected to be stopped, but not."
+}
+
 # wait for all sheep to join completely
 _wait_for_sheep()
 {
@@ -259,9 +282,7 @@ _kill_all_sheeps()
 {
     pkill -9 -f "$SHEEP_PROG $STORE"
 
-    while [ $? == 0 ]; do
-        pgrep -f "$SHEEP_PROG $STORE" > /dev/null
-    done
+    _wait_for_sheep_stop
 }
 
 _kill_sheep()
@@ -272,9 +293,7 @@ _kill_sheep()
         _die "cannot kill sheep $1"
     fi
 
-    while [ $? == 0 ]; do
-	pgrep -f "$SHEEP_PROG $STORE/$1" > /dev/null
-    done
+    _wait_for_sheep_stop $1
 
     if [[ "$DRIVER" == zoo* ]];then
         sleep 3 # wait for zookeeper ephemeral node to be deleted
-- 
1.8.1.3.566.gaa39828