[sheepdog] [PATCH v2] sheep: use queue_request when resume waiting requests

[levin li]

From: levin li <xingke.lwp at taobao.com>

When a waiting requests due to epoch inconsistency or object in recovery
is resumed, we should check it again to determine whether the current
condition can fulfill its need to make it run, so we call queue_request
to reinitialize and check the request again.

Signed-off-by: levin li <xingke.lwp at taobao.com>
---
 sheep/sdnet.c |   77 ++++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 49 insertions(+), 28 deletions(-)

diff --git a/sheep/sdnet.c b/sheep/sdnet.c
index dc39de5..79f074e 100644
--- a/sheep/sdnet.c
+++ b/sheep/sdnet.c
@@ -20,6 +20,7 @@
 
 #include "sheep_priv.h"
 
+static void queue_request(struct request *req);
 
 static int is_access_local(struct request *req, uint64_t oid)
 {
@@ -283,6 +284,18 @@ static int check_request(struct request *req)
 	return 0;
 }
 
+static void requeue_request(struct request *req, int enable)
+{
+	list_del(&req->request_list);
+
+	if (enable)
+		process_io_request(req);
+	else {
+		put_vnode_info(req->vnodes);
+		queue_request(req);
+	}
+}
+
 void resume_pending_requests(void)
 {
 	struct request *req, *n;
@@ -290,21 +303,18 @@ void resume_pending_requests(void)
 
 	list_splice_init(&sys->req_wait_for_obj_list, &pending_list);
 
-	list_for_each_entry_safe(req, n, &pending_list, request_list) {
-		list_del(&req->request_list);
-
-		if (check_request(req) < 0)
-			continue;
-		process_io_request(req);
-	}
+	list_for_each_entry_safe(req, n, &pending_list, request_list)
+		requeue_request(req, 0);
 }
 
 void resume_wait_epoch_requests(void)
 {
 	struct request *req, *t;
+	LIST_HEAD(pending_list);
 
-	list_for_each_entry_safe(req, t, &sys->wait_rw_queue,
-				 request_list) {
+	list_splice_init(&sys->wait_rw_queue, &pending_list);
+
+	list_for_each_entry_safe(req, t, &pending_list, request_list) {
 		switch (req->rp.result) {
 		/* gateway retries to send the request when
 		   its epoch changes. */
@@ -313,55 +323,66 @@ void resume_wait_epoch_requests(void)
 			put_vnode_info(req->vnodes);
 			req->vnodes = get_vnode_info();
 			setup_access_to_local_objects(req);
+			requeue_request(req, 1);
+			break;
 		/* peer retries the request locally when its epoch changes. */
 		case SD_RES_NEW_NODE_VER:
-			list_del(&req->request_list);
-			process_io_request(req);
+			requeue_request(req, 0);
 			break;
 		default:
 			break;
 		}
 	}
+
+	list_splice_init(&pending_list, &sys->wait_rw_queue);
 }
 
 void resume_wait_recovery_requests(void)
 {
 	struct request *req, *t;
+	LIST_HEAD(pending_list);
+
+	list_splice_init(&sys->wait_rw_queue, &pending_list);
+
+	list_for_each_entry_safe(req, t, &pending_list, request_list) {
+		if (req->rp.result != SD_RES_OBJ_RECOVERING)
+			continue;
 
-	list_for_each_entry_safe(req, t, &sys->wait_rw_queue,
-				 request_list) {
 		dprintf("resume wait oid %" PRIx64 "\n", req->local_oid);
-		if (req->rp.result == SD_RES_OBJ_RECOVERING) {
-			list_del(&req->request_list);
-			process_io_request(req);
-		}
+		requeue_request(req, 0);
 	}
+
+	list_splice_init(&pending_list, &sys->wait_rw_queue);
 }
 
 void resume_wait_obj_requests(uint64_t oid)
 {
 	struct request *req, *t;
+	LIST_HEAD(pending_list);
+
+	list_splice_init(&sys->wait_obj_queue, &pending_list);
+
+	list_for_each_entry_safe(req, t, &pending_list, request_list) {
+		if (req->local_oid != oid)
+			continue;
 
-	list_for_each_entry_safe(req, t, &sys->wait_obj_queue,
-			request_list) {
 		/* the object requested by a pending request has been
 		 * recovered, notify the pending request. */
-		if (req->local_oid == oid) {
-			dprintf("retry %" PRIx64 "\n", req->local_oid);
-			list_del(&req->request_list);
-			process_io_request(req);
-		}
+		dprintf("retry %" PRIx64 "\n", req->local_oid);
+		requeue_request(req, 0);
 	}
+	list_splice_init(&pending_list, &sys->wait_obj_queue);
 }
 
 void flush_wait_obj_requests(void)
 {
 	struct request *req, *n;
+	LIST_HEAD(pending_list);
 
-	list_for_each_entry_safe(req, n, &sys->wait_obj_queue, request_list) {
-		list_del(&req->request_list);
-		process_io_request(req);
-	}
+	list_splice_init(&sys->wait_obj_queue, &pending_list);
+
+	list_for_each_entry_safe(req, n, &pending_list, request_list)
+		requeue_request(req, 0);
 }
 
 static void queue_io_request(struct request *req)
-- 
1.7.10