[sheepdog] [PATCH 1/2] sheep: queue cluster and local requests directly

[Christoph Hellwig]

Now that we can process events and requests in parallel, and events aren't
queued anymore we don't need to queue requests either.

Start with offloading local and cluster requests to the workqueues directly
in this patch and leave the slightly more complex handling of I/O requests
to the next patch.

Signed-off-by: Christoph Hellwig <hch at lst.de>

diff --git a/sheep/group.c b/sheep/group.c
index ea5a69e..e0642b0 100644
--- a/sheep/group.c
+++ b/sheep/group.c
@@ -307,7 +307,7 @@ void sd_block_handler(void)
  * Must run in the main thread as it access unlocked state like
  * sys->pending_list.
  */
-static void queue_cluster_request(struct request *req)
+void queue_cluster_request(struct request *req)
 {
 	eprintf("%p %x\n", req, req->rq.opcode);
 
@@ -910,27 +910,16 @@ void process_request_event_queues(void)
 	list_for_each_entry_safe(req, n, &sys->request_queue, request_list) {
 		list_del(&req->request_list);
 
-		if (is_io_op(req->op)) {
-			list_add_tail(&req->request_list,
-				      &sys->outstanding_req_list);
-
-			if (need_consistency_check(req))
-				set_consistency_check(req);
-
-			if (req->rq.flags & SD_FLAG_CMD_IO_LOCAL)
-				queue_work(sys->io_wqueue, &req->work);
-			else
-				queue_work(sys->gateway_wqueue, &req->work);
-		} else if (is_cluster_op(req->op)) {
-			/*
-			 * Cluster requests are handed off to the cluster driver
-			 * directly from the main thread.  It's the cluster
-			 * drivers job to ensure we avoid blocking on I/O here.
-			 */
-			queue_cluster_request(req);
-		} else { /* is_local_op(req->op) */
+		list_add_tail(&req->request_list,
+			      &sys->outstanding_req_list);
+
+		if (need_consistency_check(req))
+			set_consistency_check(req);
+
+		if (req->rq.flags & SD_FLAG_CMD_IO_LOCAL)
 			queue_work(sys->io_wqueue, &req->work);
-		}
+		else
+			queue_work(sys->gateway_wqueue, &req->work);
 	}
 }
 
diff --git a/sheep/sdnet.c b/sheep/sdnet.c
index 7f501aa..bf27225 100644
--- a/sheep/sdnet.c
+++ b/sheep/sdnet.c
@@ -299,6 +299,31 @@ void resume_wait_obj_requests(uint64_t oid)
 	process_request_event_queues();
 }
 
+static void queue_io_request(struct request *req)
+{
+	if (req->rq.flags & SD_FLAG_CMD_IO_LOCAL) {
+		req->work.fn = do_io_request;
+		req->work.done = io_op_done;
+	} else {
+		req->work.fn = do_gateway_request;
+		req->work.done = io_op_done;
+	}
+
+	setup_access_to_local_objects(req);
+	if (check_request(req) < 0)
+		return;
+
+	list_add_tail(&req->request_list, &sys->request_queue);
+	process_request_event_queues();
+}
+
+static void queue_local_request(struct request *req)
+{
+	req->work.fn = do_local_request;
+	req->work.done = local_op_done;
+	queue_work(sys->io_wqueue, &req->work);
+}
+
 static void queue_request(struct request *req)
 {
 	struct sd_req *hdr = &req->rq;
@@ -356,30 +381,17 @@ static void queue_request(struct request *req)
 		req->vnodes = get_vnode_info();
 
 	if (is_io_op(req->op)) {
-		if (req->rq.flags & SD_FLAG_CMD_IO_LOCAL) {
-			req->work.fn = do_io_request;
-			req->work.done = io_op_done;
-		} else {
-			req->work.fn = do_gateway_request;
-			req->work.done = io_op_done;
-		}
-		setup_access_to_local_objects(req);
-		if (check_request(req) < 0)
-			return;
+		queue_io_request(req);
 	} else if (is_local_op(req->op)) {
-		req->work.fn = do_local_request;
-		req->work.done = local_op_done;
+		queue_local_request(req);
 	} else if (is_cluster_op(req->op)) {
-		;
+		queue_cluster_request(req);
 	} else {
 		eprintf("unknown operation %d\n", hdr->opcode);
 		rsp->result = SD_RES_SYSTEM_ERROR;
 		goto done;
 	}
 
-	list_add_tail(&req->request_list, &sys->request_queue);
-
-	process_request_event_queues();
 	return;
 done:
 	req_done(req);
diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index c31a484..96b6698 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -255,6 +255,7 @@ int create_cluster(int port, int64_t zone, int nr_vnodes);
 int leave_cluster(void);
 
 void process_request_event_queues(void);
+void queue_cluster_request(struct request *req);
 void do_io_request(struct work *work);
 void do_gateway_request(struct work *work);
 int forward_write_obj_req(struct request *req);