[sheepdog] [PATCH 1/2] sheep: use work queue for asynchronous local request

[MORITA Kazutaka]

From: MORITA Kazutaka <morita.kazutaka at lab.ntt.co.jp>

We can use our work queue framework for handling asynchrouns requests.
There is no need to create short living events.

Signed-off-by: MORITA Kazutaka <morita.kazutaka at lab.ntt.co.jp>
---
 sheep/request.c    | 73 +++++++++++++++++++++++++++++-------------------------
 sheep/sheep.c      |  4 ++-
 sheep/sheep_priv.h |  2 +-
 3 files changed, 43 insertions(+), 36 deletions(-)

diff --git a/sheep/request.c b/sheep/request.c
index 161b7db..a15fd8c 100644
--- a/sheep/request.c
+++ b/sheep/request.c
@@ -575,31 +575,11 @@ out:
 	return ret;
 }
 
-static void local_req_async_handler(int fd, int events, void *data)
-{
-	struct request *req = data;
-	struct request_iocb *iocb = req->iocb;
-
-	if (events & EPOLLERR)
-		sd_err("request handler error");
-	eventfd_xread(fd);
-
-	if (unlikely(req->rp.result != SD_RES_SUCCESS))
-		iocb->result = req->rp.result;
-
-	if (uatomic_sub_return(&iocb->count, 1) == 0)
-		eventfd_xwrite(iocb->efd, 1);
-
-	unregister_event(req->local_req_efd);
-	close(req->local_req_efd);
-	free_local_request(req);
-}
-
 worker_fn struct request_iocb *local_req_init(void)
 {
 	struct request_iocb *iocb = xzalloc(sizeof(*iocb));
 
-	iocb->efd = eventfd(0, 0);
+	iocb->efd = eventfd(0, EFD_SEMAPHORE);
 	if (iocb->efd < 0) {
 		sd_err("eventfd failed, %m");
 		free(iocb);
@@ -613,7 +593,8 @@ worker_fn int local_req_wait(struct request_iocb *iocb)
 {
 	int ret;
 
-	eventfd_xread(iocb->efd);
+	for (int i = 0; i < iocb->count; i++)
+		eventfd_xread(iocb->efd);
 
 	ret = iocb->result;
 	close(iocb->efd);
@@ -621,23 +602,47 @@ worker_fn int local_req_wait(struct request_iocb *iocb)
 	return ret;
 }
 
+struct areq_work {
+	struct sd_req rq;
+	void *data;
+	struct request_iocb *iocb;
+	int result;
+
+	struct work work;
+};
+
+static void local_req_async_work(struct work *work)
+{
+	struct areq_work *areq = container_of(work, struct areq_work, work);
+
+	areq->result = exec_local_req(&areq->rq, areq->data);
+}
+
+static void local_req_async_main(struct work *work)
+{
+	struct areq_work *areq = container_of(work, struct areq_work, work);
+
+	if (unlikely(areq->result != SD_RES_SUCCESS))
+		areq->iocb->result = areq->result;
+
+	eventfd_xwrite(areq->iocb->efd, 1);
+}
+
 worker_fn int exec_local_req_async(struct sd_req *rq, void *data,
 				   struct request_iocb *iocb)
 {
-	struct request *req;
+	struct areq_work *areq;
 
-	req = alloc_local_request(data, rq->data_length);
-	req->rq = *rq;
-	req->local_req_efd = eventfd(0, EFD_NONBLOCK);
-	if (req->local_req_efd < 0) {
-		sd_err("eventfd failed, %m");
-		return SD_RES_SYSTEM_ERROR;
-	}
+	areq = xzalloc(sizeof(*areq));
+	areq->rq = *rq;
+	areq->data = data;
+	areq->iocb = iocb;
+	areq->work.fn = local_req_async_work;
+	areq->work.done = local_req_async_main;
 
-	uatomic_inc(&iocb->count);
-	req->iocb = iocb;
-	register_event(req->local_req_efd, local_req_async_handler, req);
-	submit_local_request(req);
+	queue_work(sys->areq_wqueue, &areq->work);
+
+	iocb->count++;
 
 	return SD_RES_SUCCESS;
 }
diff --git a/sheep/sheep.c b/sheep/sheep.c
index a550f22..c6733b9 100644
--- a/sheep/sheep.c
+++ b/sheep/sheep.c
@@ -436,6 +436,7 @@ static int create_work_queues(void)
 	sys->deletion_wqueue = create_ordered_work_queue("deletion");
 	sys->block_wqueue = create_ordered_work_queue("block");
 	sys->md_wqueue = create_ordered_work_queue("md");
+	sys->areq_wqueue = create_work_queue("async_req", WQ_UNLIMITED);
 	if (sys->enable_object_cache) {
 		sys->oc_reclaim_wqueue =
 			create_ordered_work_queue("oc_reclaim");
@@ -444,7 +445,8 @@ static int create_work_queues(void)
 			return -1;
 	}
 	if (!sys->gateway_wqueue || !sys->io_wqueue || !sys->recovery_wqueue ||
-	    !sys->deletion_wqueue || !sys->block_wqueue || !sys->md_wqueue)
+	    !sys->deletion_wqueue || !sys->block_wqueue || !sys->md_wqueue ||
+	    !sys->areq_wqueue)
 			return -1;
 
 	return 0;
diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index c0a138c..e8d688b 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -106,7 +106,6 @@ struct request {
 	refcnt_t refcnt;
 	bool local;
 	int local_req_efd;
-	struct request_iocb *iocb;
 
 	uint64_t local_oid;
 
@@ -149,6 +148,7 @@ struct system_info {
 	struct work_queue *oc_reclaim_wqueue;
 	struct work_queue *oc_push_wqueue;
 	struct work_queue *md_wqueue;
+	struct work_queue *areq_wqueue;
 #ifdef HAVE_HTTP
 	struct work_queue *http_wqueue;
 #endif
-- 
1.8.1.2