[Sheepdog] [PATCH 6/6] stop performing cpg events during io reuqests

[FUJITA Tomonori]

Signed-off-by: FUJITA Tomonori <fujita.tomonori at lab.ntt.co.jp>
---
 collie/collie.h |    3 +++
 collie/group.c  |   12 ++++++++++--
 collie/net.c    |   32 ++++++++++++++++++++++++++++++++
 3 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/collie/collie.h b/collie/collie.h
index 2c21fc0..4f9283a 100644
--- a/collie/collie.h
+++ b/collie/collie.h
@@ -33,6 +33,7 @@ enum cpg_event_type {
 
 enum cpg_event_bits {
 	CPG_EVENT_SKIP = 1,
+	CPG_EVENT_IO,
 };
 
 struct cpg_event {
@@ -100,12 +101,14 @@ struct cluster_info {
 	struct list_head cpg_event_siblings;
 	struct cpg_event *cur_cevent;
 	unsigned long cpg_event_work_flags;
+	int nr_outstanding_io;
 };
 
 extern struct cluster_info *sys;
 
 int create_listen_port(int port, void *data);
 
+int is_io_request(unsigned op);
 int init_store(char *dir);
 
 int add_vdi(char *data, int data_len, uint64_t size,
diff --git a/collie/group.c b/collie/group.c
index 2361a39..35fa1a0 100644
--- a/collie/group.c
+++ b/collie/group.c
@@ -1321,14 +1321,22 @@ void start_cpg_event_work(void)
 
 	list_for_each_entry_safe(cevent, n, &sys->cpg_event_siblings, cpg_event_list) {
 		struct request *req = container_of(cevent, struct request, cev);
+		struct sd_req *hdr = &req->rq;
+
 		if (cevent->ctype != CPG_EVENT_REQUEST)
-			continue;
+			break;
+
+		if (is_io_request(hdr->opcode))
+			sys->nr_outstanding_io++;
+
+		/* TODO: we should check hdr->epoch here. */
 
 		list_del(&cevent->cpg_event_list);
 		queue_work(&req->work);
 	}
 
-	if (cpg_event_suspended() || list_empty(&sys->cpg_event_siblings))
+	if (cpg_event_suspended() || list_empty(&sys->cpg_event_siblings) ||
+	    sys->nr_outstanding_io)
 		return;
 
 	cevent = list_first_entry(&sys->cpg_event_siblings,
diff --git a/collie/net.c b/collie/net.c
index 4cb5f51..36715af 100644
--- a/collie/net.c
+++ b/collie/net.c
@@ -16,6 +16,25 @@
 
 #include "collie.h"
 
+int is_io_request(unsigned op)
+{
+	int ret = 0;
+
+	switch (op) {
+	case SD_OP_CREATE_AND_WRITE_OBJ:
+	case SD_OP_REMOVE_OBJ:
+	case SD_OP_READ_OBJ:
+	case SD_OP_WRITE_OBJ:
+	case SD_OP_SYNC_OBJ:
+		ret = 1;
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
 static void __done(struct work *work, int idx)
 {
 	struct request *req = container_of(work, struct request, work);
@@ -32,6 +51,19 @@ static void __done(struct work *work, int idx)
 		/* request is forwarded to cpg group */
 		return;
 	}
+
+	if (is_io_request(hdr->opcode)) {
+		sys->nr_outstanding_io--;
+		/*
+		 * TODO: if the request failed due to epoch unmatch,
+		 * we should retry here (adds this request to the tail
+		 * of sys->cpg_event_siblings.
+		 */
+		if (!sys->nr_outstanding_io &&
+		    !list_empty(&sys->cpg_event_siblings))
+			start_cpg_event_work();
+	}
+
 	req->done(req);
 }
 
-- 
1.6.5