[sheepdog] [PATCH v3 1/4] sheep: change gateway_forward_request() for forwarding requests to all other nodes

[Hitoshi Mitake]

8<---
v3: add new helper function init_target_nodes() for reducing unnecessary
conditional branches
8<---

This patch adds a new parameter all_node to
gateway_forward_request(). When this parameter is 1,
gateway_forward_request() forwards a request to all other nodes
instead of vnodes which store the object.

Cc: MORITA Kazutaka <morita.kazutaka at lab.ntt.co.jp>
Cc: Liu Yuan <tailai.ly at taobao.com>
Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
---
 include/sheep.h    |   13 +++++++++++
 sheep/gateway.c    |   62 +++++++++++++++++++++++++++++++++++----------------
 sheep/sheep_priv.h |    5 ++++
 3 files changed, 60 insertions(+), 20 deletions(-)

diff --git a/include/sheep.h b/include/sheep.h
index a2d0cd9..03a2090 100644
--- a/include/sheep.h
+++ b/include/sheep.h
@@ -163,6 +163,19 @@ static inline void oid_to_vnodes(struct sd_vnode *entries, int nr_entries,
 	}
 }
 
+static inline void oid_to_nodes(struct sd_vnode *entries, int nr_entries,
+				uint64_t oid, int nr_copies,
+				struct sd_node *all_nodes,
+				struct sd_node **nodes)
+{
+	int i;
+	struct sd_vnode *vnodes[SD_MAX_COPIES];
+
+	oid_to_vnodes(entries, nr_entries, oid, nr_copies, vnodes);
+	for (i = 0; i < nr_copies; i++)
+		nodes[i] = &all_nodes[vnodes[i]->node_idx];
+}
+
 static inline const char *sd_strerror(int err)
 {
 	int i;
diff --git a/sheep/gateway.c b/sheep/gateway.c
index 7b72468..19b90b9 100644
--- a/sheep/gateway.c
+++ b/sheep/gateway.c
@@ -222,10 +222,10 @@ static inline void write_info_init(struct write_info *wi)
 }
 
 static inline void
-write_info_advance(struct write_info *wi, struct sd_vnode *v,
+write_info_advance(struct write_info *wi, struct node_id *nid,
 		   struct sockfd *sfd)
 {
-	wi->ent[wi->nr_sent].nid = &v->nid;
+	wi->ent[wi->nr_sent].nid = nid;
 	wi->ent[wi->nr_sent].pfd.fd = sfd->fd;
 	wi->ent[wi->nr_sent].pfd.events = POLLIN;
 	wi->ent[wi->nr_sent].sfd = sfd;
@@ -239,17 +239,37 @@ static inline void gateway_init_fwd_hdr(struct sd_req *fwd, struct sd_req *hdr)
 	fwd->proto_ver = SD_SHEEP_PROTO_VER;
 }
 
-static int gateway_forward_request(struct request *req)
+static int init_target_nodes(struct request *req, bool all_node,
+			uint64_t oid, struct sd_node **target_nodes)
+{
+	int i, nr_to_send;
+	struct vnode_info *vinfo = req->vinfo;
+
+	if (all_node) {
+		nr_to_send = vinfo->nr_nodes;
+		for (i = 0; i < nr_to_send; i++)
+			target_nodes[i] = &vinfo->nodes[i];
+
+		return nr_to_send;
+	}
+
+	nr_to_send = get_req_copy_number(req);
+	oid_to_nodes(vinfo->vnodes, vinfo->nr_vnodes, oid, nr_to_send,
+		vinfo->nodes, target_nodes);
+
+	return nr_to_send;
+}
+
+static int gateway_forward_request(struct request *req, bool all_node)
 {
 	int i, err_ret = SD_RES_SUCCESS, ret, local = -1;
 	unsigned wlen;
-	struct sd_vnode *v;
-	struct sd_vnode *obj_vnodes[SD_MAX_COPIES];
 	uint64_t oid = req->rq.obj.oid;
-	int nr_copies;
+	int nr_to_send;
 	struct write_info wi;
 	struct sd_op_template *op;
 	struct sd_req hdr;
+	struct sd_node *target_nodes[SD_MAX_COPIES];
 
 	dprintf("%"PRIx64"\n", oid);
 
@@ -258,20 +278,19 @@ static int gateway_forward_request(struct request *req)
 
 	write_info_init(&wi);
 	wlen = hdr.data_length;
-	nr_copies = get_req_copy_number(req);
-	oid_to_vnodes(req->vinfo->vnodes, req->vinfo->nr_vnodes, oid,
-		      nr_copies, obj_vnodes);
+	nr_to_send = init_target_nodes(req, all_node, oid, target_nodes);
 
-	for (i = 0; i < nr_copies; i++) {
+	for (i = 0; i < nr_to_send; i++) {
 		struct sockfd *sfd;
+		struct node_id *nid;
 
-		v = obj_vnodes[i];
-		if (vnode_is_local(v)) {
+		if (node_is_local(target_nodes[i])) {
 			local = i;
 			continue;
 		}
 
-		sfd = sheep_get_sockfd(&v->nid);
+		nid = &target_nodes[i]->nid;
+		sfd = sheep_get_sockfd(nid);
 		if (!sfd) {
 			err_ret = SD_RES_NETWORK_ERROR;
 			break;
@@ -279,17 +298,15 @@ static int gateway_forward_request(struct request *req)
 
 		ret = send_req(sfd->fd, &hdr, req->data, &wlen);
 		if (ret) {
-			sheep_del_sockfd(&v->nid, sfd);
+			sheep_del_sockfd(nid, sfd);
 			err_ret = SD_RES_NETWORK_ERROR;
 			dprintf("fail %d\n", ret);
 			break;
 		}
-		write_info_advance(&wi, v, sfd);
+		write_info_advance(&wi, nid, sfd);
 	}
 
 	if (local != -1 && err_ret == SD_RES_SUCCESS) {
-		v = obj_vnodes[local];
-
 		assert(op);
 		ret = sheep_do_op_work(op, req);
 
@@ -314,7 +331,7 @@ int gateway_write_obj(struct request *req)
 	if (sys->enable_write_cache && !req->local && !bypass_object_cache(req))
 		return object_cache_handle_request(req);
 
-	return gateway_forward_request(req);
+	return gateway_forward_request(req, false);
 }
 
 int gateway_create_and_write_obj(struct request *req)
@@ -322,10 +339,15 @@ int gateway_create_and_write_obj(struct request *req)
 	if (sys->enable_write_cache && !req->local && !bypass_object_cache(req))
 		return object_cache_handle_request(req);
 
-	return gateway_forward_request(req);
+	return gateway_forward_request(req, false);
 }
 
 int gateway_remove_obj(struct request *req)
 {
-	return gateway_forward_request(req);
+	return gateway_forward_request(req, false);
+}
+
+int gateway_sync_vdi(struct request *req)
+{
+	return gateway_forward_request(req, true);
 }
diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index 5969c26..acad0b9 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -343,6 +343,11 @@ static inline int vnode_is_local(struct sd_vnode *v)
 	return is_myself(v->nid.addr, v->nid.port);
 }
 
+static inline int node_is_local(struct sd_node *n)
+{
+	return is_myself(n->nid.addr, n->nid.port);
+}
+
 /* gateway operations */
 int gateway_read_obj(struct request *req);
 int gateway_write_obj(struct request *req);
-- 
1.7.5.1