[sheepdog] [PATCH v2] sheep: rework send join message for corosync and local drivers

[Liu Yuan]

They can't send the whole struct cluster_info. Instead of doing tricks in
send_join_request(), we just min(SD_MAX_EVENT_BUF_SIZE, join_msg_len) for
corosync and local drivers.

This patch also fixes a bug that corosync can't work with node join because
not enough buffer allocated for join message.

Signed-off-by: Liu Yuan <namei.unix at gmail.com>
---
 include/util.h            |    3 +++
 sheep/cluster.h           |    5 ++++-
 sheep/cluster/corosync.c  |    7 ++++---
 sheep/cluster/local.c     |    1 +
 sheep/cluster/zookeeper.c |    5 ++---
 sheep/group.c             |    4 +---
 6 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/include/util.h b/include/util.h
index 218553f..6593604 100644
--- a/include/util.h
+++ b/include/util.h
@@ -54,6 +54,9 @@ static inline int after(uint32_t seq1, uint32_t seq2)
 	return (int32_t)(seq2 - seq1) < 0;
 }
 
+#define MIN(x, y) ((x) > (y) ? (y) : (x))
+#define MAX(x, y) ((x) > (y) ? (x) : (y))
+
 #define min(x, y) ({ \
 	typeof(x) _x = (x);	\
 	typeof(y) _y = (y);	\
diff --git a/sheep/cluster.h b/sheep/cluster.h
index a267443..81b5ae4 100644
--- a/sheep/cluster.h
+++ b/sheep/cluster.h
@@ -23,7 +23,10 @@
 #include "sheep.h"
 #include "config.h"
 
-/* maximum payload size sent in ->notify and ->unblock */
+/*
+ * maximum payload size sent in ->notify and ->unblock, it should be large
+ * enough to support COROSYNC_MAX_NODES * struct sd_node
+ */
 #define SD_MAX_EVENT_BUF_SIZE (128 * 1024) /* 128k */
 
 struct cluster_driver {
diff --git a/sheep/cluster/corosync.c b/sheep/cluster/corosync.c
index 45756e8..ea4421b 100644
--- a/sheep/cluster/corosync.c
+++ b/sheep/cluster/corosync.c
@@ -168,9 +168,11 @@ static int send_message(enum corosync_message_type type,
 {
 	struct iovec iov[2];
 	int ret, iov_cnt = 1;
+	size_t mlen = MIN(msg_len, SD_MAX_EVENT_BUF_SIZE);
+
 	struct corosync_message cmsg = {
 		.type = type,
-		.msg_len = msg_len,
+		.msg_len = mlen,
 		.sender = *sender,
 		.nr_nodes = nr_nodes,
 	};
@@ -182,7 +184,7 @@ static int send_message(enum corosync_message_type type,
 	iov[0].iov_len = sizeof(cmsg);
 	if (msg) {
 		iov[1].iov_base = msg;
-		iov[1].iov_len = msg_len;
+		iov[1].iov_len = mlen;
 		iov_cnt++;
 	}
 retry:
@@ -640,7 +642,6 @@ static int corosync_join(const struct sd_node *myself,
 			 void *opaque, size_t opaque_len)
 {
 	int ret;
-
 retry:
 	ret = cpg_join(cpg_handle, &cpg_group);
 	switch (ret) {
diff --git a/sheep/cluster/local.c b/sheep/cluster/local.c
index 824279c..b8cbb5c 100644
--- a/sheep/cluster/local.c
+++ b/sheep/cluster/local.c
@@ -257,6 +257,7 @@ static int add_event(enum local_event_type type, struct local_node *lnode,
 		.sender = *lnode,
 	};
 
+	buf_len = MIN(buf_len, SD_MAX_EVENT_BUF_SIZE);
 	ev.buf_len = buf_len;
 	if (buf)
 		memcpy(ev.buf, buf, buf_len);
diff --git a/sheep/cluster/zookeeper.c b/sheep/cluster/zookeeper.c
index 7ce8180..fa89c46 100644
--- a/sheep/cluster/zookeeper.c
+++ b/sheep/cluster/zookeeper.c
@@ -561,15 +561,14 @@ static void zk_watcher(zhandle_t *zh, int type, int state, const char *path,
 static int add_join_event(void *msg, size_t msglen)
 {
 	struct zk_event ev;
-	size_t msg_len = sizeof(struct cluster_info);
-	size_t len = msg_len + sizeof(struct sd_node) * SD_MAX_NODES;
+	size_t len = msglen + sizeof(struct sd_node) * SD_MAX_NODES;
 
 	if (unlikely((offsetof(struct zk_event, buf) + len) > ZK_MAX_BUF_SIZE))
 		panic("Zookeeper can't send message more than 1M");
 	ev.id = get_uniq_id();
 	ev.type = EVENT_JOIN;
 	ev.sender = this_node;
-	ev.msg_len = msg_len;
+	ev.msg_len = msglen;
 	ev.buf_len = len;
 	if (msg)
 		memcpy(ev.buf, msg, msglen);
diff --git a/sheep/group.c b/sheep/group.c
index 16f1532..4328547 100644
--- a/sheep/group.c
+++ b/sheep/group.c
@@ -773,11 +773,9 @@ main_fn bool sd_join_handler(const struct sd_node *joining,
 static int send_join_request(void)
 {
 	struct sd_node *n = &sys->this_node;
-	int len = offsetof(struct cluster_info, nodes) +
-		sys->cinfo.nr_nodes * sizeof(struct sd_node);
 
 	sd_info("%s", node_to_str(n));
-	return sys->cdrv->join(n, &sys->cinfo, len);
+	return sys->cdrv->join(n, &sys->cinfo, sizeof(sys->cinfo));
 }
 
 static void requeue_cluster_request(void)
-- 
1.7.9.5