[sheepdog] [PATCH v2] sheep: change one-shot timeout to keepalive

[Liu Yuan]

From: Liu Yuan <tailai.ly at taobao.com>

The timeout of 5s really cause trouble from our observation, we see a lot of
timeout failure when cluster is doing IO heavily.

Use keepalive means we don't fail-timeout until the other end of node is
really down.

Signed-off-by: Liu Yuan <tailai.ly at taobao.com>
---
 include/net.h   |    2 --
 lib/net.c       |   23 -----------------------
 sheep/gateway.c |    7 +------
 sheep/sdnet.c   |   35 +++++++++++++++++++++++++++++++++--
 4 files changed, 34 insertions(+), 33 deletions(-)

diff --git a/include/net.h b/include/net.h
index d97984e..83da12a 100644
--- a/include/net.h
+++ b/include/net.h
@@ -6,8 +6,6 @@
 
 #include "sheepdog_proto.h"
 
-#define DEFAULT_SOCKET_TIMEOUT 5 /* seconds */
-
 enum conn_state {
 	C_IO_HEADER = 0,
 	C_IO_DATA_INIT,
diff --git a/lib/net.c b/lib/net.c
index bebc108..c4a96ac 100644
--- a/lib/net.c
+++ b/lib/net.c
@@ -420,29 +420,6 @@ int set_nodelay(int fd)
 	return ret;
 }
 
-int set_timeout(int fd)
-{
-	int ret;
-	const struct timeval tv = {
-		.tv_sec = DEFAULT_SOCKET_TIMEOUT,
-		.tv_usec = 0,
-	};
-
-	ret = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
-	if (ret) {
-		eprintf("failed to set send timeout\n");
-		return ret;
-	}
-
-	ret = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
-	if (ret) {
-		eprintf("failed to set recv timeout\n");
-		return ret;
-	}
-
-	return 0;
-}
-
 int get_local_addr(uint8_t *bytes)
 {
 	struct ifaddrs *ifaddr, *ifa;
diff --git a/sheep/gateway.c b/sheep/gateway.c
index debe569..e92f3ed 100644
--- a/sheep/gateway.c
+++ b/sheep/gateway.c
@@ -157,18 +157,13 @@ int forward_write_obj_req(struct request *req)
 
 	ret = SD_RES_SUCCESS;
 again:
-	pollret = poll(pfds, nr_fds, DEFAULT_SOCKET_TIMEOUT * 1000);
+	pollret = poll(pfds, nr_fds, -1);
 	if (pollret < 0) {
 		if (errno == EINTR)
 			goto again;
 
 		ret = SD_RES_NETWORK_ERROR;
 		goto err;
-	} else if (pollret == 0) {
-		/* poll time out */
-		eprintf("timeout\n");
-		ret = SD_RES_NETWORK_ERROR;
-		goto err;
 	}
 
 	for (i = 0; i < nr_fds; i++) {
diff --git a/sheep/sdnet.c b/sheep/sdnet.c
index 6323ee3..8479a74 100644
--- a/sheep/sdnet.c
+++ b/sheep/sdnet.c
@@ -828,6 +828,38 @@ void del_sheep_fd(int fd)
 	}
 }
 
+/*
+ * Timeout after request is issued after 5s.
+ *
+ * Heart-beat message will be sent periodically with 1s interval.
+ * If the node of the other end of fd fails, we'll detect it in 3s
+ */
+static int set_keepalive(int fd)
+{
+	int val = 1;
+
+	if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) < 0) {
+		dprintf("%m\n");
+		return -1;
+	}
+	val = 5;
+	if (setsockopt(fd, SOL_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) {
+		dprintf("%m\n");
+		return -1;
+	}
+	val = 1;
+	if (setsockopt(fd, SOL_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) {
+		dprintf("%m\n");
+		return -1;
+	}
+	val = 3;
+	if (setsockopt(fd, SOL_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) {
+		dprintf("%m\n");
+		return -1;
+	}
+	return 0;
+}
+
 int get_sheep_fd(uint8_t *addr, uint16_t port, int node_idx, uint32_t epoch)
 {
 	int i, fd, ret;
@@ -870,9 +902,8 @@ int get_sheep_fd(uint8_t *addr, uint16_t port, int node_idx, uint32_t epoch)
 	if (fd < 0)
 		return -1;
 
-	ret = set_timeout(fd);
+	ret = set_keepalive(fd);
 	if (ret) {
-		eprintf("%m\n");
 		close(fd);
 		return -1;
 	}
-- 
1.7.10.2