From: Liu Yuan <tailai.ly at taobao.com> The timeout of 5s really cause trouble from our observation, we see a lot of timeout failure when cluster is doing IO heavily. Use keepalive means we don't fail-timeout until the other end of node is really down. Signed-off-by: Liu Yuan <tailai.ly at taobao.com> --- include/net.h | 2 -- lib/net.c | 23 ----------------------- sheep/gateway.c | 7 +------ sheep/sdnet.c | 35 +++++++++++++++++++++++++++++++++-- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/include/net.h b/include/net.h index d97984e..83da12a 100644 --- a/include/net.h +++ b/include/net.h @@ -6,8 +6,6 @@ #include "sheepdog_proto.h" -#define DEFAULT_SOCKET_TIMEOUT 5 /* seconds */ - enum conn_state { C_IO_HEADER = 0, C_IO_DATA_INIT, diff --git a/lib/net.c b/lib/net.c index bebc108..c4a96ac 100644 --- a/lib/net.c +++ b/lib/net.c @@ -420,29 +420,6 @@ int set_nodelay(int fd) return ret; } -int set_timeout(int fd) -{ - int ret; - const struct timeval tv = { - .tv_sec = DEFAULT_SOCKET_TIMEOUT, - .tv_usec = 0, - }; - - ret = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); - if (ret) { - eprintf("failed to set send timeout\n"); - return ret; - } - - ret = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); - if (ret) { - eprintf("failed to set recv timeout\n"); - return ret; - } - - return 0; -} - int get_local_addr(uint8_t *bytes) { struct ifaddrs *ifaddr, *ifa; diff --git a/sheep/gateway.c b/sheep/gateway.c index debe569..e92f3ed 100644 --- a/sheep/gateway.c +++ b/sheep/gateway.c @@ -157,18 +157,13 @@ int forward_write_obj_req(struct request *req) ret = SD_RES_SUCCESS; again: - pollret = poll(pfds, nr_fds, DEFAULT_SOCKET_TIMEOUT * 1000); + pollret = poll(pfds, nr_fds, -1); if (pollret < 0) { if (errno == EINTR) goto again; ret = SD_RES_NETWORK_ERROR; goto err; - } else if (pollret == 0) { - /* poll time out */ - eprintf("timeout\n"); - ret = SD_RES_NETWORK_ERROR; - goto err; } for (i = 0; i < nr_fds; i++) { diff --git a/sheep/sdnet.c b/sheep/sdnet.c index 6323ee3..8479a74 100644 --- a/sheep/sdnet.c +++ b/sheep/sdnet.c @@ -828,6 +828,38 @@ void del_sheep_fd(int fd) } } +/* + * Timeout after request is issued after 5s. + * + * Heart-beat message will be sent periodically with 1s interval. + * If the node of the other end of fd fails, we'll detect it in 3s + */ +static int set_keepalive(int fd) +{ + int val = 1; + + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) < 0) { + dprintf("%m\n"); + return -1; + } + val = 5; + if (setsockopt(fd, SOL_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) { + dprintf("%m\n"); + return -1; + } + val = 1; + if (setsockopt(fd, SOL_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) { + dprintf("%m\n"); + return -1; + } + val = 3; + if (setsockopt(fd, SOL_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) { + dprintf("%m\n"); + return -1; + } + return 0; +} + int get_sheep_fd(uint8_t *addr, uint16_t port, int node_idx, uint32_t epoch) { int i, fd, ret; @@ -870,9 +902,8 @@ int get_sheep_fd(uint8_t *addr, uint16_t port, int node_idx, uint32_t epoch) if (fd < 0) return -1; - ret = set_timeout(fd); + ret = set_keepalive(fd); if (ret) { - eprintf("%m\n"); close(fd); return -1; } -- 1.7.10.2 |