[sheepdog] [PATCH 2/2] sheep: retry write for EAGAIN
Liu Yuan
namei.unix at gmail.com
Thu Jan 31 07:38:55 CET 2013
From: Liu Yuan <tailai.ly at taobao.com>
Since we set timeout for write, we'll get EAGAIN even for blocking sockfd.
Signed-off-by: Liu Yuan <tailai.ly at taobao.com>
---
include/net.h | 5 ++++-
lib/net.c | 32 ++++++++++++++++++++++----------
sheep/gateway.c | 8 +++++---
sheep/sheep_priv.h | 1 +
sheep/sockfd_cache.c | 7 ++++++-
5 files changed, 38 insertions(+), 15 deletions(-)
diff --git a/include/net.h b/include/net.h
index ad90994..a68c880 100644
--- a/include/net.h
+++ b/include/net.h
@@ -7,6 +7,8 @@
#include "sheepdog_proto.h"
/*
+ * We can't always retry because if only IO NIC is down, we'll retry for ever.
+ *
* We observed that for a busy node, the response could be as long as 15s, so
* wait 30s would be a safe value. Even we are false timeouted, the gateway will
* retry the request and sockfd cache module will repair the false-closes.
@@ -51,7 +53,8 @@ int do_read(int sockfd, void *buf, int len,
int rx(struct connection *conn, enum conn_state next_state);
int tx(struct connection *conn, enum conn_state next_state);
int connect_to(const char *name, int port);
-int send_req(int sockfd, struct sd_req *hdr, void *data, unsigned int wlen);
+int send_req(int sockfd, struct sd_req *hdr, void *data, unsigned int wlen,
+ bool (*need_retry)(uint32_t), uint32_t);
int exec_req(int sockfd, struct sd_req *hdr, void *,
bool (*need_retry)(uint32_t), uint32_t);
int create_listen_ports(const char *bindaddr, int port,
diff --git a/lib/net.c b/lib/net.c
index 6504cd8..ac45fbe 100644
--- a/lib/net.c
+++ b/lib/net.c
@@ -270,7 +270,7 @@ success:
int do_read(int sockfd, void *buf, int len, bool (*need_retry)(uint32_t epoch),
uint32_t epoch)
{
- int ret;
+ int ret, repeat = MAX_RETRY_COUNT;
reread:
ret = read(sockfd, buf, len);
if (ret < 0 || !ret) {
@@ -279,12 +279,12 @@ reread:
/*
* Since we set timeout for read, we'll get EAGAIN even for
* blocking sockfd.
- *
- * Always retry if need_retry is NULL
*/
- if (errno == EAGAIN &&
- (need_retry == NULL || need_retry(epoch)))
+ if (errno == EAGAIN && repeat &&
+ (need_retry == NULL || need_retry(epoch))) {
+ repeat--;
goto reread;
+ }
sd_eprintf("failed to read from socket: %d, %d(%m)\n",
ret, errno);
@@ -312,14 +312,25 @@ static void forward_iov(struct msghdr *msg, int len)
}
-static int do_write(int sockfd, struct msghdr *msg, int len)
+static int do_write(int sockfd, struct msghdr *msg, int len,
+ bool (*need_retry)(uint32_t), uint32_t epoch)
{
- int ret;
+ int ret, repeat = MAX_RETRY_COUNT;
rewrite:
ret = sendmsg(sockfd, msg, 0);
if (ret < 0) {
if (errno == EINTR)
goto rewrite;
+ /*
+ * Since we set timeout for write, we'll get EAGAIN even for
+ * blocking sockfd.
+ */
+ if (errno == EAGAIN && repeat &&
+ (need_retry == NULL || need_retry(epoch))) {
+ repeat--;
+ goto rewrite;
+ }
+
sd_eprintf("failed to write to socket: %m\n");
return 1;
}
@@ -333,7 +344,8 @@ rewrite:
return 0;
}
-int send_req(int sockfd, struct sd_req *hdr, void *data, unsigned int wlen)
+int send_req(int sockfd, struct sd_req *hdr, void *data, unsigned int wlen,
+ bool (*need_retry)(uint32_t epoch), uint32_t epoch)
{
int ret;
struct msghdr msg;
@@ -353,7 +365,7 @@ int send_req(int sockfd, struct sd_req *hdr, void *data, unsigned int wlen)
iov[1].iov_len = wlen;
}
- ret = do_write(sockfd, &msg, sizeof(*hdr) + wlen);
+ ret = do_write(sockfd, &msg, sizeof(*hdr) + wlen, need_retry, epoch);
if (ret) {
sd_eprintf("failed to send request %x, %d: %m\n", hdr->opcode,
wlen);
@@ -378,7 +390,7 @@ int exec_req(int sockfd, struct sd_req *hdr, void *data,
rlen = hdr->data_length;
}
- if (send_req(sockfd, hdr, data, wlen))
+ if (send_req(sockfd, hdr, data, wlen, need_retry, epoch))
return 1;
ret = do_read(sockfd, rsp, sizeof(*rsp), need_retry, epoch);
diff --git a/sheep/gateway.c b/sheep/gateway.c
index b54154d..7f99c20 100644
--- a/sheep/gateway.c
+++ b/sheep/gateway.c
@@ -167,7 +167,7 @@ again:
* If IO NIC is down, epoch isn't incremented, so we can't retry
* for ever.
*/
- if (req->rq.epoch == sys_epoch() && repeat) {
+ if (sheep_need_retry(req->rq.epoch) && repeat) {
repeat--;
goto again;
}
@@ -193,7 +193,8 @@ again:
finish_one_write_err(wi, i);
goto finish_write;
}
- if (do_read(pi.pfds[i].fd, rsp, sizeof(*rsp), NULL, 0)) {
+ if (do_read(pi.pfds[i].fd, rsp, sizeof(*rsp), sheep_need_retry,
+ req->rq.epoch)) {
sd_eprintf("remote node might have gone away\n");
err_ret = SD_RES_NETWORK_ERROR;
finish_one_write_err(wi, i);
@@ -290,7 +291,8 @@ static int gateway_forward_request(struct request *req, bool all_node)
break;
}
- ret = send_req(sfd->fd, &hdr, req->data, wlen);
+ ret = send_req(sfd->fd, &hdr, req->data, wlen,
+ sheep_need_retry, req->rq.epoch);
if (ret) {
sheep_del_sockfd(nid, sfd);
err_ret = SD_RES_NETWORK_ERROR;
diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index a1e89c0..1e1a28a 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -412,6 +412,7 @@ struct sockfd *sheep_get_sockfd(const struct node_id *);
void sheep_put_sockfd(const struct node_id *, struct sockfd *);
void sheep_del_sockfd(const struct node_id *, struct sockfd *);
int sheep_exec_req(const struct node_id *nid, struct sd_req *hdr, void *data);
+bool sheep_need_retry(uint32_t epoch);
static inline bool is_object_cache_enabled(void)
{
diff --git a/sheep/sockfd_cache.c b/sheep/sockfd_cache.c
index 8aadd7c..168e9fb 100644
--- a/sheep/sockfd_cache.c
+++ b/sheep/sockfd_cache.c
@@ -521,7 +521,7 @@ int sheep_exec_req(const struct node_id *nid, struct sd_req *hdr, void *buf)
if (!sfd)
return SD_RES_NETWORK_ERROR;
- ret = exec_req(sfd->fd, hdr, buf, NULL, 0);
+ ret = exec_req(sfd->fd, hdr, buf, sheep_need_retry, hdr->epoch);
if (ret) {
sd_dprintf("remote node might have gone away\n");
sheep_del_sockfd(nid, sfd);
@@ -534,3 +534,8 @@ int sheep_exec_req(const struct node_id *nid, struct sd_req *hdr, void *buf)
sheep_put_sockfd(nid, sfd);
return ret;
}
+
+bool sheep_need_retry(uint32_t epoch)
+{
+ return sys_epoch() == epoch;
+}
--
1.7.9.5
More information about the sheepdog
mailing list