[sheepdog] [PATCH 1/3] sheep: use cached sockfd for all connections
Liu Yuan
namei.unix at gmail.com
Wed Sep 12 10:07:12 CEST 2012
From: Liu Yuan <tailai.ly at taobao.com>
This will especially boost recovery performance a bit.
Signed-off-by: Liu Yuan <tailai.ly at taobao.com>
---
sheep/group.c | 31 +++++++++++++++---------------
sheep/recovery.c | 55 ++++++++++++++++++++++++++++--------------------------
sheep/store.c | 27 +++++++++++++++++----------
3 files changed, 62 insertions(+), 51 deletions(-)
diff --git a/sheep/group.c b/sheep/group.c
index 4661110..56dee6a 100644
--- a/sheep/group.c
+++ b/sheep/group.c
@@ -623,25 +623,20 @@ static int get_vdis_from(struct sd_node *node)
struct sd_req hdr;
struct sd_rsp *rsp = (struct sd_rsp *)&hdr;
struct vdi_copy *vc = NULL;
- int fd, i, ret = SD_RES_SUCCESS;
+ struct sockfd *sfd;
+ int i, ret = SD_RES_SUCCESS;
unsigned int rlen, wlen;
- char host[128];
int count;
if (node_is_local(node))
goto out;
- addr_to_str(host, sizeof(host), node->nid.addr, 0);
-
- fd = connect_to(host, node->nid.port);
- if (fd < 0) {
- vprintf(SDOG_ERR, "unable to get the VDI bitmap from %s: %m\n", host);
- ret = SD_RES_EIO;
+ sfd = sheep_get_sockfd(&node->nid);
+ if (!sfd) {
+ ret = SD_RES_NETWORK_ERROR;
goto out;
}
- vprintf(SDOG_ERR, "%s:%d\n", host, node->nid.port);
-
rlen = SD_DATA_OBJ_SIZE; /* FIXME */
sd_init_req(&hdr, SD_OP_GET_VDI_COPIES);
hdr.epoch = sys->epoch;
@@ -655,16 +650,22 @@ static int get_vdis_from(struct sd_node *node)
goto out;
}
- ret = exec_req(fd, &hdr, (char *)vc, &wlen, &rlen);
- close(fd);
+ ret = exec_req(sfd->fd, &hdr, (char *)vc, &wlen, &rlen);
- if (ret || rsp->result != SD_RES_SUCCESS) {
- vprintf(SDOG_ERR, "unable to get the VDI bitmap (%d, %d)\n", ret,
- rsp->result);
+ if (ret) {
+ dprintf("remote node might have gone away\n");
+ sheep_del_sockfd(&node->nid, sfd);
+ ret = SD_RES_NETWORK_ERROR;
+ goto out;
+ }
+ if (rsp->result != SD_RES_SUCCESS) {
ret = rsp->result;
+ eprintf("failed %x\n", ret);
+ sheep_put_sockfd(&node->nid, sfd);
goto out;
}
+ sheep_put_sockfd(&node->nid, sfd);
count = rsp->data_length / sizeof(*vc);
for (i = 0; i < count; i++) {
set_bit(vc[i].vid, sys->vdi_inuse);
diff --git a/sheep/recovery.c b/sheep/recovery.c
index 4df5b66..ae36768 100644
--- a/sheep/recovery.c
+++ b/sheep/recovery.c
@@ -55,17 +55,16 @@ static int obj_cmp(const void *oid1, const void *oid2)
return 0;
}
-static int recover_object_from_replica(uint64_t oid,
- struct sd_vnode *entry,
+static int recover_object_from_replica(uint64_t oid, struct sd_vnode *vnode,
uint32_t epoch, uint32_t tgt_epoch)
{
struct sd_req hdr;
struct sd_rsp *rsp = (struct sd_rsp *)&hdr;
- char name[128];
unsigned wlen = 0, rlen;
- int fd, ret = -1;
+ int ret = -1;
void *buf;
struct siocb iocb = { 0 };
+ struct sockfd *sfd;
rlen = get_objsize(oid);
@@ -75,7 +74,7 @@ static int recover_object_from_replica(uint64_t oid,
goto out;
}
- if (vnode_is_local(entry)) {
+ if (vnode_is_local(vnode)) {
iocb.epoch = epoch;
iocb.length = rlen;
ret = sd_store->link(oid, &iocb, tgt_epoch);
@@ -88,11 +87,9 @@ static int recover_object_from_replica(uint64_t oid,
}
}
- addr_to_str(name, sizeof(name), entry->nid.addr, 0);
- fd = connect_to(name, entry->nid.port);
- dprintf("%s, %d\n", name, entry->nid.port);
- if (fd < 0) {
- eprintf("failed to connect to %s:%"PRIu32"\n", name, entry->nid.port);
+
+ sfd = sheep_get_sockfd(&vnode->nid);
+ if (!sfd) {
ret = -1;
goto out;
}
@@ -105,19 +102,18 @@ static int recover_object_from_replica(uint64_t oid,
hdr.obj.oid = oid;
hdr.obj.tgt_epoch = tgt_epoch;
- ret = exec_req(fd, &hdr, buf, &wlen, &rlen);
-
- close(fd);
+ ret = exec_req(sfd->fd, &hdr, buf, &wlen, &rlen);
if (ret != 0) {
eprintf("res: %"PRIx32"\n", rsp->result);
ret = -1;
+ sheep_del_sockfd(&vnode->nid, sfd);
goto out;
}
- rsp = (struct sd_rsp *)&hdr;
-
if (rsp->result == SD_RES_SUCCESS) {
+ sheep_put_sockfd(&vnode->nid, sfd);
+
iocb.epoch = epoch;
iocb.length = rlen;
iocb.buf = buf;
@@ -129,6 +125,7 @@ static int recover_object_from_replica(uint64_t oid,
} else {
eprintf("failed, res: %"PRIx32"\n", rsp->result);
ret = rsp->result;
+ sheep_del_sockfd(&vnode->nid, sfd);
goto out;
}
done:
@@ -498,18 +495,19 @@ static void finish_object_list(struct work *work)
static int fetch_object_list(struct sd_node *e, uint32_t epoch,
uint8_t *buf, size_t buf_size)
{
- int fd, ret;
unsigned wlen, rlen;
char name[128];
struct sd_list_req hdr;
- struct sd_list_rsp *rsp;
+ struct sd_list_rsp *rsp = (struct sd_list_rsp *)&hdr;
+ struct sockfd *sfd;
+ int ret;
addr_to_str(name, sizeof(name), e->nid.addr, 0);
dprintf("%s %"PRIu32"\n", name, e->nid.port);
- fd = connect_to(name, e->nid.port);
- if (fd < 0) {
+ sfd = sheep_get_sockfd(&e->nid);
+ if (!sfd) {
eprintf("%s %"PRIu32"\n", name, e->nid.port);
return -1;
}
@@ -522,17 +520,22 @@ static int fetch_object_list(struct sd_node *e, uint32_t epoch,
hdr.flags = 0;
hdr.data_length = rlen;
- ret = exec_req(fd, (struct sd_req *)&hdr, buf, &wlen, &rlen);
+ ret = exec_req(sfd->fd, (struct sd_req *)&hdr, buf, &wlen, &rlen);
- close(fd);
-
- rsp = (struct sd_list_rsp *)&hdr;
-
- if (ret || rsp->result != SD_RES_SUCCESS) {
- eprintf("failed, %"PRIu32", %"PRIu32"\n", ret, rsp->result);
+ if (ret) {
+ dprintf("remote node might have gone away\n");
+ sheep_del_sockfd(&e->nid, sfd);
+ return -1;
+ }
+ if (rsp->result != SD_RES_SUCCESS) {
+ ret = rsp->result;
+ eprintf("failed %x\n", ret);
+ sheep_put_sockfd(&e->nid, sfd);
return -1;
}
+ sheep_put_sockfd(&e->nid, sfd);
+
dprintf("%zu\n", rsp->data_length / sizeof(uint64_t));
return rsp->data_length / sizeof(uint64_t);
diff --git a/sheep/store.c b/sheep/store.c
index 74fd931..d89e875 100644
--- a/sheep/store.c
+++ b/sheep/store.c
@@ -102,17 +102,14 @@ int epoch_log_read_remote(uint32_t epoch, struct sd_node *nodes, int len)
for (i = 0; i < nr; i++) {
struct sd_req hdr;
struct sd_rsp *rsp = (struct sd_rsp *)&hdr;
- char host[128];
+ struct sockfd *sfd;
unsigned int rlen, wlen;
- int fd;
if (node_is_local(&local_nodes[i]))
continue;
- addr_to_str(host, sizeof(host), local_nodes[i].nid.addr, 0);
- fd = connect_to(host, local_nodes[i].nid.port);
- if (fd < 0) {
- vprintf(SDOG_ERR, "failed to connect to %s: %m\n", host);
+ sfd = sheep_get_sockfd(&local_nodes[i].nid);
+ if (!sfd) {
continue;
}
@@ -122,11 +119,21 @@ int epoch_log_read_remote(uint32_t epoch, struct sd_node *nodes, int len)
wlen = 0;
- ret = exec_req(fd, &hdr, nodes, &wlen, &rlen);
- close(fd);
+ ret = exec_req(sfd->fd, &hdr, nodes, &wlen, &rlen);
+ if (ret) {
+ dprintf("remote node might have gone away\n");
+ sheep_del_sockfd(&local_nodes[i].nid, sfd);
+ continue;
+ }
+ if (rsp->result != SD_RES_SUCCESS) {
+ ret = rsp->result;
+ eprintf("failed %x\n", ret);
+ sheep_put_sockfd(&local_nodes[i].nid, sfd);
+ continue;
+ }
- if (!ret && rsp->result == SD_RES_SUCCESS)
- return rsp->data_length / sizeof(*nodes);
+ sheep_put_sockfd(&local_nodes[i].nid, sfd);
+ return rsp->data_length / sizeof(*nodes);
}
/*
--
1.7.10.2
More information about the sheepdog
mailing list