[sheepdog] [PATCH 1/2] collie: parallelize parse_vdi() with work queue
Hitoshi Mitake
mitake.hitoshi at lab.ntt.co.jp
Mon Aug 5 13:29:32 CEST 2013
Current collie issues request for gathering VDI information in a
sequential manner (parse_vdi()). This way is not scalable when a
number of VDIs becomes larger.
This patch parallelize parse_vdi() with work queue. Some collie
commands which call parse_vdi() can enjoy performance improvement.
The below is an sample of collie vdi list. The test is done on 16
nodes cluster which has 3000 VDIs.
Before:
$ time sh -c "collie/collie vdi list -a 10.68.13.1 > /dev/null"
sh -c "collie/collie vdi list -a 10.68.13.1 > /dev/null" 8.81s user 0.24s system 70% cpu 12.876 total
After:
% time sh -c "collie/collie vdi list -a 10.68.13.1 > /dev/null"
sh -c "collie/collie vdi list -a 10.68.13.1 > /dev/null" 14.35s user 2.02s system 209% cpu 7.816 total
Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
---
collie/common.c | 172 ++++++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 139 insertions(+), 33 deletions(-)
diff --git a/collie/common.c b/collie/common.c
index a480418..d72d7ff 100644
--- a/collie/common.c
+++ b/collie/common.c
@@ -13,6 +13,8 @@
#include "sha1.h"
#include "sockfd_cache.h"
+#include <pthread.h>
+
char *size_to_str(uint64_t _size, char *str, int str_size)
{
const char *units[] = {"MB", "GB", "TB", "PB", "EB", "ZB", "YB"};
@@ -110,64 +112,168 @@ int sd_write_object(uint64_t oid, uint64_t cow_oid, void *data,
return SD_RES_SUCCESS;
}
+struct parse_vdi_info {
+ uint64_t oid;
+ size_t size;
+ void *data;
+ vdi_parser_func_t func;
+
+ bool succeed;
+
+ struct work work;
+ struct sd_inode inode;
+};
+
+static void parse_vdi_work(struct work *work)
+{
+ int ret;
+ struct parse_vdi_info *info = container_of(work, struct parse_vdi_info,
+ work);
+ struct sd_inode inode;
+
+ info->succeed = false;
+
+ memset(&inode, 0, sizeof(inode));
+ ret = sd_read_object(info->oid, &inode, SD_INODE_HEADER_SIZE, 0, true);
+ if (ret != SD_RES_SUCCESS) {
+ fprintf(stderr, "Failed to read inode header, oid: "
+ "%"PRIx64"\n", info->oid);
+ return;
+ }
+
+ memcpy(&info->inode, &inode, sizeof(inode));
+
+ if (SD_INODE_HEADER_SIZE < info->size) {
+ unsigned int rlen =
+ DIV_ROUND_UP(inode.vdi_size, SD_DATA_OBJ_SIZE)
+ * sizeof(inode.data_vdi_id[0]);
+ size_t size = info->size;
+
+ if (size - SD_INODE_HEADER_SIZE < rlen)
+ rlen = size - SD_INODE_HEADER_SIZE;
+
+ ret = sd_read_object(info->oid,
+ ((char *)&inode) + SD_INODE_HEADER_SIZE,
+ rlen, SD_INODE_HEADER_SIZE, true);
+
+ if (ret != SD_RES_SUCCESS) {
+ fprintf(stderr, "Failed to read inode,"
+ " oid of the inode is: %"PRIx64"\n", info->oid);
+ return;
+ }
+
+ memcpy(((char *)&info->inode) + SD_INODE_HEADER_SIZE,
+ ((char *)&inode) + SD_INODE_HEADER_SIZE, rlen);
+ }
+
+ info->succeed = true;
+}
+
+static void parse_vdi_main(struct work *work)
+{
+ struct parse_vdi_info *info = container_of(work, struct parse_vdi_info,
+ work);
+ struct sd_inode *inode;
+ uint32_t snapid;
+
+ if (!info->succeed)
+ goto out;
+
+ inode = &info->inode;
+ if (inode->name[0] == '\0') /* this VDI has been deleted */
+ return;
+
+ snapid = vdi_is_snapshot(inode) ? inode->snap_id : 0;
+ info->func(inode->vdi_id, inode->name, inode->tag, snapid, 0, inode,
+ info->data);
+
+out:
+ free(info);
+}
+
+static struct work_queue *parse_vdi_wq;
+
#define FOR_EACH_VDI(nr, vdis) \
for (nr = find_next_bit((vdis), SD_NR_VDIS, 0); \
nr < SD_NR_VDIS; \
nr = find_next_bit((vdis), SD_NR_VDIS, nr + 1))
-int parse_vdi(vdi_parser_func_t func, size_t size, void *data)
+struct parse_vdi_producer_arg {
+ size_t size;
+ vdi_parser_func_t func;
+ void *data;
+};
+
+static void *parse_vdi_producer(void *raw_arg)
{
int ret;
- unsigned long nr;
- static struct sd_inode i;
struct sd_req req;
static DECLARE_BITMAP(vdi_inuse, SD_NR_VDIS);
- unsigned int rlen = sizeof(vdi_inuse);
+ unsigned long nr;
+ struct parse_vdi_producer_arg *arg = raw_arg;
sd_init_req(&req, SD_OP_READ_VDIS);
req.data_length = sizeof(vdi_inuse);
ret = collie_exec_req(sdhost, sdport, &req, &vdi_inuse);
- if (ret < 0)
- goto out;
+ if (ret < 0) {
+ fprintf(stderr, "requesting VDI bitmap failed: %m\n");
+ return NULL;
+ }
FOR_EACH_VDI(nr, vdi_inuse) {
- uint64_t oid;
- uint32_t snapid;
+ struct parse_vdi_info *info;
- oid = vid_to_vdi_oid(nr);
+ info = xzalloc(sizeof(*info));
- memset(&i, 0, sizeof(i));
- ret = sd_read_object(oid, &i, SD_INODE_HEADER_SIZE, 0, true);
- if (ret != SD_RES_SUCCESS) {
- fprintf(stderr, "Failed to read inode header\n");
- continue;
- }
+ info->oid = vid_to_vdi_oid(nr);
+ info->size = arg->size;
+ info->func = arg->func;
+ info->data = arg->data;
- if (i.name[0] == '\0') /* this VDI has been deleted */
- continue;
+ info->work.fn = parse_vdi_work;
+ info->work.done = parse_vdi_main;
- if (size > SD_INODE_HEADER_SIZE) {
- rlen = DIV_ROUND_UP(i.vdi_size, SD_DATA_OBJ_SIZE) *
- sizeof(i.data_vdi_id[0]);
- if (rlen > size - SD_INODE_HEADER_SIZE)
- rlen = size - SD_INODE_HEADER_SIZE;
+ queue_work(parse_vdi_wq, &info->work);
- ret = sd_read_object(oid, ((char *)&i) + SD_INODE_HEADER_SIZE,
- rlen, SD_INODE_HEADER_SIZE, true);
+ /* reap results if there are ready ones */
+ event_loop(0);
+ }
- if (ret != SD_RES_SUCCESS) {
- fprintf(stderr, "Failed to read inode\n");
- continue;
- }
- }
+ work_queue_wait(parse_vdi_wq);
- snapid = vdi_is_snapshot(&i) ? i.snap_id : 0;
- func(i.vdi_id, i.name, i.tag, snapid, 0, &i, data);
+ return NULL;
+}
+
+int parse_vdi(vdi_parser_func_t func, size_t size, void *data)
+{
+ int ret;
+ pthread_t producer;
+ struct parse_vdi_producer_arg arg;
+
+ parse_vdi_wq = create_work_queue("parse vdi", WQ_DYNAMIC);
+ if (!parse_vdi_wq) {
+ fprintf(stderr, "creating work queue for parsing VDIs failed:"
+ " %m\n");
+ return -1;
}
-out:
- return ret;
+ arg.size = size;
+ arg.func = func;
+ arg.data = data;
+ ret = pthread_create(&producer, NULL, parse_vdi_producer, &arg);
+ if (ret) {
+ fprintf(stderr, "creating a thread of producer failed: %m\n");
+ return -1;
+ }
+
+ ret = pthread_join(producer, NULL);
+ if (ret) {
+ fprintf(stderr, "waiting a producer thread failed: %m");
+ return -1;
+ }
+
+ return 0;
}
int collie_exec_req(const char *host, int port, struct sd_req *hdr, void *buf)
--
1.7.10.4
More information about the sheepdog
mailing list