[Sheepdog] [PATCH v2 2/3] sheep: object list cache applied to farm and simple store

Li Wenpeng levin108 at gmail.com
Tue Mar 13 03:10:24 CET 2012


From: levin li <xingke.lwp at taobao.com>

Applied the object list cache to get_obj_list() which
walk through the rb-tree cache and return the cached list,
instead of calling sd_store->get_objlist(),
make it much faster to get the object list.

Signed-off-by: levin li <xingke.lwp at taobao.com>
---
 sheep/farm/farm.c    |   23 ++++-------------------
 sheep/farm/farm.h    |    1 +
 sheep/farm/trunk.c   |   14 ++++++++++++++
 sheep/sheep_priv.h   |    1 +
 sheep/simple_store.c |   50 +++++++++++++++++++++++++++++++++++++++++++++-----
 sheep/store.c        |   38 ++++++++++----------------------------
 6 files changed, 75 insertions(+), 52 deletions(-)

diff --git a/sheep/farm/farm.c b/sheep/farm/farm.c
index 1e073f0..6db7dab 100644
--- a/sheep/farm/farm.c
+++ b/sheep/farm/farm.c
@@ -12,6 +12,7 @@
  */
 
 #include <dirent.h>
+#include <pthread.h>
 
 #include "farm.h"
 #include "sheep_priv.h"
@@ -269,29 +270,13 @@ out:
 
 static int farm_get_objlist(struct siocb *iocb)
 {
-	struct sha1_file_hdr hdr;
-	struct trunk_entry *trunk_buf, *trunk_free = NULL;
-	unsigned char trunk_sha1[SHA1_LEN];
-	uint64_t nr_trunks, i;
 	uint64_t *objlist = (uint64_t *)iocb->buf;
-	int ret = SD_RES_NO_TAG;
 
-	if (get_trunk_sha1(iocb->epoch, trunk_sha1, 0) < 0)
-		goto out;
-
-	trunk_free = trunk_buf = trunk_file_read(trunk_sha1, &hdr);
-	if (!trunk_buf)
-		goto out;
-
-	nr_trunks = hdr.priv;
-	for (i = 0; i < nr_trunks; i++, trunk_buf++)
-		objlist[iocb->length++] = trunk_buf->oid;
+	iocb->length = trunk_get_working_objlist(objlist);
 
 	dprintf("%"PRIu32"\n", iocb->length);
-	ret = SD_RES_SUCCESS;
-out:
-	free(trunk_free);
-	return ret;
+
+	return SD_RES_SUCCESS;
 }
 
 static void *retrieve_object_from_snap(uint64_t oid, int epoch)
diff --git a/sheep/farm/farm.h b/sheep/farm/farm.h
index 86aeadd..a9a1bbf 100644
--- a/sheep/farm/farm.h
+++ b/sheep/farm/farm.h
@@ -68,6 +68,7 @@ extern int trunk_update_entry(uint64_t oid);
 extern void trunk_reset(void);
 extern void trunk_put_entry(uint64_t oid);
 extern void trunk_get_entry(uint64_t oid);
+extern int trunk_get_working_objlist(uint64_t *list);
 
 /* snap.c */
 extern int snap_init(void);
diff --git a/sheep/farm/trunk.c b/sheep/farm/trunk.c
index b9f0b3e..2949704 100644
--- a/sheep/farm/trunk.c
+++ b/sheep/farm/trunk.c
@@ -379,3 +379,17 @@ void trunk_reset(void)
 	eprintf("%s\n", trunk_entry_active_nr ? "WARN: active_list not clean" :
 						"clean");
 }
+
+int trunk_get_working_objlist(uint64_t *list)
+{
+	int nr = 0;
+	struct trunk_entry_incore *entry;
+
+	pthread_mutex_lock(&active_list_lock);
+	list_for_each_entry(entry, &trunk_active_list, active_list) {
+		list[nr++] = entry->raw.oid;
+	}
+	pthread_mutex_unlock(&active_list_lock);
+
+	return nr;
+}
diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index e93806e..c38022b 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -299,6 +299,7 @@ int read_object(struct sd_vnode *e,
 int remove_object(struct sd_vnode *e,
 		  int vnodes, int zones, uint32_t node_version,
 		  uint64_t oid, int nr);
+int merge_objlist(uint64_t *list1, int nr_list1, uint64_t *list2, int nr_list2);
 
 void del_sheep_fd(int fd);
 int get_sheep_fd(uint8_t *addr, uint16_t port, int node_idx, uint32_t epoch);
diff --git a/sheep/simple_store.c b/sheep/simple_store.c
index 7cdad31..dd3c62d 100644
--- a/sheep/simple_store.c
+++ b/sheep/simple_store.c
@@ -185,13 +185,12 @@ static int simple_store_close(uint64_t oid, struct siocb *iocb)
 	return SD_RES_SUCCESS;
 }
 
-static int simple_store_get_objlist(struct siocb *siocb)
+static int get_epoch_obj_list(int epoch, uint64_t *objlist, int *nr)
 {
 	struct strbuf buf = STRBUF_INIT;
-	int epoch = siocb->epoch;
-	uint64_t *objlist = (uint64_t *)siocb->buf;
 	DIR *dir;
 	struct dirent *d;
+	int length = 0;
 	int ret = SD_RES_SUCCESS;
 
 	strbuf_addf(&buf, "%s%08u/", obj_path, epoch);
@@ -203,7 +202,6 @@ static int simple_store_get_objlist(struct siocb *siocb)
 		ret = SD_RES_EIO;
 		goto out;
 	}
-
 	while ((d = readdir(dir))) {
 		uint64_t oid;
 		if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
@@ -213,14 +211,56 @@ static int simple_store_get_objlist(struct siocb *siocb)
 		if (oid == 0)
 			continue;
 
-		objlist[siocb->length++] = oid;
+		objlist[length++] = oid;
 	}
 	closedir(dir);
+	*nr = length;
 out:
 	strbuf_release(&buf);
 	return ret;
 }
 
+static int simple_store_get_objlist(struct siocb *siocb)
+{
+	uint64_t *objlist = (uint64_t*)siocb->buf;
+	uint64_t *buf;
+	int epoch, nr = 0, obj_nr = 0;
+	DIR *dir;
+	struct dirent *d;
+	int ret = SD_RES_SUCCESS, r;
+
+	dir = opendir(obj_path);
+	if (!dir) {
+		ret = SD_RES_EIO;
+		goto out;
+	}
+
+	buf = zalloc(1 << 22);
+	if (!buf) {
+		dprintf("no memory to allocate.\n");
+		ret = SD_RES_NO_MEM;
+		goto out;
+	}
+
+	while ((d = readdir(dir))) {
+		if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
+			continue;
+		epoch = strtoul(d->d_name, NULL, 16);
+		if (epoch == 0)
+			continue;
+
+		r = get_epoch_obj_list(epoch, buf, &obj_nr);
+		if (SD_RES_SUCCESS == r)
+			nr = merge_objlist(objlist, nr, buf, obj_nr);
+	}
+	closedir(dir);
+
+	siocb->length = nr;
+	free(buf);
+out:
+	return ret;
+}
+
 static int simple_store_link(uint64_t oid, struct siocb *iocb, int tgt_epoch)
 {
        char old[PATH_MAX], new[PATH_MAX];
diff --git a/sheep/store.c b/sheep/store.c
index 17922b2..0069eaf 100644
--- a/sheep/store.c
+++ b/sheep/store.c
@@ -177,41 +177,23 @@ out:
 	return ret;
 }
 
-static int merge_objlist(uint64_t *list1, int nr_list1,
-			 uint64_t *list2, int nr_list2);
-
 int get_obj_list(const struct sd_list_req *hdr, struct sd_list_rsp *rsp, void *data)
 {
 	uint64_t *list = (uint64_t *)data;
-	int i, nr = 0;
+	int nr = 0;
 	int res = SD_RES_SUCCESS;
-	int buf_len;
-	char *buf;
+	struct objlist_cache_entry *entry;
+	struct rb_node *p;
 
-	/* FIXME: handle larger size */
-	buf_len = (1 << 22);
-	buf = zalloc(buf_len);
-	if (!buf) {
-		eprintf("failed to allocate memory\n");
-		res = SD_RES_NO_MEM;
-		goto out;
+	pthread_rwlock_rdlock(&obj_list_cache.lock);
+	for (p = rb_first(&obj_list_cache.root); p; p = rb_next(p)) {
+		entry = rb_entry(p, struct objlist_cache_entry, node);
+		list[nr++] = entry->oid;
 	}
+	pthread_rwlock_unlock(&obj_list_cache.lock);
 
-	for (i = 1; i <= hdr->tgt_epoch; i++) {
-		struct siocb iocb = { 0 };
-
-		iocb.buf = buf;
-		iocb.length = 0;
-		iocb.epoch = i;
-		sd_store->get_objlist(&iocb);
-		nr = merge_objlist(list, nr, (uint64_t *)iocb.buf, iocb.length);
-	}
-out:
-	free(buf);
 	rsp->data_length = nr * sizeof(uint64_t);
-	for (i = 0; i < nr; i++) {
-		dprintf("oid %"PRIx64"\n", list[i]);
-	}
+
 	return res;
 }
 
@@ -1720,7 +1702,7 @@ static int request_obj_list(struct sd_node *e, uint32_t epoch,
 	return rsp->data_length / sizeof(uint64_t);
 }
 
-static int merge_objlist(uint64_t *list1, int nr_list1, uint64_t *list2, int nr_list2)
+int merge_objlist(uint64_t *list1, int nr_list1, uint64_t *list2, int nr_list2)
 {
 	int i;
 	int old_nr_list1 = nr_list1;
-- 
1.7.1




More information about the sheepdog mailing list