[Sheepdog] [PATCH 2/2] sheep: object cache applied to farm and simple store

Li Wenpeng levin108 at gmail.com
Thu Mar 1 12:16:25 CET 2012


From: levin li <xingke.lwp at taobao.com>

Applied the object list cache to get_obj_list() which
just return the cached list, instead of calling sd_store->get_objlist(),
make it much faster to get the object list.

Signed-off-by: levin li <xingke.lwp at taobao.com>
---
 sheep/farm/farm.c    |   30 +++++++++++-------------------
 sheep/farm/farm.h    |    2 ++
 sheep/farm/trunk.c   |   10 ++++++++++
 sheep/sheep_priv.h   |    1 +
 sheep/simple_store.c |   50 +++++++++++++++++++++++++++++++++++++++++++++-----
 sheep/store.c        |   37 ++++++++++---------------------------
 6 files changed, 79 insertions(+), 51 deletions(-)

diff --git a/sheep/farm/farm.c b/sheep/farm/farm.c
index e130138..7befd56 100644
--- a/sheep/farm/farm.c
+++ b/sheep/farm/farm.c
@@ -12,6 +12,7 @@
  */
 
 #include <dirent.h>
+#include <pthread.h>
 
 #include "farm.h"
 #include "sheep_priv.h"
@@ -269,29 +270,20 @@ out:
 
 static int farm_get_objlist(struct siocb *iocb)
 {
-	struct sha1_file_hdr hdr;
-	struct trunk_entry *trunk_buf, *trunk_free = NULL;
-	unsigned char trunk_sha1[SHA1_LEN];
-	uint64_t nr_trunks, i;
 	uint64_t *objlist = (uint64_t *)iocb->buf;
-	int ret = SD_RES_NO_TAG;
-
-	if (get_trunk_sha1(iocb->epoch, trunk_sha1, 0) < 0)
-		goto out;
-
-	trunk_free = trunk_buf = trunk_file_read(trunk_sha1, &hdr);
-	if (!trunk_buf)
-		goto out;
+	struct list_head *active_list = trunk_get_active_list();
+	pthread_mutex_t lock = trunk_get_active_list_lock();
+	struct trunk_entry_incore *entry;
 
-	nr_trunks = hdr.priv;
-	for (i = 0; i < nr_trunks; i++, trunk_buf++)
-		objlist[iocb->length++] = trunk_buf->oid;
+	pthread_mutex_lock(&lock);
+	list_for_each_entry(entry, active_list, active_list) {
+		objlist[iocb->length++] = entry->raw.oid;
+	}
+	pthread_mutex_unlock(&lock);
 
 	dprintf("%"PRIu32"\n", iocb->length);
-	ret = SD_RES_SUCCESS;
-out:
-	free(trunk_free);
-	return ret;
+
+	return SD_RES_SUCCESS;
 }
 
 static void *retrieve_object_from_snap(uint64_t oid, int epoch)
diff --git a/sheep/farm/farm.h b/sheep/farm/farm.h
index 86aeadd..0ecf491 100644
--- a/sheep/farm/farm.h
+++ b/sheep/farm/farm.h
@@ -68,6 +68,8 @@ extern int trunk_update_entry(uint64_t oid);
 extern void trunk_reset(void);
 extern void trunk_put_entry(uint64_t oid);
 extern void trunk_get_entry(uint64_t oid);
+extern struct list_head *trunk_get_active_list(void);
+extern pthread_mutex_t trunk_get_active_list_lock(void);
 
 /* snap.c */
 extern int snap_init(void);
diff --git a/sheep/farm/trunk.c b/sheep/farm/trunk.c
index fabae8a..8631e32 100644
--- a/sheep/farm/trunk.c
+++ b/sheep/farm/trunk.c
@@ -378,3 +378,13 @@ void trunk_reset(void)
 	eprintf("%s\n", trunk_entry_active_nr ? "WARN: active_list not clean" :
 						"clean");
 }
+
+struct list_head *trunk_get_active_list(void)
+{
+	return &trunk_active_list;
+}
+
+pthread_mutex_t trunk_get_active_list_lock(void)
+{
+	return active_list_lock;
+}
diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index db70c57..6fd838c 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -296,6 +296,7 @@ int read_object(struct sd_vnode *e,
 int remove_object(struct sd_vnode *e,
 		  int vnodes, int zones, uint32_t node_version,
 		  uint64_t oid, int nr);
+int merge_objlist(uint64_t *list1, int nr_list1, uint64_t *list2, int nr_list2);
 
 void del_sheep_fd(int fd);
 int get_sheep_fd(uint8_t *addr, uint16_t port, int node_idx, uint32_t epoch);
diff --git a/sheep/simple_store.c b/sheep/simple_store.c
index e1cfeca..b30f75f 100644
--- a/sheep/simple_store.c
+++ b/sheep/simple_store.c
@@ -184,13 +184,12 @@ static int simple_store_close(uint64_t oid, struct siocb *iocb)
 	return SD_RES_SUCCESS;
 }
 
-static int simple_store_get_objlist(struct siocb *siocb)
+static int get_epoch_obj_list(int epoch, uint64_t *objlist, int *nr)
 {
 	struct strbuf buf = STRBUF_INIT;
-	int epoch = siocb->epoch;
-	uint64_t *objlist = (uint64_t *)siocb->buf;
 	DIR *dir;
 	struct dirent *d;
+	int length = 0;
 	int ret = SD_RES_SUCCESS;
 
 	strbuf_addf(&buf, "%s%08u/", obj_path, epoch);
@@ -202,7 +201,6 @@ static int simple_store_get_objlist(struct siocb *siocb)
 		ret = SD_RES_EIO;
 		goto out;
 	}
-
 	while ((d = readdir(dir))) {
 		uint64_t oid;
 		if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
@@ -212,14 +210,56 @@ static int simple_store_get_objlist(struct siocb *siocb)
 		if (oid == 0)
 			continue;
 
-		objlist[siocb->length++] = oid;
+		objlist[length++] = oid;
 	}
 	closedir(dir);
+	*nr = length;
 out:
 	strbuf_release(&buf);
 	return ret;
 }
 
+static int simple_store_get_objlist(struct siocb *siocb)
+{
+	uint64_t *objlist = (uint64_t*)siocb->buf;
+	uint64_t *buf;
+	int epoch, nr = 0, obj_nr = 0;
+	DIR *dir;
+	struct dirent *d;
+	int ret = SD_RES_SUCCESS, r;
+
+	dir = opendir(obj_path);
+	if (!dir) {
+		ret = SD_RES_EIO;
+		goto out;
+	}
+
+	buf = zalloc(1 << 22);
+	if (!buf) {
+		dprintf("no memory to allocate.\n");
+		ret = SD_RES_NO_MEM;
+		goto out;
+	}
+
+	while ((d = readdir(dir))) {
+		if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
+			continue;
+		epoch = strtoul(d->d_name, NULL, 16);
+		if (epoch == 0)
+			continue;
+
+		r = get_epoch_obj_list(epoch, buf, &obj_nr);
+		if (SD_RES_SUCCESS == r)
+			nr = merge_objlist(objlist, nr, buf, obj_nr);
+	}
+	closedir(dir);
+
+	siocb->length = nr;
+	free(buf);
+out:
+	return ret;
+}
+
 static int simple_store_link(uint64_t oid, struct siocb *iocb, int tgt_epoch)
 {
        char old[PATH_MAX], new[PATH_MAX];
diff --git a/sheep/store.c b/sheep/store.c
index e90bf96..33e05a8 100644
--- a/sheep/store.c
+++ b/sheep/store.c
@@ -226,41 +226,24 @@ out:
 	return ret;
 }
 
-static int merge_objlist(uint64_t *list1, int nr_list1,
-			 uint64_t *list2, int nr_list2);
-
 int get_obj_list(const struct sd_list_req *hdr, struct sd_list_rsp *rsp, void *data)
 {
 	uint64_t *list = (uint64_t *)data;
 	int i, nr = 0;
 	int res = SD_RES_SUCCESS;
-	int buf_len;
-	char *buf;
-
-	/* FIXME: handle larger size */
-	buf_len = (1 << 22);
-	buf = zalloc(buf_len);
-	if (!buf) {
-		eprintf("failed to allocate memory\n");
-		res = SD_RES_NO_MEM;
-		goto out;
-	}
 
-	for (i = 1; i <= hdr->tgt_epoch; i++) {
-		struct siocb iocb = { 0 };
+	struct objlist_cache_entry *entry;
+	struct hlist_node *node;
 
-		iocb.buf = buf;
-		iocb.length = 0;
-		iocb.epoch = i;
-		sd_store->get_objlist(&iocb);
-		nr = merge_objlist(list, nr, (uint64_t *)iocb.buf, iocb.length);
+	pthread_rwlock_rdlock(&obj_list_cache.lock);
+	for (i = 0; i < obj_list_cache.hash_size; i++) {
+		hlist_for_each_entry(entry, node, obj_list_cache.hashtable + i, list)
+			list[nr++] = entry->oid;
 	}
-out:
-	free(buf);
+	pthread_rwlock_unlock(&obj_list_cache.lock);
+
 	rsp->data_length = nr * sizeof(uint64_t);
-	for (i = 0; i < nr; i++) {
-		dprintf("oid %"PRIx64"\n", list[i]);
-	}
+
 	return res;
 }
 
@@ -1733,7 +1716,7 @@ static int request_obj_list(struct sd_node *e, uint32_t epoch,
 	return rsp->data_length / sizeof(uint64_t);
 }
 
-static int merge_objlist(uint64_t *list1, int nr_list1, uint64_t *list2, int nr_list2)
+int merge_objlist(uint64_t *list1, int nr_list1, uint64_t *list2, int nr_list2)
 {
 	int i;
 	int old_nr_list1 = nr_list1;
-- 
1.7.1




More information about the sheepdog mailing list