[Sheepdog] [PATCH 2/2] sheep: object cache applied to farm and simple store

Liu Yuan namei.unix at gmail.com
Sun Mar 11 14:21:51 CET 2012


On 03/01/2012 07:16 PM, Li Wenpeng wrote:

> From: levin li <xingke.lwp at taobao.com>
> 
> Applied the object list cache to get_obj_list() which
> just return the cached list, instead of calling sd_store->get_objlist(),
> make it much faster to get the object list.
> 
> Signed-off-by: levin li <xingke.lwp at taobao.com>
> ---
>  sheep/farm/farm.c    |   30 +++++++++++-------------------
>  sheep/farm/farm.h    |    2 ++
>  sheep/farm/trunk.c   |   10 ++++++++++
>  sheep/sheep_priv.h   |    1 +
>  sheep/simple_store.c |   50 +++++++++++++++++++++++++++++++++++++++++++++-----
>  sheep/store.c        |   37 ++++++++++---------------------------
>  6 files changed, 79 insertions(+), 51 deletions(-)
> 
> diff --git a/sheep/farm/farm.c b/sheep/farm/farm.c
> index e130138..7befd56 100644
> --- a/sheep/farm/farm.c
> +++ b/sheep/farm/farm.c
> @@ -12,6 +12,7 @@
>   */
>  
>  #include <dirent.h>
> +#include <pthread.h>
>  
>  #include "farm.h"
>  #include "sheep_priv.h"
> @@ -269,29 +270,20 @@ out:
>  
>  static int farm_get_objlist(struct siocb *iocb)
>  {
> -	struct sha1_file_hdr hdr;
> -	struct trunk_entry *trunk_buf, *trunk_free = NULL;
> -	unsigned char trunk_sha1[SHA1_LEN];
> -	uint64_t nr_trunks, i;
>  	uint64_t *objlist = (uint64_t *)iocb->buf;
> -	int ret = SD_RES_NO_TAG;
> -
> -	if (get_trunk_sha1(iocb->epoch, trunk_sha1, 0) < 0)
> -		goto out;
> -
> -	trunk_free = trunk_buf = trunk_file_read(trunk_sha1, &hdr);
> -	if (!trunk_buf)
> -		goto out;
> +	struct list_head *active_list = trunk_get_active_list();

> +	pthread_mutex_t lock = trunk_get_active_list_lock();


Both active list and active_list lock are internal state of trunk
object, we should try our best to isolate internal implementation from
its users, by well-devised interfaces.

This is ugly to get pthread lock that is used by trunk.c internally.
we can simply pass a buffer to a function, say
trunk_get_working_objlist(), and get it filled with oid.

Thanks,
Yuan

> +	struct trunk_entry_incore *entry;
>  
> -	nr_trunks = hdr.priv;
> -	for (i = 0; i < nr_trunks; i++, trunk_buf++)
> -		objlist[iocb->length++] = trunk_buf->oid;
> +	pthread_mutex_lock(&lock);
> +	list_for_each_entry(entry, active_list, active_list) {
> +		objlist[iocb->length++] = entry->raw.oid;
> +	}
> +	pthread_mutex_unlock(&lock);
>  
>  	dprintf("%"PRIu32"\n", iocb->length);
> -	ret = SD_RES_SUCCESS;
> -out:
> -	free(trunk_free);
> -	return ret;
> +
> +	return SD_RES_SUCCESS;
>  }
>  
>  static void *retrieve_object_from_snap(uint64_t oid, int epoch)
> diff --git a/sheep/farm/farm.h b/sheep/farm/farm.h
> index 86aeadd..0ecf491 100644
> --- a/sheep/farm/farm.h
> +++ b/sheep/farm/farm.h
> @@ -68,6 +68,8 @@ extern int trunk_update_entry(uint64_t oid);
>  extern void trunk_reset(void);
>  extern void trunk_put_entry(uint64_t oid);
>  extern void trunk_get_entry(uint64_t oid);
> +extern struct list_head *trunk_get_active_list(void);
> +extern pthread_mutex_t trunk_get_active_list_lock(void);
>  
>  /* snap.c */
>  extern int snap_init(void);
> diff --git a/sheep/farm/trunk.c b/sheep/farm/trunk.c
> index fabae8a..8631e32 100644
> --- a/sheep/farm/trunk.c
> +++ b/sheep/farm/trunk.c
> @@ -378,3 +378,13 @@ void trunk_reset(void)
>  	eprintf("%s\n", trunk_entry_active_nr ? "WARN: active_list not clean" :
>  						"clean");
>  }
> +
> +struct list_head *trunk_get_active_list(void)
> +{
> +	return &trunk_active_list;
> +}
> +
> +pthread_mutex_t trunk_get_active_list_lock(void)
> +{
> +	return active_list_lock;
> +}
> diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
> index db70c57..6fd838c 100644
> --- a/sheep/sheep_priv.h
> +++ b/sheep/sheep_priv.h
> @@ -296,6 +296,7 @@ int read_object(struct sd_vnode *e,
>  int remove_object(struct sd_vnode *e,
>  		  int vnodes, int zones, uint32_t node_version,
>  		  uint64_t oid, int nr);
> +int merge_objlist(uint64_t *list1, int nr_list1, uint64_t *list2, int nr_list2);
>  
>  void del_sheep_fd(int fd);
>  int get_sheep_fd(uint8_t *addr, uint16_t port, int node_idx, uint32_t epoch);
> diff --git a/sheep/simple_store.c b/sheep/simple_store.c
> index e1cfeca..b30f75f 100644
> --- a/sheep/simple_store.c
> +++ b/sheep/simple_store.c
> @@ -184,13 +184,12 @@ static int simple_store_close(uint64_t oid, struct siocb *iocb)
>  	return SD_RES_SUCCESS;
>  }
>  
> -static int simple_store_get_objlist(struct siocb *siocb)
> +static int get_epoch_obj_list(int epoch, uint64_t *objlist, int *nr)
>  {
>  	struct strbuf buf = STRBUF_INIT;
> -	int epoch = siocb->epoch;
> -	uint64_t *objlist = (uint64_t *)siocb->buf;
>  	DIR *dir;
>  	struct dirent *d;
> +	int length = 0;
>  	int ret = SD_RES_SUCCESS;
>  
>  	strbuf_addf(&buf, "%s%08u/", obj_path, epoch);
> @@ -202,7 +201,6 @@ static int simple_store_get_objlist(struct siocb *siocb)
>  		ret = SD_RES_EIO;
>  		goto out;
>  	}
> -
>  	while ((d = readdir(dir))) {
>  		uint64_t oid;
>  		if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
> @@ -212,14 +210,56 @@ static int simple_store_get_objlist(struct siocb *siocb)
>  		if (oid == 0)
>  			continue;
>  
> -		objlist[siocb->length++] = oid;
> +		objlist[length++] = oid;
>  	}
>  	closedir(dir);
> +	*nr = length;
>  out:
>  	strbuf_release(&buf);
>  	return ret;
>  }
>  
> +static int simple_store_get_objlist(struct siocb *siocb)
> +{
> +	uint64_t *objlist = (uint64_t*)siocb->buf;
> +	uint64_t *buf;
> +	int epoch, nr = 0, obj_nr = 0;
> +	DIR *dir;
> +	struct dirent *d;
> +	int ret = SD_RES_SUCCESS, r;
> +
> +	dir = opendir(obj_path);
> +	if (!dir) {
> +		ret = SD_RES_EIO;
> +		goto out;
> +	}
> +
> +	buf = zalloc(1 << 22);
> +	if (!buf) {
> +		dprintf("no memory to allocate.\n");
> +		ret = SD_RES_NO_MEM;
> +		goto out;
> +	}
> +
> +	while ((d = readdir(dir))) {
> +		if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
> +			continue;
> +		epoch = strtoul(d->d_name, NULL, 16);
> +		if (epoch == 0)
> +			continue;
> +
> +		r = get_epoch_obj_list(epoch, buf, &obj_nr);
> +		if (SD_RES_SUCCESS == r)
> +			nr = merge_objlist(objlist, nr, buf, obj_nr);
> +	}
> +	closedir(dir);
> +
> +	siocb->length = nr;
> +	free(buf);
> +out:
> +	return ret;
> +}
> +
>  static int simple_store_link(uint64_t oid, struct siocb *iocb, int tgt_epoch)
>  {
>         char old[PATH_MAX], new[PATH_MAX];
> diff --git a/sheep/store.c b/sheep/store.c
> index e90bf96..33e05a8 100644
> --- a/sheep/store.c
> +++ b/sheep/store.c
> @@ -226,41 +226,24 @@ out:
>  	return ret;
>  }
>  
> -static int merge_objlist(uint64_t *list1, int nr_list1,
> -			 uint64_t *list2, int nr_list2);
> -
>  int get_obj_list(const struct sd_list_req *hdr, struct sd_list_rsp *rsp, void *data)
>  {
>  	uint64_t *list = (uint64_t *)data;
>  	int i, nr = 0;
>  	int res = SD_RES_SUCCESS;
> -	int buf_len;
> -	char *buf;
> -
> -	/* FIXME: handle larger size */
> -	buf_len = (1 << 22);
> -	buf = zalloc(buf_len);
> -	if (!buf) {
> -		eprintf("failed to allocate memory\n");
> -		res = SD_RES_NO_MEM;
> -		goto out;
> -	}
>  
> -	for (i = 1; i <= hdr->tgt_epoch; i++) {
> -		struct siocb iocb = { 0 };
> +	struct objlist_cache_entry *entry;
> +	struct hlist_node *node;
>  
> -		iocb.buf = buf;
> -		iocb.length = 0;
> -		iocb.epoch = i;
> -		sd_store->get_objlist(&iocb);
> -		nr = merge_objlist(list, nr, (uint64_t *)iocb.buf, iocb.length);
> +	pthread_rwlock_rdlock(&obj_list_cache.lock);
> +	for (i = 0; i < obj_list_cache.hash_size; i++) {
> +		hlist_for_each_entry(entry, node, obj_list_cache.hashtable + i, list)
> +			list[nr++] = entry->oid;
>  	}
> -out:
> -	free(buf);
> +	pthread_rwlock_unlock(&obj_list_cache.lock);
> +
>  	rsp->data_length = nr * sizeof(uint64_t);
> -	for (i = 0; i < nr; i++) {
> -		dprintf("oid %"PRIx64"\n", list[i]);
> -	}
> +
>  	return res;
>  }
>  
> @@ -1733,7 +1716,7 @@ static int request_obj_list(struct sd_node *e, uint32_t epoch,
>  	return rsp->data_length / sizeof(uint64_t);
>  }
>  
> -static int merge_objlist(uint64_t *list1, int nr_list1, uint64_t *list2, int nr_list2)
> +int merge_objlist(uint64_t *list1, int nr_list1, uint64_t *list2, int nr_list2)
>  {
>  	int i;
>  	int old_nr_list1 = nr_list1;





More information about the sheepdog mailing list