[sheepdog] [PATCH v2 3/3] lib: add cache version of writer and reader

[Liu Yuan]

On Mon, Feb 17, 2014 at 07:07:17PM +0800, Robin Dong wrote:
> From: Robin Dong <sanbai at taobao.com>
> 
> Add new functions to repleace default writer and reader so it can cache
> entire ext-node when set_vid_for_btree() need to access ext-node.
> 
> Signed-off-by: Robin Dong <sanbai at taobao.com>
> ---
>  lib/sd_inode.c      | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>  sheep/http/oalloc.c |   4 +-
>  2 files changed, 111 insertions(+), 2 deletions(-)
> 
> diff --git a/lib/sd_inode.c b/lib/sd_inode.c
> index 53035b0..a02d472 100644
> --- a/lib/sd_inode.c
> +++ b/lib/sd_inode.c
> @@ -216,6 +216,106 @@ static void dump_btree(read_node_fn reader, struct sd_inode *inode)
>  }
>  
>  /*
> + * This is the cache for inode and ext-node (B-tree), so we name it 'icache'.
> + * Cache of the same inode and ext-node dose not support concurrent operations
> + * so it could only be used in sd_inode_set_vid() which will be protected by
> + * distributed lock and should be released in the end of sd_inode_set_vid().
> + */
> +static write_node_fn raw_writer;
> +static read_node_fn raw_reader;

raw_{reader, writer} -> caller_{reader, writer} is easier to understand.

> +
> +/* no rationale */
> +#define NUMBER_OF_CACHE	4
> +
> +struct inode_cache {
> +	uint64_t oid;
> +	unsigned char mem[SD_INODE_DATA_INDEX_SIZE];
> +} cache_array[NUMBER_OF_CACHE];
> +static int cache_idx;
> +
> +static void icache_init(void)
> +{
> +	cache_idx = 0;
> +}
> +
> +static void icache_writeout(write_node_fn writer, int copies, int policy)
> +{
> +	int i;
> +	for (i = 0; i < cache_idx; i++) {
> +		writer(cache_array[i].oid, cache_array[i].mem,
> +		       SD_INODE_DATA_INDEX_SIZE, 0, 0, copies, policy,
> +		       false, false);
> +	}
> +}
> +
> +static void icache_release(write_node_fn writer, int copies, int policy)
> +{
> +	icache_writeout(writer, copies, policy);
> +	icache_init();
> +}
> +
> +static void icache_insert(write_node_fn writer, int copies, int policy,
> +			 uint64_t oid, void *mem)
> +{
> +	int i;
> +	for (i = 0; i < cache_idx; i++) {
> +		if (oid == cache_array[i].oid) {
> +			memcpy(cache_array[i].mem, mem,
> +			       SD_INODE_DATA_INDEX_SIZE);
> +			return;
> +		}
> +	}
> +
> +	if (cache_idx == (NUMBER_OF_CACHE - 1)) {
> +		sd_info("cache for B-tree is full, so write all out");

why user should bother with this information? I think sd_debug is good enough.

> +		icache_release(writer, copies, policy);
> +	}
> +
> +	/* insert new cache */
> +	cache_array[cache_idx].oid = oid;
> +	memcpy(cache_array[cache_idx].mem, mem, SD_INODE_DATA_INDEX_SIZE);
> +	cache_idx++;
> +}
> +
> +static void *icache_find(uint64_t oid)
> +{
> +	int i;
> +	for (i = 0; i < cache_idx; i++) {
> +		if (cache_array[i].oid == oid)
> +			return cache_array[i].mem;
> +	}
> +	return NULL;
> +}
> +
> +static int icache_writer(uint64_t id, void *mem, unsigned int len,
> +			 uint64_t offset, uint32_t flags, int copies,
> +			 int copy_policy, bool create, bool direct)
> +{
> +	/* Only try to cache entire ext-node */
> +	if (!offset && !create && !direct && len == SD_INODE_DATA_INDEX_SIZE) {
> +		icache_insert(raw_writer, copies, copy_policy, id, mem);
> +		return SD_RES_SUCCESS;
> +	}
> +	return raw_writer(id, mem, len, offset, flags, copies, copy_policy,
> +			  create, direct);
> +}
> +
> +static int icache_reader(uint64_t id, void **mem, unsigned int len,
> +			 uint64_t offset)
> +{
> +	void *data;
> +
> +	if (!offset && len == SD_INODE_DATA_INDEX_SIZE) {
> +		data = icache_find(id);
> +		if (data) {
> +			memcpy(*mem, data, len);
> +			return SD_RES_SUCCESS;
> +		}
> +	}
> +	return raw_reader(id, mem, len, offset);
> +}
> +
> +/*
>   * Search for the key in a B-tree node. If can't find it, return the position
>   * for insert operation. So we can't just use xbsearch().
>   */
> @@ -626,6 +726,13 @@ void sd_inode_set_vid(write_node_fn writer, read_node_fn reader,
>  	struct sd_extent_header *header;
>  	int idx;
>  
> +	/* save default writer and reader */
> +	raw_writer = writer;
> +	raw_reader = reader;
> +	/* use cache version of writer and reader */
> +	writer = icache_writer;
> +	reader = icache_reader;
> +
>  	for (idx = idx_start; idx <= idx_end; idx++) {
>  		if (inode->store_policy == 0)
>  			inode->data_vdi_id[idx] = vdi_id;
> @@ -641,6 +748,8 @@ void sd_inode_set_vid(write_node_fn writer, read_node_fn reader,
>  	}
>  	if (inode->store_policy != 0)
>  		dump_btree(reader, inode);
> +
> +	icache_release(raw_writer, inode->nr_copies, inode->copy_policy);
>  }
>  
>  /*
> diff --git a/sheep/http/oalloc.c b/sheep/http/oalloc.c
> index b3d9583..3f3fb8b 100644
> --- a/sheep/http/oalloc.c
> +++ b/sheep/http/oalloc.c
> @@ -178,8 +178,8 @@ int oalloc_new_finish(uint32_t vid, uint64_t start, uint64_t count)
>  		goto out;
>  	}
>  
> -	for (uint64_t i = 0; i < count; i++)
> -		INODE_SET_VID(inode, start + i, vid);
> +	sd_info("start %"PRIu64" end %"PRIu64, start, start + count - 1);

ditto.

Thanks
Yuan