[sheepdog] [PATCH 5/8] object cache: schedule the object cache in a lru list

[levin li]

From: levin li <xingke.lwp at taobao.com>

We put all the cached object into a global lru list, when
the object cache is referenced(read/write), we move the object
to the head of the lru list, then when cache reaches the max
size we can reclaim it from the end of the lru list.

Signed-off-by: levin li <xingke.lwp at taobao.com>
---
 sheep/object_cache.c |   54 +++++++++++++++++++++++++++++++++++++-------------
 1 files changed, 40 insertions(+), 14 deletions(-)

diff --git a/sheep/object_cache.c b/sheep/object_cache.c
index d4313ff..17ff190 100644
--- a/sheep/object_cache.c
+++ b/sheep/object_cache.c
@@ -306,12 +306,24 @@ static void switch_dirty_tree_and_list(struct object_cache *oc,
 /* Caller should hold the oc->lock */
 static inline void
 add_to_dirty_tree_and_list(struct object_cache *oc,
-			   struct dirty_cache_entry *entry)
+			   struct dirty_cache_entry *dirty_entry, int merge)
 {
-	if (!dirty_tree_insert(oc->active_dirty_tree, entry))
-		list_add(&entry->list, oc->active_dirty_list);
-	else
-		free(entry);
+	if (!dirty_tree_insert(oc->active_dirty_tree, dirty_entry)) {
+		list_add(&dirty_entry->list, oc->active_dirty_list);
+	} else {
+		free(dirty_entry);
+
+		if (!merge) {
+			struct object_cache_entry *entry;
+
+			entry = dirty_entry->sys_entry;
+			/* If cache isn't in reclaiming, move it
+			 * to the head of lru list */
+			cds_list_del_rcu(&entry->lru_list);
+			cds_list_add_rcu(&entry->lru_list,
+					 &sys_cache.cache_lru_list);
+		}
+	}
 }
 
 static void merge_dirty_tree_and_list(struct object_cache *oc,
@@ -324,17 +336,19 @@ static void merge_dirty_tree_and_list(struct object_cache *oc,
 
 	list_for_each_entry_safe(entry, t, inactive_dirty_list, list) {
 		del_from_dirty_tree_and_list(entry, inactive_dirty_tree);
-		add_to_dirty_tree_and_list(oc, entry);
+		add_to_dirty_tree_and_list(oc, entry, 1);
 	}
 
 	pthread_mutex_unlock(&oc->lock);
 }
 
 static inline struct dirty_cache_entry *
-alloc_cache_entry(uint32_t idx, uint64_t bmap, int create)
+alloc_cache_entry(struct object_cache_entry *oc_entry, uint32_t idx,
+		  uint64_t bmap, int create)
 {
 	struct dirty_cache_entry *entry = xzalloc(sizeof(*entry));
 
+	entry->sys_entry = oc_entry;
 	entry->idx = idx;
 	entry->bmap = bmap;
 	entry->create = create;
@@ -377,6 +391,8 @@ static int object_cache_lookup(struct object_cache *oc, uint32_t idx,
 {
 	struct strbuf buf;
 	int fd, ret = 0, flags = def_open_flags;
+	struct object_cache_entry *entry = NULL;
+	struct dirty_cache_entry *dirty_entry;
 
 	strbuf_init(&buf, PATH_MAX);
 	strbuf_addstr(&buf, cache_dir);
@@ -392,7 +408,6 @@ static int object_cache_lookup(struct object_cache *oc, uint32_t idx,
 	}
 
 	if (create) {
-		struct dirty_cache_entry *entry;
 		unsigned data_length;
 
 		if (idx_has_vdi_bit(idx))
@@ -406,11 +421,11 @@ static int object_cache_lookup(struct object_cache *oc, uint32_t idx,
 		else {
 			uint64_t bmap = UINT64_MAX;
 
-			add_to_object_cache(oc, idx);
+			entry = add_to_object_cache(oc, idx);
 
-			entry = alloc_cache_entry(idx, bmap, 1);
+			dirty_entry = alloc_cache_entry(entry, idx, bmap, 1);
 			pthread_mutex_lock(&oc->lock);
-			add_to_dirty_tree_and_list(oc, entry);
+			add_to_dirty_tree_and_list(oc, dirty_entry, 0);
 			pthread_mutex_unlock(&oc->lock);
 		}
 	}
@@ -516,22 +531,27 @@ static int object_cache_rw(struct object_cache *oc, uint32_t idx,
 {
 	struct sd_req *hdr = &req->rq;
 	uint64_t bmap = 0;
+	struct object_cache_entry *entry;
 	int ret;
 
 	dprintf("%08"PRIx32", len %"PRIu32", off %"PRIu64"\n", idx,
 		hdr->data_length, hdr->obj.offset);
 
+	pthread_mutex_lock(&oc->lock);
+	entry = object_tree_search(&oc->object_tree, idx);
+	pthread_mutex_unlock(&oc->lock);
+
 	if (hdr->flags & SD_FLAG_CMD_WRITE) {
-		struct dirty_cache_entry *entry;
+		struct dirty_cache_entry *dirty_entry;
 
 		ret = write_cache_object(oc->vid, idx, req->data,
 					 hdr->data_length, hdr->obj.offset);
 		if (ret != SD_RES_SUCCESS)
 			goto out;
 		bmap = calc_object_bmap(hdr->data_length, hdr->obj.offset);
-		entry = alloc_cache_entry(idx, bmap, 0);
+		dirty_entry = alloc_cache_entry(entry, idx, bmap, 0);
 		pthread_mutex_lock(&oc->lock);
-		add_to_dirty_tree_and_list(oc, entry);
+		add_to_dirty_tree_and_list(oc, dirty_entry, 0);
 		pthread_mutex_unlock(&oc->lock);
 	} else {
 		ret = read_cache_object(oc->vid, idx, req->data,
@@ -539,6 +559,12 @@ static int object_cache_rw(struct object_cache *oc, uint32_t idx,
 		if (ret != SD_RES_SUCCESS)
 			goto out;
 		req->rp.data_length = hdr->data_length;
+
+		if (entry) {
+			cds_list_del_rcu(&entry->lru_list);
+			cds_list_add_rcu(&entry->lru_list,
+					 &sys_cache.cache_lru_list);
+		}
 	}
 out:
 	return ret;
-- 
1.7.1