[sheepdog] [PATCH 4/6] object cache: fix false reclaim after sheep restart

[Liu Yuan]

From: Liu Yuan <tailai.ly at taobao.com>

false reclaim will drop possible dirty objects as clean object  after restart
and lead to various guest problems.

We don't know VM's cache type after restarting, so we assume that it is
writeback and mark all the objects diry to avoid false reclaim. Donot try to
reclaim at loading phase becaue cluster isn't fully working.

Signed-off-by: Liu Yuan <tailai.ly at taobao.com>
---
 sheep/object_cache.c |   41 +++++++++++++++++++++++++++--------------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/sheep/object_cache.c b/sheep/object_cache.c
index 0894b2c..b687dc3 100644
--- a/sheep/object_cache.c
+++ b/sheep/object_cache.c
@@ -420,6 +420,7 @@ static int do_reclaim_object(struct object_cache_entry *entry)
 	struct object_cache *oc = entry->oc;
 	uint64_t oid;
 	int ret = 0;
+	uint32_t size;
 
 	oid = idx_to_oid(oc->vid, entry_idx(entry));
 	pthread_rwlock_wrlock(&oc->lock);
@@ -440,8 +441,9 @@ static int do_reclaim_object(struct object_cache_entry *entry)
 		goto out;
 	}
 
+	size = uatomic_sub_return(&sys_cache.cache_size, CACHE_OBJECT_SIZE);
 	dprintf("oid %"PRIx64" reclaimed successfully, cache_size: %"PRId32"\n",
-		oid, uatomic_read(&sys_cache.cache_size));
+		oid, size);
 	del_from_object_tree_and_list(entry, &oc->object_tree);
 out:
 	pthread_rwlock_unlock(&oc->lock);
@@ -467,7 +469,6 @@ static void do_reclaim(struct work *work)
 
 		if (do_reclaim_object(entry) < 0)
 			continue;
-		uatomic_sub(&sys_cache.cache_size, CACHE_OBJECT_SIZE);
 	}
 
 	dprintf("cache reclaim complete\n");
@@ -582,8 +583,6 @@ static void add_to_object_cache(struct object_cache *oc, uint32_t idx,
 	}
 	pthread_rwlock_unlock(&oc->lock);
 	lru_move_entry(entry);
-
-	object_cache_try_to_reclaim();
 }
 
 static int object_cache_lookup(struct object_cache *oc, uint32_t idx,
@@ -621,10 +620,12 @@ static int object_cache_lookup(struct object_cache *oc, uint32_t idx,
 		data_length = SD_DATA_OBJ_SIZE;
 
 	ret = prealloc(fd, data_length);
-	if (ret < 0)
+	if (ret < 0) {
 		ret = SD_RES_EIO;
-	else
+	} else {
 		add_to_object_cache(oc, idx, writeback);
+		object_cache_try_to_reclaim();
+	}
 
 	close(fd);
 out:
@@ -724,6 +725,11 @@ static int object_cache_pull(struct object_cache *oc, uint32_t idx)
 
 		ret = create_cache_object(oc, idx, buf, rsp->data_length,
 					  rsp->obj.offset, data_length);
+		/*
+		 * We don't try to reclaim objects to avoid object ping-pong
+		 * because the pulled object is clean and likely to be reclaimed
+		 * in a full cache.
+		 */
 		if (ret == SD_RES_SUCCESS)
 			add_to_object_cache(oc, idx, false);
 		else if (ret == SD_RES_OID_EXIST)
@@ -924,12 +930,15 @@ int object_cache_handle_request(struct request *req)
 retry:
 	ret = object_cache_lookup(cache, idx, create,
 				  hdr->flags & SD_FLAG_CMD_CACHE);
-	if (ret == SD_RES_NO_CACHE) {
+	switch (ret) {
+	case SD_RES_NO_CACHE:
 		ret = object_cache_pull(cache, idx);
 		if (ret != SD_RES_SUCCESS)
 			return ret;
-	} else if (ret == SD_RES_EIO)
+		break;
+	case SD_RES_EIO:
 		return ret;
+	}
 
 	entry = get_cache_entry(cache, idx);
 	if (!entry) {
@@ -1012,7 +1021,7 @@ int object_cache_flush_vdi(const struct request *req)
 
 	cache = find_object_cache(vid, false);
 	if (!cache) {
-		dprintf("%"PRIX32" not found\n", vid);
+		dprintf("%"PRIx32" not found\n", vid);
 		return SD_RES_SUCCESS;
 	}
 
@@ -1087,9 +1096,14 @@ static int load_existing_cache_object(struct object_cache *cache)
 		if (idx == ULLONG_MAX)
 			continue;
 
-		add_to_object_cache(cache, idx, false);
-		dprintf("load cache %06" PRIx32 "/%08" PRIx32 "\n",
-			cache->vid, idx);
+		/*
+		 * We don't know VM's cache type after restarting, so we assume
+		 * that it is writeback and mark all the objects diry to avoid
+		 * false reclaim. Donot try to reclaim at loading phase becaue
+		 * cluster isn't fully working.
+		 */
+		add_to_object_cache(cache, idx, true);
+		dprintf("%"PRIx64"\n", idx_to_oid(cache->vid, idx));
 	}
 
 	closedir(dir);
@@ -1102,8 +1116,7 @@ static int load_existing_cache(void)
 {
 	DIR *dir;
 	struct dirent *d;
-	uint32_t vid;
-	struct object_cache *cache;
+	uint32_t vid; struct object_cache *cache;
 	struct strbuf vid_buf;
 	int ret = 0;
 
-- 
1.7.9.5