[Sheepdog] [PATCH v5 UPDATE] sheep: let object cache better interact with recovery logic

[Liu Yuan]

From: Liu Yuan <tailai.ly at taobao.com>

If the targeted object of the gateway request is cached already,
we can simply operate on it despite of node changes.

Signed-off-by: Liu Yuan <tailai.ly at taobao.com>
---
update:
 - use SD_FLAG_CMD_CACHE for cache operation

 sheep/group.c        |   13 ++++++++++++-
 sheep/object_cache.c |   17 +++++++++++++++++
 sheep/sheep_priv.h   |    1 +
 3 files changed, 30 insertions(+), 1 deletions(-)

diff --git a/sheep/group.c b/sheep/group.c
index 7ed8316..a9b1c50 100644
--- a/sheep/group.c
+++ b/sheep/group.c
@@ -1082,6 +1082,7 @@ do_retry:
 
 	list_for_each_entry_safe(cevent, n, &sys->cpg_event_siblings, cpg_event_list) {
 		struct request *req = container_of(cevent, struct request, cev);
+		struct sd_obj_req *hdr = (struct sd_obj_req *)&req->rq;
 
 		if (cevent->ctype == CPG_EVENT_NOTIFY)
 			continue;
@@ -1096,6 +1097,16 @@ do_retry:
 			if (copies > req->nr_zones)
 				copies = req->nr_zones;
 
+			if (hdr->flags & SD_FLAG_CMD_CACHE &&
+			    object_is_cached(hdr->oid)) {
+				/* If its gateway request and we have cache of it
+				 * we are at its service.
+				 */
+				list_add_tail(&req->r_wlist, &sys->outstanding_req_list);
+				sys->nr_outstanding_io++;
+				goto gateway_work;
+			}
+
 			if (__is_access_to_recoverying_objects(req)) {
 				if (req->rq.flags & SD_FLAG_CMD_IO_LOCAL) {
 					req->rp.result = SD_RES_NEW_NODE_VER;
@@ -1128,7 +1139,6 @@ do_retry:
 			}
 
 			if (need_consistency_check(req->rq.opcode, req->rq.flags)) {
-				struct sd_obj_req *hdr = (struct sd_obj_req *)&req->rq;
 				uint32_t vdi_id = oid_to_vid(hdr->oid);
 				struct data_object_bmap *bmap;
 
@@ -1150,6 +1160,7 @@ do_retry:
 		else if (req->rq.flags & SD_FLAG_CMD_IO_LOCAL)
 			queue_work(sys->io_wqueue, &req->work);
 		else
+gateway_work:
 			queue_work(sys->gateway_wqueue, &req->work);
 	}
 
diff --git a/sheep/object_cache.c b/sheep/object_cache.c
index 25b4cbc..847635a 100644
--- a/sheep/object_cache.c
+++ b/sheep/object_cache.c
@@ -444,6 +444,23 @@ out:
 	return ret;
 }
 
+int object_is_cached(uint64_t oid)
+{
+	uint32_t vid = oid_to_vid(oid);
+	uint32_t idx = data_oid_to_idx(oid);
+	struct object_cache *cache;
+
+	if (is_vdi_obj(oid))
+		idx |= 1 << CACHE_VDI_SHIFT;
+
+	cache = find_object_cache(vid);
+	cache->oid = oid;
+	if (object_cache_lookup(cache, idx) < 0)
+		return 0;
+	else
+		return 1; /* found it */
+}
+
 int object_cache_init(const char *p)
 {
 	int ret = 0;
diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index afee5f7..6d9a8f8 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -422,5 +422,6 @@ int object_cache_rw(struct object_cache *oc, uint32_t idx, struct request *);
 int object_cache_pull(struct object_cache *oc, uint32_t index);
 int object_cache_push(struct object_cache *oc);
 int object_cache_init(const char *p);
+int object_is_cached(uint64_t oid);
 
 #endif
-- 
1.7.8.2