[sheepdog] [PATCH v3 1/5] sheep: update inode coherence protocol for iSCSI multipath

[Hitoshi Mitake]

This patch fixes 5 bugs of inode coherence protocol, especially in
gateway request handling:
1. add a new flag for indicating requests from tgtd
   It is required because dog command can read VDIs which are used by
   tgtd. Current coherence protocol can be violated by this
   behavior. The new flag is added for preventing this problem.
2. accurate detection of inode reload
   The previous way of detecting inode reload was incorrect.
3. return SD_RES_INODE_INVALIDATED in a case of create and write
   request
4. correct conditional branch for invalidation via gateway writes
5. remove the branch of is_modified() in is_refresh_required()
   Refresh is required even if no node is in modified state. If all
   nodes are in  shared and invalidated states, invalidated nodes must
   refresh inode.

Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
---
 include/sheepdog_proto.h |  1 +
 sheep/gateway.c          | 38 ++++++++++++++++++++++++++++++++++----
 sheep/vdi.c              |  3 ---
 3 files changed, 35 insertions(+), 7 deletions(-)

v3: remove needless conditional branch in is_refresh_required()

v2: correct conditional branch for invalidation via gateway writes

diff --git a/include/sheepdog_proto.h b/include/sheepdog_proto.h
index 8b8058d..c25e9f1 100644
--- a/include/sheepdog_proto.h
+++ b/include/sheepdog_proto.h
@@ -53,6 +53,7 @@
 #define SD_FLAG_CMD_DIRECT   0x08 /* don't use object cache */
 /* return something back while sending something to sheep */
 #define SD_FLAG_CMD_PIGGYBACK   0x10
+#define SD_FLAG_CMD_TGT   0x20
 /* flags above 0x80 are sheepdog-internal */
 
 #define SD_RES_SUCCESS       0x00 /* Success */
diff --git a/sheep/gateway.c b/sheep/gateway.c
index 6379c32..a5cee5c 100644
--- a/sheep/gateway.c
+++ b/sheep/gateway.c
@@ -613,12 +613,32 @@ static int update_obj_refcnt(const struct sd_req *hdr, uint32_t *vids,
 			       false);
 }
 
+static bool is_inode_refresh_req(struct request *req)
+{
+	uint64_t oid = req->rq.obj.oid;
+	uint32_t data_length = req->rq.data_length;
+	uint32_t offset = req->rq.obj.offset;
+
+	if (!is_vdi_obj(oid))
+		return false;
+
+	if (offset)
+		return false;
+
+	if (data_length != data_vid_offset(SD_INODE_DATA_INDEX))
+		return false;
+
+	return true;
+}
+
 int gateway_read_obj(struct request *req)
 {
 	uint64_t oid = req->rq.obj.oid;
 	int ret;
 
-	if (!is_vdi_obj(oid) && is_refresh_required(oid_to_vid(oid))) {
+	if ((req->rq.flags & SD_FLAG_CMD_TGT) &&
+	    !is_inode_refresh_req(req) &&
+	    is_refresh_required(oid_to_vid(oid))) {
 		sd_debug("refresh is required: %"PRIx64, oid);
 		return SD_RES_INODE_INVALIDATED;
 	}
@@ -635,7 +655,9 @@ int gateway_read_obj(struct request *req)
 	if (ret != SD_RES_SUCCESS)
 		return ret;
 
-	validate_myself(oid_to_vid(oid));
+	if (is_inode_refresh_req(req))
+		validate_myself(oid_to_vid(oid));
+
 	return ret;
 }
 
@@ -647,7 +669,8 @@ int gateway_write_obj(struct request *req)
 	uint32_t *vids = NULL, *new_vids = req->data;
 	struct generation_reference *refs = NULL;
 
-	if (is_refresh_required(oid_to_vid(oid))) {
+	if ((req->rq.flags & SD_FLAG_CMD_TGT) &&
+	    is_refresh_required(oid_to_vid(oid))) {
 		sd_debug("refresh is required: %"PRIx64, oid);
 		return SD_RES_INODE_INVALIDATED;
 	}
@@ -678,7 +701,8 @@ int gateway_write_obj(struct request *req)
 		update_obj_refcnt(hdr, vids, new_vids, refs);
 	}
 out:
-	invalidate_other_nodes(oid_to_vid(oid));
+	if (is_data_vid_update(hdr))
+		invalidate_other_nodes(oid_to_vid(oid));
 
 	free(vids);
 	free(refs);
@@ -720,6 +744,12 @@ int gateway_create_and_write_obj(struct request *req)
 {
 	uint64_t oid = req->rq.obj.oid;
 
+	if ((req->rq.flags & SD_FLAG_CMD_TGT) &&
+	    is_refresh_required(oid_to_vid(oid))) {
+		sd_debug("refresh is required: %"PRIx64, oid);
+		return SD_RES_INODE_INVALIDATED;
+	}
+
 	if (oid_is_readonly(oid))
 		return SD_RES_READONLY;
 
diff --git a/sheep/vdi.c b/sheep/vdi.c
index f1de98c..6d0a80f 100644
--- a/sheep/vdi.c
+++ b/sheep/vdi.c
@@ -666,9 +666,6 @@ worker_fn bool is_refresh_required(uint32_t vid)
 	if (entry->lock_state != LOCK_STATE_SHARED)
 		return false;
 
-	if (!is_modified(entry))
-		return false;
-
 	for (int i = 0; i < entry->nr_participants; i++) {
 		if (node_id_cmp(&entry->participants[i], &sys->this_node.nid))
 			continue;
-- 
1.8.3.2