[sheepdog] [PATCH v2] cache: correct block size calculation for inode object

[Liu Yuan]

inode and data object have different sizes and thus cache block size is
different. Current code fail to get the right size for inode object when
its last bit of bmap is set with the fixed block size.

This patch tries to get the dynamic block size based on the oid.

Signed-off-by: Liu Yuan <namei.unix at gmail.com>
---
 include/util.h       |    3 +++
 sheep/object_cache.c |   43 ++++++++++++++++++++++++++-----------------
 sheep/plain_store.c  |    2 --
 3 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/include/util.h b/include/util.h
index 0d50f4f..97ab54d 100644
--- a/include/util.h
+++ b/include/util.h
@@ -85,6 +85,9 @@ static inline void *zalloc(size_t size)
 	_x < _y ? -1 : _x > _y ? 1 : 0;	\
 })
 
+#define sector_algined(x) ({ ((x) & (SECTOR_SIZE - 1)) == 0; })
+#define block_algined(x) ({ ((x) & (BLOCK_SIZE - 1)) == 0; })
+
 typedef void (*try_to_free_t)(size_t);
 try_to_free_t set_try_to_free_routine(try_to_free_t);
 
diff --git a/sheep/object_cache.c b/sheep/object_cache.c
index dcf6972..9ddcd50 100644
--- a/sheep/object_cache.c
+++ b/sheep/object_cache.c
@@ -28,8 +28,6 @@
 
 #define CACHE_INDEX_MASK      (CACHE_CREATE_BIT)
 
-#define CACHE_BLOCK_SIZE      ((UINT64_C(1) << 10) * 64) /* 64 KB */
-
 #define CACHE_OBJECT_SIZE (SD_DATA_OBJ_SIZE / 1024 / 1024) /* M */
 
 /* Kick background pusher if dirty_count greater than it */
@@ -116,13 +114,28 @@ static inline bool idx_has_vdi_bit(uint32_t idx)
 	return !!(idx & CACHE_VDI_BIT);
 }
 
-static uint64_t calc_object_bmap(size_t len, off_t offset)
+static inline int get_cache_block_size(uint64_t oid)
+{
+	int bsize =  DIV_ROUND_UP(get_objsize(oid),
+				  sizeof(uint64_t) * BITS_PER_BYTE);
+
+	/*
+	 * Data object block must be FS block-aliged, then we don't handle the
+	 * last partial block for data object.
+	 */
+	assert(is_vdi_obj(oid) || block_aligned(bsize));
+
+	return round_up(bsize, BLOCK_SIZE); /* To be FS friendly */
+}
+
+static uint64_t calc_object_bmap(uint64_t oid, size_t len, off_t offset)
 {
 	int start, end, nr;
 	unsigned long bmap = 0;
+	int bsize = get_cache_block_size(oid);
 
-	start = offset / CACHE_BLOCK_SIZE;
-	end = DIV_ROUND_UP(len + offset, CACHE_BLOCK_SIZE);
+	start = offset / bsize;
+	end = DIV_ROUND_UP(len + offset, bsize);
 	nr = end - start;
 
 	while (nr--)
@@ -432,7 +445,7 @@ static int write_cache_object(struct object_cache_entry *entry, void *buf,
 	}
 	write_lock_cache(oc);
 	if (writeback) {
-		entry->bmap |= calc_object_bmap(count, offset);
+		entry->bmap |= calc_object_bmap(oid, count, offset);
 		if (list_empty(&entry->dirty_list))
 			add_to_dirty_list(entry);
 	}
@@ -473,27 +486,23 @@ static int push_cache_object(uint32_t vid, uint32_t idx, uint64_t bmap,
 	unsigned data_length;
 	int ret = SD_RES_NO_MEM;
 	uint64_t oid = idx_to_oid(vid, idx);
+	int bsize = get_cache_block_size(oid);
 	int first_bit, last_bit;
 
-	sd_dprintf("%"PRIx64", create %d", oid, create);
-
 	if (!bmap) {
-		sd_dprintf("WARN: nothing to flush");
+		sd_dprintf("WARN: nothing to flush %"PRIx64, oid);
 		return SD_RES_SUCCESS;
 	}
 
 	first_bit = ffsll(bmap) - 1;
 	last_bit = fls64(bmap) - 1;
 
-	sd_dprintf("bmap:0x%"PRIx64", first_bit:%d, last_bit:%d", bmap,
-		   first_bit, last_bit);
-	offset = first_bit * CACHE_BLOCK_SIZE;
-	data_length = (last_bit - first_bit + 1) * CACHE_BLOCK_SIZE;
+	sd_dprintf("%"PRIx64" bmap(%d):0x%"PRIx64", first_bit:%d, last_bit:%d",
+		   oid, bsize, bmap, first_bit, last_bit);
+	offset = first_bit * bsize;
+	data_length = (last_bit - first_bit + 1) * bsize;
 
-	/*
-	 * CACHE_BLOCK_SIZE may not be divisible by SD_INODE_SIZE,
-	 * so (offset + data_length) could larger than SD_INODE_SIZE
-	 */
+	/* Handle last partial block for inode object */
 	if (is_vdi_obj(oid) && (offset + data_length) > SD_INODE_SIZE)
 		data_length = SD_INODE_SIZE - offset;
 
diff --git a/sheep/plain_store.c b/sheep/plain_store.c
index bebe2f3..013062c 100644
--- a/sheep/plain_store.c
+++ b/sheep/plain_store.c
@@ -13,8 +13,6 @@
 
 #include "sheep_priv.h"
 
-#define sector_algined(x) ({ ((x) & (SECTOR_SIZE - 1)) == 0; })
-
 static inline bool iocb_is_aligned(const struct siocb *iocb)
 {
 	return  sector_algined(iocb->offset) && sector_algined(iocb->length);
-- 
1.7.9.5