[sheepdog] [PATCH 6/7] sbd: add support for single major allocation scheme

[Liu Yuan]

From: Liu Yuan <tailai.ly at taobao.com>

Currently each sbd device is allocated its own major number, which leads to a
hard limit of 230-250 images mapped at once due to major_t is 12 bits.

This commit adds support for a new single-major device number allocation scheme,
meaning that we can make use of minor_t (20 bits) to map more images on a single
node.

For now we support 31 partitions for a sheep block device at most and 32768
devices for a single node.

The single major allocation looks like:

root at vm1:/home/test/sheepdog# cat /sys/bus/sbd/list
0 disk0  <--- sbd0
1 disk1
2 disk2

root at vm1:/home/test/sheepdog# ll /dev/sbd*
brw-rw---- 1 root disk 251,  0 May 24 20:27 /dev/sbd0
brw-rw---- 1 root disk 251,  1 May 24 20:27 /dev/sbd0p1 <-- partition 1 for sbd0
brw-rw---- 1 root disk 251,  2 May 24 20:27 /dev/sbd0p2 <-- partition 2 for sbd0
brw-rw---- 1 root disk 251,  3 May 24 20:27 /dev/sbd0p3 <-- partition 3 for sbd0
brw-rw---- 1 root disk 251, 32 May 24 20:28 /dev/sbd1
brw-rw---- 1 root disk 251, 33 May 24 20:28 /dev/sbd1p1
brw-rw---- 1 root disk 251, 34 May 24 20:28 /dev/sbd1p2
brw-rw---- 1 root disk 251, 64 May 24 20:22 /dev/sbd2

(major number 251 is dynamically genereged while insmod sbd.ko)

Signed-off-by: Liu Yuan <namei.unix at gmail.com>
---
 sbd/sbd.h                |  7 ++++++-
 sbd/sheep_block_device.c | 39 ++++++++++++++++++++++++++-------------
 2 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/sbd/sbd.h b/sbd/sbd.h
index ccf9b89..9f51ca2 100644
--- a/sbd/sbd.h
+++ b/sbd/sbd.h
@@ -20,7 +20,7 @@
 
 #define DRV_NAME "sbd"
 #define DEV_NAME_LEN 32
-#define SBD_MINORS_PER_MAJOR 32
+#define SBD_MINORS_SHIFT 5 /* at most 31 partitions for a single device */
 #define SECTOR_SIZE 512
 
 /* FIXME I need this hack to compile DEFINE_MUTEX successfully */
@@ -98,6 +98,11 @@ struct sheep_aiocb *sheep_aiocb_setup(struct request *req);
 int sheep_aiocb_submit(struct sheep_aiocb *aiocb);
 int sheep_handle_reply(struct sbd_device *dev);
 
+static inline int sbd_dev_id_to_minor(int id)
+{
+	return id << SBD_MINORS_SHIFT;
+}
+
 #if defined(CONFIG_DYNAMIC_DEBUG)
 
 # define _SBD_FLAGS (_DPRINTK_FLAGS_PRINT | _DPRINTK_FLAGS_INCL_MODNAME \
diff --git a/sbd/sheep_block_device.c b/sbd/sheep_block_device.c
index c7f238b..0b1cc2a 100644
--- a/sbd/sheep_block_device.c
+++ b/sbd/sheep_block_device.c
@@ -50,6 +50,8 @@
 static LIST_HEAD(sbd_dev_list);
 static DEFINE_MUTEX(dev_list_mutex);
 
+static int sbd_major;
+
 static const struct block_device_operations sbd_bd_ops = {
 	.owner		= THIS_MODULE,
 };
@@ -92,13 +94,13 @@ static int sbd_add_disk(struct sbd_device *dev)
 	struct gendisk *disk;
 	struct request_queue *rq;
 
-	disk = alloc_disk(SBD_MINORS_PER_MAJOR);
+	disk = alloc_disk(1 << SBD_MINORS_SHIFT);
 	if (!disk)
 		return -ENOMEM;
 
 	snprintf(disk->disk_name, DEV_NAME_LEN, DRV_NAME "%d", dev->id);
 	disk->major = dev->major;
-	disk->first_minor = 0;
+	disk->first_minor = dev->minor;
 	disk->fops = &sbd_bd_ops;
 	disk->private_data = dev;
 
@@ -235,29 +237,30 @@ static ssize_t sbd_add(struct bus_type *bus, const char *buf,
 
 	dev->id = new_id;
 	snprintf(name, DEV_NAME_LEN, DRV_NAME "%d", dev->id);
-	ret = register_blkdev(0, name);
-	if (ret < 0)
-		goto err_free_dev;
-	dev->major = ret;
-	dev->minor = 0;
+	dev->major = sbd_major;
+	dev->minor = sbd_dev_id_to_minor(dev->id);
 	dev->reaper = kthread_run(sbd_request_reaper, dev, "sbd_reaper");
 	if (IS_ERR(dev->reaper))
-		goto err_unreg_blkdev;
+		goto err_free_dev;
 	dev->submiter = kthread_run(sbd_request_submiter, dev, "sbd_submiter");
 	if (IS_ERR(dev->submiter))
-		goto err_unreg_blkdev;
+		goto err_stop_reaper;
 
 	ret = sbd_add_disk(dev);
 	if (ret < 0)
-		goto err_unreg_blkdev;
+		goto err_stop_kthreads;
 
 	mutex_lock(&dev_list_mutex);
 	list_add_tail(&dev->list, &sbd_dev_list);
 	mutex_unlock(&dev_list_mutex);
 
 	return count;
-err_unreg_blkdev:
-	unregister_blkdev(dev->major, name);
+err_stop_kthreads:
+	kthread_stop(dev->submiter);
+	wake_up(&dev->submiter_wq);
+err_stop_reaper:
+	kthread_stop(dev->reaper);
+	wake_up(&dev->reaper_wq);
 err_free_dev:
 	free_sbd_device(dev);
 err_put:
@@ -384,17 +387,27 @@ int __init sbd_init(void)
 {
 	int ret;
 
-	ret = sbd_sysfs_init();
+	ret = register_blkdev(0, DRV_NAME);
 	if (ret < 0)
 		return ret;
 
+	sbd_major = ret;
+
+	ret = sbd_sysfs_init();
+	if (ret < 0)
+		goto err;
+
 	pr_info("%s: Sheepdog block device loaded\n", DRV_NAME);
 	return 0;
+err:
+	unregister_blkdev(sbd_major, DRV_NAME);
+	return ret;
 }
 
 void __exit sbd_exit(void)
 {
 	sbd_sysfs_cleanup();
+	unregister_blkdev(sbd_major, DRV_NAME);
 	pr_info("%s: Sheepdog block device unloaded\n", DRV_NAME);
 }
 
-- 
1.8.1.2