[sheepdog] [PATCH v4 5/8] sheepdev: make it possible to add or remove a sheep device

levin li levin108 at gmail.com
Wed Jan 23 08:56:29 CET 2013


From: levin li <xingke.lwp at taobao.com>

This patch provides functions to create a sheep device by writing to
the proc entry, but it hasn't processed the IO requests, it just ends
the requests with -EIO.

It creates sheep devices named /dev/sheepX, but until now, it can not
handle IO request, any read/write request to this device will get EIO

Signed-off-by: levin li <xingke.lwp at taobao.com>
---
 sheepdev/device.c   | 263 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 sheepdev/proc.c     |   4 +
 sheepdev/sheepdev.h |   6 ++
 3 files changed, 273 insertions(+)

diff --git a/sheepdev/device.c b/sheepdev/device.c
index 15e149a..bcf82a2 100644
--- a/sheepdev/device.c
+++ b/sheepdev/device.c
@@ -26,22 +26,285 @@
 #include <linux/kthread.h>
 #include "sheepdev.h"
 
+static int sheepdev_major;
+spinlock_t devices_lock;
+struct list_head dev_list;
+static unsigned long *device_bitmap;
+
+static void sheepdev_get(struct sheepdev *dev)
+{
+	atomic_inc(&dev->struct_refcnt);
+}
+
+static void sheepdev_put(struct sheepdev *dev)
+{
+	if (atomic_dec_and_test(&dev->struct_refcnt))
+		kfree(dev);
+}
+
+static void sheep_end_request_directly(struct request *req, int ret)
+{
+	struct request_queue *q = req->q;
+	unsigned long flags;
+
+	spin_lock_irqsave(q->queue_lock, flags);
+	if (!list_empty(&req->queuelist))
+		list_del_init(&req->queuelist);
+	__blk_end_request_all(req, ret);
+	spin_unlock_irqrestore(q->queue_lock, flags);
+}
+
+static void remove_device(struct sheepdev *dev)
+{
+	DBPRT("remove device /dev/%s\n", dev->disk->disk_name);
+
+	blk_cleanup_queue(dev->disk->queue);
+	del_gendisk(dev->disk);
+	put_disk(dev->disk);
+
+	clear_bit(dev->minor, device_bitmap);
+	write_lock(&dev->sock_lock);
+	if (dev->sock) {
+		inet_release(dev->sock);
+		dev->sock = NULL;
+	}
+	write_unlock(&dev->sock_lock);
+
+	sheepdev_put(dev);
+}
+
+static int sheepdev_open(struct block_device *blkdev, fmode_t mode)
+{
+	struct gendisk *disk = blkdev->bd_disk;
+	struct sheepdev *dev = disk->private_data;
+
+	spin_lock(&dev->dev_lock);
+	dev->device_refcnt++;
+	spin_unlock(&dev->dev_lock);
+
+	return 0;
+}
+
+static int sheepdev_release(struct gendisk *disk, fmode_t mode)
+{
+	struct sheepdev *dev = disk->private_data;
+
+	spin_lock(&dev->dev_lock);
+	dev->device_refcnt--;
+	spin_unlock(&dev->dev_lock);
+
+	return 0;
+}
+
+static const struct block_device_operations sheepdev_ops = {
+	.owner = THIS_MODULE,
+	.open = sheepdev_open,
+	.release = sheepdev_release,
+};
+
+static void sheep_io_request(struct request_queue *rq)
+{
+	struct request *req;
+	struct gendisk *disk;
+	struct sheepdev *dev;
+
+	while ((req = blk_fetch_request(rq)) != NULL) {
+
+		spin_unlock_irq(rq->queue_lock);
+
+		disk = req->rq_disk;
+		dev = disk->private_data;
+
+		if (req->cmd_type != REQ_TYPE_FS) {
+			DBPRT("Skip non-fs request\n");
+			__blk_end_request_all(req, -EIO);
+		}
+
+		sheep_end_request_directly(req, -EIO);
+
+		spin_lock_irq(rq->queue_lock);
+	}
+}
+
+static int sheep_add_disk(struct sheepdev *dev)
+{
+	int ret;
+	struct request_queue *queue;
+
+	dev->disk = alloc_disk(SHEEP_BLKDEV_MINORS);
+	if (!dev->disk) {
+		DBPRT("allocate gendisk failure\n");
+		ret = -EBUSY;
+		return ret;
+	}
+	queue = blk_init_queue(sheep_io_request, &dev->que_lock);
+	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, queue);
+	dev->disk->major = sheepdev_major;
+	dev->disk->first_minor = dev->minor * SHEEP_BLKDEV_MINORS;
+	dev->disk->queue = queue;
+	dev->disk->fops = &sheepdev_ops;
+	dev->disk->private_data = dev;
+	snprintf(dev->disk->disk_name, sizeof(dev->disk->disk_name),
+		 SHEEP_BLKDEV_NAME"%c", dev->minor + 'a');
+
+	set_capacity(dev->disk, 0);
+	add_disk(dev->disk);
+	set_capacity(dev->disk, dev->sectors);
+
+	return 0;
+}
+
+static int sheep_dev_setup(struct sheepdev *dev)
+{
+	int ret;
+
+	ret = sheep_vdi_setup(dev);
+	if (ret)
+		return ret;
+
+	spin_lock_init(&dev->que_lock);
+	spin_lock_init(&dev->dev_lock);
+	spin_lock_init(&dev->sd_req_lock);
+	rwlock_init(&dev->creating_lock);
+	rwlock_init(&dev->sock_lock);
+	init_waitqueue_head(&dev->req_wait);
+	init_waitqueue_head(&dev->fin_wait);
+	init_waitqueue_head(&dev->creating_wait);
+	INIT_LIST_HEAD(&dev->pending_list);
+	INIT_LIST_HEAD(&dev->finish_list);
+	INIT_LIST_HEAD(&dev->dev_list);
+	INIT_LIST_HEAD(&dev->deletion_list);
+	INIT_LIST_HEAD(&dev->sd_req_list);
+
+	dev->obj_state_tree = RB_ROOT;
+	dev->req_id = 1;
+
+	ret = sheep_add_disk(dev);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int sheep_add_device(const char *addr, int port, const char *vdiname,
+		     int snapshot_id, const char *snapshot_tag)
+{
+	struct sheepdev *dev;
+	int ret = 0;
+
+	DBPRT("[%s:%d] vdiname: %s, snapshot id: %d, snapshot tag: %s\n",
+	      addr, port, vdiname, snapshot_id, snapshot_tag);
+
+	dev = kmalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	memset(dev, 0, sizeof(*dev));
+	dev->port = port;
+	dev->snapshot_id = snapshot_id;
+	memcpy(dev->ip_addr, addr, sizeof(dev->ip_addr));
+	strcpy(dev->vdiname, vdiname);
+	strcpy(dev->snapshot_tag, snapshot_tag);
+
+	spin_lock(&devices_lock);
+	dev->minor = find_next_zero_bit(device_bitmap, SHEEP_BLKDEV_MINORS, 0);
+	set_bit(dev->minor, device_bitmap);
+	spin_unlock(&devices_lock);
+
+	ret = sheep_dev_setup(dev);
+	if (ret) {
+		clear_bit(dev->minor, device_bitmap);
+		goto out;
+	} else {
+		sheepdev_get(dev);
+		spin_lock(&devices_lock);
+		list_add_tail(&dev->dev_list, &dev_list);
+		spin_unlock(&devices_lock);
+	}
+
+	return ret;
+out:
+	kfree(dev);
+	return ret;
+}
+
+int sheep_remove_device(const char *vdiname, int snapshot_id,
+			const char *snapshot_tag)
+{
+	struct sheepdev *dev, *t;
+	int ret = 0;
+
+	spin_lock(&devices_lock);
+	list_for_each_entry_safe(dev, t, &dev_list, dev_list) {
+		if (strcmp(vdiname, dev->vdiname) != 0)
+			continue;
+		if (*(dev->snapshot_tag) != '\0' &&
+		    strcmp(snapshot_tag, dev->snapshot_tag) != 0)
+			continue;
+		if (snapshot_id != 0 &&
+		    snapshot_id != dev->snapshot_id)
+			continue;
+
+		spin_unlock(&devices_lock);
+
+		spin_lock(&dev->dev_lock);
+		if (dev->device_refcnt) {
+			spin_unlock(&dev->dev_lock);
+			ret = -EBUSY;
+		} else {
+			spin_unlock(&dev->dev_lock);
+			list_del_init(&dev->dev_list);
+			remove_device(dev);
+		}
+
+		return ret;
+	}
+	spin_unlock(&devices_lock);
+
+	return -ENXIO;
+}
+
 static int __init sheep_module_init(void)
 {
 	int ret;
 
 	DBPRT("Block device driver for Sheepdog\n");
 
+	spin_lock_init(&devices_lock);
+	INIT_LIST_HEAD(&dev_list);
+	device_bitmap = kmalloc(SHEEP_BLKDEV_MINORS / 8, GFP_KERNEL);
+	if (!device_bitmap)
+		return -ENOMEM;
+	memset(device_bitmap, 0, SHEEP_BLKDEV_MINORS / 8);
+
 	ret = sheep_proc_init();
 	if (ret)
 		return ret;
 
+	sheepdev_major = register_blkdev(0, SHEEP_BLKDEV_NAME);
+	if (sheepdev_major < 0) {
+		ret = sheepdev_major;
+		goto error;
+	}
+
 	return 0;
+error:
+	sheep_proc_destroy();
+	return ret;
 }
 
 static void __exit sheep_module_exit(void)
 {
+	struct sheepdev *dev, *t;
+
+	list_for_each_entry_safe(dev, t, &dev_list, dev_list) {
+		list_del_init(&dev->dev_list);
+		remove_device(dev);
+	}
+
 	sheep_proc_destroy();
+	unregister_blkdev(sheepdev_major, SHEEP_BLKDEV_NAME);
+	kfree(device_bitmap);
 
 	DBPRT("Sheepdog Block Device Removed.\n");
 }
diff --git a/sheepdev/proc.c b/sheepdev/proc.c
index a2efbe0..e6a46f7 100644
--- a/sheepdev/proc.c
+++ b/sheepdev/proc.c
@@ -59,6 +59,8 @@ static int process_add_command(char *buf)
 		}
 	}
 
+	ret = sheep_add_device(addr, port, vdiname, snapshot_id, snapshot_tag);
+
 	return ret;
 }
 
@@ -85,6 +87,8 @@ static int process_del_command(char *buf)
 		}
 	}
 
+	ret = sheep_remove_device(vdiname, snapshot_id, snapshot_tag);
+
 	return ret;
 }
 
diff --git a/sheepdev/sheepdev.h b/sheepdev/sheepdev.h
index 91a55ea..5ef7098 100644
--- a/sheepdev/sheepdev.h
+++ b/sheepdev/sheepdev.h
@@ -100,4 +100,10 @@ int send_write_req(struct sheepdev *sheepdev, uint64_t oid, uint64_t cow_oid,
 		   int create);
 int sheep_vdi_setup(struct sheepdev *sheep_dev);
 
+/* device.c */
+int sheep_add_device(const char *addr, int port, const char *vdiname,
+		     int snapshot_id, const char *snapshot_tag);
+int sheep_remove_device(const char *vdiname, int snapshot_id,
+			const char *snapshot_tag);
+
 #endif
-- 
1.7.11.7




More information about the sheepdog mailing list