[stgt] [PATCH] tgt: Add BSG v4 backstore support to usr/bs_sg.c

Nicholas A. Bellinger nab at linux-iscsi.org
Fri Jun 25 10:30:21 CEST 2010


From: Nicholas Bellinger <nab at linux-iscsi.org>

Hi Tomo-san,

Apologies for the delay on the resubmission, I have been AFK enjoying summer time the past week.
This patch addresses your most recent comments for local scope usage of static struct sg_io_v4 usage in
bs_bsg_cmd_submit(), usage of dprintf() instead of BSG_IO(), and include a copy of include/linux/bsg.h
into the patch.

Please let me know if you have any other comments,

Best,

--nab

This patch adds support for the block layer implementation of the sg v4 interface
(BSG) -> STGT with a new struct backingstore_template bsg_bst sharing code with the
original sg_bst.  It adds for new function bs_bsg_cmd_submit() for incoming write I/O
and bs_bsg_cmd_complete() for polling read I/O using include/linux/bsg.h:struct sg_io_v4.

This patch adds BSG specific major checks in bs_sg_open() and for getting max_queue using
SG_GET_COMMAND_Q and calling SG_SET_RESERVED_SIZE following init_sg_device().

So far this has been tested with STGT/iSCSI <-> BSG <-> TCM_Loop SPC-4 iSCSI Target
Port emulation and is able to mkfs, fsck and mount a filesystem from a TCM/IBLOCK
Linux LVM kernel level backstore.  This code has also been tested with SG_IO and the
legacy codepath.

Signed-off-by: Nicholas A. Bellinger <nab at linux-iscsi.org>
---
 usr/bs_sg.c |  155 +++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 usr/bsg.h   |   94 ++++++++++++++++++++++++++++++++++++
 2 files changed, 240 insertions(+), 9 deletions(-)
 create mode 100644 usr/bsg.h

diff --git a/usr/bs_sg.c b/usr/bs_sg.c
index f01a3ef..cda5cd2 100644
--- a/usr/bs_sg.c
+++ b/usr/bs_sg.c
@@ -3,6 +3,9 @@
  *
  * Copyright (C) 2008 Alexander Nezhinsky <nezhinsky at gmail.com>
  *
+ * Added linux/block/bsg.c support using struct sg_io_v4.
+ * Copyright (C) 2010 Nicholas A. Bellinger <nab at linux-iscsi.org>
+ *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation, version 2 of the
@@ -34,6 +37,7 @@
 #include <sys/epoll.h>
 #include <scsi/sg.h>
 
+#include "bsg.h" /* Copied from include/linux/bsg.h */
 #include "list.h"
 #include "util.h"
 #include "tgtd.h"
@@ -107,7 +111,7 @@ static int bs_sg_rw(int host_no, struct scsi_cmd *cmd)
 	return SAM_STAT_CHECK_CONDITION;
 }
 
-static void set_cmd_failed(struct scsi_cmd *cmd)
+static int set_cmd_failed(struct scsi_cmd *cmd)
 {
 	int result = SAM_STAT_CHECK_CONDITION;
 	uint16_t asc = ASC_READ_ERROR;
@@ -115,6 +119,54 @@ static void set_cmd_failed(struct scsi_cmd *cmd)
 
 	scsi_set_result(cmd, result);
 	sense_data_build(cmd, key, asc);
+
+	return result;
+}
+
+static int bs_bsg_cmd_submit(struct scsi_cmd *cmd)
+{
+	struct scsi_lu *dev = cmd->dev;
+	int fd = dev->fd;
+	struct sg_io_v4 io_hdr;
+	int err = 0;
+
+	memset(&io_hdr, 0, sizeof(io_hdr));
+	/*
+	 * Following linux/include/linux/bsg.h
+	 */
+	/* [i] 'Q' to differentiate from v3 */
+	io_hdr.guard = 'Q';
+	io_hdr.protocol = BSG_PROTOCOL_SCSI;
+	io_hdr.subprotocol = BSG_SUB_PROTOCOL_SCSI_CMD;
+	io_hdr.request_len = cmd->scb_len;
+	io_hdr.request = (unsigned long )cmd->scb;
+
+	io_hdr.dout_xfer_len = scsi_get_out_length(cmd);
+	io_hdr.dout_xferp = (unsigned long)scsi_get_out_buffer(cmd);
+	io_hdr.din_xfer_len = scsi_get_in_length(cmd);
+	io_hdr.din_xferp = (unsigned long)scsi_get_in_buffer(cmd);
+
+	io_hdr.max_response_len = sizeof(cmd->sense_buffer);
+	/* SCSI: (auto)sense data */
+	io_hdr.response = (unsigned long)cmd->sense_buffer;
+	/* Using the same 2000 millisecond timeout.. */
+	io_hdr.timeout = BS_SG_TIMEOUT;
+	/* [i->o] unused internally */
+	io_hdr.usr_ptr = (unsigned long)cmd;
+	dprintf("[%d] Set io_hdr->usr_ptr from cmd: %p\n", getpid(), cmd);
+	/* Bsg does Q_AT_HEAD by default */
+	io_hdr.flags |= BSG_FLAG_Q_AT_TAIL;
+
+	dprintf("[%d] Calling graceful_write for CDB: 0x%02x\n", getpid(), cmd->scb[0]);
+	err = graceful_write(fd, &io_hdr, sizeof(io_hdr));
+	if (!err)
+		set_cmd_async(cmd);
+	else {
+		eprintf("failed to start cmd 0x%p\n", cmd);
+		return set_cmd_failed(cmd);
+	}
+
+	return 0;
 }
 
 static int bs_sg_cmd_submit(struct scsi_cmd *cmd)
@@ -150,11 +202,47 @@ static int bs_sg_cmd_submit(struct scsi_cmd *cmd)
 		set_cmd_async(cmd);
 	else {
 		eprintf("failed to start cmd 0x%p\n", cmd);
-		set_cmd_failed(cmd);
+		return set_cmd_failed(cmd);
 	}
 	return 0;
 }
 
+static void bs_bsg_cmd_complete(int fd, int events, void *data)
+{
+	struct sg_io_v4 io_hdr;
+	struct scsi_cmd *cmd;
+	int err;
+
+	dprintf("[%d] bs_bsg_cmd_complete() called!\n", getpid());
+	memset(&io_hdr, 0, sizeof(io_hdr));
+	/* [i] 'Q' to differentiate from v3 */
+	io_hdr.guard = 'Q';
+
+	err = graceful_read(fd, &io_hdr, sizeof(io_hdr));
+	if (err)
+		return;
+
+	cmd = (struct scsi_cmd *)(unsigned long)io_hdr.usr_ptr;
+	dprintf("BSG Using cmd: %p for io_hdr.usr_ptr\n", cmd);
+	/*
+	 * Check SCSI: command completion status
+	 * */
+	if (!io_hdr.device_status) {
+		scsi_set_out_resid(cmd, io_hdr.dout_resid);
+		scsi_set_in_resid(cmd, io_hdr.din_resid);
+	} else {
+		/*
+		 * NAB: Used by linux/block/bsg.c:bsg_ioctl(), is this
+		 * right..?
+		 */
+		cmd->sense_len = SCSI_SENSE_BUFFERSIZE;
+		scsi_set_out_resid_by_actual(cmd, 0);
+		scsi_set_in_resid_by_actual(cmd, 0);
+	}
+	dprintf("[%d] BSG() Calling cmd->scsi_cmd_done()\n", getpid());
+	cmd->scsi_cmd_done(cmd, io_hdr.device_status);
+}
+
 static void bs_sg_cmd_complete(int fd, int events, void *data)
 {
 	struct sg_io_hdr io_hdr;
@@ -191,10 +279,41 @@ static int chk_sg_device(char *path)
 		return -1;
 	}
 
-	if (S_ISCHR(st.st_mode) && major(st.st_rdev) == SCSI_GENERIC_MAJOR)
+	if (!S_ISCHR(st.st_mode)) {
+		eprintf("Not a character device: %s\n", path);
+		return -1;
+	}
+
+	/* Check for SG_IO major first.. */
+	if (major(st.st_rdev) == SCSI_GENERIC_MAJOR)
 		return 0;
-	else
+
+	/* This is not yet defined in include/linux/major.h.. */
+	if (major(st.st_rdev) == 254)
+		return 1;
+
+	return -1;
+}
+
+static int init_bsg_device(int fd)
+{
+	int t, err;
+
+	err = ioctl(fd, SG_GET_COMMAND_Q, &t);
+	if (err < 0) {
+		eprintf("SG_GET_COMMAND_Q for bsd failed: %d\n", err);
 		return -1;
+	}
+	eprintf("bsg: Using max_queue: %d\n", t);
+
+	t = BS_SG_RESVD_SZ;
+	err = ioctl(fd, SG_SET_RESERVED_SIZE, &t);
+	if (err < 0) {
+		eprintf("SG_SET_RESERVED_SIZE errno: %d\n", errno);
+		return -1;
+	}
+
+	return 0;
 }
 
 static int init_sg_device(int fd)
@@ -235,10 +354,11 @@ static int bs_sg_init(struct scsi_lu *lu)
 
 static int bs_sg_open(struct scsi_lu *lu, char *path, int *fd, uint64_t *size)
 {
-	int sg_fd, err;
+	void (*cmd_complete)(int, int, void *) = NULL;
+	int sg_fd, err, bsg = 0;
 
-	err = chk_sg_device(path);
-	if (err) {
+	bsg = chk_sg_device(path);
+	if (bsg < 0) {
 		eprintf("Not recognized %s as an SG device\n", path);
 		return -EINVAL;
 	}
@@ -249,13 +369,19 @@ static int bs_sg_open(struct scsi_lu *lu, char *path, int *fd, uint64_t *size)
 		return sg_fd;
 	}
 
-	err = init_sg_device(sg_fd);
+	if (bsg) {
+		cmd_complete = &bs_bsg_cmd_complete;
+		err = init_bsg_device(sg_fd);
+	} else {
+		cmd_complete = bs_sg_cmd_complete;
+		err = init_sg_device(sg_fd);
+	}
 	if (err) {
 		eprintf("Failed to initialize sg device %s\n", path);
 		return err;
 	}
 
-	err = tgt_event_add(sg_fd, EPOLLIN, bs_sg_cmd_complete, NULL);
+	err = tgt_event_add(sg_fd, EPOLLIN, cmd_complete, NULL);
 	if (err) {
 		eprintf("Failed to add sg device event %s\n", path);
 		return err;
@@ -294,6 +420,16 @@ static struct backingstore_template sg_bst = {
 	.bs_cmd_done		= bs_sg_cmd_done,
 };
 
+static struct backingstore_template bsg_bst = {
+	.bs_name		= "bsg",
+	.bs_datasize		= 0,
+	.bs_init		= bs_sg_init,
+	.bs_open		= bs_sg_open,
+	.bs_close		= bs_sg_close,
+	.bs_cmd_submit		= bs_bsg_cmd_submit,
+	.bs_cmd_done		= bs_sg_cmd_done,
+};
+
 static struct device_type_template sg_template = {
 	.type			= TYPE_PT,
 	.lu_init		= bs_sg_lu_init,
@@ -307,5 +443,6 @@ static struct device_type_template sg_template = {
 __attribute__((constructor)) static void bs_sg_constructor(void)
 {
 	register_backingstore_template(&sg_bst);
+	register_backingstore_template(&bsg_bst);
 	device_type_register(&sg_template);
 }
diff --git a/usr/bsg.h b/usr/bsg.h
new file mode 100644
index 0000000..ecb4730
--- /dev/null
+++ b/usr/bsg.h
@@ -0,0 +1,94 @@
+#ifndef BSG_H
+#define BSG_H
+
+#include <linux/types.h>
+
+#define BSG_PROTOCOL_SCSI		0
+
+#define BSG_SUB_PROTOCOL_SCSI_CMD	0
+#define BSG_SUB_PROTOCOL_SCSI_TMF	1
+#define BSG_SUB_PROTOCOL_SCSI_TRANSPORT	2
+
+/*
+ * For flags member below
+ * sg.h sg_io_hdr also has bits defined for it's flags member. However
+ * none of these bits are implemented/used by bsg. The bits below are
+ * allocated to not conflict with sg.h ones anyway.
+ */
+#define BSG_FLAG_Q_AT_TAIL 0x10 /* default, == 0 at this bit, is Q_AT_HEAD */
+
+struct sg_io_v4 {
+	__s32 guard;		/* [i] 'Q' to differentiate from v3 */
+	__u32 protocol;		/* [i] 0 -> SCSI , .... */
+	__u32 subprotocol;	/* [i] 0 -> SCSI command, 1 -> SCSI task
+				   management function, .... */
+
+	__u32 request_len;	/* [i] in bytes */
+	__u64 request;		/* [i], [*i] {SCSI: cdb} */
+	__u64 request_tag;	/* [i] {SCSI: task tag (only if flagged)} */
+	__u32 request_attr;	/* [i] {SCSI: task attribute} */
+	__u32 request_priority;	/* [i] {SCSI: task priority} */
+	__u32 request_extra;	/* [i] {spare, for padding} */
+	__u32 max_response_len;	/* [i] in bytes */
+	__u64 response;		/* [i], [*o] {SCSI: (auto)sense data} */
+
+        /* "dout_": data out (to device); "din_": data in (from device) */
+	__u32 dout_iovec_count;	/* [i] 0 -> "flat" dout transfer else
+				   dout_xfer points to array of iovec */
+	__u32 dout_xfer_len;	/* [i] bytes to be transferred to device */
+	__u32 din_iovec_count;	/* [i] 0 -> "flat" din transfer */
+	__u32 din_xfer_len;	/* [i] bytes to be transferred from device */
+	__u64 dout_xferp;	/* [i], [*i] */
+	__u64 din_xferp;	/* [i], [*o] */
+
+	__u32 timeout;		/* [i] units: millisecond */
+	__u32 flags;		/* [i] bit mask */
+	__u64 usr_ptr;		/* [i->o] unused internally */
+	__u32 spare_in;		/* [i] */
+
+	__u32 driver_status;	/* [o] 0 -> ok */
+	__u32 transport_status;	/* [o] 0 -> ok */
+	__u32 device_status;	/* [o] {SCSI: command completion status} */
+	__u32 retry_delay;	/* [o] {SCSI: status auxiliary information} */
+	__u32 info;		/* [o] additional information */
+	__u32 duration;		/* [o] time to complete, in milliseconds */
+	__u32 response_len;	/* [o] bytes of response actually written */
+	__s32 din_resid;	/* [o] din_xfer_len - actual_din_xfer_len */
+	__s32 dout_resid;	/* [o] dout_xfer_len - actual_dout_xfer_len */
+	__u64 generated_tag;	/* [o] {SCSI: transport generated task tag} */
+	__u32 spare_out;	/* [o] */
+
+	__u32 padding;
+};
+
+#ifdef __KERNEL__
+
+#if defined(CONFIG_BLK_DEV_BSG)
+struct bsg_class_device {
+	struct device *class_dev;
+	struct device *parent;
+	int minor;
+	struct request_queue *queue;
+	struct kref ref;
+	void (*release)(struct device *);
+};
+
+extern int bsg_register_queue(struct request_queue *q,
+			      struct device *parent, const char *name,
+			      void (*release)(struct device *));
+extern void bsg_unregister_queue(struct request_queue *);
+#else
+static inline int bsg_register_queue(struct request_queue *q,
+				     struct device *parent, const char *name,
+				     void (*release)(struct device *))
+{
+	return 0;
+}
+static inline void bsg_unregister_queue(struct request_queue *q)
+{
+}
+#endif
+
+#endif /* __KERNEL__ */
+
+#endif
-- 
1.5.6.5

--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html



More information about the stgt mailing list