[Stgt-devel] [PATCH 2/2] iSER throttling

[Robin Humble]

  patch 02 - limit iSER requests so there are always RDMA mempools
             available

Signed-off-by: Robin Humble <robin.humble+stgt at anu.edu.au>
--
diff -ruN ../tgt/usr/iscsi/iscsid.c ./usr/iscsi/iscsid.c
--- ../tgt/usr/iscsi/iscsid.c	2008-01-24 18:45:27.000000000 +1100
+++ ./usr/iscsi/iscsid.c	2008-01-24 18:43:34.000000000 +1100
@@ -893,6 +893,21 @@
 	}
 }
 
+static int max_queue_cmds_available(struct iscsi_connection *conn)
+{
+	int max_queue_cmd;
+	int max_safe_cmds;
+
+	max_safe_cmds = conn->tp->queue_cmds_available(conn);
+	if (max_safe_cmds < 0) {
+		eprintf("iSER max_safe_cmds %d < 0\n", max_safe_cmds );
+		return 0;
+	}
+	max_queue_cmd = min( MAX_QUEUE_CMD, max_safe_cmds );
+
+	return max_queue_cmd;
+}
+
 static int iscsi_cmd_rsp_build(struct iscsi_task *task)
 {
 	struct iscsi_connection *conn = task->conn;
@@ -908,7 +923,8 @@
 	rsp->cmd_status = scsi_get_result(&task->scmd);
 	rsp->statsn = cpu_to_be32(conn->stat_sn++);
 	rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
-	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
+	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn +
+				     max_queue_cmds_available(conn));
 
 	calc_residual(rsp, task);
 
@@ -935,7 +951,8 @@
 	rsp->cmd_status = SAM_STAT_CHECK_CONDITION;
 	rsp->statsn = cpu_to_be32(conn->stat_sn++);
 	rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
-	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
+	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn +
+				     max_queue_cmds_available(conn));
 
 	calc_residual(rsp, task);
 
@@ -989,7 +1006,8 @@
 		datalen = conn->data_inout_max_length;
 
 	rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
-	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
+	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn +
+				     max_queue_cmds_available(conn));
 
 	conn->rsp.datasize = datalen;
 	hton24(rsp->dlength, datalen);
@@ -1523,6 +1541,7 @@
 	conn->exp_stat_sn = be32_to_cpu(req->exp_statsn);
 
 	len = ntoh24(req->dlength);
+
 	task = iscsi_alloc_task(conn, 0, len);
 	if (task)
 		conn->rx_task = task;
@@ -1639,7 +1658,8 @@
 	rsp->itt = task->req.itt;
 	rsp->statsn = cpu_to_be32(conn->stat_sn++);
 	rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
-	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
+	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn +
+				     max_queue_cmds_available(conn));
 
 	return 0;
 }
@@ -1662,7 +1682,8 @@
 		rsp->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
 		rsp->statsn = cpu_to_be32(conn->stat_sn++);
 		rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
-		rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
+		rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn +
+					     max_queue_cmds_available(conn));
 
 		/* TODO: honor max_burst */
 		conn->rsp.datasize = task->len;
@@ -1686,7 +1707,8 @@
 
 	rsp->statsn = cpu_to_be32(conn->stat_sn++);
 	rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
-	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
+	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn +
+				     max_queue_cmds_available(conn));
 
 	return 0;
 }
diff -ruN ../tgt/usr/iscsi/iscsi_rdma.c ./usr/iscsi/iscsi_rdma.c
--- ../tgt/usr/iscsi/iscsi_rdma.c	2008-01-24 18:45:44.000000000 +1100
+++ ./usr/iscsi/iscsi_rdma.c	2008-01-24 18:43:34.000000000 +1100
@@ -1643,6 +1643,22 @@
 	}
 }
 
+/*
+ * we know how many iSER connections we have, and also how many areas are
+ * allocated from our mempool, so we can trivially calculate how many
+ * more cmds we can handle without running out of rdma areas.
+ */
+static int iscsi_rdma_queue_cmds_available(struct iscsi_connection *conn)
+{
+	struct conn_info *ci = RDMA_CONN(conn);
+	struct iser_device *dev = ci->dev;
+	int cmds;
+
+	cmds = (mempool_num - dev->mempool_used)/iser_conn_cnt;
+
+	return cmds;
+}
+
 static void *iscsi_rdma_alloc_data_buf(struct iscsi_connection *conn,
 				       size_t sz)
 {
@@ -1735,6 +1751,7 @@
 	.ep_show		= iscsi_rdma_show,
 	.ep_event_modify	= iscsi_rdma_event_modify,
 	.alloc_data_buf		= iscsi_rdma_alloc_data_buf,
+	.queue_cmds_available	= iscsi_rdma_queue_cmds_available,
 	.free_data_buf		= iscsi_rdma_free_data_buf,
 	.ep_getsockname		= iscsi_rdma_getsockname,
 	.ep_getpeername		= iscsi_rdma_getpeername,
diff -ruN ../tgt/usr/iscsi/iscsi_tcp.c ./usr/iscsi/iscsi_tcp.c
--- ../tgt/usr/iscsi/iscsi_tcp.c	2008-01-24 18:45:27.000000000 +1100
+++ ./usr/iscsi/iscsi_tcp.c	2008-01-24 18:43:34.000000000 +1100
@@ -36,6 +36,8 @@
 #include "tgtd.h"
 #include "util.h"
 
+#define MAX_TCP_QUEUE_CMD	128
+
 static void iscsi_tcp_event_handler(int fd, int events, void *data);
 
 static struct iscsi_transport iscsi_tcp;
@@ -322,6 +324,10 @@
 	free(task);
 }
 
+static int iscsi_tcp_queue_cmds_available(struct iscsi_connection *conn) {
+	return MAX_TCP_QUEUE_CMD;
+}
+
 static void *iscsi_tcp_alloc_data_buf(struct iscsi_connection *conn, size_t sz)
 {
 	return valloc(sz);
@@ -365,6 +371,7 @@
 	.ep_show		= iscsi_tcp_show,
 	.ep_event_modify	= iscsi_event_modify,
 	.alloc_data_buf		= iscsi_tcp_alloc_data_buf,
+	.queue_cmds_available	= iscsi_tcp_queue_cmds_available,
 	.free_data_buf		= iscsi_tcp_free_data_buf,
 	.ep_getsockname		= iscsi_tcp_getsockname,
 	.ep_getpeername		= iscsi_tcp_getpeername,
diff -ruN ../tgt/usr/iscsi/transport.h ./usr/iscsi/transport.h
--- ../tgt/usr/iscsi/transport.h	2008-01-24 18:45:27.000000000 +1100
+++ ./usr/iscsi/transport.h	2008-01-24 18:43:34.000000000 +1100
@@ -32,6 +32,7 @@
 	int (*ep_show)(struct iscsi_connection *conn, char *buf, int rest);
 	void (*ep_event_modify)(struct iscsi_connection *conn, int events);
 	void *(*alloc_data_buf)(struct iscsi_connection *conn, size_t sz);
+	int (*queue_cmds_available)(struct iscsi_connection *conn);
 	void (*free_data_buf)(struct iscsi_connection *conn, void *buf);
 	int (*ep_getsockname)(struct iscsi_connection *conn,
 			      struct sockaddr *sa, socklen_t *len);