[PATCH] iser more rdma bufs

[Pete Wyckoff]

Increase the number of slots used for RDMA transfers.  This does not
actually require more buffer space, just lists for keeping track of
outstanding operations.

It is best to minimize the number of RDMA transfers required for data
motion, however.  This can be done by increasing the maximum data transfer
length parameter in both the target and the initiator.

Signed-off-by: Pete Wyckoff <pw at osc.edu>
---
 usr/iscsi/iscsi_rdma.c |   43 +++++++++++++++++++++++++------------------
 1 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/usr/iscsi/iscsi_rdma.c b/usr/iscsi/iscsi_rdma.c
index adff476..5c308fa 100644
--- a/usr/iscsi/iscsi_rdma.c
+++ b/usr/iscsi/iscsi_rdma.c
@@ -164,6 +164,14 @@ struct conn_info {
 #define MAX_RSIZE 8192
 #define ISER_INITIAL_POST 10  /*XXX make this a global variable and set it to what gets negotiated at login*/
 
+/*
+ * Number of outstanding RDMAs per command; should instead wait for previous
+ * RDMAs to complete before starting new ones.  For RDMA write, each one
+ * will be no bigger than max burst, so we end up issuing lots when the
+ * max_recv_dlength value is very low.
+ */
+#define RDMA_PER_COMMAND 32
+
 #define MEMPOOL_SZ (64)
 /* 512K + sizeof(iscsi_task) + AHS size */
 #define MEM_SZ (roundup((1 << 19) + sizeof(struct iscsi_task) + 260+28+48, \
@@ -250,7 +258,7 @@ static int iser_match_qp(uint32_t qp_num)
 static int iser_init_comm(struct conn_info *conn, ssize_t ssize,
 			  ssize_t rsize, int id)
 {
-	int i;
+	int i, j;
 	unsigned long size;
 	uint8_t *srbuf, *listbuf;
 	struct rdmalist *rdmal;
@@ -275,8 +283,9 @@ static int iser_init_comm(struct conn_info *conn, ssize_t ssize,
 	INIT_LIST_HEAD(&conn->rdmal);
 	INIT_LIST_HEAD(&conn->rdmal_write_busy);
 
-	size = ISER_INITIAL_POST * (sizeof(struct rdmalist)
-		+ sizeof(struct sendlist) + sizeof(struct recvlist));
+	size = ISER_INITIAL_POST * (RDMA_PER_COMMAND * sizeof(struct rdmalist)
+				    + sizeof(struct sendlist)
+				    + sizeof(struct recvlist));
 	conn->listbuf = malloc(size);
 	if (!conn->listbuf) {
 		eprintf("malloc listbuf %lu\n", size);
@@ -286,31 +295,29 @@ static int iser_init_comm(struct conn_info *conn, ssize_t ssize,
 	srbuf = conn->srbuf;
 	listbuf = conn->listbuf;
 	for (i=0; i<ISER_INITIAL_POST; i++){
-		rdmal = (void *) listbuf;
-		listbuf += sizeof(*rdmal);
+
+		for (j=0; j<RDMA_PER_COMMAND; j++) {
+			rdmal = (void *) listbuf;
+			listbuf += sizeof(*rdmal);
+			rdmal->conn_index = id;
+			rdmal->free = 1;
+			list_add_tail(&rdmal->list, &conn->rdmal);
+		}
 
 		sendl = (void *) listbuf;
 		listbuf += sizeof(*sendl);
+		sendl->buf = srbuf;
+		srbuf += ssize;
+		sendl->conn_index = id;
+		sendl->free = 1;
+		list_add_tail(&sendl->list, &conn->sendl);
 
 		recvl = (void *) listbuf;
 		listbuf += sizeof(*recvl);
-
 		recvl->buf = srbuf;
 		srbuf += rsize;
-
-		sendl->buf = srbuf;
-		srbuf += ssize;
-
-		rdmal->conn_index = id;
-		sendl->conn_index = id;
 		recvl->conn_index = id;
-
-		sendl->free = 1;
-		rdmal->free = 1;
-
-		list_add_tail(&sendl->list, &conn->sendl);
 		list_add_tail(&recvl->list, &conn->recvl);
-		list_add_tail(&rdmal->list, &conn->rdmal);
 	}
 	conn->slen = ssize;
 	conn->rlen = rsize;
-- 
1.5.2.4