[Stgt-devel] [PATCH 5/6] iser iscsid changes

[Pete Wyckoff]

Handle the special cases for RDMA in core iscsid code.  These changes
all look at conn->tp->rdma and do not affect TCP code.

    - track RDMA setting at session and verify conns are compatible
    - no status collapse on final data packet
    - all data-in transmits must reenable TX for conn
    - TX state machine finishes tasks without going through epoll
    - TX handler returns status to trigger RDMA flow control

Signed-off-by: Pete Wyckoff <pw at osc.edu>
---
 usr/iscsi/iscsid.c  |   45 ++++++++++++++++++++++++++++++++++-----------
 usr/iscsi/iscsid.h  |    5 ++++-
 usr/iscsi/session.c |    2 ++
 3 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/usr/iscsi/iscsid.c b/usr/iscsi/iscsid.c
index df0fec4..b0c1b6d 100644
--- a/usr/iscsi/iscsid.c
+++ b/usr/iscsi/iscsid.c
@@ -511,8 +511,21 @@ static void login_finish(struct iscsi_connection *conn)
 			conn->state = STATE_EXIT;
 			break;
 		}
-		if (!conn->session)
+		if (!conn->session) {
 			session_create(conn);
+		} else {
+			if (conn->tp->rdma ^ conn->session->rdma) {
+				eprintf("new conn rdma %d, but session %d\n",
+					conn->tp->rdma, conn->session->rdma);
+				rsp->flags = 0;
+				rsp->status_class =
+					ISCSI_STATUS_CLS_INITIATOR_ERR;
+				rsp->status_detail =
+					ISCSI_LOGIN_STATUS_INVALID_REQUEST;
+				conn->state = STATE_EXIT;
+				break;
+			}
+		}
 		memcpy(conn->isid, conn->session->isid, sizeof(conn->isid));
 		conn->tsih = conn->session->tsih;
 		break;
@@ -965,7 +978,8 @@ static int iscsi_data_rsp_build(struct iscsi_task *task)
 
 		/* collapse status into final packet if successful */
 		if (result == SAM_STAT_GOOD &&
-		    scsi_get_data_dir(&task->scmd) != DATA_BIDIRECTIONAL) {
+		    scsi_get_data_dir(&task->scmd) != DATA_BIDIRECTIONAL &&
+		    !conn->tp->rdma) {
 			rsp->flags |= ISCSI_FLAG_DATA_STATUS;
 			rsp->cmd_status = result;
 			rsp->statsn = cpu_to_be32(conn->stat_sn++);
@@ -1720,8 +1734,9 @@ static int iscsi_scsi_cmd_tx_done(struct iscsi_connection *conn)
 		break;
 	case ISCSI_OP_SCSI_DATA_IN:
 		if (task->offset < task->len ||
-		    scsi_get_result(&task->scmd) != SAM_STAT_GOOD
-		    || scsi_get_data_dir(&task->scmd) == DATA_BIDIRECTIONAL) {
+		    scsi_get_result(&task->scmd) != SAM_STAT_GOOD ||
+		    scsi_get_data_dir(&task->scmd) == DATA_BIDIRECTIONAL ||
+		    conn->tp->rdma) {
 			dprintf("more data or sense or bidir %x\n", hdr->itt);
 			list_add_tail(&task->c_list, &task->conn->tx_clist);
 			return 0;
@@ -1996,7 +2011,7 @@ again:
 	return 0;
 }
 
-void iscsi_tx_handler(struct iscsi_connection *conn)
+int iscsi_tx_handler(struct iscsi_connection *conn)
 {
 	int ret = 0, hdigest, ddigest;
 	uint32_t crc;
@@ -2011,9 +2026,10 @@ void iscsi_tx_handler(struct iscsi_connection *conn)
 	if (conn->state == STATE_SCSI && !conn->tx_task) {
 		ret = iscsi_task_tx_start(conn);
 		if (ret)
-			return;
+			goto out;
 	}
 
+again:
 	switch (conn->tx_iostate) {
 	case IOSTATE_TX_BHS:
 		ret = do_send(conn, IOSTATE_TX_INIT_AHS);
@@ -2069,7 +2085,7 @@ void iscsi_tx_handler(struct iscsi_connection *conn)
 		ret = do_send(conn, ddigest ?
 			      IOSTATE_TX_INIT_DDIGEST : IOSTATE_TX_END);
 		if (ret < 0)
-			return;
+			goto out;
 		if (conn->tx_iostate != IOSTATE_TX_INIT_DDIGEST)
 			break;
 	case IOSTATE_TX_INIT_DDIGEST:
@@ -2089,10 +2105,14 @@ void iscsi_tx_handler(struct iscsi_connection *conn)
 		exit(1);
 	}
 
-	if (ret < 0 ||
-	    conn->tx_iostate != IOSTATE_TX_END ||
-	    conn->state == STATE_CLOSE)
-		return;
+	if (ret < 0 || conn->state == STATE_CLOSE)
+		goto out;
+
+	if (conn->tx_iostate != IOSTATE_TX_END) {
+		if (conn->tp->rdma)
+			goto again;  /* avoid event loop, just push */
+		goto out;
+	}
 
 	if (conn->tx_size) {
 		eprintf("error %d %d %d\n", conn->state, conn->tx_iostate,
@@ -2125,6 +2145,9 @@ void iscsi_tx_handler(struct iscsi_connection *conn)
 		conn->tp->ep_event_modify(conn, EPOLLIN);
 		break;
 	}
+
+out:
+	return ret;
 }
 
 static struct tgt_driver iscsi = {
diff --git a/usr/iscsi/iscsid.h b/usr/iscsi/iscsid.h
index 3fabfba..de681f7 100644
--- a/usr/iscsi/iscsid.h
+++ b/usr/iscsi/iscsid.h
@@ -91,6 +91,9 @@ struct iscsi_session {
 	struct param session_param[ISCSI_PARAM_MAX];
 
 	char *info;
+
+	/* if this session uses rdma connections */
+	int rdma;
 };
 
 struct iscsi_task {
@@ -258,7 +261,7 @@ extern void conn_add_to_session(struct iscsi_connection *conn, struct iscsi_sess
 extern char *text_key_find(struct iscsi_connection *conn, char *searchKey);
 extern void text_key_add(struct iscsi_connection *conn, char *key, char *value);
 extern void conn_read_pdu(struct iscsi_connection *conn);
-extern void iscsi_tx_handler(struct iscsi_connection *conn);
+extern int iscsi_tx_handler(struct iscsi_connection *conn);
 extern void iscsi_rx_handler(struct iscsi_connection *conn);
 extern int iscsi_scsi_cmd_execute(struct iscsi_task *task);
 
diff --git a/usr/iscsi/session.c b/usr/iscsi/session.c
index 3b1650d..028d538 100644
--- a/usr/iscsi/session.c
+++ b/usr/iscsi/session.c
@@ -128,6 +128,8 @@ int session_create(struct iscsi_connection *conn)
 	memcpy(session->isid, conn->isid, sizeof(session->isid));
 	session->tsih = last_tsih = tsih;
 
+	session->rdma = conn->tp->rdma;
+
 	conn_add_to_session(conn, session);
 
 	dprintf("session_create: %#" PRIx64 "\n", sid64(conn->isid, session->tsih));
-- 
1.5.3.4