[stgt] [PATCH] Add support for sending NOP-IN to the initiators and terminate the tcp connection upon too many failures.

Ronnie Sahlberg ronniesahlberg at gmail.com
Tue May 28 18:09:49 CEST 2013


Signed-off-by: Ronnie Sahlberg <ronniesahlberg at gmail.com>
---
 doc/tgtd.8.xml        |   29 +++++++++++++++
 usr/iscsi/iscsi_tcp.c |   95 ++++++++++++++++++++++++++++++++++++++++++++++++-
 usr/iscsi/iscsid.c    |   46 +++++++++++++++++++----
 usr/iscsi/iscsid.h    |    2 +
 usr/iscsi/transport.h |    1 +
 5 files changed, 164 insertions(+), 9 deletions(-)

diff --git a/doc/tgtd.8.xml b/doc/tgtd.8.xml
index ce734ce..21d6ea9 100644
--- a/doc/tgtd.8.xml
+++ b/doc/tgtd.8.xml
@@ -129,6 +129,35 @@
       </screen>
       </para>
     </refsect2>
+    <refsect2><title>nop_interval=<integer></title>
+      <para>
+	This option enables TGTD to send send NOP-IN to check is
+	a connected initiator is still alive.
+	This parameter specifies the interval between each NOP-IN that
+	TGTD will send in seconds.
+      </para>
+      <para>
+	This feature is disabled by default.
+      </para>
+    </refsect2>
+    <refsect2><title>nop_count=<integer></title>
+      <para>
+	When sending NOP-IN to the initiator, after this many consecutive
+	failures TGTD will terminate the connection to the client.
+	This feature can be used to read dead connections faster than waiting
+	for any TCP-Keepalive to trigger.
+      </para>
+      <para>
+	This feature is disabled by default.
+      </para>
+      <para>
+      Example: send NOP-IN every 5 seconds and abort the session after
+      6 failures.
+      <screen format="linespecific">
+	tgtd --iscsi portal=192.0.2.1:3260,nop_interval=5,nop_count=6
+      </screen>
+      </para>
+    </refsect2>
   </refsect1>
 
 
diff --git a/usr/iscsi/iscsi_tcp.c b/usr/iscsi/iscsi_tcp.c
index 0c43039..b997ffb 100644
--- a/usr/iscsi/iscsi_tcp.c
+++ b/usr/iscsi/iscsi_tcp.c
@@ -35,15 +35,26 @@
 #include "iscsid.h"
 #include "tgtd.h"
 #include "util.h"
+#include "work.h"
 
 static void iscsi_tcp_event_handler(int fd, int events, void *data);
+static struct iscsi_task *iscsi_tcp_alloc_task(struct iscsi_connection *conn,
+						size_t ext_len);
+static void iscsi_tcp_free_task(struct iscsi_task *task);
+static void iscsi_tcp_release(struct iscsi_connection *conn);
 
 static int listen_fds[8];
 static struct iscsi_transport iscsi_tcp;
+static long nop_ttt;
+static int nop_interval;
+static int nop_count;
 
 struct iscsi_tcp_connection {
-	int fd;
+	struct list_head tcp_conn_siblings;
+	int nop_inflight_count;
 
+	int fd;
+	long ttt;
 	struct iscsi_connection iscsi_conn;
 };
 
@@ -52,6 +63,65 @@ static inline struct iscsi_tcp_connection *TCP_CONN(struct iscsi_connection *con
 	return container_of(conn, struct iscsi_tcp_connection, iscsi_conn);
 }
 
+/* all iscsi connections */
+static struct list_head iscsi_tcp_conn_list;
+
+static struct tgt_work nop_work;
+
+static int iscsi_send_ping_nop_in(struct iscsi_tcp_connection *tcp_conn)
+{
+	struct iscsi_connection *conn = &tcp_conn->iscsi_conn;
+	struct iscsi_task *task = NULL;
+
+	task = iscsi_tcp_alloc_task(&tcp_conn->iscsi_conn, 0);
+	task->conn = conn;
+
+	task->tag = ISCSI_RESERVED_TAG;
+	task->req.opcode = ISCSI_OP_NOOP_IN;
+	task->req.itt = cpu_to_be32(ISCSI_RESERVED_TAG);
+	task->req.ttt = cpu_to_be32(tcp_conn->ttt);
+
+	list_add_tail(&task->c_list, &task->conn->tx_clist);
+	task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
+
+	return 0;
+}
+
+static void iscsi_tcp_nop_work_handler(void *data)
+{
+	struct iscsi_tcp_connection *tcp_conn;
+
+	list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
+		tcp_conn->nop_inflight_count++;
+		if (nop_count && tcp_conn->nop_inflight_count > nop_count) {
+			eprintf("tcp connection timed out after %d failed " \
+				"NOP-IN\n", nop_count);
+			iscsi_tcp_release(&tcp_conn->iscsi_conn);
+			/* cant/shouldnt delete tcp_conn from within the loop */
+			break;
+		}
+		nop_ttt++;
+		if (nop_ttt == ISCSI_RESERVED_TAG)
+			nop_ttt = 1;
+
+		tcp_conn->ttt = nop_ttt;
+		iscsi_send_ping_nop_in(tcp_conn);
+	}
+
+	add_work(&nop_work, nop_interval);
+}
+
+static void iscsi_tcp_nop_reply(long ttt)
+{
+	struct iscsi_tcp_connection *tcp_conn;
+
+	list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
+		if (tcp_conn->ttt != ttt)
+			continue;
+		tcp_conn->nop_inflight_count = 0;
+	}
+}
+
 static int set_keepalive(int fd)
 {
 	int ret, opt;
@@ -144,6 +214,8 @@ static void accept_connection(int afd, int events, void *data)
 		goto out;
 	}
 
+	list_add(&tcp_conn->tcp_conn_siblings, &iscsi_tcp_conn_list);
+
 	return;
 out:
 	close(fd);
@@ -279,6 +351,16 @@ int iscsi_add_portal(char *addr, int port, int tpgt)
 	return 0;
 };
 
+void iscsi_set_nop_interval(int interval)
+{
+	nop_interval = interval;
+}
+
+void iscsi_set_nop_count(int count)
+{
+	nop_count = count;
+}
+
 int iscsi_delete_portal(char *addr, int port)
 {
 	struct iscsi_portal *portal;
@@ -313,6 +395,15 @@ static int iscsi_tcp_init(void)
 		iscsi_add_portal(NULL, 3260, 1);
 	}
 
+	if (nop_interval) {
+		nop_work.func = iscsi_tcp_nop_work_handler;
+		nop_work.data = &nop_work;
+
+		add_work(&nop_work, nop_interval);
+	}
+
+	INIT_LIST_HEAD(&iscsi_tcp_conn_list);
+
 	return 0;
 }
 
@@ -370,6 +461,7 @@ static void iscsi_tcp_release(struct iscsi_connection *conn)
 
 	conn_exit(conn);
 	close(tcp_conn->fd);
+	list_del(&tcp_conn->tcp_conn_siblings);
 	free(tcp_conn);
 }
 
@@ -480,6 +572,7 @@ static struct iscsi_transport iscsi_tcp = {
 	.free_data_buf		= iscsi_tcp_free_data_buf,
 	.ep_getsockname		= iscsi_tcp_getsockname,
 	.ep_getpeername		= iscsi_tcp_getpeername,
+	.ep_nop_reply		= iscsi_tcp_nop_reply,
 };
 
 __attribute__((constructor)) static void iscsi_transport_init(void)
diff --git a/usr/iscsi/iscsid.c b/usr/iscsi/iscsid.c
index b10be1d..2161f1c 100644
--- a/usr/iscsi/iscsid.c
+++ b/usr/iscsi/iscsid.c
@@ -1660,13 +1660,9 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
 
 	dprintf("%x %x %u\n", req->ttt, req->itt, ntoh24(req->dlength));
 	if (req->ttt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
-		/*
-		 * We don't request a NOP-Out by sending a NOP-In.
-		 * See 10.18.2 in the draft 20.
-		 */
-		eprintf("initiator bug\n");
-		err = -ISCSI_REASON_PROTOCOL_ERROR;
-		goto out;
+		if ((req->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_NOOP_OUT) {
+			goto good;
+		}
 	}
 
 	if (req->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
@@ -1677,6 +1673,7 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
 		}
 	}
 
+good:
 	conn->exp_stat_sn = be32_to_cpu(req->exp_statsn);
 
 	len = ntoh24(req->dlength);
@@ -1705,8 +1702,8 @@ static int iscsi_task_rx_done(struct iscsi_connection *conn)
 
 	op = hdr->opcode & ISCSI_OPCODE_MASK;
 	switch (op) {
-	case ISCSI_OP_SCSI_CMD:
 	case ISCSI_OP_NOOP_OUT:
+	case ISCSI_OP_SCSI_CMD:
 	case ISCSI_OP_SCSI_TMFUNC:
 	case ISCSI_OP_LOGOUT:
 		err = iscsi_task_queue(task);
@@ -1799,6 +1796,28 @@ static int iscsi_logout_tx_start(struct iscsi_task *task)
 	return 0;
 }
 
+static int iscsi_noop_in_tx_start(struct iscsi_task *task)
+{
+	struct iscsi_connection *conn = task->conn;
+	struct iscsi_data_rsp *rsp = (struct iscsi_data_rsp *) &conn->rsp.bhs;
+
+	memset(rsp, 0, sizeof(*rsp));
+	rsp->opcode = ISCSI_OP_NOOP_IN;
+	rsp->flags = ISCSI_FLAG_CMD_FINAL;
+	rsp->itt = task->req.itt;
+	rsp->ttt = task->req.ttt;
+	rsp->statsn = cpu_to_be32(conn->stat_sn);
+	rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
+	rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
+
+	/* TODO: honor max_burst */
+	conn->rsp.datasize = task->len;
+	hton24(rsp->dlength, task->len);
+	conn->rsp.data = task->data;
+
+	return 0;
+}
+
 static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
 {
 	struct iscsi_connection *conn = task->conn;
@@ -1806,6 +1825,10 @@ static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
 
 	if (task->req.itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
 		*is_rsp = 0;
+
+		if (conn->tp->ep_nop_reply)
+			conn->tp->ep_nop_reply(be32_to_cpu(task->req.ttt));
+
 		iscsi_free_task(task);
 	} else {
 		*is_rsp = 1;
@@ -1918,6 +1941,9 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
 	case ISCSI_OP_SCSI_CMD:
 		err = iscsi_scsi_cmd_tx_start(task);
 		break;
+	case ISCSI_OP_NOOP_IN:
+		err = iscsi_noop_in_tx_start(task);
+		break;
 	case ISCSI_OP_NOOP_OUT:
 		err = iscsi_noop_out_tx_start(task, &is_rsp);
 		if (!is_rsp)
@@ -2413,6 +2439,10 @@ int iscsi_param_parse_portals(char *p, int do_add,
 					return -1;
 				}
 			}
+		} else if (!strncmp(p, "nop_interval", 12)) {
+			iscsi_set_nop_interval(atoi(p+13));
+		} else if (!strncmp(p, "nop_count", 9)) {
+			iscsi_set_nop_count(atoi(p+10));
 		}
 
 		p += strcspn(p, ",");
diff --git a/usr/iscsi/iscsid.h b/usr/iscsi/iscsid.h
index e7e08ae..54a5363 100644
--- a/usr/iscsi/iscsid.h
+++ b/usr/iscsi/iscsid.h
@@ -316,6 +316,8 @@ extern int iscsi_scsi_cmd_execute(struct iscsi_task *task);
 extern int iscsi_transportid(int tid, uint64_t itn_id, char *buf, int size);
 extern int iscsi_add_portal(char *addr, int port, int tpgt);
 extern int iscsi_delete_portal(char *addr, int port);
+extern void iscsi_set_nop_interval(int interval);
+extern void iscsi_set_nop_count(int count);
 extern int iscsi_param_parse_portals(char *p, int do_add, int do_delete);
 extern void iscsi_update_conn_stats_rx(struct iscsi_connection *conn, int size, int opcode);
 extern void iscsi_update_conn_stats_tx(struct iscsi_connection *conn, int size, int opcode);
diff --git a/usr/iscsi/transport.h b/usr/iscsi/transport.h
index af4af21..6573207 100644
--- a/usr/iscsi/transport.h
+++ b/usr/iscsi/transport.h
@@ -39,6 +39,7 @@ struct iscsi_transport {
 			      struct sockaddr *sa, socklen_t *len);
 	int (*ep_getpeername)(struct iscsi_connection *conn,
 			      struct sockaddr *sa, socklen_t *len);
+	void (*ep_nop_reply)(long ttt);
 };
 
 extern int iscsi_transport_register(struct iscsi_transport *);
-- 
1.7.3.1

--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html



More information about the stgt mailing list