[stgt] [PATCH] Add support for sending NOP-IN to the initiators and terminate the tcp connection upon too many failures.
Ronnie Sahlberg
ronniesahlberg at gmail.com
Tue May 28 18:09:49 CEST 2013
Signed-off-by: Ronnie Sahlberg <ronniesahlberg at gmail.com>
---
doc/tgtd.8.xml | 29 +++++++++++++++
usr/iscsi/iscsi_tcp.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++-
usr/iscsi/iscsid.c | 46 +++++++++++++++++++----
usr/iscsi/iscsid.h | 2 +
usr/iscsi/transport.h | 1 +
5 files changed, 164 insertions(+), 9 deletions(-)
diff --git a/doc/tgtd.8.xml b/doc/tgtd.8.xml
index ce734ce..21d6ea9 100644
--- a/doc/tgtd.8.xml
+++ b/doc/tgtd.8.xml
@@ -129,6 +129,35 @@
</screen>
</para>
</refsect2>
+ <refsect2><title>nop_interval=<integer></title>
+ <para>
+ This option enables TGTD to send send NOP-IN to check is
+ a connected initiator is still alive.
+ This parameter specifies the interval between each NOP-IN that
+ TGTD will send in seconds.
+ </para>
+ <para>
+ This feature is disabled by default.
+ </para>
+ </refsect2>
+ <refsect2><title>nop_count=<integer></title>
+ <para>
+ When sending NOP-IN to the initiator, after this many consecutive
+ failures TGTD will terminate the connection to the client.
+ This feature can be used to read dead connections faster than waiting
+ for any TCP-Keepalive to trigger.
+ </para>
+ <para>
+ This feature is disabled by default.
+ </para>
+ <para>
+ Example: send NOP-IN every 5 seconds and abort the session after
+ 6 failures.
+ <screen format="linespecific">
+ tgtd --iscsi portal=192.0.2.1:3260,nop_interval=5,nop_count=6
+ </screen>
+ </para>
+ </refsect2>
</refsect1>
diff --git a/usr/iscsi/iscsi_tcp.c b/usr/iscsi/iscsi_tcp.c
index 0c43039..b997ffb 100644
--- a/usr/iscsi/iscsi_tcp.c
+++ b/usr/iscsi/iscsi_tcp.c
@@ -35,15 +35,26 @@
#include "iscsid.h"
#include "tgtd.h"
#include "util.h"
+#include "work.h"
static void iscsi_tcp_event_handler(int fd, int events, void *data);
+static struct iscsi_task *iscsi_tcp_alloc_task(struct iscsi_connection *conn,
+ size_t ext_len);
+static void iscsi_tcp_free_task(struct iscsi_task *task);
+static void iscsi_tcp_release(struct iscsi_connection *conn);
static int listen_fds[8];
static struct iscsi_transport iscsi_tcp;
+static long nop_ttt;
+static int nop_interval;
+static int nop_count;
struct iscsi_tcp_connection {
- int fd;
+ struct list_head tcp_conn_siblings;
+ int nop_inflight_count;
+ int fd;
+ long ttt;
struct iscsi_connection iscsi_conn;
};
@@ -52,6 +63,65 @@ static inline struct iscsi_tcp_connection *TCP_CONN(struct iscsi_connection *con
return container_of(conn, struct iscsi_tcp_connection, iscsi_conn);
}
+/* all iscsi connections */
+static struct list_head iscsi_tcp_conn_list;
+
+static struct tgt_work nop_work;
+
+static int iscsi_send_ping_nop_in(struct iscsi_tcp_connection *tcp_conn)
+{
+ struct iscsi_connection *conn = &tcp_conn->iscsi_conn;
+ struct iscsi_task *task = NULL;
+
+ task = iscsi_tcp_alloc_task(&tcp_conn->iscsi_conn, 0);
+ task->conn = conn;
+
+ task->tag = ISCSI_RESERVED_TAG;
+ task->req.opcode = ISCSI_OP_NOOP_IN;
+ task->req.itt = cpu_to_be32(ISCSI_RESERVED_TAG);
+ task->req.ttt = cpu_to_be32(tcp_conn->ttt);
+
+ list_add_tail(&task->c_list, &task->conn->tx_clist);
+ task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
+
+ return 0;
+}
+
+static void iscsi_tcp_nop_work_handler(void *data)
+{
+ struct iscsi_tcp_connection *tcp_conn;
+
+ list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
+ tcp_conn->nop_inflight_count++;
+ if (nop_count && tcp_conn->nop_inflight_count > nop_count) {
+ eprintf("tcp connection timed out after %d failed " \
+ "NOP-IN\n", nop_count);
+ iscsi_tcp_release(&tcp_conn->iscsi_conn);
+ /* cant/shouldnt delete tcp_conn from within the loop */
+ break;
+ }
+ nop_ttt++;
+ if (nop_ttt == ISCSI_RESERVED_TAG)
+ nop_ttt = 1;
+
+ tcp_conn->ttt = nop_ttt;
+ iscsi_send_ping_nop_in(tcp_conn);
+ }
+
+ add_work(&nop_work, nop_interval);
+}
+
+static void iscsi_tcp_nop_reply(long ttt)
+{
+ struct iscsi_tcp_connection *tcp_conn;
+
+ list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
+ if (tcp_conn->ttt != ttt)
+ continue;
+ tcp_conn->nop_inflight_count = 0;
+ }
+}
+
static int set_keepalive(int fd)
{
int ret, opt;
@@ -144,6 +214,8 @@ static void accept_connection(int afd, int events, void *data)
goto out;
}
+ list_add(&tcp_conn->tcp_conn_siblings, &iscsi_tcp_conn_list);
+
return;
out:
close(fd);
@@ -279,6 +351,16 @@ int iscsi_add_portal(char *addr, int port, int tpgt)
return 0;
};
+void iscsi_set_nop_interval(int interval)
+{
+ nop_interval = interval;
+}
+
+void iscsi_set_nop_count(int count)
+{
+ nop_count = count;
+}
+
int iscsi_delete_portal(char *addr, int port)
{
struct iscsi_portal *portal;
@@ -313,6 +395,15 @@ static int iscsi_tcp_init(void)
iscsi_add_portal(NULL, 3260, 1);
}
+ if (nop_interval) {
+ nop_work.func = iscsi_tcp_nop_work_handler;
+ nop_work.data = &nop_work;
+
+ add_work(&nop_work, nop_interval);
+ }
+
+ INIT_LIST_HEAD(&iscsi_tcp_conn_list);
+
return 0;
}
@@ -370,6 +461,7 @@ static void iscsi_tcp_release(struct iscsi_connection *conn)
conn_exit(conn);
close(tcp_conn->fd);
+ list_del(&tcp_conn->tcp_conn_siblings);
free(tcp_conn);
}
@@ -480,6 +572,7 @@ static struct iscsi_transport iscsi_tcp = {
.free_data_buf = iscsi_tcp_free_data_buf,
.ep_getsockname = iscsi_tcp_getsockname,
.ep_getpeername = iscsi_tcp_getpeername,
+ .ep_nop_reply = iscsi_tcp_nop_reply,
};
__attribute__((constructor)) static void iscsi_transport_init(void)
diff --git a/usr/iscsi/iscsid.c b/usr/iscsi/iscsid.c
index b10be1d..2161f1c 100644
--- a/usr/iscsi/iscsid.c
+++ b/usr/iscsi/iscsid.c
@@ -1660,13 +1660,9 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
dprintf("%x %x %u\n", req->ttt, req->itt, ntoh24(req->dlength));
if (req->ttt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
- /*
- * We don't request a NOP-Out by sending a NOP-In.
- * See 10.18.2 in the draft 20.
- */
- eprintf("initiator bug\n");
- err = -ISCSI_REASON_PROTOCOL_ERROR;
- goto out;
+ if ((req->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_NOOP_OUT) {
+ goto good;
+ }
}
if (req->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
@@ -1677,6 +1673,7 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
}
}
+good:
conn->exp_stat_sn = be32_to_cpu(req->exp_statsn);
len = ntoh24(req->dlength);
@@ -1705,8 +1702,8 @@ static int iscsi_task_rx_done(struct iscsi_connection *conn)
op = hdr->opcode & ISCSI_OPCODE_MASK;
switch (op) {
- case ISCSI_OP_SCSI_CMD:
case ISCSI_OP_NOOP_OUT:
+ case ISCSI_OP_SCSI_CMD:
case ISCSI_OP_SCSI_TMFUNC:
case ISCSI_OP_LOGOUT:
err = iscsi_task_queue(task);
@@ -1799,6 +1796,28 @@ static int iscsi_logout_tx_start(struct iscsi_task *task)
return 0;
}
+static int iscsi_noop_in_tx_start(struct iscsi_task *task)
+{
+ struct iscsi_connection *conn = task->conn;
+ struct iscsi_data_rsp *rsp = (struct iscsi_data_rsp *) &conn->rsp.bhs;
+
+ memset(rsp, 0, sizeof(*rsp));
+ rsp->opcode = ISCSI_OP_NOOP_IN;
+ rsp->flags = ISCSI_FLAG_CMD_FINAL;
+ rsp->itt = task->req.itt;
+ rsp->ttt = task->req.ttt;
+ rsp->statsn = cpu_to_be32(conn->stat_sn);
+ rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
+ rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
+
+ /* TODO: honor max_burst */
+ conn->rsp.datasize = task->len;
+ hton24(rsp->dlength, task->len);
+ conn->rsp.data = task->data;
+
+ return 0;
+}
+
static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
{
struct iscsi_connection *conn = task->conn;
@@ -1806,6 +1825,10 @@ static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
if (task->req.itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
*is_rsp = 0;
+
+ if (conn->tp->ep_nop_reply)
+ conn->tp->ep_nop_reply(be32_to_cpu(task->req.ttt));
+
iscsi_free_task(task);
} else {
*is_rsp = 1;
@@ -1918,6 +1941,9 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
case ISCSI_OP_SCSI_CMD:
err = iscsi_scsi_cmd_tx_start(task);
break;
+ case ISCSI_OP_NOOP_IN:
+ err = iscsi_noop_in_tx_start(task);
+ break;
case ISCSI_OP_NOOP_OUT:
err = iscsi_noop_out_tx_start(task, &is_rsp);
if (!is_rsp)
@@ -2413,6 +2439,10 @@ int iscsi_param_parse_portals(char *p, int do_add,
return -1;
}
}
+ } else if (!strncmp(p, "nop_interval", 12)) {
+ iscsi_set_nop_interval(atoi(p+13));
+ } else if (!strncmp(p, "nop_count", 9)) {
+ iscsi_set_nop_count(atoi(p+10));
}
p += strcspn(p, ",");
diff --git a/usr/iscsi/iscsid.h b/usr/iscsi/iscsid.h
index e7e08ae..54a5363 100644
--- a/usr/iscsi/iscsid.h
+++ b/usr/iscsi/iscsid.h
@@ -316,6 +316,8 @@ extern int iscsi_scsi_cmd_execute(struct iscsi_task *task);
extern int iscsi_transportid(int tid, uint64_t itn_id, char *buf, int size);
extern int iscsi_add_portal(char *addr, int port, int tpgt);
extern int iscsi_delete_portal(char *addr, int port);
+extern void iscsi_set_nop_interval(int interval);
+extern void iscsi_set_nop_count(int count);
extern int iscsi_param_parse_portals(char *p, int do_add, int do_delete);
extern void iscsi_update_conn_stats_rx(struct iscsi_connection *conn, int size, int opcode);
extern void iscsi_update_conn_stats_tx(struct iscsi_connection *conn, int size, int opcode);
diff --git a/usr/iscsi/transport.h b/usr/iscsi/transport.h
index af4af21..6573207 100644
--- a/usr/iscsi/transport.h
+++ b/usr/iscsi/transport.h
@@ -39,6 +39,7 @@ struct iscsi_transport {
struct sockaddr *sa, socklen_t *len);
int (*ep_getpeername)(struct iscsi_connection *conn,
struct sockaddr *sa, socklen_t *len);
+ void (*ep_nop_reply)(long ttt);
};
extern int iscsi_transport_register(struct iscsi_transport *);
--
1.7.3.1
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
More information about the stgt
mailing list