[stgt] [PATCH] Add NOP-OUT probe support.
Ronnie Sahlberg
ronniesahlberg at gmail.com
Sun Jun 30 06:24:14 CEST 2013
Add support for setting NOP-OUT probes to detect dead initiators.
We can set the interval(how frequently to send the probes)
and the count(after how many failures will we tear down the connection)
globally from the tgtd command line.
Optionally, using tgtadm, it is possible to configure the interval/count
for individual targets.
Signed-off-by: Ronnie Sahlberg <ronniesahlberg at gmail.com>
---
doc/tgtadm.8.xml | 42 ++++++++++++
doc/tgtd.8.xml | 30 +++++++++
usr/iscsi/iscsi_tcp.c | 167 +++++++++++++++++++++++++++++++++++++++++++++++++
usr/iscsi/iscsid.c | 47 ++++++++++++--
usr/iscsi/iscsid.h | 10 +++
usr/iscsi/target.c | 14 ++++
usr/iscsi/transport.h | 1 +
usr/target.c | 4 +
usr/tgtadm.c | 4 +-
9 files changed, 310 insertions(+), 9 deletions(-)
diff --git a/doc/tgtadm.8.xml b/doc/tgtadm.8.xml
index 18813dc..315ffdf 100644
--- a/doc/tgtadm.8.xml
+++ b/doc/tgtadm.8.xml
@@ -678,6 +678,48 @@ Account information:
</refsect1>
+ <refsect1><title>NOP-OUT Probes</title>
+ <para>
+ TGTD can send NOP-OUT probes to connected initiators to determine when
+ an initiator is dead and then automatically clear and tear down the
+ TCP connection. This can either be set as a global default from the
+ tgtd command-line or it can be set for individual targets using the
+ tgtadm command.
+ </para>
+ <refsect2><title>Check the current NOP-OUT setting</title>
+ <para>
+ The tgtadm command is used to view the current setting for if/when
+ to send NOP-OUT probes to connected initiators.
+ </para>
+ <para>
+ If the target is configured to send NOP-OUT probes this will show up
+ as two parameter lines in the target printout. If the target is not
+ configured to send NOP-OUT these lines will not be printed at all.
+ </para>
+ <screen format="linespecific">
+tgtadm --lld iscsi --op show --mode target
+
+Target 1: iqn.ronnie.test
+ System information:
+ Driver: iscsi
+ State: ready
+ Nop interval: 5
+ Nop count: 5
+ I_T nexus information:
+ </screen>
+ </refsect2>
+ <refsect2><title>Setting NOP-OUT for a target</title>
+ <para>
+ The tgtadm command is used to change the NOP-OUT settings.
+ </para>
+ <screen format="linespecific">
+tgtadm --op update --mode target --tid 1 -n nop_count -v 5
+tgtadm --op update --mode target --tid 1 -n nop_interval -v 5
+ </screen>
+ </refsect2>
+ </refsect1>
+
+
<refsect1><title>iSCSI PORTALS</title>
<para>
iSCSI portals can be viewed, added and removed at runtime.
diff --git a/doc/tgtd.8.xml b/doc/tgtd.8.xml
index ce734ce..a0020c6 100644
--- a/doc/tgtd.8.xml
+++ b/doc/tgtd.8.xml
@@ -129,6 +129,36 @@
</screen>
</para>
</refsect2>
+ <refsect2><title>nop_interval=<integer></title>
+ <para>
+ This sets the default interval for sending NOP-OUT to probe for
+ connected initiators.
+ This parameter only controls the default value for targets.
+ Individual targets can be controlled using tgtadm.
+ </para>
+ <para>
+ The default value is 0 which means that the feature is disabled
+ TGTD will not send any NOP-OUT probes.
+ </para>
+ </refsect2>
+ <refsect2><title>nop_count=<integer></title>
+ <para>
+ This sets the default value for after how many failed probes TGTD
+ will consider the initiator dead and tear down the session.
+ This parameter only controls the default value for targets.
+ Individual targets can be controlled using tgtadm.
+ </para>
+ <para>
+ The default value is 0.
+ </para>
+ <para>
+ Example: send NOP-OUT every 5 seconds and abort the session after
+ 6 failures.
+ <screen format="linespecific">
+ tgtd --iscsi portal=192.0.2.1:3260,nop_interval=5,nop_count=6
+ </screen>
+ </para>
+ </refsect2>
</refsect1>
diff --git a/usr/iscsi/iscsi_tcp.c b/usr/iscsi/iscsi_tcp.c
index 0c43039..aa50780 100644
--- a/usr/iscsi/iscsi_tcp.c
+++ b/usr/iscsi/iscsi_tcp.c
@@ -35,8 +35,15 @@
#include "iscsid.h"
#include "tgtd.h"
#include "util.h"
+#include "work.h"
static void iscsi_tcp_event_handler(int fd, int events, void *data);
+static void iscsi_tcp_release(struct iscsi_connection *conn);
+static struct iscsi_task *iscsi_tcp_alloc_task(struct iscsi_connection *conn,
+ size_t ext_len);
+static void iscsi_tcp_free_task(struct iscsi_task *task);
+
+static long nop_ttt;
static int listen_fds[8];
static struct iscsi_transport iscsi_tcp;
@@ -44,6 +51,13 @@ static struct iscsi_transport iscsi_tcp;
struct iscsi_tcp_connection {
int fd;
+ struct list_head tcp_conn_siblings;
+ int nop_inflight_count;
+ int nop_interval;
+ int nop_tick;
+ int nop_count;
+ long ttt;
+
struct iscsi_connection iscsi_conn;
};
@@ -52,6 +66,110 @@ static inline struct iscsi_tcp_connection *TCP_CONN(struct iscsi_connection *con
return container_of(conn, struct iscsi_tcp_connection, iscsi_conn);
}
+static struct tgt_work nop_work;
+
+/* all iscsi connections */
+static struct list_head iscsi_tcp_conn_list;
+
+static int iscsi_send_ping_nop_in(struct iscsi_tcp_connection *tcp_conn)
+{
+ struct iscsi_connection *conn = &tcp_conn->iscsi_conn;
+ struct iscsi_task *task = NULL;
+
+ task = iscsi_tcp_alloc_task(&tcp_conn->iscsi_conn, 0);
+ task->conn = conn;
+
+ task->tag = ISCSI_RESERVED_TAG;
+ task->req.opcode = ISCSI_OP_NOOP_IN;
+ task->req.itt = cpu_to_be32(ISCSI_RESERVED_TAG);
+ task->req.ttt = cpu_to_be32(tcp_conn->ttt);
+
+ list_add_tail(&task->c_list, &task->conn->tx_clist);
+ task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
+
+ return 0;
+}
+
+static void iscsi_tcp_nop_work_handler(void *data)
+{
+ struct iscsi_tcp_connection *tcp_conn;
+
+ list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
+ if (tcp_conn->nop_interval == 0)
+ continue;
+
+ tcp_conn->nop_tick--;
+ if (tcp_conn->nop_tick > 0)
+ continue;
+
+ tcp_conn->nop_tick = tcp_conn->nop_interval;
+
+ tcp_conn->nop_inflight_count++;
+ if (tcp_conn->nop_inflight_count > tcp_conn->nop_count) {
+ eprintf("tcp connection timed out after %d failed " \
+ "NOP-OUT\n", tcp_conn->nop_count);
+ iscsi_tcp_release(&tcp_conn->iscsi_conn);
+ /* cant/shouldnt delete tcp_conn from within the loop */
+ break;
+ }
+ nop_ttt++;
+ if (nop_ttt == ISCSI_RESERVED_TAG)
+ nop_ttt = 1;
+
+ tcp_conn->ttt = nop_ttt;
+ iscsi_send_ping_nop_in(tcp_conn);
+ }
+
+ add_work(&nop_work, 1);
+}
+
+static void iscsi_tcp_nop_reply(long ttt)
+{
+ struct iscsi_tcp_connection *tcp_conn;
+
+ list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
+ if (tcp_conn->ttt != ttt)
+ continue;
+ tcp_conn->nop_inflight_count = 0;
+ }
+}
+
+int iscsi_update_target_nop_count(int tid, int count)
+{
+ struct iscsi_target *target;
+
+ list_for_each_entry(target, &iscsi_targets_list, tlist) {
+ if (target->tid != tid)
+ continue;
+ target->nop_count = count;
+ return 0;
+ }
+ return -1;
+}
+
+int iscsi_update_target_nop_interval(int tid, int interval)
+{
+ struct iscsi_target *target;
+
+ list_for_each_entry(target, &iscsi_targets_list, tlist) {
+ if (target->tid != tid)
+ continue;
+ target->nop_interval = interval;
+ return 0;
+ }
+ return -1;
+}
+
+void iscsi_set_nop_interval(int interval)
+{
+ default_nop_interval = interval;
+}
+
+void iscsi_set_nop_count(int count)
+{
+ default_nop_count = count;
+}
+
static int set_keepalive(int fd)
{
int ret, opt;
@@ -144,6 +262,8 @@ static void accept_connection(int afd, int events, void *data)
goto out;
}
+ list_add(&tcp_conn->tcp_conn_siblings, &iscsi_tcp_conn_list);
+
return;
out:
close(fd);
@@ -313,6 +433,12 @@ static int iscsi_tcp_init(void)
iscsi_add_portal(NULL, 3260, 1);
}
+ INIT_LIST_HEAD(&iscsi_tcp_conn_list);
+
+ nop_work.func = iscsi_tcp_nop_work_handler;
+ nop_work.data = &nop_work;
+ add_work(&nop_work, 1);
+
return 0;
}
@@ -328,6 +454,26 @@ static void iscsi_tcp_exit(void)
static int iscsi_tcp_conn_login_complete(struct iscsi_connection *conn)
{
+ struct iscsi_tcp_connection *tcp_conn;
+ struct iscsi_target *target;
+
+ list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings)
+ if (&tcp_conn->iscsi_conn == conn)
+ break;
+
+ if (tcp_conn == NULL)
+ return 0;
+
+ list_for_each_entry(target, &iscsi_targets_list, tlist) {
+ if (target->tid != conn->tid)
+ continue;
+
+ tcp_conn->nop_count = target->nop_count;
+ tcp_conn->nop_interval = target->nop_interval;
+ tcp_conn->nop_tick = target->nop_interval;
+ break;
+ }
+
return 0;
}
@@ -370,6 +516,7 @@ static void iscsi_tcp_release(struct iscsi_connection *conn)
conn_exit(conn);
close(tcp_conn->fd);
+ list_del(&tcp_conn->tcp_conn_siblings);
free(tcp_conn);
}
@@ -459,6 +606,25 @@ static void iscsi_tcp_conn_force_close(struct iscsi_connection *conn)
conn->tp->ep_event_modify(conn, EPOLLIN|EPOLLOUT|EPOLLERR);
}
+void iscsi_print_nop_settings(struct concat_buf *b, int tid)
+{
+ struct iscsi_target *target;
+
+ list_for_each_entry(target, &iscsi_targets_list, tlist) {
+ if (target->tid != tid)
+ continue;
+ if (target->nop_interval == 0)
+ continue;
+
+ concat_printf(b,
+ _TAB2 "Nop interval: %d\n"
+ _TAB2 "Nop count: %d\n",
+ target->nop_interval,
+ target->nop_count);
+ break;
+ }
+}
+
static struct iscsi_transport iscsi_tcp = {
.name = "iscsi",
.rdma = 0,
@@ -480,6 +646,7 @@ static struct iscsi_transport iscsi_tcp = {
.free_data_buf = iscsi_tcp_free_data_buf,
.ep_getsockname = iscsi_tcp_getsockname,
.ep_getpeername = iscsi_tcp_getpeername,
+ .ep_nop_reply = iscsi_tcp_nop_reply,
};
__attribute__((constructor)) static void iscsi_transport_init(void)
diff --git a/usr/iscsi/iscsid.c b/usr/iscsi/iscsid.c
index 260989f..005bac5 100644
--- a/usr/iscsi/iscsid.c
+++ b/usr/iscsi/iscsid.c
@@ -43,6 +43,9 @@
#define MAX_QUEUE_CMD 128
+int default_nop_interval;
+int default_nop_count;
+
LIST_HEAD(iscsi_portals_list);
char *portal_arguments;
@@ -1674,13 +1677,9 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
dprintf("%x %x %u\n", req->ttt, req->itt, ntoh24(req->dlength));
if (req->ttt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
- /*
- * We don't request a NOP-Out by sending a NOP-In.
- * See 10.18.2 in the draft 20.
- */
- eprintf("initiator bug\n");
- err = -ISCSI_REASON_PROTOCOL_ERROR;
- goto out;
+ if ((req->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_NOOP_OUT) {
+ goto good;
+ }
}
if (req->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
@@ -1691,6 +1690,7 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
}
}
+good:
conn->exp_stat_sn = be32_to_cpu(req->exp_statsn);
len = ntoh24(req->dlength);
@@ -1836,6 +1836,28 @@ static int iscsi_logout_tx_start(struct iscsi_task *task)
return 0;
}
+static int iscsi_noop_in_tx_start(struct iscsi_task *task)
+{
+ struct iscsi_connection *conn = task->conn;
+ struct iscsi_data_rsp *rsp = (struct iscsi_data_rsp *) &conn->rsp.bhs;
+
+ memset(rsp, 0, sizeof(*rsp));
+ rsp->opcode = ISCSI_OP_NOOP_IN;
+ rsp->flags = ISCSI_FLAG_CMD_FINAL;
+ rsp->itt = task->req.itt;
+ rsp->ttt = task->req.ttt;
+ rsp->statsn = cpu_to_be32(conn->stat_sn);
+ rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
+ rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
+
+ /* TODO: honor max_burst */
+ conn->rsp.datasize = task->len;
+ hton24(rsp->dlength, task->len);
+ conn->rsp.data = task->data;
+
+ return 0;
+}
+
static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
{
struct iscsi_connection *conn = task->conn;
@@ -1843,6 +1865,10 @@ static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
if (task->req.itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
*is_rsp = 0;
+
+ if (conn->tp->ep_nop_reply)
+ conn->tp->ep_nop_reply(be32_to_cpu(task->req.ttt));
+
iscsi_free_task(task);
} else {
*is_rsp = 1;
@@ -1955,6 +1981,9 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
case ISCSI_OP_SCSI_CMD:
err = iscsi_scsi_cmd_tx_start(task);
break;
+ case ISCSI_OP_NOOP_IN:
+ err = iscsi_noop_in_tx_start(task);
+ break;
case ISCSI_OP_NOOP_OUT:
err = iscsi_noop_out_tx_start(task, &is_rsp);
if (!is_rsp)
@@ -2450,6 +2479,10 @@ int iscsi_param_parse_portals(char *p, int do_add,
return -1;
}
}
+ } else if (!strncmp(p, "nop_interval", 12)) {
+ iscsi_set_nop_interval(atoi(p+13));
+ } else if (!strncmp(p, "nop_count", 9)) {
+ iscsi_set_nop_count(atoi(p+10));
}
p += strcspn(p, ",");
diff --git a/usr/iscsi/iscsid.h b/usr/iscsi/iscsid.h
index e7e08ae..0710ccb 100644
--- a/usr/iscsi/iscsid.h
+++ b/usr/iscsi/iscsid.h
@@ -239,6 +239,9 @@ struct iscsi_connection {
#define INCOMING_BUFSIZE 8192
+extern int default_nop_interval;
+extern int default_nop_count;
+
struct iscsi_target {
struct list_head tlist;
@@ -262,6 +265,8 @@ struct iscsi_target {
struct list_head isns_list;
int rdma;
+ int nop_interval;
+ int nop_count;
};
enum task_flags {
@@ -315,6 +320,11 @@ extern void iscsi_rx_handler(struct iscsi_connection *conn);
extern int iscsi_scsi_cmd_execute(struct iscsi_task *task);
extern int iscsi_transportid(int tid, uint64_t itn_id, char *buf, int size);
extern int iscsi_add_portal(char *addr, int port, int tpgt);
+extern void iscsi_print_nop_settings(struct concat_buf *b, int tid);
+extern int iscsi_update_target_nop_count(int tid, int count);
+extern int iscsi_update_target_nop_interval(int tid, int interval);
+extern void iscsi_set_nop_interval(int interval);
+extern void iscsi_set_nop_count(int count);
extern int iscsi_delete_portal(char *addr, int port);
extern int iscsi_param_parse_portals(char *p, int do_add, int do_delete);
extern void iscsi_update_conn_stats_rx(struct iscsi_connection *conn, int size, int opcode);
diff --git a/usr/iscsi/target.c b/usr/iscsi/target.c
index 916c84a..20fb1da 100644
--- a/usr/iscsi/target.c
+++ b/usr/iscsi/target.c
@@ -470,6 +470,8 @@ int iscsi_target_create(struct target *t)
INIT_LIST_HEAD(&target->sessions_list);
INIT_LIST_HEAD(&target->isns_list);
target->tid = tid;
+ target->nop_interval = default_nop_interval;
+ target->nop_count = default_nop_count;
list_add_tail(&target->tlist, &iscsi_targets_list);
isns_target_register(tgt_targetname(tid));
@@ -545,6 +547,18 @@ tgtadm_err iscsi_target_update(int mode, int op, int tid, uint64_t sid, uint64_t
break;
}
adm_err = TGTADM_SUCCESS;
+ } else if (!strncmp(name, "nop_count", 9)) {
+ err = iscsi_update_target_nop_count(tid,
+ atoi(&name[10]));
+ adm_err = !err ? TGTADM_SUCCESS :
+ TGTADM_INVALID_REQUEST;
+ break;
+ } else if (!strncmp(name, "nop_interval", 12)) {
+ err = iscsi_update_target_nop_interval(tid,
+ atoi(&name[13]));
+ adm_err = !err ? TGTADM_SUCCESS :
+ TGTADM_INVALID_REQUEST;
+ break;
}
idx = param_index_by_name(name, session_keys);
diff --git a/usr/iscsi/transport.h b/usr/iscsi/transport.h
index af4af21..f4300c1 100644
--- a/usr/iscsi/transport.h
+++ b/usr/iscsi/transport.h
@@ -39,6 +39,7 @@ struct iscsi_transport {
struct sockaddr *sa, socklen_t *len);
int (*ep_getpeername)(struct iscsi_connection *conn,
struct sockaddr *sa, socklen_t *len);
+ void (*ep_nop_reply) (long ttt);
};
extern int iscsi_transport_register(struct iscsi_transport *);
diff --git a/usr/target.c b/usr/target.c
index 9a8f2fd..23b5766 100644
--- a/usr/target.c
+++ b/usr/target.c
@@ -36,6 +36,7 @@
#include "driver.h"
#include "target.h"
#include "scsi.h"
+#include "iscsi/iscsid.h"
#include "tgtadm.h"
#include "parser.h"
#include "spc.h"
@@ -2000,6 +2001,9 @@ tgtadm_err tgt_target_show_all(struct concat_buf *b)
tgt_drivers[target->lid]->name,
target_state_name(target->target_state));
+ if (!strcmp(tgt_drivers[target->lid]->name, "iscsi"))
+ iscsi_print_nop_settings(b, target->tid);
+
concat_printf(b, _TAB1 "I_T nexus information:\n");
list_for_each_entry(nexus, &target->it_nexus_list,
diff --git a/usr/tgtadm.c b/usr/tgtadm.c
index 90803f2..ac7a386 100644
--- a/usr/tgtadm.c
+++ b/usr/tgtadm.c
@@ -707,12 +707,12 @@ int main(int argc, char **argv)
case OP_UPDATE:
rc = verify_mode_params(argc, argv, "LmotnvC");
if (rc) {
- eprintf("target mode: option '-%c' is not "
+ eprintf("target mode: option '-%c' is not " \
"allowed/supported\n", rc);
exit(EINVAL);
}
if ((!name || !value)) {
- eprintf("update operation requires 'name'"
+ eprintf("update operation requires 'name'" \
" and 'value' options\n");
exit(EINVAL);
}
--
1.7.3.1
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
More information about the stgt
mailing list