[stgt] [PATCH] Add NOP-OUT probe support.
ronnie sahlberg
ronniesahlberg at gmail.com
Fri Jul 5 02:09:47 CEST 2013
ping
On Sat, Jun 29, 2013 at 9:24 PM, Ronnie Sahlberg
<ronniesahlberg at gmail.com> wrote:
> Add support for setting NOP-OUT probes to detect dead initiators.
> We can set the interval(how frequently to send the probes)
> and the count(after how many failures will we tear down the connection)
> globally from the tgtd command line.
>
> Optionally, using tgtadm, it is possible to configure the interval/count
> for individual targets.
>
> Signed-off-by: Ronnie Sahlberg <ronniesahlberg at gmail.com>
> ---
> doc/tgtadm.8.xml | 42 ++++++++++++
> doc/tgtd.8.xml | 30 +++++++++
> usr/iscsi/iscsi_tcp.c | 167 +++++++++++++++++++++++++++++++++++++++++++++++++
> usr/iscsi/iscsid.c | 47 ++++++++++++--
> usr/iscsi/iscsid.h | 10 +++
> usr/iscsi/target.c | 14 ++++
> usr/iscsi/transport.h | 1 +
> usr/target.c | 4 +
> usr/tgtadm.c | 4 +-
> 9 files changed, 310 insertions(+), 9 deletions(-)
>
> diff --git a/doc/tgtadm.8.xml b/doc/tgtadm.8.xml
> index 18813dc..315ffdf 100644
> --- a/doc/tgtadm.8.xml
> +++ b/doc/tgtadm.8.xml
> @@ -678,6 +678,48 @@ Account information:
> </refsect1>
>
>
> + <refsect1><title>NOP-OUT Probes</title>
> + <para>
> + TGTD can send NOP-OUT probes to connected initiators to determine when
> + an initiator is dead and then automatically clear and tear down the
> + TCP connection. This can either be set as a global default from the
> + tgtd command-line or it can be set for individual targets using the
> + tgtadm command.
> + </para>
> + <refsect2><title>Check the current NOP-OUT setting</title>
> + <para>
> + The tgtadm command is used to view the current setting for if/when
> + to send NOP-OUT probes to connected initiators.
> + </para>
> + <para>
> + If the target is configured to send NOP-OUT probes this will show up
> + as two parameter lines in the target printout. If the target is not
> + configured to send NOP-OUT these lines will not be printed at all.
> + </para>
> + <screen format="linespecific">
> +tgtadm --lld iscsi --op show --mode target
> +
> +Target 1: iqn.ronnie.test
> + System information:
> + Driver: iscsi
> + State: ready
> + Nop interval: 5
> + Nop count: 5
> + I_T nexus information:
> + </screen>
> + </refsect2>
> + <refsect2><title>Setting NOP-OUT for a target</title>
> + <para>
> + The tgtadm command is used to change the NOP-OUT settings.
> + </para>
> + <screen format="linespecific">
> +tgtadm --op update --mode target --tid 1 -n nop_count -v 5
> +tgtadm --op update --mode target --tid 1 -n nop_interval -v 5
> + </screen>
> + </refsect2>
> + </refsect1>
> +
> +
> <refsect1><title>iSCSI PORTALS</title>
> <para>
> iSCSI portals can be viewed, added and removed at runtime.
> diff --git a/doc/tgtd.8.xml b/doc/tgtd.8.xml
> index ce734ce..a0020c6 100644
> --- a/doc/tgtd.8.xml
> +++ b/doc/tgtd.8.xml
> @@ -129,6 +129,36 @@
> </screen>
> </para>
> </refsect2>
> + <refsect2><title>nop_interval=<integer></title>
> + <para>
> + This sets the default interval for sending NOP-OUT to probe for
> + connected initiators.
> + This parameter only controls the default value for targets.
> + Individual targets can be controlled using tgtadm.
> + </para>
> + <para>
> + The default value is 0 which means that the feature is disabled
> + TGTD will not send any NOP-OUT probes.
> + </para>
> + </refsect2>
> + <refsect2><title>nop_count=<integer></title>
> + <para>
> + This sets the default value for after how many failed probes TGTD
> + will consider the initiator dead and tear down the session.
> + This parameter only controls the default value for targets.
> + Individual targets can be controlled using tgtadm.
> + </para>
> + <para>
> + The default value is 0.
> + </para>
> + <para>
> + Example: send NOP-OUT every 5 seconds and abort the session after
> + 6 failures.
> + <screen format="linespecific">
> + tgtd --iscsi portal=192.0.2.1:3260,nop_interval=5,nop_count=6
> + </screen>
> + </para>
> + </refsect2>
> </refsect1>
>
>
> diff --git a/usr/iscsi/iscsi_tcp.c b/usr/iscsi/iscsi_tcp.c
> index 0c43039..aa50780 100644
> --- a/usr/iscsi/iscsi_tcp.c
> +++ b/usr/iscsi/iscsi_tcp.c
> @@ -35,8 +35,15 @@
> #include "iscsid.h"
> #include "tgtd.h"
> #include "util.h"
> +#include "work.h"
>
> static void iscsi_tcp_event_handler(int fd, int events, void *data);
> +static void iscsi_tcp_release(struct iscsi_connection *conn);
> +static struct iscsi_task *iscsi_tcp_alloc_task(struct iscsi_connection *conn,
> + size_t ext_len);
> +static void iscsi_tcp_free_task(struct iscsi_task *task);
> +
> +static long nop_ttt;
>
> static int listen_fds[8];
> static struct iscsi_transport iscsi_tcp;
> @@ -44,6 +51,13 @@ static struct iscsi_transport iscsi_tcp;
> struct iscsi_tcp_connection {
> int fd;
>
> + struct list_head tcp_conn_siblings;
> + int nop_inflight_count;
> + int nop_interval;
> + int nop_tick;
> + int nop_count;
> + long ttt;
> +
> struct iscsi_connection iscsi_conn;
> };
>
> @@ -52,6 +66,110 @@ static inline struct iscsi_tcp_connection *TCP_CONN(struct iscsi_connection *con
> return container_of(conn, struct iscsi_tcp_connection, iscsi_conn);
> }
>
> +static struct tgt_work nop_work;
> +
> +/* all iscsi connections */
> +static struct list_head iscsi_tcp_conn_list;
> +
> +static int iscsi_send_ping_nop_in(struct iscsi_tcp_connection *tcp_conn)
> +{
> + struct iscsi_connection *conn = &tcp_conn->iscsi_conn;
> + struct iscsi_task *task = NULL;
> +
> + task = iscsi_tcp_alloc_task(&tcp_conn->iscsi_conn, 0);
> + task->conn = conn;
> +
> + task->tag = ISCSI_RESERVED_TAG;
> + task->req.opcode = ISCSI_OP_NOOP_IN;
> + task->req.itt = cpu_to_be32(ISCSI_RESERVED_TAG);
> + task->req.ttt = cpu_to_be32(tcp_conn->ttt);
> +
> + list_add_tail(&task->c_list, &task->conn->tx_clist);
> + task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
> +
> + return 0;
> +}
> +
> +static void iscsi_tcp_nop_work_handler(void *data)
> +{
> + struct iscsi_tcp_connection *tcp_conn;
> +
> + list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
> + if (tcp_conn->nop_interval == 0)
> + continue;
> +
> + tcp_conn->nop_tick--;
> + if (tcp_conn->nop_tick > 0)
> + continue;
> +
> + tcp_conn->nop_tick = tcp_conn->nop_interval;
> +
> + tcp_conn->nop_inflight_count++;
> + if (tcp_conn->nop_inflight_count > tcp_conn->nop_count) {
> + eprintf("tcp connection timed out after %d failed " \
> + "NOP-OUT\n", tcp_conn->nop_count);
> + iscsi_tcp_release(&tcp_conn->iscsi_conn);
> + /* cant/shouldnt delete tcp_conn from within the loop */
> + break;
> + }
> + nop_ttt++;
> + if (nop_ttt == ISCSI_RESERVED_TAG)
> + nop_ttt = 1;
> +
> + tcp_conn->ttt = nop_ttt;
> + iscsi_send_ping_nop_in(tcp_conn);
> + }
> +
> + add_work(&nop_work, 1);
> +}
> +
> +static void iscsi_tcp_nop_reply(long ttt)
> +{
> + struct iscsi_tcp_connection *tcp_conn;
> +
> + list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
> + if (tcp_conn->ttt != ttt)
> + continue;
> + tcp_conn->nop_inflight_count = 0;
> + }
> +}
> +
> +int iscsi_update_target_nop_count(int tid, int count)
> +{
> + struct iscsi_target *target;
> +
> + list_for_each_entry(target, &iscsi_targets_list, tlist) {
> + if (target->tid != tid)
> + continue;
> + target->nop_count = count;
> + return 0;
> + }
> + return -1;
> +}
> +
> +int iscsi_update_target_nop_interval(int tid, int interval)
> +{
> + struct iscsi_target *target;
> +
> + list_for_each_entry(target, &iscsi_targets_list, tlist) {
> + if (target->tid != tid)
> + continue;
> + target->nop_interval = interval;
> + return 0;
> + }
> + return -1;
> +}
> +
> +void iscsi_set_nop_interval(int interval)
> +{
> + default_nop_interval = interval;
> +}
> +
> +void iscsi_set_nop_count(int count)
> +{
> + default_nop_count = count;
> +}
> +
> static int set_keepalive(int fd)
> {
> int ret, opt;
> @@ -144,6 +262,8 @@ static void accept_connection(int afd, int events, void *data)
> goto out;
> }
>
> + list_add(&tcp_conn->tcp_conn_siblings, &iscsi_tcp_conn_list);
> +
> return;
> out:
> close(fd);
> @@ -313,6 +433,12 @@ static int iscsi_tcp_init(void)
> iscsi_add_portal(NULL, 3260, 1);
> }
>
> + INIT_LIST_HEAD(&iscsi_tcp_conn_list);
> +
> + nop_work.func = iscsi_tcp_nop_work_handler;
> + nop_work.data = &nop_work;
> + add_work(&nop_work, 1);
> +
> return 0;
> }
>
> @@ -328,6 +454,26 @@ static void iscsi_tcp_exit(void)
>
> static int iscsi_tcp_conn_login_complete(struct iscsi_connection *conn)
> {
> + struct iscsi_tcp_connection *tcp_conn;
> + struct iscsi_target *target;
> +
> + list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings)
> + if (&tcp_conn->iscsi_conn == conn)
> + break;
> +
> + if (tcp_conn == NULL)
> + return 0;
> +
> + list_for_each_entry(target, &iscsi_targets_list, tlist) {
> + if (target->tid != conn->tid)
> + continue;
> +
> + tcp_conn->nop_count = target->nop_count;
> + tcp_conn->nop_interval = target->nop_interval;
> + tcp_conn->nop_tick = target->nop_interval;
> + break;
> + }
> +
> return 0;
> }
>
> @@ -370,6 +516,7 @@ static void iscsi_tcp_release(struct iscsi_connection *conn)
>
> conn_exit(conn);
> close(tcp_conn->fd);
> + list_del(&tcp_conn->tcp_conn_siblings);
> free(tcp_conn);
> }
>
> @@ -459,6 +606,25 @@ static void iscsi_tcp_conn_force_close(struct iscsi_connection *conn)
> conn->tp->ep_event_modify(conn, EPOLLIN|EPOLLOUT|EPOLLERR);
> }
>
> +void iscsi_print_nop_settings(struct concat_buf *b, int tid)
> +{
> + struct iscsi_target *target;
> +
> + list_for_each_entry(target, &iscsi_targets_list, tlist) {
> + if (target->tid != tid)
> + continue;
> + if (target->nop_interval == 0)
> + continue;
> +
> + concat_printf(b,
> + _TAB2 "Nop interval: %d\n"
> + _TAB2 "Nop count: %d\n",
> + target->nop_interval,
> + target->nop_count);
> + break;
> + }
> +}
> +
> static struct iscsi_transport iscsi_tcp = {
> .name = "iscsi",
> .rdma = 0,
> @@ -480,6 +646,7 @@ static struct iscsi_transport iscsi_tcp = {
> .free_data_buf = iscsi_tcp_free_data_buf,
> .ep_getsockname = iscsi_tcp_getsockname,
> .ep_getpeername = iscsi_tcp_getpeername,
> + .ep_nop_reply = iscsi_tcp_nop_reply,
> };
>
> __attribute__((constructor)) static void iscsi_transport_init(void)
> diff --git a/usr/iscsi/iscsid.c b/usr/iscsi/iscsid.c
> index 260989f..005bac5 100644
> --- a/usr/iscsi/iscsid.c
> +++ b/usr/iscsi/iscsid.c
> @@ -43,6 +43,9 @@
>
> #define MAX_QUEUE_CMD 128
>
> +int default_nop_interval;
> +int default_nop_count;
> +
> LIST_HEAD(iscsi_portals_list);
>
> char *portal_arguments;
> @@ -1674,13 +1677,9 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
>
> dprintf("%x %x %u\n", req->ttt, req->itt, ntoh24(req->dlength));
> if (req->ttt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
> - /*
> - * We don't request a NOP-Out by sending a NOP-In.
> - * See 10.18.2 in the draft 20.
> - */
> - eprintf("initiator bug\n");
> - err = -ISCSI_REASON_PROTOCOL_ERROR;
> - goto out;
> + if ((req->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_NOOP_OUT) {
> + goto good;
> + }
> }
>
> if (req->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
> @@ -1691,6 +1690,7 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
> }
> }
>
> +good:
> conn->exp_stat_sn = be32_to_cpu(req->exp_statsn);
>
> len = ntoh24(req->dlength);
> @@ -1836,6 +1836,28 @@ static int iscsi_logout_tx_start(struct iscsi_task *task)
> return 0;
> }
>
> +static int iscsi_noop_in_tx_start(struct iscsi_task *task)
> +{
> + struct iscsi_connection *conn = task->conn;
> + struct iscsi_data_rsp *rsp = (struct iscsi_data_rsp *) &conn->rsp.bhs;
> +
> + memset(rsp, 0, sizeof(*rsp));
> + rsp->opcode = ISCSI_OP_NOOP_IN;
> + rsp->flags = ISCSI_FLAG_CMD_FINAL;
> + rsp->itt = task->req.itt;
> + rsp->ttt = task->req.ttt;
> + rsp->statsn = cpu_to_be32(conn->stat_sn);
> + rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
> + rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
> +
> + /* TODO: honor max_burst */
> + conn->rsp.datasize = task->len;
> + hton24(rsp->dlength, task->len);
> + conn->rsp.data = task->data;
> +
> + return 0;
> +}
> +
> static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
> {
> struct iscsi_connection *conn = task->conn;
> @@ -1843,6 +1865,10 @@ static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
>
> if (task->req.itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
> *is_rsp = 0;
> +
> + if (conn->tp->ep_nop_reply)
> + conn->tp->ep_nop_reply(be32_to_cpu(task->req.ttt));
> +
> iscsi_free_task(task);
> } else {
> *is_rsp = 1;
> @@ -1955,6 +1981,9 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
> case ISCSI_OP_SCSI_CMD:
> err = iscsi_scsi_cmd_tx_start(task);
> break;
> + case ISCSI_OP_NOOP_IN:
> + err = iscsi_noop_in_tx_start(task);
> + break;
> case ISCSI_OP_NOOP_OUT:
> err = iscsi_noop_out_tx_start(task, &is_rsp);
> if (!is_rsp)
> @@ -2450,6 +2479,10 @@ int iscsi_param_parse_portals(char *p, int do_add,
> return -1;
> }
> }
> + } else if (!strncmp(p, "nop_interval", 12)) {
> + iscsi_set_nop_interval(atoi(p+13));
> + } else if (!strncmp(p, "nop_count", 9)) {
> + iscsi_set_nop_count(atoi(p+10));
> }
>
> p += strcspn(p, ",");
> diff --git a/usr/iscsi/iscsid.h b/usr/iscsi/iscsid.h
> index e7e08ae..0710ccb 100644
> --- a/usr/iscsi/iscsid.h
> +++ b/usr/iscsi/iscsid.h
> @@ -239,6 +239,9 @@ struct iscsi_connection {
>
> #define INCOMING_BUFSIZE 8192
>
> +extern int default_nop_interval;
> +extern int default_nop_count;
> +
> struct iscsi_target {
> struct list_head tlist;
>
> @@ -262,6 +265,8 @@ struct iscsi_target {
> struct list_head isns_list;
>
> int rdma;
> + int nop_interval;
> + int nop_count;
> };
>
> enum task_flags {
> @@ -315,6 +320,11 @@ extern void iscsi_rx_handler(struct iscsi_connection *conn);
> extern int iscsi_scsi_cmd_execute(struct iscsi_task *task);
> extern int iscsi_transportid(int tid, uint64_t itn_id, char *buf, int size);
> extern int iscsi_add_portal(char *addr, int port, int tpgt);
> +extern void iscsi_print_nop_settings(struct concat_buf *b, int tid);
> +extern int iscsi_update_target_nop_count(int tid, int count);
> +extern int iscsi_update_target_nop_interval(int tid, int interval);
> +extern void iscsi_set_nop_interval(int interval);
> +extern void iscsi_set_nop_count(int count);
> extern int iscsi_delete_portal(char *addr, int port);
> extern int iscsi_param_parse_portals(char *p, int do_add, int do_delete);
> extern void iscsi_update_conn_stats_rx(struct iscsi_connection *conn, int size, int opcode);
> diff --git a/usr/iscsi/target.c b/usr/iscsi/target.c
> index 916c84a..20fb1da 100644
> --- a/usr/iscsi/target.c
> +++ b/usr/iscsi/target.c
> @@ -470,6 +470,8 @@ int iscsi_target_create(struct target *t)
> INIT_LIST_HEAD(&target->sessions_list);
> INIT_LIST_HEAD(&target->isns_list);
> target->tid = tid;
> + target->nop_interval = default_nop_interval;
> + target->nop_count = default_nop_count;
> list_add_tail(&target->tlist, &iscsi_targets_list);
>
> isns_target_register(tgt_targetname(tid));
> @@ -545,6 +547,18 @@ tgtadm_err iscsi_target_update(int mode, int op, int tid, uint64_t sid, uint64_t
> break;
> }
> adm_err = TGTADM_SUCCESS;
> + } else if (!strncmp(name, "nop_count", 9)) {
> + err = iscsi_update_target_nop_count(tid,
> + atoi(&name[10]));
> + adm_err = !err ? TGTADM_SUCCESS :
> + TGTADM_INVALID_REQUEST;
> + break;
> + } else if (!strncmp(name, "nop_interval", 12)) {
> + err = iscsi_update_target_nop_interval(tid,
> + atoi(&name[13]));
> + adm_err = !err ? TGTADM_SUCCESS :
> + TGTADM_INVALID_REQUEST;
> + break;
> }
>
> idx = param_index_by_name(name, session_keys);
> diff --git a/usr/iscsi/transport.h b/usr/iscsi/transport.h
> index af4af21..f4300c1 100644
> --- a/usr/iscsi/transport.h
> +++ b/usr/iscsi/transport.h
> @@ -39,6 +39,7 @@ struct iscsi_transport {
> struct sockaddr *sa, socklen_t *len);
> int (*ep_getpeername)(struct iscsi_connection *conn,
> struct sockaddr *sa, socklen_t *len);
> + void (*ep_nop_reply) (long ttt);
> };
>
> extern int iscsi_transport_register(struct iscsi_transport *);
> diff --git a/usr/target.c b/usr/target.c
> index 9a8f2fd..23b5766 100644
> --- a/usr/target.c
> +++ b/usr/target.c
> @@ -36,6 +36,7 @@
> #include "driver.h"
> #include "target.h"
> #include "scsi.h"
> +#include "iscsi/iscsid.h"
> #include "tgtadm.h"
> #include "parser.h"
> #include "spc.h"
> @@ -2000,6 +2001,9 @@ tgtadm_err tgt_target_show_all(struct concat_buf *b)
> tgt_drivers[target->lid]->name,
> target_state_name(target->target_state));
>
> + if (!strcmp(tgt_drivers[target->lid]->name, "iscsi"))
> + iscsi_print_nop_settings(b, target->tid);
> +
> concat_printf(b, _TAB1 "I_T nexus information:\n");
>
> list_for_each_entry(nexus, &target->it_nexus_list,
> diff --git a/usr/tgtadm.c b/usr/tgtadm.c
> index 90803f2..ac7a386 100644
> --- a/usr/tgtadm.c
> +++ b/usr/tgtadm.c
> @@ -707,12 +707,12 @@ int main(int argc, char **argv)
> case OP_UPDATE:
> rc = verify_mode_params(argc, argv, "LmotnvC");
> if (rc) {
> - eprintf("target mode: option '-%c' is not "
> + eprintf("target mode: option '-%c' is not " \
> "allowed/supported\n", rc);
> exit(EINVAL);
> }
> if ((!name || !value)) {
> - eprintf("update operation requires 'name'"
> + eprintf("update operation requires 'name'" \
> " and 'value' options\n");
> exit(EINVAL);
> }
> --
> 1.7.3.1
>
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
More information about the stgt
mailing list