[stgt] [PATCH] Add support for sending iSCSI NOP-IN

ronnie sahlberg ronniesahlberg at gmail.com
Tue Jun 11 19:13:59 CEST 2013


ping ?

On Sat, Jun 8, 2013 at 8:13 AM, Ronnie Sahlberg
<ronniesahlberg at gmail.com> wrote:
> Add target initiated NOP-IN so we can detect when an initiator is no longer available
> and tear down the connection.
> Add documentation to the manpage on how to configure the use of NOP-IN, the intervals
> between the NOP-INs and after how many consequitive failures we will tear down the connection.
>
> Disabled by default to preserve the current behaviour but can be enabled/configured on the TGTD
> command line.
>
> Signed-off-by: Ronnie Sahlberg <ronniesahlberg at gmail.com>
> ---
>  doc/tgtd.8.xml        |   29 +++++++++++++++
>  usr/iscsi/iscsi_tcp.c |   95 ++++++++++++++++++++++++++++++++++++++++++++++++-
>  usr/iscsi/iscsid.c    |   46 +++++++++++++++++++----
>  usr/iscsi/iscsid.h    |    2 +
>  usr/iscsi/transport.h |    1 +
>  5 files changed, 164 insertions(+), 9 deletions(-)
>
> diff --git a/doc/tgtd.8.xml b/doc/tgtd.8.xml
> index ce734ce..764e3fb 100644
> --- a/doc/tgtd.8.xml
> +++ b/doc/tgtd.8.xml
> @@ -129,6 +129,35 @@
>        </screen>
>        </para>
>      </refsect2>
> +    <refsect2><title>noop_interval=<integer></title>
> +      <para>
> +       This option enables TGTD to send send NOP-OUT to check is
> +       a connected initiator is still alive.
> +       This parameter specifies the interval between each NOP-OUT that
> +       TGTD will send in seconds.
> +      </para>
> +      <para>
> +       This feature is disabled by default.
> +      </para>
> +    </refsect2>
> +    <refsect2><title>noop_count=<integer></title>
> +      <para>
> +       When sending NOP-OUT to the initiator, after this many consecutive
> +       failures TGTD will terminate the connection to the client.
> +       This feature can be used to read dead connections faster than waiting
> +       for any TCP-Keepalive to trigger.
> +      </para>
> +      <para>
> +       This feature is disabled by default.
> +      </para>
> +      <para>
> +      Example: send NOP-OUT every 5 seconds and abort the session after
> +      6 failures.
> +      <screen format="linespecific">
> +       tgtd --iscsi portal=192.0.2.1:3260,noop_interval=5,noop_count=6
> +      </screen>
> +      </para>
> +    </refsect2>
>    </refsect1>
>
>
> diff --git a/usr/iscsi/iscsi_tcp.c b/usr/iscsi/iscsi_tcp.c
> index 0c43039..d886d42 100644
> --- a/usr/iscsi/iscsi_tcp.c
> +++ b/usr/iscsi/iscsi_tcp.c
> @@ -35,15 +35,26 @@
>  #include "iscsid.h"
>  #include "tgtd.h"
>  #include "util.h"
> +#include "work.h"
>
>  static void iscsi_tcp_event_handler(int fd, int events, void *data);
> +static struct iscsi_task *iscsi_tcp_alloc_task(struct iscsi_connection *conn,
> +                                               size_t ext_len);
> +static void iscsi_tcp_free_task(struct iscsi_task *task);
> +static void iscsi_tcp_release(struct iscsi_connection *conn);
>
>  static int listen_fds[8];
>  static struct iscsi_transport iscsi_tcp;
> +static long nop_ttt;
> +static int noop_interval;
> +static int noop_count;
>
>  struct iscsi_tcp_connection {
> -       int fd;
> +       struct list_head tcp_conn_siblings;
> +       int nop_inflight_count;
>
> +       int fd;
> +       long ttt;
>         struct iscsi_connection iscsi_conn;
>  };
>
> @@ -52,6 +63,65 @@ static inline struct iscsi_tcp_connection *TCP_CONN(struct iscsi_connection *con
>         return container_of(conn, struct iscsi_tcp_connection, iscsi_conn);
>  }
>
> +/* all iscsi connections */
> +static struct list_head iscsi_tcp_conn_list;
> +
> +static struct tgt_work nop_work;
> +
> +static int iscsi_send_ping_nop_in(struct iscsi_tcp_connection *tcp_conn)
> +{
> +       struct iscsi_connection *conn = &tcp_conn->iscsi_conn;
> +       struct iscsi_task *task = NULL;
> +
> +       task = iscsi_tcp_alloc_task(&tcp_conn->iscsi_conn, 0);
> +       task->conn = conn;
> +
> +       task->tag = ISCSI_RESERVED_TAG;
> +       task->req.opcode = ISCSI_OP_NOOP_IN;
> +       task->req.itt = cpu_to_be32(ISCSI_RESERVED_TAG);
> +       task->req.ttt = cpu_to_be32(tcp_conn->ttt);
> +
> +       list_add_tail(&task->c_list, &task->conn->tx_clist);
> +       task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
> +
> +       return 0;
> +}
> +
> +static void iscsi_tcp_nop_work_handler(void *data)
> +{
> +       struct iscsi_tcp_connection *tcp_conn;
> +
> +       list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
> +               tcp_conn->nop_inflight_count++;
> +               if (noop_count && tcp_conn->nop_inflight_count > noop_count) {
> +                       eprintf("tcp connection timed out after %d failed " \
> +                               "NOP-OUT\n", noop_count);
> +                       iscsi_tcp_release(&tcp_conn->iscsi_conn);
> +                       /* cant/shouldnt delete tcp_conn from within the loop */
> +                       break;
> +               }
> +               nop_ttt++;
> +               if (nop_ttt == ISCSI_RESERVED_TAG)
> +                       nop_ttt = 1;
> +
> +               tcp_conn->ttt = nop_ttt;
> +               iscsi_send_ping_nop_in(tcp_conn);
> +       }
> +
> +       add_work(&nop_work, noop_interval);
> +}
> +
> +static void iscsi_tcp_noop_reply(long ttt)
> +{
> +       struct iscsi_tcp_connection *tcp_conn;
> +
> +       list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
> +               if (tcp_conn->ttt != ttt)
> +                       continue;
> +               tcp_conn->nop_inflight_count = 0;
> +       }
> +}
> +
>  static int set_keepalive(int fd)
>  {
>         int ret, opt;
> @@ -144,6 +214,8 @@ static void accept_connection(int afd, int events, void *data)
>                 goto out;
>         }
>
> +       list_add(&tcp_conn->tcp_conn_siblings, &iscsi_tcp_conn_list);
> +
>         return;
>  out:
>         close(fd);
> @@ -279,6 +351,16 @@ int iscsi_add_portal(char *addr, int port, int tpgt)
>         return 0;
>  };
>
> +void iscsi_set_noop_interval(int interval)
> +{
> +       noop_interval = interval;
> +}
> +
> +void iscsi_set_noop_count(int count)
> +{
> +       noop_count = count;
> +}
> +
>  int iscsi_delete_portal(char *addr, int port)
>  {
>         struct iscsi_portal *portal;
> @@ -313,6 +395,15 @@ static int iscsi_tcp_init(void)
>                 iscsi_add_portal(NULL, 3260, 1);
>         }
>
> +       if (noop_interval) {
> +               nop_work.func = iscsi_tcp_nop_work_handler;
> +               nop_work.data = &nop_work;
> +
> +               add_work(&nop_work, noop_interval);
> +       }
> +
> +       INIT_LIST_HEAD(&iscsi_tcp_conn_list);
> +
>         return 0;
>  }
>
> @@ -370,6 +461,7 @@ static void iscsi_tcp_release(struct iscsi_connection *conn)
>
>         conn_exit(conn);
>         close(tcp_conn->fd);
> +       list_del(&tcp_conn->tcp_conn_siblings);
>         free(tcp_conn);
>  }
>
> @@ -480,6 +572,7 @@ static struct iscsi_transport iscsi_tcp = {
>         .free_data_buf          = iscsi_tcp_free_data_buf,
>         .ep_getsockname         = iscsi_tcp_getsockname,
>         .ep_getpeername         = iscsi_tcp_getpeername,
> +       .ep_noop_reply          = iscsi_tcp_noop_reply,
>  };
>
>  __attribute__((constructor)) static void iscsi_transport_init(void)
> diff --git a/usr/iscsi/iscsid.c b/usr/iscsi/iscsid.c
> index 260989f..656d28d 100644
> --- a/usr/iscsi/iscsid.c
> +++ b/usr/iscsi/iscsid.c
> @@ -1674,13 +1674,9 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
>
>         dprintf("%x %x %u\n", req->ttt, req->itt, ntoh24(req->dlength));
>         if (req->ttt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
> -               /*
> -                * We don't request a NOP-Out by sending a NOP-In.
> -                * See 10.18.2 in the draft 20.
> -                */
> -               eprintf("initiator bug\n");
> -               err = -ISCSI_REASON_PROTOCOL_ERROR;
> -               goto out;
> +               if ((req->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_NOOP_OUT) {
> +                       goto good;
> +               }
>         }
>
>         if (req->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
> @@ -1691,6 +1687,7 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
>                 }
>         }
>
> +good:
>         conn->exp_stat_sn = be32_to_cpu(req->exp_statsn);
>
>         len = ntoh24(req->dlength);
> @@ -1719,8 +1716,8 @@ static int iscsi_task_rx_done(struct iscsi_connection *conn)
>
>         op = hdr->opcode & ISCSI_OPCODE_MASK;
>         switch (op) {
> -       case ISCSI_OP_SCSI_CMD:
>         case ISCSI_OP_NOOP_OUT:
> +       case ISCSI_OP_SCSI_CMD:
>         case ISCSI_OP_SCSI_TMFUNC:
>         case ISCSI_OP_LOGOUT:
>                 err = iscsi_task_queue(task);
> @@ -1836,6 +1833,28 @@ static int iscsi_logout_tx_start(struct iscsi_task *task)
>         return 0;
>  }
>
> +static int iscsi_noop_in_tx_start(struct iscsi_task *task)
> +{
> +       struct iscsi_connection *conn = task->conn;
> +       struct iscsi_data_rsp *rsp = (struct iscsi_data_rsp *) &conn->rsp.bhs;
> +
> +       memset(rsp, 0, sizeof(*rsp));
> +       rsp->opcode = ISCSI_OP_NOOP_IN;
> +       rsp->flags = ISCSI_FLAG_CMD_FINAL;
> +       rsp->itt = task->req.itt;
> +       rsp->ttt = task->req.ttt;
> +       rsp->statsn = cpu_to_be32(conn->stat_sn);
> +       rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
> +       rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
> +
> +       /* TODO: honor max_burst */
> +       conn->rsp.datasize = task->len;
> +       hton24(rsp->dlength, task->len);
> +       conn->rsp.data = task->data;
> +
> +       return 0;
> +}
> +
>  static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
>  {
>         struct iscsi_connection *conn = task->conn;
> @@ -1843,6 +1862,10 @@ static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
>
>         if (task->req.itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
>                 *is_rsp = 0;
> +
> +               if (conn->tp->ep_noop_reply)
> +                       conn->tp->ep_noop_reply(be32_to_cpu(task->req.ttt));
> +
>                 iscsi_free_task(task);
>         } else {
>                 *is_rsp = 1;
> @@ -1955,6 +1978,9 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
>         case ISCSI_OP_SCSI_CMD:
>                 err = iscsi_scsi_cmd_tx_start(task);
>                 break;
> +       case ISCSI_OP_NOOP_IN:
> +               err = iscsi_noop_in_tx_start(task);
> +               break;
>         case ISCSI_OP_NOOP_OUT:
>                 err = iscsi_noop_out_tx_start(task, &is_rsp);
>                 if (!is_rsp)
> @@ -2450,6 +2476,10 @@ int iscsi_param_parse_portals(char *p, int do_add,
>                                         return -1;
>                                 }
>                         }
> +               } else if (!strncmp(p, "noop_interval", 13)) {
> +                       iscsi_set_noop_interval(atoi(p+14));
> +               } else if (!strncmp(p, "noop_count", 10)) {
> +                       iscsi_set_noop_count(atoi(p+11));
>                 }
>
>                 p += strcspn(p, ",");
> diff --git a/usr/iscsi/iscsid.h b/usr/iscsi/iscsid.h
> index e7e08ae..233a778 100644
> --- a/usr/iscsi/iscsid.h
> +++ b/usr/iscsi/iscsid.h
> @@ -316,6 +316,8 @@ extern int iscsi_scsi_cmd_execute(struct iscsi_task *task);
>  extern int iscsi_transportid(int tid, uint64_t itn_id, char *buf, int size);
>  extern int iscsi_add_portal(char *addr, int port, int tpgt);
>  extern int iscsi_delete_portal(char *addr, int port);
> +extern void iscsi_set_noop_interval(int interval);
> +extern void iscsi_set_noop_count(int count);
>  extern int iscsi_param_parse_portals(char *p, int do_add, int do_delete);
>  extern void iscsi_update_conn_stats_rx(struct iscsi_connection *conn, int size, int opcode);
>  extern void iscsi_update_conn_stats_tx(struct iscsi_connection *conn, int size, int opcode);
> diff --git a/usr/iscsi/transport.h b/usr/iscsi/transport.h
> index af4af21..42b17fe 100644
> --- a/usr/iscsi/transport.h
> +++ b/usr/iscsi/transport.h
> @@ -39,6 +39,7 @@ struct iscsi_transport {
>                               struct sockaddr *sa, socklen_t *len);
>         int (*ep_getpeername)(struct iscsi_connection *conn,
>                               struct sockaddr *sa, socklen_t *len);
> +       void (*ep_noop_reply) (long ttt);
>  };
>
>  extern int iscsi_transport_register(struct iscsi_transport *);
> --
> 1.7.3.1
>
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html



More information about the stgt mailing list