[stgt] [PATCH] Add support for sending iSCSI NOP-IN
ronnie sahlberg
ronniesahlberg at gmail.com
Tue Jun 11 19:13:59 CEST 2013
ping ?
On Sat, Jun 8, 2013 at 8:13 AM, Ronnie Sahlberg
<ronniesahlberg at gmail.com> wrote:
> Add target initiated NOP-IN so we can detect when an initiator is no longer available
> and tear down the connection.
> Add documentation to the manpage on how to configure the use of NOP-IN, the intervals
> between the NOP-INs and after how many consequitive failures we will tear down the connection.
>
> Disabled by default to preserve the current behaviour but can be enabled/configured on the TGTD
> command line.
>
> Signed-off-by: Ronnie Sahlberg <ronniesahlberg at gmail.com>
> ---
> doc/tgtd.8.xml | 29 +++++++++++++++
> usr/iscsi/iscsi_tcp.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++++-
> usr/iscsi/iscsid.c | 46 +++++++++++++++++++----
> usr/iscsi/iscsid.h | 2 +
> usr/iscsi/transport.h | 1 +
> 5 files changed, 164 insertions(+), 9 deletions(-)
>
> diff --git a/doc/tgtd.8.xml b/doc/tgtd.8.xml
> index ce734ce..764e3fb 100644
> --- a/doc/tgtd.8.xml
> +++ b/doc/tgtd.8.xml
> @@ -129,6 +129,35 @@
> </screen>
> </para>
> </refsect2>
> + <refsect2><title>noop_interval=<integer></title>
> + <para>
> + This option enables TGTD to send send NOP-OUT to check is
> + a connected initiator is still alive.
> + This parameter specifies the interval between each NOP-OUT that
> + TGTD will send in seconds.
> + </para>
> + <para>
> + This feature is disabled by default.
> + </para>
> + </refsect2>
> + <refsect2><title>noop_count=<integer></title>
> + <para>
> + When sending NOP-OUT to the initiator, after this many consecutive
> + failures TGTD will terminate the connection to the client.
> + This feature can be used to read dead connections faster than waiting
> + for any TCP-Keepalive to trigger.
> + </para>
> + <para>
> + This feature is disabled by default.
> + </para>
> + <para>
> + Example: send NOP-OUT every 5 seconds and abort the session after
> + 6 failures.
> + <screen format="linespecific">
> + tgtd --iscsi portal=192.0.2.1:3260,noop_interval=5,noop_count=6
> + </screen>
> + </para>
> + </refsect2>
> </refsect1>
>
>
> diff --git a/usr/iscsi/iscsi_tcp.c b/usr/iscsi/iscsi_tcp.c
> index 0c43039..d886d42 100644
> --- a/usr/iscsi/iscsi_tcp.c
> +++ b/usr/iscsi/iscsi_tcp.c
> @@ -35,15 +35,26 @@
> #include "iscsid.h"
> #include "tgtd.h"
> #include "util.h"
> +#include "work.h"
>
> static void iscsi_tcp_event_handler(int fd, int events, void *data);
> +static struct iscsi_task *iscsi_tcp_alloc_task(struct iscsi_connection *conn,
> + size_t ext_len);
> +static void iscsi_tcp_free_task(struct iscsi_task *task);
> +static void iscsi_tcp_release(struct iscsi_connection *conn);
>
> static int listen_fds[8];
> static struct iscsi_transport iscsi_tcp;
> +static long nop_ttt;
> +static int noop_interval;
> +static int noop_count;
>
> struct iscsi_tcp_connection {
> - int fd;
> + struct list_head tcp_conn_siblings;
> + int nop_inflight_count;
>
> + int fd;
> + long ttt;
> struct iscsi_connection iscsi_conn;
> };
>
> @@ -52,6 +63,65 @@ static inline struct iscsi_tcp_connection *TCP_CONN(struct iscsi_connection *con
> return container_of(conn, struct iscsi_tcp_connection, iscsi_conn);
> }
>
> +/* all iscsi connections */
> +static struct list_head iscsi_tcp_conn_list;
> +
> +static struct tgt_work nop_work;
> +
> +static int iscsi_send_ping_nop_in(struct iscsi_tcp_connection *tcp_conn)
> +{
> + struct iscsi_connection *conn = &tcp_conn->iscsi_conn;
> + struct iscsi_task *task = NULL;
> +
> + task = iscsi_tcp_alloc_task(&tcp_conn->iscsi_conn, 0);
> + task->conn = conn;
> +
> + task->tag = ISCSI_RESERVED_TAG;
> + task->req.opcode = ISCSI_OP_NOOP_IN;
> + task->req.itt = cpu_to_be32(ISCSI_RESERVED_TAG);
> + task->req.ttt = cpu_to_be32(tcp_conn->ttt);
> +
> + list_add_tail(&task->c_list, &task->conn->tx_clist);
> + task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
> +
> + return 0;
> +}
> +
> +static void iscsi_tcp_nop_work_handler(void *data)
> +{
> + struct iscsi_tcp_connection *tcp_conn;
> +
> + list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
> + tcp_conn->nop_inflight_count++;
> + if (noop_count && tcp_conn->nop_inflight_count > noop_count) {
> + eprintf("tcp connection timed out after %d failed " \
> + "NOP-OUT\n", noop_count);
> + iscsi_tcp_release(&tcp_conn->iscsi_conn);
> + /* cant/shouldnt delete tcp_conn from within the loop */
> + break;
> + }
> + nop_ttt++;
> + if (nop_ttt == ISCSI_RESERVED_TAG)
> + nop_ttt = 1;
> +
> + tcp_conn->ttt = nop_ttt;
> + iscsi_send_ping_nop_in(tcp_conn);
> + }
> +
> + add_work(&nop_work, noop_interval);
> +}
> +
> +static void iscsi_tcp_noop_reply(long ttt)
> +{
> + struct iscsi_tcp_connection *tcp_conn;
> +
> + list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
> + if (tcp_conn->ttt != ttt)
> + continue;
> + tcp_conn->nop_inflight_count = 0;
> + }
> +}
> +
> static int set_keepalive(int fd)
> {
> int ret, opt;
> @@ -144,6 +214,8 @@ static void accept_connection(int afd, int events, void *data)
> goto out;
> }
>
> + list_add(&tcp_conn->tcp_conn_siblings, &iscsi_tcp_conn_list);
> +
> return;
> out:
> close(fd);
> @@ -279,6 +351,16 @@ int iscsi_add_portal(char *addr, int port, int tpgt)
> return 0;
> };
>
> +void iscsi_set_noop_interval(int interval)
> +{
> + noop_interval = interval;
> +}
> +
> +void iscsi_set_noop_count(int count)
> +{
> + noop_count = count;
> +}
> +
> int iscsi_delete_portal(char *addr, int port)
> {
> struct iscsi_portal *portal;
> @@ -313,6 +395,15 @@ static int iscsi_tcp_init(void)
> iscsi_add_portal(NULL, 3260, 1);
> }
>
> + if (noop_interval) {
> + nop_work.func = iscsi_tcp_nop_work_handler;
> + nop_work.data = &nop_work;
> +
> + add_work(&nop_work, noop_interval);
> + }
> +
> + INIT_LIST_HEAD(&iscsi_tcp_conn_list);
> +
> return 0;
> }
>
> @@ -370,6 +461,7 @@ static void iscsi_tcp_release(struct iscsi_connection *conn)
>
> conn_exit(conn);
> close(tcp_conn->fd);
> + list_del(&tcp_conn->tcp_conn_siblings);
> free(tcp_conn);
> }
>
> @@ -480,6 +572,7 @@ static struct iscsi_transport iscsi_tcp = {
> .free_data_buf = iscsi_tcp_free_data_buf,
> .ep_getsockname = iscsi_tcp_getsockname,
> .ep_getpeername = iscsi_tcp_getpeername,
> + .ep_noop_reply = iscsi_tcp_noop_reply,
> };
>
> __attribute__((constructor)) static void iscsi_transport_init(void)
> diff --git a/usr/iscsi/iscsid.c b/usr/iscsi/iscsid.c
> index 260989f..656d28d 100644
> --- a/usr/iscsi/iscsid.c
> +++ b/usr/iscsi/iscsid.c
> @@ -1674,13 +1674,9 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
>
> dprintf("%x %x %u\n", req->ttt, req->itt, ntoh24(req->dlength));
> if (req->ttt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
> - /*
> - * We don't request a NOP-Out by sending a NOP-In.
> - * See 10.18.2 in the draft 20.
> - */
> - eprintf("initiator bug\n");
> - err = -ISCSI_REASON_PROTOCOL_ERROR;
> - goto out;
> + if ((req->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_NOOP_OUT) {
> + goto good;
> + }
> }
>
> if (req->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
> @@ -1691,6 +1687,7 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
> }
> }
>
> +good:
> conn->exp_stat_sn = be32_to_cpu(req->exp_statsn);
>
> len = ntoh24(req->dlength);
> @@ -1719,8 +1716,8 @@ static int iscsi_task_rx_done(struct iscsi_connection *conn)
>
> op = hdr->opcode & ISCSI_OPCODE_MASK;
> switch (op) {
> - case ISCSI_OP_SCSI_CMD:
> case ISCSI_OP_NOOP_OUT:
> + case ISCSI_OP_SCSI_CMD:
> case ISCSI_OP_SCSI_TMFUNC:
> case ISCSI_OP_LOGOUT:
> err = iscsi_task_queue(task);
> @@ -1836,6 +1833,28 @@ static int iscsi_logout_tx_start(struct iscsi_task *task)
> return 0;
> }
>
> +static int iscsi_noop_in_tx_start(struct iscsi_task *task)
> +{
> + struct iscsi_connection *conn = task->conn;
> + struct iscsi_data_rsp *rsp = (struct iscsi_data_rsp *) &conn->rsp.bhs;
> +
> + memset(rsp, 0, sizeof(*rsp));
> + rsp->opcode = ISCSI_OP_NOOP_IN;
> + rsp->flags = ISCSI_FLAG_CMD_FINAL;
> + rsp->itt = task->req.itt;
> + rsp->ttt = task->req.ttt;
> + rsp->statsn = cpu_to_be32(conn->stat_sn);
> + rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
> + rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
> +
> + /* TODO: honor max_burst */
> + conn->rsp.datasize = task->len;
> + hton24(rsp->dlength, task->len);
> + conn->rsp.data = task->data;
> +
> + return 0;
> +}
> +
> static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
> {
> struct iscsi_connection *conn = task->conn;
> @@ -1843,6 +1862,10 @@ static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
>
> if (task->req.itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
> *is_rsp = 0;
> +
> + if (conn->tp->ep_noop_reply)
> + conn->tp->ep_noop_reply(be32_to_cpu(task->req.ttt));
> +
> iscsi_free_task(task);
> } else {
> *is_rsp = 1;
> @@ -1955,6 +1978,9 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
> case ISCSI_OP_SCSI_CMD:
> err = iscsi_scsi_cmd_tx_start(task);
> break;
> + case ISCSI_OP_NOOP_IN:
> + err = iscsi_noop_in_tx_start(task);
> + break;
> case ISCSI_OP_NOOP_OUT:
> err = iscsi_noop_out_tx_start(task, &is_rsp);
> if (!is_rsp)
> @@ -2450,6 +2476,10 @@ int iscsi_param_parse_portals(char *p, int do_add,
> return -1;
> }
> }
> + } else if (!strncmp(p, "noop_interval", 13)) {
> + iscsi_set_noop_interval(atoi(p+14));
> + } else if (!strncmp(p, "noop_count", 10)) {
> + iscsi_set_noop_count(atoi(p+11));
> }
>
> p += strcspn(p, ",");
> diff --git a/usr/iscsi/iscsid.h b/usr/iscsi/iscsid.h
> index e7e08ae..233a778 100644
> --- a/usr/iscsi/iscsid.h
> +++ b/usr/iscsi/iscsid.h
> @@ -316,6 +316,8 @@ extern int iscsi_scsi_cmd_execute(struct iscsi_task *task);
> extern int iscsi_transportid(int tid, uint64_t itn_id, char *buf, int size);
> extern int iscsi_add_portal(char *addr, int port, int tpgt);
> extern int iscsi_delete_portal(char *addr, int port);
> +extern void iscsi_set_noop_interval(int interval);
> +extern void iscsi_set_noop_count(int count);
> extern int iscsi_param_parse_portals(char *p, int do_add, int do_delete);
> extern void iscsi_update_conn_stats_rx(struct iscsi_connection *conn, int size, int opcode);
> extern void iscsi_update_conn_stats_tx(struct iscsi_connection *conn, int size, int opcode);
> diff --git a/usr/iscsi/transport.h b/usr/iscsi/transport.h
> index af4af21..42b17fe 100644
> --- a/usr/iscsi/transport.h
> +++ b/usr/iscsi/transport.h
> @@ -39,6 +39,7 @@ struct iscsi_transport {
> struct sockaddr *sa, socklen_t *len);
> int (*ep_getpeername)(struct iscsi_connection *conn,
> struct sockaddr *sa, socklen_t *len);
> + void (*ep_noop_reply) (long ttt);
> };
>
> extern int iscsi_transport_register(struct iscsi_transport *);
> --
> 1.7.3.1
>
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
More information about the stgt
mailing list