[stgt] [PATCH] Add NOP-OUT probe support.
ronnie sahlberg
ronniesahlberg at gmail.com
Sun Jul 7 01:33:46 CEST 2013
ping?
On Thu, Jul 4, 2013 at 5:09 PM, ronnie sahlberg
<ronniesahlberg at gmail.com> wrote:
> ping
>
> On Sat, Jun 29, 2013 at 9:24 PM, Ronnie Sahlberg
> <ronniesahlberg at gmail.com> wrote:
>> Add support for setting NOP-OUT probes to detect dead initiators.
>> We can set the interval(how frequently to send the probes)
>> and the count(after how many failures will we tear down the connection)
>> globally from the tgtd command line.
>>
>> Optionally, using tgtadm, it is possible to configure the interval/count
>> for individual targets.
>>
>> Signed-off-by: Ronnie Sahlberg <ronniesahlberg at gmail.com>
>> ---
>> doc/tgtadm.8.xml | 42 ++++++++++++
>> doc/tgtd.8.xml | 30 +++++++++
>> usr/iscsi/iscsi_tcp.c | 167 +++++++++++++++++++++++++++++++++++++++++++++++++
>> usr/iscsi/iscsid.c | 47 ++++++++++++--
>> usr/iscsi/iscsid.h | 10 +++
>> usr/iscsi/target.c | 14 ++++
>> usr/iscsi/transport.h | 1 +
>> usr/target.c | 4 +
>> usr/tgtadm.c | 4 +-
>> 9 files changed, 310 insertions(+), 9 deletions(-)
>>
>> diff --git a/doc/tgtadm.8.xml b/doc/tgtadm.8.xml
>> index 18813dc..315ffdf 100644
>> --- a/doc/tgtadm.8.xml
>> +++ b/doc/tgtadm.8.xml
>> @@ -678,6 +678,48 @@ Account information:
>> </refsect1>
>>
>>
>> + <refsect1><title>NOP-OUT Probes</title>
>> + <para>
>> + TGTD can send NOP-OUT probes to connected initiators to determine when
>> + an initiator is dead and then automatically clear and tear down the
>> + TCP connection. This can either be set as a global default from the
>> + tgtd command-line or it can be set for individual targets using the
>> + tgtadm command.
>> + </para>
>> + <refsect2><title>Check the current NOP-OUT setting</title>
>> + <para>
>> + The tgtadm command is used to view the current setting for if/when
>> + to send NOP-OUT probes to connected initiators.
>> + </para>
>> + <para>
>> + If the target is configured to send NOP-OUT probes this will show up
>> + as two parameter lines in the target printout. If the target is not
>> + configured to send NOP-OUT these lines will not be printed at all.
>> + </para>
>> + <screen format="linespecific">
>> +tgtadm --lld iscsi --op show --mode target
>> +
>> +Target 1: iqn.ronnie.test
>> + System information:
>> + Driver: iscsi
>> + State: ready
>> + Nop interval: 5
>> + Nop count: 5
>> + I_T nexus information:
>> + </screen>
>> + </refsect2>
>> + <refsect2><title>Setting NOP-OUT for a target</title>
>> + <para>
>> + The tgtadm command is used to change the NOP-OUT settings.
>> + </para>
>> + <screen format="linespecific">
>> +tgtadm --op update --mode target --tid 1 -n nop_count -v 5
>> +tgtadm --op update --mode target --tid 1 -n nop_interval -v 5
>> + </screen>
>> + </refsect2>
>> + </refsect1>
>> +
>> +
>> <refsect1><title>iSCSI PORTALS</title>
>> <para>
>> iSCSI portals can be viewed, added and removed at runtime.
>> diff --git a/doc/tgtd.8.xml b/doc/tgtd.8.xml
>> index ce734ce..a0020c6 100644
>> --- a/doc/tgtd.8.xml
>> +++ b/doc/tgtd.8.xml
>> @@ -129,6 +129,36 @@
>> </screen>
>> </para>
>> </refsect2>
>> + <refsect2><title>nop_interval=<integer></title>
>> + <para>
>> + This sets the default interval for sending NOP-OUT to probe for
>> + connected initiators.
>> + This parameter only controls the default value for targets.
>> + Individual targets can be controlled using tgtadm.
>> + </para>
>> + <para>
>> + The default value is 0 which means that the feature is disabled
>> + TGTD will not send any NOP-OUT probes.
>> + </para>
>> + </refsect2>
>> + <refsect2><title>nop_count=<integer></title>
>> + <para>
>> + This sets the default value for after how many failed probes TGTD
>> + will consider the initiator dead and tear down the session.
>> + This parameter only controls the default value for targets.
>> + Individual targets can be controlled using tgtadm.
>> + </para>
>> + <para>
>> + The default value is 0.
>> + </para>
>> + <para>
>> + Example: send NOP-OUT every 5 seconds and abort the session after
>> + 6 failures.
>> + <screen format="linespecific">
>> + tgtd --iscsi portal=192.0.2.1:3260,nop_interval=5,nop_count=6
>> + </screen>
>> + </para>
>> + </refsect2>
>> </refsect1>
>>
>>
>> diff --git a/usr/iscsi/iscsi_tcp.c b/usr/iscsi/iscsi_tcp.c
>> index 0c43039..aa50780 100644
>> --- a/usr/iscsi/iscsi_tcp.c
>> +++ b/usr/iscsi/iscsi_tcp.c
>> @@ -35,8 +35,15 @@
>> #include "iscsid.h"
>> #include "tgtd.h"
>> #include "util.h"
>> +#include "work.h"
>>
>> static void iscsi_tcp_event_handler(int fd, int events, void *data);
>> +static void iscsi_tcp_release(struct iscsi_connection *conn);
>> +static struct iscsi_task *iscsi_tcp_alloc_task(struct iscsi_connection *conn,
>> + size_t ext_len);
>> +static void iscsi_tcp_free_task(struct iscsi_task *task);
>> +
>> +static long nop_ttt;
>>
>> static int listen_fds[8];
>> static struct iscsi_transport iscsi_tcp;
>> @@ -44,6 +51,13 @@ static struct iscsi_transport iscsi_tcp;
>> struct iscsi_tcp_connection {
>> int fd;
>>
>> + struct list_head tcp_conn_siblings;
>> + int nop_inflight_count;
>> + int nop_interval;
>> + int nop_tick;
>> + int nop_count;
>> + long ttt;
>> +
>> struct iscsi_connection iscsi_conn;
>> };
>>
>> @@ -52,6 +66,110 @@ static inline struct iscsi_tcp_connection *TCP_CONN(struct iscsi_connection *con
>> return container_of(conn, struct iscsi_tcp_connection, iscsi_conn);
>> }
>>
>> +static struct tgt_work nop_work;
>> +
>> +/* all iscsi connections */
>> +static struct list_head iscsi_tcp_conn_list;
>> +
>> +static int iscsi_send_ping_nop_in(struct iscsi_tcp_connection *tcp_conn)
>> +{
>> + struct iscsi_connection *conn = &tcp_conn->iscsi_conn;
>> + struct iscsi_task *task = NULL;
>> +
>> + task = iscsi_tcp_alloc_task(&tcp_conn->iscsi_conn, 0);
>> + task->conn = conn;
>> +
>> + task->tag = ISCSI_RESERVED_TAG;
>> + task->req.opcode = ISCSI_OP_NOOP_IN;
>> + task->req.itt = cpu_to_be32(ISCSI_RESERVED_TAG);
>> + task->req.ttt = cpu_to_be32(tcp_conn->ttt);
>> +
>> + list_add_tail(&task->c_list, &task->conn->tx_clist);
>> + task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
>> +
>> + return 0;
>> +}
>> +
>> +static void iscsi_tcp_nop_work_handler(void *data)
>> +{
>> + struct iscsi_tcp_connection *tcp_conn;
>> +
>> + list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
>> + if (tcp_conn->nop_interval == 0)
>> + continue;
>> +
>> + tcp_conn->nop_tick--;
>> + if (tcp_conn->nop_tick > 0)
>> + continue;
>> +
>> + tcp_conn->nop_tick = tcp_conn->nop_interval;
>> +
>> + tcp_conn->nop_inflight_count++;
>> + if (tcp_conn->nop_inflight_count > tcp_conn->nop_count) {
>> + eprintf("tcp connection timed out after %d failed " \
>> + "NOP-OUT\n", tcp_conn->nop_count);
>> + iscsi_tcp_release(&tcp_conn->iscsi_conn);
>> + /* cant/shouldnt delete tcp_conn from within the loop */
>> + break;
>> + }
>> + nop_ttt++;
>> + if (nop_ttt == ISCSI_RESERVED_TAG)
>> + nop_ttt = 1;
>> +
>> + tcp_conn->ttt = nop_ttt;
>> + iscsi_send_ping_nop_in(tcp_conn);
>> + }
>> +
>> + add_work(&nop_work, 1);
>> +}
>> +
>> +static void iscsi_tcp_nop_reply(long ttt)
>> +{
>> + struct iscsi_tcp_connection *tcp_conn;
>> +
>> + list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings) {
>> + if (tcp_conn->ttt != ttt)
>> + continue;
>> + tcp_conn->nop_inflight_count = 0;
>> + }
>> +}
>> +
>> +int iscsi_update_target_nop_count(int tid, int count)
>> +{
>> + struct iscsi_target *target;
>> +
>> + list_for_each_entry(target, &iscsi_targets_list, tlist) {
>> + if (target->tid != tid)
>> + continue;
>> + target->nop_count = count;
>> + return 0;
>> + }
>> + return -1;
>> +}
>> +
>> +int iscsi_update_target_nop_interval(int tid, int interval)
>> +{
>> + struct iscsi_target *target;
>> +
>> + list_for_each_entry(target, &iscsi_targets_list, tlist) {
>> + if (target->tid != tid)
>> + continue;
>> + target->nop_interval = interval;
>> + return 0;
>> + }
>> + return -1;
>> +}
>> +
>> +void iscsi_set_nop_interval(int interval)
>> +{
>> + default_nop_interval = interval;
>> +}
>> +
>> +void iscsi_set_nop_count(int count)
>> +{
>> + default_nop_count = count;
>> +}
>> +
>> static int set_keepalive(int fd)
>> {
>> int ret, opt;
>> @@ -144,6 +262,8 @@ static void accept_connection(int afd, int events, void *data)
>> goto out;
>> }
>>
>> + list_add(&tcp_conn->tcp_conn_siblings, &iscsi_tcp_conn_list);
>> +
>> return;
>> out:
>> close(fd);
>> @@ -313,6 +433,12 @@ static int iscsi_tcp_init(void)
>> iscsi_add_portal(NULL, 3260, 1);
>> }
>>
>> + INIT_LIST_HEAD(&iscsi_tcp_conn_list);
>> +
>> + nop_work.func = iscsi_tcp_nop_work_handler;
>> + nop_work.data = &nop_work;
>> + add_work(&nop_work, 1);
>> +
>> return 0;
>> }
>>
>> @@ -328,6 +454,26 @@ static void iscsi_tcp_exit(void)
>>
>> static int iscsi_tcp_conn_login_complete(struct iscsi_connection *conn)
>> {
>> + struct iscsi_tcp_connection *tcp_conn;
>> + struct iscsi_target *target;
>> +
>> + list_for_each_entry(tcp_conn, &iscsi_tcp_conn_list, tcp_conn_siblings)
>> + if (&tcp_conn->iscsi_conn == conn)
>> + break;
>> +
>> + if (tcp_conn == NULL)
>> + return 0;
>> +
>> + list_for_each_entry(target, &iscsi_targets_list, tlist) {
>> + if (target->tid != conn->tid)
>> + continue;
>> +
>> + tcp_conn->nop_count = target->nop_count;
>> + tcp_conn->nop_interval = target->nop_interval;
>> + tcp_conn->nop_tick = target->nop_interval;
>> + break;
>> + }
>> +
>> return 0;
>> }
>>
>> @@ -370,6 +516,7 @@ static void iscsi_tcp_release(struct iscsi_connection *conn)
>>
>> conn_exit(conn);
>> close(tcp_conn->fd);
>> + list_del(&tcp_conn->tcp_conn_siblings);
>> free(tcp_conn);
>> }
>>
>> @@ -459,6 +606,25 @@ static void iscsi_tcp_conn_force_close(struct iscsi_connection *conn)
>> conn->tp->ep_event_modify(conn, EPOLLIN|EPOLLOUT|EPOLLERR);
>> }
>>
>> +void iscsi_print_nop_settings(struct concat_buf *b, int tid)
>> +{
>> + struct iscsi_target *target;
>> +
>> + list_for_each_entry(target, &iscsi_targets_list, tlist) {
>> + if (target->tid != tid)
>> + continue;
>> + if (target->nop_interval == 0)
>> + continue;
>> +
>> + concat_printf(b,
>> + _TAB2 "Nop interval: %d\n"
>> + _TAB2 "Nop count: %d\n",
>> + target->nop_interval,
>> + target->nop_count);
>> + break;
>> + }
>> +}
>> +
>> static struct iscsi_transport iscsi_tcp = {
>> .name = "iscsi",
>> .rdma = 0,
>> @@ -480,6 +646,7 @@ static struct iscsi_transport iscsi_tcp = {
>> .free_data_buf = iscsi_tcp_free_data_buf,
>> .ep_getsockname = iscsi_tcp_getsockname,
>> .ep_getpeername = iscsi_tcp_getpeername,
>> + .ep_nop_reply = iscsi_tcp_nop_reply,
>> };
>>
>> __attribute__((constructor)) static void iscsi_transport_init(void)
>> diff --git a/usr/iscsi/iscsid.c b/usr/iscsi/iscsid.c
>> index 260989f..005bac5 100644
>> --- a/usr/iscsi/iscsid.c
>> +++ b/usr/iscsi/iscsid.c
>> @@ -43,6 +43,9 @@
>>
>> #define MAX_QUEUE_CMD 128
>>
>> +int default_nop_interval;
>> +int default_nop_count;
>> +
>> LIST_HEAD(iscsi_portals_list);
>>
>> char *portal_arguments;
>> @@ -1674,13 +1677,9 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
>>
>> dprintf("%x %x %u\n", req->ttt, req->itt, ntoh24(req->dlength));
>> if (req->ttt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
>> - /*
>> - * We don't request a NOP-Out by sending a NOP-In.
>> - * See 10.18.2 in the draft 20.
>> - */
>> - eprintf("initiator bug\n");
>> - err = -ISCSI_REASON_PROTOCOL_ERROR;
>> - goto out;
>> + if ((req->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_NOOP_OUT) {
>> + goto good;
>> + }
>> }
>>
>> if (req->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
>> @@ -1691,6 +1690,7 @@ static int iscsi_noop_out_rx_start(struct iscsi_connection *conn)
>> }
>> }
>>
>> +good:
>> conn->exp_stat_sn = be32_to_cpu(req->exp_statsn);
>>
>> len = ntoh24(req->dlength);
>> @@ -1836,6 +1836,28 @@ static int iscsi_logout_tx_start(struct iscsi_task *task)
>> return 0;
>> }
>>
>> +static int iscsi_noop_in_tx_start(struct iscsi_task *task)
>> +{
>> + struct iscsi_connection *conn = task->conn;
>> + struct iscsi_data_rsp *rsp = (struct iscsi_data_rsp *) &conn->rsp.bhs;
>> +
>> + memset(rsp, 0, sizeof(*rsp));
>> + rsp->opcode = ISCSI_OP_NOOP_IN;
>> + rsp->flags = ISCSI_FLAG_CMD_FINAL;
>> + rsp->itt = task->req.itt;
>> + rsp->ttt = task->req.ttt;
>> + rsp->statsn = cpu_to_be32(conn->stat_sn);
>> + rsp->exp_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn);
>> + rsp->max_cmdsn = cpu_to_be32(conn->session->exp_cmd_sn + MAX_QUEUE_CMD);
>> +
>> + /* TODO: honor max_burst */
>> + conn->rsp.datasize = task->len;
>> + hton24(rsp->dlength, task->len);
>> + conn->rsp.data = task->data;
>> +
>> + return 0;
>> +}
>> +
>> static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
>> {
>> struct iscsi_connection *conn = task->conn;
>> @@ -1843,6 +1865,10 @@ static int iscsi_noop_out_tx_start(struct iscsi_task *task, int *is_rsp)
>>
>> if (task->req.itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
>> *is_rsp = 0;
>> +
>> + if (conn->tp->ep_nop_reply)
>> + conn->tp->ep_nop_reply(be32_to_cpu(task->req.ttt));
>> +
>> iscsi_free_task(task);
>> } else {
>> *is_rsp = 1;
>> @@ -1955,6 +1981,9 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
>> case ISCSI_OP_SCSI_CMD:
>> err = iscsi_scsi_cmd_tx_start(task);
>> break;
>> + case ISCSI_OP_NOOP_IN:
>> + err = iscsi_noop_in_tx_start(task);
>> + break;
>> case ISCSI_OP_NOOP_OUT:
>> err = iscsi_noop_out_tx_start(task, &is_rsp);
>> if (!is_rsp)
>> @@ -2450,6 +2479,10 @@ int iscsi_param_parse_portals(char *p, int do_add,
>> return -1;
>> }
>> }
>> + } else if (!strncmp(p, "nop_interval", 12)) {
>> + iscsi_set_nop_interval(atoi(p+13));
>> + } else if (!strncmp(p, "nop_count", 9)) {
>> + iscsi_set_nop_count(atoi(p+10));
>> }
>>
>> p += strcspn(p, ",");
>> diff --git a/usr/iscsi/iscsid.h b/usr/iscsi/iscsid.h
>> index e7e08ae..0710ccb 100644
>> --- a/usr/iscsi/iscsid.h
>> +++ b/usr/iscsi/iscsid.h
>> @@ -239,6 +239,9 @@ struct iscsi_connection {
>>
>> #define INCOMING_BUFSIZE 8192
>>
>> +extern int default_nop_interval;
>> +extern int default_nop_count;
>> +
>> struct iscsi_target {
>> struct list_head tlist;
>>
>> @@ -262,6 +265,8 @@ struct iscsi_target {
>> struct list_head isns_list;
>>
>> int rdma;
>> + int nop_interval;
>> + int nop_count;
>> };
>>
>> enum task_flags {
>> @@ -315,6 +320,11 @@ extern void iscsi_rx_handler(struct iscsi_connection *conn);
>> extern int iscsi_scsi_cmd_execute(struct iscsi_task *task);
>> extern int iscsi_transportid(int tid, uint64_t itn_id, char *buf, int size);
>> extern int iscsi_add_portal(char *addr, int port, int tpgt);
>> +extern void iscsi_print_nop_settings(struct concat_buf *b, int tid);
>> +extern int iscsi_update_target_nop_count(int tid, int count);
>> +extern int iscsi_update_target_nop_interval(int tid, int interval);
>> +extern void iscsi_set_nop_interval(int interval);
>> +extern void iscsi_set_nop_count(int count);
>> extern int iscsi_delete_portal(char *addr, int port);
>> extern int iscsi_param_parse_portals(char *p, int do_add, int do_delete);
>> extern void iscsi_update_conn_stats_rx(struct iscsi_connection *conn, int size, int opcode);
>> diff --git a/usr/iscsi/target.c b/usr/iscsi/target.c
>> index 916c84a..20fb1da 100644
>> --- a/usr/iscsi/target.c
>> +++ b/usr/iscsi/target.c
>> @@ -470,6 +470,8 @@ int iscsi_target_create(struct target *t)
>> INIT_LIST_HEAD(&target->sessions_list);
>> INIT_LIST_HEAD(&target->isns_list);
>> target->tid = tid;
>> + target->nop_interval = default_nop_interval;
>> + target->nop_count = default_nop_count;
>> list_add_tail(&target->tlist, &iscsi_targets_list);
>>
>> isns_target_register(tgt_targetname(tid));
>> @@ -545,6 +547,18 @@ tgtadm_err iscsi_target_update(int mode, int op, int tid, uint64_t sid, uint64_t
>> break;
>> }
>> adm_err = TGTADM_SUCCESS;
>> + } else if (!strncmp(name, "nop_count", 9)) {
>> + err = iscsi_update_target_nop_count(tid,
>> + atoi(&name[10]));
>> + adm_err = !err ? TGTADM_SUCCESS :
>> + TGTADM_INVALID_REQUEST;
>> + break;
>> + } else if (!strncmp(name, "nop_interval", 12)) {
>> + err = iscsi_update_target_nop_interval(tid,
>> + atoi(&name[13]));
>> + adm_err = !err ? TGTADM_SUCCESS :
>> + TGTADM_INVALID_REQUEST;
>> + break;
>> }
>>
>> idx = param_index_by_name(name, session_keys);
>> diff --git a/usr/iscsi/transport.h b/usr/iscsi/transport.h
>> index af4af21..f4300c1 100644
>> --- a/usr/iscsi/transport.h
>> +++ b/usr/iscsi/transport.h
>> @@ -39,6 +39,7 @@ struct iscsi_transport {
>> struct sockaddr *sa, socklen_t *len);
>> int (*ep_getpeername)(struct iscsi_connection *conn,
>> struct sockaddr *sa, socklen_t *len);
>> + void (*ep_nop_reply) (long ttt);
>> };
>>
>> extern int iscsi_transport_register(struct iscsi_transport *);
>> diff --git a/usr/target.c b/usr/target.c
>> index 9a8f2fd..23b5766 100644
>> --- a/usr/target.c
>> +++ b/usr/target.c
>> @@ -36,6 +36,7 @@
>> #include "driver.h"
>> #include "target.h"
>> #include "scsi.h"
>> +#include "iscsi/iscsid.h"
>> #include "tgtadm.h"
>> #include "parser.h"
>> #include "spc.h"
>> @@ -2000,6 +2001,9 @@ tgtadm_err tgt_target_show_all(struct concat_buf *b)
>> tgt_drivers[target->lid]->name,
>> target_state_name(target->target_state));
>>
>> + if (!strcmp(tgt_drivers[target->lid]->name, "iscsi"))
>> + iscsi_print_nop_settings(b, target->tid);
>> +
>> concat_printf(b, _TAB1 "I_T nexus information:\n");
>>
>> list_for_each_entry(nexus, &target->it_nexus_list,
>> diff --git a/usr/tgtadm.c b/usr/tgtadm.c
>> index 90803f2..ac7a386 100644
>> --- a/usr/tgtadm.c
>> +++ b/usr/tgtadm.c
>> @@ -707,12 +707,12 @@ int main(int argc, char **argv)
>> case OP_UPDATE:
>> rc = verify_mode_params(argc, argv, "LmotnvC");
>> if (rc) {
>> - eprintf("target mode: option '-%c' is not "
>> + eprintf("target mode: option '-%c' is not " \
>> "allowed/supported\n", rc);
>> exit(EINVAL);
>> }
>> if ((!name || !value)) {
>> - eprintf("update operation requires 'name'"
>> + eprintf("update operation requires 'name'" \
>> " and 'value' options\n");
>> exit(EINVAL);
>> }
>> --
>> 1.7.3.1
>>
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
More information about the stgt
mailing list