[stgt] Worker Threads
Andy Grover
agrover at redhat.com
Wed Sep 21 23:29:28 CEST 2011
On 07/27/2011 07:03 PM, FUJITA Tomonori wrote:
> On Wed, 27 Jul 2011 13:03:52 -0700
> "Bennett, Jeffrey" <jab at sdsc.edu> wrote:
>
>> Is there any "side effect" by doing this change?
>
> NR_WORKER_THREADS is the number of threads that perform I/Os. So you
> run more threads. You consume more memory.
>
>> Is there any plan
>> to make this a configuration option that can be changed with the
>> tgt-adm command by the user?
>
> tgtd should adjust NR_WORKER_THREADS dynamically since the optimal
> value depends on your workload, hardware, etc. Well, until we
> implement that feature, I think that we can support the boot option to
> specify the number of I/O threads.
>
> The following patch works for you? You can specify the number in the
> following way:
Hello Tomo-san,
I haven't tried it but this looks pretty handy to me! Will you be
applying it?
Thanks -- Regards -- Andy
>
> tgtd --nr_iothreads=128
>
>
> diff --git a/usr/bs.c b/usr/bs.c
> index d72d090..6f67d15 100644
> --- a/usr/bs.c
> +++ b/usr/bs.c
> @@ -24,6 +24,7 @@
> #include <inttypes.h>
> #include <pthread.h>
> #include <stdio.h>
> +#include <stdlib.h>
> #include <unistd.h>
> #include <fcntl.h>
> #include <signal.h>
> @@ -327,6 +328,11 @@ int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
> {
> int i, ret;
>
> + info->worker_thread = zalloc(sizeof(pthread_t) * nr_threads);
> + if (!info->worker_thread)
> + return TGTADM_NOMEM;
> +
> + eprintf("%d\n", nr_threads);
> info->request_fn = rfn;
>
> INIT_LIST_HEAD(&info->pending_list);
> @@ -335,11 +341,6 @@ int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
> pthread_mutex_init(&info->pending_lock, NULL);
> pthread_mutex_init(&info->startup_lock, NULL);
>
> - if (nr_threads > ARRAY_SIZE(info->worker_thread)) {
> - eprintf("too many threads %d\n", nr_threads);
> - nr_threads = ARRAY_SIZE(info->worker_thread);
> - }
> -
> pthread_mutex_lock(&info->startup_lock);
> for (i = 0; i < nr_threads; i++) {
> ret = pthread_create(&info->worker_thread[i], NULL,
> @@ -353,6 +354,7 @@ int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
> }
> }
> pthread_mutex_unlock(&info->startup_lock);
> + info->nr_worker_threads = nr_threads;
>
> return 0;
> destroy_threads:
> @@ -367,6 +369,7 @@ destroy_threads:
> pthread_cond_destroy(&info->pending_cond);
> pthread_mutex_destroy(&info->pending_lock);
> pthread_mutex_destroy(&info->startup_lock);
> + free(info->worker_thread);
>
> return TGTADM_NOMEM;
> }
> @@ -378,13 +381,13 @@ void bs_thread_close(struct bs_thread_info *info)
> info->stop = 1;
> pthread_cond_broadcast(&info->pending_cond);
>
> - for (i = 0; info->worker_thread[i] &&
> - i < ARRAY_SIZE(info->worker_thread); i++)
> + for (i = 0; info->worker_thread[i] && i < info->nr_worker_threads; i++)
> pthread_join(info->worker_thread[i], NULL);
>
> pthread_cond_destroy(&info->pending_cond);
> pthread_mutex_destroy(&info->pending_lock);
> pthread_mutex_destroy(&info->startup_lock);
> + free(info->worker_thread);
>
> info->stop = 0;
> }
> diff --git a/usr/bs_rdwr.c b/usr/bs_rdwr.c
> index 29db403..03b869d 100644
> --- a/usr/bs_rdwr.c
> +++ b/usr/bs_rdwr.c
> @@ -148,11 +148,13 @@ static void bs_rdwr_close(struct scsi_lu *lu)
> close(lu->fd);
> }
>
> +int nr_iothreads = 16;
> +
> static int bs_rdwr_init(struct scsi_lu *lu)
> {
> struct bs_thread_info *info = BS_THREAD_I(lu);
>
> - return bs_thread_open(info, bs_rdwr_request, NR_WORKER_THREADS);
> + return bs_thread_open(info, bs_rdwr_request, nr_iothreads);
> }
>
> static void bs_rdwr_exit(struct scsi_lu *lu)
> diff --git a/usr/bs_thread.h b/usr/bs_thread.h
> index d460032..50dd1c3 100644
> --- a/usr/bs_thread.h
> +++ b/usr/bs_thread.h
> @@ -1,9 +1,8 @@
> -#define NR_WORKER_THREADS 4
> -
> typedef void (request_func_t) (struct scsi_cmd *);
>
> struct bs_thread_info {
> - pthread_t worker_thread[NR_WORKER_THREADS];
> + pthread_t *worker_thread;
> + int nr_worker_threads;
>
> /* wokers sleep on this and signaled by tgtd */
> pthread_cond_t pending_cond;
> diff --git a/usr/tgtd.c b/usr/tgtd.c
> index 5e07267..d5a512c 100644
> --- a/usr/tgtd.c
> +++ b/usr/tgtd.c
> @@ -54,13 +54,14 @@ static struct option const long_options[] =
> {
> {"foreground", no_argument, 0, 'f'},
> {"control-port", required_argument, 0, 'C'},
> + {"nr_iothreads", required_argument, 0, 't'},
> {"debug", required_argument, 0, 'd'},
> {"version", no_argument, 0, 'V'},
> {"help", no_argument, 0, 'h'},
> {0, 0, 0, 0},
> };
>
> -static char *short_options = "fC:d:Vh";
> +static char *short_options = "fC:d:t:Vh";
>
> static void usage(int status)
> {
> @@ -72,6 +73,7 @@ static void usage(int status)
> Target framework daemon, version %s\n\
> -f, --foreground make the program run in the foreground\n\
> -C, --control-port NNNN use port NNNN for the mgmt channel\n\
> + -t, --nr_iothreads NNNN specify the number of I/O threads\n\
> -d, --debug debuglevel print debugging information\n\
> -V, --version print version and exit\n\
> -h, --help display this help and exit\n\
> @@ -500,6 +502,9 @@ int main(int argc, char **argv)
> case 'C':
> control_port = atoi(optarg);
> break;
> + case 't':
> + nr_iothreads = atoi(optarg);
> + break;
> case 'd':
> is_debug = atoi(optarg);
> break;
> diff --git a/usr/tgtd.h b/usr/tgtd.h
> index 032cb1c..8f61a07 100644
> --- a/usr/tgtd.h
> +++ b/usr/tgtd.h
> @@ -210,6 +210,7 @@ enum mgmt_req_result {
>
> extern int system_active;
> extern int is_debug;
> +extern int nr_iothreads;
>
> extern int kspace_send_tsk_mgmt_res(struct mgmt_req *mreq);
> extern int kspace_send_cmd_res(uint64_t nid, int result, struct scsi_cmd *);
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe stgt" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
More information about the stgt
mailing list