[stgt] Worker Threads

[FUJITA Tomonori]

On Wed, 27 Jul 2011 13:03:52 -0700
"Bennett, Jeffrey" <jab at sdsc.edu> wrote:

> Is there any "side effect" by doing this change?

NR_WORKER_THREADS is the number of threads that perform I/Os. So you
run more threads. You consume more memory.

> Is there any plan
> to make this a configuration option that can be changed with the
> tgt-adm command by the user?

tgtd should adjust NR_WORKER_THREADS dynamically since the optimal
value depends on your workload, hardware, etc. Well, until we
implement that feature, I think that we can support the boot option to
specify the number of I/O threads.

The following patch works for you? You can specify the number in the
following way:

tgtd --nr_iothreads=128


diff --git a/usr/bs.c b/usr/bs.c
index d72d090..6f67d15 100644
--- a/usr/bs.c
+++ b/usr/bs.c
@@ -24,6 +24,7 @@
 #include <inttypes.h>
 #include <pthread.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <signal.h>
@@ -327,6 +328,11 @@ int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
 {
 	int i, ret;
 
+	info->worker_thread = zalloc(sizeof(pthread_t) * nr_threads);
+	if (!info->worker_thread)
+		return TGTADM_NOMEM;
+
+	eprintf("%d\n", nr_threads);
 	info->request_fn = rfn;
 
 	INIT_LIST_HEAD(&info->pending_list);
@@ -335,11 +341,6 @@ int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
 	pthread_mutex_init(&info->pending_lock, NULL);
 	pthread_mutex_init(&info->startup_lock, NULL);
 
-	if (nr_threads > ARRAY_SIZE(info->worker_thread)) {
-		eprintf("too many threads %d\n", nr_threads);
-		nr_threads = ARRAY_SIZE(info->worker_thread);
-	}
-
 	pthread_mutex_lock(&info->startup_lock);
 	for (i = 0; i < nr_threads; i++) {
 		ret = pthread_create(&info->worker_thread[i], NULL,
@@ -353,6 +354,7 @@ int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
 		}
 	}
 	pthread_mutex_unlock(&info->startup_lock);
+	info->nr_worker_threads = nr_threads;
 
 	return 0;
 destroy_threads:
@@ -367,6 +369,7 @@ destroy_threads:
 	pthread_cond_destroy(&info->pending_cond);
 	pthread_mutex_destroy(&info->pending_lock);
 	pthread_mutex_destroy(&info->startup_lock);
+	free(info->worker_thread);
 
 	return TGTADM_NOMEM;
 }
@@ -378,13 +381,13 @@ void bs_thread_close(struct bs_thread_info *info)
 	info->stop = 1;
 	pthread_cond_broadcast(&info->pending_cond);
 
-	for (i = 0; info->worker_thread[i] &&
-		     i < ARRAY_SIZE(info->worker_thread); i++)
+	for (i = 0; info->worker_thread[i] && i < info->nr_worker_threads; i++)
 		pthread_join(info->worker_thread[i], NULL);
 
 	pthread_cond_destroy(&info->pending_cond);
 	pthread_mutex_destroy(&info->pending_lock);
 	pthread_mutex_destroy(&info->startup_lock);
+	free(info->worker_thread);
 
 	info->stop = 0;
 }
diff --git a/usr/bs_rdwr.c b/usr/bs_rdwr.c
index 29db403..03b869d 100644
--- a/usr/bs_rdwr.c
+++ b/usr/bs_rdwr.c
@@ -148,11 +148,13 @@ static void bs_rdwr_close(struct scsi_lu *lu)
 	close(lu->fd);
 }
 
+int nr_iothreads = 16;
+
 static int bs_rdwr_init(struct scsi_lu *lu)
 {
 	struct bs_thread_info *info = BS_THREAD_I(lu);
 
-	return bs_thread_open(info, bs_rdwr_request, NR_WORKER_THREADS);
+	return bs_thread_open(info, bs_rdwr_request, nr_iothreads);
 }
 
 static void bs_rdwr_exit(struct scsi_lu *lu)
diff --git a/usr/bs_thread.h b/usr/bs_thread.h
index d460032..50dd1c3 100644
--- a/usr/bs_thread.h
+++ b/usr/bs_thread.h
@@ -1,9 +1,8 @@
-#define NR_WORKER_THREADS	4
-
 typedef void (request_func_t) (struct scsi_cmd *);
 
 struct bs_thread_info {
-	pthread_t worker_thread[NR_WORKER_THREADS];
+	pthread_t *worker_thread;
+	int nr_worker_threads;
 
 	/* wokers sleep on this and signaled by tgtd */
 	pthread_cond_t pending_cond;
diff --git a/usr/tgtd.c b/usr/tgtd.c
index 5e07267..d5a512c 100644
--- a/usr/tgtd.c
+++ b/usr/tgtd.c
@@ -54,13 +54,14 @@ static struct option const long_options[] =
 {
 	{"foreground", no_argument, 0, 'f'},
 	{"control-port", required_argument, 0, 'C'},
+	{"nr_iothreads", required_argument, 0, 't'},
 	{"debug", required_argument, 0, 'd'},
 	{"version", no_argument, 0, 'V'},
 	{"help", no_argument, 0, 'h'},
 	{0, 0, 0, 0},
 };
 
-static char *short_options = "fC:d:Vh";
+static char *short_options = "fC:d:t:Vh";
 
 static void usage(int status)
 {
@@ -72,6 +73,7 @@ static void usage(int status)
 Target framework daemon, version %s\n\
   -f, --foreground        make the program run in the foreground\n\
   -C, --control-port NNNN use port NNNN for the mgmt channel\n\
+  -t, --nr_iothreads NNNN specify the number of I/O threads\n\
   -d, --debug debuglevel  print debugging information\n\
   -V, --version           print version and exit\n\
   -h, --help              display this help and exit\n\
@@ -500,6 +502,9 @@ int main(int argc, char **argv)
 		case 'C':
 			control_port = atoi(optarg);
 			break;
+		case 't':
+			nr_iothreads = atoi(optarg);
+			break;
 		case 'd':
 			is_debug = atoi(optarg);
 			break;
diff --git a/usr/tgtd.h b/usr/tgtd.h
index 032cb1c..8f61a07 100644
--- a/usr/tgtd.h
+++ b/usr/tgtd.h
@@ -210,6 +210,7 @@ enum mgmt_req_result {
 
 extern int system_active;
 extern int is_debug;
+extern int nr_iothreads;
 
 extern int kspace_send_tsk_mgmt_res(struct mgmt_req *mreq);
 extern int kspace_send_cmd_res(uint64_t nid, int result, struct scsi_cmd *);


--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html