[stgt] tgtd memory usage

[Or Gerlitz]

FUJITA Tomonori wrote:
> On Wed, 30 Mar 2011 22:12:23 +0400
> stepan.fedorov wrote:

>> Thanks all for attension, but question remains open: what can we do,
>> to serve 1500-3000 LUNs from one node with no more then 24Gb of memory?

> Maybe the following patch works for you.

Yep, it looks something that has to do with the threads code consumes
about 10MB per thread... Stepan, you should be able to see that using
your script that allocates targets and measures the memory consumption
if applying the following trivial patch, I hope that either Tomo or
someone here will be able to address that soon

> diff --git a/usr/bs_thread.h b/usr/bs_thread.h
> index d460032..3e3ba79 100644
> --- a/usr/bs_thread.h
> +++ b/usr/bs_thread.h
> @@ -1,4 +1,4 @@
> -#define NR_WORKER_THREADS      4
> +#define NR_WORKER_THREADS      1
> 
>  typedef void (request_func_t) (struct scsi_cmd *);

Tomo, could you spare few words on the patch / approach you had in mind
to address that? I see that your patch would only call  bs_thread_open
once, e.g if (!init), where today it is called once per LUN... also I
wasn't sure to follow why you've enlarged the number of threads from 4
to 16, does this moves to global thread pool or alike?

Or.

> diff --git a/usr/bs_rdwr.c b/usr/bs_rdwr.c
> index 77d52e3..9abec59 100644
> --- a/usr/bs_rdwr.c
> +++ b/usr/bs_rdwr.c
> @@ -148,18 +148,23 @@ static void bs_rdwr_close(struct scsi_lu *lu)
>  	close(lu->fd);
>  }
>  
> +static struct bs_thread_info rdwr_bst_info;
> +
>  static int bs_rdwr_init(struct scsi_lu *lu)
>  {
> -	struct bs_thread_info *info = BS_THREAD_I(lu);
> +	static int init;
> +
> +	if (!init) {
> +		bs_thread_open(&rdwr_bst_info,
> +			       bs_rdwr_request, NR_WORKER_THREADS);
> +		init = 1;
> +	}
>  
> -	return bs_thread_open(info, bs_rdwr_request, NR_WORKER_THREADS);
> +	return 0;
>  }
>  
>  static void bs_rdwr_exit(struct scsi_lu *lu)
>  {
> -	struct bs_thread_info *info = BS_THREAD_I(lu);
> -
> -	bs_thread_close(info);
>  }
>  
>  static int bs_rdwr_cmd_done(struct scsi_cmd *cmd)
> @@ -177,6 +182,7 @@ static struct backingstore_template rdwr_bst = {
>  	.bs_cmd_submit		= bs_thread_cmd_submit,
>  	.bs_cmd_done		= bs_rdwr_cmd_done,
>  	.bs_oflags_supported    = O_SYNC | O_DIRECT,
> +	.bs_private		= (char *)&rdwr_bst_info,
>  };
>  
>  __attribute__((constructor)) static void bs_rdwr_constructor(void)
> diff --git a/usr/bs_thread.h b/usr/bs_thread.h
> index d460032..c47690e 100644
> --- a/usr/bs_thread.h
> +++ b/usr/bs_thread.h
> @@ -1,4 +1,4 @@
> -#define NR_WORKER_THREADS	4
> +#define NR_WORKER_THREADS	16
>  
>  typedef void (request_func_t) (struct scsi_cmd *);
>  
> @@ -21,7 +21,7 @@ struct bs_thread_info {
>  
>  static inline struct bs_thread_info *BS_THREAD_I(struct scsi_lu *lu)
>  {
> -	return (struct bs_thread_info *) ((char *)lu + sizeof(*lu));
> +	return (struct bs_thread_info *)lu->bst->bs_private;
>  }
>  
>  extern int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
> diff --git a/usr/tgtd.h b/usr/tgtd.h
> index 37943d2..ea2414f 100644
> --- a/usr/tgtd.h
> +++ b/usr/tgtd.h
> @@ -126,6 +126,7 @@ struct backingstore_template {
>  	int (*bs_cmd_submit)(struct scsi_cmd *cmd);
>  	int (*bs_cmd_done)(struct scsi_cmd *cmd);
>  	int bs_oflags_supported;
> +	char *bs_private;
>  
>  	struct list_head backingstore_siblings;
>  };

--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html