[Stgt-devel] sg_turs on stgt iscsi drive is very slow

[Ming Zhang]

On Sun, 2007-01-21 at 00:22 +0900, FUJITA Tomonori wrote:
> From: Ming Zhang <blackmagic02881 at gmail.com>
> Subject: [Stgt-devel] sg_turs on stgt iscsi drive is very slow
> Date: Mon, 11 Dec 2006 11:58:27 -0500
> 
> > Pulled out the linux-2.6-target git tree, compiled stgt and iet under
> > same kernel. Export a 4 disk raid0 device to ini. Ini is open-iscsi
> > r598.
> > 
> > Want to know how efficient stgt handle context switch. 
> > 
> > run  sg_turs -t -n=10000 /dev/sg0 on IET drive. results is 2521/sec.
> 
> (snip)
> 
> > run  sg_turs -t -n=10000 /dev/sg0 on stgt drive. results is 250
> > operations/sec. yes, 250.
> 
> OK. I did some tests too. As we know, aioepoll isn't effective.
> 
> The sg_turs test doesn't need AIO code. So I just tried epoll for
> event notification.
> 
> - tgt (patched)
> 
> paris:~# sg_turs -t -n=1000 /dev/sg3
> time to perform commands was 0.125131 secs; 7991.62 operations/sec
> Completed 1000 Test Unit Ready commands with 0 errors
> 
> - IET
> 
> paris:~# sg_turs -t -n=1000 /dev/sg3
> time to perform commands was 0.128747 secs; 7767.17 operations/sec
> Completed 1000 Test Unit Ready commands with 0 errors
> 
> 
> Well, not bad at all. And I heard that kevent is more effective than
> epoll.

the # looks pretty good. thanks for testing. maybe it is bounded by your
network now? could you try a open-iscsi via loopback?

what is the CPU utilization during test?

> 
> 
> Note that aioepoll affects only user-mode target drivers (now only
> iSCSI). The kernel target drivers don't need AIO code (can use
> epoll). So they should have no performance problem. And as I said
> before, I think that the iSCSI target driver is ok in real workloads.

do you have a road map for stgt? see, need to read your reply for Pete
to know OSD is already under your plan.


> 
> I've attached the patch for people who are interested:
> 
> 
> Index: usr/tgtd.c
> ===================================================================
> --- usr/tgtd.c	(revision 767)
> +++ usr/tgtd.c	(working copy)
> @@ -212,13 +212,13 @@
>  		return;
>  	}
>  
> -	iocb = iocbs;
> -	io_prep_epoll_wait(iocb, ep_fd, events, ARRAY_SIZE(events), -1);
> -	err = io_submit(ctx, 1, &iocb);
> -
> +/* 	iocb = iocbs; */
> +/* 	io_prep_epoll_wait(iocb, ep_fd, events, ARRAY_SIZE(events), -1); */
> +/* 	err = io_submit(ctx, 1, &iocb); */
>  retry:
> -	nevent = io_getevents(ctx, 1, ARRAY_SIZE(aioevents), aioevents, &timeout);
> +/* 	nevent = io_getevents(ctx, 1, ARRAY_SIZE(aioevents), aioevents, &timeout); */
>  
> +	nevent = epoll_wait(ep_fd, events, ARRAY_SIZE(events), -1);
>  	if (nevent < 0) {
>  		if (errno != EINTR) {
>  			eprintf("%m\n");
> @@ -226,18 +226,22 @@
>  		}
>  	} else if (nevent) {
>  		for (i = 0; i < nevent; i++) {
> -			if (iocb == aioevents[i].obj) {
> -				int j;
> -				for (j = 0; j < aioevents[i].res; j++) {
> -					tev = (struct tgt_event *) events[j].data.ptr;
> -					tev->handler(tev->fd, events[j].events, tev->data);
> -				}
>  
> -				err = io_submit(ctx, 1, &iocb);
> -			} else {
> -				/* FIXME */
> -				target_cmd_io_done(aioevents[i].data, 0);
> -			}
> +			tev = (struct tgt_event *) events[i].data.ptr;
> +			tev->handler(tev->fd, events[i].events, tev->data);
> +
> +/* 			if (iocb == aioevents[i].obj) { */
> +/* 				int j; */
> +/* 				for (j = 0; j < aioevents[i].res; j++) { */
> +/* 					tev = (struct tgt_event *) events[j].data.ptr; */
> +/* 					tev->handler(tev->fd, events[j].events, tev->data); */
> +/* 				} */
> +
> +/* 				err = io_submit(ctx, 1, &iocb); */
> +/* 			} else { */
> +/* 				/\* FIXME *\/ */
> +/* 				target_cmd_io_done(aioevents[i].data, 0); */
> +/* 			} */
>  		}
>  	} else
>  		schedule();
> Index: usr/bd_aio.c
> ===================================================================
> --- usr/bd_aio.c	(revision 767)
> +++ usr/bd_aio.c	(working copy)
> @@ -75,21 +75,21 @@
>  	struct iocb iocb, *io;
>  	int err;
>  
> -	*async = 1;
> +	*async = 0;
>  
> -	io = &iocb;
> -	memset(io, 0, sizeof(*io));
> +/* 	io = &iocb; */
> +/* 	memset(io, 0, sizeof(*io)); */
>  
> -	dprintf("%d %d %u %lx %" PRIx64 " %p %p\n", dev->fd, rw, datalen, *uaddr, offset,
> -		io, key);
> +/* 	dprintf("%d %d %u %lx %" PRIx64 " %p %p\n", dev->fd, rw, datalen, *uaddr, offset, */
> +/* 		io, key); */
>  
> -	if (rw == READ)
> -		io_prep_pread(io, dev->fd, (void *) *uaddr, datalen, offset);
> -	else
> -		io_prep_pwrite(io, dev->fd, (void *) *uaddr, datalen, offset);
> +/* 	if (rw == READ) */
> +/* 		io_prep_pread(io, dev->fd, (void *) *uaddr, datalen, offset); */
> +/* 	else */
> +/* 		io_prep_pwrite(io, dev->fd, (void *) *uaddr, datalen, offset); */
>  
> -	io->data = key;
> -	err = io_submit(ctx, 1, &io);
> +/* 	io->data = key; */
> +/* 	err = io_submit(ctx, 1, &io); */
>  
>  	return 0;
>  }
-- 
http://blackmagic02881.wordpress.com/