[Stgt-devel] sg_turs on stgt iscsi drive is very slow

[FUJITA Tomonori]

From: Ming Zhang <blackmagic02881 at gmail.com>
Subject: [Stgt-devel] sg_turs on stgt iscsi drive is very slow
Date: Mon, 11 Dec 2006 11:58:27 -0500

> Pulled out the linux-2.6-target git tree, compiled stgt and iet under
> same kernel. Export a 4 disk raid0 device to ini. Ini is open-iscsi
> r598.
> 
> Want to know how efficient stgt handle context switch. 
> 
> run  sg_turs -t -n=10000 /dev/sg0 on IET drive. results is 2521/sec.

(snip)

> run  sg_turs -t -n=10000 /dev/sg0 on stgt drive. results is 250
> operations/sec. yes, 250.

OK. I did some tests too. As we know, aioepoll isn't effective.

The sg_turs test doesn't need AIO code. So I just tried epoll for
event notification.

- tgt (patched)

paris:~# sg_turs -t -n=1000 /dev/sg3
time to perform commands was 0.125131 secs; 7991.62 operations/sec
Completed 1000 Test Unit Ready commands with 0 errors

- IET

paris:~# sg_turs -t -n=1000 /dev/sg3
time to perform commands was 0.128747 secs; 7767.17 operations/sec
Completed 1000 Test Unit Ready commands with 0 errors


Well, not bad at all. And I heard that kevent is more effective than
epoll.


Note that aioepoll affects only user-mode target drivers (now only
iSCSI). The kernel target drivers don't need AIO code (can use
epoll). So they should have no performance problem. And as I said
before, I think that the iSCSI target driver is ok in real workloads.

I've attached the patch for people who are interested:


Index: usr/tgtd.c
===================================================================
--- usr/tgtd.c	(revision 767)
+++ usr/tgtd.c	(working copy)
@@ -212,13 +212,13 @@
 		return;
 	}
 
-	iocb = iocbs;
-	io_prep_epoll_wait(iocb, ep_fd, events, ARRAY_SIZE(events), -1);
-	err = io_submit(ctx, 1, &iocb);
-
+/* 	iocb = iocbs; */
+/* 	io_prep_epoll_wait(iocb, ep_fd, events, ARRAY_SIZE(events), -1); */
+/* 	err = io_submit(ctx, 1, &iocb); */
 retry:
-	nevent = io_getevents(ctx, 1, ARRAY_SIZE(aioevents), aioevents, &timeout);
+/* 	nevent = io_getevents(ctx, 1, ARRAY_SIZE(aioevents), aioevents, &timeout); */
 
+	nevent = epoll_wait(ep_fd, events, ARRAY_SIZE(events), -1);
 	if (nevent < 0) {
 		if (errno != EINTR) {
 			eprintf("%m\n");
@@ -226,18 +226,22 @@
 		}
 	} else if (nevent) {
 		for (i = 0; i < nevent; i++) {
-			if (iocb == aioevents[i].obj) {
-				int j;
-				for (j = 0; j < aioevents[i].res; j++) {
-					tev = (struct tgt_event *) events[j].data.ptr;
-					tev->handler(tev->fd, events[j].events, tev->data);
-				}
 
-				err = io_submit(ctx, 1, &iocb);
-			} else {
-				/* FIXME */
-				target_cmd_io_done(aioevents[i].data, 0);
-			}
+			tev = (struct tgt_event *) events[i].data.ptr;
+			tev->handler(tev->fd, events[i].events, tev->data);
+
+/* 			if (iocb == aioevents[i].obj) { */
+/* 				int j; */
+/* 				for (j = 0; j < aioevents[i].res; j++) { */
+/* 					tev = (struct tgt_event *) events[j].data.ptr; */
+/* 					tev->handler(tev->fd, events[j].events, tev->data); */
+/* 				} */
+
+/* 				err = io_submit(ctx, 1, &iocb); */
+/* 			} else { */
+/* 				/\* FIXME *\/ */
+/* 				target_cmd_io_done(aioevents[i].data, 0); */
+/* 			} */
 		}
 	} else
 		schedule();
Index: usr/bd_aio.c
===================================================================
--- usr/bd_aio.c	(revision 767)
+++ usr/bd_aio.c	(working copy)
@@ -75,21 +75,21 @@
 	struct iocb iocb, *io;
 	int err;
 
-	*async = 1;
+	*async = 0;
 
-	io = &iocb;
-	memset(io, 0, sizeof(*io));
+/* 	io = &iocb; */
+/* 	memset(io, 0, sizeof(*io)); */
 
-	dprintf("%d %d %u %lx %" PRIx64 " %p %p\n", dev->fd, rw, datalen, *uaddr, offset,
-		io, key);
+/* 	dprintf("%d %d %u %lx %" PRIx64 " %p %p\n", dev->fd, rw, datalen, *uaddr, offset, */
+/* 		io, key); */
 
-	if (rw == READ)
-		io_prep_pread(io, dev->fd, (void *) *uaddr, datalen, offset);
-	else
-		io_prep_pwrite(io, dev->fd, (void *) *uaddr, datalen, offset);
+/* 	if (rw == READ) */
+/* 		io_prep_pread(io, dev->fd, (void *) *uaddr, datalen, offset); */
+/* 	else */
+/* 		io_prep_pwrite(io, dev->fd, (void *) *uaddr, datalen, offset); */
 
-	io->data = key;
-	err = io_submit(ctx, 1, &io);
+/* 	io->data = key; */
+/* 	err = io_submit(ctx, 1, &io); */
 
 	return 0;
 }