[stgt] [PATCH 1/2] Extract the thread loop function from iSCSI
Chandra Seetharaman
sekharan at us.ibm.com
Fri Oct 15 07:01:43 CEST 2010
In preparation of thread per connection functionality, extract the thread
function from the current context to be an independent (library like)
entity.
Signed-off-by: Chandra Seetharaman <sekharan at us.ibm.com>
---
usr/Makefile | 2 +-
usr/iscsi/iscsi_tcp.c | 20 +++-----------
usr/iscsi/iscsid.h | 8 ++----
usr/iscsi/target.c | 71 +++++++++++---------------------------------------
usr/tgtd.c | 61 +++++++++++++++++++++----------------------
usr/tgtd.h | 7 +++--
usr/thread.c | 61 +++++++++++++++++++++++++++++++++++++++++++
usr/thread.h | 17 ++++++++++++
8 files changed, 136 insertions(+), 111 deletions(-)
Index: tgt-thread/usr/thread.h
===================================================================
--- /dev/null
+++ tgt-thread/usr/thread.h
@@ -0,0 +1,17 @@
+#ifndef _USR_THREAD_H_
+#define _USR_THREAD_H_
+struct thread_info {
+ int efd;
+ pthread_rwlock_t rwlock;
+ struct list_head t_list;
+
+ int start_pthread;
+ int stop_pthread;
+
+ struct bs_finish *bsf;
+
+ struct target *target;
+};
+
+extern void *thread_fn(void *arg);
+#endif
Index: tgt-thread/usr/tgtd.h
===================================================================
--- tgt-thread.orig/usr/tgtd.h
+++ tgt-thread/usr/tgtd.h
@@ -2,6 +2,7 @@
#define __TARGET_DAEMON_H
#include "log.h"
+#include "thread.h"
#include "scsi_cmnd.h"
#define SCSI_ID_LEN 24
@@ -345,10 +346,10 @@ struct bs_finish {
void bs_sig_request_done(int fd, int events, void *data);
-int do_tgt_event_add(int efd, struct list_head *list, int fd, int events,
+int do_tgt_event_add(struct thread_info *ti, int fd, int events,
event_handler_t handler, void *data);
-void do_tgt_event_del(int efd, struct list_head *list, int fd);
-int do_tgt_event_modify(int efd, struct list_head *list, int fd, int events);
+void do_tgt_event_del(struct thread_info *ti, int fd);
+int do_tgt_event_modify(struct thread_info *ti, int fd, int events);
int call_program(const char *cmd,
void (*callback)(void *data, int result), void *data,
Index: tgt-thread/usr/iscsi/iscsid.h
===================================================================
--- tgt-thread.orig/usr/iscsi/iscsid.h
+++ tgt-thread/usr/iscsi/iscsid.h
@@ -22,12 +22,14 @@
#include <stdint.h>
#include <inttypes.h>
#include <netdb.h>
+#include <pthread.h>
#include "transport.h"
#include "list.h"
#include "param.h"
#include "log.h"
#include "tgtd.h"
+#include "thread.h"
#include "iscsi_proto.h"
#include "iscsi_if.h"
@@ -253,12 +255,8 @@ struct iscsi_target {
struct list_head isns_list;
- int efd;
- pthread_rwlock_t event_rwlock;
- struct list_head events_list;
-
pthread_t thread;
- int stop_pthread;
+ struct thread_info ti;
struct bs_finish bsfin;
};
Index: tgt-thread/usr/thread.c
===================================================================
--- /dev/null
+++ tgt-thread/usr/thread.c
@@ -0,0 +1,61 @@
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/epoll.h>
+#include <pthread.h>
+#include "list.h"
+#include "tgtd.h"
+#include "target.h"
+#include "thread.h"
+#include "util.h"
+
+void *thread_fn(void *arg)
+{
+ struct thread_info *ti = arg;
+ struct epoll_event events[1024];
+ struct event_data *tev;
+ sigset_t mask;
+ int nevent, i;
+
+ sigemptyset(&mask);
+ sigaddset(&mask, SIGUSR2);
+ pthread_sigmask(SIG_BLOCK, &mask, NULL);
+
+ if (ti->bsf) {
+ pthread_mutex_init(&ti->bsf->finished_lock, NULL);
+ INIT_LIST_HEAD(&ti->bsf->finished_list);
+ do_tgt_event_add(ti, sig_fd, EPOLLIN,
+ bs_sig_request_done, ti->bsf);
+ ti->bsf->th_id = syscall(SYS_gettid);
+ ti->target->bsf = ti->bsf;
+ eprintf("thread id %d, bsf %p, target %p\n",
+ ti->bsf->th_id, ti->target->bsf, ti->target);
+ }
+
+ while (!ti->start_pthread)
+ sleep(1);
+
+retry:
+ nevent = epoll_wait(ti->efd, events, ARRAY_SIZE(events), 1000);
+ if (nevent < 0) {
+ if (errno != EINTR) {
+ eprintf("%m\n");
+ exit(1);
+ }
+ } else if (nevent) {
+ for (i = 0; i < nevent; i++) {
+ tev = (struct event_data *) events[i].data.ptr;
+ tev->handler(tev->fd, events[i].events, tev->data);
+ }
+ }
+
+ if (!ti->stop_pthread)
+ goto retry;
+
+ close(ti->efd);
+ if (ti->bsf)
+ pthread_mutex_destroy(&ti->bsf->finished_lock);
+
+ pthread_rwlock_destroy(&ti->rwlock);
+
+ pthread_exit(NULL);
+}
Index: tgt-thread/usr/Makefile
===================================================================
--- tgt-thread.orig/usr/Makefile
+++ tgt-thread/usr/Makefile
@@ -59,7 +59,7 @@ PROGRAMS += tgtd tgtadm tgtimg
TGTD_OBJS += tgtd.o mgmt.o target.o scsi.o log.o driver.o util.o work.o \
parser.o spc.o sbc.o mmc.o osd.o scc.o smc.o \
ssc.o bs_ssc.o libssc.o \
- bs_null.o bs_sg.o bs.o libcrc32c.o
+ bs_null.o bs_sg.o bs.o libcrc32c.o thread.o
TGTD_DEP = $(TGTD_OBJS:.o=.d)
Index: tgt-thread/usr/iscsi/iscsi_tcp.c
===================================================================
--- tgt-thread.orig/usr/iscsi/iscsi_tcp.c
+++ tgt-thread/usr/iscsi/iscsi_tcp.c
@@ -171,10 +171,7 @@ static void iscsi_tcp_event_handler(int
if (tcp_conn->pthread) {
struct iscsi_target *target = conn->session->target;
- pthread_rwlock_wrlock(&target->event_rwlock);
- do_tgt_event_del(target->efd, &target->events_list,
- tcp_conn->fd);
- pthread_rwlock_unlock(&target->event_rwlock);
+ do_tgt_event_del(&target->ti, tcp_conn->fd);
/* let the main thread handle this */
tcp_conn->pthread = 0;
tgt_event_add(tcp_conn->fd, EPOLLIN,
@@ -290,13 +287,8 @@ static void iscsi_tcp_conn_nexus_init(st
tcp_conn->pthread = 1;
- pthread_rwlock_wrlock(&target->event_rwlock);
-
- do_tgt_event_add(target->efd, &target->events_list,
- tcp_conn->fd, EPOLLIN,
+ do_tgt_event_add(&target->ti, tcp_conn->fd, EPOLLIN,
iscsi_tcp_event_handler, conn);
-
- pthread_rwlock_unlock(&target->event_rwlock);
} else
conn->tp->ep_event_modify(conn, EPOLLIN);
}
@@ -375,12 +367,8 @@ static void iscsi_event_modify(struct is
int ret;
if (tcp_conn->pthread) {
- struct iscsi_target *target = conn->session->target;
-
- pthread_rwlock_rdlock(&target->event_rwlock);
- do_tgt_event_modify(target->efd, &target->events_list,
- tcp_conn->fd, events);
- pthread_rwlock_unlock(&target->event_rwlock);
+ do_tgt_event_modify(&conn->session->target->ti,
+ tcp_conn->fd, events);
} else {
ret = tgt_event_modify(tcp_conn->fd, events);
if (ret)
Index: tgt-thread/usr/iscsi/target.c
===================================================================
--- tgt-thread.orig/usr/iscsi/target.c
+++ tgt-thread/usr/iscsi/target.c
@@ -380,65 +380,18 @@ void iscsi_target_destroy(int tid)
list_del(&target->tlist);
if (target->thread) {
- target->stop_pthread = 1;
+ target->ti.stop_pthread = 1;
pthread_kill(target->thread, SIGUSR2);
pthread_join(target->thread, NULL);
- pthread_mutex_destroy(&target->bsfin.finished_lock);
- pthread_rwlock_destroy(&target->event_rwlock);
}
- close(target->efd);
free(target);
isns_target_deregister(tgt_targetname(tid));
return;
}
-static void *iscsi_thread_fn(void *arg)
-{
- struct iscsi_target *t = arg;
- struct epoll_event events[1024];
- struct event_data *tev;
- sigset_t mask;
- int nevent, i;
-
- sigemptyset(&mask);
- sigaddset(&mask, SIGUSR2);
- pthread_sigmask(SIG_BLOCK, &mask, NULL);
-
- pthread_mutex_init(&t->bsfin.finished_lock, NULL);
- INIT_LIST_HEAD(&t->bsfin.finished_list);
-
- t->bsfin.th_id = syscall(SYS_gettid);
- pthread_rwlock_wrlock(&t->event_rwlock);
-
- eprintf("Th_id: %d\n", t->bsfin.th_id);
-
- do_tgt_event_add(t->efd, &t->events_list, sig_fd, EPOLLIN,
- bs_sig_request_done, &t->bsfin);
-
- pthread_rwlock_unlock(&t->event_rwlock);
-retry:
- nevent = epoll_wait(t->efd, events, ARRAY_SIZE(events), 1000);
- if (nevent < 0) {
- if (errno != EINTR) {
- eprintf("%m\n");
- exit(1);
- }
- } else if (nevent) {
- for (i = 0; i < nevent; i++) {
- tev = (struct event_data *) events[i].data.ptr;
- tev->handler(tev->fd, events[i].events, tev->data);
- }
- }
-
- if (!t->stop_pthread)
- goto retry;
-
- pthread_exit(NULL);
-}
-
int iscsi_target_create(struct target *t)
{
int tid = t->tid;
@@ -473,8 +426,8 @@ int iscsi_target_create(struct target *t
if (!target)
return -ENOMEM;
- target->efd = epoll_create(128);
- if (target->efd < 0) {
+ target->ti.efd = epoll_create(128);
+ if (target->ti.efd < 0) {
free(target);
return -EINVAL;
}
@@ -485,8 +438,6 @@ int iscsi_target_create(struct target *t
INIT_LIST_HEAD(&target->tlist);
INIT_LIST_HEAD(&target->sessions_list);
INIT_LIST_HEAD(&target->isns_list);
- INIT_LIST_HEAD(&target->events_list);
- pthread_rwlock_init(&target->event_rwlock, NULL);
target->tid = tid;
list_add_tail(&target->tlist, &iscsi_targets_list);
@@ -495,13 +446,23 @@ int iscsi_target_create(struct target *t
if (iscsi_pthread_per_target()) {
int ret;
- ret = pthread_create(&target->thread,
- NULL, iscsi_thread_fn, target);
+ INIT_LIST_HEAD(&target->ti.t_list);
+ pthread_rwlock_init(&target->ti.rwlock, NULL);
+
+ target->ti.start_pthread = 0;
+ target->ti.bsf = &target->bsfin;
+ target->ti.target = t;
+ ret = pthread_create(&target->thread, NULL, thread_fn,
+ &target->ti);
if (!ret) {
- t->bsf = &target->bsfin;
+ target->ti.start_pthread = 1;
+
eprintf("created thread %u for target %s\n",
(unsigned)target->thread, tgt_targetname(tid));
+ } else {
+ target->ti.bsf = NULL;
+ target->ti.target = NULL;
}
}
Index: tgt-thread/usr/tgtd.c
===================================================================
--- tgt-thread.orig/usr/tgtd.c
+++ tgt-thread/usr/tgtd.c
@@ -41,14 +41,14 @@
#include "tgtd.h"
#include "driver.h"
#include "work.h"
+#include "thread.h"
#include "util.h"
unsigned long pagesize, pageshift;
int system_active = 1;
-static int ep_fd;
static char program_name[] = "tgtd";
-static LIST_HEAD(tgt_events_list);
+static struct thread_info main_ti;
static LIST_HEAD(tgt_sched_events_list);
static struct option const long_options[] =
@@ -139,7 +139,7 @@ set_rlimit:
return 0;
}
-int do_tgt_event_add(int efd, struct list_head *list, int fd, int events,
+int do_tgt_event_add(struct thread_info *ti, int fd, int events,
event_handler_t handler, void *data)
{
struct epoll_event ev;
@@ -150,6 +150,7 @@ int do_tgt_event_add(int efd, struct lis
if (!tev)
return -ENOMEM;
+ pthread_rwlock_wrlock(&ti->rwlock);
tev->data = data;
tev->handler = handler;
tev->fd = fd;
@@ -157,27 +158,22 @@ int do_tgt_event_add(int efd, struct lis
memset(&ev, 0, sizeof(ev));
ev.events = events;
ev.data.ptr = tev;
- err = epoll_ctl(efd, EPOLL_CTL_ADD, fd, &ev);
+ err = epoll_ctl(ti->efd, EPOLL_CTL_ADD, fd, &ev);
if (err) {
eprintf("Cannot add fd, %m\n");
free(tev);
} else
- list_add(&tev->e_list, list);
-
+ list_add(&tev->e_list, &ti->t_list);
+ pthread_rwlock_unlock(&ti->rwlock);
return err;
}
-static pthread_rwlock_t tgt_events_rwlock;
int tgt_event_add(int fd, int events, event_handler_t handler, void *data)
{
- int ret;
- pthread_rwlock_wrlock(&tgt_events_rwlock);
- ret = do_tgt_event_add(ep_fd, &tgt_events_list, fd, events, handler,
- data);
- pthread_rwlock_unlock(&tgt_events_rwlock);
- return ret;
+ return do_tgt_event_add(&main_ti, fd, events, handler, data);
}
+/* Assumes the list is protected */
static struct event_data *tgt_event_lookup(struct list_head *list, int fd)
{
struct event_data *tev;
@@ -189,40 +185,44 @@ static struct event_data *tgt_event_look
return NULL;
}
-void do_tgt_event_del(int efd, struct list_head *list, int fd)
+void do_tgt_event_del(struct thread_info *ti, int fd)
{
struct event_data *tev;
int ret;
- tev = tgt_event_lookup(list, fd);
+ pthread_rwlock_wrlock(&ti->rwlock);
+ tev = tgt_event_lookup(&ti->t_list, fd);
if (!tev) {
eprintf("Cannot find event %d\n", fd);
+ pthread_rwlock_unlock(&ti->rwlock);
return;
}
- ret = epoll_ctl(efd, EPOLL_CTL_DEL, fd, NULL);
+ ret = epoll_ctl(ti->efd, EPOLL_CTL_DEL, fd, NULL);
if (ret < 0)
eprintf("fail to remove epoll event, %s\n", strerror(errno));
list_del(&tev->e_list);
+ pthread_rwlock_unlock(&ti->rwlock);
free(tev);
}
void tgt_event_del(int fd)
{
- pthread_rwlock_wrlock(&tgt_events_rwlock);
- do_tgt_event_del(ep_fd, &tgt_events_list, fd);
- pthread_rwlock_unlock(&tgt_events_rwlock);
+ do_tgt_event_del(&main_ti, fd);
}
-int do_tgt_event_modify(int efd, struct list_head *list, int fd, int events)
+int do_tgt_event_modify(struct thread_info *ti, int fd, int events)
{
+ int ret;
struct epoll_event ev;
struct event_data *tev;
- tev = tgt_event_lookup(list, fd);
+ pthread_rwlock_rdlock(&ti->rwlock);
+ tev = tgt_event_lookup(&ti->t_list, fd);
if (!tev) {
eprintf("Cannot find event %d\n", fd);
+ pthread_rwlock_unlock(&ti->rwlock);
return -EINVAL;
}
@@ -230,16 +230,14 @@ int do_tgt_event_modify(int efd, struct
ev.events = events;
ev.data.ptr = tev;
- return epoll_ctl(efd, EPOLL_CTL_MOD, fd, &ev);
+ ret = epoll_ctl(ti->efd, EPOLL_CTL_MOD, fd, &ev);
+ pthread_rwlock_unlock(&ti->rwlock);
+ return ret;
}
int tgt_event_modify(int fd, int events)
{
- int ret;
- pthread_rwlock_rdlock(&tgt_events_rwlock);
- ret = do_tgt_event_modify(ep_fd, &tgt_events_list, fd, events);
- pthread_rwlock_unlock(&tgt_events_rwlock);
- return ret;
+ return do_tgt_event_modify(&main_ti, fd, events);
}
void tgt_init_sched_event(struct event_data *evt,
@@ -368,7 +366,7 @@ retry:
sched_remains = tgt_exec_scheduled();
timeout = sched_remains ? 0 : TGTD_TICK_PERIOD * 1000;
- nevent = epoll_wait(ep_fd, events, ARRAY_SIZE(events), timeout);
+ nevent = epoll_wait(main_ti.efd, events, ARRAY_SIZE(events), timeout);
if (nevent < 0) {
if (errno != EINTR) {
eprintf("%m\n");
@@ -469,7 +467,8 @@ int main(int argc, char **argv)
sigaction(SIGINT, &sa_new, &sa_old);
sigaction(SIGPIPE, &sa_new, &sa_old);
sigaction(SIGTERM, &sa_new, &sa_old);
- pthread_rwlock_init(&tgt_events_rwlock, NULL);
+ pthread_rwlock_init(&main_ti.rwlock, NULL);
+ INIT_LIST_HEAD(&main_ti.t_list);
pagesize = sysconf(_SC_PAGESIZE);
for (pageshift = 0;; pageshift++)
@@ -508,8 +507,8 @@ int main(int argc, char **argv)
}
}
- ep_fd = epoll_create(4096);
- if (ep_fd < 0) {
+ main_ti.efd = epoll_create(4096);
+ if (main_ti.efd < 0) {
fprintf(stderr, "can't create epoll fd, %m\n");
exit(1);
}
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
More information about the stgt
mailing list