[sheepdog] [PATCH v3 1/3] sheep: allow {register, unregister}_event to be called in worker thread

[Liu Yuan]

For now we can only call them in the main thread, which is designed for long
running or infrequent events. This would be inefficient if we want to deal with
short running and frequent events that register/unregister the events in the
worker thread

1. avoid to be trapped to main thread for performance
2. make sure registeration is done before some other events.

This doesn't mean we can manipulate the same event with multiple threads
simultaneously, instead we still adhere to the assumpioin that

- one event will only be manipluated by a single entity.

Signed-off-by: Liu Yuan <namei.unix at gmail.com>
---
 include/event.h      |   15 +++++++++++++++
 lib/event.c          |   14 ++++++++++++--
 shepherd/Makefile.am |    2 +-
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/include/event.h b/include/event.h
index 8f8b21f..0082e1a 100644
--- a/include/event.h
+++ b/include/event.h
@@ -27,6 +27,21 @@ void add_timer(struct timer *t, unsigned int mseconds);
 #define EVENT_PRIO_DEFAULT 0
 #define EVENT_PRIO_MIN     INT_MIN
 
+/*
+ * We can register/unregister the events in the worker thread with the
+ * assumption:
+ *
+ * - one event will only be manipluated by a single entity.
+ *
+ * By allowing register/unregister events in the work thread, we get the
+ * following benefits:
+ *
+ * 1. avoid to be trapped to main thread for better performance
+ * 2. make sure registeration is done before some other events
+ *
+ * This doesn't mean we can manipulate the same event with multiple threads
+ * simultaneously.
+ */
 static inline int register_event(int fd, event_handler_t h, void *data)
 {
 	return register_event_prio(fd, h, data, EVENT_PRIO_DEFAULT);
diff --git a/lib/event.c b/lib/event.c
index 88078f4..16aa921 100644
--- a/lib/event.c
+++ b/lib/event.c
@@ -21,6 +21,7 @@
 
 static int efd;
 static struct rb_root events_tree = RB_ROOT;
+static struct sd_lock events_lock = SD_LOCK_INITIALIZER;
 
 static void timer_handler(int fd, int events, void *data)
 {
@@ -92,8 +93,12 @@ int init_event(int nr)
 static struct event_info *lookup_event(int fd)
 {
 	struct event_info key = { .fd = fd };
+	struct event_info *ret;
 
-	return rb_search(&events_tree, &key, rb, event_cmp);
+	sd_read_lock(&events_lock);
+	ret = rb_search(&events_tree, &key, rb, event_cmp);
+	sd_unlock(&events_lock);
+	return ret;
 }
 
 int register_event_prio(int fd, event_handler_t h, void *data, int prio)
@@ -116,8 +121,11 @@ int register_event_prio(int fd, event_handler_t h, void *data, int prio)
 	if (ret) {
 		sd_err("failed to add epoll event: %m");
 		free(ei);
-	} else
+	} else {
+		sd_write_lock(&events_lock);
 		rb_insert(&events_tree, ei, rb, event_cmp);
+		sd_unlock(&events_lock);
+	}
 
 	return ret;
 }
@@ -135,7 +143,9 @@ void unregister_event(int fd)
 	if (ret)
 		sd_err("failed to delete epoll event for fd %d: %m", fd);
 
+	sd_write_lock(&events_lock);
 	rb_erase(&ei->rb, &events_tree);
+	sd_unlock(&events_lock);
 	free(ei);
 
 	/*
diff --git a/shepherd/Makefile.am b/shepherd/Makefile.am
index 33a3da6..86f45be 100644
--- a/shepherd/Makefile.am
+++ b/shepherd/Makefile.am
@@ -25,7 +25,7 @@ sbin_PROGRAMS		= shepherd
 
 shepherd_SOURCES		= shepherd.c
 
-shepherd_LDADD	  	= ../lib/libsheepdog.a
+shepherd_LDADD	  	= ../lib/libsheepdog.a -lpthread
 shepherd_DEPENDENCIES	= ../lib/libsheepdog.a
 
 EXTRA_DIST		=
-- 
1.7.9.5