[sheepdog] [PATCH 2/3] event: add deferred event register/unregister mechanism

[Hitoshi Mitake]

This patch introduces deferred event register/unregister
mechanism. Newly added APIs are:
 - deferred_register_event(): thread safe register_event()
 - deferred_register_event_prio(): thread safe register_event_prio()
 - deferred_unregister_event(): thread safe unregister_event()

These functions can be called by worker threads safely. They allocate
data structure which represents registering/unregistering event add
queue it to the list shared with the main thread. After queuing,
the main thread registers and unregisters events in a safe way.

Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
---
 include/event.h |   5 +++
 lib/event.c     | 132 ++++++++++++++++++++++++++++++++++++++++++++++++++------
 2 files changed, 124 insertions(+), 13 deletions(-)

diff --git a/include/event.h b/include/event.h
index 8f8b21f..b64b06e 100644
--- a/include/event.h
+++ b/include/event.h
@@ -32,4 +32,9 @@ static inline int register_event(int fd, event_handler_t h, void *data)
 	return register_event_prio(fd, h, data, EVENT_PRIO_DEFAULT);
 }
 
+void deferred_register_event_prio(int fd, event_handler_t h, void *data,
+				  int prio);
+void deferred_register_event(int fd, event_handler_t h, void *data);
+void deferred_unregister_event(int fd);
+
 #endif
diff --git a/lib/event.c b/lib/event.c
index 88078f4..2549dcd 100644
--- a/lib/event.c
+++ b/lib/event.c
@@ -76,19 +76,6 @@ static int event_cmp(const struct event_info *e1, const struct event_info *e2)
 	return intcmp(e1->fd, e2->fd);
 }
 
-int init_event(int nr)
-{
-	nr_events = nr;
-	events = xcalloc(nr_events, sizeof(struct epoll_event));
-
-	efd = epoll_create(nr);
-	if (efd < 0) {
-		sd_err("failed to create epoll fd");
-		return -1;
-	}
-	return 0;
-}
-
 static struct event_info *lookup_event(int fd)
 {
 	struct event_info key = { .fd = fd };
@@ -224,3 +211,122 @@ void event_loop_prio(int timeout)
 {
 	do_event_loop(timeout, true);
 }
+
+struct deferred_event_info {
+	bool is_register;	/* true: register, false: unregister */
+
+	int fd;
+	event_handler_t h;
+	void *data;
+	int prio;
+
+	struct list_node list;
+};
+
+static LIST_HEAD(deferred_event_list);
+static struct sd_mutex deferred_event_mutex = SD_MUTEX_INITIALIZER;
+
+static int deferred_event_fd;
+
+static void add_deferred_event_info(struct deferred_event_info *info)
+{
+	sd_mutex_lock(&deferred_event_mutex);
+	list_add_tail(&info->list, &deferred_event_list);
+	sd_mutex_unlock(&deferred_event_mutex);
+
+	eventfd_xwrite(deferred_event_fd, 1);
+	event_force_refresh();
+}
+
+void deferred_register_event_prio(int fd, event_handler_t h, void *data,
+				  int prio)
+{
+	struct deferred_event_info *info = xzalloc(sizeof(*info));
+
+	info->is_register = true;
+
+	info->fd = fd;
+	info->h = h;
+	info->data = data;
+	info->prio = prio;
+
+	add_deferred_event_info(info);
+}
+
+void deferred_register_event(int fd, event_handler_t h, void *data)
+{
+	deferred_register_event_prio(fd, h, data, EVENT_PRIO_DEFAULT);
+}
+
+void deferred_unregister_event(int fd)
+{
+	struct deferred_event_info *info = xzalloc(sizeof(*info));
+
+	info->is_register = false;
+	info->fd = fd;
+
+	add_deferred_event_info(info);
+}
+
+static void deferred_event_handler(int fd, int _events, void *data)
+{
+	LIST_HEAD(list);
+
+	assert(fd == deferred_event_fd);
+	eventfd_xread(fd);
+
+	sd_mutex_lock(&deferred_event_mutex);
+	list_splice_init(&deferred_event_list, &list);
+	sd_mutex_unlock(&deferred_event_mutex);
+
+	while (!list_empty(&list)) {
+		int ret;
+		struct deferred_event_info *info;
+
+		info = list_first_entry(&list, struct deferred_event_info,
+					list);
+
+		if (info->is_register) {
+			ret = register_event_prio(info->fd, info->h,
+						  info->data, info->prio);
+			if (ret < 0)
+				sd_err("failed to register event."
+				       " fd: %d, handler: %p",
+				       info->fd, info->h);
+		} else
+			unregister_event(info->fd);
+
+		list_del(&info->list);
+		free(info);
+	}
+}
+
+int init_event(int nr)
+{
+	int ret;
+
+	nr_events = nr;
+	events = xcalloc(nr_events, sizeof(struct epoll_event));
+
+	efd = epoll_create(nr);
+	if (efd < 0) {
+		sd_err("failed to create epoll fd");
+		return -1;
+	}
+
+	deferred_event_fd = eventfd(0, EFD_NONBLOCK);
+	if (deferred_event_fd < 0) {
+		sd_err("failed to create eventfd for deferred event"
+		       " register/unregister: %m");
+		return -1;
+	}
+
+	ret = register_event_prio(deferred_event_fd, deferred_event_handler,
+				  NULL, EVENT_PRIO_MAX);
+	if (ret < 0) {
+		sd_err("failed to register deferred_event_handler()");
+		return -1;
+	}
+
+	return 0;
+}
-- 
1.8.3.2