[sheepdog] [PATCH experimental 1/2] stop using timerfd and signalfd

Hitoshi Mitake mitake.hitoshi at lab.ntt.co.jp
Mon Apr 15 07:07:43 CEST 2013


From: Hitoshi Mitake <mitake.hitoshi at gmail.com>

Current DynamoRIO doesn't support timerfd and signalfd, so this patch
let sheepdog stop using them. This is a temporal, adhoc stuff.

Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
---
 collie/Makefile.am    |    2 +-
 include/event.h       |    2 +
 lib/event.c           |  109 ++++++++++++++++++++++++++++++++++++++++++++-----
 sheep/Makefile.am     |    2 +-
 sheep/cluster/local.c |   35 ++++++++++-----
 sheep/sheep.c         |   35 +++++++++++-----
 shepherd/Makefile.am  |    2 +-
 7 files changed, 151 insertions(+), 36 deletions(-)

diff --git a/collie/Makefile.am b/collie/Makefile.am
index 386aa49..4b253f2 100644
--- a/collie/Makefile.am
+++ b/collie/Makefile.am
@@ -30,7 +30,7 @@ collie_SOURCES          += debug.c
 override CFLAGS         := $(subst -pg -gstabs,,$(CFLAGS))
 endif
 
-collie_LDADD	  	= ../lib/libsheepdog.a
+collie_LDADD	  	= ../lib/libsheepdog.a -lrt
 collie_DEPENDENCIES	= ../lib/libsheepdog.a
 
 noinst_HEADERS		= treeview.h collie.h
diff --git a/include/event.h b/include/event.h
index 65277fa..de401ff 100644
--- a/include/event.h
+++ b/include/event.h
@@ -32,4 +32,6 @@ static inline int register_event(int fd, event_handler_t h, void *data)
 return register_event_prio(fd, h, data, EVENT_PRIO_DEFAULT);
 }
 
+void init_timer(void);
+
 #endif
diff --git a/lib/event.c b/lib/event.c
index 5b15925..68ec5c3 100644
--- a/lib/event.c
+++ b/lib/event.c
@@ -14,7 +14,8 @@
 #include <string.h>
 #include <unistd.h>
 #include <sys/epoll.h>
-#include <sys/timerfd.h>
+#include <signal.h>
+#include <time.h>
 
 #include "list.h"
 #include "util.h"
@@ -26,42 +27,128 @@ static LIST_HEAD(events_list);
 
 #define TICK 1
 
+struct timer_desc {
+	int pipe[2];
+	unsigned int msec;
+
+	timer_t timerid;
+	struct timer *t;
+
+	struct list_head list;
+};
+
+static LIST_HEAD(timer_list);
+
 static void timer_handler(int fd, int events, void *data)
 {
-	struct timer *t = data;
+	struct timer_desc *desc = data;
+	struct timer *t = desc->t;
 	uint64_t val;
 
+	assert(fd == desc->pipe[0]);
+
 	if (read(fd, &val, sizeof(val)) < 0)
 		return;
 
 	t->callback(t->data);
 
+	timer_delete(desc->timerid);
+	list_del(&desc->list);
+
 	unregister_event(fd);
-	close(fd);
+	close(desc->pipe[0]);
+	close(desc->pipe[1]);
+
+	free(desc);
+}
+
+static void insert_timer_desc(struct timer_desc *t)
+{
+	struct timer_desc *p;
+	struct list_head *pred;
+
+	pred = NULL;
+
+	list_for_each_entry(p, &timer_list, list) {
+		if (p->msec < t->msec)
+			continue;
+
+		pred = &p->list;
+		break;
+	}
+
+	if (!pred)
+		pred = &timer_list;
+
+	list_add(&t->list, pred);
 }
 
 void add_timer(struct timer *t, unsigned int mseconds)
 {
+	int ret;
+	struct timer_desc *desc;
 	struct itimerspec it;
-	int tfd;
 
-	tfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK);
-	if (tfd < 0) {
-		sd_eprintf("timerfd_create: %m");
-		return;
+	desc = xzalloc(sizeof(*desc));
+	desc->msec = mseconds;
+	INIT_LIST_HEAD(&desc->list);
+
+	ret = timer_create(CLOCK_MONOTONIC, NULL, &desc->timerid);
+	if (ret < 0) {
+		sd_eprintf("timer_create: %m");
+		goto free_timer_desc;
 	}
 
 	memset(&it, 0, sizeof(it));
 	it.it_value.tv_sec = mseconds / 1000;
 	it.it_value.tv_nsec = (mseconds % 1000) * 1000000;
 
-	if (timerfd_settime(tfd, 0, &it, NULL) < 0) {
+	if (timer_settime(desc->timerid, 0, &it, NULL) < 0) {
 		sd_eprintf("timerfd_settime: %m");
-		return;
+		goto del_timer;
+	}
+
+	ret = pipe(desc->pipe);
+	if (ret < 0) {
+		sd_eprintf("pipe: %m");
+		goto del_timer;
 	}
 
-	if (register_event(tfd, timer_handler, t) < 0)
+	if (register_event(desc->pipe[0], timer_handler, desc) < 0) {
 		sd_eprintf("failed to register timer fd");
+		goto close_pipe;
+	}
+
+	desc->t = t;
+	insert_timer_desc(desc);
+
+	return;
+
+close_pipe:
+	close(desc->pipe[0]);
+	close(desc->pipe[1]);
+del_timer:
+	timer_delete(desc->timerid);
+free_timer_desc:
+	free(desc);
+}
+
+static void sigalrm_handler(int signum)
+{
+	struct timer_desc *t;
+	uint64_t val = 0;
+
+	assert(signum == SIGALRM);
+	assert(!list_empty(&timer_list));
+
+	t = list_first_entry(&timer_list, struct timer_desc, list);
+	write(t->pipe[1], &val, sizeof(val));
+}
+
+void init_timer(void)
+{
+	if (install_sighandler(SIGALRM, sigalrm_handler, false) < 0)
+		panic("install_sighandler() failed: %m");
 }
 
 struct event_info {
diff --git a/sheep/Makefile.am b/sheep/Makefile.am
index 32bc1c0..84b2766 100644
--- a/sheep/Makefile.am
+++ b/sheep/Makefile.am
@@ -43,7 +43,7 @@ if BUILD_TRACE
 sheep_SOURCES		+= trace/trace.c trace/mcount.S trace/stabs.c trace/graph.c
 endif
 
-sheep_LDADD	  	= ../lib/libsheepdog.a -lpthread -lm\
+sheep_LDADD	  	= ../lib/libsheepdog.a -lpthread -lm -lrt\
 			  $(libcpg_LIBS) $(libcfg_LIBS) $(libacrd_LIBS) $(LIBS)
 sheep_DEPENDENCIES	= ../lib/libsheepdog.a
 
diff --git a/sheep/cluster/local.c b/sheep/cluster/local.c
index 142221f..ffcc9b5 100644
--- a/sheep/cluster/local.c
+++ b/sheep/cluster/local.c
@@ -28,7 +28,6 @@
 
 static const char *shmfile = "/tmp/sheepdog_shm";
 static int shmfd;
-static int sigfd;
 static int block_event_pos;
 static int nonblock_event_pos;
 static struct local_node this_node;
@@ -503,7 +502,7 @@ static void local_handler(int listen_fd, int events, void *data)
 
 	sd_dprintf("read siginfo");
 
-	ret = read(sigfd, &siginfo, sizeof(siginfo));
+	ret = read(listen_fd, &siginfo, sizeof(siginfo));
 	assert(ret == sizeof(siginfo));
 
 	shm_queue_lock();
@@ -523,9 +522,23 @@ static int local_get_local_addr(uint8_t *myaddr)
 	return 0;
 }
 
+static int sigusr1_pipe[2];
+
+static void sigusr1_handler(int signum)
+{
+	int ret;
+	struct signalfd_siginfo siginfo;
+
+	assert(signum == SIGUSR1);
+
+	siginfo.ssi_signo = SIGUSR1;
+	ret = write(sigusr1_pipe[1], &siginfo, sizeof(siginfo));
+	if (ret != sizeof(siginfo))
+		panic("write: %m");
+}
+
 static int local_init(const char *option)
 {
-	sigset_t mask;
 	int ret;
 	static struct timer t = {
 		.callback = check_pids,
@@ -537,19 +550,17 @@ static int local_init(const char *option)
 
 	shm_queue_init();
 
-	sigemptyset(&mask);
-	sigaddset(&mask, SIGUSR1);
-	sigprocmask(SIG_BLOCK, &mask, NULL);
+	ret = install_sighandler(SIGUSR1, sigusr1_handler, false);
+	if (ret < 0)
+		panic("install_sighandler: %m");
 
-	sigfd = signalfd(-1, &mask, SFD_NONBLOCK);
-	if (sigfd < 0) {
-		sd_eprintf("failed to create a signal fd: %m");
-		return -1;
-	}
+	ret = pipe(sigusr1_pipe);
+	if (ret < 0)
+		panic("pipe: %m");
 
 	add_timer(&t, PROCESS_CHECK_INTERVAL);
 
-	ret = register_event(sigfd, local_handler, NULL);
+	ret = register_event(sigusr1_pipe[0], local_handler, NULL);
 	if (ret) {
 		sd_eprintf("failed to register local event handler (%d)", ret);
 		return -1;
diff --git a/sheep/sheep.c b/sheep/sheep.c
index cda8493..9e6de4d 100644
--- a/sheep/sheep.c
+++ b/sheep/sheep.c
@@ -145,26 +145,39 @@ static void signal_handler(int listen_fd, int events, void *data)
 	}
 }
 
+static int sigterm_pipe[2];
+
+static void sigterm_handler(int signum)
+{
+	int ret;
+	struct signalfd_siginfo siginfo;
+
+	assert(signum == SIGTERM);
+
+	memset(&siginfo, 0, sizeof(siginfo));
+	siginfo.ssi_signo = SIGTERM;
+	ret = write(sigterm_pipe[1], &siginfo, sizeof(siginfo));
+	if (ret != sizeof(siginfo))
+		panic("handling SIGTERM failed: %m");
+}
+
 static int init_signal(void)
 {
-	sigset_t mask;
 	int ret;
 
 	ret = trace_init_signal();
 	if (ret)
 		return ret;
 
-	sigemptyset(&mask);
-	sigaddset(&mask, SIGTERM);
-	sigprocmask(SIG_BLOCK, &mask, NULL);
+	ret = install_sighandler(SIGTERM, sigterm_handler, true);
+	if (ret < 0)
+		panic("install_sighandler: %m");
 
-	sigfd = signalfd(-1, &mask, SFD_NONBLOCK);
-	if (sigfd < 0) {
-		sd_eprintf("failed to create a signal fd: %m");
-		return -1;
-	}
+	ret = pipe(sigterm_pipe);
+	if (ret < 0)
+		panic("pipe: %m");
 
-	ret = register_event(sigfd, signal_handler, NULL);
+	ret = register_event(sigterm_pipe[0], signal_handler, NULL);
 	if (ret) {
 		sd_eprintf("failed to register signal handler (%d)", ret);
 		return -1;
@@ -556,6 +569,8 @@ int main(int argc, char **argv)
 	sheep_info.port = port;
 	early_log_init(log_format, &sheep_info);
 
+	init_timer();
+
 	if (nr_vnodes == 0) {
 		sys->gateway_only = true;
 		sys->disk_space = 0;
diff --git a/shepherd/Makefile.am b/shepherd/Makefile.am
index f7fd998..001ef28 100644
--- a/shepherd/Makefile.am
+++ b/shepherd/Makefile.am
@@ -25,7 +25,7 @@ sbin_PROGRAMS		= shepherd
 
 shepherd_SOURCES		= shepherd.c
 
-shepherd_LDADD	  	= ../lib/libsheepdog.a
+shepherd_LDADD	  	= ../lib/libsheepdog.a -lrt
 shepherd_DEPENDENCIES	= ../lib/libsheepdog.a
 
 EXTRA_DIST		=
-- 
1.7.2.5




More information about the sheepdog mailing list