[sheepdog] [PATCH 3/6] lib: implement wrapper functions for handling EMFILE gracefully

[Hitoshi Mitake]

This patch adds wrapper functions of fd creaters for handling EMFILE
gracefully. When they face EMFILE, they reap unused sockfd and retry
creating new fds.

This patch also modifies Makefile of shepherd. Because this patch
creates a new dependency in libsheepdog, shepherd must linked with
pthread.

Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
---
 include/util.h       |  14 +++++++
 lib/util.c           | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++
 shepherd/Makefile.am |   2 +-
 3 files changed, 119 insertions(+), 1 deletion(-)

diff --git a/include/util.h b/include/util.h
index ff1f0c7..729037d 100644
--- a/include/util.h
+++ b/include/util.h
@@ -13,6 +13,9 @@
 #include <sys/eventfd.h>
 #include <pthread.h>
 #include <errno.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <fcntl.h>
 
 #include "logger.h"
 #include "list.h"
@@ -344,4 +347,15 @@ static inline bool is_stdout_console(void)
 extern mode_t sd_def_fmode;
 extern mode_t sd_def_dmode;
 
+/*
+ * The below x- prefixed wrappers are for handling EMFILE gracefully.
+ * They reap fds of sockfd cache and retry when they face EMFILE.
+ */
+int xsocket(int domain, int type, int protocol);
+int xaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen);
+int xeventfd(int initval, int flags);
+int xopen(const char *pathname, int flags, ...);
+int xtimerfd_create(clockid_t clock_id, int flags);
+int xepoll_create(int size);
+
 #endif
diff --git a/lib/util.c b/lib/util.c
index 9781fd8..b88f297 100644
--- a/lib/util.c
+++ b/lib/util.c
@@ -22,8 +22,12 @@
 #include <signal.h>
 #include <sys/xattr.h>
 #include <fcntl.h>
+#include <stdarg.h>
+#include <sys/timerfd.h>
+#include <sys/epoll.h>
 
 #include "util.h"
+#include "sockfd_cache.h"
 
 mode_t sd_def_dmode = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP;
 mode_t sd_def_fmode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP;
@@ -735,3 +739,103 @@ void list_sort(void *priv, struct list_head *head,
 
 	merge_and_restore_back_links(priv, cmp, head, part[max_lev], list);
 }
+
+int xsocket(int domain, int type, int protocol)
+{
+	int fd;
+
+retry:
+	fd = socket(domain, type, protocol);
+	if (fd < 0) {
+		if (errno == EINTR)
+			goto retry;
+
+		if (errno == EMFILE && sockfd_shrink())
+			goto retry;
+	}
+
+	return fd;
+}
+
+int xaccept(int sockfd, struct sockaddr *addr, socklen_t *addrlen)
+{
+	int fd;
+
+retry:
+	fd = accept(sockfd, addr, addrlen);
+	if (fd < 0) {
+		if (errno == EINTR)
+			goto retry;
+
+		if (errno == EMFILE && sockfd_shrink())
+			goto retry;
+	}
+
+	return fd;
+}
+
+int xeventfd(int initval, int flags)
+{
+	int fd;
+
+retry:
+	fd = eventfd(initval, flags);
+	if (fd < 0 && errno == EMFILE) {
+		if (sockfd_shrink())
+			goto retry;
+	}
+
+	return fd;
+}
+
+int xopen(const char *pathname, int flags, ...)
+{
+	mode_t mode = 0;
+	va_list ap;
+	int fd;
+
+	va_start(ap, flags);
+	if (flags & O_CREAT)
+		mode = va_arg(ap, mode_t);
+
+retry:
+	fd = open(pathname, flags, mode);
+	if (fd < 0) {
+		if (errno == EINTR)
+			goto retry;
+
+		if (errno == EMFILE && sockfd_shrink())
+			goto retry;
+	}
+
+	va_end(ap);
+	return fd;
+}
+
+int xtimerfd_create(clockid_t clock_id, int flags)
+{
+	int fd;
+
+retry:
+	fd = timerfd_create(clock_id, flags);
+	if (fd < 0) {
+		if (errno == EMFILE && sockfd_shrink())
+			goto retry;
+	}
+
+	return fd;
+}
+
+int xepoll_create(int size)
+{
+	int fd;
+
+retry:
+	fd = epoll_create(size);
+	if (fd < 0) {
+		if (errno == EMFILE && sockfd_shrink())
+			goto retry;
+	}
+
+	return fd;
+}
diff --git a/shepherd/Makefile.am b/shepherd/Makefile.am
index f7fd998..01ccfa9 100644
--- a/shepherd/Makefile.am
+++ b/shepherd/Makefile.am
@@ -25,7 +25,7 @@ sbin_PROGRAMS		= shepherd
 
 shepherd_SOURCES		= shepherd.c
 
-shepherd_LDADD	  	= ../lib/libsheepdog.a
+shepherd_LDADD	  	= ../lib/libsheepdog.a -lpthread
 shepherd_DEPENDENCIES	= ../lib/libsheepdog.a
 
 EXTRA_DIST		=
-- 
1.8.1.2