[Sheepdog] [PATCH 2/2] Use RCU to avoid race on current_vnode_info
Yunkai Zhang
yunkai.me at gmail.com
Sat May 5 23:36:56 CEST 2012
From: Yunkai Zhang <qiushu.zyk at taobao.com>
We have discussed this issue in maillist:
http://lists.wpkg.org/pipermail/sheepdog/2012-May/003315.html
This patch depends on a third-party usersapce RCU library, we
can get it from here: http://lttng.org/urcu.
Signed-off-by: Yunkai Zhang <yunkai.me at gmail.com>
---
configure.ac | 3 ++-
sheep/Makefile.am | 7 +++++--
sheep/group.c | 39 ++++++++++++++++++++++++++++++---------
sheep/sheep.c | 5 +++++
sheep/work.c | 3 +++
5 files changed, 45 insertions(+), 12 deletions(-)
diff --git a/configure.ac b/configure.ac
index 7a5eedf..b7af48f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -77,6 +77,7 @@ AM_MISSING_PROG(AUTOM4TE, autom4te, $missing_dir)
# Checks for libraries.
AC_CHECK_LIB([socket], [socket])
+PKG_CHECK_MODULES([liburcu],[liburcu])
# Checks for header files.
AC_FUNC_ALLOCA
@@ -86,7 +87,7 @@ AC_HEADER_SYS_WAIT
AC_CHECK_HEADERS([arpa/inet.h fcntl.h limits.h netdb.h netinet/in.h stdint.h \
stdlib.h string.h sys/ioctl.h sys/param.h sys/socket.h \
sys/time.h syslog.h unistd.h sys/types.h getopt.h malloc.h \
- sys/sockio.h utmpx.h])
+ sys/sockio.h utmpx.h urcu.h])
# Checks for typedefs, structures, and compiler characteristics.
AC_C_CONST
diff --git a/sheep/Makefile.am b/sheep/Makefile.am
index 7448ae1..bca365c 100644
--- a/sheep/Makefile.am
+++ b/sheep/Makefile.am
@@ -20,7 +20,8 @@ MAINTAINERCLEANFILES = Makefile.in
AM_CFLAGS =
INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include \
- $(libcpg_CFLAGS) $(libcfg_CFLAGS) $(libacrd_CFLAGS)
+ $(libcpg_CFLAGS) $(libcfg_CFLAGS) $(libacrd_CFLAGS) \
+ $(liburcu_CFLAGS)
sbin_PROGRAMS = sheep
@@ -47,7 +48,9 @@ sheep_SOURCES += trace/trace.c trace/mcount.S trace/stabs.c trace/graph.c
endif
sheep_LDADD = ../lib/libsheepdog.a -lpthread \
- $(libcpg_LIBS) $(libcfg_LIBS) $(libacrd_LIBS) $(LIBS)
+ $(libcpg_LIBS) $(libcfg_LIBS) $(libacrd_LIBS) $(LIBS) \
+ $(liburcu_LIBS)
+
sheep_DEPENDENCIES = ../lib/libsheepdog.a
diff --git a/sheep/group.c b/sheep/group.c
index c7fd387..15d5866 100644
--- a/sheep/group.c
+++ b/sheep/group.c
@@ -13,6 +13,7 @@
#include <stdlib.h>
#include <unistd.h>
#include <netdb.h>
+#include <urcu.h>
#include <arpa/inet.h>
#include <sys/time.h>
#include <sys/epoll.h>
@@ -36,7 +37,7 @@ struct vnode_info {
struct sd_vnode entries[SD_MAX_VNODES];
int nr_vnodes;
int nr_zones;
- int refcnt;
+ struct rcu_head rcu;
};
struct join_message {
@@ -155,15 +156,27 @@ int get_max_nr_copies_from(struct sd_node *nodes, int nr_nodes)
struct vnode_info *get_vnode_info(void)
{
+ struct vnode_info *vnodes, *p;
+
assert(current_vnode_info);
- current_vnode_info->refcnt++;
- return current_vnode_info;
+
+ vnodes = zalloc(sizeof(*vnodes));
+ if (!vnodes)
+ panic("failed to allocate memory\n");
+
+ rcu_read_lock();
+
+ p = rcu_dereference(current_vnode_info);
+ memcpy(vnodes, p, sizeof(*p));
+
+ rcu_read_unlock();
+
+ return vnodes;
}
void put_vnode_info(struct vnode_info *vnodes)
{
- if (vnodes && --vnodes->refcnt == 0)
- free(vnodes);
+ free(vnodes);
}
struct sd_vnode *oid_to_vnode(struct vnode_info *vnode_info, uint64_t oid,
@@ -176,10 +189,16 @@ struct sd_vnode *oid_to_vnode(struct vnode_info *vnode_info, uint64_t oid,
return &vnode_info->entries[n];
}
+static void vnode_info_reclaim(struct rcu_head *head)
+{
+ struct vnode_info *vnodes;
+ vnodes = container_of(head, struct vnode_info, rcu);
+ free(vnodes);
+}
static int update_vnode_info(void)
{
- struct vnode_info *vnode_info;
+ struct vnode_info *vnode_info, *old_vnode_info;
vnode_info = zalloc(sizeof(*vnode_info));
if (!vnode_info) {
@@ -190,10 +209,12 @@ static int update_vnode_info(void)
vnode_info->nr_vnodes = nodes_to_vnodes(sys->nodes, sys->nr_nodes,
vnode_info->entries);
vnode_info->nr_zones = get_zones_nr_from(sys->nodes, sys->nr_nodes);
- vnode_info->refcnt = 1;
- put_vnode_info(current_vnode_info);
- current_vnode_info = vnode_info;
+ old_vnode_info = current_vnode_info;
+ rcu_assign_pointer(current_vnode_info, vnode_info);
+ if (old_vnode_info)
+ call_rcu(&old_vnode_info->rcu, vnode_info_reclaim);
+
return 0;
}
diff --git a/sheep/sheep.c b/sheep/sheep.c
index 6645e8d..28b6311 100644
--- a/sheep/sheep.c
+++ b/sheep/sheep.c
@@ -17,6 +17,7 @@
#include <unistd.h>
#include <time.h>
#include <signal.h>
+#include <urcu.h>
#include <linux/limits.h>
#include <sys/syslog.h>
@@ -246,6 +247,9 @@ int main(int argc, char **argv)
exit(1);
}
+ rcu_init();
+ rcu_register_thread();
+
sys->event_wqueue = init_work_queue(1);
sys->gateway_wqueue = init_work_queue(nr_gateway_worker);
sys->io_wqueue = init_work_queue(nr_io_worker);
@@ -274,5 +278,6 @@ int main(int argc, char **argv)
log_close();
+ rcu_unregister_thread();
return 0;
}
diff --git a/sheep/work.c b/sheep/work.c
index 8564cb2..84ce3c2 100644
--- a/sheep/work.c
+++ b/sheep/work.c
@@ -19,6 +19,7 @@
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
+#include <urcu.h>
#include <stdlib.h>
#include <syscall.h>
#include <sys/types.h>
@@ -172,6 +173,7 @@ static void *worker_routine(void *arg)
/* started this thread */
pthread_mutex_unlock(&wi->startup_lock);
+ rcu_register_thread();
while (!(wi->q.wq_state & WQ_DEAD)) {
pthread_mutex_lock(&wi->pending_lock);
@@ -200,6 +202,7 @@ retest:
eventfd_write(efd, value);
}
+ rcu_unregister_thread();
pthread_exit(NULL);
}
--
1.7.7.6
More information about the sheepdog
mailing list