[Sheepdog] [PATCH 2/2] Use RCU to avoid race on current_vnode_info
Yunkai Zhang
yunkai.me at gmail.com
Sun May 6 00:18:39 CEST 2012
Please ignore this patch, I will give v2 later.
On Sun, May 6, 2012 at 5:36 AM, Yunkai Zhang <yunkai.me at gmail.com> wrote:
> From: Yunkai Zhang <qiushu.zyk at taobao.com>
>
> We have discussed this issue in maillist:
> http://lists.wpkg.org/pipermail/sheepdog/2012-May/003315.html
>
> This patch depends on a third-party usersapce RCU library, we
> can get it from here: http://lttng.org/urcu.
>
> Signed-off-by: Yunkai Zhang <yunkai.me at gmail.com>
> ---
> configure.ac | 3 ++-
> sheep/Makefile.am | 7 +++++--
> sheep/group.c | 39 ++++++++++++++++++++++++++++++---------
> sheep/sheep.c | 5 +++++
> sheep/work.c | 3 +++
> 5 files changed, 45 insertions(+), 12 deletions(-)
>
> diff --git a/configure.ac b/configure.ac
> index 7a5eedf..b7af48f 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -77,6 +77,7 @@ AM_MISSING_PROG(AUTOM4TE, autom4te, $missing_dir)
>
> # Checks for libraries.
> AC_CHECK_LIB([socket], [socket])
> +PKG_CHECK_MODULES([liburcu],[liburcu])
>
> # Checks for header files.
> AC_FUNC_ALLOCA
> @@ -86,7 +87,7 @@ AC_HEADER_SYS_WAIT
> AC_CHECK_HEADERS([arpa/inet.h fcntl.h limits.h netdb.h netinet/in.h stdint.h \
> stdlib.h string.h sys/ioctl.h sys/param.h sys/socket.h \
> sys/time.h syslog.h unistd.h sys/types.h getopt.h malloc.h \
> - sys/sockio.h utmpx.h])
> + sys/sockio.h utmpx.h urcu.h])
>
> # Checks for typedefs, structures, and compiler characteristics.
> AC_C_CONST
> diff --git a/sheep/Makefile.am b/sheep/Makefile.am
> index 7448ae1..bca365c 100644
> --- a/sheep/Makefile.am
> +++ b/sheep/Makefile.am
> @@ -20,7 +20,8 @@ MAINTAINERCLEANFILES = Makefile.in
> AM_CFLAGS =
>
> INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include \
> - $(libcpg_CFLAGS) $(libcfg_CFLAGS) $(libacrd_CFLAGS)
> + $(libcpg_CFLAGS) $(libcfg_CFLAGS) $(libacrd_CFLAGS) \
> + $(liburcu_CFLAGS)
>
> sbin_PROGRAMS = sheep
>
> @@ -47,7 +48,9 @@ sheep_SOURCES += trace/trace.c trace/mcount.S trace/stabs.c trace/graph.c
> endif
>
> sheep_LDADD = ../lib/libsheepdog.a -lpthread \
> - $(libcpg_LIBS) $(libcfg_LIBS) $(libacrd_LIBS) $(LIBS)
> + $(libcpg_LIBS) $(libcfg_LIBS) $(libacrd_LIBS) $(LIBS) \
> + $(liburcu_LIBS)
> +
> sheep_DEPENDENCIES = ../lib/libsheepdog.a
>
>
> diff --git a/sheep/group.c b/sheep/group.c
> index c7fd387..15d5866 100644
> --- a/sheep/group.c
> +++ b/sheep/group.c
> @@ -13,6 +13,7 @@
> #include <stdlib.h>
> #include <unistd.h>
> #include <netdb.h>
> +#include <urcu.h>
> #include <arpa/inet.h>
> #include <sys/time.h>
> #include <sys/epoll.h>
> @@ -36,7 +37,7 @@ struct vnode_info {
> struct sd_vnode entries[SD_MAX_VNODES];
> int nr_vnodes;
> int nr_zones;
> - int refcnt;
> + struct rcu_head rcu;
> };
>
> struct join_message {
> @@ -155,15 +156,27 @@ int get_max_nr_copies_from(struct sd_node *nodes, int nr_nodes)
>
> struct vnode_info *get_vnode_info(void)
> {
> + struct vnode_info *vnodes, *p;
> +
> assert(current_vnode_info);
> - current_vnode_info->refcnt++;
> - return current_vnode_info;
> +
> + vnodes = zalloc(sizeof(*vnodes));
> + if (!vnodes)
> + panic("failed to allocate memory\n");
> +
> + rcu_read_lock();
> +
> + p = rcu_dereference(current_vnode_info);
> + memcpy(vnodes, p, sizeof(*p));
> +
> + rcu_read_unlock();
> +
> + return vnodes;
> }
>
> void put_vnode_info(struct vnode_info *vnodes)
> {
> - if (vnodes && --vnodes->refcnt == 0)
> - free(vnodes);
> + free(vnodes);
> }
>
> struct sd_vnode *oid_to_vnode(struct vnode_info *vnode_info, uint64_t oid,
> @@ -176,10 +189,16 @@ struct sd_vnode *oid_to_vnode(struct vnode_info *vnode_info, uint64_t oid,
> return &vnode_info->entries[n];
> }
>
> +static void vnode_info_reclaim(struct rcu_head *head)
> +{
> + struct vnode_info *vnodes;
> + vnodes = container_of(head, struct vnode_info, rcu);
> + free(vnodes);
> +}
>
> static int update_vnode_info(void)
> {
> - struct vnode_info *vnode_info;
> + struct vnode_info *vnode_info, *old_vnode_info;
>
> vnode_info = zalloc(sizeof(*vnode_info));
> if (!vnode_info) {
> @@ -190,10 +209,12 @@ static int update_vnode_info(void)
> vnode_info->nr_vnodes = nodes_to_vnodes(sys->nodes, sys->nr_nodes,
> vnode_info->entries);
> vnode_info->nr_zones = get_zones_nr_from(sys->nodes, sys->nr_nodes);
> - vnode_info->refcnt = 1;
>
> - put_vnode_info(current_vnode_info);
> - current_vnode_info = vnode_info;
> + old_vnode_info = current_vnode_info;
> + rcu_assign_pointer(current_vnode_info, vnode_info);
> + if (old_vnode_info)
> + call_rcu(&old_vnode_info->rcu, vnode_info_reclaim);
> +
> return 0;
> }
>
> diff --git a/sheep/sheep.c b/sheep/sheep.c
> index 6645e8d..28b6311 100644
> --- a/sheep/sheep.c
> +++ b/sheep/sheep.c
> @@ -17,6 +17,7 @@
> #include <unistd.h>
> #include <time.h>
> #include <signal.h>
> +#include <urcu.h>
> #include <linux/limits.h>
> #include <sys/syslog.h>
>
> @@ -246,6 +247,9 @@ int main(int argc, char **argv)
> exit(1);
> }
>
> + rcu_init();
> + rcu_register_thread();
> +
> sys->event_wqueue = init_work_queue(1);
> sys->gateway_wqueue = init_work_queue(nr_gateway_worker);
> sys->io_wqueue = init_work_queue(nr_io_worker);
> @@ -274,5 +278,6 @@ int main(int argc, char **argv)
>
> log_close();
>
> + rcu_unregister_thread();
> return 0;
> }
> diff --git a/sheep/work.c b/sheep/work.c
> index 8564cb2..84ce3c2 100644
> --- a/sheep/work.c
> +++ b/sheep/work.c
> @@ -19,6 +19,7 @@
> #include <stdio.h>
> #include <unistd.h>
> #include <fcntl.h>
> +#include <urcu.h>
> #include <stdlib.h>
> #include <syscall.h>
> #include <sys/types.h>
> @@ -172,6 +173,7 @@ static void *worker_routine(void *arg)
> /* started this thread */
> pthread_mutex_unlock(&wi->startup_lock);
>
> + rcu_register_thread();
> while (!(wi->q.wq_state & WQ_DEAD)) {
>
> pthread_mutex_lock(&wi->pending_lock);
> @@ -200,6 +202,7 @@ retest:
> eventfd_write(efd, value);
> }
>
> + rcu_unregister_thread();
> pthread_exit(NULL);
> }
>
> --
> 1.7.7.6
>
--
Yunkai Zhang
Work at Taobao
More information about the sheepdog
mailing list