[sheepdog] [PATCH v2] tests: add a DynamoRIO client for testing the jounaling mechanism

Hitoshi Mitake mitake.hitoshi at gmail.com
Sun Jun 30 20:36:03 CEST 2013


From: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>

This patch adds a DynamoRIO (often called DR) client for testing the
jounaling mechanism. Because of its nature, the recoverying path is
the most important and hard to test part of the journaling
mechanism. They need to be tested well.

But testing targetted recovery paths with traditional tests/ stuff is
hard because:
1. killing sheeps with kill commands doesn't take into account the
   internal state
2. inserting exit()s into sheep manually is a painful work

So this patch implements a fault injection mechanism with DR. DR
provides rich functionalities of transparent dynamic
instrumentation. One of the functionalities makes inserting function
calls before and after system calls possible. With this mechanism, the
fault injection mechanism lets sheep exit at suitable timings for
testing recovery paths of the journaling.

How to use:
0. preparation
   $ cd
   $ svn checkout http://dynamorio.googlecode.com/svn/trunk/ dynamorio
   $ cd dynamorio
   $ mkdir build
   $ cd build
   $ cmake ..
   $ make

(This patch assumes the source code of DR is store in $HOME/dynamorio,
and the build is done in $HOME/dynamorio/build)

1. build the DR client
   $ cd tests/fi/journaling/
   $ cmake .
   $ make

2. run tests with preset scenarios
   $ ./01.sh 	  # for testing recovery of object store
     		  # after this, actual completion of recovery can be
		    checked via sheep.log

The fault injection implemented with this patch is so slack and not
capable for exhaustive testing. This is only a supoprted care. But I
believe it is useful.

With this patch, I tested the recovery path for object store and
checked it work well.

Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
---

v2: DR now supports signalfd(), so we can use the DR client with
    shepherd (timerfd() is not supported yet, but its only user is the
    local cluster driver).

 tests/fi/.gitignore                |    6 +
 tests/fi/journaling/01.sh          |   21 ++
 tests/fi/journaling/CMakeLists.txt |   10 +
 tests/fi/journaling/journaling.c   |  358 ++++++++++++++++++++++++++++++++++++
 4 files changed, 395 insertions(+), 0 deletions(-)
 create mode 100644 tests/fi/.gitignore
 create mode 100755 tests/fi/journaling/01.sh
 create mode 100644 tests/fi/journaling/CMakeLists.txt
 create mode 100644 tests/fi/journaling/journaling.c

diff --git a/tests/fi/.gitignore b/tests/fi/.gitignore
new file mode 100644
index 0000000..09c1215
--- /dev/null
+++ b/tests/fi/.gitignore
@@ -0,0 +1,6 @@
+*/CMakeCache.txt
+*/CMakeFiles
+*/Makefile
+*/cmake_install.cmake
+*/ldscript
+*/*.so
diff --git a/tests/fi/journaling/01.sh b/tests/fi/journaling/01.sh
new file mode 100755
index 0000000..dd2fb45
--- /dev/null
+++ b/tests/fi/journaling/01.sh
@@ -0,0 +1,21 @@
+#! /bin/bash
+
+# fault injection for testing journaling with object store
+
+sudo killall -KILL sheep
+sudo killall -KILL shepherd
+
+sudo rm -rf /tmp/sheepdog*
+sudo mkdir -p /tmp/sheepdog/0
+
+sudo shepherd
+
+sudo ~/dynamorio/build/bin64/drrun -c libjournaling.so 1 --\
+ sheep -d -c shepherd:127.0.0.1 -p 7000 -j size=64 /tmp/sheepdog/0
+
+sleep 3
+
+collie cluster format -c 1
+collie vdi create test 100M
+
+sudo sheep -d -c shepherd:127.0.0.1 -p 7000 -j size=64 /tmp/sheepdog/0
diff --git a/tests/fi/journaling/CMakeLists.txt b/tests/fi/journaling/CMakeLists.txt
new file mode 100644
index 0000000..fcfabbd
--- /dev/null
+++ b/tests/fi/journaling/CMakeLists.txt
@@ -0,0 +1,10 @@
+cmake_minimum_required(VERSION 2.8)
+
+SET(DynamoRIO_DIR "~/dynamorio/exports/cmake")
+find_package(DynamoRIO)
+
+add_library(journaling SHARED journaling.c)
+configure_DynamoRIO_client(journaling)
+
+use_DynamoRIO_extension(journaling drwrap)
+use_DynamoRIO_extension(journaling drcontainers)
diff --git a/tests/fi/journaling/journaling.c b/tests/fi/journaling/journaling.c
new file mode 100644
index 0000000..4c604df
--- /dev/null
+++ b/tests/fi/journaling/journaling.c
@@ -0,0 +1,358 @@
+/*
+ * Copyright (C) 2013 Nippon Telegraph and Telephone Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * DynamoRIO client for fault injection which can be used to test
+ * journaling mechanism
+ */
+
+#include "dr_api.h"
+#include "drwrap.h"
+#include "drmgr.h"
+#include "hashtable.h"
+#include "dr_tools.h"
+
+#include <string.h>
+#include <syscall.h>
+
+#include <stdint.h>
+
+struct journal_descriptor {
+	uint32_t magic;
+	uint16_t flag;
+	uint16_t reserved;
+	union {
+		uint32_t epoch;
+		uint64_t oid;
+	};
+	uint64_t offset;
+	uint64_t size;
+	uint8_t create;
+	uint8_t pad[475];
+} __packed;
+
+#define JOURNAL_DESC_MAGIC 0xfee1900d
+
+#define JF_STORE 0
+#define JF_REMOVE_OBJ 2
+
+enum scenario_id {
+	SID_UNDEF = -1,
+
+	SID_DO_NOTHING = 0,
+	SID_DEATH_AFTER_STORE,
+};
+
+enum scenario_id sid = SID_UNDEF;
+
+static int tls_idx;
+static file_t log_file = INVALID_FILE;
+
+static int jfile_fds[2];
+
+#define fi_printf(fmt, args...) do {					\
+		if (log_file == INVALID_FILE)				\
+			dr_printf("%s(%d), " fmt,			\
+				__func__, __LINE__, ## args);		\
+		else							\
+			dr_fprintf(log_file, "%s(%d), " fmt,		\
+				__func__, __LINE__, ## args);		\
+	} while (0)
+
+#define die(fmt, args...) do {						\
+		fi_printf("FATAL %s(%d), " fmt,				\
+			__func__, __LINE__, ## args);			\
+	} while (0)
+
+static void *xmalloc(size_t size)
+{
+	void *ret;
+
+	ret = __wrap_malloc(size);
+	if (!ret)
+		die("allocating memory with __wrap_malloc() failed\n");
+
+	return ret;
+}
+
+static void *xzalloc(size_t size)
+{
+	void *ret;
+
+	ret = xmalloc(size);
+	memset(ret, 0, size);
+
+	return ret;
+}
+
+static void *xcalloc(size_t size, size_t nmnb)
+{
+	void *ret;
+	size_t length = size * nmnb;
+
+	ret = __wrap_malloc(length);
+	if (!ret)
+		die("allocating memory with __wrap_malloc() failed\n");
+	memset(ret, 0, length);
+
+	return ret;
+}
+
+static void xfree(void *ptr)
+{
+	__wrap_free(ptr);
+}
+
+enum thread_state {
+	THREAD_STATE_DEFAULT,
+
+	THREAD_STATE_OPENING_JFILE_0,
+	THREAD_STATE_OPENING_JFILE_1,
+
+	THREAD_STATE_WRITING_JFILE,
+};
+
+enum pwrite_state {
+	PWRITE_WRITING_STORE,
+};
+
+struct per_thread_journal_state {
+	enum thread_state state;
+	int using_fd;
+
+	enum pwrite_state pwrite_state;
+};
+
+static void thread_init_event(void *drcontext)
+{
+	struct per_thread_journal_state *new_jstate;
+
+	new_jstate = xzalloc(sizeof(*new_jstate));
+
+	drmgr_set_tls_field(drcontext, tls_idx, new_jstate);
+}
+
+static void thread_exit_event(void *drcontext)
+{
+	struct per_thread_journal_state *jstate;
+
+	jstate = (struct per_thread_journal_state *)
+		drmgr_get_tls_field(drcontext, tls_idx);
+	xfree(jstate);
+}
+
+static void pre_open(void *drcontext)
+{
+	const char *path;
+	struct per_thread_journal_state *jstate;
+
+	jstate = (struct per_thread_journal_state *)
+		drmgr_get_tls_field(drcontext, tls_idx);
+
+	path = (const char *)dr_syscall_get_param(drcontext, 0);
+
+	if (strstr(path, "journal_file0")) {
+		fi_printf("journal_file0 is opened\n");
+		DR_ASSERT(jstate->state == THREAD_STATE_DEFAULT);
+		jstate->state = THREAD_STATE_OPENING_JFILE_0;
+	} else if (strstr(path, "journal_file1")) {
+		fi_printf("journal_file1 is opened\n");
+		DR_ASSERT(jstate->state == THREAD_STATE_DEFAULT);
+		jstate->state = THREAD_STATE_OPENING_JFILE_1;
+	}
+}
+
+static void pre_close(void *drcontext)
+{
+}
+
+static void pre_read(void *drcontext)
+{
+}
+
+static void pre_write(void *drcontext)
+{
+}
+
+static void pre_pwrite(void *drcontext)
+{
+	int fd;
+	struct per_thread_journal_state *jstate;
+	struct journal_descriptor *jd;
+
+	fd = (int)dr_syscall_get_param(drcontext, 0);
+	if (fd != jfile_fds[0] && fd != jfile_fds[1])
+		return;
+
+	jstate = (struct per_thread_journal_state *)
+		drmgr_get_tls_field(drcontext, tls_idx);
+
+	fi_printf("writing journal\n");
+	jstate->using_fd = fd;
+	jstate->state = THREAD_STATE_WRITING_JFILE;
+
+	jd = (struct journal_descriptor *)dr_syscall_get_param(drcontext, 1);
+	DR_ASSERT(jd->magic == JOURNAL_DESC_MAGIC);
+	if (jd->flag == JF_STORE)
+		jstate->pwrite_state = PWRITE_WRITING_STORE;
+	else if (jd->flag == JF_REMOVE_OBJ)
+		fi_printf("FIXME: testing object removal is not supported yet");
+	else
+		die("unknown journal flag: %d\n", jd->flag);
+}
+
+static bool pre_syscall(void *drcontext, int sysnum)
+{
+	switch (sysnum) {
+	case SYS_open:
+		pre_open(drcontext);
+		break;
+	case SYS_close:
+		pre_close(drcontext);
+		break;
+	case SYS_read:
+		pre_read(drcontext);
+		break;
+	case SYS_write:
+		pre_write(drcontext);
+		break;
+	case SYS_pwrite64:
+		pre_pwrite(drcontext);
+		break;
+	default:
+		break;
+	}
+
+	return true;
+}
+
+static void post_open(void *drcontext)
+{
+	int fd;
+	struct per_thread_journal_state *jstate;
+
+	jstate = (struct per_thread_journal_state *)
+		drmgr_get_tls_field(drcontext, tls_idx);
+
+	if (jstate->state == THREAD_STATE_DEFAULT)
+		return;
+
+	fd = (int)dr_syscall_get_result(drcontext);
+
+	if (jstate->state == THREAD_STATE_OPENING_JFILE_0) {
+		fi_printf("fd of jfile0: %d\n", fd);
+		jfile_fds[0] = fd;
+	} else if (jstate->state == THREAD_STATE_OPENING_JFILE_1) {
+		fi_printf("fd of jfile1: %d\n", fd);
+		jfile_fds[1] = fd;
+	}
+
+	jstate->state = THREAD_STATE_DEFAULT;
+}
+
+static void post_close(void *drcontext)
+{
+}
+
+static void post_read(void *drcontext)
+{
+}
+
+static void post_write(void *drcontext)
+{
+}
+
+static void post_pwrite(void *drcontext)
+{
+	int fd;
+	struct per_thread_journal_state *jstate;
+
+	jstate = (struct per_thread_journal_state *)
+		drmgr_get_tls_field(drcontext, tls_idx);
+
+	if (jstate->state != THREAD_STATE_WRITING_JFILE)
+		return;
+
+	fd = jstate->using_fd;
+	DR_ASSERT(fd == jfile_fds[0] || fd == jfile_fds[1]);
+
+	switch (sid) {
+	case SID_DEATH_AFTER_STORE:
+		if (jstate->pwrite_state != PWRITE_WRITING_STORE)
+			return;
+
+		fi_printf("scenario is death after writing normal store,"
+			" exiting\n");
+		exit(1);
+		break;
+	default:
+		die("invalid SID: %d\n", sid);
+		break;
+	}
+}
+
+static void post_syscall(void *drcontext, int sysnum)
+{
+	switch (sysnum) {
+	case SYS_open:
+		post_open(drcontext);
+		break;
+	case SYS_close:
+		post_close(drcontext);
+		break;
+	case SYS_read:
+		post_read(drcontext);
+		break;
+	case SYS_write:
+		post_write(drcontext);
+		break;
+	case SYS_pwrite64:
+		post_pwrite(drcontext);
+		break;
+	}
+}
+
+static bool pre_syscall_filter(void *drcontext, int sysnum)
+{
+	return true;
+}
+
+static bool post_syscall_filter(void *drcontext, int sysnum)
+{
+	return true;
+}
+
+DR_EXPORT void dr_init(client_id_t id)
+{
+	const char *option;
+
+	option = dr_get_options(id);
+	fi_printf("the passed option to this client: %s\n", option);
+	sid = atoi(option);
+	fi_printf("sid: %d\n", sid);
+
+	log_file = dr_open_file("journaling_fi.log", DR_FILE_WRITE_APPEND);
+	if (log_file == INVALID_FILE)
+		die("opening journaling_fi.log failed\n");
+
+	dr_register_filter_syscall_event(pre_syscall_filter);
+	drmgr_init();
+
+	tls_idx = drmgr_register_tls_field();
+	drmgr_register_pre_syscall_event(pre_syscall);
+	drmgr_register_post_syscall_event(post_syscall);
+
+	drmgr_register_thread_init_event(thread_init_event);
+	drmgr_register_thread_exit_event(thread_exit_event);
+
+	jfile_fds[0] = -1;
+	jfile_fds[1] = -1;
+}
-- 
1.7.5.1




More information about the sheepdog mailing list