[sheepdog] [PATCH v4 05/10] sheep: store the VDI copies data in disk in case of sheep restarts

[levin li]

From: levin li <xingke.lwp at taobao.com>

If we only store the copies data in memory, then after restart the
copies data is lost, and we can only try to read the inode data to
get the copies data, which is expensive

Signed-off-by: levin li <xingke.lwp at taobao.com>
---
 sheep/sheep_priv.h |    2 +
 sheep/store.c      |   21 +++++++++++
 sheep/vdi.c        |   99 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 119 insertions(+), 3 deletions(-)

diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index 86c4c56..3f763c4 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -182,6 +182,7 @@ extern char *obj_path;
 extern char *mnt_path;
 extern char *jrnl_path;
 extern char *epoch_path;
+extern char *vdi_path;
 extern mode_t def_fmode;
 extern mode_t def_dmode;
 extern struct sd_node joining_nodes[];
@@ -205,6 +206,7 @@ int get_obj_copy_number(uint64_t oid);
 int get_max_copy_number(void);
 int get_req_copy_number(struct request *req);
 int add_vdi_copies(uint32_t vid, int nr_copies);
+int load_vdi_copies(void);
 int vdi_exist(uint32_t vid);
 int add_vdi(struct vdi_iocb *iocb, uint32_t *new_vid);
 
diff --git a/sheep/store.c b/sheep/store.c
index b093916..bebb2c0 100644
--- a/sheep/store.c
+++ b/sheep/store.c
@@ -40,6 +40,7 @@ char *obj_path;
 char *mnt_path;
 char *jrnl_path;
 char *epoch_path;
+char *vdi_path;
 static char *config_path;
 
 mode_t def_dmode = S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP;
@@ -342,6 +343,18 @@ static int init_epoch_path(const char *base_path)
 	return init_path(epoch_path, &new);
 }
 
+#define VDI_PATCH "/vdi/"
+
+static int init_vdi_path(const char *base_path)
+{
+	int new;
+
+	vdi_path = xzalloc(strlen(base_path) + strlen(VDI_PATCH) + 1);
+	sprintf(vdi_path, "%s" VDI_PATCH, base_path);
+
+	return init_path(vdi_path, &new);
+}
+
 static int init_mnt_path(const char *base_path)
 {
 	int ret;
@@ -491,6 +504,10 @@ int init_store(const char *d, int enable_write_cache)
 	if (ret)
 		return ret;
 
+	ret = init_vdi_path(d);
+	if (ret)
+		return ret;
+
 	ret = init_mnt_path(d);
 	if (ret)
 		return ret;
@@ -507,6 +524,10 @@ int init_store(const char *d, int enable_write_cache)
 	if (ret)
 		return ret;
 
+	ret = load_vdi_copies();
+	if (ret)
+		return ret;
+
 	if (!sys->gateway_only) {
 		ret = init_store_driver();
 		if (ret)
diff --git a/sheep/vdi.c b/sheep/vdi.c
index 25c4449..0581ad6 100644
--- a/sheep/vdi.c
+++ b/sheep/vdi.c
@@ -12,6 +12,8 @@
 #include <stdlib.h>
 #include <pthread.h>
 #include <sys/time.h>
+#include <sys/file.h>
+#include <dirent.h>
 
 #include "sheepdog_proto.h"
 #include "sheep_priv.h"
@@ -107,9 +109,58 @@ int get_max_copy_number(void)
 	return uatomic_read(&max_copies);
 }
 
+static int read_vdi_copies(uint32_t vid, unsigned int *nr_copies)
+{
+	char path[PATH_MAX];
+	int fd, flags = O_RDONLY, ret;
+
+	sprintf(path, "%s%" PRIx32, vdi_path, vid);
+
+	fd = open(path, flags, def_fmode);
+	if (fd < 0) {
+		eprintf("fail: %m\n");
+		return SD_RES_EIO;
+	}
+
+	ret = xpread(fd, nr_copies, sizeof(*nr_copies), 0);
+	close(fd);
+	if (ret < 0) {
+		eprintf("pread: %m\n");
+		return SD_RES_EIO;
+	}
+
+	return SD_RES_SUCCESS;
+}
+
+static int log_vdi_copies(uint32_t vid, unsigned int nr_copies)
+{
+	int fd, flags, ret;
+	char path[PATH_MAX];
+
+	sprintf(path, "%s%" PRIx32, vdi_path, vid);
+
+	flags = O_CREAT | O_TRUNC | O_DSYNC | O_RDWR;
+
+	fd = open(path, flags, def_fmode);
+	if (fd < 0) {
+		dprintf("fail: %m\n");
+		return SD_RES_EIO;
+	}
+
+	ret = xpwrite(fd, (void *)&nr_copies, sizeof(nr_copies), 0);
+	close(fd);
+	if (ret < 0) {
+		dprintf("%m\n");
+		return SD_RES_EIO;
+	}
+
+	return SD_RES_SUCCESS;
+}
+
 int add_vdi_copies(uint32_t vid, int nr_copies)
 {
 	struct vdi_copy_entry *entry, *old;
+	int ret;
 
 	entry = xzalloc(sizeof(*entry));
 	entry->vid = vid;
@@ -121,8 +172,11 @@ int add_vdi_copies(uint32_t vid, int nr_copies)
 	old = vdi_copy_insert(&vdi_copy_root, entry);
 	if (old) {
 		free(entry);
-		entry = old;
-		entry->nr_copies = nr_copies;
+		if (old->nr_copies == nr_copies) {
+			pthread_rwlock_unlock(&vdi_copy_lock);
+			return SD_RES_SUCCESS;
+		}
+		old->nr_copies = nr_copies;
 	}
 
 	if (uatomic_read(&max_copies) == 0 ||
@@ -130,7 +184,46 @@ int add_vdi_copies(uint32_t vid, int nr_copies)
 		uatomic_set(&max_copies, nr_copies);
 	pthread_rwlock_unlock(&vdi_copy_lock);
 
-	return SD_RES_SUCCESS;
+	ret = log_vdi_copies(vid, nr_copies);
+
+	return ret;
+}
+
+int load_vdi_copies(void)
+{
+	DIR *dir;
+	struct dirent *d;
+	uint32_t vid;
+	struct strbuf vid_buf;
+	int ret = 0;
+	unsigned int nr_copies;
+
+	strbuf_init(&vid_buf, PATH_MAX);
+	strbuf_addstr(&vid_buf, vdi_path);
+
+	dir = opendir(vid_buf.buf);
+	if (!dir) {
+		dprintf("%m\n");
+		ret = -1;
+		goto out;
+	}
+
+	while ((d = readdir(dir))) {
+		if (!strncmp(d->d_name, ".", 1))
+			continue;
+		vid = strtoul(d->d_name, NULL, 16);
+		if (vid == ULLONG_MAX)
+			continue;
+		ret = read_vdi_copies(vid, &nr_copies);
+		dprintf("load %" PRIx32 ", %d\n", vid, nr_copies);
+		if (ret != SD_RES_SUCCESS)
+			goto out;
+		add_vdi_copies(vid, nr_copies);
+	}
+
+out:
+	strbuf_release(&vid_buf);
+	return ret;
 }
 
 int vdi_exist(uint32_t vid)
-- 
1.7.1