[Sheepdog] [PATCH v6 14/17] sheep: add cluster snapshot/restore support
Liu Yuan
namei.unix at gmail.com
Thu Jan 12 14:37:25 CET 2012
From: Liu Yuan <tailai.ly at taobao.com>
This kind of snapshot is supposed be triggered by user, _not_ by recovery
code. I don't think we need to restore to the state at the beginning of
the recovery. So this work only permits us to restore cluster to the
snapshot initiated by end users, thought it is quite easy to implement
to restore to the snapshots forcibly taken by recovery path.
Signed-off-by: Liu Yuan <tailai.ly at taobao.com>
---
include/sheep.h | 4 ++
include/sheepdog_proto.h | 1 +
sheep/farm/farm.c | 135 ++++++++++++++++++++++++++++++++++++++++++++++
sheep/ops.c | 61 +++++++++++++++++++++
sheep/sheep_priv.h | 4 ++
5 files changed, 205 insertions(+), 0 deletions(-)
diff --git a/include/sheep.h b/include/sheep.h
index 785c8ae..4f2f1dc 100644
--- a/include/sheep.h
+++ b/include/sheep.h
@@ -39,6 +39,9 @@
#define SD_OP_GET_VDI_ATTR 0x89
#define SD_OP_RECOVER 0x8a
#define SD_OP_GET_STORE_LIST 0x90
+#define SD_OP_SNAPSHOT 0x91
+#define SD_OP_RESTORE 0x92
+#define SD_OP_GET_SNAP_FILE 0x93
#define SD_FLAG_CMD_IO_LOCAL 0x0010
#define SD_FLAG_CMD_RECOVERY 0x0020
@@ -265,6 +268,7 @@ static inline const char *sd_strerror(int err)
{SD_RES_HALT, "IO has halted as there are too few living nodes"},
{SD_RES_MANUAL_RECOVER, "Cluster is running/halted and cannot be manually recovered"},
{SD_RES_NO_STORE, "Targeted backend store is not found"},
+ {SD_RES_NO_SUPPORT, "Operation is not supported"},
{SD_RES_OLD_NODE_VER, "Remote node has an old epoch"},
{SD_RES_NEW_NODE_VER, "Remote node has a new epoch"},
diff --git a/include/sheepdog_proto.h b/include/sheepdog_proto.h
index 9cc5a99..429f5cd 100644
--- a/include/sheepdog_proto.h
+++ b/include/sheepdog_proto.h
@@ -60,6 +60,7 @@
#define SD_RES_HALT 0x19 /* Sheepdog is stopped doing IO */
#define SD_RES_MANUAL_RECOVER 0x1A /* Users should not manually recover this cluster */
#define SD_RES_NO_STORE 0x20 /* No targeted backend store */
+#define SD_RES_NO_SUPPORT 0x21 /* Operation is not supported by backend store */
/*
* Object ID rules
diff --git a/sheep/farm/farm.c b/sheep/farm/farm.c
index ccf4c49..4b1eedc 100644
--- a/sheep/farm/farm.c
+++ b/sheep/farm/farm.c
@@ -427,6 +427,138 @@ out:
return ret;
}
+static int farm_snapshot(struct siocb *iocb)
+{
+ unsigned char snap_sha1[SHA1_LEN];
+ void *buffer;
+ int log_nr, ret = SD_RES_EIO, epoch;
+
+ buffer = snap_log_read(&log_nr, 1);
+ if (!buffer)
+ goto out;
+
+ epoch = log_nr + 1;
+ dprintf("user epoch %d\n", epoch);
+ if (snap_file_write(epoch, snap_sha1, 1) < 0)
+ goto out;
+
+ if (snap_log_write(epoch, snap_sha1, 1) < 0)
+ goto out;
+
+ ret = SD_RES_SUCCESS;
+out:
+ free(buffer);
+ return ret;
+}
+
+static int cleanup_working_dir(void)
+{
+ DIR *dir;
+ struct dirent *d;
+
+ dprintf("try clean up working dir\n");
+ dir = opendir(obj_path);
+ if (!dir)
+ return -1;
+
+ while ((d = readdir(dir))) {
+ char p[PATH_MAX];
+ if (!strncmp(d->d_name, ".", 1))
+ continue;
+ snprintf(p, sizeof(p), "%s%s", obj_path, d->d_name);
+ if (unlink(p) < 0) {
+ eprintf("%s:%m\n", p);
+ continue;
+ }
+ dprintf("remove file %s\n", d->d_name);
+ }
+ closedir(dir);
+ return 0;
+}
+
+static int restore_objects_from_snap(int epoch)
+{
+ struct sha1_file_hdr hdr;
+ struct trunk_entry *trunk_buf, *trunk_free = NULL;
+ unsigned char trunk_sha1[SHA1_LEN];
+ uint64_t nr_trunks, i;
+ int ret = SD_RES_EIO;
+
+ if (get_trunk_sha1(epoch, trunk_sha1, 1) < 0)
+ goto out;
+
+ trunk_free = trunk_buf = trunk_file_read(trunk_sha1, &hdr);
+ if (!trunk_buf)
+ goto out;
+
+ nr_trunks = hdr.priv;
+ ret = SD_RES_SUCCESS;
+ for (i = 0; i < nr_trunks; i++, trunk_buf++) {
+ struct sha1_file_hdr h;
+ struct siocb io = { 0 };
+ uint64_t oid;
+ void *buffer = NULL;
+
+ oid = trunk_buf->oid;
+ buffer = sha1_file_read(trunk_buf->sha1, &h);
+ if (!buffer) {
+ eprintf("oid %"PRIx64" not restored\n", oid);
+ goto out;
+ }
+ io.length = h.size;
+ io.buf = buffer;
+ ret = farm_atomic_put(oid, &io);
+ if (ret != SD_RES_SUCCESS) {
+ eprintf("oid %"PRIx64" not restored\n", oid);
+ goto out;
+ } else
+ dprintf("oid %"PRIx64" restored\n", oid);
+
+ free(buffer);
+ }
+out:
+ free(trunk_free);
+ return ret;
+}
+
+static int farm_restore(struct siocb *iocb)
+{
+ int ret = SD_RES_EIO, epoch = iocb->epoch;
+
+ dprintf("try recover user epoch %d\n", epoch);
+
+ if (cleanup_working_dir() < 0) {
+ eprintf("failed to clean up the working dir %m\n");
+ goto out;
+ }
+
+ ret = restore_objects_from_snap(epoch);
+ if (ret != SD_RES_SUCCESS)
+ goto out;
+out:
+ return ret;
+}
+
+static int farm_get_snap_file(struct siocb *iocb)
+{
+ int ret = SD_RES_EIO;
+ void *buffer = NULL;
+ size_t size;
+ int nr;
+
+ dprintf("try get snap file\n");
+ buffer = snap_log_read(&nr, 1);
+ if (!buffer)
+ goto out;
+ size = nr * sizeof(struct snap_log);
+ memcpy(iocb->buf, buffer, size);
+ iocb->length = size;
+ ret = SD_RES_SUCCESS;
+out:
+ free(buffer);
+ return ret;
+}
+
struct store_driver farm = {
.name = "farm",
.init = farm_init,
@@ -439,6 +571,9 @@ struct store_driver farm = {
.atomic_put = farm_atomic_put,
.begin_recover = farm_begin_recover,
.end_recover = farm_end_recover,
+ .snapshot = farm_snapshot,
+ .restore = farm_restore,
+ .get_snap_file = farm_get_snap_file,
};
add_store_driver(farm);
diff --git a/sheep/ops.c b/sheep/ops.c
index 3f65477..58a1956 100644
--- a/sheep/ops.c
+++ b/sheep/ops.c
@@ -407,6 +407,49 @@ out:
return ret;
}
+static int cluster_snapshot(const struct sd_req *req, struct sd_rsp *rsp,
+ void *data)
+{
+ int ret;
+ struct siocb iocb = { 0 };
+
+ if (sd_store->snapshot)
+ ret = sd_store->snapshot(&iocb);
+ else
+ ret = SD_RES_NO_SUPPORT;
+
+ return ret;
+}
+
+static int cluster_restore(const struct sd_req *req, struct sd_rsp *rsp,
+ void *data)
+{
+ const struct sd_obj_req *hdr = (const struct sd_obj_req *)req;
+ int ret;
+ struct siocb iocb = { .epoch = hdr->tgt_epoch };
+
+ if (sd_store->restore)
+ ret = sd_store->restore(&iocb);
+ else
+ ret = SD_RES_NO_SUPPORT;
+ return ret;
+}
+
+static int local_get_snap_file(const struct sd_req *req, struct sd_rsp *rsp,
+ void *data)
+{
+ int ret;
+ struct siocb iocb = { .buf = data };
+
+ if (sd_store->get_snap_file) {
+ ret = sd_store->get_snap_file(&iocb);
+ rsp->data_length = iocb.length;
+ } else
+ ret = SD_RES_NO_SUPPORT;
+
+ return ret;
+}
+
static struct sd_op_template sd_ops[] = {
/* cluster operations */
@@ -457,6 +500,18 @@ static struct sd_op_template sd_ops[] = {
.process_main = cluster_manual_recover,
},
+ [SD_OP_SNAPSHOT] = {
+ .type = SD_OP_TYPE_CLUSTER,
+ .force = 1,
+ .process_main = cluster_snapshot,
+ },
+
+ [SD_OP_RESTORE] = {
+ .type = SD_OP_TYPE_CLUSTER,
+ .force = 1,
+ .process_main = cluster_restore,
+ },
+
/* local operations */
[SD_OP_GET_STORE_LIST] = {
.type = SD_OP_TYPE_LOCAL,
@@ -503,6 +558,12 @@ static struct sd_op_template sd_ops[] = {
.process_work = local_get_epoch,
},
+ [SD_OP_GET_SNAP_FILE] = {
+ .type = SD_OP_TYPE_LOCAL,
+ .force = 1,
+ .process_work = local_get_snap_file,
+ },
+
/* I/O operations */
[SD_OP_CREATE_AND_WRITE_OBJ] = {
.type = SD_OP_TYPE_IO,
diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index 153fdc9..45f44e0 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -179,6 +179,10 @@ struct store_driver {
int (*atomic_put)(uint64_t oid, struct siocb *);
int (*begin_recover)(struct siocb *);
int (*end_recover)(struct siocb *);
+ /* Operations for snapshot */
+ int (*snapshot)(struct siocb *);
+ int (*restore)(struct siocb *);
+ int (*get_snap_file)(struct siocb *);
};
extern struct list_head store_drivers;
--
1.7.8.2
More information about the sheepdog
mailing list