[sheepdog] [PATCH] sheep, dog: add vnodes fixed options.
Hitoshi Mitake
mitake.hitoshi at gmail.com
Wed Dec 24 15:43:12 CET 2014
At Fri, 19 Dec 2014 10:07:23 +0900,
Saeki Masaki wrote:
>
> In the current sheepdog, vnodes is recalculated at the time of
> node increase or decrease.
>
> In the auto recovery, first get the object from the other node,
> finally delete the object do not need.
> During auto recovery run, available disk decreases.
> In the worst case, it exhaust available disk.
>
> Add the following new commands and options.
> 1. option to specify vnodes in sheep. (-V, --vnodes)
> - The old days it has been implemented as -v(--vnodes) option.
> - For now -v option is used to print version. so add in -V (large).
> $ sheep -V 100 /var/lib/sheepdog
> If -V is specified, vnodes strategy of sheep is 'fixed'
> (default value is 'auto')
>
> 2. option to dog cluster format with vnode fixed. (-V, --fixedvnodes)
> $ dog cluster format -V
> If 'fixed' and 'auto' vnodes strategy are mixed,
> cluster format command fails.
> (different sheep of vnodes strategy can not be mixed in the cluster)
>
> 3. dog command to change the vnodes
> $ dog node vnode set <vnodes>
> After changing the vnodes, new epoch are created and auto recovery will start.
>
> If you want to operate the vnodes fixed,
> it is necessary to manage the vnodes with capacity of
> the data store in each node.
> So you should use this option carefully.
>
> For example of using fixed vnodes strategy:
>
> 1) start sheep with fixed vnodes strategy.
> $ sheep -V 100 /var/lib/sheepdog
> $ sheep -V 110 /var/lib/sheepdog
> $ dog node list
> Id Host:Port V-Nodes Zone
> 0 172.16.4.205:7000 100 1812140204
> 1 172.16.4.206:7000 120 1828917420
>
> 2) format the cluster with fixed vnodes strategy.
> $ dog cluster format -V
>
> 3) check vnodes strategy of cluster.
> $ dog cluster info -v
> Cluster status: running, auto-recovery enabled
> Cluster store: plain with 3 redundancy policy
> Cluster vnodes strategy: fixed
> Cluster vnode mode: node
> Cluster created at Wed Dec 17 18:20:10 2014
>
> Epoch Time Version [Host:Port:V-Nodes,,,]
> 2014-12-17 18:20:10 1 [172.16.4.205:7000:100, 172.16.4.206:7000:120]
>
> 4) change of vnodes.
> $ dog node vnodes set 140
> $ dog node list
> Id Host:Port V-Nodes Zone
> 0 172.16.4.205:7000 140 1812140204
> 1 172.16.4.206:7000 120 1828917420
Saeki-san, thanks a lot for this patch. The change seems good to me,
but rebase is required for the latest master. Could you rebase and
send v2? In addition, I have some opinions, mainly related to trivial
coding styles. I'm glad if you can take them in v2.
>
> Signed-off-by Masaki Saeki <masaki.saeki at po.ntts.co.jp>
You need ':' between Signed-off-by and your name.
>
> ---
> dog/cluster.c | 82 ++++++++++++++++++++++++++++++++++++++--------
> dog/node.c | 67 +++++++++++++++++++++++++++++++++++++
> include/internal_proto.h | 3 ++
> include/sheep.h | 8 ++++
> include/sheepdog_proto.h | 2 +
> sheep/config.c | 15 ++++++++-
> sheep/group.c | 62 ++++++++++++++++++++++++++++++++---
> sheep/ops.c | 82 ++++++++++++++++++++++++++++++++++++++++++++++
> sheep/sheep.c | 31 ++++++++++++++++-
> 9 files changed, 330 insertions(+), 22 deletions(-)
>
> diff --git a/dog/cluster.c b/dog/cluster.c
> index 20f190b..c92141e 100644
> --- a/dog/cluster.c
> +++ b/dog/cluster.c
> @@ -15,6 +15,7 @@
> #include <sys/time.h>
>
> #include "dog.h"
> +#include "sheep.h"
> #include "farm/farm.h"
>
> static struct sd_option cluster_options[] = {
> @@ -27,6 +28,7 @@ static struct sd_option cluster_options[] = {
> "do not serve write request if number of nodes is not sufficient"},
> {'z', "block_size_shift", true, "specify the shift num of default"
> " data object size"},
> + {'V', "fixedvnodes", false, "disable automatic vnodes calculation"},
> { 0, NULL, false, NULL },
> };
>
> @@ -38,6 +40,7 @@ static struct cluster_cmd_data {
> bool force;
> bool strict;
> char name[STORE_LEN];
> + bool fixed_vnodes;
> } cluster_cmd_data;
>
> #define DEFAULT_STORE "plain"
> @@ -87,6 +90,41 @@ static int cluster_format(int argc, char **argv)
> struct timeval tv;
> char store_name[STORE_LEN];
> static DECLARE_BITMAP(vdi_inuse, SD_NR_VDIS);
> + struct sd_node *n;
> +
> + rb_for_each_entry(n, &sd_nroot, rb) {
> + struct sd_req info_req;
> + struct sd_rsp *info_rsp = (struct sd_rsp *)&info_req;
> + struct cluster_info cinfo;
> +
> + sd_init_req(&info_req, SD_OP_CLUSTER_INFO);
> + info_req.data_length = sizeof(cinfo);
> + ret = dog_exec_req(&n->nid, &info_req, &cinfo);
> + if (ret < 0) {
> + sd_err("Fail to execute request");
> + return EXIT_FAILURE;
> + }
> + if (info_rsp->result != SD_RES_SUCCESS) {
> + sd_err("%s", sd_strerror(info_rsp->result));
> + return EXIT_FAILURE;
> + }
> +
> + if (n->nr_vnodes != 0) {
> + if ((cinfo.flags & SD_CLUSTER_FLAG_AUTO_VNODES)
> + && cluster_cmd_data.fixed_vnodes) {
> + sd_err("Can not apply the option of '-V', "
> + "because there are vnode strategy of sheep "
> + "is auto in the cluster");
> + return EXIT_FAILURE;
> + } else if (!(cinfo.flags & SD_CLUSTER_FLAG_AUTO_VNODES)
> + && !cluster_cmd_data.fixed_vnodes) {
> + sd_err("Need to specify the option of '-V', "
> + "because there are vnode strategy of sheep "
> + "is fixed in the cluster");
> + return EXIT_FAILURE;
> + }
> + }
> + }
>
> if (cluster_cmd_data.copies > sd_nodes_nr) {
> char info[1024];
> @@ -132,6 +170,11 @@ static int cluster_format(int argc, char **argv)
> hdr.cluster.flags |= SD_CLUSTER_FLAG_DISKMODE;
> #endif
>
> + if (cluster_cmd_data.fixed_vnodes)
> + hdr.cluster.flags &= ~SD_CLUSTER_FLAG_AUTO_VNODES;
> + else
> + hdr.cluster.flags |= SD_CLUSTER_FLAG_AUTO_VNODES;
> +
> printf("using backend %s store\n", store_name);
> ret = dog_exec_req(&sd_nid, &hdr, store_name);
> if (ret < 0)
> @@ -160,14 +203,15 @@ static void print_nodes(const struct epoch_log *logs, uint16_t flags)
> if (entry->disks[nr_disk].disk_id == 0)
> break;
> }
> - printf("%s%s(%d)",
> - (i == 0) ? "" : ", ",
> - addr_to_str(entry->nid.addr, entry->nid.port),
> - nr_disk);
> + printf("%s%s:%d(%d)",
> + (i == 0) ? "" : ", ",
> + addr_to_str(entry->nid.addr, entry->nid.port),
> + entry->nr_vnodes, nr_disk);
> } else
> - printf("%s%s",
> - (i == 0) ? "" : ", ",
> - addr_to_str(entry->nid.addr, entry->nid.port));
> + printf("%s%s:%d",
> + (i == 0) ? "" : ", ",
> + addr_to_str(entry->nid.addr, entry->nid.port),
> + entry->nr_vnodes);
> }
> }
>
> @@ -232,6 +276,15 @@ retry:
> }
> printf("%s with %s redundancy policy\n",
> logs->drv_name, copy);
> +
> + /* show vnode strategy */
> + if (!raw_output)
> + printf("Cluster vnodes strategy: ");
> + if (logs->flags & SD_CLUSTER_FLAG_AUTO_VNODES)
> + printf("auto\n");
> + else
> + printf("fixed\n");
> +
> } else
> printf("%s\n", sd_strerror(rsp->result));
>
> @@ -239,15 +292,16 @@ retry:
> if (!raw_output)
> printf("Cluster vnode mode: ");
> if (logs->flags & SD_CLUSTER_FLAG_DISKMODE)
> - printf("disk");
> + printf("disk\n");
> else
> - printf("node");
> + printf("node\n");
> }
>
> if (!raw_output && rsp->data_length > 0) {
> ct = logs[0].ctime >> 32;
> - printf("\nCluster created at %s\n", ctime(&ct));
> - printf("Epoch Time Version\n");
> + printf("Cluster created at %s\n", ctime(&ct));
> + printf("Epoch Time Version
> [Host:Port:V-Nodes,,,]");
The above change will break existing tests. Could you create a patch
for updating tests? You can send it as another patch.
> + printf("\n");
> }
>
> nr_logs = rsp->data_length / (sizeof(struct epoch_log)
> @@ -761,7 +815,7 @@ failure:
> static struct subcommand cluster_cmd[] = {
> {"info", NULL, "aprhvT", "show cluster information",
> NULL, CMD_NEED_NODELIST, cluster_info, cluster_options},
> - {"format", NULL, "bctaphzT", "create a Sheepdog store",
> + {"format", NULL, "bctaphzTV", "create a Sheepdog store",
> NULL, CMD_NEED_NODELIST, cluster_format, cluster_options},
> {"shutdown", NULL, "aphT", "stop Sheepdog",
> NULL, 0, cluster_shutdown, cluster_options},
> @@ -823,9 +877,9 @@ static int cluster_parser(int ch, const char *opt)
> " Please set shift bit larger than 20");
> exit(EXIT_FAILURE);
> }
> -
> cluster_cmd_data.block_size_shift = block_size_shift;
> -
> + case 'V':
> + cluster_cmd_data.fixed_vnodes = true;
> break;
> }
>
> diff --git a/dog/node.c b/dog/node.c
> index a4e9142..b9d441a 100644
> --- a/dog/node.c
> +++ b/dog/node.c
> @@ -625,6 +625,71 @@ static int node_log(int argc, char **argv)
> return do_generic_subcommand(node_log_cmd, argc, argv);
> }
>
> +static int do_vnodes_set(const struct node_id *nid, int *nr_vnodes)
> +{
> + int ret = 0;
> + struct sd_req hdr;
> + struct sd_rsp *rsp = (struct sd_rsp *)&hdr;
> +
> + sd_init_req(&hdr, SD_OP_SET_VNODES);
> + hdr.flags = SD_FLAG_CMD_WRITE;
> + hdr.data_length = sizeof(nr_vnodes);
> +
> + ret = dog_exec_req(nid, &hdr, nr_vnodes);
> + if (ret < 0)
> + return EXIT_SYSFAIL;
> +
> + if (rsp->result != SD_RES_SUCCESS)
> + return EXIT_FAILURE;
> +
> + return ret;
> +}
> +
> +static int node_vnodes_set(int argc, char **argv)
> +{
> + int ret = 0;
> + char *p;
> + int32_t nr_vnodes = strtol(argv[optind], &p, 10);
> +
> + if (argv[optind] == p || nr_vnodes < 1 || nr_vnodes > UINT16_MAX
> + || *p != '\0') {
> + sd_err("Invalid number of vnodes '%s': must be an integer "
> + "between 1 and %u",
> + argv[optind], UINT16_MAX);
> + exit(EXIT_USAGE);
> + }
> +
> + ret = do_vnodes_set(&sd_nid, &nr_vnodes);
> +
> + switch (ret) {
> + case EXIT_FAILURE:
> + case EXIT_SYSFAIL:
> + sd_err("Failed to execute request");
> + ret = -1;
> + break;
> + case EXIT_SUCCESS:
> + /* do nothing */
> + break;
> + default:
> + sd_err("unknown return code of do_vnodes_set(): %d", ret);
> + ret = -1;
> + break;
> + }
> +
> + return ret;
> +}
> +
> +static struct subcommand node_vnodes_cmd[] = {
> + {"set", "<num of vnodes>", NULL, "set new vnodes",
> + NULL, CMD_NEED_ARG, node_vnodes_set},
> + {NULL},
> +};
> +
> +static int node_vnodes(int argc, char **argv)
> +{
> + return do_generic_subcommand(node_vnodes_cmd, argc, argv);
> +}
> +
> static struct subcommand node_cmd[] = {
> {"kill", "<node id>", "aprhlT", "kill node", NULL,
> CMD_NEED_NODELIST, node_kill, node_options},
> @@ -640,6 +705,8 @@ static struct subcommand node_cmd[] = {
> 0, node_stat, node_options},
> {"log", NULL, "aphT", "show or set log level of the node", node_log_cmd,
> CMD_NEED_ARG, node_log},
> + {"vnodes", "<num of vnodes>", "aph", "set new vnodes", node_vnodes_cmd,
> + CMD_NEED_ARG, node_vnodes},
> {NULL,},
> };
>
> diff --git a/include/internal_proto.h b/include/internal_proto.h
> index 3f5d77f..f280d6d 100644
> --- a/include/internal_proto.h
> +++ b/include/internal_proto.h
> @@ -111,6 +111,8 @@
> #define SD_OP_VDI_STATE_SNAPSHOT_CTL 0xC7
> #define SD_OP_INODE_COHERENCE 0xC8
> #define SD_OP_READ_DEL_VDIS 0xC9
> +#define SD_OP_SET_VNODES 0xCC
> +#define SD_OP_GET_VNODES 0xCD
>
> /* internal flags for hdr.flags, must be above 0x80 */
> #define SD_FLAG_CMD_RECOVERY 0x0080
> @@ -143,6 +145,7 @@
>
> #define SD_CLUSTER_FLAG_STRICT 0x0001 /* Strict mode for write */
> #define SD_CLUSTER_FLAG_DISKMODE 0x0002 /* Disk mode for cluster */
> +#define SD_CLUSTER_FLAG_AUTO_VNODES 0x0004 /* Cluster vnodes strategy */
>
> enum sd_status {
> SD_STATUS_OK = 1,
> diff --git a/include/sheep.h b/include/sheep.h
> index 22524c1..fe6f066 100644
> --- a/include/sheep.h
> +++ b/include/sheep.h
> @@ -149,6 +149,9 @@ static inline const char *sd_strerror(int err)
> "IO has halted as there are not enough living nodes",
> [SD_RES_READONLY] = "Object is read-only",
> [SD_RES_INODE_INVALIDATED] = "Inode object is invalidated",
> + [SD_RES_INVALID_VNODES_STRATEGY] =
> + "Invalid cluster vnodes strategy",
> + [SD_RES_GATEWAY_MODE] = "Targeted node is gateway mode",
>
> /* from internal_proto.h */
> [SD_RES_OLD_NODE_VER] = "Request has an old epoch",
> @@ -328,4 +331,9 @@ static inline bool is_cluster_diskmode(const struct cluster_info *cinfo)
> return (cinfo->flags & SD_CLUSTER_FLAG_DISKMODE) > 0;
> }
>
> +static inline bool is_cluster_autovnodes(const struct cluster_info *cinfo)
> +{
> + return (cinfo->flags & SD_CLUSTER_FLAG_AUTO_VNODES) > 0;
> +}
> +
> #endif
> diff --git a/include/sheepdog_proto.h b/include/sheepdog_proto.h
> index 4f0c48c..28ededd 100644
> --- a/include/sheepdog_proto.h
> +++ b/include/sheepdog_proto.h
> @@ -86,6 +86,8 @@
> #define SD_RES_INCOMPLETE 0x1B /* Object (in kv) is incomplete uploading */
> #define SD_RES_COLLECTING_CINFO 0x1C /* sheep is collecting cluster wide status, not ready for operation */
> #define SD_RES_INODE_INVALIDATED 0x1D /* inode object in client is invalidated, refreshing is required */
> +#define SD_RES_GATEWAY_MODE 0x1E /* Target node is gateway mode */
> +#define SD_RES_INVALID_VNODES_STRATEGY 0x1F /* Invalid vnodes strategy */
>
> /* errors above 0x80 are sheepdog-internal */
>
> diff --git a/sheep/config.c b/sheep/config.c
> index 383a1ed..4a1e600 100644
> --- a/sheep/config.c
> +++ b/sheep/config.c
> @@ -62,7 +62,12 @@ static int get_cluster_config(struct cluster_info *cinfo)
> {
> cinfo->ctime = config.ctime;
> cinfo->nr_copies = config.copies;
> - cinfo->flags = config.flags;
> + if (config.ctime > 0) {
> + cinfo->flags = config.flags;
> + } else {
> + cinfo->flags = (config.flags & ~SD_CLUSTER_FLAG_AUTO_VNODES) |
> + (cinfo->flags & SD_CLUSTER_FLAG_AUTO_VNODES);
> + }
Could you eliminate the braces? In sheepdog coding style, braces of
if, for, while statements should be removed they don't have more than
two statements.
# seems that script/checkpatch.pl doesn't work well for this style...
> cinfo->copy_policy = config.copy_policy;
> memcpy(cinfo->store, config.store, sizeof(config.store));
>
> @@ -121,6 +126,14 @@ int init_config_file(void)
> }
>
> reload:
> + if ((config.flags & SD_CLUSTER_FLAG_AUTO_VNODES) !=
> + (sys->cinfo.flags & SD_CLUSTER_FLAG_AUTO_VNODES)
> + && !sys->gateway_only
> + && config.ctime > 0) {
> + sd_err("Designation of before a restart and a vnodes option is different.");
> + return -1;
> + }
> +
> ret = 0;
> get_cluster_config(&sys->cinfo);
> if ((config.flags & SD_CLUSTER_FLAG_DISKMODE) !=
> diff --git a/sheep/group.c b/sheep/group.c
> index 095b7c5..b33e514 100644
> --- a/sheep/group.c
> +++ b/sheep/group.c
> @@ -145,7 +145,8 @@ struct vnode_info *alloc_vnode_info(const struct rb_root *nroot)
> vnode_info->nr_nodes++;
> }
>
> - recalculate_vnodes(&vnode_info->nroot);
> + if (is_cluster_autovnodes(&sys->cinfo))
> + recalculate_vnodes(&vnode_info->nroot);
>
> if (is_cluster_diskmode(&sys->cinfo))
> disks_to_vnodes(&vnode_info->nroot, &vnode_info->vroot);
> @@ -1098,6 +1099,20 @@ static bool cluster_join_check(const struct cluster_info *cinfo)
> if (!cluster_ctime_check(cinfo))
> return false;
>
> + if (cinfo->ctime > 0 && sys->this_node.nr_vnodes != 0) {
> + if (!is_cluster_autovnodes(&sys->cinfo)
> + && is_cluster_autovnodes(cinfo)) {
> + sd_err("failed to join for vnodes strategy unmatch. "
> + " cluster:fixed, joined:auto");
> + return false;
> + } else if (is_cluster_autovnodes(&sys->cinfo)
> + && !is_cluster_autovnodes(cinfo)) {
> + sd_err("failed to join for vnodes strategy unmatch. "
> + " cluster:auto, joined:fixed");
> + return false;
> + }
> + }
> +
> /*
> * Sheepdog's recovery code assumes every node have the same epoch
> * history. But we don't check epoch history of joining node because:
> @@ -1119,6 +1134,14 @@ main_fn void sd_accept_handler(const struct sd_node *joined,
> {
> const struct cluster_info *cinfo = opaque;
> struct sd_node *n;
> + uint16_t flags;
> +
> + if (node_is_local(joined) && sys->gateway_only
> + && sys->cinfo.ctime <= 0) {
> + flags = cinfo->flags & SD_CLUSTER_FLAG_AUTO_VNODES;
> + } else {
> + flags = sys->cinfo.flags & SD_CLUSTER_FLAG_AUTO_VNODES;
> + }
The brace problem, too.
>
> if (node_is_local(joined) && !cluster_join_check(cinfo)) {
> sd_err("failed to join Sheepdog");
> @@ -1127,6 +1150,9 @@ main_fn void sd_accept_handler(const struct sd_node *joined,
>
> cluster_info_copy(&sys->cinfo, cinfo);
>
> + sys->cinfo.flags &= ~SD_CLUSTER_FLAG_AUTO_VNODES;
> + sys->cinfo.flags |= flags;
> +
> sd_debug("join %s", node_to_str(joined));
> rb_for_each_entry(n, nroot, rb) {
> sd_debug("%s", node_to_str(n));
> @@ -1191,7 +1217,7 @@ main_fn void sd_leave_handler(const struct sd_node *left,
> remove_node_from_participants(&left->nid);
> }
>
> -static void update_node_size(struct sd_node *node)
> +static void update_node_info(struct sd_node *node)
> {
> struct vnode_info *cur_vinfo = get_vnode_info();
> struct sd_node *n = rb_search(&cur_vinfo->nroot, node, rb, node_cmp);
> @@ -1199,6 +1225,11 @@ static void update_node_size(struct sd_node *node)
> if (unlikely(!n))
> panic("can't find %s", node_to_str(node));
> n->space = node->space;
> +
> + if (!is_cluster_autovnodes(&sys->cinfo)) {
> + n->nr_vnodes = node->nr_vnodes;
> + }
The brace problem.
That's all. Thanks.
Hitoshi
More information about the sheepdog
mailing list