[sheepdog] [PATCH] sheep/cluster: remove ephemeral node when zookeeper master leaves

[Hitoshi Mitake]

At Wed, 30 Jul 2014 18:43:59 +0800,
Ruoyu wrote:
> 
> The zookeeper node /sheepdog/master/000000xxxx is ephemeral, but when
> the sheep process which is master in the zookeeper cluster leaves,
> zookeeper will not remove the ephemeral node immediately. If the sheep
> restarts at once, the sheepdog cluster will be corrupted.
> 
> This patch fixes it by trying removing the ephemeral node when
> sheep leaves the cluster.
> 
> Typo for MASTER_ZNODE is also fixed in the patch.
> 
> Signed-off-by: Ruoyu <liangry at ucweb.com>
> ---
>  sheep/cluster/zookeeper.c | 28 ++++++++++++++++++----------
>  1 file changed, 18 insertions(+), 10 deletions(-)

Applied, thanks.

BTW, from next time, could you prepare an individual patch for typo
fixing? Separating patches as much as possible is helpful for reading
git log and finding incorrect changes.

Thanks,
Hitoshi

> 
> diff --git a/sheep/cluster/zookeeper.c b/sheep/cluster/zookeeper.c
> index 96acce6..c99d5d0 100644
> --- a/sheep/cluster/zookeeper.c
> +++ b/sheep/cluster/zookeeper.c
> @@ -30,10 +30,11 @@
>  #define BASE_ZNODE "/sheepdog"
>  #define QUEUE_ZNODE BASE_ZNODE "/queue"
>  #define MEMBER_ZNODE BASE_ZNODE "/member"
> -#define MASTER_ZNONE BASE_ZNODE "/master"
> +#define MASTER_ZNODE BASE_ZNODE "/master"
>  #define LOCK_ZNODE BASE_ZNODE "/lock"
>  
>  static int zk_timeout = SESSION_TIMEOUT;
> +static int my_master_seq;
>  
>  /* structure for distributed lock */
>  struct cluster_lock {
> @@ -641,7 +642,7 @@ static inline void build_node_list(void)
>  static int zk_queue_init(void)
>  {
>  	RETURN_IF_ERROR(zk_init_node(BASE_ZNODE), "path %s", BASE_ZNODE);
> -	RETURN_IF_ERROR(zk_init_node(MASTER_ZNONE), "path %s", MASTER_ZNONE);
> +	RETURN_IF_ERROR(zk_init_node(MASTER_ZNODE), "path %s", MASTER_ZNODE);
>  	RETURN_IF_ERROR(zk_init_node(QUEUE_ZNODE), "path %s", QUEUE_ZNODE);
>  	RETURN_IF_ERROR(zk_init_node(MEMBER_ZNODE), "path %s", MEMBER_ZNODE);
>  	return ZOK;
> @@ -722,7 +723,7 @@ static void zk_watcher(zhandle_t *zh, int type, int state, const char *path,
>  			return;
>  		}
>  
> -		ret = sscanf(path, MASTER_ZNONE "/%s", str);
> +		ret = sscanf(path, MASTER_ZNODE "/%s", str);
>  		if (ret == 1) {
>  			zk_compete_master();
>  			return;
> @@ -816,17 +817,17 @@ static int zk_find_master(int *master_seq, char *master_name)
>  	char master_compete_path[MAX_NODE_STR_LEN];
>  
>  	if (*master_seq < 0) {
> -		RETURN_IF_ERROR(zk_get_least_seq(MASTER_ZNONE,
> +		RETURN_IF_ERROR(zk_get_least_seq(MASTER_ZNODE,
>  						 master_compete_path,
>  						 MAX_NODE_STR_LEN, master_name,
>  						 &len), "");
> -		sscanf(master_compete_path, MASTER_ZNONE "/%"PRId32,
> +		sscanf(master_compete_path, MASTER_ZNODE "/%"PRId32,
>  		       master_seq);
>  		return ZOK;
>  	} else {
>  		while (true) {
>  			snprintf(master_compete_path, len,
> -				 MASTER_ZNONE "/%010"PRId32, *master_seq);
> +				 MASTER_ZNODE "/%010"PRId32, *master_seq);
>  			rc = zk_get_data(master_compete_path, master_name,
>  					 &len);
>  			switch (rc) {
> @@ -855,7 +856,7 @@ static int zk_verify_last_sheep_join(int seq, int *last_sheep)
>  	char path[MAX_NODE_STR_LEN], name[MAX_NODE_STR_LEN];
>  
>  	for (*last_sheep = seq - 1; *last_sheep >= 0; (*last_sheep)--) {
> -		snprintf(path, MAX_NODE_STR_LEN, MASTER_ZNONE "/%010"PRId32,
> +		snprintf(path, MAX_NODE_STR_LEN, MASTER_ZNODE "/%010"PRId32,
>  			 *last_sheep);
>  		rc = zk_get_data(path, name, &len);
>  		switch (rc) {
> @@ -913,18 +914,18 @@ static void zk_compete_master(void)
>  			if (uatomic_is_true(&stop))
>  				goto out_unlock;
>  			/* duplicate sequential node has no side-effect */
> -			rc = zk_create_seq_node(MASTER_ZNONE "/",
> +			rc = zk_create_seq_node(MASTER_ZNODE "/",
>  						node_to_str(&this_node.node),
>  						MAX_NODE_STR_LEN,
>  						my_compete_path,
>  						MAX_NODE_STR_LEN, true);
>  		} while (rc == ZOPERATIONTIMEOUT || rc == ZCONNECTIONLOSS);
> -		CHECK_ZK_RC(rc, MASTER_ZNONE "/");
> +		CHECK_ZK_RC(rc, MASTER_ZNODE "/");
>  		if (rc != ZOK)
>  			goto out_unlock;
>  
>  		sd_debug("my compete path: %s", my_compete_path);
> -		sscanf(my_compete_path, MASTER_ZNONE "/%"PRId32,
> +		sscanf(my_compete_path, MASTER_ZNODE "/%"PRId32,
>  		       &my_seq);
>  	}
>  
> @@ -952,6 +953,7 @@ static void zk_compete_master(void)
>  	}
>  success:
>  	uatomic_set_true(&is_master);
> +	my_master_seq = master_seq;
>  	sd_debug("success");
>  out_unlock:
>  	sd_rw_unlock(&zk_compete_master_lock);
> @@ -986,6 +988,12 @@ static int zk_leave(void)
>  	sd_info("leaving from cluster");
>  	uatomic_set_true(&stop);
>  
> +	if (uatomic_is_true(&is_master)) {
> +		snprintf(path, sizeof(path), MASTER_ZNODE "/%010"PRId32,
> +				my_master_seq);
> +		zk_delete_node(path, -1);
> +	}
> +
>  	snprintf(path, sizeof(path), MEMBER_ZNODE"/%s",
>  		 node_to_str(&this_node.node));
>  	add_event(EVENT_LEAVE, &this_node, NULL, 0);
> -- 
> 1.8.3.2
> 
> 
> -- 
> sheepdog mailing list
> sheepdog at lists.wpkg.org
> http://lists.wpkg.org/mailman/listinfo/sheepdog