[sheepdog] [PATCH 1/2] zookeeper: remove master from zookeeper

[Liu Yuan]

On Tue, Aug 06, 2013 at 05:18:11AM -0700, Kai Zhang wrote:
> In the old implementation, zookeeper driver needs master to handle join requests
> and avoid errors during concurrent joining.
> 
> However, as join requests can be handled by any joined node, we have cheeper
> way to avoid errors during concurrent joining.
> 
> In this patch, we use ephemeral znode to simulate distributed lock.
> Before a node send join request, it have to acquire the lock.
> And under the protection of the lock, zk_member_empty() is safe because the
> membership it stable.
> 
> Because the znode is ephemeral, the lock will automatically released if the
> node is down.
> 
> Signed-off-by: Kai Zhang <kyle at zelin.io>
> ---
>  sheep/cluster/zookeeper.c |  218 ++++++++-------------------------------------
>  1 file changed, 37 insertions(+), 181 deletions(-)
> 
> diff --git a/sheep/cluster/zookeeper.c b/sheep/cluster/zookeeper.c
> index 6e632fc..357824a 100644
> --- a/sheep/cluster/zookeeper.c
> +++ b/sheep/cluster/zookeeper.c
> @@ -30,7 +30,7 @@
>  #define BASE_ZNODE "/sheepdog"
>  #define QUEUE_ZNODE BASE_ZNODE "/queue"
>  #define MEMBER_ZNODE BASE_ZNODE "/member"
> -#define MASTER_ZNONE BASE_ZNODE "/master"
> +#define JOIN_LOCK_ZNODE BASE_ZNODE "/join_lock"
>  
>  /* iterate child znodes */
>  #define FOR_EACH_ZNODE(parent, path, strs)			       \
> @@ -72,15 +72,11 @@ static struct sd_node sd_nodes[SD_MAX_NODES];
>  static size_t nr_sd_nodes;
>  static struct rb_root zk_node_root = RB_ROOT;
>  static pthread_rwlock_t zk_tree_lock = PTHREAD_RWLOCK_INITIALIZER;
> -static pthread_rwlock_t zk_compete_master_lock = PTHREAD_RWLOCK_INITIALIZER;
>  static LIST_HEAD(zk_block_list);
> -static uatomic_bool is_master;
>  static uatomic_bool stop;
>  static bool joined;
>  static bool first_push = true;
>  
> -static void zk_compete_master(void);
> -
>  static struct zk_node *zk_tree_insert(struct zk_node *new)
>  {
>  	struct rb_node **p = &zk_node_root.rb_node;
> @@ -496,7 +492,6 @@ static inline void build_node_list(void)
>  static int zk_queue_init(void)
>  {
>  	RETURN_IF_ERROR(zk_init_node(BASE_ZNODE), "path %s", BASE_ZNODE);
> -	RETURN_IF_ERROR(zk_init_node(MASTER_ZNONE), "path %s", MASTER_ZNONE);
>  	RETURN_IF_ERROR(zk_init_node(QUEUE_ZNODE), "path %s", QUEUE_ZNODE);
>  	RETURN_IF_ERROR(zk_init_node(MEMBER_ZNODE), "path %s", MEMBER_ZNODE);
>  	return ZOK;
> @@ -562,12 +557,6 @@ static void zk_watcher(zhandle_t *zh, int type, int state, const char *path,
>  	} else if (type == ZOO_DELETED_EVENT) {
>  		struct zk_node *n;
>  
> -		ret = sscanf(path, MASTER_ZNONE "/%s", str);
> -		if (ret == 1) {
> -			zk_compete_master();
> -			return;
> -		}
> -
>  		ret = sscanf(path, MEMBER_ZNODE "/%s", str);
>  		if (ret != 1)
>  			return;
> @@ -605,186 +594,46 @@ static int add_join_event(void *msg, size_t msg_len)
>  	return zk_queue_push(&ev);
>  }
>  
> -static int zk_get_least_seq(const char *parent, char *least_seq_path,
> -			    int path_len, void *buf, int *buf_len)
> +static int zk_acquire_lock(const char *path)
>  {
> -	char path[MAX_NODE_STR_LEN], *p, *tmp;
> -	struct String_vector strs;
> -	int rc, least_seq = INT_MAX , seq;
> -
> +	int rc;
>  	while (true) {
> -		RETURN_IF_ERROR(zk_get_children(parent, &strs), "");
> -
> -		FOR_EACH_ZNODE(parent, path, &strs) {
> -			p = strrchr(path, '/');
> -			seq = strtol(++p, &tmp, 10);
> -			if (seq < least_seq)
> -				least_seq = seq;
> -		}
> -
> -		snprintf(path, MAX_NODE_STR_LEN, "%s/%010"PRId32,
> -			 parent, least_seq);
> -		rc = zk_get_data(path, buf, buf_len);
> +		rc = zk_create_node(path, "", 0, &ZOO_OPEN_ACL_UNSAFE,
> +				    ZOO_EPHEMERAL, NULL, 0);
>  		switch (rc) {
>  		case ZOK:
> -			strncpy(least_seq_path, path, path_len);
> -			return ZOK;
> -		case ZNONODE:
> +			return rc;
> +		case ZNODEEXISTS:
> +			sleep(1);

why sleep here?

Thanks
Yuan