[sheepdog] [PATCH v1 2/5] sheep: add disk information into sd_node

[Liu Yuan]

On Tue, Apr 29, 2014 at 10:53:42AM +0800, Robin Dong wrote:
> From: Robin Dong <sanbai at taobao.com>
> 
> Add disk information into sd_node therefore it will be sended
> to cluster shared region (corosync/zookeeper, etc.) and be
> aware of by all nodes in this cluster.
> 
> The new size of sd_node is 592 bytes and the max size of zookeeper
> is 983080, so the SD_MAX_NODES is 983080/(592 * 2) = 830.
> 
> Signed-off-by: Robin Dong <sanbai at taobao.com>
> ---
>  include/internal_proto.h | 21 ++++++++++++++--
>  sheep/group.c            |  9 ++++++-
>  sheep/md.c               | 64 +++++++++++++++++++++++++++++++-----------------
>  sheep/sheep_priv.h       | 24 ++++++++++++++++++
>  4 files changed, 92 insertions(+), 26 deletions(-)
> 
> diff --git a/include/internal_proto.h b/include/internal_proto.h
> index 7db3769..3a47b85 100644
> --- a/include/internal_proto.h
> +++ b/include/internal_proto.h
> @@ -42,7 +42,11 @@
>   * Currently, only zookeeper driver support SD_MAX_NODES nodes because
>   * its message buffer size is large enough to hold nodes[SD_MAX_NODES].
>   */
> -#define SD_MAX_NODES 6144
> +#ifdef HAVE_DISKVNODES
> +	#define SD_MAX_NODES 830
> +#else
> +	#define SD_MAX_NODES 6144
> +#endif
>  #define SD_DEFAULT_VNODES 128
>  
>  /*
> @@ -148,7 +152,17 @@ struct node_id {
>  	uint8_t pad[4];
>  };
>  
> -#define SD_NODE_SIZE 80
> +struct disk_info {
> +	uint64_t disk_id;
> +	uint64_t disk_space;
> +};
> +
> +#ifdef HAVE_DISKVNODES
> +	#define DISK_MAX     32
> +	#define SD_NODE_SIZE (80 + sizeof(struct disk_info) * DISK_MAX)
> +#else
> +	#define SD_NODE_SIZE 80
> +#endif
>  
>  struct sd_node {
>  	struct rb_node  rb;
> @@ -156,6 +170,9 @@ struct sd_node {
>  	uint16_t	nr_vnodes;
>  	uint32_t	zone;
>  	uint64_t        space;
> +#ifdef HAVE_DISKVNODES
> +	struct disk_info disks[DISK_MAX];
> +#endif
>  };
>  
>  /*
> diff --git a/sheep/group.c b/sheep/group.c
> index 4114dfb..b5cc484 100644
> --- a/sheep/group.c
> +++ b/sheep/group.c
> @@ -965,7 +965,12 @@ static void update_node_size(struct sd_node *node)
>  	if (unlikely(!n))
>  		panic("can't find %s", node_to_str(node));
>  	n->space = node->space;
> -
> +#ifdef HAVE_DISKVNODES
> +	memset(n->disks, 0, sizeof(struct disk_info) * DISK_MAX);
> +	for (int i = 0; i < DISK_MAX; i++)
> +		if (node->disks[i].disk_id)
> +			n->disks[i] = node->disks[i];
> +#endif
>  	put_vnode_info(cur_vinfo);
>  }
>  
> @@ -1026,6 +1031,8 @@ int create_cluster(int port, int64_t zone, int nr_vnodes,
>  
>  	sys->this_node.space = sys->disk_space;
>  
> +	update_node_disks();
> +
>  	sys->cinfo.epoch = get_latest_epoch();
>  	if (sys->cinfo.epoch) {
>  		sys->cinfo.nr_nodes = epoch_log_read(sys->cinfo.epoch,
> diff --git a/sheep/md.c b/sheep/md.c
> index 8154ee6..bec8853 100644
> --- a/sheep/md.c
> +++ b/sheep/md.c
> @@ -17,27 +17,7 @@
>  
>  #define NONE_EXIST_PATH "/all/disks/are/broken/,ps/əʌo7/!"
>  
> -struct disk {
> -	struct rb_node rb;
> -	char path[PATH_MAX];
> -	uint64_t space;
> -};
> -
> -struct vdisk {
> -	struct rb_node rb;
> -	struct disk *disk;
> -	uint64_t hash;
> -};
> -
> -struct md {
> -	struct rb_root vroot;
> -	struct rb_root root;
> -	struct sd_rw_lock lock;
> -	uint64_t space;
> -	uint32_t nr_disks;
> -};
> -
> -static struct md md = {
> +struct md md = {
>  	.vroot = RB_ROOT,
>  	.root = RB_ROOT,
>  	.lock = SD_RW_LOCK_INITIALIZER,
> @@ -87,7 +67,7 @@ static struct vdisk *oid_to_vdisk(uint64_t oid)
>  	return hval_to_vdisk(sd_hash_oid(oid));
>  }
>  
> -static void create_vdisks(struct disk *disk)
> +static void create_vdisks(const struct disk *disk)
>  {
>  	uint64_t hval = sd_hash(disk->path, strlen(disk->path));
>  	int nr = vdisk_number(disk);
> @@ -751,6 +731,42 @@ static inline void md_del_disk(const char *path)
>  	md_remove_disk(disk);
>  }
>  
> +#ifdef HAVE_DISKVNODES
> +void update_node_disks(void)
> +{
> +	const struct disk *disk;
> +	int i = 0;
> +	bool rb_empty = false;

this is not necessary. RB_EMPTY_ROOT is good enough.

> +
> +	if (!sys)
> +		return;
> +
> +	memset(sys->this_node.disks, 0, sizeof(struct disk_info) * DISK_MAX);
> +	sd_read_lock(&md.lock);
> +	rb_for_each_entry(disk, &md.root, rb) {
> +		sys->this_node.disks[i].disk_id =
> +			sd_hash(disk->path, strlen(disk->path));
> +		sys->this_node.disks[i].disk_space = disk->space;
> +		i++;
> +	}
> +	sd_rw_unlock(&md.lock);
> +
> +	if (md.vroot.rb_node == NULL)
> +		rb_empty = true;
> +	sd_write_lock(&md.lock);
> +	rb_for_each_entry(disk, &md.root, rb) {
> +		if (!rb_empty)

use RB_EMPTY_ROOT macro directly for if condition.

Thanks,
Yuan