[sheepdog] [PATCH] sheep/http: use a dynamic buffer size for read/write buffer allocation

[Robin Dong]

2014/1/12 Liu Yuan <namei.unix at gmail.com>

> - make the max of buffer size as 1000MB (reason inlined in the source)
> - allocate the buffer size based on the request length
>
> Signed-off-by: Liu Yuan <namei.unix at gmail.com>
> ---
>  sheep/http/kv.c | 16 +++++++++++-----
>  1 file changed, 11 insertions(+), 5 deletions(-)
>
> diff --git a/sheep/http/kv.c b/sheep/http/kv.c
> index dd5ca56..6c73d30 100644
> --- a/sheep/http/kv.c
> +++ b/sheep/http/kv.c
> @@ -673,7 +673,11 @@ static int vdi_read_write(uint32_t vid, char *data,
> size_t length,
>         return local_req_wait(iocb);
>  }
>
> -#define READ_WRITE_BUFFER (SD_DATA_OBJ_SIZE * 25) /* no rationale */
> +/*
> + * For a 10Gb NIC, we have bandwidth over 1000 MB/s, so we can have this
> as the
> + * upper limit for PUT/GET bandwidth.
> + */
> +#define MAX_RW_BUFFER (SD_DATA_OBJ_SIZE * 250)
>

I think 1GB buffer is too large for some old computer.
And , 1GB buffer is not the essential precondition for reaching the
bandwidth of 1000 MB/s


>  static int onode_populate_extents(struct kv_onode *onode,
>                                   struct http_request *req)
> @@ -683,6 +687,7 @@ static int onode_populate_extents(struct kv_onode
> *onode,
>         int ret;
>         char *data_buf = NULL;
>         uint32_t data_vid = onode->data_vid;
> +       uint64_t write_buffer_size = MIN(MAX_RW_BUFFER, req->data_length);
>
>         count = DIV_ROUND_UP(req->data_length, SD_DATA_OBJ_SIZE);
>         sys->cdrv->lock(data_vid);
> @@ -694,11 +699,11 @@ static int onode_populate_extents(struct kv_onode
> *onode,
>                 goto out;
>         }
>
> -       data_buf = xmalloc(READ_WRITE_BUFFER);
> +       data_buf = xmalloc(write_buffer_size);
>         offset = start * SD_DATA_OBJ_SIZE;
>         total = req->data_length;
>         while (done < total) {
> -               size = http_request_read(req, data_buf, READ_WRITE_BUFFER);
> +               size = http_request_read(req, data_buf, write_buffer_size);
>                 ret = vdi_read_write(data_vid, data_buf, size, offset,
> false);
>                 if (ret != SD_RES_SUCCESS) {
>                         sd_err("Failed to write data object for %s, %s",
> @@ -863,15 +868,16 @@ static int onode_read_extents(struct kv_onode
> *onode, struct http_request *req)
>         uint64_t size, total, total_size, offset, done = 0, i;
>         int ret;
>         char *data_buf = NULL;
> +       uint64_t read_buffer_size = MIN(MAX_RW_BUFFER, onode->size);
>
> -       data_buf = xmalloc(READ_WRITE_BUFFER);
> +       data_buf = xmalloc(read_buffer_size);
>         total_size = onode->size;
>         for (i = 0; i < onode->nr_extent; i++) {
>                 ext = onode->o_extent + i;
>                 total = min(ext->count * SD_DATA_OBJ_SIZE, total_size);
>                 offset = ext->start * SD_DATA_OBJ_SIZE;
>                 while (done < total) {
> -                       size = MIN(total - done, READ_WRITE_BUFFER);
> +                       size = MIN(total - done, read_buffer_size);
>                         ret = vdi_read_write(onode->data_vid, data_buf,
>                                              size, offset, true);
>                         if (ret != SD_RES_SUCCESS) {
> --
> 1.8.1.2
>
> --
> sheepdog mailing list
> sheepdog at lists.wpkg.org
> http://lists.wpkg.org/mailman/listinfo/sheepdog
>



-- 
--
Best Regard
Robin Dong
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.wpkg.org/pipermail/sheepdog/attachments/20140115/eefdf21d/attachment-0004.html>