[sheepdog] [PATCH v2 1/2] sheep/http: add option 'buffer=' to adjust size of buffer for kv

[Robin Dong]

From: Robin Dong <sanbai at taobao.com>

In some testing environment, large size of buffer will make
the performance of swift bad because kv will split buffer into too many
requests and send out, which lead to congestion in network; but small size
of buffer will make the speed of receiving http request slow. After all,
we found out that 8MB is the best value for our environment.

Even the 8MB is good, the size of buffer is difficult to choose in different
environment (different hardware of servers, different number of nodes, etc.),
so we should add this option to let users choose buffer size.

Signed-off-by: Robin Dong <sanbai at taobao.com>
---
v1-->v2:
  1. use 32M as default value
  2. support 'buffer=64M' notation instead of 'buffer=64

 sheep/http/http.c | 21 +++++++++++++++++++++
 sheep/http/http.h |  4 ++++
 sheep/http/kv.c   |  8 ++++----
 sheep/sheep.c     |  5 +++--
 4 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/sheep/http/http.c b/sheep/http/http.c
index cf658c6..6a04252 100644
--- a/sheep/http/http.c
+++ b/sheep/http/http.c
@@ -346,6 +346,26 @@ static int http_opt_port_parser(const char *s)
 	return 0;
 }
 
+static int http_opt_buffer_parser(const char *s)
+{
+	const uint64_t max_buffer_size = SD_DATA_OBJ_SIZE * 256;
+	const uint64_t min_buffer_size = SD_DATA_OBJ_SIZE;
+	uint64_t buffer_size;
+
+	if (option_parse_size(s, &buffer_size) < 0)
+		return -1;
+	if (buffer_size < min_buffer_size || buffer_size > max_buffer_size) {
+		sd_err("Invalid buffer option '%s': size must be "
+		       "between %"PRIu64"M and %"PRIu64"M", s,
+		       min_buffer_size / 1024 / 1024,
+		       max_buffer_size / 1024 / 1024);
+		       return -1;
+	}
+	kv_rw_buffer = round_up(buffer_size, SD_DATA_OBJ_SIZE);
+	sd_info("kv_rw_buffer: %"PRIu64, kv_rw_buffer);
+	return 0;
+}
+
 static int http_opt_default_parser(const char *s)
 {
 	struct http_driver *hdrv;
@@ -375,6 +395,7 @@ static int http_opt_default_parser(const char *s)
 static struct option_parser http_opt_parsers[] = {
 	{ "host=", http_opt_host_parser },
 	{ "port=", http_opt_port_parser },
+	{ "buffer=", http_opt_buffer_parser },
 	{ "", http_opt_default_parser },
 	{ NULL, NULL },
 };
diff --git a/sheep/http/http.h b/sheep/http/http.h
index 2d837c6..5bcd528 100644
--- a/sheep/http/http.h
+++ b/sheep/http/http.h
@@ -115,6 +115,10 @@ int http_request_writef(struct http_request *req, const char *fmt, ...);
 #define SD_MAX_BUCKET_NAME 256
 #define SD_MAX_OBJECT_NAME 1024
 
+/* This default value shows best performance in test */
+#define DEFAULT_KV_RW_BUFFER (SD_DATA_OBJ_SIZE * 8)
+extern uint64_t kv_rw_buffer;
+
 /* Account operations */
 int kv_create_account(const char *account);
 int kv_read_account_meta(struct http_request *req, const char *account);
diff --git a/sheep/http/kv.c b/sheep/http/kv.c
index 6b51b7b..de8c6af 100644
--- a/sheep/http/kv.c
+++ b/sheep/http/kv.c
@@ -16,6 +16,8 @@
 #include "sheep_priv.h"
 #include "http.h"
 
+uint64_t kv_rw_buffer = DEFAULT_KV_RW_BUFFER;
+
 struct kv_bnode {
 	char name[SD_MAX_BUCKET_NAME];
 	uint64_t object_count;
@@ -644,8 +646,6 @@ static int vdi_read_write(uint32_t vid, char *data, size_t length,
 	return local_req_wait(iocb);
 }
 
-#define MAX_RW_BUFFER (SD_DATA_OBJ_SIZE * 25) /* No rationale yet */
-
 static int onode_populate_extents(struct kv_onode *onode,
 				  struct http_request *req)
 {
@@ -654,7 +654,7 @@ static int onode_populate_extents(struct kv_onode *onode,
 	int ret;
 	char *data_buf = NULL;
 	uint32_t data_vid = onode->data_vid;
-	uint64_t write_buffer_size = MIN(MAX_RW_BUFFER, req->data_length);
+	uint64_t write_buffer_size = MIN(kv_rw_buffer, req->data_length);
 
 	count = DIV_ROUND_UP(req->data_length, SD_DATA_OBJ_SIZE);
 	sys->cdrv->lock(data_vid);
@@ -836,7 +836,7 @@ static int onode_read_extents(struct kv_onode *onode, struct http_request *req)
 	uint64_t off = req->offset, len = req->data_length;
 	int ret;
 	char *data_buf = NULL;
-	uint64_t read_buffer_size = MIN(MAX_RW_BUFFER, onode->size);
+	uint64_t read_buffer_size = MIN(kv_rw_buffer, onode->size);
 
 	data_buf = xmalloc(read_buffer_size);
 	total_size = len;
diff --git a/sheep/sheep.c b/sheep/sheep.c
index e18dabf..7ec3658 100644
--- a/sheep/sheep.c
+++ b/sheep/sheep.c
@@ -47,10 +47,11 @@ static const char http_help[] =
 "Available arguments:\n"
 "\thost=: specify a host to communicate with http server (default: localhost)\n"
 "\tport=: specify a port to communicate with http server (default: 8000)\n"
+"\tbuffer=: specify buffer size for http request (default: 32M)\n"
 "\tswift: enable swift API\n"
-"Example:\n\t$ sheep -r host=localhost,port=7001,swift ...\n"
+"Example:\n\t$ sheep -r host=localhost,port=7001,buffer=64M,swift ...\n"
 "This tries to enable Swift API and use localhost:7001 to\n"
-"communicate with http server.\n";
+"communicate with http server, using 64MB buffer.\n";
 
 static const char myaddr_help[] =
 "Example:\n\t$ sheep -y 192.168.1.1:7000 ...\n"
-- 
1.7.12.4