[sheepdog] [PATCH v1 2/3] sheepfs: implement the http interface of sheepfs by using libcurl

[Liu Yuan]

On Thu, Jan 23, 2014 at 01:57:52PM +0800, Robin Dong wrote:
> From: Robin Dong <sanbai at taobao.com>
> 
> Use libcurl to call the interface of http for object-storage and implement
> the read/write operation.
> 
> Signed-off-by: Robin Dong <sanbai at taobao.com>
> ---
>  sheepfs/Makefile.am |   2 +-
>  sheepfs/core.c      |   1 +
>  sheepfs/http.c      | 269 +++++++++++++++++++++++++++++++++++++++++++++++++++-
>  sheepfs/sheepfs.h   |   5 +
>  4 files changed, 275 insertions(+), 2 deletions(-)
> 
> diff --git a/sheepfs/Makefile.am b/sheepfs/Makefile.am
> index 64f84f2..a5e84ba 100644
> --- a/sheepfs/Makefile.am
> +++ b/sheepfs/Makefile.am
> @@ -26,7 +26,7 @@ sbin_PROGRAMS		= sheepfs
>  sheepfs_SOURCES		= core.c cluster.c vdi.c shadow_file.c volume.c node.c \
>  			  config.c http.c
>  
> -sheepfs_LDADD	  	= ../lib/libsheepdog.a $(fuse_LIBS) $(LIBS) -lpthread
> +sheepfs_LDADD	  	= ../lib/libsheepdog.a $(fuse_LIBS) $(LIBS) -lpthread -lcurl
>  sheepfs_DEPENDENCIES	= ../lib/libsheepdog.a
>  
>  noinst_HEADERS		= sheepfs.h
> diff --git a/sheepfs/core.c b/sheepfs/core.c
> index 2f900bc..f80270b 100644
> --- a/sheepfs/core.c
> +++ b/sheepfs/core.c
> @@ -79,6 +79,7 @@ static struct sheepfs_file_operation {
>  	[OP_HTTP_ADDRESS]   = { http_address_read, http_address_write,
>  				http_address_get_size },
>  	[OP_HTTP_OBJECT]    = { NULL, http_object_write },
> +	[OP_OBJECT]         = { object_read, NULL, object_get_size },
>  };
>  
>  __printf(3, 4)
> diff --git a/sheepfs/http.c b/sheepfs/http.c
> index e220e9c..2762407 100644
> --- a/sheepfs/http.c
> +++ b/sheepfs/http.c
> @@ -18,6 +18,7 @@
>  #include <stdlib.h>
>  #include <stdio.h>
>  #include <time.h>
> +#include <curl/curl.h>
>  
>  #include "strbuf.h"
>  #include "sheepfs.h"
> @@ -27,6 +28,9 @@
>  #define PATH_HTTP_ADDRESS	"/http/address"
>  #define PATH_HTTP_OBJECT	"/http/object"
>  
> +#define HTTP_SIZE_NAME		"user.object.size"
> +#define HTTP_SIZE_SIZE		sizeof(uint64_t)
> +
>  int create_http_layout(void)
>  {
>  	if (shadow_dir_create(PATH_HTTP) < 0)
> @@ -64,8 +68,271 @@ size_t http_address_get_size(const char *path)
>  	return 0;
>  }
>  
> +static uint64_t curl_get_object_size(const char *url)
> +{
> +	CURL *curl;
> +	CURLcode res;
> +	double content_length;
> +
> +	curl = curl_easy_init();
> +	if (!curl) {
> +		sd_err("Failed to init curl");
> +		goto out;
> +	}
> +
> +	curl_easy_setopt(curl, CURLOPT_FAILONERROR, true);
> +	curl_easy_setopt(curl, CURLOPT_NOBODY, true);
> +	curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD");
> +	curl_easy_setopt(curl, CURLOPT_URL, url);
> +	res = curl_easy_perform(curl);
> +	if (res == CURLE_OK) {
> +		res = curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD,
> +					&content_length);
> +		if (res != CURLE_OK) {
> +			sd_err("Failed to get size of object %s",
> +			       curl_easy_strerror(res));
> +			content_length = 0;
> +		}
> +	}
> +out:
> +	curl_easy_cleanup(curl);
> +	return (uint64_t)content_length;
> +}
> +
> +struct buffer_s {
> +	char *mem;
> +	size_t current_size;
> +	size_t total_size;
> +};
> +
> +static size_t write_cb(void *contents, size_t size, size_t nmemb, void *userp)
> +{
> +	size_t real_size = size * nmemb;
> +	struct buffer_s *buff = (struct buffer_s *)userp;
> +
> +	if ((buff->current_size + real_size) > buff->total_size)
> +		real_size = buff->total_size - buff->current_size;
> +	memcpy(buff->mem + buff->current_size, contents, real_size);
> +	buff->current_size += real_size;
> +	return real_size;
> +}
> +
> +static size_t curl_read_object(const char *url, char *buf, size_t size,
> +			       off_t offset)
> +{
> +	CURL *curl;
> +	CURLcode res;
> +	char header[PATH_MAX];
> +	double content_length;
> +	struct buffer_s buffer = { 0 };
> +	struct curl_slist *headers = NULL;
> +
> +	curl = curl_easy_init();
> +	if (!curl) {
> +		sd_err("Failed to init curl");

sd_err is nonsense for sheepfs right now. use sheepfs_pr

Thanks
Yuan