[sheepdog] [PATCH v3 3/3] sheep/http: add lock to protect container and object
Robin Dong
robin.k.dong at gmail.com
Wed Dec 11 11:14:43 CET 2013
From: Robin Dong <sanbai at taobao.com>
Add locks to protect containers and objects when users create/delete
containers or create/delete objects in the same time.
Signed-off-by: Robin Dong <sanbai at taobao.com>
---
sheep/http/kv.c | 252 ++++++++++++++++++++++++++++++++---------------------
sheep/http/swift.c | 10 ++-
2 files changed, 162 insertions(+), 100 deletions(-)
diff --git a/sheep/http/kv.c b/sheep/http/kv.c
index fb66dfa..4d690cb 100644
--- a/sheep/http/kv.c
+++ b/sheep/http/kv.c
@@ -218,22 +218,22 @@ static int kv_get_account(const char *account, uint32_t *nr_buckets)
ret = lookup_vdi(account, &account_vid);
if (ret != SD_RES_SUCCESS)
- return ret;
+ goto out;
/* read account vdi out */
oid = vid_to_vdi_oid(account_vid);
ret = sd_read_object(oid, (char *)&inode, sizeof(struct sd_inode), 0);
if (ret != SD_RES_SUCCESS) {
sd_err("Failed to read inode header %lx", oid);
- return ret;
+ goto out;
}
struct list_buckets_arg arg = {NULL, NULL, NULL, 0};
traverse_btree(sheep_bnode_reader, &inode, list_buckets_cb, &arg);
if (nr_buckets)
*nr_buckets = arg.bucket_counter;
-
- return SD_RES_SUCCESS;
+out:
+ return ret;
}
int kv_read_account(const char *account, uint32_t *nr_buckets)
@@ -528,21 +528,22 @@ out:
}
static int kv_get_bucket(struct sd_inode *account_inode, const char *account,
- const char *bucket)
+ const char *bucket, uint32_t *account_vid)
{
char vdi_name[SD_MAX_VDI_LEN];
uint64_t oid;
- uint32_t account_vid, bucket_vid;
+ uint32_t bucket_vid;
int ret;
- ret = lookup_vdi(account, &account_vid);
+ ret = lookup_vdi(account, account_vid);
if (ret != SD_RES_SUCCESS) {
sd_err("Failed to find account %s", account);
return -1;
}
+ sys->cdrv->lock(*account_vid);
/* read account vdi out */
- oid = vid_to_vdi_oid(account_vid);
+ oid = vid_to_vdi_oid(*account_vid);
ret = sd_read_object(oid, (char *)account_inode,
sizeof(struct sd_inode), 0);
if (ret != SD_RES_SUCCESS)
@@ -559,15 +560,19 @@ int kv_create_bucket(const char *account, const char *bucket)
{
struct sd_inode inode;
uint64_t hval, i;
+ uint32_t account_vid;
int ret;
- ret = kv_get_bucket(&inode, account, bucket);
+ ret = kv_get_bucket(&inode, account, bucket, &account_vid);
/* if bucket is exists, return SD_RES_VDI_EXIST */
if (!ret) {
sd_err("bucket %s is exists.", bucket);
- return SD_RES_VDI_EXIST;
- } else if (ret < 0)
- return -1;
+ ret = SD_RES_VDI_EXIST;
+ goto out;
+ } else if (ret < 0) {
+ ret = -1;
+ goto out;
+ }
sd_debug("read account inode success");
@@ -581,7 +586,7 @@ int kv_create_bucket(const char *account, const char *bucket)
continue;
} else if (ret < 0) {
sd_err("Failed to add bucket");
- return ret;
+ goto out;
}
/* add bucket success */
sd_debug("add bucket success");
@@ -590,9 +595,12 @@ int kv_create_bucket(const char *account, const char *bucket)
if (i >= MAX_BUCKETS) {
sd_err("Containers in vdi %s is full!", account);
- return -1;
+ ret = -1;
+ goto out;
}
- return 0;
+out:
+ sys->cdrv->unlock(account_vid);
+ return ret;
}
int kv_read_bucket(const char *account, const char *bucket)
@@ -612,12 +620,13 @@ int kv_delete_bucket(const char *account, const char *bucket)
{
struct sd_inode inode;
uint64_t hval, i;
+ uint32_t account_vid;
int ret;
- ret = kv_get_bucket(&inode, account, bucket);
+ ret = kv_get_bucket(&inode, account, bucket, &account_vid);
if (ret) {
sd_err("Failed to get bucket");
- return ret;
+ goto out;
}
hval = sd_hash(bucket, strlen(bucket));
@@ -629,7 +638,7 @@ int kv_delete_bucket(const char *account, const char *bucket)
continue;
} else if (ret < 0) {
sd_err("Failed to delete bucket %d", ret);
- return ret;
+ goto out;
}
/* delete bucket success */
sd_debug("delete bucket success");
@@ -638,9 +647,12 @@ int kv_delete_bucket(const char *account, const char *bucket)
if (i >= MAX_BUCKETS) {
sd_err("Can't find bucket %s", bucket);
- return SD_RES_NO_VDI;
+ ret = SD_RES_NO_VDI;
+ goto out;
}
- return SD_RES_SUCCESS;
+out:
+ sys->cdrv->unlock(account_vid);
+ return ret;
}
int kv_list_buckets(struct http_request *req, const char *account,
@@ -659,17 +671,20 @@ int kv_list_buckets(struct http_request *req, const char *account,
/* read account vdi out */
oid = vid_to_vdi_oid(account_vid);
+ sys->cdrv->lock(account_vid);
ret = sd_read_object(oid, (char *)&account_inode,
sizeof(struct sd_inode), 0);
if (ret != SD_RES_SUCCESS) {
sd_err("Failed to read account inode header %lx", oid);
- return ret;
+ goto out;
}
struct list_buckets_arg arg = {req, opaque, cb, 0};
traverse_btree(sheep_bnode_reader, &account_inode,
list_buckets_cb, &arg);
- return SD_RES_SUCCESS;
+out:
+ sys->cdrv->unlock(account_vid);
+ return ret;
}
/*
@@ -867,13 +882,72 @@ out:
return ret;
}
+static int kv_create_extent_onode(struct http_request *req, uint32_t data_vid,
+ struct kv_onode *onode, ssize_t *total_size)
+{
+ ssize_t size;
+ uint64_t start = 0, count, limit, block;
+ int ret;
+ char *data_buf = NULL;
+
+ count = (req->data_length + SD_DATA_OBJ_SIZE + 1) / SD_DATA_OBJ_SIZE;
+ sys->cdrv->lock(data_vid);
+ ret = oalloc_new_prepare(data_vid, &start, count);
+ sys->cdrv->unlock(data_vid);
+ if (ret != SD_RES_SUCCESS) {
+ sd_err("Failed to prepare allocation of %lu bytes!",
+ req->data_length);
+ ret = -1;
+ goto out;
+ }
+
+ /* receive and write data at first, then write onode */
+ data_buf = xmalloc(SD_DATA_OBJ_SIZE);
+
+ sd_debug("start: %lu, count: %lu", start, count);
+ for (block = start, limit = start + count; block < limit; block++) {
+ sd_debug("block: %lu, limit: %lu", block, limit);
+ size = http_request_read(req, data_buf, SD_DATA_OBJ_SIZE);
+ *total_size += size;
+ ret = sd_write_object(vid_to_data_oid(data_vid, block),
+ data_buf, size, 0, true);
+ if (ret != SD_RES_SUCCESS) {
+ sd_err("Failed to write data object for %" PRIx32" %s",
+ data_vid, sd_strerror(ret));
+ goto out;
+ }
+ if (size < SD_DATA_OBJ_SIZE)
+ break;
+ }
+
+ sd_debug("DATA_LENGTH: %lu, total size: %lu, last blocks: %lu",
+ req->data_length, *total_size, start);
+
+ sd_debug("finish start: %lu, count: %lu", start, count);
+ sys->cdrv->lock(data_vid);
+ ret = oalloc_new_finish(data_vid, start, count);
+ sys->cdrv->unlock(data_vid);
+ if (ret != SD_RES_SUCCESS) {
+ sd_err("Failed to finish allocation of %lu bytes!",
+ req->data_length);
+ ret = -1;
+ goto out;
+ }
+
+ onode->o_extent[0].start = start;
+ onode->o_extent[0].count = count;
+ onode->hdr.nr_extent = 1;
+out:
+ return ret;
+}
+
int kv_create_object(struct http_request *req, const char *account,
const char *bucket, const char *name)
{
struct kv_onode *onode;
ssize_t size, total_size = 0;
int ret;
- uint64_t hval, start = 0, count, block, limit;
+ uint64_t hval;
uint32_t vid, data_vid;
struct timeval tv;
char vdi_name[SD_MAX_VDI_LEN];
@@ -904,54 +978,10 @@ int kv_create_object(struct http_request *req, const char *account,
}
total_size = size;
} else {
- sd_debug("data_length: %lu, %lu", req->data_length,
- SD_DATA_OBJ_SIZE);
- count = (req->data_length + SD_DATA_OBJ_SIZE + 1) /
- SD_DATA_OBJ_SIZE;
- ret = oalloc_new_prepare(data_vid, &start, count);
- if (ret != SD_RES_SUCCESS) {
- sd_err("Failed to prepare allocation of %lu bytes!",
- req->data_length);
- ret = -1;
- goto out;
- }
-
- /* receive and write data at first, then write onode */
- data_buf = xmalloc(SD_DATA_OBJ_SIZE);
-
- sd_debug("start: %lu, count: %lu", start, count);
- for (block = start, limit = start + count;
- block < limit; block++) {
- sd_debug("block: %lu, limit: %lu", block, limit);
- size = http_request_read(req, data_buf,
- SD_DATA_OBJ_SIZE);
- total_size += size;
- ret = sd_write_object(vid_to_data_oid(data_vid, block),
- data_buf, size, 0, true);
- if (ret != SD_RES_SUCCESS) {
- sd_err("Failed to write data object for %"
- PRIx32" %s", data_vid, sd_strerror(ret));
- goto out;
- }
- if (size < SD_DATA_OBJ_SIZE)
- break;
- }
-
- sd_debug("DATA_LENGTH: %lu, total size: %lu, last blocks: %lu",
- req->data_length, total_size, start);
-
- sd_debug("finish start: %lu, count: %lu", start, count);
- ret = oalloc_new_finish(data_vid, start, count);
- if (ret != SD_RES_SUCCESS) {
- sd_err("Failed to finish allocation of %lu bytes!",
- req->data_length);
- ret = -1;
+ /* for extented onode */
+ ret = kv_create_extent_onode(req, data_vid, onode, &total_size);
+ if (ret != SD_RES_SUCCESS)
goto out;
- }
-
- onode->o_extent[0].start = start;
- onode->o_extent[0].count = count;
- onode->hdr.nr_extent = 1;
}
/* after write data, we write onode now */
@@ -963,6 +993,8 @@ int kv_create_object(struct http_request *req, const char *account,
onode->hdr.size = total_size;
onode->hdr.data_vid = data_vid;
+ /* lock vid which stores onode */
+ sys->cdrv->lock(vid);
hval = sd_hash(name, strlen(name));
for (int i = 0; i < MAX_DATA_OBJS; i++) {
uint32_t idx = (hval + i) % MAX_DATA_OBJS;
@@ -982,6 +1014,7 @@ int kv_create_object(struct http_request *req, const char *account,
/* no free space to create a object */
http_response_header(req, SERVICE_UNAVAILABLE);
out:
+ sys->cdrv->unlock(vid);
free(onode);
free(data_buf);
return ret;
@@ -1037,11 +1070,11 @@ static int do_kv_read_object(struct http_request *req, const char *obj_name,
case SD_RES_NO_OBJ:
sd_info("object %s doesn't exist", obj_name);
http_response_header(req, NOT_FOUND);
- return -1;
+ goto out;
default:
sd_err("failed to read %s, %s", req->uri, sd_strerror(ret));
http_response_header(req, INTERNAL_SERVER_ERROR);
- return -1;
+ goto out;
}
if (strcmp(onode->hdr.name, obj_name) == 0) {
@@ -1050,22 +1083,25 @@ static int do_kv_read_object(struct http_request *req, const char *obj_name,
if (onode->hdr.inlined)
http_request_write(req, onode->data, onode->hdr.size);
else {
+ sys->cdrv->unlock(vid);
ret = kv_read_extent_onode(req, onode);
- if (ret) {
+ sys->cdrv->lock(vid);
+ if (ret != SD_RES_SUCCESS) {
sd_err("Failed to read extent onode");
- return -1;
+ goto out;
}
}
- }
-
- return 0;
+ } else
+ ret = SD_RES_OBJ_TAKEN;
+out:
+ return ret;
}
int kv_read_object(struct http_request *req, const char *account,
const char *bucket, const char *object)
{
- struct kv_onode *onode;
- int ret;
+ struct kv_onode *onode = NULL;
+ int ret, i;
uint64_t hval;
uint32_t vid;
char vdi_name[SD_MAX_VDI_LEN];
@@ -1073,25 +1109,33 @@ int kv_read_object(struct http_request *req, const char *account,
snprintf(vdi_name, SD_MAX_VDI_LEN, "%s/%s", account, bucket);
ret = lookup_bucket(req, vdi_name, &vid);
if (ret < 0)
- return ret;
+ goto out;
onode = xzalloc(sizeof(*onode));
+ sys->cdrv->lock(vid);
hval = sd_hash(object, strlen(object));
- for (int i = 0; i < MAX_DATA_OBJS; i++) {
+ for (i = 0; i < MAX_DATA_OBJS; i++) {
uint32_t idx = (hval + i) % MAX_DATA_OBJS;
- do_kv_read_object(req, object, onode, vid, idx);
- if (req->status != UNKNOWN) {
- free(onode);
- return 0;
+ ret = do_kv_read_object(req, object, onode, vid, idx);
+ switch (ret) {
+ case SD_RES_SUCCESS:
+ goto out;
+ case SD_RES_OBJ_TAKEN:
+ break;
+ default:
+ ret = -1;
+ goto out;
}
}
+ if (i >= MAX_DATA_OBJS)
+ ret = -1;
+out:
+ sys->cdrv->unlock(vid);
free(onode);
-
- http_response_header(req, NOT_FOUND);
- return -1;
+ return ret;
}
static int do_kv_update_object(struct http_request *req, const char *obj_name,
@@ -1194,11 +1238,11 @@ static int do_kv_delete_object(struct http_request *req, const char *obj_name,
case SD_RES_NO_OBJ:
sd_info("object %s doesn't exist", obj_name);
http_response_header(req, NOT_FOUND);
- return -1;
+ goto out;
default:
sd_err("failed to read %s, %s", req->uri, sd_strerror(ret));
http_response_header(req, INTERNAL_SERVER_ERROR);
- return -1;
+ goto out;
}
if (strcmp(name, obj_name) == 0) {
@@ -1236,7 +1280,8 @@ static int do_kv_delete_object(struct http_request *req, const char *obj_name,
sd_err("failed to free start %lu count %lu",
ext[i].start, ext[i].count);
}
- }
+ } else
+ ret = SD_RES_NO_OBJ;
out:
free(ext);
free(onode);
@@ -1246,7 +1291,7 @@ out:
int kv_delete_object(struct http_request *req, const char *account,
const char *bucket, const char *object)
{
- int ret;
+ int ret, i;
uint64_t hval;
uint32_t vid;
char vdi_name[SD_MAX_VDI_LEN];
@@ -1256,17 +1301,28 @@ int kv_delete_object(struct http_request *req, const char *account,
if (ret < 0)
return ret;
+ sys->cdrv->lock(vid);
hval = sd_hash(object, strlen(object));
- for (int i = 0; i < MAX_DATA_OBJS; i++) {
+ for (i = 0; i < MAX_DATA_OBJS; i++) {
uint32_t idx = (hval + i) % MAX_DATA_OBJS;
- do_kv_delete_object(req, object, vid, idx);
- if (req->status != UNKNOWN)
- return 0;
+ ret = do_kv_delete_object(req, object, vid, idx);
+ switch (ret) {
+ case SD_RES_SUCCESS:
+ goto out;
+ case SD_RES_NO_OBJ:
+ break;
+ default:
+ ret = -1;
+ goto out;
+ }
}
- http_response_header(req, NOT_FOUND);
- return -1;
+ if (i >= MAX_DATA_OBJS)
+ ret = -1;
+out:
+ sys->cdrv->unlock(vid);
+ return ret;
}
int kv_list_objects(struct http_request *req, const char *account,
diff --git a/sheep/http/swift.c b/sheep/http/swift.c
index c76388c..d9f5a76 100644
--- a/sheep/http/swift.c
+++ b/sheep/http/swift.c
@@ -178,7 +178,10 @@ static void swift_head_object(struct http_request *req, const char *account,
static void swift_get_object(struct http_request *req, const char *account,
const char *container, const char *object)
{
- kv_read_object(req, account, container, object);
+ int ret;
+ ret = kv_read_object(req, account, container, object);
+ if (ret)
+ http_response_header(req, NOT_FOUND);
}
static void swift_put_object(struct http_request *req, const char *account,
@@ -196,7 +199,10 @@ static void swift_post_object(struct http_request *req, const char *account,
static void swift_delete_object(struct http_request *req, const char *account,
const char *container, const char *object)
{
- kv_delete_object(req, account, container, object);
+ int ret;
+ ret = kv_delete_object(req, account, container, object);
+ if (ret)
+ http_response_header(req, NOT_FOUND);
}
/* Swift driver interfaces */
--
1.7.12.4
More information about the sheepdog
mailing list