[Sheepdog] [PATCH 5/9] farm: add trunk object
Liu Yuan
namei.unix at gmail.com
Wed Dec 21 10:17:37 CET 2011
From: Liu Yuan <tailai.ly at taobao.com>
trunk object is meta data that describes the structure of the data objects
at the timeline of snapshot being taken.
todo:
- add more description
Signed-off-by: Liu Yuan <tailai.ly at taobao.com>
---
sheep/farm.h | 26 +++++++
sheep/farm/trunk.c | 205 ++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 231 insertions(+), 0 deletions(-)
create mode 100644 sheep/farm/trunk.c
diff --git a/sheep/farm.h b/sheep/farm.h
index 51ede20..d808e3c 100644
--- a/sheep/farm.h
+++ b/sheep/farm.h
@@ -18,11 +18,17 @@
#include "sheepdog_proto.h"
#include "sheep.h"
#include "logger.h"
+#include "strbuf.h"
#define SHA1_LEN 20
#define HEX_LEN 40
#define NAME_LEN HEX_LEN
+#define TAG_LEN 6
+#define TAG_DATA "data\0\0"
+#define TAG_TRUNK "trunk\0"
+#define TAG_SNAP "snap\0\0"
+
struct sha1_file_hdr {
char tag[TAG_LEN];
uint64_t size;
@@ -30,6 +36,20 @@ struct sha1_file_hdr {
uint64_t reserved;
};
+struct trunk_entry {
+ uint64_t oid;
+ unsigned char sha1[SHA1_LEN];
+};
+
+struct __trunk_entry {
+ struct trunk_entry raw;
+ int flags;
+ struct list_head active_list;
+ struct hlist_node hash;
+};
+
+extern char *epoch_path;
+
/* sha1_file.c */
extern char *sha1_to_path(const unsigned char *sha1);
extern int sha1_file_write(unsigned char *buf, unsigned len, unsigned char *outsha1);
@@ -37,4 +57,10 @@ extern void * sha1_file_read(const unsigned char *sha1, struct sha1_file_hdr *hd
extern char * sha1_to_hex(const unsigned char *sha1);
extern int get_sha1_hex(const char *hex, unsigned char *sha1);
+/* trunk.c */
+extern int trunk_init(void);
+extern int trunk_file_write(unsigned char *outsha1);
+extern void *trunk_file_read(unsigned char *sha1, struct sha1_file_hdr *);
+extern int trunk_update_entry(uint64_t oid);
+
#endif
diff --git a/sheep/farm/trunk.c b/sheep/farm/trunk.c
new file mode 100644
index 0000000..c106304
--- /dev/null
+++ b/sheep/farm/trunk.c
@@ -0,0 +1,205 @@
+#include <pthread.h>
+#include <dirent.h>
+
+#include "farm.h"
+#include "strbuf.h"
+#include "list.h"
+#include "slabs.h"
+#include "util.h"
+#include "sheepdog_proto.h"
+
+#define TRUNK_ENTRY_DIRTY 0x00000001
+
+#define trunk_entry_size() sizeof(struct __trunk_entry)
+
+#define HASH_BITS 10
+#define HASH_SIZE (1 << HASH_BITS)
+
+static LIST_HEAD(trunk_active_list); /* no lock protection */
+static struct hlist_head trunk_hashtable[HASH_SIZE];
+static pthread_mutex_t hashtable_lock[HASH_SIZE] = { [0 ... HASH_SIZE - 1] = PTHREAD_MUTEX_INITIALIZER };
+static int trunk_slab_clsid;
+static unsigned int trunk_entry_active_nr;
+
+static inline int trunk_entry_is_dirty(struct __trunk_entry *entry)
+{
+ return entry->flags & TRUNK_ENTRY_DIRTY;
+}
+
+static inline void mark_trunk_entry_dirty(struct __trunk_entry *entry)
+{
+ entry->flags |= TRUNK_ENTRY_DIRTY;
+}
+
+static inline int hash(uint64_t oid)
+{
+ return hash_64(oid, HASH_BITS);
+}
+
+static inline struct __trunk_entry *alloc_trunk_entry(void)
+{
+ return (struct __trunk_entry *)slabs_alloc(trunk_slab_clsid);
+}
+
+static inline void free_trunk_entry(struct __trunk_entry *entry)
+{
+ return slabs_free(entry, trunk_slab_clsid);
+}
+
+static struct __trunk_entry *lookup_trunk_entry(uint64_t oid, int create)
+{
+ int h = hash(oid);
+ struct hlist_head *head = trunk_hashtable + h;
+ struct __trunk_entry *entry = NULL;
+ struct hlist_node *node;
+
+ pthread_mutex_lock(&hashtable_lock[h]);
+ if (hlist_empty(head))
+ goto not_found;
+
+ hlist_for_each_entry(entry, node, head, hash) {
+ if (entry->raw.oid == oid) {
+ dprintf("found node %lx\n", oid);
+ goto out;
+ }
+ }
+not_found:
+ if (create) {
+ entry = alloc_trunk_entry();
+ entry->raw.oid = oid;
+ hlist_add_head(&entry->hash, head);
+ trunk_entry_active_nr++;
+ list_add(&entry->active_list, &trunk_active_list);
+ dprintf("add node %lx\n", oid);
+ }
+out:
+ pthread_mutex_unlock(&hashtable_lock[h]);
+ return entry;
+}
+
+int trunk_init(void)
+{
+ DIR *dir;
+ struct dirent *d;
+ uint64_t oid;
+
+ trunk_slab_clsid = slabs_clsid(trunk_entry_size());
+ if (trunk_slab_clsid < 0)
+ panic("failed to get a sane id");
+ dprintf("slab class id: %u\n", trunk_slab_clsid);
+
+ dir = opendir(obj_path);
+ if (!dir)
+ return -1;
+
+ while ((d = readdir(dir))) {
+ if (!strncmp(d->d_name, ".", 1))
+ continue;
+ oid = strtoull(d->d_name, NULL, 16);
+ if (oid == 0)
+ continue;
+ lookup_trunk_entry(oid, 1);
+ }
+ closedir(dir);
+ return 0;
+}
+
+static int fill_entry_new_sha1(struct __trunk_entry *entry)
+{
+ struct strbuf buf = STRBUF_INIT;
+ int fd, ret = 0;
+ struct sha1_file_hdr hdr = { .priv = 0 };
+
+ memcpy(hdr.tag, TAG_DATA, TAG_LEN);
+ strbuf_addstr(&buf, obj_path);
+ strbuf_addf(&buf, "%016" PRIx64, entry->raw.oid);
+ fd = open(buf.buf, O_RDONLY);
+ dprintf("open %s\n", buf.buf);
+ strbuf_reset(&buf);
+
+ if (fd < 0) {
+ ret = -1;
+ goto out;
+ }
+ if (!strbuf_read(&buf, fd, SD_DATA_OBJ_SIZE) == SD_DATA_OBJ_SIZE) {
+ ret = -1;
+ close(fd);
+ goto out;
+ }
+ hdr.size = buf.len;
+ strbuf_insert(&buf, 0, &hdr, sizeof(hdr));
+
+ if (sha1_file_write((void *)buf.buf, buf.len, entry->raw.sha1) < 0) {
+ ret = -1;
+ close(fd);
+ goto out;
+ }
+ dprintf("dirty data sha1: %s\n", sha1_to_hex(entry->raw.sha1));
+out:
+ strbuf_release(&buf);
+ return ret;
+}
+
+static inline int trunk_entry_no_sha1(struct __trunk_entry *entry)
+{
+ return !strlen((char *)entry->raw.sha1);
+}
+
+int trunk_file_write(unsigned char *outsha1)
+{
+ struct strbuf buf;
+ uint64_t data_size = sizeof(struct trunk_entry) * trunk_entry_active_nr;
+ struct sha1_file_hdr hdr = { .size = data_size,
+ .priv = trunk_entry_active_nr };
+ struct __trunk_entry *entry;
+ int ret = 0;
+
+ memcpy(hdr.tag, TAG_TRUNK, TAG_LEN);
+ strbuf_init(&buf, sizeof(hdr) + data_size);
+
+ strbuf_add(&buf, &hdr, sizeof(hdr));
+ list_for_each_entry(entry, &trunk_active_list, active_list) {
+ if (trunk_entry_no_sha1(entry) || trunk_entry_is_dirty(entry)) {
+ if (fill_entry_new_sha1(entry) < 0) {
+ ret = -1;
+ goto out;
+ }
+ }
+ strbuf_add(&buf, &entry->raw, sizeof(struct trunk_entry));
+ }
+ if (sha1_file_write((void *)buf.buf, buf.len, outsha1) < 0) {
+ ret = -1;
+ goto out;
+ }
+ dprintf("trunk sha1: %s\n", sha1_to_hex(outsha1));
+out:
+ strbuf_release(&buf);
+ return ret;
+}
+
+void *trunk_file_read(unsigned char *sha1, struct sha1_file_hdr *outhdr)
+{
+ void *buffer;
+
+ dprintf("%s\n", sha1_to_hex(sha1));
+ buffer = sha1_file_read(sha1, outhdr);
+ if (!buffer)
+ return NULL;
+ if (strcmp(outhdr->tag, TAG_TRUNK) != 0) {
+ free(buffer);
+ return NULL;
+ }
+
+ return buffer;
+}
+
+int trunk_update_entry(uint64_t oid)
+{
+ struct __trunk_entry *entry;
+
+ entry = lookup_trunk_entry(oid, 1);
+ if (!trunk_entry_is_dirty(entry))
+ mark_trunk_entry_dirty(entry);
+
+ return 0;
+}
--
1.7.8.rc3
More information about the sheepdog
mailing list