[sheepdog] [PATCH v3 06/12] collie/farm: implement sha1_file

Kai Zhang kyle at zelin.io
Tue May 14 09:51:51 CEST 2013


moved sha1_file from sheep/farm to collie/farm and made minor modification.

Signed-off-by: Kai Zhang <kyle at zelin.io>
---
 collie/Makefile.am      |    2 +-
 collie/farm/farm.c      |   24 ++++
 collie/farm/farm.h      |   23 ++++
 collie/farm/sha1_file.c |  273 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 321 insertions(+), 1 deletions(-)
 create mode 100644 collie/farm/farm.c
 create mode 100644 collie/farm/sha1_file.c

diff --git a/collie/Makefile.am b/collie/Makefile.am
index 9c30495..d185512 100644
--- a/collie/Makefile.am
+++ b/collie/Makefile.am
@@ -23,7 +23,7 @@ INCLUDES		= -I$(top_builddir)/include -I$(top_srcdir)/include
 
 sbin_PROGRAMS		= collie
 
-collie_SOURCES		= farm/object_rb_tree.c \
+collie_SOURCES		= farm/object_rb_tree.c farm/sha1_file.c farm/farm.c \
 			  collie.c common.c treeview.c vdi.c node.c cluster.c
 
 if BUILD_TRACE
diff --git a/collie/farm/farm.c b/collie/farm/farm.c
new file mode 100644
index 0000000..283cd9b
--- /dev/null
+++ b/collie/farm/farm.c
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2011 Taobao Inc.
+ * Copyright (C) 2013 Zelin Inc.
+ *
+ * Liu Yuan <namei.unix at gmail.com>
+ * Kai Zhang <kyle at zelin.io>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "farm.h"
+
+static char farm_obj_dir[PATH_MAX];
+static char farm_dir[PATH_MAX];
+
+inline char *get_object_directory(void)
+{
+	return farm_obj_dir;
+}
diff --git a/collie/farm/farm.h b/collie/farm/farm.h
index 3d08396..ca108ca 100644
--- a/collie/farm/farm.h
+++ b/collie/farm/farm.h
@@ -20,11 +20,34 @@
 #include "strbuf.h"
 #include "sha1.h"
 
+#define TAG_LEN         6
+#define TAG_DATA        "data\0\0"
+#define TAG_TRUNK       "trunk\0"
+#define TAG_SNAP        "snap\0\0"
+
 enum obj_type {
 	OBJECT = 1,
 	VDI,
 };
 
+struct sha1_file_hdr {
+	char tag[TAG_LEN];
+	uint64_t size;
+	uint64_t priv;
+	uint64_t reserved;
+};
+
+/* farm.c */
+int farm_init(const char *path);
+char *get_object_directory(void);
+
+/* sha1_file.c */
+int sha1_file_write(unsigned char *buf, unsigned len, unsigned char *);
+void *sha1_file_read(const unsigned char *sha1, struct sha1_file_hdr *);
+char *sha1_to_hex(const unsigned char *sha1);
+int get_sha1_hex(const char *hex, unsigned char *sha1);
+int sha1_file_try_delete(const unsigned char *sha1);
+
 /* object_rb_tree.c */
 int get_obj_nr(void);
 void obj_tree_insert(uint64_t oid, enum obj_type type, int nr_copies);
diff --git a/collie/farm/sha1_file.c b/collie/farm/sha1_file.c
new file mode 100644
index 0000000..6b62914
--- /dev/null
+++ b/collie/farm/sha1_file.c
@@ -0,0 +1,273 @@
+/*
+ * Copyright (C) 2011 Taobao Inc.
+ *
+ * Liu Yuan <namei.unix at gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ *   sha1_file provide us some useful features:
+ *
+ *   - Regardless of object type, all objects are all in deflated with zlib,
+ *     and have a header that not only specifies their tag, but also size
+ *     information about the data in the object.
+ *
+ *   - the general consistency of an object can always be tested independently
+ *     of the contents or the type of the object: all objects can be validated
+ *     by verifying that their hashes match the content of the file.
+ */
+#include <sys/types.h>
+#include <sys/xattr.h>
+
+#include "farm.h"
+#include "util.h"
+
+static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
+{
+	int i;
+	for (i = 0; i < SHA1_LEN; i++) {
+		static const char hex[] = "0123456789abcdef";
+		unsigned int val = sha1[i];
+		char *pos = pathbuf + i*2 + (i > 0);
+		*pos++ = hex[val >> 4];
+		*pos = hex[val & 0xf];
+	}
+}
+
+static char *sha1_to_path(const unsigned char *sha1)
+{
+	static char buf[PATH_MAX];
+	const char *objdir;
+	int len;
+
+	objdir = get_object_directory();
+	len = strlen(objdir);
+
+	/* '/' + sha1(2) + '/' + sha1(38) + '\0' */
+	memcpy(buf, objdir, len);
+	buf[len] = '/';
+	buf[len+3] = '/';
+	buf[len+42] = '\0';
+	fill_sha1_path(buf + len + 1, sha1);
+	return buf;
+}
+
+#define CNAME	"user.farm.count"
+#define CSIZE	sizeof(uint32_t)
+
+static void get_sha1_file(char *name)
+{
+	uint32_t count;
+	if (getxattr(name, CNAME, &count, CSIZE) < 0) {
+		if (errno == ENODATA) {
+			count = 1;
+			if (setxattr(name, CNAME, &count, CSIZE, 0) < 0)
+				panic("%m");
+			return;
+		} else
+			panic("%m");
+	}
+	count++;
+	if (setxattr(name, CNAME, &count, CSIZE, 0) < 0)
+		panic("%m");
+}
+
+static int put_sha1_file(char *name)
+{
+	uint32_t count;
+
+	if (getxattr(name, CNAME, &count, CSIZE) < 0) {
+		if (errno == ENOENT) {
+			sd_dprintf("sha1 file doesn't exist");
+			return -1;
+		} else
+			panic("%m");
+	}
+
+	count--;
+	if (count == 0) {
+		if (unlink(name) < 0) {
+			sd_dprintf("%m");
+			return -1;
+		}
+		sd_dprintf("%s deleted", name);
+	} else {
+		if (setxattr(name, CNAME, &count, CSIZE, 0) < 0)
+			panic("%m");
+	}
+	return 0;
+}
+
+static int sha1_buffer_write(const unsigned char *sha1,
+			     void *buf, unsigned int size)
+{
+	char *filename = sha1_to_path(sha1);
+	int fd, ret = 0, len;
+
+	fd = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0666);
+	if (fd < 0) {
+		if (errno != EEXIST) {
+			sd_eprintf("failed to open file %s with error: %m",
+				   filename);
+			ret = -1;
+		}
+		goto err_open;
+	}
+	len = xwrite(fd, buf, size);
+	if (len != size) {
+		sd_dprintf("%m");
+		close(fd);
+		return -1;
+	}
+
+	close(fd);
+	get_sha1_file(filename);
+err_open:
+	return ret;
+}
+
+int sha1_file_write(unsigned char *buf, unsigned len, unsigned char *outsha1)
+{
+	unsigned char sha1[SHA1_LEN];
+	struct sha1_ctx c;
+
+	sha1_init(&c);
+	sha1_update(&c, buf, len);
+	sha1_final(&c, sha1);
+
+	if (sha1_buffer_write(sha1, buf, len) < 0)
+		return -1;
+	if (outsha1)
+		memcpy(outsha1, sha1, SHA1_LEN);
+	return 0;
+}
+
+static void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
+{
+	char *filename = sha1_to_path(sha1);
+	int fd = open(filename, O_RDONLY);
+	struct stat st;
+	void *map;
+
+	if (fd < 0) {
+		perror(filename);
+		return NULL;
+	}
+	if (fstat(fd, &st) < 0) {
+		sd_dprintf("%m");
+		close(fd);
+		return NULL;
+	}
+	map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	close(fd);
+	if (map == MAP_FAILED) {
+		sd_dprintf("%m");
+		return NULL;
+	}
+	*size = st.st_size;
+	return map;
+}
+
+static void *unpack_sha1_file(void *map, unsigned long mapsize,
+			      struct sha1_file_hdr *hdr)
+{
+	int hdr_len;
+	char *buf;
+
+	memcpy(hdr, map, sizeof(*hdr));
+	hdr_len = sizeof(*hdr);
+	buf = valloc(hdr->size);
+	if (!buf) {
+		sd_dprintf("%m");
+		return NULL;
+	}
+
+	memcpy(buf, (char *)map + hdr_len, mapsize - hdr_len);
+	return buf;
+}
+
+static int verify_sha1_file(const unsigned char *sha1,
+			    void *buf, unsigned long len)
+{
+	unsigned char tmp[SHA1_LEN];
+	struct sha1_ctx c;
+
+	sha1_init(&c);
+	sha1_update(&c, buf, len);
+	sha1_final(&c, tmp);
+
+	if (memcmp((char *)tmp, (char *)sha1, SHA1_LEN) != 0) {
+		sd_dprintf("failed, %s != %s", sha1_to_hex(sha1),
+			   sha1_to_hex(tmp));
+		return -1;
+	}
+	return 0;
+}
+
+void *sha1_file_read(const unsigned char *sha1, struct sha1_file_hdr *hdr)
+{
+	unsigned long mapsize;
+	void *map, *buf;
+
+	map = map_sha1_file(sha1, &mapsize);
+	if (map) {
+		if (verify_sha1_file(sha1, map, mapsize) < 0)
+			return NULL;
+		buf = unpack_sha1_file(map, mapsize, hdr);
+		munmap(map, mapsize);
+		return buf;
+	}
+	return NULL;
+}
+
+int sha1_file_try_delete(const unsigned char *sha1)
+{
+	char *filename = sha1_to_path(sha1);
+
+	return put_sha1_file(filename);
+}
+
+static unsigned hexval(char c)
+{
+	if (c >= '0' && c <= '9')
+		return c - '0';
+	if (c >= 'a' && c <= 'f')
+		return c - 'a' + 10;
+	if (c >= 'A' && c <= 'F')
+		return c - 'A' + 10;
+	return ~0;
+}
+
+int get_sha1_hex(const char *hex, unsigned char *sha1)
+{
+	int i;
+	for (i = 0; i < SHA1_LEN; i++) {
+		unsigned int val = (hexval(hex[0]) << 4) | hexval(hex[1]);
+		if (val & ~0xff)
+			return -1;
+		*sha1++ = val;
+		hex += 2;
+	}
+	return 0;
+}
+
+char *sha1_to_hex(const unsigned char *sha1)
+{
+	static char buffer[50];
+	static const char hex[] = "0123456789abcdef";
+	char *buf = buffer;
+	int i;
+
+	for (i = 0; i < SHA1_LEN; i++) {
+		unsigned int val = *sha1++;
+		*buf++ = hex[val >> 4];
+		*buf++ = hex[val & 0xf];
+	}
+	return buffer;
+}
-- 
1.7.1




More information about the sheepdog mailing list