[sheepdog] [PATCH 2/2] lib: call rmdir(2) and unlink(2) in worker threads during purging directory

[Hitoshi Mitake]

purge_directory() can cause amount of disk I/O because of rmdir(2)
and unlink(2). Because they can slow down main thread significantly,
it should be done in worker threads for avoiding long request
blocking.

Cc: Philip Crotwell <crotwell at seis.sc.edu>
Cc: Masahiro Tsuji <tuji at atworks.co.jp>
Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
---
 lib/util.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 63 insertions(+), 9 deletions(-)

diff --git a/lib/util.c b/lib/util.c
index 82cf28c..d4592a3 100644
--- a/lib/util.c
+++ b/lib/util.c
@@ -350,6 +350,43 @@ char *chomp(char *str)
 	return str;
 }
 
+struct purge_work {
+	struct work work;
+
+	bool is_dir;
+	char path[PATH_MAX];
+};
+
+static void purge_work_fn(struct work *work)
+{
+	struct purge_work *pw = container_of(work, struct purge_work, work);
+	int ret;
+
+	if (pw->is_dir)
+		ret = rmdir_r(pw->path);
+	else
+		ret = unlink(pw->path);
+
+	if (ret)
+		sd_err("failed to remove %s %s: %m",
+		       pw->is_dir ? "directory" : "file", pw->path);
+	/*
+	 * We cannot check and do something even above rmdir_r() and unlink()
+	 * cause error. Actually, sd_store->cleanup() (typical user of
+	 * purge_directory())call of cluster_recovery_completion() ignores its
+	 * error code.
+	 */
+}
+
+static void purge_work_done(struct work *work)
+{
+	struct purge_work *pw = container_of(work, struct purge_work, work);
+
+	sd_debug("purging %s %s done",
+		 pw->is_dir ? "directory" : "file", pw->path);
+	free(pw);
+}
+
 /* Purge directory recursively */
 int purge_directory(const char *dir_path)
 {
@@ -376,15 +413,32 @@ int purge_directory(const char *dir_path)
 			sd_err("failed to stat %s: %m", path);
 			goto out;
 		}
-		if (S_ISDIR(s.st_mode))
-			ret = rmdir_r(path);
-		else
-			ret = unlink(path);
-
-		if (ret != 0) {
-			sd_err("failed to remove %s %s: %m",
-			       S_ISDIR(s.st_mode) ? "directory" : "file", path);
-			goto out;
+
+		if (util_wqueue) {
+			struct purge_work *w;
+
+			w = xzalloc(sizeof(*w));
+
+			w->work.fn = purge_work_fn;
+			w->work.done = purge_work_done;
+
+			w->is_dir = S_ISDIR(s.st_mode);
+			strcpy(w->path, path);
+
+			queue_work(util_wqueue, &w->work);
+		} else {
+			if (S_ISDIR(s.st_mode))
+				ret = rmdir_r(path);
+			else
+				ret = unlink(path);
+
+			if (ret != 0) {
+				sd_err("failed to remove %s %s: %m",
+				       S_ISDIR(s.st_mode) ?
+				       "directory" : "file",
+				       path);
+				goto out;
+			}
 		}
 	}
 out:
-- 
1.9.1