[sheepdog] [PATCH 2/2] lib: call rmdir(2) and unlink(2) in worker threads during purging directory

[Liu Yuan]

On Thu, Oct 09, 2014 at 12:16:26AM -0600, Hitoshi Mitake wrote:
> purge_directory() can cause amount of disk I/O because of rmdir(2)
> and unlink(2). Because they can slow down main thread significantly,
> it should be done in worker threads for avoiding long request
> blocking.
> 
> Cc: Philip Crotwell <crotwell at seis.sc.edu>
> Cc: Masahiro Tsuji <tuji at atworks.co.jp>
> Signed-off-by: Hitoshi Mitake <mitake.hitoshi at lab.ntt.co.jp>
> ---
>  lib/util.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--------
>  1 file changed, 63 insertions(+), 9 deletions(-)
> 
> diff --git a/lib/util.c b/lib/util.c
> index 82cf28c..d4592a3 100644
> --- a/lib/util.c
> +++ b/lib/util.c
> @@ -350,6 +350,43 @@ char *chomp(char *str)
>  	return str;
>  }
>  
> +struct purge_work {
> +	struct work work;
> +
> +	bool is_dir;
> +	char path[PATH_MAX];
> +};
> +
> +static void purge_work_fn(struct work *work)
> +{
> +	struct purge_work *pw = container_of(work, struct purge_work, work);
> +	int ret;
> +
> +	if (pw->is_dir)
> +		ret = rmdir_r(pw->path);
> +	else
> +		ret = unlink(pw->path);
> +
> +	if (ret)
> +		sd_err("failed to remove %s %s: %m",
> +		       pw->is_dir ? "directory" : "file", pw->path);
> +	/*
> +	 * We cannot check and do something even above rmdir_r() and unlink()
> +	 * cause error. Actually, sd_store->cleanup() (typical user of
> +	 * purge_directory())call of cluster_recovery_completion() ignores its
> +	 * error code.
> +	 */
> +}
> +
> +static void purge_work_done(struct work *work)
> +{
> +	struct purge_work *pw = container_of(work, struct purge_work, work);
> +
> +	sd_debug("purging %s %s done",
> +		 pw->is_dir ? "directory" : "file", pw->path);
> +	free(pw);
> +}
> +
>  /* Purge directory recursively */
>  int purge_directory(const char *dir_path)
>  {
> @@ -376,15 +413,32 @@ int purge_directory(const char *dir_path)
>  			sd_err("failed to stat %s: %m", path);
>  			goto out;
>  		}
> -		if (S_ISDIR(s.st_mode))
> -			ret = rmdir_r(path);
> -		else
> -			ret = unlink(path);
> -
> -		if (ret != 0) {
> -			sd_err("failed to remove %s %s: %m",
> -			       S_ISDIR(s.st_mode) ? "directory" : "file", path);
> -			goto out;
> +
> +		if (util_wqueue) {
> +			struct purge_work *w;
> +
> +			w = xzalloc(sizeof(*w));
> +
> +			w->work.fn = purge_work_fn;
> +			w->work.done = purge_work_done;
> +
> +			w->is_dir = S_ISDIR(s.st_mode);
> +			strcpy(w->path, path);
> +
> +			queue_work(util_wqueue, &w->work);

why not pass sys->io_queue directly to queue_work? By the way, unlink and rmdir
should be a very fast operation because Linux has a large dentry cache. Probably
it is not a bottleneck.

Thanks
Yuan