[stgt] [PATCH] new timer-based work scheduling
Alexander Nezhinsky
alexandern at Voltaire.COM
Sun Jan 9 17:02:20 CET 2011
Re-implementing the time-based work scheduler. This patch implements
a timer-based scheme.
Work items are scheduled from various application contexts, and put on
a queue. The current time from gettimeofday() is used instead of
a jiffie-based mechanism used previosuly. The work item is stamped with
the expiration time, obtained as the current time plus the timeout period.
A global timer (one per process) is registered and fired periodically,
few times a second. When the timer signal is handled, a word is written
to a dedicated pipe, whose fd is registered with the event loop.
The event handler reads from the pipe, and examines the inactive work
queue. All items that have expired (the immediate current time ise used
for comparison, not the timer or pipe event handling times) are moved
to the active list and processed one after another. The items that
might have accumulated and expired in the meantime are not handled
and are postponed until the next timer event.
The new scheme handles the timer event through the standard event loop's
file descritor registration and polling. This removes the previosly used
call to schedule() function which had to update jiffies.
There is still no guarantee about the exact work execution time.
This scheme is suitable for tasks with seconds-scale resolution, so that
firing the timer few times per second provides a satisfactory accuracy.
The assumption is that all event handlers take small periods of time.
If not, in theory, the timer handlers may be delayed indefinitely,
but then the entire event loop processing gets stalled.
The immediate clients of this mechansim is the iSNS code and the new
iSER code to be based on it instead of the previously proposed
custom timer.
Signed-off-by: Alexander Nezhinsky <alexandern at voltaire.com>
---
usr/tgtd.c | 13 ++++-
usr/work.c | 177 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------
usr/work.h | 8 ++-
3 files changed, 171 insertions(+), 27 deletions(-)
diff --git a/usr/tgtd.c b/usr/tgtd.c
index 2fd4959..5b000d3 100644
--- a/usr/tgtd.c
+++ b/usr/tgtd.c
@@ -329,6 +329,8 @@ static int tgt_exec_scheduled(void)
return work_remains;
}
+#define TGTD_EVENT_TIMEOUT 2000
+
static void event_loop(void)
{
int nevent, i, sched_remains, timeout;
@@ -337,7 +339,7 @@ static void event_loop(void)
retry:
sched_remains = tgt_exec_scheduled();
- timeout = sched_remains ? 0 : TGTD_TICK_PERIOD * 1000;
+ timeout = sched_remains ? 0 : TGTD_EVENT_TIMEOUT;
nevent = epoll_wait(ep_fd, events, ARRAY_SIZE(events), timeout);
if (nevent < 0) {
@@ -350,8 +352,7 @@ retry:
tev = (struct event_data *) events[i].data.ptr;
tev->handler(tev->fd, events[i].events, tev->data);
}
- } else
- schedule();
+ }
if (system_active)
goto retry;
@@ -517,12 +518,18 @@ int main(int argc, char **argv)
}
}
+ err = work_timer_start();
+ if (err)
+ exit(1);
+
bs_init();
event_loop();
lld_exit();
+ work_timer_stop();
+
ipc_exit();
log_close();
diff --git a/usr/work.c b/usr/work.c
index 3080a59..f825581 100644
--- a/usr/work.c
+++ b/usr/work.c
@@ -1,8 +1,9 @@
/*
- * bogus scheduler
+ * work scheduler, loosely timer-based
*
* Copyright (C) 2006-2007 FUJITA Tomonori <tomof at acm.org>
* Copyright (C) 2006-2007 Mike Christie <michaelc at cs.wisc.edu>
+ * Copyright (C) 2011 Alexander Nezhinsky <alexandern at voltaire.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
@@ -21,36 +22,167 @@
*/
#include <stdlib.h>
#include <stdint.h>
+#include <signal.h>
+#include <sys/epoll.h>
#include "list.h"
#include "util.h"
#include "log.h"
#include "work.h"
+#include "tgtd.h"
+
+#define time_before(w1, w2) timercmp(w1, w2, <)
+
+#define WORK_TIMER_INT_SEC 0
+#define WORK_TIMER_INT_MSEC 250
+#define WORK_TIMER_INT_USEC (WORK_TIMER_INT_MSEC * 1000)
+
+static struct itimerval work_timer = {
+ {WORK_TIMER_INT_SEC, WORK_TIMER_INT_USEC},
+ {WORK_TIMER_INT_SEC, WORK_TIMER_INT_USEC}
+};
+
+static int timer_started;
+static int timer_pending;
+static int timer_fd[2] = {0, 0};
-static unsigned int jiffies;
static LIST_HEAD(active_work_list);
static LIST_HEAD(inactive_work_list);
+static void execute_work(void);
+
+static inline void work_timer_schedule_evt(void)
+{
+ unsigned int n = 0;
+ int err;
+
+ if (timer_pending)
+ return;
+
+ timer_pending = 1;
+
+ err = write(timer_fd[1], &n, sizeof(n));
+ if (err < 0)
+ eprintf("Failed to write to pipe, %m\n");
+}
+
+static void work_timer_sig_handler(int data)
+{
+ work_timer_schedule_evt();
+}
+
+static void work_timer_evt_handler(int fd, int events, void *data)
+{
+ unsigned int n;
+ int err;
+
+ err = read(timer_fd[0], &n, sizeof(n));
+ if (err < 0) {
+ eprintf("Failed to read from pipe, %m\n");
+ return;
+ }
+
+ timer_pending = 0;
+
+ execute_work();
+}
+
+int work_timer_start(void)
+{
+ struct sigaction s;
+ int err;
+
+ if (timer_started)
+ return 0;
+
+ timer_started = 1;
+
+ sigemptyset(&s.sa_mask);
+ sigaddset(&s.sa_mask, SIGALRM);
+ s.sa_flags = 0;
+ s.sa_handler = work_timer_sig_handler;
+ err = sigaction(SIGALRM, &s, NULL);
+ if (err) {
+ eprintf("Failed to setup timer handler\n");
+ goto timer_err;
+ }
+
+ err = setitimer(ITIMER_REAL, &work_timer, 0);
+ if (err) {
+ eprintf("Failed to set timer\n");
+ goto timer_err;
+ }
+
+ err = pipe(timer_fd);
+ if (err) {
+ eprintf("Failed to open timer pipe\n");
+ goto timer_err;
+ }
+
+ err = tgt_event_add(timer_fd[0], EPOLLIN,
+ work_timer_evt_handler, NULL);
+ if (err) {
+ eprintf("failed to add timer event, fd:%d\n", timer_fd[0]);
+ goto timer_err;
+ }
+
+ dprintf("started, timeout: %d sec %d msec\n",
+ WORK_TIMER_INT_SEC, WORK_TIMER_INT_MSEC);
+ return 0;
+
+timer_err:
+ work_timer_stop();
+ return err;
+}
+
+int work_timer_stop(void)
+{
+ int err;
+
+ if (!timer_started)
+ return 0;
+
+ timer_started = 0;
+
+ tgt_event_del(timer_fd[0]);
+
+ if (timer_fd[0] > 0)
+ close(timer_fd[0]);
+ if (timer_fd[1] > 0)
+ close(timer_fd[1]);
+
+ err = setitimer(ITIMER_REAL, 0, 0);
+ if (err)
+ eprintf("Failed to stop timer\n");
+ else
+ dprintf("Timer stopped\n");
+
+ return err;
+}
+
void add_work(struct tgt_work *work, unsigned int second)
{
- unsigned int when;
struct tgt_work *ent;
+ int err;
if (second) {
- when = second / TGTD_TICK_PERIOD;
- if (!when)
- when = 1;
-
- work->when = when + jiffies;
+ err = gettimeofday(&work->when, NULL);
+ if (err) {
+ eprintf("gettimeofday failed, %m\n");
+ return;
+ }
+ work->when.tv_sec += second;
list_for_each_entry(ent, &inactive_work_list, entry) {
- if (before(work->when, ent->when))
+ if (time_before(&work->when, &ent->when))
break;
}
list_add_tail(&work->entry, &ent->entry);
- } else
+ } else {
list_add_tail(&work->entry, &active_work_list);
+ work_timer_schedule_evt();
+ }
}
void del_work(struct tgt_work *work)
@@ -58,20 +190,24 @@ void del_work(struct tgt_work *work)
list_del_init(&work->entry);
}
-/*
- * this function is called only when the system is idle. So this
- * scheduler is pretty bogus. Your job would be delayed unexpectedly.
- */
-void schedule(void)
+static void execute_work()
{
+ struct timeval cur_time;
struct tgt_work *work, *n;
+ int err;
+
+ err = gettimeofday(&cur_time, NULL);
+ if (err) {
+ eprintf("gettimeofday failed, %m\n");
+ return;
+ }
list_for_each_entry_safe(work, n, &inactive_work_list, entry) {
- if (after(jiffies, work->when)) {
- list_del(&work->entry);
- list_add_tail(&work->entry, &active_work_list);
- } else
+ if (time_before(&cur_time, &work->when))
break;
+
+ list_del(&work->entry);
+ list_add_tail(&work->entry, &active_work_list);
}
while (!list_empty(&active_work_list)) {
@@ -80,6 +216,5 @@ void schedule(void)
list_del_init(&work->entry);
work->func(work->data);
}
-
- jiffies++;
}
+
diff --git a/usr/work.h b/usr/work.h
index 3d5e75e..c43243b 100644
--- a/usr/work.h
+++ b/usr/work.h
@@ -1,16 +1,18 @@
#ifndef __SCHED_H
#define __SCHED_H
-#define TGTD_TICK_PERIOD 2
+#include <sys/time.h>
struct tgt_work {
struct list_head entry;
void (*func)(void *);
void *data;
- unsigned int when;
+ struct timeval when;
};
-extern void schedule(void);
+extern int work_timer_start(void);
+extern int work_timer_stop(void);
+
extern void add_work(struct tgt_work *work, unsigned int second);
extern void del_work(struct tgt_work *work);
--
1.7.3
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
More information about the stgt
mailing list