[Xen-devel] [PATCH] libxl: Add new "notify-only" childproc mode

[George Dunlap <george.dunlap@xxxxxxxxxx>]

libxl needs to be able to know when processes it forks have completed.

At the moment, libxl has two basic mode (with some variations).  In
one mode -- libxl_sigchld_owner_libxl* -- libxl sets up its own
SIGCHLD signal handler, and also handles the event loop that allows
libxl to safely block until the child in question is finished (using a
self-pipe for the SIGCHLD handler to notify the waiters that it's time
to look for reaped children).

In the other mode, libxl does not set up the SIGCHLD handler, nor does
it do anything with processing the event loop; it expects the library
caller to handle the event loop itself.

The golang runtime manages its own processes, and thus must use
SIGCHLD itself; and it has an easy way for other users to get SIGCHLD
notifications.  However, because its event loop is hidden away behind
abstractions, it's not easy to hook into; and there's no need -- the
golang runtime assumes that C function calls may block, and handles
everything behind the scenes.

Introduce a new mode, libxl_sigchld_owner_notify, in which libxl sets
up the SIGCHLD event handling machinery, but relies on the caller to
tell it when a SIGCHLD happens.

Call these two actions "notify" (for the self-pipe notification
machinery) and "handler" (for the actual SIGCHLD handler).

Move the actual notification handler into its own function.  Have
sigchld_handler() call this function, and provide a new external
function, libxl_childproc_sigchld_notify(), for library users to call
when a SIGCHLD happens.

Rename chldmode_ours() to chldmode_notify(), and use it to determine
whether to set up the notification chain.

When setting up the notification chain, do everything except setting
up the signal handler in "notify-only" mode.

defer_sigchld() and release_sigchld() do two things: they modify the
signal handler, and grab and release locks.  Refactor these so that
they grab and release the locks correctly in "notify-only" mode, but
don't tweak the signal handler unless it's been set up.

With the golang bindings ported to use this change, domain creation
works.

Signed-off-by: George Dunlap <george.dunlap@xxxxxxxxxx>
---


CC: Ian Jackson <ian.jackson@xxxxxxxxxx>
CC: Wei Liu <wl@xxxxxxx>
CC: Anthony Perard <anthony.perard@xxxxxxxxxx>
CC: Nick Rosbrook <rosbrookn@xxxxxxxxxxxx>
---
 tools/libxl/libxl_event.h    | 32 +++++++++++++++++-
 tools/libxl/libxl_fork.c     | 64 +++++++++++++++++++++++++-----------
 tools/libxl/libxl_internal.h |  4 +--
 3 files changed, 78 insertions(+), 22 deletions(-)

diff --git a/tools/libxl/libxl_event.h b/tools/libxl/libxl_event.h
index d1517f7456..8001571cf1 100644
--- a/tools/libxl/libxl_event.h
+++ b/tools/libxl/libxl_event.h
@@ -441,7 +441,7 @@ void libxl_osevent_occurred_timeout(libxl_ctx *ctx, void 
*for_libxl)
  *
  *     A program which does this must call libxl_childproc_setmode.
  *     There are three options:
- * 
+ *
  *     libxl_sigchld_owner_libxl:
  *
  *       While any libxl operation which might use child processes
@@ -466,6 +466,14 @@ void libxl_osevent_occurred_timeout(libxl_ctx *ctx, void 
*for_libxl)
  *       has multiple libxl ctx's, it must call libxl_childproc_exited
  *       on each ctx.)
  *
+ *     libxl_sigchld_owner_mainloop_notify:
+ *
+ *       The application must install a SIGCHLD handler, but will
+ *       notify libxl when a sigchld happened by calling
+ *       libxl_childproc_sigchld_notify.  libxl will arrange to reap
+ *       only its own children.  This needs to be called only once,
+ *       even for applications which have multiple libxl ctx's.
+ *
  *     libxl_sigchld_owner_libxl_always:
  *
  *       The application expects this libxl ctx to reap all of the
@@ -494,6 +502,12 @@ typedef enum {
      * arrange to (un)block or catch SIGCHLD. */
     libxl_sigchld_owner_mainloop,
 
+    /* Application promises to discover when SIGCHLD occurs and call
+     * libxl_childproc_sigchld_notify (perhaps from within a signal
+     * handler).  libxl will not itself arrange to (un)block or catch
+     * SIGCHLD. */
+    libxl_sigchld_owner_mainloop_notify,
+
     /* libxl owns SIGCHLD all the time, and the application is
      * relying on libxl's event loop for reaping its children too. */
     libxl_sigchld_owner_libxl_always,
@@ -590,6 +604,22 @@ int libxl_childproc_reaped(libxl_ctx *ctx, pid_t, int 
status)
 void libxl_childproc_sigchld_occurred(libxl_ctx *ctx)
                            LIBXL_EXTERNAL_CALLERS_ONLY;
 
+/*
+ * This function is for an application which owns SIGCHLD but still
+ * expects libxl to handle its own event loops.
+ *
+ * Such an the application must notify libxl, by calling this
+ * function, that a SIGCHLD occurred.  libxl will then notify all
+ * appropriate event loops to check for reaped children..
+ *
+ * May be called only by an application which has called setmode with
+ * chldowner == libxl_sigchld_owner_mainloop_notify.
+ *
+ * MAY be called from within a signal handler.
+ */
+void libxl_childproc_sigchld_notify(void)
+                           LIBXL_EXTERNAL_CALLERS_ONLY;
+
 
 /*
  * An application which initialises a libxl_ctx in a parent process
diff --git a/tools/libxl/libxl_fork.c b/tools/libxl/libxl_fork.c
index 0f1b6b518c..f350246576 100644
--- a/tools/libxl/libxl_fork.c
+++ b/tools/libxl/libxl_fork.c
@@ -78,7 +78,7 @@ static void atfork_unlock(void)
 int libxl__atfork_init(libxl_ctx *ctx)
 {
     int r, rc;
-    
+
     atfork_lock();
     if (atfork_registered) { rc = 0; goto out; }
 
@@ -227,12 +227,9 @@ static pid_t checked_waitpid(libxl__egc *egc, pid_t want, 
int *status)
 static void sigchld_selfpipe_handler(libxl__egc *egc, libxl__ev_fd *ev,
                                      int fd, short events, short revents);
 
-static void sigchld_handler(int signo)
+static void sigchld_notify(void)
 {
-    /* This function has to be reentrant!  Luckily it is. */
-
     libxl_ctx *notify;
-    int esave = errno;
 
     int r = pthread_mutex_lock(&sigchld_defer_mutex);
     assert(!r);
@@ -244,10 +241,28 @@ static void sigchld_handler(int signo)
 
     r = pthread_mutex_unlock(&sigchld_defer_mutex);
     assert(!r);
+}
+
+static void sigchld_handler(int signo)
+{
+    /* This function has to be reentrant!  Luckily it is. */
+
+    int esave = errno;
+
+    sigchld_notify();
 
     errno = esave;
 }
 
+void libxl_childproc_sigchld_notify(void)
+{
+    /*
+     * XXX: We don't need a ctx here for functionality sake; should we
+     * take one just so we can make sure we're in the right mode?
+     */
+    sigchld_notify();
+}
+
 static void sigchld_sethandler_raw(void (*handler)(int), struct sigaction *old)
 {
     struct sigaction ours;
@@ -288,9 +303,8 @@ static void sigchld_handler_when_deferred(int signo)
 
 static void defer_sigchld(void)
 {
-    assert(sigchld_installed);
-
-    sigchld_sethandler_raw(sigchld_handler_when_deferred, 0);
+    if ( sigchld_installed )
+        sigchld_sethandler_raw(sigchld_handler_when_deferred, 0);
 
     /* Now _this thread_ cannot any longer be interrupted by the
      * signal, so we can take the mutex without risk of deadlock.  If
@@ -303,12 +317,12 @@ static void defer_sigchld(void)
 
 static void release_sigchld(void)
 {
-    assert(sigchld_installed);
-
     int r = pthread_mutex_unlock(&sigchld_defer_mutex);
     assert(!r);
 
-    sigchld_sethandler_raw(sigchld_handler, 0);
+    if ( sigchld_installed )
+        sigchld_sethandler_raw(sigchld_handler, 0);
+
     if (sigchld_occurred_while_deferred) {
         sigchld_occurred_while_deferred = 0;
         /* We might get another SIGCHLD here, in which case
@@ -326,7 +340,7 @@ static void sigchld_removehandler_core(void) /* idempotent 
*/
 {
     struct sigaction was;
     int r;
-    
+
     if (!sigchld_installed)
         return;
 
@@ -375,6 +389,11 @@ static void sigchld_user_remove(libxl_ctx *ctx) /* 
idempotent */
 
 void libxl__sigchld_notneeded(libxl__gc *gc) /* non-reentrant, idempotent */
 {
+    /*
+     * NB that we don't need to special-case
+     * libxl_sigchld_owner_mainloop_notify; sigchld_removehandler_core
+     * will DTRT if no signal handler has been set up.
+     */
     sigchld_user_remove(CTX);
     libxl__ev_fd_deregister(gc, &CTX->sigchld_selfpipe_efd);
 }
@@ -399,7 +418,9 @@ int libxl__sigchld_needed(libxl__gc *gc) /* non-reentrant, 
idempotent */
     if (!CTX->sigchld_user_registered) {
         atfork_lock();
 
-        sigchld_installhandler_core();
+        if (CTX->childproc_hooks->chldowner != 
libxl_sigchld_owner_mainloop_notify) {
+            sigchld_installhandler_core();
+        }
 
         defer_sigchld();
 
@@ -416,13 +437,15 @@ int libxl__sigchld_needed(libxl__gc *gc) /* 
non-reentrant, idempotent */
     return rc;
 }
 
-static bool chldmode_ours(libxl_ctx *ctx, bool creating)
+/* Do we need the sigchld notification machinery? */
+static bool chldmode_notify(libxl_ctx *ctx, bool creating)
 {
     switch (ctx->childproc_hooks->chldowner) {
     case libxl_sigchld_owner_libxl:
         return creating || !LIBXL_LIST_EMPTY(&ctx->children);
     case libxl_sigchld_owner_mainloop:
         return 0;
+    case libxl_sigchld_owner_mainloop_notify:
     case libxl_sigchld_owner_libxl_always:
     case libxl_sigchld_owner_libxl_always_selective_reap:
         return 1;
@@ -432,7 +455,7 @@ static bool chldmode_ours(libxl_ctx *ctx, bool creating)
 
 static void perhaps_sigchld_notneeded(libxl__gc *gc)
 {
-    if (!chldmode_ours(CTX, 0))
+    if (!chldmode_notify(CTX, 0))
         libxl__sigchld_notneeded(gc);
 }
 
@@ -440,7 +463,7 @@ static int perhaps_sigchld_needed(libxl__gc *gc, bool 
creating)
 {
     int rc;
 
-    if (chldmode_ours(CTX, creating)) {
+    if (chldmode_notify(CTX, creating)) {
         rc = libxl__sigchld_needed(gc);
         if (rc) return rc;
     }
@@ -556,13 +579,16 @@ static void sigchld_selfpipe_handler(libxl__egc *egc, 
libxl__ev_fd *ev,
     int e = libxl__self_pipe_eatall(selfpipe);
     if (e) LIBXL__EVENT_DISASTER(egc, "read sigchld pipe", e, 0);
 
-    if (CTX->childproc_hooks->chldowner
-        == libxl_sigchld_owner_libxl_always_selective_reap) {
+    switch (CTX->childproc_hooks->chldowner) {
+    case libxl_sigchld_owner_libxl_always_selective_reap:
+    case libxl_sigchld_owner_mainloop_notify:
         childproc_checkall(egc);
         return;
+    default:
+        ;
     }
 
-    while (chldmode_ours(CTX, 0) /* in case the app changes the mode */) {
+    while (chldmode_notify(CTX, 0) /* in case the app changes the mode */) {
         int status;
         pid_t pid = checked_waitpid(egc, -1, &status);
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index b5adbfe4b7..bfbee9451e 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -161,7 +161,7 @@
 #endif
   /* all of these macros preserve errno (saving and restoring) */
 
-/* 
+/*
  * A macro to help retain the first failure in "do as much as you can"
  * situations.  Note the hard-coded use of the variable name `rc`.
  */
@@ -1367,7 +1367,7 @@ typedef struct {
     const char *shim_cmdline;
     const char *pv_cmdline;
 
-    /* 
+    /*
      * dm_runas: If set, pass qemu the `-runas` parameter with this
      *  string as an argument
      * dm_kill_uid: If set, the devicemodel should be killed by
-- 
2.24.1


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel