[Xen-devel] [PATCH 7 of 8 [RFC]] libxl: introduce libxl_domain_move_memory

[Dario Faggioli <dario.faggioli@xxxxxxxxxx>]

and make use of it in `xl node-affinity' (introduced by one of
the previous patches). This way, users of both, library and cmd
line can ask deallocation and reallocation of the whole memory
pages belonging to a given domain.

In `xl node-affinity', if the '-M' option is used, the above
happens right after changing the NUMA node affinity of the domain
itself. This produces the move of the domain from the (set of)
NUMA node(s) where it belongs now, to somewhere else.

In libxl, what was needed was a way of requesting for domain
suspension, without triggering the whole save/send part, and
avoiding its (potential) asynchronicity. Perhaps, in future,
and if we want that, the _whole_ suspend+move operation can
be made asynchronous, but not the single pieces of it. That is
achieved by calling directly the core of the suspend routine,
with a simplified save context. Of course, this brings no
change to the actual suspend/resume and save/restore
behaviour, wrt the current one.

XXX: Update man pages and documentation still to be done.

Signed-off-by: Dario Faggioli <dario.faggioli@xxxxxxxxxx>

diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -853,6 +853,81 @@ int libxl_domain_unpause(libxl_ctx *ctx,
     return rc;
 }
 
+/*
+ * This function will suspend the domain and invoke xc_domain_move_memory().
+ * The xc_ call will deallocate and reallocate all the memory of the domain.
+ * Notice that this function _does_not_ resume the domain on its own, and
+ * that needs to be done manually by the caller.
+ *
+ * This means that, if the allocation policy (e.g., the node affinity)
+ * for the domain changed (right) before calling this function, at the
+ * end of it all the domain's memory will be compliant with that policy.
+ */
+int libxl_move_memory(libxl_ctx *ctx, uint32_t domid)
+{
+    GC_INIT(ctx);
+    libxl__domain_suspend_state *dss;
+    int rc = 0;
+
+    libxl_domain_type type = libxl__domain_type(gc, domid);
+    if (type == LIBXL_DOMAIN_TYPE_INVALID)
+        abort();
+
+    /*
+     * First of all, we need to suspend the domain. We use the core
+     * suspen code from libxl_dom.c, calling it directly instea of having
+     * libxc calling back into it. For that reason, we need a dss, although
+     * only some of the fields are relevant in our case.
+     */
+    GCNEW(dss);
+    dss->domid = domid;
+    dss->hvm = type == LIBXL_DOMAIN_TYPE_HVM;
+    dss->suspend_eventchn = -1;
+    dss->guest_responded = 0;
+    dss->dm_savefile = NULL;
+
+     /* Try to initialize the suspend event channel */
+    dss->xce = xc_evtchn_open(NULL, 0);
+    if (dss->xce == NULL) {
+        rc = ERROR_FAIL;
+        goto out;
+    } else {
+        int port = xs_suspend_evtchn_port(domid);
+
+        if (port >= 0) {
+            dss->suspend_eventchn =
+                xc_suspend_evtchn_init(ctx->xch, dss->xce, domid, port);
+
+            if (dss->suspend_eventchn < 0)
+                LOG(WARN, "Suspend event channel initialization failed");
+        }
+    }
+
+    if (!libxl__do_domain_suspend(gc, dss))
+    {
+        rc = ERROR_GUEST_TIMEDOUT;
+        goto out;
+    }
+
+    rc = xc_domain_move_memory(ctx->xch, domid);
+    if (rc) {
+        LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "moving memory");
+        rc = ERROR_BADFAIL;
+        goto out;
+    }
+
+ out:
+    /* Tear down the suspend event channel (if successfully initialized) */
+    if (dss->suspend_eventchn > 0)
+        xc_suspend_evtchn_release(CTX->xch, dss->xce, domid,
+                                  dss->suspend_eventchn);
+    if (dss->xce != NULL)
+        xc_evtchn_close(dss->xce);
+
+    GC_FREE;
+    return rc;
+}
+
 int libxl__domain_pvcontrol_available(libxl__gc *gc, uint32_t domid)
 {
     libxl_ctx *ctx = libxl__gc_owner(gc);
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -950,6 +950,8 @@ int libxl_flask_getenforce(libxl_ctx *ct
 int libxl_flask_setenforce(libxl_ctx *ctx, int mode);
 int libxl_flask_loadpolicy(libxl_ctx *ctx, void *policy, uint32_t size);
 
+int libxl_move_memory(libxl_ctx *ctx, uint32_t domid);
+
 /* misc */
 
 /* Each of these sets or clears the flag according to whether the
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -991,11 +991,8 @@ int libxl__domain_resume_device_model(li
     return 0;
 }
 
-int libxl__domain_suspend_common_callback(void *user)
+int libxl__do_domain_suspend(libxl__gc *gc, libxl__domain_suspend_state *dss)
 {
-    libxl__save_helper_state *shs = user;
-    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
-    STATE_AO_GC(dss->ao);
     unsigned long hvm_s_state = 0, hvm_pvdrv = 0;
     int ret;
     char *state = "suspend";
@@ -1125,6 +1122,16 @@ int libxl__domain_suspend_common_callbac
     return 1;
 }
 
+
+int libxl__domain_suspend_common_callback(void *user)
+{
+    libxl__save_helper_state *shs = user;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
+    STATE_AO_GC(dss->ao);
+
+    return libxl__do_domain_suspend(gc, dss);
+}
+
 static inline char *physmap_path(libxl__gc *gc, uint32_t domid,
         char *phys_offset, char *node)
 {
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2541,6 +2541,9 @@ struct libxl__domain_create_state {
 void libxl__xc_domain_saverestore_async_callback_done(libxl__egc *egc,
                            libxl__save_helper_state *shs, int return_value);
 
+_hidden int libxl__do_domain_suspend(libxl__gc *gc,
+                                     libxl__domain_suspend_state *dss);
+
 _hidden int libxl__domain_suspend_common_callback(void *data);
 _hidden void libxl__domain_suspend_common_switch_qemu_logdirty
                                (int domid, unsigned int enable, void *data);
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -4654,13 +4654,31 @@ static void nodeaffinity(uint32_t domid,
 
 int main_nodeaffinity(int argc, char **argv)
 {
-    int opt;
-
-    SWITCH_FOREACH_OPT(opt, "", NULL, "node-affinity", 2) {
-        /* No options */
+    int movemem = 0;
+    int opt = 0;
+
+    SWITCH_FOREACH_OPT(opt, "M", NULL, "node-affinity", 2) {
+    case 'M':
+        movemem = 1;
+        break;
+    }
+
+    if (argc - optind > 3) {
+        help("nodeaffinity");
+        return 2;
     }
 
     nodeaffinity(find_domain(argv[optind]), argv[optind+1]);
+
+    if (movemem) {
+        int rc = libxl_move_memory(ctx, find_domain(argv[optind]));
+
+        if (rc < 0)
+            fprintf(stderr, "Failed to save domain, trying to resume\n");
+
+        libxl_domain_resume(ctx, find_domain(argv[optind]), 1, NULL);
+    }
+
     return 0;
 }
 
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -217,7 +217,10 @@ struct cmd_spec cmd_table[] = {
     { "node-affinity",
       &main_nodeaffinity, 0, 1,
       "Set the NUMA node affinity for the domain",
-      "<Domain> [<NODEs|all|none>]",
+      "[options] <Domain> [<NODEs|all|none>]",
+      "-M        Move the memory to th nodes corresponding to CPUs\n"
+      "          (involves suspending/resuming the domain, so some\n"
+      "          downtime is to be expected)",
     },
     { "vcpu-set",
       &main_vcpuset, 0, 1,

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel