|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [Xen-devel] [PATCH RFC v2 19/23] libxl/migration: implement the receiver side of postcopy live migration
From: Joshua Otto <jtotto@xxxxxxxxxxxx>
To make the libxl receiver capable of supporting postcopy live
migration:
- As was done for the libxl stream writer, add a symmetric callback
chain through the stream reader that reads the sequence of xl records
necessary to resume the guest and enter the postcopy phase. This
chain is very similar to the checkpoint chain.
- Add a new postcopy path through the domain creation sequence that
permits the xc memory postcopy phase to proceed in parallel to the
libxl domain creation and resumption sequence.
- Add a out-parameter to libxl_domain_create_restore(),
'postcopy_resumed', that callers can test to determine whether or not
further action is required on their-part post-migration to get the
guest running.
A subsequent patch will introduce a mechanism by which library clients
can _induce_ a postcopy live migration.
Signed-off-by: Joshua Otto <jtotto@xxxxxxxxxxxx>
---
tools/libxl/libxl.h | 28 +++++-
tools/libxl/libxl_create.c | 178 +++++++++++++++++++++++++++++++++--
tools/libxl/libxl_internal.h | 45 ++++++++-
tools/libxl/libxl_stream_read.c | 57 +++++++++++
tools/ocaml/libs/xl/xenlight_stubs.c | 2 +-
tools/xl/xl_vmcontrol.c | 2 +-
6 files changed, 297 insertions(+), 15 deletions(-)
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 5e48862..70441cf 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1308,6 +1308,7 @@ int libxl_domain_create_new(libxl_ctx *ctx,
libxl_domain_config *d_config,
int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
uint32_t *domid, int restore_fd,
int send_back_fd,
+ bool *postcopy_resumed, /* OUT */
const libxl_domain_restore_params *params,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
@@ -1327,8 +1328,9 @@ static inline int libxl_domain_create_restore_0x040200(
libxl_domain_restore_params_init(¶ms);
- ret = libxl_domain_create_restore(
- ctx, d_config, domid, restore_fd, -1, ¶ms, ao_how,
aop_console_how);
+ ret = libxl_domain_create_restore(ctx, d_config, domid, restore_fd,
+ -1, NULL, ¶ms, ao_how,
+ aop_console_how);
libxl_domain_restore_params_dispose(¶ms);
return ret;
@@ -1348,11 +1350,31 @@ static inline int libxl_domain_create_restore_0x040400(
LIBXL_EXTERNAL_CALLERS_ONLY
{
return libxl_domain_create_restore(ctx, d_config, domid, restore_fd,
- -1, params, ao_how, aop_console_how);
+ -1, NULL, params, ao_how,
+ aop_console_how);
}
#define libxl_domain_create_restore libxl_domain_create_restore_0x040400
+#elif defined(LIBXL_API_VERSION) && LIBXL_API_VERSION >= 0x040700 \
+ && LIBXL_API_VERSION < 0x040900
+
+static inline int libxl_domain_create_restore_0x040700(
+ libxl_ctx *ctx, libxl_domain_config *d_config,
+ uint32_t *domid, int restore_fd,
+ int send_back_fd,
+ const libxl_domain_restore_params *params,
+ const libxl_asyncop_how *ao_how,
+ const libxl_asyncprogress_how *aop_console_how)
+ LIBXL_EXTERNAL_CALLERS_ONLY
+{
+ return libxl_domain_create_restore(ctx, d_config, domid, restore_fd,
+ -1, NULL, params, ao_how,
+ aop_console_how);
+}
+
+#define libxl_domain_create_restore libxl_domain_create_restore_0x040700
+
#endif
int libxl_domain_soft_reset(libxl_ctx *ctx,
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 1354689..227fdfb 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -748,8 +748,24 @@ static void domcreate_bootloader_done(libxl__egc *egc,
libxl__bootloader_state *bl,
int rc);
+/*
+ * If a postcopy migration is initiated by the sending side during a live
+ * migration, this function returns control of the stream to the stream reader
+ * so it can finish the libxl stream.
+ */
static void domcreate_postcopy_transition_callback(void *user);
+/*
+ * When the stream reader postcopy transition completes, this callback is
+ * invoked. It transfers control of the restore stream back to the helper.
+ */
+void domcreate_postcopy_transition_complete_callback(
+ libxl__egc *egc, libxl__stream_read_state *srs, int rc);
+
+static void domcreate_postcopy_stream_done(libxl__egc *egc,
+ libxl__stream_read_state *srs,
+ int ret);
+
static void domcreate_launch_dm(libxl__egc *egc, libxl__multidev *aodevs,
int ret);
@@ -776,6 +792,10 @@ static void domcreate_destruction_cb(libxl__egc *egc,
libxl__domain_destroy_state *dds,
int rc);
+static void domcreate_report_result(libxl__egc *egc,
+ libxl__domain_create_state *dcs,
+ int rc);
+
static void initiate_domain_create(libxl__egc *egc,
libxl__domain_create_state *dcs)
{
@@ -1111,6 +1131,15 @@ static void domcreate_bootloader_done(libxl__egc *egc,
callbacks->postcopy_transition =
domcreate_postcopy_transition_callback;
+ /*
+ * When the stream reader is finished reading the postcopy
+ * transition, we'll find out in the
+ * domcreate_postcopy_transition_complete_callback(), where we'll
+ * hand control of the stream back to the libxc helper.
+ */
+ dcs->srs.postcopy_transition_callback =
+ domcreate_postcopy_transition_complete_callback;
+
libxl__stream_read_start(egc, &dcs->srs);
}
return;
@@ -1124,8 +1153,81 @@ static void domcreate_bootloader_done(libxl__egc *egc,
static void domcreate_postcopy_transition_callback(void *user)
{
- /* XXX we're not ready to deal with this yet */
- assert(0);
+ libxl__save_helper_state *shs = user;
+ libxl__domain_create_state *dcs = shs->caller_state;
+ libxl__stream_read_state *srs = &dcs->srs;
+
+ libxl__stream_read_start_postcopy_transition(shs->egc, srs);
+}
+
+void domcreate_postcopy_transition_complete_callback(
+ libxl__egc *egc, libxl__stream_read_state *srs, int rc)
+{
+ libxl__domain_create_state *dcs = srs->dcs;
+
+ if (!rc)
+ srs->completion_callback = domcreate_postcopy_stream_done;
+
+ /*
+ * If all is well (for now) we'll find out about the eventual termination
+ * of the restore helper/stream through domcreate_postcopy_stream_done().
+ * Otherwise, we'll find out sooner through domcreate_stream_done().
+ */
+ libxl__xc_domain_saverestore_async_callback_done(egc, &srs->shs, !rc);
+
+ if (!rc) {
+ /* In parallel, resume the guest. */
+ dcs->postcopy.active = true;
+ dcs->postcopy.resume.state = DCS_POSTCOPY_RESUME_INPROGRESS;
+ dcs->postcopy.stream.state = DCS_POSTCOPY_STREAM_INPROGRESS;
+ domcreate_stream_done(egc, srs, 0);
+ }
+}
+
+static void domcreate_postcopy_stream_done(libxl__egc *egc,
+ libxl__stream_read_state *srs,
+ int ret)
+{
+ libxl__domain_create_state *dcs = srs->dcs;
+
+ EGC_GC;
+
+ assert(dcs->postcopy.stream.state == DCS_POSTCOPY_STREAM_INPROGRESS);
+
+ switch (dcs->postcopy.resume.state) {
+ case DCS_POSTCOPY_RESUME_INPROGRESS:
+ if (ret) {
+ /*
+ * The stream failed, and the resumption is still in progress.
+ * Stash our return code for resumption to find later.
+ */
+ dcs->postcopy.stream.state = DCS_POSTCOPY_STREAM_FAILED;
+ dcs->postcopy.stream.rc = ret;
+ } else {
+ /*
+ * We've successfully completed, but the resumption is still
humming
+ * away.
+ */
+ dcs->postcopy.stream.state =
DCS_POSTCOPY_STREAM_SUCCESS;
+
+ /* Just let it finish. Nothing to do for now. */
+ LOG(INFO, "Postcopy stream completed _before_ domain
unpaused");
+ }
+ break;
+ case DCS_POSTCOPY_RESUME_FAILED:
+ /* The resumption failed first, so report its result. */
+ dcs->callback(egc, dcs, dcs->postcopy.resume.rc, dcs->guest_domid);
+ break;
+ case DCS_POSTCOPY_RESUME_SUCCESS:
+ /*
+ * This is the expected case - resumption completed, and some time
later
+ * the final postcopy pages were migrated and the stream wrapped up.
+ * We're now totally done!
+ */
+ LOG(INFO, "Postcopy stream completed after domain unpaused");
+ dcs->callback(egc, dcs, ret, dcs->guest_domid);
+ break;
+ }
}
void libxl__srm_callout_callback_restore_results(xen_pfn_t store_mfn,
@@ -1582,7 +1684,8 @@ static void domcreate_complete(libxl__egc *egc,
}
dcs->guest_domid = -1;
}
- dcs->callback(egc, dcs, rc, dcs->guest_domid);
+
+ domcreate_report_result(egc, dcs, rc);
}
static void domcreate_destruction_cb(libxl__egc *egc,
@@ -1595,7 +1698,63 @@ static void domcreate_destruction_cb(libxl__egc *egc,
if (rc)
LOGD(ERROR, dds->domid, "unable to destroy domain following failed
creation");
- dcs->callback(egc, dcs, ERROR_FAIL, dcs->guest_domid);
+ domcreate_report_result(egc, dcs, ERROR_FAIL);
+}
+
+static void domcreate_report_result(libxl__egc *egc,
+ libxl__domain_create_state *dcs,
+ int rc)
+{
+ EGC_GC;
+
+ if (!dcs->postcopy.active) {
+ /*
+ * If we aren't presently in the process of completing a postcopy
+ * resumption (the norm), everything is all cleaned up and we can
report
+ * our result directly.
+ */
+ LOG(INFO, "No postcopy at all");
+ dcs->callback(egc, dcs, rc, dcs->guest_domid);
+ } else {
+ switch (dcs->postcopy.stream.state) {
+ case DCS_POSTCOPY_STREAM_INPROGRESS:
+ case DCS_POSTCOPY_STREAM_SUCCESS:
+ /* If we haven't yet failed, try to unpause the guest. */
+ rc = rc ?: libxl_domain_unpause(CTX, dcs->guest_domid);
+ if (dcs->postcopy_resumed)
+ *dcs->postcopy_resumed = !rc;
+
+ if (dcs->postcopy.stream.state == DCS_POSTCOPY_STREAM_SUCCESS) {
+ /*
+ * The stream finished successfully, so we can report our local
+ * result as the overall result.
+ */
+ dcs->callback(egc, dcs, rc, dcs->guest_domid);
+ LOG(INFO, "Postcopy domain unpaused after stream completed");
+ } else if (rc) {
+ /*
+ * The stream isn't done yet, but we failed. Tell it to bail,
+ * and stash our return code for the postcopy stream completion
+ * callback to find.
+ */
+ dcs->postcopy.resume.state = DCS_POSTCOPY_RESUME_FAILED;
+ dcs->postcopy.resume.rc = rc;
+
+ libxl__stream_read_abort(egc, &dcs->srs, -1);
+ } else {
+ dcs->postcopy.resume.state = DCS_POSTCOPY_RESUME_SUCCESS;
+ LOG(INFO, "Postcopy domain unpaused before stream completed");
+ }
+ break;
+ case DCS_POSTCOPY_STREAM_FAILED:
+ /*
+ * The stream failed. Now that we're done, tie things up by
+ * reporting the stream's result.
+ */
+ dcs->callback(egc, dcs, dcs->postcopy.stream.rc, dcs->guest_domid);
+ break;
+ }
+ }
}
/*----- application-facing domain creation interface -----*/
@@ -1619,6 +1778,7 @@ static void domain_create_cb(libxl__egc *egc,
static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config,
uint32_t *domid, int restore_fd, int send_back_fd,
+ bool *postcopy_resumed, /* OUT */
const libxl_domain_restore_params *params,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
@@ -1627,6 +1787,9 @@ static int do_domain_create(libxl_ctx *ctx,
libxl_domain_config *d_config,
libxl__app_domain_create_state *cdcs;
int rc;
+ if (postcopy_resumed)
+ *postcopy_resumed = false;
+
GCNEW(cdcs);
cdcs->dcs.ao = ao;
cdcs->dcs.guest_config = d_config;
@@ -1641,6 +1804,7 @@ static int do_domain_create(libxl_ctx *ctx,
libxl_domain_config *d_config,
&cdcs->dcs.restore_fdfl);
if (rc < 0) goto out_err;
}
+ cdcs->dcs.postcopy_resumed = postcopy_resumed;
cdcs->dcs.callback = domain_create_cb;
cdcs->dcs.domid_soft_reset = INVALID_DOMID;
@@ -1862,13 +2026,13 @@ int libxl_domain_create_new(libxl_ctx *ctx,
libxl_domain_config *d_config,
const libxl_asyncprogress_how *aop_console_how)
{
unset_disk_colo_restore(d_config);
- return do_domain_create(ctx, d_config, domid, -1, -1, NULL,
+ return do_domain_create(ctx, d_config, domid, -1, -1, NULL, NULL,
ao_how, aop_console_how);
}
int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config,
uint32_t *domid, int restore_fd,
- int send_back_fd,
+ int send_back_fd, bool *postcopy_resumed,
const libxl_domain_restore_params *params,
const libxl_asyncop_how *ao_how,
const libxl_asyncprogress_how *aop_console_how)
@@ -1880,7 +2044,7 @@ int libxl_domain_create_restore(libxl_ctx *ctx,
libxl_domain_config *d_config,
}
return do_domain_create(ctx, d_config, domid, restore_fd, send_back_fd,
- params, ao_how, aop_console_how);
+ postcopy_resumed, params, ao_how, aop_console_how);
}
int libxl_domain_soft_reset(libxl_ctx *ctx,
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index c8ea3ba..54ad16a 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3127,9 +3127,15 @@ struct libxl__stream_read_state {
void (*completion_callback)(libxl__egc *egc,
libxl__stream_read_state *srs,
int rc);
- void (*checkpoint_callback)(libxl__egc *egc,
- libxl__stream_read_state *srs,
- int rc);
+ /* Checkpointing and postcopy live migration are mutually exclusive. */
+ union {
+ void (*checkpoint_callback)(libxl__egc *egc,
+ libxl__stream_read_state *srs,
+ int rc);
+ void (*postcopy_transition_callback)(libxl__egc *egc,
+ libxl__stream_read_state *srs,
+ int rc);
+ };
/* Private */
int rc;
bool running;
@@ -3143,10 +3149,12 @@ struct libxl__stream_read_state {
LIBXL_STAILQ_HEAD(, libxl__sr_record_buf) record_queue; /* NOGC */
enum {
SRS_PHASE_NORMAL,
+ SRS_PHASE_POSTCOPY_TRANSITION,
SRS_PHASE_CHECKPOINT_BUFFERING,
SRS_PHASE_CHECKPOINT_UNBUFFERING,
SRS_PHASE_CHECKPOINT_STATE
} phase;
+ bool postcopy_transitioned;
bool recursion_guard;
/* Only used while actively reading a record from the stream. */
@@ -3160,6 +3168,9 @@ struct libxl__stream_read_state {
_hidden void libxl__stream_read_init(libxl__stream_read_state *stream);
_hidden void libxl__stream_read_start(libxl__egc *egc,
libxl__stream_read_state *stream);
+_hidden void libxl__stream_read_start_postcopy_transition(
+ libxl__egc *egc,
+ libxl__stream_read_state *stream);
_hidden void libxl__stream_read_start_checkpoint(libxl__egc *egc,
libxl__stream_read_state
*stream);
_hidden void libxl__stream_read_checkpoint_state(libxl__egc *egc,
@@ -3709,8 +3720,36 @@ struct libxl__domain_create_state {
int restore_fd, libxc_fd;
int restore_fdfl; /* original flags of restore_fd */
int send_back_fd;
+ bool *postcopy_resumed;
libxl_domain_restore_params restore_params;
uint32_t domid_soft_reset;
+ struct {
+ /*
+ * Is a postcopy resumption in progress? (i.e. does the rest of this
+ * state have any meaning?)
+ */
+ bool active;
+
+ struct {
+ enum {
+ DCS_POSTCOPY_RESUME_INPROGRESS,
+ DCS_POSTCOPY_RESUME_FAILED,
+ DCS_POSTCOPY_RESUME_SUCCESS
+ } state;
+
+ int rc;
+ } resume;
+
+ struct {
+ enum {
+ DCS_POSTCOPY_STREAM_INPROGRESS,
+ DCS_POSTCOPY_STREAM_FAILED,
+ DCS_POSTCOPY_STREAM_SUCCESS
+ } state;
+
+ int rc;
+ } stream;
+ } postcopy;
libxl__domain_create_cb *callback;
libxl_asyncprogress_how aop_console_how;
/* private to domain_create */
diff --git a/tools/libxl/libxl_stream_read.c b/tools/libxl/libxl_stream_read.c
index 4cb553e..8e9b720 100644
--- a/tools/libxl/libxl_stream_read.c
+++ b/tools/libxl/libxl_stream_read.c
@@ -35,6 +35,7 @@
* Undefined undef undef undef undef
* Idle false undef 0 0
* Active true NORMAL 0/1 0/partial
+ * Active true POSTCOPY_TRANSITION 0/1 0/partial
* Active true CHECKPOINT_BUFFERING any 0/partial
* Active true CHECKPOINT_UNBUFFERING any 0
* Active true CHECKPOINT_STATE 0/1 0/partial
@@ -133,6 +134,8 @@
/* Success/error/cleanup handling. */
static void stream_complete(libxl__egc *egc,
libxl__stream_read_state *stream, int rc);
+static void postcopy_transition_done(libxl__egc *egc,
+ libxl__stream_read_state *stream, int rc);
static void checkpoint_done(libxl__egc *egc,
libxl__stream_read_state *stream, int rc);
static void stream_done(libxl__egc *egc,
@@ -222,6 +225,7 @@ void libxl__stream_read_init(libxl__stream_read_state
*stream)
FILLZERO(stream->hdr);
LIBXL_STAILQ_INIT(&stream->record_queue);
stream->phase = SRS_PHASE_NORMAL;
+ stream->postcopy_transitioned = false;
stream->recursion_guard = false;
stream->incoming_record = NULL;
FILLZERO(stream->emu_dc);
@@ -299,6 +303,26 @@ void libxl__stream_read_start(libxl__egc *egc,
stream_complete(egc, stream, rc);
}
+void libxl__stream_read_start_postcopy_transition(
+ libxl__egc *egc,
+ libxl__stream_read_state *stream)
+{
+ int checkpointed_stream = stream->dcs->restore_params.checkpointed_stream;
+
+ assert(stream->running);
+ assert(checkpointed_stream == LIBXL_CHECKPOINTED_STREAM_NONE);
+ assert(stream->phase == SRS_PHASE_NORMAL);
+ assert(!stream->postcopy_transitioned);
+
+ stream->phase = SRS_PHASE_POSTCOPY_TRANSITION;
+
+ /*
+ * Libxc has handed control of the fd to us. Start reading some
+ * libxl records out of it.
+ */
+ stream_continue(egc, stream);
+}
+
void libxl__stream_read_start_checkpoint(libxl__egc *egc,
libxl__stream_read_state *stream)
{
@@ -397,6 +421,7 @@ static void stream_continue(libxl__egc *egc,
switch (stream->phase) {
case SRS_PHASE_NORMAL:
+ case SRS_PHASE_POSTCOPY_TRANSITION:
case SRS_PHASE_CHECKPOINT_STATE:
/*
* Normal phase (regular migration or restore from file):
@@ -576,6 +601,13 @@ static bool process_record(libxl__egc *egc,
LOG(DEBUG, "Record: %u, length %u", rec->hdr.type, rec->hdr.length);
+ if (stream->postcopy_transitioned &&
+ rec->hdr.type != REC_TYPE_END) {
+ rc = ERROR_FAIL;
+ LOG(ERROR, "Received non-end record after postcopy transition");
+ goto err;
+ }
+
switch (rec->hdr.type) {
case REC_TYPE_END:
@@ -627,6 +659,15 @@ static bool process_record(libxl__egc *egc,
write_emulator_blob(egc, stream, rec);
break;
+ case REC_TYPE_POSTCOPY_TRANSITION_END:
+ if (stream->phase != SRS_PHASE_POSTCOPY_TRANSITION) {
+ LOG(ERROR, "Unexpected POSTCOPY_TRANSITION_END record in stream");
+ rc = ERROR_FAIL;
+ goto err;
+ }
+ postcopy_transition_done(egc, stream, 0);
+ break;
+
case REC_TYPE_CHECKPOINT_END:
if (!stream_in_checkpoint(stream)) {
LOG(ERROR, "Unexpected CHECKPOINT_END record in stream");
@@ -761,6 +802,13 @@ static void stream_complete(libxl__egc *egc,
*/
checkpoint_done(egc, stream, rc);
break;
+ case SRS_PHASE_POSTCOPY_TRANSITION:
+ assert(rc);
+
+ /*
+ * To deal with errors during the postcopy transition, we use the same
+ * strategy as during checkpoints.
+ */
case SRS_PHASE_CHECKPOINT_STATE:
assert(rc);
@@ -777,6 +825,15 @@ static void stream_complete(libxl__egc *egc,
}
}
+static void postcopy_transition_done(libxl__egc *egc,
+ libxl__stream_read_state *stream, int rc)
+{
+ assert(stream->phase == SRS_PHASE_POSTCOPY_TRANSITION);
+ stream->postcopy_transitioned = true;
+ stream->phase = SRS_PHASE_NORMAL;
+ stream->postcopy_transition_callback(egc, stream, rc);
+}
+
static void checkpoint_done(libxl__egc *egc,
libxl__stream_read_state *stream, int rc)
{
diff --git a/tools/ocaml/libs/xl/xenlight_stubs.c
b/tools/ocaml/libs/xl/xenlight_stubs.c
index 98b52b9..3ef5a1e 100644
--- a/tools/ocaml/libs/xl/xenlight_stubs.c
+++ b/tools/ocaml/libs/xl/xenlight_stubs.c
@@ -538,7 +538,7 @@ value stub_libxl_domain_create_restore(value ctx, value
domain_config, value par
caml_enter_blocking_section();
ret = libxl_domain_create_restore(CTX, &c_dconfig, &c_domid, restore_fd,
- -1, &c_params, ao_how, NULL);
+ -1, NULL, &c_params, ao_how, NULL);
caml_leave_blocking_section();
free(ao_how);
diff --git a/tools/xl/xl_vmcontrol.c b/tools/xl/xl_vmcontrol.c
index 89c2b25..47ba9f3 100644
--- a/tools/xl/xl_vmcontrol.c
+++ b/tools/xl/xl_vmcontrol.c
@@ -882,7 +882,7 @@ start:
ret = libxl_domain_create_restore(ctx, &d_config,
&domid, restore_fd,
- send_back_fd, ¶ms,
+ send_back_fd, NULL, ¶ms,
0, autoconnect_console_how);
libxl_domain_restore_params_dispose(¶ms);
--
2.7.4
_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxxx
https://lists.xenproject.org/mailman/listinfo/xen-devel
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |