From mboxrd@z Thu Jan 1 00:00:00 1970 From: Andrew Cooper Subject: [PATCH 23/27] tools/libxl: [RFC] Write checkpoint records into the stream Date: Mon, 15 Jun 2015 14:44:36 +0100 Message-ID: <1434375880-30914-24-git-send-email-andrew.cooper3@citrix.com> References: <1434375880-30914-1-git-send-email-andrew.cooper3@citrix.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <1434375880-30914-1-git-send-email-andrew.cooper3@citrix.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: Xen-devel Cc: Wei Liu , Yang Hongyang , Ian Jackson , Ian Campbell , Andrew Cooper List-Id: xen-devel@lists.xenproject.org when signalled to do so by libxl__remus_domain_checkpoint_callback() Signed-off-by: Andrew Cooper CC: Ian Campbell CC: Ian Jackson CC: Wei Liu --- tools/libxl/libxl_dom.c | 16 +++--- tools/libxl/libxl_internal.h | 7 +++ tools/libxl/libxl_stream_write.c | 111 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 121 insertions(+), 13 deletions(-) diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c index 06bfaab..3597a91 100644 --- a/tools/libxl/libxl_dom.c +++ b/tools/libxl/libxl_dom.c @@ -1867,8 +1867,8 @@ static void remus_devices_preresume_cb(libxl__egc *egc, /*----- remus asynchronous checkpoint callback -----*/ -static void remus_checkpoint_dm_saved(libxl__egc *egc, - libxl__domain_suspend_state *dss, int rc); +static void remus_checkpoint_stream_written( + libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); static void remus_devices_commit_cb(libxl__egc *egc, libxl__remus_devices_state *rds, int rc); @@ -1882,16 +1882,11 @@ static void libxl__remus_domain_checkpoint_callback(void *data) libxl__egc *egc = dss->shs.egc; STATE_AO_GC(dss->ao); - /* This would go into tailbuf. */ - if (dss->hvm) { - libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved); - } else { - remus_checkpoint_dm_saved(egc, dss, 0); - } + libxl__stream_write_start_checkpoint(egc, &dss->sws); } -static void remus_checkpoint_dm_saved(libxl__egc *egc, - libxl__domain_suspend_state *dss, int rc) +static void remus_checkpoint_stream_written( + libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { /* Convenience aliases */ libxl__remus_devices_state *const rds = &dss->rds; @@ -2036,6 +2031,7 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss) callbacks->suspend = libxl__remus_domain_suspend_callback; callbacks->postcopy = libxl__remus_domain_resume_callback; callbacks->checkpoint = libxl__remus_domain_checkpoint_callback; + dss->sws.checkpoint_callback = remus_checkpoint_stream_written; } else callbacks->suspend = libxl__domain_suspend_callback; diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 82cd792..bf1c377 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -2879,17 +2879,24 @@ struct libxl__stream_write_state { void (*completion_callback)(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); + void (*checkpoint_callback)(libxl__egc *egc, + libxl__domain_suspend_state *dss, + int rc); /* Private */ int rc; int joined_rc; size_t padding; bool running; + bool in_checkpoint; libxl__datacopier_state dc; }; _hidden void libxl__stream_write_start(libxl__egc *egc, libxl__stream_write_state *stream); +_hidden void libxl__stream_write_start_checkpoint( + libxl__egc *egc, libxl__stream_write_state *stream); + _hidden void libxl__stream_write_abort(libxl__egc *egc, libxl__stream_write_state *stream, int rc); diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c index d28a8a5..40f2cb7 100644 --- a/tools/libxl/libxl_stream_write.c +++ b/tools/libxl/libxl_stream_write.c @@ -23,6 +23,9 @@ * - libxl__stream_write_start() * - Start writing a stream from the start. * + * - libxl__stream_write_start() + * - Write the records which form a checkpoint into a stream. + * * In normal operation, there are two tasks running at once; this stream * processing, and the the libxl-save-helper. check_stream_finished() is used * to join all the tasks in both success and error cases. @@ -39,6 +42,12 @@ * - Toolstack record * - if (hvm), Qemu record * - End record + * + * For checkpointed stream, there is a second loop which is triggered by a + * save-helper checkpoint callback. It writes: + * - Toolstack record + * - if (hvm), Qemu record + * - Checkpoint end record */ static const uint8_t zero_padding[1U << REC_ALIGN_ORDER] = { 0 }; @@ -81,6 +90,16 @@ static void end_record_done(libxl__egc *egc, libxl__datacopier_state *dc, int onwrite, int errnoval); +/* Event callbacks unique to checkpointed streams. */ +static void checkpoint_done(libxl__egc *egc, + libxl__stream_write_state *stream, + int rc); +static void write_checkpoint_end_record(libxl__egc *egc, + libxl__stream_write_state *stream); +static void checkpoint_end_record_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval); + void libxl__stream_write_start(libxl__egc *egc, libxl__stream_write_state *stream) { @@ -119,6 +138,16 @@ void libxl__stream_write_start(libxl__egc *egc, stream_failed(egc, stream, ret); } +void libxl__stream_write_start_checkpoint(libxl__egc *egc, + libxl__stream_write_state *stream) +{ + assert(stream->running); + assert(!stream->in_checkpoint); + stream->in_checkpoint = true; + + write_toolstack_record(egc, stream); +} + void libxl__stream_write_abort(libxl__egc *egc, libxl__stream_write_state *stream, int rc) { @@ -130,6 +159,7 @@ static void stream_success(libxl__egc *egc, libxl__stream_write_state *stream) stream->rc = 0; stream->running = false; + assert(!stream->in_checkpoint); stream_done(egc, stream); } @@ -139,6 +169,15 @@ static void stream_failed(libxl__egc *egc, assert(rc); stream->rc = rc; + /* + *If we are in a checkpoint, pass the failure to libxc, which will come + * back around to us via libxl__xc_domain_save_done(). + */ + if (stream->in_checkpoint) { + checkpoint_done(egc, stream, rc); + return; + } + if (stream->running) { stream->running = false; stream_done(egc, stream); @@ -151,6 +190,7 @@ static void stream_done(libxl__egc *egc, libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); assert(!stream->running); + assert(!stream->in_checkpoint); check_stream_finished(egc, dss, stream->rc, "stream"); } @@ -335,8 +375,12 @@ static void toolstack_record_done(libxl__egc *egc, if (dss->type == LIBXL_DOMAIN_TYPE_HVM) write_emulator_record(egc, stream); - else - write_end_record(egc, stream); + else { + if (stream->in_checkpoint) + write_checkpoint_end_record(egc, stream); + else + write_end_record(egc, stream); + } return; @@ -473,7 +517,10 @@ static void emulator_padding_done(libxl__egc *egc, goto err; } - write_end_record(egc, stream); + if (stream->in_checkpoint) + write_checkpoint_end_record(egc, stream); + else + write_end_record(egc, stream); return; err: @@ -526,6 +573,64 @@ static void end_record_done(libxl__egc *egc, stream_failed(egc, stream, ret); } +static void checkpoint_done(libxl__egc *egc, + libxl__stream_write_state *stream, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); + + assert(stream->in_checkpoint); + stream->in_checkpoint = false; + stream->checkpoint_callback(egc, dss, rc); +} + +static void write_checkpoint_end_record(libxl__egc *egc, + libxl__stream_write_state *stream) +{ + libxl__datacopier_state *dc = &stream->dc; + STATE_AO_GC(stream->ao); + struct libxl_sr_rec_hdr rec = { REC_TYPE_CHECKPOINT_END, 0 }; + int ret = 0; + + assert(stream->in_checkpoint); + + dc->copywhat = "checkpoint record"; + dc->writewhat = "save/migration stream"; + dc->callback = checkpoint_end_record_done; + + ret = libxl__datacopier_start(dc); + if (ret) + goto err; + + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +static void checkpoint_end_record_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval) +{ + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); + STATE_AO_GC(stream->ao); + int ret = 0; + + if (onwrite || errnoval) { + ret = ERROR_FAIL; + goto err; + } + + checkpoint_done(egc, stream, 0); + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + /* * Local variables: * mode: C -- 1.7.10.4