From mboxrd@z Thu Jan 1 00:00:00 1970 From: Yang Hongyang Subject: [PATCH v2 4/6] tools/libxl: move remus code into libxl_remus.c Date: Wed, 3 Jun 2015 16:01:31 +0800 Message-ID: <1433318493-24561-5-git-send-email-yanghy@cn.fujitsu.com> References: <1433318493-24561-1-git-send-email-yanghy@cn.fujitsu.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <1433318493-24561-1-git-send-email-yanghy@cn.fujitsu.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: xen-devel@lists.xen.org Cc: wei.liu2@citrix.com, ian.campbell@citrix.com, wency@cn.fujitsu.com, andrew.cooper3@citrix.com, yunhong.jiang@intel.com, eddie.dong@intel.com, guijianfeng@cn.fujitsu.com, rshriram@cs.ubc.ca, ian.jackson@eu.citrix.com List-Id: xen-devel@lists.xenproject.org move remus code into libxl_remus.c. Signed-off-by: Yang Hongyang CC: Ian Campbell CC: Ian Jackson CC: Wei Liu --- tools/libxl/Makefile | 2 +- tools/libxl/libxl.c | 55 +------- tools/libxl/libxl_dom.c | 206 +---------------------------- tools/libxl/libxl_internal.h | 11 ++ tools/libxl/libxl_remus.c | 304 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 318 insertions(+), 260 deletions(-) create mode 100644 tools/libxl/libxl_remus.c diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index 3f98d62..8535eaa 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -56,7 +56,7 @@ else LIBXL_OBJS-y += libxl_nonetbuffer.o endif -LIBXL_OBJS-y += libxl_remus_device.o libxl_remus_disk_drbd.o +LIBXL_OBJS-y += libxl_remus.o libxl_remus_device.o libxl_remus_disk_drbd.o LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o libxl_psr.o LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o libxl_libfdt_compat.o diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 77c6a36..0f9248e 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -792,10 +792,6 @@ out: return ptr; } -static void libxl__remus_setup_done(libxl__egc *egc, - libxl__remus_devices_state *rds, int rc); -static void libxl__remus_setup_failed(libxl__egc *egc, - libxl__remus_devices_state *rds, int rc); static void remus_failover_cb(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); @@ -844,63 +840,14 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, assert(info); - /* Convenience aliases */ - libxl__remus_devices_state *const rds = &dss->rds; - - if (libxl_defbool_val(info->netbuf)) { - if (!libxl__netbuffer_enabled(gc)) { - LOG(ERROR, "Remus: No support for network buffering"); - rc = ERROR_FAIL; - goto out; - } - rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF); - } - - if (libxl_defbool_val(info->diskbuf)) - rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD); - - rds->ao = ao; - rds->domid = domid; - rds->callback = libxl__remus_setup_done; - /* Point of no return */ - libxl__remus_devices_setup(egc, rds); + libxl__remus_setup(egc, dss); return AO_INPROGRESS; out: return AO_ABORT(rc); } -static void libxl__remus_setup_done(libxl__egc *egc, - libxl__remus_devices_state *rds, int rc) -{ - libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); - STATE_AO_GC(dss->ao); - - if (!rc) { - libxl__domain_save(egc, dss); - return; - } - - LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d", - dss->domid, rc); - rds->callback = libxl__remus_setup_failed; - libxl__remus_devices_teardown(egc, rds); -} - -static void libxl__remus_setup_failed(libxl__egc *egc, - libxl__remus_devices_state *rds, int rc) -{ - libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); - STATE_AO_GC(dss->ao); - - if (rc) - LOG(ERROR, "Remus: failed to teardown device after setup failed" - " for guest with domid %u, rc %d", dss->domid, rc); - - dss->callback(egc, dss, rc); -} - static void remus_failover_cb(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c index 701e9f7..0f81081 100644 --- a/tools/libxl/libxl_dom.c +++ b/tools/libxl/libxl_dom.c @@ -1409,189 +1409,6 @@ int libxl__toolstack_save(uint32_t domid, uint8_t **buf, return 0; } -/*----- remus callbacks -----*/ -static void remus_domain_suspend_callback_common_done(libxl__egc *egc, - libxl__domain_suspend_state *dss, int ok); -static void remus_devices_postsuspend_cb(libxl__egc *egc, - libxl__remus_devices_state *rds, - int rc); -static void remus_devices_preresume_cb(libxl__egc *egc, - libxl__remus_devices_state *rds, - int rc); - -static void libxl__remus_domain_suspend_callback(void *data) -{ - libxl__save_helper_state *shs = data; - libxl__egc *egc = shs->egc; - libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); - - dss->callback_common_done = remus_domain_suspend_callback_common_done; - libxl__domain_suspend(egc, dss); -} - -static void remus_domain_suspend_callback_common_done(libxl__egc *egc, - libxl__domain_suspend_state *dss, int ok) -{ - if (!ok) - goto out; - - libxl__remus_devices_state *const rds = &dss->rds; - rds->callback = remus_devices_postsuspend_cb; - libxl__remus_devices_postsuspend(egc, rds); - return; - -out: - libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); -} - -static void remus_devices_postsuspend_cb(libxl__egc *egc, - libxl__remus_devices_state *rds, - int rc) -{ - int ok = 0; - libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); - - if (rc) - goto out; - - ok = 1; - -out: - libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); -} - -static void libxl__remus_domain_resume_callback(void *data) -{ - libxl__save_helper_state *shs = data; - libxl__egc *egc = shs->egc; - libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); - STATE_AO_GC(dss->ao); - - libxl__remus_devices_state *const rds = &dss->rds; - rds->callback = remus_devices_preresume_cb; - libxl__remus_devices_preresume(egc, rds); -} - -static void remus_devices_preresume_cb(libxl__egc *egc, - libxl__remus_devices_state *rds, - int rc) -{ - int ok = 0; - libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); - STATE_AO_GC(dss->ao); - - if (rc) - goto out; - - /* Resumes the domain and the device model */ - rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1); - if (rc) - goto out; - - ok = 1; - -out: - libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); -} - -/*----- remus asynchronous checkpoint callback -----*/ - -static void remus_checkpoint_dm_saved(libxl__egc *egc, - libxl__domain_suspend_state *dss, int rc); -static void remus_devices_commit_cb(libxl__egc *egc, - libxl__remus_devices_state *rds, - int rc); -static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, - const struct timeval *requested_abs); - -static void libxl__remus_domain_checkpoint_callback(void *data) -{ - libxl__save_helper_state *shs = data; - libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); - libxl__egc *egc = dss->shs.egc; - STATE_AO_GC(dss->ao); - - /* This would go into tailbuf. */ - if (dss->hvm) { - libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved); - } else { - remus_checkpoint_dm_saved(egc, dss, 0); - } -} - -static void remus_checkpoint_dm_saved(libxl__egc *egc, - libxl__domain_suspend_state *dss, int rc) -{ - /* Convenience aliases */ - libxl__remus_devices_state *const rds = &dss->rds; - - STATE_AO_GC(dss->ao); - - if (rc) { - LOG(ERROR, "Failed to save device model. Terminating Remus.."); - goto out; - } - - rds->callback = remus_devices_commit_cb; - libxl__remus_devices_commit(egc, rds); - - return; - -out: - libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); -} - -static void remus_devices_commit_cb(libxl__egc *egc, - libxl__remus_devices_state *rds, - int rc) -{ - libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); - - STATE_AO_GC(dss->ao); - - if (rc) { - LOG(ERROR, "Failed to do device commit op." - " Terminating Remus.."); - goto out; - } - - /* - * At this point, we have successfully checkpointed the guest and - * committed it at the backup. We'll come back after the checkpoint - * interval to checkpoint the guest again. Until then, let the guest - * continue execution. - */ - - /* Set checkpoint interval timeout */ - rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout, - remus_next_checkpoint, - dss->interval); - - if (rc) - goto out; - - return; - -out: - libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); -} - -static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, - const struct timeval *requested_abs) -{ - libxl__domain_suspend_state *dss = - CONTAINER_OF(ev, *dss, checkpoint_timeout); - - STATE_AO_GC(dss->ao); - - /* - * Time to checkpoint the guest again. We return 1 to libxc - * (xc_domain_save.c). in order to continue executing the infinite loop - * (suspend, checkpoint, resume) in xc_domain_save(). - */ - libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1); -} - /*----- main code for suspending, in order of execution -----*/ static void domain_save_done(libxl__egc *egc, @@ -1811,10 +1628,6 @@ static void save_device_model_datacopier_done(libxl__egc *egc, dss->save_dm_callback(egc, dss, our_rc); } -static void remus_teardown_done(libxl__egc *egc, - libxl__remus_devices_state *rds, - int rc); - static void domain_save_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { @@ -1840,24 +1653,7 @@ static void domain_save_done(libxl__egc *egc, * from sending checkpoints. Teardown the network buffers and * release netlink resources. This is an async op. */ - LOG(WARN, "Remus: Domain suspend terminated with rc %d," - " teardown Remus devices...", rc); - dss->rds.callback = remus_teardown_done; - libxl__remus_devices_teardown(egc, &dss->rds); -} - -static void remus_teardown_done(libxl__egc *egc, - libxl__remus_devices_state *rds, - int rc) -{ - libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); - STATE_AO_GC(dss->ao); - - if (rc) - LOG(ERROR, "Remus: failed to teardown device for guest with domid %u," - " rc %d", dss->domid, rc); - - dss->callback(egc, dss, rc); + libxl__remus_teardown(egc, dss, rc); } /*==================== Miscellaneous ====================*/ diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 164c448..dc6b62b 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -3193,6 +3193,17 @@ _hidden void libxl__domain_suspend(libxl__egc *egc, /* used by libxc to suspend the guest during migration */ _hidden void libxl__domain_suspend_callback(void *data); +/* Remus callbacks for save */ +_hidden void libxl__remus_domain_suspend_callback(void *data); +_hidden void libxl__remus_domain_resume_callback(void *data); +_hidden void libxl__remus_domain_checkpoint_callback(void *data); +/* Remus setup and teardown*/ +_hidden void libxl__remus_setup(libxl__egc *egc, + libxl__domain_suspend_state *dss); +_hidden void libxl__remus_teardown(libxl__egc *egc, + libxl__domain_suspend_state *dss, + int rc); + /* * Convenience macros. */ diff --git a/tools/libxl/libxl_remus.c b/tools/libxl/libxl_remus.c new file mode 100644 index 0000000..51261e0 --- /dev/null +++ b/tools/libxl/libxl_remus.c @@ -0,0 +1,304 @@ +/* + * Copyright (C) 2009 Citrix Ltd. + * Author Yang Hongyang + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2 and later. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +/*----- Remus setup and teardown -----*/ + +static void remus_setup_done(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc); +static void remus_setup_failed(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc); + +void libxl__remus_setup(libxl__egc *egc, libxl__domain_suspend_state *dss) +{ + /* Convenience aliases */ + libxl__remus_devices_state *const rds = &dss->rds; + const libxl_domain_remus_info *const info = dss->remus; + + STATE_AO_GC(dss->ao); + + if (libxl_defbool_val(info->netbuf)) { + if (!libxl__netbuffer_enabled(gc)) { + LOG(ERROR, "Remus: No support for network buffering"); + goto out; + } + rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VIF); + } + + if (libxl_defbool_val(info->diskbuf)) + rds->device_kind_flags |= (1 << LIBXL__DEVICE_KIND_VBD); + + rds->ao = ao; + rds->domid = dss->domid; + rds->callback = remus_setup_done; + + libxl__remus_devices_setup(egc, rds); + return; + +out: + dss->callback(egc, dss, ERROR_FAIL); +} + +static void remus_setup_done(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (!rc) { + libxl__domain_save(egc, dss); + return; + } + + LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d", + dss->domid, rc); + rds->callback = remus_setup_failed; + libxl__remus_devices_teardown(egc, rds); +} + +static void remus_setup_failed(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + LOG(ERROR, "Remus: failed to teardown device after setup failed" + " for guest with domid %u, rc %d", dss->domid, rc); + + dss->callback(egc, dss, rc); +} + +static void remus_teardown_done(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); +void libxl__remus_teardown(libxl__egc *egc, + libxl__domain_suspend_state *dss, + int rc) +{ + EGC_GC; + + LOG(WARN, "Remus: Domain suspend terminated with rc %d," + " teardown Remus devices...", rc); + dss->rds.callback = remus_teardown_done; + libxl__remus_devices_teardown(egc, &dss->rds); +} + +static void remus_teardown_done(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + LOG(ERROR, "Remus: failed to teardown device for guest with domid %u," + " rc %d", dss->domid, rc); + + dss->callback(egc, dss, rc); +} + +/*----- remus callbacks -----*/ +static void remus_domain_suspend_callback_common_done(libxl__egc *egc, + libxl__domain_suspend_state *dss, int ok); +static void remus_devices_postsuspend_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); +static void remus_devices_preresume_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); + +void libxl__remus_domain_suspend_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__egc *egc = shs->egc; + libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); + + dss->callback_common_done = remus_domain_suspend_callback_common_done; + libxl__domain_suspend(egc, dss); +} + +static void remus_domain_suspend_callback_common_done(libxl__egc *egc, + libxl__domain_suspend_state *dss, int ok) +{ + if (!ok) + goto out; + + libxl__remus_devices_state *const rds = &dss->rds; + rds->callback = remus_devices_postsuspend_cb; + libxl__remus_devices_postsuspend(egc, rds); + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + +static void remus_devices_postsuspend_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + + if (rc) + goto out; + + ok = 1; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + +void libxl__remus_domain_resume_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__egc *egc = shs->egc; + libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); + STATE_AO_GC(dss->ao); + + libxl__remus_devices_state *const rds = &dss->rds; + rds->callback = remus_devices_preresume_cb; + libxl__remus_devices_preresume(egc, rds); +} + +static void remus_devices_preresume_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + goto out; + + /* Resumes the domain and the device model */ + rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1); + if (rc) + goto out; + + ok = 1; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + +/*----- remus asynchronous checkpoint callback -----*/ + +static void remus_checkpoint_dm_saved(libxl__egc *egc, + libxl__domain_suspend_state *dss, int rc); +static void remus_devices_commit_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, + const struct timeval *requested_abs); + +void libxl__remus_domain_checkpoint_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); + libxl__egc *egc = dss->shs.egc; + STATE_AO_GC(dss->ao); + + /* This would go into tailbuf. */ + if (dss->hvm) { + libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved); + } else { + remus_checkpoint_dm_saved(egc, dss, 0); + } +} + +static void remus_checkpoint_dm_saved(libxl__egc *egc, + libxl__domain_suspend_state *dss, int rc) +{ + /* Convenience aliases */ + libxl__remus_devices_state *const rds = &dss->rds; + + STATE_AO_GC(dss->ao); + + if (rc) { + LOG(ERROR, "Failed to save device model. Terminating Remus.."); + goto out; + } + + rds->callback = remus_devices_commit_cb; + libxl__remus_devices_commit(egc, rds); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void remus_devices_commit_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + + STATE_AO_GC(dss->ao); + + if (rc) { + LOG(ERROR, "Failed to do device commit op." + " Terminating Remus.."); + goto out; + } + + /* + * At this point, we have successfully checkpointed the guest and + * committed it at the backup. We'll come back after the checkpoint + * interval to checkpoint the guest again. Until then, let the guest + * continue execution. + */ + + /* Set checkpoint interval timeout */ + rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout, + remus_next_checkpoint, + dss->interval); + + if (rc) + goto out; + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, + const struct timeval *requested_abs) +{ + libxl__domain_suspend_state *dss = + CONTAINER_OF(ev, *dss, checkpoint_timeout); + + STATE_AO_GC(dss->ao); + + /* + * Time to checkpoint the guest again. We return 1 to libxc + * (xc_domain_save.c). in order to continue executing the infinite loop + * (suspend, checkpoint, resume) in xc_domain_save(). + */ + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1); +} +/* + * Local variables: + * mode: C + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ -- 1.9.1