From 420c3029381c300979017b4f1df293fa2f314b76 Mon Sep 17 00:00:00 2001 From: eillon Date: Sun, 23 Nov 2025 22:53:33 +0800 Subject: [PATCH 1/3] Migration: support the basic framework of URMA migration URMA migration is a feature that doing live migration via urma protocol to fully utilize the high bandwidth brought by the UB architecture. This feature can speed up the live migration, reduce CPU usage during migration, and indirectly affect the migration downtime. We can use --enable-urma-migration/--disable-urma-migration during the compile time to control whether to support the feature. Currently, the feature is auto supported on aarch64 architecture. This feature can be used during migration as this: (monitor) migrate urma://IP --- Kconfig.host | 3 + meson.build | 9 ++ meson_options.txt | 3 + migration/meson.build | 4 + migration/migration-stats.c | 5 +- migration/migration-stats.h | 4 + migration/migration.c | 58 +++++++++++++ migration/migration.h | 14 ++++ migration/options.c | 14 ++++ migration/options.h | 3 + migration/ram.c | 25 +++++- migration/savevm.c | 5 ++ migration/trace-events | 2 +- migration/urma.c | 109 ++++++++++++++++++++++++ migration/urma.h | 154 ++++++++++++++++++++++++++++++++++ qapi/migration.json | 6 +- scripts/meson-buildoptions.sh | 5 ++ util/qemu-sockets.c | 8 ++ 18 files changed, 425 insertions(+), 6 deletions(-) create mode 100644 migration/urma.c create mode 100644 migration/urma.h diff --git a/Kconfig.host b/Kconfig.host index f60ea6cef9..b03d5c6c0d 100644 --- a/Kconfig.host +++ b/Kconfig.host @@ -57,5 +57,8 @@ config HV_BALLOON_POSSIBLE config UB bool +config URMA_MIGRATION + bool + config HAM_MIGRATION bool \ No newline at end of file diff --git a/meson.build b/meson.build index 458d8981cc..60752d9d77 100644 --- a/meson.build +++ b/meson.build @@ -595,6 +595,12 @@ else have_ham_migration = false endif +# urma migration +have_urma_migration = get_option('urma_migration') \ + .require(targetos == 'linux', error_message: 'urma_migration is supported only on Linux') \ + .require(cpu == 'aarch64', error_message: 'urma_migration is supported only on aarch64') \ + .allowed() + # vhost have_vhost_user = get_option('vhost_user') \ .disable_auto_if(targetos != 'linux') \ @@ -2304,6 +2310,7 @@ config_host_data.set('CONFIG_VHOST_USER', have_vhost_user) config_host_data.set('CONFIG_VHOST_CRYPTO', have_vhost_user_crypto) config_host_data.set('CONFIG_UB', have_ub) config_host_data.set('CONFIG_HAM_MIGRATION', have_ham_migration) +config_host_data.set('CONFIG_URMA_MIGRATION', have_urma_migration) config_host_data.set('CONFIG_VHOST_VDPA', have_vhost_vdpa) config_host_data.set('CONFIG_VMNET', vmnet.found()) config_host_data.set('CONFIG_VHOST_USER_BLK_SERVER', have_vhost_user_blk_server) @@ -3018,6 +3025,7 @@ host_kconfig = \ (x11.found() ? ['CONFIG_X11=y'] : []) + \ (have_ub ? ['CONFIG_UB=y'] : []) + \ (have_ham_migration ? ['CONFIG_HAM_MIGRATION=y'] : []) + \ + (have_urma_migration ? ['CONFIG_URMA_MIGRATION=y'] : []) + \ (have_vhost_user ? ['CONFIG_VHOST_USER=y'] : []) + \ (have_vhost_vdpa ? ['CONFIG_VHOST_VDPA=y'] : []) + \ (have_vhost_kernel ? ['CONFIG_VHOST_KERNEL=y'] : []) + \ @@ -4249,6 +4257,7 @@ summary_info += {'QOM debugging': get_option('qom_cast_debug')} summary_info += {'Relocatable install': get_option('relocatable')} summary_info += {'ub support': have_ub} summary_info += {'ham migration support': have_ham_migration} +summary_info += {'urma migration support': have_urma_migration} summary_info += {'vhost-kernel support': have_vhost_kernel} summary_info += {'vhost-net support': have_vhost_net} summary_info += {'vhost-user support': have_vhost_user} diff --git a/meson_options.txt b/meson_options.txt index ea83306b8a..a9227bce69 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -294,6 +294,9 @@ option('sndio', type: 'feature', value: 'auto', option('ub', type: 'feature', value: 'auto', description: 'unify bus support') +option('urma_migration', type: 'feature', value: 'auto', + description: 'live migration via urma protocol support') + option('ham_migration', type: 'feature', value: 'auto', description: 'live migration via memory semantics') diff --git a/migration/meson.build b/migration/meson.build index e8934218c5..ec8cb1a7e9 100644 --- a/migration/meson.build +++ b/migration/meson.build @@ -31,6 +31,10 @@ system_ss.add(files( 'threadinfo.c', ), gnutls) +if have_urma_migration + system_ss.add(files('urma.c')) +endif + if have_ham_migration system_ss.add(files('ham.c')) endif diff --git a/migration/migration-stats.c b/migration/migration-stats.c index f690b98a03..fa32f2ee7e 100644 --- a/migration/migration-stats.c +++ b/migration/migration-stats.c @@ -64,7 +64,8 @@ uint64_t migration_transferred_bytes(void) uint64_t multifd = stat64_get(&mig_stats.multifd_bytes); uint64_t rdma = stat64_get(&mig_stats.rdma_bytes); uint64_t qemu_file = stat64_get(&mig_stats.qemu_file_transferred); + uint64_t urma = stat64_get(&mig_stats.urma_bytes); - trace_migration_transferred_bytes(qemu_file, multifd, rdma); - return qemu_file + multifd + rdma; + trace_migration_transferred_bytes(qemu_file, multifd, rdma, urma); + return qemu_file + multifd + rdma + urma; } diff --git a/migration/migration-stats.h b/migration/migration-stats.h index 05290ade76..24911ce067 100644 --- a/migration/migration-stats.h +++ b/migration/migration-stats.h @@ -97,6 +97,10 @@ typedef struct { * Number of bytes sent through RDMA. */ Stat64 rdma_bytes; + /* + * Number of bytes sent through urma. + */ + Stat64 urma_bytes; /* * Number of pages transferred that were full of zeros. */ diff --git a/migration/migration.c b/migration/migration.c index 9e71a2566b..837e0471cb 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -282,6 +282,12 @@ void migration_incoming_state_destroy(void) { struct MigrationIncomingState *mis = migration_incoming_get_current(); +#ifdef CONFIG_URMA_MIGRATION + if (migrate_urma()) { + urma_migration_cleanup(); + } +#endif + multifd_recv_cleanup(); compress_threads_load_cleanup(); /* @@ -498,6 +504,17 @@ bool migrate_uri_parse(const char *uri, MigrationChannel **channel, return false; } addr->transport = MIGRATION_ADDRESS_TYPE_RDMA; +#ifdef CONFIG_URMA_MIGRATION + } else if (strstart(uri, "urma:", NULL) || strstart(uri, "hcom:", NULL)) { + SocketAddress *saddr = socket_parse(uri, errp); + if (!saddr) { + return false; + } + addr->u.socket.type = saddr->type; + addr->u.socket.u = saddr->u; + addr->transport = MIGRATION_ADDRESS_TYPE_URMA; + g_free(saddr); +#endif } else if (strstart(uri, "tcp:", NULL) || strstart(uri, "unix:", NULL) || strstart(uri, "vsock:", NULL) || @@ -596,6 +613,10 @@ static void qemu_start_incoming_migration(const char *uri, bool has_channels, return; } rdma_start_incoming_migration(&addr->u.rdma, errp); +#endif +#ifdef CONFIG_URMA_MIGRATION + } else if (addr->transport == MIGRATION_ADDRESS_TYPE_URMA) { + urma_start_incoming_migration(&addr->u.socket, errp); #endif } else if (addr->transport == MIGRATION_ADDRESS_TYPE_EXEC) { exec_start_incoming_migration(addr->u.exec.args, errp); @@ -688,6 +709,16 @@ process_incoming_migration_co(void *opaque) migrate_set_state(&mis->state, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_ACTIVE); +#ifdef CONFIG_URMA_MIGRATION + if (migrate_urma()) { + if (qemu_exchange_urma_info(qemu_file_get_return_path(mis->from_src_file), + migrate_get_current()->urma_ctx, + true)) { + goto fail; + } + } +#endif + mis->loadvm_co = qemu_coroutine_self(); ret = qemu_loadvm_state(mis->from_src_file); mis->loadvm_co = NULL; @@ -739,6 +770,12 @@ fail: MIGRATION_STATUS_FAILED); qemu_fclose(mis->from_src_file); +#ifdef CONFIG_URMA_MIGRATION + if (migrate_urma()) { + urma_migration_cleanup(); + } +#endif + multifd_recv_cleanup(); compress_threads_load_cleanup(); @@ -1313,6 +1350,12 @@ static void migrate_fd_cleanup(MigrationState *s) qemu_savevm_state_cleanup(); +#ifdef CONFIG_URMA_MIGRATION + if (migrate_urma()) { + urma_migration_cleanup(); + } +#endif + if (s->to_dst_file) { QEMUFile *tmp; @@ -1623,6 +1666,7 @@ int migrate_init(MigrationState *s, Error **errp) s->threshold_size = 0; s->switchover_acked = false; s->rdma_migration = false; + s->urma_migration = false; s->iteration_num = 0; /* * set mig_stats memory to zero for a new migration @@ -2044,6 +2088,10 @@ void qmp_migrate(const char *uri, bool has_channels, #ifdef CONFIG_RDMA } else if (addr->transport == MIGRATION_ADDRESS_TYPE_RDMA) { rdma_start_outgoing_migration(s, &addr->u.rdma, &local_err); +#endif +#ifdef CONFIG_URMA_MIGRATION + } else if (addr->transport == MIGRATION_ADDRESS_TYPE_URMA) { + urma_start_outgoing_migration(s, &addr->u.socket, &local_err); #endif } else if (addr->transport == MIGRATION_ADDRESS_TYPE_EXEC) { exec_start_outgoing_migration(s, addr->u.exec.args, &local_err); @@ -3409,6 +3457,13 @@ static void *migration_thread(void *opaque) qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_ACTIVE); +#ifdef CONFIG_URMA_MIGRATION + if (migrate_urma()) { + qemu_exchange_urma_info(qemu_file_get_return_path(s->to_dst_file), s->urma_ctx, false); + qemu_urma_import(s->urma_ctx); + } +#endif + s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start; trace_migration_thread_setup_complete(); @@ -3449,6 +3504,9 @@ out: object_unref(OBJECT(s)); rcu_unregister_thread(); migration_threads_remove(thread); +#ifdef CONFIG_URMA_MIGRATION + record_migration_log(s); +#endif #ifdef CONFIG_HAM_MIGRATION ham_migrate_cleanup(); #endif diff --git a/migration/migration.h b/migration/migration.h index 46f0c37fec..0fa6c377a8 100644 --- a/migration/migration.h +++ b/migration/migration.h @@ -26,6 +26,9 @@ #include "qom/object.h" #include "postcopy-ram.h" #include "sysemu/runstate.h" +#ifdef CONFIG_URMA_MIGRATION +#include "urma.h" +#endif struct PostcopyBlocktimeContext; @@ -470,6 +473,17 @@ struct MigrationState { bool switchover_acked; /* Is this a rdma migration */ bool rdma_migration; + + /* Is this a urma migration */ + bool urma_migration; +#ifdef CONFIG_URMA_MIGRATION + URMAContext *urma_ctx; +#endif + int64_t urma_init_time; + int64_t urma_exchange_time; + int64_t last_memcpy_time; + int64_t ram_reg_time; + int64_t dev_mig_time; /* Number of migration iterations */ uint64_t iteration_num; }; diff --git a/migration/options.c b/migration/options.c index 01c9a93adb..c2695aee65 100644 --- a/migration/options.c +++ b/migration/options.c @@ -336,6 +336,15 @@ bool migrate_postcopy_ram(void) return s->capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM]; } +#ifdef CONFIG_URMA_MIGRATION +bool migrate_urma(void) +{ + MigrationState *s = migrate_get_current(); + + return s->urma_migration; +} +#endif + bool migrate_use_ldst(void) { MigrationState *s = migrate_get_current(); @@ -360,6 +369,11 @@ bool migrate_release_ram(void) bool migrate_return_path(void) { MigrationState *s = migrate_get_current(); +#ifdef CONFIG_URMA_MIGRATION + if (migrate_urma()) { + return false; + } +#endif return s->capabilities[MIGRATION_CAPABILITY_RETURN_PATH]; } diff --git a/migration/options.h b/migration/options.h index 92cc79cb26..78f4af0ac8 100644 --- a/migration/options.h +++ b/migration/options.h @@ -38,6 +38,9 @@ bool migrate_pause_before_switchover(void); bool migrate_postcopy_blocktime(void); bool migrate_postcopy_preempt(void); bool migrate_postcopy_ram(void); +#ifdef CONFIG_URMA_MIGRATION +bool migrate_urma(void); +#endif bool migrate_rdma_pin_all(void); bool migrate_release_ram(void); bool migrate_return_path(void); diff --git a/migration/ram.c b/migration/ram.c index f8623153cf..2abedadc7c 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -1216,8 +1216,19 @@ static bool control_save_page(PageSearchStatus *pss, { int ret; +#ifdef CONFIG_URMA_MIGRATION + if (migrate_urma()) { + ret = urma_control_save_page(pss->pss_channel, pss->block->offset, offset, + TARGET_PAGE_SIZE); + } else { + ret = rdma_control_save_page(pss->pss_channel, pss->block->offset, offset, + TARGET_PAGE_SIZE); + } +#else ret = rdma_control_save_page(pss->pss_channel, pss->block->offset, offset, TARGET_PAGE_SIZE); +#endif + if (ret == RAM_SAVE_CONTROL_NOT_SUPP) { return false; } @@ -3678,6 +3689,8 @@ static int ram_save_complete(QEMUFile *f, void *opaque) RAMState **temp = opaque; RAMState *rs = *temp; int ret = 0; + MigrationState *s = migrate_get_current(); + int64_t start_time = 0; rs->last_stage = !migration_in_colo_state(); @@ -3694,6 +3707,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque) /* try transferring iterative blocks of memory */ + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); /* flush all remaining blocks regardless of rate limiting */ qemu_mutex_lock(&rs->bitmap_mutex); while (true) { @@ -3751,7 +3765,16 @@ static int ram_save_complete(QEMUFile *f, void *opaque) qemu_put_be64(f, RAM_SAVE_FLAG_MULTIFD_FLUSH); } qemu_put_be64(f, RAM_SAVE_FLAG_EOS); - return qemu_fflush(f); + ret = qemu_fflush(f); + +#ifdef CONFIG_URMA_MIGRATION + if (migrate_urma()) { + ret |= qemu_flush_urma_write(s->urma_ctx); + } +#endif + + s->last_memcpy_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - start_time; + return ret; } static void ram_state_pending_estimate(void *opaque, uint64_t *must_precopy, diff --git a/migration/savevm.c b/migration/savevm.c index be3bfc1078..4b847060d1 100644 --- a/migration/savevm.c +++ b/migration/savevm.c @@ -1545,6 +1545,9 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f, int vmdesc_len; SaveStateEntry *se; int ret; + int64_t start_time; + + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { if (se->vmsd && se->vmsd->early_setup) { @@ -1602,6 +1605,8 @@ int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f, trace_vmstate_downtime_checkpoint("src-non-iterable-saved"); + ms->dev_mig_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - start_time; + return 0; } diff --git a/migration/trace-events b/migration/trace-events index f0e1cb80c7..cd041bc308 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -193,7 +193,7 @@ process_incoming_migration_co_postcopy_end_main(void) "" postcopy_preempt_enabled(bool value) "%d" # migration-stats -migration_transferred_bytes(uint64_t qemu_file, uint64_t multifd, uint64_t rdma) "qemu_file %" PRIu64 " multifd %" PRIu64 " RDMA %" PRIu64 +migration_transferred_bytes(uint64_t qemu_file, uint64_t multifd, uint64_t rdma, uint64_t urma) "qemu_file %" PRIu64 " multifd %" PRIu64 " RDMA %" PRIu64 " URMA %" PRIu64 # channel.c migration_set_incoming_channel(void *ioc, const char *ioctype) "ioc=%p ioctype=%s" diff --git a/migration/urma.c b/migration/urma.c new file mode 100644 index 0000000000..8fb85e4123 --- /dev/null +++ b/migration/urma.c @@ -0,0 +1,109 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2025. All rights reserved. + * + * Description: Support vm migration using the protocol and interfaces provided by the URMA component. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License along + * with this program; if not, see . + * + */ + +#include "qemu/osdep.h" +#include "qapi/error.h" + +#include "urma.h" +#include "migration.h" +#include "multifd.h" +#include "migration-stats.h" +#include "qemu-file.h" +#include "ram.h" +#include "rdma.h" +#include "qemu/error-report.h" +#include "qemu/main-loop.h" +#include "qemu/module.h" +#include "qemu/rcu.h" +#include "qemu/sockets.h" +#include "qemu/bitmap.h" +#include "qemu/coroutine.h" +#include "exec/memory.h" +#include +#include +#include +#include "trace.h" +#include "qom/object.h" +#include "options.h" +#include +#include "qemu/log.h" +#include +#include "socket.h" +#include "exec/target_page.h" +#include +#include "crypto/random.h" + +int qemu_flush_urma_write(URMAContext *urma) +{ + /* TODO */ + return -EINVAL; +} + +int qemu_urma_import(URMAContext *urma) +{ + /* TODO */ + return -EINVAL; +} + +int qemu_exchange_urma_info(QEMUFile *f, URMAContext *urma, bool server) +{ + /* TODO */ + return -EINVAL; +} + +void urma_start_outgoing_migration(void *opaque, + SocketAddress *saddr, + Error **errp) +{ + /* TODO */ + return; +} + +void urma_start_incoming_migration(SocketAddress *saddr, + Error **errp) +{ + /* TODO */ + return; +} + +void urma_migration_cleanup(void) +{ + /* TODO */ + return; +} + +int urma_control_save_page(QEMUFile *f, ram_addr_t block_offset, + ram_addr_t offset, size_t size) +{ + /* TODO */ + return RAM_SAVE_CONTROL_NOT_SUPP; +} + +void record_migration_log(MigrationState *s) +{ + qemu_log("qmp urma resource initialization and connection cost time: %ld(ms)\n", s->urma_init_time); + qemu_log("qmp urma exchange info cost time: %ld(ms)\n", s->urma_exchange_time); + qemu_log("qmp ram registration cost time: %ld(ms)\n", s->ram_reg_time); + qemu_log("qmp device migration cost time: %ld(ms)\n", s->dev_mig_time); + qemu_log("qmp last memcpy cost time: %ld(ms)\n", s->last_memcpy_time); + qemu_log("qmp downtime %ld(ms)\n", s->downtime); + qemu_log("qmp setup time %ld(ms)\n", s->setup_time); + qemu_log("qmp total time %ld(ms)\n", s->total_time); +} diff --git a/migration/urma.h b/migration/urma.h new file mode 100644 index 0000000000..62cd8cb489 --- /dev/null +++ b/migration/urma.h @@ -0,0 +1,154 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. + * + * Description: Support vm migration using the protocol and interfaces provided by the URMA component. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License along + * with this program; if not, see . + * + */ + +#ifndef QEMU_MIGRATION_URMA_H +#define QEMU_MIGRATION_URMA_H + +#include "qemu/sockets.h" +#include "exec/memory.h" + +#include +#include + +#define URMA_SO_PATH "liburma.so.0" +#define URMA_TOKEN_LEN 32 + +typedef struct QEMU_PACKED URMADestBlock { + uint64_t remote_host_addr; + uint64_t offset; + uint64_t length; + uint32_t remote_rkey; + uint32_t padding; +} URMADestBlock; + +typedef struct URMALocalBlock { + char *block_name; + uint8_t *local_host_addr; /* local virtual address */ + uint64_t remote_host_addr; /* remote virtual address */ + uint64_t offset; + uint64_t length; + int index; /* which block are we */ + unsigned int src_index; /* (Only used on dest) */ + bool is_ram_block; + int nb_chunks; + unsigned long *transit_bitmap; + unsigned long *unregister_bitmap; + + urma_target_seg_t *local_tseg; /* tseg for non-chunk-level registration */ + urma_token_t local_seg_token; + urma_seg_t remote_seg; /* remote seg for non-chunk-level registration */ + urma_token_t remote_seg_token; + urma_target_seg_t *import_tseg; /* Imported target segment for read/write/atomic */ +} URMALocalBlock; + +typedef struct URMALocalBlocks { + int nb_blocks; + bool init; /* main memory init complete */ + URMALocalBlock *block; +} URMALocalBlocks; + + +typedef struct URMAContext { + char *host; + int port; + int id; + + int is_incoming; + + /* number of outstanding writes */ + int nb_sent; + + /* number of polling writes */ + int nb_polling; + + /* store info about current buffer so that we can + merge it with future sends */ + uint64_t current_addr; + uint64_t current_length; + /* index of ram block the current buffer belongs to */ + int current_index; + /* index of the chunk in the current ram block */ + int current_chunk; + + bool pin_all; + + GHashTable *blockmap; + + /* + * Description of ram blocks used throughout the code. + */ + URMALocalBlocks local_ram_blocks; + + + URMADestBlock *dest_blocks; + + /* urma info */ + urma_context_t *urma_ctx; + urma_device_attr_t dev_attr; + + urma_jfce_t *jfce; + urma_jfc_t *jfc; + urma_jfs_t *jfs; + urma_jfr_t *jfr; + uint64_t rid; + urma_token_t jfr_token; + bool event_mode; + int max_jfs_depth; + + int client_sockfd; + int listen_fd; + + urma_jfr_id_t remote_jfr_id; + urma_token_t rjfr_token; + urma_target_jetty_t *tjfr; +} URMAContext; + +typedef struct seg_jfr_info_t { + /* Common */ + urma_eid_t eid; + uint32_t uasid; + /* segment */ + uint64_t seg_va; + uint64_t seg_len; + uint32_t seg_flag; + uint32_t seg_token_id; + urma_token_t seg_token; + /* jfr */ + urma_jfr_id_t jfr_id; + urma_token_t jfr_token; + + /* bond info */ + urma_bond_seg_info_out_t seg_bond_info; + urma_bond_id_info_out_t jfr_bond_info; +} __attribute__((packed)) seg_jfr_info_t; + + +void urma_start_outgoing_migration(void *opaque, SocketAddress *saddr, + Error **errp); +void urma_start_incoming_migration(SocketAddress *saddr, Error **errp); +int urma_control_save_page(QEMUFile *f, ram_addr_t block_offset, + ram_addr_t offset, size_t size); +int qemu_flush_urma_write(URMAContext *urma); +int qemu_exchange_urma_info(QEMUFile *f, URMAContext *urma, bool server); +int qemu_urma_import(URMAContext *urma); +void urma_migration_cleanup(void); +void record_migration_log(MigrationState *s); + +#endif diff --git a/qapi/migration.json b/qapi/migration.json index a22e6df695..cfa2e2c4e3 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -1771,7 +1771,7 @@ # Since 8.2 ## { 'enum': 'MigrationAddressType', - 'data': [ 'socket', 'exec', 'rdma', 'file' ] } + 'data': [ 'socket', 'exec', 'rdma', 'file', 'urma', 'hcom' ] } ## # @FileMigrationArgs: @@ -1810,7 +1810,9 @@ 'socket': 'SocketAddress', 'exec': 'MigrationExecCommand', 'rdma': 'InetSocketAddress', - 'file': 'FileMigrationArgs' } } + 'file': 'FileMigrationArgs', + 'hcom': 'SocketAddress', + 'urma': 'SocketAddress' } } ## # @MigrationChannelType: diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 3b5a146afd..fd3eecbb5e 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -54,6 +54,8 @@ meson_options_help() { printf "%s\n" ' --enable-tsan enable thread sanitizer' printf "%s\n" ' --enable-ub enable unify bus' printf "%s\n" ' --disable-ub disable unify bus' + printf "%s\n" ' --enable-urma-migration enable urma migration' + printf "%s\n" ' --disable-urma-migration disable urma migration' printf "%s\n" ' --enable-ham-migration enable ham migration' printf "%s\n" ' --disable-ham-migration disable ham migration' printf "%s\n" ' --firmwarepath=VALUES search PATH for firmware files [share/qemu-' @@ -195,6 +197,7 @@ meson_options_help() { printf "%s\n" ' u2f U2F emulation support' printf "%s\n" ' ub unify bus support' printf "%s\n" ' ham-migration live migration via memory semantics' + printf "%s\n" ' urma-migration live migration via urma protocol support' printf "%s\n" ' usb-redir libusbredir support' printf "%s\n" ' vde vde network backend support' printf "%s\n" ' vdi vdi image format support' @@ -523,6 +526,8 @@ _meson_option_parse() { --disable-ub) printf "%s" -Dub=disabled ;; --enable-ham-migration) printf "%s" -Dham_migration=enabled ;; --disable-ham-migration) printf "%s" -Dham_migration=disabled ;; + --enable-urma-migration) printf "%s" -Durma_migration=enabled ;; + --disable-urma-migration) printf "%s" -Durma_migration=disabled ;; --enable-usb-redir) printf "%s" -Dusb_redir=enabled ;; --disable-usb-redir) printf "%s" -Dusb_redir=disabled ;; --enable-vde) printf "%s" -Dvde=enabled ;; diff --git a/util/qemu-sockets.c b/util/qemu-sockets.c index 83e84b1186..3ed8997753 100644 --- a/util/qemu-sockets.c +++ b/util/qemu-sockets.c @@ -1122,12 +1122,20 @@ SocketAddress *socket_parse(const char *str, Error **errp) if (inet_parse(&addr->u.inet, str + strlen("tcp:"), errp)) { goto fail; } +#ifdef CONFIG_URMA_MIGTAION + } else if (strstart(str, "urma:", NULL) || strstart(str, "hcom:", NULL)) { + addr->type = SOCKET_ADDRESS_TYPE_INET; + if (inet_parse(&addr->u.inet, str + strlen("urma:"), errp)) { + goto fail; + } +#endif } else { addr->type = SOCKET_ADDRESS_TYPE_INET; if (inet_parse(&addr->u.inet, str, errp)) { goto fail; } } + return addr; fail: -- Gitee From 32025a765654c95955fa84b7b9adcdd81ed1a580 Mon Sep 17 00:00:00 2001 From: eillon Date: Sun, 23 Nov 2025 23:08:35 +0800 Subject: [PATCH 2/3] Migration: support send data through urma protocol during migration Supprot prepare urma during migration init and send data through urma write. --- migration/urma.c | 1240 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1229 insertions(+), 11 deletions(-) diff --git a/migration/urma.c b/migration/urma.c index 8fb85e4123..80af7d8fca 100644 --- a/migration/urma.c +++ b/migration/urma.c @@ -50,50 +50,1268 @@ #include #include "crypto/random.h" -int qemu_flush_urma_write(URMAContext *urma) +#define URMA_REG_CHUNK_SHIFT 24 /* 16 MB */ + +/* Do not merge data if larger than this. */ +#define URMA_CHUNK_MERGE_MAX (1 << URMA_REG_CHUNK_SHIFT) + +#define URMA_MAX_POLL_TIME 100000000 /* ms */ + +#define URMA_DEV_DEFAULT_NAME "bonding_dev_0" /* default use bonding dev */ +#define URMA_DEV_DEFAULT_IDX 0 + +void *handle_urma = NULL; +static const char *urma_dev_name = URMA_DEV_DEFAULT_NAME; +static int urma_dev_idx = URMA_DEV_DEFAULT_IDX; + +urma_status_t (*urma_init_p)(urma_init_attr_t *conf); +urma_status_t (*urma_uninit_p)(void); +urma_device_t **(*urma_get_device_list_p)(int *num_devices); +void (*urma_free_device_list_p)(urma_device_t **device_list); +urma_eid_info_t *(*urma_get_eid_list_p)(urma_device_t *dev, uint32_t *cnt); +void (*urma_free_eid_list_p)(urma_eid_info_t *eid_list); +urma_status_t (*urma_query_device_p)(urma_device_t *dev, urma_device_attr_t *dev_attr); +urma_context_t *(*urma_create_context_p)(urma_device_t *dev, uint32_t eid_index); +urma_status_t (*urma_delete_context_p)(urma_context_t *ctx); +urma_jfc_t *(*urma_create_jfc_p)(urma_context_t *ctx, urma_jfc_cfg_t *jfc_cfg); +urma_status_t (*urma_delete_jfc_p)(urma_jfc_t *jfc); +urma_jfs_t *(*urma_create_jfs_p)(urma_context_t *ctx, urma_jfs_cfg_t *jfs_cfg); +urma_status_t (*urma_delete_jfs_p)(urma_jfs_t *jfs); +urma_jfr_t *(*urma_create_jfr_p)(urma_context_t *ctx, urma_jfr_cfg_t *jfr_cfg); +urma_status_t (*urma_delete_jfr_p)(urma_jfr_t *jfr); +urma_target_jetty_t *(*urma_import_jfr_p)(urma_context_t *ctx, urma_rjfr_t *rjfr, urma_token_t *token_value); +urma_status_t (*urma_unimport_jfr_p)(urma_target_jetty_t *target_jfr); +urma_status_t (*urma_advise_jfr_p)(urma_jfs_t *jfs, urma_target_jetty_t *tjfr); +urma_jfce_t *(*urma_create_jfce_p)(urma_context_t *ctx); +urma_status_t (*urma_delete_jfce_p)(urma_jfce_t *jfce); +urma_target_seg_t *(*urma_register_seg_p)(urma_context_t *ctx, urma_seg_cfg_t *seg_cfg); +urma_status_t (*urma_unregister_seg_p)(urma_target_seg_t *target_seg); +urma_target_seg_t *(*urma_import_seg_p)( + urma_context_t *ctx, urma_seg_t *seg, urma_token_t *token_value, uint64_t addr, urma_import_seg_flag_t flag); +urma_status_t (*urma_unimport_seg_p)(urma_target_seg_t *tseg); +urma_status_t (*urma_write_p)(urma_jfs_t *jfs, urma_target_jetty_t *target_jfr, urma_target_seg_t *dst_tseg, + urma_target_seg_t *src_tseg, uint64_t dst, uint64_t src, uint32_t len, urma_jfs_wr_flag_t flag, uint64_t user_ctx); +int (*urma_poll_jfc_p)(urma_jfc_t *jfc, int cr_cnt, urma_cr_t *cr); +urma_status_t (*urma_user_ctl_p)(urma_context_t *ctx, urma_user_ctl_in_t *in, urma_user_ctl_out_t *out); +urma_status_t (*urma_set_context_opt_p)(urma_context_t *ctx, urma_opt_name_t opt_name, + const void *opt_value, size_t opt_len); + +typedef struct dl_functions { + const char *func_name; + void **func; +} dl_functions; + +dl_functions urma_dlfunc_list[] = { + {.func_name = "urma_init", .func = (void **)&urma_init_p}, + {.func_name = "urma_uninit", .func = (void **)&urma_uninit_p}, + {.func_name = "urma_get_device_list", .func = (void **)&urma_get_device_list_p}, + {.func_name = "urma_free_device_list", .func = (void **)&urma_free_device_list_p}, + {.func_name = "urma_get_eid_list", .func = (void **)&urma_get_eid_list_p}, + {.func_name = "urma_free_eid_list", .func = (void **)&urma_free_eid_list_p}, + {.func_name = "urma_query_device", .func = (void **)&urma_query_device_p}, + {.func_name = "urma_create_context", .func = (void **)&urma_create_context_p}, + {.func_name = "urma_delete_context", .func = (void **)&urma_delete_context_p}, + {.func_name = "urma_create_jfc", .func = (void **)&urma_create_jfc_p}, + {.func_name = "urma_delete_jfc", .func = (void **)&urma_delete_jfc_p}, + {.func_name = "urma_create_jfs", .func = (void **)&urma_create_jfs_p}, + {.func_name = "urma_delete_jfs", .func = (void **)&urma_delete_jfs_p}, + {.func_name = "urma_create_jfr", .func = (void **)&urma_create_jfr_p}, + {.func_name = "urma_delete_jfr", .func = (void **)&urma_delete_jfr_p}, + {.func_name = "urma_import_jfr", .func = (void **)&urma_import_jfr_p}, + {.func_name = "urma_unimport_jfr", .func = (void **)&urma_unimport_jfr_p}, + {.func_name = "urma_advise_jfr", .func = (void **)&urma_advise_jfr_p}, + {.func_name = "urma_create_jfce", .func = (void **)&urma_create_jfce_p}, + {.func_name = "urma_delete_jfce", .func = (void **)&urma_delete_jfce_p}, + {.func_name = "urma_register_seg", .func = (void **)&urma_register_seg_p}, + {.func_name = "urma_unregister_seg", .func = (void **)&urma_unregister_seg_p}, + {.func_name = "urma_import_seg", .func = (void **)&urma_import_seg_p}, + {.func_name = "urma_unimport_seg", .func = (void **)&urma_unimport_seg_p}, + {.func_name = "urma_write", .func = (void **)&urma_write_p}, + {.func_name = "urma_poll_jfc", .func = (void **)&urma_poll_jfc_p}, + {.func_name = "urma_user_ctl", .func = (void **)&urma_user_ctl_p}, + {.func_name = "urma_set_context_opt", .func = (void **)&urma_set_context_opt_p}, +}; + +static void urma_dlfunc_list_set_null(void) +{ + for (int i = 0; i < ARRAY_SIZE(urma_dlfunc_list); i++) { + *urma_dlfunc_list[i].func = NULL; + } +} + +static void urma_dlfunc_close(void) +{ + if (handle_urma) { + (void)dlclose(handle_urma); + handle_urma = NULL; + } + urma_dlfunc_list_set_null(); +} + +static int migrate_get_urma_dlfunc(Error **errp) +{ + char *error = NULL; + + urma_dlfunc_list_set_null(); + handle_urma = dlopen(URMA_SO_PATH, RTLD_LAZY | RTLD_GLOBAL); + if (!handle_urma) { + qemu_log("dlopen error: %s", dlerror()); + return -1; + } + + for (int i = 0; i < ARRAY_SIZE(urma_dlfunc_list); i++) { + *urma_dlfunc_list[i].func = dlsym(handle_urma, urma_dlfunc_list[i].func_name); + if ((error = dlerror()) != NULL) { + qemu_log("dlsym error: %s while getting %s", error, urma_dlfunc_list[i].func_name); + urma_dlfunc_close(); + return -1; + } + } + + return 0; +} + +static int urma_dlfunc_init(Error **errp) +{ + int r; + + r = migrate_get_urma_dlfunc(errp); + if (r < 0) { + qemu_log("dlsym error, open urma dlfunc failed\n"); + return r; + } + + return r; +} + +static inline uint64_t urma_ram_chunk_index(const uint8_t *start, + const uint8_t *host) +{ + return ((uintptr_t) host - (uintptr_t) start) >> URMA_REG_CHUNK_SHIFT; +} + +static inline uint8_t *urma_ram_chunk_start(const URMALocalBlock *ram_block, + uint64_t i) +{ + return (uint8_t *)(uintptr_t)(ram_block->local_host_addr + + (i << URMA_REG_CHUNK_SHIFT)); +} + +static inline uint8_t *urma_ram_chunk_end(const URMALocalBlock *ram_block, + uint64_t i) +{ + uint8_t *result = urma_ram_chunk_start(ram_block, i) + + (1UL << URMA_REG_CHUNK_SHIFT); + + if (result > (ram_block->local_host_addr + ram_block->length)) { + result = ram_block->local_host_addr + ram_block->length; + } + + return result; +} + +static void urma_add_block(URMAContext *urma, const char *block_name, + void *host_addr, + ram_addr_t block_offset, uint64_t length) +{ + URMALocalBlocks *local = &urma->local_ram_blocks; + URMALocalBlock *block; + URMALocalBlock *old = local->block; + + local->block = g_new0(URMALocalBlock, local->nb_blocks + 1); + + if (local->nb_blocks) { + int x; + if (urma->blockmap) { + for (x = 0; x < local->nb_blocks; x++) { + g_hash_table_remove(urma->blockmap, + (void *)(uintptr_t)old[x].offset); + g_hash_table_insert(urma->blockmap, + (void *)(uintptr_t)old[x].offset, + &local->block[x]); + } + } + memcpy(local->block, old, sizeof(URMALocalBlock) * local->nb_blocks); + g_free(old); + } + + block = &local->block[local->nb_blocks]; + + block->block_name = g_strdup(block_name); + block->local_host_addr = host_addr; + block->offset = block_offset; + block->length = length; + block->index = local->nb_blocks; + block->src_index = ~0U; /* Filled in by the receipt of the block list */ + block->nb_chunks = urma_ram_chunk_index(host_addr, host_addr + length) + 1UL; + block->transit_bitmap = bitmap_new(block->nb_chunks); + bitmap_clear(block->transit_bitmap, 0, block->nb_chunks); + block->unregister_bitmap = bitmap_new(block->nb_chunks); + bitmap_clear(block->unregister_bitmap, 0, block->nb_chunks); + + block->is_ram_block = local->init ? false : true; + + if (urma->blockmap) { + g_hash_table_insert(urma->blockmap, (void *)(uintptr_t)block_offset, block); + } + + local->nb_blocks++; +} + +static int qemu_urma_init_one_block(RAMBlock *rb, void *opaque) +{ + const char *block_name = qemu_ram_get_idstr(rb); + void *host_addr = qemu_ram_get_host_addr(rb); + ram_addr_t block_offset = qemu_ram_get_offset(rb); + ram_addr_t length = qemu_ram_get_used_length(rb); + urma_add_block(opaque, block_name, host_addr, block_offset, length); + return 0; +} + +static void qemu_urma_free_blocks(URMAContext *urma) +{ + URMALocalBlocks *local = &urma->local_ram_blocks; + int i; + + for (i = 0; i < local->nb_blocks; i++) { + URMALocalBlock *block = &local->block[i]; + + if (urma->blockmap) { + g_hash_table_remove(urma->blockmap, (void *)(uintptr_t)block->offset); + } + + g_free(block->transit_bitmap); + block->transit_bitmap = NULL; + + g_free(block->unregister_bitmap); + block->unregister_bitmap = NULL; + + g_free(block->block_name); + block->block_name = NULL; + } + + g_free(local->block); + local->block = NULL; + local->nb_blocks = 0; + + g_free(urma->dest_blocks); + urma->dest_blocks = NULL; +} + +static int qemu_urma_init_ram_blocks(URMAContext *urma) +{ + URMALocalBlocks *local = &urma->local_ram_blocks; + int ret; + + if (urma->blockmap != NULL) { + qemu_log("Ram blocks have been inited before! blockmap is %p\n", urma->blockmap); + return -EINVAL; + } + + memset(local, 0, sizeof *local); + ret = foreach_not_ignored_block(qemu_urma_init_one_block, urma); + if (ret) { + qemu_log("do qemu_urma_init_one_block failed, %d\n", ret); + return ret; + } + + urma->dest_blocks = g_new0(URMADestBlock, + urma->local_ram_blocks.nb_blocks); + local->init = true; + + /* Build the hash that maps from offset to RAMBlock */ + urma->blockmap = g_hash_table_new(g_direct_hash, g_direct_equal); + for (int i = 0; i < urma->local_ram_blocks.nb_blocks; i++) { + g_hash_table_insert(urma->blockmap, + (void *)(uintptr_t)urma->local_ram_blocks.block[i].offset, + &urma->local_ram_blocks.block[i]); + } + + return 0; +} + +static void qemu_urma_data_free(URMAContext *urma) +{ + if (urma == NULL) { + return; + } + + g_free(urma->host); + g_free(urma); +} + +static URMAContext *qemu_urma_data_init(InetSocketAddress *saddr) +{ + URMAContext *urma = NULL; + + urma = g_new0(URMAContext, 1); + urma->current_index = -1; + urma->current_chunk = -1; + + urma->host = g_strdup(saddr->host); + urma->port = atoi(saddr->port); + + return urma; +} + +static int qemu_get_urma_eid_index(urma_device_t *dev) +{ + urma_eid_info_t *eid_list; + uint32_t eid_cnt; + int i, eid_index = -1; + + eid_list = urma_get_eid_list_p(dev, &eid_cnt); + if (eid_list == NULL) { + return -1; + } + + for (i = 0; eid_list != NULL && i < eid_cnt; i++) { + qemu_log("device_name :%s (eid%d: "EID_FMT").\n", dev->name, eid_list[i].eid_index, EID_ARGS(eid_list[i].eid)); + } + + if (eid_cnt > 0) { + if (urma_dev_idx >= 0 && urma_dev_idx < eid_cnt) { + eid_index = eid_list[urma_dev_idx].eid_index; + } else { + qemu_log("Invalid urma_dev_idx, use the first one.\n"); + eid_index = eid_list[0].eid_index; + } + + qemu_log("Use the eid%d: "EID_FMT".\n", eid_index, EID_ARGS(eid_list[eid_index].eid)); + } + + urma_free_eid_list_p(eid_list); + return eid_index; +} + +static urma_device_t *qemu_get_urma_device(URMAContext *ctx) +{ + int i, device_num = 0; + urma_device_t *urma_dev = NULL; + urma_device_t **device_list = urma_get_device_list_p(&device_num); + + if (device_list == NULL || device_num == 0) { + qemu_log("Failed to get device list, errno: %d\n", errno); + return NULL; + } + + for (i = 0; i < device_num; i++) { + if (urma_dev_name != NULL && strcmp(device_list[i]->name, urma_dev_name) == 0) { + urma_dev = device_list[i]; + break; + } + } + + /* If the specified device cannot be found, use the first device */ + if (urma_dev == NULL) { + qemu_log("Cannot find the device %s, use the first device\n", urma_dev_name); + urma_dev = device_list[0]; + } + + urma_free_device_list_p(device_list); + return urma_dev; +} + +static int qemu_get_random_u32(uint32_t *rand_value) { - /* TODO */ + char random_char[URMA_TOKEN_LEN / CHAR_BIT]; + Error *local_err = NULL; + + if (qcrypto_random_bytes(random_char, sizeof(random_char), &local_err)) { + qemu_log("cannot get qcrypto random bytes, %s\n", error_get_pretty(local_err)); + error_free(local_err); + return -EINVAL; + } + + memcpy(rand_value, random_char, sizeof(uint32_t)); + + return 0; +} + +static int qemu_urma_init_context(URMAContext *ctx) +{ + int eid_index, ret; + urma_context_aggr_mode_t aggr_mode = URMA_AGGR_MODE_BALANCE; + + ctx->event_mode = false; + + urma_device_t *urma_dev = qemu_get_urma_device(ctx); + if (urma_dev == NULL) { + qemu_log("URMA: urma get device failed, errno: %d\n", errno); + return -EINVAL; + } + + ret = urma_query_device_p(urma_dev, &ctx->dev_attr); + if (ret) { + qemu_log("URMA: Failed to query device %s, ret: %d, errno: %d\n", urma_dev->name, ret, errno); + return ret; + } + + eid_index = qemu_get_urma_eid_index(urma_dev); + if (eid_index < 0) { + qemu_log("URMA: Failed to get eid index, ret: %d, errno: %d.\n", eid_index, errno); + return eid_index; + } + + ctx->urma_ctx = urma_create_context_p(urma_dev, (uint32_t)eid_index); + if (ctx->urma_ctx == NULL) { + qemu_log("URMA: Failed to create instance with eid: %d, errno: %d.\n", eid_index, errno); + return -EINVAL; + } + + ret = urma_set_context_opt_p(ctx->urma_ctx, URMA_OPT_AGGR_MODE, &aggr_mode, sizeof(aggr_mode)); + if (ret) { + qemu_log("URMA: Failed to do urma_set_context_opt, ret: %d, errno: %d\n", ret, errno); + } + + ctx->jfce = urma_create_jfce_p(ctx->urma_ctx); + if (ctx->jfce == NULL) { + qemu_log("URMA: Failed to create jfce, errno: %d.\n", errno); + goto err_del_ctx; + } + + urma_jfc_cfg_t jfc_cfg = { + .depth = ctx->dev_attr.dev_cap.max_jfc_depth, + .flag = {.value = 0}, + .jfce = ctx->jfce, + .user_ctx = (uint64_t)NULL, + }; + ctx->jfc = urma_create_jfc_p(ctx->urma_ctx, &jfc_cfg); + if (ctx->jfc == NULL) { + qemu_log("URMA: Failed to create jfc, errno: %d\n", errno); + goto err_del_jfce; + } + + urma_jfs_cfg_t jfs_cfg = { + .depth = ctx->dev_attr.dev_cap.max_jfs_depth, + .trans_mode = URMA_TM_RM, + .priority = URMA_MAX_PRIORITY, /* Highest priority */ + .max_sge = 1, + .max_inline_data = 0, + .rnr_retry = URMA_TYPICAL_RNR_RETRY, + .err_timeout = URMA_TYPICAL_ERR_TIMEOUT, + .jfc = ctx->jfc, + .flag.bs.multi_path = 1, + .user_ctx = (uint64_t)NULL + }; + ctx->jfs = urma_create_jfs_p(ctx->urma_ctx, &jfs_cfg); + if (ctx->jfs == NULL) { + qemu_log("URMA: Failed to create jfs, errno: %d\n", errno); + goto err_del_jfc; + } + + ctx->max_jfs_depth = ctx->dev_attr.dev_cap.max_jfs_depth; + + if (qemu_get_random_u32(&ctx->jfr_token.token) < 0) { + qemu_log("get jfr random token failed, errno: %d\n", errno); + goto err_del_jfs; + } + + urma_jfr_cfg_t jfr_cfg = { + .depth = ctx->dev_attr.dev_cap.max_jfr_depth, + .max_sge = 1, + .flag.bs.tag_matching = URMA_NO_TAG_MATCHING, + .trans_mode = URMA_TM_RM, + .min_rnr_timer = URMA_TYPICAL_MIN_RNR_TIMER, + .jfc = ctx->jfc, + .token_value = ctx->jfr_token, + .id = 0 + }; + ctx->jfr = urma_create_jfr_p(ctx->urma_ctx, &jfr_cfg); + if (ctx->jfr == NULL) { + qemu_log("Failed to create jfr, errno: %d\n", errno); + goto err_del_jfs; + } + + qemu_log("init urma context success.\n"); + return 0; + +err_del_jfs: + urma_delete_jfs_p(ctx->jfs); +err_del_jfc: + urma_delete_jfc_p(ctx->jfc); +err_del_jfce: + urma_delete_jfce_p(ctx->jfce); +err_del_ctx: + (void)urma_delete_context_p(ctx->urma_ctx); + return -EINVAL; } +static void qemu_urma_cleanup_context(URMAContext *ctx) +{ + if (!ctx) { + return; + } + + if (ctx->tjfr) { + urma_unimport_jfr_p(ctx->tjfr); + ctx->tjfr = NULL; + } + + if (ctx->jfr) { + urma_delete_jfr_p(ctx->jfr); + ctx->jfr = NULL; + } + + if (ctx->jfs) { + urma_delete_jfs_p(ctx->jfs); + ctx->jfs = NULL; + } + + if (ctx->jfc) { + urma_delete_jfc_p(ctx->jfc); + ctx->jfc = NULL; + } + + if (ctx->jfce) { + urma_delete_jfce_p(ctx->jfce); + ctx->jfce = NULL; + } + + if (ctx->urma_ctx) { + (void)urma_delete_context_p(ctx->urma_ctx); + ctx->urma_ctx = NULL; + } + + qemu_log("clean up urma context success.\n"); +} + +static int urma_init_lib(void) +{ + int ret; + urma_init_attr_t init_attr = { + .uasid = 0, + }; + + ret = urma_init_p(&init_attr); + if (ret != URMA_SUCCESS) { + qemu_log("URMA: urma_init failed, ret: %d, errno: %d\n", ret, errno); + return ret; + } + + return 0; +} + +static void qemu_urma_unreg_ram_blocks(URMAContext *urma) +{ + int i; + URMALocalBlocks *local = &urma->local_ram_blocks; + + for (i = 0; i < local->nb_blocks; i++) { + URMALocalBlock *block = &local->block[i]; + + if (block->local_tseg) { + urma_unregister_seg_p(block->local_tseg); + block->local_tseg = NULL; + } + } + + ram_block_discard_disable(false); + + qemu_log("unreg all ram blocks success.\n"); +} + +static int qemu_urma_reg_whole_ram_blocks(URMAContext *urma) +{ + int i; + int64_t start_time; + URMALocalBlocks *local = &urma->local_ram_blocks; + MigrationState *s = migrate_get_current(); + urma_reg_seg_flag_t flag = { + .bs.token_policy = URMA_TOKEN_PLAIN_TEXT, + .bs.cacheable = URMA_NON_CACHEABLE, + .bs.reserved = 0 + }; + + if (!urma->is_incoming) { + flag.bs.access = URMA_ACCESS_LOCAL_ONLY; + } else { + flag.bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE | URMA_ACCESS_ATOMIC; + } + + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + + /* disable memory ballon before register seg */ + ram_block_discard_disable(true); + + for (i = 0; i < local->nb_blocks; i++) { + URMALocalBlock *block = &local->block[i]; + if (qemu_get_random_u32(&block->local_seg_token.token) < 0) { + qemu_log("get segment random token failed, errno: %d\n", errno); + goto err; + } + + urma_seg_cfg_t seg_cfg = { + .va = (uint64_t)block->local_host_addr, + .len = block->length, + .token_value = block->local_seg_token, + .flag = flag, + .user_ctx = (uintptr_t)NULL, + .iova = 0 + }; + + block->local_tseg = urma_register_seg_p(urma->urma_ctx, &seg_cfg); + if (block->local_tseg == NULL) { + qemu_log("URMA: Failed to register RAM block: %s, va: %p, size: %ld\n", block->block_name, block->local_host_addr, block->length); + goto err; + } + } + + qemu_log("reigster all ram blocks success.\n"); + s->ram_reg_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - start_time; + + return 0; + +err: + qemu_urma_unreg_ram_blocks(urma); + return -EINVAL; +} + +static void qemu_urma_cleanup(URMAContext *urma) +{ + if (urma == NULL) { + return; + } + + qemu_urma_unreg_ram_blocks(urma); + qemu_urma_free_blocks(urma); + qemu_urma_cleanup_context(urma); + + urma_uninit_p(); + qemu_log("clean up urma info success.\n"); +} + +static int qemu_urma_init_all(URMAContext *urma, bool pin_all) +{ + int ret; + + urma->pin_all = pin_all; + urma->nb_polling = 0; + + ret = urma_init_lib(); + if (ret) { + goto err; + } + + ret = qemu_urma_init_context(urma); + if (ret) { + goto err; + } + + ret = qemu_urma_init_ram_blocks(urma); + if (ret) { + goto err; + } + + if (urma->pin_all) { + ret = qemu_urma_reg_whole_ram_blocks(urma); + if (ret) { + goto err; + } + } + + qemu_log("prepare all urma info success.\n"); + return 0; +err: + qemu_log("Get error during prepare urma info, ret: %d, errno: %d\n", ret, errno); + qemu_urma_cleanup(urma); + return ret; +} + +static int get_ubbond_seg_info(urma_target_seg_t *tseg, urma_bond_seg_info_out_t *seg_info_out) +{ + urma_bond_seg_info_in_t seg_info_in = { + .tseg = tseg, + }; + urma_user_ctl_in_t user_ctl_in = { + .opcode = URMA_USER_CTL_BOND_GET_SEG_INFO, + .addr = (uint64_t)&seg_info_in, + .len = sizeof(urma_bond_seg_info_in_t), + }; + urma_user_ctl_out_t user_ctl_out = { + .addr = (uint64_t)seg_info_out, + .len = sizeof(urma_bond_seg_info_out_t), + }; + + if (urma_user_ctl_p(tseg->urma_ctx, &user_ctl_in, &user_ctl_out)) { + qemu_log("urma_user_ctl: get seg info failed, errno: %d\n", errno); + return -1; + } + + qemu_log("get ubbond seg info success.\n"); + return 0; +} + +static int add_ubbond_seg_info(urma_context_t *ctx, urma_bond_add_remote_seg_info_in_t *rseg_info_in) +{ + urma_user_ctl_in_t user_ctl_in = { + .opcode = URMA_USER_CTL_BOND_ADD_REMOTE_SEG_INFO, + .addr = (uint64_t)rseg_info_in, + .len = sizeof(urma_bond_add_remote_seg_info_in_t) + }; + urma_user_ctl_out_t user_ctl_out = {0}; + + if (urma_user_ctl_p(ctx, &user_ctl_in, &user_ctl_out)) { + qemu_log("urma_user_ctl: set seg info failed, errno: %d\n", errno); + return -1; + } + + qemu_log("add ubbond seg info success.\n"); + return 0; +} + +static int get_ubbond_jfr_info(urma_jfr_t *jfr, urma_bond_id_info_out_t *info_out) +{ + urma_bond_id_info_in_t in = { + .jfr = jfr, + .type = URMA_JFR, + }; + urma_user_ctl_in_t user_ctl_in = { + .opcode = URMA_USER_CTL_BOND_GET_ID_INFO, + .addr = (uint64_t)&in, + .len = sizeof(urma_bond_id_info_in_t), + }; + urma_user_ctl_out_t user_ctl_out = { + .addr = (uint64_t)info_out, + .len = sizeof(urma_bond_id_info_out_t), + }; + + if (urma_user_ctl_p(jfr->urma_ctx, &user_ctl_in, &user_ctl_out)) { + qemu_log("urma_user_ctl: get jfr info failed, errno: %d\n", errno); + return -1; + } + + qemu_log("get ubbond jfr info success.\n"); + return 0; +} + +static int add_ubbond_jfr_info(urma_context_t *ctx, urma_bond_id_info_out_t *info) +{ + urma_user_ctl_in_t user_ctl_in = { + .opcode = URMA_USER_CTL_BOND_ADD_RJFR_ID_INFO, + .addr = (uint64_t)info, + .len = sizeof(urma_bond_id_info_out_t), + }; + urma_user_ctl_out_t user_ctl_out = { + .addr = 0, + .len = 0, + }; + + if (urma_user_ctl_p(ctx, &user_ctl_in, &user_ctl_out)) { + qemu_log("urma_user_ctl: set jfr info failed, errno: %d\n", errno); + return -1; + } + + qemu_log("add ubbond jfr info success.\n"); + return 0; +} + +static void pack_seg_jfr_info(seg_jfr_info_t *info, URMAContext *ctx, URMALocalBlock *block) +{ + urma_bond_seg_info_out_t seg_bond_info; + urma_bond_id_info_out_t jfr_bond_info; + + (void)memset(info, 0, sizeof(seg_jfr_info_t)); + info->eid = ctx->urma_ctx->eid; + info->uasid = ctx->urma_ctx->uasid; + info->seg_va = block->local_tseg->seg.ubva.va; + info->seg_len = block->local_tseg->seg.len; + info->seg_flag = block->local_tseg->seg.attr.value; + info->seg_token_id = block->local_tseg->seg.token_id; + info->seg_token.token = block->local_seg_token.token; + info->jfr_id = ctx->jfr->jfr_id; + info->jfr_token.token = ctx->jfr_token.token; + + get_ubbond_seg_info(block->local_tseg, &seg_bond_info); + get_ubbond_jfr_info(ctx->jfr, &jfr_bond_info); + memcpy(&info->seg_bond_info, &seg_bond_info, sizeof(urma_bond_seg_info_out_t)); + memcpy(&info->jfr_bond_info, &jfr_bond_info, sizeof(urma_bond_id_info_out_t)); +} + +static void unpack_seg_jfr_info(seg_jfr_info_t *info, URMAContext *ctx, URMALocalBlock *block) +{ + urma_bond_seg_info_out_t seg_bond_info; + urma_bond_id_info_out_t jfr_bond_info; + + block->remote_seg.ubva.eid = info->eid; + block->remote_seg.ubva.uasid = info->uasid; + block->remote_seg.ubva.va = info->seg_va; + block->remote_seg.len = info->seg_len; + block->remote_seg.attr.value = info->seg_flag; + block->remote_seg.token_id = info->seg_token_id; + block->remote_seg_token.token = info->seg_token.token; + ctx->remote_jfr_id = info->jfr_id; + ctx->rjfr_token.token = info->jfr_token.token; + + memcpy(&seg_bond_info, &info->seg_bond_info, sizeof(urma_bond_seg_info_out_t)); + memcpy(&jfr_bond_info, &info->jfr_bond_info, sizeof(urma_bond_id_info_out_t)); + add_ubbond_seg_info(ctx->urma_ctx, &seg_bond_info); + add_ubbond_jfr_info(ctx->urma_ctx, &jfr_bond_info); +} + +static urma_target_jetty_t *qemu_import_jfr(URMAContext *ctx) +{ + urma_rjfr_t remote_jfr = { + .jfr_id = ctx->remote_jfr_id, + .trans_mode = URMA_TM_RM + }; + urma_target_jetty_t *tjfr = urma_import_jfr_p(ctx->urma_ctx, &remote_jfr, &ctx->rjfr_token); + if (tjfr == NULL) { + qemu_log("Failed to do urma_import_jfr, errno: %d\n", errno); + return NULL; + } + + if (urma_advise_jfr_p(ctx->jfs, tjfr) != URMA_SUCCESS) { + qemu_log("Failed to advise jfr, errno: %d\n", errno); + (void)urma_unimport_jfr_p(tjfr); + return NULL; + } + + return tjfr; +} + +static void qemu_urma_search_ram_block(URMAContext *urma, + uintptr_t block_offset, + uint64_t offset, + uint64_t length, + uint64_t *block_index, + uint64_t *chunk_index) +{ + uint64_t current_addr = block_offset + offset; + URMALocalBlock *block = g_hash_table_lookup(urma->blockmap, + (void *) block_offset); + assert(block); + assert(current_addr >= block->offset); + assert((current_addr + length) <= (block->offset + block->length)); + + *block_index = block->index; + *chunk_index = urma_ram_chunk_index(block->local_host_addr, + block->local_host_addr + (current_addr - block->offset)); +} + +static inline int qemu_urma_buffer_mergable(URMAContext *urma, + uint64_t offset, uint64_t len) +{ + URMALocalBlock *block; + uint8_t *host_addr; + uint8_t *chunk_end; + + if (urma->current_index < 0) { + return 0; + } + + if (urma->current_chunk < 0) { + return 0; + } + + block = &(urma->local_ram_blocks.block[urma->current_index]); + host_addr = block->local_host_addr + (offset - block->offset); + chunk_end = urma_ram_chunk_end(block, urma->current_chunk); + + if (urma->current_length == 0) { + return 0; + } + + /* + * Only merge into chunk sequentially. + */ + if (offset != (urma->current_addr + urma->current_length)) { + return 0; + } + + if (offset < block->offset) { + return 0; + } + + if ((offset + len) > (block->offset + block->length)) { + return 0; + } + + if ((host_addr + len) > chunk_end) { + return 0; + } + + return 1; +} + +static int poll_jfc_wait(URMAContext *ctx, urma_cr_t *cr) +{ + int i, j = 0, cnt = 0; + + for (i = 0; i < URMA_MAX_POLL_TIME; i++) { + if (ctx->nb_polling == 0) { + return 0; + } + + cnt = urma_poll_jfc_p(ctx->jfc, ctx->nb_polling, cr); + if (cnt < 0) { + goto err; + } else if (cnt > 0) { + for (j = 0; j < cnt; j++) { + if (cr[j].status != URMA_CR_SUCCESS) { + goto err; + } + } + ctx->nb_polling -= cnt; + } + + usleep(1); + } + +err: + qemu_log("urma_poll_jfc err: loop num: %d, cnt: %d, status: %d, nb_polling: %d, errno: %d\n", + i, cnt, cr[j].status, ctx->nb_polling, errno); + return -EINVAL; +} + +int qemu_flush_urma_write(URMAContext *urma) +{ + urma_cr_t *cr = NULL; + + if (!urma) { + qemu_log("enter qemu_flush_urma_write when the urma is uninitialized!\n"); + return -EINVAL; + } + + cr = g_new0(urma_cr_t, urma->max_jfs_depth); + if (cr == NULL) { + qemu_log("Failed to alloc urma cr\n"); + return -EINVAL; + } + + if (poll_jfc_wait(urma, cr) != 0) { + qemu_log("Failed to poll jfc, errno: %d\n", errno); + g_free(cr); + return -EINVAL; + } + + g_free(cr); + return 0; +} + +static int qemu_urma_write_one(URMAContext *urma, + int current_index, uint64_t current_addr, + uint64_t length) +{ + uintptr_t local_addr, remote_addr, offset; + URMALocalBlock *block = &(urma->local_ram_blocks.block[current_index]); + urma_jfs_wr_flag_t flag = { 0 }; + + if (block->is_ram_block) { + offset = current_addr - block->offset; + local_addr = (uintptr_t)(block->local_host_addr + offset); + remote_addr = (uintptr_t)(block->remote_seg.ubva.va + offset); + flag.bs.complete_enable = 1; + + if (urma_write_p(urma->jfs, urma->tjfr, block->import_tseg, block->local_tseg, + remote_addr, local_addr, length, + flag, (uintptr_t)urma->rid) != URMA_SUCCESS) { + qemu_log("Failed to do urma_write, local addr: %lx, remote addr: %lx, size: %lx, errno: %d\n", + local_addr, remote_addr, length, errno); + return -EINVAL; + } + + urma->nb_polling++; + if (urma->nb_polling >= urma->max_jfs_depth) { + if (qemu_flush_urma_write(urma) < 0) { + qemu_log("Failed to flush urma write, errno: %d\n", errno); + return -EINVAL; + } + } + } + + stat64_add(&mig_stats.normal_pages, length / qemu_target_page_size()); + stat64_add(&mig_stats.urma_bytes, length); + ram_transferred_add(length); + + return 0; +} + +static int qemu_urma_write_flush(URMAContext *urma) +{ + int ret; + + if (!urma->current_length) { + return 0; + } + + ret = qemu_urma_write_one(urma, urma->current_index, urma->current_addr, + urma->current_length); + if (ret < 0) { + return ret; + } + + urma->nb_sent++; + urma->current_length = 0; + urma->current_addr = 0; + + return 0; +} + +static int qemu_urma_write(URMAContext *urma, + uint64_t block_offset, uint64_t offset, + uint64_t len) +{ + uint64_t current_addr = block_offset + offset; + uint64_t index = urma->current_index; + uint64_t chunk = urma->current_chunk; + int ret; + + /* If we cannot merge it, we flush the current buffer first. */ + if (!qemu_urma_buffer_mergable(urma, current_addr, len)) { + ret = qemu_urma_write_flush(urma); + if (ret) { + return ret; + } + urma->current_length = 0; + urma->current_addr = current_addr; + qemu_urma_search_ram_block(urma, block_offset, + offset, len, &index, &chunk); + urma->current_index = index; + urma->current_chunk = chunk; + } + + /* merge it */ + urma->current_length += len; + + /* flush it if buffer is too large */ + if (urma->current_length >= URMA_CHUNK_MERGE_MAX) { + return qemu_urma_write_flush(urma); + } + + return 0; +} + +static int qemu_urma_save_page(QEMUFile *f, ram_addr_t block_offset, + ram_addr_t offset, size_t size) +{ + MigrationState *s = migrate_get_current(); + URMAContext *urma = s->urma_ctx; + int ret; + + if (!urma) { + return -EINVAL; + } + + if (size > 0) { + /* + * Add this page to the current 'chunk'. If the chunk + * is full, or the page doesn't belong to the current chunk, + * an actual urma write will occur and a new chunk will be formed. + */ + ret = qemu_urma_write(urma, block_offset, offset, size); + if (ret < 0) { + qemu_log("urma write failed, block offset: %lx, offset: %lx, size: %lx, ret: %d, errno: %d\n", + block_offset, offset, size, ret, errno); + return ret; + } + } + + return RAM_SAVE_CONTROL_DELAYED; +} + +static void qemu_urma_unimport(URMAContext *urma) +{ + int i; + URMALocalBlocks *local_block = &urma->local_ram_blocks; + + for (i = 0; i < local_block->nb_blocks; i++) { + URMALocalBlock *block = &local_block->block[i]; + if (block->import_tseg) { + urma_unimport_seg_p(block->import_tseg); + block->import_tseg = NULL; + } + } + + if (urma->tjfr) { + urma_unimport_jfr_p(urma->tjfr); + urma->tjfr = NULL; + } + + qemu_log("unimport all blocks and jfr success.\n"); +} + int qemu_urma_import(URMAContext *urma) { - /* TODO */ + int i; + URMALocalBlocks *local_block = &urma->local_ram_blocks; + urma_import_seg_flag_t flag = { + .bs.cacheable = URMA_NON_CACHEABLE, + .bs.access = URMA_ACCESS_READ | URMA_ACCESS_WRITE | URMA_ACCESS_ATOMIC, + .bs.mapping = URMA_SEG_NOMAP, + .bs.reserved = 0 + }; + + for (i = 0; i < local_block->nb_blocks; i++) { + URMALocalBlock *block = &local_block->block[i]; + + block->import_tseg = urma_import_seg_p(urma->urma_ctx, &block->remote_seg, &block->remote_seg_token, 0, flag); + if (block->import_tseg == NULL) { + qemu_log("Failed to import segment, block name: %s, va: %p, size: %ld, errono: %d\n", + block->block_name, block->local_host_addr, block->length, errno); + goto err; + } + } + + urma->tjfr = qemu_import_jfr(urma); + if (urma->tjfr == NULL) { + qemu_log("Failed to import jfr, errno: %d\n", errno); + goto err; + } + + qemu_log("import all blocks and jfr success.\n"); + return 0; + +err: + qemu_urma_unimport(urma); return -EINVAL; } int qemu_exchange_urma_info(QEMUFile *f, URMAContext *urma, bool server) { - /* TODO */ - return -EINVAL; + int i; + URMALocalBlocks *local_block = &urma->local_ram_blocks; + seg_jfr_info_t local = {0}, remote = {0}; + MigrationState *s = migrate_get_current(); + int64_t start_time; + + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + + qemu_log("start to exchange urma segment info.\n"); + + for (i = 0; i < local_block->nb_blocks; i++) { + URMALocalBlock *block = &local_block->block[i]; + + if (server) { + pack_seg_jfr_info(&local, urma, block); + qemu_put_buffer(f, (uint8_t *)&local, sizeof(seg_jfr_info_t)); + if (qemu_fflush(f) < 0) { + qemu_log("Failed to flush qemu file, errno: %d\n", errno); + return -EINVAL; + } + } else { + if (qemu_get_buffer(f, (uint8_t *)&remote, sizeof(seg_jfr_info_t)) != sizeof(seg_jfr_info_t)) { + qemu_log("get urma info failed, block name: %s, errno: %d\n", block->block_name, errno); + return -EINVAL; + } + unpack_seg_jfr_info(&remote, urma, block); + } + } + + s->urma_exchange_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - start_time; + return 0; } void urma_start_outgoing_migration(void *opaque, SocketAddress *saddr, Error **errp) { - /* TODO */ + MigrationState *s = opaque; + URMAContext *urma = NULL; + int ret; + int64_t start_time; + + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + + ret = urma_dlfunc_init(errp); + if (ret < 0) { + goto err; + } + + urma = qemu_urma_data_init(&saddr->u.inet); + if (urma == NULL) { + qemu_log("migration: qemu_urma_data_init failed\n"); + goto err; + } + + ret = qemu_urma_init_all(urma, true); + if (ret) { + qemu_log("migration: qemu_urma_init_all failed, ret: %d\n", ret); + goto err; + } + + s->urma_init_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - start_time; + s->urma_migration = true; + s->urma_ctx = urma; + socket_start_outgoing_migration(s, saddr, errp); + + qemu_log("migration: start urma_start_outgoing_migration\n"); return; + +err: + error_setg(errp, "migration: urma start outgoing migration failed"); + qemu_urma_data_free(urma); } void urma_start_incoming_migration(SocketAddress *saddr, Error **errp) { - /* TODO */ + MigrationState *s = migrate_get_current(); + URMAContext *urma; + int ret; + + urma = qemu_urma_data_init(&saddr->u.inet); + if (urma == NULL) { + qemu_log("migration: qemu_urma_data_init failed\n"); + goto err; + } + + ret = urma_dlfunc_init(errp); + if (ret < 0) { + goto err; + } + + urma->is_incoming = true; + + ret = qemu_urma_init_all(urma, true); + if (ret) { + qemu_log("migration: qemu_urma_init_all failed, ret: %d\n", ret); + goto err; + } + + s->urma_migration = true; + s->urma_ctx = urma; + socket_start_incoming_migration(saddr, errp); + + qemu_log("migration: start urma_start_incoming_migration\n"); return; + +err: + error_setg(errp, "migration: urma start incoming migration failed"); + qemu_urma_data_free(urma); } void urma_migration_cleanup(void) { - /* TODO */ - return; + MigrationState *s = migrate_get_current(); + + if (s->urma_ctx == NULL) { + return; + } + + qemu_urma_unimport(s->urma_ctx); + qemu_urma_cleanup(s->urma_ctx); + qemu_urma_data_free(s->urma_ctx); + s->urma_ctx = NULL; + + qemu_log("urma migration cleanup success.\n"); } int urma_control_save_page(QEMUFile *f, ram_addr_t block_offset, ram_addr_t offset, size_t size) { - /* TODO */ - return RAM_SAVE_CONTROL_NOT_SUPP; + int ret; + + ret = qemu_urma_save_page(f, block_offset, offset, size); + + if (ret != RAM_SAVE_CONTROL_DELAYED && + ret != RAM_SAVE_CONTROL_NOT_SUPP) { + if (ret < 0) { + qemu_file_set_error(f, ret); + } + } + return ret; } void record_migration_log(MigrationState *s) -- Gitee From 7ad8437796591aaa074be1eabbe3aa31f34063c4 Mon Sep 17 00:00:00 2001 From: GQX <2290721782@qq.com> Date: Mon, 17 Nov 2025 14:41:49 +0800 Subject: [PATCH 3/3] Migration: support onecopy migration If onecopy is enabled, migration will skip the iteration phase and enter to the completion phase:suspend the source vm and copy all vm rams to the destination host. This can improve the UB bandwidth utilization of urma migration. So we can enable the onecopy feature when the UB bandwidth is very large. For example, when the UB bandwidth is 200GBps and the expected migration downtime is less than 500ms, we can enable the onecopy feature when the VM size is less than 80GB (some time for device save/load). --- migration/options.c | 9 +++++++++ migration/options.h | 1 + qapi/migration.json | 7 ++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/migration/options.c b/migration/options.c index c2695aee65..a1c29293d3 100644 --- a/migration/options.c +++ b/migration/options.c @@ -220,6 +220,7 @@ Property migration_properties[] = { DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD), DEFINE_PROP_MIG_CAP("x-background-snapshot", MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT), + DEFINE_PROP_MIG_CAP("x-onecopy", MIGRATION_CAPABILITY_ONECOPY), #ifdef CONFIG_LINUX DEFINE_PROP_MIG_CAP("x-zero-copy-send", MIGRATION_CAPABILITY_ZERO_COPY_SEND), @@ -345,6 +346,13 @@ bool migrate_urma(void) } #endif +bool migrate_onecopy_ram(void) +{ + MigrationState *s = migrate_get_current(); + + return s->capabilities[MIGRATION_CAPABILITY_ONECOPY]; +} + bool migrate_use_ldst(void) { MigrationState *s = migrate_get_current(); @@ -498,6 +506,7 @@ INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot, MIGRATION_CAPABILITY_X_COLO, MIGRATION_CAPABILITY_VALIDATE_UUID, MIGRATION_CAPABILITY_ZERO_COPY_SEND, + MIGRATION_CAPABILITY_ONECOPY, MIGRATION_CAPABILITY_LDST); static bool migrate_incoming_started(void) diff --git a/migration/options.h b/migration/options.h index 78f4af0ac8..59bfb7e854 100644 --- a/migration/options.h +++ b/migration/options.h @@ -41,6 +41,7 @@ bool migrate_postcopy_ram(void); #ifdef CONFIG_URMA_MIGRATION bool migrate_urma(void); #endif +bool migrate_onecopy_ram(void); bool migrate_rdma_pin_all(void); bool migrate_release_ram(void); bool migrate_return_path(void); diff --git a/qapi/migration.json b/qapi/migration.json index cfa2e2c4e3..9b0d42b6a5 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -535,6 +535,10 @@ # and can result in more stable read performance. Requires KVM # with accelerator property "dirty-ring-size" set. (Since 8.1) # +# @onecopy: If enabled, live migration will skip the iteration phase +# and enter to the completion phase:suspend the source VM and +# synchronize all VM states to the destination host. (Since 8.2) +# # Features: # # @deprecated: Member @block is deprecated. Use blockdev-mirror with @@ -550,7 +554,7 @@ { 'enum': 'MigrationCapability', 'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks', { 'name': 'compress', 'features': [ 'deprecated' ] }, - 'events', 'postcopy-ram', 'ldst', + 'events', 'postcopy-ram', 'onecopy', 'ldst', { 'name': 'x-colo', 'features': [ 'unstable' ] }, 'release-ram', { 'name': 'block', 'features': [ 'deprecated' ] }, @@ -614,6 +618,7 @@ # {"state": true, "capability": "events"}, # {"state": false, "capability": "postcopy-ram"}, # {"state": false, "capability": "x-colo"}, +# {"state": false, "capability": "onecopy"}, # {"state": false, "capability": "ldst"} # ]} ## -- Gitee