| Message ID | 1603365127-14202-18-git-send-email-kwankhede@nvidia.com |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Add migration support for VFIO devices | expand |
On Thu, 22 Oct 2020 16:42:07 +0530 Kirti Wankhede <kwankhede@nvidia.com> wrote: > Added amount of bytes transferred to the VM at destination by all VFIO > devices > > Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com> > Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com> > --- > hw/vfio/common.c | 20 ++++++++++++++++++++ > hw/vfio/migration.c | 10 ++++++++++ > include/qemu/vfio-helpers.h | 3 +++ > migration/migration.c | 14 ++++++++++++++ > monitor/hmp-cmds.c | 6 ++++++ > qapi/migration.json | 17 +++++++++++++++++ > 6 files changed, 70 insertions(+) > > diff --git a/hw/vfio/common.c b/hw/vfio/common.c > index 9c879e5c0f62..8d0758eda9fa 100644 > --- a/hw/vfio/common.c > +++ b/hw/vfio/common.c > @@ -39,6 +39,7 @@ > #include "trace.h" > #include "qapi/error.h" > #include "migration/migration.h" > +#include "qemu/vfio-helpers.h" > > VFIOGroupList vfio_group_list = > QLIST_HEAD_INITIALIZER(vfio_group_list); > @@ -292,6 +293,25 @@ const MemoryRegionOps vfio_region_ops = { > * Device state interfaces > */ > > +bool vfio_mig_active(void) > +{ > + VFIOGroup *group; > + VFIODevice *vbasedev; > + > + if (QLIST_EMPTY(&vfio_group_list)) { > + return false; > + } > + > + QLIST_FOREACH(group, &vfio_group_list, next) { > + QLIST_FOREACH(vbasedev, &group->device_list, next) { > + if (vbasedev->migration_blocker) { > + return false; > + } > + } > + } > + return true; > +} > + > static bool vfio_devices_all_stopped_and_saving(VFIOContainer *container) > { > VFIOGroup *group; > diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c > index 77ee60a43ea5..b23e21c6de2b 100644 > --- a/hw/vfio/migration.c > +++ b/hw/vfio/migration.c > @@ -28,6 +28,7 @@ > #include "pci.h" > #include "trace.h" > #include "hw/hw.h" > +#include "qemu/vfio-helpers.h" > > /* > * Flags to be used as unique delimiters for VFIO devices in the migration > @@ -45,6 +46,8 @@ > #define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL) > #define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL) > > +static int64_t bytes_transferred; > + > static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count, > off_t off, bool iswrite) > { > @@ -255,6 +258,7 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size) > *size = data_size; > } > > + bytes_transferred += data_size; > return ret; > } > > @@ -776,6 +780,7 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data) > case MIGRATION_STATUS_CANCELLING: > case MIGRATION_STATUS_CANCELLED: > case MIGRATION_STATUS_FAILED: > + bytes_transferred = 0; > ret = vfio_migration_set_state(vbasedev, > ~(VFIO_DEVICE_STATE_SAVING | VFIO_DEVICE_STATE_RESUMING), > VFIO_DEVICE_STATE_RUNNING); > @@ -862,6 +867,11 @@ err: > > /* ---------------------------------------------------------------------- */ > > +int64_t vfio_mig_bytes_transferred(void) > +{ > + return bytes_transferred; > +} > + > int vfio_migration_probe(VFIODevice *vbasedev, Error **errp) > { > VFIOContainer *container = vbasedev->group->container; > diff --git a/include/qemu/vfio-helpers.h b/include/qemu/vfio-helpers.h > index 4491c8e1a6e9..7f7a46e6ef2d 100644 > --- a/include/qemu/vfio-helpers.h > +++ b/include/qemu/vfio-helpers.h > @@ -29,4 +29,7 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar, > int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, > int irq_type, Error **errp); > > +bool vfio_mig_active(void); > +int64_t vfio_mig_bytes_transferred(void); > + > #endif I don't think vfio-helpers is the right place for this, this header is specifically for using util/vfio-helpers.c. Would include/hw/vfio/vfio-common.h work? > diff --git a/migration/migration.c b/migration/migration.c > index 0575ecb37953..8b2865d25ef4 100644 > --- a/migration/migration.c > +++ b/migration/migration.c > @@ -56,6 +56,7 @@ > #include "net/announce.h" > #include "qemu/queue.h" > #include "multifd.h" > +#include "qemu/vfio-helpers.h" > > #define MAX_THROTTLE (128 << 20) /* Migration transfer speed throttling */ > > @@ -1002,6 +1003,17 @@ static void populate_disk_info(MigrationInfo *info) > } > } > > +static void populate_vfio_info(MigrationInfo *info) > +{ > +#ifdef CONFIG_LINUX Use CONFIG_VFIO? I get a build failure on qemu-system-avr /usr/bin/ld: /tmp/tmp.3QbqxgbENl/build/../migration/migration.c:1012: undefined reference to `vfio_mig_bytes_transferred'. Thanks, Alex > + if (vfio_mig_active()) { > + info->has_vfio = true; > + info->vfio = g_malloc0(sizeof(*info->vfio)); > + info->vfio->transferred = vfio_mig_bytes_transferred(); > + } > +#endif > +} > + > static void fill_source_migration_info(MigrationInfo *info) > { > MigrationState *s = migrate_get_current(); > @@ -1026,6 +1038,7 @@ static void fill_source_migration_info(MigrationInfo *info) > populate_time_info(info, s); > populate_ram_info(info, s); > populate_disk_info(info); > + populate_vfio_info(info); > break; > case MIGRATION_STATUS_COLO: > info->has_status = true; > @@ -1034,6 +1047,7 @@ static void fill_source_migration_info(MigrationInfo *info) > case MIGRATION_STATUS_COMPLETED: > populate_time_info(info, s); > populate_ram_info(info, s); > + populate_vfio_info(info); > break; > case MIGRATION_STATUS_FAILED: > info->has_status = true; > diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c > index 9789f4277f50..56e9bad33d94 100644 > --- a/monitor/hmp-cmds.c > +++ b/monitor/hmp-cmds.c > @@ -357,6 +357,12 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict) > } > monitor_printf(mon, "]\n"); > } > + > + if (info->has_vfio) { > + monitor_printf(mon, "vfio device transferred: %" PRIu64 " kbytes\n", > + info->vfio->transferred >> 10); > + } > + > qapi_free_MigrationInfo(info); > } > > diff --git a/qapi/migration.json b/qapi/migration.json > index a5da513c9e05..3c7582052725 100644 > --- a/qapi/migration.json > +++ b/qapi/migration.json > @@ -147,6 +147,18 @@ > 'active', 'postcopy-active', 'postcopy-paused', > 'postcopy-recover', 'completed', 'failed', 'colo', > 'pre-switchover', 'device', 'wait-unplug' ] } > +## > +# @VfioStats: > +# > +# Detailed VFIO devices migration statistics > +# > +# @transferred: amount of bytes transferred to the target VM by VFIO devices > +# > +# Since: 5.2 > +# > +## > +{ 'struct': 'VfioStats', > + 'data': {'transferred': 'int' } } > > ## > # @MigrationInfo: > @@ -208,11 +220,16 @@ > # > # @socket-address: Only used for tcp, to know what the real port is (Since 4.0) > # > +# @vfio: @VfioStats containing detailed VFIO devices migration statistics, > +# only returned if VFIO device is present, migration is supported by all > +# VFIO devices and status is 'active' or 'completed' (since 5.2) > +# > # Since: 0.14.0 > ## > { 'struct': 'MigrationInfo', > 'data': {'*status': 'MigrationStatus', '*ram': 'MigrationStats', > '*disk': 'MigrationStats', > + '*vfio': 'VfioStats', > '*xbzrle-cache': 'XBZRLECacheStats', > '*total-time': 'int', > '*expected-downtime': 'int',
On 10/23/2020 3:48 AM, Alex Williamson wrote: > On Thu, 22 Oct 2020 16:42:07 +0530 > Kirti Wankhede <kwankhede@nvidia.com> wrote: > >> Added amount of bytes transferred to the VM at destination by all VFIO >> devices >> >> Signed-off-by: Kirti Wankhede <kwankhede@nvidia.com> >> Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com> >> --- >> hw/vfio/common.c | 20 ++++++++++++++++++++ >> hw/vfio/migration.c | 10 ++++++++++ >> include/qemu/vfio-helpers.h | 3 +++ >> migration/migration.c | 14 ++++++++++++++ >> monitor/hmp-cmds.c | 6 ++++++ >> qapi/migration.json | 17 +++++++++++++++++ >> 6 files changed, 70 insertions(+) >> >> diff --git a/hw/vfio/common.c b/hw/vfio/common.c >> index 9c879e5c0f62..8d0758eda9fa 100644 >> --- a/hw/vfio/common.c >> +++ b/hw/vfio/common.c >> @@ -39,6 +39,7 @@ >> #include "trace.h" >> #include "qapi/error.h" >> #include "migration/migration.h" >> +#include "qemu/vfio-helpers.h" >> >> VFIOGroupList vfio_group_list = >> QLIST_HEAD_INITIALIZER(vfio_group_list); >> @@ -292,6 +293,25 @@ const MemoryRegionOps vfio_region_ops = { >> * Device state interfaces >> */ >> >> +bool vfio_mig_active(void) >> +{ >> + VFIOGroup *group; >> + VFIODevice *vbasedev; >> + >> + if (QLIST_EMPTY(&vfio_group_list)) { >> + return false; >> + } >> + >> + QLIST_FOREACH(group, &vfio_group_list, next) { >> + QLIST_FOREACH(vbasedev, &group->device_list, next) { >> + if (vbasedev->migration_blocker) { >> + return false; >> + } >> + } >> + } >> + return true; >> +} >> + >> static bool vfio_devices_all_stopped_and_saving(VFIOContainer *container) >> { >> VFIOGroup *group; >> diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c >> index 77ee60a43ea5..b23e21c6de2b 100644 >> --- a/hw/vfio/migration.c >> +++ b/hw/vfio/migration.c >> @@ -28,6 +28,7 @@ >> #include "pci.h" >> #include "trace.h" >> #include "hw/hw.h" >> +#include "qemu/vfio-helpers.h" >> >> /* >> * Flags to be used as unique delimiters for VFIO devices in the migration >> @@ -45,6 +46,8 @@ >> #define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL) >> #define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL) >> >> +static int64_t bytes_transferred; >> + >> static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count, >> off_t off, bool iswrite) >> { >> @@ -255,6 +258,7 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size) >> *size = data_size; >> } >> >> + bytes_transferred += data_size; >> return ret; >> } >> >> @@ -776,6 +780,7 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data) >> case MIGRATION_STATUS_CANCELLING: >> case MIGRATION_STATUS_CANCELLED: >> case MIGRATION_STATUS_FAILED: >> + bytes_transferred = 0; >> ret = vfio_migration_set_state(vbasedev, >> ~(VFIO_DEVICE_STATE_SAVING | VFIO_DEVICE_STATE_RESUMING), >> VFIO_DEVICE_STATE_RUNNING); >> @@ -862,6 +867,11 @@ err: >> >> /* ---------------------------------------------------------------------- */ >> >> +int64_t vfio_mig_bytes_transferred(void) >> +{ >> + return bytes_transferred; >> +} >> + >> int vfio_migration_probe(VFIODevice *vbasedev, Error **errp) >> { >> VFIOContainer *container = vbasedev->group->container; >> diff --git a/include/qemu/vfio-helpers.h b/include/qemu/vfio-helpers.h >> index 4491c8e1a6e9..7f7a46e6ef2d 100644 >> --- a/include/qemu/vfio-helpers.h >> +++ b/include/qemu/vfio-helpers.h >> @@ -29,4 +29,7 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar, >> int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, >> int irq_type, Error **errp); >> >> +bool vfio_mig_active(void); >> +int64_t vfio_mig_bytes_transferred(void); >> + >> #endif > > > I don't think vfio-helpers is the right place for this, this header is > specifically for using util/vfio-helpers.c. Would > include/hw/vfio/vfio-common.h work? > > Yes, works with CONFIG_VFIO check. Changing it. >> diff --git a/migration/migration.c b/migration/migration.c >> index 0575ecb37953..8b2865d25ef4 100644 >> --- a/migration/migration.c >> +++ b/migration/migration.c >> @@ -56,6 +56,7 @@ >> #include "net/announce.h" >> #include "qemu/queue.h" >> #include "multifd.h" >> +#include "qemu/vfio-helpers.h" >> >> #define MAX_THROTTLE (128 << 20) /* Migration transfer speed throttling */ >> >> @@ -1002,6 +1003,17 @@ static void populate_disk_info(MigrationInfo *info) >> } >> } >> >> +static void populate_vfio_info(MigrationInfo *info) >> +{ >> +#ifdef CONFIG_LINUX > > Use CONFIG_VFIO? I get a build failure on qemu-system-avr > > /usr/bin/ld: /tmp/tmp.3QbqxgbENl/build/../migration/migration.c:1012: > undefined reference to `vfio_mig_bytes_transferred'. Thanks, > Ok Changing it. > Alex > >> + if (vfio_mig_active()) { >> + info->has_vfio = true; >> + info->vfio = g_malloc0(sizeof(*info->vfio)); >> + info->vfio->transferred = vfio_mig_bytes_transferred(); >> + } >> +#endif >> +} >> + >> static void fill_source_migration_info(MigrationInfo *info) >> { >> MigrationState *s = migrate_get_current(); >> @@ -1026,6 +1038,7 @@ static void fill_source_migration_info(MigrationInfo *info) >> populate_time_info(info, s); >> populate_ram_info(info, s); >> populate_disk_info(info); >> + populate_vfio_info(info); >> break; >> case MIGRATION_STATUS_COLO: >> info->has_status = true; >> @@ -1034,6 +1047,7 @@ static void fill_source_migration_info(MigrationInfo *info) >> case MIGRATION_STATUS_COMPLETED: >> populate_time_info(info, s); >> populate_ram_info(info, s); >> + populate_vfio_info(info); >> break; >> case MIGRATION_STATUS_FAILED: >> info->has_status = true; >> diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c >> index 9789f4277f50..56e9bad33d94 100644 >> --- a/monitor/hmp-cmds.c >> +++ b/monitor/hmp-cmds.c >> @@ -357,6 +357,12 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict) >> } >> monitor_printf(mon, "]\n"); >> } >> + >> + if (info->has_vfio) { >> + monitor_printf(mon, "vfio device transferred: %" PRIu64 " kbytes\n", >> + info->vfio->transferred >> 10); >> + } >> + >> qapi_free_MigrationInfo(info); >> } >> >> diff --git a/qapi/migration.json b/qapi/migration.json >> index a5da513c9e05..3c7582052725 100644 >> --- a/qapi/migration.json >> +++ b/qapi/migration.json >> @@ -147,6 +147,18 @@ >> 'active', 'postcopy-active', 'postcopy-paused', >> 'postcopy-recover', 'completed', 'failed', 'colo', >> 'pre-switchover', 'device', 'wait-unplug' ] } >> +## >> +# @VfioStats: >> +# >> +# Detailed VFIO devices migration statistics >> +# >> +# @transferred: amount of bytes transferred to the target VM by VFIO devices >> +# >> +# Since: 5.2 >> +# >> +## >> +{ 'struct': 'VfioStats', >> + 'data': {'transferred': 'int' } } >> >> ## >> # @MigrationInfo: >> @@ -208,11 +220,16 @@ >> # >> # @socket-address: Only used for tcp, to know what the real port is (Since 4.0) >> # >> +# @vfio: @VfioStats containing detailed VFIO devices migration statistics, >> +# only returned if VFIO device is present, migration is supported by all >> +# VFIO devices and status is 'active' or 'completed' (since 5.2) >> +# >> # Since: 0.14.0 >> ## >> { 'struct': 'MigrationInfo', >> 'data': {'*status': 'MigrationStatus', '*ram': 'MigrationStats', >> '*disk': 'MigrationStats', >> + '*vfio': 'VfioStats', >> '*xbzrle-cache': 'XBZRLECacheStats', >> '*total-time': 'int', >> '*expected-downtime': 'int', >
diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 9c879e5c0f62..8d0758eda9fa 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -39,6 +39,7 @@ #include "trace.h" #include "qapi/error.h" #include "migration/migration.h" +#include "qemu/vfio-helpers.h" VFIOGroupList vfio_group_list = QLIST_HEAD_INITIALIZER(vfio_group_list); @@ -292,6 +293,25 @@ const MemoryRegionOps vfio_region_ops = { * Device state interfaces */ +bool vfio_mig_active(void) +{ + VFIOGroup *group; + VFIODevice *vbasedev; + + if (QLIST_EMPTY(&vfio_group_list)) { + return false; + } + + QLIST_FOREACH(group, &vfio_group_list, next) { + QLIST_FOREACH(vbasedev, &group->device_list, next) { + if (vbasedev->migration_blocker) { + return false; + } + } + } + return true; +} + static bool vfio_devices_all_stopped_and_saving(VFIOContainer *container) { VFIOGroup *group; diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c index 77ee60a43ea5..b23e21c6de2b 100644 --- a/hw/vfio/migration.c +++ b/hw/vfio/migration.c @@ -28,6 +28,7 @@ #include "pci.h" #include "trace.h" #include "hw/hw.h" +#include "qemu/vfio-helpers.h" /* * Flags to be used as unique delimiters for VFIO devices in the migration @@ -45,6 +46,8 @@ #define VFIO_MIG_FLAG_DEV_SETUP_STATE (0xffffffffef100003ULL) #define VFIO_MIG_FLAG_DEV_DATA_STATE (0xffffffffef100004ULL) +static int64_t bytes_transferred; + static inline int vfio_mig_access(VFIODevice *vbasedev, void *val, int count, off_t off, bool iswrite) { @@ -255,6 +258,7 @@ static int vfio_save_buffer(QEMUFile *f, VFIODevice *vbasedev, uint64_t *size) *size = data_size; } + bytes_transferred += data_size; return ret; } @@ -776,6 +780,7 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data) case MIGRATION_STATUS_CANCELLING: case MIGRATION_STATUS_CANCELLED: case MIGRATION_STATUS_FAILED: + bytes_transferred = 0; ret = vfio_migration_set_state(vbasedev, ~(VFIO_DEVICE_STATE_SAVING | VFIO_DEVICE_STATE_RESUMING), VFIO_DEVICE_STATE_RUNNING); @@ -862,6 +867,11 @@ err: /* ---------------------------------------------------------------------- */ +int64_t vfio_mig_bytes_transferred(void) +{ + return bytes_transferred; +} + int vfio_migration_probe(VFIODevice *vbasedev, Error **errp) { VFIOContainer *container = vbasedev->group->container; diff --git a/include/qemu/vfio-helpers.h b/include/qemu/vfio-helpers.h index 4491c8e1a6e9..7f7a46e6ef2d 100644 --- a/include/qemu/vfio-helpers.h +++ b/include/qemu/vfio-helpers.h @@ -29,4 +29,7 @@ void qemu_vfio_pci_unmap_bar(QEMUVFIOState *s, int index, void *bar, int qemu_vfio_pci_init_irq(QEMUVFIOState *s, EventNotifier *e, int irq_type, Error **errp); +bool vfio_mig_active(void); +int64_t vfio_mig_bytes_transferred(void); + #endif diff --git a/migration/migration.c b/migration/migration.c index 0575ecb37953..8b2865d25ef4 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -56,6 +56,7 @@ #include "net/announce.h" #include "qemu/queue.h" #include "multifd.h" +#include "qemu/vfio-helpers.h" #define MAX_THROTTLE (128 << 20) /* Migration transfer speed throttling */ @@ -1002,6 +1003,17 @@ static void populate_disk_info(MigrationInfo *info) } } +static void populate_vfio_info(MigrationInfo *info) +{ +#ifdef CONFIG_LINUX + if (vfio_mig_active()) { + info->has_vfio = true; + info->vfio = g_malloc0(sizeof(*info->vfio)); + info->vfio->transferred = vfio_mig_bytes_transferred(); + } +#endif +} + static void fill_source_migration_info(MigrationInfo *info) { MigrationState *s = migrate_get_current(); @@ -1026,6 +1038,7 @@ static void fill_source_migration_info(MigrationInfo *info) populate_time_info(info, s); populate_ram_info(info, s); populate_disk_info(info); + populate_vfio_info(info); break; case MIGRATION_STATUS_COLO: info->has_status = true; @@ -1034,6 +1047,7 @@ static void fill_source_migration_info(MigrationInfo *info) case MIGRATION_STATUS_COMPLETED: populate_time_info(info, s); populate_ram_info(info, s); + populate_vfio_info(info); break; case MIGRATION_STATUS_FAILED: info->has_status = true; diff --git a/monitor/hmp-cmds.c b/monitor/hmp-cmds.c index 9789f4277f50..56e9bad33d94 100644 --- a/monitor/hmp-cmds.c +++ b/monitor/hmp-cmds.c @@ -357,6 +357,12 @@ void hmp_info_migrate(Monitor *mon, const QDict *qdict) } monitor_printf(mon, "]\n"); } + + if (info->has_vfio) { + monitor_printf(mon, "vfio device transferred: %" PRIu64 " kbytes\n", + info->vfio->transferred >> 10); + } + qapi_free_MigrationInfo(info); } diff --git a/qapi/migration.json b/qapi/migration.json index a5da513c9e05..3c7582052725 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -147,6 +147,18 @@ 'active', 'postcopy-active', 'postcopy-paused', 'postcopy-recover', 'completed', 'failed', 'colo', 'pre-switchover', 'device', 'wait-unplug' ] } +## +# @VfioStats: +# +# Detailed VFIO devices migration statistics +# +# @transferred: amount of bytes transferred to the target VM by VFIO devices +# +# Since: 5.2 +# +## +{ 'struct': 'VfioStats', + 'data': {'transferred': 'int' } } ## # @MigrationInfo: @@ -208,11 +220,16 @@ # # @socket-address: Only used for tcp, to know what the real port is (Since 4.0) # +# @vfio: @VfioStats containing detailed VFIO devices migration statistics, +# only returned if VFIO device is present, migration is supported by all +# VFIO devices and status is 'active' or 'completed' (since 5.2) +# # Since: 0.14.0 ## { 'struct': 'MigrationInfo', 'data': {'*status': 'MigrationStatus', '*ram': 'MigrationStats', '*disk': 'MigrationStats', + '*vfio': 'VfioStats', '*xbzrle-cache': 'XBZRLECacheStats', '*total-time': 'int', '*expected-downtime': 'int',