diff --git a/.mailmap b/.mailmap index 15bec72470..7f817d9f42 100644 --- a/.mailmap +++ b/.mailmap @@ -74,6 +74,7 @@ Aleksandar Markovic Aleksandar Rikalo Aleksandar Rikalo +Alex Williamson Alexander Graf Ani Sinha Anthony Liguori Anthony Liguori diff --git a/MAINTAINERS b/MAINTAINERS index d4e7fd965b..36eef27b41 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2286,7 +2286,7 @@ S: Maintained F: hw/usb/dev-serial.c VFIO -M: Alex Williamson +M: Alex Williamson M: Cédric Le Goater S: Supported F: hw/vfio/* @@ -2298,7 +2298,7 @@ F: migration/vfio-stub.c F: tests/functional/aarch64/test_device_passthrough.py vfio-igd -M: Alex Williamson +M: Alex Williamson M: Cédric Le Goater M: Tomita Moeko S: Supported diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 56031925c4..f9254ae654 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -525,7 +525,8 @@ static int do_kvm_destroy_vcpu(CPUState *cpu) } /* If I am the CPU that created coalesced_mmio_ring, then discard it */ - if (s->coalesced_mmio_ring == (void *)cpu->kvm_run + PAGE_SIZE) { + if (s->coalesced_mmio_ring == + (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE) { s->coalesced_mmio_ring = NULL; } diff --git a/docs/system/devices/vfio-user.rst b/docs/system/devices/vfio-user.rst index 30c2215f4e..e10a6d0822 100644 --- a/docs/system/devices/vfio-user.rst +++ b/docs/system/devices/vfio-user.rst @@ -20,7 +20,7 @@ Presuming a suitable ``vfio-user`` server has opened a socket at .. code-block:: console --device '{"driver": "vfio-user-pci","socket": {"path": "/tmp/vfio-user.sock", "type": "unix"}}' + --device '{"driver": "vfio-user-pci","socket": {"path": "/tmp/vfio-user.sock", "type": "unix"}}' See `libvfio-user `_ for further information. diff --git a/hw/vfio-user/device.c b/hw/vfio-user/device.c index 0609a7dc25..64ef35b320 100644 --- a/hw/vfio-user/device.c +++ b/hw/vfio-user/device.c @@ -134,7 +134,7 @@ static int vfio_user_device_io_get_region_info(VFIODevice *vbasedev, VFIOUserFDs fds = { 0, 1, fd}; int ret; - if (info->index > vbasedev->num_regions) { + if (info->index > vbasedev->num_initial_regions) { return -EINVAL; } diff --git a/hw/vfio/ccw.c b/hw/vfio/ccw.c index 9560b8d851..4d9588e7aa 100644 --- a/hw/vfio/ccw.c +++ b/hw/vfio/ccw.c @@ -484,9 +484,9 @@ static bool vfio_ccw_get_region(VFIOCCWDevice *vcdev, Error **errp) * We always expect at least the I/O region to be present. We also * may have a variable number of regions governed by capabilities. */ - if (vdev->num_regions < VFIO_CCW_CONFIG_REGION_INDEX + 1) { + if (vdev->num_initial_regions < VFIO_CCW_CONFIG_REGION_INDEX + 1) { error_setg(errp, "vfio: too few regions (%u), expected at least %u", - vdev->num_regions, VFIO_CCW_CONFIG_REGION_INDEX + 1); + vdev->num_initial_regions, VFIO_CCW_CONFIG_REGION_INDEX + 1); return false; } diff --git a/hw/vfio/container-legacy.c b/hw/vfio/container-legacy.c index a3615d7b5d..8e9639603e 100644 --- a/hw/vfio/container-legacy.c +++ b/hw/vfio/container-legacy.c @@ -121,14 +121,14 @@ unmap_exit: return ret; } -static int vfio_legacy_dma_unmap_one(const VFIOContainer *bcontainer, +static int vfio_legacy_dma_unmap_one(const VFIOLegacyContainer *container, hwaddr iova, uint64_t size, - IOMMUTLBEntry *iotlb) + uint32_t flags, IOMMUTLBEntry *iotlb) { - const VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(bcontainer); + const VFIOContainer *bcontainer = VFIO_IOMMU(container); struct vfio_iommu_type1_dma_unmap unmap = { .argsz = sizeof(unmap), - .flags = 0, + .flags = flags, .iova = iova, .size = size, }; @@ -170,25 +170,28 @@ static int vfio_legacy_dma_unmap(const VFIOContainer *bcontainer, hwaddr iova, uint64_t size, IOMMUTLBEntry *iotlb, bool unmap_all) { + const VFIOLegacyContainer *container = VFIO_IOMMU_LEGACY(bcontainer); + uint32_t flags = 0; int ret; if (unmap_all) { - /* The unmap ioctl doesn't accept a full 64-bit span. */ - Int128 llsize = int128_rshift(int128_2_64(), 1); + if (container->unmap_all_supported) { + flags = VFIO_DMA_UNMAP_FLAG_ALL; + } else { + /* The unmap ioctl doesn't accept a full 64-bit span. */ + Int128 llsize = int128_rshift(int128_2_64(), 1); + size = int128_get64(llsize); - ret = vfio_legacy_dma_unmap_one(bcontainer, 0, int128_get64(llsize), - iotlb); + ret = vfio_legacy_dma_unmap_one(container, 0, size, flags, iotlb); + if (ret) { + return ret; + } - if (ret == 0) { - ret = vfio_legacy_dma_unmap_one(bcontainer, int128_get64(llsize), - int128_get64(llsize), iotlb); + iova = size; } - - } else { - ret = vfio_legacy_dma_unmap_one(bcontainer, iova, size, iotlb); } - return ret; + return vfio_legacy_dma_unmap_one(container, iova, size, flags, iotlb); } static int vfio_legacy_dma_map(const VFIOContainer *bcontainer, hwaddr iova, @@ -519,6 +522,9 @@ static bool vfio_legacy_setup(VFIOContainer *bcontainer, Error **errp) vfio_get_info_iova_range(info, bcontainer); + ret = ioctl(container->fd, VFIO_CHECK_EXTENSION, VFIO_UNMAP_ALL); + container->unmap_all_supported = !!ret; + vfio_get_iommu_info_migration(container, info); return true; } diff --git a/hw/vfio/cpr-legacy.c b/hw/vfio/cpr-legacy.c index 80af7469d0..7184c93991 100644 --- a/hw/vfio/cpr-legacy.c +++ b/hw/vfio/cpr-legacy.c @@ -52,8 +52,6 @@ static int vfio_legacy_cpr_dma_map(const VFIOContainer *bcontainer, .size = size, }; - g_assert(cpr_is_incoming()); - if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map)) { return -errno; } @@ -228,22 +226,32 @@ void vfio_cpr_giommu_remap(VFIOContainer *bcontainer, memory_region_iommu_replay(giommu->iommu_mr, &giommu->n); } +static int vfio_cpr_rdm_remap(MemoryRegionSection *section, void *opaque) +{ + RamDiscardListener *rdl = opaque; + + return rdl->notify_populate(rdl, section); +} + /* * In old QEMU, VFIO_DMA_UNMAP_FLAG_VADDR may fail on some mapping after * succeeding for others, so the latter have lost their vaddr. Call this - * to restore vaddr for a section with a RamDiscardManager. + * to restore vaddr for populated parts in a section with a RamDiscardManager. * - * The ram discard listener already exists. Call its populate function + * The ram discard listener already exists. Call its replay_populated function * directly, which calls vfio_legacy_cpr_dma_map. */ -bool vfio_cpr_ram_discard_register_listener(VFIOContainer *bcontainer, - MemoryRegionSection *section) +bool vfio_cpr_ram_discard_replay_populated(VFIOContainer *bcontainer, + MemoryRegionSection *section) { + RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr); VFIORamDiscardListener *vrdl = vfio_find_ram_discard_listener(bcontainer, section); g_assert(vrdl); - return vrdl->listener.notify_populate(&vrdl->listener, section) == 0; + return ram_discard_manager_replay_populated(rdm, section, + vfio_cpr_rdm_remap, + &vrdl->listener) == 0; } int vfio_cpr_group_get_device_fd(int d, const char *name) diff --git a/hw/vfio/device.c b/hw/vfio/device.c index 64f8750389..8b63e765ac 100644 --- a/hw/vfio/device.c +++ b/hw/vfio/device.c @@ -205,10 +205,19 @@ int vfio_device_get_region_info(VFIODevice *vbasedev, int index, int fd = -1; int ret; - /* check cache */ - if (vbasedev->reginfo[index] != NULL) { - *info = vbasedev->reginfo[index]; - return 0; + /* + * We only set up the region info cache for the initial number of regions. + * + * Since a VFIO device may later increase the number of regions then use + * such regions with an index past ->num_initial_regions, don't attempt to + * use the info cache in those cases. + */ + if (index < vbasedev->num_initial_regions) { + /* check cache */ + if (vbasedev->reginfo[index] != NULL) { + *info = vbasedev->reginfo[index]; + return 0; + } } *info = g_malloc0(argsz); @@ -236,10 +245,12 @@ retry: goto retry; } - /* fill cache */ - vbasedev->reginfo[index] = *info; - if (vbasedev->region_fds != NULL) { - vbasedev->region_fds[index] = fd; + if (index < vbasedev->num_initial_regions) { + /* fill cache */ + vbasedev->reginfo[index] = *info; + if (vbasedev->region_fds != NULL) { + vbasedev->region_fds[index] = fd; + } } return 0; @@ -257,7 +268,7 @@ int vfio_device_get_region_info_type(VFIODevice *vbasedev, uint32_t type, { int i; - for (i = 0; i < vbasedev->num_regions; i++) { + for (i = 0; i < vbasedev->num_initial_regions; i++) { struct vfio_info_cap_header *hdr; struct vfio_region_info_cap_type *cap_type; @@ -466,7 +477,7 @@ void vfio_device_prepare(VFIODevice *vbasedev, VFIOContainer *bcontainer, int i; vbasedev->num_irqs = info->num_irqs; - vbasedev->num_regions = info->num_regions; + vbasedev->num_initial_regions = info->num_regions; vbasedev->flags = info->flags; vbasedev->reset_works = !!(info->flags & VFIO_DEVICE_FLAGS_RESET); @@ -476,10 +487,10 @@ void vfio_device_prepare(VFIODevice *vbasedev, VFIOContainer *bcontainer, QLIST_INSERT_HEAD(&vfio_device_list, vbasedev, global_next); vbasedev->reginfo = g_new0(struct vfio_region_info *, - vbasedev->num_regions); + vbasedev->num_initial_regions); if (vbasedev->use_region_fds) { - vbasedev->region_fds = g_new0(int, vbasedev->num_regions); - for (i = 0; i < vbasedev->num_regions; i++) { + vbasedev->region_fds = g_new0(int, vbasedev->num_initial_regions); + for (i = 0; i < vbasedev->num_initial_regions; i++) { vbasedev->region_fds[i] = -1; } } @@ -489,7 +500,7 @@ void vfio_device_unprepare(VFIODevice *vbasedev) { int i; - for (i = 0; i < vbasedev->num_regions; i++) { + for (i = 0; i < vbasedev->num_initial_regions; i++) { g_free(vbasedev->reginfo[i]); if (vbasedev->region_fds != NULL && vbasedev->region_fds[i] != -1) { close(vbasedev->region_fds[i]); diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c index 68470d552e..bb5775aa71 100644 --- a/hw/vfio/iommufd.c +++ b/hw/vfio/iommufd.c @@ -62,21 +62,8 @@ static int iommufd_cdev_unmap(const VFIOContainer *bcontainer, { const VFIOIOMMUFDContainer *container = VFIO_IOMMU_IOMMUFD(bcontainer); - /* unmap in halves */ if (unmap_all) { - Int128 llsize = int128_rshift(int128_2_64(), 1); - int ret; - - ret = iommufd_backend_unmap_dma(container->be, container->ioas_id, - 0, int128_get64(llsize)); - - if (ret == 0) { - ret = iommufd_backend_unmap_dma(container->be, container->ioas_id, - int128_get64(llsize), - int128_get64(llsize)); - } - - return ret; + size = UINT64_MAX; } /* TODO: Handle dma_unmap_bitmap with iotlb args (migration) */ @@ -560,10 +547,9 @@ static bool iommufd_cdev_attach(const char *name, VFIODevice *vbasedev, continue; } - if (!cpr_is_incoming()) { + if (!cpr_is_incoming() || + (vbasedev->cpr.ioas_id == container->ioas_id)) { res = iommufd_cdev_attach_container(vbasedev, container, &err); - } else if (vbasedev->cpr.ioas_id == container->ioas_id) { - res = true; } else { continue; } @@ -602,7 +588,6 @@ skip_ioas_alloc: container->be = vbasedev->iommufd; container->ioas_id = ioas_id; QLIST_INIT(&container->hwpt_list); - vbasedev->cpr.ioas_id = ioas_id; bcontainer = VFIO_IOMMU(container); vfio_address_space_insert(space, bcontainer); @@ -636,6 +621,8 @@ skip_ioas_alloc: bcontainer->initialized = true; found_container: + vbasedev->cpr.ioas_id = container->ioas_id; + ret = ioctl(devfd, VFIO_DEVICE_GET_INFO, &dev_info); if (ret) { error_setg_errno(errp, errno, "error getting device info"); @@ -663,7 +650,8 @@ found_container: vfio_iommufd_cpr_register_device(vbasedev); trace_iommufd_cdev_device_info(vbasedev->name, devfd, vbasedev->num_irqs, - vbasedev->num_regions, vbasedev->flags); + vbasedev->num_initial_regions, + vbasedev->flags); return true; err_listener_register: diff --git a/hw/vfio/listener.c b/hw/vfio/listener.c index c6bb58f520..2d7d3a4645 100644 --- a/hw/vfio/listener.c +++ b/hw/vfio/listener.c @@ -577,8 +577,8 @@ void vfio_container_region_add(VFIOContainer *bcontainer, if (!vfio_ram_discard_register_listener(bcontainer, section, &err)) { goto fail; } - } else if (!vfio_cpr_ram_discard_register_listener(bcontainer, - section)) { + } else if (!vfio_cpr_ram_discard_replay_populated(bcontainer, + section)) { error_setg(&err, "vfio_cpr_ram_discard_register_listener for %s failed", memory_region_name(section->mr)); @@ -715,6 +715,7 @@ static void vfio_listener_region_del(MemoryListener *listener, bool unmap_all = false; if (int128_eq(llsize, int128_2_64())) { + assert(!iova); unmap_all = true; llsize = int128_zero(); } diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c index 06b06afc2b..8b8bc5a421 100644 --- a/hw/vfio/pci.c +++ b/hw/vfio/pci.c @@ -2975,9 +2975,9 @@ bool vfio_pci_populate_device(VFIOPCIDevice *vdev, Error **errp) return false; } - if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) { + if (vbasedev->num_initial_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) { error_setg(errp, "unexpected number of io regions %u", - vbasedev->num_regions); + vbasedev->num_initial_regions); return false; } diff --git a/include/hw/vfio/vfio-container-legacy.h b/include/hw/vfio/vfio-container-legacy.h index 74a72df018..ffd594e80d 100644 --- a/include/hw/vfio/vfio-container-legacy.h +++ b/include/hw/vfio/vfio-container-legacy.h @@ -30,6 +30,7 @@ struct VFIOLegacyContainer { int fd; /* /dev/vfio/vfio, empowered by the attached groups */ unsigned iommu_type; + bool unmap_all_supported; QLIST_HEAD(, VFIOGroup) group_list; VFIOContainerCPR cpr; }; diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h index 81f4e24e22..4606da500a 100644 --- a/include/hw/vfio/vfio-cpr.h +++ b/include/hw/vfio/vfio-cpr.h @@ -68,7 +68,7 @@ bool vfio_cpr_container_match(struct VFIOLegacyContainer *container, void vfio_cpr_giommu_remap(struct VFIOContainer *bcontainer, MemoryRegionSection *section); -bool vfio_cpr_ram_discard_register_listener( +bool vfio_cpr_ram_discard_replay_populated( struct VFIOContainer *bcontainer, MemoryRegionSection *section); void vfio_cpr_save_vector_fd(struct VFIOPCIDevice *vdev, const char *name, diff --git a/include/hw/vfio/vfio-device.h b/include/hw/vfio/vfio-device.h index 7e9aed6d3c..0fe6c60ba2 100644 --- a/include/hw/vfio/vfio-device.h +++ b/include/hw/vfio/vfio-device.h @@ -74,7 +74,7 @@ typedef struct VFIODevice { VFIODeviceOps *ops; VFIODeviceIOOps *io_ops; unsigned int num_irqs; - unsigned int num_regions; + unsigned int num_initial_regions; unsigned int flags; VFIOMigration *migration; Error *migration_blocker;