From 971e25424c0e2cbbea28cc7e6d09a4569ca6cb06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 17:16:23 +0200 Subject: [PATCH 01/41] system/ramblock: Remove obsolete comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This comment was added almost 5 years ago in commit 41aa4e9fd84 ("ram_addr: Split RAMBlock definition"). Clearly it got ignored: $ git grep -l system/ramblock.h hw/display/virtio-gpu-udmabuf.c hw/hyperv/hv-balloon.c hw/virtio/vhost-user.c migration/dirtyrate.c migration/file.c migration/multifd-nocomp.c migration/multifd-qatzip.c migration/multifd-qpl.c migration/multifd-uadk.c migration/multifd-zero-page.c migration/multifd-zlib.c migration/multifd-zstd.c migration/multifd.c migration/postcopy-ram.c system/ram-block-attributes.c target/i386/kvm/tdx.c tests/qtest/fuzz/generic_fuzz.c At this point it seems saner to just remove it. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: David Hildenbrand Reviewed-by: Richard Henderson Acked-by: Peter Xu Message-Id: <20251002032812.26069-2-philmd@linaro.org> --- include/system/ramblock.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/system/ramblock.h b/include/system/ramblock.h index 87e847e184..8999206592 100644 --- a/include/system/ramblock.h +++ b/include/system/ramblock.h @@ -11,11 +11,6 @@ * */ -/* - * This header is for use by exec.c and memory.c ONLY. Do not include it. - * The functions declared here will be removed soon. - */ - #ifndef SYSTEM_RAMBLOCK_H #define SYSTEM_RAMBLOCK_H From 34f9b0ad082268e347cee5c4903940e4ea67614f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 17:12:12 +0200 Subject: [PATCH 02/41] system/ramblock: Move ram_block_is_pmem() declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move ramblock_is_pmem() along with the RAM Block API exposed by the "system/ramblock.h" header. Rename as ram_block_is_pmem() to keep API prefix consistency. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Alex Bennée Reviewed-by: Richard Henderson Acked-by: Peter Xu Message-Id: <20251002032812.26069-3-philmd@linaro.org> --- include/system/ram_addr.h | 2 -- include/system/ramblock.h | 5 +++++ migration/ram.c | 3 ++- system/physmem.c | 5 +++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 15a1b1a4fa..53c0c8c385 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -99,8 +99,6 @@ static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr, return host_addr_offset >> TARGET_PAGE_BITS; } -bool ramblock_is_pmem(RAMBlock *rb); - /** * qemu_ram_alloc_from_file, * qemu_ram_alloc_from_fd: Allocate a ram block from the specified backing diff --git a/include/system/ramblock.h b/include/system/ramblock.h index 8999206592..7059b20d91 100644 --- a/include/system/ramblock.h +++ b/include/system/ramblock.h @@ -108,4 +108,9 @@ void ram_block_attributes_destroy(RamBlockAttributes *attr); int ram_block_attributes_state_change(RamBlockAttributes *attr, uint64_t offset, uint64_t size, bool to_discard); +/** + * ram_block_is_pmem: Whether the RAM block is of persistent memory + */ +bool ram_block_is_pmem(RAMBlock *rb); + #endif diff --git a/migration/ram.c b/migration/ram.c index 9aac89638a..189931d24d 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -53,6 +53,7 @@ #include "qemu/rcu_queue.h" #include "migration/colo.h" #include "system/cpu-throttle.h" +#include "system/ramblock.h" #include "savevm.h" #include "qemu/iov.h" #include "multifd.h" @@ -4370,7 +4371,7 @@ static bool ram_has_postcopy(void *opaque) { RAMBlock *rb; RAMBLOCK_FOREACH_NOT_IGNORED(rb) { - if (ramblock_is_pmem(rb)) { + if (ram_block_is_pmem(rb)) { info_report("Block: %s, host: %p is a nvdimm memory, postcopy" "is not supported now!", rb->idstr, rb->host); return false; diff --git a/system/physmem.c b/system/physmem.c index dbb2a4e017..cb3e5fcfdc 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -43,6 +43,7 @@ #include "system/kvm.h" #include "system/tcg.h" #include "system/qtest.h" +#include "system/ramblock.h" #include "qemu/timer.h" #include "qemu/config-file.h" #include "qemu/error-report.h" @@ -1802,7 +1803,7 @@ void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length) #ifdef CONFIG_LIBPMEM /* The lack of support for pmem should not block the sync */ - if (ramblock_is_pmem(block)) { + if (ram_block_is_pmem(block)) { void *addr = ramblock_ptr(block, start); pmem_persist(addr, length); return; @@ -3941,7 +3942,7 @@ int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start, return ret; } -bool ramblock_is_pmem(RAMBlock *rb) +bool ram_block_is_pmem(RAMBlock *rb) { return rb->flags & RAM_PMEM; } From 8fe6ce40190872b4ccb4a3d601f4361b4bcdb0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 16:25:59 +0200 Subject: [PATCH 03/41] system/ramblock: Move ram_block_discard_*_range() declarations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep RAM blocks API in the same header: "system/ramblock.h". Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Acked-by: Peter Xu Message-Id: <20251002032812.26069-4-philmd@linaro.org> --- accel/kvm/kvm-all.c | 1 + hw/hyperv/hv-balloon-our_range_memslots.c | 1 + hw/virtio/virtio-balloon.c | 1 + hw/virtio/virtio-mem.c | 1 + include/exec/cpu-common.h | 3 --- include/system/ramblock.h | 4 ++++ 6 files changed, 8 insertions(+), 3 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 9060599cd7..e3c8472340 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -32,6 +32,7 @@ #include "system/runstate.h" #include "system/cpus.h" #include "system/accel-blocker.h" +#include "system/ramblock.h" #include "accel/accel-ops.h" #include "qemu/bswap.h" #include "exec/tswap.h" diff --git a/hw/hyperv/hv-balloon-our_range_memslots.c b/hw/hyperv/hv-balloon-our_range_memslots.c index 1505a395cf..1fc95e1648 100644 --- a/hw/hyperv/hv-balloon-our_range_memslots.c +++ b/hw/hyperv/hv-balloon-our_range_memslots.c @@ -8,6 +8,7 @@ */ #include "qemu/osdep.h" +#include "system/ramblock.h" #include "hv-balloon-internal.h" #include "hv-balloon-our_range_memslots.h" #include "trace.h" diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c index db787d00b3..02cdd807d7 100644 --- a/hw/virtio/virtio-balloon.c +++ b/hw/virtio/virtio-balloon.c @@ -23,6 +23,7 @@ #include "hw/qdev-properties.h" #include "hw/boards.h" #include "system/balloon.h" +#include "system/ramblock.h" #include "hw/virtio/virtio-balloon.h" #include "system/address-spaces.h" #include "qapi/error.h" diff --git a/hw/virtio/virtio-mem.c b/hw/virtio/virtio-mem.c index c46f6f9c3e..1de2d3de52 100644 --- a/hw/virtio/virtio-mem.c +++ b/hw/virtio/virtio-mem.c @@ -17,6 +17,7 @@ #include "qemu/units.h" #include "system/numa.h" #include "system/system.h" +#include "system/ramblock.h" #include "system/reset.h" #include "system/runstate.h" #include "hw/virtio/virtio.h" diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index b96ac49844..df520f15d3 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -163,9 +163,6 @@ void cpu_flush_icache_range(hwaddr start, hwaddr len); typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque); int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque); -int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length); -int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start, - size_t length); /* Returns: 0 on success, -1 on error */ int cpu_memory_rw_debug(CPUState *cpu, vaddr addr, diff --git a/include/system/ramblock.h b/include/system/ramblock.h index 7059b20d91..530c5a2e4c 100644 --- a/include/system/ramblock.h +++ b/include/system/ramblock.h @@ -103,6 +103,10 @@ struct RamBlockAttributes { QLIST_HEAD(, RamDiscardListener) rdl_list; }; +int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length); +int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start, + size_t length); + RamBlockAttributes *ram_block_attributes_create(RAMBlock *ram_block); void ram_block_attributes_destroy(RamBlockAttributes *attr); int ram_block_attributes_state_change(RamBlockAttributes *attr, uint64_t offset, From fe11c4957b8120c48ed650e7c970ea004d8432f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 16:18:18 +0200 Subject: [PATCH 04/41] system/ramblock: Rename @start -> @offset in ram_block_discard_range() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename @start as @offset, since it express an offset within a RAMBlock. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Acked-by: Peter Xu Message-Id: <20251002032812.26069-5-philmd@linaro.org> --- include/system/ramblock.h | 6 ++++-- system/physmem.c | 28 ++++++++++++++-------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/include/system/ramblock.h b/include/system/ramblock.h index 530c5a2e4c..85cceff6bc 100644 --- a/include/system/ramblock.h +++ b/include/system/ramblock.h @@ -103,8 +103,10 @@ struct RamBlockAttributes { QLIST_HEAD(, RamDiscardListener) rdl_list; }; -int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length); -int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start, +/* @offset: the offset within the RAMBlock */ +int ram_block_discard_range(RAMBlock *rb, uint64_t offset, size_t length); +/* @offset: the offset within the RAMBlock */ +int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t offset, size_t length); RamBlockAttributes *ram_block_attributes_create(RAMBlock *ram_block); diff --git a/system/physmem.c b/system/physmem.c index cb3e5fcfdc..a37592fd32 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -3795,18 +3795,18 @@ int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque) } /* - * Unmap pages of memory from start to start+length such that + * Unmap pages of memory from offset to offset+length such that * they a) read as 0, b) Trigger whatever fault mechanism * the OS provides for postcopy. * The pages must be unmapped by the end of the function. * Returns: 0 on success, none-0 on failure * */ -int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length) +int ram_block_discard_range(RAMBlock *rb, uint64_t offset, size_t length) { int ret = -1; - uint8_t *host_startaddr = rb->host + start; + uint8_t *host_startaddr = rb->host + offset; if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) { error_report("%s: Unaligned start address: %p", @@ -3814,7 +3814,7 @@ int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length) goto err; } - if ((start + length) <= rb->max_length) { + if ((offset + length) <= rb->max_length) { bool need_madvise, need_fallocate; if (!QEMU_IS_ALIGNED(length, rb->page_size)) { error_report("%s: Unaligned length: %zx", __func__, length); @@ -3865,11 +3865,11 @@ int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length) } ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, - start + rb->fd_offset, length); + offset + rb->fd_offset, length); if (ret) { ret = -errno; error_report("%s: Failed to fallocate %s:%" PRIx64 "+%" PRIx64 - " +%zx (%d)", __func__, rb->idstr, start, + " +%zx (%d)", __func__, rb->idstr, offset, rb->fd_offset, length, ret); goto err; } @@ -3877,7 +3877,7 @@ int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length) ret = -ENOSYS; error_report("%s: fallocate not available/file" "%s:%" PRIx64 "+%" PRIx64 " +%zx (%d)", __func__, - rb->idstr, start, rb->fd_offset, length, ret); + rb->idstr, offset, rb->fd_offset, length, ret); goto err; #endif } @@ -3897,13 +3897,13 @@ int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length) ret = -errno; error_report("%s: Failed to discard range " "%s:%" PRIx64 " +%zx (%d)", - __func__, rb->idstr, start, length, ret); + __func__, rb->idstr, offset, length, ret); goto err; } #else ret = -ENOSYS; error_report("%s: MADVISE not available %s:%" PRIx64 " +%zx (%d)", - __func__, rb->idstr, start, length, ret); + __func__, rb->idstr, offset, length, ret); goto err; #endif } @@ -3911,14 +3911,14 @@ int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length) need_madvise, need_fallocate, ret); } else { error_report("%s: Overrun block '%s' (%" PRIu64 "/%zx/" RAM_ADDR_FMT")", - __func__, rb->idstr, start, length, rb->max_length); + __func__, rb->idstr, offset, length, rb->max_length); } err: return ret; } -int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start, +int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t offset, size_t length) { int ret = -1; @@ -3926,17 +3926,17 @@ int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start, #ifdef CONFIG_FALLOCATE_PUNCH_HOLE /* ignore fd_offset with guest_memfd */ ret = fallocate(rb->guest_memfd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, - start, length); + offset, length); if (ret) { ret = -errno; error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)", - __func__, rb->idstr, start, length, ret); + __func__, rb->idstr, offset, length, ret); } #else ret = -ENOSYS; error_report("%s: fallocate not available %s:%" PRIx64 " +%zx (%d)", - __func__, rb->idstr, start, length, ret); + __func__, rb->idstr, offset, length, ret); #endif return ret; From 6feb119d32ed1b8853e407495fad33795b9c2137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 17:36:46 +0200 Subject: [PATCH 05/41] system/ramblock: Move RAMBlock helpers out of "system/ram_addr.h" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Acked-by: Peter Xu Message-Id: <20251002032812.26069-6-philmd@linaro.org> --- include/system/ram_addr.h | 11 ----------- include/system/ramblock.h | 11 +++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 53c0c8c385..6b528338ef 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -80,17 +80,6 @@ static inline bool clear_bmap_test_and_clear(RAMBlock *rb, uint64_t page) return bitmap_test_and_clear(rb->clear_bmap, page >> shift, 1); } -static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) -{ - return (b && b->host && offset < b->used_length) ? true : false; -} - -static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) -{ - assert(offset_in_ramblock(block, offset)); - return (char *)block->host + offset; -} - static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr, RAMBlock *rb) { diff --git a/include/system/ramblock.h b/include/system/ramblock.h index 85cceff6bc..76694fe1b5 100644 --- a/include/system/ramblock.h +++ b/include/system/ramblock.h @@ -119,4 +119,15 @@ int ram_block_attributes_state_change(RamBlockAttributes *attr, uint64_t offset, */ bool ram_block_is_pmem(RAMBlock *rb); +static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) +{ + return b && b->host && (offset < b->used_length); +} + +static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) +{ + assert(offset_in_ramblock(block, offset)); + return (char *)block->host + offset; +} + #endif From bb02dcf2d4c0ec446fb53516eea76087f8b7a3ed Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 22 Sep 2025 12:29:40 -0700 Subject: [PATCH 06/41] system/memory: Split address_space_write_rom_internal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 2dbaf58bbe7 we conditionally skipped the increment of buf because ubsan warns incrementing NULL, and buf is always NULL for FLUSH_CACHE. However, the existence of the test for NULL caused Coverity to warn that the memcpy in the WRITE_DATA case lacked a test for NULL. Duplicate address_space_write_rom_internal into the two callers, dropping enum write_rom_type, and simplify. This eliminates buf in the flush case, and eliminates the conditional increment of buf in the write case. Coverity: CID 1621220 Fixes: 2dbaf58bbe7 ("system/physmem: Silence warning from ubsan") Signed-off-by: Richard Henderson Reviewed-by: Thomas Huth Message-ID: <20250922192940.2908002-1-richard.henderson@linaro.org> Signed-off-by: Philippe Mathieu-Daudé --- system/physmem.c | 90 ++++++++++++++++++++---------------------------- 1 file changed, 37 insertions(+), 53 deletions(-) diff --git a/system/physmem.c b/system/physmem.c index a37592fd32..34216fa538 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -3186,63 +3186,33 @@ void cpu_physical_memory_rw(hwaddr addr, void *buf, buf, len, is_write); } -enum write_rom_type { - WRITE_DATA, - FLUSH_CACHE, -}; - -static inline MemTxResult address_space_write_rom_internal(AddressSpace *as, - hwaddr addr, - MemTxAttrs attrs, - const void *ptr, - hwaddr len, - enum write_rom_type type) -{ - hwaddr l; - uint8_t *ram_ptr; - hwaddr addr1; - MemoryRegion *mr; - const uint8_t *buf = ptr; - - RCU_READ_LOCK_GUARD(); - while (len > 0) { - l = len; - mr = address_space_translate(as, addr, &addr1, &l, true, attrs); - - if (!memory_region_supports_direct_access(mr)) { - l = memory_access_size(mr, l, addr1); - } else { - /* ROM/RAM case */ - ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1); - switch (type) { - case WRITE_DATA: - memcpy(ram_ptr, buf, l); - invalidate_and_set_dirty(mr, addr1, l); - break; - case FLUSH_CACHE: - flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l); - break; - } - } - len -= l; - addr += l; - if (buf) { - buf += l; - } - } - return MEMTX_OK; -} - /* used for ROM loading : can write in RAM and ROM */ MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, const void *buf, hwaddr len) { - return address_space_write_rom_internal(as, addr, attrs, - buf, len, WRITE_DATA); + RCU_READ_LOCK_GUARD(); + while (len > 0) { + hwaddr addr1, l = len; + MemoryRegion *mr = address_space_translate(as, addr, &addr1, &l, + true, attrs); + + if (!memory_region_supports_direct_access(mr)) { + l = memory_access_size(mr, l, addr1); + } else { + /* ROM/RAM case */ + void *ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1); + memcpy(ram_ptr, buf, l); + invalidate_and_set_dirty(mr, addr1, l); + } + len -= l; + addr += l; + buf += l; + } + return MEMTX_OK; } -void cpu_flush_icache_range(hwaddr start, hwaddr len) +void cpu_flush_icache_range(hwaddr addr, hwaddr len) { /* * This function should do the same thing as an icache flush that was @@ -3254,9 +3224,23 @@ void cpu_flush_icache_range(hwaddr start, hwaddr len) return; } - address_space_write_rom_internal(&address_space_memory, - start, MEMTXATTRS_UNSPECIFIED, - NULL, len, FLUSH_CACHE); + RCU_READ_LOCK_GUARD(); + while (len > 0) { + hwaddr addr1, l = len; + MemoryRegion *mr = address_space_translate(&address_space_memory, + addr, &addr1, &l, true, + MEMTXATTRS_UNSPECIFIED); + + if (!memory_region_supports_direct_access(mr)) { + l = memory_access_size(mr, l, addr1); + } else { + /* ROM/RAM case */ + void *ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1); + flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l); + } + len -= l; + addr += l; + } } /* From 7d4c9d2cb89818ef800718a4f462b34a21ee65cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 20:27:48 +0200 Subject: [PATCH 07/41] docs/devel/loads-stores: Stop mentioning cpu_physical_memory_write_rom() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the documentation after commit 3c8133f9737 ("Rename cpu_physical_memory_write_rom() to address_space_write_rom()"). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-2-philmd@linaro.org> --- docs/devel/loads-stores.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/devel/loads-stores.rst b/docs/devel/loads-stores.rst index 9471bac859..f9b565da57 100644 --- a/docs/devel/loads-stores.rst +++ b/docs/devel/loads-stores.rst @@ -474,7 +474,7 @@ This function is intended for use by the GDB stub and similar code. It takes a virtual address, converts it to a physical address via an MMU lookup using the current settings of the specified CPU, and then performs the access (using ``address_space_rw`` for -reads or ``cpu_physical_memory_write_rom`` for writes). +reads or ``address_space_write_rom`` for writes). This means that if the access is a write to a ROM then this function will modify the contents (whereas a normal guest CPU access would ignore the write attempt). From 839976e9da4b577aca284847b7e965332f2ca687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 14:36:19 +0200 Subject: [PATCH 08/41] system/memory: Factor address_space_is_io() out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Factor address_space_is_io() out of cpu_physical_memory_is_io(). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-3-philmd@linaro.org> --- include/system/memory.h | 9 +++++++++ system/physmem.c | 21 ++++++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/include/system/memory.h b/include/system/memory.h index 08daf0fc59..f222743b6f 100644 --- a/include/system/memory.h +++ b/include/system/memory.h @@ -3047,6 +3047,15 @@ static inline MemoryRegion *address_space_translate(AddressSpace *as, bool address_space_access_valid(AddressSpace *as, hwaddr addr, hwaddr len, bool is_write, MemTxAttrs attrs); +/** + * address_space_is_io: check whether an guest physical addresses + * whithin an address space is I/O memory. + * + * @as: #AddressSpace to be accessed + * @addr: address within that address space + */ +bool address_space_is_io(AddressSpace *as, hwaddr addr); + /* address_space_map: map a physical memory region into a host virtual address * * May map a subset of the requested range, given by and returned in @plen. diff --git a/system/physmem.c b/system/physmem.c index 34216fa538..5574c424f0 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -3356,6 +3356,17 @@ bool address_space_access_valid(AddressSpace *as, hwaddr addr, return flatview_access_valid(fv, addr, len, is_write, attrs); } +bool address_space_is_io(AddressSpace *as, hwaddr addr) +{ + MemoryRegion *mr; + + RCU_READ_LOCK_GUARD(); + mr = address_space_translate(as, addr, &addr, NULL, false, + MEMTXATTRS_UNSPECIFIED); + + return !(memory_region_is_ram(mr) || memory_region_is_romd(mr)); +} + static hwaddr flatview_extend_translation(FlatView *fv, hwaddr addr, hwaddr target_len, @@ -3752,15 +3763,7 @@ int cpu_memory_rw_debug(CPUState *cpu, vaddr addr, bool cpu_physical_memory_is_io(hwaddr phys_addr) { - MemoryRegion*mr; - hwaddr l = 1; - - RCU_READ_LOCK_GUARD(); - mr = address_space_translate(&address_space_memory, - phys_addr, &phys_addr, &l, false, - MEMTXATTRS_UNSPECIFIED); - - return !(memory_region_is_ram(mr) || memory_region_is_romd(mr)); + return address_space_is_io(&address_space_memory, phys_addr); } int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque) From 6ffaa9219616d12bc6c3e7cb26a77f7432ac0c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:35:28 +0200 Subject: [PATCH 09/41] target/i386/arch_memory_mapping: Use address_space_memory_is_io() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since all functions have an address space argument, it is trivial to replace cpu_physical_memory_is_io() by address_space_memory_is_io(). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-4-philmd@linaro.org> --- target/i386/arch_memory_mapping.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/target/i386/arch_memory_mapping.c b/target/i386/arch_memory_mapping.c index a2398c2173..560f4689ab 100644 --- a/target/i386/arch_memory_mapping.c +++ b/target/i386/arch_memory_mapping.c @@ -35,7 +35,7 @@ static void walk_pte(MemoryMappingList *list, AddressSpace *as, } start_paddr = (pte & ~0xfff) & ~(0x1ULL << 63); - if (cpu_physical_memory_is_io(start_paddr)) { + if (address_space_is_io(as, start_paddr)) { /* I/O region */ continue; } @@ -65,7 +65,7 @@ static void walk_pte2(MemoryMappingList *list, AddressSpace *as, } start_paddr = pte & ~0xfff; - if (cpu_physical_memory_is_io(start_paddr)) { + if (address_space_is_io(as, start_paddr)) { /* I/O region */ continue; } @@ -100,7 +100,7 @@ static void walk_pde(MemoryMappingList *list, AddressSpace *as, if (pde & PG_PSE_MASK) { /* 2 MB page */ start_paddr = (pde & ~0x1fffff) & ~(0x1ULL << 63); - if (cpu_physical_memory_is_io(start_paddr)) { + if (address_space_is_io(as, start_paddr)) { /* I/O region */ continue; } @@ -142,7 +142,7 @@ static void walk_pde2(MemoryMappingList *list, AddressSpace *as, */ high_paddr = ((hwaddr)(pde & 0x1fe000) << 19); start_paddr = (pde & ~0x3fffff) | high_paddr; - if (cpu_physical_memory_is_io(start_paddr)) { + if (address_space_is_io(as, start_paddr)) { /* I/O region */ continue; } @@ -203,7 +203,7 @@ static void walk_pdpe(MemoryMappingList *list, AddressSpace *as, if (pdpe & PG_PSE_MASK) { /* 1 GB page */ start_paddr = (pdpe & ~0x3fffffff) & ~(0x1ULL << 63); - if (cpu_physical_memory_is_io(start_paddr)) { + if (address_space_is_io(as, start_paddr)) { /* I/O region */ continue; } From 1e440937d699514207946c36c353a7a616f8b805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:28:06 +0200 Subject: [PATCH 10/41] hw/s390x/sclp: Use address_space_memory_is_io() in sclp_service_call() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When cpu_address_space_init() isn't called during vCPU creation, its single address space is the global &address_space_memory. As s390x boards don't call cpu_address_space_init(), cpu->as points to &address_space_memory. We can then replace cpu_physical_memory_is_io() by the semantically equivalent address_space_memory_is_io() call. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Reviewed-by: Eric Farman Message-Id: <20251002084203.63899-5-philmd@linaro.org> --- hw/s390x/sclp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c index 9718564fa4..16057356b1 100644 --- a/hw/s390x/sclp.c +++ b/hw/s390x/sclp.c @@ -16,6 +16,7 @@ #include "qemu/units.h" #include "qapi/error.h" #include "hw/boards.h" +#include "system/memory.h" #include "hw/s390x/sclp.h" #include "hw/s390x/event-facility.h" #include "hw/s390x/s390-pci-bus.h" @@ -308,7 +309,7 @@ int sclp_service_call(S390CPU *cpu, uint64_t sccb, uint32_t code) if (env->psw.mask & PSW_MASK_PSTATE) { return -PGM_PRIVILEGED; } - if (cpu_physical_memory_is_io(sccb)) { + if (address_space_is_io(CPU(cpu)->as, sccb)) { return -PGM_ADDRESSING; } if ((sccb & ~0x1fffUL) == 0 || (sccb & ~0x1fffUL) == env->psa From ec1eb357cb86eee74d63940154db1e1bfa86026a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:36:08 +0200 Subject: [PATCH 11/41] system/physmem: Remove cpu_physical_memory_is_io() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are no more uses of the legacy cpu_physical_memory_is_io() method. Remove it. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-6-philmd@linaro.org> --- include/exec/cpu-common.h | 2 -- system/physmem.c | 5 ----- 2 files changed, 7 deletions(-) diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index df520f15d3..55911c1d9f 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -149,8 +149,6 @@ void *cpu_physical_memory_map(hwaddr addr, void cpu_physical_memory_unmap(void *buffer, hwaddr len, bool is_write, hwaddr access_len); -bool cpu_physical_memory_is_io(hwaddr phys_addr); - /* Coalesced MMIO regions are areas where write operations can be reordered. * This usually implies that write operations are side-effect free. This allows * batching which can make a major impact on performance when using diff --git a/system/physmem.c b/system/physmem.c index 5574c424f0..c8a1fda55b 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -3761,11 +3761,6 @@ int cpu_memory_rw_debug(CPUState *cpu, vaddr addr, return 0; } -bool cpu_physical_memory_is_io(hwaddr phys_addr) -{ - return address_space_is_io(&address_space_memory, phys_addr); -} - int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque) { RAMBlock *block; From c2cac27dba3db3348563154f9b4b436ecde7b09a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:40:33 +0200 Subject: [PATCH 12/41] system/physmem: Pass address space argument to cpu_flush_icache_range() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename cpu_flush_icache_range() as address_space_flush_icache_range(), passing an address space by argument. The single caller, rom_reset(), already operates on an address space. Use it. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-7-philmd@linaro.org> --- hw/core/loader.c | 2 +- include/exec/cpu-common.h | 2 -- include/system/memory.h | 2 ++ system/physmem.c | 5 ++--- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/hw/core/loader.c b/hw/core/loader.c index 524af6f14a..477661a025 100644 --- a/hw/core/loader.c +++ b/hw/core/loader.c @@ -1242,7 +1242,7 @@ static void rom_reset(void *unused) * that the instruction cache for that new region is clear, so that the * CPU definitely fetches its instructions from the just written data. */ - cpu_flush_icache_range(rom->addr, rom->datasize); + address_space_flush_icache_range(rom->as, rom->addr, rom->datasize); trace_loader_write_rom(rom->name, rom->addr, rom->datasize, rom->isrom); } diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index 55911c1d9f..2e5aa684e9 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -156,8 +156,6 @@ void cpu_physical_memory_unmap(void *buffer, hwaddr len, */ void qemu_flush_coalesced_mmio_buffer(void); -void cpu_flush_icache_range(hwaddr start, hwaddr len); - typedef int (RAMBlockIterFunc)(RAMBlock *rb, void *opaque); int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque); diff --git a/include/system/memory.h b/include/system/memory.h index f222743b6f..3bd5ffa5e0 100644 --- a/include/system/memory.h +++ b/include/system/memory.h @@ -2995,6 +2995,8 @@ void address_space_cache_invalidate(MemoryRegionCache *cache, */ void address_space_cache_destroy(MemoryRegionCache *cache); +void address_space_flush_icache_range(AddressSpace *as, hwaddr addr, hwaddr len); + /* address_space_get_iotlb_entry: translate an address into an IOTLB * entry. Should be called from an RCU critical section. */ diff --git a/system/physmem.c b/system/physmem.c index c8a1fda55b..018b8f3157 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -3212,7 +3212,7 @@ MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr, return MEMTX_OK; } -void cpu_flush_icache_range(hwaddr addr, hwaddr len) +void address_space_flush_icache_range(AddressSpace *as, hwaddr addr, hwaddr len) { /* * This function should do the same thing as an icache flush that was @@ -3227,8 +3227,7 @@ void cpu_flush_icache_range(hwaddr addr, hwaddr len) RCU_READ_LOCK_GUARD(); while (len > 0) { hwaddr addr1, l = len; - MemoryRegion *mr = address_space_translate(&address_space_memory, - addr, &addr1, &l, true, + MemoryRegion *mr = address_space_translate(as, addr, &addr1, &l, true, MEMTXATTRS_UNSPECIFIED); if (!memory_region_supports_direct_access(mr)) { From 0d4e15e8f80e77a5bf22e3620ed990331f05256d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 30 Sep 2025 05:57:57 +0200 Subject: [PATCH 13/41] hw/s390x/sclp: Replace [cpu_physical_memory -> address_space]_r/w() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpu_physical_memory_read() and cpu_physical_memory_write() are legacy (see commit b7ecba0f6f6), replace by address_space_read() and address_space_write(). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Reviewed-by: Eric Farman Message-Id: <20251002084203.63899-8-philmd@linaro.org> --- hw/s390x/sclp.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c index 16057356b1..51e88ba8f1 100644 --- a/hw/s390x/sclp.c +++ b/hw/s390x/sclp.c @@ -304,6 +304,8 @@ int sclp_service_call(S390CPU *cpu, uint64_t sccb, uint32_t code) SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp); SCCBHeader header; g_autofree SCCB *work_sccb = NULL; + AddressSpace *as = CPU(cpu)->as; + const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED; /* first some basic checks on program checks */ if (env->psw.mask & PSW_MASK_PSTATE) { @@ -318,7 +320,7 @@ int sclp_service_call(S390CPU *cpu, uint64_t sccb, uint32_t code) } /* the header contains the actual length of the sccb */ - cpu_physical_memory_read(sccb, &header, sizeof(SCCBHeader)); + address_space_read(as, sccb, attrs, &header, sizeof(SCCBHeader)); /* Valid sccb sizes */ if (be16_to_cpu(header.length) < sizeof(SCCBHeader)) { @@ -331,7 +333,7 @@ int sclp_service_call(S390CPU *cpu, uint64_t sccb, uint32_t code) * the host has checked the values */ work_sccb = g_malloc0(be16_to_cpu(header.length)); - cpu_physical_memory_read(sccb, work_sccb, be16_to_cpu(header.length)); + address_space_read(as, sccb, attrs, work_sccb, be16_to_cpu(header.length)); if (!sclp_command_code_valid(code)) { work_sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND); @@ -345,8 +347,7 @@ int sclp_service_call(S390CPU *cpu, uint64_t sccb, uint32_t code) sclp_c->execute(sclp, work_sccb, code); out_write: - cpu_physical_memory_write(sccb, work_sccb, - be16_to_cpu(work_sccb->h.length)); + address_space_write(as, sccb, attrs, work_sccb, be16_to_cpu(header.length)); sclp_c->service_interrupt(sclp, sccb); From 2fd38d9c2944efef3b96fe9047b3c0f4d4e25eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:43:45 +0200 Subject: [PATCH 14/41] target/s390x/mmu: Replace [cpu_physical_memory -> address_space]_rw() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When cpu_address_space_init() isn't called during vCPU creation, its single address space is the global &address_space_memory. As s390x boards don't call cpu_address_space_init(), cpu->as points to &address_space_memory. We can then replace cpu_physical_memory_rw() by the semantically equivalent address_space_rw() call. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-9-philmd@linaro.org> --- target/s390x/mmu_helper.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/target/s390x/mmu_helper.c b/target/s390x/mmu_helper.c index 00946e9c0f..487c41bf93 100644 --- a/target/s390x/mmu_helper.c +++ b/target/s390x/mmu_helper.c @@ -23,6 +23,7 @@ #include "kvm/kvm_s390x.h" #include "system/kvm.h" #include "system/tcg.h" +#include "system/memory.h" #include "exec/page-protection.h" #include "exec/target_page.h" #include "hw/hw.h" @@ -522,6 +523,7 @@ int s390_cpu_pv_mem_rw(S390CPU *cpu, unsigned int offset, void *hostbuf, int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf, int len, bool is_write) { + const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED; int currlen, nr_pages, i; target_ulong *pages; uint64_t tec; @@ -542,11 +544,13 @@ int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf, if (ret) { trigger_access_exception(&cpu->env, ret, tec); } else if (hostbuf != NULL) { + AddressSpace *as = CPU(cpu)->as; + /* Copy data by stepping through the area page by page */ for (i = 0; i < nr_pages; i++) { currlen = MIN(len, TARGET_PAGE_SIZE - (laddr % TARGET_PAGE_SIZE)); - cpu_physical_memory_rw(pages[i] | (laddr & ~TARGET_PAGE_MASK), - hostbuf, currlen, is_write); + address_space_rw(as, pages[i] | (laddr & ~TARGET_PAGE_MASK), + attrs, hostbuf, currlen, is_write); laddr += currlen; hostbuf += currlen; len -= currlen; From 04979ddde45f1064aa98317172b0a5b99b4b4d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:51:33 +0200 Subject: [PATCH 15/41] target/i386/whpx: Replace legacy cpu_physical_memory_rw() call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get the vCPU address space and convert the legacy cpu_physical_memory_rw() by address_space_rw(). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-10-philmd@linaro.org> --- target/i386/whpx/whpx-all.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/target/i386/whpx/whpx-all.c b/target/i386/whpx/whpx-all.c index 2a85168ed5..256761834c 100644 --- a/target/i386/whpx/whpx-all.c +++ b/target/i386/whpx/whpx-all.c @@ -788,8 +788,11 @@ static HRESULT CALLBACK whpx_emu_mmio_callback( void *ctx, WHV_EMULATOR_MEMORY_ACCESS_INFO *ma) { - cpu_physical_memory_rw(ma->GpaAddress, ma->Data, ma->AccessSize, - ma->Direction); + CPUState *cs = (CPUState *)ctx; + AddressSpace *as = cpu_addressspace(cs, MEMTXATTRS_UNSPECIFIED); + + address_space_rw(as, ma->GpaAddress, MEMTXATTRS_UNSPECIFIED, + ma->Data, ma->AccessSize, ma->Direction); return S_OK; } From 12a65afbbfd8bef09cafb1e0e7883c30ddc1c956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:47:18 +0200 Subject: [PATCH 16/41] target/i386/kvm: Replace legacy cpu_physical_memory_rw() call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get the vCPU address space and convert the legacy cpu_physical_memory_rw() by address_space_rw(). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-11-philmd@linaro.org> --- target/i386/kvm/xen-emu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/target/i386/kvm/xen-emu.c b/target/i386/kvm/xen-emu.c index 284c5ef6f6..52de019834 100644 --- a/target/i386/kvm/xen-emu.c +++ b/target/i386/kvm/xen-emu.c @@ -21,6 +21,7 @@ #include "system/address-spaces.h" #include "xen-emu.h" #include "trace.h" +#include "system/memory.h" #include "system/runstate.h" #include "hw/pci/msi.h" @@ -75,6 +76,7 @@ static bool kvm_gva_to_gpa(CPUState *cs, uint64_t gva, uint64_t *gpa, static int kvm_gva_rw(CPUState *cs, uint64_t gva, void *_buf, size_t sz, bool is_write) { + AddressSpace *as = cpu_addressspace(cs, MEMTXATTRS_UNSPECIFIED); uint8_t *buf = (uint8_t *)_buf; uint64_t gpa; size_t len; @@ -87,7 +89,7 @@ static int kvm_gva_rw(CPUState *cs, uint64_t gva, void *_buf, size_t sz, len = sz; } - cpu_physical_memory_rw(gpa, buf, len, is_write); + address_space_rw(as, gpa, MEMTXATTRS_UNSPECIFIED, buf, len, is_write); buf += len; sz -= len; From 5699f6a4409af985d45cd67a0695a97655fec8bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:53:21 +0200 Subject: [PATCH 17/41] target/i386/nvmm: Inline cpu_physical_memory_rw() in nvmm_mem_callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-12-philmd@linaro.org> --- target/i386/nvmm/nvmm-all.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/target/i386/nvmm/nvmm-all.c b/target/i386/nvmm/nvmm-all.c index ed42425167..2e442baf4b 100644 --- a/target/i386/nvmm/nvmm-all.c +++ b/target/i386/nvmm/nvmm-all.c @@ -15,6 +15,7 @@ #include "accel/accel-ops.h" #include "system/nvmm.h" #include "system/cpus.h" +#include "system/memory.h" #include "system/runstate.h" #include "qemu/main-loop.h" #include "qemu/error-report.h" @@ -516,7 +517,9 @@ nvmm_io_callback(struct nvmm_io *io) static void nvmm_mem_callback(struct nvmm_mem *mem) { - cpu_physical_memory_rw(mem->gpa, mem->data, mem->size, mem->write); + /* TODO: Get CPUState via mem->vcpu? */ + address_space_rw(&address_space_memory, mem->gpa, MEMTXATTRS_UNSPECIFIED, + mem->data, mem->size, mem->write); /* Needed, otherwise infinite loop. */ current_cpu->vcpu_dirty = false; From 94c460835df5b810309a5ae4c89f0cddddbd9f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:55:07 +0200 Subject: [PATCH 18/41] hw/xen/hvm: Inline cpu_physical_memory_rw() in rw_phys_req_item() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpu_physical_memory_rw() is legacy, replace by address_space_rw(). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-13-philmd@linaro.org> --- hw/xen/xen-hvm-common.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c index 78e0bc8f64..52e2cce397 100644 --- a/hw/xen/xen-hvm-common.c +++ b/hw/xen/xen-hvm-common.c @@ -12,6 +12,7 @@ #include "hw/xen/xen-bus.h" #include "hw/boards.h" #include "hw/xen/arch_hvm.h" +#include "system/memory.h" #include "system/runstate.h" #include "system/system.h" #include "system/xen.h" @@ -279,8 +280,8 @@ static void do_outp(uint32_t addr, * memory, as part of the implementation of an ioreq. * * Equivalent to - * cpu_physical_memory_rw(addr + (req->df ? -1 : +1) * req->size * i, - * val, req->size, 0/1) + * address_space_rw(as, addr + (req->df ? -1 : +1) * req->size * i, + * attrs, val, req->size, 0/1) * except without the integer overflow problems. */ static void rw_phys_req_item(hwaddr addr, @@ -295,7 +296,8 @@ static void rw_phys_req_item(hwaddr addr, } else { addr += offset; } - cpu_physical_memory_rw(addr, val, req->size, rw); + address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED, + val, req->size, rw); } static inline void read_phys_req_item(hwaddr addr, From be481476bae495191c1184b368531401a5584904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:57:57 +0200 Subject: [PATCH 19/41] system/physmem: Un-inline cpu_physical_memory_read/write() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to remove cpu_physical_memory_rw() in a pair of commits, and due to a cyclic dependency between "exec/cpu-common.h" and "system/memory.h", un-inline cpu_physical_memory_read() and cpu_physical_memory_write() as a prerequired step. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-14-philmd@linaro.org> --- include/exec/cpu-common.h | 12 ++---------- system/physmem.c | 10 ++++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index 2e5aa684e9..ed35a18704 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -133,16 +133,8 @@ void cpu_destroy_address_spaces(CPUState *cpu); void cpu_physical_memory_rw(hwaddr addr, void *buf, hwaddr len, bool is_write); -static inline void cpu_physical_memory_read(hwaddr addr, - void *buf, hwaddr len) -{ - cpu_physical_memory_rw(addr, buf, len, false); -} -static inline void cpu_physical_memory_write(hwaddr addr, - const void *buf, hwaddr len) -{ - cpu_physical_memory_rw(addr, (void *)buf, len, true); -} +void cpu_physical_memory_read(hwaddr addr, void *buf, hwaddr len); +void cpu_physical_memory_write(hwaddr addr, const void *buf, hwaddr len); void *cpu_physical_memory_map(hwaddr addr, hwaddr *plen, bool is_write); diff --git a/system/physmem.c b/system/physmem.c index 018b8f3157..4dc66ac2c7 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -3186,6 +3186,16 @@ void cpu_physical_memory_rw(hwaddr addr, void *buf, buf, len, is_write); } +void cpu_physical_memory_read(hwaddr addr, void *buf, hwaddr len) +{ + cpu_physical_memory_rw(addr, buf, len, false); +} + +void cpu_physical_memory_write(hwaddr addr, const void *buf, hwaddr len) +{ + cpu_physical_memory_rw(addr, (void *)buf, len, true); +} + /* used for ROM loading : can write in RAM and ROM */ MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, From 5b7502234c7ede51ab6e684b6abc64d5fee3c1a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 30 Sep 2025 10:14:35 +0200 Subject: [PATCH 20/41] system/physmem: Avoid cpu_physical_memory_rw when is_write is constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following the mechanical changes of commit adeefe01671 ("Avoid cpu_physical_memory_rw() with a constant is_write argument"), replace: - cpu_physical_memory_rw(, is_write=false) -> address_space_read() - cpu_physical_memory_rw(, is_write=true) -> address_space_write() Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-15-philmd@linaro.org> --- scripts/coccinelle/exec_rw_const.cocci | 12 ------------ system/physmem.c | 6 ++++-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/scripts/coccinelle/exec_rw_const.cocci b/scripts/coccinelle/exec_rw_const.cocci index 1a20296951..35ab79e6d7 100644 --- a/scripts/coccinelle/exec_rw_const.cocci +++ b/scripts/coccinelle/exec_rw_const.cocci @@ -62,18 +62,6 @@ symbol true, false; + address_space_write(E1, E2, E3, E4, E5) ) -// Avoid uses of cpu_physical_memory_rw() with a constant is_write argument. -@@ -expression E1, E2, E3; -@@ -( -- cpu_physical_memory_rw(E1, E2, E3, false) -+ cpu_physical_memory_read(E1, E2, E3) -| -- cpu_physical_memory_rw(E1, E2, E3, true) -+ cpu_physical_memory_write(E1, E2, E3) -) - // Remove useless cast @@ expression E1, E2, E3, E4, E5, E6; diff --git a/system/physmem.c b/system/physmem.c index 4dc66ac2c7..8d65b4c125 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -3188,12 +3188,14 @@ void cpu_physical_memory_rw(hwaddr addr, void *buf, void cpu_physical_memory_read(hwaddr addr, void *buf, hwaddr len) { - cpu_physical_memory_rw(addr, buf, len, false); + address_space_read(&address_space_memory, addr, + MEMTXATTRS_UNSPECIFIED, buf, len); } void cpu_physical_memory_write(hwaddr addr, const void *buf, hwaddr len) { - cpu_physical_memory_rw(addr, (void *)buf, len, true); + address_space_write(&address_space_memory, addr, + MEMTXATTRS_UNSPECIFIED, buf, len); } /* used for ROM loading : can write in RAM and ROM */ From e98ae807c42e5c9f5089c947f7c6ef444248a946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 15:58:17 +0200 Subject: [PATCH 21/41] system/physmem: Remove legacy cpu_physical_memory_rw() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy cpu_physical_memory_rw() method is no more used, remove it. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-16-philmd@linaro.org> --- docs/devel/loads-stores.rst | 4 +--- include/exec/cpu-common.h | 2 -- scripts/coccinelle/exec_rw_const.cocci | 10 ---------- system/physmem.c | 7 ------- 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/docs/devel/loads-stores.rst b/docs/devel/loads-stores.rst index f9b565da57..c906c6509e 100644 --- a/docs/devel/loads-stores.rst +++ b/docs/devel/loads-stores.rst @@ -460,10 +460,8 @@ For new code they are better avoided: ``cpu_physical_memory_write`` -``cpu_physical_memory_rw`` - Regexes for git grep: - - ``\`` + - ``\`` ``cpu_memory_rw_debug`` ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index ed35a18704..67e15c8e50 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -131,8 +131,6 @@ void cpu_address_space_init(CPUState *cpu, int asidx, */ void cpu_destroy_address_spaces(CPUState *cpu); -void cpu_physical_memory_rw(hwaddr addr, void *buf, - hwaddr len, bool is_write); void cpu_physical_memory_read(hwaddr addr, void *buf, hwaddr len); void cpu_physical_memory_write(hwaddr addr, const void *buf, hwaddr len); void *cpu_physical_memory_map(hwaddr addr, diff --git a/scripts/coccinelle/exec_rw_const.cocci b/scripts/coccinelle/exec_rw_const.cocci index 35ab79e6d7..4c02c94e04 100644 --- a/scripts/coccinelle/exec_rw_const.cocci +++ b/scripts/coccinelle/exec_rw_const.cocci @@ -21,13 +21,6 @@ expression E1, E2, E3, E4, E5; + address_space_rw(E1, E2, E3, E4, E5, true) | -- cpu_physical_memory_rw(E1, E2, E3, 0) -+ cpu_physical_memory_rw(E1, E2, E3, false) -| -- cpu_physical_memory_rw(E1, E2, E3, 1) -+ cpu_physical_memory_rw(E1, E2, E3, true) -| - - cpu_physical_memory_map(E1, E2, 0) + cpu_physical_memory_map(E1, E2, false) | @@ -81,9 +74,6 @@ type T; + address_space_write_rom(E1, E2, E3, E4, E5) | -- cpu_physical_memory_rw(E1, (T *)(E2), E3, E4) -+ cpu_physical_memory_rw(E1, E2, E3, E4) -| - cpu_physical_memory_read(E1, (T *)(E2), E3) + cpu_physical_memory_read(E1, E2, E3) | diff --git a/system/physmem.c b/system/physmem.c index 8d65b4c125..36f327ca94 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -3179,13 +3179,6 @@ MemTxResult address_space_set(AddressSpace *as, hwaddr addr, return error; } -void cpu_physical_memory_rw(hwaddr addr, void *buf, - hwaddr len, bool is_write) -{ - address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED, - buf, len, is_write); -} - void cpu_physical_memory_read(hwaddr addr, void *buf, hwaddr len) { address_space_read(&address_space_memory, addr, From 41d5e37e0e2080e9d351f75ebd60d21e0b4e115e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 16:03:33 +0200 Subject: [PATCH 22/41] hw/virtio/vhost: Replace legacy cpu_physical_memory_*map() calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use VirtIODevice::dma_as address space to convert the legacy cpu_physical_memory_[un]map() calls to address_space_[un]map(). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-17-philmd@linaro.org> --- hw/virtio/vhost.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index c120ef38b9..266a11514a 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -27,6 +27,7 @@ #include "migration/blocker.h" #include "migration/qemu-file-types.h" #include "system/dma.h" +#include "system/memory.h" #include "trace.h" /* enabled until disconnected backend stabilizes */ @@ -455,7 +456,8 @@ static void *vhost_memory_map(struct vhost_dev *dev, hwaddr addr, hwaddr *plen, bool is_write) { if (!vhost_dev_has_iommu(dev)) { - return cpu_physical_memory_map(addr, plen, is_write); + return address_space_map(dev->vdev->dma_as, addr, plen, is_write, + MEMTXATTRS_UNSPECIFIED); } else { return (void *)(uintptr_t)addr; } @@ -466,7 +468,8 @@ static void vhost_memory_unmap(struct vhost_dev *dev, void *buffer, hwaddr access_len) { if (!vhost_dev_has_iommu(dev)) { - cpu_physical_memory_unmap(buffer, len, is_write, access_len); + address_space_unmap(dev->vdev->dma_as, buffer, len, is_write, + access_len); } } From f9f713f1b4a272826f4acc4d57483e9dd1faa3f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 16:06:27 +0200 Subject: [PATCH 23/41] hw/virtio/virtio: Replace legacy cpu_physical_memory_map() call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Propagate VirtIODevice::dma_as to virtqueue_undo_map_desc() in order to replace the legacy cpu_physical_memory_unmap() call by address_space_unmap(). Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251002084203.63899-18-philmd@linaro.org> --- hw/virtio/virtio.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index be73753b59..153ee0a0cf 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -32,6 +32,7 @@ #include "hw/virtio/virtio-access.h" #include "system/dma.h" #include "system/iothread.h" +#include "system/memory.h" #include "system/runstate.h" #include "virtio-qmp.h" @@ -1632,7 +1633,8 @@ out: * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to * yet. */ -static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num, +static void virtqueue_undo_map_desc(AddressSpace *as, + unsigned int out_num, unsigned int in_num, struct iovec *iov) { unsigned int i; @@ -1640,7 +1642,7 @@ static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num, for (i = 0; i < out_num + in_num; i++) { int is_write = i >= out_num; - cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0); + address_space_unmap(as, iov->iov_base, iov->iov_len, is_write, 0); iov++; } } @@ -1842,7 +1844,7 @@ done: return elem; err_undo_map: - virtqueue_undo_map_desc(out_num, in_num, iov); + virtqueue_undo_map_desc(vdev->dma_as, out_num, in_num, iov); goto done; } @@ -1992,7 +1994,7 @@ done: return elem; err_undo_map: - virtqueue_undo_map_desc(out_num, in_num, iov); + virtqueue_undo_map_desc(vdev->dma_as, out_num, in_num, iov); goto done; } From 2fb5ae9832728f766b66120fd23c7cf9db9d3ec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 30 Sep 2025 06:50:01 +0200 Subject: [PATCH 24/41] system/ram_addr: Remove unnecessary 'exec/cpu-common.h' header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in "system/ram_addr.h" requires definitions from "exec/cpu-common.h", remove it. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-2-philmd@linaro.org> --- include/system/ram_addr.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 6b528338ef..f74a0ecee5 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -29,7 +29,6 @@ #include "qemu/rcu.h" #include "exec/hwaddr.h" -#include "exec/cpu-common.h" extern uint64_t total_dirty_pages; From a55e8ffe47ebe0d7da7b5008b5efdfa652545a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 1 Oct 2025 09:52:40 +0200 Subject: [PATCH 25/41] accel/kvm: Include missing 'exec/target_page.h' header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "exec/target_page.h" header is indirectly pulled from "system/ram_addr.h". Include it explicitly, in order to avoid unrelated issues when refactoring "system/ram_addr.h": accel/kvm/kvm-all.c: In function ‘kvm_init’: accel/kvm/kvm-all.c:2636:12: error: ‘TARGET_PAGE_SIZE’ undeclared (first use in this function); did you mean ‘TARGET_PAGE_BITS’? 2636 | assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size()); | ^~~~~~~~~~~~~~~~ Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-3-philmd@linaro.org> --- accel/kvm/kvm-all.c | 1 + 1 file changed, 1 insertion(+) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index e3c8472340..08b2b5a371 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -36,6 +36,7 @@ #include "accel/accel-ops.h" #include "qemu/bswap.h" #include "exec/tswap.h" +#include "exec/target_page.h" #include "system/memory.h" #include "system/ram_addr.h" #include "qemu/event_notifier.h" From d08bc190a0302d1fad8ca71a3ba13c51f7337bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 1 Oct 2025 09:54:51 +0200 Subject: [PATCH 26/41] hw/s390x/s390-stattrib: Include missing 'exec/target_page.h' header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "exec/target_page.h" header is indirectly pulled from "system/ram_addr.h". Include it explicitly, in order to avoid unrelated issues when refactoring "system/ram_addr.h": hw/s390x/s390-stattrib-kvm.c: In function ‘kvm_s390_stattrib_set_stattr’: hw/s390x/s390-stattrib-kvm.c:89:57: error: ‘TARGET_PAGE_SIZE’ undeclared (first use in this function); did you mean ‘TARGET_PAGE_BITS’? 89 | unsigned long max = s390_get_memory_limit(s390ms) / TARGET_PAGE_SIZE; | ^~~~~~~~~~~~~~~~ | TARGET_PAGE_BITS Since "system/ram_addr.h" is actually not needed, remove it. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Reviewed-by: Eric Farman Reviewed-by: Thomas Huth Message-Id: <20251001175448.18933-4-philmd@linaro.org> --- hw/s390x/s390-stattrib-kvm.c | 2 +- hw/s390x/s390-stattrib.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/s390x/s390-stattrib-kvm.c b/hw/s390x/s390-stattrib-kvm.c index e1fee361dc..73df1f600b 100644 --- a/hw/s390x/s390-stattrib-kvm.c +++ b/hw/s390x/s390-stattrib-kvm.c @@ -10,13 +10,13 @@ */ #include "qemu/osdep.h" +#include "exec/target_page.h" #include "hw/s390x/s390-virtio-ccw.h" #include "migration/qemu-file.h" #include "hw/s390x/storage-attributes.h" #include "qemu/error-report.h" #include "system/kvm.h" #include "system/memory_mapping.h" -#include "system/ram_addr.h" #include "kvm/kvm_s390x.h" #include "qapi/error.h" diff --git a/hw/s390x/s390-stattrib.c b/hw/s390x/s390-stattrib.c index 13a678a803..aa18537291 100644 --- a/hw/s390x/s390-stattrib.c +++ b/hw/s390x/s390-stattrib.c @@ -11,12 +11,12 @@ #include "qemu/osdep.h" #include "qemu/units.h" +#include "exec/target_page.h" #include "migration/qemu-file.h" #include "migration/register.h" #include "hw/qdev-properties.h" #include "hw/s390x/storage-attributes.h" #include "qemu/error-report.h" -#include "system/ram_addr.h" #include "qapi/error.h" #include "qobject/qdict.h" #include "cpu.h" From edd1f91d38dfc341cac02529fcd315609e959763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 1 Oct 2025 09:56:41 +0200 Subject: [PATCH 27/41] hw/vfio/listener: Include missing 'exec/target_page.h' header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "exec/target_page.h" header is indirectly pulled from "system/ram_addr.h". Include it explicitly, in order to avoid unrelated issues when refactoring "system/ram_addr.h": hw/vfio/listener.c: In function ‘vfio_ram_discard_register_listener’: hw/vfio/listener.c:258:28: error: implicit declaration of function ‘qemu_target_page_size’; did you mean ‘qemu_ram_pagesize’? 258 | int target_page_size = qemu_target_page_size(); | ^~~~~~~~~~~~~~~~~~~~~ Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Reviewed-by: Cédric Le Goater Message-Id: <20251001175448.18933-5-philmd@linaro.org> --- hw/vfio/listener.c | 1 + 1 file changed, 1 insertion(+) diff --git a/hw/vfio/listener.c b/hw/vfio/listener.c index a2c19a3cec..b5cefc9395 100644 --- a/hw/vfio/listener.c +++ b/hw/vfio/listener.c @@ -25,6 +25,7 @@ #endif #include +#include "exec/target_page.h" #include "hw/vfio/vfio-device.h" #include "hw/vfio/pci.h" #include "system/address-spaces.h" From 6204f64260b78cc4080ca7b1ee221718273d6566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 30 Sep 2025 10:33:33 +0200 Subject: [PATCH 28/41] target/arm/tcg/mte: Include missing 'exec/target_page.h' header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "exec/target_page.h" header is indirectly pulled from "system/ram_addr.h". Include it explicitly, in order to avoid unrelated issues when refactoring "system/ram_addr.h": target/arm/tcg/mte_helper.c:815:23: error: use of undeclared identifier 'TARGET_PAGE_MASK' 815 | prev_page = ptr & TARGET_PAGE_MASK; | ^ target/arm/tcg/mte_helper.c:816:29: error: use of undeclared identifier 'TARGET_PAGE_SIZE' 816 | next_page = prev_page + TARGET_PAGE_SIZE; | ^ Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-6-philmd@linaro.org> --- target/arm/tcg/mte_helper.c | 1 + 1 file changed, 1 insertion(+) diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c index 302e899287..7d80244788 100644 --- a/target/arm/tcg/mte_helper.c +++ b/target/arm/tcg/mte_helper.c @@ -21,6 +21,7 @@ #include "qemu/log.h" #include "cpu.h" #include "internals.h" +#include "exec/target_page.h" #include "exec/page-protection.h" #ifdef CONFIG_USER_ONLY #include "user/cpu_loop.h" From 97480ca692e94bb790190a43bb122bd0752b8f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 30 Sep 2025 09:20:38 +0200 Subject: [PATCH 29/41] hw: Remove unnecessary 'system/ram_addr.h' header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit None of these files require definition exposed by "system/ram_addr.h", remove its inclusion. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Reviewed-by: Jagannathan Raman Reviewed-by: Cédric Le Goater Reviewed-by: Eric Farman Reviewed-by: Thomas Huth Acked-by: Michael S. Tsirkin Message-Id: <20251001175448.18933-7-philmd@linaro.org> --- hw/ppc/spapr.c | 1 - hw/ppc/spapr_caps.c | 1 - hw/ppc/spapr_pci.c | 1 - hw/remote/memory.c | 1 - hw/remote/proxy-memory-listener.c | 1 - hw/s390x/s390-virtio-ccw.c | 1 - hw/vfio/spapr.c | 1 - hw/virtio/virtio-mem.c | 1 - 8 files changed, 8 deletions(-) diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index 82fb23beaa..97ab6bebd2 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -77,7 +77,6 @@ #include "hw/virtio/virtio-scsi.h" #include "hw/virtio/vhost-scsi-common.h" -#include "system/ram_addr.h" #include "system/confidential-guest-support.h" #include "hw/usb.h" #include "qemu/config-file.h" diff --git a/hw/ppc/spapr_caps.c b/hw/ppc/spapr_caps.c index f2f5722d8a..0f94c192fd 100644 --- a/hw/ppc/spapr_caps.c +++ b/hw/ppc/spapr_caps.c @@ -27,7 +27,6 @@ #include "qapi/error.h" #include "qapi/visitor.h" #include "system/hw_accel.h" -#include "system/ram_addr.h" #include "target/ppc/cpu.h" #include "target/ppc/mmu-hash64.h" #include "cpu-models.h" diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c index 1ac1185825..f9095552e8 100644 --- a/hw/ppc/spapr_pci.c +++ b/hw/ppc/spapr_pci.c @@ -34,7 +34,6 @@ #include "hw/pci/pci_host.h" #include "hw/ppc/spapr.h" #include "hw/pci-host/spapr.h" -#include "system/ram_addr.h" #include #include "trace.h" #include "qemu/error-report.h" diff --git a/hw/remote/memory.c b/hw/remote/memory.c index 00193a552f..8195aa5fb8 100644 --- a/hw/remote/memory.c +++ b/hw/remote/memory.c @@ -11,7 +11,6 @@ #include "qemu/osdep.h" #include "hw/remote/memory.h" -#include "system/ram_addr.h" #include "qapi/error.h" static void remote_sysmem_reset(void) diff --git a/hw/remote/proxy-memory-listener.c b/hw/remote/proxy-memory-listener.c index 30ac74961d..e1a52d24f0 100644 --- a/hw/remote/proxy-memory-listener.c +++ b/hw/remote/proxy-memory-listener.c @@ -12,7 +12,6 @@ #include "qemu/range.h" #include "system/memory.h" #include "exec/cpu-common.h" -#include "system/ram_addr.h" #include "qapi/error.h" #include "qemu/error-report.h" #include "hw/remote/mpqemu-link.h" diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c index d0c6e80cb0..ad2c48188a 100644 --- a/hw/s390x/s390-virtio-ccw.c +++ b/hw/s390x/s390-virtio-ccw.c @@ -13,7 +13,6 @@ #include "qemu/osdep.h" #include "qapi/error.h" -#include "system/ram_addr.h" #include "system/confidential-guest-support.h" #include "hw/boards.h" #include "hw/s390x/sclp.h" diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c index 8d9d68da4e..0f23681a3f 100644 --- a/hw/vfio/spapr.c +++ b/hw/vfio/spapr.c @@ -17,7 +17,6 @@ #include "hw/vfio/vfio-container-legacy.h" #include "hw/hw.h" -#include "system/ram_addr.h" #include "qemu/error-report.h" #include "qapi/error.h" #include "trace.h" diff --git a/hw/virtio/virtio-mem.c b/hw/virtio/virtio-mem.c index 1de2d3de52..15ba6799f2 100644 --- a/hw/virtio/virtio-mem.c +++ b/hw/virtio/virtio-mem.c @@ -25,7 +25,6 @@ #include "hw/virtio/virtio-mem.h" #include "qapi/error.h" #include "qapi/visitor.h" -#include "system/ram_addr.h" #include "migration/misc.h" #include "hw/boards.h" #include "hw/qdev-properties.h" From 81aef73696c37c1680412e52d0a63c9c1fdd0466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 13:31:32 +0200 Subject: [PATCH 30/41] system/physmem: Un-inline cpu_physical_memory_get_dirty_flag() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid maintaining large functions in header, rely on the linker to optimize at linking time. cpu_physical_memory_get_dirty() doesn't involve any CPU, remove the 'cpu_' prefix. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-8-philmd@linaro.org> --- include/system/ram_addr.h | 46 +-------------------------------------- system/physmem.c | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index f74a0ecee5..f8a307d1a3 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -142,46 +142,6 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) -static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, - ram_addr_t length, - unsigned client) -{ - DirtyMemoryBlocks *blocks; - unsigned long end, page; - unsigned long idx, offset, base; - bool dirty = false; - - assert(client < DIRTY_MEMORY_NUM); - - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - - WITH_RCU_READ_LOCK_GUARD() { - blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - unsigned long num = next - base; - unsigned long found = find_next_bit(blocks->blocks[idx], - num, offset); - if (found < num) { - dirty = true; - break; - } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; - } - } - - return dirty; -} - static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, ram_addr_t length, unsigned client) @@ -221,11 +181,7 @@ static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, return dirty; } -static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, - unsigned client) -{ - return cpu_physical_memory_get_dirty(addr, 1, client); -} +bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) { diff --git a/system/physmem.c b/system/physmem.c index 36f327ca94..3d81272f9f 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -899,6 +899,50 @@ void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length) } } +static bool physical_memory_get_dirty(ram_addr_t start, ram_addr_t length, + unsigned client) +{ + DirtyMemoryBlocks *blocks; + unsigned long end, page; + unsigned long idx, offset, base; + bool dirty = false; + + assert(client < DIRTY_MEMORY_NUM); + + end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; + page = start >> TARGET_PAGE_BITS; + + WITH_RCU_READ_LOCK_GUARD() { + blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); + + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + base = page - offset; + while (page < end) { + unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); + unsigned long num = next - base; + unsigned long found = find_next_bit(blocks->blocks[idx], + num, offset); + if (found < num) { + dirty = true; + break; + } + + page = next; + idx++; + offset = 0; + base += DIRTY_MEMORY_BLOCK_SIZE; + } + } + + return dirty; +} + +bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client) +{ + return physical_memory_get_dirty(addr, 1, client); +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, From 3a0539afcbdf83aa919e32a36107bbe357ae9ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 13:33:02 +0200 Subject: [PATCH 31/41] system/physmem: Un-inline cpu_physical_memory_is_clean() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-9-philmd@linaro.org> --- include/system/ram_addr.h | 9 +-------- system/physmem.c | 9 +++++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index f8a307d1a3..cdf25c315b 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -183,14 +183,7 @@ static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); -static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) -{ - bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); - bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); - bool migration = - cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); - return !(vga && code && migration); -} +bool cpu_physical_memory_is_clean(ram_addr_t addr); static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, ram_addr_t length, diff --git a/system/physmem.c b/system/physmem.c index 3d81272f9f..e555b3196f 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -943,6 +943,15 @@ bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client) return physical_memory_get_dirty(addr, 1, client); } +bool cpu_physical_memory_is_clean(ram_addr_t addr) +{ + bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); + bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); + bool migration = + cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); + return !(vga && code && migration); +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, From 84a8e6399bc06550e41f5b2f7b94e4a2213b2ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 13:35:49 +0200 Subject: [PATCH 32/41] system/physmem: Un-inline cpu_physical_memory_range_includes_clean() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid maintaining large functions in header, rely on the linker to optimize at linking time. cpu_physical_memory_all_dirty() doesn't involve any CPU, remove the 'cpu_' prefix. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-10-philmd@linaro.org> --- include/system/ram_addr.h | 62 ++------------------------------------- system/physmem.c | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 59 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index cdf25c315b..2dcca260b2 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -142,69 +142,13 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) -static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, - ram_addr_t length, - unsigned client) -{ - DirtyMemoryBlocks *blocks; - unsigned long end, page; - unsigned long idx, offset, base; - bool dirty = true; - - assert(client < DIRTY_MEMORY_NUM); - - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - - RCU_READ_LOCK_GUARD(); - - blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - unsigned long num = next - base; - unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset); - if (found < num) { - dirty = false; - break; - } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; - } - - return dirty; -} - bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); bool cpu_physical_memory_is_clean(ram_addr_t addr); -static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, - ram_addr_t length, - uint8_t mask) -{ - uint8_t ret = 0; - - if (mask & (1 << DIRTY_MEMORY_VGA) && - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { - ret |= (1 << DIRTY_MEMORY_VGA); - } - if (mask & (1 << DIRTY_MEMORY_CODE) && - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { - ret |= (1 << DIRTY_MEMORY_CODE); - } - if (mask & (1 << DIRTY_MEMORY_MIGRATION) && - !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { - ret |= (1 << DIRTY_MEMORY_MIGRATION); - } - return ret; -} +uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, + ram_addr_t length, + uint8_t mask); static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) diff --git a/system/physmem.c b/system/physmem.c index e555b3196f..144fd7303b 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -952,6 +952,66 @@ bool cpu_physical_memory_is_clean(ram_addr_t addr) return !(vga && code && migration); } +static bool physical_memory_all_dirty(ram_addr_t start, ram_addr_t length, + unsigned client) +{ + DirtyMemoryBlocks *blocks; + unsigned long end, page; + unsigned long idx, offset, base; + bool dirty = true; + + assert(client < DIRTY_MEMORY_NUM); + + end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; + page = start >> TARGET_PAGE_BITS; + + RCU_READ_LOCK_GUARD(); + + blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); + + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + base = page - offset; + while (page < end) { + unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); + unsigned long num = next - base; + unsigned long found = find_next_zero_bit(blocks->blocks[idx], + num, offset); + if (found < num) { + dirty = false; + break; + } + + page = next; + idx++; + offset = 0; + base += DIRTY_MEMORY_BLOCK_SIZE; + } + + return dirty; +} + +uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, + ram_addr_t length, + uint8_t mask) +{ + uint8_t ret = 0; + + if (mask & (1 << DIRTY_MEMORY_VGA) && + !physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { + ret |= (1 << DIRTY_MEMORY_VGA); + } + if (mask & (1 << DIRTY_MEMORY_CODE) && + !physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { + ret |= (1 << DIRTY_MEMORY_CODE); + } + if (mask & (1 << DIRTY_MEMORY_MIGRATION) && + !physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { + ret |= (1 << DIRTY_MEMORY_MIGRATION); + } + return ret; +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, From 62c889eb7dfd4da5e30e9496496917bad53851fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 13:38:52 +0200 Subject: [PATCH 33/41] system/physmem: Un-inline cpu_physical_memory_set_dirty_flag() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-11-philmd@linaro.org> --- include/system/ram_addr.h | 19 +------------------ system/physmem.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 2dcca260b2..81d26eb149 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -150,24 +150,7 @@ uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, ram_addr_t length, uint8_t mask); -static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, - unsigned client) -{ - unsigned long page, idx, offset; - DirtyMemoryBlocks *blocks; - - assert(client < DIRTY_MEMORY_NUM); - - page = addr >> TARGET_PAGE_BITS; - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - - RCU_READ_LOCK_GUARD(); - - blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); - - set_bit_atomic(offset, blocks->blocks[idx]); -} +void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, diff --git a/system/physmem.c b/system/physmem.c index 144fd7303b..50235e8853 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -1012,6 +1012,24 @@ uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, return ret; } +void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) +{ + unsigned long page, idx, offset; + DirtyMemoryBlocks *blocks; + + assert(client < DIRTY_MEMORY_NUM); + + page = addr >> TARGET_PAGE_BITS; + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + + RCU_READ_LOCK_GUARD(); + + blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]); + + set_bit_atomic(offset, blocks->blocks[idx]); +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, From 6290580e9b712d5cbaa1cc2a314d07cd4a6c283d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 13:40:29 +0200 Subject: [PATCH 34/41] system/physmem: Un-inline cpu_physical_memory_set_dirty_range() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-12-philmd@linaro.org> --- include/system/ram_addr.h | 53 ++------------------------------------- system/physmem.c | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 51 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 81d26eb149..ca5ae84244 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -152,57 +152,8 @@ uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); -static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, - ram_addr_t length, - uint8_t mask) -{ - DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; - unsigned long end, page; - unsigned long idx, offset, base; - int i; - - if (!mask && !xen_enabled()) { - return; - } - - end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; - page = start >> TARGET_PAGE_BITS; - - WITH_RCU_READ_LOCK_GUARD() { - for (i = 0; i < DIRTY_MEMORY_NUM; i++) { - blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]); - } - - idx = page / DIRTY_MEMORY_BLOCK_SIZE; - offset = page % DIRTY_MEMORY_BLOCK_SIZE; - base = page - offset; - while (page < end) { - unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); - - if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], - offset, next - page); - } - if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], - offset, next - page); - } - if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { - bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], - offset, next - page); - } - - page = next; - idx++; - offset = 0; - base += DIRTY_MEMORY_BLOCK_SIZE; - } - } - - if (xen_enabled()) { - xen_hvm_modified_memory(start, length); - } -} +void cpu_physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, + uint8_t mask); #if !defined(_WIN32) diff --git a/system/physmem.c b/system/physmem.c index 50235e8853..bcd9d6fb8f 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -1030,6 +1030,57 @@ void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) set_bit_atomic(offset, blocks->blocks[idx]); } +void cpu_physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, + uint8_t mask) +{ + DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; + unsigned long end, page; + unsigned long idx, offset, base; + int i; + + if (!mask && !xen_enabled()) { + return; + } + + end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; + page = start >> TARGET_PAGE_BITS; + + WITH_RCU_READ_LOCK_GUARD() { + for (i = 0; i < DIRTY_MEMORY_NUM; i++) { + blocks[i] = qatomic_rcu_read(&ram_list.dirty_memory[i]); + } + + idx = page / DIRTY_MEMORY_BLOCK_SIZE; + offset = page % DIRTY_MEMORY_BLOCK_SIZE; + base = page - offset; + while (page < end) { + unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); + + if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { + bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], + offset, next - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { + bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], + offset, next - page); + } + if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { + bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], + offset, next - page); + } + + page = next; + idx++; + offset = 0; + base += DIRTY_MEMORY_BLOCK_SIZE; + } + } + + if (xen_enabled()) { + xen_hvm_modified_memory(start, length); + } +} + /* Note: start and end must be within the same ram block. */ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, From 3037def8b8e91ed3ce3f711dc646e07e0ef894dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 1 Oct 2025 19:06:19 +0200 Subject: [PATCH 35/41] system/physmem: Remove _WIN32 #ifdef'ry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit fb3ecb7ea40 ("exec: Exclude non portable function for MinGW") guarded cpu_physical_memory_set_dirty_lebitmap() within _WIN32 #ifdef'ry because of the non-portable ffsl() call, which was later replaced for the same reason by commit 7224f66ec3c ("exec: replace ffsl with ctzl"); we don't need that anymore. Reported-by: Richard Henderson Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-13-philmd@linaro.org> --- include/system/ram_addr.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index ca5ae84244..fbf57a05b2 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -155,8 +155,6 @@ void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); void cpu_physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, uint8_t mask); -#if !defined(_WIN32) - /* * Contrary to cpu_physical_memory_sync_dirty_bitmap() this function returns * the number of dirty pages in @bitmap passed as argument. On the other hand, @@ -265,7 +263,6 @@ uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, return num_dirty; } -#endif /* not _WIN32 */ static inline void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length) From 198c5f707f43d970b22e9b52a0ec573aa595ef35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 13:43:30 +0200 Subject: [PATCH 36/41] system/physmem: Un-inline cpu_physical_memory_set_dirty_lebitmap() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Remove the now unneeded "system/xen.h" header. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-14-philmd@linaro.org> --- include/system/ram_addr.h | 102 +------------------------------------ system/physmem.c | 103 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 101 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index fbf57a05b2..49e9a9c66d 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -19,7 +19,6 @@ #ifndef SYSTEM_RAM_ADDR_H #define SYSTEM_RAM_ADDR_H -#include "system/xen.h" #include "system/tcg.h" #include "exec/cputlb.h" #include "exec/ramlist.h" @@ -161,108 +160,9 @@ void cpu_physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, * cpu_physical_memory_sync_dirty_bitmap() returns newly dirtied pages that * weren't set in the global migration bitmap. */ -static inline uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_addr_t start, - ram_addr_t pages) -{ - unsigned long i, j; - unsigned long page_number, c, nbits; - hwaddr addr; - ram_addr_t ram_addr; - uint64_t num_dirty = 0; - unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; - unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE; - unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); - - /* start address is aligned at the start of a word? */ - if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && - (hpratio == 1)) { - unsigned long **blocks[DIRTY_MEMORY_NUM]; - unsigned long idx; - unsigned long offset; - long k; - long nr = BITS_TO_LONGS(pages); - - idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; - offset = BIT_WORD((start >> TARGET_PAGE_BITS) % - DIRTY_MEMORY_BLOCK_SIZE); - - WITH_RCU_READ_LOCK_GUARD() { - for (i = 0; i < DIRTY_MEMORY_NUM; i++) { - blocks[i] = - qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks; - } - - for (k = 0; k < nr; k++) { - if (bitmap[k]) { - unsigned long temp = leul_to_cpu(bitmap[k]); - - nbits = ctpopl(temp); - qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); - - if (global_dirty_tracking) { - qatomic_or( - &blocks[DIRTY_MEMORY_MIGRATION][idx][offset], - temp); - if (unlikely( - global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { - total_dirty_pages += nbits; - } - } - - num_dirty += nbits; - - if (tcg_enabled()) { - qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], - temp); - } - } - - if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { - offset = 0; - idx++; - } - } - } - - if (xen_enabled()) { - xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); - } - } else { - uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; - - if (!global_dirty_tracking) { - clients &= ~(1 << DIRTY_MEMORY_MIGRATION); - } - - /* - * bitmap-traveling is faster than memory-traveling (for addr...) - * especially when most of the memory is not dirty. - */ - for (i = 0; i < len; i++) { - if (bitmap[i] != 0) { - c = leul_to_cpu(bitmap[i]); - nbits = ctpopl(c); - if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { - total_dirty_pages += nbits; - } - num_dirty += nbits; - do { - j = ctzl(c); - c &= ~(1ul << j); - page_number = (i * HOST_LONG_BITS + j) * hpratio; - addr = page_number * TARGET_PAGE_SIZE; - ram_addr = start + addr; - cpu_physical_memory_set_dirty_range(ram_addr, - TARGET_PAGE_SIZE * hpratio, clients); - } while (c != 0); - } - } - } - - return num_dirty; -} + ram_addr_t pages); static inline void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length) diff --git a/system/physmem.c b/system/physmem.c index bcd9d6fb8f..e85552483a 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -1205,6 +1205,109 @@ bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, return false; } +uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, + ram_addr_t start, + ram_addr_t pages) +{ + unsigned long i, j; + unsigned long page_number, c, nbits; + hwaddr addr; + ram_addr_t ram_addr; + uint64_t num_dirty = 0; + unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; + unsigned long hpratio = qemu_real_host_page_size() / TARGET_PAGE_SIZE; + unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); + + /* start address is aligned at the start of a word? */ + if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && + (hpratio == 1)) { + unsigned long **blocks[DIRTY_MEMORY_NUM]; + unsigned long idx; + unsigned long offset; + long k; + long nr = BITS_TO_LONGS(pages); + + idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; + offset = BIT_WORD((start >> TARGET_PAGE_BITS) % + DIRTY_MEMORY_BLOCK_SIZE); + + WITH_RCU_READ_LOCK_GUARD() { + for (i = 0; i < DIRTY_MEMORY_NUM; i++) { + blocks[i] = + qatomic_rcu_read(&ram_list.dirty_memory[i])->blocks; + } + + for (k = 0; k < nr; k++) { + if (bitmap[k]) { + unsigned long temp = leul_to_cpu(bitmap[k]); + + nbits = ctpopl(temp); + qatomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); + + if (global_dirty_tracking) { + qatomic_or( + &blocks[DIRTY_MEMORY_MIGRATION][idx][offset], + temp); + if (unlikely( + global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { + total_dirty_pages += nbits; + } + } + + num_dirty += nbits; + + if (tcg_enabled()) { + qatomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], + temp); + } + } + + if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { + offset = 0; + idx++; + } + } + } + + if (xen_enabled()) { + xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS); + } + } else { + uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL + : DIRTY_CLIENTS_NOCODE; + + if (!global_dirty_tracking) { + clients &= ~(1 << DIRTY_MEMORY_MIGRATION); + } + + /* + * bitmap-traveling is faster than memory-traveling (for addr...) + * especially when most of the memory is not dirty. + */ + for (i = 0; i < len; i++) { + if (bitmap[i] != 0) { + c = leul_to_cpu(bitmap[i]); + nbits = ctpopl(c); + if (unlikely(global_dirty_tracking & GLOBAL_DIRTY_DIRTY_RATE)) { + total_dirty_pages += nbits; + } + num_dirty += nbits; + do { + j = ctzl(c); + c &= ~(1ul << j); + page_number = (i * HOST_LONG_BITS + j) * hpratio; + addr = page_number * TARGET_PAGE_SIZE; + ram_addr = start + addr; + cpu_physical_memory_set_dirty_range(ram_addr, + TARGET_PAGE_SIZE * hpratio, clients); + } while (c != 0); + } + } + } + + return num_dirty; +} + static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end, uint16_t section); static subpage_t *subpage_init(FlatView *fv, hwaddr base); From 57f3d859bf947cdef418dec14445b1f8935f2d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 13:53:07 +0200 Subject: [PATCH 37/41] system/physmem: Un-inline cpu_physical_memory_dirty_bits_cleared() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid maintaining large functions in header, rely on the linker to optimize at linking time. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-15-philmd@linaro.org> --- include/system/ram_addr.h | 10 +--------- system/physmem.c | 7 +++++++ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 49e9a9c66d..54b5f5ec16 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -19,8 +19,6 @@ #ifndef SYSTEM_RAM_ADDR_H #define SYSTEM_RAM_ADDR_H -#include "system/tcg.h" -#include "exec/cputlb.h" #include "exec/ramlist.h" #include "system/ramblock.h" #include "system/memory.h" @@ -164,14 +162,8 @@ uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_addr_t start, ram_addr_t pages); -static inline void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, - ram_addr_t length) -{ - if (tcg_enabled()) { - tlb_reset_dirty_range_all(start, length); - } +void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length); -} bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, unsigned client); diff --git a/system/physmem.c b/system/physmem.c index e85552483a..24b654fb3f 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -899,6 +899,13 @@ void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length) } } +void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length) +{ + if (tcg_enabled()) { + tlb_reset_dirty_range_all(start, length); + } +} + static bool physical_memory_get_dirty(ram_addr_t start, ram_addr_t length, unsigned client) { From 1085a44ebb55346b30888ce1b7b18565a719f78b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 13:55:15 +0200 Subject: [PATCH 38/41] system/physmem: Reduce cpu_physical_memory_clear_dirty_range() scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpu_physical_memory_clear_dirty_range() is now only called within system/physmem.c, by qemu_ram_resize(). Reduce its scope by making it internal to this file. Since it doesn't involve any CPU, remove the 'cpu_' prefix. As it operates on a range, rename @start as @addr. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-16-philmd@linaro.org> --- include/system/ram_addr.h | 9 --------- system/physmem.c | 9 ++++++++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 54b5f5ec16..cafd258580 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -175,15 +175,6 @@ bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, ram_addr_t length); -static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, - ram_addr_t length) -{ - cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); - cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); - cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); -} - - /* Called with RCU critical section */ static inline uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, diff --git a/system/physmem.c b/system/physmem.c index 24b654fb3f..079bdbff30 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -1137,6 +1137,13 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, return dirty; } +static void physical_memory_clear_dirty_range(ram_addr_t addr, ram_addr_t length) +{ + cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_MIGRATION); + cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_VGA); + cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_CODE); +} + DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client) { @@ -2071,7 +2078,7 @@ int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp) ram_block_notify_resize(block->host, oldsize, newsize); } - cpu_physical_memory_clear_dirty_range(block->offset, block->used_length); + physical_memory_clear_dirty_range(block->offset, block->used_length); block->used_length = newsize; cpu_physical_memory_set_dirty_range(block->offset, block->used_length, DIRTY_CLIENTS_ALL); From 8bf3a883088b65430f58aeb3d2d1db87f01eabec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 29 Sep 2025 13:58:03 +0200 Subject: [PATCH 39/41] system/physmem: Reduce cpu_physical_memory_sync_dirty_bitmap() scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cpu_physical_memory_sync_dirty_bitmap() is now only called within system/physmem.c, by ramblock_sync_dirty_bitmap(). Reduce its scope by making it internal to this file. Since it doesn't involve any CPU, remove the 'cpu_' prefix. Remove the now unneeded "qemu/rcu.h" and "system/memory.h" headers. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Message-Id: <20251001175448.18933-17-philmd@linaro.org> --- include/system/ram_addr.h | 79 --------------------------------------- migration/ram.c | 77 +++++++++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 80 deletions(-) diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index cafd258580..d2d088bbea 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -21,10 +21,7 @@ #include "exec/ramlist.h" #include "system/ramblock.h" -#include "system/memory.h" #include "exec/target_page.h" -#include "qemu/rcu.h" - #include "exec/hwaddr.h" extern uint64_t total_dirty_pages; @@ -175,80 +172,4 @@ bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, ram_addr_t length); -/* Called with RCU critical section */ -static inline -uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb, - ram_addr_t start, - ram_addr_t length) -{ - ram_addr_t addr; - unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS); - uint64_t num_dirty = 0; - unsigned long *dest = rb->bmap; - - /* start address and length is aligned at the start of a word? */ - if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) == - (start + rb->offset) && - !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) { - int k; - int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); - unsigned long * const *src; - unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; - unsigned long offset = BIT_WORD((word * BITS_PER_LONG) % - DIRTY_MEMORY_BLOCK_SIZE); - unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); - - src = qatomic_rcu_read( - &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; - - for (k = page; k < page + nr; k++) { - if (src[idx][offset]) { - unsigned long bits = qatomic_xchg(&src[idx][offset], 0); - unsigned long new_dirty; - new_dirty = ~dest[k]; - dest[k] |= bits; - new_dirty &= bits; - num_dirty += ctpopl(new_dirty); - } - - if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { - offset = 0; - idx++; - } - } - if (num_dirty) { - cpu_physical_memory_dirty_bits_cleared(start, length); - } - - if (rb->clear_bmap) { - /* - * Postpone the dirty bitmap clear to the point before we - * really send the pages, also we will split the clear - * dirty procedure into smaller chunks. - */ - clear_bmap_set(rb, start >> TARGET_PAGE_BITS, - length >> TARGET_PAGE_BITS); - } else { - /* Slow path - still do that in a huge chunk */ - memory_region_clear_dirty_bitmap(rb->mr, start, length); - } - } else { - ram_addr_t offset = rb->offset; - - for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { - if (cpu_physical_memory_test_and_clear_dirty( - start + addr + offset, - TARGET_PAGE_SIZE, - DIRTY_MEMORY_MIGRATION)) { - long k = (start + addr) >> TARGET_PAGE_BITS; - if (!test_and_set_bit(k, dest)) { - num_dirty++; - } - } - } - } - - return num_dirty; -} - #endif diff --git a/migration/ram.c b/migration/ram.c index 189931d24d..f7c165dfb3 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -936,11 +936,86 @@ bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start) return false; } +/* Called with RCU critical section */ +static uint64_t physical_memory_sync_dirty_bitmap(RAMBlock *rb, + ram_addr_t start, + ram_addr_t length) +{ + ram_addr_t addr; + unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS); + uint64_t num_dirty = 0; + unsigned long *dest = rb->bmap; + + /* start address and length is aligned at the start of a word? */ + if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) == + (start + rb->offset) && + !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) { + int k; + int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); + unsigned long * const *src; + unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; + unsigned long offset = BIT_WORD((word * BITS_PER_LONG) % + DIRTY_MEMORY_BLOCK_SIZE); + unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); + + src = qatomic_rcu_read( + &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; + + for (k = page; k < page + nr; k++) { + if (src[idx][offset]) { + unsigned long bits = qatomic_xchg(&src[idx][offset], 0); + unsigned long new_dirty; + new_dirty = ~dest[k]; + dest[k] |= bits; + new_dirty &= bits; + num_dirty += ctpopl(new_dirty); + } + + if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { + offset = 0; + idx++; + } + } + if (num_dirty) { + cpu_physical_memory_dirty_bits_cleared(start, length); + } + + if (rb->clear_bmap) { + /* + * Postpone the dirty bitmap clear to the point before we + * really send the pages, also we will split the clear + * dirty procedure into smaller chunks. + */ + clear_bmap_set(rb, start >> TARGET_PAGE_BITS, + length >> TARGET_PAGE_BITS); + } else { + /* Slow path - still do that in a huge chunk */ + memory_region_clear_dirty_bitmap(rb->mr, start, length); + } + } else { + ram_addr_t offset = rb->offset; + + for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { + if (cpu_physical_memory_test_and_clear_dirty( + start + addr + offset, + TARGET_PAGE_SIZE, + DIRTY_MEMORY_MIGRATION)) { + long k = (start + addr) >> TARGET_PAGE_BITS; + if (!test_and_set_bit(k, dest)) { + num_dirty++; + } + } + } + } + + return num_dirty; +} + /* Called with RCU critical section */ static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb) { uint64_t new_dirty_pages = - cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length); + physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length); rs->migration_dirty_pages += new_dirty_pages; rs->num_dirty_pages_period += new_dirty_pages; From aa60bdb700f4afa8577a495c734870516910864f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 30 Sep 2025 09:08:44 +0200 Subject: [PATCH 40/41] system/physmem: Drop 'cpu_' prefix in Physical Memory API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The functions related to the Physical Memory API declared in "system/ram_addr.h" do not operate on vCPU. Remove the 'cpu_' prefix. Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Reviewed-by: Cédric Le Goater Message-Id: <20251001175448.18933-18-philmd@linaro.org> --- accel/kvm/kvm-all.c | 2 +- accel/tcg/cputlb.c | 12 +++++----- hw/vfio/container-legacy.c | 8 +++---- hw/vfio/container.c | 4 ++-- include/system/ram_addr.h | 24 +++++++++---------- migration/ram.c | 4 ++-- system/memory.c | 8 +++---- system/memory_ldst.c.inc | 2 +- system/physmem.c | 48 ++++++++++++++++++------------------- target/arm/tcg/mte_helper.c | 2 +- tests/tsan/ignore.tsan | 4 ++-- 11 files changed, 59 insertions(+), 59 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 08b2b5a371..a7ece7db96 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -758,7 +758,7 @@ static void kvm_slot_sync_dirty_pages(KVMSlot *slot) ram_addr_t start = slot->ram_start_offset; ram_addr_t pages = slot->memory_size / qemu_real_host_page_size(); - cpu_physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages); + physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages); } static void kvm_slot_reset_dirty_pages(KVMSlot *slot) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 2a6aa01c57..a721235dea 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -858,7 +858,7 @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu, can be detected */ void tlb_protect_code(ram_addr_t ram_addr) { - cpu_physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK, + physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK, TARGET_PAGE_SIZE, DIRTY_MEMORY_CODE); } @@ -867,7 +867,7 @@ void tlb_protect_code(ram_addr_t ram_addr) tested for self modifying code */ void tlb_unprotect_code(ram_addr_t ram_addr) { - cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE); + physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE); } @@ -1085,7 +1085,7 @@ void tlb_set_page_full(CPUState *cpu, int mmu_idx, if (prot & PAGE_WRITE) { if (section->readonly) { write_flags |= TLB_DISCARD_WRITE; - } else if (cpu_physical_memory_is_clean(iotlb)) { + } else if (physical_memory_is_clean(iotlb)) { write_flags |= TLB_NOTDIRTY; } } @@ -1341,7 +1341,7 @@ static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size); - if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) { + if (!physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) { tb_invalidate_phys_range_fast(cpu, ram_addr, size, retaddr); } @@ -1349,10 +1349,10 @@ static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size, * Set both VGA and migration bits for simplicity and to remove * the notdirty callback faster. */ - cpu_physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE); + physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE); /* We remove the notdirty callback only if the code has been flushed. */ - if (!cpu_physical_memory_is_clean(ram_addr)) { + if (!physical_memory_is_clean(ram_addr)) { trace_memory_notdirty_set_dirty(mem_vaddr); tlb_set_dirty(cpu, mem_vaddr); } diff --git a/hw/vfio/container-legacy.c b/hw/vfio/container-legacy.c index 629ff23b0b..765be7f024 100644 --- a/hw/vfio/container-legacy.c +++ b/hw/vfio/container-legacy.c @@ -92,7 +92,7 @@ static int vfio_dma_unmap_bitmap(const VFIOLegacyContainer *container, bitmap = (struct vfio_bitmap *)&unmap->data; /* - * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of + * physical_memory_set_dirty_lebitmap() supports pages in bitmap of * qemu_real_host_page_size to mark those dirty. Hence set bitmap_pgsize * to qemu_real_host_page_size. */ @@ -108,7 +108,7 @@ static int vfio_dma_unmap_bitmap(const VFIOLegacyContainer *container, ret = ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, unmap); if (!ret) { - cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, + physical_memory_set_dirty_lebitmap(vbmap.bitmap, iotlb->translated_addr, vbmap.pages); } else { error_report("VFIO_UNMAP_DMA with DIRTY_BITMAP : %m"); @@ -266,7 +266,7 @@ static int vfio_legacy_query_dirty_bitmap(const VFIOContainer *bcontainer, range->size = size; /* - * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of + * physical_memory_set_dirty_lebitmap() supports pages in bitmap of * qemu_real_host_page_size to mark those dirty. Hence set bitmap's pgsize * to qemu_real_host_page_size. */ @@ -485,7 +485,7 @@ static void vfio_get_iommu_info_migration(VFIOLegacyContainer *container, header); /* - * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of + * physical_memory_set_dirty_lebitmap() supports pages in bitmap of * qemu_real_host_page_size to mark those dirty. */ if (cap_mig->pgsize_bitmap & qemu_real_host_page_size()) { diff --git a/hw/vfio/container.c b/hw/vfio/container.c index 41de343924..3fb19a1c8a 100644 --- a/hw/vfio/container.c +++ b/hw/vfio/container.c @@ -255,7 +255,7 @@ int vfio_container_query_dirty_bitmap(const VFIOContainer *bcontainer, int ret; if (!bcontainer->dirty_pages_supported && !all_device_dirty_tracking) { - cpu_physical_memory_set_dirty_range(translated_addr, size, + physical_memory_set_dirty_range(translated_addr, size, tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE); return 0; @@ -280,7 +280,7 @@ int vfio_container_query_dirty_bitmap(const VFIOContainer *bcontainer, goto out; } - dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, + dirty_pages = physical_memory_set_dirty_lebitmap(vbmap.bitmap, translated_addr, vbmap.pages); diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index d2d088bbea..3894a84fb9 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -136,39 +136,39 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) -bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); +bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); -bool cpu_physical_memory_is_clean(ram_addr_t addr); +bool physical_memory_is_clean(ram_addr_t addr); -uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, +uint8_t physical_memory_range_includes_clean(ram_addr_t start, ram_addr_t length, uint8_t mask); -void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); +void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); -void cpu_physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, +void physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, uint8_t mask); /* - * Contrary to cpu_physical_memory_sync_dirty_bitmap() this function returns + * Contrary to physical_memory_sync_dirty_bitmap() this function returns * the number of dirty pages in @bitmap passed as argument. On the other hand, - * cpu_physical_memory_sync_dirty_bitmap() returns newly dirtied pages that + * physical_memory_sync_dirty_bitmap() returns newly dirtied pages that * weren't set in the global migration bitmap. */ -uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, +uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_addr_t start, ram_addr_t pages); -void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length); +void physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length); -bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, +bool physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, unsigned client); -DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty +DirtyBitmapSnapshot *physical_memory_snapshot_and_clear_dirty (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client); -bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, +bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, ram_addr_t length); diff --git a/migration/ram.c b/migration/ram.c index f7c165dfb3..ba01654228 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -977,7 +977,7 @@ static uint64_t physical_memory_sync_dirty_bitmap(RAMBlock *rb, } } if (num_dirty) { - cpu_physical_memory_dirty_bits_cleared(start, length); + physical_memory_dirty_bits_cleared(start, length); } if (rb->clear_bmap) { @@ -996,7 +996,7 @@ static uint64_t physical_memory_sync_dirty_bitmap(RAMBlock *rb, ram_addr_t offset = rb->offset; for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { - if (cpu_physical_memory_test_and_clear_dirty( + if (physical_memory_test_and_clear_dirty( start + addr + offset, TARGET_PAGE_SIZE, DIRTY_MEMORY_MIGRATION)) { diff --git a/system/memory.c b/system/memory.c index 41797ceef4..5c14e56ebb 100644 --- a/system/memory.c +++ b/system/memory.c @@ -2271,7 +2271,7 @@ void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr, hwaddr size) { assert(mr->ram_block); - cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, + physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, size, memory_region_get_dirty_log_mask(mr)); } @@ -2375,7 +2375,7 @@ DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *snapshot; assert(mr->ram_block); memory_region_sync_dirty_bitmap(mr, false); - snapshot = cpu_physical_memory_snapshot_and_clear_dirty(mr, addr, size, client); + snapshot = physical_memory_snapshot_and_clear_dirty(mr, addr, size, client); memory_global_after_dirty_log_sync(); return snapshot; } @@ -2384,7 +2384,7 @@ bool memory_region_snapshot_get_dirty(MemoryRegion *mr, DirtyBitmapSnapshot *sna hwaddr addr, hwaddr size) { assert(mr->ram_block); - return cpu_physical_memory_snapshot_get_dirty(snap, + return physical_memory_snapshot_get_dirty(snap, memory_region_get_ram_addr(mr) + addr, size); } @@ -2422,7 +2422,7 @@ void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr, hwaddr size, unsigned client) { assert(mr->ram_block); - cpu_physical_memory_test_and_clear_dirty( + physical_memory_test_and_clear_dirty( memory_region_get_ram_addr(mr) + addr, size, client); } diff --git a/system/memory_ldst.c.inc b/system/memory_ldst.c.inc index 7f32d3d9ff..333da209d1 100644 --- a/system/memory_ldst.c.inc +++ b/system/memory_ldst.c.inc @@ -287,7 +287,7 @@ void glue(address_space_stl_notdirty, SUFFIX)(ARG1_DECL, dirty_log_mask = memory_region_get_dirty_log_mask(mr); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); - cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, + physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr, 4, dirty_log_mask); r = MEMTX_OK; } diff --git a/system/physmem.c b/system/physmem.c index 079bdbff30..347119b8c2 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -899,7 +899,7 @@ void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length) } } -void cpu_physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length) +void physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length) { if (tcg_enabled()) { tlb_reset_dirty_range_all(start, length); @@ -945,17 +945,17 @@ static bool physical_memory_get_dirty(ram_addr_t start, ram_addr_t length, return dirty; } -bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client) +bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client) { return physical_memory_get_dirty(addr, 1, client); } -bool cpu_physical_memory_is_clean(ram_addr_t addr) +bool physical_memory_is_clean(ram_addr_t addr) { - bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); - bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); + bool vga = physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); + bool code = physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); bool migration = - cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); + physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); return !(vga && code && migration); } @@ -998,7 +998,7 @@ static bool physical_memory_all_dirty(ram_addr_t start, ram_addr_t length, return dirty; } -uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, +uint8_t physical_memory_range_includes_clean(ram_addr_t start, ram_addr_t length, uint8_t mask) { @@ -1019,7 +1019,7 @@ uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, return ret; } -void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) +void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) { unsigned long page, idx, offset; DirtyMemoryBlocks *blocks; @@ -1037,7 +1037,7 @@ void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client) set_bit_atomic(offset, blocks->blocks[idx]); } -void cpu_physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, +void physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, uint8_t mask) { DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; @@ -1089,7 +1089,7 @@ void cpu_physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, } /* Note: start and end must be within the same ram block. */ -bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, +bool physical_memory_test_and_clear_dirty(ram_addr_t start, ram_addr_t length, unsigned client) { @@ -1131,7 +1131,7 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, } if (dirty) { - cpu_physical_memory_dirty_bits_cleared(start, length); + physical_memory_dirty_bits_cleared(start, length); } return dirty; @@ -1139,12 +1139,12 @@ bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, static void physical_memory_clear_dirty_range(ram_addr_t addr, ram_addr_t length) { - cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_MIGRATION); - cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_VGA); - cpu_physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_CODE); + physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_MIGRATION); + physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_VGA); + physical_memory_test_and_clear_dirty(addr, length, DIRTY_MEMORY_CODE); } -DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty +DirtyBitmapSnapshot *physical_memory_snapshot_and_clear_dirty (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client) { DirtyMemoryBlocks *blocks; @@ -1191,14 +1191,14 @@ DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty } } - cpu_physical_memory_dirty_bits_cleared(start, length); + physical_memory_dirty_bits_cleared(start, length); memory_region_clear_dirty_bitmap(mr, offset, length); return snap; } -bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, +bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, ram_addr_t start, ram_addr_t length) { @@ -1219,7 +1219,7 @@ bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, return false; } -uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, +uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap, ram_addr_t start, ram_addr_t pages) { @@ -1312,7 +1312,7 @@ uint64_t cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, page_number = (i * HOST_LONG_BITS + j) * hpratio; addr = page_number * TARGET_PAGE_SIZE; ram_addr = start + addr; - cpu_physical_memory_set_dirty_range(ram_addr, + physical_memory_set_dirty_range(ram_addr, TARGET_PAGE_SIZE * hpratio, clients); } while (c != 0); } @@ -2080,7 +2080,7 @@ int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp) physical_memory_clear_dirty_range(block->offset, block->used_length); block->used_length = newsize; - cpu_physical_memory_set_dirty_range(block->offset, block->used_length, + physical_memory_set_dirty_range(block->offset, block->used_length, DIRTY_CLIENTS_ALL); memory_region_set_size(block->mr, unaligned_size); if (block->resized) { @@ -2285,7 +2285,7 @@ static void ram_block_add(RAMBlock *new_block, Error **errp) ram_list.version++; qemu_mutex_unlock_ramlist(); - cpu_physical_memory_set_dirty_range(new_block->offset, + physical_memory_set_dirty_range(new_block->offset, new_block->used_length, DIRTY_CLIENTS_ALL); @@ -3134,19 +3134,19 @@ static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr, addr += ramaddr; /* No early return if dirty_log_mask is or becomes 0, because - * cpu_physical_memory_set_dirty_range will still call + * physical_memory_set_dirty_range will still call * xen_modified_memory. */ if (dirty_log_mask) { dirty_log_mask = - cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask); + physical_memory_range_includes_clean(addr, length, dirty_log_mask); } if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) { assert(tcg_enabled()); tb_invalidate_phys_range(NULL, addr, addr + length - 1); dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE); } - cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask); + physical_memory_set_dirty_range(addr, length, dirty_log_mask); } void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size) diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c index 7d80244788..077ff4b2b2 100644 --- a/target/arm/tcg/mte_helper.c +++ b/target/arm/tcg/mte_helper.c @@ -189,7 +189,7 @@ uint8_t *allocation_tag_mem_probe(CPUARMState *env, int ptr_mmu_idx, */ if (tag_access == MMU_DATA_STORE) { ram_addr_t tag_ra = memory_region_get_ram_addr(mr) + xlat; - cpu_physical_memory_set_dirty_flag(tag_ra, DIRTY_MEMORY_MIGRATION); + physical_memory_set_dirty_flag(tag_ra, DIRTY_MEMORY_MIGRATION); } return memory_region_get_ram_ptr(mr) + xlat; diff --git a/tests/tsan/ignore.tsan b/tests/tsan/ignore.tsan index 423e482d2f..8fa00a2c49 100644 --- a/tests/tsan/ignore.tsan +++ b/tests/tsan/ignore.tsan @@ -4,7 +4,7 @@ # The eventual goal would be to fix these warnings. # TSan is not happy about setting/getting of dirty bits, -# for example, cpu_physical_memory_set_dirty_range, -# and cpu_physical_memory_get_dirty. +# for example, physical_memory_set_dirty_range, +# and physical_memory_get_dirty. src:bitops.c src:bitmap.c From 4db362f68c7cd28a6b1aa2f9ada68aa3556f9613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 30 Sep 2025 09:08:54 +0200 Subject: [PATCH 41/41] system/physmem: Extract API out of 'system/ram_addr.h' header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Very few files use the Physical Memory API. Declare its methods in their own header: "system/physmem.h". Signed-off-by: Philippe Mathieu-Daudé Reviewed-by: Richard Henderson Reviewed-by: Cédric Le Goater Message-Id: <20251001175448.18933-19-philmd@linaro.org> --- MAINTAINERS | 1 + accel/kvm/kvm-all.c | 2 +- accel/tcg/cputlb.c | 1 + hw/vfio/container-legacy.c | 2 +- hw/vfio/container.c | 1 + hw/vfio/listener.c | 1 - include/system/physmem.h | 54 +++++++++++++++++++++++++++++++++++++ include/system/ram_addr.h | 40 --------------------------- migration/ram.c | 1 + system/memory.c | 1 + system/physmem.c | 1 + target/arm/tcg/mte_helper.c | 2 +- 12 files changed, 63 insertions(+), 44 deletions(-) create mode 100644 include/system/physmem.h diff --git a/MAINTAINERS b/MAINTAINERS index 75e1fa5c30..43d8d0bab5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3223,6 +3223,7 @@ S: Supported F: include/system/ioport.h F: include/exec/memop.h F: include/system/memory.h +F: include/system/physmem.h F: include/system/ram_addr.h F: include/system/ramblock.h F: include/system/memory_mapping.h diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index a7ece7db96..58802f7c3c 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -32,13 +32,13 @@ #include "system/runstate.h" #include "system/cpus.h" #include "system/accel-blocker.h" +#include "system/physmem.h" #include "system/ramblock.h" #include "accel/accel-ops.h" #include "qemu/bswap.h" #include "exec/tswap.h" #include "exec/target_page.h" #include "system/memory.h" -#include "system/ram_addr.h" #include "qemu/event_notifier.h" #include "qemu/main-loop.h" #include "trace.h" diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index a721235dea..7214d41cb5 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -25,6 +25,7 @@ #include "accel/tcg/probe.h" #include "exec/page-protection.h" #include "system/memory.h" +#include "system/physmem.h" #include "accel/tcg/cpu-ldst-common.h" #include "accel/tcg/cpu-mmu-index.h" #include "exec/cputlb.h" diff --git a/hw/vfio/container-legacy.c b/hw/vfio/container-legacy.c index 765be7f024..a3615d7b5d 100644 --- a/hw/vfio/container-legacy.c +++ b/hw/vfio/container-legacy.c @@ -25,7 +25,7 @@ #include "hw/vfio/vfio-device.h" #include "system/address-spaces.h" #include "system/memory.h" -#include "system/ram_addr.h" +#include "system/physmem.h" #include "qemu/error-report.h" #include "qemu/range.h" #include "system/reset.h" diff --git a/hw/vfio/container.c b/hw/vfio/container.c index 3fb19a1c8a..9ddec300e3 100644 --- a/hw/vfio/container.c +++ b/hw/vfio/container.c @@ -20,6 +20,7 @@ #include "qemu/error-report.h" #include "hw/vfio/vfio-container.h" #include "hw/vfio/vfio-device.h" /* vfio_device_reset_handler */ +#include "system/physmem.h" #include "system/reset.h" #include "vfio-helpers.h" diff --git a/hw/vfio/listener.c b/hw/vfio/listener.c index b5cefc9395..c6bb58f520 100644 --- a/hw/vfio/listener.c +++ b/hw/vfio/listener.c @@ -30,7 +30,6 @@ #include "hw/vfio/pci.h" #include "system/address-spaces.h" #include "system/memory.h" -#include "system/ram_addr.h" #include "hw/hw.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" diff --git a/include/system/physmem.h b/include/system/physmem.h new file mode 100644 index 0000000000..879f6eae38 --- /dev/null +++ b/include/system/physmem.h @@ -0,0 +1,54 @@ +/* + * QEMU physical memory interfaces (target independent). + * + * Copyright (c) 2003 Fabrice Bellard + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ +#ifndef QEMU_SYSTEM_PHYSMEM_H +#define QEMU_SYSTEM_PHYSMEM_H + +#include "exec/hwaddr.h" +#include "exec/ramlist.h" + +#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) +#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) + +bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); + +bool physical_memory_is_clean(ram_addr_t addr); + +uint8_t physical_memory_range_includes_clean(ram_addr_t start, + ram_addr_t length, + uint8_t mask); + +void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); + +void physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, + uint8_t mask); + +/* + * Contrary to physical_memory_sync_dirty_bitmap() this function returns + * the number of dirty pages in @bitmap passed as argument. On the other hand, + * physical_memory_sync_dirty_bitmap() returns newly dirtied pages that + * weren't set in the global migration bitmap. + */ +uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap, + ram_addr_t start, + ram_addr_t pages); + +void physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length); + +bool physical_memory_test_and_clear_dirty(ram_addr_t start, + ram_addr_t length, + unsigned client); + +DirtyBitmapSnapshot * +physical_memory_snapshot_and_clear_dirty(MemoryRegion *mr, hwaddr offset, + hwaddr length, unsigned client); + +bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, + ram_addr_t start, + ram_addr_t length); + +#endif diff --git a/include/system/ram_addr.h b/include/system/ram_addr.h index 3894a84fb9..683485980c 100644 --- a/include/system/ram_addr.h +++ b/include/system/ram_addr.h @@ -19,7 +19,6 @@ #ifndef SYSTEM_RAM_ADDR_H #define SYSTEM_RAM_ADDR_H -#include "exec/ramlist.h" #include "system/ramblock.h" #include "exec/target_page.h" #include "exec/hwaddr.h" @@ -133,43 +132,4 @@ static inline void qemu_ram_block_writeback(RAMBlock *block) qemu_ram_msync(block, 0, block->used_length); } -#define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) -#define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) - -bool physical_memory_get_dirty_flag(ram_addr_t addr, unsigned client); - -bool physical_memory_is_clean(ram_addr_t addr); - -uint8_t physical_memory_range_includes_clean(ram_addr_t start, - ram_addr_t length, - uint8_t mask); - -void physical_memory_set_dirty_flag(ram_addr_t addr, unsigned client); - -void physical_memory_set_dirty_range(ram_addr_t start, ram_addr_t length, - uint8_t mask); - -/* - * Contrary to physical_memory_sync_dirty_bitmap() this function returns - * the number of dirty pages in @bitmap passed as argument. On the other hand, - * physical_memory_sync_dirty_bitmap() returns newly dirtied pages that - * weren't set in the global migration bitmap. - */ -uint64_t physical_memory_set_dirty_lebitmap(unsigned long *bitmap, - ram_addr_t start, - ram_addr_t pages); - -void physical_memory_dirty_bits_cleared(ram_addr_t start, ram_addr_t length); - -bool physical_memory_test_and_clear_dirty(ram_addr_t start, - ram_addr_t length, - unsigned client); - -DirtyBitmapSnapshot *physical_memory_snapshot_and_clear_dirty - (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client); - -bool physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap, - ram_addr_t start, - ram_addr_t length); - #endif diff --git a/migration/ram.c b/migration/ram.c index ba01654228..5eef2efc78 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -53,6 +53,7 @@ #include "qemu/rcu_queue.h" #include "migration/colo.h" #include "system/cpu-throttle.h" +#include "system/physmem.h" #include "system/ramblock.h" #include "savevm.h" #include "qemu/iov.h" diff --git a/system/memory.c b/system/memory.c index 5c14e56ebb..8b84661ae3 100644 --- a/system/memory.c +++ b/system/memory.c @@ -25,6 +25,7 @@ #include "qemu/target-info.h" #include "qom/object.h" #include "trace.h" +#include "system/physmem.h" #include "system/ram_addr.h" #include "system/kvm.h" #include "system/runstate.h" diff --git a/system/physmem.c b/system/physmem.c index 347119b8c2..a340ca3e61 100644 --- a/system/physmem.c +++ b/system/physmem.c @@ -43,6 +43,7 @@ #include "system/kvm.h" #include "system/tcg.h" #include "system/qtest.h" +#include "system/physmem.h" #include "system/ramblock.h" #include "qemu/timer.h" #include "qemu/config-file.h" diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c index 077ff4b2b2..b96c953f80 100644 --- a/target/arm/tcg/mte_helper.c +++ b/target/arm/tcg/mte_helper.c @@ -27,7 +27,7 @@ #include "user/cpu_loop.h" #include "user/page-protection.h" #else -#include "system/ram_addr.h" +#include "system/physmem.h" #endif #include "accel/tcg/cpu-ldst.h" #include "accel/tcg/probe.h"