From 8eb6d39e22a2f9ea1a415d95d3c3ac5d5752b6d7 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 11 Jul 2025 08:45:01 -0700 Subject: [PATCH 1/6] qom: qom-list-get MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using qom-list and qom-get to get all the nodes and property values in a QOM tree can take multiple seconds because it requires 1000's of individual QOM requests. Some managers fetch the entire tree or a large subset of it when starting a new VM, and this cost is a substantial fraction of start up time. Define the qom-list-get command, which fetches all the properties and values for a list of paths. This can be much faster than qom-list plus qom-get. When getting an entire QOM tree, I measured a 10x speedup in elapsed time. Signed-off-by: Steve Sistare Tested-by: Philippe Mathieu-Daudé Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Markus Armbruster Message-ID: <1752248703-217318-2-git-send-email-steven.sistare@oracle.com> Signed-off-by: Markus Armbruster --- qapi/qom.json | 50 +++++++++++++++++++++++++++++++++++++++++++ qom/qom-qmp-cmds.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/qapi/qom.json b/qapi/qom.json index 96d56df6cd..830cb2ffe7 100644 --- a/qapi/qom.json +++ b/qapi/qom.json @@ -47,6 +47,34 @@ '*description': 'str', '*default-value': 'any' } } +## +# @ObjectPropertyValue: +# +# @name: the name of the property. +# +# @type: the type of the property, as described in `ObjectPropertyInfo`. +# +# @value: the value of the property. Absent when the property cannot +# be read. +# +# Since 10.1 +## +{ 'struct': 'ObjectPropertyValue', + 'data': { 'name': 'str', + 'type': 'str', + '*value': 'any' } } + +## +# @ObjectPropertiesValues: +# +# @properties: a list of properties. +# +# Since 10.1 +## +{ 'struct': 'ObjectPropertiesValues', + 'data': { 'properties': [ 'ObjectPropertyValue' ] }} + + ## # @qom-list: # @@ -124,6 +152,28 @@ 'returns': 'any', 'allow-preconfig': true } +## +# @qom-list-get: +# +# List properties and their values for each object path in the input +# list. +# +# @paths: The absolute or partial path for each object, as described +# in `qom-get`. +# +# Errors: +# - If any path is not valid or is ambiguous +# +# Returns: A list where each element is the result for the +# corresponding element of @paths. +# +# Since 10.1 +## +{ 'command': 'qom-list-get', + 'data': { 'paths': [ 'str' ] }, + 'returns': [ 'ObjectPropertiesValues' ], + 'allow-preconfig': true } + ## # @qom-set: # diff --git a/qom/qom-qmp-cmds.c b/qom/qom-qmp-cmds.c index 293755f409..57f1898cf6 100644 --- a/qom/qom-qmp-cmds.c +++ b/qom/qom-qmp-cmds.c @@ -69,6 +69,59 @@ ObjectPropertyInfoList *qmp_qom_list(const char *path, Error **errp) return props; } +static void qom_list_add_property_value(Object *obj, ObjectProperty *prop, + ObjectPropertyValueList **props) +{ + ObjectPropertyValue *item = g_new0(ObjectPropertyValue, 1); + + QAPI_LIST_PREPEND(*props, item); + + item->name = g_strdup(prop->name); + item->type = g_strdup(prop->type); + item->value = object_property_get_qobject(obj, prop->name, NULL); +} + +static ObjectPropertyValueList *qom_get_property_value_list(const char *path, + Error **errp) +{ + Object *obj; + ObjectProperty *prop; + ObjectPropertyIterator iter; + ObjectPropertyValueList *props = NULL; + + obj = qom_resolve_path(path, errp); + if (obj == NULL) { + return NULL; + } + + object_property_iter_init(&iter, obj); + while ((prop = object_property_iter_next(&iter))) { + qom_list_add_property_value(obj, prop, &props); + } + + return props; +} + +ObjectPropertiesValuesList *qmp_qom_list_get(strList *paths, Error **errp) +{ + ObjectPropertiesValuesList *head = NULL, **tail = &head; + strList *path; + + for (path = paths; path; path = path->next) { + ObjectPropertiesValues *item = g_new0(ObjectPropertiesValues, 1); + + QAPI_LIST_APPEND(tail, item); + + item->properties = qom_get_property_value_list(path->value, errp); + if (!item->properties) { + qapi_free_ObjectPropertiesValuesList(head); + return NULL; + } + } + + return head; +} + void qmp_qom_set(const char *path, const char *property, QObject *value, Error **errp) { From 4ece9b61c90ce8cd5edb28ce1ba9e14992382fb0 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 11 Jul 2025 08:45:02 -0700 Subject: [PATCH 2/6] python: use qom-list-get Use qom-list-get to speed up the qom-tree command. Signed-off-by: Steve Sistare Acked-by: Markus Armbruster Message-ID: <1752248703-217318-3-git-send-email-steven.sistare@oracle.com> Tested-by: Markus Armbruster [Lint picked off to mollify make check-minreqs] Signed-off-by: Markus Armbruster --- python/qemu/utils/qom.py | 45 +++++++++++++++------------ python/qemu/utils/qom_common.py | 55 +++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 20 deletions(-) diff --git a/python/qemu/utils/qom.py b/python/qemu/utils/qom.py index 426a0f245f..05e5f14179 100644 --- a/python/qemu/utils/qom.py +++ b/python/qemu/utils/qom.py @@ -31,8 +31,7 @@ QOM commands: ## import argparse - -from qemu.qmp import ExecuteError +from typing import List from .qom_common import QOMCommand @@ -224,28 +223,34 @@ class QOMTree(QOMCommand): super().__init__(args) self.path = args.path - def _list_node(self, path: str) -> None: - print(path) - items = self.qom_list(path) - for item in items: - if item.child: - continue - try: - rsp = self.qmp.cmd('qom-get', path=path, - property=item.name) - print(f" {item.name}: {rsp} ({item.type})") - except ExecuteError as err: - print(f" {item.name}: ({item.type})") - print('') - for item in items: - if not item.child: - continue + def _list_nodes(self, paths: List[str]) -> None: + all_paths_props = self.qom_list_get(paths) + i = 0 + + for props in all_paths_props: + path = paths[i] + i = i + 1 + print(path) if path == '/': path = '' - self._list_node(f"{path}/{item.name}") + newpaths = [] + + for item in props.properties: + if item.child: + newpaths += [f"{path}/{item.name}"] + else: + value = item.value + if value is None: + value = "" + print(f" {item.name}: {value} ({item.type})") + + print('') + + if newpaths: + self._list_nodes(newpaths) def run(self) -> int: - self._list_node(self.path) + self._list_nodes([self.path]) return 0 diff --git a/python/qemu/utils/qom_common.py b/python/qemu/utils/qom_common.py index dd2c8b1908..ab21a4d364 100644 --- a/python/qemu/utils/qom_common.py +++ b/python/qemu/utils/qom_common.py @@ -65,6 +65,52 @@ class ObjectPropertyInfo: return self.type.startswith('link<') +class ObjectPropertyValue: + """ + Represents a property return from e.g. qom-tree-get + """ + def __init__(self, name: str, type_: str, value: object): + self.name = name + self.type = type_ + self.value = value + + @classmethod + def make(cls, value: Dict[str, Any]) -> 'ObjectPropertyValue': + """ + Build an ObjectPropertyValue from a Dict with an unknown shape. + """ + assert value.keys() >= {'name', 'type'} + assert value.keys() <= {'name', 'type', 'value'} + return cls(value['name'], value['type'], value.get('value')) + + @property + def child(self) -> bool: + """Is this property a child property?""" + return self.type.startswith('child<') + + +class ObjectPropertiesValues: + """ + Represents the return type from e.g. qom-list-get + """ + # pylint: disable=too-few-public-methods + + def __init__(self, properties: List[ObjectPropertyValue]) -> None: + self.properties = properties + + @classmethod + def make(cls, value: Dict[str, Any]) -> 'ObjectPropertiesValues': + """ + Build an ObjectPropertiesValues from a Dict with an unknown shape. + """ + assert value.keys() == {'properties'} + props = [ObjectPropertyValue(item['name'], + item['type'], + item.get('value')) + for item in value['properties']] + return cls(props) + + CommandT = TypeVar('CommandT', bound='QOMCommand') @@ -145,6 +191,15 @@ class QOMCommand: assert isinstance(rsp, list) return [ObjectPropertyInfo.make(x) for x in rsp] + def qom_list_get(self, paths: List[str]) -> List[ObjectPropertiesValues]: + """ + :return: a strongly typed list from the 'qom-list-get' command. + """ + rsp = self.qmp.cmd('qom-list-get', paths=paths) + # qom-list-get returns List[ObjectPropertiesValues] + assert isinstance(rsp, list) + return [ObjectPropertiesValues.make(x) for x in rsp] + @classmethod def command_runner( cls: Type[CommandT], From 3dd93992ffbd86a520e5f887d17179fd00ff0928 Mon Sep 17 00:00:00 2001 From: Steve Sistare Date: Fri, 11 Jul 2025 08:45:03 -0700 Subject: [PATCH 3/6] tests/qtest/qom-test: unit test for qom-list-get MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a unit test for qom-list-get. Signed-off-by: Steve Sistare Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Markus Armbruster Message-ID: <1752248703-217318-4-git-send-email-steven.sistare@oracle.com> Signed-off-by: Markus Armbruster --- tests/qtest/qom-test.c | 116 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 1 deletion(-) diff --git a/tests/qtest/qom-test.c b/tests/qtest/qom-test.c index 27d70bc11c..4ade1c728c 100644 --- a/tests/qtest/qom-test.c +++ b/tests/qtest/qom-test.c @@ -11,11 +11,119 @@ #include "qobject/qdict.h" #include "qobject/qlist.h" +#include "qobject/qstring.h" #include "qemu/cutils.h" #include "libqtest.h" +#define RAM_NAME "node0" +#define RAM_SIZE 65536 + static int verbosity_level; +/* + * Verify that the /object/RAM_NAME 'size' property is RAM_SIZE. + */ +static void test_list_get_value(QTestState *qts) +{ + QDict *args = qdict_new(); + g_autoptr(QDict) response = NULL; + g_autoptr(QList) paths = qlist_new(); + QListEntry *entry, *prop_entry; + const char *prop_name; + QList *properties, *return_list; + QDict *obj; + + qlist_append_str(paths, "/objects/" RAM_NAME); + qdict_put_obj(args, "paths", QOBJECT(qlist_copy(paths))); + response = qtest_qmp(qts, "{ 'execute': 'qom-list-get'," + " 'arguments': %p }", args); + g_assert(response); + g_assert(qdict_haskey(response, "return")); + return_list = qobject_to(QList, qdict_get(response, "return")); + + entry = QTAILQ_FIRST(&return_list->head); + obj = qobject_to(QDict, qlist_entry_obj(entry)); + g_assert(qdict_haskey(obj, "properties")); + properties = qobject_to(QList, qdict_get(obj, "properties")); + + QLIST_FOREACH_ENTRY(properties, prop_entry) { + QDict *prop = qobject_to(QDict, qlist_entry_obj(prop_entry)); + + g_assert(qdict_haskey(prop, "name")); + g_assert(qdict_haskey(prop, "value")); + + prop_name = qdict_get_str(prop, "name"); + if (!strcmp(prop_name, "type")) { + g_assert_cmpstr(qdict_get_str(prop, "value"), ==, + "memory-backend-ram"); + + } else if (!strcmp(prop_name, "size")) { + g_assert_cmpint(qdict_get_int(prop, "value"), ==, RAM_SIZE); + } + } +} + +static void test_list_get(QTestState *qts, QList *paths) +{ + QListEntry *entry, *prop_entry, *path_entry; + g_autoptr(QDict) response = NULL; + QDict *args = qdict_new(); + QDict *prop; + QList *return_list; + + if (verbosity_level >= 2) { + g_test_message("Obtaining properties for paths:"); + QLIST_FOREACH_ENTRY(paths, path_entry) { + QString *qstr = qobject_to(QString, qlist_entry_obj(path_entry)); + g_test_message(" %s", qstring_get_str(qstr)); + } + } + + qdict_put_obj(args, "paths", QOBJECT(qlist_copy(paths))); + response = qtest_qmp(qts, "{ 'execute': 'qom-list-get'," + " 'arguments': %p }", args); + g_assert(response); + g_assert(qdict_haskey(response, "return")); + return_list = qobject_to(QList, qdict_get(response, "return")); + g_assert(!qlist_empty(return_list)); + + path_entry = QTAILQ_FIRST(&paths->head); + QLIST_FOREACH_ENTRY(return_list, entry) { + QDict *obj = qobject_to(QDict, qlist_entry_obj(entry)); + g_assert(qdict_haskey(obj, "properties")); + QList *properties = qobject_to(QList, qdict_get(obj, "properties")); + bool has_child = false; + + QLIST_FOREACH_ENTRY(properties, prop_entry) { + prop = qobject_to(QDict, qlist_entry_obj(prop_entry)); + g_assert(qdict_haskey(prop, "name")); + g_assert(qdict_haskey(prop, "type")); + has_child |= strstart(qdict_get_str(prop, "type"), "child<", NULL); + } + + if (has_child) { + /* build a list of child paths */ + QString *qstr = qobject_to(QString, qlist_entry_obj(path_entry)); + const char *path = qstring_get_str(qstr); + g_autoptr(QList) child_paths = qlist_new(); + + QLIST_FOREACH_ENTRY(properties, prop_entry) { + prop = qobject_to(QDict, qlist_entry_obj(prop_entry)); + if (strstart(qdict_get_str(prop, "type"), "child<", NULL)) { + g_autofree char *child_path = g_strdup_printf( + "%s/%s", path, qdict_get_str(prop, "name")); + qlist_append_str(child_paths, child_path); + } + } + + /* fetch props for all children with one qom-list-get call */ + test_list_get(qts, child_paths); + } + + path_entry = QTAILQ_NEXT(path_entry, next); + } +} + static void test_properties(QTestState *qts, const char *path, bool recurse) { char *child_path; @@ -85,8 +193,10 @@ static void test_machine(gconstpointer data) const char *machine = data; QDict *response; QTestState *qts; + g_autoptr(QList) paths = qlist_new(); - qts = qtest_initf("-machine %s", machine); + qts = qtest_initf("-machine %s -object memory-backend-ram,id=%s,size=%d", + machine, RAM_NAME, RAM_SIZE); if (g_test_slow()) { /* Make sure we can get the machine class properties: */ @@ -101,6 +211,10 @@ static void test_machine(gconstpointer data) test_properties(qts, "/machine", true); + qlist_append_str(paths, "/machine"); + test_list_get(qts, paths); + test_list_get_value(qts); + response = qtest_qmp(qts, "{ 'execute': 'quit' }"); g_assert(qdict_haskey(response, "return")); qobject_unref(response); From 5dd68db9cd005aeb2870d8754fd90e1b6af793a7 Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Tue, 15 Jul 2025 17:28:47 -0400 Subject: [PATCH 4/6] docs: Bump sphinx to 6.2.1 sphinx 5.3.0 fails with Python 3.13.1: ../docs/meson.build:37: WARNING: /home/me/qemu/build/pyvenv/bin/sphinx-build: Extension error: Could not import extension sphinx.builders.epub3 (exception: No module named 'imghdr') ../docs/meson.build:39:6: ERROR: Problem encountered: Install a Python 3 version of python-sphinx and the readthedoc theme Bump sphinx to 6.2.1 and also sphinx_rtd_theme as required for the new sphinx version. (jsnow note: this patch bumps the recommended version for Sphinx to install when it is missing, but allows old versions to be used if they are present and functional. The version used for building docs on readthedocs is pinned to the recommended version, 6.2.1.) Signed-off-by: Akihiko Odaki Tested-by: Markus Armbruster Reviewed-by: John Snow Signed-off-by: John Snow Message-ID: <20250715212848.171879-2-jsnow@redhat.com> Reviewed-by: Thomas Huth Signed-off-by: Markus Armbruster --- docs/requirements.txt | 4 ++-- pythondeps.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 02583f209a..87f7afcb2e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ # Used by readthedocs.io # Should be in sync with the "installed" key of pythondeps.toml -sphinx==5.3.0 -sphinx_rtd_theme==1.1.1 +sphinx==6.2.1 +sphinx_rtd_theme==1.2.2 diff --git a/pythondeps.toml b/pythondeps.toml index 7884ab521d..b2eec940ce 100644 --- a/pythondeps.toml +++ b/pythondeps.toml @@ -24,8 +24,8 @@ pycotap = { accepted = ">=1.1.0", installed = "1.3.1" } [docs] # Please keep the installed versions in sync with docs/requirements.txt -sphinx = { accepted = ">=3.4.3", installed = "5.3.0", canary = "sphinx-build" } -sphinx_rtd_theme = { accepted = ">=0.5", installed = "1.1.1" } +sphinx = { accepted = ">=3.4.3", installed = "6.2.1", canary = "sphinx-build" } +sphinx_rtd_theme = { accepted = ">=0.5", installed = "1.2.2" } [testdeps] qemu.qmp = { accepted = ">=0.0.3", installed = "0.0.3" } From b417dd98f442a11f8a7fa79ae7e78279a42483cb Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Tue, 15 Jul 2025 17:28:48 -0400 Subject: [PATCH 5/6] MAINTAINERS: Add docs/requirements.txt Add docs/requirements.txt to "Sphinx documentation configuration and build machinery". Signed-off-by: Akihiko Odaki Signed-off-by: John Snow Message-ID: <20250715212848.171879-3-jsnow@redhat.com> Reviewed-by: Thomas Huth Signed-off-by: Markus Armbruster --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index bc0af3e8dd..a462345618 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4428,6 +4428,7 @@ M: Peter Maydell S: Maintained F: docs/conf.py F: docs/*/conf.py +F: docs/requirements.txt F: docs/sphinx/ F: docs/_templates/ F: docs/devel/docs.rst From 64e4375b2b1efb634876a119a7378c50be5d195b Mon Sep 17 00:00:00 2001 From: John Snow Date: Tue, 15 Jul 2025 18:25:48 -0400 Subject: [PATCH 6/6] python: fix editable installs for modern pip/setuptools The way editable installs work has changed at some point since Fedora 40 was released. Generally, we should be opting to use pyproject.toml installs (PEP517/518) - but those are not fully supported until v61 of setuptools, and CentOS Stream 9 ships v53. Until that time, we can make use of a transitional feature in pip/setuptools to use "legacy" editable installs, which is enough to fix "make check-dev" on modern local workstations for now. By using the environment variable approach to configure pip, we avoid any problems for older versions of pip that don't recognize this option, so it's harmless. The config-settings option first appeared in v23 of pip. editable_mode was first supported by setuptools in v64. (I'm not currently precisely aware of when the default behavior of '-e' switched away from 'compat', but it appears to be a joint effect between setuptools and pip versions.) Version information for supported build platforms: distro python3 pip setuptools sphinx -------------------------------------------------------- centos_stream_9 3.9.23 21.3.1 53.0.0 3.4.3 ubuntu_22_04 3.10.12 22.0.2 59.6.0 4.3.2 ** pyproject.toml installs supported as of here ** freebsd 3.11.13 23.3.2 63.1.0 5.3.0 debian_12 3.11.2 23.0.1 66.1.1 5.3.0 ubuntu_24_04 3.12.3 24.0 68.1.2 7.2.6 centos_stream_10 3.12.11 23.3.2 69.0.3 7.2.6 fedora_41 3.13.5 24.2 69.2.0 7.3.7 alpine_3_19 3.11.13 23.3.1 70.3.0 6.2.1 alpine_3_20 3.12.11 24.0 70.3.0 7.2.6 alpine_3_21 3.12.11 24.3.1 70.3.0 8.1.3 ubuntu_24_10 3.12.7 24.2 74.1.2 7.4.7 fedora_42 3.13.5 24.3.1 74.1.3 8.1.3 ubuntu_25_04 3.13.3 25.0 75.8.0 8.1.3 macports 3.13.5 25.1.1 78.1.1 8.2.3 openbsd 3.12.11 25.1.1 79.0.1 8.2.3 alpine_3_22 3.12.11 25.1.1 80.9.0 8.2.3 homebrew 3.13.5 --- 80.9.0 8.2.3 pkgsrc_current 3.12.11 25.1.1 80.9.0 8.2.3 Signed-off-by: John Snow Message-ID: <20250715222548.198888-1-jsnow@redhat.com> Tested-by: Markus Armbruster Signed-off-by: Markus Armbruster --- python/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/Makefile b/python/Makefile index 764b79ccb2..32aedce413 100644 --- a/python/Makefile +++ b/python/Makefile @@ -68,7 +68,7 @@ $(QEMU_MINVENV_DIR) $(QEMU_MINVENV_DIR)/bin/activate: setup.cfg tests/minreqs.tx echo "INSTALL -r tests/minreqs.txt $(QEMU_MINVENV_DIR)";\ $(PIP_INSTALL) -r tests/minreqs.txt 1>/dev/null; \ echo "INSTALL -e qemu $(QEMU_MINVENV_DIR)"; \ - $(PIP_INSTALL) -e . 1>/dev/null; \ + PIP_CONFIG_SETTINGS="editable_mode=compat" $(PIP_INSTALL) -e . 1>/dev/null; \ ) @touch $(QEMU_MINVENV_DIR) @@ -103,7 +103,7 @@ check-dev: dev-venv .PHONY: develop develop: - $(PIP_INSTALL) -e .[devel] + PIP_CONFIG_SETTINGS="editable_mode=compat" $(PIP_INSTALL) -e .[devel] .PHONY: check check: