Version in base suite: 21.1.0-3 Version in overlay suite: 21.1.0-3+deb12u1 Base version: ironic_21.1.0-3+deb12u1 Target version: ironic_21.4.4-0+deb12u1 Base file: /srv/ftp-master.debian.org/ftp/pool/main/i/ironic/ironic_21.1.0-3+deb12u1.dsc Target file: /srv/ftp-master.debian.org/policy/pool/main/i/ironic/ironic_21.4.4-0+deb12u1.dsc .gitreview | 1 api-ref/source/baremetal-api-v1-nodes-inventory.inc | 44 api-ref/source/baremetal-api-v1-nodes.inc | 26 api-ref/source/baremetal-api-v1-ports.inc | 5 api-ref/source/baremetal-api-v1-shards.inc | 56 api-ref/source/index.rst | 2 api-ref/source/parameters.yaml | 57 api-ref/source/samples/node-inventory-response.json | 31 api-ref/source/samples/shards-list-response.json | 12 debian/changelog | 38 debian/control | 4 debian/patches/CVE-2025-44021_OSSA-2025-001_Disallow_unsafe_image_file_paths.patch | 15 debian/patches/CVE-2026-44916_Use_sandbox_rendering_for_jinja2.patch | 33 debian/patches/CVE-2026-44917_disable-driver_info-level-pxe_template-override.patch | 200 debian/patches/CVE-2026-44919_move_file_url_validation_up_into_deploy_utils_main_path.patch | 198 debian/patches/CVE-2026-46447_Sanitize-kernel_append_parms.patch | 4328 ++++++++++ debian/patches/CVE-2026-48681-directory_transversal_ISO9660_support.patch | 388 debian/patches/fix-initial_grub_cfg.template.patch | 16 debian/patches/py3.11_fix_unit_tests.patch | 70 debian/patches/series | 6 debian/rules | 12 debian/tests/unittests | 2 devstack/lib/ironic | 54 devstack/plugin.sh | 3 devstack/settings | 11 devstack/tools/ironic/scripts/cirros-partition.sh | 2 devstack/upgrade/upgrade.sh | 2 doc/source/admin/anaconda-deploy-interface.rst | 40 doc/source/admin/drivers/ibmc.rst | 2 doc/source/admin/drivers/ilo.rst | 8 doc/source/admin/drivers/irmc.rst | 70 doc/source/admin/drivers/redfish.rst | 31 doc/source/admin/fast-track.rst | 13 doc/source/admin/hardware-burn-in.rst | 7 doc/source/admin/interfaces/deploy.rst | 40 doc/source/admin/metrics.rst | 34 doc/source/admin/retirement.rst | 21 doc/source/admin/secure-rbac.rst | 27 doc/source/admin/security.rst | 101 doc/source/admin/troubleshooting.rst | 158 doc/source/contributor/dev-quickstart.rst | 7 doc/source/contributor/ironic-boot-from-volume.rst | 3 doc/source/contributor/releasing.rst | 52 doc/source/contributor/webapi-version-history.rst | 18 doc/source/install/configure-glance-images.rst | 22 doc/source/install/include/common-prerequisites.inc | 10 doc/source/install/refarch/common.rst | 5 doc/source/user/creating-images.rst | 33 driver-requirements.txt | 7 ironic/api/controllers/v1/__init__.py | 14 ironic/api/controllers/v1/allocation.py | 13 ironic/api/controllers/v1/node.py | 82 ironic/api/controllers/v1/port.py | 73 ironic/api/controllers/v1/portgroup.py | 4 ironic/api/controllers/v1/ramdisk.py | 8 ironic/api/controllers/v1/shard.py | 59 ironic/api/controllers/v1/utils.py | 25 ironic/api/controllers/v1/versions.py | 6 ironic/cmd/status.py | 41 ironic/common/checksum_utils.py | 250 ironic/common/cinder.py | 71 ironic/common/context.py | 7 ironic/common/exception.py | 39 ironic/common/glance_service/image_service.py | 3 ironic/common/image_format_inspector.py | 1038 ++ ironic/common/image_service.py | 80 ironic/common/images.py | 223 ironic/common/keystone.py | 24 ironic/common/neutron.py | 6 ironic/common/policy.py | 109 ironic/common/pxe_utils.py | 59 ironic/common/qemu_img.py | 89 ironic/common/release_mappings.py | 71 ironic/common/rpc.py | 7 ironic/common/rpc_service.py | 27 ironic/common/states.py | 3 ironic/common/swift.py | 42 ironic/common/utils.py | 23 ironic/conductor/base_manager.py | 21 ironic/conductor/cleaning.py | 31 ironic/conductor/manager.py | 115 ironic/conductor/periodics.py | 17 ironic/conductor/steps.py | 6 ironic/conductor/task_manager.py | 11 ironic/conductor/utils.py | 19 ironic/conf/__init__.py | 8 ironic/conf/conductor.py | 100 ironic/conf/default.py | 38 ironic/conf/disk_utils.py | 33 ironic/conf/fake.py | 85 ironic/conf/glance.py | 1 ironic/conf/inventory.py | 34 ironic/conf/irmc.py | 15 ironic/conf/opts.py | 6 ironic/conf/sensor_data.py | 89 ironic/db/api.py | 39 ironic/db/sqlalchemy/__init__.py | 2 ironic/db/sqlalchemy/alembic/versions/0ac0f39bc5aa_add_node_inventory_table.py | 46 ironic/db/sqlalchemy/alembic/versions/2581ebaf0cb2_initial_migration.py | 16 ironic/db/sqlalchemy/alembic/versions/2aac7e0872f6_add_deploy_templates.py | 8 ironic/db/sqlalchemy/alembic/versions/48d6c242bb9b_add_node_tags.py | 4 ironic/db/sqlalchemy/alembic/versions/4dbec778866e_create_node_shard.py | 31 ironic/db/sqlalchemy/alembic/versions/5ea1b0d310e_added_port_group_table_and_altered_ports.py | 4 ironic/db/sqlalchemy/alembic/versions/82c315d60161_add_bios_settings.py | 4 ironic/db/sqlalchemy/alembic/versions/9ef41f07cb58_add_node_history_table.py | 4 ironic/db/sqlalchemy/alembic/versions/b4130a7fc904_create_nodetraits_table.py | 4 ironic/db/sqlalchemy/alembic/versions/dd67b91a1981_add_allocations_table.py | 5 ironic/db/sqlalchemy/api.py | 1159 +- ironic/db/sqlalchemy/models.py | 74 ironic/drivers/irmc.py | 10 ironic/drivers/modules/agent_base.py | 27 ironic/drivers/modules/ansible/playbooks/roles/deploy/tasks/write.yaml | 2 ironic/drivers/modules/boot_mode_utils.py | 2 ironic/drivers/modules/console_utils.py | 2 ironic/drivers/modules/deploy_utils.py | 281 ironic/drivers/modules/fake.py | 63 ironic/drivers/modules/image_cache.py | 105 ironic/drivers/modules/image_utils.py | 47 ironic/drivers/modules/inspect_utils.py | 166 ironic/drivers/modules/inspector.py | 12 ironic/drivers/modules/ipmitool.py | 6 ironic/drivers/modules/irmc/common.py | 238 ironic/drivers/modules/irmc/inspect.py | 99 ironic/drivers/modules/irmc/management.py | 295 ironic/drivers/modules/irmc/power.py | 64 ironic/drivers/modules/irmc/vendor.py | 75 ironic/drivers/modules/network/neutron.py | 18 ironic/drivers/modules/pxe.py | 15 ironic/drivers/modules/pxe_base.py | 20 ironic/drivers/modules/pxe_grub_config.template | 5 ironic/drivers/modules/redfish/boot.py | 19 ironic/drivers/modules/redfish/firmware_utils.py | 2 ironic/drivers/modules/redfish/management.py | 11 ironic/drivers/modules/redfish/raid.py | 28 ironic/drivers/modules/redfish/utils.py | 3 ironic/objects/__init__.py | 1 ironic/objects/node.py | 8 ironic/objects/node_inventory.py | 89 ironic/objects/port.py | 29 ironic/objects/portgroup.py | 8 ironic/tests/unit/api/controllers/v1/test_allocation.py | 11 ironic/tests/unit/api/controllers/v1/test_node.py | 223 ironic/tests/unit/api/controllers/v1/test_port.py | 88 ironic/tests/unit/api/controllers/v1/test_ramdisk.py | 2 ironic/tests/unit/api/controllers/v1/test_root.py | 4 ironic/tests/unit/api/controllers/v1/test_shard.py | 80 ironic/tests/unit/api/test_acl.py | 24 ironic/tests/unit/api/test_rbac_project_scoped.yaml | 490 + ironic/tests/unit/api/test_rbac_system_scoped.yaml | 418 ironic/tests/unit/cmd/test_status.py | 82 ironic/tests/unit/common/test_checksum_utils.py | 203 ironic/tests/unit/common/test_cinder.py | 54 ironic/tests/unit/common/test_format_inspector.py | 668 + ironic/tests/unit/common/test_glance_service.py | 57 ironic/tests/unit/common/test_image_service.py | 141 ironic/tests/unit/common/test_images.py | 349 ironic/tests/unit/common/test_neutron.py | 1 ironic/tests/unit/common/test_qemu_img.py | 146 ironic/tests/unit/common/test_release_mappings.py | 10 ironic/tests/unit/common/test_rpc_service.py | 81 ironic/tests/unit/common/test_utils.py | 6 ironic/tests/unit/conductor/mgr_utils.py | 16 ironic/tests/unit/conductor/test_allocations.py | 2 ironic/tests/unit/conductor/test_base_manager.py | 6 ironic/tests/unit/conductor/test_cleaning.py | 21 ironic/tests/unit/conductor/test_manager.py | 187 ironic/tests/unit/conductor/test_utils.py | 55 ironic/tests/unit/db/sqlalchemy/test_migrations.py | 354 ironic/tests/unit/db/test_api.py | 29 ironic/tests/unit/db/test_conductor.py | 13 ironic/tests/unit/db/test_node_inventory.py | 36 ironic/tests/unit/db/test_nodes.py | 63 ironic/tests/unit/db/test_ports.py | 39 ironic/tests/unit/db/test_shard.py | 46 ironic/tests/unit/db/utils.py | 32 ironic/tests/unit/drivers/modules/irmc/test_common.py | 171 ironic/tests/unit/drivers/modules/irmc/test_inspect.py | 132 ironic/tests/unit/drivers/modules/irmc/test_management.py | 357 ironic/tests/unit/drivers/modules/irmc/test_power.py | 88 ironic/tests/unit/drivers/modules/network/test_common.py | 4 ironic/tests/unit/drivers/modules/network/test_neutron.py | 12 ironic/tests/unit/drivers/modules/redfish/test_boot.py | 44 ironic/tests/unit/drivers/modules/redfish/test_management.py | 10 ironic/tests/unit/drivers/modules/redfish/test_raid.py | 41 ironic/tests/unit/drivers/modules/redfish/test_utils.py | 8 ironic/tests/unit/drivers/modules/test_agent_base.py | 27 ironic/tests/unit/drivers/modules/test_deploy_utils.py | 751 + ironic/tests/unit/drivers/modules/test_image_cache.py | 276 ironic/tests/unit/drivers/modules/test_image_utils.py | 192 ironic/tests/unit/drivers/modules/test_inspect_utils.py | 229 ironic/tests/unit/drivers/modules/test_inspector.py | 43 ironic/tests/unit/drivers/modules/test_ipmitool.py | 10 ironic/tests/unit/drivers/modules/test_pxe.py | 51 ironic/tests/unit/drivers/modules/test_snmp.py | 4 ironic/tests/unit/drivers/pxe_grub_config.template | 4 ironic/tests/unit/drivers/test_fake_hardware.py | 29 ironic/tests/unit/drivers/third_party_driver_mock_specs.py | 2 ironic/tests/unit/objects/test_node_inventory.py | 49 ironic/tests/unit/objects/test_objects.py | 7 ironic/tests/unit/objects/test_port.py | 12 ironic/tests/unit/objects/test_portgroup.py | 15 redfish-interop-profiles/OpenStackIronicProfile.v1_0_0.json | 221 releasenotes/config.yaml | 5 releasenotes/notes/2061160-5e080a17ae31fb53.yaml | 8 releasenotes/notes/add-allocations-table-check-38f1c9eef189b411.yaml | 8 releasenotes/notes/add-node-inventory-7cde961b14caa11e.yaml | 5 releasenotes/notes/add-service-role-support-8e9390769508ca99.yaml | 13 releasenotes/notes/address-qemu-issues-1bbead8bb70b76fb.yaml | 108 releasenotes/notes/allocations-charset-5384d1ea00964bdd.yaml | 23 releasenotes/notes/catch-all-cleaning-exceptions-1317a534a1c9db56.yaml | 8 releasenotes/notes/change-c9c01700dcfd599b.yaml | 9 releasenotes/notes/checksum-before-conversion-66d273b94fa2ba4d.yaml | 44 releasenotes/notes/cinder-2019892-6b5a9de5c5f05aa6.yaml | 16 releasenotes/notes/cleaning-error-5c13c33c58404b97.yaml | 8 releasenotes/notes/conductor-metric-collector-support-1b8b8c71f9f59da4.yaml | 39 releasenotes/notes/console-pid-file-6108d2775ef947fe.yaml | 6 releasenotes/notes/cross-link-1ffd1a4958f14fd7.yaml | 5 releasenotes/notes/fakedelay-7eac23ad8881a736.yaml | 8 releasenotes/notes/file-symlink-b65bd6b407bd1683.yaml | 6 releasenotes/notes/fix-allocation-exception-on-list-c04e93fb9cace218.yaml | 8 releasenotes/notes/fix-console-port-conflict-6dc19688079e2c7f.yaml | 8 releasenotes/notes/fix-context-image-hardlink-16f452974abc7327.yaml | 7 releasenotes/notes/fix-eject-media-dvd-b1994446ea71be9c.yaml | 8 releasenotes/notes/fix-grub2-uefi-config-path-f1b4c5083cc97ee5.yaml | 14 releasenotes/notes/fix-inspectwait-finished-at-4b817af4bf4c30c2.yaml | 5 releasenotes/notes/fix-irmc-enforcing-snmpv3-with-fips-e45971d363925ec3.yaml | 6 releasenotes/notes/fix-irmc-s6-2.00-http-incompatibility-61a31d12aa33fbd8.yaml | 19 releasenotes/notes/fix-irmc-s6-2.00-ipmi-incompatibility-118484a424df02b1.yaml | 15 releasenotes/notes/fix-nonetype-object-is-not-iterable-0592926d890d6c11.yaml | 7 releasenotes/notes/fix-online-version-migration-db432a7b239647fa.yaml | 14 releasenotes/notes/fix-overlooked-irmc-ipmi-incompatibility-patch-situation-c246d2b59b2e8a78.yaml | 8 releasenotes/notes/fix-power-off-token-wipe-e7d605997f00d39d.yaml | 6 releasenotes/notes/fix-self-owned-node-policy-fc2dae357879dc33.yaml | 7 releasenotes/notes/fix-system-scope-triggered-clean-22ada9b920c08365.yaml | 12 releasenotes/notes/fix_anaconda-70f4268edc255ff4.yaml | 5 releasenotes/notes/fix_anaconda_pxe-6c75d42872424fec.yaml | 6 releasenotes/notes/fix_secure_boot_with_anaconda_deploy-84d7c1e3bbfa40f2.yaml | 4 releasenotes/notes/handle-missing-ethernetinterfaces-attr-7e52f7259fe66762.yaml | 9 releasenotes/notes/irmc-add-snmp-auth-protocols-3ff7597cea7ef9dd.yaml | 5 releasenotes/notes/irmc-align-with-ironic-default-boot-mode-dde6f65ea084c9e6.yaml | 5 releasenotes/notes/irmc-change-boot-interface-order-e76f5018da116a90.yaml | 26 releasenotes/notes/ironic-antelope-prelude-0b77964469f56b13.yaml | 14 releasenotes/notes/limit-boot-to-disk-calls-lenovo-39763bfc98f602d8.yaml | 13 releasenotes/notes/lockutils-default-logging-8c38b8c0ac71043f.yaml | 8 releasenotes/notes/no-recalculate-653e524fd6160e72.yaml | 5 releasenotes/notes/node-iso-external_http_url-c5e3fa9ae4960dd6.yaml | 5 releasenotes/notes/permit-conductor-to-start-without-neutron-networks-d4aa21654f9c07bf.yaml | 9 releasenotes/notes/prepare-for-sqlalchemy-20-e817f340f261b1a2.yaml | 7 releasenotes/notes/redfish-fix-raid-creation-f437066b1301c032.yaml | 6 releasenotes/notes/service-project-service-role-fix-e4d1a8c23856926a.yaml | 41 releasenotes/notes/shard-support-a26f8d2ab5cca582.yaml | 14 releasenotes/notes/virtual-media-publisher-id-injection-c88674a31634f852.yaml | 6 releasenotes/notes/wait_hash_ring_reset-ef8bd548659e9906.yaml | 13 releasenotes/notes/wipe-agent-token-upon-cleaning-timeout-c9add514fad1b02c.yaml | 7 releasenotes/source/index.rst | 1 releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po | 527 - releasenotes/source/yoga.rst | 6 releasenotes/source/zed.rst | 6 reno.yaml | 4 requirements.txt | 6 setup.cfg | 1 setup.py | 4 test-requirements.txt | 2 tools/benchmark/do_not_run_create_benchmark_data.py | 63 tools/benchmark/generate-statistics.py | 112 tox.ini | 26 zuul.d/ironic-jobs.yaml | 35 zuul.d/project.yaml | 65 268 files changed, 18809 insertions(+), 2146 deletions(-) dpkg-source: warning: cannot verify inline signature for /srv/release.debian.org/tmp/tmprgfkqzgc/ironic_21.1.0-3+deb12u1.dsc: no acceptable signature found dpkg-source: warning: cannot verify inline signature for /srv/release.debian.org/tmp/tmprgfkqzgc/ironic_21.4.4-0+deb12u1.dsc: no acceptable signature found diff -Nru ironic-21.1.0/.gitreview ironic-21.4.4/.gitreview --- ironic-21.1.0/.gitreview 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/.gitreview 2024-10-11 15:42:16.000000000 +0000 @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/ironic.git +defaultbranch=stable/2023.1 diff -Nru ironic-21.1.0/api-ref/source/baremetal-api-v1-nodes-inventory.inc ironic-21.4.4/api-ref/source/baremetal-api-v1-nodes-inventory.inc --- ironic-21.1.0/api-ref/source/baremetal-api-v1-nodes-inventory.inc 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/api-ref/source/baremetal-api-v1-nodes-inventory.inc 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,44 @@ +.. -*- rst -*- + +============== +Node inventory +============== + +.. versionadded:: 1.81 + +Given a Node identifier, the API provides access to the introspection data +associated to the Node via ``v1/nodes/{node_ident}/inventory`` endpoint. + +The format inventory comes from ironic-python-agent and is currently documented +in the `agent inventory documentation +`_. + +Show Node Inventory +=================== + +.. rest_method:: GET /v1/nodes/{node_ident}/inventory + +Normal response code: 200 + +Error codes: + - 404 (NodeNotFound, InventoryNotRecorded) + +Request +------- + +.. rest_parameters:: parameters.yaml + + - node_ident: node_ident + +Response +-------- + +.. rest_parameters:: parameters.yaml + + - inventory: n_inventory + - plugin_data: n_plugin_data + +**Example of inventory from a node:** + +.. literalinclude:: samples/node-inventory-response.json + :language: javascript diff -Nru ironic-21.1.0/api-ref/source/baremetal-api-v1-nodes.inc ironic-21.4.4/api-ref/source/baremetal-api-v1-nodes.inc --- ironic-21.1.0/api-ref/source/baremetal-api-v1-nodes.inc 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/api-ref/source/baremetal-api-v1-nodes.inc 2024-10-11 15:42:16.000000000 +0000 @@ -104,6 +104,9 @@ .. versionadded:: 1.65 Introduced the ``lessee`` field. +.. versionadded:: 1.82 + Introduced the ``shard`` field. + Normal response codes: 201 Error codes: 400,403,406 @@ -135,6 +138,7 @@ - owner: owner - description: req_n_description - lessee: lessee + - shard: shard - automated_clean: req_automated_clean - bios_interface: req_bios_interface - chassis_uuid: req_chassis_uuid @@ -161,7 +165,7 @@ or "". The list and example below are representative of the response as of API -microversion 1.48. +microversion 1.81. .. rest_parameters:: parameters.yaml @@ -213,6 +217,7 @@ - conductor: conductor - owner: owner - lessee: lessee + - shard: shard - description: n_description - allocation_uuid: allocation_uuid - automated_clean: automated_clean @@ -280,6 +285,9 @@ .. versionadded:: 1.65 Introduced the ``lessee`` field. +.. versionadded:: 1.82 + Introduced the ``shard`` field. Introduced the ``sharded`` request parameter. + Normal response codes: 200 Error codes: 400,403,406 @@ -300,6 +308,8 @@ - fault: r_fault - owner: owner - lessee: lessee + - shard: req_shard + - sharded: req_sharded - description_contains: r_description_contains - fields: fields - limit: limit @@ -371,6 +381,9 @@ .. versionadded:: 1.65 Introduced the ``lessee`` field. +.. versionadded:: 1.82 + Introduced the ``shard`` field. Introduced the ``sharded`` request parameter. + Normal response codes: 200 Error codes: 400,403,406 @@ -391,6 +404,8 @@ - conductor: r_conductor - owner: owner - lessee: lessee + - shard: req_shard + - sharded: req_sharded - description_contains: r_description_contains - limit: limit - marker: marker @@ -450,6 +465,7 @@ - protected_reason: protected_reason - owner: owner - lessee: lessee + - shard: shard - description: n_description - conductor: conductor - allocation_uuid: allocation_uuid @@ -508,6 +524,9 @@ .. versionadded:: 1.66 Introduced the ``network_data`` field. +.. versionadded:: 1.82 + Introduced the ``shard`` field. + Normal response codes: 200 Error codes: 400,403,404,406 @@ -573,6 +592,7 @@ - protected_reason: protected_reason - owner: owner - lessee: lessee + - shard: shard - description: n_description - conductor: conductor - allocation_uuid: allocation_uuid @@ -600,6 +620,9 @@ .. versionadded:: 1.51 Introduced the ability to set/unset a node's description. +.. versionadded:: 1.82 + Introduced the ability to set/unset a node's shard. + Normal response codes: 200 Error codes: 400,403,404,406,409 @@ -670,6 +693,7 @@ - protected_reason: protected_reason - owner: owner - lessee: lessee + - shard: shard - description: n_description - conductor: conductor - allocation_uuid: allocation_uuid diff -Nru ironic-21.1.0/api-ref/source/baremetal-api-v1-ports.inc ironic-21.4.4/api-ref/source/baremetal-api-v1-ports.inc --- ironic-21.1.0/api-ref/source/baremetal-api-v1-ports.inc 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/api-ref/source/baremetal-api-v1-ports.inc 2024-10-11 15:42:16.000000000 +0000 @@ -49,6 +49,10 @@ .. versionadded:: 1.53 Added the ``is_smartnic`` field. +.. versionadded:: 1.82 + Added the ability to filter ports based on the shard of the node they are + associated with. + Normal response code: 200 Request @@ -60,6 +64,7 @@ - node_uuid: r_port_node_uuid - portgroup: r_port_portgroup_ident - address: r_port_address + - shard: r_port_shard - fields: fields - limit: limit - marker: marker diff -Nru ironic-21.1.0/api-ref/source/baremetal-api-v1-shards.inc ironic-21.4.4/api-ref/source/baremetal-api-v1-shards.inc --- ironic-21.1.0/api-ref/source/baremetal-api-v1-shards.inc 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/api-ref/source/baremetal-api-v1-shards.inc 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,56 @@ +.. -*- rst -*- + +====== +Shards +====== + +This section describes an API endpoint returning the population of shards +among nodes in the Bare Metal Service. Shards are a way to group nodes in the +Bare Metal service. They are used by API clients to separate nodes into groups, +allowing horizontal scaling. + +Shards are not directly added and removed from the Bare Metal service. Instead, +operators can configure a node into a given shard by setting the ``shard`` key +to any unique string value representing the shard. + +.. note:: + The Bare Metal Service does not use shards directly. It instead relies on + API clients and external services to use shards to group nodes into smaller + areas of responsibility. + + +Shards +====== + +.. rest_method:: GET /v1/shards + +.. versionadded:: 1.82 + +The ``/v1/shards`` endpoint exists to allow querying the distribution of nodes +between all defined shards. + +Normal response codes: 200 + +Error response codes: 400 403 404 + +Request +------- + +No request parameters are accepted by this endpoint. + +Response +-------- + +Returns a list of shards and the count of nodes assigned to each. The +list is sorted by descending count. + +.. rest_parameters:: parameters.yaml + + - name: shard_name + - count: shard_count + +Response Example +---------------- + +.. literalinclude:: samples/shards-list-response.json + :language: javascript diff -Nru ironic-21.1.0/api-ref/source/index.rst ironic-21.4.4/api-ref/source/index.rst --- ironic-21.1.0/api-ref/source/index.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/api-ref/source/index.rst 2024-10-11 15:42:16.000000000 +0000 @@ -28,6 +28,8 @@ .. include:: baremetal-api-v1-node-allocation.inc .. include:: baremetal-api-v1-deploy-templates.inc .. include:: baremetal-api-v1-nodes-history.inc +.. include:: baremetal-api-v1-nodes-inventory.inc +.. include:: baremetal-api-v1-shards.inc .. NOTE(dtantsur): keep chassis close to the end since it's semi-deprecated .. include:: baremetal-api-v1-chassis.inc .. NOTE(dtantsur): keep misc last, since it covers internal API diff -Nru ironic-21.1.0/api-ref/source/parameters.yaml ironic-21.4.4/api-ref/source/parameters.yaml --- ironic-21.1.0/api-ref/source/parameters.yaml 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/api-ref/source/parameters.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -343,13 +343,17 @@ description: | Filter the list of returned Ports, and only return the ones associated with this specific node (name or UUID), or an empty set if not found. + This filter takes precedence over all other filters, and cannot be set at + the same time as node_uuid or portgroup. in: query required: false type: string r_port_node_uuid: description: | Filter the list of returned Ports, and only return the ones associated - with this specific node UUID, or an empty set if not found. + with this specific node UUID, or an empty set if not found. This filter + takes precedence over all other filters, and cannot be set at the same + time as node or portgroup. in: query required: false type: string @@ -357,9 +361,18 @@ description: | Filter the list of returned Ports, and only return the ones associated with this specific Portgroup (name or UUID), or an empty set if not found. + This filter takes precedence over all other filters, and cannot be set at + the same time as node_uuid or node. in: query required: false type: string +r_port_shard: + description: | + Filter the list of returned Ports, and only return the ones associated + with nodes in this specific shard(s), or an empty set if not found. + in: query + required: false + type: array r_portgroup_address: description: | Filter the list of returned Portgroups, and only return the ones with the @@ -1191,6 +1204,18 @@ in: body required: true type: array +n_inventory: + description: | + Inventory of this node. + in: body + required: false + type: JSON +n_plugin_data: + description: | + Plugin data of this node. + in: body + required: false + type: JSON n_portgroups: description: | Links to the collection of portgroups on this node. @@ -1795,6 +1820,20 @@ in: body required: false type: string +req_shard: + description: | + Filter the list of returned Nodes, and only return the ones associated + with nodes in this specific shard(s), or an empty set if not found. + in: body + required: false + type: array +req_sharded: + description: | + When true, filter the list of returned Nodes, and only return the ones with + a non-null ``shard`` value. When false, the inverse filter is performed. + in: body + required: false + type: boolean req_standalone_ports_supported: description: | Indicates whether ports that are members of this portgroup can be @@ -1920,6 +1959,22 @@ Indicates whether node is currently booted with secure_boot turned on. in: body type: boolean +shard: + description: | + A string indicating the shard this node belongs to. + in: body + type: string +shard_count: + description: | + The number of nodes with this current string as their assigned shard value. + in: body + type: integer +shard_name: + description: | + The name of the shard. A value of "None" indicates the count of nodes with + an empty shard value. + in: body + type: string standalone_ports_supported: description: | Indicates whether ports that are members of this portgroup can be diff -Nru ironic-21.1.0/api-ref/source/samples/node-inventory-response.json ironic-21.4.4/api-ref/source/samples/node-inventory-response.json --- ironic-21.1.0/api-ref/source/samples/node-inventory-response.json 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/api-ref/source/samples/node-inventory-response.json 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,31 @@ +{ + "inventory": { + "interfaces":[ + { + "name":"eth0", + "mac_address":"52:54:00:90:35:d6", + "ipv4_address":"192.168.122.128", + "ipv6_address":"fe80::5054:ff:fe90:35d6%eth0", + "has_carrier":true, + "lldp":null, + "vendor":"0x1af4", + "product":"0x0001" + } + ], + "cpu":{ + "model_name":"QEMU Virtual CPU version 2.5+", + "frequency":null, + "count":1, + "architecture":"x86_64" + } + }, + "plugin_data":{ + "macs":[ + "52:54:00:90:35:d6" + ], + "local_gb":10, + "cpus":1, + "cpu_arch":"x86_64", + "memory_mb":2048 + } +} diff -Nru ironic-21.1.0/api-ref/source/samples/shards-list-response.json ironic-21.4.4/api-ref/source/samples/shards-list-response.json --- ironic-21.1.0/api-ref/source/samples/shards-list-response.json 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/api-ref/source/samples/shards-list-response.json 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,12 @@ +{ + "shards": [ + { + "count": 47, + "name": "example_shard1", + }, + { + "count": 46, + "name": "example_shard2" + } + ] +} diff -Nru ironic-21.1.0/debian/changelog ironic-21.4.4/debian/changelog --- ironic-21.1.0/debian/changelog 2026-04-30 08:41:21.000000000 +0000 +++ ironic-21.4.4/debian/changelog 2024-11-08 15:10:43.000000000 +0000 @@ -1,3 +1,41 @@ +ironic (1:21.4.4-0+deb12u1) bookworm-security; urgency=medium + + * New upstream point release. Fixed CVE-2024-44082. + * CVE-2026-44917: Ironic does not validate the location of + node.driver_info[pxe_template], allowing a user who can set it to expose + arbitrary files on an internal Ironic network, such as the servicing, + provisioning, or cleaning networks. Applied upstream patch: + - CVE-2026-44917_disable-driver_info-level-pxe_template-override.patch + * CVE-2026-46447: A user with access to add or modify node.driver_info or + node.instance_info can create a crafted value to enable iPXE script + execution during the boot process. Applied upstream patch: + - CVE-2026-46447_Sanitize-kernel_append_parms.patch + * CVE-2026-48681: A maliciously crafted ISO image can cause Ironic to perform + path traversal and overwrite files on a conductor's disk. Applied upstream + patch: + - CVE-2026-48681-directory_transversal_ISO9660_support.patch + (Closes: #1138842) + * CVE-2026-44919: during image handling, an infinite loop in checksum + calculations can occur via the file:///dev/zero URL. Add upstream patch: + move_file_url_validation_up_into_deploy_utils_main_path.patch. + (Closes: #1136655). + * CVE-2026-44916: instance_info['ks_template'] is rendered without + sandboxing. An attacker with sufficient access, an ironic deployment with + the anaconda deploy interface, a node with the anaconda deployment + interface set by an admin, and a malicious template could result in + conductor internal data being rendered and if the infrastucture operator is + allowing traffic egress for the provisioning network, could have sensitive + internal data exfiled out of the environment. Applied upstream patch: + - CVE-2026-44916_Use_sandbox_rendering_for_jinja2.patch + (Closes: #1136005). + * CVE-2026-42997 / OSSA-2026-010: Credential Forwarding to Arbitrary + Endpoints via Ironic’s idrac Configuration molds Feature. Add upstream + patch validate_molds_url_against_swift_in_keystone_catalog.patch. + (Closes: #1135898). + * (build-)depends on python3-oslo.messaging >= 14.0.3-0+deb12u1~. + + -- Thomas Goirand Fri, 08 Nov 2024 16:10:43 +0100 + ironic (1:21.1.0-3+deb12u1) bookworm; urgency=medium * CVE-2026-42510 / OSSA-2026-008: Command Injection in Ironic IPMI Console diff -Nru ironic-21.1.0/debian/control ironic-21.4.4/debian/control --- ironic-21.1.0/debian/control 2026-04-30 08:41:21.000000000 +0000 +++ ironic-21.4.4/debian/control 2024-11-08 15:10:43.000000000 +0000 @@ -47,7 +47,7 @@ python3-oslo.context, python3-oslo.db (>= 9.1.0), python3-oslo.log, - python3-oslo.messaging, + python3-oslo.messaging (>= 14.0.3-0+deb12u1~), python3-oslo.middleware, python3-oslo.policy (>= 3.7.0), python3-oslo.reports, @@ -213,7 +213,7 @@ python3-oslo.context, python3-oslo.db (>= 9.1.0), python3-oslo.log, - python3-oslo.messaging, + python3-oslo.messaging (>= 14.0.3-0+deb12u1~), python3-oslo.middleware, python3-oslo.policy (>= 3.7.0), python3-oslo.rootwrap, diff -Nru ironic-21.1.0/debian/patches/CVE-2025-44021_OSSA-2025-001_Disallow_unsafe_image_file_paths.patch ironic-21.4.4/debian/patches/CVE-2025-44021_OSSA-2025-001_Disallow_unsafe_image_file_paths.patch --- ironic-21.1.0/debian/patches/CVE-2025-44021_OSSA-2025-001_Disallow_unsafe_image_file_paths.patch 2026-04-30 08:41:21.000000000 +0000 +++ ironic-21.4.4/debian/patches/CVE-2025-44021_OSSA-2025-001_Disallow_unsafe_image_file_paths.patch 2024-11-08 15:10:43.000000000 +0000 @@ -56,7 +56,7 @@ =================================================================== --- ironic.orig/ironic/common/image_service.py +++ ironic/ironic/common/image_service.py -@@ -242,14 +242,32 @@ class FileImageService(BaseImageService) +@@ -273,14 +273,32 @@ class FileImageService(BaseImageService) :param image_href: Image reference. :raises: exception.ImageRefValidationFailed if source image file @@ -95,18 +95,19 @@ =================================================================== --- ironic.orig/ironic/conf/conductor.py +++ ironic/ironic/conf/conductor.py -@@ -19,6 +19,7 @@ from oslo_config import cfg +@@ -19,6 +19,8 @@ from oslo_config import cfg from oslo_config import types from ironic.common.i18n import _ +from ironic.conf import types as ir_types ++ opts = [ cfg.IntOpt('workers_pool_size', -@@ -384,6 +385,20 @@ opts = [ - 'is a global setting applying to all requests this ' - 'conductor receives, regardless of access rights. ' - 'The concurrent clean limit cannot be disabled.')), +@@ -414,6 +416,20 @@ opts = [ + 'functionality by setting this option to True will ' + 'create a more secure environment, however it may ' + 'break users in an unexpected fashion.')), + cfg.ListOpt('file_url_allowed_paths', + default=['/var/lib/ironic', '/shared/html', '/templates', + '/opt/cache/files', '/vagrant'], @@ -188,7 +189,7 @@ =================================================================== --- ironic.orig/ironic/tests/unit/common/test_image_service.py +++ ironic/ironic/tests/unit/common/test_image_service.py -@@ -452,20 +452,58 @@ class FileImageServiceTestCase(base.Test +@@ -482,20 +482,58 @@ class FileImageServiceTestCase(base.Test def setUp(self): super(FileImageServiceTestCase, self).setUp() self.service = image_service.FileImageService() diff -Nru ironic-21.1.0/debian/patches/CVE-2026-44916_Use_sandbox_rendering_for_jinja2.patch ironic-21.4.4/debian/patches/CVE-2026-44916_Use_sandbox_rendering_for_jinja2.patch --- ironic-21.1.0/debian/patches/CVE-2026-44916_Use_sandbox_rendering_for_jinja2.patch 2026-04-30 08:41:21.000000000 +0000 +++ ironic-21.4.4/debian/patches/CVE-2026-44916_Use_sandbox_rendering_for_jinja2.patch 2024-11-08 15:10:43.000000000 +0000 @@ -28,11 +28,11 @@ Origin: upstream, https://review.opendev.org/c/openstack/ironic/+/987778 Last-Update: 2026-05-08 -Index: ironic/ironic/common/utils.py -=================================================================== ---- ironic.orig/ironic/common/utils.py -+++ ironic/ironic/common/utils.py -@@ -31,6 +31,7 @@ import tempfile +diff --git a/ironic/common/utils.py b/ironic/common/utils.py +index 04e6d04..1fcc245 100644 +--- a/ironic/common/utils.py ++++ b/ironic/common/utils.py +@@ -31,6 +31,7 @@ import time import jinja2 @@ -40,7 +40,7 @@ from oslo_concurrency import processutils from oslo_log import log as logging from oslo_serialization import jsonutils -@@ -481,6 +482,8 @@ def render_template(template, params, is +@@ -481,6 +482,8 @@ :param strict: Enable strict template rendering. Default is False :returns: Rendered template :raises: jinja2.exceptions.UndefinedError @@ -49,7 +49,7 @@ """ if is_file: tmpl_path, tmpl_name = os.path.split(template) -@@ -488,11 +491,9 @@ def render_template(template, params, is +@@ -488,11 +491,9 @@ else: tmpl_name = 'template' loader = jinja2.DictLoader({tmpl_name: template}) @@ -62,11 +62,11 @@ loader=loader, autoescape=jinja2.select_autoescape(), undefined=jinja2.StrictUndefined if strict else jinja2.Undefined -Index: ironic/ironic/tests/unit/common/test_utils.py -=================================================================== ---- ironic.orig/ironic/tests/unit/common/test_utils.py -+++ ironic/ironic/tests/unit/common/test_utils.py -@@ -589,6 +589,11 @@ class JinjaTemplatingTestCase(base.TestC +diff --git a/ironic/tests/unit/common/test_utils.py b/ironic/tests/unit/common/test_utils.py +index 96c6612..b28281d 100644 +--- a/ironic/tests/unit/common/test_utils.py ++++ b/ironic/tests/unit/common/test_utils.py +@@ -595,6 +595,11 @@ self.template = '{{ foo }} {{ bar }}' self.params = {'foo': 'spam', 'bar': 'ham'} self.expected = 'spam ham' @@ -78,7 +78,7 @@ def test_render_string(self): self.assertEqual(self.expected, -@@ -615,6 +620,46 @@ class JinjaTemplatingTestCase(base.TestC +@@ -621,6 +626,46 @@ self.params)) jinja_fsl_mock.assert_called_once_with('/path/to') @@ -125,10 +125,11 @@ class ValidateConductorGroupTestCase(base.TestCase): def test_validate_conductor_group_success(self): -Index: ironic/releasenotes/notes/fix-bug-2148307-ddf0b4d69244e86e.yaml -=================================================================== +diff --git a/releasenotes/notes/fix-bug-2148307-ddf0b4d69244e86e.yaml b/releasenotes/notes/fix-bug-2148307-ddf0b4d69244e86e.yaml +new file mode 100644 +index 0000000..c905254 --- /dev/null -+++ ironic/releasenotes/notes/fix-bug-2148307-ddf0b4d69244e86e.yaml ++++ b/releasenotes/notes/fix-bug-2148307-ddf0b4d69244e86e.yaml @@ -0,0 +1,8 @@ +--- +fixes: diff -Nru ironic-21.1.0/debian/patches/CVE-2026-44917_disable-driver_info-level-pxe_template-override.patch ironic-21.4.4/debian/patches/CVE-2026-44917_disable-driver_info-level-pxe_template-override.patch --- ironic-21.1.0/debian/patches/CVE-2026-44917_disable-driver_info-level-pxe_template-override.patch 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/debian/patches/CVE-2026-44917_disable-driver_info-level-pxe_template-override.patch 2024-11-08 15:10:43.000000000 +0000 @@ -0,0 +1,200 @@ +Author: Julia Kreger +Date: Thu, 7 May 2026 08:48:48 -0700 +Description: CVE-2026-44917: disable driver_info level pxe_template override + A vulnerability report was filed pointing out a flaw in the pxe_template + override logic where a direct file path was supplied. The original usage + context of this minimally documented feature was that an operator, i.e. + the owner of the ironic deployment could leverage a direct file path to + a template on disk. This should instead have utilized the file:/// URL + provider, but research suggests this feature has largely not been used. + . + As a result, consensus has been reached amongst security maintainers + for the Ironic project to disable and remove this functionality. + . + Where this issue became a vulnerability for Ironic was the evolution + of the usage and Role Based Access Control model where we began to + separate the overall operator of the system from the administrative + manager of the system. + . + The resulting vector was that an authenticated and authorized user + could potentially request a template a sensitive file to be sourced + as the PXE template. This file would then be written to disk and + utilized IF the ironic-conductor service could access it. The + malicious authenticated and authorized user could then, if the + environment was misconfigured, or operating with "flat" networking, + it could be possible to guess the underlying file path on the + tftpboot/httpboot network bootendpoints, and retrieve the rendered + output before the deployment failed and the rendered output is removed. + . + This is tracked as CVE-2026-44917, and the underlying feature is + expected to be removed during the 2027.2. + . +Bug: https://launchpad.net/bugs/2148319 +Change-Id: I52daa344b4d417eee09c28b53703fea792e4367b +Signed-off-by: Julia Kreger +Origin: upstream, pre-OSSA mailing list +Last-Update: 2026-06-01 + +diff --git a/ironic/conf/pxe.py b/ironic/conf/pxe.py +index d96712824..3fb5a1f4d 100644 +--- a/ironic/conf/pxe.py ++++ b/ironic/conf/pxe.py +@@ -209,6 +209,13 @@ opts = [ + '$pybasedir', 'drivers/modules/initial_grub_cfg.template'), + help=_('On ironic-conductor node, the path to the initial grub' + 'configuration template for grub network boot.')), ++ cfg.BoolOpt('enable_insecure_template_override', ++ default=False, ++ help=_('If node level pxe_template override is permitted to ' ++ 'be used in this Ironic deployment. This is an ' ++ 'insecure pattern filed under CVE-2026-44917 and ' ++ 'the feature this guards this is expected to be ' ++ 'removed in Ironic release 2027.2.')), + ] + + +diff --git a/ironic/drivers/modules/deploy_utils.py b/ironic/drivers/modules/deploy_utils.py +index f4f1ce0f4..d921329a6 100644 +--- a/ironic/drivers/modules/deploy_utils.py ++++ b/ironic/drivers/modules/deploy_utils.py +@@ -464,9 +464,22 @@ def get_ipxe_config_template(node): + # loaders by architecture as they are all consistent. Where as PXE + # could need to be grub for one arch, PXELINUX for another. + configured_template = CONF.pxe.ipxe_config_template +- override_template = node.driver_info.get('pxe_template') +- if override_template: +- configured_template = override_template ++ insecure_override_template = node.driver_info.get('pxe_template') ++ if CONF.pxe.enable_insecure_template_override: ++ # TODO(TheJulia): Remove the node level pxe_template setting in ++ # a future release as it is inhernetly insecure. ++ if insecure_override_template: ++ configured_template = insecure_override_template ++ elif insecure_override_template: ++ raise exception.InvalidParameterValue(_( ++ 'The node\'s driver_info field pxe_template override value is ' ++ 'insecure (CVE-2026-44917) and should not be used. The ' ++ 'appropriate approach is to utilize [pxe]ipxe_template_by_arch ' ++ 'configuration in ironic.conf to match the baremetal node\'s ' ++ 'architecture. Please work with your Ironic operator to remedy ' ++ 'your usage and configuration. Default templates may be ' ++ 'leveraged by deleting the pxe_template value in the driver_info ' ++ 'field.')) + return configured_template or get_pxe_config_template(node) + + +@@ -481,7 +494,22 @@ def get_pxe_config_template(node): + :param node: A single Node. + :returns: The PXE config template file name. + """ +- config_template = node.driver_info.get("pxe_template", None) ++ config_template = None ++ insecure_override_template = node.driver_info.get("pxe_template", None) ++ if CONF.pxe.enable_insecure_template_override: ++ # TODO(TheJulia): Remove the node level pxe_template setting in ++ # a future release as it is inhernetly insecure. ++ config_template = insecure_override_template ++ elif insecure_override_template: ++ raise exception.InvalidParameterValue(_( ++ 'The node\'s driver_info field pxe_template override value is ' ++ 'insecure (CVE-2026-44917) and should not be used. The ' ++ 'appropriate approach is to utilize [pxe]pxe_template_by_arch ' ++ 'configuration in ironic.conf to match the baremetal node\'s ' ++ 'architecture. Please work with your Ironic operator to remedy ' ++ 'your usage and configuration. Default templates may be ' ++ 'leveraged by deleting the pxe_template value in the driver_info ' ++ 'field.')) + if config_template is None: + cpu_arch = node.properties.get('cpu_arch') + config_template = CONF.pxe.pxe_config_template_by_arch.get(cpu_arch) +diff --git a/ironic/tests/unit/drivers/modules/test_deploy_utils.py b/ironic/tests/unit/drivers/modules/test_deploy_utils.py +index 185bea6d0..8668faa40 100644 +--- a/ironic/tests/unit/drivers/modules/test_deploy_utils.py ++++ b/ironic/tests/unit/drivers/modules/test_deploy_utils.py +@@ -423,6 +423,8 @@ class GetPxeBootConfigTestCase(db_base.DbTestCase): + self.assertEqual('bios-template', result) + + def test_get_pxe_config_template_per_node(self): ++ cfg.CONF.set_override('enable_insecure_template_override', True, ++ group='pxe') + node = obj_utils.create_test_node( + self.context, driver='fake-hardware', + driver_info={"pxe_template": "fake-template"}, +@@ -430,6 +432,16 @@ class GetPxeBootConfigTestCase(db_base.DbTestCase): + result = utils.get_pxe_config_template(node) + self.assertEqual('fake-template', result) + ++ def test_get_pxe_config_template_per_node_disabled(self): ++ self.assertFalse(cfg.CONF.pxe.enable_insecure_template_override) ++ node = obj_utils.create_test_node( ++ self.context, driver='fake-hardware', ++ driver_info={"pxe_template": "fake-template"}, ++ ) ++ self.assertRaisesRegex( ++ exception.InvalidParameterValue, 'CVE-2026-44917', ++ utils.get_pxe_config_template, node) ++ + def test_get_ipxe_config_template(self): + node = obj_utils.create_test_node( + self.context, driver='fake-hardware') +@@ -456,12 +468,23 @@ class GetPxeBootConfigTestCase(db_base.DbTestCase): + utils.get_ipxe_config_template(node)) + + def test_get_ipxe_config_template_override_pxe_fallback(self): ++ cfg.CONF.set_override('enable_insecure_template_override', True, ++ group='pxe') + node = obj_utils.create_test_node( + self.context, driver='fake-hardware', + driver_info={'pxe_template': 'magical'}) + self.assertEqual('magical', + utils.get_ipxe_config_template(node)) + ++ def test_get_ipxe_config_template_override_pxe_fallback_disabled(self): ++ self.assertFalse(cfg.CONF.pxe.enable_insecure_template_override) ++ node = obj_utils.create_test_node( ++ self.context, driver='fake-hardware', ++ driver_info={'pxe_template': 'magical'}) ++ self.assertRaisesRegex( ++ exception.InvalidParameterValue, 'CVE-2026-44917', ++ utils.get_ipxe_config_template, node) ++ + + @mock.patch('time.sleep', lambda sec: None) + class OtherFunctionTestCase(db_base.DbTestCase): +diff --git a/releasenotes/notes/security-bug-2148319-49974afdcd38d9c0.yaml b/releasenotes/notes/security-bug-2148319-49974afdcd38d9c0.yaml +new file mode 100644 +index 000000000..67edfc8f4 +--- /dev/null ++++ b/releasenotes/notes/security-bug-2148319-49974afdcd38d9c0.yaml +@@ -0,0 +1,28 @@ ++--- ++security: ++ - | ++ A vulnerability was discovered in an minimally documented feature of ++ Ironic where an absolute path to a ``pxe_template`` override value could ++ be defined by an authenticated and privilged API user. The Ironic team has ++ chosen to immediately deprecate and remove this functionality. To provide ++ an immediate security fix, this functionality is now disabled by default. ++ The functionality can be re-enabled via the ++ ``[pxe]enable_insecure_template_override`` configuration option which ++ was added to ironic.conf with a default value of ``False``. ++ This issue is tracked as ++ `bug 2148319 `_. ++fixes: ++ - | ++ Fixes a vulnerability (CVE-2026-44917) which was identified inhandling ++ of pxe_template overrides where an authenticated and authorized user ++ could request an override template via direct file path which would ++ bypass file URL handling guards introduced in OSSA-2025-001. This ++ feature was minimally documented through only a release note, and ++ does not appear to have actual use. This functionality is being ++ disabled by default, and will be promptly removed from Ironic's ++ current development branch. ++deprecations: ++ - | ++ The node ``driver_info`` field value ``pxe_template`` has been ++ deprecated and is expected to be removed in the future Ironic ++ 2027.2 release. +-- +2.47.3 + diff -Nru ironic-21.1.0/debian/patches/CVE-2026-44919_move_file_url_validation_up_into_deploy_utils_main_path.patch ironic-21.4.4/debian/patches/CVE-2026-44919_move_file_url_validation_up_into_deploy_utils_main_path.patch --- ironic-21.1.0/debian/patches/CVE-2026-44919_move_file_url_validation_up_into_deploy_utils_main_path.patch 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/debian/patches/CVE-2026-44919_move_file_url_validation_up_into_deploy_utils_main_path.patch 2024-11-08 15:10:43.000000000 +0000 @@ -0,0 +1,198 @@ +Author: Julia Kreger +Date: Thu, 30 Apr 2026 15:45:54 -0700 +Description: CVE-2026-44919: move file url validation up into deploy_utils main path + An issue was discovered where we were executing checksums + prior to doing file path guard logic. We've moved the check + into the same area of the code where we do all other url checks + for consistency. + . + This issue is tracked as CVE-2026-44919. + . + As a side note, on this specific branch, we may have accidently + had slightly better enforcing logic because we were previously + testing for this case in this branch and got a different error + because the overall flow now pre-flights the check far in + advance, but that was really only because the test itself + mocked out cache which was the vulnerable method to the current + CVE. +Bug: https://launchpad.net/bugs/2150332 +Bug-Debian: https://bugs.debian.org/1136655 +Change-Id: I09fa51801cf40da6f7be31014cca63ad1c253a5a +Signed-off-by: Julia Kreger +Origin: upstream, https://review.opendev.org/c/openstack/ironic/+/988480 +Last-Update: 2026-05-16 + +diff --git a/ironic/common/image_service.py b/ironic/common/image_service.py +index 1633b86..d4d05ba 100644 +--- a/ironic/common/image_service.py ++++ b/ironic/common/image_service.py +@@ -276,8 +276,9 @@ + doesn't exist, is in a blocked path, or is not in an allowed path. + :returns: Path to image file if it exists and is allowed. + """ ++ # TODO(TheJulia): Validate there are *THREE* slashes in the file path ++ # URL, otherwise urlparse doesn't split it properly. + image_path = urlparse.urlparse(image_href).path +- + # Check if the path is in the blocklist + rpath = os.path.abspath(image_path) + +diff --git a/ironic/drivers/modules/deploy_utils.py b/ironic/drivers/modules/deploy_utils.py +index f4f1ce0..ab8afcc 100644 +--- a/ironic/drivers/modules/deploy_utils.py ++++ b/ironic/drivers/modules/deploy_utils.py +@@ -1316,6 +1316,8 @@ + # and gets replaced at various points in this sequence. + instance_info['image_url'] = None + ++ is_file_url = image_source.startswith('file://') ++ + if service_utils.is_glance_image(image_source): + glance = image_service.GlanceImageService(context=task.context) + image_info = glance.show(image_source) +@@ -1357,8 +1359,7 @@ + if not iwdi and boot_option != 'local': + instance_info['kernel'] = image_info['properties']['kernel_id'] + instance_info['ramdisk'] = image_info['properties']['ramdisk_id'] +- elif (image_source.startswith('file://') +- or image_download_source == 'local'): ++ elif (is_file_url or image_download_source == 'local'): + # In this case, we're explicitly downloading (or copying a file) + # hosted locally so IPA can download it directly from Ironic. + +@@ -1366,7 +1367,12 @@ + # based deploy source since we don't want to, nor should we be in + # in the business of copying large numbers of files as it is a + # huge performance impact. +- ++ if is_file_url: ++ # In this case, we need to validate the URL first before ++ # moving on to _cache_and_convert_image, because it's whole ++ # existence is to download, checksum, convert, etc. ++ image_service.FileImageService().validate_href( ++ image_href=image_source) + _cache_and_convert_image(task, instance_info) + else: + # This is the "all other cases" logic for aspects like the user +diff --git a/ironic/tests/unit/drivers/modules/test_deploy_utils.py b/ironic/tests/unit/drivers/modules/test_deploy_utils.py +index 185bea6..dfe4a51 100644 +--- a/ironic/tests/unit/drivers/modules/test_deploy_utils.py ++++ b/ironic/tests/unit/drivers/modules/test_deploy_utils.py +@@ -2366,6 +2366,54 @@ + self.assertEqual('https://image-url/file', + task.node.instance_info['image_source']) + ++ @mock.patch.object(os.path, 'isfile', autospec=True) ++ @mock.patch.object(utils, '_cache_and_convert_image', autospec=True) ++ def test_build_instance_info_for_deploy_file_url_valid( ++ self, mock_cache_image, mock_isfile): ++ i_info = self.node.instance_info ++ driver_internal_info = self.node.driver_internal_info ++ i_info['image_source'] = 'file:///var/lib/ironic/files/foo/bar' ++ driver_internal_info['is_whole_disk_image'] = True ++ self.node.instance_info = i_info ++ self.node.driver_internal_info = driver_internal_info ++ self.node.save() ++ mock_isfile.return_value = True ++ with task_manager.acquire( ++ self.context, self.node.uuid, shared=False) as task: ++ ++ utils.build_instance_info_for_deploy(task) ++ mock_cache_image.assert_called_once_with( ++ mock.ANY, ++ {'configdrive': 'TG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQ=', ++ 'image_url': None, ++ 'foo': 'bar', ++ 'image_source': 'file:///var/lib/ironic/files/foo/bar', ++ 'image_type': 'whole-disk'}) ++ ++ @mock.patch.object(utils, 'cache_instance_image', autospec=True) ++ def test_build_instance_info_for_deploy_file_url_invalid( ++ self, mock_cache_image): ++ mock_cache_image.return_value = ('fake', '/tmp/foo', 'qcow2') ++ i_info = self.node.instance_info ++ driver_internal_info = self.node.driver_internal_info ++ url = 'file:///dev/zero' ++ i_info['image_source'] = url ++ self.node.instance_info = i_info ++ driver_internal_info['is_whole_disk_image'] = True ++ self.node.driver_internal_info = driver_internal_info ++ self.node.save() ++ ++ with task_manager.acquire( ++ self.context, self.node.uuid, shared=False) as task: ++ self.assertRaisesRegex( ++ exception.ImageRefValidationFailed, ++ 'Validation of image href file:///dev/zero failed, reason: ' ++ 'Security: Path /dev/zero is not allowed for image source ' ++ 'file URLs', ++ utils.build_instance_info_for_deploy, task) ++ ++ mock_cache_image.assert_not_called() ++ + + class TestBuildInstanceInfoForHttpProvisioning(db_base.DbTestCase): + def setUp(self): +@@ -2508,41 +2556,6 @@ + + @mock.patch.object(image_service.HttpImageService, 'validate_href', + autospec=True) +- def test_build_instance_info_file_image(self, validate_href_mock): +- i_info = self.node.instance_info +- driver_internal_info = self.node.driver_internal_info +- i_info['image_source'] = 'file://image-ref' +- i_info['image_checksum'] = 'aa' +- i_info['root_gb'] = 10 +- driver_internal_info['is_whole_disk_image'] = True +- self.node.instance_info = i_info +- self.node.driver_internal_info = driver_internal_info +- self.node.save() +- +- expected_url = ( +- 'http://172.172.24.10:8080/agent_images/%s' % self.node.uuid) +- +- with task_manager.acquire( +- self.context, self.node.uuid, shared=False) as task: +- +- info = utils.build_instance_info_for_deploy(task) +- +- self.assertEqual(expected_url, info['image_url']) +- self.assertEqual('sha256', info['image_os_hash_algo']) +- self.assertEqual('fake-checksum', info['image_os_hash_value']) +- self.assertEqual('raw', info['image_disk_format']) +- self.cache_image_mock.assert_called_once_with( +- task.context, task.node, force_raw=True, +- expected_format=None, +- expected_checksum='aa', +- expected_checksum_algo=None) +- self.checksum_mock.assert_called_once_with( +- self.fake_path, algorithm='sha256') +- validate_href_mock.assert_called_once_with( +- mock.ANY, expected_url, False) +- +- @mock.patch.object(image_service.HttpImageService, 'validate_href', +- autospec=True) + def test_build_instance_info_local_image(self, validate_href_mock): + cfg.CONF.set_override('image_download_source', 'local', group='agent') + i_info = self.node.instance_info +diff --git a/releasenotes/notes/fix-file-url-validation-5fb819ef5ae74ddc.yaml b/releasenotes/notes/fix-file-url-validation-5fb819ef5ae74ddc.yaml +new file mode 100644 +index 0000000..0b7b49a +--- /dev/null ++++ b/releasenotes/notes/fix-file-url-validation-5fb819ef5ae74ddc.yaml +@@ -0,0 +1,15 @@ ++--- ++fixes: ++ - | ++ Fixes an issue in the url handling logic of instance deploy logic where ++ file paths validity was not checked upfront, and the conductor service ++ would directly begin calculating checksums. This highlighted a DoS issue ++ where an attacker could exhaust conductor threads by attempting to request, ++ "file:///dev/zero", which would never return the thread. This is because ++ file URLs don't require downloading first, and validity checking logic ++ was all in the file access logic, not checksum logic. ++ ++ Ironic now explicitly invokes the URL/validity checking logic prior to ++ calling the checksum logic which effectively prevents this issue. ++ ++ This issue is tracked as CVE-2026-44919. diff -Nru ironic-21.1.0/debian/patches/CVE-2026-46447_Sanitize-kernel_append_parms.patch ironic-21.4.4/debian/patches/CVE-2026-46447_Sanitize-kernel_append_parms.patch --- ironic-21.1.0/debian/patches/CVE-2026-46447_Sanitize-kernel_append_parms.patch 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/debian/patches/CVE-2026-46447_Sanitize-kernel_append_parms.patch 2024-11-08 15:10:43.000000000 +0000 @@ -0,0 +1,4328 @@ +Author: Clif Houck +Date: Tue, 5 May 2026 17:21:57 -0500 +Description: CVE-2026-46447: Ensure kernel_append_params are valid kernel parameters + By defining a kernel command line grammar and attemping to parse + kernel_append_params. A successful parse indicates the input contained + in kernel_append_params are valid kernel parameters. Unsuccessful + parsing will raise and be rejected. + . + This parsing can be disabled through a new conductor configuration + option: disable_kernel_parameter_parsing which is False by default. + . + Basic kernel parameter sanitization (ie filtering newlines) is always + applied in kernel_append_params since they are never valid for + inclusion. + . + Future patches should extend kernel parameter parsing to all areas of + Ironic's code base in order to guarantee valid kernel parameters being + passed along. + . + NOTE: This patch is back-ported from stable/2026.{1,2} and slightly + weakens the kernel command line grammar by not including init arguments. + Lark's stand-alone LALR(1) parser can't handle the ambiguity introduced. + . + This commit addresses CVE-2026-46447. +Bug: https://launchpad.net/bugs/2150624 +Change-Id: I31ee960f6f055e39dd248f54ac853e21838632df +Signed-off-by: Clif Houck +Origin: upstream, pre-OSSA mailing list +Last-Update: 2026-06-01 + +diff --git a/LICENSE b/LICENSE +index 68c771a09..2902e77a7 100644 +--- a/LICENSE ++++ b/LICENSE +@@ -174,3 +174,5 @@ + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + ++ironic/common/kernel_parameter_parser/kernel_parameter_parser.py is governed ++by the MPL v2.0 license. See that file for more information. +diff --git a/ironic/common/kernel_parameter_parser/__init__.py b/ironic/common/kernel_parameter_parser/__init__.py +new file mode 100644 +index 000000000..e69de29bb +diff --git a/ironic/common/kernel_parameter_parser/grammar.g b/ironic/common/kernel_parameter_parser/grammar.g +new file mode 100644 +index 000000000..790971383 +--- /dev/null ++++ b/ironic/common/kernel_parameter_parser/grammar.g +@@ -0,0 +1,38 @@ ++// NOTE(clif): This grammar is used by lark to generate the parser found in ++// kernel_parameter_parser.py ++// ++// The following assumes your current working directory is the same as this ++// grammar file and the lark python library is available. Check the existing ++// generated parser to find the lark ++// version used. ++// ++// Use this command to regenerate the parser: ++// ++// $ python -m lark.tools.standalone grammar.g > kernel_parameter_parser.py ++// ++// The generated file (kernel_parameter_parser.py) will note at the beginning ++// of the file which version of lark was used to generate it. Additionally ++// note that lark requires python >= 3.8. Which means the stand-alone parser ++// *should* be fine for any recent-ish version of Ironic. ++ ++?start: kernel_command_line ++ ++kernel_command_line: parameter_list ++ ++parameter_list: parameter?(" "+ parameter)* ++ ++parameter: key ++ | key_value_pair ++ ++key_value_pair: key"="value ++ ++key: /[A-Za-z0-9_\-\.]+/ ++ ++value: bare_value ++ | quoted_value ++ ++quoted_value: "\"" value_with_spaces "\"" ++ ++bare_value: /[\!\#-\\.0-9:-\@A-Za-z\[-~]+/ ++ ++value_with_spaces: /[\!\#-\\.0-9:-\@A-Za-z\[-~ ]+/ +diff --git a/ironic/common/kernel_parameter_parser/kernel_parameter_parser.py b/ironic/common/kernel_parameter_parser/kernel_parameter_parser.py +new file mode 100644 +index 000000000..233df08f1 +--- /dev/null ++++ b/ironic/common/kernel_parameter_parser/kernel_parameter_parser.py +@@ -0,0 +1,3572 @@ ++# The file was automatically generated by Lark v1.3.1 ++__version__ = "1.3.1" ++ ++# ++# ++# Lark Stand-alone Generator Tool ++# ---------------------------------- ++# Generates a stand-alone LALR(1) parser ++# ++# Git: https://github.com/erezsh/lark ++# Author: Erez Shinan (erezshin@gmail.com) ++# ++# ++# >>> LICENSE ++# ++# This tool and its generated code use a separate license from Lark, ++# and are subject to the terms of the Mozilla Public License, v. 2.0. ++# If a copy of the MPL was not distributed with this ++# file, You can obtain one at https://mozilla.org/MPL/2.0/. ++# ++# If you wish to purchase a commercial license for this tool and its ++# generated code, you may contact me via email or otherwise. ++# ++# If MPL2 is incompatible with your free or open-source project, ++# contact me and we'll work it out. ++# ++# ++ ++from copy import deepcopy ++from abc import ABC, abstractmethod ++from types import ModuleType ++from typing import ( ++ TypeVar, Generic, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, ++ Union, Iterable, IO, TYPE_CHECKING, overload, Sequence, ++ Pattern as REPattern, ClassVar, Set, Mapping ++) ++ ++ ++class LarkError(Exception): ++ pass ++ ++ ++class ConfigurationError(LarkError, ValueError): ++ pass ++ ++ ++def assert_config(value, options: Collection, msg='Got %r, expected one of %s'): ++ if value not in options: ++ raise ConfigurationError(msg % (value, options)) ++ ++ ++class GrammarError(LarkError): ++ pass ++ ++ ++class ParseError(LarkError): ++ pass ++ ++ ++class LexError(LarkError): ++ pass ++ ++T = TypeVar('T') ++ ++class UnexpectedInput(LarkError): ++ #-- ++ line: int ++ column: int ++ pos_in_stream = None ++ state: Any ++ _terminals_by_name = None ++ interactive_parser: 'InteractiveParser' ++ ++ def get_context(self, text: str, span: int=40) -> str: ++ #-- ++ pos = self.pos_in_stream or 0 ++ start = max(pos - span, 0) ++ end = pos + span ++ if not isinstance(text, bytes): ++ before = text[start:pos].rsplit('\n', 1)[-1] ++ after = text[pos:end].split('\n', 1)[0] ++ return before + after + '\n' + ' ' * len(before.expandtabs()) + '^\n' ++ else: ++ before = text[start:pos].rsplit(b'\n', 1)[-1] ++ after = text[pos:end].split(b'\n', 1)[0] ++ return (before + after + b'\n' + b' ' * len(before.expandtabs()) + b'^\n').decode("ascii", "backslashreplace") ++ ++ def match_examples(self, parse_fn: 'Callable[[str], Tree]', ++ examples: Union[Mapping[T, Iterable[str]], Iterable[Tuple[T, Iterable[str]]]], ++ token_type_match_fallback: bool=False, ++ use_accepts: bool=True ++ ) -> Optional[T]: ++ #-- ++ assert self.state is not None, "Not supported for this exception" ++ ++ if isinstance(examples, Mapping): ++ examples = examples.items() ++ ++ candidate = (None, False) ++ for i, (label, example) in enumerate(examples): ++ assert not isinstance(example, str), "Expecting a list" ++ ++ for j, malformed in enumerate(example): ++ try: ++ parse_fn(malformed) ++ except UnexpectedInput as ut: ++ if ut.state == self.state: ++ if ( ++ use_accepts ++ and isinstance(self, UnexpectedToken) ++ and isinstance(ut, UnexpectedToken) ++ and ut.accepts != self.accepts ++ ): ++ logger.debug("Different accepts with same state[%d]: %s != %s at example [%s][%s]" % ++ (self.state, self.accepts, ut.accepts, i, j)) ++ continue ++ if ( ++ isinstance(self, (UnexpectedToken, UnexpectedEOF)) ++ and isinstance(ut, (UnexpectedToken, UnexpectedEOF)) ++ ): ++ if ut.token == self.token: ## ++ ++ logger.debug("Exact Match at example [%s][%s]" % (i, j)) ++ return label ++ ++ if token_type_match_fallback: ++ ## ++ ++ if (ut.token.type == self.token.type) and not candidate[-1]: ++ logger.debug("Token Type Fallback at example [%s][%s]" % (i, j)) ++ candidate = label, True ++ ++ if candidate[0] is None: ++ logger.debug("Same State match at example [%s][%s]" % (i, j)) ++ candidate = label, False ++ ++ return candidate[0] ++ ++ def _format_expected(self, expected): ++ if self._terminals_by_name: ++ d = self._terminals_by_name ++ expected = [d[t_name].user_repr() if t_name in d else t_name for t_name in expected] ++ return "Expected one of: \n\t* %s\n" % '\n\t* '.join(expected) ++ ++ ++class UnexpectedEOF(ParseError, UnexpectedInput): ++ #-- ++ expected: 'List[Token]' ++ ++ def __init__(self, expected, state=None, terminals_by_name=None): ++ super(UnexpectedEOF, self).__init__() ++ ++ self.expected = expected ++ self.state = state ++ from .lexer import Token ++ self.token = Token("", "") ## ++ ++ self.pos_in_stream = -1 ++ self.line = -1 ++ self.column = -1 ++ self._terminals_by_name = terminals_by_name ++ ++ ++ def __str__(self): ++ message = "Unexpected end-of-input. " ++ message += self._format_expected(self.expected) ++ return message ++ ++ ++class UnexpectedCharacters(LexError, UnexpectedInput): ++ #-- ++ ++ allowed: Set[str] ++ considered_tokens: Set[Any] ++ ++ def __init__(self, seq, lex_pos, line, column, allowed=None, considered_tokens=None, state=None, token_history=None, ++ terminals_by_name=None, considered_rules=None): ++ super(UnexpectedCharacters, self).__init__() ++ ++ ## ++ ++ self.line = line ++ self.column = column ++ self.pos_in_stream = lex_pos ++ self.state = state ++ self._terminals_by_name = terminals_by_name ++ ++ self.allowed = allowed ++ self.considered_tokens = considered_tokens ++ self.considered_rules = considered_rules ++ self.token_history = token_history ++ ++ if isinstance(seq, bytes): ++ self.char = seq[lex_pos:lex_pos + 1].decode("ascii", "backslashreplace") ++ else: ++ self.char = seq[lex_pos] ++ self._context = self.get_context(seq) ++ ++ ++ def __str__(self): ++ message = "No terminal matches '%s' in the current parser context, at line %d col %d" % (self.char, self.line, self.column) ++ message += '\n\n' + self._context ++ if self.allowed: ++ message += self._format_expected(self.allowed) ++ if self.token_history: ++ message += '\nPrevious tokens: %s\n' % ', '.join(repr(t) for t in self.token_history) ++ return message ++ ++ ++class UnexpectedToken(ParseError, UnexpectedInput): ++ #-- ++ ++ expected: Set[str] ++ considered_rules: Set[str] ++ ++ def __init__(self, token, expected, considered_rules=None, state=None, interactive_parser=None, terminals_by_name=None, token_history=None): ++ super(UnexpectedToken, self).__init__() ++ ++ ## ++ ++ self.line = getattr(token, 'line', '?') ++ self.column = getattr(token, 'column', '?') ++ self.pos_in_stream = getattr(token, 'start_pos', None) ++ self.state = state ++ ++ self.token = token ++ self.expected = expected ## ++ ++ self._accepts = NO_VALUE ++ self.considered_rules = considered_rules ++ self.interactive_parser = interactive_parser ++ self._terminals_by_name = terminals_by_name ++ self.token_history = token_history ++ ++ ++ @property ++ def accepts(self) -> Set[str]: ++ if self._accepts is NO_VALUE: ++ self._accepts = self.interactive_parser and self.interactive_parser.accepts() ++ return self._accepts ++ ++ def __str__(self): ++ message = ("Unexpected token %r at line %s, column %s.\n%s" ++ % (self.token, self.line, self.column, self._format_expected(self.accepts or self.expected))) ++ if self.token_history: ++ message += "Previous tokens: %r\n" % self.token_history ++ ++ return message ++ ++ ++ ++class VisitError(LarkError): ++ #-- ++ ++ obj: 'Union[Tree, Token]' ++ orig_exc: Exception ++ ++ def __init__(self, rule, obj, orig_exc): ++ message = 'Error trying to process rule "%s":\n\n%s' % (rule, orig_exc) ++ super(VisitError, self).__init__(message) ++ ++ self.rule = rule ++ self.obj = obj ++ self.orig_exc = orig_exc ++ ++ ++class MissingVariableError(LarkError): ++ pass ++ ++ ++import sys, re ++import logging ++from dataclasses import dataclass ++from typing import Generic, AnyStr ++ ++logger: logging.Logger = logging.getLogger("lark") ++logger.addHandler(logging.StreamHandler()) ++## ++ ++## ++ ++logger.setLevel(logging.CRITICAL) ++ ++ ++NO_VALUE = object() ++ ++T = TypeVar("T") ++ ++ ++def classify(seq: Iterable, key: Optional[Callable] = None, value: Optional[Callable] = None) -> Dict: ++ d: Dict[Any, Any] = {} ++ for item in seq: ++ k = key(item) if (key is not None) else item ++ v = value(item) if (value is not None) else item ++ try: ++ d[k].append(v) ++ except KeyError: ++ d[k] = [v] ++ return d ++ ++ ++def _deserialize(data: Any, namespace: Dict[str, Any], memo: Dict) -> Any: ++ if isinstance(data, dict): ++ if '__type__' in data: ## ++ ++ class_ = namespace[data['__type__']] ++ return class_.deserialize(data, memo) ++ elif '@' in data: ++ return memo[data['@']] ++ return {key:_deserialize(value, namespace, memo) for key, value in data.items()} ++ elif isinstance(data, list): ++ return [_deserialize(value, namespace, memo) for value in data] ++ return data ++ ++ ++_T = TypeVar("_T", bound="Serialize") ++ ++class Serialize: ++ #-- ++ ++ def memo_serialize(self, types_to_memoize: List) -> Any: ++ memo = SerializeMemoizer(types_to_memoize) ++ return self.serialize(memo), memo.serialize() ++ ++ def serialize(self, memo = None) -> Dict[str, Any]: ++ if memo and memo.in_types(self): ++ return {'@': memo.memoized.get(self)} ++ ++ fields = getattr(self, '__serialize_fields__') ++ res = {f: _serialize(getattr(self, f), memo) for f in fields} ++ res['__type__'] = type(self).__name__ ++ if hasattr(self, '_serialize'): ++ self._serialize(res, memo) ++ return res ++ ++ @classmethod ++ def deserialize(cls: Type[_T], data: Dict[str, Any], memo: Dict[int, Any]) -> _T: ++ namespace = getattr(cls, '__serialize_namespace__', []) ++ namespace = {c.__name__:c for c in namespace} ++ ++ fields = getattr(cls, '__serialize_fields__') ++ ++ if '@' in data: ++ return memo[data['@']] ++ ++ inst = cls.__new__(cls) ++ for f in fields: ++ try: ++ setattr(inst, f, _deserialize(data[f], namespace, memo)) ++ except KeyError as e: ++ raise KeyError("Cannot find key for class", cls, e) ++ ++ if hasattr(inst, '_deserialize'): ++ inst._deserialize() ++ ++ return inst ++ ++ ++class SerializeMemoizer(Serialize): ++ #-- ++ ++ __serialize_fields__ = 'memoized', ++ ++ def __init__(self, types_to_memoize: List) -> None: ++ self.types_to_memoize = tuple(types_to_memoize) ++ self.memoized = Enumerator() ++ ++ def in_types(self, value: Serialize) -> bool: ++ return isinstance(value, self.types_to_memoize) ++ ++ def serialize(self) -> Dict[int, Any]: ## ++ ++ return _serialize(self.memoized.reversed(), None) ++ ++ @classmethod ++ def deserialize(cls, data: Dict[int, Any], namespace: Dict[str, Any], memo: Dict[Any, Any]) -> Dict[int, Any]: ## ++ ++ return _deserialize(data, namespace, memo) ++ ++ ++try: ++ import regex ++ _has_regex = True ++except ImportError: ++ _has_regex = False ++ ++if sys.version_info >= (3, 11): ++ import re._parser as sre_parse ++ import re._constants as sre_constants ++else: ++ import sre_parse ++ import sre_constants ++ ++categ_pattern = re.compile(r'\\p{[A-Za-z_]+}') ++ ++def get_regexp_width(expr: str) -> Union[Tuple[int, int], List[int]]: ++ if _has_regex: ++ ## ++ ++ ## ++ ++ ## ++ ++ regexp_final = re.sub(categ_pattern, 'A', expr) ++ else: ++ if re.search(categ_pattern, expr): ++ raise ImportError('`regex` module must be installed in order to use Unicode categories.', expr) ++ regexp_final = expr ++ try: ++ ## ++ ++ return [int(x) for x in sre_parse.parse(regexp_final).getwidth()] ++ except sre_constants.error: ++ if not _has_regex: ++ raise ValueError(expr) ++ else: ++ ## ++ ++ ## ++ ++ c = regex.compile(regexp_final) ++ ## ++ ++ ## ++ ++ MAXWIDTH = getattr(sre_parse, "MAXWIDTH", sre_constants.MAXREPEAT) ++ if c.match('') is None: ++ ## ++ ++ return 1, int(MAXWIDTH) ++ else: ++ return 0, int(MAXWIDTH) ++ ++ ++@dataclass(frozen=True) ++class TextSlice(Generic[AnyStr]): ++ #-- ++ text: AnyStr ++ start: int ++ end: int ++ ++ def __post_init__(self): ++ if not isinstance(self.text, (str, bytes)): ++ raise TypeError("text must be str or bytes") ++ ++ if self.start < 0: ++ object.__setattr__(self, 'start', self.start + len(self.text)) ++ assert self.start >=0 ++ ++ if self.end is None: ++ object.__setattr__(self, 'end', len(self.text)) ++ elif self.end < 0: ++ object.__setattr__(self, 'end', self.end + len(self.text)) ++ assert self.end <= len(self.text) ++ ++ @classmethod ++ def cast_from(cls, text: 'TextOrSlice') -> 'TextSlice[AnyStr]': ++ if isinstance(text, TextSlice): ++ return text ++ ++ return cls(text, 0, len(text)) ++ ++ def is_complete_text(self): ++ return self.start == 0 and self.end == len(self.text) ++ ++ def __len__(self): ++ return self.end - self.start ++ ++ def count(self, substr: AnyStr): ++ return self.text.count(substr, self.start, self.end) ++ ++ def rindex(self, substr: AnyStr): ++ return self.text.rindex(substr, self.start, self.end) ++ ++ ++TextOrSlice = Union[AnyStr, 'TextSlice[AnyStr]'] ++LarkInput = Union[AnyStr, TextSlice[AnyStr], Any] ++ ++ ++ ++class Meta: ++ ++ empty: bool ++ line: int ++ column: int ++ start_pos: int ++ end_line: int ++ end_column: int ++ end_pos: int ++ orig_expansion: 'List[TerminalDef]' ++ match_tree: bool ++ ++ def __init__(self): ++ self.empty = True ++ ++ ++_Leaf_T = TypeVar("_Leaf_T") ++Branch = Union[_Leaf_T, 'Tree[_Leaf_T]'] ++ ++ ++class Tree(Generic[_Leaf_T]): ++ #-- ++ ++ data: str ++ children: 'List[Branch[_Leaf_T]]' ++ ++ def __init__(self, data: str, children: 'List[Branch[_Leaf_T]]', meta: Optional[Meta]=None) -> None: ++ self.data = data ++ self.children = children ++ self._meta = meta ++ ++ @property ++ def meta(self) -> Meta: ++ if self._meta is None: ++ self._meta = Meta() ++ return self._meta ++ ++ def __repr__(self): ++ return 'Tree(%r, %r)' % (self.data, self.children) ++ ++ __match_args__ = ("data", "children") ++ ++ def _pretty_label(self): ++ return self.data ++ ++ def _pretty(self, level, indent_str): ++ yield f'{indent_str*level}{self._pretty_label()}' ++ if len(self.children) == 1 and not isinstance(self.children[0], Tree): ++ yield f'\t{self.children[0]}\n' ++ else: ++ yield '\n' ++ for n in self.children: ++ if isinstance(n, Tree): ++ yield from n._pretty(level+1, indent_str) ++ else: ++ yield f'{indent_str*(level+1)}{n}\n' ++ ++ def pretty(self, indent_str: str=' ') -> str: ++ #-- ++ return ''.join(self._pretty(0, indent_str)) ++ ++ def __rich__(self, parent:Optional['rich.tree.Tree']=None) -> 'rich.tree.Tree': ++ #-- ++ return self._rich(parent) ++ ++ def _rich(self, parent): ++ if parent: ++ tree = parent.add(f'[bold]{self.data}[/bold]') ++ else: ++ import rich.tree ++ tree = rich.tree.Tree(self.data) ++ ++ for c in self.children: ++ if isinstance(c, Tree): ++ c._rich(tree) ++ else: ++ tree.add(f'[green]{c}[/green]') ++ ++ return tree ++ ++ def __eq__(self, other): ++ try: ++ return self.data == other.data and self.children == other.children ++ except AttributeError: ++ return False ++ ++ def __ne__(self, other): ++ return not (self == other) ++ ++ def __hash__(self) -> int: ++ return hash((self.data, tuple(self.children))) ++ ++ def iter_subtrees(self) -> 'Iterator[Tree[_Leaf_T]]': ++ #-- ++ queue = [self] ++ subtrees = dict() ++ for subtree in queue: ++ subtrees[id(subtree)] = subtree ++ queue += [c for c in reversed(subtree.children) ++ if isinstance(c, Tree) and id(c) not in subtrees] ++ ++ del queue ++ return reversed(list(subtrees.values())) ++ ++ def iter_subtrees_topdown(self): ++ #-- ++ stack = [self] ++ stack_append = stack.append ++ stack_pop = stack.pop ++ while stack: ++ node = stack_pop() ++ if not isinstance(node, Tree): ++ continue ++ yield node ++ for child in reversed(node.children): ++ stack_append(child) ++ ++ def find_pred(self, pred: 'Callable[[Tree[_Leaf_T]], bool]') -> 'Iterator[Tree[_Leaf_T]]': ++ #-- ++ return filter(pred, self.iter_subtrees()) ++ ++ def find_data(self, data: str) -> 'Iterator[Tree[_Leaf_T]]': ++ #-- ++ return self.find_pred(lambda t: t.data == data) ++ ++ ++from functools import wraps, update_wrapper ++from inspect import getmembers, getmro ++ ++_Return_T = TypeVar('_Return_T') ++_Return_V = TypeVar('_Return_V') ++_Leaf_T = TypeVar('_Leaf_T') ++_Leaf_U = TypeVar('_Leaf_U') ++_R = TypeVar('_R') ++_FUNC = Callable[..., _Return_T] ++_DECORATED = Union[_FUNC, type] ++ ++class _DiscardType: ++ #-- ++ ++ def __repr__(self): ++ return "lark.visitors.Discard" ++ ++Discard = _DiscardType() ++ ++## ++ ++ ++class _Decoratable: ++ #-- ++ ++ @classmethod ++ def _apply_v_args(cls, visit_wrapper): ++ mro = getmro(cls) ++ assert mro[0] is cls ++ libmembers = {name for _cls in mro[1:] for name, _ in getmembers(_cls)} ++ for name, value in getmembers(cls): ++ ++ ## ++ ++ if name.startswith('_') or (name in libmembers and name not in cls.__dict__): ++ continue ++ if not callable(value): ++ continue ++ ++ ## ++ ++ if isinstance(cls.__dict__[name], _VArgsWrapper): ++ continue ++ ++ setattr(cls, name, _VArgsWrapper(cls.__dict__[name], visit_wrapper)) ++ return cls ++ ++ def __class_getitem__(cls, _): ++ return cls ++ ++ ++class Transformer(_Decoratable, ABC, Generic[_Leaf_T, _Return_T]): ++ #-- ++ __visit_tokens__ = True ## ++ ++ ++ def __init__(self, visit_tokens: bool=True) -> None: ++ self.__visit_tokens__ = visit_tokens ++ ++ def _call_userfunc(self, tree, new_children=None): ++ ## ++ ++ children = new_children if new_children is not None else tree.children ++ try: ++ f = getattr(self, tree.data) ++ except AttributeError: ++ return self.__default__(tree.data, children, tree.meta) ++ else: ++ try: ++ wrapper = getattr(f, 'visit_wrapper', None) ++ if wrapper is not None: ++ return f.visit_wrapper(f, tree.data, children, tree.meta) ++ else: ++ return f(children) ++ except GrammarError: ++ raise ++ except Exception as e: ++ raise VisitError(tree.data, tree, e) ++ ++ def _call_userfunc_token(self, token): ++ try: ++ f = getattr(self, token.type) ++ except AttributeError: ++ return self.__default_token__(token) ++ else: ++ try: ++ return f(token) ++ except GrammarError: ++ raise ++ except Exception as e: ++ raise VisitError(token.type, token, e) ++ ++ def _transform_children(self, children): ++ for c in children: ++ if isinstance(c, Tree): ++ res = self._transform_tree(c) ++ elif self.__visit_tokens__ and isinstance(c, Token): ++ res = self._call_userfunc_token(c) ++ else: ++ res = c ++ ++ if res is not Discard: ++ yield res ++ ++ def _transform_tree(self, tree): ++ children = list(self._transform_children(tree.children)) ++ return self._call_userfunc(tree, children) ++ ++ def transform(self, tree: Tree[_Leaf_T]) -> _Return_T: ++ #-- ++ res = list(self._transform_children([tree])) ++ if not res: ++ return None ## ++ ++ assert len(res) == 1 ++ return res[0] ++ ++ def __mul__( ++ self: 'Transformer[_Leaf_T, Tree[_Leaf_U]]', ++ other: 'Union[Transformer[_Leaf_U, _Return_V], TransformerChain[_Leaf_U, _Return_V,]]' ++ ) -> 'TransformerChain[_Leaf_T, _Return_V]': ++ #-- ++ return TransformerChain(self, other) ++ ++ def __default__(self, data, children, meta): ++ #-- ++ return Tree(data, children, meta) ++ ++ def __default_token__(self, token): ++ #-- ++ return token ++ ++ ++def merge_transformers(base_transformer=None, **transformers_to_merge): ++ #-- ++ if base_transformer is None: ++ base_transformer = Transformer() ++ for prefix, transformer in transformers_to_merge.items(): ++ for method_name in dir(transformer): ++ method = getattr(transformer, method_name) ++ if not callable(method): ++ continue ++ if method_name.startswith("_") or method_name == "transform": ++ continue ++ prefixed_method = prefix + "__" + method_name ++ if hasattr(base_transformer, prefixed_method): ++ raise AttributeError("Cannot merge: method '%s' appears more than once" % prefixed_method) ++ ++ setattr(base_transformer, prefixed_method, method) ++ ++ return base_transformer ++ ++ ++class InlineTransformer(Transformer): ## ++ ++ def _call_userfunc(self, tree, new_children=None): ++ ## ++ ++ children = new_children if new_children is not None else tree.children ++ try: ++ f = getattr(self, tree.data) ++ except AttributeError: ++ return self.__default__(tree.data, children, tree.meta) ++ else: ++ return f(*children) ++ ++ ++class TransformerChain(Generic[_Leaf_T, _Return_T]): ++ ++ transformers: 'Tuple[Union[Transformer, TransformerChain], ...]' ++ ++ def __init__(self, *transformers: 'Union[Transformer, TransformerChain]') -> None: ++ self.transformers = transformers ++ ++ def transform(self, tree: Tree[_Leaf_T]) -> _Return_T: ++ for t in self.transformers: ++ tree = t.transform(tree) ++ return cast(_Return_T, tree) ++ ++ def __mul__( ++ self: 'TransformerChain[_Leaf_T, Tree[_Leaf_U]]', ++ other: 'Union[Transformer[_Leaf_U, _Return_V], TransformerChain[_Leaf_U, _Return_V]]' ++ ) -> 'TransformerChain[_Leaf_T, _Return_V]': ++ return TransformerChain(*self.transformers + (other,)) ++ ++ ++class Transformer_InPlace(Transformer[_Leaf_T, _Return_T]): ++ #-- ++ def _transform_tree(self, tree): ## ++ ++ return self._call_userfunc(tree) ++ ++ def transform(self, tree: Tree[_Leaf_T]) -> _Return_T: ++ for subtree in tree.iter_subtrees(): ++ subtree.children = list(self._transform_children(subtree.children)) ++ ++ return self._transform_tree(tree) ++ ++ ++class Transformer_NonRecursive(Transformer[_Leaf_T, _Return_T]): ++ #-- ++ ++ def transform(self, tree: Tree[_Leaf_T]) -> _Return_T: ++ ## ++ ++ rev_postfix = [] ++ q: List[Branch[_Leaf_T]] = [tree] ++ while q: ++ t = q.pop() ++ rev_postfix.append(t) ++ if isinstance(t, Tree): ++ q += t.children ++ ++ ## ++ ++ stack: List = [] ++ for x in reversed(rev_postfix): ++ if isinstance(x, Tree): ++ size = len(x.children) ++ if size: ++ args = stack[-size:] ++ del stack[-size:] ++ else: ++ args = [] ++ ++ res = self._call_userfunc(x, args) ++ if res is not Discard: ++ stack.append(res) ++ ++ elif self.__visit_tokens__ and isinstance(x, Token): ++ res = self._call_userfunc_token(x) ++ if res is not Discard: ++ stack.append(res) ++ else: ++ stack.append(x) ++ ++ result, = stack ## ++ ++ ## ++ ++ ## ++ ++ ## ++ ++ return cast(_Return_T, result) ++ ++ ++class Transformer_InPlaceRecursive(Transformer[_Leaf_T, _Return_T]): ++ #-- ++ def _transform_tree(self, tree): ++ tree.children = list(self._transform_children(tree.children)) ++ return self._call_userfunc(tree) ++ ++ ++## ++ ++ ++class VisitorBase: ++ def _call_userfunc(self, tree): ++ return getattr(self, tree.data, self.__default__)(tree) ++ ++ def __default__(self, tree): ++ #-- ++ return tree ++ ++ def __class_getitem__(cls, _): ++ return cls ++ ++ ++class Visitor(VisitorBase, ABC, Generic[_Leaf_T]): ++ #-- ++ ++ def visit(self, tree: Tree[_Leaf_T]) -> Tree[_Leaf_T]: ++ #-- ++ for subtree in tree.iter_subtrees(): ++ self._call_userfunc(subtree) ++ return tree ++ ++ def visit_topdown(self, tree: Tree[_Leaf_T]) -> Tree[_Leaf_T]: ++ #-- ++ for subtree in tree.iter_subtrees_topdown(): ++ self._call_userfunc(subtree) ++ return tree ++ ++ ++class Visitor_Recursive(VisitorBase, Generic[_Leaf_T]): ++ #-- ++ ++ def visit(self, tree: Tree[_Leaf_T]) -> Tree[_Leaf_T]: ++ #-- ++ for child in tree.children: ++ if isinstance(child, Tree): ++ self.visit(child) ++ ++ self._call_userfunc(tree) ++ return tree ++ ++ def visit_topdown(self,tree: Tree[_Leaf_T]) -> Tree[_Leaf_T]: ++ #-- ++ self._call_userfunc(tree) ++ ++ for child in tree.children: ++ if isinstance(child, Tree): ++ self.visit_topdown(child) ++ ++ return tree ++ ++ ++class Interpreter(_Decoratable, ABC, Generic[_Leaf_T, _Return_T]): ++ #-- ++ ++ def visit(self, tree: Tree[_Leaf_T]) -> _Return_T: ++ ## ++ ++ ## ++ ++ ## ++ ++ return self._visit_tree(tree) ++ ++ def _visit_tree(self, tree: Tree[_Leaf_T]): ++ f = getattr(self, tree.data) ++ wrapper = getattr(f, 'visit_wrapper', None) ++ if wrapper is not None: ++ return f.visit_wrapper(f, tree.data, tree.children, tree.meta) ++ else: ++ return f(tree) ++ ++ def visit_children(self, tree: Tree[_Leaf_T]) -> List: ++ return [self._visit_tree(child) if isinstance(child, Tree) else child ++ for child in tree.children] ++ ++ def __getattr__(self, name): ++ return self.__default__ ++ ++ def __default__(self, tree): ++ return self.visit_children(tree) ++ ++ ++_InterMethod = Callable[[Type[Interpreter], _Return_T], _R] ++ ++def visit_children_decor(func: _InterMethod) -> _InterMethod: ++ #-- ++ @wraps(func) ++ def inner(cls, tree): ++ values = cls.visit_children(tree) ++ return func(cls, values) ++ return inner ++ ++## ++ ++ ++def _apply_v_args(obj, visit_wrapper): ++ try: ++ _apply = obj._apply_v_args ++ except AttributeError: ++ return _VArgsWrapper(obj, visit_wrapper) ++ else: ++ return _apply(visit_wrapper) ++ ++ ++class _VArgsWrapper: ++ #-- ++ base_func: Callable ++ ++ def __init__(self, func: Callable, visit_wrapper: Callable[[Callable, str, list, Any], Any]): ++ if isinstance(func, _VArgsWrapper): ++ func = func.base_func ++ self.base_func = func ++ self.visit_wrapper = visit_wrapper ++ update_wrapper(self, func) ++ ++ def __call__(self, *args, **kwargs): ++ return self.base_func(*args, **kwargs) ++ ++ def __get__(self, instance, owner=None): ++ try: ++ ## ++ ++ ## ++ ++ g = type(self.base_func).__get__ ++ except AttributeError: ++ return self ++ else: ++ return _VArgsWrapper(g(self.base_func, instance, owner), self.visit_wrapper) ++ ++ def __set_name__(self, owner, name): ++ try: ++ f = type(self.base_func).__set_name__ ++ except AttributeError: ++ return ++ else: ++ f(self.base_func, owner, name) ++ ++ ++def _vargs_inline(f, _data, children, _meta): ++ return f(*children) ++def _vargs_meta_inline(f, _data, children, meta): ++ return f(meta, *children) ++def _vargs_meta(f, _data, children, meta): ++ return f(meta, children) ++def _vargs_tree(f, data, children, meta): ++ return f(Tree(data, children, meta)) ++ ++ ++def v_args(inline: bool = False, meta: bool = False, tree: bool = False, wrapper: Optional[Callable] = None) -> Callable[[_DECORATED], _DECORATED]: ++ #-- ++ if tree and (meta or inline): ++ raise ValueError("Visitor functions cannot combine 'tree' with 'meta' or 'inline'.") ++ ++ func = None ++ if meta: ++ if inline: ++ func = _vargs_meta_inline ++ else: ++ func = _vargs_meta ++ elif inline: ++ func = _vargs_inline ++ elif tree: ++ func = _vargs_tree ++ ++ if wrapper is not None: ++ if func is not None: ++ raise ValueError("Cannot use 'wrapper' along with 'tree', 'meta' or 'inline'.") ++ func = wrapper ++ ++ def _visitor_args_dec(obj): ++ return _apply_v_args(obj, func) ++ return _visitor_args_dec ++ ++ ++ ++TOKEN_DEFAULT_PRIORITY = 0 ++ ++ ++class Symbol(Serialize): ++ __slots__ = ('name',) ++ ++ name: str ++ is_term: ClassVar[bool] = NotImplemented ++ ++ def __init__(self, name: str) -> None: ++ self.name = name ++ ++ def __eq__(self, other): ++ if not isinstance(other, Symbol): ++ return NotImplemented ++ return self.is_term == other.is_term and self.name == other.name ++ ++ def __ne__(self, other): ++ return not (self == other) ++ ++ def __hash__(self): ++ return hash(self.name) ++ ++ def __repr__(self): ++ return '%s(%r)' % (type(self).__name__, self.name) ++ ++ fullrepr = property(__repr__) ++ ++ def renamed(self, f): ++ return type(self)(f(self.name)) ++ ++ ++class Terminal(Symbol): ++ __serialize_fields__ = 'name', 'filter_out' ++ ++ is_term: ClassVar[bool] = True ++ ++ def __init__(self, name: str, filter_out: bool = False) -> None: ++ self.name = name ++ self.filter_out = filter_out ++ ++ @property ++ def fullrepr(self): ++ return '%s(%r, %r)' % (type(self).__name__, self.name, self.filter_out) ++ ++ def renamed(self, f): ++ return type(self)(f(self.name), self.filter_out) ++ ++ ++class NonTerminal(Symbol): ++ __serialize_fields__ = 'name', ++ ++ is_term: ClassVar[bool] = False ++ ++ def serialize(self, memo=None) -> Dict[str, Any]: ++ ## ++ ++ ## ++ ++ return {'name': str(self.name), '__type__': 'NonTerminal'} ++ ++ ++class RuleOptions(Serialize): ++ __serialize_fields__ = 'keep_all_tokens', 'expand1', 'priority', 'template_source', 'empty_indices' ++ ++ keep_all_tokens: bool ++ expand1: bool ++ priority: Optional[int] ++ template_source: Optional[str] ++ empty_indices: Tuple[bool, ...] ++ ++ def __init__(self, keep_all_tokens: bool=False, expand1: bool=False, priority: Optional[int]=None, template_source: Optional[str]=None, empty_indices: Tuple[bool, ...]=()) -> None: ++ self.keep_all_tokens = keep_all_tokens ++ self.expand1 = expand1 ++ self.priority = priority ++ self.template_source = template_source ++ self.empty_indices = empty_indices ++ ++ def __repr__(self): ++ return 'RuleOptions(%r, %r, %r, %r)' % ( ++ self.keep_all_tokens, ++ self.expand1, ++ self.priority, ++ self.template_source ++ ) ++ ++ ++class Rule(Serialize): ++ #-- ++ __slots__ = ('origin', 'expansion', 'alias', 'options', 'order', '_hash') ++ ++ __serialize_fields__ = 'origin', 'expansion', 'order', 'alias', 'options' ++ __serialize_namespace__ = Terminal, NonTerminal, RuleOptions ++ ++ origin: NonTerminal ++ expansion: Sequence[Symbol] ++ order: int ++ alias: Optional[str] ++ options: RuleOptions ++ _hash: int ++ ++ def __init__(self, origin: NonTerminal, expansion: Sequence[Symbol], ++ order: int=0, alias: Optional[str]=None, options: Optional[RuleOptions]=None): ++ self.origin = origin ++ self.expansion = expansion ++ self.alias = alias ++ self.order = order ++ self.options = options or RuleOptions() ++ self._hash = hash((self.origin, tuple(self.expansion))) ++ ++ def _deserialize(self): ++ self._hash = hash((self.origin, tuple(self.expansion))) ++ ++ def __str__(self): ++ return '<%s : %s>' % (self.origin.name, ' '.join(x.name for x in self.expansion)) ++ ++ def __repr__(self): ++ return 'Rule(%r, %r, %r, %r)' % (self.origin, self.expansion, self.alias, self.options) ++ ++ def __hash__(self): ++ return self._hash ++ ++ def __eq__(self, other): ++ if not isinstance(other, Rule): ++ return False ++ return self.origin == other.origin and self.expansion == other.expansion ++ ++ ++ ++from contextlib import suppress ++from copy import copy ++ ++try: ## ++ ++ has_interegular = bool(interegular) ++except NameError: ++ has_interegular = False ++ ++class Pattern(Serialize, ABC): ++ #-- ++ ++ value: str ++ flags: Collection[str] ++ raw: Optional[str] ++ type: ClassVar[str] ++ ++ def __init__(self, value: str, flags: Collection[str] = (), raw: Optional[str] = None) -> None: ++ self.value = value ++ self.flags = frozenset(flags) ++ self.raw = raw ++ ++ def __repr__(self): ++ return repr(self.to_regexp()) ++ ++ ## ++ ++ def __hash__(self): ++ return hash((type(self), self.value, self.flags)) ++ ++ def __eq__(self, other): ++ return type(self) == type(other) and self.value == other.value and self.flags == other.flags ++ ++ @abstractmethod ++ def to_regexp(self) -> str: ++ raise NotImplementedError() ++ ++ @property ++ @abstractmethod ++ def min_width(self) -> int: ++ raise NotImplementedError() ++ ++ @property ++ @abstractmethod ++ def max_width(self) -> int: ++ raise NotImplementedError() ++ ++ def _get_flags(self, value): ++ for f in self.flags: ++ value = ('(?%s:%s)' % (f, value)) ++ return value ++ ++ ++class PatternStr(Pattern): ++ __serialize_fields__ = 'value', 'flags', 'raw' ++ ++ type: ClassVar[str] = "str" ++ ++ def to_regexp(self) -> str: ++ return self._get_flags(re.escape(self.value)) ++ ++ @property ++ def min_width(self) -> int: ++ return len(self.value) ++ ++ @property ++ def max_width(self) -> int: ++ return len(self.value) ++ ++ ++class PatternRE(Pattern): ++ __serialize_fields__ = 'value', 'flags', 'raw', '_width' ++ ++ type: ClassVar[str] = "re" ++ ++ def to_regexp(self) -> str: ++ return self._get_flags(self.value) ++ ++ _width = None ++ def _get_width(self): ++ if self._width is None: ++ self._width = get_regexp_width(self.to_regexp()) ++ return self._width ++ ++ @property ++ def min_width(self) -> int: ++ return self._get_width()[0] ++ ++ @property ++ def max_width(self) -> int: ++ return self._get_width()[1] ++ ++ ++class TerminalDef(Serialize): ++ #-- ++ __serialize_fields__ = 'name', 'pattern', 'priority' ++ __serialize_namespace__ = PatternStr, PatternRE ++ ++ name: str ++ pattern: Pattern ++ priority: int ++ ++ def __init__(self, name: str, pattern: Pattern, priority: int = TOKEN_DEFAULT_PRIORITY) -> None: ++ assert isinstance(pattern, Pattern), pattern ++ self.name = name ++ self.pattern = pattern ++ self.priority = priority ++ ++ def __repr__(self): ++ return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern) ++ ++ def user_repr(self) -> str: ++ if self.name.startswith('__'): ## ++ ++ return self.pattern.raw or self.name ++ else: ++ return self.name ++ ++_T = TypeVar('_T', bound="Token") ++ ++class Token(str): ++ #-- ++ __slots__ = ('type', 'start_pos', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') ++ ++ __match_args__ = ('type', 'value') ++ ++ type: str ++ start_pos: Optional[int] ++ value: Any ++ line: Optional[int] ++ column: Optional[int] ++ end_line: Optional[int] ++ end_column: Optional[int] ++ end_pos: Optional[int] ++ ++ ++ @overload ++ def __new__( ++ cls, ++ type: str, ++ value: Any, ++ start_pos: Optional[int] = None, ++ line: Optional[int] = None, ++ column: Optional[int] = None, ++ end_line: Optional[int] = None, ++ end_column: Optional[int] = None, ++ end_pos: Optional[int] = None ++ ) -> 'Token': ++ ... ++ ++ @overload ++ def __new__( ++ cls, ++ type_: str, ++ value: Any, ++ start_pos: Optional[int] = None, ++ line: Optional[int] = None, ++ column: Optional[int] = None, ++ end_line: Optional[int] = None, ++ end_column: Optional[int] = None, ++ end_pos: Optional[int] = None ++ ) -> 'Token': ... ++ ++ def __new__(cls, *args, **kwargs): ++ if "type_" in kwargs: ++ warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning) ++ ++ if "type" in kwargs: ++ raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.") ++ kwargs["type"] = kwargs.pop("type_") ++ ++ return cls._future_new(*args, **kwargs) ++ ++ ++ @classmethod ++ def _future_new(cls, type, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): ++ inst = super(Token, cls).__new__(cls, value) ++ ++ inst.type = type ++ inst.start_pos = start_pos ++ inst.value = value ++ inst.line = line ++ inst.column = column ++ inst.end_line = end_line ++ inst.end_column = end_column ++ inst.end_pos = end_pos ++ return inst ++ ++ @overload ++ def update(self, type: Optional[str] = None, value: Optional[Any] = None) -> 'Token': ++ ... ++ ++ @overload ++ def update(self, type_: Optional[str] = None, value: Optional[Any] = None) -> 'Token': ++ ... ++ ++ def update(self, *args, **kwargs): ++ if "type_" in kwargs: ++ warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning) ++ ++ if "type" in kwargs: ++ raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.") ++ kwargs["type"] = kwargs.pop("type_") ++ ++ return self._future_update(*args, **kwargs) ++ ++ def _future_update(self, type: Optional[str] = None, value: Optional[Any] = None) -> 'Token': ++ return Token.new_borrow_pos( ++ type if type is not None else self.type, ++ value if value is not None else self.value, ++ self ++ ) ++ ++ @classmethod ++ def new_borrow_pos(cls: Type[_T], type_: str, value: Any, borrow_t: 'Token') -> _T: ++ return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) ++ ++ def __reduce__(self): ++ return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column)) ++ ++ def __repr__(self): ++ return 'Token(%r, %r)' % (self.type, self.value) ++ ++ def __deepcopy__(self, memo): ++ return Token(self.type, self.value, self.start_pos, self.line, self.column) ++ ++ def __eq__(self, other): ++ if isinstance(other, Token) and self.type != other.type: ++ return False ++ ++ return str.__eq__(self, other) ++ ++ __hash__ = str.__hash__ ++ ++ ++class LineCounter: ++ #-- ++ ++ __slots__ = 'char_pos', 'line', 'column', 'line_start_pos', 'newline_char' ++ ++ def __init__(self, newline_char): ++ self.newline_char = newline_char ++ self.char_pos = 0 ++ self.line = 1 ++ self.column = 1 ++ self.line_start_pos = 0 ++ ++ def __eq__(self, other): ++ if not isinstance(other, LineCounter): ++ return NotImplemented ++ ++ return self.char_pos == other.char_pos and self.newline_char == other.newline_char ++ ++ def feed(self, token: TextOrSlice, test_newline=True): ++ #-- ++ if test_newline: ++ newlines = token.count(self.newline_char) ++ if newlines: ++ self.line += newlines ++ self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1 ++ ++ self.char_pos += len(token) ++ self.column = self.char_pos - self.line_start_pos + 1 ++ ++ ++class UnlessCallback: ++ def __init__(self, scanner: 'Scanner'): ++ self.scanner = scanner ++ ++ def __call__(self, t: Token): ++ res = self.scanner.fullmatch(t.value) ++ if res is not None: ++ t.type = res ++ return t ++ ++ ++class CallChain: ++ def __init__(self, callback1, callback2, cond): ++ self.callback1 = callback1 ++ self.callback2 = callback2 ++ self.cond = cond ++ ++ def __call__(self, t): ++ t2 = self.callback1(t) ++ return self.callback2(t) if self.cond(t2) else t2 ++ ++ ++def _get_match(re_, regexp, s, flags): ++ m = re_.match(regexp, s, flags) ++ if m: ++ return m.group(0) ++ ++def _create_unless(terminals, g_regex_flags, re_, use_bytes): ++ tokens_by_type = classify(terminals, lambda t: type(t.pattern)) ++ assert len(tokens_by_type) <= 2, tokens_by_type.keys() ++ embedded_strs = set() ++ callback = {} ++ for retok in tokens_by_type.get(PatternRE, []): ++ unless = [] ++ for strtok in tokens_by_type.get(PatternStr, []): ++ if strtok.priority != retok.priority: ++ continue ++ s = strtok.pattern.value ++ if s == _get_match(re_, retok.pattern.to_regexp(), s, g_regex_flags): ++ unless.append(strtok) ++ if strtok.pattern.flags <= retok.pattern.flags: ++ embedded_strs.add(strtok) ++ if unless: ++ callback[retok.name] = UnlessCallback(Scanner(unless, g_regex_flags, re_, use_bytes=use_bytes)) ++ ++ new_terminals = [t for t in terminals if t not in embedded_strs] ++ return new_terminals, callback ++ ++ ++class Scanner: ++ def __init__(self, terminals, g_regex_flags, re_, use_bytes): ++ self.terminals = terminals ++ self.g_regex_flags = g_regex_flags ++ self.re_ = re_ ++ self.use_bytes = use_bytes ++ ++ self.allowed_types = {t.name for t in self.terminals} ++ ++ self._mres = self._build_mres(terminals, len(terminals)) ++ ++ def _build_mres(self, terminals, max_size): ++ ## ++ ++ ## ++ ++ ## ++ ++ mres = [] ++ while terminals: ++ pattern = u'|'.join(u'(?P<%s>%s)' % (t.name, t.pattern.to_regexp()) for t in terminals[:max_size]) ++ if self.use_bytes: ++ pattern = pattern.encode('latin-1') ++ try: ++ mre = self.re_.compile(pattern, self.g_regex_flags) ++ except AssertionError: ## ++ ++ return self._build_mres(terminals, max_size // 2) ++ ++ mres.append(mre) ++ terminals = terminals[max_size:] ++ return mres ++ ++ def match(self, text: TextSlice, pos): ++ for mre in self._mres: ++ m = mre.match(text.text, pos, text.end) ++ if m: ++ return m.group(0), m.lastgroup ++ ++ ++ def fullmatch(self, text: str) -> Optional[str]: ++ for mre in self._mres: ++ m = mre.fullmatch(text) ++ if m: ++ return m.lastgroup ++ return None ++ ++def _regexp_has_newline(r: str): ++ #-- ++ return '\n' in r or '\\n' in r or '\\s' in r or '[^' in r or ('(?s' in r and '.' in r) ++ ++ ++class LexerState: ++ #-- ++ ++ __slots__ = 'text', 'line_ctr', 'last_token' ++ ++ text: TextSlice ++ line_ctr: LineCounter ++ last_token: Optional[Token] ++ ++ def __init__(self, text: TextSlice, line_ctr: Optional[LineCounter] = None, last_token: Optional[Token]=None): ++ if isinstance(text, TextSlice): ++ if line_ctr is None: ++ line_ctr = LineCounter(b'\n' if isinstance(text.text, bytes) else '\n') ++ ++ if text.start > 0: ++ ## ++ ++ line_ctr.feed(TextSlice(text.text, 0, text.start)) ++ ++ if not (text.start <= line_ctr.char_pos <= text.end): ++ raise ValueError("LineCounter.char_pos is out of bounds") ++ ++ self.text = text ++ self.line_ctr = line_ctr ++ self.last_token = last_token ++ ++ ++ def __eq__(self, other): ++ if not isinstance(other, LexerState): ++ return NotImplemented ++ ++ return self.text == other.text and self.line_ctr == other.line_ctr and self.last_token == other.last_token ++ ++ def __copy__(self): ++ return type(self)(self.text, copy(self.line_ctr), self.last_token) ++ ++ ++class LexerThread: ++ #-- ++ ++ def __init__(self, lexer: 'Lexer', lexer_state: Optional[LexerState]): ++ self.lexer = lexer ++ self.state = lexer_state ++ ++ @classmethod ++ def from_text(cls, lexer: 'Lexer', text_or_slice: TextOrSlice) -> 'LexerThread': ++ text = TextSlice.cast_from(text_or_slice) ++ return cls(lexer, LexerState(text)) ++ ++ @classmethod ++ def from_custom_input(cls, lexer: 'Lexer', text: Any) -> 'LexerThread': ++ return cls(lexer, LexerState(text)) ++ ++ def lex(self, parser_state): ++ if self.state is None: ++ raise TypeError("Cannot lex: No text assigned to lexer state") ++ return self.lexer.lex(self.state, parser_state) ++ ++ def __copy__(self): ++ return type(self)(self.lexer, copy(self.state)) ++ ++ _Token = Token ++ ++ ++_Callback = Callable[[Token], Token] ++ ++class Lexer(ABC): ++ #-- ++ @abstractmethod ++ def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]: ++ return NotImplemented ++ ++ def make_lexer_state(self, text: str): ++ #-- ++ return LexerState(TextSlice.cast_from(text)) ++ ++ ++def _check_regex_collisions(terminal_to_regexp: Dict[TerminalDef, str], comparator, strict_mode, max_collisions_to_show=8): ++ if not comparator: ++ comparator = interegular.Comparator.from_regexes(terminal_to_regexp) ++ ++ ## ++ ++ ## ++ ++ max_time = 2 if strict_mode else 0.2 ++ ++ ## ++ ++ if comparator.count_marked_pairs() >= max_collisions_to_show: ++ return ++ for group in classify(terminal_to_regexp, lambda t: t.priority).values(): ++ for a, b in comparator.check(group, skip_marked=True): ++ assert a.priority == b.priority ++ ## ++ ++ comparator.mark(a, b) ++ ++ ## ++ ++ message = f"Collision between Terminals {a.name} and {b.name}. " ++ try: ++ example = comparator.get_example_overlap(a, b, max_time).format_multiline() ++ except ValueError: ++ ## ++ ++ example = "No example could be found fast enough. However, the collision does still exists" ++ if strict_mode: ++ raise LexError(f"{message}\n{example}") ++ logger.warning("%s The lexer will choose between them arbitrarily.\n%s", message, example) ++ if comparator.count_marked_pairs() >= max_collisions_to_show: ++ logger.warning("Found 8 regex collisions, will not check for more.") ++ return ++ ++ ++class AbstractBasicLexer(Lexer): ++ terminals_by_name: Dict[str, TerminalDef] ++ ++ @abstractmethod ++ def __init__(self, conf: 'LexerConf', comparator=None) -> None: ++ ... ++ ++ @abstractmethod ++ def next_token(self, lex_state: LexerState, parser_state: Any = None) -> Token: ++ ... ++ ++ def lex(self, state: LexerState, parser_state: Any) -> Iterator[Token]: ++ with suppress(EOFError): ++ while True: ++ yield self.next_token(state, parser_state) ++ ++ ++class BasicLexer(AbstractBasicLexer): ++ terminals: Collection[TerminalDef] ++ ignore_types: FrozenSet[str] ++ newline_types: FrozenSet[str] ++ user_callbacks: Dict[str, _Callback] ++ callback: Dict[str, _Callback] ++ re: ModuleType ++ ++ def __init__(self, conf: 'LexerConf', comparator=None) -> None: ++ terminals = list(conf.terminals) ++ assert all(isinstance(t, TerminalDef) for t in terminals), terminals ++ ++ self.re = conf.re_module ++ ++ if not conf.skip_validation: ++ ## ++ ++ terminal_to_regexp = {} ++ for t in terminals: ++ regexp = t.pattern.to_regexp() ++ try: ++ self.re.compile(regexp, conf.g_regex_flags) ++ except self.re.error: ++ raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern)) ++ ++ if t.pattern.min_width == 0: ++ raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern)) ++ if t.pattern.type == "re": ++ terminal_to_regexp[t] = regexp ++ ++ if not (set(conf.ignore) <= {t.name for t in terminals}): ++ raise LexError("Ignore terminals are not defined: %s" % (set(conf.ignore) - {t.name for t in terminals})) ++ ++ if has_interegular: ++ _check_regex_collisions(terminal_to_regexp, comparator, conf.strict) ++ elif conf.strict: ++ raise LexError("interegular must be installed for strict mode. Use `pip install 'lark[interegular]'`.") ++ ++ ## ++ ++ self.newline_types = frozenset(t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp())) ++ self.ignore_types = frozenset(conf.ignore) ++ ++ terminals.sort(key=lambda x: (-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name)) ++ self.terminals = terminals ++ self.user_callbacks = conf.callbacks ++ self.g_regex_flags = conf.g_regex_flags ++ self.use_bytes = conf.use_bytes ++ self.terminals_by_name = conf.terminals_by_name ++ ++ self._scanner: Optional[Scanner] = None ++ ++ def _build_scanner(self) -> Scanner: ++ terminals, self.callback = _create_unless(self.terminals, self.g_regex_flags, self.re, self.use_bytes) ++ assert all(self.callback.values()) ++ ++ for type_, f in self.user_callbacks.items(): ++ if type_ in self.callback: ++ ## ++ ++ self.callback[type_] = CallChain(self.callback[type_], f, lambda t: t.type == type_) ++ else: ++ self.callback[type_] = f ++ ++ return Scanner(terminals, self.g_regex_flags, self.re, self.use_bytes) ++ ++ @property ++ def scanner(self) -> Scanner: ++ if self._scanner is None: ++ self._scanner = self._build_scanner() ++ return self._scanner ++ ++ def match(self, text, pos): ++ return self.scanner.match(text, pos) ++ ++ def next_token(self, lex_state: LexerState, parser_state: Any = None) -> Token: ++ line_ctr = lex_state.line_ctr ++ while line_ctr.char_pos < lex_state.text.end: ++ res = self.match(lex_state.text, line_ctr.char_pos) ++ if not res: ++ allowed = self.scanner.allowed_types - self.ignore_types ++ if not allowed: ++ allowed = {""} ++ raise UnexpectedCharacters(lex_state.text.text, line_ctr.char_pos, line_ctr.line, line_ctr.column, ++ allowed=allowed, token_history=lex_state.last_token and [lex_state.last_token], ++ state=parser_state, terminals_by_name=self.terminals_by_name) ++ ++ value, type_ = res ++ ++ ignored = type_ in self.ignore_types ++ t = None ++ if not ignored or type_ in self.callback: ++ t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) ++ line_ctr.feed(value, type_ in self.newline_types) ++ if t is not None: ++ t.end_line = line_ctr.line ++ t.end_column = line_ctr.column ++ t.end_pos = line_ctr.char_pos ++ if t.type in self.callback: ++ t = self.callback[t.type](t) ++ if not ignored: ++ if not isinstance(t, Token): ++ raise LexError("Callbacks must return a token (returned %r)" % t) ++ lex_state.last_token = t ++ return t ++ ++ ## ++ ++ raise EOFError(self) ++ ++ ++class ContextualLexer(Lexer): ++ lexers: Dict[int, AbstractBasicLexer] ++ root_lexer: AbstractBasicLexer ++ ++ BasicLexer: Type[AbstractBasicLexer] = BasicLexer ++ ++ def __init__(self, conf: 'LexerConf', states: Dict[int, Collection[str]], always_accept: Collection[str]=()) -> None: ++ terminals = list(conf.terminals) ++ terminals_by_name = conf.terminals_by_name ++ ++ trad_conf = copy(conf) ++ trad_conf.terminals = terminals ++ ++ if has_interegular and not conf.skip_validation: ++ comparator = interegular.Comparator.from_regexes({t: t.pattern.to_regexp() for t in terminals}) ++ else: ++ comparator = None ++ lexer_by_tokens: Dict[FrozenSet[str], AbstractBasicLexer] = {} ++ self.lexers = {} ++ for state, accepts in states.items(): ++ key = frozenset(accepts) ++ try: ++ lexer = lexer_by_tokens[key] ++ except KeyError: ++ accepts = set(accepts) | set(conf.ignore) | set(always_accept) ++ lexer_conf = copy(trad_conf) ++ lexer_conf.terminals = [terminals_by_name[n] for n in accepts if n in terminals_by_name] ++ lexer = self.BasicLexer(lexer_conf, comparator) ++ lexer_by_tokens[key] = lexer ++ ++ self.lexers[state] = lexer ++ ++ assert trad_conf.terminals is terminals ++ trad_conf.skip_validation = True ## ++ ++ self.root_lexer = self.BasicLexer(trad_conf, comparator) ++ ++ def lex(self, lexer_state: LexerState, parser_state: 'ParserState') -> Iterator[Token]: ++ try: ++ while True: ++ lexer = self.lexers[parser_state.position] ++ yield lexer.next_token(lexer_state, parser_state) ++ except EOFError: ++ pass ++ except UnexpectedCharacters as e: ++ ## ++ ++ ## ++ ++ try: ++ last_token = lexer_state.last_token ## ++ ++ token = self.root_lexer.next_token(lexer_state, parser_state) ++ raise UnexpectedToken(token, e.allowed, state=parser_state, token_history=[last_token], terminals_by_name=self.root_lexer.terminals_by_name) ++ except UnexpectedCharacters: ++ raise e ## ++ ++ ++ ++ ++_ParserArgType: 'TypeAlias' = 'Literal["earley", "lalr", "cyk", "auto"]' ++_LexerArgType: 'TypeAlias' = 'Union[Literal["auto", "basic", "contextual", "dynamic", "dynamic_complete"], Type[Lexer]]' ++_LexerCallback = Callable[[Token], Token] ++ParserCallbacks = Dict[str, Callable] ++ ++class LexerConf(Serialize): ++ __serialize_fields__ = 'terminals', 'ignore', 'g_regex_flags', 'use_bytes', 'lexer_type' ++ __serialize_namespace__ = TerminalDef, ++ ++ terminals: Collection[TerminalDef] ++ re_module: ModuleType ++ ignore: Collection[str] ++ postlex: 'Optional[PostLex]' ++ callbacks: Dict[str, _LexerCallback] ++ g_regex_flags: int ++ skip_validation: bool ++ use_bytes: bool ++ lexer_type: Optional[_LexerArgType] ++ strict: bool ++ ++ def __init__(self, terminals: Collection[TerminalDef], re_module: ModuleType, ignore: Collection[str]=(), postlex: 'Optional[PostLex]'=None, ++ callbacks: Optional[Dict[str, _LexerCallback]]=None, g_regex_flags: int=0, skip_validation: bool=False, use_bytes: bool=False, strict: bool=False): ++ self.terminals = terminals ++ self.terminals_by_name = {t.name: t for t in self.terminals} ++ assert len(self.terminals) == len(self.terminals_by_name) ++ self.ignore = ignore ++ self.postlex = postlex ++ self.callbacks = callbacks or {} ++ self.g_regex_flags = g_regex_flags ++ self.re_module = re_module ++ self.skip_validation = skip_validation ++ self.use_bytes = use_bytes ++ self.strict = strict ++ self.lexer_type = None ++ ++ def _deserialize(self): ++ self.terminals_by_name = {t.name: t for t in self.terminals} ++ ++ def __deepcopy__(self, memo=None): ++ return type(self)( ++ deepcopy(self.terminals, memo), ++ self.re_module, ++ deepcopy(self.ignore, memo), ++ deepcopy(self.postlex, memo), ++ deepcopy(self.callbacks, memo), ++ deepcopy(self.g_regex_flags, memo), ++ deepcopy(self.skip_validation, memo), ++ deepcopy(self.use_bytes, memo), ++ ) ++ ++class ParserConf(Serialize): ++ __serialize_fields__ = 'rules', 'start', 'parser_type' ++ ++ rules: List['Rule'] ++ callbacks: ParserCallbacks ++ start: List[str] ++ parser_type: _ParserArgType ++ ++ def __init__(self, rules: List['Rule'], callbacks: ParserCallbacks, start: List[str]): ++ assert isinstance(start, list) ++ self.rules = rules ++ self.callbacks = callbacks ++ self.start = start ++ ++ ++from functools import partial, wraps ++from itertools import product ++ ++ ++class ExpandSingleChild: ++ def __init__(self, node_builder): ++ self.node_builder = node_builder ++ ++ def __call__(self, children): ++ if len(children) == 1: ++ return children[0] ++ else: ++ return self.node_builder(children) ++ ++ ++ ++class PropagatePositions: ++ def __init__(self, node_builder, node_filter=None): ++ self.node_builder = node_builder ++ self.node_filter = node_filter ++ ++ def __call__(self, children): ++ res = self.node_builder(children) ++ ++ if isinstance(res, Tree): ++ ## ++ ++ ## ++ ++ ## ++ ++ ## ++ ++ ++ res_meta = res.meta ++ ++ first_meta = self._pp_get_meta(children) ++ if first_meta is not None: ++ if not hasattr(res_meta, 'line'): ++ ## ++ ++ res_meta.line = getattr(first_meta, 'container_line', first_meta.line) ++ res_meta.column = getattr(first_meta, 'container_column', first_meta.column) ++ res_meta.start_pos = getattr(first_meta, 'container_start_pos', first_meta.start_pos) ++ res_meta.empty = False ++ ++ res_meta.container_line = getattr(first_meta, 'container_line', first_meta.line) ++ res_meta.container_column = getattr(first_meta, 'container_column', first_meta.column) ++ res_meta.container_start_pos = getattr(first_meta, 'container_start_pos', first_meta.start_pos) ++ ++ last_meta = self._pp_get_meta(reversed(children)) ++ if last_meta is not None: ++ if not hasattr(res_meta, 'end_line'): ++ res_meta.end_line = getattr(last_meta, 'container_end_line', last_meta.end_line) ++ res_meta.end_column = getattr(last_meta, 'container_end_column', last_meta.end_column) ++ res_meta.end_pos = getattr(last_meta, 'container_end_pos', last_meta.end_pos) ++ res_meta.empty = False ++ ++ res_meta.container_end_line = getattr(last_meta, 'container_end_line', last_meta.end_line) ++ res_meta.container_end_column = getattr(last_meta, 'container_end_column', last_meta.end_column) ++ res_meta.container_end_pos = getattr(last_meta, 'container_end_pos', last_meta.end_pos) ++ ++ return res ++ ++ def _pp_get_meta(self, children): ++ for c in children: ++ if self.node_filter is not None and not self.node_filter(c): ++ continue ++ if isinstance(c, Tree): ++ if not c.meta.empty: ++ return c.meta ++ elif isinstance(c, Token): ++ return c ++ elif hasattr(c, '__lark_meta__'): ++ return c.__lark_meta__() ++ ++def make_propagate_positions(option): ++ if callable(option): ++ return partial(PropagatePositions, node_filter=option) ++ elif option is True: ++ return PropagatePositions ++ elif option is False: ++ return None ++ ++ raise ConfigurationError('Invalid option for propagate_positions: %r' % option) ++ ++ ++class ChildFilter: ++ def __init__(self, to_include, append_none, node_builder): ++ self.node_builder = node_builder ++ self.to_include = to_include ++ self.append_none = append_none ++ ++ def __call__(self, children): ++ filtered = [] ++ ++ for i, to_expand, add_none in self.to_include: ++ if add_none: ++ filtered += [None] * add_none ++ if to_expand: ++ filtered += children[i].children ++ else: ++ filtered.append(children[i]) ++ ++ if self.append_none: ++ filtered += [None] * self.append_none ++ ++ return self.node_builder(filtered) ++ ++ ++class ChildFilterLALR(ChildFilter): ++ #-- ++ ++ def __call__(self, children): ++ filtered = [] ++ for i, to_expand, add_none in self.to_include: ++ if add_none: ++ filtered += [None] * add_none ++ if to_expand: ++ if filtered: ++ filtered += children[i].children ++ else: ## ++ ++ filtered = children[i].children ++ else: ++ filtered.append(children[i]) ++ ++ if self.append_none: ++ filtered += [None] * self.append_none ++ ++ return self.node_builder(filtered) ++ ++ ++class ChildFilterLALR_NoPlaceholders(ChildFilter): ++ #-- ++ def __init__(self, to_include, node_builder): ++ self.node_builder = node_builder ++ self.to_include = to_include ++ ++ def __call__(self, children): ++ filtered = [] ++ for i, to_expand in self.to_include: ++ if to_expand: ++ if filtered: ++ filtered += children[i].children ++ else: ## ++ ++ filtered = children[i].children ++ else: ++ filtered.append(children[i]) ++ return self.node_builder(filtered) ++ ++ ++def _should_expand(sym): ++ return not sym.is_term and sym.name.startswith('_') ++ ++ ++def maybe_create_child_filter(expansion, keep_all_tokens, ambiguous, _empty_indices: List[bool]): ++ ## ++ ++ if _empty_indices: ++ assert _empty_indices.count(False) == len(expansion) ++ s = ''.join(str(int(b)) for b in _empty_indices) ++ empty_indices = [len(ones) for ones in s.split('0')] ++ assert len(empty_indices) == len(expansion)+1, (empty_indices, len(expansion)) ++ else: ++ empty_indices = [0] * (len(expansion)+1) ++ ++ to_include = [] ++ nones_to_add = 0 ++ for i, sym in enumerate(expansion): ++ nones_to_add += empty_indices[i] ++ if keep_all_tokens or not (sym.is_term and sym.filter_out): ++ to_include.append((i, _should_expand(sym), nones_to_add)) ++ nones_to_add = 0 ++ ++ nones_to_add += empty_indices[len(expansion)] ++ ++ if _empty_indices or len(to_include) < len(expansion) or any(to_expand for i, to_expand,_ in to_include): ++ if _empty_indices or ambiguous: ++ return partial(ChildFilter if ambiguous else ChildFilterLALR, to_include, nones_to_add) ++ else: ++ ## ++ ++ return partial(ChildFilterLALR_NoPlaceholders, [(i, x) for i,x,_ in to_include]) ++ ++ ++class AmbiguousExpander: ++ #-- ++ def __init__(self, to_expand, tree_class, node_builder): ++ self.node_builder = node_builder ++ self.tree_class = tree_class ++ self.to_expand = to_expand ++ ++ def __call__(self, children): ++ def _is_ambig_tree(t): ++ return hasattr(t, 'data') and t.data == '_ambig' ++ ++ ## ++ ++ ## ++ ++ ## ++ ++ ## ++ ++ ambiguous = [] ++ for i, child in enumerate(children): ++ if _is_ambig_tree(child): ++ if i in self.to_expand: ++ ambiguous.append(i) ++ ++ child.expand_kids_by_data('_ambig') ++ ++ if not ambiguous: ++ return self.node_builder(children) ++ ++ expand = [child.children if i in ambiguous else (child,) for i, child in enumerate(children)] ++ return self.tree_class('_ambig', [self.node_builder(list(f)) for f in product(*expand)]) ++ ++ ++def maybe_create_ambiguous_expander(tree_class, expansion, keep_all_tokens): ++ to_expand = [i for i, sym in enumerate(expansion) ++ if keep_all_tokens or ((not (sym.is_term and sym.filter_out)) and _should_expand(sym))] ++ if to_expand: ++ return partial(AmbiguousExpander, to_expand, tree_class) ++ ++ ++class AmbiguousIntermediateExpander: ++ #-- ++ ++ def __init__(self, tree_class, node_builder): ++ self.node_builder = node_builder ++ self.tree_class = tree_class ++ ++ def __call__(self, children): ++ def _is_iambig_tree(child): ++ return hasattr(child, 'data') and child.data == '_iambig' ++ ++ def _collapse_iambig(children): ++ #-- ++ ++ ## ++ ++ ## ++ ++ if children and _is_iambig_tree(children[0]): ++ iambig_node = children[0] ++ result = [] ++ for grandchild in iambig_node.children: ++ collapsed = _collapse_iambig(grandchild.children) ++ if collapsed: ++ for child in collapsed: ++ child.children += children[1:] ++ result += collapsed ++ else: ++ new_tree = self.tree_class('_inter', grandchild.children + children[1:]) ++ result.append(new_tree) ++ return result ++ ++ collapsed = _collapse_iambig(children) ++ if collapsed: ++ processed_nodes = [self.node_builder(c.children) for c in collapsed] ++ return self.tree_class('_ambig', processed_nodes) ++ ++ return self.node_builder(children) ++ ++ ++ ++def inplace_transformer(func): ++ @wraps(func) ++ def f(children): ++ ## ++ ++ tree = Tree(func.__name__, children) ++ return func(tree) ++ return f ++ ++ ++def apply_visit_wrapper(func, name, wrapper): ++ if wrapper is _vargs_meta or wrapper is _vargs_meta_inline: ++ raise NotImplementedError("Meta args not supported for internal transformer; use YourTransformer().transform(parser.parse()) instead") ++ ++ @wraps(func) ++ def f(children): ++ return wrapper(func, name, children, None) ++ return f ++ ++ ++class ParseTreeBuilder: ++ def __init__(self, rules, tree_class, propagate_positions=False, ambiguous=False, maybe_placeholders=False): ++ self.tree_class = tree_class ++ self.propagate_positions = propagate_positions ++ self.ambiguous = ambiguous ++ self.maybe_placeholders = maybe_placeholders ++ ++ self.rule_builders = list(self._init_builders(rules)) ++ ++ def _init_builders(self, rules): ++ propagate_positions = make_propagate_positions(self.propagate_positions) ++ ++ for rule in rules: ++ options = rule.options ++ keep_all_tokens = options.keep_all_tokens ++ expand_single_child = options.expand1 ++ ++ wrapper_chain = list(filter(None, [ ++ (expand_single_child and not rule.alias) and ExpandSingleChild, ++ maybe_create_child_filter(rule.expansion, keep_all_tokens, self.ambiguous, options.empty_indices if self.maybe_placeholders else None), ++ propagate_positions, ++ self.ambiguous and maybe_create_ambiguous_expander(self.tree_class, rule.expansion, keep_all_tokens), ++ self.ambiguous and partial(AmbiguousIntermediateExpander, self.tree_class) ++ ])) ++ ++ yield rule, wrapper_chain ++ ++ def create_callback(self, transformer=None): ++ callbacks = {} ++ ++ default_handler = getattr(transformer, '__default__', None) ++ if default_handler: ++ def default_callback(data, children): ++ return default_handler(data, children, None) ++ else: ++ default_callback = self.tree_class ++ ++ for rule, wrapper_chain in self.rule_builders: ++ ++ user_callback_name = rule.alias or rule.options.template_source or rule.origin.name ++ try: ++ f = getattr(transformer, user_callback_name) ++ wrapper = getattr(f, 'visit_wrapper', None) ++ if wrapper is not None: ++ f = apply_visit_wrapper(f, user_callback_name, wrapper) ++ elif isinstance(transformer, Transformer_InPlace): ++ f = inplace_transformer(f) ++ except AttributeError: ++ f = partial(default_callback, user_callback_name) ++ ++ for w in wrapper_chain: ++ f = w(f) ++ ++ if rule in callbacks: ++ raise GrammarError("Rule '%s' already exists" % (rule,)) ++ ++ callbacks[rule] = f ++ ++ return callbacks ++ ++ ++ ++class Action: ++ def __init__(self, name): ++ self.name = name ++ def __str__(self): ++ return self.name ++ def __repr__(self): ++ return str(self) ++ ++Shift = Action('Shift') ++Reduce = Action('Reduce') ++ ++StateT = TypeVar("StateT") ++ ++class ParseTableBase(Generic[StateT]): ++ states: Dict[StateT, Dict[str, Tuple]] ++ start_states: Dict[str, StateT] ++ end_states: Dict[str, StateT] ++ ++ def __init__(self, states, start_states, end_states): ++ self.states = states ++ self.start_states = start_states ++ self.end_states = end_states ++ ++ def serialize(self, memo): ++ tokens = Enumerator() ++ ++ states = { ++ state: {tokens.get(token): ((1, arg.serialize(memo)) if action is Reduce else (0, arg)) ++ for token, (action, arg) in actions.items()} ++ for state, actions in self.states.items() ++ } ++ ++ return { ++ 'tokens': tokens.reversed(), ++ 'states': states, ++ 'start_states': self.start_states, ++ 'end_states': self.end_states, ++ } ++ ++ @classmethod ++ def deserialize(cls, data, memo): ++ tokens = data['tokens'] ++ states = { ++ state: {tokens[token]: ((Reduce, Rule.deserialize(arg, memo)) if action==1 else (Shift, arg)) ++ for token, (action, arg) in actions.items()} ++ for state, actions in data['states'].items() ++ } ++ return cls(states, data['start_states'], data['end_states']) ++ ++class ParseTable(ParseTableBase['State']): ++ #-- ++ pass ++ ++ ++class IntParseTable(ParseTableBase[int]): ++ #-- ++ ++ @classmethod ++ def from_ParseTable(cls, parse_table: ParseTable): ++ enum = list(parse_table.states) ++ state_to_idx: Dict['State', int] = {s:i for i,s in enumerate(enum)} ++ int_states = {} ++ ++ for s, la in parse_table.states.items(): ++ la = {k:(v[0], state_to_idx[v[1]]) if v[0] is Shift else v ++ for k,v in la.items()} ++ int_states[ state_to_idx[s] ] = la ++ ++ ++ start_states = {start:state_to_idx[s] for start, s in parse_table.start_states.items()} ++ end_states = {start:state_to_idx[s] for start, s in parse_table.end_states.items()} ++ return cls(int_states, start_states, end_states) ++ ++ ++ ++class ParseConf(Generic[StateT]): ++ __slots__ = 'parse_table', 'callbacks', 'start', 'start_state', 'end_state', 'states' ++ ++ parse_table: ParseTableBase[StateT] ++ callbacks: ParserCallbacks ++ start: str ++ ++ start_state: StateT ++ end_state: StateT ++ states: Dict[StateT, Dict[str, tuple]] ++ ++ def __init__(self, parse_table: ParseTableBase[StateT], callbacks: ParserCallbacks, start: str): ++ self.parse_table = parse_table ++ ++ self.start_state = self.parse_table.start_states[start] ++ self.end_state = self.parse_table.end_states[start] ++ self.states = self.parse_table.states ++ ++ self.callbacks = callbacks ++ self.start = start ++ ++class ParserState(Generic[StateT]): ++ __slots__ = 'parse_conf', 'lexer', 'state_stack', 'value_stack' ++ ++ parse_conf: ParseConf[StateT] ++ lexer: LexerThread ++ state_stack: List[StateT] ++ value_stack: list ++ ++ def __init__(self, parse_conf: ParseConf[StateT], lexer: LexerThread, state_stack=None, value_stack=None): ++ self.parse_conf = parse_conf ++ self.lexer = lexer ++ self.state_stack = state_stack or [self.parse_conf.start_state] ++ self.value_stack = value_stack or [] ++ ++ @property ++ def position(self) -> StateT: ++ return self.state_stack[-1] ++ ++ ## ++ ++ def __eq__(self, other) -> bool: ++ if not isinstance(other, ParserState): ++ return NotImplemented ++ return len(self.state_stack) == len(other.state_stack) and self.position == other.position ++ ++ def __copy__(self): ++ return self.copy() ++ ++ def copy(self, deepcopy_values=True) -> 'ParserState[StateT]': ++ return type(self)( ++ self.parse_conf, ++ self.lexer, ## ++ ++ copy(self.state_stack), ++ deepcopy(self.value_stack) if deepcopy_values else copy(self.value_stack), ++ ) ++ ++ def feed_token(self, token: Token, is_end=False) -> Any: ++ state_stack = self.state_stack ++ value_stack = self.value_stack ++ states = self.parse_conf.states ++ end_state = self.parse_conf.end_state ++ callbacks = self.parse_conf.callbacks ++ ++ while True: ++ state = state_stack[-1] ++ try: ++ action, arg = states[state][token.type] ++ except KeyError: ++ expected = {s for s in states[state].keys() if s.isupper()} ++ raise UnexpectedToken(token, expected, state=self, interactive_parser=None) ++ ++ assert arg != end_state ++ ++ if action is Shift: ++ ## ++ ++ assert not is_end ++ state_stack.append(arg) ++ value_stack.append(token if token.type not in callbacks else callbacks[token.type](token)) ++ return ++ else: ++ ## ++ ++ rule = arg ++ size = len(rule.expansion) ++ if size: ++ s = value_stack[-size:] ++ del state_stack[-size:] ++ del value_stack[-size:] ++ else: ++ s = [] ++ ++ value = callbacks[rule](s) if callbacks else s ++ ++ _action, new_state = states[state_stack[-1]][rule.origin.name] ++ assert _action is Shift ++ state_stack.append(new_state) ++ value_stack.append(value) ++ ++ if is_end and state_stack[-1] == end_state: ++ return value_stack[-1] ++ ++ ++class LALR_Parser(Serialize): ++ def __init__(self, parser_conf: ParserConf, debug: bool=False, strict: bool=False): ++ analysis = LALR_Analyzer(parser_conf, debug=debug, strict=strict) ++ analysis.compute_lalr() ++ callbacks = parser_conf.callbacks ++ ++ self._parse_table = analysis.parse_table ++ self.parser_conf = parser_conf ++ self.parser = _Parser(analysis.parse_table, callbacks, debug) ++ ++ @classmethod ++ def deserialize(cls, data, memo, callbacks, debug=False): ++ inst = cls.__new__(cls) ++ inst._parse_table = IntParseTable.deserialize(data, memo) ++ inst.parser = _Parser(inst._parse_table, callbacks, debug) ++ return inst ++ ++ def serialize(self, memo: Any = None) -> Dict[str, Any]: ++ return self._parse_table.serialize(memo) ++ ++ def parse_interactive(self, lexer: LexerThread, start: str): ++ return self.parser.parse(lexer, start, start_interactive=True) ++ ++ def parse(self, lexer, start, on_error=None): ++ try: ++ return self.parser.parse(lexer, start) ++ except UnexpectedInput as e: ++ if on_error is None: ++ raise ++ ++ while True: ++ if isinstance(e, UnexpectedCharacters): ++ s = e.interactive_parser.lexer_thread.state ++ p = s.line_ctr.char_pos ++ ++ if not on_error(e): ++ raise e ++ ++ if isinstance(e, UnexpectedCharacters): ++ ## ++ ++ if p == s.line_ctr.char_pos: ++ s.line_ctr.feed(s.text.text[p:p+1]) ++ ++ try: ++ return e.interactive_parser.resume_parse() ++ except UnexpectedToken as e2: ++ if (isinstance(e, UnexpectedToken) ++ and e.token.type == e2.token.type == '$END' ++ and e.interactive_parser == e2.interactive_parser): ++ ## ++ ++ raise e2 ++ e = e2 ++ except UnexpectedCharacters as e2: ++ e = e2 ++ ++ ++class _Parser: ++ parse_table: ParseTableBase ++ callbacks: ParserCallbacks ++ debug: bool ++ ++ def __init__(self, parse_table: ParseTableBase, callbacks: ParserCallbacks, debug: bool=False): ++ self.parse_table = parse_table ++ self.callbacks = callbacks ++ self.debug = debug ++ ++ def parse(self, lexer: LexerThread, start: str, value_stack=None, state_stack=None, start_interactive=False): ++ parse_conf = ParseConf(self.parse_table, self.callbacks, start) ++ parser_state = ParserState(parse_conf, lexer, state_stack, value_stack) ++ if start_interactive: ++ return InteractiveParser(self, parser_state, parser_state.lexer) ++ return self.parse_from_state(parser_state) ++ ++ ++ def parse_from_state(self, state: ParserState, last_token: Optional[Token]=None): ++ #-- ++ try: ++ token = last_token ++ for token in state.lexer.lex(state): ++ assert token is not None ++ state.feed_token(token) ++ ++ end_token = Token.new_borrow_pos('$END', '', token) if token else Token('$END', '', 0, 1, 1) ++ return state.feed_token(end_token, True) ++ except UnexpectedInput as e: ++ try: ++ e.interactive_parser = InteractiveParser(self, state, state.lexer) ++ except NameError: ++ pass ++ raise e ++ except Exception as e: ++ if self.debug: ++ print("") ++ print("STATE STACK DUMP") ++ print("----------------") ++ for i, s in enumerate(state.state_stack): ++ print('%d)' % i , s) ++ print("") ++ ++ raise ++ ++ ++class InteractiveParser: ++ #-- ++ def __init__(self, parser, parser_state: ParserState, lexer_thread: LexerThread): ++ self.parser = parser ++ self.parser_state = parser_state ++ self.lexer_thread = lexer_thread ++ self.result = None ++ ++ @property ++ def lexer_state(self) -> LexerThread: ++ warnings.warn("lexer_state will be removed in subsequent releases. Use lexer_thread instead.", DeprecationWarning) ++ return self.lexer_thread ++ ++ def feed_token(self, token: Token): ++ #-- ++ return self.parser_state.feed_token(token, token.type == '$END') ++ ++ def iter_parse(self) -> Iterator[Token]: ++ #-- ++ for token in self.lexer_thread.lex(self.parser_state): ++ yield token ++ self.result = self.feed_token(token) ++ ++ def exhaust_lexer(self) -> List[Token]: ++ #-- ++ return list(self.iter_parse()) ++ ++ ++ def feed_eof(self, last_token=None): ++ #-- ++ eof = Token.new_borrow_pos('$END', '', last_token) if last_token is not None else self.lexer_thread._Token('$END', '', 0, 1, 1) ++ return self.feed_token(eof) ++ ++ ++ def __copy__(self): ++ #-- ++ return self.copy() ++ ++ def copy(self, deepcopy_values=True): ++ return type(self)( ++ self.parser, ++ self.parser_state.copy(deepcopy_values=deepcopy_values), ++ copy(self.lexer_thread), ++ ) ++ ++ def __eq__(self, other): ++ if not isinstance(other, InteractiveParser): ++ return False ++ ++ return self.parser_state == other.parser_state and self.lexer_thread == other.lexer_thread ++ ++ def as_immutable(self): ++ #-- ++ p = copy(self) ++ return ImmutableInteractiveParser(p.parser, p.parser_state, p.lexer_thread) ++ ++ def pretty(self): ++ #-- ++ out = ["Parser choices:"] ++ for k, v in self.choices().items(): ++ out.append('\t- %s -> %r' % (k, v)) ++ out.append('stack size: %s' % len(self.parser_state.state_stack)) ++ return '\n'.join(out) ++ ++ def choices(self): ++ #-- ++ return self.parser_state.parse_conf.parse_table.states[self.parser_state.position] ++ ++ def accepts(self): ++ #-- ++ accepts = set() ++ conf_no_callbacks = copy(self.parser_state.parse_conf) ++ ## ++ ++ ## ++ ++ conf_no_callbacks.callbacks = {} ++ for t in self.choices(): ++ if t.isupper(): ## ++ ++ new_cursor = self.copy(deepcopy_values=False) ++ new_cursor.parser_state.parse_conf = conf_no_callbacks ++ try: ++ new_cursor.feed_token(self.lexer_thread._Token(t, '')) ++ except UnexpectedToken: ++ pass ++ else: ++ accepts.add(t) ++ return accepts ++ ++ def resume_parse(self): ++ #-- ++ return self.parser.parse_from_state(self.parser_state, last_token=self.lexer_thread.state.last_token) ++ ++ ++ ++class ImmutableInteractiveParser(InteractiveParser): ++ #-- ++ ++ result = None ++ ++ def __hash__(self): ++ return hash((self.parser_state, self.lexer_thread)) ++ ++ def feed_token(self, token): ++ c = copy(self) ++ c.result = InteractiveParser.feed_token(c, token) ++ return c ++ ++ def exhaust_lexer(self): ++ #-- ++ cursor = self.as_mutable() ++ cursor.exhaust_lexer() ++ return cursor.as_immutable() ++ ++ def as_mutable(self): ++ #-- ++ p = copy(self) ++ return InteractiveParser(p.parser, p.parser_state, p.lexer_thread) ++ ++ ++ ++def _wrap_lexer(lexer_class): ++ future_interface = getattr(lexer_class, '__future_interface__', 0) ++ if future_interface == 2: ++ return lexer_class ++ elif future_interface == 1: ++ class CustomLexerWrapper1(Lexer): ++ def __init__(self, lexer_conf): ++ self.lexer = lexer_class(lexer_conf) ++ def lex(self, lexer_state, parser_state): ++ if isinstance(lexer_state.text, TextSlice) and not lexer_state.text.is_complete_text(): ++ raise TypeError("Interface=1 Custom Lexer don't support TextSlice") ++ lexer_state.text = lexer_state.text ++ return self.lexer.lex(lexer_state, parser_state) ++ return CustomLexerWrapper1 ++ elif future_interface == 0: ++ class CustomLexerWrapper0(Lexer): ++ def __init__(self, lexer_conf): ++ self.lexer = lexer_class(lexer_conf) ++ ++ def lex(self, lexer_state, parser_state): ++ if isinstance(lexer_state.text, TextSlice): ++ if not lexer_state.text.is_complete_text(): ++ raise TypeError("Interface=0 Custom Lexer don't support TextSlice") ++ return self.lexer.lex(lexer_state.text.text) ++ return self.lexer.lex(lexer_state.text) ++ return CustomLexerWrapper0 ++ else: ++ raise ValueError(f"Unknown __future_interface__ value {future_interface}, integer 0-2 expected") ++ ++ ++def _deserialize_parsing_frontend(data, memo, lexer_conf, callbacks, options): ++ parser_conf = ParserConf.deserialize(data['parser_conf'], memo) ++ cls = (options and options._plugins.get('LALR_Parser')) or LALR_Parser ++ parser = cls.deserialize(data['parser'], memo, callbacks, options.debug) ++ parser_conf.callbacks = callbacks ++ return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser) ++ ++ ++_parser_creators: 'Dict[str, Callable[[LexerConf, Any, Any], Any]]' = {} ++ ++ ++class ParsingFrontend(Serialize): ++ __serialize_fields__ = 'lexer_conf', 'parser_conf', 'parser' ++ ++ lexer_conf: LexerConf ++ parser_conf: ParserConf ++ options: Any ++ ++ def __init__(self, lexer_conf: LexerConf, parser_conf: ParserConf, options, parser=None): ++ self.parser_conf = parser_conf ++ self.lexer_conf = lexer_conf ++ self.options = options ++ ++ ## ++ ++ if parser: ## ++ ++ self.parser = parser ++ else: ++ create_parser = _parser_creators.get(parser_conf.parser_type) ++ assert create_parser is not None, "{} is not supported in standalone mode".format( ++ parser_conf.parser_type ++ ) ++ self.parser = create_parser(lexer_conf, parser_conf, options) ++ ++ ## ++ ++ lexer_type = lexer_conf.lexer_type ++ self.skip_lexer = False ++ if lexer_type in ('dynamic', 'dynamic_complete'): ++ assert lexer_conf.postlex is None ++ self.skip_lexer = True ++ return ++ ++ if isinstance(lexer_type, type): ++ assert issubclass(lexer_type, Lexer) ++ self.lexer = _wrap_lexer(lexer_type)(lexer_conf) ++ elif isinstance(lexer_type, str): ++ create_lexer = { ++ 'basic': create_basic_lexer, ++ 'contextual': create_contextual_lexer, ++ }[lexer_type] ++ self.lexer = create_lexer(lexer_conf, self.parser, lexer_conf.postlex, options) ++ else: ++ raise TypeError("Bad value for lexer_type: {lexer_type}") ++ ++ if lexer_conf.postlex: ++ self.lexer = PostLexConnector(self.lexer, lexer_conf.postlex) ++ ++ def _verify_start(self, start=None): ++ if start is None: ++ start_decls = self.parser_conf.start ++ if len(start_decls) > 1: ++ raise ConfigurationError("Lark initialized with more than 1 possible start rule. Must specify which start rule to parse", start_decls) ++ start ,= start_decls ++ elif start not in self.parser_conf.start: ++ raise ConfigurationError("Unknown start rule %s. Must be one of %r" % (start, self.parser_conf.start)) ++ return start ++ ++ def _make_lexer_thread(self, text: Optional[LarkInput]) -> Union[LarkInput, LexerThread, None]: ++ cls = (self.options and self.options._plugins.get('LexerThread')) or LexerThread ++ if self.skip_lexer: ++ return text ++ if text is None: ++ return cls(self.lexer, None) ++ if isinstance(text, (str, bytes, TextSlice)): ++ return cls.from_text(self.lexer, text) ++ return cls.from_custom_input(self.lexer, text) ++ ++ def parse(self, text: Optional[LarkInput], start=None, on_error=None): ++ if self.lexer_conf.lexer_type in ("dynamic", "dynamic_complete"): ++ if isinstance(text, TextSlice) and not text.is_complete_text(): ++ raise TypeError(f"Lexer {self.lexer_conf.lexer_type} does not support text slices.") ++ ++ chosen_start = self._verify_start(start) ++ kw = {} if on_error is None else {'on_error': on_error} ++ stream = self._make_lexer_thread(text) ++ return self.parser.parse(stream, chosen_start, **kw) ++ ++ def parse_interactive(self, text: Optional[TextOrSlice]=None, start=None): ++ ## ++ ++ ## ++ ++ chosen_start = self._verify_start(start) ++ if self.parser_conf.parser_type != 'lalr': ++ raise ConfigurationError("parse_interactive() currently only works with parser='lalr' ") ++ stream = self._make_lexer_thread(text) ++ return self.parser.parse_interactive(stream, chosen_start) ++ ++ ++def _validate_frontend_args(parser, lexer) -> None: ++ assert_config(parser, ('lalr', 'earley', 'cyk')) ++ if not isinstance(lexer, type): ## ++ ++ expected = { ++ 'lalr': ('basic', 'contextual'), ++ 'earley': ('basic', 'dynamic', 'dynamic_complete'), ++ 'cyk': ('basic', ), ++ }[parser] ++ assert_config(lexer, expected, 'Parser %r does not support lexer %%r, expected one of %%s' % parser) ++ ++ ++def _get_lexer_callbacks(transformer, terminals): ++ result = {} ++ for terminal in terminals: ++ callback = getattr(transformer, terminal.name, None) ++ if callback is not None: ++ result[terminal.name] = callback ++ return result ++ ++class PostLexConnector: ++ def __init__(self, lexer, postlexer): ++ self.lexer = lexer ++ self.postlexer = postlexer ++ ++ def lex(self, lexer_state, parser_state): ++ i = self.lexer.lex(lexer_state, parser_state) ++ return self.postlexer.process(i) ++ ++ ++ ++def create_basic_lexer(lexer_conf, parser, postlex, options) -> BasicLexer: ++ cls = (options and options._plugins.get('BasicLexer')) or BasicLexer ++ return cls(lexer_conf) ++ ++def create_contextual_lexer(lexer_conf: LexerConf, parser, postlex, options) -> ContextualLexer: ++ cls = (options and options._plugins.get('ContextualLexer')) or ContextualLexer ++ parse_table: ParseTableBase[int] = parser._parse_table ++ states: Dict[int, Collection[str]] = {idx:list(t.keys()) for idx, t in parse_table.states.items()} ++ always_accept: Collection[str] = postlex.always_accept if postlex else () ++ return cls(lexer_conf, states, always_accept=always_accept) ++ ++def create_lalr_parser(lexer_conf: LexerConf, parser_conf: ParserConf, options=None) -> LALR_Parser: ++ debug = options.debug if options else False ++ strict = options.strict if options else False ++ cls = (options and options._plugins.get('LALR_Parser')) or LALR_Parser ++ return cls(parser_conf, debug=debug, strict=strict) ++ ++_parser_creators['lalr'] = create_lalr_parser ++ ++ ++ ++ ++class PostLex(ABC): ++ @abstractmethod ++ def process(self, stream: Iterator[Token]) -> Iterator[Token]: ++ return stream ++ ++ always_accept: Iterable[str] = () ++ ++class LarkOptions(Serialize): ++ #-- ++ ++ start: List[str] ++ debug: bool ++ strict: bool ++ transformer: 'Optional[Transformer]' ++ propagate_positions: Union[bool, str] ++ maybe_placeholders: bool ++ cache: Union[bool, str] ++ cache_grammar: bool ++ regex: bool ++ g_regex_flags: int ++ keep_all_tokens: bool ++ tree_class: Optional[Callable[[str, List], Any]] ++ parser: _ParserArgType ++ lexer: _LexerArgType ++ ambiguity: 'Literal["auto", "resolve", "explicit", "forest"]' ++ postlex: Optional[PostLex] ++ priority: 'Optional[Literal["auto", "normal", "invert"]]' ++ lexer_callbacks: Dict[str, Callable[[Token], Token]] ++ use_bytes: bool ++ ordered_sets: bool ++ edit_terminals: Optional[Callable[[TerminalDef], TerminalDef]] ++ import_paths: 'List[Union[str, Callable[[Union[None, str, PackageResource], str], Tuple[str, str]]]]' ++ source_path: Optional[str] ++ ++ OPTIONS_DOC = r""" ++ **=== General Options ===** ++ ++ start ++ The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start") ++ debug ++ Display debug information and extra warnings. Use only when debugging (Default: ``False``) ++ When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed. ++ strict ++ Throw an exception on any potential ambiguity, including shift/reduce conflicts, and regex collisions. ++ transformer ++ Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster) ++ propagate_positions ++ Propagates positional attributes into the 'meta' attribute of all tree branches. ++ Sets attributes: (line, column, end_line, end_column, start_pos, end_pos, ++ container_line, container_column, container_end_line, container_end_column) ++ Accepts ``False``, ``True``, or a callable, which will filter which nodes to ignore when propagating. ++ maybe_placeholders ++ When ``True``, the ``[]`` operator returns ``None`` when not matched. ++ When ``False``, ``[]`` behaves like the ``?`` operator, and returns no value at all. ++ (default= ``True``) ++ cache ++ Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now. ++ ++ - When ``False``, does nothing (default) ++ - When ``True``, caches to a temporary file in the local directory ++ - When given a string, caches to the path pointed by the string ++ cache_grammar ++ For use with ``cache`` option. When ``True``, the unanalyzed grammar is also included in the cache. ++ Useful for classes that require the ``Lark.grammar`` to be present (e.g. Reconstructor). ++ (default= ``False``) ++ regex ++ When True, uses the ``regex`` module instead of the stdlib ``re``. ++ g_regex_flags ++ Flags that are applied to all terminals (both regex and strings) ++ keep_all_tokens ++ Prevent the tree builder from automagically removing "punctuation" tokens (Default: ``False``) ++ tree_class ++ Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``. ++ ++ **=== Algorithm Options ===** ++ ++ parser ++ Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley"). ++ (there is also a "cyk" option for legacy) ++ lexer ++ Decides whether or not to use a lexer stage ++ ++ - "auto" (default): Choose for me based on the parser ++ - "basic": Use a basic lexer ++ - "contextual": Stronger lexer (only works with parser="lalr") ++ - "dynamic": Flexible and powerful (only with parser="earley") ++ - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible. ++ ambiguity ++ Decides how to handle ambiguity in the parse. Only relevant if parser="earley" ++ ++ - "resolve": The parser will automatically choose the simplest derivation ++ (it chooses consistently: greedy for tokens, non-greedy for rules) ++ - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest). ++ - "forest": The parser will return the root of the shared packed parse forest. ++ ++ **=== Misc. / Domain Specific Options ===** ++ ++ postlex ++ Lexer post-processing (Default: ``None``) Only works with the basic and contextual lexers. ++ priority ++ How priorities should be evaluated - "auto", ``None``, "normal", "invert" (Default: "auto") ++ lexer_callbacks ++ Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution. ++ use_bytes ++ Accept an input of type ``bytes`` instead of ``str``. ++ ordered_sets ++ Should Earley use ordered-sets to achieve stable output (~10% slower than regular sets. Default: True) ++ edit_terminals ++ A callback for editing the terminals before parse. ++ import_paths ++ A List of either paths or loader functions to specify from where grammars are imported ++ source_path ++ Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading ++ **=== End of Options ===** ++ """ ++ if __doc__: ++ __doc__ += OPTIONS_DOC ++ ++ ++ ## ++ ++ ## ++ ++ ## ++ ++ ## ++ ++ ## ++ ++ ## ++ ++ _defaults: Dict[str, Any] = { ++ 'debug': False, ++ 'strict': False, ++ 'keep_all_tokens': False, ++ 'tree_class': None, ++ 'cache': False, ++ 'cache_grammar': False, ++ 'postlex': None, ++ 'parser': 'earley', ++ 'lexer': 'auto', ++ 'transformer': None, ++ 'start': 'start', ++ 'priority': 'auto', ++ 'ambiguity': 'auto', ++ 'regex': False, ++ 'propagate_positions': False, ++ 'lexer_callbacks': {}, ++ 'maybe_placeholders': True, ++ 'edit_terminals': None, ++ 'g_regex_flags': 0, ++ 'use_bytes': False, ++ 'ordered_sets': True, ++ 'import_paths': [], ++ 'source_path': None, ++ '_plugins': {}, ++ } ++ ++ def __init__(self, options_dict: Dict[str, Any]) -> None: ++ o = dict(options_dict) ++ ++ options = {} ++ for name, default in self._defaults.items(): ++ if name in o: ++ value = o.pop(name) ++ if isinstance(default, bool) and name not in ('cache', 'use_bytes', 'propagate_positions'): ++ value = bool(value) ++ else: ++ value = default ++ ++ options[name] = value ++ ++ if isinstance(options['start'], str): ++ options['start'] = [options['start']] ++ ++ self.__dict__['options'] = options ++ ++ ++ assert_config(self.parser, ('earley', 'lalr', 'cyk', None)) ++ ++ if self.parser == 'earley' and self.transformer: ++ raise ConfigurationError('Cannot specify an embedded transformer when using the Earley algorithm. ' ++ 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)') ++ ++ if self.cache_grammar and not self.cache: ++ raise ConfigurationError('cache_grammar cannot be set when cache is disabled') ++ ++ if o: ++ raise ConfigurationError("Unknown options: %s" % o.keys()) ++ ++ def __getattr__(self, name: str) -> Any: ++ try: ++ return self.__dict__['options'][name] ++ except KeyError as e: ++ raise AttributeError(e) ++ ++ def __setattr__(self, name: str, value: str) -> None: ++ assert_config(name, self.options.keys(), "%r isn't a valid option. Expected one of: %s") ++ self.options[name] = value ++ ++ def serialize(self, memo = None) -> Dict[str, Any]: ++ return self.options ++ ++ @classmethod ++ def deserialize(cls, data: Dict[str, Any], memo: Dict[int, Union[TerminalDef, Rule]]) -> "LarkOptions": ++ return cls(data) ++ ++ ++## ++ ++## ++ ++_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class', '_plugins'} ++ ++_VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None) ++_VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest') ++ ++ ++_T = TypeVar('_T', bound="Lark") ++ ++class Lark(Serialize): ++ #-- ++ ++ source_path: str ++ source_grammar: str ++ grammar: 'Grammar' ++ options: LarkOptions ++ lexer: Lexer ++ parser: 'ParsingFrontend' ++ terminals: Collection[TerminalDef] ++ ++ __serialize_fields__ = ['parser', 'rules', 'options'] ++ ++ def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None: ++ self.options = LarkOptions(options) ++ re_module: types.ModuleType ++ ++ ## ++ ++ if self.options.cache_grammar: ++ self.__serialize_fields__ = self.__serialize_fields__ + ['grammar'] ++ ++ ## ++ ++ use_regex = self.options.regex ++ if use_regex: ++ if _has_regex: ++ re_module = regex ++ else: ++ raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.') ++ else: ++ re_module = re ++ ++ ## ++ ++ if self.options.source_path is None: ++ try: ++ self.source_path = grammar.name ## ++ ++ except AttributeError: ++ self.source_path = '' ++ else: ++ self.source_path = self.options.source_path ++ ++ ## ++ ++ try: ++ read = grammar.read ## ++ ++ except AttributeError: ++ pass ++ else: ++ grammar = read() ++ ++ cache_fn = None ++ cache_sha256 = None ++ if isinstance(grammar, str): ++ self.source_grammar = grammar ++ if self.options.use_bytes: ++ if not grammar.isascii(): ++ raise ConfigurationError("Grammar must be ascii only, when use_bytes=True") ++ ++ if self.options.cache: ++ if self.options.parser != 'lalr': ++ raise ConfigurationError("cache only works with parser='lalr' for now") ++ ++ unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals', '_plugins') ++ options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable) ++ from . import __version__ ++ s = grammar + options_str + __version__ + str(sys.version_info[:2]) ++ cache_sha256 = sha256_digest(s) ++ ++ if isinstance(self.options.cache, str): ++ cache_fn = self.options.cache ++ else: ++ if self.options.cache is not True: ++ raise ConfigurationError("cache argument must be bool or str") ++ ++ try: ++ username = getpass.getuser() ++ except Exception: ++ ## ++ ++ ## ++ ++ ## ++ ++ username = "unknown" ++ ++ ++ cache_fn = tempfile.gettempdir() + "/.lark_%s_%s_%s_%s_%s.tmp" % ( ++ "cache_grammar" if self.options.cache_grammar else "cache", username, cache_sha256, *sys.version_info[:2]) ++ ++ old_options = self.options ++ try: ++ with FS.open(cache_fn, 'rb') as f: ++ logger.debug('Loading grammar from cache: %s', cache_fn) ++ ## ++ ++ for name in (set(options) - _LOAD_ALLOWED_OPTIONS): ++ del options[name] ++ file_sha256 = f.readline().rstrip(b'\n') ++ cached_used_files = pickle.load(f) ++ if file_sha256 == cache_sha256.encode('utf8') and verify_used_files(cached_used_files): ++ cached_parser_data = pickle.load(f) ++ self._load(cached_parser_data, **options) ++ return ++ except FileNotFoundError: ++ ## ++ ++ pass ++ except Exception: ## ++ ++ logger.exception("Failed to load Lark from cache: %r. We will try to carry on.", cache_fn) ++ ++ ## ++ ++ ## ++ ++ self.options = old_options ++ ++ ++ ## ++ ++ self.grammar, used_files = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens) ++ else: ++ assert isinstance(grammar, Grammar) ++ self.grammar = grammar ++ ++ ++ if self.options.lexer == 'auto': ++ if self.options.parser == 'lalr': ++ self.options.lexer = 'contextual' ++ elif self.options.parser == 'earley': ++ if self.options.postlex is not None: ++ logger.info("postlex can't be used with the dynamic lexer, so we use 'basic' instead. " ++ "Consider using lalr with contextual instead of earley") ++ self.options.lexer = 'basic' ++ else: ++ self.options.lexer = 'dynamic' ++ elif self.options.parser == 'cyk': ++ self.options.lexer = 'basic' ++ else: ++ assert False, self.options.parser ++ lexer = self.options.lexer ++ if isinstance(lexer, type): ++ assert issubclass(lexer, Lexer) ## ++ ++ else: ++ assert_config(lexer, ('basic', 'contextual', 'dynamic', 'dynamic_complete')) ++ if self.options.postlex is not None and 'dynamic' in lexer: ++ raise ConfigurationError("Can't use postlex with a dynamic lexer. Use basic or contextual instead") ++ ++ if self.options.ambiguity == 'auto': ++ if self.options.parser == 'earley': ++ self.options.ambiguity = 'resolve' ++ else: ++ assert_config(self.options.parser, ('earley', 'cyk'), "%r doesn't support disambiguation. Use one of these parsers instead: %s") ++ ++ if self.options.priority == 'auto': ++ self.options.priority = 'normal' ++ ++ if self.options.priority not in _VALID_PRIORITY_OPTIONS: ++ raise ConfigurationError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS)) ++ if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS: ++ raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS)) ++ ++ if self.options.parser is None: ++ terminals_to_keep = '*' ## ++ ++ elif self.options.postlex is not None: ++ terminals_to_keep = set(self.options.postlex.always_accept) ++ else: ++ terminals_to_keep = set() ++ ++ ## ++ ++ self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep) ++ ++ if self.options.edit_terminals: ++ for t in self.terminals: ++ self.options.edit_terminals(t) ++ ++ self._terminals_dict = {t.name: t for t in self.terminals} ++ ++ ## ++ ++ if self.options.priority == 'invert': ++ for rule in self.rules: ++ if rule.options.priority is not None: ++ rule.options.priority = -rule.options.priority ++ for term in self.terminals: ++ term.priority = -term.priority ++ ## ++ ++ ## ++ ++ ## ++ ++ elif self.options.priority is None: ++ for rule in self.rules: ++ if rule.options.priority is not None: ++ rule.options.priority = None ++ for term in self.terminals: ++ term.priority = 0 ++ ++ ## ++ ++ self.lexer_conf = LexerConf( ++ self.terminals, re_module, self.ignore_tokens, self.options.postlex, ++ self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes, strict=self.options.strict ++ ) ++ ++ if self.options.parser: ++ self.parser = self._build_parser() ++ elif lexer: ++ self.lexer = self._build_lexer() ++ ++ if cache_fn: ++ logger.debug('Saving grammar to cache: %s', cache_fn) ++ try: ++ with FS.open(cache_fn, 'wb') as f: ++ assert cache_sha256 is not None ++ f.write(cache_sha256.encode('utf8') + b'\n') ++ pickle.dump(used_files, f) ++ self.save(f, _LOAD_ALLOWED_OPTIONS) ++ except IOError as e: ++ logger.exception("Failed to save Lark to cache: %r.", cache_fn, e) ++ ++ if __doc__: ++ __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC ++ ++ def _build_lexer(self, dont_ignore: bool=False) -> BasicLexer: ++ lexer_conf = self.lexer_conf ++ if dont_ignore: ++ from copy import copy ++ lexer_conf = copy(lexer_conf) ++ lexer_conf.ignore = () ++ return BasicLexer(lexer_conf) ++ ++ def _prepare_callbacks(self) -> None: ++ self._callbacks = {} ++ ## ++ ++ if self.options.ambiguity != 'forest': ++ self._parse_tree_builder = ParseTreeBuilder( ++ self.rules, ++ self.options.tree_class or Tree, ++ self.options.propagate_positions, ++ self.options.parser != 'lalr' and self.options.ambiguity == 'explicit', ++ self.options.maybe_placeholders ++ ) ++ self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer) ++ self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals)) ++ ++ def _build_parser(self) -> "ParsingFrontend": ++ self._prepare_callbacks() ++ _validate_frontend_args(self.options.parser, self.options.lexer) ++ parser_conf = ParserConf(self.rules, self._callbacks, self.options.start) ++ return _construct_parsing_frontend( ++ self.options.parser, ++ self.options.lexer, ++ self.lexer_conf, ++ parser_conf, ++ options=self.options ++ ) ++ ++ def save(self, f, exclude_options: Collection[str] = ()) -> None: ++ #-- ++ if self.options.parser != 'lalr': ++ raise NotImplementedError("Lark.save() is only implemented for the LALR(1) parser.") ++ data, m = self.memo_serialize([TerminalDef, Rule]) ++ if exclude_options: ++ data["options"] = {n: v for n, v in data["options"].items() if n not in exclude_options} ++ pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL) ++ ++ @classmethod ++ def load(cls: Type[_T], f) -> _T: ++ #-- ++ inst = cls.__new__(cls) ++ return inst._load(f) ++ ++ def _deserialize_lexer_conf(self, data: Dict[str, Any], memo: Dict[int, Union[TerminalDef, Rule]], options: LarkOptions) -> LexerConf: ++ lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo) ++ lexer_conf.callbacks = options.lexer_callbacks or {} ++ lexer_conf.re_module = regex if options.regex else re ++ lexer_conf.use_bytes = options.use_bytes ++ lexer_conf.g_regex_flags = options.g_regex_flags ++ lexer_conf.skip_validation = True ++ lexer_conf.postlex = options.postlex ++ return lexer_conf ++ ++ def _load(self: _T, f: Any, **kwargs) -> _T: ++ if isinstance(f, dict): ++ d = f ++ else: ++ d = pickle.load(f) ++ memo_json = d['memo'] ++ data = d['data'] ++ ++ assert memo_json ++ memo = SerializeMemoizer.deserialize(memo_json, {'Rule': Rule, 'TerminalDef': TerminalDef}, {}) ++ if 'grammar' in data: ++ self.grammar = Grammar.deserialize(data['grammar'], memo) ++ options = dict(data['options']) ++ if (set(kwargs) - _LOAD_ALLOWED_OPTIONS) & set(LarkOptions._defaults): ++ raise ConfigurationError("Some options are not allowed when loading a Parser: {}" ++ .format(set(kwargs) - _LOAD_ALLOWED_OPTIONS)) ++ options.update(kwargs) ++ self.options = LarkOptions.deserialize(options, memo) ++ self.rules = [Rule.deserialize(r, memo) for r in data['rules']] ++ self.source_path = '' ++ _validate_frontend_args(self.options.parser, self.options.lexer) ++ self.lexer_conf = self._deserialize_lexer_conf(data['parser'], memo, self.options) ++ self.terminals = self.lexer_conf.terminals ++ self._prepare_callbacks() ++ self._terminals_dict = {t.name: t for t in self.terminals} ++ self.parser = _deserialize_parsing_frontend( ++ data['parser'], ++ memo, ++ self.lexer_conf, ++ self._callbacks, ++ self.options, ## ++ ++ ) ++ return self ++ ++ @classmethod ++ def _load_from_dict(cls, data, memo, **kwargs): ++ inst = cls.__new__(cls) ++ return inst._load({'data': data, 'memo': memo}, **kwargs) ++ ++ @classmethod ++ def open(cls: Type[_T], grammar_filename: str, rel_to: Optional[str]=None, **options) -> _T: ++ #-- ++ if rel_to: ++ basepath = os.path.dirname(rel_to) ++ grammar_filename = os.path.join(basepath, grammar_filename) ++ with open(grammar_filename, encoding='utf8') as f: ++ return cls(f, **options) ++ ++ @classmethod ++ def open_from_package(cls: Type[_T], package: str, grammar_path: str, search_paths: 'Sequence[str]'=[""], **options) -> _T: ++ #-- ++ package_loader = FromPackageLoader(package, search_paths) ++ full_path, text = package_loader(None, grammar_path) ++ options.setdefault('source_path', full_path) ++ options.setdefault('import_paths', []) ++ options['import_paths'].append(package_loader) ++ return cls(text, **options) ++ ++ def __repr__(self): ++ return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source_path, self.options.parser, self.options.lexer) ++ ++ ++ def lex(self, text: TextOrSlice, dont_ignore: bool=False) -> Iterator[Token]: ++ #-- ++ lexer: Lexer ++ if not hasattr(self, 'lexer') or dont_ignore: ++ lexer = self._build_lexer(dont_ignore) ++ else: ++ lexer = self.lexer ++ lexer_thread = LexerThread.from_text(lexer, text) ++ stream = lexer_thread.lex(None) ++ if self.options.postlex: ++ return self.options.postlex.process(stream) ++ return stream ++ ++ def get_terminal(self, name: str) -> TerminalDef: ++ #-- ++ return self._terminals_dict[name] ++ ++ def parse_interactive(self, text: Optional[LarkInput]=None, start: Optional[str]=None) -> 'InteractiveParser': ++ #-- ++ return self.parser.parse_interactive(text, start=start) ++ ++ def parse(self, text: LarkInput, start: Optional[str]=None, on_error: 'Optional[Callable[[UnexpectedInput], bool]]'=None) -> 'ParseTree': ++ #-- ++ if on_error is not None and self.options.parser != 'lalr': ++ raise NotImplementedError("The on_error option is only implemented for the LALR(1) parser.") ++ return self.parser.parse(text, start=start, on_error=on_error) ++ ++ ++ ++ ++class DedentError(LarkError): ++ pass ++ ++class Indenter(PostLex, ABC): ++ #-- ++ paren_level: int ++ indent_level: List[int] ++ ++ def __init__(self) -> None: ++ self.paren_level = 0 ++ self.indent_level = [0] ++ assert self.tab_len > 0 ++ ++ def handle_NL(self, token: Token) -> Iterator[Token]: ++ if self.paren_level > 0: ++ return ++ ++ yield token ++ ++ indent_str = token.rsplit('\n', 1)[1] ## ++ ++ indent = indent_str.count(' ') + indent_str.count('\t') * self.tab_len ++ ++ if indent > self.indent_level[-1]: ++ self.indent_level.append(indent) ++ yield Token.new_borrow_pos(self.INDENT_type, indent_str, token) ++ else: ++ while indent < self.indent_level[-1]: ++ self.indent_level.pop() ++ yield Token.new_borrow_pos(self.DEDENT_type, indent_str, token) ++ ++ if indent != self.indent_level[-1]: ++ raise DedentError('Unexpected dedent to column %s. Expected dedent to %s' % (indent, self.indent_level[-1])) ++ ++ def _process(self, stream): ++ token = None ++ for token in stream: ++ if token.type == self.NL_type: ++ yield from self.handle_NL(token) ++ else: ++ yield token ++ ++ if token.type in self.OPEN_PAREN_types: ++ self.paren_level += 1 ++ elif token.type in self.CLOSE_PAREN_types: ++ self.paren_level -= 1 ++ assert self.paren_level >= 0 ++ ++ while len(self.indent_level) > 1: ++ self.indent_level.pop() ++ yield Token.new_borrow_pos(self.DEDENT_type, '', token) if token else Token(self.DEDENT_type, '', 0, 0, 0, 0, 0, 0) ++ ++ assert self.indent_level == [0], self.indent_level ++ ++ def process(self, stream): ++ self.paren_level = 0 ++ self.indent_level = [0] ++ return self._process(stream) ++ ++ ## ++ ++ @property ++ def always_accept(self): ++ return (self.NL_type,) ++ ++ @property ++ @abstractmethod ++ def NL_type(self) -> str: ++ #-- ++ raise NotImplementedError() ++ ++ @property ++ @abstractmethod ++ def OPEN_PAREN_types(self) -> List[str]: ++ #-- ++ raise NotImplementedError() ++ ++ @property ++ @abstractmethod ++ def CLOSE_PAREN_types(self) -> List[str]: ++ #-- ++ raise NotImplementedError() ++ ++ @property ++ @abstractmethod ++ def INDENT_type(self) -> str: ++ #-- ++ raise NotImplementedError() ++ ++ @property ++ @abstractmethod ++ def DEDENT_type(self) -> str: ++ #-- ++ raise NotImplementedError() ++ ++ @property ++ @abstractmethod ++ def tab_len(self) -> int: ++ #-- ++ raise NotImplementedError() ++ ++ ++class PythonIndenter(Indenter): ++ #-- ++ ++ NL_type = '_NEWLINE' ++ OPEN_PAREN_types = ['LPAR', 'LSQB', 'LBRACE'] ++ CLOSE_PAREN_types = ['RPAR', 'RSQB', 'RBRACE'] ++ INDENT_type = '_INDENT' ++ DEDENT_type = '_DEDENT' ++ tab_len = 8 ++ ++ ++import pickle, zlib, base64 ++DATA = ( ++{'parser': {'lexer_conf': {'terminals': [{'@': 0}, {'@': 1}, {'@': 2}, {'@': 3}, {'@': 4}, {'@': 5}], 'ignore': [], 'g_regex_flags': 0, 'use_bytes': False, 'lexer_type': 'contextual', '__type__': 'LexerConf'}, 'parser_conf': {'rules': [{'@': 6}, {'@': 7}, {'@': 8}, {'@': 9}, {'@': 10}, {'@': 11}, {'@': 12}, {'@': 13}, {'@': 14}, {'@': 15}, {'@': 16}, {'@': 17}, {'@': 18}, {'@': 19}, {'@': 20}, {'@': 21}, {'@': 22}, {'@': 23}, {'@': 24}], 'start': ['start'], 'parser_type': 'lalr', '__type__': 'ParserConf'}, 'parser': {'tokens': {0: '__ANON_0', 1: 'SPACE', 2: '$END', 3: 'EQUAL', 4: '__ANON_2', 5: 'value_with_spaces', 6: '__parameter_list_plus_0', 7: '__parameter_list_star_1', 8: 'DBLQUOTE', 9: 'bare_value', 10: 'value', 11: '__ANON_1', 12: 'quoted_value', 13: 'key', 14: 'parameter', 15: 'key_value_pair', 16: 'kernel_command_line', 17: 'parameter_list', 18: 'start'}, 'states': {0: {0: (1, {'@': 21}), 1: (1, {'@': 21})}, 1: {2: (1, {'@': 16}), 1: (1, {'@': 16})}, 2: {2: (1, {'@': 17}), 1: (1, {'@': 17})}, 3: {2: (1, {'@': 6})}, 4: {2: (1, {'@': 19}), 1: (1, {'@': 19})}, 5: {1: (1, {'@': 15}), 3: (1, {'@': 15}), 2: (1, {'@': 15})}, 6: {4: (0, 24), 5: (0, 8)}, 7: {1: (0, 0), 6: (0, 22), 7: (0, 20), 2: (1, {'@': 9})}, 8: {8: (0, 12)}, 9: {2: (1, {'@': 7})}, 10: {2: (1, {'@': 23}), 1: (1, {'@': 23})}, 11: {2: (1, {'@': 13}), 1: (1, {'@': 13})}, 12: {2: (1, {'@': 18}), 1: (1, {'@': 18})}, 13: {0: (1, {'@': 22}), 1: (1, {'@': 22})}, 14: {2: (1, {'@': 24}), 1: (1, {'@': 24})}, 15: {}, 16: {9: (0, 1), 10: (0, 19), 11: (0, 4), 8: (0, 6), 12: (0, 2)}, 17: {1: (0, 0), 6: (0, 21), 2: (1, {'@': 10})}, 18: {3: (0, 16), 2: (1, {'@': 12}), 1: (1, {'@': 12})}, 19: {2: (1, {'@': 14}), 1: (1, {'@': 14})}, 20: {1: (0, 0), 6: (0, 21), 2: (1, {'@': 8})}, 21: {13: (0, 18), 14: (0, 14), 0: (0, 5), 1: (0, 13), 15: (0, 11)}, 22: {13: (0, 18), 0: (0, 5), 14: (0, 10), 1: (0, 13), 15: (0, 11)}, 23: {16: (0, 3), 13: (0, 18), 14: (0, 7), 1: (0, 0), 0: (0, 5), 6: (0, 22), 15: (0, 11), 17: (0, 9), 18: (0, 15), 7: (0, 17), 2: (1, {'@': 11})}, 24: {8: (1, {'@': 20})}}, 'start_states': {'start': 23}, 'end_states': {'start': 15}}, '__type__': 'ParsingFrontend'}, 'rules': [{'@': 6}, {'@': 7}, {'@': 8}, {'@': 9}, {'@': 10}, {'@': 11}, {'@': 12}, {'@': 13}, {'@': 14}, {'@': 15}, {'@': 16}, {'@': 17}, {'@': 18}, {'@': 19}, {'@': 20}, {'@': 21}, {'@': 22}, {'@': 23}, {'@': 24}], 'options': {'debug': False, 'strict': False, 'keep_all_tokens': False, 'tree_class': None, 'cache': False, 'cache_grammar': False, 'postlex': None, 'parser': 'lalr', 'lexer': 'contextual', 'transformer': None, 'start': ['start'], 'priority': 'normal', 'ambiguity': 'auto', 'regex': False, 'propagate_positions': False, 'lexer_callbacks': {}, 'maybe_placeholders': False, 'edit_terminals': None, 'g_regex_flags': 0, 'use_bytes': False, 'ordered_sets': True, 'import_paths': [], 'source_path': None, '_plugins': {}}, '__type__': 'Lark'} ++) ++MEMO = ( ++{0: {'name': 'SPACE', 'pattern': {'value': ' ', 'flags': [], 'raw': '" "', '__type__': 'PatternStr'}, 'priority': 0, '__type__': 'TerminalDef'}, 1: {'name': 'EQUAL', 'pattern': {'value': '=', 'flags': [], 'raw': '"="', '__type__': 'PatternStr'}, 'priority': 0, '__type__': 'TerminalDef'}, 2: {'name': '__ANON_0', 'pattern': {'value': '[A-Za-z0-9_\\-\\.]+', 'flags': [], 'raw': '/[A-Za-z0-9_\\-\\.]+/', '_width': [1, 18446744073709551616], '__type__': 'PatternRE'}, 'priority': 0, '__type__': 'TerminalDef'}, 3: {'name': 'DBLQUOTE', 'pattern': {'value': '"', 'flags': [], 'raw': '"\\""', '__type__': 'PatternStr'}, 'priority': 0, '__type__': 'TerminalDef'}, 4: {'name': '__ANON_1', 'pattern': {'value': '[\\!\\#-\\\\.0-9:-\\@A-Za-z\\[-~]+', 'flags': [], 'raw': '/[\\!\\#-\\\\.0-9:-\\@A-Za-z\\[-~]+/', '_width': [1, 18446744073709551616], '__type__': 'PatternRE'}, 'priority': 0, '__type__': 'TerminalDef'}, 5: {'name': '__ANON_2', 'pattern': {'value': '[\\!\\#-\\\\.0-9:-\\@A-Za-z\\[-~ ]+', 'flags': [], 'raw': '/[\\!\\#-\\\\.0-9:-\\@A-Za-z\\[-~ ]+/', '_width': [1, 18446744073709551616], '__type__': 'PatternRE'}, 'priority': 0, '__type__': 'TerminalDef'}, 6: {'origin': {'name': 'start', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'kernel_command_line', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': True, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 7: {'origin': {'name': 'kernel_command_line', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'parameter_list', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 8: {'origin': {'name': 'parameter_list', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'parameter', '__type__': 'NonTerminal'}, {'name': '__parameter_list_star_1', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 9: {'origin': {'name': 'parameter_list', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'parameter', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 10: {'origin': {'name': 'parameter_list', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__parameter_list_star_1', '__type__': 'NonTerminal'}], 'order': 2, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 11: {'origin': {'name': 'parameter_list', '__type__': 'NonTerminal'}, 'expansion': [], 'order': 3, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 12: {'origin': {'name': 'parameter', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'key', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 13: {'origin': {'name': 'parameter', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'key_value_pair', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 14: {'origin': {'name': 'key_value_pair', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'key', '__type__': 'NonTerminal'}, {'name': 'EQUAL', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'value', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 15: {'origin': {'name': 'key', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__ANON_0', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 16: {'origin': {'name': 'value', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'bare_value', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 17: {'origin': {'name': 'value', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'quoted_value', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 18: {'origin': {'name': 'quoted_value', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'DBLQUOTE', 'filter_out': True, '__type__': 'Terminal'}, {'name': 'value_with_spaces', '__type__': 'NonTerminal'}, {'name': 'DBLQUOTE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 19: {'origin': {'name': 'bare_value', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__ANON_1', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 20: {'origin': {'name': 'value_with_spaces', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__ANON_2', 'filter_out': False, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 21: {'origin': {'name': '__parameter_list_plus_0', '__type__': 'NonTerminal'}, 'expansion': [{'name': 'SPACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 22: {'origin': {'name': '__parameter_list_plus_0', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__parameter_list_plus_0', '__type__': 'NonTerminal'}, {'name': 'SPACE', 'filter_out': True, '__type__': 'Terminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 23: {'origin': {'name': '__parameter_list_star_1', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__parameter_list_plus_0', '__type__': 'NonTerminal'}, {'name': 'parameter', '__type__': 'NonTerminal'}], 'order': 0, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}, 24: {'origin': {'name': '__parameter_list_star_1', '__type__': 'NonTerminal'}, 'expansion': [{'name': '__parameter_list_star_1', '__type__': 'NonTerminal'}, {'name': '__parameter_list_plus_0', '__type__': 'NonTerminal'}, {'name': 'parameter', '__type__': 'NonTerminal'}], 'order': 1, 'alias': None, 'options': {'keep_all_tokens': False, 'expand1': False, 'priority': None, 'template_source': None, 'empty_indices': (), '__type__': 'RuleOptions'}, '__type__': 'Rule'}} ++) ++Shift = 0 ++Reduce = 1 ++def Lark_StandAlone(**kwargs): ++ return Lark._load_from_dict(DATA, MEMO, **kwargs) +diff --git a/ironic/common/kernel_parameters.py b/ironic/common/kernel_parameters.py +new file mode 100644 +index 000000000..ca0cc924e +--- /dev/null ++++ b/ironic/common/kernel_parameters.py +@@ -0,0 +1,159 @@ ++# Licensed under the Apache License, Version 2.0 (the "License"); you may ++# not use this file except in compliance with the License. You may obtain ++# a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations ++# under the License. ++ ++from dataclasses import dataclass ++ ++from ironic.common.exception import InvalidParameterValue ++from ironic.common.i18n import _ ++from ironic.common.kernel_parameter_parser.kernel_parameter_parser \ ++ import Lark_StandAlone ++from ironic.common.kernel_parameter_parser.kernel_parameter_parser \ ++ import LarkError ++from ironic.common.kernel_parameter_parser.kernel_parameter_parser \ ++ import Transformer ++from ironic.common.kernel_parameter_parser.kernel_parameter_parser \ ++ import UnexpectedInput ++ ++ ++def sanitize_kernel_command_line(command_line: str) -> str: ++ """Applies filtering to a command line to sanitize it. ++ ++ NOTE: This does not guarantee a correct or safe kernel command line, ++ for stronger guarantees of correctness and safety use ++ KernelCommandLine.parse(). ++ ++ :param command_line: A string containing a kernel command line or ++ individual parameters. ++ :returns: A filtered string which should be safer for use. ++ """ ++ return ''.join(c for c in command_line if c not in {'\n', '\r', '\0'}) ++ ++ ++KernelParameterParser = Lark_StandAlone(debug=True) ++ ++ ++@dataclass(frozen=True) ++class ParameterKey: ++ key: str ++ ++ def __str__(self): ++ return self.key ++ ++ ++@dataclass(frozen=True) ++class ParameterValue: ++ value: str ++ ++ def __str__(self): ++ if ' ' in self.value: ++ return f"\"{self.value}\"" ++ return self.value ++ ++ ++@dataclass(frozen=True) ++class KernelParameter: ++ key: ParameterKey ++ value: ParameterValue ++ ++ def __str__(self): ++ if len(self.value.value) > 0: ++ return f"{self.key.key}={self.value.value}" ++ return self.key.key ++ ++ ++_INIT_ARG_PREAMBLE = " -- " ++ ++ ++# NOTE(clif): We're handling init args here instead of inside the grammar ++# because Lark's stand-alone LALR(1) parser can't handle it. ++def _divide_command_line_by_init_args(command_line: str) -> tuple[str, str]: ++ index = command_line.rfind(_INIT_ARG_PREAMBLE) ++ if index == -1: ++ return (command_line, '') ++ return (command_line[:index], ++ command_line[index + len(_INIT_ARG_PREAMBLE):]) ++ ++ ++@dataclass(frozen=True) ++class KernelCommandLine: ++ parameters: dict[str, list[KernelParameter]] ++ init_args: str ++ ++ def __str__(self): ++ output = ' '.join( ++ ' '.join(str(param) for param in param_list) ++ for param_list in self.parameters.values()) ++ if len(self.init_args) > 0: ++ output += _INIT_ARG_PREAMBLE + self.init_args ++ return output ++ ++ @classmethod ++ def parse(cls, command_line: str): ++ try: ++ cmd_line, init_args = \ ++ _divide_command_line_by_init_args(command_line) ++ tree = KernelParameterParser.parse(cmd_line) ++ kcl = KernelParameterTransformer().transform(tree) ++ return KernelCommandLine(kcl.parameters, init_args) ++ except (LarkError, UnexpectedInput) as e: ++ raise InvalidParameterValue( ++ _('Kernel command line did not parse: "%s" -- %s') ++ % (command_line, str(e))) from None ++ ++ ++class KernelParameterTransformer(Transformer): ++ def kernel_command_line(self, items): ++ # NOTE(clif) adding init arguments to the grammar is too much for ++ # Lark's stand-alone LALR(1) parser. Therefore it isn't part of the ++ # back-ported grammar. ++ return KernelCommandLine(items[0], '') ++ ++ def parameter_list(self, items): ++ parameters = {} ++ for item in items: ++ if item.key.key in parameters.keys(): ++ parameters[item.key.key].append(item) ++ else: ++ parameters[item.key.key] = [item] ++ return parameters ++ ++ def parameter(self, items): ++ if isinstance(items[0], ParameterKey): ++ return KernelParameter(items[0], ParameterValue("")) ++ return items[0] ++ ++ def key_value_pair(self, items): ++ key = items[0] ++ value = items[1] ++ return KernelParameter(key, value) ++ ++ def key(self, items): ++ return ParameterKey(items[0].value) ++ ++ def value(self, items): ++ return ParameterValue(items[0]) ++ ++ def quoted_value(self, items): ++ # Strip " characters from literal. ++ return items[0].value[1:-1] ++ ++ def bare_value(self, items): ++ return items[0].value ++ ++ def value_with_spaces(self, items): ++ return items[0].value ++ ++ def init_suffix(self, items): ++ return items[0] ++ ++ def init_arguments(self, items): ++ return items[0] +diff --git a/ironic/common/pxe_utils.py b/ironic/common/pxe_utils.py +index a4fd438ec..951d86854 100644 +--- a/ironic/common/pxe_utils.py ++++ b/ironic/common/pxe_utils.py +@@ -936,6 +936,8 @@ def build_pxe_config_options(task, pxe_info, service=False, + as kernel command-line arguments. + :returns: A dictionary of pxe options to be used in the pxe bootfile + template. ++ ++ :raises: InvalidParameterValue via get_kernel_append_params + """ + node = task.node + mode = deploy_utils.rescue_or_deploy_mode(node) +diff --git a/ironic/conf/conductor.py b/ironic/conf/conductor.py +index 9f5e9fe45..7a81d05df 100644 +--- a/ironic/conf/conductor.py ++++ b/ironic/conf/conductor.py +@@ -430,6 +430,18 @@ opts = [ + 'here are validated as absolute paths and will be rejected' + 'if they contain path traversal mechanisms, such as "..".' + )), ++ cfg.BoolOpt('disable_kernel_parameter_parsing', ++ default=False, ++ # Normally such an option would be mutable, but this is, ++ # a security guard and operators should not expect to change ++ # this option under normal circumstances. ++ mutable=False, ++ help=_('Disable parsing of kernel parameters. Kernel ' ++ 'parameter parsing allows Ironic to detect and prevent ' ++ 'malformed kernel parameters before they are passed to ' ++ 'nodes. Malformed kernel parameters can pose a ' ++ 'security risk therefore it is not recommended to ' ++ 'disable this option unless absolutely necessary.')), + ] + + +diff --git a/ironic/drivers/utils.py b/ironic/drivers/utils.py +index fc5cdcf0d..594574ca3 100644 +--- a/ironic/drivers/utils.py ++++ b/ironic/drivers/utils.py +@@ -23,6 +23,7 @@ from oslo_utils import timeutils + + from ironic.common import exception + from ironic.common.i18n import _ ++from ironic.common import kernel_parameters + from ironic.common import states + from ironic.common import swift + from ironic.conductor import utils +@@ -406,11 +407,30 @@ def get_kernel_append_params(node, default): + + :param node: Node object. + :param default: Default value. ++ ++ :raises: InvalidParameterValue if kernel_append_params is an invalid ++ string to append to a kernel command line. + """ + for location in ('instance_info', 'driver_info'): + result = getattr(node, location).get('kernel_append_params') + if result is not None: +- return result.replace('%default%', default or '') ++ result = result.replace('%default%', default or '') ++ ++ if not CONF.conductor.disable_kernel_parameter_parsing: ++ # NOTE(clif) Attempt to parse the append params. Failure to ++ # parse indicates malformed kernel parameters and should be ++ # rejected. parse() will raise if parsing fails. ++ try: ++ kernel_parameters.KernelCommandLine.parse(result) ++ except exception.InvalidParameterValue: ++ raise exception.InvalidParameterValue( ++ _('node\'s %s[\'kernel_append_params\'] contains ' ++ 'malformed kernel command line') % location) ++ ++ # NOTE(clif) Always run basic sanitization on kernel_append_params ++ result = kernel_parameters.sanitize_kernel_command_line(result) ++ ++ return result + + return default + +diff --git a/ironic/tests/unit/common/test_kernel_parameters.py b/ironic/tests/unit/common/test_kernel_parameters.py +new file mode 100644 +index 000000000..b2acdc5a3 +--- /dev/null ++++ b/ironic/tests/unit/common/test_kernel_parameters.py +@@ -0,0 +1,230 @@ ++# Licensed under the Apache License, Version 2.0 (the "License"); you may ++# not use this file except in compliance with the License. You may obtain ++# a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations ++# under the License. ++ ++from ddt import data ++from ddt import ddt ++from ddt import unpack ++ ++from ironic.common.exception import InvalidParameterValue ++import ironic.common.kernel_parameters as kp ++from ironic.tests import base ++ ++ ++def annotate(name, *args): ++ class AnnotatedList(list): ++ pass ++ ++ al = AnnotatedList([*args]) ++ al.__name__ = name ++ return al ++ ++ ++def generate_invalid_characters_to_test(): ++ invalid_characters_to_test = [ ++ chr(c) for c in range(0, 32) ++ ] ++ invalid_characters_to_test.extend([ ++ "\n", ++ "\r", ++ chr(127), ++ ]) ++ invalid_characters_to_test.extend([ ++ chr(c) for c in range(128, 160) ++ ]) ++ return invalid_characters_to_test ++ ++ ++INVALID_CHARACTERS = generate_invalid_characters_to_test() ++ ++ ++class KernelParamTryout(base.TestCase): ++ def test_if_can_parse(self): ++ result = kp.KernelCommandLine.parse("quiet") ++ self.assertIsNotNone(result) ++ ++ ++@ddt ++class KernelParametersTestCase(base.TestCase): ++ @data( ++ annotate( ++ "Filtering newlines", ++ "quiet\n", ++ "quiet" ++ ), ++ annotate( ++ "Filtering carraige returns", ++ "qu\riet", ++ "quiet" ++ ), ++ annotate( ++ "Filtering NULL", ++ "\0quiet", ++ "quiet" ++ ), ++ annotate( ++ "Nothing needs changing - a real valid kernel cmdline", ++ ("BOOT_IMAGE=(hd5,gpt2)/vmlinuz-6.19.9-200.fc43.x86_64 " ++ "root=UUID=217c8a40-4956-11f1-9c98-d8bbc1c85452 ro " ++ "rootflags=subvol=root " ++ "rd.luks.uuid=luks-3a516752-4956-11f1-aa13-d8bbc1c85452 " ++ "rhgb quiet rd.driver.blacklist=nouveau,nova_core " ++ "modprobe.blacklist=nouveau,nova_core"), ++ ("BOOT_IMAGE=(hd5,gpt2)/vmlinuz-6.19.9-200.fc43.x86_64 " ++ "root=UUID=217c8a40-4956-11f1-9c98-d8bbc1c85452 ro " ++ "rootflags=subvol=root " ++ "rd.luks.uuid=luks-3a516752-4956-11f1-aa13-d8bbc1c85452 " ++ "rhgb quiet rd.driver.blacklist=nouveau,nova_core " ++ "modprobe.blacklist=nouveau,nova_core") ++ ), ++ ) ++ @unpack ++ def test_sanitize_kernel_command_line( ++ self, command_line: str, expected_result: str): ++ self.assertEqual( ++ expected_result, ++ kp.sanitize_kernel_command_line(command_line)) ++ ++ @data( ++ annotate( ++ "Single key=value pair", ++ "BOOT_IMAGE=(hd5,gpt2)/vmlinuz-6.19.9-200.fc43.x86_64", ++ kp.KernelCommandLine({ ++ 'BOOT_IMAGE': [kp.KernelParameter( ++ kp.ParameterKey('BOOT_IMAGE'), ++ kp.ParameterValue( ++ '(hd5,gpt2)/vmlinuz-6.19.9-200.fc43.x86_64') ++ )], ++ }, "") ++ ), ++ annotate( ++ "Single key", ++ "quiet", ++ kp.KernelCommandLine({ ++ 'quiet': [kp.KernelParameter( ++ kp.ParameterKey('quiet'), ++ kp.ParameterValue(''), ++ )], ++ }, "") ++ ), ++ annotate( ++ "Two parameters", ++ "quiet BOOT_IMAGE=(hd5,gpt2)/vmlinuz-6.19.9-200.fc43.x86_64", ++ kp.KernelCommandLine({ ++ 'quiet': [kp.KernelParameter( ++ kp.ParameterKey('quiet'), ++ kp.ParameterValue(''), ++ )], ++ 'BOOT_IMAGE': [kp.KernelParameter( ++ kp.ParameterKey('BOOT_IMAGE'), ++ kp.ParameterValue( ++ '(hd5,gpt2)/vmlinuz-6.19.9-200.fc43.x86_64') ++ )], ++ }, "") ++ ), ++ annotate( ++ "A real linux kernel cmdline", ++ ("BOOT_IMAGE=(hd5,gpt2)/vmlinuz-6.19.9-200.fc43.x86_64 " ++ "root=UUID=217c8a40-4956-11f1-9c98-d8bbc1c85452 ro " ++ "rootflags=subvol=root " ++ "rd.luks.uuid=luks-3a516752-4956-11f1-aa13-d8bbc1c85452 " ++ "rhgb quiet rd.driver.blacklist=nouveau,nova_core " ++ "modprobe.blacklist=nouveau,nova_core"), ++ kp.KernelCommandLine({ ++ 'BOOT_IMAGE': [kp.KernelParameter( ++ kp.ParameterKey('BOOT_IMAGE'), ++ kp.ParameterValue( ++ '(hd5,gpt2)/vmlinuz-6.19.9-200.fc43.x86_64') ++ )], ++ 'root': [kp.KernelParameter( ++ kp.ParameterKey('root'), ++ kp.ParameterValue( ++ 'UUID=217c8a40-4956-11f1-9c98-d8bbc1c85452'), ++ )], ++ 'ro': [kp.KernelParameter( ++ kp.ParameterKey('ro'), ++ kp.ParameterValue(''), ++ )], ++ 'rootflags': [kp.KernelParameter( ++ kp.ParameterKey('rootflags'), ++ kp.ParameterValue('subvol=root'), ++ )], ++ 'rd.luks.uuid': [kp.KernelParameter( ++ kp.ParameterKey('rd.luks.uuid'), ++ kp.ParameterValue( ++ 'luks-3a516752-4956-11f1-aa13-d8bbc1c85452'), ++ )], ++ 'rhgb': [kp.KernelParameter( ++ kp.ParameterKey('rhgb'), ++ kp.ParameterValue(''), ++ )], ++ 'quiet': [kp.KernelParameter( ++ kp.ParameterKey('quiet'), ++ kp.ParameterValue(''), ++ )], ++ 'rd.driver.blacklist': [kp.KernelParameter( ++ kp.ParameterKey('rd.driver.blacklist'), ++ kp.ParameterValue('nouveau,nova_core'), ++ )], ++ 'modprobe.blacklist': [kp.KernelParameter( ++ kp.ParameterKey('modprobe.blacklist'), ++ kp.ParameterValue('nouveau,nova_core'), ++ )], ++ }, "") ++ ), ++ annotate( ++ "Multiple parameters with the same key", ++ "initrd=/initramfs-linux.img initrd=ramdisk", ++ kp.KernelCommandLine({ ++ 'initrd': [ ++ kp.KernelParameter( ++ kp.ParameterKey('initrd'), ++ kp.ParameterValue('/initramfs-linux.img') ++ ), ++ kp.KernelParameter( ++ kp.ParameterKey('initrd'), ++ kp.ParameterValue('ramdisk') ++ ) ++ ]}, "") ++ ), ++ annotate( ++ "init arguments", ++ "quiet -- some init args", ++ kp.KernelCommandLine({ ++ 'quiet': [kp.KernelParameter( ++ kp.ParameterKey('quiet'), ++ kp.ParameterValue(''), ++ )], ++ }, "some init args") ++ ), ++ ) ++ @unpack ++ def test_kernel_command_line_parsing( ++ self, command_line: str, expected_result: kp.KernelCommandLine): ++ result = kp.KernelCommandLine.parse(command_line) ++ # Assert parsing the command line spits out the expected ++ # object. ++ self.assertEqual(expected_result, result) ++ # Assert rendering the object back to a string matches the initial ++ # command line string. ++ self.assertEqual(command_line, str(result)) ++ ++ @data( ++ *[annotate( ++ f"character ordinal {ord(c)} shouldn't parse", ++ f"ro{c}quiet",) for c in INVALID_CHARACTERS] ++ ) ++ @unpack ++ def test_invalid_kernel_command_lines_fail_to_parse( ++ self, command_line: str): ++ self.assertRaises(InvalidParameterValue, ++ kp.KernelCommandLine.parse, ++ command_line) +diff --git a/ironic/tests/unit/drivers/test_utils.py b/ironic/tests/unit/drivers/test_utils.py +index f2e79e827..dc07f4dca 100644 +--- a/ironic/tests/unit/drivers/test_utils.py ++++ b/ironic/tests/unit/drivers/test_utils.py +@@ -13,10 +13,14 @@ + # License for the specific language governing permissions and limitations + # under the License. + ++from dataclasses import dataclass + import datetime + import os + from unittest import mock + ++from ddt import data ++from ddt import ddt ++from ddt import unpack + from oslo_config import cfg + from oslo_utils import timeutils + +@@ -32,6 +36,15 @@ from ironic.tests.unit.db import base as db_base + from ironic.tests.unit.objects import utils as obj_utils + + ++def annotate(name, *args): ++ class AnnotatedList(list): ++ pass ++ ++ al = AnnotatedList([*args]) ++ al.__name__ = name ++ return al ++ ++ + class UtilsTestCase(db_base.DbTestCase): + + def setUp(self): +@@ -222,6 +235,90 @@ class UtilsTestCase(db_base.DbTestCase): + self.assertEqual("0a1b2c3d4f", mac_clean) + + ++@ddt ++class GetKernelAppendParamsTestCase(tests_base.TestCase): ++ @dataclass(frozen=True) ++ class FauxTestNode: ++ instance_info: dict ++ driver_info: dict ++ ++ @data( ++ annotate( ++ "valid params in instance_info", ++ FauxTestNode({'kernel_append_params': 'quiet ro'}, ++ {}), ++ '', ++ 'quiet ro', ++ False, ++ False ++ ), ++ annotate( ++ "valid params in driver_info", ++ FauxTestNode({}, ++ {'kernel_append_params': 'quiet ro'}), ++ '', ++ 'quiet ro', ++ False, ++ False ++ ), ++ annotate( ++ "params in default", ++ FauxTestNode({}, {}), ++ 'quiet ro', ++ 'quiet ro', ++ False, ++ False ++ ), ++ annotate( ++ "invalid params in instance_info raises", ++ FauxTestNode({'kernel_append_params': 'bad\nparams'}, {}), ++ '', ++ '', ++ True, ++ False ++ ++ ), ++ annotate( ++ "invalid params in driver_info raises", ++ FauxTestNode({}, {'kernel_append_params': 'bad\nparams'}), ++ '', ++ '', ++ True, ++ False ++ ), ++ annotate( ++ "parsing disabled - but newline is filtered", ++ FauxTestNode({}, {'kernel_append_params': 'quiet\n ro'}), ++ '', ++ 'quiet ro', ++ False, ++ True, ++ ), ++ ) ++ @unpack ++ def test_get_kernel_append_params( ++ self, ++ test_node: FauxTestNode, ++ default: str, ++ expected_result: str, ++ should_raise: bool, ++ disable_kernel_parameter_parsing: bool): ++ cfg.CONF.set_override('disable_kernel_parameter_parsing', ++ disable_kernel_parameter_parsing, ++ 'conductor') ++ if should_raise: ++ self.assertRaises( ++ exception.InvalidParameterValue, ++ driver_utils.get_kernel_append_params, ++ test_node, ++ default) ++ else: ++ self.assertEqual( ++ expected_result, ++ driver_utils.get_kernel_append_params(test_node, ++ default)) ++ ++ + class UtilsRamdiskLogsTestCase(tests_base.TestCase): + + def setUp(self): +diff --git a/releasenotes/notes/sanitize-kernel-append-params-8b2953a9d903d0f6.yaml b/releasenotes/notes/sanitize-kernel-append-params-8b2953a9d903d0f6.yaml +new file mode 100644 +index 000000000..90e136f76 +--- /dev/null ++++ b/releasenotes/notes/sanitize-kernel-append-params-8b2953a9d903d0f6.yaml +@@ -0,0 +1,14 @@ ++--- ++security: ++ - | ++ Fixes a security issue where a malicious 'kernel_append_params' in a node's ++ 'instance_info' or 'driver_info' could cause an attacker to take control of ++ a node's initial boot through boot script injection. The fix includes ++ sanitization of 'kernel_append_params' to prevent such injection. Strict ++ parsing of kernel parameters is now in place as well and enabled by ++ default. If an operator needs to disable such strict parsing they may do ++ so by setting the configuration option ++ conductor.disable_kernel_parameter_parsing to 'True'. However, this is ++ discouraged as it weakens the security posture of Ironic. This fix ++ addresses CVE-2026-46447. This back-port utilizes a generated, stand-alone ++ lark parser governed by the MPL v2.0 license. +diff --git a/tools/bandit.yml b/tools/bandit.yml +index 028d1a214..96d6f5396 100644 +--- a/tools/bandit.yml ++++ b/tools/bandit.yml +@@ -88,7 +88,6 @@ tests: + + # (optional) list skipped test IDs here, eg '[B101, B406]': + skips: +- - B104 + - B604 + + ### (optional) plugin settings - some test plugins require configuration data +diff --git a/tox.ini b/tox.ini +index 249f5e0f7..9c6084d30 100644 +--- a/tox.ini ++++ b/tox.ini +@@ -35,6 +35,7 @@ deps = {[testenv]deps} + commands = {toxinidir}/tools/states_to_dot.py -f {toxinidir}/doc/source/images/states.svg --format svg + + [testenv:pep8] ++basepython = py310 + usedevelop = False + deps= + hacking>=4.1.0,<5.0.0 # Apache-2.0 +@@ -132,7 +133,7 @@ commands = {posargs} + # [W503] Line break before binary operator. + ignore = E129,E741,W503 + filename = *.py,app.wsgi +-exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build ++exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,./ironic/common/kernel_parameter_parser/* + import-order-style = pep8 + application-import-names = ironic + max-complexity=19 +-- +2.54.0 + diff -Nru ironic-21.1.0/debian/patches/CVE-2026-48681-directory_transversal_ISO9660_support.patch ironic-21.4.4/debian/patches/CVE-2026-48681-directory_transversal_ISO9660_support.patch --- ironic-21.1.0/debian/patches/CVE-2026-48681-directory_transversal_ISO9660_support.patch 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/debian/patches/CVE-2026-48681-directory_transversal_ISO9660_support.patch 2024-11-08 15:10:43.000000000 +0000 @@ -0,0 +1,388 @@ +Author: Julia Kreger +Date: Wed, 20 May 2026 15:31:38 -0700 +Description: CVE-2026-48681: directory transversal ISO9660 support + A vulnerability was identified in Ironic's handling of + ISO images where Ironic contains support to patch ISO9660 + virtual media contents to include key data items like + configuration drive data and other required metadata. + . + Anyhow, the issue here was the Ironic service was trusting + that the submitted contents were valid, and a directory + transversal attempt could be embedded within a modified + configuration drive ISO contents submitted to Ironic. + This is a case where an attacker would take a path in an ISO, + and attempt to directly modify it to reach another path on the + filesystem which was within the confines and path structure they + were working with. i.e. while "../foo" is not a valid file or + directory name in ISO9660, it can still be represented, injected, + and read by the pycdlib library. + . + The code on all paths which perform this type of ISO content + interaction have been patched to explicitly check the path for + transversal attempts and internally raises an InvalidContent + exception. + . + Impacted features: + * Virtual Media ISO patching code path for pre-generated + deployment ISOs as opposed to Ironic generated ISOs from + a kernel/ramdisk. + * Anaconda deployment interfacce where a user could + impact the resulting pathing on the node being deployed + at deploy time. +Related-Bug: https://launchpad.net/bugs/2148333 +Change-Id: I09ba308dc5088260594a502d43413d2069616703 +Signed-off-by: Julia Kreger +Origin: upstream, pre-OSSA mailing list +Last-Update: 2026-06-01 + +Index: ironic/ironic/common/exception.py +=================================================================== +--- ironic.orig/ironic/common/exception.py ++++ ironic/ironic/common/exception.py +@@ -900,3 +900,9 @@ class ImageChecksumFileReadFailure(Inval + _msg_fmt = _("Failed to read the file from local storage " + "to perform a checksum operation.") + code = http_client.SERVICE_UNAVAILABLE ++ ++ ++class InvalidContent(Invalid): ++ """Invalid or malicious content has been provided to the conductor.""" ++ _msg_fmt = _("Invalid or potentially malicious content has been provided " ++ "to the conductor and the conductor will not proceed.") +Index: ironic/ironic/common/images.py +=================================================================== +--- ironic.orig/ironic/common/images.py ++++ ironic/ironic/common/images.py +@@ -736,15 +736,22 @@ def _extract_iso(extract_iso, extract_di + iso.open(extract_iso) + + for dirname, dirlist, filelist in iso.walk(iso_path='/'): ++ # NOTE(TheJulia): This code is confusing because the way the ++ # walk method returns data, basically a list of tuples to ++ # provide a structural mapping which a consumer of the walk ++ # data can use to understand the structure. + dir_path = dirname.lstrip('/') ++ utils.check_iso_path(extract_dir, dir_path) + for dir_iso in dirlist: ++ utils.check_iso_path(extract_dir, ++ os.path.join(dir_path, dir_iso)) + os.makedirs(os.path.join(extract_dir, dir_path, dir_iso)) + for file in filelist: +- file_path = os.path.join(extract_dir, dirname, file) ++ utils.check_iso_path(extract_dir, dir_path, file) ++ file_path = os.path.join(dirname, file) + iso.get_file_from_iso( + os.path.join(extract_dir, dir_path, file), + iso_path=file_path) +- + iso.close() + + +Index: ironic/ironic/common/kickstart_utils.py +=================================================================== +--- ironic.orig/ironic/common/kickstart_utils.py ++++ ironic/ironic/common/kickstart_utils.py +@@ -50,6 +50,11 @@ def _get_config_drive_dict_from_iso( + # server. + posix_file_path = posix_file_path.lstrip('/') + target_file_path = os.path.join(target_path, posix_file_path) ++ real_target_file_path = os.path.realpath(target_file_path) ++ if not target_file_path.startswith(real_target_file_path): ++ LOG.error('Discovered transversal attempt while reading ' ++ 'the configuration drive contents.') ++ raise exception.InvalidContent() + b_buf = io.BytesIO() + iso_reader.get_file_from_iso_fp( + iso_path=iso_file_path, outfp=b_buf +Index: ironic/ironic/common/utils.py +=================================================================== +--- ironic.orig/ironic/common/utils.py ++++ ironic/ironic/common/utils.py +@@ -701,3 +701,38 @@ def stop_after_retries(option, group=Non + return retry_state.attempt_number >= num_retries + 1 + + return should_stop ++ ++ ++def check_iso_path(base_folder, folder, file=None): ++ """Sanity check an ISO path, folder, and file structure. ++ ++ :param base_folder: The target folder for path operations. ++ :param folder: The folder being evaluated in the ISO. ++ :param file: An optional file to also evaluate for path ++ transversal attempts. ++ :raises: InvalidContent when an inconsistency is detected. ++ """ ++ if folder.startswith('/'): ++ # If we're here, we were handed the base folder path with a leading /. ++ # Any caller of this method should pre-emptively strip it. ++ raise exception.InvalidContent() ++ ++ target_folder = os.path.join(base_folder, folder) ++ resolved_folder = os.path.realpath(target_folder) ++ if not resolved_folder.startswith(base_folder): ++ # supplied folder path has something like ../ in the data set ++ # or resolution has resulted in a change in the value. ++ # Possible risk: if the temp folder is being used with a symlink... ++ LOG.error('ISO path evaluation identified a folder based ' ++ 'transversal attempt.') ++ raise exception.InvalidContent() ++ if file: ++ target_file_path = os.path.join(resolved_folder, file) ++ resolved_file_path = os.path.realpath(target_file_path) ++ # Check that the folder itself doesn't change, ++ # and then check that the resolved path matches ++ if (not target_file_path.startswith(resolved_folder) ++ or target_file_path != resolved_file_path): ++ LOG.error('ISO path evaluation identified a file name based ' ++ 'transversal attempt.') ++ raise exception.InvalidContent() +Index: ironic/ironic/tests/unit/common/test_images.py +=================================================================== +--- ironic.orig/ironic/tests/unit/common/test_images.py ++++ ironic/ironic/tests/unit/common/test_images.py +@@ -24,6 +24,7 @@ from unittest import mock + from oslo_concurrency import processutils + from oslo_config import cfg + from oslo_utils import fileutils ++import pycdlib + + from ironic.common import exception + from ironic.common.glance_service import service_utils as glance_utils +@@ -663,6 +664,116 @@ class FsImageTestCase(base.TestCase): + options) + self.assertEqual(expected_cfg, cfg) + ++ @mock.patch.object(os, 'makedirs', autospec=True) ++ @mock.patch('pycdlib.PyCdlib', autospec=True) ++ def test__extract_iso(self, mock_pycdlib_cls, mock_makedirs): ++ mock_iso = mock_pycdlib_cls.return_value ++ mock_iso.walk.return_value = [ ++ ('/', ['BOOT'], ['README.TXT']), ++ ('/BOOT', ['GRUB'], ['BOOTX64.EFI']), ++ ('/BOOT/GRUB', [], ['GRUB.CFG']), ++ ] ++ ++ images._extract_iso('/path/to/image.iso', '/extract') ++ ++ mock_iso.open.assert_called_once_with('/path/to/image.iso') ++ mock_iso.walk.assert_called_once_with(iso_path='/') ++ mock_makedirs.assert_any_call( ++ os.path.join('/extract', '', 'BOOT')) ++ mock_makedirs.assert_any_call( ++ os.path.join('/extract', 'BOOT', 'GRUB')) ++ mock_iso.get_file_from_iso.assert_any_call( ++ os.path.join('/extract', '', 'README.TXT'), ++ iso_path=os.path.join('/extract', '/', 'README.TXT')) ++ mock_iso.get_file_from_iso.assert_any_call( ++ os.path.join('/extract', 'BOOT', 'BOOTX64.EFI'), ++ iso_path=os.path.join('/extract', '/BOOT', ++ 'BOOTX64.EFI')) ++ mock_iso.get_file_from_iso.assert_any_call( ++ os.path.join('/extract', 'BOOT/GRUB', 'GRUB.CFG'), ++ iso_path=os.path.join('/extract', '/BOOT/GRUB', ++ 'GRUB.CFG')) ++ self.assertEqual(3, mock_iso.get_file_from_iso.call_count) ++ mock_iso.close.assert_called_once() ++ ++ @mock.patch('pycdlib.PyCdlib', autospec=True) ++ def test__extract_iso_empty(self, mock_pycdlib_cls): ++ mock_iso = mock_pycdlib_cls.return_value ++ mock_iso.walk.return_value = [ ++ ('/', [], []), ++ ] ++ ++ images._extract_iso('/path/to/empty.iso', '/extract') ++ ++ mock_iso.open.assert_called_once_with('/path/to/empty.iso') ++ mock_iso.walk.assert_called_once_with(iso_path='/') ++ mock_iso.get_file_from_iso.assert_not_called() ++ mock_iso.close.assert_called_once() ++ ++ @mock.patch('pycdlib.PyCdlib', autospec=True) ++ def test__extract_iso_open_fails(self, mock_pycdlib_cls): ++ mock_iso = mock_pycdlib_cls.return_value ++ mock_iso.open.side_effect = ( ++ pycdlib.pycdlibexception.PyCdlibInvalidInput( ++ msg='Could not open file')) ++ ++ self.assertRaises( ++ pycdlib.pycdlibexception.PyCdlibInvalidInput, ++ images._extract_iso, ++ '/path/to/bad.iso', '/extract') ++ mock_iso.walk.assert_not_called() ++ mock_iso.close.assert_not_called() ++ ++ @mock.patch.object(os, 'makedirs', autospec=True) ++ @mock.patch('pycdlib.PyCdlib', autospec=True) ++ def test__extract_iso_invalid_file(self, mock_pycdlib_cls, mock_makedirs): ++ mock_iso = mock_pycdlib_cls.return_value ++ mock_iso.walk.return_value = [ ++ ('/', ['BOOT'], ['README.TXT']), ++ ('/BOOT', ['GRUB'], ['../TX64.EFI']), ++ ('/BOOT/GRUB', [], ['GRUB.CFG']), ++ ] ++ ++ self.assertRaises(exception.InvalidContent, ++ images._extract_iso, ++ '/path/to/image.iso', '/extract') ++ ++ mock_iso.open.assert_called_once_with('/path/to/image.iso') ++ mock_iso.walk.assert_called_once_with(iso_path='/') ++ mock_makedirs.assert_any_call( ++ os.path.join('/extract', '', 'BOOT')) ++ mock_makedirs.assert_any_call( ++ os.path.join('/extract', 'BOOT', 'GRUB')) ++ mock_iso.get_file_from_iso.assert_any_call( ++ os.path.join('/extract', '', 'README.TXT'), ++ iso_path=os.path.join('/extract', '/', 'README.TXT')) ++ self.assertEqual(1, mock_iso.get_file_from_iso.call_count) ++ ++ @mock.patch.object(os, 'makedirs', autospec=True) ++ @mock.patch('pycdlib.PyCdlib', autospec=True) ++ def test__extract_iso_invalid_folder(self, mock_pycdlib_cls, ++ mock_makedirs): ++ mock_iso = mock_pycdlib_cls.return_value ++ mock_iso.walk.return_value = [ ++ ('/', ['BOOT'], ['README.TXT']), ++ ('/../T', ['GRUB'], ['BOOTX64.EFI']), ++ ('/BOOT/GRUB', [], ['GRUB.CFG']), ++ ] ++ ++ self.assertRaises(exception.InvalidContent, ++ images._extract_iso, ++ '/path/to/image.iso', '/extract') ++ ++ mock_iso.open.assert_called_once_with('/path/to/image.iso') ++ mock_iso.walk.assert_called_once_with(iso_path='/') ++ mock_makedirs.assert_any_call( ++ os.path.join('/extract', '', 'BOOT')) ++ self.assertEqual(1, mock_makedirs.call_count) ++ mock_iso.get_file_from_iso.assert_any_call( ++ os.path.join('/extract', '', 'README.TXT'), ++ iso_path=os.path.join('/extract', '/', 'README.TXT')) ++ self.assertEqual(1, mock_iso.get_file_from_iso.call_count) ++ + @mock.patch.object(os.path, 'relpath', autospec=True) + @mock.patch.object(os, 'walk', autospec=True) + @mock.patch.object(images, '_extract_iso', autospec=True) +Index: ironic/ironic/tests/unit/common/test_kickstart_utils.py +=================================================================== +--- ironic.orig/ironic/tests/unit/common/test_kickstart_utils.py ++++ ironic/ironic/tests/unit/common/test_kickstart_utils.py +@@ -17,6 +17,7 @@ import os + from unittest import mock + + from oslo_config import cfg ++import pycdlib + + from ironic.common import kickstart_utils as ks_utils + from ironic.conductor import task_manager +@@ -130,3 +131,85 @@ echo $CONTENT | /usr/bin/base64 --decode + task.node.save() + self.assertEqual(expected, ks_utils.prepare_config_drive(task)) + mock_get.assert_called_with('http://server/fake-configdrive-url') ++ ++ @mock.patch.object(pycdlib, 'PyCdlib', autospec=True) ++ def test_read_iso9600_config_drive(self, mock_pycdlib_cls): ++ mock_iso = mock_pycdlib_cls.return_value ++ mock_iso.walk.return_value = [ ++ ('/', [], ['FILE1.TXT;1']), ++ ] ++ mock_record = mock.Mock() ++ mock_iso.get_record.return_value = mock_record ++ mock_iso.full_path_from_dirrecord.return_value = ( ++ '/openstack/latest/user_data' ++ ) ++ ++ def fake_get_file(iso_path, outfp): ++ outfp.write(b'test user_data') ++ ++ mock_iso.get_file_from_iso_fp.side_effect = fake_get_file ++ ++ result = ks_utils.read_iso9600_config_drive(b'fake-iso') ++ ++ mock_iso.open.assert_called_once() ++ mock_iso.walk.assert_called_once_with(iso_path='/') ++ mock_iso.get_record.assert_called_once_with( ++ iso_path='/FILE1.TXT;1' ++ ) ++ mock_iso.full_path_from_dirrecord.assert_called_once_with( ++ mock_record, rockridge=True ++ ) ++ mock_iso.get_file_from_iso_fp.assert_called_once() ++ mock_iso.close.assert_called_once() ++ ++ expected_path = ( ++ '/var/lib/cloud/seed/config_drive' ++ '/openstack/latest/user_data' ++ ) ++ self.assertIn(expected_path, result) ++ self.assertEqual('test user_data', result[expected_path]) ++ ++ @mock.patch.object(pycdlib, 'PyCdlib', autospec=True) ++ def test_read_iso9600_config_drive_pycdlib_exception( ++ self, mock_pycdlib_cls): ++ mock_iso = mock_pycdlib_cls.return_value ++ mock_iso.open.side_effect = ( ++ pycdlib.pycdlibexception.PyCdlibInvalidInput( ++ msg='bad iso' ++ ) ++ ) ++ ++ result = ks_utils.read_iso9600_config_drive(b'bad-data') ++ ++ mock_iso.open.assert_called_once() ++ mock_iso.walk.assert_not_called() ++ self.assertEqual({}, result) ++ ++ @mock.patch.object(pycdlib, 'PyCdlib', autospec=True) ++ def test_read_iso9600_config_drive_invalid_file(self, mock_pycdlib_cls): ++ mock_iso = mock_pycdlib_cls.return_value ++ mock_iso.walk.return_value = [ ++ ('/', [], ['../E1.TXT;1']), ++ ] ++ mock_record = mock.Mock() ++ mock_iso.get_record.return_value = mock_record ++ mock_iso.full_path_from_dirrecord.return_value = ( ++ '../E1.TXT' ++ ) ++ ++ def fake_get_file(iso_path, outfp): ++ outfp.write(b'test user_data') ++ ++ mock_iso.get_file_from_iso_fp.side_effect = fake_get_file ++ ++ returned = ks_utils.read_iso9600_config_drive(b'fake-iso') ++ self.assertEqual({}, returned) ++ mock_iso.open.assert_called_once() ++ mock_iso.walk.assert_called_once_with(iso_path='/') ++ mock_iso.get_record.assert_called_once_with( ++ iso_path='/../E1.TXT;1' ++ ) ++ mock_iso.full_path_from_dirrecord.assert_called_once_with( ++ mock_record, rockridge=True ++ ) ++ mock_iso.get_file_from_iso_fp.assert_not_called() +Index: ironic/releasenotes/notes/bug-2148333-b3a74b813eea7dab.yaml +=================================================================== +--- /dev/null ++++ ironic/releasenotes/notes/bug-2148333-b3a74b813eea7dab.yaml +@@ -0,0 +1,17 @@ ++--- ++security: ++ - | ++ Fixes CVE-2026-48681 which was a lack of file path validation when ++ interacting with ISO9660 files in the kickstart/anaconda driver, ++ and administrative deployment ISOs which are admin-provided. ++ Ironic now explicitly rejects the submitted files when such ++ a case has been detected. ++fixes: ++ - | ++ Fixes path handling around ISO9660 file handling as denoted in ++ `bug 2148333 `_ ++ under CVE-2026-48681 where a malicious user with deployment privilges ++ could craft an ISO9660 formatted configuration drive or deployment ramdisk ++ which includes intentional path manipulation which is non-compliant with ++ the underlying standard. Ironic now explicitly looks for such attempts ++ and rejects the content. diff -Nru ironic-21.1.0/debian/patches/fix-initial_grub_cfg.template.patch ironic-21.4.4/debian/patches/fix-initial_grub_cfg.template.patch --- ironic-21.1.0/debian/patches/fix-initial_grub_cfg.template.patch 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/debian/patches/fix-initial_grub_cfg.template.patch 2024-11-08 15:10:43.000000000 +0000 @@ -0,0 +1,16 @@ +Description: Fix initial_grub_cfg.template + The default grub.cfg happen /srv/tftp, but tftp-hpa is, in Debian, + already doing a chroot in there. +Author: Thomas Goirand +Forwarded: no +Last-Update: 2023-05-10 + +--- ironic-21.1.0.orig/ironic/drivers/modules/initial_grub_cfg.template ++++ ironic-21.1.0/ironic/drivers/modules/initial_grub_cfg.template +@@ -3,5 +3,5 @@ set timeout=5 + set hidden_timeout_quiet=false + + menuentry "initial" { +-configfile {{ tftp_root }}/$net_default_mac.conf ++configfile /$net_default_mac.conf + } diff -Nru ironic-21.1.0/debian/patches/py3.11_fix_unit_tests.patch ironic-21.4.4/debian/patches/py3.11_fix_unit_tests.patch --- ironic-21.1.0/debian/patches/py3.11_fix_unit_tests.patch 2026-04-30 08:41:21.000000000 +0000 +++ ironic-21.4.4/debian/patches/py3.11_fix_unit_tests.patch 1970-01-01 00:00:00.000000000 +0000 @@ -1,70 +0,0 @@ -Description: Fix unit tests for Python 3.11 - Mocks can no longer be provided as the specs for other Mocks. - See https://github.com/python/cpython/issues/87644 and - https://docs.python.org/3.11/whatsnew/3.11.html for more info. -Author: Riccardo Pittau -Date: Wed, 07 Dec 2022 15:11:09 +0100 -Change-Id: If7c10d9bfd0bb410b3bc5180b737439c92e515da -Bug-Debian: https://bugs.debian.org/1024783 -Origin: upstream, https://review.opendev.org/c/openstack/ironic/+/866861 -Last-Update: 2022-12-09 - -diff --git a/ironic/tests/unit/drivers/modules/irmc/test_inspect.py b/ironic/tests/unit/drivers/modules/irmc/test_inspect.py -index 5c66cb9..da91ec6 100644 ---- a/ironic/tests/unit/drivers/modules/irmc/test_inspect.py -+++ b/ironic/tests/unit/drivers/modules/irmc/test_inspect.py -@@ -204,8 +204,8 @@ - _inspect_hardware_mock.return_value = (inspected_props, - inspected_macs, - new_traits) -- new_port_mock1 = mock.MagicMock(spec=objects.Port) -- new_port_mock2 = mock.MagicMock(spec=objects.Port) -+ new_port_mock1 = objects.Port -+ new_port_mock2 = objects.Port - - port_mock.side_effect = [new_port_mock1, new_port_mock2] - -@@ -220,11 +220,11 @@ - port_mock.assert_has_calls([ - mock.call(task.context, address=inspected_macs[0], - node_id=node_id), -+ mock.call.create(), - mock.call(task.context, address=inspected_macs[1], -- node_id=node_id) -- ]) -- new_port_mock1.create.assert_called_once_with() -- new_port_mock2.create.assert_called_once_with() -+ node_id=node_id), -+ mock.call.create() -+ ], any_order=False) - - self.assertTrue(info_mock.called) - task.node.refresh() -@@ -259,8 +259,9 @@ - _inspect_hardware_mock.return_value = (inspected_props, - inspected_macs, - new_traits) -- new_port_mock1 = mock.MagicMock(spec=objects.Port) -- new_port_mock2 = mock.MagicMock(spec=objects.Port) -+ -+ new_port_mock1 = objects.Port -+ new_port_mock2 = objects.Port - - port_mock.side_effect = [new_port_mock1, new_port_mock2] - -@@ -276,11 +277,11 @@ - port_mock.assert_has_calls([ - mock.call(task.context, address=inspected_macs[0], - node_id=node_id), -+ mock.call.create(), - mock.call(task.context, address=inspected_macs[1], -- node_id=node_id) -- ]) -- new_port_mock1.create.assert_called_once_with() -- new_port_mock2.create.assert_called_once_with() -+ node_id=node_id), -+ mock.call.create() -+ ], any_order=False) - - self.assertTrue(info_mock.called) - task.node.refresh() diff -Nru ironic-21.1.0/debian/patches/series ironic-21.4.4/debian/patches/series --- ironic-21.1.0/debian/patches/series 2026-04-30 08:41:21.000000000 +0000 +++ ironic-21.4.4/debian/patches/series 2024-11-08 15:10:43.000000000 +0000 @@ -1,6 +1,10 @@ adds-alembic.ini-in-MANIFEST.in.patch -py3.11_fix_unit_tests.patch +fix-initial_grub_cfg.template.patch CVE-2025-44021_OSSA-2025-001_Disallow_unsafe_image_file_paths.patch CVE-2026-42510_Shell-quote_console_command_passed_to_socat.patch CVE-2026-42997_OSSN-2026-010_validate_molds_url_against_swift_in_keystone_catalog.patch CVE-2026-44916_Use_sandbox_rendering_for_jinja2.patch +CVE-2026-44919_move_file_url_validation_up_into_deploy_utils_main_path.patch +CVE-2026-44917_disable-driver_info-level-pxe_template-override.patch +CVE-2026-46447_Sanitize-kernel_append_parms.patch +CVE-2026-48681-directory_transversal_ISO9660_support.patch diff -Nru ironic-21.1.0/debian/rules ironic-21.4.4/debian/rules --- ironic-21.1.0/debian/rules 2026-04-30 08:41:21.000000000 +0000 +++ ironic-21.4.4/debian/rules 2024-11-08 15:10:43.000000000 +0000 @@ -39,9 +39,15 @@ # for the moment until we upgrade with the above patch. # - ironic.tests.unit.drivers.modules.test_pxe.PXEAnacondaDeployTestCase.test_reboot_to_instance # Try to access to /etc/dnsmasq.d/hostsdir.d/ironic-52:54:00:cf:2d:31.conf which apparently doesn't exist. +# We're disabling this because of our patch: +# ironic.tests.unit.common.test_pxe_utils.TestPXEUtils.test_place_common_config +# No clue why these are failing: +# ironic.tests.unit.drivers.modules.network.test_common.TestNeutronVifPortIDMixin.test_port_changed_client_id +# ironic.tests.unit.drivers.modules.network.test_common.TestNeutronVifPortIDMixin.test_port_changed_client_id_fail +# Make jenkins crash: +# common\.test_image_service\.FileImageServiceTestCase\.test_validate_href_(empty_allowlist|not_in_allowlist|in_allowlist) ifeq (,$(findstring nocheck, $(DEB_BUILD_OPTIONS))) - pkgos-dh_auto_test --no-py2 'ironic\.tests\.unit\.(?!(.*api\.controllers\.v1\.test_chassis\.TestPost\.test_create_chassis_unicode_description.*|.*common\.test_glance_service\.TestGlanceImageService\.test_show_makes_datetimes.*|.*drivers\.modules\.irmc\.test_raid\.IRMCRaidConfigurationInternalMethodsTestCase\.test__commit_raid_config_with_logical_drives.*|.*drivers\.modules\.test_pxe\.PXEAnacondaDeployTestCase\.test_reboot_to_instance.*))' - + pkgos-dh_auto_test --no-py2 'ironic\.tests\.unit\.(?!(.*api\.controllers\.v1\.test_chassis\.TestPost\.test_create_chassis_unicode_description.*|.*common\.test_glance_service\.TestGlanceImageService\.test_show_makes_datetimes.*|.*drivers\.modules\.irmc\.test_raid\.IRMCRaidConfigurationInternalMethodsTestCase\.test__commit_raid_config_with_logical_drives.*|.*drivers\.modules\.test_pxe\.PXEAnacondaDeployTestCase\.test_reboot_to_instance.*|common\.test_pxe_utils\.TestPXEUtils\.test_place_common_config|drivers\.modules\.network.test_common\.TestNeutronVifPortIDMixin\.test_port_changed_client_id|drivers\.modules\.network.test_common\.TestNeutronVifPortIDMixin\.test_port_changed_client_id_fail|common\.test_image_service\.FileImageServiceTestCase\.test_validate_href_(empty_allowlist|not_in_allowlist|in_allowlist)|conductor\.test_manager\.SensorsTestCase\.test__sensors_conductor))' endif mkdir -p $(CURDIR)/debian/ironic-common/usr/share/ironic-common @@ -75,7 +81,7 @@ pkgos-readd-keystone-authtoken-missing-options $(CURDIR)/debian/ironic-common/usr/share/ironic-common/ironic.conf keystone_authtoken ironic # We're using python3, the default isn't good, it's using py2. - pkgos-fix-config-default $(CURDIR)/debian/ironic-common/usr/share/ironic-common/ironic.conf DEFAULT pybasedir /usr/lib/python3/site-packages/ironic + pkgos-fix-config-default $(CURDIR)/debian/ironic-common/usr/share/ironic-common/ironic.conf DEFAULT pybasedir /usr/lib/python3/dist-packages/ironic # Fix the default httpboot path, some of it is in the postinst # pkgos-fix-config-default $(CURDIR)/debian/ironic-common/usr/share/ironic-common/ironic.conf deploy http_root /var/lib/ironic/httpboot diff -Nru ironic-21.1.0/debian/tests/unittests ironic-21.4.4/debian/tests/unittests --- ironic-21.1.0/debian/tests/unittests 2026-04-30 08:41:21.000000000 +0000 +++ ironic-21.4.4/debian/tests/unittests 2024-11-08 15:10:43.000000000 +0000 @@ -2,4 +2,4 @@ set -e -pkgos-dh_auto_test --no-py2 'ironic\.tests\.unit\.(?!(.*api\.controllers\.v1\.test_chassis\.TestPost\.test_create_chassis_unicode_description.*|.*common\.test_glance_service\.TestGlanceImageService\.test_show_makes_datetimes.*|.*drivers\.modules\.irmc\.test_raid\.IRMCRaidConfigurationInternalMethodsTestCase\.test__commit_raid_config_with_logical_drives.*|.*drivers\.modules\.test_pxe\.PXEAnacondaDeployTestCase\.test_reboot_to_instance.*))' +pkgos-dh_auto_test --no-py2 'ironic\.tests\.unit\.(?!(.*api\.controllers\.v1\.test_chassis\.TestPost\.test_create_chassis_unicode_description.*|.*common\.test_glance_service\.TestGlanceImageService\.test_show_makes_datetimes.*|.*drivers\.modules\.irmc\.test_raid\.IRMCRaidConfigurationInternalMethodsTestCase\.test__commit_raid_config_with_logical_drives.*|.*drivers\.modules\.test_pxe\.PXEAnacondaDeployTestCase\.test_reboot_to_instance.*|common\.test_pxe_utils\.TestPXEUtils\.test_place_common_config|drivers\.modules\.network.test_common\.TestNeutronVifPortIDMixin\.test_port_changed_client_id|drivers\.modules\.network.test_common\.TestNeutronVifPortIDMixin\.test_port_changed_client_id_fail|common\.test_image_service\.FileImageServiceTestCase\.test_validate_href_(empty_allowlist|not_in_allowlist|in_allowlist)|conductor\.test_manager\.SensorsTestCase\.test__sensors_conductor))' diff -Nru ironic-21.1.0/devstack/lib/ironic ironic-21.4.4/devstack/lib/ironic --- ironic-21.1.0/devstack/lib/ironic 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/devstack/lib/ironic 2024-10-11 15:42:16.000000000 +0000 @@ -586,7 +586,7 @@ TEMPEST_BAREMETAL_MAX_MICROVERSION=${TEMPEST_BAREMETAL_MAX_MICROVERSION:-} # TODO(TheJulia): This PHYSICAL_NETWORK needs to be refactored in -# our devstack plugin. It is used by the neutron-legacy integration, +# our devstack plugin. It is used by the neutron integration, # however they want to name the new variable for the current neutron # plugin NEUTRON_PHYSICAL_NETWORK. For now we'll do some magic and # change it later once we migrate our jobs. @@ -594,7 +594,7 @@ PHYSICAL_NETWORK=${NEUTRON_PHYSICAL_NETWORK:-${PHYSICAL_NETWORK:-}} # Ramdisk ISO image for Ramdisk Virtual Media/iPXE testing -IRONIC_RAMDISK_IMAGE=${IRONIC_RAMDISK_IMAGE:-http://tinycorelinux.net/10.x/x86/archive/10.0/Core-10.0.iso} +IRONIC_RAMDISK_IMAGE=${IRONIC_RAMDISK_IMAGE:-http://tinycorelinux.net/13.x/x86/archive/13.0/Core-13.0.iso} IRONIC_LOADER_PATHS=${IRONIC_LOADER_PATHS:-} @@ -679,6 +679,14 @@ die $LINENO "Grub2 Bootloader and Shim file missing." fi fi + + # NOTE(tkajinam) Use local mirror in CI + if [ -f /etc/ci/mirror_info.sh ]; then + source /etc/ci/mirror_info.sh + CENTOS_MIRROR_HOST="http://${NODEPOOL_MIRROR_HOST}/centos-stream" + IRONIC_GRUB2_SHIM_FILE=$(echo $IRONIC_GRUB2_SHIM_FILE | sed "s|https://mirror.stream.centos.org|${CENTOS_MIRROR_HOST}|g") + IRONIC_GRUB2_FILE=$(echo $IRONIC_GRUB2_FILE | sed "s|https://mirror.stream.centos.org|${CENTOS_MIRROR_HOST}|g") + fi fi # TODO(pas-ha) find a way to (cross-)sign the custom CA bundle used by tls-proxy @@ -1625,7 +1633,7 @@ function configure_ironic_api { iniset $IRONIC_CONF_FILE DEFAULT auth_strategy $IRONIC_AUTH_STRATEGY configure_keystone_authtoken_middleware $IRONIC_CONF_FILE ironic - + iniset $IRONIC_CONF_FILE keystone_authtoken service_token_roles_required True if [[ "$IRONIC_USE_WSGI" == "True" ]]; then iniset $IRONIC_CONF_FILE oslo_middleware enable_proxy_headers_parsing True elif is_service_enabled tls-proxy; then @@ -1649,15 +1657,9 @@ # NOTE(TheJulia): Below are services which we know, as of late 2021, which support # explicit scope based ops *and* have knobs. - # Needed: Neutron, swift, nova ?service_catalog? - # Neutron - https://review.opendev.org/c/openstack/devstack/+/797450 if [[ "$service_config_section" == "inspector" ]] && [[ "$IRONIC_INSPECTOR_ENFORCE_SCOPE" == "True" ]]; then use_system_scope="True" - elif [[ "$service_config_section" == "cinder" ]] && [[ "${CINDER_ENFORCE_SCOPE:-False}" == "True" ]]; then - use_system_scope="True" - elif [[ "$service_config_section" == "glance" ]] && [[ "${GLANCE_ENFORCE_SCOPE:-False}" == "True" ]]; then - use_system_scope="True" fi if [[ "$use_system_scope" == "True" ]]; then @@ -1924,6 +1926,11 @@ # NOTE(rloo): We're not upgrading but want to make sure this command works, # even though we're not parsing the output of this command. $IRONIC_BIN_DIR/ironic-status upgrade check + + $IRONIC_BIN_DIR/ironic-status upgrade check && ret_val=$? || ret_val=$? + if [ $ret_val -gt 1 ] ; then + die $LINENO "The `ironic-status upgrade check` command returned an error. Cannot proceed." + fi } # _ironic_bm_vm_names() - Generates list of names for baremetal VMs. @@ -2377,6 +2384,9 @@ local ironic_node_disk=$IRONIC_VM_SPECS_DISK local ironic_ephemeral_disk=$IRONIC_VM_EPHEMERAL_DISK local ironic_node_arch=x86_64 + if [[ ! -f $IRONIC_VM_MACS_CSV_FILE ]]; then + touch $IRONIC_VM_MACS_CSV_FILE + fi local ironic_hwinfo_file=$IRONIC_VM_MACS_CSV_FILE if is_deployed_by_ipmi; then @@ -2944,8 +2954,16 @@ sudo mkdir -p $efiboot_mount/efi/boot - sudo cp "$IRONIC_GRUB2_SHIM_FILE" $efiboot_mount/efi/boot/bootx64.efi - sudo cp "$IRONIC_GRUB2_FILE" $efiboot_mount/efi/boot/grubx64.efi + if [[ "$IRONIC_GRUB2_SHIM_FILE" =~ "http".* ]]; then + sudo wget "$IRONIC_GRUB2_SHIM_FILE" -O $efiboot_mount/efi/boot/bootx64.efi + else + sudo cp "$IRONIC_GRUB2_SHIM_FILE" $efiboot_mount/efi/boot/bootx64.efi + fi + if [[ "$IRONIC_GRUB2_FILE" =~ "http".* ]]; then + sudo wget "$IRONIC_GRUB2_FILE" -O $efiboot_mount/efi/boot/grubx64.efi + else + sudo cp "$IRONIC_GRUB2_FILE" $efiboot_mount/efi/boot/grubx64.efi + fi sudo umount $efiboot_mount @@ -2982,7 +3000,7 @@ # NOTE(dtantsur): this is likely incorrect efi_grub_path=EFI/BOOT/grub.cfg fi - iniset $IRONIC_CONF_FILE DEFAULT grub_config_path $efi_grub_path + iniset $IRONIC_CONF_FILE DEFAULT grub_config_path ${IRONIC_GRUB2_CONFIG_PATH:-$efi_grub_path} } # build deploy kernel+ramdisk, then upload them to glance @@ -3334,6 +3352,18 @@ sudo ovs-vsctl set interface phy-brbm-infra options:peer=phy-infra-brbm } +function downgrade_dnsmasq { + # NOTE(TheJulia): The intent here is to use dnsmasq version + # which does not crash on segfaults or sigabort when configuration + # is updated. See https://bugs.launchpad.net/ironic/+bug/2026757 + sudo dpkg -r dnsmasq-base + git clone http://thekelleys.org.uk/git/dnsmasq.git -b v2.85 --depth 1 + pushd dnsmasq + sed -i 's|^PREFIX.*|PREFIX = /usr|' Makefile + sudo make install + popd +} + # Restore xtrace + pipefail $_XTRACE_IRONIC $_PIPEFAIL_IRONIC diff -Nru ironic-21.1.0/devstack/plugin.sh ironic-21.4.4/devstack/plugin.sh --- ironic-21.1.0/devstack/plugin.sh 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/devstack/plugin.sh 2024-10-11 15:42:16.000000000 +0000 @@ -15,12 +15,13 @@ echo_summary "Installing Ironic" if ! is_service_enabled nova; then - source $RC_DIR/lib/nova_plugins/functions-libvirt + source $TOP_DIR/lib/nova_plugins/functions-libvirt install_libvirt fi install_ironic install_ironicclient cleanup_ironic_config_files + downgrade_dnsmasq elif [[ "$2" == "post-config" ]]; then # stack/post-config - Called after the layer 1 and 2 services have been diff -Nru ironic-21.1.0/devstack/settings ironic-21.4.4/devstack/settings --- ironic-21.1.0/devstack/settings 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/devstack/settings 2024-10-11 15:42:16.000000000 +0000 @@ -8,9 +8,12 @@ # Get the smallest local MTU local_mtu=$(ip link show | sed -ne 's/.*mtu \([0-9]\+\).*/\1/p' | sort -n | head -1) # 50 bytes is overhead for vxlan (which is greater than GRE -# allowing us to use either overlay option with this MTU. +# allowing us to use either overlay option with this MTU). # However, if traffic is flowing over IPv6 tunnels, then -# The overhead is essentially another 100 bytes. In order to +# The overhead is essentially another 78 bytes. In order to # handle both cases, lets go ahead and drop the maximum by -# 100 bytes. -PUBLIC_BRIDGE_MTU=${OVERRIDE_PUBLIC_BRIDGE_MTU:-$((local_mtu - 100))} +# 78 bytes, while not going below 1280 to make IPv6 work at all. +PUBLIC_BRIDGE_MTU=${OVERRIDE_PUBLIC_BRIDGE_MTU:-$((local_mtu - 78))} +if [ $PUBLIC_BRIDGE_MTU -lt 1280 ]; then + PUBLIC_BRIDGE_MTU=1280 +fi diff -Nru ironic-21.1.0/devstack/tools/ironic/scripts/cirros-partition.sh ironic-21.4.4/devstack/tools/ironic/scripts/cirros-partition.sh --- ironic-21.1.0/devstack/tools/ironic/scripts/cirros-partition.sh 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/devstack/tools/ironic/scripts/cirros-partition.sh 2024-10-11 15:42:16.000000000 +0000 @@ -8,7 +8,7 @@ guestfish_args="--verbose" fi -CIRROS_VERSION=${CIRROS_VERSION:-0.5.2} +CIRROS_VERSION=${CIRROS_VERSION:-0.6.1} CIRROS_ARCH=${CIRROS_ARCH:-x86_64} # TODO(dtantsur): use the image cached on infra images in the CI DISK_URL=http://download.cirros-cloud.net/${CIRROS_VERSION}/cirros-${CIRROS_VERSION}-${CIRROS_ARCH}-disk.img diff -Nru ironic-21.1.0/devstack/upgrade/upgrade.sh ironic-21.4.4/devstack/upgrade/upgrade.sh --- ironic-21.1.0/devstack/upgrade/upgrade.sh 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/devstack/upgrade/upgrade.sh 2024-10-11 15:42:16.000000000 +0000 @@ -40,7 +40,7 @@ source $TARGET_DEVSTACK_DIR/stackrc source $TARGET_DEVSTACK_DIR/lib/tls source $TARGET_DEVSTACK_DIR/lib/nova -source $TARGET_DEVSTACK_DIR/lib/neutron-legacy +source $TARGET_DEVSTACK_DIR/lib/neutron source $TARGET_DEVSTACK_DIR/lib/apache source $TARGET_DEVSTACK_DIR/lib/keystone diff -Nru ironic-21.1.0/doc/source/admin/anaconda-deploy-interface.rst ironic-21.4.4/doc/source/admin/anaconda-deploy-interface.rst --- ironic-21.1.0/doc/source/admin/anaconda-deploy-interface.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/anaconda-deploy-interface.rst 2024-10-11 15:42:16.000000000 +0000 @@ -271,16 +271,44 @@ ``liveimg`` which is used as the base operating system image to start with. +Configuration Considerations +---------------------------- + +When using the ``anaconda`` deployment interface, some configuration +parameters may need to be adjusted in your environment. This is in large +part due to the general defaults being set to much lower values for image +based deployments, but the way the anaconda deployment interface works, +you may need to make some adjustments. + +* ``[conductor]deploy_callback_timeout`` likely needs to be adjusted + for most ``anaconda`` deployment interface users. By default this + is a timer which looks for "agents" which have not checked in with + Ironic, or agents which may have crashed or failed after they + started. If the value is reached, then the current operation is failed. + This value should be set to a number of seconds which exceeds your + average anaconda deployment time. +* ``[pxe]boot_retry_timeout`` can also be triggered and result in + an anaconda deployment in progress getting reset as it is intended + to reboot nodes which might have failed their initial PXE operation. + Depending on sizes of images, and the exact nature of what was deployed, + it may be necessary to ensure this is a much higher value. + Limitations ----------- -This deploy interface has only been tested with Red Hat based operating systems -that use anaconda. Other systems are not supported. +* This deploy interface has only been tested with Red Hat based operating + systems that use anaconda. Other systems are not supported. + +* Runtime TLS certifiate injection into ramdisks is not supported. Assets + such as ``ramdisk`` or a ``stage2`` ramdisk image need to have trusted + Certificate Authority certificates present within the images *or* the + Ironic API endpoint utilized should utilize a known trusted Certificate + Authority. -Runtime TLS certifiate injection into ramdisks is not supported. Assets such -as ``ramdisk`` or a ``stage2`` ramdisk image need to have trusted Certificate -Authority certificates present within the images *or* the Ironic API endpoint -utilized should utilize a known trusted Certificate Authority. +* The ``anaconda`` tooling deploying the instance/workload does not + heartbeat to Ironic like the ``ironic-python-agent`` driven ramdisks. + As such, you may need to adjust some timers. See + `Configuration Considerations`_ for some details on this. .. _`anaconda`: https://fedoraproject.org/wiki/Anaconda .. _`ks.cfg.template`: https://opendev.org/openstack/ironic/src/branch/master/ironic/drivers/modules/ks.cfg.template diff -Nru ironic-21.1.0/doc/source/admin/drivers/ibmc.rst ironic-21.4.4/doc/source/admin/drivers/ibmc.rst --- ironic-21.1.0/doc/source/admin/drivers/ibmc.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/drivers/ibmc.rst 2024-10-11 15:42:16.000000000 +0000 @@ -312,6 +312,6 @@ get_raid_controller_list GET Query RAID controller summary info ======================== ============ ====================================== -.. _Huawei iBMC: https://e.huawei.com/en/products/cloud-computing-dc/servers/accessories/ibmc +.. _Huawei iBMC: https://e.huawei.com/en/products/computing/kunpeng/accessories/ibmc .. _TLS: https://en.wikipedia.org/wiki/Transport_Layer_Security .. _HUAWEI iBMC Client library: https://pypi.org/project/python-ibmcclient/ diff -Nru ironic-21.1.0/doc/source/admin/drivers/ilo.rst ironic-21.4.4/doc/source/admin/drivers/ilo.rst --- ironic-21.1.0/doc/source/admin/drivers/ilo.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/drivers/ilo.rst 2024-10-11 15:42:16.000000000 +0000 @@ -2211,6 +2211,14 @@ ``ilo`` vendor interface for Gen10 and Gen10 Plus servers. See :ref:`node-vendor-passthru-methods` for more information. +Anaconda based deployment +^^^^^^^^^^^^^^^^^^^^^^^^^ +Deployment with ``anaconda`` deploy interface is supported by ``ilo`` and +``ilo5`` hardware type and works with ``ilo-pxe`` and ``ilo-ipxe`` +boot interfaces. See :doc:`/admin/anaconda-deploy-interface` for +more information. + + .. _`ssacli documentation`: https://support.hpe.com/hpsc/doc/public/display?docId=c03909334 .. _`proliant-tools`: https://docs.openstack.org/diskimage-builder/latest/elements/proliant-tools/README.html .. _`HPE iLO4 User Guide`: https://h20566.www2.hpe.com/hpsc/doc/public/display?docId=c03334051 diff -Nru ironic-21.1.0/doc/source/admin/drivers/irmc.rst ironic-21.4.4/doc/source/admin/drivers/irmc.rst --- ironic-21.1.0/doc/source/admin/drivers/irmc.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/drivers/irmc.rst 2024-10-11 15:42:16.000000000 +0000 @@ -123,11 +123,29 @@ the iRMC with administrator privileges. - ``driver_info/irmc_password`` property to be ``password`` for irmc_username. - - ``properties/capabilities`` property to be ``boot_mode:uefi`` if - UEFI boot is required. - - ``properties/capabilities`` property to be ``secure_boot:true`` if - UEFI Secure Boot is required. Please refer to `UEFI Secure Boot Support`_ - for more information. + + .. note:: + Fujitsu server equipped with iRMC S6 2.00 or later version of firmware + disables IPMI over LAN by default. However user may be able to enable IPMI + via BMC settings. + To handle this change, ``irmc`` hardware type first tries IPMI and, + if IPMI operation fails, ``irmc`` hardware type uses Redfish API of Fujitsu + server to provide Ironic functionalities. + So if user deploys Fujitsu server with iRMC S6 2.00 or later, user needs + to set Redfish related parameters in ``driver_info``. + + - ``driver_info/redifsh_address`` property to be ``IP address`` or + ``hostname`` of the iRMC. You can prefix it with protocol (e.g. + ``https://``). If you don't provide protocol, Ironic assumes HTTPS + (i.e. add ``https://`` prefix). + iRMC with S6 2.00 or later only support HTTPS connection to Redfish API. + - ``driver_info/redfish_username`` to be user name of iRMC with administrative + privileges + - ``driver_info/redfish_password`` to be password of ``redfish_username`` + - ``driver_info/redfish_verify_ca`` accepts values those accepted in + ``driver_info/irmc_verify_ca`` + - ``driver_info/redfish_auth_type`` to be one of ``basic``, ``session`` or + ``auto`` * If ``port`` in ``[irmc]`` section of ``/etc/ironic/ironic.conf`` or ``driver_info/irmc_port`` is set to 443, ``driver_info/irmc_verify_ca`` @@ -191,6 +209,22 @@ - ``driver_info/irmc_snmp_priv_password`` property to be the privacy protocol pass phrase. The length of pass phrase should be at least 8 characters. + +Configuration via ``properties`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Each node is configured for ``irmc`` hardware type by setting the following + ironic node object's properties: + + - ``properties/capabilities`` property to be ``boot_mode:uefi`` if + UEFI boot is required, or ``boot_mode:bios`` if Legacy BIOS is required. + If this is not set, ``default_boot_mode`` at ``[default]`` section in + ``ironic.conf`` will be used. + - ``properties/capabilities`` property to be ``secure_boot:true`` if + UEFI Secure Boot is required. Please refer to `UEFI Secure Boot Support`_ + for more information. + + Configuration via ``ironic.conf`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -199,6 +233,25 @@ - ``port``: Port to be used for iRMC operations; either 80 or 443. The default value is 443. Optional. + + .. note:: + Since iRMC S6 2.00, iRMC firmware doesn't support HTTP connection to + REST API. If you deploy server with iRMS S6 2.00 and later, please + set ``port`` to 443. + + ``irmc`` hardware type provides ``verify_step`` named + ``verify_http_https_connection_and_fw_version`` to check HTTP(S) + connection to iRMC REST API. If HTTP(S) connection is successfully + established, then it fetches and caches iRMC firmware version. + If HTTP(S) connection to iRMC REST API failed, Ironic node's state + moves to ``enroll`` with suggestion put in log message. + Default priority of this verify step is 10. + + If operator updates iRMC firmware version of node, operator should + run ``cache_irmc_firmware_version`` node vendor passthru method + to update iRMC firmware version stored in + ``driver_internal_info/irmc_fw_version``. + - ``auth_method``: Authentication method for iRMC operations; either ``basic`` or ``digest``. The default value is ``basic``. Optional. - ``client_timeout``: Timeout (in seconds) for iRMC @@ -229,9 +282,10 @@ and ``v2c``. The default value is ``public``. Optional. - ``snmp_security``: SNMP security name required for version ``v3``. Optional. - - ``snmp_auth_proto``: The SNMPv3 auth protocol. The valid value and the - default value are both ``sha``. We will add more supported valid values - in the future. Optional. + - ``snmp_auth_proto``: The SNMPv3 auth protocol. If using iRMC S4 or S5, the + valid value of this option is only ``sha``. If using iRMC S6, the valid + values are ``sha256``, ``sha384`` and ``sha512``. The default value is + ``sha``. Optional. - ``snmp_priv_proto``: The SNMPv3 privacy protocol. The valid value and the default value are both ``aes``. We will add more supported valid values in the future. Optional. diff -Nru ironic-21.1.0/doc/source/admin/drivers/redfish.rst ironic-21.4.4/doc/source/admin/drivers/redfish.rst --- ironic-21.1.0/doc/source/admin/drivers/redfish.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/drivers/redfish.rst 2024-10-11 15:42:16.000000000 +0000 @@ -670,6 +670,37 @@ this may be sign of an unexpected condition, and please consider contacting the Ironic developer community for assistance. +Redfish Interoperability Profile +================================ + +Ironic projects provides Redfish Interoperability Profile located in +``redfish-interop-profiles`` folder at source code root. The Redfish +Interoperability Profile is a JSON document written in a particular format +that serves two purposes. + +* It enables the creation of a human-readable document that merges the + profile requirements with the Redfish schema into a single document + for developers or users. +* It allows a conformance test utility to test a Redfish Service + implementation for conformance with the profile. + +The JSON document structure is intended to align easily with JSON payloads +retrieved from Redfish Service implementations, to allow for easy comparisons +and conformance testing. Many of the properties defined within this structure +have assumed default values that correspond with the most common use case, so +that those properties can be omitted from the document for brevity. + +Validation of Profiles using DMTF tool +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An open source utility has been created by the Redfish Forum to verify that +a Redfish Service implementation conforms to the requirements included in a +Redfish Interoperability Profile. The Redfish Interop Validator is available +for download from the DMTF's organization on Github at +https://github.com/DMTF/Redfish-Interop-Validator. Refer to instructions in +README on how to configure and run validation. + + .. _Redfish: http://redfish.dmtf.org/ .. _Sushy: https://opendev.org/openstack/sushy .. _TLS: https://en.wikipedia.org/wiki/Transport_Layer_Security diff -Nru ironic-21.1.0/doc/source/admin/fast-track.rst ironic-21.4.4/doc/source/admin/fast-track.rst --- ironic-21.1.0/doc/source/admin/fast-track.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/fast-track.rst 2024-10-11 15:42:16.000000000 +0000 @@ -15,6 +15,19 @@ the ``noop`` networking. The case where inspection, cleaning and provisioning networks are different is not supported. +.. note:: + Fast track mode is very sensitive to long-running processes on the conductor + side that may prevent agent heartbeats from being registered. + + For example, converting a large image to the raw format may take long enough + to reach the fast track timeout. In this case, you can either :ref:`use raw + images ` or move the conversion to the agent side with: + + .. code-block:: ini + + [DEFAULT] + force_raw_images = False + Enabling ======== diff -Nru ironic-21.1.0/doc/source/admin/hardware-burn-in.rst ironic-21.4.4/doc/source/admin/hardware-burn-in.rst --- ironic-21.1.0/doc/source/admin/hardware-burn-in.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/hardware-burn-in.rst 2024-10-11 15:42:16.000000000 +0000 @@ -108,6 +108,13 @@ baremetal node clean --clean-steps '[{"step": "burnin_disk", \ "interface": "deploy"}]' $NODE_NAME_OR_UUID +In order to launch a parallel SMART self test on all devices after the +disk burn-in (which will fail the step if any of the tests fail), set: + +.. code-block:: console + + baremetal node set --driver-info agent_burnin_fio_disk_smart_test=True \ + $NODE_NAME_OR_UUID Network burn-in =============== diff -Nru ironic-21.1.0/doc/source/admin/interfaces/deploy.rst ironic-21.4.4/doc/source/admin/interfaces/deploy.rst --- ironic-21.1.0/doc/source/admin/interfaces/deploy.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/interfaces/deploy.rst 2024-10-11 15:42:16.000000000 +0000 @@ -81,6 +81,46 @@ ``FollowSymLinks`` if you are using Apache HTTP server, or ``disable_symlinks`` if Nginx HTTP server is in use. +.. _stream_raw_images: + +Streaming raw images +-------------------- + +The Bare Metal service is capable of streaming raw images directly to the +target disk of a node, without caching them in the node's RAM. When the source +image is not already raw, the conductor will convert the image and calculate +the new checksum. + +.. note:: + If no algorithm is specified via the ``image_os_hash_algo`` field, or if + this field is set to ``md5``, SHA256 is used for the updated checksum. + +For HTTP or local file images that are already raw, you need to explicitly set +the disk format to prevent the checksum from being unnecessarily re-calculated. +For example: + +.. code-block:: shell + + baremetal node set \ + --instance-info image_source=http://server/myimage.img \ + --instance-info image_os_hash_algo=sha512 \ + --instance-info image_os_hash_value= \ + --instance-info image_disk_format=raw + +To disable this feature and cache images in the node's RAM, set + +.. code-block:: ini + + [agent] + stream_raw_images = False + +To disable the conductor-side conversion completely, set + +.. code-block:: ini + + [DEFAULT] + force_raw_images = False + .. _ansible-deploy: Ansible deploy diff -Nru ironic-21.1.0/doc/source/admin/metrics.rst ironic-21.4.4/doc/source/admin/metrics.rst --- ironic-21.1.0/doc/source/admin/metrics.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/metrics.rst 2024-10-11 15:42:16.000000000 +0000 @@ -17,8 +17,11 @@ emitted from the Bare Metal service, including ironic-api, ironic-conductor, and ironic-python-agent. By default, none of the services will emit metrics. -Configuring the Bare Metal Service to Enable Metrics -==================================================== +It is important to stress that not only statsd is supported for metrics +collection and transmission. This is covered later on in our documentation. + +Configuring the Bare Metal Service to Enable Metrics with Statsd +================================================================ Enabling metrics in ironic-api and ironic-conductor --------------------------------------------------- @@ -62,6 +65,30 @@ agent_statsd_host = 198.51.100.2 agent_statsd_port = 8125 +.. Note:: + Use of a different metrics backend with the agent is not presently + supported. + +Transmission to the Message Bus Notifier +======================================== + +Regardless if you're using Ceilometer, +`ironic-prometheus-exporter `_, +or some scripting you wrote to consume the message bus notifications, +metrics data can be sent to the message bus notifier from the timer methods +*and* additional gauge counters by utilizing the ``[metrics]backend`` +configuration option and setting it to ``collector``. When this is the case, +Information is cached locally and periodically sent along with the general sensor +data update to the messaging notifier, which can consumed off of the message bus, +or via notifier plugin (such as is done with ironic-prometheus-exporter). + +.. NOTE:: + Transmission of timer data only works for the Conductor or ``single-process`` + Ironic service model. A separate webserver process presently does not have + the capability of triggering the call to retrieve and transmit the data. + +.. NOTE:: + This functionality requires ironic-lib version 5.4.0 to be installed. Types of Metrics Emitted ======================== @@ -79,6 +106,9 @@ or have been removed between releases, refer to the `ironic release notes `_. +Additional conductor metrics in the form of counts will also be generated in +limited locations where petinant to the activity of the conductor. + .. note:: With the default statsd configuration, each timing metric may create additional metrics due to how statsd handles timing metrics. For more diff -Nru ironic-21.1.0/doc/source/admin/retirement.rst ironic-21.4.4/doc/source/admin/retirement.rst --- ironic-21.1.0/doc/source/admin/retirement.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/retirement.rst 2024-10-11 15:42:16.000000000 +0000 @@ -23,6 +23,27 @@ such as cleaning, to happen (this marks an important difference to nodes which have the ``maintenance`` flag set). +Requirements +============ + +The use of the retirement feature requires that automated cleaning +be enabled. The default ``[conductor]automated_clean`` setting must +not be disabled as the retirement feature is only engaged upon +the completion of cleaning as it sets forth the expectation of removing +sensitive data from a node. + +If you're uncomfortable with full cleaning, but want to make use of the +the retirement feature, a compromise may be to explore use of metadata +erasure, however this will leave additional data on disk which you may +wish to erase completely. Please consult the configuration for the +``[deploy]erase_devices_metadata_priority`` and +``[deploy]erase_devices_priority`` settings, and do note that +clean steps can be manually invoked through manual cleaning should you +wish to trigger the ``erase_devices`` clean step to completely wipe +all data from storage devices. Alternatively, automated cleaning can +also be enabled on an individual node level using the +``baremetal node set --automated-clean `` command. + How to use ========== diff -Nru ironic-21.1.0/doc/source/admin/secure-rbac.rst ironic-21.4.4/doc/source/admin/secure-rbac.rst --- ironic-21.1.0/doc/source/admin/secure-rbac.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/secure-rbac.rst 2024-10-11 15:42:16.000000000 +0000 @@ -280,3 +280,30 @@ This functionality can be disabled by setting ``[api]project_admin_can_manage_own_nodes`` to ``False``. + +Can I use a service role? +------------------------- + +In later versions of Ironic, the ``service`` role has been added to enable +delineation of accounts and access to Ironic's API. As Ironic's API was +largely originally intended as an "admin" API service, the service role +enables similar levels of access as a project-scoped user with the +``admin`` or ``manager`` roles. + +In terms of access, this is likely best viewed as a user with the +``manager`` role, but with slight elevation in privilege to enable +usage of the service via a service account. + +A project scoped user with the ``service`` role is able to create +baremetal nodes, but is not able to delete them. To disable the +ability to create nodes, set the +``[api]project_admin_can_manage_own_nodes`` setting to ``False``. +The nodes which can be accessed/managed in the project scope also align +with the ``owner`` and ``lessee`` access model, and thus if nodes are not +matching the user's ``project_id``, then Ironic's API will appear not to +have any enrolled baremetal nodes. + +With the system scope, a user with the ``service`` role is able to +create baremetal nodes, but also, not delete them. The access rights +are modeled such an ``admin`` scoped is needed to delete baremetal +nodes from Ironic. diff -Nru ironic-21.1.0/doc/source/admin/security.rst ironic-21.4.4/doc/source/admin/security.rst --- ironic-21.1.0/doc/source/admin/security.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/security.rst 2024-10-11 15:42:16.000000000 +0000 @@ -19,6 +19,40 @@ .. TODO: add "Multi-tenancy Considerations" section +Image Checksums +=============== + +Ironic has long provided a capacity to supply and check a checksum for disk +images being deployed. However, one aspect which Ironic has not asserted is +"Why?" in terms of "Is it for security?" or "Is it for data integrity?". + +The answer is both to ensure a higher level of security with remote +image files, *and* provide faster feedback should a image being transferred +happens to be corrupted. + +Normally checksums are verified by the ``ironic-python-agent`` **OR** the +deployment interface responsible for overall deployment operation. That being +said, not *every* deployment interface relies on disk images which have +checksums, and those deployment interfaces are for specific use cases which +Ironic users leverage, outside of the "general" use case capabilities provided +by the ``direct`` deployment interface. + +.. NOTE:: + Use of the node ``instance_info/image_checksum`` field is discouraged + for integrated OpenStack Users as usage of the matching Glance Image + Service field is also deprecated. That being said, Ironic retains this + feature by popular demand while also enabling also retain simplified + operator interaction. + The newer field values supported by Glance are also specifically + supported by Ironic as ``instance_info/image_os_hash_value`` for + checksum values and ``instance_info/image_os_hash_algo`` field for + the checksum algorithm. + +.. WARNING:: + Setting a checksum value to a URL is supported, *however* doing this is + making a "tradeoff" with security as the remote checksum *can* change. + Conductor support this functionality can be disabled using the + :oslo.config:option:`conductor.disable_support_for_checksum_files` setting. REST API: user roles and policy settings ======================================== @@ -275,3 +309,70 @@ # Access IPA ramdisk functions "baremetal:driver:ipa_lookup": "rule:is_admin" + +Disk Images +=========== + +Ironic relies upon the ``qemu-img`` tool to convert images from a supplied +disk image format, to a ``raw`` format in order to write the contents of a +disk image to the remote device. + +By default, only ``qcow2`` format is supported for this operation, however there +have been reports other formats work when so enabled using the +``[conductor]permitted_image_formats`` configuration option. + + +Ironic takes several steps by default. + +#. Ironic checks and compares supplied metadata with a remote authoritative + source, such as the Glance Image Service, if available. +#. Ironic attempts to "fingerprint" the file type based upon available + metadata and file structure. A file format which is not known to the image + format inspection code may be evaluated as "raw", which means the image + would not be passed through ``qemu-img``. When in doubt, use a ``raw`` + image which you can verify is in the desirable and expected state. +#. The image then has a set of safety and sanity checks executed which look + for unknown or unsafe feature usage in the base format which could permit + an attacker to potentially leverage functionality in ``qemu-img`` which + should not be utilized. This check, by default, occurs only through images + which transverse *through* the conductor. +#. Ironic then checks if the fingerprint values and metadata values match. + If they do not match, the requested image is rejected and the operation + fails. +#. The image is then provided to the ``ironic-python-agent``. + +Images which are considered "pass-through", as in they are supplied by an +API user as a URL, or are translated to a temporary URL via available +service configuration, are supplied as a URL to the +``ironic-python-agent``. + +Ironic can be configured to intercept this interaction and have the conductor +download and inspect these items before the ``ironic-python-agent`` will do so, +however this can increase the temporary disk utilization of the Conductor +along with network traffic to facilitate the transfer. This check is disabled +by default, but can be enabled using the +``[conductor]conductor_always_validates_images`` configuration option. + +An option exists which forces all files to be served from the conductor, and +thus force image inspection before involvement of the ``ironic-python-agent`` +is the use of the ``[agent]image_download_source`` configuration parameter +when set to ``local`` which proxies all disk images through the conductor. +This setting is also available in the node ``driver_info`` and +``instance_info`` fields. + +Mitigating Factors to disk images +--------------------------------- + +In a fully integrated OpenStack context, Ironic requires images to be set to +"public" in the Image Service. + +A direct API user with sufficient elevated access rights *can* submit a URL +for the baremetal node ``instance_info`` dictionary field with an +``image_source`` key value set to a URL. To do so explicitly requires +elevated (trusted) access rights of a System scoped Member, +or Project scoped Owner-Member, or a Project scoped Lessee-Admin via +the ``baremetal:node:update_instance_info`` policy permission rule. +Before the Wallaby release of OpenStack, this was restricted to +``admin`` and ``baremetal_admin`` roles and remains similarly restrictive +in the newer "Secure RBAC" model. +>>>>>>> 8491abb92 (Harden all image handling and conversion code) diff -Nru ironic-21.1.0/doc/source/admin/troubleshooting.rst ironic-21.4.4/doc/source/admin/troubleshooting.rst --- ironic-21.1.0/doc/source/admin/troubleshooting.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/admin/troubleshooting.rst 2024-10-11 15:42:16.000000000 +0000 @@ -1010,3 +1010,161 @@ This was an infrastructure operator requested feature from actual lessons learned in the operation of Ironic in large scale production. The defaults may not be suitable for the largest scale operators. + +Why do I have an error that an NVMe Partition is not a block device? +==================================================================== + +In some cases, you can encounter an error that suggests a partition that has +been created on an NVMe block device, is not a block device. + +Example: + + lsblk: /dev/nvme0n1p2: not a block device + +What has happened is the partition contains a partition table inside of it +which is confusing the NVMe device interaction. While basically valid in +some cases to have nested partition tables, for example, with software +raid, in the NVMe case the driver and possibly the underlying device gets +quite confused. This is in part because partitions in NVMe devices are higher +level abstracts. + +The way this occurs is you likely had a ``whole-disk`` image, and it was +configured as a partition image. If using glance, your image properties +may have a ``img_type`` field, which should be ``whole-disk``, or you +have a ``kernel_id`` and ``ramdisk_id`` value in the glance image +``properties`` field. Definition of a kernel and ramdisk value also +indicates that the image is of a ``partition`` image type. This is because +a ``whole-disk`` image is bootable from the contents within the image, +and partition images are unable to be booted without a kernel, and ramdisk. + +If you are using Ironic in standalone mode, the optional +``instance_info/image_type`` setting may be advisable to be checked. +Very similar to Glance usage above, if you have set Ironic's node level +``instance_info/kernel`` and ``instance_info/ramdisk`` parameters, Ironic +will proceed with deploying an image as if it is a partition image, and +create a partition table on the new block device, and then write the +contents of the image into the newly created partition. + +.. NOTE:: + As a general reminder, the Ironic community recommends the use of + whole disk images over the use of partition images. + +Why can't I use Secure Erase/Wipe with RAID controllers? +======================================================== + +Situations have been reported where an infrastructure operator is expecting +particular device types to be Secure Erased or Wiped when they are behind a +RAID controller. + +For example, the server may have NVMe devices attached to a RAID controller +which could be in pass-through or single disk volume mode. The same scenario +exists basically regardless of the disk/storage medium/type. + +The basic reason why is that RAID controllers essentially act as command +translators with a buffer cache. They tend to offer a simplified protocol +to the Operating System, and interact with the storage device in whatever +protocol is native to the device. This is the root of the underlying +problem. + +Protocols such as SCSI are rooted in quite a bit of computing history, +but never evolved to include primitives like Secure Erase which evolved in +the `ATA protocol `_. + +The closest primitives in SCSI to ATA Secure Erase is the ``FORMAT UNIT`` +and ``UNMAP`` commands. + +``FORMAT UNIT`` might be a viable solution, and a tool named +`sg_format `_ exists, +but there has not been a sufficient call upstream to implement this and +test it sufficiently that the Ironic community would be comfortable +shipping such a capability. The possibility also exists that a RAID +controller might not translate this command through to an end device, +just as some RAID controllers know how to handle and pass through +ATA commands to disk devices which support them. It is entirely dependent +upon the hardware configuration scenario. + +The ``UNMAP`` command is similar to the ATA ``TRIM`` command. Unfortunately +the SCSI protocol requires this be performed at block level, and similar to +``FORMAT UNIT``, it may not be supported or just passed through. + +If your interested in working on this area, or are willing to help test, +please feel free to contact the +:doc:`Ironic development community `. +An additional option is the creation of your own +`custom Hardware Manager `_ +which can contain your preferred logic, however this does require some Python +development experience. + +One last item of note, depending on the RAID controller, the BMC, and a number +of other variables, you may be able to leverage the `RAID `_ +configuration interface to delete volumes/disks, and recreate them. This may +have the same effect as a clean disk, however that too is RAID controller +dependent behavior. + +I'm in "clean failed" state, what do I do? +========================================== + +There is only one way to exit the ``clean failed`` state. But before we visit +the answer as to **how**, we need to stress the importance of attempting to +understand **why** cleaning failed. On the simple side of things, this may be +as simple as a DHCP failure, but on a complex side of things, it could be that +a cleaning action failed against the underlying hardware, possibly due to +a hardware failure. + +As such, we encourage everyone to attempt to understand **why** before exiting +the ``clean failed`` state, because you could potentially make things worse +for yourself. For example if firmware updates were being performed, you may +need to perform a rollback operation against the physical server, depending on +what, and how the firmware was being updated. Unfortunately this also borders +the territory of "no simple answer". + +This can be counter balanced with sometimes there is a transient networking +failure and a DHCP address was not obtained. An example of this would be +suggested by the ``last_error`` field indicating something about "Timeout +reached while cleaning the node", however we recommend following several +basic troubleshooting steps: + +* Consult the ``last_error`` field on the node, utilizing the + ``baremetal node show `` command. +* If the version of ironic supports the feature, consult the node history + log, ``baremetal node history list`` and + ``baremetal node history get ``. +* Consult the acutal console screen of the physical machine. *If* the ramdisk + booted, you will generally want to investigate the controller logs and see + if an uploaded agent log is being stored on the conductor responsible for + the baremetal node. Consult `Retrieving logs from the deploy ramdisk`_. + If the node did not boot for some reason, you can typically just retry + at this point and move on. + +How to get out of the state, once you've understood **why** you reached it +in the first place, is to utilize the ``baremetal node manage `` +command. This returns the node to ``manageable`` state, from where you can +retry "cleaning" through automated cleaning with the ``provide`` command, +or manual cleaning with ``clean`` command. or the next appropriate action +in the workflow process you are attempting to follow, which may be +ultimately be decommissioning the node because it could have failed and is +being removed or replaced. + +Ironic says my Image is Invalid +=============================== + +As a result of security fixes which were added to Ironic, resulting from the +security posture of the ``qemu-img`` utility, Ironic enforces certain aspects +related to image files. + +* Enforces that the file format of a disk image matches what Ironic is + told by an API user. Any mismatch will result in the image being declared + as invalid. A mismatch with the file contents and what is stored in the + Image service will necessitate uploading a new image as that property + cannot be changed in the image service *after* creation of an image. +* Enforces that the input file format to be written is ``qcow2`` or ``raw``. + This can be extended by modifying ``[conductor]permitted_image_formats`` in + ``ironic.conf``. +* Performs safety and sanity check assessment against the file, which can be + disabled by modifying ``[conductor]disable_deep_image_inspection`` and + setting it to ``True``. Doing so is not considered safe and should only + be done by operators accepting the inherent risk that the image they + are attempting to use may have a bad or malicious structure. + Image safety checks are generally performed as the deployment process begins + and stages artifacts, however a late stage check is performed when + needed by the ironic-python-agent. diff -Nru ironic-21.1.0/doc/source/contributor/dev-quickstart.rst ironic-21.4.4/doc/source/contributor/dev-quickstart.rst --- ironic-21.1.0/doc/source/contributor/dev-quickstart.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/contributor/dev-quickstart.rst 2024-10-11 15:42:16.000000000 +0000 @@ -131,6 +131,13 @@ ``tools/test-setup.sh`` to set up the database the same way as setup in the OpenStack test systems. +.. note:: + If you encounter issues executing unit tests, specifically where errors + may indicate that a field is too long, check your database's default + character encoding. Debian specifically sets MariaDB to ``utf8mb4`` + which utilizes 4 byte encoded unicode characters by default, which is + incompatible by default. + Additional Tox Targets ---------------------- diff -Nru ironic-21.1.0/doc/source/contributor/ironic-boot-from-volume.rst ironic-21.4.4/doc/source/contributor/ironic-boot-from-volume.rst --- ironic-21.1.0/doc/source/contributor/ironic-boot-from-volume.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/contributor/ironic-boot-from-volume.rst 2024-10-11 15:42:16.000000000 +0000 @@ -125,7 +125,8 @@ volume with tempest in the environment:: cd /opt/stack/tempest - tox -e all-plugin -- ironic_tempest_plugin.tests.scenario.test_baremetal_boot_from_volume + tox -e venv-tempest -- pip install (path to the ironic-tempest-plugin directory) + tox -e all -- ironic_tempest_plugin.tests.scenario.test_baremetal_boot_from_volume Please note that the storage interface will only indicate errors based upon the state of the node and the configuration present. As such a node does not diff -Nru ironic-21.1.0/doc/source/contributor/releasing.rst ironic-21.4.4/doc/source/contributor/releasing.rst --- ironic-21.1.0/doc/source/contributor/releasing.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/contributor/releasing.rst 2024-10-11 15:42:16.000000000 +0000 @@ -5,7 +5,7 @@ Since the responsibility for releases will move between people, we document that process here. -A full list of projects that ironic manages is available in the `governance +A full list of projects that Ironic manages is available in the `governance site`_. .. _`governance site`: https://governance.openstack.org/reference/projects/ironic.html @@ -33,7 +33,7 @@ What do we have to release? =========================== -The ironic project has a number of deliverables under its governance. The +The Ironic project has a number of deliverables under its governance. The ultimate source of truth for this is `projects.yaml `__ in the governance repository. These deliverables have varying release models, @@ -41,7 +41,7 @@ `__ in the releases repository. -In general, ironic deliverables follow the `cycle-with-intermediary +In general, Ironic deliverables follow the `cycle-with-intermediary `__ release model. @@ -125,26 +125,30 @@ * ironic-inspector * ironic-python-agent -They are also released on a regular cadence as opposed to on-demand, namely -three times a release cycle (roughly a release every 2 months). One of the -releases corresponds to the coordinated OpenStack released and receives a -``stable/NAME`` branch. The other two happen during the cycle and receive a -``bugfix/X.Y`` branch, where ``X.Y`` consists of the major and the minor -component of the version (e.g. ``bugfix/8.1`` for 8.1.0). +These projects receive releases every six months as part of the coordinated +OpenStack release that happens semi-annually. These releases can be +found in a ``stable/NAME`` branch. + +They are also evaluated for additional bugfix releases between scheduled +stable releases at the two and four month milestone between stable releases +(roughly every 2 months). These releases can be found in a ``bugfix/X.Y`` +branch. A bugfix release is only created if there are significant +beneficial changes and a known downstream operator or distributor will consume +the release. To leave some version space for releases from these branches, releases of these projects from the master branch always increase either the major or the minor version. -Currently releases from bugfix branches cannot be automated and must be done by -the release team manually. +Currently releases and retirements from bugfix branches cannot be automated and +must be done by the release team manually. -After the creation of a bugfix branch it is utmost important to update the -upper-constraints link for the tests in the tox.ini file, plus override the -branch for the requirements project to be sure to use the correct +After the creation of a bugfix branch it is of the utmost importance to update +the upper-constraints link for the tests in the tox.ini file, plus override +the branch for the requirements project to be sure to use the correct upper-constraints; for example see the following change: -https://review.opendev.org/c/openstack/ironic-python-agent/+/841290 +https://review.opendev.org/c/openstack/Ironic-python-agent/+/841290 Things to do before releasing ============================= @@ -155,7 +159,7 @@ Combine release notes if necessary (for example, a release note for a feature and another release note to add to that feature may be combined). -* For ironic releases only, not ironic-inspector releases: if any new API +* For Ironic releases only, not Ironic-inspector releases: if any new API microversions have been added since the last release, update the REST API version history (``doc/source/contributor/webapi-version-history.rst``) to indicate that they were part of the new release. @@ -196,7 +200,7 @@ deliverable (i.e. subproject) grouped by release cycles. * The ``_independent`` directory contains yaml files for deliverables that - are not bound to (official) cycles (e.g. ironic-python-agent-builder). + are not bound to (official) cycles (e.g. Ironic-python-agent-builder). * To check the changes we're about to release we can use the tox environment ``list-unreleased-changes``, with this syntax: @@ -209,7 +213,7 @@ not stable/ussuri or stable/train). For example, assuming we're in the main directory of the releases repository, - to check the changes in the ussuri series for ironic-python-agent + to check the changes in the ussuri series for Ironic-python-agent type: .. code-block:: bash @@ -239,12 +243,12 @@ The ``--intermediate-branch`` option is used to create an intermediate bugfix branch following the - `new release model for ironic projects `_. + `new release model for Ironic projects `_. To propose the release, use the script to update the deliverable file, then commit the change, and propose it for review. - For example, to propose a minor release for ironic in the master branch + For example, to propose a minor release for Ironic in the master branch (current development branch), considering that the code name of the future stable release is wallaby, use: @@ -256,7 +260,7 @@ deliverable, the new version and the branch, if applicable. A good commit message title should also include the same, for example - "Release ironic 1.2.3 for ussuri" + "Release Ironic 1.2.3 for ussuri" * As an optional step, we can use ``tox -e list-changes`` to double-check the changes before submitting them for review. @@ -306,7 +310,7 @@ We need to submit patches for changes in the stable branch to: -* update the ironic devstack plugin to point at the branched tarball for IPA. +* update the Ironic devstack plugin to point at the branched tarball for IPA. An example of this patch is `here `_. * set appropriate defaults for ``TEMPEST_BAREMETAL_MIN_MICROVERSION`` and @@ -320,7 +324,7 @@ need to make these changes. Note that we need to wait until *after* the switch in grenade is made to test the latest release (N) with master (e.g. `for stable/queens `_). - Doing these changes sooner -- after the ironic release and before the switch + Doing these changes sooner -- after the Ironic release and before the switch when grenade is testing the prior release (N-1) with master, will cause the tests to fail. (You may want to ask/remind infra/qa team, as to when they will do this switch.) @@ -331,7 +335,7 @@ only support upgrades from the most recent named release to master. * remove any DB migration scripts from ``ironic.cmd.dbsync.ONLINE_MIGRATIONS`` - and remove the corresponding code from ironic. (These migration scripts + and remove the corresponding code from Ironic. (These migration scripts are used to migrate from an old release to this latest release; they shouldn't be needed after that.) diff -Nru ironic-21.1.0/doc/source/contributor/webapi-version-history.rst ironic-21.4.4/doc/source/contributor/webapi-version-history.rst --- ironic-21.1.0/doc/source/contributor/webapi-version-history.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/contributor/webapi-version-history.rst 2024-10-11 15:42:16.000000000 +0000 @@ -2,10 +2,26 @@ REST API Version History ======================== +1.82 (Antelope) +---------------------- + +This version signifies the addition of node sharding endpoints. + +- Adds support for get, set, and delete of shard key on Node object. +- Adds support for ``GET /v1/shards`` which returns a list of all shards and + the count of nodes assigned to each. + +1.81 (Antelope) +---------------------- + +Add endpoint to retrieve introspection data for nodes via the REST API. + +* ``GET /v1/nodes/{node_ident}/inventory/`` + 1.80 (Zed, 21.1) ---------------------- -This verison is a signifier of additional RBAC functionality allowing +This version is a signifier of additional RBAC functionality allowing a project scoped ``admin`` to create or delete nodes in Ironic. 1.79 (Zed, 21.0) diff -Nru ironic-21.1.0/doc/source/install/configure-glance-images.rst ironic-21.4.4/doc/source/install/configure-glance-images.rst --- ironic-21.1.0/doc/source/install/configure-glance-images.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/install/configure-glance-images.rst 2024-10-11 15:42:16.000000000 +0000 @@ -3,6 +3,24 @@ Add images to the Image service =============================== +Supported Image Formats +~~~~~~~~~~~~~~~~~~~~~~~ + +Ironic officially supports and tests use of ``qcow2`` formatted images as well +as ``raw`` format images. Other types of disk images, like ``vdi``, and single +file ``vmdk`` files have been reported by users as working in their specific +cases, but are not tested upstream. We advise operators to convert the image +and properly upload the image to Glance. + +Ironic enforces the list of supported and permitted image formats utilizing +the ``[conductor]permitted_image_formats`` option in ironic.conf. This setting +defaults to "raw" and "qcow2". + +A detected format mismatch between Glance and what the actual contents of +the disk image file are detected as will result in a failed deployment. +To correct such a situation, the image must be re-uploaded with the +declared ``--disk-format`` or actual image file format corrected. + Instance (end-user) images ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -11,6 +29,10 @@ Load all the created images into the Image service, and note the image UUIDs in the Image service for each one as it is generated. +.. note:: + Images from Glance used by Ironic must be flagged as ``public``, which + requires administrative privileges with the Glance image service to set. + - For *whole disk images* just upload the image: .. code-block:: console diff -Nru ironic-21.1.0/doc/source/install/include/common-prerequisites.inc ironic-21.4.4/doc/source/install/include/common-prerequisites.inc --- ironic-21.1.0/doc/source/install/include/common-prerequisites.inc 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/install/include/common-prerequisites.inc 2024-10-11 15:42:16.000000000 +0000 @@ -22,8 +22,16 @@ .. code-block:: console # mysql -u root -p - mysql> CREATE DATABASE ironic CHARACTER SET utf8; + mysql> CREATE DATABASE ironic CHARACTER SET utf8mb3; mysql> GRANT ALL PRIVILEGES ON ironic.* TO 'ironic'@'localhost' \ IDENTIFIED BY 'IRONIC_DBPASSWORD'; mysql> GRANT ALL PRIVILEGES ON ironic.* TO 'ironic'@'%' \ IDENTIFIED BY 'IRONIC_DBPASSWORD'; + +.. note:: + When creating the database to house Ironic, specifically on MySQL/MariaDB, + the character set *cannot* be 4 byte Unicode characters. This is due to + an internal structural constraint. UTF8, in these database platforms, + has traditionally meant ``utf8mb3``, short for "UTF-8, 3 byte encoding", + however the platforms are expected to move to ``utf8mb4`` which is + incompatible with Ironic. diff -Nru ironic-21.1.0/doc/source/install/refarch/common.rst ironic-21.4.4/doc/source/install/refarch/common.rst --- ironic-21.1.0/doc/source/install/refarch/common.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/install/refarch/common.rst 2024-10-11 15:42:16.000000000 +0000 @@ -277,9 +277,8 @@ In both cases a cached image is converted to raw if ``force_raw_images`` is ``True`` (the default). - .. note:: - ``image_download_source`` can also be provided in the node's - ``driver_info`` or ``instance_info``. See :ref:`image_download_source`. + See :ref:`image_download_source` and :ref:`stream_raw_images` for more + details. * When network boot is used, the instance image kernel and ramdisk are cached locally while the instance is active. diff -Nru ironic-21.1.0/doc/source/user/creating-images.rst ironic-21.4.4/doc/source/user/creating-images.rst --- ironic-21.1.0/doc/source/user/creating-images.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/doc/source/user/creating-images.rst 2024-10-11 15:42:16.000000000 +0000 @@ -27,6 +27,39 @@ images that are built for legacy boot mode (not UEFI), with Ubuntu being an exception (they publish images that work in both modes). +Supported Disk Image Formats +---------------------------- + +The following formats are tested by Ironic and are expected to work as +long as no unknown or unsafe special features are being used + +* raw - A file containing bytes as they would exist on a disk or other + block storage device. This is the simplest format. +* qcow2 - An updated file format based upon the `QEMU `_ + Copy-on-Write format. + +A special mention exists for ``iso`` formatted "CD" images. While Ironic uses +the ISO9660 filesystems in some of it's processes for aspects such as virtual +media, it does *not* support writing them to the remote block storage device. + +Image formats we believe may work due to third party reports, but do not test: + +* vmdk - A file format derived from the image format originally created + by VMware for their hypervisor product line. Specifically we believe + a single file VMDK formatted image should work. As there are + are several subformats, some of which will not work and may result + in unexpected behavior such as failed deployments. +* vdi - A file format used by + `Oracle VM Virtualbox `_ hypervisor. + +As Ironic does not support these formats, their usage is normally blocked +due security considerations by default. Please consult with your Ironic Operator. + +It is important to highlight that Ironic enforces and matches the file type +based upon signature, and not file extension. If there is a mismatch, +the input and or remote service records such as in the Image service +must be corrected. + disk-image-builder ------------------ diff -Nru ironic-21.1.0/driver-requirements.txt ironic-21.4.4/driver-requirements.txt --- ironic-21.1.0/driver-requirements.txt 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/driver-requirements.txt 2024-10-11 15:42:16.000000000 +0000 @@ -4,9 +4,12 @@ # python projects they should package as optional dependencies for Ironic. # These are available on pypi -proliantutils>=2.14.0 +# NOTE(TheJulia): Proliantutils 2.16.0 moves to pysnmp-lextudio +# however that breaks on imports and with testing due to collission +# with pysnmp. +proliantutils>=2.14.0,<2.16.0 pysnmp>=4.3.0,<5.0.0 -python-scciclient>=0.12.2 +python-scciclient>=0.12.2,<0.14.0 python-dracclient>=5.1.0,<9.0.0 python-xclarityclient>=0.1.6 diff -Nru ironic-21.1.0/ironic/api/controllers/v1/__init__.py ironic-21.4.4/ironic/api/controllers/v1/__init__.py --- ironic-21.1.0/ironic/api/controllers/v1/__init__.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/api/controllers/v1/__init__.py 2024-10-11 15:42:16.000000000 +0000 @@ -36,6 +36,7 @@ from ironic.api.controllers.v1 import port from ironic.api.controllers.v1 import portgroup from ironic.api.controllers.v1 import ramdisk +from ironic.api.controllers.v1 import shard from ironic.api.controllers.v1 import utils from ironic.api.controllers.v1 import versions from ironic.api.controllers.v1 import volume @@ -182,6 +183,16 @@ 'deploy_templates', '', bookmark=True) ] + if utils.allow_shards_endpoint(): + v1['shards'] = [ + link.make_link('self', + api.request.public_url, + 'shards', ''), + link.make_link('bookmark', + api.request.public_url, + 'shards', '', + bookmark=True) + ] return v1 @@ -200,7 +211,8 @@ 'conductors': conductor.ConductorsController(), 'allocations': allocation.AllocationsController(), 'events': event.EventsController(), - 'deploy_templates': deploy_template.DeployTemplatesController() + 'deploy_templates': deploy_template.DeployTemplatesController(), + 'shards': shard.ShardController(), } @method.expose() diff -Nru ironic-21.1.0/ironic/api/controllers/v1/allocation.py ironic-21.4.4/ironic/api/controllers/v1/allocation.py --- ironic-21.1.0/ironic/api/controllers/v1/allocation.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/api/controllers/v1/allocation.py 2024-10-11 15:42:16.000000000 +0000 @@ -271,11 +271,22 @@ :fields: fields :owner: r_owner """ - owner = api_utils.check_list_policy('allocation', owner) + requestor = api_utils.check_list_policy('allocation', owner) self._check_allowed_allocation_fields(fields) if owner is not None and not api_utils.allow_allocation_owner(): + # Requestor has asked for an owner field/column match, but + # their client version does not support it. raise exception.NotAcceptable() + if (owner is not None + and requestor is not None + and owner != requestor): + # The requestor is asking about other owner's records. + # Naughty! + raise exception.NotAuthorized() + + if requestor is not None: + owner = requestor return self._get_allocations_collection(node, resource_class, state, owner, marker, limit, diff -Nru ironic-21.1.0/ironic/api/controllers/v1/node.py ironic-21.4.4/ironic/api/controllers/v1/node.py --- ironic-21.1.0/ironic/api/controllers/v1/node.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/api/controllers/v1/node.py 2024-10-11 15:42:16.000000000 +0000 @@ -48,6 +48,7 @@ from ironic.conductor import steps as conductor_steps import ironic.conf from ironic.drivers import base as driver_base +from ironic.drivers.modules import inspect_utils from ironic import objects @@ -179,6 +180,7 @@ 'retired': {'type': ['string', 'boolean', 'null']}, 'retired_reason': {'type': ['string', 'null']}, 'secure_boot': {'type': ['string', 'boolean', 'null']}, + 'shard': {'type': ['string', 'null']}, 'storage_interface': {'type': ['string', 'null']}, 'uuid': {'type': ['string', 'null']}, 'vendor_interface': {'type': ['string', 'null']}, @@ -266,6 +268,7 @@ 'resource_class', 'retired', 'retired_reason', + 'shard', 'storage_interface', 'vendor_interface' ] @@ -1382,6 +1385,7 @@ 'retired', 'retired_reason', 'secure_boot', + 'shard', 'storage_interface', 'target_power_state', 'target_provision_state', @@ -1944,6 +1948,26 @@ node.uuid, event, detail=True) +class NodeInventoryController(rest.RestController): + + def __init__(self, node_ident): + super(NodeInventoryController).__init__() + self.node_ident = node_ident + + @METRICS.timer('NodeInventoryController.get') + @method.expose() + @args.validate(node_ident=args.uuid_or_name) + def get(self): + """Node inventory of the node. + + :param node_ident: the UUID of a node. + """ + node = api_utils.check_node_policy_and_retrieve( + 'baremetal:node:inventory:get', self.node_ident) + return inspect_utils.get_introspection_data(node, + api.request.context) + + class NodesController(rest.RestController): """REST controller for Nodes.""" @@ -1990,6 +2014,7 @@ 'bios': bios.NodeBiosController, 'allocation': allocation.NodeAllocationController, 'history': NodeHistoryController, + 'inventory': NodeInventoryController, } @pecan.expose() @@ -2013,7 +2038,9 @@ or (remainder[0] == 'allocation' and not api_utils.allow_allocations()) or (remainder[0] == 'history' - and not api_utils.allow_node_history())): + and not api_utils.allow_node_history()) + or (remainder[0] == 'inventory' + and not api_utils.allow_node_inventory())): pecan.abort(http_client.NOT_FOUND) if remainder[0] == 'traits' and not api_utils.allow_traits(): # NOTE(mgoddard): Returning here will ensure we exhibit the @@ -2045,7 +2072,8 @@ fields=None, fault=None, conductor_group=None, detail=None, conductor=None, owner=None, lessee=None, project=None, - description_contains=None): + description_contains=None, shard=None, + sharded=None): if self.from_chassis and not chassis_uuid: raise exception.MissingParameterValue( _("Chassis id not specified.")) @@ -2065,6 +2093,12 @@ # The query parameters for the 'next' URL parameters = {} + + # note(JayF): This is where you resolve differences between the name + # of the filter in the API and the name of the filter in the DB API. + # In the case of lists (args.string_list), you need to append _in to + # the filter name in order to exercise the list-aware logic in the + # lower level. possible_filters = { 'maintenance': maintenance, 'chassis_uuid': chassis_uuid, @@ -2076,10 +2110,12 @@ 'conductor_group': conductor_group, 'owner': owner, 'lessee': lessee, + 'shard_in': shard, 'project': project, 'description_contains': description_contains, 'retired': retired, - 'instance_uuid': instance_uuid + 'instance_uuid': instance_uuid, + 'sharded': sharded } filters = {} for key, value in possible_filters.items(): @@ -2098,7 +2134,7 @@ # map the name for the call, as we did not pickup a specific # list of fields to return. obj_fields = fields - # NOTE(TheJulia): When a data set of the nodeds list is being + # NOTE(TheJulia): When a data set of the nodes list is being # requested, this method takes approximately 3-3.5% of the time # when requesting specific fields aligning with Nova's sync # process. (Local DB though) @@ -2223,14 +2259,15 @@ fault=args.string, conductor_group=args.string, detail=args.boolean, conductor=args.string, owner=args.string, description_contains=args.string, - lessee=args.string, project=args.string) + lessee=args.string, project=args.string, + shard=args.string_list, sharded=args.boolean) def get_all(self, chassis_uuid=None, instance_uuid=None, associated=None, maintenance=None, retired=None, provision_state=None, marker=None, limit=None, sort_key='id', sort_dir='asc', driver=None, fields=None, resource_class=None, fault=None, conductor_group=None, detail=None, conductor=None, owner=None, description_contains=None, lessee=None, - project=None): + project=None, shard=None, sharded=None): """Retrieve a list of nodes. :param chassis_uuid: Optional UUID of a chassis, to get only nodes for @@ -2265,15 +2302,20 @@ :param owner: Optional string value that set the owner whose nodes are to be retrurned. :param lessee: Optional string value that set the lessee whose nodes - are to be returned. + are to be returned. :param project: Optional string value that set the project - lessee or owner - whose nodes are to be returned. + :param shard: Optional string value that set the shards whose nodes are + to be returned. :param fields: Optional, a list with a specified set of fields of the resource to be returned. :param fault: Optional string value to get only nodes with that fault. :param description_contains: Optional string value to get only nodes with description field contains matching value. + :param sharded: Optional boolean whether to return a list of + nodes with or without a shard set. May be combined + with other parameters. """ project = api_utils.check_list_policy('node', project) @@ -2288,6 +2330,9 @@ api_utils.check_allow_filter_by_conductor(conductor) api_utils.check_allow_filter_by_owner(owner) api_utils.check_allow_filter_by_lessee(lessee) + api_utils.check_allow_filter_by_shard(shard) + # Sharded is guarded by the same API version as shard + api_utils.check_allow_filter_by_shard(sharded) fields = api_utils.get_request_return_fields(fields, detail, _DEFAULT_RETURN_FIELDS) @@ -2304,8 +2349,8 @@ detail=detail, conductor=conductor, owner=owner, lessee=lessee, - project=project, - **extra_args) + shard=shard, sharded=sharded, + project=project, **extra_args) @METRICS.timer('NodesController.detail') @method.expose() @@ -2317,13 +2362,15 @@ resource_class=args.string, fault=args.string, conductor_group=args.string, conductor=args.string, owner=args.string, description_contains=args.string, - lessee=args.string, project=args.string) + lessee=args.string, project=args.string, + shard=args.string_list, sharded=args.boolean) def detail(self, chassis_uuid=None, instance_uuid=None, associated=None, maintenance=None, retired=None, provision_state=None, marker=None, limit=None, sort_key='id', sort_dir='asc', driver=None, resource_class=None, fault=None, conductor_group=None, conductor=None, owner=None, - description_contains=None, lessee=None, project=None): + description_contains=None, lessee=None, project=None, + shard=None, sharded=None): """Retrieve a list of nodes with detail. :param chassis_uuid: Optional UUID of a chassis, to get only nodes for @@ -2360,9 +2407,13 @@ are to be returned. :param project: Optional string value that set the project - lessee or owner - whose nodes are to be returned. + :param shard: Optional - set the shards whose nodes are to be returned. :param description_contains: Optional string value to get only nodes with description field contains matching value. + :param sharded: Optional boolean whether to return a list of + nodes with or without a shard set. May be combined + with other parameters. """ project = api_utils.check_list_policy('node', project) @@ -2380,6 +2431,9 @@ raise exception.HTTPNotFound() api_utils.check_allow_filter_by_conductor(conductor) + api_utils.check_allow_filter_by_shard(shard) + # Sharded is guarded by the same API version as shard + api_utils.check_allow_filter_by_shard(sharded) extra_args = {'description_contains': description_contains} return self._get_nodes_collection(chassis_uuid, instance_uuid, @@ -2393,8 +2447,8 @@ conductor_group=conductor_group, conductor=conductor, owner=owner, lessee=lessee, - project=project, - **extra_args) + project=project, shard=shard, + sharded=sharded, **extra_args) @METRICS.timer('NodesController.validate') @method.expose() @@ -2619,6 +2673,8 @@ policy_checks.append('baremetal:node:update:name') elif p['path'].startswith('/retired'): policy_checks.append('baremetal:node:update:retired') + elif p['path'].startswith('/shard'): + policy_checks.append('baremetal:node:update:shard') else: generic_update = True # always do at least one check diff -Nru ironic-21.1.0/ironic/api/controllers/v1/port.py ironic-21.4.4/ironic/api/controllers/v1/port.py --- ironic-21.1.0/ironic/api/controllers/v1/port.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/api/controllers/v1/port.py 2024-10-11 15:42:16.000000000 +0000 @@ -123,9 +123,9 @@ 'local_link_connection', 'physical_network', 'pxe_enabled', + 'node_uuid', ) ) - api_utils.populate_node_uuid(rpc_port, port) if rpc_port.portgroup_id: pg = objects.Portgroup.get(api.request.context, rpc_port.portgroup_id) port['portgroup_uuid'] = pg.uuid @@ -163,24 +163,13 @@ def list_convert_with_links(rpc_ports, limit, url, fields=None, **kwargs): ports = [] for rpc_port in rpc_ports: - try: - port = convert_with_links(rpc_port, fields=fields, - sanitize=False) - except exception.NodeNotFound: - # NOTE(dtantsur): node was deleted after we fetched the port - # list, meaning that the port was also deleted. Skip it. - LOG.debug('Skipping port %s as its node was deleted', - rpc_port.uuid) + port = convert_with_links(rpc_port, fields=fields, + sanitize=False) + # NOTE(dtantsur): node was deleted after we fetched the port + # list, meaning that the port was also deleted. Skip it. + if port['node_uuid'] is None: continue - except exception.PortgroupNotFound: - # NOTE(dtantsur): port group was deleted after we fetched the - # port list, it may mean that the port was deleted too, but - # we don't know it. Pretend that the port group was removed. - LOG.debug('Removing port group UUID from port %s as the port ' - 'group was deleted', rpc_port.uuid) - rpc_port.portgroup_id = None - port = convert_with_links(rpc_port, fields=fields, - sanitize=False) + ports.append(port) return collection.list_convert_with_links( items=ports, @@ -210,7 +199,7 @@ self.parent_portgroup_ident = portgroup_ident def _get_ports_collection(self, node_ident, address, portgroup_ident, - marker, limit, sort_key, sort_dir, + shard, marker, limit, sort_key, sort_dir, resource_url=None, fields=None, detail=None, project=None): """Retrieve a collection of ports. @@ -221,6 +210,8 @@ this MAC address. :param portgroup_ident: UUID or name of a portgroup, to get only ports for that portgroup. + :param shard: A comma-separated shard list, to get only ports for those + shards :param marker: pagination marker for large data sets. :param limit: maximum number of resources to return in a single result. This value cannot be larger than the value of max_limit @@ -253,8 +244,12 @@ node_ident = self.parent_node_ident or node_ident portgroup_ident = self.parent_portgroup_ident or portgroup_ident - if node_ident and portgroup_ident: - raise exception.OperationNotPermitted() + exclusive_filters = 0 + for i in [node_ident, portgroup_ident, shard]: + if i: + exclusive_filters += 1 + if exclusive_filters > 1: + raise exception.OperationNotPermitted() if portgroup_ident: # FIXME: Since all we need is the portgroup ID, we can @@ -281,6 +276,11 @@ project=project) elif address: ports = self._get_ports_by_address(address, project=project) + elif shard: + ports = objects.Port.list_by_node_shards(api.request.context, + shard, limit, + marker_obj, sort_key, + sort_dir, project=project) else: ports = objects.Port.list(api.request.context, limit, marker_obj, sort_key=sort_key, @@ -351,10 +351,11 @@ address=args.mac_address, marker=args.uuid, limit=args.integer, sort_key=args.string, sort_dir=args.string, fields=args.string_list, - portgroup=args.uuid_or_name, detail=args.boolean) + portgroup=args.uuid_or_name, detail=args.boolean, + shard=args.string_list) def get_all(self, node=None, node_uuid=None, address=None, marker=None, limit=None, sort_key='id', sort_dir='asc', fields=None, - portgroup=None, detail=None): + portgroup=None, detail=None, shard=None): """Retrieve a list of ports. Note that the 'node_uuid' interface is deprecated in favour @@ -377,6 +378,8 @@ of the resource to be returned. :param portgroup: UUID or name of a portgroup, to get only ports for that portgroup. + :param shard: Optional, a list of shard ids to filter by, only ports + associated with nodes in these shards will be returned. :raises: NotAcceptable, HTTPNotFound """ project = api_utils.check_port_list_policy( @@ -396,6 +399,8 @@ if portgroup and not api_utils.allow_portgroups_subcontrollers(): raise exception.NotAcceptable() + api_utils.check_allow_filter_by_shard(shard) + fields = api_utils.get_request_return_fields(fields, detail, _DEFAULT_RETURN_FIELDS) @@ -408,8 +413,9 @@ raise exception.NotAcceptable() return self._get_ports_collection(node_uuid or node, address, - portgroup, marker, limit, sort_key, - sort_dir, resource_url='ports', + portgroup, shard, marker, limit, + sort_key, sort_dir, + resource_url='ports', fields=fields, detail=detail, project=project) @@ -418,10 +424,11 @@ @args.validate(node=args.uuid_or_name, node_uuid=args.uuid, address=args.mac_address, marker=args.uuid, limit=args.integer, sort_key=args.string, - sort_dir=args.string, - portgroup=args.uuid_or_name) + sort_dir=args.string, portgroup=args.uuid_or_name, + shard=args.string_list) def detail(self, node=None, node_uuid=None, address=None, marker=None, - limit=None, sort_key='id', sort_dir='asc', portgroup=None): + limit=None, sort_key='id', sort_dir='asc', portgroup=None, + shard=None): """Retrieve a list of ports with detail. Note that the 'node_uuid' interface is deprecated in favour @@ -435,6 +442,8 @@ this MAC address. :param portgroup: UUID or name of a portgroup, to get only ports for that portgroup. + :param shard: comma separated list of shards, to only get ports + associated with nodes in those shards. :param marker: pagination marker for large data sets. :param limit: maximum number of resources to return in a single result. This value cannot be larger than the value of max_limit @@ -452,6 +461,8 @@ if portgroup and not api_utils.allow_portgroups_subcontrollers(): raise exception.NotAcceptable() + api_utils.check_allow_filter_by_shard(shard) + if not node_uuid and node: # We're invoking this interface using positional notation, or # explicitly using 'node'. Try and determine which one. @@ -466,8 +477,8 @@ raise exception.HTTPNotFound() return self._get_ports_collection(node_uuid or node, address, - portgroup, marker, limit, sort_key, - sort_dir, + portgroup, shard, marker, limit, + sort_key, sort_dir, resource_url='ports/detail', project=project) @@ -661,7 +672,7 @@ context, port_dict['portgroup_uuid']) else: portgroup = None - except exception.PortGroupNotFound as e: + except exception.PortgroupNotFound as e: # Change error code because 404 (NotFound) is inappropriate # response for a PATCH request to change a Port e.code = http_client.BAD_REQUEST # BadRequest diff -Nru ironic-21.1.0/ironic/api/controllers/v1/portgroup.py ironic-21.4.4/ironic/api/controllers/v1/portgroup.py --- ironic-21.1.0/ironic/api/controllers/v1/portgroup.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/api/controllers/v1/portgroup.py 2024-10-11 15:42:16.000000000 +0000 @@ -90,10 +90,10 @@ 'mode', 'name', 'properties', - 'standalone_ports_supported' + 'standalone_ports_supported', + 'node_uuid' ) ) - api_utils.populate_node_uuid(rpc_portgroup, portgroup) url = api.request.public_url portgroup['ports'] = [ link.make_link('self', url, 'portgroups', diff -Nru ironic-21.1.0/ironic/api/controllers/v1/ramdisk.py ironic-21.4.4/ironic/api/controllers/v1/ramdisk.py --- ironic-21.1.0/ironic/api/controllers/v1/ramdisk.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/api/controllers/v1/ramdisk.py 2024-10-11 15:42:16.000000000 +0000 @@ -57,6 +57,8 @@ # explicit True statement for newer agents to lock the setting # and behavior into place. 'agent_token_required': True, + 'disable_deep_image_inspection': CONF.conductor.disable_deep_image_inspection, # noqa + 'permitted_image_formats': CONF.conductor.permitted_image_formats, } @@ -131,13 +133,17 @@ else: node = objects.Node.get_by_port_addresses( api.request.context, valid_addresses) - except exception.NotFound: + except exception.NotFound as e: # NOTE(dtantsur): we are reraising the same exception to make sure # we don't disclose the difference between nodes that are not found # at all and nodes in a wrong state by different error messages. + LOG.error('No node has been found during lookup: %s', e) raise exception.NotFound() if CONF.api.restrict_lookup and not self.lookup_allowed(node): + LOG.error('Lookup is not allowed for node %(node)s in the ' + 'provision state %(state)s', + {'node': node.uuid, 'state': node.provision_state}) raise exception.NotFound() if api_utils.allow_agent_token(): diff -Nru ironic-21.1.0/ironic/api/controllers/v1/shard.py ironic-21.4.4/ironic/api/controllers/v1/shard.py --- ironic-21.1.0/ironic/api/controllers/v1/shard.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/api/controllers/v1/shard.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,59 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ironic_lib import metrics_utils +from oslo_config import cfg +import pecan +from webob import exc as webob_exc + +from ironic import api +from ironic.api.controllers.v1 import utils as api_utils +from ironic.api import method +from ironic.common.i18n import _ + + +CONF = cfg.CONF + +METRICS = metrics_utils.get_metrics_logger(__name__) + + +class ShardController(pecan.rest.RestController): + """REST controller for shards.""" + + @pecan.expose() + def _route(self, argv, request=None): + if not api_utils.allow_shards_endpoint(): + msg = _("The API version does not allow shards") + if api.request.method in "GET": + raise webob_exc.HTTPNotFound(msg) + else: + raise webob_exc.HTTPMethodNotAllowed(msg) + return super(ShardController, self)._route(argv, request) + + @METRICS.timer('ShardController.get_all') + @method.expose() + def get_all(self): + """Retrieve a list of shards. + + :returns: A list of shards. + """ + api_utils.check_policy('baremetal:shards:get') + + return { + 'shards': api.request.dbapi.get_shard_list(), + } + + @METRICS.timer('ShardController.get_one') + @method.expose() + def get_one(self, __): + """Explicitly do not support getting one.""" + pecan.abort(404) diff -Nru ironic-21.1.0/ironic/api/controllers/v1/utils.py ironic-21.4.4/ironic/api/controllers/v1/utils.py --- ironic-21.1.0/ironic/api/controllers/v1/utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/api/controllers/v1/utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -807,6 +807,7 @@ 'network_data': versions.MINOR_66_NODE_NETWORK_DATA, 'boot_mode': versions.MINOR_75_NODE_BOOT_MODE, 'secure_boot': versions.MINOR_75_NODE_BOOT_MODE, + 'shard': versions.MINOR_82_NODE_SHARD } for field in V31_FIELDS: @@ -1065,6 +1066,20 @@ 'opr': versions.MINOR_65_NODE_LESSEE}) +def check_allow_filter_by_shard(shard): + """Check if filtering nodes by shard is allowed. + + Version 1.82 of the API allows filtering nodes by shard. + """ + if (shard is not None and api.request.version.minor + < versions.MINOR_82_NODE_SHARD): + raise exception.NotAcceptable(_( + "Request not acceptable. The minimal required API version " + "should be %(base)s.%(opr)s") % + {'base': versions.BASE_VERSION, + 'opr': versions.MINOR_82_NODE_SHARD}) + + def initial_node_provision_state(): """Return node state to use by default when creating new nodes. @@ -1341,6 +1356,11 @@ return api.request.version.minor >= versions.MINOR_78_NODE_HISTORY +def allow_node_inventory(): + """Check if node inventory is allowed.""" + return api.request.version.minor >= versions.MINOR_81_NODE_INVENTORY + + def get_request_return_fields(fields, detail, default_fields, check_detail_version=allow_detail_query, check_fields_version=None): @@ -1948,3 +1968,8 @@ elif target != "clean": raise exception.BadRequest( _("disable_ramdisk is supported only with manual cleaning")) + + +def allow_shards_endpoint(): + """Check if shards endpoint is available.""" + return api.request.version.minor >= versions.MINOR_82_NODE_SHARD diff -Nru ironic-21.1.0/ironic/api/controllers/v1/versions.py ironic-21.4.4/ironic/api/controllers/v1/versions.py --- ironic-21.1.0/ironic/api/controllers/v1/versions.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/api/controllers/v1/versions.py 2024-10-11 15:42:16.000000000 +0000 @@ -118,6 +118,8 @@ # v1.78: Add node history endpoint # v1.79: Change allocation behaviour to prefer node name match # v1.80: Marker to represent self service node creation/deletion +# v1.81: Add node inventory +# v1.82: Add node sharding capability MINOR_0_JUNO = 0 MINOR_1_INITIAL_VERSION = 1 MINOR_2_AVAILABLE_STATE = 2 @@ -199,6 +201,8 @@ MINOR_78_NODE_HISTORY = 78 MINOR_79_ALLOCATION_NODE_NAME = 79 MINOR_80_PROJECT_CREATE_DELETE_NODE = 80 +MINOR_81_NODE_INVENTORY = 81 +MINOR_82_NODE_SHARD = 82 # When adding another version, update: # - MINOR_MAX_VERSION @@ -206,7 +210,7 @@ # explanation of what changed in the new version # - common/release_mappings.py, RELEASE_MAPPING['master']['api'] -MINOR_MAX_VERSION = MINOR_80_PROJECT_CREATE_DELETE_NODE +MINOR_MAX_VERSION = MINOR_82_NODE_SHARD # String representations of the minor and maximum versions _MIN_VERSION_STRING = '{}.{}'.format(BASE_VERSION, MINOR_1_INITIAL_VERSION) diff -Nru ironic-21.1.0/ironic/cmd/status.py ironic-21.4.4/ironic/cmd/status.py --- ironic-21.1.0/ironic/cmd/status.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/cmd/status.py 2024-10-11 15:42:16.000000000 +0000 @@ -19,7 +19,7 @@ from oslo_db.sqlalchemy import utils from oslo_upgradecheck import common_checks from oslo_upgradecheck import upgradecheck -from sqlalchemy import exc as sa_exc +import sqlalchemy from ironic.cmd import dbsync from ironic.common.i18n import _ @@ -50,7 +50,7 @@ # when a table is missing, so lets catch it, since it is fatal. msg = dbsync.DBCommand().check_obj_versions( ignore_missing_tables=True) - except sa_exc.NoSuchTableError as e: + except sqlalchemy.exc.NoSuchTableError as e: msg = ('Database table missing. Please ensure you have ' 'updated the database schema. Not Found: %s' % e) return upgradecheck.Result(upgradecheck.Code.FAILURE, details=msg) @@ -94,6 +94,41 @@ else: return upgradecheck.Result(upgradecheck.Code.SUCCESS) + def _check_allocations_table(self): + msg = None + engine = enginefacade.reader.get_engine() + if 'mysql' not in str(engine.url): + # This test only applies to mysql and database schema + # selection. + return upgradecheck.Result(upgradecheck.Code.SUCCESS) + res = engine.execute("show create table allocations") + results = str(res.all()).lower() + if 'utf8' not in results: + msg = ('The Allocations table is is not using UTF8 encoding. ' + 'This is corrected in later versions of Ironic, where ' + 'the table character set schema is automatically ' + 'migrated. Continued use of a non-UTF8 character ' + 'set may produce unexpected results.') + + if 'innodb' not in results: + warning = ('The engine used by MySQL for the allocations ' + 'table is not the intended engine for the Ironic ' + 'database tables to use. This may have been a result ' + 'of an error with the table creation schema. This ' + 'may require Database Administrator intervention ' + 'and downtime to dump, modify the table engine to ' + 'utilize InnoDB, and reload the allocations table to ' + 'utilize the InnoDB engine.') + if msg: + msg = msg + ' Additionally: ' + warning + else: + msg = warning + + if msg: + return upgradecheck.Result(upgradecheck.Code.WARNING, details=msg) + else: + return upgradecheck.Result(upgradecheck.Code.SUCCESS) + # A tuple of check tuples of (, ). # The name of the check will be used in the output of this command. # The check function takes no arguments and returns an @@ -105,6 +140,8 @@ _upgrade_checks = ( (_('Object versions'), _check_obj_versions), (_('Database Index Status'), _check_db_indexes), + (_('Allocations Name Field Length Check'), + _check_allocations_table), # Victoria -> Wallaby migration (_('Policy File JSON to YAML Migration'), (common_checks.check_policy_json, {'conf': CONF})), diff -Nru ironic-21.1.0/ironic/common/checksum_utils.py ironic-21.4.4/ironic/common/checksum_utils.py --- ironic-21.1.0/ironic/common/checksum_utils.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/common/checksum_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,250 @@ +# Copyright (c) 2024 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import re +import time +from urllib import parse as urlparse + +from oslo_log import log as logging +from oslo_utils import fileutils + +from ironic.common import exception +from ironic.common.i18n import _ +from ironic.common import image_service +from ironic.conf import CONF + +LOG = logging.getLogger(__name__) + + +# REGEX matches for Checksum file payloads +# If this list requires changes, it should be changed in +# ironic-python-agent (extensions/standby.py) as well. + +MD5_MATCH = r"^([a-fA-F\d]{32})\s" # MD5 at beginning of line +MD5_MATCH_END = r"\s([a-fA-F\d]{32})$" # MD5 at end of line +MD5_MATCH_ONLY = r"^([a-fA-F\d]{32})$" # MD5 only +SHA256_MATCH = r"^([a-fA-F\d]{64})\s" # SHA256 at beginning of line +SHA256_MATCH_END = r"\s([a-fA-F\d]{64})$" # SHA256 at end of line +SHA256_MATCH_ONLY = r"^([a-fA-F\d]{64})$" # SHA256 only +SHA512_MATCH = r"^([a-fA-F\d]{128})\s" # SHA512 at beginning of line +SHA512_MATCH_END = r"\s([a-fA-F\d]{128})$" # SHA512 at end of line +SHA512_MATCH_ONLY = r"^([a-fA-F\d]{128})$" # SHA512 only +FILENAME_MATCH_END = r"\s[*]?{filename}$" # Filename binary/text end of line +FILENAME_MATCH_PARENTHESES = r"\s\({filename}\)\s" # CentOS images + +CHECKSUM_MATCHERS = (MD5_MATCH, MD5_MATCH_END, SHA256_MATCH, SHA256_MATCH_END, + SHA512_MATCH, SHA512_MATCH_END) +CHECKSUM_ONLY_MATCHERS = (MD5_MATCH_ONLY, SHA256_MATCH_ONLY, SHA512_MATCH_ONLY) +FILENAME_MATCHERS = (FILENAME_MATCH_END, FILENAME_MATCH_PARENTHESES) + + +def validate_checksum(path, checksum, checksum_algo=None): + """Validate image checksum. + + :param path: File path in the form of a string to calculate a checksum + which is compared to the checksum field. + :param checksum: The supplied checksum value, a string, which will be + compared to the file. + :param checksum_algo: The checksum type of the algorithm. + :raises: ImageChecksumError if the supplied data cannot be parsed or + if the supplied value does not match the supplied checksum + value. + """ + # TODO(TheJilia): At some point, we likely need to compare + # the incoming checksum algorithm upfront, ut if one is invoked which + # is not supported, hashlib will raise ValueError. + use_checksum_algo = None + if ":" in checksum: + # A form of communicating the checksum algorithm is to delimit the + # type from the value. See ansible deploy interface where this + # is most evident. + split_checksum = checksum.split(":") + use_checksum = split_checksum[1] + use_checksum_algo = split_checksum[0] + else: + use_checksum = checksum + if not use_checksum_algo: + use_checksum_algo = checksum_algo + # If we have a zero length value, but we split it, we have + # invalid input. Also, checksum is what we expect, algorithm is + # optional. This guards against the split of a value which is + # image_checksum = "sha256:" which is a potential side effect of + # splitting the string. + if use_checksum == '': + raise exception.ImageChecksumError() + + # Make everything lower case since we don't expect mixed case, + # but we may have human originated input on the supplied algorithm. + try: + if not use_checksum_algo: + # This is backwards compatible support for a bare checksum. + calculated = compute_image_checksum(path) + else: + calculated = compute_image_checksum(path, + use_checksum_algo.lower()) + except ValueError: + # ValueError is raised when an invalid/unsupported/unknown + # checksum algorithm is invoked. + LOG.error("Failed to generate checksum for file %(path)s, possible " + "invalid checksum algorithm: %(algo)s", + {"path": path, + "algo": use_checksum_algo}) + raise exception.ImageChecksumAlgorithmFailure() + except OSError: + LOG.error("Failed to read file %(path)s to compute checksum.", + {"path": path}) + raise exception.ImageChecksumFileReadFailure() + if (use_checksum is not None + and calculated.lower() != use_checksum.lower()): + LOG.error("We were supplied a checksum value of %(supplied)s, but " + "calculated a value of %(value)s. This is a fatal error.", + {"supplied": use_checksum, + "value": calculated}) + raise exception.ImageChecksumError() + + +def compute_image_checksum(image_path, algorithm='md5'): + """Compute checksum by given image path and algorithm. + + :param image_path: The path to the file to undergo checksum calculation. + :param algorithm: The checksum algorithm to utilize. Defaults + to 'md5' due to historical support reasons in Ironic. + :returns: The calculated checksum value. + :raises: ValueError when the checksum algorithm is not supported + by the system. + """ + + time_start = time.time() + LOG.debug('Start computing %(algo)s checksum for image %(image)s.', + {'algo': algorithm, 'image': image_path}) + + checksum = fileutils.compute_file_checksum(image_path, + algorithm=algorithm) + time_elapsed = time.time() - time_start + LOG.debug('Computed %(algo)s checksum for image %(image)s in ' + '%(delta).2f seconds, checksum value: %(checksum)s.', + {'algo': algorithm, 'image': image_path, 'delta': time_elapsed, + 'checksum': checksum}) + return checksum + + +def get_checksum_and_algo(instance_info): + """Get and return the image checksum and algo. + + :param instance_info: The node instance info, or newly updated/generated + instance_info value. + :returns: A tuple containing two values, a checksum and algorithm, + if available. + """ + checksum_algo = None + if 'image_os_hash_value' in instance_info.keys(): + # A value set by image_os_hash_value supersedes other + # possible uses as it is specific. + checksum = instance_info.get('image_os_hash_value') + checksum_algo = instance_info.get('image_os_hash_algo') + else: + checksum = instance_info.get('image_checksum') + if is_checksum_url(checksum): + image_source = instance_info.get('image_source') + checksum = get_checksum_from_url(checksum, image_source) + + # NOTE(TheJulia): This is all based on SHA-2 lengths. + # SHA-3 would require a hint and it would not be a fixed length. + # That said, SHA-2 is still valid and has not been withdrawn. + checksum_len = len(checksum) + if checksum_len == 128: + # SHA2-512 is 512 bits, 128 characters. + checksum_algo = "sha512" + elif checksum_len == 64: + checksum_algo = "sha256" + + return checksum, checksum_algo + + +def is_checksum_url(checksum): + """Identify if checksum is not a url. + + :param checksum: The user supplied checksum value. + :returns: True if the checksum is a url, otherwise False. + :raises: ImageChecksumURLNotSupported should the conductor have this + support disabled. + """ + if (checksum.startswith('http://') or checksum.startswith('https://')): + if CONF.conductor.disable_support_for_checksum_files: + raise exception.ImageChecksumURLNotSupported() + return True + else: + return False + + +def get_checksum_from_url(checksum, image_source): + """Gets a checksum value based upon a remote checksum URL file. + + :param checksum: The URL to the checksum URL content. + :param image_soource: The image source utilized to match with + the contents of the URL payload file. + :raises: ImageDownloadFailed when the checksum file cannot be + accessed or cannot be parsed. + """ + + LOG.debug('Attempting to download checksum from: %(checksum)s.', + {'checksum': checksum}) + + # Directly invoke the image service and get the checksum data. + resp = image_service.HttpImageService.get(checksum) + checksum_url = str(checksum) + + # NOTE(TheJulia): The rest of this method is taken from + # ironic-python-agent. If a change is required here, it may + # be required in ironic-python-agent (extensions/standby.py). + lines = [line.strip() for line in resp.split('\n') if line.strip()] + if not lines: + raise exception.ImageDownloadFailed(image_href=checksum, + reason=_('Checksum file empty.')) + elif len(lines) == 1: + # Special case - checksums file with only the checksum itself + if ' ' not in lines[0]: + for matcher in CHECKSUM_ONLY_MATCHERS: + checksum = re.findall(matcher, lines[0]) + if checksum: + return checksum[0] + raise exception.ImageDownloadFailed( + image_href=checksum_url, + reason=( + _("Invalid checksum file (No valid checksum found)"))) + # FIXME(dtantsur): can we assume the same name for all images? + expected_fname = os.path.basename(urlparse.urlparse( + image_source).path) + for line in lines: + # Ignore comment lines + if line.startswith("#"): + continue + + # Ignore checksums for other files + for matcher in FILENAME_MATCHERS: + if re.findall(matcher.format(filename=expected_fname), line): + break + else: + continue + + for matcher in CHECKSUM_MATCHERS: + checksum = re.findall(matcher, line) + if checksum: + return checksum[0] + + raise exception.ImageDownloadFailed( + image_href=checksum, + reason=(_("Checksum file does not contain name %s") + % expected_fname)) diff -Nru ironic-21.1.0/ironic/common/cinder.py ironic-21.4.4/ironic/common/cinder.py --- ironic-21.1.0/ironic/common/cinder.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/cinder.py 2024-10-11 15:42:16.000000000 +0000 @@ -19,6 +19,7 @@ from cinderclient.v3 import client from oslo_log import log +from ironic.common import context as ironic_context from ironic.common import exception from ironic.common.i18n import _ from ironic.common import keystone @@ -39,30 +40,67 @@ return _CINDER_SESSION -def get_client(context): +def get_client(context, auth_from_config=False): """Get a cinder client connection. :param context: request context, instance of ironic.common.context.RequestContext + :param auth_from_config: (boolean) When True, use auth values from + conf parameters :returns: A cinder client. """ service_auth = keystone.get_auth('cinder') session = _get_cinder_session() + # Used the service cached session to get the endpoint + # because getting an endpoint requires auth! + endpoint = keystone.get_endpoint('cinder', session=session, + auth=service_auth) + project_id = None + if hasattr(service_auth, 'get_project_id'): + # Being moderately defensive so we don't have to mock this in + # every cinder unit test. + project_id = service_auth.get_project_id(session) + if project_id and project_id in str(endpoint): + # We've found the project ID in the endpoint URL due to the + # endpoint configuration, however this is not required in + # more modern versions of cinder, and if you attempt to access + # a resource with a project ID in the URL, and with a service + # token, the request gets a Bad Request response. + # This works starting at Cinder microversion 3.67 - Yoga. + endpoint = endpoint.replace("/%s" % project_id, "") + + if not context: + context = ironic_context.RequestContext(auth_token=None) + + user_auth = None + if CONF.cinder.auth_type != 'none' and context.auth_token: + user_auth = keystone.get_service_auth( + context, endpoint, service_auth, + only_service_auth=auth_from_config) + + if auth_from_config: + # If we are here, then we've been requested to *only* use our supplied + sess = keystone.get_session('cinder', timeout=CONF.cinder.timeout, + auth=service_auth) + else: + sess = keystone.get_session('cinder', timeout=CONF.cinder.timeout, + auth=user_auth or service_auth) + + # Re-determine the endpoint so we can work with versions prior to + # Yoga, becuase the endpoint, based upon configuration, may require + # project_id specific URLs. + if user_auth: + endpoint = keystone.get_endpoint('cinder', session=sess, + auth=user_auth) - # TODO(pas-ha) use versioned endpoint data to select required - # cinder api version - cinder_url = keystone.get_endpoint('cinder', session=session, - auth=service_auth) - # TODO(pas-ha) investigate possibility of passing a user context here, - # similar to what neutron/glance-related code does # NOTE(pas-ha) cinderclient has both 'connect_retries' (passed to # ksa.Adapter) and 'retries' (used in its subclass of ksa.Adapter) options. # The first governs retries on establishing the HTTP connection, # the second governs retries on OverLimit exceptions from API. # The description of [cinder]/retries fits the first, # so this is what we pass. - return client.Client(session=session, auth=service_auth, - endpoint_override=cinder_url, + return client.Client(session=sess, auth=user_auth, + endpoint_override=endpoint, connect_retries=CONF.cinder.retries, global_request_id=context.global_id) @@ -134,17 +172,21 @@ return {label: json.dumps(data)} -def _init_client(task): +def _init_client(task, auth_from_config=False): """Obtain cinder client and return it for use. :param task: TaskManager instance representing the operation. + :param auth_from_config: If we should source our authentication parameters + from the configured service as opposed to request + context. :returns: A cinder client. :raises: StorageError If an exception is encountered creating the client. """ node = task.node try: - return get_client(task.context) + return get_client(task.context, + auth_from_config=auth_from_config) except Exception as e: msg = (_('Failed to initialize cinder client for operations on node ' '%(uuid)s: %(err)s') % {'uuid': node.uuid, 'err': e}) @@ -238,8 +280,9 @@ }] """ node = task.node + LOG.debug('Initializing volume attach for node %(node)s.', + {'node': node.uuid}) client = _init_client(task) - connected = [] for volume_id in volume_list: try: @@ -367,8 +410,10 @@ LOG.error(msg) raise exception.StorageError(msg) - client = _init_client(task) + client = _init_client(task, auth_from_config=False) node = task.node + LOG.debug('Initializing volume detach for node %(node)s.', + {'node': node.uuid}) for volume_id in volume_list: try: diff -Nru ironic-21.1.0/ironic/common/context.py ironic-21.4.4/ironic/common/context.py --- ironic-21.1.0/ironic/common/context.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/context.py 2024-10-11 15:42:16.000000000 +0000 @@ -12,9 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_config import cfg from oslo_context import context +CONF = cfg.CONF + + class RequestContext(context.RequestContext): """Extends security contexts from the oslo.context library.""" @@ -44,6 +48,9 @@ 'project_name': self.project_name, 'is_public_api': self.is_public_api, }) + if (CONF.rbac_service_role_elevated_access + and CONF.rbac_service_project_name is not None): + policy_values['config.service_project_name'] = CONF.rbac_service_project_name # noqa return policy_values def ensure_thread_contain_context(self): diff -Nru ironic-21.1.0/ironic/common/exception.py ironic-21.4.4/ironic/common/exception.py --- ironic-21.1.0/ironic/common/exception.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/exception.py 2024-10-11 15:42:16.000000000 +0000 @@ -120,6 +120,10 @@ "for the same node already exists.") +class NodeInventoryAlreadyExists(Conflict): + _msg_fmt = _("A node inventory with ID %(id)s already exists.") + + class VifAlreadyAttached(Conflict): _msg_fmt = _("Unable to attach VIF because VIF %(vif)s is already " "attached to Ironic %(object_type)s %(object_uuid)s") @@ -828,6 +832,10 @@ _msg_fmt = _("Node history record %(history)s could not be found.") +class NodeInventoryNotFound(NotFound): + _msg_fmt = _("Node inventory record for node %(node)s could not be found.") + + class IncorrectConfiguration(IronicException): _msg_fmt = _("Supplied configuration is incorrect and must be fixed. " "Error: %(error)s") @@ -861,3 +869,34 @@ "The concurrent action limit for %(task_type)s " "has been reached. Please contact your administrator " "and try again later.") + + +class SwiftObjectStillExists(IronicException): + _msg_fmt = _("Clean up failed for swift object %(obj)s during deletion" + " of node %(node)s.") + + +class InvalidImage(ImageUnacceptable): + _msg_fmt = _("The requested image is not valid for use.") + + +class ImageChecksumError(InvalidImage): + """Exception indicating checksum failed to match.""" + _msg_fmt = _("The supplied image checksum is invalid or does not match.") + + +class ImageChecksumAlgorithmFailure(InvalidImage): + """Cannot load the requested or required checksum algorithm.""" + _msg_fmt = _("The requested image checksum algorithm cannot be loaded.") + + +class ImageChecksumURLNotSupported(InvalidImage): + """Exception indicating we cannot support the remote checksum file.""" + _msg_fmt = _("Use of remote checksum files is not supported.") + + +class ImageChecksumFileReadFailure(InvalidImage): + """An OSError was raised when trying to read the file.""" + _msg_fmt = _("Failed to read the file from local storage " + "to perform a checksum operation.") + code = http_client.SERVICE_UNAVAILABLE diff -Nru ironic-21.1.0/ironic/common/glance_service/image_service.py ironic-21.4.4/ironic/common/glance_service/image_service.py --- ironic-21.1.0/ironic/common/glance_service/image_service.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/glance_service/image_service.py 2024-10-11 15:42:16.000000000 +0000 @@ -33,6 +33,7 @@ from ironic.common.i18n import _ from ironic.common import keystone from ironic.common import swift +from ironic.common import utils from ironic.conf import CONF TempUrlCacheElement = collections.namedtuple('TempUrlCacheElement', @@ -114,7 +115,7 @@ @tenacity.retry( retry=tenacity.retry_if_exception_type( exception.GlanceConnectionFailed), - stop=tenacity.stop_after_attempt(CONF.glance.num_retries + 1), + stop=utils.stop_after_retries('num_retries', group='glance'), wait=tenacity.wait_fixed(1), reraise=True ) diff -Nru ironic-21.1.0/ironic/common/image_format_inspector.py ironic-21.4.4/ironic/common/image_format_inspector.py --- ironic-21.1.0/ironic/common/image_format_inspector.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/common/image_format_inspector.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,1038 @@ +# Copyright 2020 Red Hat, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +This is a python implementation of virtual disk format inspection routines +gathered from various public specification documents, as well as qemu disk +driver code. It attempts to store and parse the minimum amount of data +required, and in a streaming-friendly manner to collect metadata about +complex-format images. +""" + +import struct + +from oslo_log import log as logging +from oslo_utils import units + +LOG = logging.getLogger(__name__) + + +def chunked_reader(fileobj, chunk_size=512): + while True: + chunk = fileobj.read(chunk_size) + if not chunk: + break + yield chunk + + +class CaptureRegion(object): + """Represents a region of a file we want to capture. + + A region of a file we want to capture requires a byte offset into + the file and a length. This is expected to be used by a data + processing loop, calling capture() with the most recently-read + chunk. This class handles the task of grabbing the desired region + of data across potentially multiple fractional and unaligned reads. + + :param offset: Byte offset into the file starting the region + :param length: The length of the region + """ + + def __init__(self, offset, length): + self.offset = offset + self.length = length + self.data = b'' + + @property + def complete(self): + """Returns True when we have captured the desired data.""" + return self.length == len(self.data) + + def capture(self, chunk, current_position): + """Process a chunk of data. + + This should be called for each chunk in the read loop, at least + until complete returns True. + + :param chunk: A chunk of bytes in the file + :param current_position: The position of the file processed by the + read loop so far. Note that this will be + the position in the file *after* the chunk + being presented. + """ + read_start = current_position - len(chunk) + if (read_start <= self.offset <= current_position + or self.offset <= read_start <= (self.offset + self.length)): + if read_start < self.offset: + lead_gap = self.offset - read_start + else: + lead_gap = 0 + self.data += chunk[lead_gap:] + self.data = self.data[:self.length] + + +class ImageFormatError(Exception): + """An unrecoverable image format error that aborts the process.""" + pass + + +class TraceDisabled(object): + """A logger-like thing that swallows tracing when we do not want it.""" + + def debug(self, *a, **k): + pass + + info = debug + warning = debug + error = debug + + +class FileInspector(object): + """A stream-based disk image inspector. + + This base class works on raw images and is subclassed for more + complex types. It is to be presented with the file to be examined + one chunk at a time, during read processing and will only store + as much data as necessary to determine required attributes of + the file. + """ + + def __init__(self, tracing=False): + self._total_count = 0 + + # NOTE(danms): The logging in here is extremely verbose for a reason, + # but should never really be enabled at that level at runtime. To + # retain all that work and assist in future debug, we have a separate + # debug flag that can be passed from a manual tool to turn it on. + if tracing: + self._log = logging.getLogger(str(self)) + else: + self._log = TraceDisabled() + self._capture_regions = {} + + def _capture(self, chunk, only=None): + for name, region in self._capture_regions.items(): + if only and name not in only: + continue + if not region.complete: + region.capture(chunk, self._total_count) + + def eat_chunk(self, chunk): + """Call this to present chunks of the file to the inspector.""" + pre_regions = set(self._capture_regions.keys()) + + # Increment our position-in-file counter + self._total_count += len(chunk) + + # Run through the regions we know of to see if they want this + # data + self._capture(chunk) + + # Let the format do some post-read processing of the stream + self.post_process() + + # Check to see if the post-read processing added new regions + # which may require the current chunk. + new_regions = set(self._capture_regions.keys()) - pre_regions + if new_regions: + self._capture(chunk, only=new_regions) + + def post_process(self): + """Post-read hook to process what has been read so far. + + This will be called after each chunk is read and potentially captured + by the defined regions. If any regions are defined by this call, + those regions will be presented with the current chunk in case it + is within one of the new regions. + """ + pass + + def region(self, name): + """Get a CaptureRegion by name.""" + return self._capture_regions[name] + + def new_region(self, name, region): + """Add a new CaptureRegion by name.""" + if self.has_region(name): + # This is a bug, we tried to add the same region twice + raise ImageFormatError('Inspector re-added region %s' % name) + self._capture_regions[name] = region + + def has_region(self, name): + """Returns True if named region has been defined.""" + return name in self._capture_regions + + @property + def format_match(self): + """Returns True if the file appears to be the expected format.""" + return True + + @property + def virtual_size(self): + """Returns the virtual size of the disk image, or zero if unknown.""" + return self._total_count + + @property + def actual_size(self): + """Returns the total size of the file, usually smaller than virtual_size. + + NOTE: this will only be accurate if the entire file is read and processed. + """ # noqa + return self._total_count + + @property + def complete(self): + """Returns True if we have all the information needed.""" + return all(r.complete for r in self._capture_regions.values()) + + def __str__(self): + """The string name of this file format.""" + return 'raw' + + @property + def context_info(self): + """Return info on amount of data held in memory for auditing. + + This is a dict of region:sizeinbytes items that the inspector + uses to examine the file. + """ + return {name: len(region.data) for name, region in + self._capture_regions.items()} + + @classmethod + def from_file(cls, filename): + """Read as much of a file as necessary to complete inspection. + + NOTE: Because we only read as much of the file as necessary, the + actual_size property will not reflect the size of the file, but the + amount of data we read before we satisfied the inspector. + + Raises ImageFormatError if we cannot parse the file. + """ + inspector = cls() + with open(filename, 'rb') as f: + for chunk in chunked_reader(f): + inspector.eat_chunk(chunk) + if inspector.complete: + # No need to eat any more data + break + if not inspector.complete or not inspector.format_match: + raise ImageFormatError('File is not in requested format') + return inspector + + def safety_check(self): + """Perform some checks to determine if this file is safe. + + Returns True if safe, False otherwise. It may raise ImageFormatError + if safety cannot be guaranteed because of parsing or other errors. + """ + return True + + +# The qcow2 format consists of a big-endian 72-byte header, of which +# only a small portion has information we care about: +# +# Dec Hex Name +# 0 0x00 Magic 4-bytes 'QFI\xfb' +# 4 0x04 Version (uint32_t, should always be 2 for modern files) +# . . . +# 8 0x08 Backing file offset (uint64_t) +# 24 0x18 Size in bytes (unint64_t) +# . . . +# 72 0x48 Incompatible features bitfield (6 bytes) +# +# https://gitlab.com/qemu-project/qemu/-/blob/master/docs/interop/qcow2.txt +class QcowInspector(FileInspector): + """QEMU QCOW2 Format + + This should only require about 32 bytes of the beginning of the file + to determine the virtual size, and 104 bytes to perform the safety check. + """ + + BF_OFFSET = 0x08 + BF_OFFSET_LEN = 8 + I_FEATURES = 0x48 + I_FEATURES_LEN = 8 + I_FEATURES_DATAFILE_BIT = 3 + I_FEATURES_MAX_BIT = 4 + + def __init__(self, *a, **k): + super(QcowInspector, self).__init__(*a, **k) + self.new_region('header', CaptureRegion(0, 512)) + + def _qcow_header_data(self): + magic, version, bf_offset, bf_sz, cluster_bits, size = ( + struct.unpack('>4sIQIIQ', self.region('header').data[:32])) + return magic, size + + @property + def has_header(self): + return self.region('header').complete + + @property + def virtual_size(self): + if not self.region('header').complete: + return 0 + if not self.format_match: + return 0 + magic, size = self._qcow_header_data() + return size + + @property + def format_match(self): + if not self.region('header').complete: + return False + magic, size = self._qcow_header_data() + return magic == b'QFI\xFB' + + @property + def has_backing_file(self): + if not self.region('header').complete: + return None + if not self.format_match: + return False + bf_offset_bytes = self.region('header').data[ + self.BF_OFFSET:self.BF_OFFSET + self.BF_OFFSET_LEN] + # nonzero means "has a backing file" + bf_offset, = struct.unpack('>Q', bf_offset_bytes) + return bf_offset != 0 + + @property + def has_unknown_features(self): + if not self.region('header').complete: + return None + if not self.format_match: + return False + i_features = self.region('header').data[ + self.I_FEATURES:self.I_FEATURES + self.I_FEATURES_LEN] + + # This is the maximum byte number we should expect any bits to be set + max_byte = self.I_FEATURES_MAX_BIT // 8 + + # The flag bytes are in big-endian ordering, so if we process + # them in index-order, they're reversed + for i, byte_num in enumerate(reversed(range(self.I_FEATURES_LEN))): + if byte_num == max_byte: + # If we're in the max-allowed byte, allow any bits less than + # the maximum-known feature flag bit to be set + allow_mask = ((1 << self.I_FEATURES_MAX_BIT) - 1) + elif byte_num > max_byte: + # If we're above the byte with the maximum known feature flag + # bit, then we expect all zeroes + allow_mask = 0x0 + else: + # Any earlier-than-the-maximum byte can have any of the flag + # bits set + allow_mask = 0xFF + + if i_features[i] & ~allow_mask: + LOG.warning('Found unknown feature bit in byte %i: %s/%s', + byte_num, bin(i_features[byte_num] & ~allow_mask), + bin(allow_mask)) + return True + + return False + + @property + def has_data_file(self): + if not self.region('header').complete: + return None + if not self.format_match: + return False + i_features = self.region('header').data[ + self.I_FEATURES:self.I_FEATURES + self.I_FEATURES_LEN] + + # First byte of bitfield, which is i_features[7] + byte = self.I_FEATURES_LEN - 1 - self.I_FEATURES_DATAFILE_BIT // 8 + # Third bit of bitfield, which is 0x04 + bit = 1 << (self.I_FEATURES_DATAFILE_BIT - 1 % 8) + return bool(i_features[byte] & bit) + + def __str__(self): + return 'qcow2' + + def safety_check(self): + return (not self.has_backing_file + and not self.has_data_file + and not self.has_unknown_features) + + +class QEDInspector(FileInspector): + def __init__(self, tracing=False): + super().__init__(tracing) + self.new_region('header', CaptureRegion(0, 512)) + + @property + def format_match(self): + if not self.region('header').complete: + return False + return self.region('header').data.startswith(b'QED\x00') + + def safety_check(self): + # QED format is not supported by anyone, but we want to detect it + # and mark it as just always unsafe. + return False + + +# The VHD (or VPC as QEMU calls it) format consists of a big-endian +# 512-byte "footer" at the beginning of the file with various +# information, most of which does not matter to us: +# +# Dec Hex Name +# 0 0x00 Magic string (8-bytes, always 'conectix') +# 40 0x28 Disk size (uint64_t) +# +# https://github.com/qemu/qemu/blob/master/block/vpc.c +class VHDInspector(FileInspector): + """Connectix/MS VPC VHD Format + + This should only require about 512 bytes of the beginning of the file + to determine the virtual size. + """ + + def __init__(self, *a, **k): + super(VHDInspector, self).__init__(*a, **k) + self.new_region('header', CaptureRegion(0, 512)) + + @property + def format_match(self): + return self.region('header').data.startswith(b'conectix') + + @property + def virtual_size(self): + if not self.region('header').complete: + return 0 + + if not self.format_match: + return 0 + + return struct.unpack('>Q', self.region('header').data[40:48])[0] + + def __str__(self): + return 'vhd' + + +# The VHDX format consists of a complex dynamic little-endian +# structure with multiple regions of metadata and data, linked by +# offsets with in the file (and within regions), identified by MSFT +# GUID strings. The header is a 320KiB structure, only a few pieces of +# which we actually need to capture and interpret: +# +# Dec Hex Name +# 0 0x00000 Identity (Technically 9-bytes, padded to 64KiB, the first +# 8 bytes of which are 'vhdxfile') +# 196608 0x30000 The Region table (64KiB of a 32-byte header, followed +# by up to 2047 36-byte region table entry structures) +# +# The region table header includes two items we need to read and parse, +# which are: +# +# 196608 0x30000 4-byte signature ('regi') +# 196616 0x30008 Entry count (uint32-t) +# +# The region table entries follow the region table header immediately +# and are identified by a 16-byte GUID, and provide an offset of the +# start of that region. We care about the "metadata region", identified +# by the METAREGION class variable. The region table entry is (offsets +# from the beginning of the entry, since it could be in multiple places): +# +# 0 0x00000 16-byte MSFT GUID +# 16 0x00010 Offset of the actual metadata region (uint64_t) +# +# When we find the METAREGION table entry, we need to grab that offset +# and start examining the region structure at that point. That +# consists of a metadata table of structures, which point to places in +# the data in an unstructured space that follows. The header is +# (offsets relative to the region start): +# +# 0 0x00000 8-byte signature ('metadata') +# . . . +# 16 0x00010 2-byte entry count (up to 2047 entries max) +# +# This header is followed by the specified number of metadata entry +# structures, identified by GUID: +# +# 0 0x00000 16-byte MSFT GUID +# 16 0x00010 4-byte offset (uint32_t, relative to the beginning of +# the metadata region) +# +# We need to find the "Virtual Disk Size" metadata item, identified by +# the GUID in the VIRTUAL_DISK_SIZE class variable, grab the offset, +# add it to the offset of the metadata region, and examine that 8-byte +# chunk of data that follows. +# +# The "Virtual Disk Size" is a naked uint64_t which contains the size +# of the virtual disk, and is our ultimate target here. +# +# https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-vhdx/83e061f8-f6e2-4de1-91bd-5d518a43d477 +class VHDXInspector(FileInspector): + """MS VHDX Format + + This requires some complex parsing of the stream. The first 256KiB + of the image is stored to get the header and region information, + and then we capture the first metadata region to read those + records, find the location of the virtual size data and parse + it. This needs to store the metadata table entries up until the + VDS record, which may consist of up to 2047 32-byte entries at + max. Finally, it must store a chunk of data at the offset of the + actual VDS uint64. + + """ + METAREGION = '8B7CA206-4790-4B9A-B8FE-575F050F886E' + VIRTUAL_DISK_SIZE = '2FA54224-CD1B-4876-B211-5DBED83BF4B8' + VHDX_METADATA_TABLE_MAX_SIZE = 32 * 2048 # From qemu + + def __init__(self, *a, **k): + super(VHDXInspector, self).__init__(*a, **k) + self.new_region('ident', CaptureRegion(0, 32)) + self.new_region('header', CaptureRegion(192 * 1024, 64 * 1024)) + + def post_process(self): + # After reading a chunk, we may have the following conditions: + # + # 1. We may have just completed the header region, and if so, + # we need to immediately read and calculate the location of + # the metadata region, as it may be starting in the same + # read we just did. + # 2. We may have just completed the metadata region, and if so, + # we need to immediately calculate the location of the + # "virtual disk size" record, as it may be starting in the + # same read we just did. + if self.region('header').complete and not self.has_region('metadata'): + region = self._find_meta_region() + if region: + self.new_region('metadata', region) + elif self.has_region('metadata') and not self.has_region('vds'): + region = self._find_meta_entry(self.VIRTUAL_DISK_SIZE) + if region: + self.new_region('vds', region) + + @property + def format_match(self): + return self.region('ident').data.startswith(b'vhdxfile') + + @staticmethod + def _guid(buf): + """Format a MSFT GUID from the 16-byte input buffer.""" + guid_format = '= 2048: + raise ImageFormatError('Region count is %i (limit 2047)' % count) + + # Process the regions until we find the metadata one; grab the + # offset and return + self._log.debug('Region entry first is %x', region_entry_first) + self._log.debug('Region entries %i', count) + meta_offset = 0 + for i in range(0, count): + entry_start = region_entry_first + (i * 32) + entry_end = entry_start + 32 + entry = self.region('header').data[entry_start:entry_end] + self._log.debug('Entry offset is %x', entry_start) + + # GUID is the first 16 bytes + guid = self._guid(entry[:16]) + if guid == self.METAREGION: + # This entry is the metadata region entry + meta_offset, meta_len, meta_req = struct.unpack( + '= 2048: + raise ImageFormatError( + 'Metadata item count is %i (limit 2047)' % count) + + for i in range(0, count): + entry_offset = 32 + (i * 32) + guid = self._guid(meta_buffer[entry_offset:entry_offset + 16]) + if guid == desired_guid: + # Found the item we are looking for by id. + # Stop our region from capturing + item_offset, item_length, _reserved = struct.unpack( + ' 1: + all_formats = [str(inspector) for inspector in detections] + raise ImageFormatError( + 'Multiple formats detected: %s' % ', '.join(all_formats)) + + return inspectors['raw'] if not detections else detections[0] diff -Nru ironic-21.1.0/ironic/common/image_service.py ironic-21.4.4/ironic/common/image_service.py --- ironic-21.1.0/ironic/common/image_service.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/image_service.py 2024-10-11 15:42:16.000000000 +0000 @@ -34,10 +34,6 @@ from ironic.conf import CONF IMAGE_CHUNK_SIZE = 1024 * 1024 # 1mb -# NOTE(kaifeng) Image will be truncated to 2GiB by sendfile, -# we use a large chunk size here for a better performance -# while keep the chunk size less than the size limit. -SENDFILE_CHUNK_SIZE = 1024 * 1024 * 1024 # 1Gb LOG = log.getLogger(__name__) @@ -233,6 +229,41 @@ 'no_cache': no_cache, } + @staticmethod + def get(image_href): + """Downloads content and returns the response text. + + :param image_href: Image reference. + :raises: exception.ImageRefValidationFailed if GET request returned + response code not equal to 200. + :raises: exception.ImageDownloadFailed if: + * IOError happened during file write; + * GET request failed. + """ + + try: + + verify = strutils.bool_from_string(CONF.webserver_verify_ca, + strict=True) + except ValueError: + verify = CONF.webserver_verify_ca + + try: + response = requests.get(image_href, stream=False, verify=verify, + timeout=CONF.webserver_connection_timeout) + if response.status_code != http_client.OK: + raise exception.ImageRefValidationFailed( + image_href=image_href, + reason=_("Got HTTP code %s instead of 200 in response " + "to GET request.") % response.status_code) + + return response.text + + except (OSError, requests.ConnectionError, requests.RequestException, + IOError) as e: + raise exception.ImageDownloadFailed(image_href=image_href, + reason=str(e)) + class FileImageService(BaseImageService): """Provides retrieval of disk images available locally on the conductor.""" @@ -264,26 +295,31 @@ """ source_image_path = self.validate_href(image_href) dest_image_path = image_file.name - local_device = os.stat(dest_image_path).st_dev try: - # We should have read and write access to source file to create - # hard link to it. - if (local_device == os.stat(source_image_path).st_dev - and os.access(source_image_path, os.R_OK | os.W_OK)): - image_file.close() - os.remove(dest_image_path) - os.link(source_image_path, dest_image_path) + image_file.close() + os.remove(dest_image_path) + + # NOTE(dtantsur): os.link is supposed to follow symlinks, but it + # does not: https://github.com/python/cpython/issues/81793 + real_image_path = os.path.realpath(source_image_path) + try: + os.link(real_image_path, dest_image_path) + except OSError as exc: + orig = (f' (real path {real_image_path})' + if real_image_path != source_image_path + else '') + + LOG.debug('Could not create a link from %(src)s%(orig)s to ' + '%(dest)s, will copy the content instead. ' + 'Error: %(exc)s.', + {'src': source_image_path, 'dest': dest_image_path, + 'orig': orig, 'exc': exc}) else: - filesize = os.path.getsize(source_image_path) - offset = 0 - with open(source_image_path, 'rb') as input_img: - while offset < filesize: - count = min(SENDFILE_CHUNK_SIZE, filesize - offset) - nbytes_out = os.sendfile(image_file.fileno(), - input_img.fileno(), - offset, - count) - offset += nbytes_out + return + + # NOTE(dtantsur): starting with Python 3.8, copyfile() uses + # efficient copying (i.e. sendfile) under the hood. + shutil.copyfile(source_image_path, dest_image_path) except Exception as e: raise exception.ImageDownloadFailed(image_href=image_href, reason=str(e)) diff -Nru ironic-21.1.0/ironic/common/images.py ironic-21.4.4/ironic/common/images.py --- ironic-21.1.0/ironic/common/images.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/images.py 2024-10-11 15:42:16.000000000 +0000 @@ -23,16 +23,18 @@ import shutil import time -from ironic_lib import disk_utils from oslo_concurrency import processutils from oslo_log import log as logging from oslo_utils import fileutils import pycdlib +from ironic.common import checksum_utils from ironic.common import exception from ironic.common.glance_service import service_utils as glance_utils from ironic.common.i18n import _ +from ironic.common import image_format_inspector from ironic.common import image_service as service +from ironic.common import qemu_img from ironic.common import utils from ironic.conf import CONF @@ -175,7 +177,8 @@ def create_isolinux_image_for_bios( - output_file, kernel, ramdisk, kernel_params=None, inject_files=None): + output_file, kernel, ramdisk, kernel_params=None, inject_files=None, + publisher_id=None): """Creates an isolinux image on the specified file. Copies the provided kernel, ramdisk to a directory, generates the isolinux @@ -191,6 +194,8 @@ as the kernel cmdline. :param inject_files: Mapping of local source file paths to their location on the final ISO image. + :param publisher_id: A value to set as the publisher identifier string + in the ISO image to be generated. :raises: ImageCreationFailed, if image creation failed while copying files or while running command to generate iso. """ @@ -237,9 +242,12 @@ isolinux_cfg = os.path.join(tmpdir, ISOLINUX_CFG) utils.write_to_file(isolinux_cfg, cfg) + # Set a publisher ID value to a string. + pub_id = str(publisher_id) + try: utils.execute('mkisofs', '-r', '-V', _label(files_info), - '-J', '-l', '-no-emul-boot', + '-J', '-l', '-publisher', pub_id, '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '-b', ISOLINUX_BIN, '-o', output_file, tmpdir) except processutils.ProcessExecutionError as e: @@ -249,7 +257,7 @@ def create_esp_image_for_uefi( output_file, kernel, ramdisk, deploy_iso=None, esp_image=None, - kernel_params=None, inject_files=None): + kernel_params=None, inject_files=None, publisher_id=None): """Creates an ESP image on the specified file. Copies the provided kernel, ramdisk and EFI system partition image (ESP) to @@ -271,6 +279,8 @@ as the kernel cmdline. :param inject_files: Mapping of local source file paths to their location on the final ISO image. + :param publisher_id: A value to set as the publisher identifier string + in the ISO image to be generated. :raises: ImageCreationFailed, if image creation failed while copying files or while running command to generate iso. """ @@ -337,10 +347,18 @@ utils.write_to_file(grub_cfg, grub_conf) # Create the boot_iso. + if publisher_id: + args = ('mkisofs', '-r', '-V', _label(files_info), + '-l', '-publisher', publisher_id, '-e', e_img_rel_path, + '-no-emul-boot', '-o', output_file, + tmpdir) + else: + args = ('mkisofs', '-r', '-V', _label(files_info), + '-l', '-e', e_img_rel_path, + '-no-emul-boot', '-o', output_file, + tmpdir) try: - utils.execute('mkisofs', '-r', '-V', _label(files_info), - '-l', '-e', e_img_rel_path, '-no-emul-boot', - '-o', output_file, tmpdir) + utils.execute(*args) except processutils.ProcessExecutionError as e: LOG.exception("Creating ISO image failed.") @@ -369,31 +387,25 @@ {'image_href': image_href, 'time': time.time() - start}) -def fetch(context, image_href, path, force_raw=False): +def fetch(context, image_href, path, force_raw=False, + checksum=None, checksum_algo=None): with fileutils.remove_path_on_error(path): fetch_into(context, image_href, path) - + if (not CONF.conductor.disable_file_checksum + and checksum): + checksum_utils.validate_checksum(path, checksum, checksum_algo) if force_raw: image_to_raw(image_href, path, "%s.part" % path) def get_source_format(image_href, path): - data = disk_utils.qemu_img_info(path) - - fmt = data.file_format - if fmt is None: + try: + img_format = image_format_inspector.detect_file_format(path) + except image_format_inspector.ImageFormatError: raise exception.ImageUnacceptable( - reason=_("'qemu-img info' parsing failed."), + reason=_("parsing of the image failed."), image_id=image_href) - - backing_file = data.backing_file - if backing_file is not None: - raise exception.ImageUnacceptable( - image_id=image_href, - reason=_("fmt=%(fmt)s backed by: %(backing_file)s") % - {'fmt': fmt, 'backing_file': backing_file}) - - return fmt + return str(img_format) def force_raw_will_convert(image_href, path_tmp): @@ -406,24 +418,46 @@ def image_to_raw(image_href, path, path_tmp): with fileutils.remove_path_on_error(path_tmp): - fmt = get_source_format(image_href, path_tmp) + if not CONF.conductor.disable_deep_image_inspection: + fmt = safety_check_image(path_tmp) - if fmt != "raw": + if fmt not in CONF.conductor.permitted_image_formats: + LOG.error("Security: The requested image %(image_href)s " + "is of format image %(format)s and is not in " + "the [conductor]permitted_image_formats list.", + {'image_href': image_href, + 'format': fmt}) + raise exception.InvalidImage() + else: + fmt = get_source_format(image_href, path) + LOG.warning("Security: Image safety checking has been disabled. " + "This is unsafe operation. Attempting to continue " + "the detected format %(img_fmt)s for %(path)s.", + {'img_fmt': fmt, + 'path': path}) + + if fmt != "raw" and fmt != "iso": + # When the target format is NOT raw, we need to convert it. + # however, we don't need nor want to do that when we have + # an ISO image. If we have an ISO because it was requested, + # we have correctly fingerprinted it. Prior to proper + # image detection, we thought we had a raw image, and we + # would end up asking for a raw image to be made a raw image. staged = "%s.converted" % path utils.is_memory_insufficient(raise_if_fail=True) LOG.debug("%(image)s was %(format)s, converting to raw", {'image': image_href, 'format': fmt}) with fileutils.remove_path_on_error(staged): - disk_utils.convert_image(path_tmp, staged, 'raw') + qemu_img.convert_image(path_tmp, staged, 'raw', + source_format=fmt) os.unlink(path_tmp) - - data = disk_utils.qemu_img_info(staged) - if data.file_format != "raw": + new_fmt = get_source_format(image_href, staged) + if new_fmt != "raw": raise exception.ImageConvertFailed( image_id=image_href, reason=_("Converted to raw, but format is " - "now %s") % data.file_format) + "now %s") % new_fmt) os.rename(staged, path) else: @@ -454,11 +488,11 @@ the original image scaled by the configuration value `raw_image_growth_factor`. """ - data = disk_utils.qemu_img_info(path) + data = image_format_inspector.detect_file_format(path) if not estimate: return data.virtual_size growth_factor = CONF.raw_image_growth_factor - return int(min(data.disk_size * growth_factor, data.virtual_size)) + return int(min(data.actual_size * growth_factor, data.virtual_size)) def get_image_properties(context, image_href, properties="all"): @@ -498,7 +532,7 @@ def create_boot_iso(context, output_filename, kernel_href, ramdisk_href, deploy_iso_href=None, esp_image_href=None, root_uuid=None, kernel_params=None, boot_mode=None, - inject_files=None): + inject_files=None, publisher_id=None): """Creates a bootable ISO image for a node. Given the hrefs for kernel, ramdisk, root partition's UUID and @@ -524,6 +558,8 @@ :boot_mode: the boot mode in which the deploy is to happen. :param inject_files: Mapping of local source file paths to their location on the final ISO image. + :param publisher_id: A value to set as the publisher identifier string + in the ISO image to be generated. :raises: ImageCreationFailed, if creating boot ISO failed. """ with utils.tempdir() as tmpdir: @@ -560,12 +596,14 @@ create_esp_image_for_uefi( output_filename, kernel_path, ramdisk_path, deploy_iso=deploy_iso_path, esp_image=esp_image_path, - kernel_params=params, inject_files=inject_files) + kernel_params=params, inject_files=inject_files, + publisher_id=publisher_id) else: create_isolinux_image_for_bios( output_filename, kernel_path, ramdisk_path, - kernel_params=params, inject_files=inject_files) + kernel_params=params, inject_files=inject_files, + publisher_id=publisher_id) IMAGE_TYPE_PARTITION = 'partition' @@ -657,10 +695,24 @@ # NOTE(TheJulia): I don't really like this pattern, *but* # the wholedisk image support is similar. return + # NOTE(TheJulia): Files should have been caught almost exclusively + # before with the Content-Length check. + # When the ISO is mounted and the webserver mount point url is + # checked here, it has both 'Content-Length' and 'Content-Type' + # due to which it always returns False. Hence switched the conditions. + if ('Content-Type' in headers + and str(headers['Content-Type']).startswith('text/html')): + LOG.debug('Evaluated %(url)s to determine if it is a URL to a path ' + 'or a file. A Content-Type header was returned with a text ' + 'content, which suggests a file list was returned.', + {'url': image_source}) + return True # When issuing a head request, folders have no length # A list can be generated by the server.. This is a solid # hint. - if 'Content-Length' in headers: + if ('Content-Type' in headers + and (str(headers['Content-Type']) != 'text/html') + and 'Content-Length' in headers): LOG.debug('Evaluated %(url)s to determine if it is a URL to a path ' 'or a file. A Content-Length header was returned ' 'suggesting file.', @@ -668,16 +720,6 @@ # NOTE(TheJulia): Files on a webserver have a length which is returned # when headres are queried. return False - if ('Content-Type' in headers - and str(headers['Content-Type']).startswith('text') - and 'Content-Length' not in headers): - LOG.debug('Evaluated %(url)s to determine if it is a URL to a path ' - 'or a file. A Content-Type header was returned with a text ' - 'content, which suggests a file list was returned.', - {'url': image_source}) - return True - # NOTE(TheJulia): Files should have been caught almost exclusively - # before with the Content-Length check. if image_source.endswith('/'): # If all else fails, looks like a URL, and the server didn't give # us any hints. @@ -763,3 +805,92 @@ # present in deploy iso. This path varies for different OS vendors. # e_img_rel_path: is required by mkisofs to generate boot iso. return uefi_path_info, e_img_rel_path, grub_rel_path + + +def __node_or_image_cache(node): + """A helper for logging to determine if image cache or node uuid.""" + if not node: + return 'image cache' + else: + return node.uuid + + +def safety_check_image(image_path, node=None): + """Performs a safety check on the supplied image. + + This method triggers the image format inspector's to both identify the + type of the supplied file and safety check logic to identify if there + are any known unsafe features being leveraged, and return the detected + file format in the form of a string for the caller. + + :param image_path: A fully qualified path to an image which needs to + be evaluated for safety. + :param node: A Node object, optional. When supplied logging indicates the + node which triggered this issue, but the node is not + available in all invocation cases. + :returns: a string representing the the image type which is used. + :raises: InvalidImage when the supplied image is detected as unsafe, + or the image format inspector has failed to parse the supplied + image's contents. + """ + id_string = __node_or_image_cache(node) + try: + img_class = image_format_inspector.detect_file_format(image_path) + if not img_class.safety_check(): + LOG.error("Security: The requested image for " + "deployment of node %(node)s fails safety sanity " + "checking.", + {'node': id_string}) + raise exception.InvalidImage() + image_format_name = str(img_class) + except image_format_inspector.ImageFormatError: + LOG.error("Security: The requested user image for the " + "deployment node %(node)s failed to be able " + "to be parsed by the image format checker.", + {'node': id_string}) + raise exception.InvalidImage() + return image_format_name + + +def check_if_image_format_is_permitted(img_format, + expected_format=None, + node=None): + """Checks image format consistency. + + :params img_format: The determined image format by name. + :params expected_format: Optional, the expected format based upon + supplied configuration values. + :params node: A node object or None implying image cache. + :raises: InvalidImage if the requested image format is not permitted + by configuration, or the expected_format does not match the + determined format. + """ + + id_string = __node_or_image_cache(node) + if img_format not in CONF.conductor.permitted_image_formats: + LOG.error("Security: The requested deploy image for node %(node)s " + "is of format image %(format)s and is not in the " + "[conductor]permitted_image_formats list.", + {'node': id_string, + 'format': img_format}) + raise exception.InvalidImage() + if expected_format is not None and img_format != expected_format: + if expected_format in ['ari', 'aki']: + # In this case, we have an ari or aki, meaning we're pulling + # down a kernel/ramdisk, and this is rooted in a misunderstanding. + # They should be raw. The detector should be detecting this *as* + # raw anyway, so the data just mismatches from a common + # misunderstanding, and that is okay in this case as they are not + # passed to qemu-img. + # TODO(TheJulia): Add a log entry to warn here at some point in + # the future as we begin to shift the perception around this. + # See: https://bugs.launchpad.net/ironic/+bug/2074090 + return + LOG.error("Security: The requested deploy image for node %(node)s " + "has a format (%(format)s) which does not match the " + "expected image format (%(expected)s) based upon " + "supplied or retrieved information.", + {'node': id_string, + 'format': img_format, + 'expected': expected_format}) + raise exception.InvalidImage() diff -Nru ironic-21.1.0/ironic/common/keystone.py ironic-21.4.4/ironic/common/keystone.py --- ironic-21.1.0/ironic/common/keystone.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/keystone.py 2024-10-11 15:42:16.000000000 +0000 @@ -125,7 +125,8 @@ return result -def get_service_auth(context, endpoint, service_auth): +def get_service_auth(context, endpoint, service_auth, + only_service_auth=False): """Create auth plugin wrapping both user and service auth. When properly configured and using auth_token middleware, @@ -134,8 +135,25 @@ Ideally we would use the plugin provided by auth_token middleware however this plugin isn't serialized yet. + + :param context: The RequestContext instance from which the user + auth_token is extracted. + :param endpoint: The requested endpoint to be utilized. + :param service_auth: The service authenticaiton credentals to be + used. + :param only_service_auth: Boolean, default False. When set to True, + the resulting Service token pair is generated + as if it originates from the user itself. + Useful to cast admin level operations which are + launched by Ironic itself, as opposed to user + initiated requests. + :returns: Returns a service token via the ServiceTokenAuthWrapper + class. """ - # TODO(pas-ha) use auth plugin from context when it is available - user_auth = token_endpoint.Token(endpoint, context.auth_token) + user_auth = None + if not only_service_auth: + user_auth = token_endpoint.Token(endpoint, context.auth_token) + else: + user_auth = service_auth return service_token.ServiceTokenAuthWrapper(user_auth=user_auth, service_auth=service_auth) diff -Nru ironic-21.1.0/ironic/common/neutron.py ironic-21.4.4/ironic/common/neutron.py --- ironic-21.1.0/ironic/common/neutron.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/neutron.py 2024-10-11 15:42:16.000000000 +0000 @@ -70,7 +70,11 @@ user_auth = None if (not auth_from_config and CONF.neutron.auth_type != 'none' - and context.auth_token): + and context.auth_token and not context.system_scope): + # If we have a token, we *should* use the user's auth, however we + # can only do so *if* it is a project scoped request. If it is + # system scoped, we cannot leverage user auth data to make the next + # request. user_auth = keystone.get_service_auth(context, endpoint, service_auth) sess = keystone.get_session('neutron', timeout=CONF.neutron.timeout, diff -Nru ironic-21.1.0/ironic/common/policy.py ironic-21.4.4/ironic/common/policy.py --- ironic-21.1.0/ironic/common/policy.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/policy.py 2024-10-11 15:42:16.000000000 +0000 @@ -49,15 +49,22 @@ # authorization that system administrators typically have. This persona, or # check string, typically isn't used by default, but it's existence it useful # in the event a deployment wants to offload some administrative action from -# system administrator to system members -SYSTEM_MEMBER = 'role:member and system_scope:all' - -# Generic policy check string for read-only access to system-level resources. -# This persona is useful for someone who needs access for auditing or even -# support. These uses are also able to view project-specific resources where -# applicable (e.g., listing all volumes in the deployment, regardless of the -# project they belong to). -SYSTEM_READER = 'role:reader and system_scope:all' +# system administrator to system members. +# The rule:service_role match here is to enable an elevated level of API +# access for a specialized service role and users with appropriate +# service role access. +SYSTEM_MEMBER = '(role:member and system_scope:all) or rule:service_role' # noqa + +# Generic policy check string for read-only access to system-level +# resources. This persona is useful for someone who needs access +# for auditing or even support. These uses are also able to view +# project-specific resources where applicable (e.g., listing all +# volumes in the deployment, regardless of the project they belong to). +# The rule:service_role match here is to enable an elevated level of API +# access for a specialized service role and users with appropriate +# role access, specifically because 'service" role is outside of the RBAC +# model defaults and does not imply reader access. +SYSTEM_READER = '(role:reader and system_scope:all) or (role:service and system_scope:all) or rule:service_role' # noqa # This check string is reserved for actions that require the highest level of # authorization on a project or resources within the project (e.g., setting the @@ -83,15 +90,23 @@ PROJECT_READER = ('role:reader and ' '(project_id:%(node.owner)s or project_id:%(node.lessee)s)') +# This check string is used for granting access to other services which need +# to communicate with Ironic, for example, Nova-Compute to provision nodes, +# or Ironic-Inspector to create nodes. The idea behind a service role is +# one which has restricted access to perform operations, that are limited +# to remote automated and inter-operation processes. +SYSTEM_SERVICE = ('role:service and system_scope:all') +PROJECT_SERVICE = ('role:service and project_id:%(node.owner)s') + # The following are common composite check strings that are useful for # protecting APIs designed to operate with multiple scopes (e.g., a system # administrator should be able to delete any baremetal host in the deployment, # a project member should only be able to delete hosts in their project). SYSTEM_OR_PROJECT_MEMBER = ( - '(' + SYSTEM_MEMBER + ') or (' + PROJECT_MEMBER + ')' + '(' + SYSTEM_MEMBER + ') or (' + PROJECT_MEMBER + ') or (' + SYSTEM_SERVICE + ')' # noqa ) SYSTEM_OR_PROJECT_READER = ( - '(' + SYSTEM_READER + ') or (' + PROJECT_READER + ')' + '(' + SYSTEM_READER + ') or (' + PROJECT_READER + ') or (' + PROJECT_SERVICE + ')' # noqa ) PROJECT_OWNER_ADMIN = ('role:admin and project_id:%(node.owner)s') @@ -109,28 +124,36 @@ ALLOCATION_OWNER_MEMBER = ('role:member and project_id:%(allocation.owner)s') ALLOCATION_OWNER_READER = ('role:reader and project_id:%(allocation.owner)s') +# Used for general operations like changing provision state. SYSTEM_OR_OWNER_MEMBER_AND_LESSEE_ADMIN = ( - '(' + SYSTEM_MEMBER + ') or (' + PROJECT_OWNER_MEMBER + ') or (' + PROJECT_LESSEE_ADMIN + ') or (' + PROJECT_LESSEE_MANAGER + ')' # noqa + '(' + SYSTEM_MEMBER + ') or (' + SYSTEM_SERVICE + ') or (' + PROJECT_OWNER_MEMBER + ') or (' + PROJECT_LESSEE_ADMIN + ') or (' + PROJECT_LESSEE_MANAGER + ') or (' + PROJECT_SERVICE + ')' # noqa ) +# Used for creation and deletion of network ports. SYSTEM_ADMIN_OR_OWNER_ADMIN = ( - '(' + SYSTEM_ADMIN + ') or (' + PROJECT_OWNER_ADMIN + ') or (' + PROJECT_OWNER_MANAGER + ')' # noqa + '(' + SYSTEM_ADMIN + ') or (' + SYSTEM_SERVICE + ') or (' + PROJECT_OWNER_ADMIN + ') or (' + PROJECT_OWNER_MANAGER + ') or (' + PROJECT_SERVICE + ')' # noqa ) +# Used to map system members, and owner admins to the same access rights. +# This is actions such as update driver interfaces, delete ports. SYSTEM_MEMBER_OR_OWNER_ADMIN = ( - '(' + SYSTEM_MEMBER + ') or (' + PROJECT_OWNER_ADMIN + ') or (' + PROJECT_OWNER_MANAGER + ')' # noqa + '(' + SYSTEM_MEMBER + ') or (' + SYSTEM_SERVICE + ') or (' + PROJECT_OWNER_ADMIN + ') or (' + PROJECT_OWNER_MANAGER + ') or (' + PROJECT_SERVICE + ')' # noqa ) +# Used to map "member" only rights, i.e. those of "users using a deployment" SYSTEM_MEMBER_OR_OWNER_MEMBER = ( - '(' + SYSTEM_MEMBER + ') or (' + PROJECT_OWNER_MEMBER + ')' + '(' + SYSTEM_MEMBER + ') or (' + SYSTEM_SERVICE + ') or (' + PROJECT_OWNER_MEMBER + ') or (' + PROJECT_SERVICE + ')' # noqa ) +# Used throughout to map where authenticated readers +# should be able to read API objects. SYSTEM_OR_OWNER_READER = ( - '(' + SYSTEM_READER + ') or (' + PROJECT_OWNER_READER + ')' + '(' + SYSTEM_READER + ') or (' + SYSTEM_SERVICE + ') or (' + PROJECT_OWNER_READER + ') or (' + PROJECT_SERVICE + ')' # noqa ) +# Mainly used for targets/connectors SYSTEM_MEMBER_OR_OWNER_LESSEE_ADMIN = ( - '(' + SYSTEM_MEMBER + ') or (' + PROJECT_OWNER_ADMIN + ') or (' + PROJECT_OWNER_MANAGER + ') or (' + PROJECT_LESSEE_ADMIN + ') or (' + PROJECT_LESSEE_MANAGER + ')' # noqa + '(' + SYSTEM_MEMBER + ') or (' + SYSTEM_SERVICE + ') or (' + PROJECT_OWNER_ADMIN + ') or (' + PROJECT_OWNER_MANAGER + ') or (' + PROJECT_LESSEE_ADMIN + ') or (' + PROJECT_LESSEE_MANAGER + ') or (' + PROJECT_SERVICE + ')' # noqa ) @@ -152,7 +175,10 @@ # Special purpose aliases for things like "ability to access the API # as a reader, or permission checking that does not require node # owner relationship checking -API_READER = ('role:reader') +API_READER = ('(role:reader) or (role:service)') + +# Used for ability to view target properties of a volume, which is +# considered highly restricted. TARGET_PROPERTIES_READER = ( '(' + SYSTEM_READER + ') or (role:admin)' ) @@ -189,6 +215,13 @@ policy.RuleDefault('show_instance_secrets', '!', description='Show or mask secrets within instance information in API responses'), # noqa + # NOTE(TheJulia): This is a special rule to allow customization of the + # service role check. The config.service_project_name is a reserved + # target check field which is loaded from configuration to the + # check context in ironic/common/context.py. + policy.RuleDefault('service_role', + 'role:service and project_name:%(config.service_project_name)s', # noqa + description='Rule to match service role usage with a service project, delineated as a separate rule to enable customization.'), # noqa # Roles likely to be overridden by operator # TODO(TheJulia): Lets nuke demo from high orbit. policy.RuleDefault('is_member', @@ -436,7 +469,7 @@ node_policies = [ policy.DocumentedRuleDefault( name='baremetal:node:create', - check_str=SYSTEM_ADMIN, + check_str='(' + SYSTEM_ADMIN + ') or (' + SYSTEM_SERVICE + ')', scope_types=['system', 'project'], description='Create Node records', operations=[{'path': '/nodes', 'method': 'POST'}], @@ -444,8 +477,8 @@ ), policy.DocumentedRuleDefault( name='baremetal:node:create:self_owned_node', - check_str=('role:admin'), - scope_types=['project'], + check_str=('(role:admin) or (role:service)'), + scope_types=['system', 'project'], description='Create node records which will be tracked ' 'as owned by the associated user project.', operations=[{'path': '/nodes', 'method': 'POST'}], @@ -463,7 +496,7 @@ policy.DocumentedRuleDefault( name='baremetal:node:list_all', check_str=SYSTEM_READER, - scope_types=['system'], + scope_types=['system', 'project'], description='Retrieve multiple Node records', operations=[{'path': '/nodes', 'method': 'GET'}, {'path': '/nodes/detail', 'method': 'GET'}], @@ -674,7 +707,7 @@ policy.DocumentedRuleDefault( name='baremetal:node:delete:self_owned_node', check_str=PROJECT_ADMIN, - scope_types=['project'], + scope_types=['system', 'project'], description='Delete node records which are associated with ' 'the requesting project.', operations=[{'path': '/nodes/{node_ident}', 'method': 'DELETE'}], @@ -954,8 +987,34 @@ # operating context. deprecated_rule=deprecated_node_get ), - - + policy.DocumentedRuleDefault( + name='baremetal:node:inventory:get', + check_str=SYSTEM_OR_OWNER_READER, + scope_types=['system', 'project'], + description='Retrieve introspection data for a node.', + operations=[ + {'path': '/nodes/{node_ident}/inventory', 'method': 'GET'}, + ], + # This rule fallsback to deprecated_node_get in order to provide a + # mechanism so the additional policies only engage in an updated + # operating context. + deprecated_rule=deprecated_node_get + ), + policy.DocumentedRuleDefault( + name='baremetal:node:update:shard', + check_str=SYSTEM_ADMIN, + scope_types=['system', 'project'], + description='Governs if node shard field can be updated via ' + 'the API clients.', + operations=[{'path': '/nodes/{node_ident}', 'method': 'PATCH'}], + ), + policy.DocumentedRuleDefault( + name='baremetal:shards:get', + check_str=SYSTEM_READER, + scope_types=['system', 'project'], + description='Governs if shards can be read via the API clients.', + operations=[{'path': '/shards', 'method': 'GET'}], + ), ] deprecated_port_reason = """ diff -Nru ironic-21.1.0/ironic/common/pxe_utils.py ironic-21.4.4/ironic/common/pxe_utils.py --- ironic-21.1.0/ironic/common/pxe_utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/pxe_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -265,6 +265,9 @@ :param root_dir: Directory in which the image must be placed :param label: Name of the image """ + if label in ['ks_template', 'ks_cfg', 'stage2']: + path = os.path.join(CONF.deploy.http_root, node_uuid) + ensure_tree(path) if label == 'ks_template': return os.path.join(CONF.deploy.http_root, node_uuid, 'ks.cfg.template') @@ -661,11 +664,12 @@ """ ctx = task.context node = task.node + boot_option = deploy_utils.get_boot_option(node) image_info = {} # NOTE(pas-ha) do not report image kernel and ramdisk for # local boot or whole disk images so that they are not cached if (node.driver_internal_info.get('is_whole_disk_image') - or deploy_utils.get_boot_option(node) == 'local'): + or boot_option == 'local'): return image_info root_dir = _get_root_dir(ipxe_enabled) i_info = node.instance_info @@ -694,7 +698,9 @@ # like is done with basically Glance. labels = ('kernel', 'ramdisk') - if not isap: + if boot_option != 'kickstart': + anaconda_labels = () + elif not isap: anaconda_labels = ('stage2', 'ks_template', 'ks_cfg') else: # When a path is used, a stage2 ramdisk can be determiend @@ -717,6 +723,7 @@ # TODO(TheJulia): Add functionality to look/grab the hints file # for anaconda and just run with the entire path. + if 'stage2' in anaconda_labels: # stage2: installer stage2 squashfs image # ks_template: anaconda kickstart template # ks_cfg - rendered ks_template @@ -740,30 +747,32 @@ else: node.set_driver_internal_info( 'stage2', str(image_properties['stage2_id'])) - # NOTE(TheJulia): A kickstart template is entirely independent - # of the stage2 ramdisk. In the end, it was the configuration which - # told anaconda how to execute. - if i_info.get('ks_template'): - # If the value is set, we always overwrite it, in the event - # a rebuild is occuring or something along those lines. - node.set_driver_internal_info('ks_template', - i_info['ks_template']) - else: - _get_image_properties() - # ks_template is an optional property on the image - if image_properties and 'ks_template' in image_properties: - node.set_driver_internal_info( - 'ks_template', str(image_properties['ks_template'])) + + if 'ks_template' in anaconda_labels: + # NOTE(TheJulia): A kickstart template is entirely independent + # of the stage2 ramdisk. In the end, it was the configuration which + # told anaconda how to execute. + if i_info.get('ks_template'): + # If the value is set, we always overwrite it, in the event + # a rebuild is occuring or something along those lines. + node.set_driver_internal_info('ks_template', + i_info['ks_template']) else: - # If not defined, default to the overall system default - # kickstart template, as opposed to a user supplied - # template. - node.set_driver_internal_info( - 'ks_template', - 'file://' + os.path.abspath( - CONF.anaconda.default_ks_template + _get_image_properties() + # ks_template is an optional property on the image + if image_properties and 'ks_template' in image_properties: + node.set_driver_internal_info( + 'ks_template', str(image_properties['ks_template'])) + else: + # If not defined, default to the overall system default + # kickstart template, as opposed to a user supplied + # template. + node.set_driver_internal_info( + 'ks_template', + 'file://' + os.path.abspath( + CONF.anaconda.default_ks_template + ) ) - ) node.save() @@ -1157,7 +1166,7 @@ ks_file.flush() try: utils.execute( - 'ksvalidator', ks_file.name, check_on_exit=[0], attempts=1 + 'ksvalidator', ks_file.name, check_exit_code=[0], attempts=1 ) except processutils.ProcessExecutionError as e: msg = (_("The kickstart file generated does not pass validation. " diff -Nru ironic-21.1.0/ironic/common/qemu_img.py ironic-21.4.4/ironic/common/qemu_img.py --- ironic-21.1.0/ironic/common/qemu_img.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/common/qemu_img.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,89 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from oslo_concurrency import processutils +from oslo_utils import units +import tenacity + +from ironic.common import utils +from ironic.conf import CONF + + +LOG = logging.getLogger(__name__) + +# Limit the memory address space to 1 GiB when running qemu-img +QEMU_IMG_LIMITS = None + + +def _qemu_img_limits(): + # NOTE(TheJulia): If you make *any* chance to this code, you may need + # to make an identitical or similar change to ironic-python-agent. + global QEMU_IMG_LIMITS + if QEMU_IMG_LIMITS is None: + QEMU_IMG_LIMITS = processutils.ProcessLimits( + address_space=CONF.disk_utils.image_convert_memory_limit + * units.Mi) + return QEMU_IMG_LIMITS + + +def _retry_on_res_temp_unavailable(exc): + if (isinstance(exc, processutils.ProcessExecutionError) + and ('Resource temporarily unavailable' in exc.stderr + or 'Cannot allocate memory' in exc.stderr)): + return True + return False + + +@tenacity.retry( + retry=tenacity.retry_if_exception(_retry_on_res_temp_unavailable), + stop=tenacity.stop_after_attempt(CONF.disk_utils.image_convert_attempts), + reraise=True) +def convert_image(source, dest, out_format, run_as_root=False, cache=None, + out_of_order=False, sparse_size=None, + source_format='qcow2'): + # NOTE(TheJulia): If you make *any* chance to this code, you may need + # to make an identitical or similar change to ironic-python-agent. + """Convert image to other format.""" + cmd = ['qemu-img', 'convert', '-f', source_format, '-O', out_format] + if cache is not None: + cmd += ['-t', cache] + if sparse_size is not None: + cmd += ['-S', sparse_size] + if out_of_order: + cmd.append('-W') + cmd += [source, dest] + # NOTE(TheJulia): Statically set the MALLOC_ARENA_MAX to prevent leaking + # and the creation of new malloc arenas which will consume the system + # memory. If limited to 1, qemu-img consumes ~250 MB of RAM, but when + # another thread tries to access a locked section of memory in use with + # another thread, then by default a new malloc arena is created, + # which essentially balloons the memory requirement of the machine. + # Default for qemu-img is 8 * nCPU * ~250MB (based on defaults + + # thread/code/process/library overhead. In other words, 64 GB. Limiting + # this to 3 keeps the memory utilization in happy cases below the overall + # threshold which is in place in case a malicious image is attempted to + # be passed through qemu-img. + env_vars = {'MALLOC_ARENA_MAX': '3'} + try: + utils.execute(*cmd, run_as_root=run_as_root, + prlimit=_qemu_img_limits(), + use_standard_locale=True, + env_variables=env_vars) + except processutils.ProcessExecutionError as e: + if ('Resource temporarily unavailable' in e.stderr + or 'Cannot allocate memory' in e.stderr): + LOG.debug('Failed to convert image, retrying. Error: %s', e) + # Sync disk caches before the next attempt + utils.execute('sync') + raise diff -Nru ironic-21.1.0/ironic/common/release_mappings.py ironic-21.4.4/ironic/common/release_mappings.py --- ironic-21.1.0/ironic/common/release_mappings.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/release_mappings.py 2024-10-11 15:42:16.000000000 +0000 @@ -510,7 +510,7 @@ 'VolumeTarget': ['1.0'], } }, - 'master': { + '21.2': { 'api': '1.80', 'rpc': '1.55', 'objects': { @@ -518,6 +518,7 @@ 'BIOSSetting': ['1.1'], 'Node': ['1.36'], 'NodeHistory': ['1.0'], + 'NodeInventory': ['1.0'], 'Conductor': ['1.3'], 'Chassis': ['1.3'], 'Deployment': ['1.0'], @@ -530,6 +531,69 @@ 'VolumeTarget': ['1.0'], } }, + '21.3': { + 'api': '1.81', + 'rpc': '1.55', + 'objects': { + 'Allocation': ['1.1'], + 'BIOSSetting': ['1.1'], + 'Node': ['1.36'], + 'NodeHistory': ['1.0'], + 'NodeInventory': ['1.0'], + 'Conductor': ['1.3'], + 'Chassis': ['1.3'], + 'Deployment': ['1.0'], + 'DeployTemplate': ['1.1'], + 'Port': ['1.11'], + 'Portgroup': ['1.4'], + 'Trait': ['1.0'], + 'TraitList': ['1.0'], + 'VolumeConnector': ['1.0'], + 'VolumeTarget': ['1.0'], + } + }, + '21.4': { + 'api': '1.82', + 'rpc': '1.55', + 'objects': { + 'Allocation': ['1.1'], + 'BIOSSetting': ['1.1'], + 'Node': ['1.37'], + 'NodeHistory': ['1.0'], + 'NodeInventory': ['1.0'], + 'Conductor': ['1.3'], + 'Chassis': ['1.3'], + 'Deployment': ['1.0'], + 'DeployTemplate': ['1.1'], + 'Port': ['1.11'], + 'Portgroup': ['1.5'], + 'Trait': ['1.0'], + 'TraitList': ['1.0'], + 'VolumeConnector': ['1.0'], + 'VolumeTarget': ['1.0'], + } + }, + 'master': { + 'api': '1.82', + 'rpc': '1.55', + 'objects': { + 'Allocation': ['1.1'], + 'BIOSSetting': ['1.1'], + 'Node': ['1.37'], + 'NodeHistory': ['1.0'], + 'NodeInventory': ['1.0'], + 'Conductor': ['1.3'], + 'Chassis': ['1.3'], + 'Deployment': ['1.0'], + 'DeployTemplate': ['1.1'], + 'Port': ['1.11'], + 'Portgroup': ['1.5'], + 'Trait': ['1.0'], + 'TraitList': ['1.0'], + 'VolumeConnector': ['1.0'], + 'VolumeTarget': ['1.0'], + } + }, } # NOTE(xek): Assign each named release to the appropriate semver. @@ -542,12 +606,11 @@ # # Just after we do a new named release, delete the oldest named # release (that we are no longer supporting for a rolling upgrade). -# -# There should be at most two named mappings here. -# NOTE(mgoddard): remove yoga prior to the antelope release. RELEASE_MAPPING['yoga'] = RELEASE_MAPPING['20.1'] RELEASE_MAPPING['zed'] = RELEASE_MAPPING['21.1'] +RELEASE_MAPPING['antelope'] = RELEASE_MAPPING['21.4'] +RELEASE_MAPPING['2023.1'] = RELEASE_MAPPING['21.4'] # List of available versions with named versions first; 'master' is excluded. RELEASE_VERSIONS = sorted(set(RELEASE_MAPPING) - {'master'}, reverse=True) diff -Nru ironic-21.1.0/ironic/common/rpc.py ironic-21.4.4/ironic/common/rpc.py --- ironic-21.1.0/ironic/common/rpc.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/rpc.py 2024-10-11 15:42:16.000000000 +0000 @@ -122,10 +122,9 @@ def get_client(target, version_cap=None, serializer=None): assert TRANSPORT is not None serializer = RequestContextSerializer(serializer) - return messaging.RPCClient(TRANSPORT, - target, - version_cap=version_cap, - serializer=serializer) + return messaging.get_rpc_client( + TRANSPORT, target, version_cap=version_cap, + serializer=serializer) def get_server(target, endpoints, serializer=None): diff -Nru ironic-21.1.0/ironic/common/rpc_service.py ironic-21.4.4/ironic/common/rpc_service.py --- ironic-21.1.0/ironic/common/rpc_service.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/rpc_service.py 2024-10-11 15:42:16.000000000 +0000 @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import datetime import signal import sys import time @@ -24,6 +25,7 @@ import oslo_messaging as messaging from oslo_service import service from oslo_utils import importutils +from oslo_utils import timeutils from ironic.common import context from ironic.common import rpc @@ -93,6 +95,26 @@ 'transport': CONF.rpc_transport}) def stop(self): + initial_time = timeutils.utcnow() + extend_time = initial_time + datetime.timedelta( + seconds=CONF.hash_ring_reset_interval) + + try: + self.manager.del_host(deregister=self.deregister) + except Exception as e: + LOG.exception('Service error occurred when cleaning up ' + 'the RPC manager. Error: %s', e) + + if self.manager.get_online_conductor_count() > 1: + # Delay stopping the server until the hash ring has been + # reset on the cluster + stop_time = timeutils.utcnow() + if stop_time < extend_time: + stop_wait = max(0, (extend_time - stop_time).seconds) + LOG.info('Waiting %(stop_wait)s seconds for hash ring reset.', + {'stop_wait': stop_wait}) + time.sleep(stop_wait) + try: if self.rpcserver is not None: self.rpcserver.stop() @@ -100,11 +122,6 @@ except Exception as e: LOG.exception('Service error occurred when stopping the ' 'RPC server. Error: %s', e) - try: - self.manager.del_host(deregister=self.deregister) - except Exception as e: - LOG.exception('Service error occurred when cleaning up ' - 'the RPC manager. Error: %s', e) super(RPCService, self).stop(graceful=True) LOG.info('Stopped RPC server for service %(service)s on host ' diff -Nru ironic-21.1.0/ironic/common/states.py ironic-21.4.4/ironic/common/states.py --- ironic-21.1.0/ironic/common/states.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/states.py 2024-10-11 15:42:16.000000000 +0000 @@ -269,6 +269,9 @@ FASTTRACK_LOOKUP_ALLOWED_STATES = frozenset(_FASTTRACK_LOOKUP_ALLOWED_STATES) """States where API lookups are permitted with fast track enabled.""" +FAILURE_STATES = frozenset((DEPLOYFAIL, CLEANFAIL, INSPECTFAIL, + RESCUEFAIL, UNRESCUEFAIL, ADOPTFAIL)) + ############## # Power states diff -Nru ironic-21.1.0/ironic/common/swift.py ironic-21.4.4/ironic/common/swift.py --- ironic-21.1.0/ironic/common/swift.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/swift.py 2024-10-11 15:42:16.000000000 +0000 @@ -111,6 +111,31 @@ return obj_uuid + def create_object_from_data(self, object, data, container): + """Uploads a given string to Swift. + + :param object: The name of the object in Swift + :param data: string data to put in the object + :param container: The name of the container for the object. + Defaults to the value set in the configuration options. + :returns: The Swift UUID of the object + :raises: utils.Error, if any operation with Swift fails. + """ + try: + self.connection.put_container(container) + except swift_exceptions.ClientException as e: + operation = _("put container") + raise exception.SwiftOperationError(operation=operation, error=e) + + try: + obj_uuid = self.connection.create_object( + container, object, data=data) + except swift_exceptions.ClientException as e: + operation = _("put object") + raise exception.SwiftOperationError(operation=operation, error=e) + + return obj_uuid + def get_temp_url(self, container, obj, timeout): """Returns the temp url for the given Swift object. @@ -143,6 +168,23 @@ (parse_result.scheme, parse_result.netloc, url_path, None, None, None)) + def get_object(self, object, container): + """Downloads a given object from Swift. + + :param object: The name of the object in Swift + :param container: The name of the container for the object. + Defaults to the value set in the configuration options. + :returns: Swift object + :raises: utils.Error, if the Swift operation fails. + """ + try: + obj = self.connection.download_object(object, container=container) + except swift_exceptions.ClientException as e: + operation = _("get object") + raise exception.SwiftOperationError(operation=operation, error=e) + + return obj + def delete_object(self, container, obj): """Deletes the given Swift object. diff -Nru ironic-21.1.0/ironic/common/utils.py ironic-21.4.4/ironic/common/utils.py --- ironic-21.1.0/ironic/common/utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/common/utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -575,8 +575,12 @@ def wrap_ipv6(ip): """Wrap the address in square brackets if it's an IPv6 address.""" - if ipaddress.ip_address(ip).version == 6: - return "[%s]" % ip + try: + if ipaddress.ip_address(ip).version == 6: + return "[%s]" % ip + except ValueError: + pass + return ip @@ -681,3 +685,18 @@ except Exception: pass return False + + +def stop_after_retries(option, group=None): + """A tenacity retry helper that stops after retries specified in conf.""" + # NOTE(dtantsur): fetch the option inside of the nested call, otherwise it + # cannot be changed in runtime. + def should_stop(retry_state): + if group: + conf = getattr(CONF, group) + else: + conf = CONF + num_retries = getattr(conf, option) + return retry_state.attempt_number >= num_retries + 1 + + return should_stop diff -Nru ironic-21.1.0/ironic/conductor/base_manager.py ironic-21.4.4/ironic/conductor/base_manager.py --- ironic-21.1.0/ironic/conductor/base_manager.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conductor/base_manager.py 2024-10-11 15:42:16.000000000 +0000 @@ -88,10 +88,14 @@ # clear all locks held by this conductor before registering self.dbapi.clear_node_reservations_for_conductor(self.host) - def init_host(self, admin_context=None): + def init_host(self, admin_context=None, start_consoles=True, + start_allocations=True): """Initialize the conductor host. :param admin_context: the admin context to pass to periodic tasks. + :param start_consoles: If consoles should be started in intialization. + :param start_allocations: If allocations should be started in + initialization. :raises: RuntimeError when conductor is already running. :raises: NoDriversLoaded when no drivers are enabled on the conductor. :raises: DriverNotFound if a driver is enabled that does not exist. @@ -189,8 +193,9 @@ # Start consoles if it set enabled in a greenthread. try: - self._spawn_worker(self._start_consoles, - ironic_context.get_admin_context()) + if start_consoles: + self._spawn_worker(self._start_consoles, + ironic_context.get_admin_context()) except exception.NoFreeConductorWorker: LOG.warning('Failed to start worker for restarting consoles.') @@ -207,8 +212,9 @@ # Resume allocations that started before the restart. try: - self._spawn_worker(self._resume_allocations, - ironic_context.get_admin_context()) + if start_allocations: + self._spawn_worker(self._resume_allocations, + ironic_context.get_admin_context()) except exception.NoFreeConductorWorker: LOG.warning('Failed to start worker for resuming allocations.') @@ -328,6 +334,10 @@ self._started = False + def get_online_conductor_count(self): + """Return a count of currently online conductors""" + return len(self.dbapi.get_online_conductors()) + def _register_and_validate_hardware_interfaces(self, hardware_types): """Register and validate hardware interfaces for this conductor. @@ -539,6 +549,7 @@ try: with task_manager.acquire(context, node_uuid, shared=False, purpose='start console') as task: + notify_utils.emit_console_notification( task, 'console_restore', obj_fields.NotificationStatus.START) diff -Nru ironic-21.1.0/ironic/conductor/cleaning.py ironic-21.4.4/ironic/conductor/cleaning.py --- ironic-21.1.0/ironic/conductor/cleaning.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conductor/cleaning.py 2024-10-11 15:42:16.000000000 +0000 @@ -114,8 +114,9 @@ try: conductor_steps.set_node_cleaning_steps( task, disable_ramdisk=disable_ramdisk) - except (exception.InvalidParameterValue, - exception.NodeCleaningFailure) as e: + except Exception as e: + # Catch all exceptions and follow the error handling + # path so things are cleaned up properly. msg = (_('Cannot clean node %(node)s: %(msg)s') % {'node': node.uuid, 'msg': e}) return utils.cleaning_error_handler(task, msg) @@ -247,12 +248,21 @@ task.process_event(event) +def get_last_error(node): + last_error = _('By request, the clean operation was aborted') + if node.clean_step: + last_error += ( + _(' during or after the completion of step "%s"') + % conductor_steps.step_id(node.clean_step) + ) + return last_error + + @task_manager.require_exclusive_lock -def do_node_clean_abort(task, step_name=None): +def do_node_clean_abort(task): """Internal method to abort an ongoing operation. :param task: a TaskManager instance with an exclusive lock - :param step_name: The name of the clean step. """ node = task.node try: @@ -270,12 +280,13 @@ set_fail_state=False) return + last_error = get_last_error(node) info_message = _('Clean operation aborted for node %s') % node.uuid - last_error = _('By request, the clean operation was aborted') - if step_name: - msg = _(' after the completion of step "%s"') % step_name - last_error += msg - info_message += msg + if node.clean_step: + info_message += ( + _(' during or after the completion of step "%s"') + % node.clean_step + ) node.last_error = last_error node.clean_step = None @@ -317,7 +328,7 @@ target_state = None task.process_event('fail', target_state=target_state) - do_node_clean_abort(task, step_name) + do_node_clean_abort(task) return LOG.debug('The cleaning operation for node %(node)s was ' diff -Nru ironic-21.1.0/ironic/conductor/manager.py ironic-21.4.4/ironic/conductor/manager.py --- ironic-21.1.0/ironic/conductor/manager.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conductor/manager.py 2024-10-11 15:42:16.000000000 +0000 @@ -73,6 +73,7 @@ from ironic.drivers import base as drivers_base from ironic.drivers.modules import deploy_utils from ironic.drivers.modules import image_cache +from ironic.drivers.modules import inspect_utils from ironic import objects from ironic.objects import base as objects_base from ironic.objects import fields @@ -97,6 +98,8 @@ def __init__(self, host, topic): super(ConductorManager, self).__init__(host, topic) + # NOTE(TheJulia): This is less a metric-able count, but a means to + # sort out nodes and prioritise a subset (of non-responding nodes). self.power_state_sync_count = collections.defaultdict(int) @METRICS.timer('ConductorManager._clean_up_caches') @@ -1348,7 +1351,8 @@ callback=self._spawn_worker, call_args=(cleaning.do_node_clean_abort, task), err_handler=utils.provisioning_error_handler, - target_state=target_state) + target_state=target_state, + last_error=cleaning.get_last_error(node)) return if node.provision_state == states.RESCUEWAIT: @@ -1432,6 +1436,11 @@ finally: waiters.wait_for_all(futures) + # report a count of the nodes + METRICS.send_gauge( + 'ConductorManager.PowerSyncNodesCount', + len(nodes)) + def _sync_power_state_nodes_task(self, context, nodes): """Invokes power state sync on nodes from synchronized queue. @@ -1450,6 +1459,7 @@ can do here to avoid failing a brand new deploy to a node that we've locked here, though. """ + # FIXME(comstud): Since our initial state checks are outside # of the lock (to try to avoid the lock), some checks are # repeated after grabbing the lock so we can unlock quickly. @@ -1496,6 +1506,12 @@ LOG.info("During sync_power_state, node %(node)s was not " "found and presumed deleted by another process.", {'node': node_uuid}) + # TODO(TheJulia): The chance exists that we orphan a node + # in power_state_sync_count, albeit it is not much data, + # it could eventually cause the memory footprint to grow + # on an exceptionally large ironic deployment. We should + # make sure we clean it up at some point, but overall given + # minimal impact, it is definite low hanging fruit. except exception.NodeLocked: LOG.info("During sync_power_state, node %(node)s was " "already locked by another process. Skip.", @@ -1512,6 +1528,7 @@ # regular power state checking, maintenance is still a required # condition. filters={'maintenance': True, 'fault': faults.POWER_FAILURE}, + node_count_metric_name='ConductorManager.PowerSyncRecoveryNodeCount', ) def _power_failure_recovery(self, task, context): """Periodic task to check power states for nodes in maintenance. @@ -1773,10 +1790,6 @@ if task.node.console_enabled: notify_utils.emit_console_notification( task, 'console_restore', fields.NotificationStatus.START) - # NOTE(kaifeng) Clear allocated_ipmi_terminal_port if exists, - # so current conductor can allocate a new free port from local - # resources. - task.node.del_driver_internal_info('allocated_ipmi_terminal_port') try: task.driver.console.start_console(task) except Exception as err: @@ -1858,6 +1871,7 @@ predicate=lambda n, m: n.conductor_affinity != m.conductor.id, limit=lambda: CONF.conductor.periodic_max_workers, shared_task=False, + node_count_metric_name='ConductorManager.SyncLocalStateNodeCount', ) def _sync_local_state(self, task, context): """Perform any actions necessary to sync local state. @@ -2023,6 +2037,26 @@ node.console_enabled = False notify_utils.emit_console_notification( task, 'console_set', fields.NotificationStatus.END) + # Destroy Swift Inventory entries for this node + try: + inspect_utils.clean_up_swift_entries(task) + except exception.SwiftObjectStillExists as e: + if node.maintenance: + # Maintenance -> Allow orphaning + LOG.warning('Swift object orphaned during destruction of ' + 'node %(node)s: %(e)s', + {'node': node.uuid, 'e': e}) + else: + LOG.error('Swift object cannot be orphaned without ' + 'maintenance mode during destruction of node ' + '%(node)s: %(e)s', {'node': node.uuid, 'e': e}) + raise + except Exception as err: + LOG.error('Failed to delete Swift entries related ' + 'to the node %(node)s: %(err)s.', + {'node': node.uuid, 'err': err}) + raise + node.destroy() LOG.info('Successfully deleted node %(node)s.', {'node': node.uuid}) @@ -2203,18 +2237,16 @@ """ LOG.debug('RPC set_console_mode called for node %(node)s with ' 'enabled %(enabled)s', {'node': node_id, 'enabled': enabled}) - - with task_manager.acquire(context, node_id, shared=False, + with task_manager.acquire(context, node_id, shared=True, purpose='setting console mode') as task: node = task.node - task.driver.console.validate(task) - if enabled == node.console_enabled: op = 'enabled' if enabled else 'disabled' LOG.info("No console action was triggered because the " "console is already %s", op) else: + task.upgrade_lock() node.last_error = None node.save() task.spawn_after(self._spawn_worker, @@ -2625,14 +2657,63 @@ # Yield on every iteration eventlet.sleep(0) + def _sensors_conductor(self, context): + """Called to collect and send metrics "sensors" for the conductor.""" + # populate the message which will be sent to ceilometer + # or other data consumer + message = {'message_id': uuidutils.generate_uuid(), + 'timestamp': datetime.datetime.utcnow(), + 'hostname': self.host} + + try: + ev_type = 'ironic.metrics' + message['event_type'] = ev_type + '.update' + sensors_data = METRICS.get_metrics_data() + except AttributeError: + # TODO(TheJulia): Remove this at some point, but right now + # don't inherently break on version mismatches when people + # disregard requriements. + LOG.warning( + 'get_sensors_data has been configured to collect ' + 'conductor metrics, however the installed ironic-lib ' + 'library lacks the functionality. Please update ' + 'ironic-lib to a minimum of version 5.4.0.') + except Exception as e: + LOG.exception( + "An unknown error occured while attempting to collect " + "sensor data from within the conductor. Error: %(error)s", + {'error': e}) + else: + message['payload'] = ( + self._filter_out_unsupported_types(sensors_data)) + if message['payload']: + self.sensors_notifier.info( + context, ev_type, message) + @METRICS.timer('ConductorManager._send_sensor_data') - @periodics.periodic(spacing=CONF.conductor.send_sensor_data_interval, - enabled=CONF.conductor.send_sensor_data) + @periodics.periodic(spacing=CONF.sensor_data.interval, + enabled=CONF.sensor_data.send_sensor_data) def _send_sensor_data(self, context): """Periodically collects and transmits sensor data notifications.""" + if CONF.sensor_data.enable_for_conductor: + if CONF.sensor_data.workers == 1: + # Directly call the sensors_conductor when only one + # worker is permitted, so we collect data serially + # instead. + self._sensors_conductor(context) + else: + # Also, do not apply the general threshold limit to + # the self collection of "sensor" data from the conductor, + # as were not launching external processes, we're just reading + # from an internal data structure, if we can. + self._spawn_worker(self._sensors_conductor, context) + if not CONF.sensor_data.enable_for_nodes: + # NOTE(TheJulia): If node sensor data is not required, then + # skip the rest of this method. + return filters = {} - if not CONF.conductor.send_sensor_data_for_undeployed_nodes: + if not CONF.sensor_data.enable_for_undeployed_nodes: filters['provision_state'] = states.ACTIVE nodes = queue.Queue() @@ -2640,7 +2721,7 @@ filters=filters): nodes.put_nowait(node_info) - number_of_threads = min(CONF.conductor.send_sensor_data_workers, + number_of_threads = min(CONF.sensor_data.workers, nodes.qsize()) futures = [] for thread_number in range(number_of_threads): @@ -2656,7 +2737,7 @@ break done, not_done = waiters.wait_for_all( - futures, timeout=CONF.conductor.send_sensor_data_wait_timeout) + futures, timeout=CONF.sensor_data.wait_timeout) if not_done: LOG.warning("%d workers for send sensors data did not complete", len(not_done)) @@ -2665,13 +2746,14 @@ """Filters out sensor data types that aren't specified in the config. Removes sensor data types that aren't specified in - CONF.conductor.send_sensor_data_types. + CONF.sensor_data.data_types. :param sensors_data: dict containing sensor types and the associated data :returns: dict with unsupported sensor types removed """ - allowed = set(x.lower() for x in CONF.conductor.send_sensor_data_types) + allowed = set(x.lower() for x in + CONF.sensor_data.data_types) if 'all' in allowed: return sensors_data @@ -3469,7 +3551,6 @@ self.conductor.id): # Another conductor has taken over, skipping continue - LOG.debug('Taking over allocation %s', allocation.uuid) allocations.do_allocate(context, allocation) except Exception: diff -Nru ironic-21.1.0/ironic/conductor/periodics.py ironic-21.4.4/ironic/conductor/periodics.py --- ironic-21.1.0/ironic/conductor/periodics.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conductor/periodics.py 2024-10-11 15:42:16.000000000 +0000 @@ -18,6 +18,7 @@ import eventlet from futurist import periodics +from ironic_lib import metrics_utils from oslo_log import log from ironic.common import exception @@ -29,6 +30,9 @@ LOG = log.getLogger(__name__) +METRICS = metrics_utils.get_metrics_logger(__name__) + + def periodic(spacing, enabled=True, **kwargs): """A decorator to define a periodic task. @@ -46,7 +50,7 @@ def node_periodic(purpose, spacing, enabled=True, filters=None, predicate=None, predicate_extra_fields=(), limit=None, - shared_task=True): + shared_task=True, node_count_metric_name=None): """A decorator to define a periodic task to act on nodes. Defines a periodic task that fetches the list of nodes mapped to the @@ -84,6 +88,9 @@ iteration to determine the limit. :param shared_task: if ``True``, the task will have a shared lock. It is recommended to start with a shared lock and upgrade it only if needed. + :param node_count_metric_name: A string value to identify a metric + representing the count of matching nodes to be recorded upon the + completion of the periodic. """ node_type = collections.namedtuple( 'Node', @@ -116,10 +123,11 @@ else: local_limit = limit assert local_limit is None or local_limit > 0 - + node_count = 0 nodes = manager.iter_nodes(filters=filters, fields=predicate_extra_fields) for (node_uuid, *other) in nodes: + node_count += 1 if predicate is not None: node = node_type(node_uuid, *other) if accepts_manager: @@ -158,6 +166,11 @@ local_limit -= 1 if not local_limit: return + if node_count_metric_name: + # Send post-run metrics. + METRICS.send_gauge( + node_count_metric_name, + node_count) return wrapper diff -Nru ironic-21.1.0/ironic/conductor/steps.py ironic-21.4.4/ironic/conductor/steps.py --- ironic-21.1.0/ironic/conductor/steps.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conductor/steps.py 2024-10-11 15:42:16.000000000 +0000 @@ -194,9 +194,9 @@ sort_step_key=sort_key, prio_overrides=csp_override) - LOG.debug("cleaning_steps after applying " - "clean_step_priority_override for node %(node)s: %(step)s", - task.node.uuid, cleaning_steps) + LOG.debug('cleaning_steps after applying ' + 'clean_step_priority_override for node %(node)s: %(steps)s', + {'node': task.node.uuid, 'steps': cleaning_steps}) else: cleaning_steps = _get_steps(task, CLEANING_INTERFACE_PRIORITY, 'get_clean_steps', enabled=enabled, diff -Nru ironic-21.1.0/ironic/conductor/task_manager.py ironic-21.4.4/ironic/conductor/task_manager.py --- ironic-21.1.0/ironic/conductor/task_manager.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conductor/task_manager.py 2024-10-11 15:42:16.000000000 +0000 @@ -527,7 +527,8 @@ self.release_resources() def process_event(self, event, callback=None, call_args=None, - call_kwargs=None, err_handler=None, target_state=None): + call_kwargs=None, err_handler=None, target_state=None, + last_error=None): """Process the given event for the task's current state. :param event: the name of the event to process @@ -540,6 +541,8 @@ prev_target_state) :param target_state: if specified, the target provision state for the node. Otherwise, use the target state from the fsm + :param last_error: last error to set on the node together with + the state transition. :raises: InvalidState if the event is not allowed by the associated state machine """ @@ -572,13 +575,15 @@ # set up the async worker if callback: - # clear the error if we're going to start work in a callback - self.node.last_error = None + # update the error if we're going to start work in a callback + self.node.last_error = last_error if call_args is None: call_args = () if call_kwargs is None: call_kwargs = {} self.spawn_after(callback, *call_args, **call_kwargs) + elif last_error is not None: + self.node.last_error = last_error # publish the state transition by saving the Node self.node.save() diff -Nru ironic-21.1.0/ironic/conductor/utils.py ironic-21.4.4/ironic/conductor/utils.py --- ironic-21.1.0/ironic/conductor/utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conductor/utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -297,14 +297,23 @@ node = task.node if _can_skip_state_change(task, new_state): + # NOTE(TheJulia): Even if we are not changing the power state, + # we need to wipe the token out, just in case for some reason + # the power was turned off outside of our interaction/management. + if new_state in (states.POWER_OFF, states.SOFT_POWER_OFF, + states.REBOOT, states.SOFT_REBOOT): + wipe_internal_info_on_power_off(node) + node.save() return target_state = _calculate_target_state(new_state) # Set the target_power_state and clear any last_error, if we're # starting a new operation. This will expose to other processes - # and clients that work is in progress. - node['target_power_state'] = target_state - node['last_error'] = None + # and clients that work is in progress. Keep the last_error intact + # if the power action happens as a result of a failure. + node.target_power_state = target_state + if node.provision_state not in states.FAILURE_STATES: + node.last_error = None node.timestamp_driver_internal_info('last_power_state_change') # NOTE(dtantsur): wipe token on shutting down, otherwise a reboot in # fast-track (or an accidentally booted agent) will cause subsequent @@ -479,9 +488,9 @@ node.del_driver_internal_info('cleaning_reboot') node.del_driver_internal_info('cleaning_polling') node.del_driver_internal_info('skip_current_clean_step') - # We don't need to keep the old agent URL + # We don't need to keep the old agent URL, or token # as it should change upon the next cleaning attempt. - node.del_driver_internal_info('agent_url') + wipe_token_and_url(task) # For manual cleaning, the target provision state is MANAGEABLE, whereas # for automated cleaning, it is AVAILABLE. manual_clean = node.target_provision_state == states.MANAGEABLE diff -Nru ironic-21.1.0/ironic/conf/__init__.py ironic-21.4.4/ironic/conf/__init__.py --- ironic-21.1.0/ironic/conf/__init__.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conf/__init__.py 2024-10-11 15:42:16.000000000 +0000 @@ -27,13 +27,16 @@ from ironic.conf import default from ironic.conf import deploy from ironic.conf import dhcp +from ironic.conf import disk_utils from ironic.conf import dnsmasq from ironic.conf import drac +from ironic.conf import fake from ironic.conf import glance from ironic.conf import healthcheck from ironic.conf import ibmc from ironic.conf import ilo from ironic.conf import inspector +from ironic.conf import inventory from ironic.conf import ipmi from ironic.conf import irmc from ironic.conf import metrics @@ -43,6 +46,7 @@ from ironic.conf import nova from ironic.conf import pxe from ironic.conf import redfish +from ironic.conf import sensor_data from ironic.conf import service_catalog from ironic.conf import snmp from ironic.conf import swift @@ -63,12 +67,15 @@ deploy.register_opts(CONF) drac.register_opts(CONF) dhcp.register_opts(CONF) +disk_utils.register_opts(CONF) dnsmasq.register_opts(CONF) +fake.register_opts(CONF) glance.register_opts(CONF) healthcheck.register_opts(CONF) ibmc.register_opts(CONF) ilo.register_opts(CONF) inspector.register_opts(CONF) +inventory.register_opts(CONF) ipmi.register_opts(CONF) irmc.register_opts(CONF) metrics.register_opts(CONF) @@ -78,6 +85,7 @@ nova.register_opts(CONF) pxe.register_opts(CONF) redfish.register_opts(CONF) +sensor_data.register_opts(CONF) service_catalog.register_opts(CONF) snmp.register_opts(CONF) swift.register_opts(CONF) diff -Nru ironic-21.1.0/ironic/conf/conductor.py ironic-21.4.4/ironic/conf/conductor.py --- ironic-21.1.0/ironic/conf/conductor.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conf/conductor.py 2024-10-11 15:42:16.000000000 +0000 @@ -97,41 +97,6 @@ cfg.IntOpt('node_locked_retry_interval', default=1, help=_('Seconds to sleep between node lock attempts.')), - cfg.BoolOpt('send_sensor_data', - default=False, - help=_('Enable sending sensor data message via the ' - 'notification bus')), - cfg.IntOpt('send_sensor_data_interval', - default=600, - min=1, - help=_('Seconds between conductor sending sensor data message ' - 'to ceilometer via the notification bus.')), - cfg.IntOpt('send_sensor_data_workers', - default=4, min=1, - help=_('The maximum number of workers that can be started ' - 'simultaneously for send data from sensors periodic ' - 'task.')), - cfg.IntOpt('send_sensor_data_wait_timeout', - default=300, - help=_('The time in seconds to wait for send sensors data ' - 'periodic task to be finished before allowing periodic ' - 'call to happen again. Should be less than ' - 'send_sensor_data_interval value.')), - cfg.ListOpt('send_sensor_data_types', - default=['ALL'], - help=_('List of comma separated meter types which need to be' - ' sent to Ceilometer. The default value, "ALL", is a ' - 'special value meaning send all the sensor data.')), - cfg.BoolOpt('send_sensor_data_for_undeployed_nodes', - default=False, - help=_('The default for sensor data collection is to only ' - 'collect data for machines that are deployed, however ' - 'operators may desire to know if there are failures ' - 'in hardware that is not presently in use. ' - 'When set to true, the conductor will collect sensor ' - 'information from all nodes when sensor data ' - 'collection is enabled via the send_sensor_data ' - 'setting.')), cfg.IntOpt('sync_local_state_interval', default=180, help=_('When conductors join or leave the cluster, existing ' @@ -384,6 +349,71 @@ 'is a global setting applying to all requests this ' 'conductor receives, regardless of access rights. ' 'The concurrent clean limit cannot be disabled.')), + cfg.BoolOpt('disable_deep_image_inspection', + default=False, + # Normally such an option would be mutable, but this is, + # a security guard and operators should not expect to change + # this option under normal circumstances. + mutable=False, + help=_('Security Option to permit an operator to disable ' + 'file content inspections. Under normal conditions, ' + 'the conductor will inspect requested image contents ' + 'which are transferred through the conductor. ' + 'Disabling this option is not advisable and opens ' + 'the risk of unsafe images being processed which may ' + 'allow an attacker to leverage unsafe features in ' + 'various disk image formats to perform a variety of ' + 'unsafe and potentially compromising actions. ' + 'This option is *not* mutable, and ' + 'requires a service restart to change.')), + cfg.BoolOpt('conductor_always_validates_images', + default=False, + # Normally mutable, however from a security context we do want + # all logging to be generated from this option to be changed, + # and as such is set to False to force a conductor restart. + mutable=False, + help=_('Security Option to enable the conductor to *always* ' + 'inspect the image content of any requested deploy, ' + 'even if the deployment would have normally bypassed ' + 'the conductor\'s cache. When this is set to False, ' + 'the Ironic-Python-Agent is responsible ' + 'for any necessary image checks. Setting this to ' + 'True will result in a higher utilization of ' + 'resources (disk space, network traffic) ' + 'as the conductor will evaluate *all* images. ' + 'This option is *not* mutable, and requires a ' + 'service restart to change. This option requires ' + '[conductor]disable_deep_image_inspection to be set ' + 'to False.')), + cfg.ListOpt('permitted_image_formats', + default=['raw', 'qcow2', 'iso'], + mutable=True, + help=_('The supported list of image formats which are ' + 'permitted for deployment with Ironic. If an image ' + 'format outside of this list is detected, the image ' + 'validation logic will fail the deployment process.')), + cfg.BoolOpt('disable_file_checksum', + default=False, + mutable=False, + help=_('Deprecated Security option: In the default case, ' + 'image files have their checksums verified before ' + 'undergoing additional conductor side actions such ' + 'as image conversion. ' + 'Enabling this option opens the risk of files being ' + 'replaced at the source without the user\'s ' + 'knowledge.'), + deprecated_for_removal=True), + cfg.BoolOpt('disable_support_for_checksum_files', + default=False, + mutable=False, + help=_('Security option: By default Ironic will attempt to ' + 'retrieve a remote checksum file via HTTP(S) URL in ' + 'order to validate an image download. This is ' + 'functionality aligning with ironic-python-agent ' + 'support for standalone users. Disabling this ' + 'functionality by setting this option to True will ' + 'create a more secure environment, however it may ' + 'break users in an unexpected fashion.')), ] diff -Nru ironic-21.1.0/ironic/conf/default.py ironic-21.4.4/ironic/conf/default.py --- ironic-21.1.0/ironic/conf/default.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conf/default.py 2024-10-11 15:42:16.000000000 +0000 @@ -216,7 +216,7 @@ 'common/isolinux_config.template'), help=_('Template file for isolinux configuration file.')), cfg.StrOpt('grub_config_path', - default='/boot/grub/grub.cfg', + default='EFI/BOOT/grub.cfg', help=_('GRUB2 configuration file location on the UEFI ISO ' 'images produced by ironic. The default value is ' 'usually incorrect and should not be relied on. ' @@ -430,6 +430,40 @@ 'with images.')), ] +rbac_opts = [ + cfg.BoolOpt('rbac_service_role_elevated_access', + default=False, + help=_('Enable elevated access for users with service role ' + 'belonging to the \'rbac_service_project_name\' ' + 'project when using default policy. The default ' + 'setting of disabled causes all service role ' + 'requests to be scoped to the project the service ' + 'account belongs to.')), + cfg.StrOpt('rbac_service_project_name', + default='service', + help=_('The project name utilized for Role Based Access ' + 'Control checks for the reserved `service` project. ' + 'This project is utilized for services to have ' + 'accounts for cross-service communication. Often ' + 'these accounts require higher levels of access, and ' + 'effectively this permits accounts from the service ' + 'to not be restricted to project scoping ' + 'of responses. i.e. The service project user with a ' + '`service` role will be able to see nodes across all ' + 'projects, similar to System scoped access. If not ' + 'set to a value, and all service role access will ' + 'be filtered matching an `owner` or `lessee`, if ' + 'applicable. If an operator wishes to make behavior ' + 'visible for all service role users across ' + 'all projects, then a custom policy must be used ' + 'to override the default "service_role" rule. ' + 'It should be noted that the value of "service" ' + 'is a default convention for OpenStack deployments, ' + 'but the requsite access and details around ' + 'end configuration are largely up to an operator ' + 'if they are doing an OpenStack deployment manually.')), +] + def list_opts(): _default_opt_lists = [ @@ -446,6 +480,7 @@ service_opts, utils_opts, webserver_opts, + rbac_opts ] full_opt_list = [] for options in _default_opt_lists: @@ -467,3 +502,4 @@ conf.register_opts(service_opts) conf.register_opts(utils_opts) conf.register_opts(webserver_opts) + conf.register_opts(rbac_opts) diff -Nru ironic-21.1.0/ironic/conf/disk_utils.py ironic-21.4.4/ironic/conf/disk_utils.py --- ironic-21.1.0/ironic/conf/disk_utils.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/conf/disk_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +# NOTE(TheJulia): If you make *any* chance to this code, you may need +# to make an identitical or similar change to ironic-python-agent. +# These options were originally taken from ironic-lib upon the decision +# to move the qemu-img image conversion calls into the projects in +# order to simplify fixes related to them. +opts = [ + cfg.IntOpt('image_convert_memory_limit', + default=2048, + help='Memory limit for "qemu-img convert" in MiB. Implemented ' + 'via the address space resource limit.'), + cfg.IntOpt('image_convert_attempts', + default=3, + help='Number of attempts to convert an image.'), +] + + +def register_opts(conf): + conf.register_opts(opts, group='disk_utils') diff -Nru ironic-21.1.0/ironic/conf/fake.py ironic-21.4.4/ironic/conf/fake.py --- ironic-21.1.0/ironic/conf/fake.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/conf/fake.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,85 @@ +# +# Copyright 2022 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from ironic.common.i18n import _ + +opts = [ + cfg.StrOpt('power_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'power driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('boot_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'boot driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('deploy_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'deploy driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('vendor_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'vendor driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('management_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'management driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('inspect_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'inspect driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('raid_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'raid driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('bios_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'bios driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('storage_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'storage driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('rescue_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'rescue driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), +] + + +def register_opts(conf): + conf.register_opts(opts, group='fake') diff -Nru ironic-21.1.0/ironic/conf/glance.py ironic-21.4.4/ironic/conf/glance.py --- ironic-21.1.0/ironic/conf/glance.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conf/glance.py 2024-10-11 15:42:16.000000000 +0000 @@ -114,6 +114,7 @@ 'will determine how many containers are created.')), cfg.IntOpt('num_retries', default=0, + mutable=True, help=_('Number of retries when downloading an image from ' 'glance.')), ] diff -Nru ironic-21.1.0/ironic/conf/inventory.py ironic-21.4.4/ironic/conf/inventory.py --- ironic-21.1.0/ironic/conf/inventory.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/conf/inventory.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,34 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from ironic.common.i18n import _ + +opts = [ + cfg.StrOpt('data_backend', + help=_('The storage backend for storing introspection data.'), + choices=[('none', _('introspection data will not be stored')), + ('database', _('introspection data stored in an SQL ' + 'database')), + ('swift', _('introspection data stored in Swift'))], + default='database'), + cfg.StrOpt('swift_data_container', + default='introspection_data_container', + help=_('The Swift introspection data container to store ' + 'the inventory data.')), +] + + +def register_opts(conf): + conf.register_opts(opts, group='inventory') diff -Nru ironic-21.1.0/ironic/conf/irmc.py ironic-21.4.4/ironic/conf/irmc.py --- ironic-21.1.0/ironic/conf/irmc.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conf/irmc.py 2024-10-11 15:42:16.000000000 +0000 @@ -81,9 +81,20 @@ help='SNMP polling interval in seconds'), cfg.StrOpt('snmp_auth_proto', default='sha', - choices=[('sha', _('Secure Hash Algorithm 1'))], + choices=[('sha', _('Secure Hash Algorithm 1, supported in iRMC ' + 'S4 and S5.')), + ('sha256', ('Secure Hash Algorithm 2 with 256 bits ' + 'digest, only supported in iRMC S6.')), + ('sha384', ('Secure Hash Algorithm 2 with 384 bits ' + 'digest, only supported in iRMC S6.')), + ('sha512', ('Secure Hash Algorithm 2 with 512 bits ' + 'digest, only supported in iRMC S6.'))], help=_("SNMPv3 message authentication protocol ID. " - "Required for version 'v3'. 'sha' is supported.")), + "Required for version 'v3'. The valid options are " + "'sha', 'sha256', 'sha384' and 'sha512', while 'sha' is " + "the only supported protocol in iRMC S4 and S5, and " + "from iRMC S6, 'sha256', 'sha384' and 'sha512' are " + "supported, but 'sha' is not supported any more.")), cfg.StrOpt('snmp_priv_proto', default='aes', choices=[('aes', _('Advanced Encryption Standard'))], diff -Nru ironic-21.1.0/ironic/conf/opts.py ironic-21.4.4/ironic/conf/opts.py --- ironic-21.1.0/ironic/conf/opts.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/conf/opts.py 2024-10-11 15:42:16.000000000 +0000 @@ -27,11 +27,13 @@ ('database', ironic.conf.database.opts), ('deploy', ironic.conf.deploy.opts), ('dhcp', ironic.conf.dhcp.opts), + ('disk_utils', ironic.conf.disk_utils.opts), ('drac', ironic.conf.drac.opts), ('glance', ironic.conf.glance.list_opts()), ('healthcheck', ironic.conf.healthcheck.opts), ('ilo', ironic.conf.ilo.opts), ('inspector', ironic.conf.inspector.list_opts()), + ('inventory', ironic.conf.inventory.opts), ('ipmi', ironic.conf.ipmi.opts), ('irmc', ironic.conf.irmc.opts), ('anaconda', ironic.conf.anaconda.opts), @@ -42,6 +44,7 @@ ('nova', ironic.conf.nova.list_opts()), ('pxe', ironic.conf.pxe.opts), ('redfish', ironic.conf.redfish.opts), + ('sensor_data', ironic.conf.sensor_data.opts), ('service_catalog', ironic.conf.service_catalog.list_opts()), ('snmp', ironic.conf.snmp.opts), ('swift', ironic.conf.swift.list_opts()), @@ -88,5 +91,8 @@ 'openstack=WARNING', # Policy logging is not necessarily useless, but very verbose 'oslo_policy=WARNING', + # Concurrency lock logging is not bad, but exceptionally noisy + # and typically not needed in debugging Ironic itself. + 'oslo_concurrency.lockutils=WARNING', ] ) diff -Nru ironic-21.1.0/ironic/conf/sensor_data.py ironic-21.4.4/ironic/conf/sensor_data.py --- ironic-21.1.0/ironic/conf/sensor_data.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/conf/sensor_data.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,89 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from ironic.common.i18n import _ + +opts = [ + cfg.BoolOpt('send_sensor_data', + default=False, + deprecated_group='conductor', + deprecated_name='send_sensor_data', + help=_('Enable sending sensor data message via the ' + 'notification bus.')), + cfg.IntOpt('interval', + default=600, + min=1, + deprecated_group='conductor', + deprecated_name='send_sensor_data_interval', + help=_('Seconds between conductor sending sensor data message ' + 'via the notification bus. This was originally for ' + 'consumption via ceilometer, but the data may also ' + 'be consumed via a plugin like ' + 'ironic-prometheus-exporter or any other message bus ' + 'data collector.')), + cfg.IntOpt('workers', + default=4, min=1, + deprecated_group='conductor', + deprecated_name='send_sensor_data_workers', + help=_('The maximum number of workers that can be started ' + 'simultaneously for send data from sensors periodic ' + 'task.')), + cfg.IntOpt('wait_timeout', + default=300, + deprecated_group='conductor', + deprecated_name='send_sensor_data_wait_timeout', + help=_('The time in seconds to wait for send sensors data ' + 'periodic task to be finished before allowing periodic ' + 'call to happen again. Should be less than ' + 'send_sensor_data_interval value.')), + cfg.ListOpt('data_types', + default=['ALL'], + deprecated_group='conductor', + deprecated_name='send_sensor_data_types', + help=_('List of comma separated meter types which need to be ' + 'sent to Ceilometer. The default value, "ALL", is a ' + 'special value meaning send all the sensor data. ' + 'This setting only applies to baremetal sensor data ' + 'being processed through the conductor.')), + cfg.BoolOpt('enable_for_undeployed_nodes', + default=False, + deprecated_group='conductor', + deprecated_name='send_sensor_data_for_undeployed_nodes', + help=_('The default for sensor data collection is to only ' + 'collect data for machines that are deployed, however ' + 'operators may desire to know if there are failures ' + 'in hardware that is not presently in use. ' + 'When set to true, the conductor will collect sensor ' + 'information from all nodes when sensor data ' + 'collection is enabled via the send_sensor_data ' + 'setting.')), + cfg.BoolOpt('enable_for_conductor', + default=True, + help=_('If to include sensor metric data for the Conductor ' + 'process itself in the message payload for sensor ' + 'data which allows operators to gather instance ' + 'counts of actions and states to better manage ' + 'the deployment.')), + cfg.BoolOpt('enable_for_nodes', + default=True, + help=_('If to transmit any sensor data for any nodes under ' + 'this conductor\'s management. This option superceeds ' + 'the ``send_sensor_data_for_undeployed_nodes`` ' + 'setting.')), +] + + +def register_opts(conf): + conf.register_opts(opts, group='sensor_data') diff -Nru ironic-21.1.0/ironic/db/api.py ironic-21.4.4/ironic/db/api.py --- ironic-21.1.0/ironic/db/api.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/api.py 2024-10-11 15:42:16.000000000 +0000 @@ -72,6 +72,7 @@ :reserved_by_any_of: [conductor1, conductor2] :resource_class: resource class name :retired: True | False + :shard_in: shard (multiple possibilities) :provision_state: provision state of node :provision_state_in: provision state of node (multiple possibilities) @@ -106,6 +107,7 @@ :provisioned_before: nodes with provision_updated_at field before this interval in seconds + :shard: nodes with the given shard :param limit: Maximum number of nodes to return. :param marker: the last item of the previous page; we return the next result set. @@ -295,6 +297,14 @@ """ @abc.abstractmethod + def get_ports_by_shards(self, shards, limit=None, marker=None, + sort_key=None, sort_dir=None): + """Return a list of ports contained in the provided shards. + + :param shard_ids: A list of shards to filter ports by. + """ + + @abc.abstractmethod def get_ports_by_node_id(self, node_id, limit=None, marker=None, sort_key=None, sort_dir=None): """List all the ports for a given node. @@ -1425,3 +1435,32 @@ count operation. This can be a single provision state value or a list of values. """ + + @abc.abstractmethod + def create_node_inventory(self, values): + """Create a new inventory record. + + :param values: Dict of values. + """ + + @abc.abstractmethod + def destroy_node_inventory_by_node_id(self, inventory_node_id): + """Destroy a inventory record. + + :param inventory_uuid: The uuid of a inventory record + """ + + @abc.abstractmethod + def get_node_inventory_by_node_id(self, node_id): + """Get the node inventory for a given node. + + :param node_id: The integer node ID. + :returns: An inventory of a node. + """ + + @abc.abstractmethod + def get_shard_list(self): + """Retrieve a list of shards. + + :returns: list of dicts containing shard names and count + """ diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/__init__.py ironic-21.4.4/ironic/db/sqlalchemy/__init__.py --- ironic-21.1.0/ironic/db/sqlalchemy/__init__.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/__init__.py 2024-10-11 15:42:16.000000000 +0000 @@ -12,7 +12,7 @@ from oslo_db.sqlalchemy import enginefacade -# NOTE(dtantsur): we want sqlite as close to a real database as possible. # FIXME(stephenfin): we need to remove reliance on autocommit semantics ASAP # since it's not compatible with SQLAlchemy 2.0 +# NOTE(dtantsur): we want sqlite as close to a real database as possible. enginefacade.configure(sqlite_fk=True, __autocommit=True) diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/0ac0f39bc5aa_add_node_inventory_table.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/0ac0f39bc5aa_add_node_inventory_table.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/0ac0f39bc5aa_add_node_inventory_table.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/0ac0f39bc5aa_add_node_inventory_table.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +"""add node inventory table + +Revision ID: 0ac0f39bc5aa +Revises: 9ef41f07cb58 +Create Date: 2022-10-25 17:15:38.181544 + +""" + +from alembic import op +from oslo_db.sqlalchemy import types as db_types +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '0ac0f39bc5aa' +down_revision = '9ef41f07cb58' + + +def upgrade(): + op.create_table('node_inventory', + sa.Column('version', sa.String(length=15), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('inventory_data', db_types.JsonEncodedDict( + mysql_as_long=True).impl, nullable=True), + sa.Column('plugin_data', db_types.JsonEncodedDict( + mysql_as_long=True).impl, nullable=True), + sa.Column('node_id', sa.Integer(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ), + sa.Index('inventory_node_id_idx', 'node_id'), + mysql_engine='InnoDB', + mysql_charset='UTF8MB3') diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/2581ebaf0cb2_initial_migration.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/2581ebaf0cb2_initial_migration.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/2581ebaf0cb2_initial_migration.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/2581ebaf0cb2_initial_migration.py 2024-10-11 15:42:16.000000000 +0000 @@ -38,8 +38,8 @@ sa.Column('drivers', sa.Text(), nullable=True), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('hostname', name='uniq_conductors0hostname'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8' + mysql_charset='UTF8MB3', + mysql_engine='InnoDB', ) op.create_table( 'chassis', @@ -51,8 +51,8 @@ sa.Column('description', sa.String(length=255), nullable=True), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('uuid', name='uniq_chassis0uuid'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8' + mysql_engine='InnoDB', + mysql_charset='UTF8MB3' ) op.create_table( 'nodes', @@ -77,8 +77,8 @@ sa.ForeignKeyConstraint(['chassis_id'], ['chassis.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('uuid', name='uniq_nodes0uuid'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8' + mysql_engine='InnoDB', + mysql_charset='UTF8MB3' ) op.create_index('node_instance_uuid', 'nodes', ['instance_uuid'], unique=False) @@ -95,7 +95,7 @@ sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('address', name='uniq_ports0address'), sa.UniqueConstraint('uuid', name='uniq_ports0uuid'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8' + mysql_engine='InnoDB', + mysql_charset='UTF8MB3' ) # end Alembic commands diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/2aac7e0872f6_add_deploy_templates.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/2aac7e0872f6_add_deploy_templates.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/2aac7e0872f6_add_deploy_templates.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/2aac7e0872f6_add_deploy_templates.py 2024-10-11 15:42:16.000000000 +0000 @@ -39,8 +39,8 @@ sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('uuid', name='uniq_deploytemplates0uuid'), sa.UniqueConstraint('name', name='uniq_deploytemplates0name'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8' + mysql_engine='InnoDB', + mysql_charset='UTF8MB3' ) op.create_table( @@ -62,6 +62,6 @@ sa.Index('deploy_template_id', 'deploy_template_id'), sa.Index('deploy_template_steps_interface_idx', 'interface'), sa.Index('deploy_template_steps_step_idx', 'step'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8' + mysql_engine='InnoDB', + mysql_charset='UTF8MB3' ) diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/48d6c242bb9b_add_node_tags.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/48d6c242bb9b_add_node_tags.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/48d6c242bb9b_add_node_tags.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/48d6c242bb9b_add_node_tags.py 2024-10-11 15:42:16.000000000 +0000 @@ -36,7 +36,7 @@ sa.Column('tag', sa.String(length=255), nullable=False), sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ), sa.PrimaryKeyConstraint('node_id', 'tag'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8' + mysql_engine='InnoDB', + mysql_charset='UTF8MB3' ) op.create_index('node_tags_idx', 'node_tags', ['tag'], unique=False) diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/4dbec778866e_create_node_shard.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/4dbec778866e_create_node_shard.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/4dbec778866e_create_node_shard.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/4dbec778866e_create_node_shard.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""create node.shard + +Revision ID: 4dbec778866e +Revises: 0ac0f39bc5aa +Create Date: 2022-11-10 14:20:59.175355 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '4dbec778866e' +down_revision = '0ac0f39bc5aa' + + +def upgrade(): + op.add_column('nodes', sa.Column('shard', sa.String(length=255), + nullable=True)) + op.create_index('shard_idx', 'nodes', ['shard'], unique=False) diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/5ea1b0d310e_added_port_group_table_and_altered_ports.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/5ea1b0d310e_added_port_group_table_and_altered_ports.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/5ea1b0d310e_added_port_group_table_and_altered_ports.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/5ea1b0d310e_added_port_group_table_and_altered_ports.py 2024-10-11 15:42:16.000000000 +0000 @@ -42,8 +42,8 @@ sa.UniqueConstraint('address', name='uniq_portgroups0address'), sa.UniqueConstraint('name', name='uniq_portgroups0name'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8') + mysql_engine='InnoDB', + mysql_charset='UTF8MB3') op.add_column(u'ports', sa.Column('local_link_connection', sa.Text(), nullable=True)) op.add_column(u'ports', sa.Column('portgroup_id', sa.Integer(), diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/82c315d60161_add_bios_settings.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/82c315d60161_add_bios_settings.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/82c315d60161_add_bios_settings.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/82c315d60161_add_bios_settings.py 2024-10-11 15:42:16.000000000 +0000 @@ -37,6 +37,6 @@ sa.Column('version', sa.String(length=15), nullable=True), sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ), sa.PrimaryKeyConstraint('node_id', 'name'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8' + mysql_engine='InnoDB', + mysql_charset='UTF8MB3' ) diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/9ef41f07cb58_add_node_history_table.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/9ef41f07cb58_add_node_history_table.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/9ef41f07cb58_add_node_history_table.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/9ef41f07cb58_add_node_history_table.py 2024-10-11 15:42:16.000000000 +0000 @@ -48,5 +48,5 @@ sa.Index('history_node_id_idx', 'node_id'), sa.Index('history_uuid_idx', 'uuid'), sa.Index('history_conductor_idx', 'conductor'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8') + mysql_engine='InnoDB', + mysql_charset='UTF8MB3') diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/b4130a7fc904_create_nodetraits_table.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/b4130a7fc904_create_nodetraits_table.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/b4130a7fc904_create_nodetraits_table.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/b4130a7fc904_create_nodetraits_table.py 2024-10-11 15:42:16.000000000 +0000 @@ -37,7 +37,7 @@ sa.Column('trait', sa.String(length=255), nullable=False), sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ), sa.PrimaryKeyConstraint('node_id', 'trait'), - mysql_ENGINE='InnoDB', - mysql_DEFAULT_CHARSET='UTF8' + mysql_engine='InnoDB', + mysql_charset='UTF8MB3' ) op.create_index('node_traits_idx', 'node_traits', ['trait'], unique=False) diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/dd67b91a1981_add_allocations_table.py ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/dd67b91a1981_add_allocations_table.py --- ironic-21.1.0/ironic/db/sqlalchemy/alembic/versions/dd67b91a1981_add_allocations_table.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/alembic/versions/dd67b91a1981_add_allocations_table.py 2024-10-11 15:42:16.000000000 +0000 @@ -48,7 +48,10 @@ sa.ForeignKeyConstraint(['node_id'], ['nodes.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('name', name='uniq_allocations0name'), - sa.UniqueConstraint('uuid', name='uniq_allocations0uuid') + sa.UniqueConstraint('uuid', name='uniq_allocations0uuid'), + mysql_engine='InnoDB', + mysql_charset='UTF8MB3' + ) op.add_column('nodes', sa.Column('allocation_id', sa.Integer(), nullable=True)) diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/api.py ironic-21.4.4/ironic/db/sqlalchemy/api.py --- ironic-21.1.0/ironic/db/sqlalchemy/api.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/api.py 2024-10-11 15:42:16.000000000 +0000 @@ -19,9 +19,11 @@ import json import threading +from oslo_concurrency import lockutils from oslo_db import api as oslo_db_api from oslo_db import exception as db_exc from oslo_db.sqlalchemy import enginefacade +from oslo_db.sqlalchemy import orm as sa_orm from oslo_db.sqlalchemy import utils as db_utils from oslo_log import log from oslo_utils import netutils @@ -31,8 +33,7 @@ from osprofiler import sqlalchemy as osp_sqlalchemy import sqlalchemy as sa from sqlalchemy import or_ -from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound -from sqlalchemy.orm import joinedload +from sqlalchemy.exc import NoResultFound, MultipleResultsFound from sqlalchemy.orm import Load from sqlalchemy.orm import selectinload from sqlalchemy import sql @@ -53,6 +54,10 @@ _CONTEXT = threading.local() + +RESERVATION_SEMAPHORE = "reserve_node_db_lock" +synchronized = lockutils.synchronized_with_prefix('ironic-') + # NOTE(mgoddard): We limit the number of traits per node to 50 as this is the # maximum number of traits per resource provider allowed in placement. MAX_TRAITS_PER_NODE = 50 @@ -80,104 +85,43 @@ return session -def _get_node_query_with_all_for_single_node(): - """Return a query object for the Node joined with all relevant fields. +def _get_node_select(): + """Returns a SQLAlchemy Select Object for Nodes. - This method utilizes a joined load query which creates a result set - where corresponding traits, and tags, are joined together in the result - set. - - This is more efficent from a Queries Per Second standpoint with the - backend database, as they are not separate distinct queries which - are being executed by the client. - - The downside of this, is the relationship of tags and traits to nodes - is that there may be multiple tags and traits for each node. Ultimately - this style of query forces SQLAlchemy to de-duplicate the result set - because the database returns the nodes portion of the result set for - every trait, tag, or other table field the query is joined with. - This looks like: - - node1, tag1, trait1 - node1, tag1, trait2 - node1, tag1, trait3 - node1, tag2, trait1 - - Et cetra, to create: - - node1, [tag1, tag2], [trait1, trait 2, trait3] - - Where joins are super in-efficent for Ironic, is where nodes are being - enumerated, as the above result set pattern is not just for one node, but - potentially thousands of nodes. In that case, we should use the - _get_node_query_with_all_for_list helper to return a more appropriate - query object which will be more efficient for the end user. + This method returns a pre-formatted select object which models + the entire Node object, allowing callers to operate on a node like + they would have with an SQLAlchemy ORM Query Object. - :returns: a query object. - """ - # NOTE(TheJulia): This *likely* ought to be selectinload, however - # it is a very common hit pattern for Ironic to query just the node. - # In those sorts of locations, the performance issues are less noticable - # to end users. *IF/WHEN* we change it to be selectinload for nodes, - # the resulting DB load will see a queries per second increase, which - # we should be careful about. - - # NOTE(TheJulia): Basic benchmark difference - # Test data creation: 67.202 seconds. - # 2.43 seconds to obtain all nodes from SQLAlchemy (10k nodes) - # 5.15 seconds to obtain all nodes *and* have node objects (10k nodes) - return (model_query(models.Node) - .options(joinedload('tags')) - .options(joinedload('traits'))) - - -def _get_node_query_with_all_for_list(): - """Return a query object for the Node with queried extra fields. - - This method returns a query object joining tags and traits in a pattern - where the result set is first built, and then the resulting associations - are queried separately and the objects are reconciled by SQLAlchemy to - build the composite objects based upon the associations. - - This results in the following query pattern when the query is executed: - - select $fields from nodes where x; - # SQLAlchemy creates a list of associated node IDs. - select $fields from tags where node_id in ('1', '3', '37268'); - select $fields from traits where node_id in ('1', '3', '37268'); - - SQLAlchemy then returns a result set where the tags and traits are - composited together efficently as opposed to having to deduplicate - the result set. This shifts additional load to the database which - was previously a high overhead operation with-in the conductor... - which results in a slower conductor. + This object *also* performs two additional select queries, in the form + of a selectin operation, to achieve the same results of a Join query, + but without the join query itself, and the client side load. + + This method is best utilized when retrieving lists of nodes. - :returns: a query object. + Select objects in this fashion were added as a result of SQLAlchemy 1.4 + in preparation for SQLAlchemy 2.0's release to provide a unified + select interface. + + :returns: a select object """ - # NOTE(TheJulia): When comparing CI rubs *with* this being the default - # for all general list operations, at 10k nodes, this pattern appears - # to be on-par with a 5% variability between the two example benchmark - # tests. That being said, the test *does* not include tags or traits - # in it's test data set so client side deduplication is not measured. - - # NOTE(TheJulia): Basic benchmark difference - # tests data creation: 67.117 seconds - # 2.32 seconds to obtain all nodes from SQLAlchemy (10k nodes) - # 4.99 seconds to obtain all nodes *and* have node objects (10k nodes) - # If this holds true, the required record deduplication with joinedload - # may be basically the same amount of overhead as requesting the tags - # and traits separately. - return (model_query(models.Node) - .options(selectinload('tags')) - .options(selectinload('traits'))) + # NOTE(TheJulia): This returns a query in the SQLAlchemy 1.4->2.0 + # migration style as query model loading is deprecated. + + # This must use selectinload to avoid later need to invokededuplication. + return (sa.select(models.Node) + .options(selectinload(models.Node.tags), + selectinload(models.Node.traits))) -def _get_deploy_template_query_with_steps(): - """Return a query object for the DeployTemplate joined with steps. - :returns: a query object. +def _get_deploy_template_select_with_steps(): + """Return a select object for the DeployTemplate joined with steps. + + :returns: a select object. """ - return model_query(models.DeployTemplate).options(joinedload('steps')) + return sa.select( + models.DeployTemplate + ).options(selectinload(models.DeployTemplate.steps)) def model_query(model, *args, **kwargs): @@ -209,6 +153,26 @@ raise exception.InvalidIdentity(identity=value) +def add_identity_where(op, model, value): + """Adds an identity filter to operation for where method. + + Filters results by ID, if supplied value is a valid integer. + Otherwise attempts to filter results by UUID. + + :param op: Initial operation to add filter to. + i.e. a update or delete statement. + :param model: The SQLAlchemy model to apply. + :param value: Value for filtering results by. + :return: Modified query. + """ + if strutils.is_int_like(value): + return op.where(model.id == value) + elif uuidutils.is_uuid_like(value): + return op.where(model.uuid == value) + else: + raise exception.InvalidIdentity(identity=value) + + def add_port_filter(query, value): """Adds a port-specific filter to a query. @@ -281,7 +245,7 @@ if netutils.is_valid_mac(value): return query.filter_by(address=value) else: - return add_identity_filter(query, value) + return add_identity_where(query, models.Portgroup, value) def add_portgroup_filter_by_node(query, value): @@ -332,9 +296,11 @@ def _paginate_query(model, limit=None, marker=None, sort_key=None, - sort_dir=None, query=None): - if not query: - query = model_query(model) + sort_dir=None, query=None, return_base_tuple=False): + # NOTE(TheJulia): We can't just ask for the bool of query if it is + # populated, so we need to ask if it is None. + if query is None: + query = sa.select(model) sort_keys = ['id'] if sort_key and sort_key not in sort_keys: sort_keys.insert(0, sort_key) @@ -345,7 +311,34 @@ raise exception.InvalidParameterValue( _('The sort_key value "%(key)s" is an invalid field for sorting') % {'key': sort_key}) - return query.all() + with _session_for_read() as session: + # NOTE(TheJulia): SQLAlchemy 2.0 no longer returns pre-uniqued result + # sets in ORM mode, so we need to explicitly ask for it to be unique + # before returning it to the caller. + if isinstance(query, sa_orm.Query): + # The classic "Legacy" ORM query object result set which is + # deprecated in advance of SQLAlchemy 2.0. + # TODO(TheJulia): Calls of this style basically need to be + # eliminated in ironic as returning this way does not allow + # commit or rollback in enginefacade to occur until the returned + # object is garbage collected as ORM Query objects allow + # for DB interactions to occur after the fact, so it remains + # connected to the DB.. + return query.all() + else: + # In this case, we have a sqlalchemy.sql.selectable.Select + # (most likely) which utilizes the unified select interface. + res = session.execute(query).fetchall() + if len(res) == 0: + # Return an empty list instead of a class with no objects. + return [] + if return_base_tuple: + # The caller expects a tuple, lets just give it to them. + return res + # Everything is a tuple in a resultset from the unified interface + # but for objects, our model expects just object access, + # so we extract and return them. + return [r[0] for r in res] def _filter_active_conductors(query, interval=None): @@ -405,10 +398,11 @@ 'uuid', 'id', 'fault', 'conductor_group', 'owner', 'lessee', 'instance_uuid'} _NODE_IN_QUERY_FIELDS = {'%s_in' % field: field - for field in ('uuid', 'provision_state')} + for field in ('uuid', 'provision_state', 'shard')} _NODE_NON_NULL_FILTERS = {'associated': 'instance_uuid', 'reserved': 'reservation', - 'with_power_state': 'power_state'} + 'with_power_state': 'power_state', + 'sharded': 'shard'} _NODE_FILTERS = ({'chassis_uuid', 'reserved_by_any_of', 'provisioned_before', 'inspection_started_before', 'description_contains', 'project'} @@ -514,15 +508,16 @@ else: columns = [getattr(models.Node, c) for c in columns] - query = model_query(*columns, base_model=models.Node) + query = sa.select(*columns) query = self._add_nodes_filters(query, filters) return _paginate_query(models.Node, limit, marker, - sort_key, sort_dir, query) + sort_key, sort_dir, query, + return_base_tuple=True) def get_node_list(self, filters=None, limit=None, marker=None, sort_key=None, sort_dir=None, fields=None): if not fields: - query = _get_node_query_with_all_for_list() + query = _get_node_select() query = self._add_nodes_filters(query, filters) return _paginate_query(models.Node, limit, marker, sort_key, sort_dir, query) @@ -559,24 +554,25 @@ # with SQLAlchemy. traits_found = True use_columns.remove('traits') - # Generate the column object list so SQLAlchemy only fulfills # the requested columns. use_columns = [getattr(models.Node, c) for c in use_columns] - # In essence, traits (and anything else needed to generate the # composite objects) need to be reconciled without using a join # as multiple rows can be generated in the result set being returned # from the database server. In this case, with traits, we use # a selectinload pattern. if traits_found: - query = model_query(models.Node).options( - Load(models.Node).load_only(*use_columns), - selectinload(models.Node.traits)) + query = sa.select(models.Node).options( + selectinload(models.Node.traits), + Load(models.Node).load_only(*use_columns) + ) else: - query = model_query(models.Node).options( - Load(models.Node).load_only(*use_columns)) - + # Note for others, if you ask for a whole model, it is + # modeled, i.e. you can access it as an object. + query = sa.select(models.NodeBase).options( + Load(models.Node).load_only(*use_columns) + ) query = self._add_nodes_filters(query, filters) return _paginate_query(models.Node, limit, marker, sort_key, sort_dir, query) @@ -597,19 +593,20 @@ raise exception.NodeNotFound( _("Nodes cannot be found: %s") % ', '.join(missing)) - query = model_query(models.Node.uuid, models.Node.name).filter( - sql.or_(models.Node.uuid.in_(uuids), - models.Node.name.in_(names)) - ) - if project: - query = query.filter((models.Node.owner == project) - | (models.Node.lessee == project)) - - for row in query: - if row[0] in idents: - mapping[row[0]] = row[0] - if row[1] and row[1] in idents: - mapping[row[1]] = row[0] + with _session_for_read() as session: + query = session.query(models.Node.uuid, models.Node.name).filter( + sql.or_(models.Node.uuid.in_(uuids), + models.Node.name.in_(names)) + ) + if project: + query = query.filter((models.Node.owner == project) + | (models.Node.lessee == project)) + + for row in query: + if row[0] in idents: + mapping[row[0]] = row[0] + if row[1] and row[1] in idents: + mapping[row[1]] = row[0] missing = idents - set(mapping) if missing: @@ -618,40 +615,85 @@ return mapping + @synchronized(RESERVATION_SEMAPHORE, fair=True) + def _reserve_node_place_lock(self, tag, node_id, node): + try: + # NOTE(TheJulia): We explicitly do *not* synch the session + # so the other actions in the conductor do not become aware + # that the lock is in place and believe they hold the lock. + # This necessitates an overall lock in the code side, so + # we avoid conditions where two separate threads can believe + # they hold locks at the same time. + with _session_for_write() as session: + res = session.execute( + sa.update(models.Node). + where(models.Node.id == node.id). + where(models.Node.reservation == None). # noqa + values(reservation=tag). + execution_options(synchronize_session=False)) + session.flush() + node = self._get_node_by_id_no_joins(node.id) + # NOTE(TheJulia): In SQLAlchemy 2.0 style, we don't + # magically get a changed node as they moved from the + # many ways to do things to singular ways to do things. + if res.rowcount != 1: + # Nothing updated and node exists. Must already be + # locked. + raise exception.NodeLocked(node=node.uuid, + host=node.reservation) + except NoResultFound: + # In the event that someone has deleted the node on + # another thread. + raise exception.NodeNotFound(node=node_id) + @oslo_db_api.retry_on_deadlock def reserve_node(self, tag, node_id): - with _session_for_write(): - query = _get_node_query_with_all_for_single_node() - query = add_identity_filter(query, node_id) - count = query.filter_by(reservation=None).update( - {'reservation': tag}, synchronize_session=False) + with _session_for_read() as session: try: + # TODO(TheJulia): Figure out a good way to query + # this so that we do it as light as possible without + # the full object invocation, which will speed lock + # activities. Granted, this is all at the DB level + # so maybe that is okay in the grand scheme of things. + query = session.query(models.Node) + query = add_identity_filter(query, node_id) node = query.one() - if count != 1: - # Nothing updated and node exists. Must already be - # locked. - raise exception.NodeLocked(node=node.uuid, - host=node['reservation']) - return node except NoResultFound: raise exception.NodeNotFound(node=node_id) + if node.reservation: + # Fail fast, instead of attempt the update. + raise exception.NodeLocked(node=node.uuid, + host=node.reservation) + self._reserve_node_place_lock(tag, node_id, node) + # Return a node object as that is the contract for this method. + return self.get_node_by_id(node.id) @oslo_db_api.retry_on_deadlock def release_node(self, tag, node_id): - with _session_for_write(): - query = model_query(models.Node) - query = add_identity_filter(query, node_id) - # be optimistic and assume we usually release a reservation - count = query.filter_by(reservation=tag).update( - {'reservation': None}, synchronize_session=False) - try: - if count != 1: - node = query.one() - if node['reservation'] is None: + with _session_for_read() as session: + try: + query = session.query(models.Node) + query = add_identity_filter(query, node_id) + node = query.one() + except NoResultFound: + raise exception.NodeNotFound(node=node_id) + with _session_for_write() as session: + try: + res = session.execute( + sa.update(models.Node). + where(models.Node.id == node.id). + where(models.Node.reservation == tag). + values(reservation=None). + execution_options(synchronize_session=False) + ) + node = self.get_node_by_id(node.id) + if res.rowcount != 1: + if node.reservation is None: raise exception.NodeNotLocked(node=node.uuid) else: raise exception.NodeLocked(node=node.uuid, host=node['reservation']) + session.flush() except NoResultFound: raise exception.NodeNotFound(node=node_id) @@ -677,47 +719,68 @@ node = models.Node() node.update(values) - with _session_for_write() as session: - try: + try: + with _session_for_write() as session: session.add(node) + # Set tags & traits to [] for new created node + # NOTE(mgoddard): We need to set the tags and traits fields in + # the session context, otherwise SQLAlchemy will try and fail + # to lazy load the attributes, resulting in an exception being + # raised. + node['tags'] = [] + node['traits'] = [] session.flush() - except db_exc.DBDuplicateEntry as exc: - if 'name' in exc.columns: - raise exception.DuplicateName(name=values['name']) - elif 'instance_uuid' in exc.columns: - raise exception.InstanceAssociated( - instance_uuid=values['instance_uuid'], - node=values['uuid']) - raise exception.NodeAlreadyExists(uuid=values['uuid']) - # Set tags & traits to [] for new created node - # NOTE(mgoddard): We need to set the tags and traits fields in the - # session context, otherwise SQLAlchemy will try and fail to lazy - # load the attributes, resulting in an exception being raised. - node['tags'] = [] - node['traits'] = [] + except db_exc.DBDuplicateEntry as exc: + if 'name' in exc.columns: + raise exception.DuplicateName(name=values['name']) + elif 'instance_uuid' in exc.columns: + raise exception.InstanceAssociated( + instance_uuid=values['instance_uuid'], + node=values['uuid']) + raise exception.NodeAlreadyExists(uuid=values['uuid']) return node + def _get_node_by_id_no_joins(self, node_id): + # TODO(TheJulia): Maybe replace with this with a minimal + # "get these three fields" thing. + try: + with _session_for_read() as session: + # Explicitly load NodeBase as the invocation of the + # priamary model object reesults in the join query + # triggering. + return session.execute( + sa.select(models.NodeBase).filter_by(id=node_id).limit(1) + ).scalars().first() + except NoResultFound: + raise exception.NodeNotFound(node=node_id) + def get_node_by_id(self, node_id): - query = _get_node_query_with_all_for_single_node() - query = query.filter_by(id=node_id) try: - return query.one() + query = _get_node_select() + with _session_for_read() as session: + return session.scalars( + query.filter_by(id=node_id).limit(1) + ).unique().one() except NoResultFound: raise exception.NodeNotFound(node=node_id) def get_node_by_uuid(self, node_uuid): - query = _get_node_query_with_all_for_single_node() - query = query.filter_by(uuid=node_uuid) try: - return query.one() + query = _get_node_select() + with _session_for_read() as session: + return session.scalars( + query.filter_by(uuid=node_uuid).limit(1) + ).unique().one() except NoResultFound: raise exception.NodeNotFound(node=node_uuid) def get_node_by_name(self, node_name): - query = _get_node_query_with_all_for_single_node() - query = query.filter_by(name=node_name) try: - return query.one() + query = _get_node_select() + with _session_for_read() as session: + return session.scalars( + query.filter_by(name=node_name).limit(1) + ).unique().one() except NoResultFound: raise exception.NodeNotFound(node=node_name) @@ -725,20 +788,19 @@ if not uuidutils.is_uuid_like(instance): raise exception.InvalidUUID(uuid=instance) - query = _get_node_query_with_all_for_single_node() - query = query.filter_by(instance_uuid=instance) - try: - result = query.one() + query = _get_node_select() + with _session_for_read() as session: + return session.scalars( + query.filter_by(instance_uuid=instance).limit(1) + ).unique().one() except NoResultFound: - raise exception.InstanceNotFound(instance=instance) - - return result + raise exception.InstanceNotFound(instance_uuid=instance) @oslo_db_api.retry_on_deadlock def destroy_node(self, node_id): with _session_for_write() as session: - query = model_query(models.Node) + query = session.query(models.Node) query = add_identity_filter(query, node_id) try: @@ -756,47 +818,53 @@ if uuidutils.is_uuid_like(node_id): node_id = node_ref['id'] - port_query = model_query(models.Port) + port_query = session.query(models.Port) port_query = add_port_filter_by_node(port_query, node_id) port_query.delete() - portgroup_query = model_query(models.Portgroup) + portgroup_query = session.query(models.Portgroup) portgroup_query = add_portgroup_filter_by_node(portgroup_query, node_id) portgroup_query.delete() # Delete all tags attached to the node - tag_query = model_query(models.NodeTag).filter_by(node_id=node_id) + tag_query = session.query(models.NodeTag).filter_by( + node_id=node_id) tag_query.delete() # Delete all traits attached to the node - trait_query = model_query( + trait_query = session.query( models.NodeTrait).filter_by(node_id=node_id) trait_query.delete() - volume_connector_query = model_query( + volume_connector_query = session.query( models.VolumeConnector).filter_by(node_id=node_id) volume_connector_query.delete() - volume_target_query = model_query( + volume_target_query = session.query( models.VolumeTarget).filter_by(node_id=node_id) volume_target_query.delete() # delete all bios attached to the node - bios_settings_query = model_query( + bios_settings_query = session.query( models.BIOSSetting).filter_by(node_id=node_id) bios_settings_query.delete() # delete all allocations for this node - allocation_query = model_query( + allocation_query = session.query( models.Allocation).filter_by(node_id=node_id) allocation_query.delete() # delete all history for this node - history_query = model_query( + history_query = session.query( models.NodeHistory).filter_by(node_id=node_id) history_query.delete() + # delete all inventory for this node + inventory_query = session.query( + models.NodeInventory).filter_by(node_id=node_id) + inventory_query.delete() + query.delete() def update_node(self, node_id, values): @@ -821,10 +889,10 @@ @oslo_db_api.retry_on_deadlock def _do_update_node(self, node_id, values): - with _session_for_write(): + with _session_for_write() as session: # NOTE(mgoddard): Don't issue a joined query for the update as this # does not work with PostgreSQL. - query = model_query(models.Node) + query = session.query(models.Node) query = add_identity_filter(query, node_id) try: ref = query.with_for_update().one() @@ -836,20 +904,26 @@ if values['provision_state'] == states.INSPECTING: values['inspection_started_at'] = timeutils.utcnow() values['inspection_finished_at'] = None - elif (ref.provision_state == states.INSPECTING + elif ((ref.provision_state == states.INSPECTING + or ref.provision_state == states.INSPECTWAIT) and values['provision_state'] == states.MANAGEABLE): values['inspection_finished_at'] = timeutils.utcnow() values['inspection_started_at'] = None - elif (ref.provision_state == states.INSPECTING + elif ((ref.provision_state == states.INSPECTING + or ref.provision_state == states.INSPECTWAIT) and values['provision_state'] == states.INSPECTFAIL): values['inspection_started_at'] = None ref.update(values) - # Return the updated node model joined with all relevant fields. - query = _get_node_query_with_all_for_single_node() - query = add_identity_filter(query, node_id) - return query.one() + # Return the updated node model joined with all relevant fields. + query = _get_node_select() + query = add_identity_filter(query, node_id) + # FIXME(TheJulia): This entire method needs to be re-written to + # use the proper execution format for SQLAlchemy 2.0. Likely + # A query, independent update, and a re-query on the transaction. + with _session_for_read() as session: + return session.execute(query).one()[0] def get_port_by_id(self, port_id): query = model_query(models.Port).filter_by(id=port_id) @@ -886,7 +960,7 @@ def get_port_list(self, limit=None, marker=None, sort_key=None, sort_dir=None, owner=None, project=None): - query = model_query(models.Port) + query = sa.select(models.Port) if owner: query = add_port_filter_by_node_owner(query, owner) elif project: @@ -894,11 +968,22 @@ return _paginate_query(models.Port, limit, marker, sort_key, sort_dir, query) + def get_ports_by_shards(self, shards, limit=None, marker=None, + sort_key=None, sort_dir=None): + shard_node_ids = sa.select(models.Node) \ + .where(models.Node.shard.in_(shards)) \ + .with_only_columns(models.Node.id) + with _session_for_read() as session: + query = session.query(models.Port).filter( + models.Port.node_id.in_(shard_node_ids)) + ports = _paginate_query( + models.Port, limit, marker, sort_key, sort_dir, query) + return ports + def get_ports_by_node_id(self, node_id, limit=None, marker=None, sort_key=None, sort_dir=None, owner=None, project=None): - query = model_query(models.Port) - query = query.filter_by(node_id=node_id) + query = sa.select(models.Port).where(models.Port.node_id == node_id) if owner: query = add_port_filter_by_node_owner(query, owner) elif project: @@ -909,8 +994,8 @@ def get_ports_by_portgroup_id(self, portgroup_id, limit=None, marker=None, sort_key=None, sort_dir=None, owner=None, project=None): - query = model_query(models.Port) - query = query.filter_by(portgroup_id=portgroup_id) + query = sa.select(models.Port).where( + models.Port.portgroup_id == portgroup_id) if owner: query = add_port_filter_by_node_owner(query, owner) elif project: @@ -925,15 +1010,15 @@ port = models.Port() port.update(values) - with _session_for_write() as session: - try: + try: + with _session_for_write() as session: session.add(port) session.flush() - except db_exc.DBDuplicateEntry as exc: - if 'address' in exc.columns: - raise exception.MACAlreadyExists(mac=values['address']) - raise exception.PortAlreadyExists(uuid=values['uuid']) - return port + except db_exc.DBDuplicateEntry as exc: + if 'address' in exc.columns: + raise exception.MACAlreadyExists(mac=values['address']) + raise exception.PortAlreadyExists(uuid=values['uuid']) + return port @oslo_db_api.retry_on_deadlock def update_port(self, port_id, values): @@ -941,10 +1026,9 @@ if 'uuid' in values: msg = _("Cannot overwrite UUID for an existing Port.") raise exception.InvalidParameterValue(err=msg) - try: with _session_for_write() as session: - query = model_query(models.Port) + query = session.query(models.Port) query = add_port_filter(query, port_id) ref = query.one() ref.update(values) @@ -960,8 +1044,8 @@ @oslo_db_api.retry_on_deadlock def destroy_port(self, port_id): - with _session_for_write(): - query = model_query(models.Port) + with _session_for_write() as session: + query = session.query(models.Port) query = add_port_filter(query, port_id) count = query.delete() if count == 0: @@ -1001,7 +1085,7 @@ def get_portgroup_list(self, limit=None, marker=None, sort_key=None, sort_dir=None, project=None): - query = model_query(models.Portgroup) + query = sa.select(models.Portgroup) if project: query = add_portgroup_filter_by_node_project(query, project) return _paginate_query(models.Portgroup, limit, marker, @@ -1009,8 +1093,8 @@ def get_portgroups_by_node_id(self, node_id, limit=None, marker=None, sort_key=None, sort_dir=None, project=None): - query = model_query(models.Portgroup) - query = query.filter_by(node_id=node_id) + query = sa.select(models.Portgroup) + query = query.where(models.Portgroup.node_id == node_id) if project: query = add_portgroup_filter_by_node_project(query, project) return _paginate_query(models.Portgroup, limit, marker, @@ -1046,7 +1130,7 @@ with _session_for_write() as session: try: - query = model_query(models.Portgroup) + query = session.query(models.Portgroup) query = add_portgroup_filter(query, portgroup_id) ref = query.one() ref.update(values) @@ -1067,34 +1151,40 @@ def destroy_portgroup(self, portgroup_id): def portgroup_not_empty(session): """Checks whether the portgroup does not have ports.""" - - query = model_query(models.Port) - query = add_port_filter_by_portgroup(query, portgroup_id) - - return query.count() != 0 + with _session_for_read() as session: + return session.scalar( + sa.select( + sa.func.count(models.Port.id) + ).where(models.Port.portgroup_id == portgroup_id)) != 0 with _session_for_write() as session: if portgroup_not_empty(session): raise exception.PortgroupNotEmpty(portgroup=portgroup_id) - query = model_query(models.Portgroup, session=session) - query = add_identity_filter(query, portgroup_id) + query = sa.delete(models.Portgroup) + query = add_identity_where(query, models.Portgroup, portgroup_id) - count = query.delete() + count = session.execute(query).rowcount if count == 0: raise exception.PortgroupNotFound(portgroup=portgroup_id) def get_chassis_by_id(self, chassis_id): - query = model_query(models.Chassis).filter_by(id=chassis_id) + query = sa.select(models.Chassis).where( + models.Chassis.id == chassis_id) + try: - return query.one() + with _session_for_read() as session: + return session.execute(query).one()[0] except NoResultFound: raise exception.ChassisNotFound(chassis=chassis_id) def get_chassis_by_uuid(self, chassis_uuid): - query = model_query(models.Chassis).filter_by(uuid=chassis_uuid) + query = sa.select(models.Chassis).where( + models.Chassis.uuid == chassis_uuid) + try: - return query.one() + with _session_for_read() as session: + return session.execute(query).one()[0] except NoResultFound: raise exception.ChassisNotFound(chassis=chassis_uuid) @@ -1110,13 +1200,13 @@ chassis = models.Chassis() chassis.update(values) - with _session_for_write() as session: - try: + try: + with _session_for_write() as session: session.add(chassis) session.flush() - except db_exc.DBDuplicateEntry: - raise exception.ChassisAlreadyExists(uuid=values['uuid']) - return chassis + except db_exc.DBDuplicateEntry: + raise exception.ChassisAlreadyExists(uuid=values['uuid']) + return chassis @oslo_db_api.retry_on_deadlock def update_chassis(self, chassis_id, values): @@ -1125,9 +1215,9 @@ msg = _("Cannot overwrite UUID for an existing Chassis.") raise exception.InvalidParameterValue(err=msg) - with _session_for_write(): - query = model_query(models.Chassis) - query = add_identity_filter(query, chassis_id) + with _session_for_write() as session: + query = session.query(models.Chassis) + query = add_identity_where(query, models.Chassis, chassis_id) count = query.update(values) if count != 1: @@ -1137,19 +1227,14 @@ @oslo_db_api.retry_on_deadlock def destroy_chassis(self, chassis_id): - def chassis_not_empty(): - """Checks whether the chassis does not have nodes.""" - - query = model_query(models.Node) + with _session_for_write() as session: + query = session.query(models.Node) query = add_node_filter_by_chassis(query, chassis_id) - return query.count() != 0 - - with _session_for_write(): - if chassis_not_empty(): + if query.count() != 0: raise exception.ChassisNotEmpty(chassis=chassis_id) - query = model_query(models.Chassis) + query = session.query(models.Chassis) query = add_identity_filter(query, chassis_id) count = query.delete() @@ -1159,7 +1244,7 @@ @oslo_db_api.retry_on_deadlock def register_conductor(self, values, update_existing=False): with _session_for_write() as session: - query = (model_query(models.Conductor) + query = (session.query(models.Conductor) .filter_by(hostname=values['hostname'])) try: ref = query.one() @@ -1183,39 +1268,46 @@ def get_conductor(self, hostname, online=True): try: - query = model_query(models.Conductor).filter_by(hostname=hostname) + query = sa.select(models.Conductor).where( + models.Conductor.hostname == hostname) if online is not None: - query = query.filter_by(online=online) - return query.one() + query = query.where(models.Conductor.online == online) + with _session_for_read() as session: + res = session.execute(query).one()[0] + return res except NoResultFound: raise exception.ConductorNotFound(conductor=hostname) @oslo_db_api.retry_on_deadlock def unregister_conductor(self, hostname): - with _session_for_write(): - query = (model_query(models.Conductor) - .filter_by(hostname=hostname, online=True)) - count = query.update({'online': False}) + with _session_for_write() as session: + query = sa.update(models.Conductor).where( + models.Conductor.hostname == hostname, + models.Conductor.online == True).values( # noqa + online=False) + count = session.execute(query).rowcount if count == 0: raise exception.ConductorNotFound(conductor=hostname) @oslo_db_api.retry_on_deadlock def touch_conductor(self, hostname): - with _session_for_write(): - query = (model_query(models.Conductor) - .filter_by(hostname=hostname)) - # since we're not changing any other field, manually set updated_at - # and since we're heartbeating, make sure that online=True - count = query.update({'updated_at': timeutils.utcnow(), - 'online': True}) - if count == 0: - raise exception.ConductorNotFound(conductor=hostname) + with _session_for_write() as session: + query = sa.update(models.Conductor).where( + models.Conductor.hostname == hostname + ).values({ + 'updated_at': timeutils.utcnow(), + 'online': True} + ).execution_options(synchronize_session=False) + res = session.execute(query) + count = res.rowcount + if count == 0: + raise exception.ConductorNotFound(conductor=hostname) @oslo_db_api.retry_on_deadlock def clear_node_reservations_for_conductor(self, hostname): nodes = [] - with _session_for_write(): - query = (model_query(models.Node) + with _session_for_write() as session: + query = (session.query(models.Node) .filter(models.Node.reservation.ilike(hostname))) nodes = [node['uuid'] for node in query] query.update({'reservation': None}, synchronize_session=False) @@ -1229,8 +1321,8 @@ @oslo_db_api.retry_on_deadlock def clear_node_target_power_state(self, hostname): nodes = [] - with _session_for_write(): - query = (model_query(models.Node) + with _session_for_write() as session: + query = (session.query(models.Node) .filter(models.Node.reservation.ilike(hostname))) query = query.filter(models.Node.target_power_state != sql.null()) nodes = [node['uuid'] for node in query] @@ -1248,46 +1340,51 @@ '%(nodes)s', {'nodes': nodes}) def get_active_hardware_type_dict(self, use_groups=False): - query = (model_query(models.ConductorHardwareInterfaces, - models.Conductor) - .join(models.Conductor)) - result = _filter_active_conductors(query) - - d2c = collections.defaultdict(set) - for iface_row, cdr_row in result: - hw_type = iface_row['hardware_type'] - if use_groups: - key = '%s:%s' % (cdr_row['conductor_group'], hw_type) - else: - key = hw_type - d2c[key].add(cdr_row['hostname']) + with _session_for_read() as session: + query = (session.query(models.ConductorHardwareInterfaces, + models.Conductor) + .join(models.Conductor)) + result = _filter_active_conductors(query) + + d2c = collections.defaultdict(set) + for iface_row, cdr_row in result: + hw_type = iface_row['hardware_type'] + if use_groups: + key = '%s:%s' % (cdr_row['conductor_group'], hw_type) + else: + key = hw_type + d2c[key].add(cdr_row['hostname']) return d2c def get_offline_conductors(self, field='hostname'): - field = getattr(models.Conductor, field) - interval = CONF.conductor.heartbeat_timeout - limit = timeutils.utcnow() - datetime.timedelta(seconds=interval) - result = (model_query(field) - .filter(models.Conductor.updated_at < limit)) - return [row[0] for row in result] + with _session_for_read() as session: + field = getattr(models.Conductor, field) + interval = CONF.conductor.heartbeat_timeout + limit = timeutils.utcnow() - datetime.timedelta(seconds=interval) + result = (session.query(field) + .filter(models.Conductor.updated_at < limit)) + return [row[0] for row in result] def get_online_conductors(self): - query = model_query(models.Conductor.hostname) - query = _filter_active_conductors(query) - return [row[0] for row in query] + with _session_for_read() as session: + query = session.query(models.Conductor.hostname) + query = _filter_active_conductors(query) + return [row[0] for row in query] def list_conductor_hardware_interfaces(self, conductor_id): - query = (model_query(models.ConductorHardwareInterfaces) - .filter_by(conductor_id=conductor_id)) - return query.all() + with _session_for_read() as session: + query = (session.query(models.ConductorHardwareInterfaces) + .filter_by(conductor_id=conductor_id)) + return query.all() def list_hardware_type_interfaces(self, hardware_types): - query = (model_query(models.ConductorHardwareInterfaces) - .filter(models.ConductorHardwareInterfaces.hardware_type - .in_(hardware_types))) + with _session_for_read() as session: + query = (session.query(models.ConductorHardwareInterfaces) + .filter(models.ConductorHardwareInterfaces.hardware_type + .in_(hardware_types))) - query = _filter_active_conductors(query) - return query.all() + query = _filter_active_conductors(query) + return query.all() @oslo_db_api.retry_on_deadlock def register_conductor_hardware_interfaces(self, conductor_id, interfaces): @@ -1307,22 +1404,23 @@ @oslo_db_api.retry_on_deadlock def unregister_conductor_hardware_interfaces(self, conductor_id): - with _session_for_write(): - query = (model_query(models.ConductorHardwareInterfaces) + with _session_for_write() as session: + query = (session.query(models.ConductorHardwareInterfaces) .filter_by(conductor_id=conductor_id)) query.delete() @oslo_db_api.retry_on_deadlock def touch_node_provisioning(self, node_id): - with _session_for_write(): - query = model_query(models.Node) + with _session_for_write() as session: + query = session.query(models.Node) query = add_identity_filter(query, node_id) count = query.update({'provision_updated_at': timeutils.utcnow()}) if count == 0: raise exception.NodeNotFound(node=node_id) - def _check_node_exists(self, node_id): - if not model_query(models.Node).filter_by(id=node_id).scalar(): + def _check_node_exists(self, session, node_id): + if not session.query(models.Node).where( + models.Node.id == node_id).scalar(): raise exception.NodeNotFound(node=node_id) @oslo_db_api.retry_on_deadlock @@ -1341,24 +1439,25 @@ @oslo_db_api.retry_on_deadlock def unset_node_tags(self, node_id): - self._check_node_exists(node_id) - with _session_for_write(): - model_query(models.NodeTag).filter_by(node_id=node_id).delete() + with _session_for_write() as session: + self._check_node_exists(session, node_id) + session.query(models.NodeTag).filter_by(node_id=node_id).delete() def get_node_tags_by_node_id(self, node_id): - self._check_node_exists(node_id) - result = (model_query(models.NodeTag) - .filter_by(node_id=node_id) - .all()) + with _session_for_read() as session: + self._check_node_exists(session, node_id) + result = (session.query(models.NodeTag) + .filter_by(node_id=node_id) + .all()) return result @oslo_db_api.retry_on_deadlock def add_node_tag(self, node_id, tag): - node_tag = models.NodeTag(tag=tag, node_id=node_id) - - self._check_node_exists(node_id) try: with _session_for_write() as session: + node_tag = models.NodeTag(tag=tag, node_id=node_id) + + self._check_node_exists(session, node_id) session.add(node_tag) session.flush() except db_exc.DBDuplicateEntry: @@ -1369,26 +1468,33 @@ @oslo_db_api.retry_on_deadlock def delete_node_tag(self, node_id, tag): - self._check_node_exists(node_id) - with _session_for_write(): - result = model_query(models.NodeTag).filter_by( + with _session_for_write() as session: + self._check_node_exists(session, node_id) + result = session.query(models.NodeTag).filter_by( node_id=node_id, tag=tag).delete() - if not result: - raise exception.NodeTagNotFound(node_id=node_id, tag=tag) + if not result: + raise exception.NodeTagNotFound(node_id=node_id, tag=tag) def node_tag_exists(self, node_id, tag): - self._check_node_exists(node_id) - q = model_query(models.NodeTag).filter_by(node_id=node_id, tag=tag) - return model_query(q.exists()).scalar() + with _session_for_read() as session: + self._check_node_exists(session, node_id) + q = session.query(models.NodeTag).filter_by( + node_id=node_id, tag=tag) + return session.query(q.exists()).scalar() def get_node_by_port_addresses(self, addresses): - q = _get_node_query_with_all_for_single_node() + q = _get_node_select() q = q.distinct().join(models.Port) q = q.filter(models.Port.address.in_(addresses)) try: - return q.one() + # FIXME(TheJulia): This needs to be updated to be + # an explicit query to identify the node for SQLAlchemy. + with _session_for_read() as session: + # Always return the first element, since we always + # get a tuple from sqlalchemy. + return session.execute(q).one()[0] except NoResultFound: raise exception.NodeNotFound( _('Node with port addresses %s was not found') @@ -1400,7 +1506,7 @@ def get_volume_connector_list(self, limit=None, marker=None, sort_key=None, sort_dir=None, project=None): - query = model_query(models.VolumeConnector) + query = sa.select(models.VolumeConnector) if project: query = add_volume_conn_filter_by_node_project(query, project) return _paginate_query(models.VolumeConnector, limit, marker, @@ -1424,7 +1530,8 @@ def get_volume_connectors_by_node_id(self, node_id, limit=None, marker=None, sort_key=None, sort_dir=None, project=None): - query = model_query(models.VolumeConnector).filter_by(node_id=node_id) + query = sa.select(models.VolumeConnector).where( + models.VolumeConnector.node_id == node_id) if project: add_volume_conn_filter_by_node_project(query, project) return _paginate_query(models.VolumeConnector, limit, marker, @@ -1458,7 +1565,7 @@ try: with _session_for_write() as session: - query = model_query(models.VolumeConnector) + query = session.query(models.VolumeConnector) query = add_identity_filter(query, ident) ref = query.one() orig_type = ref['type'] @@ -1476,8 +1583,8 @@ @oslo_db_api.retry_on_deadlock def destroy_volume_connector(self, ident): - with _session_for_write(): - query = model_query(models.VolumeConnector) + with _session_for_write() as session: + query = session.query(models.VolumeConnector) query = add_identity_filter(query, ident) count = query.delete() if count == 0: @@ -1485,14 +1592,15 @@ def get_volume_target_list(self, limit=None, marker=None, sort_key=None, sort_dir=None, project=None): - query = model_query(models.VolumeTarget) + query = sa.select(models.VolumeTarget) if project: query = add_volume_target_filter_by_node_project(query, project) return _paginate_query(models.VolumeTarget, limit, marker, sort_key, sort_dir, query) def get_volume_target_by_id(self, db_id): - query = model_query(models.VolumeTarget).filter_by(id=db_id) + query = model_query(models.VolumeTarget).where( + models.VolumeTarget.id == db_id) try: return query.one() except NoResultFound: @@ -1508,7 +1616,8 @@ def get_volume_targets_by_node_id(self, node_id, limit=None, marker=None, sort_key=None, sort_dir=None, project=None): - query = model_query(models.VolumeTarget).filter_by(node_id=node_id) + query = sa.select(models.VolumeTarget).where( + models.VolumeTarget.node_id == node_id) if project: add_volume_target_filter_by_node_project(query, project) return _paginate_query(models.VolumeTarget, limit, marker, sort_key, @@ -1517,7 +1626,8 @@ def get_volume_targets_by_volume_id(self, volume_id, limit=None, marker=None, sort_key=None, sort_dir=None, project=None): - query = model_query(models.VolumeTarget).filter_by(volume_id=volume_id) + query = sa.select(models.VolumeTarget).where( + models.VolumeTarget.volume_id == volume_id) if project: query = add_volume_target_filter_by_node_project(query, project) return _paginate_query(models.VolumeTarget, limit, marker, sort_key, @@ -1550,7 +1660,7 @@ try: with _session_for_write() as session: - query = model_query(models.VolumeTarget) + query = session.query(models.VolumeTarget) query = add_identity_filter(query, ident) ref = query.one() orig_boot_index = ref['boot_index'] @@ -1565,8 +1675,8 @@ @oslo_db_api.retry_on_deadlock def destroy_volume_target(self, ident): - with _session_for_write(): - query = model_query(models.VolumeTarget) + with _session_for_write() as session: + query = session.query(models.VolumeTarget) query = add_identity_filter(query, ident) count = query.delete() if count == 0: @@ -1586,6 +1696,8 @@ if not versions: return [] + if model_name == 'Node': + model_name = 'NodeBase' model = models.get_class(model_name) # NOTE(rloo): .notin_ does not handle null: @@ -1614,7 +1726,11 @@ """ object_versions = release_mappings.get_object_versions() table_missing_ok = False - for model in models.Base.__subclasses__(): + models_to_check = models.Base.__subclasses__() + # We need to append Node to the list as it is a subclass of + # NodeBase, which is intentional to delineate excess queries. + models_to_check.append(models.Node) + for model in models_to_check: if model.__name__ not in object_versions: continue @@ -1688,13 +1804,15 @@ mapping = release_mappings.RELEASE_MAPPING['master']['objects'] total_to_migrate = 0 total_migrated = 0 - - sql_models = [model for model in models.Base.__subclasses__() + all_models = models.Base.__subclasses__() + all_models.append(models.Node) + sql_models = [model for model in all_models if model.__name__ in mapping] - for model in sql_models: - version = mapping[model.__name__][0] - query = model_query(model).filter(model.version != version) - total_to_migrate += query.count() + with _session_for_read() as session: + for model in sql_models: + version = mapping[model.__name__][0] + query = session.query(model).filter(model.version != version) + total_to_migrate += query.count() if not total_to_migrate: return total_to_migrate, 0 @@ -1716,10 +1834,13 @@ max_to_migrate = max_count or total_to_migrate for model in sql_models: + use_node_id = False + if (not hasattr(model, 'id') and hasattr(model, 'node_id')): + use_node_id = True version = mapping[model.__name__][0] num_migrated = 0 - with _session_for_write(): - query = model_query(model).filter(model.version != version) + with _session_for_write() as session: + query = session.query(model).filter(model.version != version) # NOTE(rloo) Caution here; after doing query.count(), it is # possible that the value is different in the # next invocation of the query. @@ -1729,16 +1850,30 @@ # max_to_migrate objects. ids = [] for obj in query.slice(0, max_to_migrate): - ids.append(obj['id']) - num_migrated = ( - model_query(model). - filter(sql.and_(model.id.in_(ids), - model.version != version)). - update({model.version: version}, - synchronize_session=False)) + if not use_node_id: + ids.append(obj['id']) + else: + # BIOSSettings, NodeTrait, NodeTag do not have id + # columns, fallback to node_id as they both have + # it. + ids.append(obj['node_id']) + if not use_node_id: + num_migrated = ( + session.query(model). + filter(sql.and_(model.id.in_(ids), + model.version != version)). + update({model.version: version}, + synchronize_session=False)) + else: + num_migrated = ( + session.query(model). + filter(sql.and_(model.node_id.in_(ids), + model.version != version)). + update({model.version: version}, + synchronize_session=False)) else: num_migrated = ( - model_query(model). + session.query(model). filter(model.version != version). update({model.version: version}, synchronize_session=False)) @@ -1789,15 +1924,16 @@ @oslo_db_api.retry_on_deadlock def unset_node_traits(self, node_id): - self._check_node_exists(node_id) - with _session_for_write(): - model_query(models.NodeTrait).filter_by(node_id=node_id).delete() + with _session_for_write() as session: + self._check_node_exists(session, node_id) + session.query(models.NodeTrait).filter_by(node_id=node_id).delete() def get_node_traits_by_node_id(self, node_id): - self._check_node_exists(node_id) - result = (model_query(models.NodeTrait) - .filter_by(node_id=node_id) - .all()) + with _session_for_read() as session: + self._check_node_exists(session, node_id) + result = (session.query(models.NodeTrait) + .filter_by(node_id=node_id) + .all()) return result @oslo_db_api.retry_on_deadlock @@ -1805,13 +1941,14 @@ node_trait = models.NodeTrait(trait=trait, node_id=node_id, version=version) - self._check_node_exists(node_id) try: with _session_for_write() as session: + self._check_node_exists(session, node_id) + session.add(node_trait) session.flush() - num_traits = (model_query(models.NodeTrait) + num_traits = (session.query(models.NodeTrait) .filter_by(node_id=node_id).count()) self._verify_max_traits_per_node(node_id, num_traits) except db_exc.DBDuplicateEntry: @@ -1822,25 +1959,26 @@ @oslo_db_api.retry_on_deadlock def delete_node_trait(self, node_id, trait): - self._check_node_exists(node_id) - with _session_for_write(): - result = model_query(models.NodeTrait).filter_by( + with _session_for_write() as session: + self._check_node_exists(session, node_id) + result = session.query(models.NodeTrait).filter_by( node_id=node_id, trait=trait).delete() - if not result: - raise exception.NodeTraitNotFound(node_id=node_id, trait=trait) + if not result: + raise exception.NodeTraitNotFound(node_id=node_id, trait=trait) def node_trait_exists(self, node_id, trait): - self._check_node_exists(node_id) - q = model_query( - models.NodeTrait).filter_by(node_id=node_id, trait=trait) - return model_query(q.exists()).scalar() + with _session_for_read() as session: + self._check_node_exists(session, node_id) + q = session.query( + models.NodeTrait).filter_by(node_id=node_id, trait=trait) + return session.query(q.exists()).scalar() @oslo_db_api.retry_on_deadlock def create_bios_setting_list(self, node_id, settings, version): - self._check_node_exists(node_id) bios_settings = [] with _session_for_write() as session: + self._check_node_exists(session, node_id) try: for setting in settings: bios_setting = models.BIOSSetting( @@ -1867,12 +2005,12 @@ @oslo_db_api.retry_on_deadlock def update_bios_setting_list(self, node_id, settings, version): - self._check_node_exists(node_id) bios_settings = [] with _session_for_write() as session: + self._check_node_exists(session, node_id) try: for setting in settings: - query = model_query(models.BIOSSetting).filter_by( + query = session.query(models.BIOSSetting).filter_by( node_id=node_id, name=setting['name']) ref = query.one() ref.update({'value': setting['value'], @@ -1898,11 +2036,11 @@ @oslo_db_api.retry_on_deadlock def delete_bios_setting_list(self, node_id, names): - self._check_node_exists(node_id) missing_bios_settings = [] - with _session_for_write(): + with _session_for_write() as session: + self._check_node_exists(session, node_id) for name in names: - count = model_query(models.BIOSSetting).filter_by( + count = session.query(models.BIOSSetting).filter_by( node_id=node_id, name=name).delete() if count == 0: missing_bios_settings.append(name) @@ -1911,20 +2049,22 @@ node=node_id, names=','.join(missing_bios_settings)) def get_bios_setting(self, node_id, name): - self._check_node_exists(node_id) - query = model_query(models.BIOSSetting).filter_by( - node_id=node_id, name=name) - try: - ref = query.one() - except NoResultFound: - raise exception.BIOSSettingNotFound(node=node_id, name=name) + with _session_for_read() as session: + self._check_node_exists(session, node_id) + query = session.query(models.BIOSSetting).filter_by( + node_id=node_id, name=name) + try: + ref = query.one() + except NoResultFound: + raise exception.BIOSSettingNotFound(node=node_id, name=name) return ref def get_bios_setting_list(self, node_id): - self._check_node_exists(node_id) - result = (model_query(models.BIOSSetting) - .filter_by(node_id=node_id) - .all()) + with _session_for_read() as session: + self._check_node_exists(session, node_id) + result = (session.query(models.BIOSSetting) + .filter_by(node_id=node_id) + .all()) return result def get_allocation_by_id(self, allocation_id): @@ -1934,11 +2074,13 @@ :returns: An allocation. :raises: AllocationNotFound """ - query = model_query(models.Allocation).filter_by(id=allocation_id) - try: - return query.one() - except NoResultFound: - raise exception.AllocationNotFound(allocation=allocation_id) + with _session_for_read() as session: + query = session.query(models.Allocation).filter_by( + id=allocation_id) + try: + return query.one() + except NoResultFound: + raise exception.AllocationNotFound(allocation=allocation_id) def get_allocation_by_uuid(self, allocation_uuid): """Return an allocation representation. @@ -1947,11 +2089,13 @@ :returns: An allocation. :raises: AllocationNotFound """ - query = model_query(models.Allocation).filter_by(uuid=allocation_uuid) - try: - return query.one() - except NoResultFound: - raise exception.AllocationNotFound(allocation=allocation_uuid) + with _session_for_read() as session: + query = session.query(models.Allocation).filter_by( + uuid=allocation_uuid) + try: + return query.one() + except NoResultFound: + raise exception.AllocationNotFound(allocation=allocation_uuid) def get_allocation_by_name(self, name): """Return an allocation representation. @@ -1960,11 +2104,12 @@ :returns: An allocation. :raises: AllocationNotFound """ - query = model_query(models.Allocation).filter_by(name=name) - try: - return query.one() - except NoResultFound: - raise exception.AllocationNotFound(allocation=name) + with _session_for_read() as session: + query = session.query(models.Allocation).filter_by(name=name) + try: + return query.one() + except NoResultFound: + raise exception.AllocationNotFound(allocation=name) def get_allocation_list(self, filters=None, limit=None, marker=None, sort_key=None, sort_dir=None): @@ -1983,8 +2128,9 @@ (asc, desc) :returns: A list of allocations. """ - query = self._add_allocations_filters(model_query(models.Allocation), - filters) + query = self._add_allocations_filters( + sa.select(models.Allocation), + filters) return _paginate_query(models.Allocation, limit, marker, sort_key, sort_dir, query) @@ -2039,16 +2185,16 @@ # initialized, but set them to None just in case. instance_uuid = node_uuid = None - with _session_for_write() as session: - try: - query = model_query(models.Allocation, session=session) + try: + with _session_for_write() as session: + query = session.query(models.Allocation) query = add_identity_filter(query, allocation_id) ref = query.one() ref.update(values) instance_uuid = ref.uuid if values.get('node_id') and update_node: - node = model_query(models.Node, session=session).filter_by( + node = session.query(models.Node).filter_by( id=ref.node_id).with_for_update().one() node_uuid = node.uuid if node.instance_uuid and node.instance_uuid != ref.uuid: @@ -2060,20 +2206,26 @@ 'instance_uuid': instance_uuid, 'instance_info': iinfo}) session.flush() - except NoResultFound: - raise exception.AllocationNotFound(allocation=allocation_id) - except db_exc.DBDuplicateEntry as exc: - if 'name' in exc.columns: - raise exception.AllocationDuplicateName( - name=values['name']) - elif 'instance_uuid' in exc.columns: - # Case when the allocation UUID is already used on some - # node as instance_uuid. - raise exception.InstanceAssociated( - instance_uuid=instance_uuid, node=node_uuid) - else: - raise - return ref + # Perform a separate read so the commit closes out on the write + # transaction. + with _session_for_read() as session: + query = session.query(models.Allocation) + query = add_identity_filter(query, allocation_id) + ref = query.one() + except NoResultFound: + raise exception.AllocationNotFound(allocation=allocation_id) + except db_exc.DBDuplicateEntry as exc: + if 'name' in exc.columns: + raise exception.AllocationDuplicateName( + name=values['name']) + elif 'instance_uuid' in exc.columns: + # Case when the allocation UUID is already used on some + # node as instance_uuid. + raise exception.InstanceAssociated( + instance_uuid=instance_uuid, node=node_uuid) + else: + raise + return ref @oslo_db_api.retry_on_deadlock def take_over_allocation(self, allocation_id, old_conductor_id, @@ -2091,9 +2243,9 @@ :returns: True if the take over was successful, False otherwise. :raises: AllocationNotFound """ - with _session_for_write() as session: - try: - query = model_query(models.Allocation, session=session) + try: + with _session_for_write() as session: + query = session.query(models.Allocation) query = add_identity_filter(query, allocation_id) # NOTE(dtantsur): the FOR UPDATE clause locks the allocation ref = query.with_for_update().one() @@ -2103,10 +2255,10 @@ ref.update({'conductor_affinity': new_conductor_id}) session.flush() - except NoResultFound: - raise exception.AllocationNotFound(allocation=allocation_id) - else: - return True + except NoResultFound: + raise exception.AllocationNotFound(allocation=allocation_id) + else: + return True @oslo_db_api.retry_on_deadlock def destroy_allocation(self, allocation_id): @@ -2116,17 +2268,21 @@ :raises: AllocationNotFound """ with _session_for_write() as session: - query = model_query(models.Allocation) + query = session.query(models.Allocation) query = add_identity_filter(query, allocation_id) try: - ref = query.one() + # NOTE(TheJulia): We explicitly need to indicate we intend + # to update the record so we block until the other users of + # the row are free, such as the process to match the + # allocation to a node. + ref = query.with_for_update().one() except NoResultFound: raise exception.AllocationNotFound(allocation=allocation_id) allocation_id = ref['id'] - node_query = model_query(models.Node, session=session).filter_by( + node_query = session.query(models.Node).filter_by( allocation_id=allocation_id) node_query.update({'allocation_id': None, 'instance_uuid': None}) @@ -2179,7 +2335,7 @@ return step.interface, step.step, sortable_args, step.priority # List all existing steps for the template. - current_steps = (model_query(models.DeployTemplateStep) + current_steps = (session.query(models.DeployTemplateStep) .filter_by(deploy_template_id=template_id)) # List the new steps for the template. @@ -2203,7 +2359,7 @@ # Delete and create steps in bulk as necessary. if step_ids_to_delete: - ((model_query(models.DeployTemplateStep) + ((session.query(models.DeployTemplateStep) .filter(models.DeployTemplateStep.id.in_(step_ids_to_delete))) .delete(synchronize_session=False)) if steps_to_create: @@ -2219,70 +2375,81 @@ with _session_for_write() as session: # NOTE(mgoddard): Don't issue a joined query for the update as # this does not work with PostgreSQL. - query = model_query(models.DeployTemplate) + query = session.query(models.DeployTemplate) query = add_identity_filter(query, template_id) - try: - ref = query.with_for_update().one() - except NoResultFound: - raise exception.DeployTemplateNotFound( - template=template_id) - + ref = query.with_for_update().one() # First, update non-step columns. steps = values.pop('steps', None) ref.update(values) - # If necessary, update steps. if steps is not None: self._update_deploy_template_steps(session, ref.id, steps) + session.flush() + with _session_for_read() as session: # Return the updated template joined with all relevant fields. - query = _get_deploy_template_query_with_steps() + query = _get_deploy_template_select_with_steps() query = add_identity_filter(query, template_id) - return query.one() + return session.execute(query).one()[0] except db_exc.DBDuplicateEntry as e: if 'name' in e.columns: raise exception.DeployTemplateDuplicateName( name=values['name']) raise + except NoResultFound: + # TODO(TheJulia): What would unified core raise?!? + raise exception.DeployTemplateNotFound( + template=template_id) @oslo_db_api.retry_on_deadlock def destroy_deploy_template(self, template_id): - with _session_for_write(): - model_query(models.DeployTemplateStep).filter_by( + with _session_for_write() as session: + session.query(models.DeployTemplateStep).filter_by( deploy_template_id=template_id).delete() - count = model_query(models.DeployTemplate).filter_by( + count = session.query(models.DeployTemplate).filter_by( id=template_id).delete() if count == 0: raise exception.DeployTemplateNotFound(template=template_id) def _get_deploy_template(self, field, value): """Helper method for retrieving a deploy template.""" - query = (_get_deploy_template_query_with_steps() - .filter_by(**{field: value})) + query = (_get_deploy_template_select_with_steps() + .where(field == value)) try: - return query.one() + # FIXME(TheJulia): This needs to be fixed for SQLAlchemy 2.0 + with _session_for_read() as session: + return session.execute(query).one()[0] except NoResultFound: raise exception.DeployTemplateNotFound(template=value) def get_deploy_template_by_id(self, template_id): - return self._get_deploy_template('id', template_id) + return self._get_deploy_template(models.DeployTemplate.id, + template_id) def get_deploy_template_by_uuid(self, template_uuid): - return self._get_deploy_template('uuid', template_uuid) + return self._get_deploy_template(models.DeployTemplate.uuid, + template_uuid) def get_deploy_template_by_name(self, template_name): - return self._get_deploy_template('name', template_name) + return self._get_deploy_template(models.DeployTemplate.name, + template_name) def get_deploy_template_list(self, limit=None, marker=None, sort_key=None, sort_dir=None): - query = _get_deploy_template_query_with_steps() + query = model_query(models.DeployTemplate).options( + selectinload(models.DeployTemplate.steps)) return _paginate_query(models.DeployTemplate, limit, marker, sort_key, sort_dir, query) def get_deploy_template_list_by_names(self, names): - query = (_get_deploy_template_query_with_steps() - .filter(models.DeployTemplate.name.in_(names))) - return query.all() + query = _get_deploy_template_select_with_steps() + with _session_for_read() as session: + res = session.execute( + query.where( + models.DeployTemplate.name.in_(names) + ) + ).all() + return [r[0] for r in res] @oslo_db_api.retry_on_deadlock def create_node_history(self, values): @@ -2300,8 +2467,8 @@ @oslo_db_api.retry_on_deadlock def destroy_node_history_by_uuid(self, history_uuid): - with _session_for_write(): - query = model_query(models.NodeHistory).filter_by( + with _session_for_write() as session: + query = session.query(models.NodeHistory).filter_by( uuid=history_uuid) count = query.delete() if count == 0: @@ -2329,7 +2496,7 @@ def get_node_history_by_node_id(self, node_id, limit=None, marker=None, sort_key=None, sort_dir=None): query = model_query(models.NodeHistory) - query = query.filter_by(node_id=node_id) + query = query.where(models.NodeHistory.node_id == node_id) return _paginate_query(models.NodeHistory, limit, marker, sort_key, sort_dir, query) @@ -2396,6 +2563,9 @@ # Uses input entry list, selects entries matching those ids # then deletes them and does not synchronize the session so # sqlalchemy doesn't do extra un-necessary work. + # NOTE(TheJulia): This is "legacy" syntax, but it is still + # valid and under the hood SQLAlchemy rewrites the form into + # a delete syntax. session.query( models.NodeHistory ).filter( @@ -2414,13 +2584,70 @@ # literally have the DB do *all* of the world, so no # client side ops occur. The column is also indexed, # which means this will be an index based response. - # TODO(TheJulia): This might need to be revised for - # SQLAlchemy 2.0 as it should be a scaler select and count - # instead. - return session.query( - models.Node.provision_state - ).filter( - or_( - models.Node.provision_state == v for v in state + return session.scalar( + sa.select( + sa.func.count(models.Node.id) + ).filter( + or_( + models.Node.provision_state == v for v in state + ) ) - ).count() + ) + + @oslo_db_api.retry_on_deadlock + def create_node_inventory(self, values): + inventory = models.NodeInventory() + inventory.update(values) + with _session_for_write() as session: + try: + session.add(inventory) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.NodeInventoryAlreadyExists( + id=values['id']) + return inventory + + @oslo_db_api.retry_on_deadlock + def destroy_node_inventory_by_node_id(self, node_id): + with _session_for_write() as session: + query = session.query(models.NodeInventory).filter_by( + node_id=node_id) + count = query.delete() + if count == 0: + raise exception.NodeInventoryNotFound( + node=node_id) + + def get_node_inventory_by_node_id(self, node_id): + query = model_query(models.NodeInventory).filter_by(node_id=node_id) + try: + return query.one() + except NoResultFound: + raise exception.NodeInventoryNotFound(node=node_id) + + def get_shard_list(self): + """Return a list of shards. + + :returns: A list of dicts containing the keys name and count. + """ + # Note(JayF): This should never be a large enough list to require + # pagination. Furthermore, it wouldn't really be a sensible + # thing to paginate as the data it's fetching can mutate. + # So we just aren't even going to try. + shard_list = [] + with _session_for_read() as session: + res = session.execute( + # Note(JayF): SQLAlchemy counts are notoriously slow because + # sometimes they will use a subquery. Be careful + # before changing this to use any magic. + sa.text( + "SELECT count(id), shard from nodes group by shard;" + )).fetchall() + + if res: + res.sort(key=lambda x: x[0], reverse=True) + for shard in res: + shard_list.append( + {"name": str(shard[1]), "count": shard[0]} + ) + + return shard_list diff -Nru ironic-21.1.0/ironic/db/sqlalchemy/models.py ironic-21.4.4/ironic/db/sqlalchemy/models.py --- ironic-21.1.0/ironic/db/sqlalchemy/models.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/db/sqlalchemy/models.py 2024-10-11 15:42:16.000000000 +0000 @@ -19,16 +19,18 @@ """ from os import path +from typing import List from urllib import parse as urlparse from oslo_db import options as db_options from oslo_db.sqlalchemy import models from oslo_db.sqlalchemy import types as db_types +from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy import Boolean, Column, DateTime, false, Index from sqlalchemy import ForeignKey, Integer from sqlalchemy import schema, String, Text -from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import orm +from sqlalchemy.orm import declarative_base from ironic.common import exception from ironic.common.i18n import _ @@ -116,8 +118,8 @@ default = Column(Boolean, default=False, nullable=False) -class Node(Base): - """Represents a bare metal node.""" +class NodeBase(Base): + """Represents a base bare metal node.""" __tablename__ = 'nodes' __table_args__ = ( @@ -132,6 +134,7 @@ Index('reservation_idx', 'reservation'), Index('conductor_group_idx', 'conductor_group'), Index('resource_class_idx', 'resource_class'), + Index('shard_idx', 'shard'), table_args()) id = Column(Integer, primary_key=True) uuid = Column(String(36)) @@ -212,6 +215,34 @@ boot_mode = Column(String(16), nullable=True) secure_boot = Column(Boolean, nullable=True) + shard = Column(String(255), nullable=True) + + +class Node(NodeBase): + """Represents a bare metal node.""" + + # NOTE(TheJulia): The purpose of the delineation between NodeBase and Node + # is to facilitate a hard delineation for queries where we do not need to + # populate additional information needlessly which would normally populate + # from the access of the property. In this case, Traits and Tags. + # The other reason we do this, is because these are generally "joined" + # data structures, we cannot de-duplicate node objects with unhashable dict + # data structures. + + # NOTE(TheJulia): The choice of selectin lazy population is intentional + # as it causes a subselect to occur, skipping the need for deduplication + # in general. This puts a slightly higher query load on the DB server, but + # means *far* less gets shipped over the wire in the end. + traits: orm.Mapped[List['NodeTrait']] = orm.relationship( # noqa + "NodeTrait", + back_populates="node", + lazy="selectin") + + tags: orm.Mapped[List['NodeTag']] = orm.relationship( # noqa + "NodeTag", + back_populates="node", + lazy="selectin") + class Port(Base): """Represents a network port of a bare metal node.""" @@ -235,6 +266,15 @@ is_smartnic = Column(Boolean, nullable=True, default=False) name = Column(String(255), nullable=True) + _node_uuid = orm.relationship( + "Node", + viewonly=True, + primaryjoin="(Node.id == Port.node_id)", + lazy="selectin", + ) + node_uuid = association_proxy( + "_node_uuid", "uuid", creator=lambda _i: Node(uuid=_i)) + class Portgroup(Base): """Represents a group of network ports of a bare metal node.""" @@ -256,6 +296,15 @@ mode = Column(String(255)) properties = Column(db_types.JsonEncodedDict) + _node_uuid = orm.relationship( + "Node", + viewonly=True, + primaryjoin="(Node.id == Portgroup.node_id)", + lazy="selectin", + ) + node_uuid = association_proxy( + "_node_uuid", "uuid", creator=lambda _i: Node(uuid=_i)) + class NodeTag(Base): """Represents a tag of a bare metal node.""" @@ -270,7 +319,6 @@ node = orm.relationship( "Node", - backref='tags', primaryjoin='and_(NodeTag.node_id == Node.id)', foreign_keys=node_id ) @@ -327,7 +375,6 @@ trait = Column(String(255), primary_key=True, nullable=False) node = orm.relationship( "Node", - backref='traits', primaryjoin='and_(NodeTrait.node_id == Node.id)', foreign_keys=node_id ) @@ -389,6 +436,10 @@ uuid = Column(String(36)) name = Column(String(255), nullable=False) extra = Column(db_types.JsonEncodedDict) + steps: orm.Mapped[List['DeployTemplateStep']] = orm.relationship( # noqa + "DeployTemplateStep", + back_populates="deploy_template", + lazy="selectin") class DeployTemplateStep(Base): @@ -409,7 +460,6 @@ priority = Column(Integer, nullable=False) deploy_template = orm.relationship( "DeployTemplate", - backref='steps', primaryjoin=( 'and_(DeployTemplateStep.deploy_template_id == ' 'DeployTemplate.id)'), @@ -437,6 +487,18 @@ node_id = Column(Integer, ForeignKey('nodes.id'), nullable=True) +class NodeInventory(Base): + """Represents an inventory of a baremetal node.""" + __tablename__ = 'node_inventory' + __table_args__ = ( + Index('inventory_node_id_idx', 'node_id'), + table_args()) + id = Column(Integer, primary_key=True) + inventory_data = Column(db_types.JsonEncodedDict(mysql_as_long=True)) + plugin_data = Column(db_types.JsonEncodedDict(mysql_as_long=True)) + node_id = Column(Integer, ForeignKey('nodes.id'), nullable=True) + + def get_class(model_name): """Returns the model class with the specified name. diff -Nru ironic-21.1.0/ironic/drivers/irmc.py ironic-21.4.4/ironic/drivers/irmc.py --- ironic-21.1.0/ironic/drivers/irmc.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/irmc.py 2024-10-11 15:42:16.000000000 +0000 @@ -27,6 +27,7 @@ from ironic.drivers.modules.irmc import management from ironic.drivers.modules.irmc import power from ironic.drivers.modules.irmc import raid +from ironic.drivers.modules.irmc import vendor from ironic.drivers.modules import noop from ironic.drivers.modules import pxe @@ -48,8 +49,8 @@ """List of supported boot interfaces.""" # NOTE: Support for pxe boot is deprecated, and will be # removed from the list in the future. - return [boot.IRMCVirtualMediaBoot, boot.IRMCPXEBoot, - ipxe.iPXEBoot, pxe.PXEBoot] + return [boot.IRMCVirtualMediaBoot, ipxe.iPXEBoot, + boot.IRMCPXEBoot, pxe.PXEBoot] @property def supported_console_interfaces(self): @@ -77,3 +78,8 @@ def supported_raid_interfaces(self): """List of supported raid interfaces.""" return [noop.NoRAID, raid.IRMCRAID, agent.AgentRAID] + + @property + def supported_vendor_interfaces(self): + """List of supported vendor interfaces.""" + return [noop.NoVendor, vendor.IRMCVendorPassthru] diff -Nru ironic-21.1.0/ironic/drivers/modules/agent_base.py ironic-21.4.4/ironic/drivers/modules/agent_base.py --- ironic-21.1.0/ironic/drivers/modules/agent_base.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/agent_base.py 2024-10-11 15:42:16.000000000 +0000 @@ -100,7 +100,7 @@ FASTTRACK_HEARTBEAT_ALLOWED = frozenset(_FASTTRACK_HEARTBEAT_ALLOWED) -@METRICS.timer('post_clean_step_hook') +@METRICS.timer('AgentBase.post_clean_step_hook') def post_clean_step_hook(interface, step): """Decorator method for adding a post clean step hook. @@ -128,7 +128,7 @@ return decorator -@METRICS.timer('post_deploy_step_hook') +@METRICS.timer('AgentBase.post_deploy_step_hook') def post_deploy_step_hook(interface, step): """Decorator method for adding a post deploy step hook. @@ -279,7 +279,7 @@ return last_command -@METRICS.timer('log_and_raise_deployment_error') +@METRICS.timer('AgentBase.log_and_raise_deployment_error') def log_and_raise_deployment_error(task, msg, collect_logs=True, exc=None): """Helper method to log the error and raise exception. @@ -1334,11 +1334,28 @@ try: persistent = True + # NOTE(TheJulia): We *really* only should be doing this in bios + # boot mode. In UEFI this might just get disregarded, or cause + # issues/failures. if node.driver_info.get('force_persistent_boot_device', 'Default') == 'Never': persistent = False - deploy_utils.try_set_boot_device(task, boot_devices.DISK, - persistent=persistent) + + vendor = task.node.properties.get('vendor', None) + if not (vendor and vendor.lower() == 'lenovo' + and target_boot_mode == 'uefi'): + # Lenovo hardware is modeled on a "just update" + # UEFI nvram model of use, and if multiple actions + # get requested, you can end up in cases where NVRAM + # changes are deleted as the host "restores" to the + # backup. For more information see + # https://bugs.launchpad.net/ironic/+bug/2053064 + # NOTE(TheJulia): We likely just need to do this with + # all hosts in uefi mode, but libvirt VMs don't handle + # nvram only changes *and* this pattern is known to generally + # work for Ironic operators. + deploy_utils.try_set_boot_device(task, boot_devices.DISK, + persistent=persistent) except Exception as e: msg = (_("Failed to change the boot device to %(boot_dev)s " "when deploying node %(node)s: %(error)s") % diff -Nru ironic-21.1.0/ironic/drivers/modules/ansible/playbooks/roles/deploy/tasks/write.yaml ironic-21.4.4/ironic/drivers/modules/ansible/playbooks/roles/deploy/tasks/write.yaml --- ironic-21.1.0/ironic/drivers/modules/ansible/playbooks/roles/deploy/tasks/write.yaml 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/ansible/playbooks/roles/deploy/tasks/write.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -1,6 +1,6 @@ - name: convert and write become: yes - command: qemu-img convert -t directsync -O host_device /tmp/{{ inventory_hostname }}.img {{ ironic_image_target }} + command: qemu-img convert -f {{ ironic.image.disk_format }} -t directsync -O host_device /tmp/{{ inventory_hostname }}.img {{ ironic_image_target }} async: 1200 poll: 10 when: ironic.image.disk_format != 'raw' diff -Nru ironic-21.1.0/ironic/drivers/modules/boot_mode_utils.py ironic-21.4.4/ironic/drivers/modules/boot_mode_utils.py --- ironic-21.1.0/ironic/drivers/modules/boot_mode_utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/boot_mode_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -137,7 +137,7 @@ elif ironic_boot_mode != bm_boot_mode: msg = (_("Boot mode %(node_boot_mode)s currently configured " "on node %(uuid)s does not match the boot mode " - "%(ironic_boot_mode)s requested for provisioning." + "%(ironic_boot_mode)s requested for provisioning. " "Attempting to set node boot mode to %(ironic_boot_mode)s.") % {'uuid': node.uuid, 'node_boot_mode': bm_boot_mode, 'ironic_boot_mode': ironic_boot_mode}) diff -Nru ironic-21.1.0/ironic/drivers/modules/console_utils.py ironic-21.4.4/ironic/drivers/modules/console_utils.py --- ironic-21.1.0/ironic/drivers/modules/console_utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/console_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -90,7 +90,7 @@ with open(pid_path, 'r') as f: pid_str = f.readline() return int(pid_str) - except (IOError, ValueError): + except (IOError, ValueError, FileNotFoundError): raise exception.NoConsolePid(pid_path=pid_path) diff -Nru ironic-21.1.0/ironic/drivers/modules/deploy_utils.py ironic-21.4.4/ironic/drivers/modules/deploy_utils.py --- ironic-21.1.0/ironic/drivers/modules/deploy_utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/deploy_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -16,7 +16,6 @@ import os import re -import time from ironic_lib import metrics_utils from ironic_lib import utils as il_utils @@ -25,6 +24,8 @@ from oslo_utils import fileutils from oslo_utils import strutils +from ironic.common import checksum_utils +from ironic.common import context from ironic.common import exception from ironic.common import faults from ironic.common.glance_service import service_utils @@ -66,6 +67,7 @@ DISK_LAYOUT_PARAMS = ('root_gb', 'swap_mb', 'ephemeral_gb') + # All functions are called from deploy() directly or indirectly. # They are split for stub-out. @@ -211,7 +213,9 @@ 'missing_info': missing_info}) -def fetch_images(ctx, cache, images_info, force_raw=True): +def fetch_images(ctx, cache, images_info, force_raw=True, + expected_format=None, expected_checksum=None, + expected_checksum_algo=None): """Check for available disk space and fetch images using ImageCache. :param ctx: context @@ -219,7 +223,15 @@ :param images_info: list of tuples (image href, destination path) :param force_raw: boolean value, whether to convert the image to raw format + :param expected_format: The expected format of the image. + :param expected_checksum: The expected image checksum, to be used if we + need to convert the image to raw prior to deploying. + :param expected_checksum_algo: The checksum algo in use, if separately + set. :raises: InstanceDeployFailure if unable to find enough disk space + :raises: InvalidImage if the supplied image metadata or contents are + deemed to be invalid, unsafe, or not matching the expectations + asserted by configuration supplied or set. """ try: @@ -231,8 +243,17 @@ # if disk space is used between the check and actual download. # This is probably unavoidable, as we can't control other # (probably unrelated) processes + image_list = [] for href, path in images_info: - cache.fetch_image(href, path, ctx=ctx, force_raw=force_raw) + # NOTE(TheJulia): Href in this case can be an image UUID or a URL. + image_format = cache.fetch_image( + href, path, ctx=ctx, + force_raw=force_raw, + expected_format=expected_format, + expected_checksum=expected_checksum, + expected_checksum_algo=expected_checksum_algo) + image_list.append((href, path, image_format)) + return image_list def set_failed_state(task, msg, collect_logs=True): @@ -998,7 +1019,8 @@ @METRICS.timer('cache_instance_image') -def cache_instance_image(ctx, node, force_raw=None): +def cache_instance_image(ctx, node, force_raw=None, expected_format=None, + expected_checksum=None, expected_checksum_algo=None): """Fetch the instance's image from Glance This method pulls the disk image and writes them to the appropriate @@ -1007,8 +1029,16 @@ :param ctx: context :param node: an ironic node object :param force_raw: whether convert image to raw format + :param expected_format: The expected format of the disk image contents. + :param expected_checksum: The expected image checksum, to be used if we + need to convert the image to raw prior to deploying. + :param expected_checksum_algo: The checksum algo in use, if separately + set. :returns: a tuple containing the uuid of the image and the path in the filesystem where image is cached. + :raises: InvalidImage if the requested image is invalid and cannot be + used for deployed based upon contents of the image or the metadata + surrounding the image not matching the configured image. """ # NOTE(dtantsur): applying the default here to make the option mutable if force_raw is None: @@ -1022,10 +1052,11 @@ LOG.debug("Fetching image %(image)s for node %(uuid)s", {'image': uuid, 'uuid': node.uuid}) - fetch_images(ctx, InstanceImageCache(), [(uuid, image_path)], - force_raw) - - return (uuid, image_path) + image_list = fetch_images(ctx, InstanceImageCache(), [(uuid, image_path)], + force_raw, expected_format=expected_format, + expected_checksum=expected_checksum, + expected_checksum_algo=expected_checksum_algo) + return (uuid, image_path, image_list[0][2]) @METRICS.timer('destroy_images') @@ -1042,17 +1073,11 @@ @METRICS.timer('compute_image_checksum') def compute_image_checksum(image_path, algorithm='md5'): """Compute checksum by given image path and algorithm.""" - time_start = time.time() - LOG.debug('Start computing %(algo)s checksum for image %(image)s.', - {'algo': algorithm, 'image': image_path}) - checksum = fileutils.compute_file_checksum(image_path, - algorithm=algorithm) - time_elapsed = time.time() - time_start - LOG.debug('Computed %(algo)s checksum for image %(image)s in ' - '%(delta).2f seconds, checksum value: %(checksum)s.', - {'algo': algorithm, 'image': image_path, 'delta': time_elapsed, - 'checksum': checksum}) - return checksum + # NOTE(TheJulia): This likely wouldn't be removed, but if we do + # significant refactoring we could likely just change everything + # over to the images common code, if we don't need the metrics + # data anymore. + return checksum_utils.compute_image_checksum(image_path, algorithm) def remove_http_instance_symlink(node_uuid): @@ -1066,13 +1091,39 @@ destroy_images(node.uuid) -def _validate_image_url(node, url, secret=False): +def _validate_image_url(node, url, secret=False, inspect_image=None, + expected_format=None): """Validates image URL through the HEAD request. :param url: URL to be validated :param secret: if URL is secret (e.g. swift temp url), it will not be shown in logs. - """ + :param inspect_image: If the requested URL should have extensive + content checking applied. Defaults to the value provided by + the [conductor]conductor_always_validates_images configuration + parameter setting, but is also able to be turned off by supplying + False where needed to perform a redirect or URL head request only. + :param expected_format: The expected image format, if known, for + the image inspection logic. + :returns: Returns a dictionary with basic information about the + requested image if image introspection is + """ + if inspect_image is not None: + # The caller has a bit more context and we can rely upon it, + # for example if it knows we cannot or should not inspect + # the image contents. + inspect = inspect_image + elif not CONF.conductor.disable_deep_image_inspection: + inspect = CONF.conductor.conductor_always_validates_images + else: + # If we're here, file inspection has been explicitly disabled. + inspect = False + + # NOTE(TheJulia): This method gets used in two different ways. + # The first is as a "i did a thing, let me make sure my url works." + # The second is to validate a remote URL is valid. In the remote case + # we will grab the file and proceed from there. + image_info = {} try: # NOTE(TheJulia): This method only validates that an exception # is NOT raised. In other words, that the endpoint does not @@ -1084,14 +1135,58 @@ LOG.error("The specified URL is not a valid HTTP(S) URL or is " "not reachable for node %(node)s: %(msg)s", {'node': node.uuid, 'msg': e}) + if inspect: + LOG.info("Inspecting image contents for %(node)s with url %(url)s. " + "Expecting user supplied format: %(expected)s", + {'node': node.uuid, + 'expected': expected_format, + 'url': url}) + # Utilizes the file cache since it knows how to pull files down + # and handles pathing and caching and all that fun, however with + # force_raw set as false. + + # The goal here being to get the file we would normally just point + # IPA at, be it via swift transfer *or* direct URL request, and + # perform the safety check on it before allowing it to proceed. + ctx = context.get_admin_context() + # NOTE(TheJulia): Because we're using the image cache here, we + # let it run the image validation checking as it's normal course + # of action, and save what it tells us the image format is. + # if there *was* a mismatch, it will raise the error. + + # NOTE(TheJulia): We don't need to supply the checksum here, because + # we are not converting the image. The net result is the deploy + # interface or remote agent has the responsibility to checksum the + # image. + _, image_path, img_format = cache_instance_image( + ctx, + node, + force_raw=False, + expected_format=expected_format) + # NOTE(TheJulia): We explicitly delete this file because it has no use + # in the cache after this point. + il_utils.unlink_without_raise(image_path) + image_info['disk_format'] = img_format + return image_info def _cache_and_convert_image(task, instance_info, image_info=None): """Cache an image locally and covert it to RAW if needed.""" # Ironic cache and serve images from httpboot server force_raw = direct_deploy_should_convert_raw_image(task.node) - _, image_path = cache_instance_image(task.context, task.node, - force_raw=force_raw) + + if image_info is None: + initial_format = instance_info.get('image_disk_format') + else: + initial_format = image_info.get('disk_format') + checksum, checksum_algo = checksum_utils.get_checksum_and_algo( + instance_info) + _, image_path, img_format = cache_instance_image( + task.context, task.node, + force_raw=force_raw, + expected_format=initial_format, + expected_checksum=checksum, + expected_checksum_algo=checksum_algo) if force_raw or image_info is None: if force_raw: instance_info['image_disk_format'] = 'raw' @@ -1108,21 +1203,30 @@ # sha256. if image_info is None: os_hash_algo = instance_info.get('image_os_hash_algo') + hash_value = instance_info.get('image_os_hash_value') + old_checksum = instance_info.get('image_checksum') else: os_hash_algo = image_info.get('os_hash_algo') + hash_value = image_info.get('os_hash_value') + old_checksum = image_info.get('checksum') + + if initial_format != instance_info['image_disk_format']: + if not os_hash_algo or os_hash_algo == 'md5': + LOG.debug("Checksum algorithm for image %(image)s for node " + "%(node)s is set to '%(algo)s', changing to sha256", + {'algo': os_hash_algo, 'node': task.node.uuid, + 'image': image_path}) + os_hash_algo = 'sha256' + + LOG.debug('Recalculating checksum for image %(image)s for node ' + '%(node)s due to image conversion', + {'image': image_path, 'node': task.node.uuid}) + instance_info['image_checksum'] = None + hash_value = checksum_utils.compute_image_checksum(image_path, + os_hash_algo) + else: + instance_info['image_checksum'] = old_checksum - if not os_hash_algo or os_hash_algo == 'md5': - LOG.debug("Checksum algorithm for image %(image)s for node " - "%(node)s is set to '%(algo)s', changing to 'sha256'", - {'algo': os_hash_algo, 'node': task.node.uuid, - 'image': image_path}) - os_hash_algo = 'sha256' - - LOG.debug('Recalculating checksum for image %(image)s for node ' - '%(node)s due to image conversion', - {'image': image_path, 'node': task.node.uuid}) - instance_info['image_checksum'] = None - hash_value = compute_image_checksum(image_path, os_hash_algo) instance_info['image_os_hash_algo'] = os_hash_algo instance_info['image_os_hash_value'] = hash_value else: @@ -1165,7 +1269,11 @@ task.node.uuid]) if file_extension: http_image_url = http_image_url + file_extension - _validate_image_url(task.node, http_image_url, secret=False) + # We don't inspect the image in our url check because we just need to do + # an quick path validity check here, we should be checking contents way + # earlier on in this method. + _validate_image_url(task.node, http_image_url, secret=False, + inspect_image=False) instance_info['image_url'] = http_image_url @@ -1190,29 +1298,57 @@ instance_info = node.instance_info iwdi = node.driver_internal_info.get('is_whole_disk_image') image_source = instance_info['image_source'] + + # Flag if we know the source is a path, used for Anaconda + # deploy interface where you can just tell anaconda to + # consume artifacts from a path. In this case, we are not + # doing any image conversions, we're just passing through + # a URL in the form of configuration. isap = node.driver_internal_info.get('is_source_a_path') + # If our url ends with a /, i.e. we have been supplied with a path, # we can only deploy this in limited cases for drivers and tools # which are aware of such. i.e. anaconda. image_download_source = get_image_download_source(node) boot_option = get_boot_option(task.node) + # There is no valid reason this should already be set, and + # and gets replaced at various points in this sequence. + instance_info['image_url'] = None + if service_utils.is_glance_image(image_source): glance = image_service.GlanceImageService(context=task.context) image_info = glance.show(image_source) LOG.debug('Got image info: %(info)s for node %(node)s.', {'info': image_info, 'node': node.uuid}) + # Values are explicitly set into the instance info field + # so IPA have the values available. + instance_info['image_checksum'] = image_info['checksum'] + instance_info['image_os_hash_algo'] = image_info['os_hash_algo'] + instance_info['image_os_hash_value'] = image_info['os_hash_value'] if image_download_source == 'swift': + # In this case, we are getting a file *from* swift for a glance + # image which is backed by swift. IPA downloads the file directly + # from swift, but cannot get any metadata related to it otherwise. swift_temp_url = glance.swift_temp_url(image_info) - _validate_image_url(node, swift_temp_url, secret=True) + image_format = image_info.get('disk_format') + # In the process of validating the URL is valid, we will perform + # the requisite safety checking of the asset as we can end up + # converting it in the agent, or needing the disk format value + # to be correct for the Ansible deployment interface. + validate_results = _validate_image_url( + node, swift_temp_url, secret=True, + expected_format=image_format) instance_info['image_url'] = swift_temp_url - instance_info['image_checksum'] = image_info['checksum'] - instance_info['image_disk_format'] = image_info['disk_format'] - instance_info['image_os_hash_algo'] = image_info['os_hash_algo'] - instance_info['image_os_hash_value'] = image_info['os_hash_value'] + instance_info['image_disk_format'] = \ + validate_results.get('disk_format', image_format) else: + # In this case, we're directly downloading the glance image and + # hosting it locally for retrieval by the IPA. _cache_and_convert_image(task, instance_info, image_info) + # We're just populating extra information for a glance backed image in + # case a deployment interface driver needs them at some point. instance_info['image_container_format'] = ( image_info['container_format']) instance_info['image_tags'] = image_info.get('tags', []) @@ -1223,20 +1359,80 @@ instance_info['ramdisk'] = image_info['properties']['ramdisk_id'] elif (image_source.startswith('file://') or image_download_source == 'local'): + # In this case, we're explicitly downloading (or copying a file) + # hosted locally so IPA can download it directly from Ironic. + # NOTE(TheJulia): Intentionally only supporting file:/// as image # based deploy source since we don't want to, nor should we be in # in the business of copying large numbers of files as it is a # huge performance impact. + _cache_and_convert_image(task, instance_info) else: + # This is the "all other cases" logic for aspects like the user + # has supplied us a direct URL to reference. In cases like the + # anaconda deployment interface where we might just have a path + # and not a file, or where a user may be supplying a full URL to + # a remotely hosted image, we at a minimum need to check if the url + # is valid, and address any redirects upfront. try: - _validate_image_url(node, image_source) + # NOTE(TheJulia): In the case we're here, we not doing an + # integrated image based deploy, but we may also be doing + # a path based anaconda base deploy, in which case we have + # no backing image, but we need to check for a URL + # redirection. So, if the source is a path (i.e. isap), + # we don't need to inspect the image as there is no image + # in the case for the deployment to drive. + validated_results = {} + if isap: + # This is if the source is a path url, such as one used by + # anaconda templates to to rely upon bootstrapping defaults. + _validate_image_url(node, image_source, inspect_image=False) + else: + # When not isap, we can just let _validate_image_url make a + # the required decision on if contents need to be sampled, + # or not. We try to pass the image_disk_format which may be + # declared by the user, and if not we set expected_format to + # None. + validate_results = _validate_image_url( + node, + image_source, + expected_format=instance_info.get('image_disk_format', + None)) # image_url is internal, and used by IPA and some boot templates. # in most cases, it needs to come from image_source explicitly. + if 'disk_format' in validated_results: + # Ensure IPA has the value available, so write what we detect, + # if anything. This is also an item which might be needful + # with ansible deploy interface, when used in standalone mode. + instance_info['image_disk_format'] = \ + validate_results.get('disk_format') instance_info['image_url'] = image_source except exception.ImageRefIsARedirect as e: + # At this point, we've got a redirect response from the webserver, + # and we're going to try to handle it as a single redirect action, + # as requests, by default, only lets a single redirect to occur. + # This is likely a URL pathing fix, like a trailing / on a path, + # or move to HTTPS from a user supplied HTTP url. if e.redirect_url: + # Since we've got a redirect, we need to carry the rest of the + # request logic as well, which includes recording a disk + # format, if applicable. instance_info['image_url'] = e.redirect_url + # We need to save the image_source back out so it caches + instance_info['image_source'] = e.redirect_url + task.node.instance_info = instance_info + if not isap: + # The redirect doesn't relate to a path being used, so + # the target is a filename, likely cause is webserver + # telling the client to use HTTPS. + validated_results = _validate_image_url( + node, e.redirect_url, + expected_format=instance_info.get('image_disk_format', + None)) + if 'disk_format' in validated_results: + instance_info['image_disk_format'] = \ + validated_results.get('disk_format') else: raise @@ -1251,7 +1447,6 @@ # Call central parsing so we retain things like config drives. i_info = parse_instance_info(node, image_deploy=False) instance_info.update(i_info) - return instance_info diff -Nru ironic-21.1.0/ironic/drivers/modules/fake.py ironic-21.4.4/ironic/drivers/modules/fake.py --- ironic-21.1.0/ironic/drivers/modules/fake.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/fake.py 2024-10-11 15:42:16.000000000 +0000 @@ -24,6 +24,9 @@ on separate vendor_passthru methods. """ +import random +import time + from oslo_log import log from ironic.common import boot_devices @@ -32,6 +35,7 @@ from ironic.common.i18n import _ from ironic.common import indicator_states from ironic.common import states +from ironic.conf import CONF from ironic.drivers import base from ironic import objects @@ -39,6 +43,34 @@ LOG = log.getLogger(__name__) +def parse_sleep_range(sleep_range): + if not sleep_range: + return 0, 0 + + sleep_split = sleep_range.split(',') + if len(sleep_split) == 1: + a = sleep_split[0] + b = sleep_split[0] + else: + a = sleep_split[0] + b = sleep_split[1] + return int(a), int(b) + + +def sleep(sleep_range): + earliest, latest = parse_sleep_range(sleep_range) + if earliest == 0 and latest == 0: + # no sleep + return + if earliest == latest: + # constant sleep + sleep = earliest + else: + # triangular random sleep, weighted towards the earliest + sleep = random.triangular(earliest, latest, earliest) + time.sleep(sleep) + + class FakePower(base.PowerInterface): """Example implementation of a simple power interface.""" @@ -49,12 +81,15 @@ pass def get_power_state(self, task): + sleep(CONF.fake.power_delay) return task.node.power_state def reboot(self, task, timeout=None): + sleep(CONF.fake.power_delay) pass def set_power_state(self, task, power_state, timeout=None): + sleep(CONF.fake.power_delay) if power_state not in [states.POWER_ON, states.POWER_OFF, states.SOFT_REBOOT, states.SOFT_POWER_OFF]: raise exception.InvalidParameterValue( @@ -81,15 +116,19 @@ pass def prepare_ramdisk(self, task, ramdisk_params, mode='deploy'): + sleep(CONF.fake.boot_delay) pass def clean_up_ramdisk(self, task, mode='deploy'): + sleep(CONF.fake.boot_delay) pass def prepare_instance(self, task): + sleep(CONF.fake.boot_delay) pass def clean_up_instance(self, task): + sleep(CONF.fake.boot_delay) pass @@ -108,18 +147,23 @@ @base.deploy_step(priority=100) def deploy(self, task): + sleep(CONF.fake.deploy_delay) return None def tear_down(self, task): + sleep(CONF.fake.deploy_delay) return states.DELETED def prepare(self, task): + sleep(CONF.fake.deploy_delay) pass def clean_up(self, task): + sleep(CONF.fake.deploy_delay) pass def take_over(self, task): + sleep(CONF.fake.deploy_delay) pass @@ -140,6 +184,7 @@ @base.passthru(['POST'], description=_("Test if the value of bar is baz")) def first_method(self, task, http_method, bar): + sleep(CONF.fake.vendor_delay) return True if bar == 'baz' else False @@ -161,16 +206,19 @@ @base.passthru(['POST'], description=_("Test if the value of bar is kazoo")) def second_method(self, task, http_method, bar): + sleep(CONF.fake.vendor_delay) return True if bar == 'kazoo' else False @base.passthru(['POST'], async_call=False, description=_("Test if the value of bar is meow")) def third_method_sync(self, task, http_method, bar): + sleep(CONF.fake.vendor_delay) return True if bar == 'meow' else False @base.passthru(['POST'], require_exclusive_lock=False, description=_("Test if the value of bar is woof")) def fourth_method_shared_lock(self, task, http_method, bar): + sleep(CONF.fake.vendor_delay) return True if bar == 'woof' else False @@ -211,17 +259,21 @@ return [boot_devices.PXE] def set_boot_device(self, task, device, persistent=False): + sleep(CONF.fake.management_delay) if device not in self.get_supported_boot_devices(task): raise exception.InvalidParameterValue(_( "Invalid boot device %s specified.") % device) def get_boot_device(self, task): + sleep(CONF.fake.management_delay) return {'boot_device': boot_devices.PXE, 'persistent': False} def get_sensors_data(self, task): + sleep(CONF.fake.management_delay) return {} def get_supported_indicators(self, task, component=None): + sleep(CONF.fake.management_delay) indicators = { components.CHASSIS: { 'led-0': { @@ -248,6 +300,7 @@ if not component or component == c} def get_indicator_state(self, task, component, indicator): + sleep(CONF.fake.management_delay) indicators = self.get_supported_indicators(task) if component not in indicators: raise exception.InvalidParameterValue(_( @@ -271,6 +324,7 @@ pass def inspect_hardware(self, task): + sleep(CONF.fake.inspect_delay) return states.MANAGEABLE @@ -282,9 +336,11 @@ def create_configuration(self, task, create_root_volume=True, create_nonroot_volumes=True): + sleep(CONF.fake.raid_delay) pass def delete_configuration(self, task): + sleep(CONF.fake.raid_delay) pass @@ -302,6 +358,7 @@ 'to contain a dictionary with name/value pairs'), 'required': True}}) def apply_configuration(self, task, settings): + sleep(CONF.fake.bios_delay) # Note: the implementation of apply_configuration in fake interface # is just for testing purpose, for real driver implementation, please # refer to develop doc at https://docs.openstack.org/ironic/latest/ @@ -328,6 +385,7 @@ @base.clean_step(priority=0) def factory_reset(self, task): + sleep(CONF.fake.bios_delay) # Note: the implementation of factory_reset in fake interface is # just for testing purpose, for real driver implementation, please # refer to develop doc at https://docs.openstack.org/ironic/latest/ @@ -340,6 +398,7 @@ @base.clean_step(priority=0) def cache_bios_settings(self, task): + sleep(CONF.fake.bios_delay) # Note: the implementation of cache_bios_settings in fake interface # is just for testing purpose, for real driver implementation, please # refer to develop doc at https://docs.openstack.org/ironic/latest/ @@ -357,9 +416,11 @@ return {} def attach_volumes(self, task): + sleep(CONF.fake.storage_delay) pass def detach_volumes(self, task): + sleep(CONF.fake.storage_delay) pass def should_write_image(self, task): @@ -376,7 +437,9 @@ pass def rescue(self, task): + sleep(CONF.fake.rescue_delay) return states.RESCUE def unrescue(self, task): + sleep(CONF.fake.rescue_delay) return states.ACTIVE diff -Nru ironic-21.1.0/ironic/drivers/modules/image_cache.py ironic-21.4.4/ironic/drivers/modules/image_cache.py --- ironic-21.1.0/ironic/drivers/modules/image_cache.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/image_cache.py 2024-10-11 15:42:16.000000000 +0000 @@ -65,7 +65,9 @@ if master_dir is not None: fileutils.ensure_tree(master_dir) - def fetch_image(self, href, dest_path, ctx=None, force_raw=True): + def fetch_image(self, href, dest_path, ctx=None, force_raw=True, + expected_format=None, expected_checksum=None, + expected_checksum_algo=None): """Fetch image by given href to the destination path. Does nothing if destination path exists and is up to date with cache @@ -80,16 +82,33 @@ :param ctx: context :param force_raw: boolean value, whether to convert the image to raw format + :param expected_format: The expected image format. + :param expected_checksum: The expected image checksum + :param expected_checksum_algo: The expected image checksum algorithm, + if needed/supplied. """ img_download_lock_name = 'download-image' if self.master_dir is None: # NOTE(ghe): We don't share images between instances/hosts + # NOTE(TheJulia): These is a weird code path, because master_dir + # has to be None, which by default it never should be unless + # an operator forces it to None, which is a path we just never + # expect. + # TODO(TheJulia): This may be dead-ish code and likely needs + # to be removed. Likely originated *out* of just the iscsi + # deployment interface and local image caching. if not CONF.parallel_image_downloads: with lockutils.lock(img_download_lock_name): - _fetch(ctx, href, dest_path, force_raw) + _fetch(ctx, href, dest_path, force_raw, + expected_format=expected_format, + expected_checksum=expected_checksum, + expected_checksum_algo=expected_checksum_algo) else: with _concurrency_semaphore: - _fetch(ctx, href, dest_path, force_raw) + _fetch(ctx, href, dest_path, force_raw, + expected_format=expected_format, + expected_checksum=expected_checksum, + expected_checksum_algo=expected_checksum_algo) return # TODO(ghe): have hard links and counts the same behaviour in all fs @@ -140,13 +159,18 @@ {'href': href}) self._download_image( href, master_path, dest_path, img_info, - ctx=ctx, force_raw=force_raw) + ctx=ctx, force_raw=force_raw, + expected_format=expected_format, + expected_checksum=expected_checksum, + expected_checksum_algo=expected_checksum_algo) # NOTE(dtantsur): we increased cache size - time to clean up self.clean_up() def _download_image(self, href, master_path, dest_path, img_info, - ctx=None, force_raw=True): + ctx=None, force_raw=True, expected_format=None, + expected_checksum=None, + expected_checksum_algo=None): """Download image by href and store at a given path. This method should be called with uuid-specific lock taken. @@ -158,6 +182,9 @@ :param ctx: context :param force_raw: boolean value, whether to convert the image to raw format + :param expected_format: The expected original format for the image. + :param expected_checksum: The expected image checksum. + :param expected_checksum_algo: The expected image checksum algorithm. :raise ImageDownloadFailed: when the image cache and the image HTTP or TFTP location are on different file system, causing hard link to fail. @@ -169,7 +196,9 @@ try: with _concurrency_semaphore: - _fetch(ctx, href, tmp_path, force_raw) + _fetch(ctx, href, tmp_path, force_raw, expected_format, + expected_checksum=expected_checksum, + expected_checksum_algo=expected_checksum_algo) if img_info.get('no_cache'): LOG.debug("Caching is disabled for image %s", href) @@ -333,31 +362,59 @@ return stat.f_frsize * stat.f_bavail -def _fetch(context, image_href, path, force_raw=False): +def _fetch(context, image_href, path, force_raw=False, expected_format=None, + expected_checksum=None, expected_checksum_algo=None): """Fetch image and convert to raw format if needed.""" path_tmp = "%s.part" % path - images.fetch(context, image_href, path_tmp, force_raw=False) + images.fetch(context, image_href, path_tmp, force_raw=False, + checksum=expected_checksum, + checksum_algo=expected_checksum_algo) + # By default, the image format is unknown + image_format = None + disable_dii = CONF.conductor.disable_deep_image_inspection + if not disable_dii: + if not expected_format: + # Call of last resort to check the image format. Caching other + # artifacts like kernel/ramdisks are not going to have an expected + # format known even if they are not passed to qemu-img. + remote_image_format = images.image_show( + context, + image_href).get('disk_format') + else: + remote_image_format = expected_format + image_format = images.safety_check_image(path_tmp) + images.check_if_image_format_is_permitted( + image_format, remote_image_format) + # Notes(yjiang5): If glance can provide the virtual size information, # then we can firstly clean cache and then invoke images.fetch(). - if force_raw: - if images.force_raw_will_convert(image_href, path_tmp): - required_space = images.converted_size(path_tmp, estimate=False) - directory = os.path.dirname(path_tmp) + if (force_raw + and ((disable_dii + and images.force_raw_will_convert(image_href, path_tmp)) + or (not disable_dii and image_format != 'raw'))): + # NOTE(TheJulia): What is happening here is the rest of the logic + # is hinged on force_raw, but also we don't need to take the entire + # path *if* the image on disk is *already* raw. Depending on settings, + # the path differs slightly because if we have deep image inspection, + # we can just rely upon the inspection image format, otherwise we + # need to ask the image format. + + required_space = images.converted_size(path_tmp, estimate=False) + directory = os.path.dirname(path_tmp) + try: + _clean_up_caches(directory, required_space) + except exception.InsufficientDiskSpace: + + # try again with an estimated raw size instead of the full size + required_space = images.converted_size(path_tmp, estimate=True) try: _clean_up_caches(directory, required_space) except exception.InsufficientDiskSpace: - - # try again with an estimated raw size instead of the full size - required_space = images.converted_size(path_tmp, estimate=True) - try: - _clean_up_caches(directory, required_space) - except exception.InsufficientDiskSpace: - LOG.warning('Not enough space for estimated image size. ' - 'Consider lowering ' - '[DEFAULT]raw_image_growth_factor=%s', - CONF.raw_image_growth_factor) - raise - + LOG.error('Not enough space for estimated image size. ' + 'Consider lowering ' + '[DEFAULT]raw_image_growth_factor=%s', + CONF.raw_image_growth_factor) + raise images.image_to_raw(image_href, path, path_tmp) else: os.rename(path_tmp, path) diff -Nru ironic-21.1.0/ironic/drivers/modules/image_utils.py ironic-21.4.4/ironic/drivers/modules/image_utils.py --- ironic-21.1.0/ironic/drivers/modules/image_utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/image_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -24,6 +24,7 @@ from ironic_lib import utils as ironic_utils from oslo_log import log +from oslo_utils import uuidutils from ironic.common import exception from ironic.common.glance_service import service_utils @@ -211,6 +212,16 @@ try: os.link(image_file, published_file) os.chmod(image_file, self._file_permission) + try: + utils.execute( + '/usr/sbin/restorecon', '-i', '-R', 'v', public_dir) + except FileNotFoundError as exc: + LOG.debug( + "Could not restore SELinux context on " + "%(public_dir)s, restorecon command not found.\n" + "Error: %(error)s", + {'public_dir': public_dir, + 'error': exc}) except OSError as exc: LOG.debug( @@ -484,6 +495,10 @@ img_handler = ImageHandler(task.node.driver) boot_mode = boot_mode_utils.get_boot_mode(task.node) + if not is_ramdisk_boot: + publisher_id = uuidutils.generate_uuid() + else: + publisher_id = None with tempfile.TemporaryDirectory(dir=CONF.tempdir) as boot_file_dir: @@ -506,6 +521,7 @@ else: kernel_params = driver_utils.get_kernel_append_params( task.node, default=img_handler.kernel_params) + kernel_params += " ir_pub_id=%s" % publisher_id if params: kernel_params = ' '.join( @@ -523,19 +539,32 @@ 'ramdisk_href': ramdisk_href, 'bootloader_href': bootloader_href, 'params': kernel_params}) - images.create_boot_iso( - task.context, boot_iso_tmp_file, - kernel_href, ramdisk_href, - esp_image_href=bootloader_href, - root_uuid=root_uuid, - kernel_params=kernel_params, - boot_mode=boot_mode, - inject_files=inject_files) + + if publisher_id: + images.create_boot_iso( + task.context, boot_iso_tmp_file, + kernel_href, ramdisk_href, + esp_image_href=bootloader_href, + root_uuid=root_uuid, + kernel_params=kernel_params, + boot_mode=boot_mode, + inject_files=inject_files, + publisher_id=publisher_id) + else: + images.create_boot_iso( + task.context, boot_iso_tmp_file, + kernel_href, ramdisk_href, + esp_image_href=bootloader_href, + root_uuid=root_uuid, + kernel_params=kernel_params, + boot_mode=boot_mode, + inject_files=inject_files) iso_object_name = _get_name(task.node, prefix='boot', suffix='.iso') + node_http_url = task.node.driver_info.get("external_http_url") image_url = img_handler.publish_image( - boot_iso_tmp_file, iso_object_name) + boot_iso_tmp_file, iso_object_name, node_http_url) LOG.debug("Created ISO %(name)s in object store for node %(node)s, " "exposed as temporary URL " diff -Nru ironic-21.1.0/ironic/drivers/modules/inspect_utils.py ironic-21.4.4/ironic/drivers/modules/inspect_utils.py --- ironic-21.1.0/ironic/drivers/modules/inspect_utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/inspect_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -15,11 +15,16 @@ from oslo_log import log as logging from oslo_utils import netutils +import swiftclient.exceptions from ironic.common import exception +from ironic.common import swift +from ironic.conf import CONF from ironic import objects +from ironic.objects import node_inventory LOG = logging.getLogger(__name__) +_OBJECT_NAME_PREFIX = 'inspector_data' def create_ports_if_not_exist(task, macs): @@ -51,3 +56,164 @@ except exception.MACAlreadyExists: LOG.info("Port already exists for MAC address %(address)s " "for node %(node)s", {'address': mac, 'node': node.uuid}) + + +def clean_up_swift_entries(task): + """Delete swift entries containing introspection data. + + Delete swift entries related to the node in task.node containing + introspection data. The entries are + ``inspector_data--inventory`` for hardware inventory and + similar for ``-plugin`` containing the rest of the introspection data. + + :param task: A TaskManager instance. + """ + if CONF.inventory.data_backend != 'swift': + return + swift_api = swift.SwiftAPI() + swift_object_name = '%s-%s' % (_OBJECT_NAME_PREFIX, task.node.uuid) + container = CONF.inventory.swift_data_container + inventory_obj_name = swift_object_name + '-inventory' + plugin_obj_name = swift_object_name + '-plugin' + try: + swift_api.delete_object(inventory_obj_name, container) + except swiftclient.exceptions.ClientException as e: + if e.http_status == 404: + # 404 -> entry did not exist - acceptable. + pass + else: + LOG.error("Object %(obj)s related to node %(node)s " + "failed to be deleted with expection: %(e)s", + {'obj': inventory_obj_name, 'node': task.node.uuid, + 'e': e}) + raise exception.SwiftObjectStillExists(obj=inventory_obj_name, + node=task.node.uuid) + try: + swift_api.delete_object(plugin_obj_name, container) + except swiftclient.exceptions.ClientException as e: + if e.http_status == 404: + # 404 -> entry did not exist - acceptable. + pass + else: + LOG.error("Object %(obj)s related to node %(node)s " + "failed to be deleted with exception: %(e)s", + {'obj': plugin_obj_name, 'node': task.node.uuid, + 'e': e}) + raise exception.SwiftObjectStillExists(obj=plugin_obj_name, + node=task.node.uuid) + + +def store_introspection_data(node, introspection_data, context): + """Store introspection data. + + Store the introspection data for a node. Either to database + or swift as configured. + + :param node: the Ironic node that the introspection data is about + :param introspection_data: the data to store + :param context: an admin context + """ + # If store_data == 'none', do not store the data + store_data = CONF.inventory.data_backend + if store_data == 'none': + LOG.debug('Introspection data storage is disabled, the data will ' + 'not be saved for node %(node)s', {'node': node.uuid}) + return + inventory_data = introspection_data.pop("inventory") + plugin_data = introspection_data + if store_data == 'database': + node_inventory.NodeInventory( + context, + node_id=node.id, + inventory_data=inventory_data, + plugin_data=plugin_data).create() + LOG.info('Introspection data was stored in database for node ' + '%(node)s', {'node': node.uuid}) + if store_data == 'swift': + swift_object_name = _store_introspection_data_in_swift( + node_uuid=node.uuid, + inventory_data=inventory_data, + plugin_data=plugin_data) + LOG.info('Introspection data was stored for node %(node)s in Swift' + ' object %(obj_name)s-inventory and %(obj_name)s-plugin', + {'node': node.uuid, 'obj_name': swift_object_name}) + + +def _node_inventory_convert(node_inventory): + inventory_data = node_inventory['inventory_data'] + plugin_data = node_inventory['plugin_data'] + return {"inventory": inventory_data, "plugin_data": plugin_data} + + +def get_introspection_data(node, context): + """Get introspection data. + + Retrieve the introspection data for a node. Either from database + or swift as configured. + + :param node_id: the Ironic node that the required data is about + :param context: an admin context + :returns: dictionary with ``inventory`` and ``plugin_data`` fields + """ + store_data = CONF.inventory.data_backend + if store_data == 'none': + raise exception.NodeInventoryNotFound(node=node.uuid) + if store_data == 'database': + node_inventory = objects.NodeInventory.get_by_node_id( + context, node.id) + return _node_inventory_convert(node_inventory) + if store_data == 'swift': + try: + node_inventory = _get_introspection_data_from_swift(node.uuid) + except exception.SwiftObjectNotFoundError: + raise exception.NodeInventoryNotFound(node=node.uuid) + return node_inventory + + +def _store_introspection_data_in_swift(node_uuid, inventory_data, plugin_data): + """Uploads introspection data to Swift. + + :param data: data to store in Swift + :param node_id: ID of the Ironic node that the data came from + :returns: name of the Swift object that the data is stored in + """ + swift_api = swift.SwiftAPI() + swift_object_name = '%s-%s' % (_OBJECT_NAME_PREFIX, node_uuid) + container = CONF.inventory.swift_data_container + swift_api.create_object_from_data(swift_object_name + '-inventory', + inventory_data, + container) + swift_api.create_object_from_data(swift_object_name + '-plugin', + plugin_data, + container) + return swift_object_name + + +def _get_introspection_data_from_swift(node_uuid): + """Get introspection data from Swift. + + :param node_uuid: UUID of the Ironic node that the data came from + :returns: dictionary with ``inventory`` and ``plugin_data`` fields + """ + swift_api = swift.SwiftAPI() + swift_object_name = '%s-%s' % (_OBJECT_NAME_PREFIX, node_uuid) + container = CONF.inventory.swift_data_container + inv_obj = swift_object_name + '-inventory' + plug_obj = swift_object_name + '-plugin' + try: + inventory_data = swift_api.get_object(inv_obj, container) + except exception.SwiftOperationError: + LOG.error("Failed to retrieve object %(obj)s from swift", + {'obj': inv_obj}) + raise exception.SwiftObjectNotFoundError(obj=inv_obj, + container=container, + operation='get') + try: + plugin_data = swift_api.get_object(plug_obj, container) + except exception.SwiftOperationError: + LOG.error("Failed to retrieve object %(obj)s from swift", + {'obj': plug_obj}) + raise exception.SwiftObjectNotFoundError(obj=plug_obj, + container=container, + operation='get') + return {"inventory": inventory_data, "plugin_data": plugin_data} diff -Nru ironic-21.1.0/ironic/drivers/modules/inspector.py ironic-21.4.4/ironic/drivers/modules/inspector.py --- ironic-21.1.0/ironic/drivers/modules/inspector.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/inspector.py 2024-10-11 15:42:16.000000000 +0000 @@ -339,7 +339,8 @@ task.node.uuid) try: - status = _get_client(task.context).get_introspection(node.uuid) + inspector_client = _get_client(task.context) + status = inspector_client.get_introspection(node.uuid) except Exception: # NOTE(dtantsur): get_status should not normally raise # let's assume it's a transient failure and retry later @@ -363,6 +364,15 @@ _inspection_error_handler(task, error) elif status.is_finished: _clean_up(task) + store_data = CONF.inventory.data_backend + if store_data == 'none': + LOG.debug('Introspection data storage is disabled, the data will ' + 'not be saved for node %(node)s', {'node': node.uuid}) + return + introspection_data = inspector_client.get_introspection_data( + node.uuid, processed=True) + inspect_utils.store_introspection_data(node, introspection_data, + task.context) def _clean_up(task): diff -Nru ironic-21.1.0/ironic/drivers/modules/ipmitool.py ironic-21.4.4/ironic/drivers/modules/ipmitool.py --- ironic-21.1.0/ironic/drivers/modules/ipmitool.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/ipmitool.py 2024-10-11 15:42:16.000000000 +0000 @@ -1556,6 +1556,9 @@ created :raises: ConsoleSubprocessFailed when invoking the subprocess failed """ + # Dealloc allocated port if any, so the same host can never has + # duplicated port. + _release_allocated_port(task) driver_info = _parse_driver_info(task.node) if not driver_info['port']: driver_info['port'] = _allocate_port(task) @@ -1611,6 +1614,9 @@ created :raises: ConsoleSubprocessFailed when invoking the subprocess failed """ + # Dealloc allocated port if any, so the same host can never has + # duplicated port. + _release_allocated_port(task) driver_info = _parse_driver_info(task.node) if not driver_info['port']: driver_info['port'] = _allocate_port( diff -Nru ironic-21.1.0/ironic/drivers/modules/irmc/common.py ironic-21.4.4/ironic/drivers/modules/irmc/common.py --- ironic-21.1.0/ironic/drivers/modules/irmc/common.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/irmc/common.py 2024-10-11 15:42:16.000000000 +0000 @@ -15,9 +15,12 @@ """ Common functionalities shared between different iRMC modules. """ +import json import os +import re from oslo_log import log as logging +from oslo_serialization import jsonutils from oslo_utils import importutils from oslo_utils import strutils @@ -31,6 +34,29 @@ elcm = importutils.try_import('scciclient.irmc.elcm') LOG = logging.getLogger(__name__) + + +IRMC_OS_NAME_R = re.compile(r'iRMC\s+S\d+') +IRMC_OS_NAME_NUM_R = re.compile(r'\d+$') +IRMC_FW_VER_R = re.compile(r'\d(\.\d+)*\w*') +IRMC_FW_VER_NUM_R = re.compile(r'\d(\.\d+)*') + +IPMI_ENABLED_BY_DEFAULT_RANGES = { + # iRMC S4 enables IPMI over LAN by default + '4': None, + # iRMC S5 enables IPMI over LAN by default + '5': None, + # iRMC S6 disables IPMI over LAN by default from version 2.00 + '6': {'upper': '2.00'}} + +ELCM_STATUS_PATH = '/rest/v1/Oem/eLCM/eLCMStatus' + +# List of xxx_interface & implementation pair which uses SNMP internally +# and iRMC driver supports +INTERFACE_IMPL_LIST_WITH_SNMP = { + 'inspect_interface': {'irmc', }, + 'power_interface': {'irmc', }} + REQUIRED_PROPERTIES = { 'irmc_address': _("IP address or hostname of the iRMC. Required."), 'irmc_username': _("Username for the iRMC with administrator privileges. " @@ -83,7 +109,9 @@ SNMP_V3_OPTIONAL_PROPERTIES = { 'irmc_snmp_auth_proto': _("SNMPv3 message authentication protocol ID. " "Required for version 'v3'. " - "'sha' is supported."), + "If using iRMC S4/S5, only 'sha' is supported." + "If using iRMC S6, the valid options are " + "'sha256', 'sha384', 'sha512'."), 'irmc_snmp_priv_proto': _("SNMPv3 message privacy (encryption) protocol " "ID. Required for version 'v3'. " "'aes' is supported."), @@ -212,6 +240,12 @@ "v2c": snmp.SNMP_V2C, "v3": snmp.SNMP_V3} + for int_name, impl_list in INTERFACE_IMPL_LIST_WITH_SNMP.items(): + if getattr(node, int_name) in impl_list: + break + else: + return snmp_info + if snmp_info['irmc_snmp_version'].lower() not in valid_versions: raise exception.InvalidParameterValue(_( "Value '%s' is not supported for 'irmc_snmp_version'.") % @@ -243,7 +277,8 @@ def _parse_snmp_v3_info(node, info): snmp_info = {} missing_info = [] - valid_values = {'irmc_snmp_auth_proto': ['sha'], + valid_values = {'irmc_snmp_auth_proto': ['sha', 'sha256', 'sha384', + 'sha512'], 'irmc_snmp_priv_proto': ['aes']} valid_protocols = {'irmc_snmp_auth_proto': snmp.snmp_auth_protocols, 'irmc_snmp_priv_proto': snmp.snmp_priv_protocols} @@ -433,3 +468,202 @@ raise exception.IRMCOperationError( operation=_("setting secure boot mode"), error=irmc_exception) + + +def check_elcm_license(node): + """Connect to iRMC and return status of eLCM license + + This function connects to iRMC REST API and check whether eLCM + license is active. This function can be used to check connection to + iRMC REST API. + + :param node: An ironic node object + :returns: dictionary whose keys are 'active' and 'status_code'. + value of 'active' is boolean showing if eLCM license is active + and value of 'status_code' is int which is HTTP return code + from iRMC REST API access + :raises: InvalidParameterValue if invalid value is contained + in the 'driver_info' property. + :raises: MissingParameterValue if some mandatory key is missing + in the 'driver_info' property. + :raises: IRMCOperationError if the operation fails. + """ + try: + d_info = parse_driver_info(node) + # GET to /rest/v1/Oem/eLCM/eLCMStatus returns + # JSON data like this: + # + # { + # "eLCMStatus":{ + # "EnabledAndLicenced":"true", + # "SDCardMounted":"false" + # } + # } + # + # EnabledAndLicenced tells whether eLCM license is valid + # + r = elcm.elcm_request(d_info, 'GET', ELCM_STATUS_PATH) + + # If r.status_code is 200, it means success and r.text is JSON. + # If it is 500, it means there is problem at iRMC side + # and iRMC cannot return eLCM status. + # If it was 401, elcm_request raises SCCIClientError. + # Otherwise, r.text may not be JSON. + if r.status_code == 200: + license_active = strutils.bool_from_string( + jsonutils.loads(r.text)['eLCMStatus']['EnabledAndLicenced'], + strict=True) + else: + license_active = False + + return {'active': license_active, 'status_code': r.status_code} + except (scci.SCCIError, + json.JSONDecodeError, + TypeError, + KeyError, + ValueError) as irmc_exception: + LOG.error("Failed to check eLCM license status for node $(node)s", + {'node': node.uuid}) + raise exception.IRMCOperationError( + operation='checking eLCM license status', + error=irmc_exception) + + +def set_irmc_version(task): + """Fetch and save iRMC firmware version. + + This function should be called before calling any other functions which + need to check node's iRMC firmware version. + + Set `/` to driver_internal_info['irmc_fw_version'] + + :param node: An ironic node object + :raises: InvalidParameterValue if invalid value is contained + in the 'driver_info' property. + :raises: MissingParameterValue if some mandatory key is missing + in the 'driver_info' property. + :raises: IRMCOperationError if the operation fails. + :raises: NodeLocked if the target node is already locked. + """ + + node = task.node + try: + report = get_irmc_report(node) + irmc_os, fw_version = scci.get_irmc_version_str(report) + + fw_ver = node.driver_internal_info.get('irmc_fw_version') + if fw_ver != '/'.join([irmc_os, fw_version]): + task.upgrade_lock(purpose='saving firmware version') + node.set_driver_internal_info('irmc_fw_version', + f"{irmc_os}/{fw_version}") + node.save() + except scci.SCCIError as irmc_exception: + LOG.error("Failed to fetch iRMC FW version for node %s", + node.uuid) + raise exception.IRMCOperationError( + operation=_("fetching irmc fw version "), + error=irmc_exception) + + +def _version_lt(v1, v2): + v1_l = v1.split('.') + v2_l = v2.split('.') + if len(v1_l) <= len(v2_l): + v1_l.extend(['0'] * (len(v2_l) - len(v1_l))) + else: + v2_l.extend(['0'] * (len(v1_l) - len(v2_l))) + + for i in range(len(v1_l)): + if int(v1_l[i]) < int(v2_l[i]): + return True + elif int(v1_l[i]) > int(v2_l[i]): + return False + else: + return False + + +def _version_le(v1, v2): + v1_l = v1.split('.') + v2_l = v2.split('.') + if len(v1_l) <= len(v2_l): + v1_l.extend(['0'] * (len(v2_l) - len(v1_l))) + else: + v2_l.extend(['0'] * (len(v1_l) - len(v2_l))) + + for i in range(len(v1_l)): + if int(v1_l[i]) < int(v2_l[i]): + return True + elif int(v1_l[i]) > int(v2_l[i]): + return False + else: + return True + + +def within_version_ranges(node, version_ranges): + """Read saved iRMC FW version and check if it is within the passed ranges. + + :param node: An ironic node object + :param version_ranges: A Python dictionary containing version ranges in the + next format: : , where is a string representing + iRMC OS number (e.g. '4') and is a dictionaries indicating + the specific firmware version ranges under the iRMC OS number . + + The dictionary used in only has two keys: 'min' and 'upper', + and value of each key is a string representing iRMC firmware version + number or None. Both keys can be absent and their value can be None. + + It is acceptable to not set ranges for a (for example set + to None, {}, etc...), in this case, this function only + checks if the node's iRMC OS number matches the . + + Valid example: + {'3': None, # all version of iRMC S3 matches + '4': {}, # all version of iRMC S4 matches + # all version of iRMC S5 matches + '5': {'min': None, 'upper': None}, + # iRMC S6 whose version is >=1.20 matches + '6': {'min': '1.20', 'upper': None}, + # iRMC S7 whose version is + # 5.51<= (version) <8.23 matches + '7': {'min': '5.51', 'upper': '8.23'}} + + :returns: True if node's iRMC FW is in range, False if not or + fails to parse firmware version + """ + + try: + fw_version = node.driver_internal_info.get('irmc_fw_version', '') + irmc_os, irmc_ver = fw_version.split('/') + + if IRMC_OS_NAME_R.match(irmc_os) and IRMC_FW_VER_R.match(irmc_ver): + os_num = IRMC_OS_NAME_NUM_R.search(irmc_os).group(0) + fw_num = IRMC_FW_VER_NUM_R.search(irmc_ver).group(0) + + if os_num not in version_ranges: + return False + + v_range = version_ranges[os_num] + + # An OS number with no ranges setted means no need to check + # specific version, all the version under this OS number is valid. + if not v_range: + return True + + # Specific range is setted, check if the node's + # firmware version is within it. + min_ver = v_range.get('min') + upper_ver = v_range.get('upper') + flag = True + if min_ver: + flag = _version_le(min_ver, fw_num) + if flag and upper_ver: + flag = _version_lt(fw_num, upper_ver) + return flag + + except Exception: + # All exceptions are ignored + pass + + LOG.warning('Failed to parse iRMC firmware version on node %(uuid)s: ' + '%(fw_ver)s', {'uuid': node.uuid, 'fw_ver': fw_version}) + return False diff -Nru ironic-21.1.0/ironic/drivers/modules/irmc/inspect.py ironic-21.4.4/ironic/drivers/modules/irmc/inspect.py --- ironic-21.1.0/ironic/drivers/modules/irmc/inspect.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/irmc/inspect.py 2024-10-11 15:42:16.000000000 +0000 @@ -32,7 +32,7 @@ from ironic.drivers.modules import snmp from ironic import objects -scci = importutils.try_import('scciclient.irmc.scci') +irmc = importutils.try_import('scciclient.irmc') LOG = logging.getLogger(__name__) @@ -122,6 +122,39 @@ if c == NODE_CLASS_OID_VALUE['primary']] +def _get_capabilities_properties_without_ipmi(d_info, cap_props, + current_cap, props): + capabilities = {} + snmp_client = snmp.SNMPClient( + address=d_info['irmc_address'], + port=d_info['irmc_snmp_port'], + version=d_info['irmc_snmp_version'], + read_community=d_info['irmc_snmp_community'], + user=d_info.get('irmc_snmp_user'), + auth_proto=d_info.get('irmc_snmp_auth_proto'), + auth_key=d_info.get('irmc_snmp_auth_password'), + priv_proto=d_info.get('irmc_snmp_priv_proto'), + priv_key=d_info.get('irmc_snmp_priv_password')) + + if 'rom_firmware_version' in cap_props: + capabilities['rom_firmware_version'] = \ + irmc.snmp.get_bios_firmware_version(snmp_client) + + if 'irmc_firmware_version' in cap_props: + capabilities['irmc_firmware_version'] = \ + irmc.snmp.get_irmc_firmware_version(snmp_client) + + if 'server_model' in cap_props: + capabilities['server_model'] = irmc.snmp.get_server_model( + snmp_client) + + capabilities = utils.get_updated_capabilities(current_cap, capabilities) + if capabilities: + props['capabilities'] = capabilities + + return props + + def _inspect_hardware(node, existing_traits=None, **kwargs): """Inspect the node and get hardware information. @@ -161,39 +194,51 @@ try: report = irmc_common.get_irmc_report(node) - props = scci.get_essential_properties( + props = irmc.scci.get_essential_properties( report, IRMCInspect.ESSENTIAL_PROPERTIES) d_info = irmc_common.parse_driver_info(node) - capabilities = scci.get_capabilities_properties( - d_info, - capabilities_props, - gpu_ids, - fpga_ids=fpga_ids, - **kwargs) - if capabilities: - if capabilities.get('pci_gpu_devices') == 0: - capabilities.pop('pci_gpu_devices') - - cpu_fpga = capabilities.pop('cpu_fpga', 0) - if cpu_fpga == 0 and 'CUSTOM_CPU_FPGA' in new_traits: - new_traits.remove('CUSTOM_CPU_FPGA') - elif cpu_fpga != 0 and 'CUSTOM_CPU_FPGA' not in new_traits: - new_traits.append('CUSTOM_CPU_FPGA') - - # Ironic no longer supports trusted boot - capabilities.pop('trusted_boot', None) - capabilities = utils.get_updated_capabilities( - node.properties.get('capabilities'), capabilities) + if (getattr(node, 'power_interface') == 'ipmitool' + or node.driver_internal_info.get('irmc_ipmi_succeed')): + capabilities = irmc.scci.get_capabilities_properties( + d_info, + capabilities_props, + gpu_ids, + fpga_ids=fpga_ids, + **kwargs) if capabilities: - props['capabilities'] = capabilities + if capabilities.get('pci_gpu_devices') == 0: + capabilities.pop('pci_gpu_devices') + + cpu_fpga = capabilities.pop('cpu_fpga', 0) + if cpu_fpga == 0 and 'CUSTOM_CPU_FPGA' in new_traits: + new_traits.remove('CUSTOM_CPU_FPGA') + elif cpu_fpga != 0 and 'CUSTOM_CPU_FPGA' not in new_traits: + new_traits.append('CUSTOM_CPU_FPGA') + + # Ironic no longer supports trusted boot + capabilities.pop('trusted_boot', None) + capabilities = utils.get_updated_capabilities( + node.properties.get('capabilities', ''), capabilities) + if capabilities: + props['capabilities'] = capabilities + + else: + props = _get_capabilities_properties_without_ipmi( + d_info, capabilities_props, + node.properties.get('capabilities', ''), props) macs = _get_mac_addresses(node) - except (scci.SCCIInvalidInputError, - scci.SCCIClientError, + except (irmc.scci.SCCIInvalidInputError, + irmc.scci.SCCIClientError, exception.SNMPFailure) as e: + advice = "" + if ("SNMP operation" in str(e)): + advice = ("The SNMP related parameters' value may be different " + "with the server, please check if you have set them " + "correctly.") error = (_("Inspection failed for node %(node_id)s " - "with the following error: %(error)s") % - {'node_id': node.uuid, 'error': e}) + "with the following error: %(error)s. (advice)s") % + {'node_id': node.uuid, 'error': e, 'advice': advice}) raise exception.HardwareInspectionFailure(error=error) return props, macs, new_traits diff -Nru ironic-21.1.0/ironic/drivers/modules/irmc/management.py ironic-21.4.4/ironic/drivers/modules/irmc/management.py --- ironic-21.1.0/ironic/drivers/modules/irmc/management.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/irmc/management.py 2024-10-11 15:42:16.000000000 +0000 @@ -27,9 +27,10 @@ from ironic.conductor import utils as manager_utils from ironic import conf from ironic.drivers import base +from ironic.drivers.modules import boot_mode_utils from ironic.drivers.modules import ipmitool from ironic.drivers.modules.irmc import common as irmc_common -from ironic.drivers import utils as driver_utils +from ironic.drivers.modules.redfish import management as redfish_management irmc = importutils.try_import('scciclient.irmc') @@ -204,7 +205,8 @@ manager_utils.node_power_action(task, states.POWER_ON) -class IRMCManagement(ipmitool.IPMIManagement): +class IRMCManagement(ipmitool.IPMIManagement, + redfish_management.RedfishManagement): def get_properties(self): """Return the properties of the interface. @@ -224,9 +226,32 @@ :raises: InvalidParameterValue if required parameters are invalid. :raises: MissingParameterValue if a required parameter is missing. """ - irmc_common.parse_driver_info(task.node) - irmc_common.update_ipmi_properties(task) - super(IRMCManagement, self).validate(task) + if (getattr(task.node, 'power_interface') == 'ipmitool' + or task.node.driver_internal_info.get('irmc_ipmi_succeed')): + irmc_common.parse_driver_info(task.node) + irmc_common.update_ipmi_properties(task) + super(IRMCManagement, self).validate(task) + else: + irmc_common.parse_driver_info(task.node) + super(ipmitool.IPMIManagement, self).validate(task) + + def get_supported_boot_devices(self, task): + """Get list of supported boot devices + + Actual code is delegated to IPMIManagement or RedfishManagement + based on iRMC firmware version. + + :param task: A TaskManager instance + :returns: A list with the supported boot devices defined + in :mod:`ironic.common.boot_devices`. + + """ + if (getattr(task.node, 'power_interface') == 'ipmitool' + or task.node.driver_internal_info.get('irmc_ipmi_succeed')): + return super(IRMCManagement, self).get_supported_boot_devices(task) + else: + return super(ipmitool.IPMIManagement, + self).get_supported_boot_devices(task) @METRICS.timer('IRMCManagement.set_boot_device') @task_manager.require_exclusive_lock @@ -245,39 +270,114 @@ specified. :raises: MissingParameterValue if a required parameter is missing. :raises: IPMIFailure on an error from ipmitool. + :raises: RedfishConnectionError on Redfish operation failure. + :raises: RedfishError on Redfish operation failure. + """ + if (getattr(task.node, 'power_interface') == 'ipmitool' + or task.node.driver_internal_info.get('irmc_ipmi_succeed')): + if device not in self.get_supported_boot_devices(task): + raise exception.InvalidParameterValue(_( + "Invalid boot device %s specified.") % device) + + uefi_mode = ( + boot_mode_utils.get_boot_mode(task.node) == 'uefi') + + # disable 60 secs timer + timeout_disable = "0x00 0x08 0x03 0x08" + ipmitool.send_raw(task, timeout_disable) + + # note(naohirot): + # Set System Boot Options : ipmi cmd '0x08', bootparam '0x05' + # + # $ ipmitool raw 0x00 0x08 0x05 data1 data2 0x00 0x00 0x00 + # + # data1 : '0xe0' persistent + uefi + # '0xc0' persistent + bios + # '0xa0' next only + uefi + # '0x80' next only + bios + # data2 : boot device defined in the dict _BOOTPARAM5_DATA2 + + bootparam5 = '0x00 0x08 0x05 %s %s 0x00 0x00 0x00' + if persistent: + data1 = '0xe0' if uefi_mode else '0xc0' + else: + data1 = '0xa0' if uefi_mode else '0x80' + data2 = _BOOTPARAM5_DATA2[device] + cmd8 = bootparam5 % (data1, data2) + ipmitool.send_raw(task, cmd8) + else: + if device not in self.get_supported_boot_devices(task): + raise exception.InvalidParameterValue(_( + "Invalid boot device %s specified. " + "Current iRMC firmware condition doesn't support IPMI " + "but Redfish.") % device) + super(ipmitool.IPMIManagement, self).set_boot_device( + task, device, persistent) + + def get_boot_device(self, task): + """Get the current boot device for the task's node. + + Returns the current boot device of the node. + + :param task: a task from TaskManager. + :raises: InvalidParameterValue if an invalid boot device is + specified. + :raises: MissingParameterValue if a required parameter is missing. + :raises: IPMIFailure on an error from ipmitool. + :raises: RedfishConnectionError on Redfish operation failure. + :raises: RedfishError on Redfish operation failure. + :returns: a dictionary containing: + + :boot_device: the boot device, one of + :mod:`ironic.common.boot_devices` or None if it is unknown. + :persistent: Whether the boot device will persist to all + future boots or not, None if it is unknown. """ - if device not in self.get_supported_boot_devices(task): - raise exception.InvalidParameterValue(_( - "Invalid boot device %s specified.") % device) - - uefi_mode = ( - driver_utils.get_node_capability(task.node, 'boot_mode') == 'uefi') - - # disable 60 secs timer - timeout_disable = "0x00 0x08 0x03 0x08" - ipmitool.send_raw(task, timeout_disable) - - # note(naohirot): - # Set System Boot Options : ipmi cmd '0x08', bootparam '0x05' - # - # $ ipmitool raw 0x00 0x08 0x05 data1 data2 0x00 0x00 0x00 - # - # data1 : '0xe0' persistent + uefi - # '0xc0' persistent + bios - # '0xa0' next only + uefi - # '0x80' next only + bios - # data2 : boot device defined in the dict _BOOTPARAM5_DATA2 - - bootparam5 = '0x00 0x08 0x05 %s %s 0x00 0x00 0x00' - if persistent: - data1 = '0xe0' if uefi_mode else '0xc0' + if (getattr(task.node, 'power_interface') == 'ipmitool' + or task.node.driver_internal_info.get('irmc_ipmi_succeed')): + return super(IRMCManagement, self).get_boot_device(task) else: - data1 = '0xa0' if uefi_mode else '0x80' - data2 = _BOOTPARAM5_DATA2[device] + return super( + ipmitool.IPMIManagement, self).get_boot_device(task) + + def get_supported_boot_modes(self, task): + """Get a list of the supported boot modes. + + IRMCManagement class doesn't support this method + + :param task: a task from TaskManager. + :raises: UnsupportedDriverExtension if requested operation is + not supported by the driver + """ + raise exception.UnsupportedDriverExtension( + driver=task.node.driver, extension='get_supported_boot_modes') + + def set_boot_mode(self, task, mode): + """Set the boot mode for a node. + + IRMCManagement class doesn't support this method - cmd8 = bootparam5 % (data1, data2) - ipmitool.send_raw(task, cmd8) + :param task: a task from TaskManager. + :param mode: The boot mode, one of + :mod:`ironic.common.boot_modes`. + :raises: UnsupportedDriverExtension if requested operation is + not supported by the driver + """ + raise exception.UnsupportedDriverExtension( + driver=task.node.driver, extension='set_boot_mode') + + def get_boot_mode(self, task): + """Get the current boot mode for a node. + + IRMCManagement class doesn't support this method + + :param task: a task from TaskManager. + :raises: UnsupportedDriverExtension if requested operation is + not supported by the driver + """ + raise exception.UnsupportedDriverExtension( + driver=task.node.driver, extension='get_boot_mode') @METRICS.timer('IRMCManagement.get_sensors_data') def get_sensors_data(self, task): @@ -330,7 +430,14 @@ if sensor_method == 'scci': return _get_sensors_data(task) elif sensor_method == 'ipmitool': - return super(IRMCManagement, self).get_sensors_data(task) + if (getattr(task.node, 'power_interface') == 'ipmitool' + or task.node.driver_internal_info.get('irmc_ipmi_succeed')): + return super(IRMCManagement, self).get_sensors_data(task) + else: + raise exception.InvalidParameterValue(_( + "Invalid sensor method %s specified. " + "IPMI operation doesn't work on current iRMC " + "condition.") % sensor_method) @METRICS.timer('IRMCManagement.inject_nmi') @task_manager.require_exclusive_lock @@ -401,3 +508,121 @@ not supported by the driver or the hardware """ return irmc_common.set_secure_boot_mode(task.node, state) + + def get_supported_indicators(self, task, component=None): + """Get a map of the supported indicators (e.g. LEDs). + + IRMCManagement class doesn't support this method + + :param task: a task from TaskManager. + :param component: If not `None`, return indicator information + for just this component, otherwise return indicators for + all existing components. + :raises: UnsupportedDriverExtension if requested operation is + not supported by the driver + + """ + raise exception.UnsupportedDriverExtension( + driver=task.node.driver, extension='get_supported_indicators') + + def set_indicator_state(self, task, component, indicator, state): + """Set indicator on the hardware component to the desired state. + + IRMCManagement class doesn't support this method + + :param task: A task from TaskManager. + :param component: The hardware component, one of + :mod:`ironic.common.components`. + :param indicator: Indicator ID (as reported by + `get_supported_indicators`). + :state: Desired state of the indicator, one of + :mod:`ironic.common.indicator_states`. + :raises: UnsupportedDriverExtension if requested operation is + not supported by the driver + """ + raise exception.UnsupportedDriverExtension( + driver=task.node.driver, extension='set_indicator_state') + + def get_indicator_state(self, task, component, indicator): + """Get current state of the indicator of the hardware component. + + IRMCManagement class doesn't support this method + + :param task: A task from TaskManager. + :param component: The hardware component, one of + :mod:`ironic.common.components`. + :param indicator: Indicator ID (as reported by + `get_supported_indicators`). + :raises: UnsupportedDriverExtension if requested operation is + not supported by the driver + """ + raise exception.UnsupportedDriverExtension( + driver=task.node.driver, extension='get_indicator_state') + + def detect_vendor(self, task): + """Detects and returns the hardware vendor. + + :param task: A task from TaskManager. + :raises: InvalidParameterValue if a required parameter is missing + :raises: MissingParameterValue if a required parameter is missing + :raises: RedfishError on Redfish operation error. + :raises: PasswordFileFailedToCreate from creating or writing to the + temporary file during IPMI operation. + :raises: processutils.ProcessExecutionError from executing ipmi command + :returns: String representing the BMC reported Vendor or + Manufacturer, otherwise returns None. + """ + if (getattr(task.node, 'power_interface') == 'ipmitool' + or task.node.driver_internal_info.get('irmc_ipmi_succeed')): + return super(IRMCManagement, self).detect_vendor(task) + else: + return super(ipmitool.IPMIManagement, self).detect_vendor(task) + + def get_mac_addresses(self, task): + """Get MAC address information for the node. + + IRMCManagement class doesn't support this method + + :param task: A TaskManager instance containing the node to act on. + :raises: UnsupportedDriverExtension + """ + raise exception.UnsupportedDriverExtension( + driver=task.node.driver, extension='get_mac_addresses') + + @base.verify_step(priority=10) + def verify_http_https_connection_and_fw_version(self, task): + """Check http(s) connection to iRMC and save fw version + + :param task' A task from TaskManager + 'raises: IRMCOperationError + """ + error_msg_https = ('Access to REST API returns unexpected ' + 'status code. Check driver_info parameter ' + 'related to iRMC driver') + error_msg_http = ('Access to REST API returns unexpected ' + 'status code. Check driver_info parameter ' + 'or version of iRMC because iRMC does not ' + 'support HTTP connection to iRMC REST API ' + 'since iRMC S6 2.00.') + try: + # Check connection to iRMC + elcm_license = irmc_common.check_elcm_license(task.node) + + # On iRMC S6 2.00, access to REST API through HTTP returns 404 + if elcm_license.get('status_code') not in (200, 500): + port = task.node.driver_info.get( + 'irmc_port', CONF.irmc.get('port')) + if port == 80: + e_msg = error_msg_http + else: + e_msg = error_msg_https + raise exception.IRMCOperationError( + operation='establishing connection to REST API', + error=e_msg) + + irmc_common.set_irmc_version(task) + except (exception.InvalidParameterValue, + exception.MissingParameterValue) as irmc_exception: + raise exception.IRMCOperationError( + operation='configuration validation', + error=irmc_exception) diff -Nru ironic-21.1.0/ironic/drivers/modules/irmc/power.py ironic-21.4.4/ironic/drivers/modules/irmc/power.py --- ironic-21.1.0/ironic/drivers/modules/irmc/power.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/irmc/power.py 2024-10-11 15:42:16.000000000 +0000 @@ -29,6 +29,7 @@ from ironic.drivers.modules import ipmitool from ironic.drivers.modules.irmc import boot as irmc_boot from ironic.drivers.modules.irmc import common as irmc_common +from ironic.drivers.modules.redfish import power as redfish_power from ironic.drivers.modules import snmp scci = importutils.try_import('scciclient.irmc.scci') @@ -203,14 +204,17 @@ _wait_power_state(task, states.SOFT_REBOOT, timeout=timeout) except exception.SNMPFailure as snmp_exception: + advice = ("The SNMP related parameters' value may be different with " + "the server, please check if you have set them correctly.") LOG.error("iRMC failed to acknowledge the target state " - "for node %(node_id)s. Error: %(error)s", - {'node_id': node.uuid, 'error': snmp_exception}) + "for node %(node_id)s. Error: %(error)s. %(advice)s", + {'node_id': node.uuid, 'error': snmp_exception, + 'advice': advice}) raise exception.IRMCOperationError(operation=target_state, error=snmp_exception) -class IRMCPower(base.PowerInterface): +class IRMCPower(redfish_power.RedfishPower, base.PowerInterface): """Interface for power-related actions.""" def get_properties(self): @@ -233,7 +237,19 @@ is missing or invalid on the node. :raises: MissingParameterValue if a required parameter is missing. """ - irmc_common.parse_driver_info(task.node) + # validate method of power interface is called at very first point + # in verifying. + # We take try-fallback approach against iRMC S6 2.00 and later + # incompatibility in which iRMC firmware disables IPMI by default. + # get_power_state method first try IPMI and if fails try Redfish + # along with setting irmc_ipmi_succeed flag to indicate if IPMI works. + if (task.node.driver_internal_info.get('irmc_ipmi_succeed') + or (task.node.driver_internal_info.get('irmc_ipmi_succeed') + is None)): + irmc_common.parse_driver_info(task.node) + else: + irmc_common.parse_driver_info(task.node) + super(IRMCPower, self).validate(task) @METRICS.timer('IRMCPower.get_power_state') def get_power_state(self, task): @@ -241,14 +257,40 @@ :param task: a TaskManager instance containing the node to act on. :returns: a power state. One of :mod:`ironic.common.states`. - :raises: InvalidParameterValue if required ipmi parameters are missing. - :raises: MissingParameterValue if a required parameter is missing. - :raises: IPMIFailure on an error from ipmitool (from _power_status - call). + :raises: InvalidParameterValue if required parameters are incorrect. + :raises: MissingParameterValue if required parameters are missing. + :raises: IRMCOperationError If IPMI or Redfish operation fails """ - irmc_common.update_ipmi_properties(task) - ipmi_power = ipmitool.IPMIPower() - return ipmi_power.get_power_state(task) + # If IPMI operation failed, iRMC may not enable/support IPMI, + # so fallback to Redfish. + # get_power_state is called at verifying and is called periodically + # so this method is good choice to determine IPMI enablement. + try: + irmc_common.update_ipmi_properties(task) + ipmi_power = ipmitool.IPMIPower() + pw_state = ipmi_power.get_power_state(task) + if (task.node.driver_internal_info.get('irmc_ipmi_succeed') + is not True): + task.upgrade_lock(purpose='update irmc_ipmi_succeed flag', + retry=True) + task.node.set_driver_internal_info('irmc_ipmi_succeed', True) + task.node.save() + task.downgrade_lock() + return pw_state + except exception.IPMIFailure: + if (task.node.driver_internal_info.get('irmc_ipmi_succeed') + is not False): + task.upgrade_lock(purpose='update irmc_ipmi_succeed flag', + retry=True) + task.node.set_driver_internal_info('irmc_ipmi_succeed', False) + task.node.save() + task.downgrade_lock() + try: + return super(IRMCPower, self).get_power_state(task) + except (exception.RedfishConnectionError, + exception.RedfishError): + raise exception.IRMCOperationError( + operation='IPMI try and Redfish fallback operation') @METRICS.timer('IRMCPower.set_power_state') @task_manager.require_exclusive_lock diff -Nru ironic-21.1.0/ironic/drivers/modules/irmc/vendor.py ironic-21.4.4/ironic/drivers/modules/irmc/vendor.py --- ironic-21.1.0/ironic/drivers/modules/irmc/vendor.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/irmc/vendor.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,75 @@ +# Copyright 2022 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Vendor interface of iRMC driver +""" + +from ironic.common import exception +from ironic.common.i18n import _ +from ironic.drivers import base +from ironic.drivers.modules.irmc import common as irmc_common + + +class IRMCVendorPassthru(base.VendorInterface): + def get_properties(self): + """Return the properties of the interface. + + :returns: Dictionary of : entries. + """ + return irmc_common.COMMON_PROPERTIES + + def validate(self, task, method=None, **kwargs): + """Validate vendor-specific actions. + + This method validates whether the 'driver_info' property of the + supplied node contains the required information for this driver. + + :param task: An instance of TaskManager. + :param method: Name of vendor passthru method + :raises: InvalidParameterValue if invalid value is contained + in the 'driver_info' property. + :raises: MissingParameterValue if some mandatory key is missing + in the 'driver_info' property. + """ + irmc_common.parse_driver_info(task.node) + + @base.passthru(['POST'], + async_call=True, + description='Connect to iRMC and fetch iRMC firmware ' + 'version and, if firmware version has not been cached ' + 'in or actual firmware version is different from one in ' + 'driver_internal_info/irmc_fw_version, store firmware ' + 'version in driver_internal_info/irmc_fw_version.', + attach=False, + require_exclusive_lock=False) + def cache_irmc_firmware_version(self, task, **kwargs): + """Fetch and save iRMC firmware version. + + This method connects to iRMC and fetch iRMC firmware verison. + If fetched firmware version is not cached in or is different from + one in driver_internal_info/irmc_fw_version, store fetched version + in driver_internal_info/irmc_fw_version. + + :param task: An instance of TaskManager. + :raises: IRMCOperationError if some error occurs + """ + try: + irmc_common.set_irmc_version(task) + except (exception.IRMCOperationError, + exception.InvalidParameterValue, + exception.MissingParameterValue, + exception.NodeLocked) as e: + raise exception.IRMCOperationError( + operation=_('caching firmware version'), error=e) diff -Nru ironic-21.1.0/ironic/drivers/modules/network/neutron.py ironic-21.4.4/ironic/drivers/modules/network/neutron.py --- ironic-21.1.0/ironic/drivers/modules/network/neutron.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/network/neutron.py 2024-10-11 15:42:16.000000000 +0000 @@ -33,22 +33,6 @@ base.NetworkInterface): """Neutron v2 network interface""" - def __init__(self): - failures = [] - cleaning_net = CONF.neutron.cleaning_network - if not cleaning_net: - failures.append('cleaning_network') - - provisioning_net = CONF.neutron.provisioning_network - if not provisioning_net: - failures.append('provisioning_network') - - if failures: - raise exception.DriverLoadError( - driver=self.__class__.__name__, - reason=(_('The following [neutron] group configuration ' - 'options are missing: %s') % ', '.join(failures))) - def validate(self, task): """Validates the network interface. @@ -57,6 +41,8 @@ is invalid. :raises: MissingParameterValue, if some parameters are missing. """ + # NOTE(TheJulia): These are the minimal networks needed for + # the neutron network interface to function. self.get_cleaning_network_uuid(task) self.get_provisioning_network_uuid(task) diff -Nru ironic-21.1.0/ironic/drivers/modules/pxe.py ironic-21.4.4/ironic/drivers/modules/pxe.py --- ironic-21.1.0/ironic/drivers/modules/pxe.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/pxe.py 2024-10-11 15:42:16.000000000 +0000 @@ -27,6 +27,7 @@ from ironic.conf import CONF from ironic.drivers import base from ironic.drivers.modules import agent_base +from ironic.drivers.modules import boot_mode_utils from ironic.drivers.modules import deploy_utils from ironic.drivers.modules import pxe_base LOG = logging.getLogger(__name__) @@ -114,21 +115,11 @@ def reboot_to_instance(self, task): node = task.node try: - # anaconda deploy will install the bootloader and the node is ready - # to boot from disk. - - deploy_utils.try_set_boot_device(task, boot_devices.DISK) - except Exception as e: - msg = (_("Failed to change the boot device to %(boot_dev)s " - "when deploying node %(node)s. Error: %(error)s") % - {'boot_dev': boot_devices.DISK, 'node': node.uuid, - 'error': e}) - agent_base.log_and_raise_deployment_error(task, msg) - - try: task.process_event('resume') self.clean_up(task) manager_utils.node_power_action(task, states.POWER_OFF) + deploy_utils.try_set_boot_device(task, boot_devices.DISK) + boot_mode_utils.configure_secure_boot_if_needed(task) task.driver.network.remove_provisioning_network(task) task.driver.network.configure_tenant_networks(task) manager_utils.node_power_action(task, states.POWER_ON) diff -Nru ironic-21.1.0/ironic/drivers/modules/pxe_base.py ironic-21.4.4/ironic/drivers/modules/pxe_base.py --- ironic-21.1.0/ironic/drivers/modules/pxe_base.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/pxe_base.py 2024-10-11 15:42:16.000000000 +0000 @@ -231,11 +231,12 @@ :returns: None """ boot_mode_utils.sync_boot_mode(task) - boot_mode_utils.configure_secure_boot_if_needed(task) - node = task.node - boot_option = deploy_utils.get_boot_option(node) boot_device = None + boot_option = deploy_utils.get_boot_option(node) + if boot_option != "kickstart": + boot_mode_utils.configure_secure_boot_if_needed(task) + instance_image_info = {} if boot_option == "ramdisk" or boot_option == "kickstart": instance_image_info = pxe_utils.get_instance_image_info( @@ -283,6 +284,19 @@ # NOTE(pas-ha) do not re-set boot device on ACTIVE nodes # during takeover if boot_device and task.node.provision_state != states.ACTIVE: + vendor = task.node.properties.get('vendor', None) + boot_mode = boot_mode_utils.get_boot_mode(task.node) + if (task.node.provision_state == states.DEPLOYING + and vendor and vendor.lower() == 'lenovo' + and boot_mode == 'uefi' + and boot_device == boot_devices.DISK): + # Lenovo hardware is modeled on a "just update" + # UEFI nvram model of use, and if multiple actions + # get requested, you can end up in cases where NVRAM + # changes are deleted as the host "restores" to the + # backup. For more information see + # https://bugs.launchpad.net/ironic/+bug/2053064 + return manager_utils.node_set_boot_device(task, boot_device, persistent=True) diff -Nru ironic-21.1.0/ironic/drivers/modules/pxe_grub_config.template ironic-21.4.4/ironic/drivers/modules/pxe_grub_config.template --- ironic-21.1.0/ironic/drivers/modules/pxe_grub_config.template 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/pxe_grub_config.template 2024-10-11 15:42:16.000000000 +0000 @@ -15,3 +15,8 @@ menuentry "boot_whole_disk" { linuxefi chain.c32 mbr:{{ DISK_IDENTIFIER }} } + +menuentry "boot_anaconda" { + linuxefi {{ pxe_options.aki_path }} text {{ pxe_options.pxe_append_params|default("", true) }} inst.ks={{ pxe_options.ks_cfg_url }} {% if pxe_options.repo_url %}inst.repo={{ pxe_options.repo_url }}{% else %}inst.stage2={{ pxe_options.stage2_url }}{% endif %} + initrdefi {{ pxe_options.ari_path }} +} diff -Nru ironic-21.1.0/ironic/drivers/modules/redfish/boot.py ironic-21.4.4/ironic/drivers/modules/redfish/boot.py --- ironic-21.1.0/ironic/drivers/modules/redfish/boot.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/redfish/boot.py 2024-10-11 15:42:16.000000000 +0000 @@ -233,8 +233,23 @@ for manager in managers: for v_media in manager.virtual_media.get_members(): if boot_device and boot_device not in v_media.media_types: - continue - + # NOTE(iurygregory): this conditional allows v_media that only + # support DVD MediaType and NOT CD to also be used. + # if v_media.media_types contains sushy.VIRTUAL_MEDIA_DVD + # we follow the usual steps of checking if v_media is inserted + # and eject it. Otherwise we skip to the + # next v_media device, if any. + # This is needed to add support to Cisco UCSB and UCSX blades + # reference: https://bugs.launchpad.net/ironic/+bug/2039042 + if (boot_device == sushy.VIRTUAL_MEDIA_CD + and sushy.VIRTUAL_MEDIA_DVD in v_media.media_types): + LOG.debug('While looking for %(requested_device)s virtual ' + 'media device, found %(available_device)s ' + 'instead. Attempting to use it to eject media.', + {'requested_device': sushy.VIRTUAL_MEDIA_CD, + 'available_device': sushy.VIRTUAL_MEDIA_DVD}) + else: + continue inserted = v_media.inserted if inserted: diff -Nru ironic-21.1.0/ironic/drivers/modules/redfish/firmware_utils.py ironic-21.4.4/ironic/drivers/modules/redfish/firmware_utils.py --- ironic-21.1.0/ironic/drivers/modules/redfish/firmware_utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/redfish/firmware_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -29,7 +29,7 @@ LOG = log.getLogger(__name__) _UPDATE_FIRMWARE_SCHEMA = { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-04/schema#", "title": "update_firmware clean step schema", "type": "array", # list of firmware update images diff -Nru ironic-21.1.0/ironic/drivers/modules/redfish/management.py ironic-21.4.4/ironic/drivers/modules/redfish/management.py --- ironic-21.1.0/ironic/drivers/modules/redfish/management.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/redfish/management.py 2024-10-11 15:42:16.000000000 +0000 @@ -1197,9 +1197,18 @@ :raises: RedfishError on an error from the Sushy library :returns: A list of MAC addresses for the node """ + system = redfish_utils.get_system(task.node) try: - system = redfish_utils.get_system(task.node) return list(redfish_utils.get_enabled_macs(task, system)) + # NOTE(janders) we should handle MissingAttributeError separately + # from other SushyErrors - some servers (e.g. some Cisco UCSB and UCSX + # blades) are missing EthernetInterfaces attribute yet could be + # provisioned successfully if MAC information is provided manually AND + # this exception is caught and handled accordingly. + except sushy.exceptions.MissingAttributeError as exc: + LOG.warning('Cannot get MAC addresses for node %(node)s: %(exc)s', + {'node': task.node.uuid, 'exc': exc}) + # if the exception is not a MissingAttributeError, raise it except sushy.exceptions.SushyError as exc: msg = (_('Failed to get network interface information on node ' '%(node)s: %(exc)s') diff -Nru ironic-21.1.0/ironic/drivers/modules/redfish/raid.py ironic-21.4.4/ironic/drivers/modules/redfish/raid.py --- ironic-21.1.0/ironic/drivers/modules/redfish/raid.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/redfish/raid.py 2024-10-11 15:42:16.000000000 +0000 @@ -43,7 +43,6 @@ 'min_disks': 1, 'max_disks': 1000, 'type': 'simple', - 'volume_type': 'NonRedundant', 'raid_type': 'RAID0', 'overhead': 0 }, @@ -51,7 +50,6 @@ 'min_disks': 2, 'max_disks': 2, 'type': 'simple', - 'volume_type': 'Mirrored', 'raid_type': 'RAID1', 'overhead': 1 }, @@ -59,7 +57,6 @@ 'min_disks': 3, 'max_disks': 1000, 'type': 'simple', - 'volume_type': 'StripedWithParity', 'raid_type': 'RAID5', 'overhead': 1 }, @@ -67,25 +64,21 @@ 'min_disks': 4, 'max_disks': 1000, 'type': 'simple', - 'volume_type': 'StripedWithParity', 'raid_type': 'RAID6', 'overhead': 2 }, '1+0': { 'type': 'spanned', - 'volume_type': 'SpannedMirrors', 'raid_type': 'RAID10', 'span_type': '1' }, '5+0': { 'type': 'spanned', - 'volume_type': 'SpannedStripesWithParity', 'raid_type': 'RAID50', 'span_type': '5' }, '6+0': { 'type': 'spanned', - 'volume_type': 'SpannedStripesWithParity', 'raid_type': 'RAID60', 'span_type': '6' } @@ -618,17 +611,17 @@ def _construct_volume_payload( node, storage, raid_controller, physical_disks, raid_level, size_bytes, disk_name=None, span_length=None, span_depth=None): - payload = {'Encrypted': False, - 'VolumeType': RAID_LEVELS[raid_level]['volume_type'], - 'RAIDType': RAID_LEVELS[raid_level]['raid_type'], - 'CapacityBytes': size_bytes} + payload = { + 'RAIDType': RAID_LEVELS[raid_level]['raid_type'], + 'CapacityBytes': size_bytes + } if physical_disks: payload['Links'] = { "Drives": [{"@odata.id": _drive_path(storage, d)} for d in physical_disks] } LOG.debug('Payload for RAID logical disk creation on node %(node_uuid)s: ' - '%(payload)r', {'node': node.uuid, 'payload': payload}) + '%(payload)r', {'node_uuid': node.uuid, 'payload': payload}) return payload @@ -1120,7 +1113,9 @@ raid_configs['pending'].setdefault(controller, []).append( logical_disk) - node.set_driver_internal_info('raid_configs', raid_configs) + # Store only when async operation + if reboot_required: + node.set_driver_internal_info('raid_configs', raid_configs) return raid_configs, reboot_required @@ -1147,8 +1142,7 @@ controller_id = storage.identity iter_volumes = iter(storage.volumes.get_members()) for volume in iter_volumes: - if (volume.raid_type or volume.volume_type not in - [None, sushy.VOLUME_TYPE_RAW_DEVICE]): + if volume.raid_type: if controller_id not in vols_to_delete: vols_to_delete[controller_id] = [] apply_time = self._get_apply_time( @@ -1182,7 +1176,9 @@ response.task_monitor_uri) reboot_required = True - node.set_driver_internal_info('raid_configs', raid_configs) + # Store only when async operation + if reboot_required: + node.set_driver_internal_info('raid_configs', raid_configs) return raid_configs, reboot_required diff -Nru ironic-21.1.0/ironic/drivers/modules/redfish/utils.py ironic-21.4.4/ironic/drivers/modules/redfish/utils.py --- ironic-21.1.0/ironic/drivers/modules/redfish/utils.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/drivers/modules/redfish/utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -28,6 +28,7 @@ from ironic.common import exception from ironic.common.i18n import _ +from ironic.common import utils from ironic.conf import CONF sushy = importutils.try_import('sushy') @@ -97,7 +98,7 @@ 'info': missing_info}) # Validate the Redfish address - address = driver_info['redfish_address'] + address = utils.wrap_ipv6(driver_info['redfish_address']) try: parsed = rfc3986.uri_reference(address) except TypeError: diff -Nru ironic-21.1.0/ironic/objects/__init__.py ironic-21.4.4/ironic/objects/__init__.py --- ironic-21.1.0/ironic/objects/__init__.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/objects/__init__.py 2024-10-11 15:42:16.000000000 +0000 @@ -32,6 +32,7 @@ __import__('ironic.objects.deployment') __import__('ironic.objects.node') __import__('ironic.objects.node_history') + __import__('ironic.objects.node_inventory') __import__('ironic.objects.port') __import__('ironic.objects.portgroup') __import__('ironic.objects.trait') diff -Nru ironic-21.1.0/ironic/objects/node.py ironic-21.4.4/ironic/objects/node.py --- ironic-21.1.0/ironic/objects/node.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/objects/node.py 2024-10-11 15:42:16.000000000 +0000 @@ -78,7 +78,8 @@ # Version 1.34: Add lessee field # Version 1.35: Add network_data field # Version 1.36: Add boot_mode and secure_boot fields - VERSION = '1.36' + # Version 1.37: Add shard field + VERSION = '1.37' dbapi = db_api.get_instance() @@ -170,6 +171,7 @@ 'network_data': object_fields.FlexibleDictField(nullable=True), 'boot_mode': object_fields.StringField(nullable=True), 'secure_boot': object_fields.BooleanField(nullable=True), + 'shard': object_fields.StringField(nullable=True), } def as_dict(self, secure=False, mask_configdrive=True): @@ -656,6 +658,8 @@ should be set to empty dict (or removed). Version 1.36: boot_mode, secure_boot were was added. Defaults are None. For versions prior to this, it should be set to None or removed. + Version 1.37: shard was added. Default is None. For versions prior to + this, it should be set to None or removed. :param target_version: the desired version of the object :param remove_unavailable_fields: True to remove fields that are @@ -671,7 +675,7 @@ ('automated_clean', 28), ('protected_reason', 29), ('owner', 30), ('allocation_id', 31), ('description', 32), ('retired_reason', 33), ('lessee', 34), ('boot_mode', 36), - ('secure_boot', 36)] + ('secure_boot', 36), ('shard', 37)] for name, minor in fields: self._adjust_field_to_version(name, None, target_version, diff -Nru ironic-21.1.0/ironic/objects/node_inventory.py ironic-21.4.4/ironic/objects/node_inventory.py --- ironic-21.1.0/ironic/objects/node_inventory.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/objects/node_inventory.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,89 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_versionedobjects import base as object_base + +from ironic.db import api as dbapi +from ironic.objects import base +from ironic.objects import fields as object_fields + + +@base.IronicObjectRegistry.register +class NodeInventory(base.IronicObject, object_base.VersionedObjectDictCompat): + # Version 1.0: Initial version + VERSION = '1.0' + + dbapi = dbapi.get_instance() + + fields = { + 'id': object_fields.IntegerField(), + 'node_id': object_fields.IntegerField(nullable=True), + 'inventory_data': object_fields.FlexibleDictField(nullable=True), + 'plugin_data': object_fields.FlexibleDictField(nullable=True), + } + + @classmethod + def _from_node_object(cls, context, node): + """Convert a node into a virtual `NodeInventory` object.""" + result = cls(context) + result._update_from_node_object(node) + return result + + def _update_from_node_object(self, node): + """Update the NodeInventory object from the node.""" + for src, dest in self.node_mapping.items(): + setattr(self, dest, getattr(node, src, None)) + for src, dest in self.instance_info_mapping.items(): + setattr(self, dest, node.instance_info.get(src)) + + @classmethod + def get_by_node_id(cls, context, node_id): + """Get a NodeInventory object by its node ID. + + :param cls: the :class:`NodeInventory` + :param context: Security context + :param uuid: The UUID of a NodeInventory. + :returns: A :class:`NodeInventory` object. + :raises: NodeInventoryNotFound + + """ + db_inventory = cls.dbapi.get_node_inventory_by_node_id(node_id) + inventory = cls._from_db_object(context, cls(), db_inventory) + return inventory + + def create(self, context=None): + """Create a NodeInventory record in the DB. + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: NodeHistory(context) + """ + values = self.do_version_changes_for_db() + db_inventory = self.dbapi.create_node_inventory(values) + self._from_db_object(self._context, self, db_inventory) + + def destroy(self, context=None): + """Delete the NodeInventory from the DB. + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: NodeInventory(context) + :raises: NodeInventoryNotFound + """ + self.dbapi.destroy_node_inventory_by_node_id(self.node_id) + self.obj_reset_changes() diff -Nru ironic-21.1.0/ironic/objects/port.py ironic-21.4.4/ironic/objects/port.py --- ironic-21.1.0/ironic/objects/port.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/objects/port.py 2024-10-11 15:42:16.000000000 +0000 @@ -44,7 +44,8 @@ # change) # Version 1.9: Add support for Smart NIC port # Version 1.10: Add name field - VERSION = '1.10' + # Version 1.11: Add node_uuid field + VERSION = '1.11' dbapi = dbapi.get_instance() @@ -52,6 +53,7 @@ 'id': object_fields.IntegerField(), 'uuid': object_fields.UUIDField(nullable=True), 'node_id': object_fields.IntegerField(nullable=True), + 'node_uuid': object_fields.UUIDField(nullable=True), 'address': object_fields.MACAddressField(nullable=True), 'extra': object_fields.FlexibleDictField(nullable=True), 'local_link_connection': object_fields.FlexibleDictField( @@ -297,6 +299,27 @@ project=project) return cls._from_db_object_list(context, db_ports) + @classmethod + def list_by_node_shards(cls, context, shards, limit=None, marker=None, + sort_key=None, sort_dir=None, project=None): + """Return a list of Port objects associated with nodes in shards + + :param context: Security context. + :param shards: a list of shards + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param project: a node owner or lessee to match against + :returns: a list of :class:`Port` object. + + """ + db_ports = cls.dbapi.get_ports_by_shards(shards, limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_ports) + # NOTE(xek): We don't want to enable RPC on this call just yet. Remotable # methods can be used in the future to replace current explicit RPC calls. # Implications of calling new remote procedures should be thought through. @@ -377,6 +400,10 @@ """ values = self.do_version_changes_for_db() db_port = self.dbapi.create_port(values) + # NOTE(hjensas): To avoid lazy load issue (DetachedInstanceError) in + # sqlalchemy, get new port the port from the DB to ensure the node_uuid + # via association_proxy relationship is loaded. + db_port = self.dbapi.get_port_by_id(db_port['id']) self._from_db_object(self._context, self, db_port) # NOTE(xek): We don't want to enable RPC on this call just yet. Remotable diff -Nru ironic-21.1.0/ironic/objects/portgroup.py ironic-21.4.4/ironic/objects/portgroup.py --- ironic-21.1.0/ironic/objects/portgroup.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/objects/portgroup.py 2024-10-11 15:42:16.000000000 +0000 @@ -36,7 +36,8 @@ # Version 1.4: Migrate/copy extra['vif_port_id'] to # internal_info['tenant_vif_port_id'] (not an explicit db # change) - VERSION = '1.4' + # Version 1.5: Add node_uuid field + VERSION = '1.5' dbapi = dbapi.get_instance() @@ -45,6 +46,7 @@ 'uuid': object_fields.UUIDField(nullable=True), 'name': object_fields.StringField(nullable=True), 'node_id': object_fields.IntegerField(nullable=True), + 'node_uuid': object_fields.UUIDField(nullable=True), 'address': object_fields.MACAddressField(nullable=True), 'extra': object_fields.FlexibleDictField(nullable=True), 'internal_info': object_fields.FlexibleDictField(nullable=True), @@ -261,6 +263,10 @@ """ values = self.do_version_changes_for_db() db_portgroup = self.dbapi.create_portgroup(values) + # NOTE(hjensas): To avoid lazy load issue (DetachedInstanceError) in + # sqlalchemy, get new port the port from the DB to ensure the node_uuid + # via association_proxy relationship is loaded. + db_portgroup = self.dbapi.get_portgroup_by_id(db_portgroup['id']) self._from_db_object(self._context, self, db_portgroup) # NOTE(xek): We don't want to enable RPC on this call just yet. Remotable diff -Nru ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_allocation.py ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_allocation.py --- ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_allocation.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_allocation.py 2024-10-11 15:42:16.000000000 +0000 @@ -28,6 +28,7 @@ from ironic.api.controllers import base as api_base from ironic.api.controllers import v1 as api_v1 from ironic.api.controllers.v1 import notification_utils +from ironic.api.controllers.v1 import utils as v1_api_utils from ironic.common import exception from ironic.common import policy from ironic.conductor import rpcapi @@ -420,6 +421,16 @@ self.assertEqual(http_client.NOT_ACCEPTABLE, response.status_code) self.assertTrue(response.json['error_message']) + @mock.patch.object(v1_api_utils, 'check_list_policy', autospec=True) + def test_get_all_by_owner_not_allowed_mismatch(self, mock_check): + mock_check.return_value = '54321' + response = self.get_json("/allocations?owner=12345", + headers={api_base.Version.string: '1.60'}, + expect_errors=True) + self.assertEqual('application/json', response.content_type) + self.assertEqual(http_client.FORBIDDEN, response.status_code) + self.assertTrue(response.json['error_message']) + def test_get_all_by_node_name(self): for i in range(5): if i < 3: diff -Nru ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_node.py ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_node.py --- ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_node.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_node.py 2024-10-11 15:42:16.000000000 +0000 @@ -21,6 +21,7 @@ import tempfile from unittest import mock from urllib import parse as urlparse +import uuid import fixtures from oslo_config import cfg @@ -43,6 +44,8 @@ from ironic.common import policy from ironic.common import states from ironic.conductor import rpcapi +from ironic.drivers.modules import inspect_utils +from ironic.drivers.modules import inspector from ironic import objects from ironic.objects import fields as obj_fields from ironic import tests as tests_root @@ -51,6 +54,7 @@ from ironic.tests.unit.api import utils as test_api_utils from ironic.tests.unit.objects import utils as obj_utils +CONF = inspector.CONF with open( os.path.join( @@ -7912,3 +7916,222 @@ self.assertIn('nodes/%s/history' % self.node.uuid, ret['next']) self.assertIn('limit=1', ret['next']) self.assertIn('marker=%s' % result_uuid, ret['next']) + + +class TestNodeInventory(test_api_base.BaseApiTest): + fake_inventory_data = {"cpu": "amd"} + fake_plugin_data = {"disks": [{"name": "/dev/vda"}]} + + def setUp(self): + super(TestNodeInventory, self).setUp() + self.version = "1.81" + self.node = obj_utils.create_test_node( + self.context, + provision_state=states.AVAILABLE, name='node-81') + self.node.save() + self.node.obj_reset_changes() + + def _add_inventory(self): + self.inventory = objects.NodeInventory( + node_id=self.node.id, inventory_data=self.fake_inventory_data, + plugin_data=self.fake_plugin_data) + self.inventory.create() + + def test_get_old_version(self): + ret = self.get_json('/nodes/%s/inventory' % self.node.uuid, + headers={api_base.Version.string: "1.80"}, + expect_errors=True) + self.assertEqual(http_client.NOT_FOUND, ret.status_code) + + def test_get_inventory_no_inventory(self): + ret = self.get_json('/nodes/%s/inventory' % self.node.uuid, + headers={api_base.Version.string: self.version}, + expect_errors=True) + self.assertEqual(http_client.NOT_FOUND, ret.status_code) + + def test_get_inventory(self): + self._add_inventory() + CONF.set_override('data_backend', 'database', + group='inventory') + ret = self.get_json('/nodes/%s/inventory' % self.node.uuid, + headers={api_base.Version.string: self.version}) + self.assertEqual({'inventory': self.fake_inventory_data, + 'plugin_data': self.fake_plugin_data}, ret) + + @mock.patch.object(inspect_utils, 'get_introspection_data', + autospec=True) + def test_get_inventory_exception(self, mock_get_data): + CONF.set_override('data_backend', 'database', + group='inventory') + mock_get_data.side_effect = [ + exception.NodeInventoryNotFound] + ret = self.get_json('/nodes/%s/inventory' % self.node.uuid, + headers={api_base.Version.string: self.version}, + expect_errors=True) + self.assertEqual(http_client.NOT_FOUND, ret.status_int) + + @mock.patch.object(inspect_utils, '_get_introspection_data_from_swift', + autospec=True) + def test_get_inventory_swift(self, mock_get_data): + CONF.set_override('data_backend', 'swift', + group='inventory') + mock_get_data.return_value = {"inventory": self.fake_inventory_data, + "plugin_data": self.fake_plugin_data} + ret = self.get_json('/nodes/%s/inventory' % self.node.uuid, + headers={api_base.Version.string: self.version}) + self.assertEqual({'inventory': self.fake_inventory_data, + 'plugin_data': self.fake_plugin_data}, ret) + + +class TestNodeShardGets(test_api_base.BaseApiTest): + def setUp(self): + super(TestNodeShardGets, self).setUp() + p = mock.patch.object(rpcapi.ConductorAPI, 'get_topic_for', + autospec=True) + self.mock_gtf = p.start() + self.mock_gtf.return_value = 'test-topic' + self.addCleanup(p.stop) + self.mock_get_conductor_for = self.useFixture( + fixtures.MockPatchObject(rpcapi.ConductorAPI, 'get_conductor_for', + autospec=True)).mock + self.mock_get_conductor_for.return_value = 'fake.conductor' + self.node = obj_utils.create_test_node(self.context, shard='foo') + self.headers = {api_base.Version.string: '1.82'} + + def test_get_node_shard_field(self): + result = self.get_json( + '/nodes/%s' % self.node.uuid, headers=self.headers) + self.assertEqual('foo', result['shard']) + + def test_get_node_shard_field_fails_wrong_version(self): + headers = {api_base.Version.string: '1.80'} + result = self.get_json('/nodes/%s' % self.node.uuid, headers=headers) + self.assertNotIn('shard', result) + + def test_filtering_by_shard(self): + result = self.get_json( + '/nodes?shard=foo', fields='shard', headers=self.headers) + self.assertEqual(1, len(result['nodes'])) + self.assertEqual('foo', result['nodes'][0]['shard']) + + def test_filtering_by_shard_fails_wrong_version(self): + headers = {api_base.Version.string: '1.80'} + + result = self.get_json('/nodes?shard=foo', + expect_errors=True, headers=headers) + self.assertEqual(http_client.NOT_ACCEPTABLE, result.status_code) + + def test_filtering_by_single_shard_detail(self): + result = self.get_json('/nodes/detail?shard=foo', headers=self.headers) + self.assertEqual(1, len(result['nodes'])) + self.assertEqual('foo', result['nodes'][0]['shard']) + + def test_filtering_by_multi_shard_detail(self): + obj_utils.create_test_node( + self.context, uuid=uuid.uuid4(), shard='bar') + result = self.get_json( + '/nodes?shard=foo,bar', headers=self.headers) + self.assertEqual(2, len(result['nodes'])) + + def test_filtering_by_shard_detail_fails_wrong_version(self): + headers = {api_base.Version.string: '1.80'} + + result = self.get_json('/nodes/detail?shard=foo', + expect_errors=True, headers=headers) + self.assertEqual(http_client.NOT_ACCEPTABLE, result.status_code) + + def test_filtering_by_sharded(self): + obj_utils.create_test_node(self.context, uuid=uuid.uuid4()) + obj_utils.create_test_node(self.context, uuid=uuid.uuid4()) + # We now have one node in shard foo (setUp) and two unsharded. + result_true = self.get_json( + '/nodes?sharded=true', headers=self.headers) + result_false = self.get_json( + '/nodes?sharded=false', headers=self.headers) + self.assertEqual(1, len(result_true['nodes'])) + self.assertEqual(2, len(result_false['nodes'])) + + +@mock.patch.object(rpcapi.ConductorAPI, 'create_node', + lambda _api, _ctx, node, _topic: _create_node_locally(node)) +class TestNodeShardPost(test_api_base.BaseApiTest): + def setUp(self): + super(TestNodeShardPost, self).setUp() + p = mock.patch.object(rpcapi.ConductorAPI, 'get_topic_for', + autospec=True) + self.mock_gtf = p.start() + self.mock_gtf.return_value = 'test-topic' + self.addCleanup(p.stop) + self.chassis = obj_utils.create_test_chassis(self.context) + + def test_create_node_with_shard(self): + shard = 'foo' + ndict = test_api_utils.post_get_test_node(shard=shard) + headers = {api_base.Version.string: '1.82'} + response = self.post_json('/nodes', ndict, headers=headers) + self.assertEqual(http_client.CREATED, response.status_int) + + result = self.get_json('/nodes/%s' % ndict['uuid'], headers=headers) + self.assertEqual(ndict['uuid'], result['uuid']) + self.assertEqual(shard, result['shard']) + + def test_create_node_with_shard_fail_wrong_version(self): + headers = {api_base.Version.string: '1.80'} + shard = 'foo' + ndict = test_api_utils.post_get_test_node(shard=shard) + response = self.post_json( + '/nodes', ndict, expect_errors=True, headers=headers) + self.assertEqual(http_client.NOT_ACCEPTABLE, response.status_int) + + +class TestNodeShardPatch(test_api_base.BaseApiTest): + def setUp(self): + super(TestNodeShardPatch, self).setUp() + self.node = obj_utils.create_test_node(self.context, name='node-57.1') + p = mock.patch.object(rpcapi.ConductorAPI, 'get_topic_for', + autospec=True) + self.mock_gtf = p.start() + self.mock_gtf.return_value = 'test-topic' + self.addCleanup(p.stop) + p = mock.patch.object(rpcapi.ConductorAPI, 'update_node', + autospec=True) + self.mock_update_node = p.start() + self.addCleanup(p.stop) + + def test_node_add_shard(self): + self.mock_update_node.return_value = self.node + (self + .mock_update_node + .return_value + .updated_at) = "2013-12-03T06:20:41.184720+00:00" + headers = {api_base.Version.string: '1.82'} + shard = 'shard1' + body = [{ + 'path': '/shard', + 'value': shard, + 'op': 'add', + }] + + response = self.patch_json( + '/nodes/%s' % self.node.uuid, body, headers=headers) + self.assertEqual(http_client.OK, response.status_code) + self.mock_update_node.assert_called_once() + + def test_node_add_shard_fail_wrong_version(self): + self.mock_update_node.return_value = self.node + (self + .mock_update_node + .return_value + .updated_at) = "2013-12-03T06:20:41.184720+00:00" + headers = {api_base.Version.string: '1.80'} + shard = 'shard1' + body = [{ + 'path': '/shard', + 'value': shard, + 'op': 'add', + }] + + response = self.patch_json('/nodes/%s' % self.node.uuid, + body, expect_errors=True, headers=headers) + self.mock_update_node.assert_not_called() + self.assertEqual(http_client.NOT_ACCEPTABLE, response.status_code) diff -Nru ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_port.py ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_port.py --- ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_port.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_port.py 2024-10-11 15:42:16.000000000 +0000 @@ -15,7 +15,6 @@ import datetime from http import client as http_client -import types from unittest import mock from urllib import parse as urlparse @@ -194,7 +193,7 @@ mock_request.context = 'fake-context' mock_list.return_value = [] self.controller._get_ports_collection(None, None, None, None, None, - None, 'asc', + None, None, 'asc', resource_url='ports') mock_list.assert_called_once_with('fake-context', 1000, None, project=None, sort_dir='asc', @@ -236,53 +235,6 @@ # never expose the node_id self.assertNotIn('node_id', data['ports'][0]) - # NOTE(jlvillal): autospec=True doesn't work on staticmethods: - # https://bugs.python.org/issue23078 - @mock.patch.object(objects.Node, 'get_by_id', spec_set=types.FunctionType) - def test_list_with_deleted_node(self, mock_get_node): - # check that we don't end up with HTTP 400 when node deletion races - # with listing ports - see https://launchpad.net/bugs/1748893 - obj_utils.create_test_port(self.context, node_id=self.node.id) - mock_get_node.side_effect = exception.NodeNotFound('boom') - data = self.get_json('/ports') - self.assertEqual([], data['ports']) - - # NOTE(jlvillal): autospec=True doesn't work on staticmethods: - # https://bugs.python.org/issue23078 - @mock.patch.object(objects.Node, 'get_by_id', - spec_set=types.FunctionType) - def test_list_detailed_with_deleted_node(self, mock_get_node): - # check that we don't end up with HTTP 400 when node deletion races - # with listing ports - see https://launchpad.net/bugs/1748893 - port = obj_utils.create_test_port(self.context, node_id=self.node.id) - port2 = obj_utils.create_test_port(self.context, node_id=self.node.id, - uuid=uuidutils.generate_uuid(), - address='66:44:55:33:11:22') - mock_get_node.side_effect = [exception.NodeNotFound('boom'), self.node] - data = self.get_json('/ports/detail') - # The "correct" port is still returned - self.assertEqual(1, len(data['ports'])) - self.assertIn(data['ports'][0]['uuid'], {port.uuid, port2.uuid}) - self.assertEqual(self.node.uuid, data['ports'][0]['node_uuid']) - - # NOTE(jlvillal): autospec=True doesn't work on staticmethods: - # https://bugs.python.org/issue23078 - @mock.patch.object(objects.Portgroup, 'get', spec_set=types.FunctionType) - def test_list_with_deleted_port_group(self, mock_get_pg): - # check that we don't end up with HTTP 400 when port group deletion - # races with listing ports - see https://launchpad.net/bugs/1748893 - portgroup = obj_utils.create_test_portgroup(self.context, - node_id=self.node.id) - port = obj_utils.create_test_port(self.context, node_id=self.node.id, - portgroup_id=portgroup.id) - mock_get_pg.side_effect = exception.PortgroupNotFound('boom') - data = self.get_json( - '/ports/detail', - headers={api_base.Version.string: str(api_v1.max_version())} - ) - self.assertEqual(port.uuid, data['ports'][0]["uuid"]) - self.assertIsNone(data['ports'][0]["portgroup_uuid"]) - @mock.patch.object(policy, 'authorize', spec=True) def test_list_non_admin_forbidden(self, mock_authorize): def mock_authorize_function(rule, target, creds): @@ -1129,6 +1081,44 @@ response.json['error_message']) +class TestListPortsByShard(test_api_base.BaseApiTest): + def setUp(self): + super(TestListPortsByShard, self).setUp() + self.headers = { + api_base.Version.string: '1.%s' % versions.MINOR_82_NODE_SHARD + } + + def _create_port_with_shard(self, shard, address): + node = obj_utils.create_test_node(self.context, owner='12345', + shard=shard, + uuid=uuidutils.generate_uuid()) + return obj_utils.create_test_port(self.context, name='port_%s' % shard, + node_id=node.id, address=address, + uuid=uuidutils.generate_uuid()) + + def test_get_by_shard_single_fail_api_version(self): + self._create_port_with_shard('test_shard', 'aa:bb:cc:dd:ee:ff') + data = self.get_json('/ports?shard=test_shard', expect_errors=True) + self.assertEqual(406, data.status_int) + + def test_get_by_shard_single(self): + port = self._create_port_with_shard('test_shard', 'aa:bb:cc:dd:ee:ff') + data = self.get_json('/ports?shard=test_shard', headers=self.headers) + self.assertEqual(port.uuid, data['ports'][0]["uuid"]) + + def test_get_by_shard_multi(self): + bad_shard_address = 'ee:ee:ee:ee:ee:ee' + self._create_port_with_shard('shard1', 'aa:bb:cc:dd:ee:ff') + self._create_port_with_shard('shard2', 'ab:bb:cc:dd:ee:ff') + self._create_port_with_shard('shard3', bad_shard_address) + + res = self.get_json('/ports?shard=shard1,shard2', headers=self.headers) + self.assertEqual(2, len(res['ports'])) + print(res['ports'][0]) + self.assertNotEqual(res['ports'][0]['address'], bad_shard_address) + self.assertNotEqual(res['ports'][1]['address'], bad_shard_address) + + @mock.patch.object(rpcapi.ConductorAPI, 'update_port', autospec=True, side_effect=_rpcapi_update_port) class TestPatch(test_api_base.BaseApiTest): diff -Nru ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_ramdisk.py ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_ramdisk.py --- ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_ramdisk.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_ramdisk.py 2024-10-11 15:42:16.000000000 +0000 @@ -79,6 +79,8 @@ 'heartbeat_timeout': CONF.api.ramdisk_heartbeat_timeout, 'agent_token': mock.ANY, 'agent_token_required': True, + 'disable_deep_image_inspection': CONF.conductor.disable_deep_image_inspection, # noqa + 'permitted_image_formats': CONF.conductor.permitted_image_formats, } self.assertEqual(expected_config, data['config']) self.assertIsNotNone(data['config']['agent_token']) diff -Nru ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_root.py ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_root.py --- ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_root.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_root.py 2024-10-11 15:42:16.000000000 +0000 @@ -147,6 +147,10 @@ {'href': 'http://localhost/v1/ports/', 'rel': 'self'}, {'href': 'http://localhost/ports/', 'rel': 'bookmark'} ], + 'shards': [ + {'href': 'http://localhost/v1/shards/', 'rel': 'self'}, + {'href': 'http://localhost/shards/', 'rel': 'bookmark'} + ], 'volume': [ {'href': 'http://localhost/v1/volume/', 'rel': 'self'}, {'href': 'http://localhost/volume/', 'rel': 'bookmark'} diff -Nru ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_shard.py ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_shard.py --- ironic-21.1.0/ironic/tests/unit/api/controllers/v1/test_shard.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/api/controllers/v1/test_shard.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,80 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests for the API /shards/ methods. +""" + +from http import client as http_client +import uuid + +from ironic.api.controllers import base as api_base +from ironic.api.controllers import v1 as api_v1 +from ironic.tests.unit.api import base as test_api_base +from ironic.tests.unit.objects import utils as obj_utils + + +class TestListShards(test_api_base.BaseApiTest): + headers = {api_base.Version.string: str(api_v1.max_version())} + + def _create_test_shard(self, name, count): + for i in range(count): + obj_utils.create_test_node( + self.context, uuid=uuid.uuid4(), shard=name) + + def test_empty(self): + data = self.get_json('/shards', headers=self.headers) + self.assertEqual([], data['shards']) + + def test_one_shard(self): + shard = 'shard1' + count = 1 + self._create_test_shard(shard, count) + data = self.get_json('/shards', headers=self.headers) + self.assertEqual(shard, data['shards'][0]['name']) + self.assertEqual(count, data['shards'][0]['count']) + + def test_multiple_shards(self): + for i in range(0, 6): + self._create_test_shard('shard{}'.format(i), i) + data = self.get_json('/shards', headers=self.headers) + self.assertEqual(5, len(data['shards'])) + + def test_nodes_but_no_shards(self): + self._create_test_shard(None, 5) + data = self.get_json('/shards', headers=self.headers) + self.assertEqual("None", data['shards'][0]['name']) + self.assertEqual(5, data['shards'][0]['count']) + + def test_fail_wrong_version(self): + headers = {api_base.Version.string: '1.80'} + self._create_test_shard('shard1', 1) + result = self.get_json( + '/shards', expect_errors=True, headers=headers) + self.assertEqual(http_client.NOT_FOUND, result.status_int) + + def test_fail_get_one(self): + # We do not implement a get /v1/shards/ endpoint + # validate it errors properly + self._create_test_shard('shard1', 1) + result = self.get_json( + '/shards/shard1', expect_errors=True, headers=self.headers) + self.assertEqual(http_client.NOT_FOUND, result.status_int) + + def test_fail_post(self): + result = self.post_json( + '/shards', {}, expect_errors=True, headers=self.headers) + self.assertEqual(http_client.METHOD_NOT_ALLOWED, result.status_int) + + def test_fail_put(self): + result = self.put_json( + '/shards', {}, expect_errors=True, headers=self.headers) + self.assertEqual(http_client.METHOD_NOT_ALLOWED, result.status_int) diff -Nru ironic-21.1.0/ironic/tests/unit/api/test_acl.py ironic-21.4.4/ironic/tests/unit/api/test_acl.py --- ironic-21.1.0/ironic/tests/unit/api/test_acl.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/api/test_acl.py 2024-10-11 15:42:16.000000000 +0000 @@ -77,12 +77,14 @@ def _fake_process_request(self, request, auth_token_request): pass - def _test_request(self, path, params=None, headers=None, method='get', + def _test_request(self, path, params=None, headers=None, method='get', # noqa: C901, E501 body=None, assert_status=None, assert_dict_contains=None, assert_list_length=None, deprecated=None, - self_manage_nodes=True): + self_manage_nodes=True, + enable_service_project=False, + service_project='service'): path = path.format(**self.format_data) self.mock_auth.side_effect = self._fake_process_request @@ -92,6 +94,13 @@ 'project_admin_can_manage_own_nodes', False, 'api') + if enable_service_project: + cfg.CONF.set_override('rbac_service_role_elevated_access', True) + if service_project != 'service': + # Enable us to sort of gracefully test a name variation + # with existing ddt test modeling. + cfg.CONF.set_override('rbac_service_project_name', + service_project) # always request the latest api version version = api_versions.max_version_string() @@ -286,6 +295,8 @@ db_utils.create_test_node_trait( node_id=fake_db_node['id']) fake_history = db_utils.create_test_history(node_id=fake_db_node.id) + fake_inventory = db_utils.create_test_inventory( + node_id=fake_db_node.id) # dedicated node for portgroup addition test to avoid # false positives with test runners. db_utils.create_test_node( @@ -309,6 +320,7 @@ 'volume_target_ident': fake_db_volume_target['uuid'], 'volume_connector_ident': fake_db_volume_connector['uuid'], 'history_ident': fake_history['uuid'], + 'node_inventory': fake_inventory, }) @@ -415,6 +427,8 @@ resource_class="CUSTOM_TEST") owned_node_history = db_utils.create_test_history( node_id=owned_node.id) + owned_node_inventory = db_utils.create_test_inventory( + node_id=owned_node.id) # Leased nodes leased_node = db_utils.create_test_node( @@ -445,6 +459,8 @@ leased_node_history = db_utils.create_test_history( node_id=leased_node.id) + leased_node_inventory = db_utils.create_test_inventory( + node_id=leased_node.id) # Random objects that shouldn't be project visible other_node = db_utils.create_test_node( @@ -480,7 +496,9 @@ 'owner_allocation': fake_owner_allocation['uuid'], 'lessee_allocation': fake_leased_allocation['uuid'], 'owned_history_ident': owned_node_history['uuid'], - 'lessee_history_ident': leased_node_history['uuid']}) + 'lessee_history_ident': leased_node_history['uuid'], + 'owned_inventory': owned_node_inventory, + 'leased_inventory': leased_node_inventory}) @ddt.file_data('test_rbac_project_scoped.yaml') @ddt.unpack diff -Nru ironic-21.1.0/ironic/tests/unit/api/test_rbac_project_scoped.yaml ironic-21.4.4/ironic/tests/unit/api/test_rbac_project_scoped.yaml --- ironic-21.1.0/ironic/tests/unit/api/test_rbac_project_scoped.yaml 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/api/test_rbac_project_scoped.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -74,6 +74,14 @@ X-Auth-Token: 'third-party-admin-token' X-Project-Id: ae64129e-b188-4662-b014-4127f4366ee6 X-Roles: admin,manager,member,reader + service_headers: &service_headers + X-Auth-Token: 'service-token' + X-Project-Id: ae64129e-b188-4662-b014-4127f4366ee6 + X-Roles: service + service_headers_owner_project: &service_headers_owner_project + X-Auth-Token: 'service-token' + X-Project-Id: 70e5e25a-2ca2-4cb1-8ae8-7d8739cee205 + X-Roles: service owner_project_id: &owner_project_id 70e5e25a-2ca2-4cb1-8ae8-7d8739cee205 lessee_project_id: &lessee_project_id f11853c7-fa9c-4db3-a477-c9d8e0dbbf13 owned_node_ident: &owned_node_ident f11853c7-fa9c-4db3-a477-c9d8e0dbbf13 @@ -100,6 +108,22 @@ assert_status: 503 self_manage_nodes: True +service_nodes_cannot_post_nodes: + path: '/v1/nodes' + method: post + headers: *service_headers + body: *node_post_body + assert_status: 403 + self_manage_nodes: False + +service_nodes_can_post_nodes: + path: '/v1/nodes' + method: post + headers: *service_headers + body: *node_post_body + assert_status: 503 + self_manage_nodes: True + owner_manager_cannot_post_nodes: path: '/v1/nodes' method: post @@ -716,6 +740,18 @@ assert_status: 503 self_manage_nodes: True +service_cannot_delete_owner_admin_nodes: + path: '/v1/nodes/{owner_node_ident}' + method: delete + headers: *service_headers + assert_status: 404 + +service_can_delete_nodes_in_own_project: + path: '/v1/nodes/{owner_node_ident}' + method: delete + headers: *service_headers_owner_project + assert_status: 403 + owner_manager_cannot_delete_nodes: path: '/v1/nodes/{owner_node_ident}' method: delete @@ -1306,7 +1342,6 @@ body: *provision_body assert_status: 503 - lessee_member_cannot_change_provision_state: path: '/v1/nodes/{lessee_node_ident}/states/provision' method: put @@ -1321,6 +1356,20 @@ body: *provision_body assert_status: 404 +service_can_change_provision_state_for_own_nodes: + path: '/v1/nodes/{owner_node_ident}/states/provision' + method: put + headers: *service_headers_owner_project + body: *provision_body + assert_status: 503 + +service_cannot_change_provision_state: + path: '/v1/nodes/{owner_node_ident}/states/provision' + method: put + headers: *service_headers + body: *provision_body + assert_status: 404 + # Raid configuration owner_admin_can_set_raid_config: @@ -1363,6 +1412,13 @@ body: *raid_body assert_status: 503 +owner_member_can_set_raid_config: + path: '/v1/nodes/{lessee_node_ident}/states/raid' + method: put + headers: *service_headers_owner_project + body: *raid_body + assert_status: 503 + lessee_member_cannot_set_raid_config: path: '/v1/nodes/{lessee_node_ident}/states/raid' method: put @@ -1377,6 +1433,14 @@ body: *raid_body assert_status: 404 +service_cannot_set_raid_config: + path: '/v1/nodes/{lessee_node_ident}/states/raid' + method: put + headers: *service_headers + body: *raid_body + assert_status: 404 + + # Console owner_admin_can_get_console: @@ -1391,6 +1455,12 @@ headers: *owner_manager_headers assert_status: 503 +owner_service_can_get_console: + path: '/v1/nodes/{owner_node_ident}/states/console' + method: get + headers: *service_headers_owner_project + assert_status: 503 + lessee_admin_cannot_get_console: path: '/v1/nodes/{lessee_node_ident}/states/console' method: get @@ -1476,6 +1546,20 @@ body: *console_body_put assert_status: 403 +owner_service_can_set_console: + path: '/v1/nodes/{owner_node_ident}/states/console' + method: put + headers: *service_headers_owner_project + body: *console_body_put + assert_status: 503 + +service_cannot_set_console: + path: '/v1/nodes/{owner_node_ident}/states/console' + method: put + headers: *service_headers + body: *console_body_put + assert_status: 404 + # Vendor Passthru - https://docs.openstack.org/api-ref/baremetal/?expanded=#node-vendor-passthru-nodes # owner/lessee vendor passthru methods inaccessible @@ -1494,6 +1578,12 @@ headers: *owner_manager_headers assert_status: 403 +owner_service_cannot_get_vendor_passthru_methods: + path: '/v1/nodes/{owner_node_ident}/vendor_passthru/methods' + method: get + headers: *service_headers_owner_project + assert_status: 403 + owner_member_cannot_get_vendor_passthru_methods: path: '/v1/nodes/{owner_node_ident}/vendor_passthru/methods' method: get @@ -1543,6 +1633,12 @@ headers: *owner_manager_headers assert_status: 403 +owner_service_cannot_get_vendor_passthru: + path: '/v1/nodes/{owner_node_ident}/vendor_passthru?method=test' + method: get + headers: *service_headers_owner_project + assert_status: 403 + owner_member_cannot_get_vendor_passthru: path: '/v1/nodes/{owner_node_ident}/vendor_passthru?method=test' method: get @@ -1593,6 +1689,12 @@ headers: *owner_manager_headers assert_status: 403 +owner_service_cannot_post_vendor_passthru: + path: '/v1/nodes/{owner_node_ident}/vendor_passthru?method=test' + method: post + headers: *service_headers_owner_project + assert_status: 403 + owner_member_cannot_post_vendor_passthru: path: '/v1/nodes/{owner_node_ident}/vendor_passthru?method=test' method: post @@ -1643,6 +1745,12 @@ headers: *owner_manager_headers assert_status: 403 +owner_service_cannot_put_vendor_passthru: + path: '/v1/nodes/{owner_node_ident}/vendor_passthru?method=test' + method: put + headers: *service_headers_owner_project + assert_status: 403 + owner_member_cannot_put_vendor_passthru: path: '/v1/nodes/{owner_node_ident}/vendor_passthru?method=test' method: put @@ -1693,6 +1801,12 @@ headers: *owner_manager_headers assert_status: 403 +owner_service_cannot_delete_vendor_passthru: + path: '/v1/nodes/{owner_node_ident}/vendor_passthru?method=test' + method: delete + headers: *service_headers_owner_project + assert_status: 403 + owner_member_cannot_delete_vendor_passthru: path: '/v1/nodes/{owner_node_ident}/vendor_passthru?method=test' method: delete @@ -1737,6 +1851,12 @@ headers: *owner_reader_headers assert_status: 200 +owner_reader_get_traits: + path: '/v1/nodes/{owner_node_ident}/traits' + method: get + headers: *service_headers_owner_project + assert_status: 200 + lessee_reader_get_traits: path: '/v1/nodes/{lessee_node_ident}/traits' method: get @@ -1766,6 +1886,13 @@ assert_status: 503 body: *traits_body +owner_service_can_put_traits: + path: '/v1/nodes/{owner_node_ident}/traits' + method: put + headers: *service_headers_owner_project + assert_status: 503 + body: *traits_body + owner_member_cannot_put_traits: path: '/v1/nodes/{owner_node_ident}/traits' method: put @@ -1801,6 +1928,13 @@ assert_status: 404 body: *traits_body +third_party_admin_cannot_put_traits: + path: '/v1/nodes/{lessee_node_ident}/traits' + method: put + headers: *service_headers + assert_status: 404 + body: *traits_body + owner_admin_can_delete_traits: path: '/v1/nodes/{owner_node_ident}/traits/{trait}' method: delete @@ -1917,6 +2051,21 @@ body: &vif_body id: ee21d58f-5de2-4956-85ff-33935ea1ca00 +service_can_post_vifs_for_own_project: + path: '/v1/nodes/{owner_node_ident}/vifs' + method: post + headers: *service_headers_owner_project + assert_status: 503 + body: *vif_body + +service_cannot_post_vifs_for_other_project: + path: '/v1/nodes/{owner_node_ident}/vifs' + method: post + headers: *service_headers + # NOTE(TheJulia): This is a 404 because the node should not be visible. + assert_status: 404 + body: *vif_body + owner_manager_can_post_vifs: path: '/v1/nodes/{owner_node_ident}/vifs' method: post @@ -2015,6 +2164,18 @@ headers: *third_party_admin_headers assert_status: 404 +service_can_delete_vifs: + path: '/v1/nodes/{owner_node_ident}/vifs/{vif_ident}' + method: delete + headers: *service_headers_owner_project + assert_status: 503 + +service_cannot_delete_other_nodes_vifs: + path: '/v1/nodes/{owner_node_ident}/vifs/{vif_ident}' + method: delete + headers: *service_headers + assert_status: 404 + # Indicators - https://docs.openstack.org/api-ref/baremetal/#indicators-management owner_readers_can_get_indicators: path: '/v1/nodes/{owner_node_ident}/management/indicators' @@ -2078,6 +2239,14 @@ assert_list_length: portgroups: 2 +owner_service_can_list_portgroups: + path: '/v1/portgroups' + method: get + headers: *service_headers_owner_project + assert_status: 200 + assert_list_length: + portgroups: 2 + lessee_reader_can_list_portgroups: path: '/v1/portgroups' method: get @@ -2122,6 +2291,13 @@ node_uuid: 1ab63b9e-66d7-4cd7-8618-dddd0f9f7881 assert_status: 201 +owner_service_can_add_portgroup: + path: '/v1/portgroups' + method: post + headers: *service_headers_owner_project + body: *owner_portgroup_body + assert_status: 201 + owner_manager_can_add_portgroup: path: '/v1/portgroups' method: post @@ -2237,6 +2413,12 @@ headers: *owner_member_headers assert_status: 403 +owner_service_can_delete_portgroup: + path: '/v1/portgroups/{owner_portgroup_ident}' + method: delete + headers: *service_headers_owner_project + assert_status: 503 + lessee_admin_cannot_delete_portgroup: path: '/v1/portgroups/{lessee_portgroup_ident}' method: delete @@ -2261,6 +2443,12 @@ headers: *third_party_admin_headers assert_status: 404 +service_cannot_delete_portgroup: + path: '/v1/portgroups/{lessee_portgroup_ident}' + method: delete + headers: *service_headers + assert_status: 404 + # Portgroups by node - https://docs.openstack.org/api-ref/baremetal/#listing-portgroups-by-node-nodes-portgroups owner_reader_can_get_node_portgroups: @@ -2281,6 +2469,13 @@ headers: *third_party_admin_headers assert_status: 404 +service_cannot_get_portgroups: + path: '/v1/nodes/{lessee_node_ident}/portgroups' + method: get + headers: *service_headers + assert_status: 404 + + # Ports - https://docs.openstack.org/api-ref/baremetal/#ports-ports # Based on ports_* tests @@ -2294,6 +2489,15 @@ assert_list_length: ports: 3 +owner_service_can_list_ports: + path: '/v1/ports' + method: get + headers: *service_headers_owner_project + assert_status: 200 + # Two ports owned, one on the leased node. 1 invisible. + assert_list_length: + ports: 3 + lessee_reader_can_list_ports: path: '/v1/ports' method: get @@ -2316,6 +2520,12 @@ headers: *owner_reader_headers assert_status: 200 +owner_service_can_read_port: + path: '/v1/ports/{owner_port_ident}' + method: get + headers: *service_headers_owner_project + assert_status: 200 + lessee_reader_can_read_port: path: '/v1/ports/{lessee_port_ident}' method: get @@ -2362,6 +2572,13 @@ body: *other_node_add_port_body assert_status: 403 +owner_service_cannot_add_ports_to_other_nodes: + path: '/v1/ports' + method: post + headers: *service_headers_owner_project + body: *other_node_add_port_body + assert_status: 403 + owner_member_cannot_add_port: path: '/v1/ports' method: post @@ -2399,6 +2616,20 @@ body: *lessee_port_body assert_status: 403 +service_can_add_port: + path: '/v1/ports' + method: post + headers: *service_headers_owner_project + body: *owner_port_body + assert_status: 503 + +service_cannot_add_ports_to_other_project: + path: '/v1/ports' + method: post + headers: *service_headers + body: *owner_port_body + assert_status: 403 + owner_admin_can_modify_port: path: '/v1/ports/{owner_port_ident}' method: patch @@ -2416,6 +2647,13 @@ body: *port_patch_body assert_status: 503 +owner_service_can_modify_port: + path: '/v1/ports/{owner_port_ident}' + method: patch + headers: *service_headers_owner_project + body: *port_patch_body + assert_status: 503 + owner_member_cannot_modify_port: path: '/v1/ports/{owner_port_ident}' method: patch @@ -2463,6 +2701,12 @@ headers: *owner_manager_headers assert_status: 503 +owner_service_can_delete_port: + path: '/v1/ports/{owner_port_ident}' + method: delete + headers: *service_headers_owner_project + assert_status: 503 + owner_member_cannot_delete_port: path: '/v1/ports/{owner_port_ident}' method: delete @@ -2503,6 +2747,14 @@ assert_list_length: ports: 2 +owner_service_can_get_node_ports: + path: '/v1/nodes/{owner_node_ident}/ports' + method: get + headers: *service_headers_owner_project + assert_status: 200 + assert_list_length: + ports: 2 + lessee_reader_can_get_node_port: path: '/v1/nodes/{lessee_node_ident}/ports' method: get @@ -2517,6 +2769,12 @@ headers: *third_party_admin_headers assert_status: 404 +service_cannot_get_ports: + path: '/v1/nodes/{lessee_node_ident}/ports' + method: get + headers: *service_headers + assert_status: 404 + # Ports by portgroup - https://docs.openstack.org/api-ref/baremetal/#listing-ports-by-portgroup-portgroup-ports # Based on portgroups_ports_get* tests @@ -2527,6 +2785,12 @@ headers: *owner_reader_headers assert_status: 200 +owner_service_cam_get_ports_by_portgroup: + path: '/v1/portgroups/{owner_portgroup_ident}/ports' + method: get + headers: *service_headers_owner_project + assert_status: 200 + lessee_reader_can_get_ports_by_portgroup: path: '/v1/portgroups/{lessee_portgroup_ident}/ports' method: get @@ -2539,6 +2803,13 @@ headers: *third_party_admin_headers assert_status: 404 +service_cannot_get_ports_by_portgroup: + path: '/v1/portgroups/{other_portgroup_ident}/ports' + method: get + headers: *service_headers + assert_status: 404 + + # Volume(s) - https://docs.openstack.org/api-ref/baremetal/#volume-volume # TODO(TheJulia): volumes will likely need some level of exhaustive testing. # i.e. ensure that the volume is permissible. However this may not be possible @@ -2587,6 +2858,13 @@ assert_status: 201 body: *volume_connector_body +owner_service_can_post_volume_connector: + path: '/v1/volume/connectors' + method: post + headers: *service_headers_owner_project + assert_status: 201 + body: *volume_connector_body + lessee_admin_cannot_post_volume_connector: path: '/v1/volume/connectors' method: post @@ -2608,6 +2886,13 @@ assert_status: 403 body: *volume_connector_body +service_admin_cannot_post_volume_connector: + path: '/v1/volume/connectors' + method: post + headers: *service_headers + assert_status: 403 + body: *volume_connector_body + owner_reader_can_get_volume_connector: path: '/v1/volume/connectors/{volume_connector_ident}' method: get @@ -2698,6 +2983,12 @@ headers: *owner_manager_headers assert_status: 503 +owner_service_can_delete_volume_connectors: + path: '/v1/volume/connectors/{volume_connector_ident}' + method: delete + headers: *service_headers_owner_project + assert_status: 503 + lessee_admin_can_delete_volume_connectors: path: '/v1/volume/connectors/{volume_connector_ident}' method: delete @@ -2716,6 +3007,12 @@ headers: *third_party_admin_headers assert_status: 404 +service_cannot_delete_volume_connector: + path: '/v1/volume/connectors/{volume_connector_ident}' + method: delete + headers: *service_headers + assert_status: 404 + # Volume targets # TODO(TheJulia): Create at least 3 targets. @@ -2776,6 +3073,13 @@ boot_index: 2 volume_id: 'test-id' +owner_service_create_volume_target: + path: '/v1/volume/targets' + method: post + headers: *service_headers_owner_project + assert_status: 201 + body: *volume_target_body + owner_manager_create_volume_target: path: '/v1/volume/targets' method: post @@ -2826,6 +3130,13 @@ headers: *owner_member_headers assert_status: 503 +owner_service_can_patch_volume_target: + path: '/v1/volume/targets/{volume_target_ident}' + method: patch + body: *volume_target_patch + headers: *service_headers_owner_project + assert_status: 503 + lessee_admin_can_patch_volume_target: path: '/v1/volume/targets/{volume_target_ident}' method: patch @@ -2854,6 +3165,13 @@ headers: *third_party_admin_headers assert_status: 404 +service_cannot_patch_volume_target: + path: '/v1/volume/targets/{volume_target_ident}' + method: patch + body: *volume_target_patch + headers: *service_headers + assert_status: 404 + owner_admin_can_delete_volume_target: path: '/v1/volume/targets/{volume_target_ident}' method: delete @@ -2866,6 +3184,12 @@ headers: *owner_manager_headers assert_status: 503 +owner_manager_can_delete_volume_target: + path: '/v1/volume/targets/{volume_target_ident}' + method: delete + headers: *service_headers_owner_project + assert_status: 503 + lessee_admin_can_delete_volume_target: path: '/v1/volume/targets/{volume_target_ident}' method: delete @@ -2896,6 +3220,12 @@ headers: *third_party_admin_headers assert_status: 404 +service_cannot_delete_volume_target: + path: '/v1/volume/targets/{volume_target_ident}' + method: delete + headers: *service_headers + assert_status: 404 + # Get Volumes by Node - https://docs.openstack.org/api-ref/baremetal/#listing-volume-resources-by-node-nodes-volume owner_reader_can_get_volume_connectors: @@ -2904,6 +3234,12 @@ headers: *owner_reader_headers assert_status: 200 +owner_service_can_get_volume_connectors: + path: '/v1/nodes/{owner_node_ident}/volume/connectors' + method: get + headers: *service_headers_owner_project + assert_status: 200 + lessee_reader_can_get_node_volume_connectors: path: '/v1/nodes/{lessee_node_ident}/volume/connectors' method: get @@ -2916,12 +3252,24 @@ headers: *third_party_admin_headers assert_status: 404 +service_cannot_get_node_volume_connectors: + path: '/v1/nodes/{lessee_node_ident}/volume/connectors' + method: get + headers: *service_headers + assert_status: 404 + owner_reader_can_get_node_volume_targets: path: '/v1/nodes/{owner_node_ident}/volume/targets' method: get headers: *owner_reader_headers assert_status: 200 +owner_service_can_read_get_node_volume_targets: + path: '/v1/nodes/{owner_node_ident}/volume/targets' + method: get + headers: *service_headers_owner_project + assert_status: 200 + lessee_reader_can_get_node_volume_targets: path: '/v1/nodes/{lessee_node_ident}/volume/targets' method: get @@ -2934,6 +3282,12 @@ headers: *third_party_admin_headers assert_status: 404 +service_cannot_read_node_volume_targets: + path: '/v1/nodes/{lessee_node_ident}/volume/targets' + method: get + headers: *service_headers + assert_status: 404 + # Drivers - https://docs.openstack.org/api-ref/baremetal/#drivers-drivers # This is a system scoped endpoint, everything should fail in this section. @@ -2956,6 +3310,12 @@ headers: *third_party_admin_headers assert_status: 500 +service_cannot_get_drivers: + path: '/v1/drivers' + method: get + headers: *service_headers + assert_status: 500 + # Driver vendor passthru - https://docs.openstack.org/api-ref/baremetal/#driver-vendor-passthru-drivers # This is a system scoped endpoint, everything should fail in this section. @@ -2978,6 +3338,12 @@ headers: *third_party_admin_headers assert_status: 500 +service_cannot_get_drivers_vendor_passthru: + path: '/v1/drivers/{driver_name}/vendor_passthru/methods' + method: get + headers: *service_headers + assert_status: 500 + # Node Bios - https://docs.openstack.org/api-ref/baremetal/#node-bios-nodes owner_reader_can_get_bios_setttings: @@ -2998,6 +3364,18 @@ headers: *third_party_admin_headers assert_status: 404 +service_can_get_bios_setttings_owner_project: + path: '/v1/nodes/{owner_node_ident}/bios' + method: get + headers: *service_headers_owner_project + assert_status: 200 + +service_cannot_get_bios_setttings: + path: '/v1/nodes/{owner_node_ident}/bios' + method: get + headers: *service_headers + assert_status: 404 + # Conductors - https://docs.openstack.org/api-ref/baremetal/#allocations-allocations # This is a system scoped endpoint, everything should fail in this section. @@ -3271,7 +3649,7 @@ third_party_admin_cannot_post_deploy_template: path: '/v1/deploy_templates' method: post - body: + body: &deploy_template name: 'CUSTOM_TEST_TEMPLATE' steps: - interface: 'deploy' @@ -3281,6 +3659,19 @@ headers: *third_party_admin_headers assert_status: 500 +service_cannot_get_deploy_templates: + path: '/v1/deploy_templates' + method: get + headers: *service_headers + assert_status: 500 + +service_cannot_post_deploy_template: + path: '/v1/deploy_templates' + method: post + body: *deploy_template + headers: *service_headers + assert_status: 500 + # Chassis endpoints - https://docs.openstack.org/api-ref/baremetal/#chassis-chassis # This is a system scoped endpoint, everything should fail in this section. @@ -3311,6 +3702,20 @@ description: 'test-chassis' assert_status: 500 +service_cannot_access_chassis: + path: '/v1/chassis' + method: get + headers: *service_headers + assert_status: 500 + +service_cannot_create_chassis: + path: '/v1/chassis' + method: post + headers: *service_headers + body: + description: 'test-chassis' + assert_status: 500 + # Node history entries node_history_get_admin: @@ -3337,6 +3742,20 @@ assert_list_length: history: 1 +node_history_get_service: + path: '/v1/nodes/{owner_node_ident}/history' + method: get + headers: *service_headers_owner_project + assert_status: 200 + assert_list_length: + history: 1 + +node_history_get_service_cannot_be_retrieved: + path: '/v1/nodes/{owner_node_ident}/history' + method: get + headers: *service_headers + assert_status: 404 + node_history_get_entry_admin: path: '/v1/nodes/{owner_node_ident}/history/{owned_history_ident}' method: get @@ -3391,6 +3810,12 @@ headers: *lessee_reader_headers assert_status: 404 +owner_service_node_history_get_entry_reader: + path: '/v1/nodes/{owner_node_ident}/history/{owned_history_ident}' + method: get + headers: *service_headers_owner_project + assert_status: 200 + third_party_admin_cannot_get_node_history: path: '/v1/nodes/{owner_node_ident}' method: get @@ -3402,3 +3827,64 @@ method: get headers: *third_party_admin_headers assert_status: 404 + +node_history_get_entry_service: + path: '/v1/nodes/{owner_node_ident}/history/{owned_history_ident}' + method: get + headers: *service_headers + assert_status: 404 + +# Node inventory support + +node_inventory_get_admin: + path: '/v1/nodes/{owner_node_ident}/inventory' + method: get + headers: *owner_admin_headers + assert_status: 200 + +node_inventory_get_member: + path: '/v1/nodes/{owner_node_ident}/inventory' + method: get + headers: *owner_member_headers + assert_status: 200 + +node_inventory_get_reader: + path: '/v1/nodes/{owner_node_ident}/inventory' + method: get + headers: *owner_reader_headers + assert_status: 200 + +lessee_node_inventory_get_admin: + path: '/v1/nodes/{node_ident}/inventory' + method: get + headers: *lessee_admin_headers + assert_status: 404 + +lessee_node_inventory_get_member: + path: '/v1/nodes/{node_ident}/inventory' + method: get + headers: *lessee_member_headers + assert_status: 404 + +lessee_node_inventory_get_reader: + path: '/v1/nodes/{node_ident}/inventory' + method: get + headers: *lessee_reader_headers + assert_status: 404 + +# Shard support - system scoped req'd to set on a node or view via /v1/shards +shard_get_shards_disallowed: + path: '/v1/shards' + method: get + headers: *owner_reader_headers + assert_status: 403 + +shard_patch_set_node_shard_disallowed: + path: '/v1/nodes/{owner_node_ident}' + method: patch + headers: *owner_admin_headers + body: + - op: replace + path: /shard + value: 'TestShard' + assert_status: 403 diff -Nru ironic-21.1.0/ironic/tests/unit/api/test_rbac_system_scoped.yaml ironic-21.4.4/ironic/tests/unit/api/test_rbac_system_scoped.yaml --- ironic-21.1.0/ironic/tests/unit/api/test_rbac_system_scoped.yaml 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/api/test_rbac_system_scoped.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -23,6 +23,15 @@ X-Project-ID: a1111111111111111111111111111111 X-Roles: admin X-Project-Name: 'other-project' + service_headers: &service_headers + X-Auth-Token: 'baremetal-service-token' + X-Roles: service + OpenStack-System-Scope: all + service_project_headers: &service_project_headers + X-Auth-Toke: 'service-project-token' + X-Roles: 'service' + X-Project-Name: 'service' + X-Project-ID: c11111111111111111111111111111 owner_project_id: &owner_project_id '{owner_project_id}' other_project_id: &other_project_id '{other_project_id}' node_ident: &node_ident '{node_ident}' @@ -52,6 +61,13 @@ body: *node_post_body assert_status: 403 +nodes_post_service: + path: '/v1/nodes' + method: post + headers: *service_headers + body: *node_post_body + assert_status: 503 + nodes_get_node_admin: path: '/v1/nodes/{node_ident}' method: get @@ -92,6 +108,42 @@ nodes: 3 assert_status: 200 +nodes_get_service: + path: '/v1/nodes' + method: get + headers: *service_headers + assert_list_length: + nodes: 3 + assert_status: 200 + +nodes_get_service_project: + path: '/v1/nodes' + method: get + headers: *service_project_headers + assert_list_length: + nodes: 3 + assert_status: 200 + enable_service_project: true + +nodes_get_service_project_disabled: + path: '/v1/nodes' + method: get + headers: *service_project_headers + assert_list_length: + nodes: 0 + assert_status: 200 + enable_service_project: false + +nodes_get_service_project_admin: + path: '/v1/nodes' + method: get + headers: *service_project_headers + assert_list_length: + nodes: 0 + assert_status: 200 + service_project: admin + enable_service_project: true + nodes_get_other_admin: path: '/v1/nodes' method: get @@ -119,6 +171,12 @@ headers: *reader_headers assert_status: 200 +nodes_detail_get_service: + path: '/v1/nodes/detail' + method: get + headers: *service_headers + assert_status: 200 + nodes_node_ident_get_admin: path: '/v1/nodes/{node_ident}' method: get @@ -174,6 +232,21 @@ body: *extra_patch assert_status: 503 +nodes_node_ident_patch_service: + path: '/v1/nodes/{node_ident}' + method: patch + headers: *service_headers + body: *extra_patch + assert_status: 503 + +nodes_node_ident_patch_service_project: + path: '/v1/nodes/{node_ident}' + method: patch + headers: *service_project_headers + body: *extra_patch + assert_status: 503 + enable_service_project: true + nodes_node_ident_patch_reader: path: '/v1/nodes/{node_ident}' method: patch @@ -187,6 +260,12 @@ headers: *admin_headers assert_status: 503 +nodes_node_ident_delete_service: + path: '/v1/nodes/{node_ident}' + method: delete + headers: *service_headers + assert_status: 403 + nodes_node_ident_delete_member: path: '/v1/nodes/{node_ident}' method: delete @@ -216,6 +295,19 @@ headers: *scoped_member_headers assert_status: 503 +nodes_validate_get_service: + path: '/v1/nodes/{node_ident}/validate' + method: get + headers: *service_headers + assert_status: 503 + +nodes_validate_get_service_project: + path: '/v1/nodes/{node_ident}/validate' + method: get + headers: *service_project_headers + assert_status: 503 + enable_service_project: true + nodes_validate_get_reader: path: '/v1/nodes/{node_ident}/validate' method: get @@ -337,7 +429,6 @@ body: {} assert_status: 403 - nodes_states_get_admin: path: '/v1/nodes/{node_ident}/states' method: get @@ -448,6 +539,13 @@ body: *provision_body assert_status: 403 +nodes_states_provision_put_service: + path: '/v1/nodes/{node_ident}/states/provision' + method: put + headers: *service_headers + body: *provision_body + assert_status: 503 + nodes_states_raid_put_admin: path: '/v1/nodes/{node_ident}/states/raid' method: put @@ -486,12 +584,18 @@ headers: *scoped_member_headers assert_status: 503 -nodes_states_console_get_admin: +nodes_states_console_get_reader: path: '/v1/nodes/{node_ident}/states/console' method: get headers: *reader_headers assert_status: 403 +nodes_states_console_get_service: + path: '/v1/nodes/{node_ident}/states/console' + method: get + headers: *service_headers + assert_status: 503 + nodes_states_console_put_admin: path: '/v1/nodes/{node_ident}/states/console' method: put @@ -514,6 +618,13 @@ body: *console_body_put assert_status: 403 +nodes_states_console_put_service: + path: '/v1/nodes/{node_ident}/states/console' + method: put + headers: *service_headers + body: *console_body_put + assert_status: 503 + # Node Traits - https://docs.openstack.org/api-ref/baremetal/?expanded=#node-vendor-passthru-nodes # Calls conductor upon the get as a task is required. @@ -729,6 +840,12 @@ headers: *reader_headers assert_status: 503 +nodes_vifs_get_service: + path: '/v1/nodes/{node_ident}/vifs' + method: get + headers: *service_headers + assert_status: 503 + nodes_vifs_post_admin: path: '/v1/nodes/{node_ident}/vifs' method: post @@ -751,6 +868,21 @@ assert_status: 403 body: *vif_body +nodes_vifs_post_service: + path: '/v1/nodes/{node_ident}/vifs' + method: post + headers: *service_headers + assert_status: 503 + body: *vif_body + +nodes_vifs_post_service_project: + path: '/v1/nodes/{node_ident}/vifs' + method: post + headers: *service_project_headers + assert_status: 503 + body: *vif_body + enable_service_project: true + # This calls the conductor, hence not status 403. nodes_vifs_node_vif_ident_delete_admin: path: '/v1/nodes/{node_ident}/vifs/{vif_ident}' @@ -770,6 +902,12 @@ headers: *reader_headers assert_status: 403 +nodes_vifs_node_vif_ident_delete_service: + path: '/v1/nodes/{node_ident}/vifs/{vif_ident}' + method: delete + headers: *service_headers + assert_status: 503 + # Indicators - https://docs.openstack.org/api-ref/baremetal/#indicators-management nodes_management_indicators_get_allow: @@ -934,6 +1072,26 @@ headers: *reader_headers assert_status: 200 +nodes_portgroups_get_service: + path: '/v1/nodes/{node_ident}/portgroups' + method: get + headers: *service_headers + assert_status: 200 + +nodes_portgroups_get_service_project: + path: '/v1/nodes/{node_ident}/portgroups' + method: get + headers: *service_project_headers + assert_status: 200 + enable_service_project: true + +nodes_portgroups_get_service_project: + path: '/v1/nodes/{node_ident}/portgroups' + method: get + headers: *service_project_headers + assert_status: 404 + enable_service_project: false + nodes_portgroups_detail_get_admin: path: '/v1/nodes/{node_ident}/portgroups/detail' method: get @@ -952,6 +1110,25 @@ headers: *reader_headers assert_status: 200 +nodes_portgroups_detail_get_service: + path: '/v1/nodes/{node_ident}/portgroups/detail' + method: get + headers: *service_headers + assert_status: 200 + +nodes_portgroups_detail_get_service_project: + path: '/v1/nodes/{node_ident}/portgroups/detail' + method: get + headers: *service_project_headers + assert_status: 200 + enable_service_project: true + +nodes_portgroups_detail_get_service_project: + path: '/v1/nodes/{node_ident}/portgroups/detail' + method: get + headers: *service_project_headers + assert_status: 404 + # Ports - https://docs.openstack.org/api-ref/baremetal/#ports-ports ports_get_admin: @@ -959,6 +1136,33 @@ method: get headers: *admin_headers assert_status: 200 + assert_list_length: + ports: 1 + +ports_get_service: + path: '/v1/ports' + method: get + headers: *service_headers + assert_status: 200 + assert_list_length: + ports: 1 + +ports_get_service_project: + path: '/v1/ports' + method: get + headers: *service_project_headers + assert_status: 200 + assert_list_length: + ports: 1 + enable_service_project: true + +ports_get_service_project_disabled: + path: '/v1/ports' + method: get + headers: *service_project_headers + assert_status: 200 + assert_list_length: + ports: 0 ports_get_member: path: '/v1/ports' @@ -1182,6 +1386,20 @@ headers: *reader_headers assert_status: 200 +volume_get_service: + path: '/v1/volume' + method: get + headers: *service_headers + assert_status: 200 + +volume_get_service_project: + path: '/v1/volume' + method: get + headers: *service_project_headers + assert_status: 200 + assert_list_length: + enable_service_project: true + # Volume connectors volume_connectors_get_admin: @@ -1202,6 +1420,30 @@ headers: *reader_headers assert_status: 200 +volume_connectors_get_service: + path: '/v1/volume/connectors' + method: get + headers: *service_headers + assert_status: 200 + +volume_connectors_get_service_project: + path: '/v1/volume/connectors' + method: get + headers: *service_project_headers + assert_status: 200 + assert_list_length: + connectors: 1 + enable_service_project: true + +volume_connectors_get_service_project_disable: + path: '/v1/volume/connectors' + method: get + headers: *service_project_headers + assert_status: 200 + assert_list_length: + connectors: 0 + enable_service_project: false + # NOTE(TheJulia): This ends up returning a 400 due to the # UUID not already being in ironic. volume_connectors_post_admin: @@ -1230,6 +1472,21 @@ assert_status: 403 body: *volume_connector_body +volume_connectors_post_service: + path: '/v1/volume/connectors' + method: post + headers: *service_headers + assert_status: 201 + body: *volume_connector_body + +volume_connectors_post_service_project: + path: '/v1/volume/connectors' + method: post + headers: *service_project_headers + assert_status: 201 + body: *volume_connector_body + enable_service_project: true + volume_volume_connector_id_get_admin: path: '/v1/volume/connectors/{volume_connector_ident}' method: get @@ -1272,6 +1529,13 @@ body: *connector_patch_body assert_status: 403 +volume_volume_connector_id_patch_service: + path: '/v1/volume/connectors/{volume_connector_ident}' + method: patch + headers: *service_headers + body: *connector_patch_body + assert_status: 503 + volume_volume_connector_id_delete_admin: path: '/v1/volume/connectors/{volume_connector_ident}' method: delete @@ -1290,6 +1554,12 @@ headers: *reader_headers assert_status: 403 +volume_volume_connector_id_delete_service: + path: '/v1/volume/connectors/{volume_connector_ident}' + method: delete + headers: *service_headers + assert_status: 503 + # Volume targets volume_targets_get_admin: @@ -1310,6 +1580,12 @@ headers: *reader_headers assert_status: 200 +volume_targets_get_service: + path: '/v1/volume/targets' + method: get + headers: *service_headers + assert_status: 200 + # NOTE(TheJulia): Because we can't seem to get the uuid # to load from an existing uuid, since we're not subsituting # it, this will return with 400 due to the ID not matching. @@ -1335,6 +1611,53 @@ boot_index: 2 volume_id: 'test-id2' +volume_targets_post_service: + path: '/v1/volume/targets' + method: post + headers: *service_headers + assert_status: 201 + body: + node_uuid: 1be26c0b-03f2-4d2e-ae87-c02d7f33c123 + volume_type: iscsi + boot_index: 2 + volume_id: 'test-id2' + +volume_targets_post_service_project: + path: '/v1/volume/targets' + method: post + headers: *service_project_headers + assert_status: 201 + body: + node_uuid: 1be26c0b-03f2-4d2e-ae87-c02d7f33c123 + volume_type: iscsi + boot_index: 2 + volume_id: 'test-id2' + enable_service_project: true + +volume_targets_post_service_project_disabled: + path: '/v1/volume/targets' + method: post + headers: *service_project_headers + assert_status: 403 + body: + node_uuid: 1be26c0b-03f2-4d2e-ae87-c02d7f33c123 + volume_type: iscsi + boot_index: 2 + volume_id: 'test-id2' + +volume_targets_post_service_project_admin: + path: '/v1/volume/targets' + method: post + headers: *service_project_headers + assert_status: 403 + body: + node_uuid: 1be26c0b-03f2-4d2e-ae87-c02d7f33c123 + volume_type: iscsi + boot_index: 2 + volume_id: 'test-id2' + service_project: admin + enable_service_project: true + volume_targets_post_reader: path: '/v1/volume/targets' method: post @@ -1360,6 +1683,12 @@ headers: *reader_headers assert_status: 200 +volume_volume_target_id_get_service: + path: '/v1/volume/targets/{volume_target_ident}' + method: get + headers: *service_headers + assert_status: 200 + # NOTE(TheJulia): This triggers a call to the conductor and # thus will fail, but does not return a 403 which means success. volume_volume_target_id_patch_admin: @@ -1386,6 +1715,21 @@ headers: *reader_headers assert_status: 403 +volume_volume_target_id_patch_service: + path: '/v1/volume/targets/{volume_target_ident}' + method: patch + body: *volume_target_patch + headers: *service_headers + assert_status: 503 + +volume_volume_target_id_patch_service: + path: '/v1/volume/targets/{volume_target_ident}' + method: patch + body: *volume_target_patch + headers: *service_project_headers + assert_status: 503 + enable_service_project: true + volume_volume_target_id_delete_admin: path: '/v1/volume/targets/{volume_target_ident}' method: delete @@ -1404,6 +1748,12 @@ headers: *reader_headers assert_status: 403 +volume_volume_target_id_delete_service: + path: '/v1/volume/targets/{volume_target_ident}' + method: delete + headers: *service_headers + assert_status: 503 + # Get Volumes by Node - https://docs.openstack.org/api-ref/baremetal/#listing-volume-resources-by-node-nodes-volume nodes_volume_get_admin: @@ -2002,6 +2352,12 @@ headers: *reader_headers assert_status: 200 +chassis_get_service: + path: '/v1/chassis' + method: get + headers: *service_headers + assert_status: 200 + chassis_detail_get_admin: path: '/v1/chassis/detail' method: get @@ -2080,6 +2436,12 @@ headers: *reader_headers assert_status: 403 +chassis_chassis_id_delete_service: + path: '/v1/chassis/{chassis_ident}' + method: delete + headers: *service_headers + assert_status: 403 + # Node history entries node_history_get_admin: @@ -2106,6 +2468,14 @@ assert_list_length: history: 1 +node_history_get_service: + path: '/v1/nodes/{node_ident}/history' + method: get + headers: *service_headers + assert_status: 200 + assert_list_length: + history: 1 + node_history_get_entry_admin: path: '/v1/nodes/{node_ident}/history/{history_ident}' method: get @@ -2123,3 +2493,47 @@ method: get headers: *reader_headers assert_status: 200 + +# Node inventory support + +node_inventory_get_admin: + path: '/v1/nodes/{node_ident}/inventory' + method: get + headers: *admin_headers + assert_status: 200 + +node_inventory_get_reader: + path: '/v1/nodes/{node_ident}/inventory' + method: get + headers: *reader_headers + assert_status: 200 + +node_history_get_entry_service: + path: '/v1/nodes/{node_ident}/history/{history_ident}' + method: get + headers: *service_headers + assert_status: 200 + +# Shard support +shard_get_shards: + path: '/v1/shards' + method: get + headers: *reader_headers + assert_status: 200 + +shard_patch_set_node_shard: + path: '/v1/nodes/{node_ident}' + method: patch + headers: *admin_headers + body: &replace_shard + - op: replace + path: /shard + value: 'TestShard' + assert_status: 503 + +shard_patch_set_node_shard_disallowed: + path: '/v1/nodes/{node_ident}' + method: patch + headers: *scoped_member_headers + body: *replace_shard + assert_status: 403 diff -Nru ironic-21.1.0/ironic/tests/unit/cmd/test_status.py ironic-21.4.4/ironic/tests/unit/cmd/test_status.py --- ironic-21.1.0/ironic/tests/unit/cmd/test_status.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/cmd/test_status.py 2024-10-11 15:42:16.000000000 +0000 @@ -14,6 +14,7 @@ from unittest import mock +from oslo_db import sqlalchemy from oslo_upgradecheck.upgradecheck import Code from ironic.cmd import dbsync @@ -38,3 +39,84 @@ check_result = self.cmd._check_obj_versions() self.assertEqual(Code.FAILURE, check_result.code) self.assertEqual(msg, check_result.details) + + def test__check_allocations_table_ok(self): + check_result = self.cmd._check_allocations_table() + self.assertEqual(Code.SUCCESS, + check_result.code) + + @mock.patch.object(sqlalchemy.enginefacade.reader, + 'get_engine', autospec=True) + def test__check_allocations_table_latin1(self, mock_reader): + mock_engine = mock.Mock() + mock_res = mock.Mock() + mock_res.all.return_value = ( + '... ENGINE=InnoDB DEFAULT CHARSET=latin1', + ) + mock_engine.url = '..mysql..' + mock_engine.execute.return_value = mock_res + mock_reader.return_value = mock_engine + check_result = self.cmd._check_allocations_table() + self.assertEqual(Code.WARNING, + check_result.code) + expected_msg = ('The Allocations table is is not using UTF8 ' + 'encoding. This is corrected in later versions ' + 'of Ironic, where the table character set schema ' + 'is automatically migrated. Continued use of a ' + 'non-UTF8 character set may produce unexpected ' + 'results.') + self.assertEqual(expected_msg, check_result.details) + + @mock.patch.object(sqlalchemy.enginefacade.reader, + 'get_engine', autospec=True) + def test__check_allocations_table_myiasm(self, mock_reader): + mock_engine = mock.Mock() + mock_res = mock.Mock() + mock_engine.url = '..mysql..' + mock_res.all.return_value = ( + '... ENGINE=MyIASM DEFAULT CHARSET=utf8', + ) + mock_engine.execute.return_value = mock_res + mock_reader.return_value = mock_engine + check_result = self.cmd._check_allocations_table() + self.assertEqual(Code.WARNING, + check_result.code) + expected_msg = ('The engine used by MySQL for the allocations ' + 'table is not the intended engine for the Ironic ' + 'database tables to use. This may have been a ' + 'result of an error with the table creation schema. ' + 'This may require Database Administrator ' + 'intervention and downtime to dump, modify the ' + 'table engine to utilize InnoDB, and reload the ' + 'allocations table to utilize the InnoDB engine.') + self.assertEqual(expected_msg, check_result.details) + + @mock.patch.object(sqlalchemy.enginefacade.reader, + 'get_engine', autospec=True) + def test__check_allocations_table_myiasm_both(self, mock_reader): + mock_engine = mock.Mock() + mock_res = mock.Mock() + mock_engine.url = '..mysql..' + mock_res.all.return_value = ( + '... ENGINE=MyIASM DEFAULT CHARSET=latin1', + ) + mock_engine.execute.return_value = mock_res + mock_reader.return_value = mock_engine + check_result = self.cmd._check_allocations_table() + self.assertEqual(Code.WARNING, + check_result.code) + expected_msg = ('The Allocations table is is not using UTF8 ' + 'encoding. This is corrected in later versions ' + 'of Ironic, where the table character set schema ' + 'is automatically migrated. Continued use of a ' + 'non-UTF8 character set may produce unexpected ' + 'results. Additionally: ' + 'The engine used by MySQL for the allocations ' + 'table is not the intended engine for the Ironic ' + 'database tables to use. This may have been a ' + 'result of an error with the table creation schema. ' + 'This may require Database Administrator ' + 'intervention and downtime to dump, modify the ' + 'table engine to utilize InnoDB, and reload the ' + 'allocations table to utilize the InnoDB engine.') + self.assertEqual(expected_msg, check_result.details) diff -Nru ironic-21.1.0/ironic/tests/unit/common/test_checksum_utils.py ironic-21.4.4/ironic/tests/unit/common/test_checksum_utils.py --- ironic-21.1.0/ironic/tests/unit/common/test_checksum_utils.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/common/test_checksum_utils.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,203 @@ +# coding=utf-8 + +# Copyright 2024 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import mock + +from oslo_config import cfg + +from ironic.common import checksum_utils +from ironic.common import exception +from ironic.common import image_service +from ironic.tests import base + +CONF = cfg.CONF + + +@mock.patch.object(checksum_utils, 'compute_image_checksum', + autospec=True) +class IronicChecksumUtilsValidateTestCase(base.TestCase): + + def test_validate_checksum(self, mock_compute): + mock_compute.return_value = 'f00' + checksum_utils.validate_checksum('path', 'f00', 'algo') + mock_compute.assert_called_once_with('path', 'algo') + + def test_validate_checksum_mixed_case(self, mock_compute): + mock_compute.return_value = 'f00' + checksum_utils.validate_checksum('path', 'F00', 'ALGO') + mock_compute.assert_called_once_with('path', 'algo') + + def test_validate_checksum_mixed_md5(self, mock_compute): + mock_compute.return_value = 'f00' + checksum_utils.validate_checksum('path', 'F00') + mock_compute.assert_called_once_with('path') + + def test_validate_checksum_mismatch(self, mock_compute): + mock_compute.return_value = 'a00' + self.assertRaises(exception.ImageChecksumError, + checksum_utils.validate_checksum, + 'path', 'f00', 'algo') + mock_compute.assert_called_once_with('path', 'algo') + + def test_validate_checksum_hashlib_not_supports_algo(self, mock_compute): + mock_compute.side_effect = ValueError() + self.assertRaises(exception.ImageChecksumAlgorithmFailure, + checksum_utils.validate_checksum, + 'path', 'f00', 'algo') + mock_compute.assert_called_once_with('path', 'algo') + + def test_validate_checksum_file_not_found(self, mock_compute): + mock_compute.side_effect = OSError() + self.assertRaises(exception.ImageChecksumFileReadFailure, + checksum_utils.validate_checksum, + 'path', 'f00', 'algo') + mock_compute.assert_called_once_with('path', 'algo') + + def test_validate_checksum_mixed_case_delimited(self, mock_compute): + mock_compute.return_value = 'f00' + checksum_utils.validate_checksum('path', 'algo:F00') + mock_compute.assert_called_once_with('path', 'algo') + + +class IronicChecksumUtilsTestCase(base.TestCase): + + def test_is_checksum_url_string(self): + self.assertFalse(checksum_utils.is_checksum_url('f00')) + + def test_is_checksum_url_file(self): + self.assertFalse(checksum_utils.is_checksum_url('file://foo')) + + def test_is_checksum_url(self): + urls = ['http://foo.local/file', + 'https://foo.local/file'] + for url in urls: + self.assertTrue(checksum_utils.is_checksum_url(url)) + + def test_get_checksum_and_algo_image_checksum(self): + value = 'c46f2c98efe1cd246be1796cd842246e' + i_info = {'image_checksum': value} + csum, algo = checksum_utils.get_checksum_and_algo(i_info) + self.assertEqual(value, csum) + self.assertIsNone(algo) + + def test_get_checksum_and_algo_image_checksum_glance(self): + value = 'c46f2c98efe1cd246be1796cd842246e' + i_info = {'image_os_hash_value': value, + 'image_os_hash_algo': 'foobar'} + csum, algo = checksum_utils.get_checksum_and_algo(i_info) + self.assertEqual(value, csum) + self.assertEqual('foobar', algo) + + def test_get_checksum_and_algo_image_checksum_sha256(self): + value = 'a' * 64 + i_info = {'image_checksum': value} + csum, algo = checksum_utils.get_checksum_and_algo(i_info) + self.assertEqual(value, csum) + self.assertEqual('sha256', algo) + + def test_get_checksum_and_algo_image_checksum_sha512(self): + value = 'f' * 128 + i_info = {'image_checksum': value} + csum, algo = checksum_utils.get_checksum_and_algo(i_info) + self.assertEqual(value, csum) + self.assertEqual('sha512', algo) + + @mock.patch.object(checksum_utils, 'get_checksum_from_url', autospec=True) + def test_get_checksum_and_algo_image_checksum_http_url(self, mock_get): + value = 'http://checksum-url' + i_info = { + 'image_checksum': value, + 'image_source': 'image-ref' + } + mock_get.return_value = 'f' * 64 + csum, algo = checksum_utils.get_checksum_and_algo(i_info) + mock_get.assert_called_once_with(value, 'image-ref') + self.assertEqual('f' * 64, csum) + self.assertEqual('sha256', algo) + + @mock.patch.object(checksum_utils, 'get_checksum_from_url', autospec=True) + def test_get_checksum_and_algo_image_checksum_https_url(self, mock_get): + value = 'https://checksum-url' + i_info = { + 'image_checksum': value, + 'image_source': 'image-ref' + } + mock_get.return_value = 'f' * 128 + csum, algo = checksum_utils.get_checksum_and_algo(i_info) + mock_get.assert_called_once_with(value, 'image-ref') + self.assertEqual('f' * 128, csum) + self.assertEqual('sha512', algo) + + +@mock.patch.object(image_service.HttpImageService, 'get', + autospec=True) +class IronicChecksumUtilsGetChecksumTestCase(base.TestCase): + + def test_get_checksum_from_url_empty_response(self, mock_get): + mock_get.return_value = '' + error = ('Failed to download image https://checksum-url, ' + 'reason: Checksum file empty.') + self.assertRaisesRegex(exception.ImageDownloadFailed, + error, + checksum_utils.get_checksum_from_url, + 'https://checksum-url', + 'https://image-url/file') + mock_get.assert_called_once_with('https://checksum-url') + + def test_get_checksum_from_url_one_line(self, mock_get): + mock_get.return_value = 'a' * 32 + csum = checksum_utils.get_checksum_from_url( + 'https://checksum-url', 'https://image-url/file') + mock_get.assert_called_once_with('https://checksum-url') + self.assertEqual('a' * 32, csum) + + def test_get_checksum_from_url_nomatch_line(self, mock_get): + mock_get.return_value = 'foobar' + # For some reason assertRaisesRegex really doesn't like + # the error. Easiest path is just to assertTrue the compare. + exc = self.assertRaises(exception.ImageDownloadFailed, + checksum_utils.get_checksum_from_url, + 'https://checksum-url', + 'https://image-url/file') + self.assertTrue( + 'Invalid checksum file (No valid checksum found' in str(exc)) + mock_get.assert_called_once_with('https://checksum-url') + + def test_get_checksum_from_url_multiline(self, mock_get): + test_csum = ('f2ca1bb6c7e907d06dafe4687e579fce76b37e4e9' + '3b7605022da52e6ccc26fd2') + mock_get.return_value = ('fee f00\n%s file\nbar fee\nf00' % test_csum) + # For some reason assertRaisesRegex really doesn't like + # the error. Easiest path is just to assertTrue the compare. + checksum = checksum_utils.get_checksum_from_url( + 'https://checksum-url', + 'https://image-url/file') + self.assertEqual(test_csum, checksum) + mock_get.assert_called_once_with('https://checksum-url') + + def test_get_checksum_from_url_multiline_no_file(self, mock_get): + test_csum = 'a' * 64 + error = ("Failed to download image https://checksum-url, reason: " + "Checksum file does not contain name file") + mock_get.return_value = ('f00\n%s\nbar\nf00' % test_csum) + # For some reason assertRaisesRegex really doesn't like + # the error. Easiest path is just to assertTrue the compare. + self.assertRaisesRegex(exception.ImageDownloadFailed, + error, + checksum_utils.get_checksum_from_url, + 'https://checksum-url', + 'https://image-url/file') + mock_get.assert_called_once_with('https://checksum-url') diff -Nru ironic-21.1.0/ironic/tests/unit/common/test_cinder.py ironic-21.4.4/ironic/tests/unit/common/test_cinder.py --- ironic-21.1.0/ironic/tests/unit/common/test_cinder.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/common/test_cinder.py 2024-10-11 15:42:16.000000000 +0000 @@ -58,8 +58,7 @@ @mock.patch('ironic.common.keystone.get_adapter', autospec=True) @mock.patch('ironic.common.keystone.get_service_auth', autospec=True, return_value=mock.sentinel.sauth) -@mock.patch('ironic.common.keystone.get_auth', autospec=True, - return_value=mock.sentinel.auth) +@mock.patch('ironic.common.keystone.get_auth', autospec=True) @mock.patch('ironic.common.keystone.get_session', autospec=True, return_value=mock.sentinel.session) @mock.patch.object(cinderclient.Client, '__init__', autospec=True, @@ -74,8 +73,11 @@ cinder._CINDER_SESSION = None self.context = context.RequestContext(global_request_id='global') - def _assert_client_call(self, init_mock, url, auth=mock.sentinel.auth): - cinder.get_client(self.context) + def _assert_client_call(self, init_mock, url, auth=mock.sentinel.auth, + auth_from_config=None): + if not auth_from_config: + self.context.auth_token = 'meow' + cinder.get_client(self.context, auth_from_config=auth_from_config) init_mock.assert_called_once_with( mock.ANY, session=mock.sentinel.session, @@ -86,15 +88,45 @@ def test_get_client(self, mock_client_init, mock_session, mock_auth, mock_sauth, mock_adapter): - + mock_auth.return_value = mock_auth_obj = mock.Mock() + mock_auth_obj.get_project_id.return_value = '1111' + mock_adapter.return_value = mock_adapter_obj = mock.Mock() + mock_adapter_obj.get_endpoint.side_effect = iter([ + 'cinder_url/1111', + 'cinder_url']) + self._assert_client_call(mock_client_init, 'cinder_url', + auth=mock.sentinel.sauth) + mock_session.assert_has_calls([ + mock.call('cinder'), + mock.call('cinder', timeout=1, auth=mock.sentinel.sauth)]) + mock_auth.assert_called_once_with('cinder') + mock_adapter.assert_has_calls([ + mock.call('cinder', session=mock.sentinel.session, + auth=mock_auth_obj), + + mock.call('cinder', session=mock.sentinel.session, + auth=mock.sentinel.sauth)]) + self.assertTrue(mock_sauth.called) + + def test_get_client_service_token(self, mock_client_init, mock_session, + mock_auth, mock_sauth, mock_adapter): + mock_auth.return_value = mock_auth_obj = mock.Mock() + mock_auth_obj.get_project_id.return_value = '1111' mock_adapter.return_value = mock_adapter_obj = mock.Mock() - mock_adapter_obj.get_endpoint.return_value = 'cinder_url' - self._assert_client_call(mock_client_init, 'cinder_url') - mock_session.assert_called_once_with('cinder') + mock_adapter_obj.get_endpoint.side_effect = iter([ + 'cinder_url/1111', + 'cinder_url']) + self._assert_client_call( + mock_client_init, + 'cinder_url', + auth=None, + auth_from_config=True) + mock_session.assert_has_calls([ + mock.call('cinder'), + mock.call('cinder', timeout=1, auth=mock_auth_obj)]) mock_auth.assert_called_once_with('cinder') - mock_adapter.assert_called_once_with('cinder', - session=mock.sentinel.session, - auth=mock.sentinel.auth) + mock_adapter.assert_called_once_with( + 'cinder', session=mock.sentinel.session, auth=mock_auth_obj) self.assertFalse(mock_sauth.called) diff -Nru ironic-21.1.0/ironic/tests/unit/common/test_format_inspector.py ironic-21.4.4/ironic/tests/unit/common/test_format_inspector.py --- ironic-21.1.0/ironic/tests/unit/common/test_format_inspector.py 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/ironic/tests/unit/common/test_format_inspector.py 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,668 @@ +# Copyright 2020 Red Hat, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import io +import os +import re +import struct +import subprocess +import tempfile +from unittest import mock + +from oslo_utils import units + +from ironic.common import image_format_inspector as format_inspector +from ironic.tests import base as test_base + + +TEST_IMAGE_PREFIX = 'ironic-unittest-formatinspector-' + + +def get_size_from_qemu_img(filename): + output = subprocess.check_output('qemu-img info "%s"' % filename, + shell=True) + for line in output.split(b'\n'): + m = re.search(b'^virtual size: .* .([0-9]+) bytes', line.strip()) + if m: + return int(m.group(1)) + + raise Exception('Could not find virtual size with qemu-img') + + +class TestFormatInspectors(test_base.TestCase): + + block_execute = False + + def setUp(self): + super(TestFormatInspectors, self).setUp() + self._created_files = [] + + def tearDown(self): + super(TestFormatInspectors, self).tearDown() + for fn in self._created_files: + try: + os.remove(fn) + except Exception: + pass + + def _create_iso(self, image_size, subformat='9660'): + """Create an ISO file of the given size. + + :param image_size: The size of the image to create in bytes + :param subformat: The subformat to use, if any + """ + + # these tests depend on mkisofs + # being installed and in the path, + # if it is not installed, skip + try: + subprocess.check_output('mkisofs --version', shell=True) + except Exception: + self.skipTest('mkisofs not installed') + + size = image_size // units.Mi + base_cmd = "mkisofs" + if subformat == 'udf': + # depending on the distribution mkisofs may not support udf + # and may be provided by genisoimage instead. As a result we + # need to check if the command supports udf via help + # instead of checking the installed version. + # mkisofs --help outputs to stderr so we need to + # redirect it to stdout to use grep. + try: + subprocess.check_output( + 'mkisofs --help 2>&1 | grep udf', shell=True) + except Exception: + self.skipTest('mkisofs does not support udf format') + base_cmd += " -udf" + prefix = TEST_IMAGE_PREFIX + prefix += '-%s-' % subformat + fn = tempfile.mktemp(prefix=prefix, suffix='.iso') + self._created_files.append(fn) + subprocess.check_output( + 'dd if=/dev/zero of=%s bs=1M count=%i' % (fn, size), + shell=True) + # We need to use different file as input and output as the behavior + # of mkisofs is version dependent if both the input and the output + # are the same and can cause test failures + out_fn = "%s.iso" % fn + subprocess.check_output( + '%s -V "TEST" -o %s %s' % (base_cmd, out_fn, fn), + shell=True) + self._created_files.append(out_fn) + return out_fn + + def _create_img( + self, fmt, size, subformat=None, options=None, + backing_file=None): + """Create an image file of the given format and size. + + :param fmt: The format to create + :param size: The size of the image to create in bytes + :param subformat: The subformat to use, if any + :param options: A dictionary of options to pass to the format + :param backing_file: The backing file to use, if any + """ + + if fmt == 'iso': + return self._create_iso(size, subformat) + + if fmt == 'vhd': + # QEMU calls the vhd format vpc + fmt = 'vpc' + + # these tests depend on qemu-img being installed and in the path, + # if it is not installed, skip. we also need to ensure that the + # format is supported by qemu-img, this can vary depending on the + # distribution so we need to check if the format is supported via + # the help output. + try: + subprocess.check_output( + 'qemu-img --help | grep %s' % fmt, shell=True) + except Exception: + self.skipTest( + 'qemu-img not installed or does not support %s format' % fmt) + + if options is None: + options = {} + opt = '' + prefix = TEST_IMAGE_PREFIX + + if subformat: + options['subformat'] = subformat + prefix += subformat + '-' + + if options: + opt += '-o ' + ','.join('%s=%s' % (k, v) + for k, v in options.items()) + + if backing_file is not None: + opt += ' -b %s -F raw' % backing_file + + fn = tempfile.mktemp(prefix=prefix, + suffix='.%s' % fmt) + self._created_files.append(fn) + subprocess.check_output( + 'qemu-img create -f %s %s %s %i' % (fmt, opt, fn, size), + shell=True) + return fn + + def _create_allocated_vmdk(self, size_mb, subformat=None): + # We need a "big" VMDK file to exercise some parts of the code of the + # format_inspector. A way to create one is to first create an empty + # file, and then to convert it with the -S 0 option. + + if subformat is None: + # Matches qemu-img default, see `qemu-img convert -O vmdk -o help` + subformat = 'monolithicSparse' + + prefix = TEST_IMAGE_PREFIX + prefix += '-%s-' % subformat + fn = tempfile.mktemp(prefix=prefix, suffix='.vmdk') + self._created_files.append(fn) + raw = tempfile.mktemp(prefix=prefix, suffix='.raw') + self._created_files.append(raw) + + # Create a file with pseudo-random data, otherwise it will get + # compressed in the streamOptimized format + subprocess.check_output( + 'dd if=/dev/urandom of=%s bs=1M count=%i' % (raw, size_mb), + shell=True) + + # Convert it to VMDK + subprocess.check_output( + 'qemu-img convert -f raw -O vmdk -o subformat=%s -S 0 %s %s' % ( + subformat, raw, fn), + shell=True) + return fn + + def _test_format_at_block_size(self, format_name, img, block_size): + fmt = format_inspector.get_inspector(format_name)() + self.assertIsNotNone(fmt, + 'Did not get format inspector for %s' % ( + format_name)) + wrapper = format_inspector.InfoWrapper(open(img, 'rb'), fmt) + + while True: + chunk = wrapper.read(block_size) + if not chunk: + break + + wrapper.close() + return fmt + + def _test_format_at_image_size(self, format_name, image_size, + subformat=None): + """Test the format inspector for the given format at the given image size. + + :param format_name: The format to test + :param image_size: The size of the image to create in bytes + :param subformat: The subformat to use, if any + """ # noqa + img = self._create_img(format_name, image_size, subformat=subformat) + + # Some formats have internal alignment restrictions making this not + # always exactly like image_size, so get the real value for comparison + virtual_size = get_size_from_qemu_img(img) + + # Read the format in various sizes, some of which will read whole + # sections in a single read, others will be completely unaligned, etc. + block_sizes = [64 * units.Ki, 1 * units.Mi] + # ISO images have a 32KB system area at the beginning of the image + # as a result reading that in 17 or 512 byte blocks takes too long, + # causing the test to fail. The 64KiB block size is enough to read + # the system area and header in a single read. the 1MiB block size + # adds very little time to the test so we include it. + if format_name != 'iso': + block_sizes.extend([17, 512]) + for block_size in block_sizes: + fmt = self._test_format_at_block_size(format_name, img, block_size) + self.assertTrue(fmt.format_match, + 'Failed to match %s at size %i block %i' % ( + format_name, image_size, block_size)) + self.assertEqual(virtual_size, fmt.virtual_size, + ('Failed to calculate size for %s at size %i ' + 'block %i') % (format_name, image_size, + block_size)) + memory = sum(fmt.context_info.values()) + self.assertLess(memory, 512 * units.Ki, + 'Format used more than 512KiB of memory: %s' % ( + fmt.context_info)) + + def _test_format(self, format_name, subformat=None): + # Try a few different image sizes, including some odd and very small + # sizes + for image_size in (512, 513, 2057, 7): + self._test_format_at_image_size(format_name, image_size * units.Mi, + subformat=subformat) + + def test_qcow2(self): + self._test_format('qcow2') + + def test_iso_9660(self): + self._test_format('iso', subformat='9660') + + def test_iso_udf(self): + self._test_format('iso', subformat='udf') + + def _generate_bad_iso(self): + # we want to emulate a malicious user who uploads a an + # ISO file has a qcow2 header in the system area + # of the ISO file + # we will create a qcow2 image and an ISO file + # and then copy the qcow2 header to the ISO file + # e.g. + # mkisofs -o orig.iso /etc/resolv.conf + # qemu-img create orig.qcow2 -f qcow2 64M + # dd if=orig.qcow2 of=outcome bs=32K count=1 + # dd if=orig.iso of=outcome bs=32K skip=1 seek=1 + + qcow = self._create_img('qcow2', 10 * units.Mi) + iso = self._create_iso(64 * units.Mi, subformat='9660') + # first ensure the files are valid + iso_fmt = self._test_format_at_block_size('iso', iso, 4 * units.Ki) + self.assertTrue(iso_fmt.format_match) + qcow_fmt = self._test_format_at_block_size('qcow2', qcow, 4 * units.Ki) + self.assertTrue(qcow_fmt.format_match) + # now copy the qcow2 header to an ISO file + prefix = TEST_IMAGE_PREFIX + prefix += '-bad-' + fn = tempfile.mktemp(prefix=prefix, suffix='.iso') + self._created_files.append(fn) + subprocess.check_output( + 'dd if=%s of=%s bs=32K count=1' % (qcow, fn), + shell=True) + subprocess.check_output( + 'dd if=%s of=%s bs=32K skip=1 seek=1' % (iso, fn), + shell=True) + return qcow, iso, fn + + def test_bad_iso_qcow2(self): + + _, _, fn = self._generate_bad_iso() + + iso_check = self._test_format_at_block_size('iso', fn, 4 * units.Ki) + qcow_check = self._test_format_at_block_size('qcow2', fn, 4 * units.Ki) + # this system area of the ISO file is not considered part of the format + # the qcow2 header is in the system area of the ISO file + # so the ISO file is still valid + self.assertTrue(iso_check.format_match) + # the qcow2 header is in the system area of the ISO file + # but that will be parsed by the qcow2 format inspector + # and it will match + self.assertTrue(qcow_check.format_match) + # if we call format_inspector.detect_file_format it should detect + # and raise an exception because both match internally. + e = self.assertRaises( + format_inspector.ImageFormatError, + format_inspector.detect_file_format, fn) + self.assertIn('Multiple formats detected', str(e)) + + def test_vhd(self): + self._test_format('vhd') + + # NOTE(TheJulia): This is not a supported format, and we know this + # test can timeout due to some of the inner workings. Overall the + # code voered by this is being moved to oslo in the future, so this + # test being in ironic is also not the needful. + # def test_vhdx(self): + # self._test_format('vhdx') + + def test_vmdk(self): + self._test_format('vmdk') + + def test_vmdk_stream_optimized(self): + self._test_format('vmdk', 'streamOptimized') + + def test_from_file_reads_minimum(self): + img = self._create_img('qcow2', 10 * units.Mi) + file_size = os.stat(img).st_size + fmt = format_inspector.QcowInspector.from_file(img) + # We know everything we need from the first 512 bytes of a QCOW image, + # so make sure that we did not read the whole thing when we inspect + # a local file. + self.assertLess(fmt.actual_size, file_size) + + def test_qed_always_unsafe(self): + img = self._create_img('qed', 10 * units.Mi) + fmt = format_inspector.get_inspector('qed').from_file(img) + self.assertTrue(fmt.format_match) + self.assertFalse(fmt.safety_check()) + + def _test_vmdk_bad_descriptor_offset(self, subformat=None): + format_name = 'vmdk' + image_size = 10 * units.Mi + descriptorOffsetAddr = 0x1c + BAD_ADDRESS = 0x400 + img = self._create_img(format_name, image_size, subformat=subformat) + + # Corrupt the header + fd = open(img, 'r+b') + fd.seek(descriptorOffsetAddr) + fd.write(struct.pack('`_. + - | + Ironic *always* inspects the supplied user image content for safety prior + to deployment of a node should the image pass through the conductor, + even if the image is supplied in ``raw`` format. This is utilized to + identify the format of the image and the overall safety + of the image, such that source images with unknown or unsafe feature + usage are explicitly rejected. This can be disabled by setting + ``[conductor]disable_deep_image_inspection`` to ``True``. + This is the result of CVE-2024-44082 tracked as + `bug 2071740 `_. + - | + Ironic can also inspect images which would normally be provided as a URL + for direct download by the ``ironic-python-agent`` ramdisk. This is not + enabled by default as it will increase the overall network traffic and + disk space utilization of the conductor. This level of inspection can be + enabled by setting ``[conductor]conductor_always_validates_images`` to + ``True``. Once the ``ironic-python-agent`` ramdisk has been updated, + it will perform similar image security checks independently, should an + image conversion be required. + This is the result of CVE-2024-44082 tracked as + `bug 2071740 `_. + - | + Ironic now explicitly enforces a list of permitted image types for + deployment via the ``[conductor]permitted_image_formats`` setting, + which defaults to "raw", "qcow2", and "iso". + While the project has classically always declared permissible + images as "qcow2" and "raw", it was previously possible to supply other + image formats known to ``qemu-img``, and the utility would attempt to + convert the images. The "iso" support is required for "boot from ISO" + ramdisk support. + - | + Ironic now explicitly passes the source input format to executions of + ``qemu-img`` to limit the permitted qemu disk image drivers which may + evaluate an image to prevent any mismatched format attacks against + ``qemu-img``. + - | + The ``ansible`` deploy interface example playbooks now supply an input + format to execution of ``qemu-img``. If you are using customized + playbooks, please add "-f {{ ironic.image.disk_format }}" to your + invocations of ``qemu-img``. If you do not do so, ``qemu-img`` will + automatically try and guess which can lead to known security issues + with the incorrect source format driver. + - | + Operators who have implemented any custom deployment drivers or additional + functionality like machine snapshot, should review their downstream code + to ensure they are properly invoking ``qemu-img``. If there are any + questions or concerns, please reach out to the Ironic project developers. + - | + Operators are reminded that they should utilize cleaning in their + environments. Disabling any security features such as cleaning or image + inspection are at **your** **own** **risk**. Should you have any issues + with security related features, please don't hesitate to open a bug with + the project. + - | + The ``[conductor]disable_deep_image_inspection`` setting is + conveyed to the ``ironic-python-agent`` ramdisks automatically, and + will prevent those operating ramdisks from performing deep inspection + of images before they are written. + - The ``[conductor]permitted_image_formats`` setting is conveyed to the + ``ironic-python-agent`` ramdisks automatically. Should a need arise + to explicitly permit an additional format, that should take place in + the Ironic service configuration. +fixes: + - | + Fixes multiple issues in the handling of images as it relates to the + execution of the ``qemu-img`` utility, which is used for image format + conversion, where a malicious user could craft a disk image to potentially + extract information from an ``ironic-conductor`` process's operating + environment. + + Ironic now explicitly enforces a list of approved image + formats as a ``[conductor]permitted_image_formats`` list, which mirrors + the image formats the Ironic project has historically tested and expressed + as known working. Testing is not based upon file extension, but upon + content fingerprinting of the disk image files. + This is tracked as CVE-2024-44082 via + `bug 2071740 `_. +upgrade: + - | + When upgrading Ironic to address the ``qemu-img`` image conversion + security issues, the ``ironic-python-agent`` ramdisks will also need + to be upgraded. + - | + When upgrading Ironic to address the ``qemu-img`` image conversion + security issues, the ``[conductor]conductor_always_validates_images`` + setting may be set to ``True`` as a short term remedy while + ``ironic-python-agent`` ramdisks are being updated. Alternatively it + may be advisable to also set the ``[agent]image_download_source`` + setting to ``local`` to minimize redundant network data transfers. + - | + As a result of security fixes to address ``qemu-img`` image conversion + security issues, a new configuration parameter has been added to + Ironic, ``[conductor]permitted_image_formats`` with a default value of + "raw,qcow2,iso". Raw and qcow2 format disk images are the image formats + the Ironic community has consistently stated as what is supported + and expected for use with Ironic. These formats also match the formats + which the Ironic community tests. Operators who leverage other disk image + formats, may need to modify this setting further. diff -Nru ironic-21.1.0/releasenotes/notes/allocations-charset-5384d1ea00964bdd.yaml ironic-21.4.4/releasenotes/notes/allocations-charset-5384d1ea00964bdd.yaml --- ironic-21.1.0/releasenotes/notes/allocations-charset-5384d1ea00964bdd.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/allocations-charset-5384d1ea00964bdd.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,23 @@ +--- +fixes: + - | + Fixes an missing MySQL/MariaDB character set configuration and default + table type encoding for the ``allocations`` database table. Previously, + If Ironic's database was attempted to be populated on a machine which + was using 4 byte character encoding, such as MySQL/MariaDB on Debian + based systems, then the database schema creation would fail. +upgrade: + - This upgrade updates the default character set to utilized in the + database tables when using MySQL/MariaDB. Previously, the default + for Ironic was ``UTF8``, however we now explicitly set ``UTF8MB3`` + which is short for "3 byte UTF8" encoding. The exception to this + is the ``allocations`` table, which would just rely upon the database + default. This was done as Ironic's database schema is incompatible + with MySQL/MariaDB's ``UTF8MB4``, or "4 byte UTF8" character encoding + and storage constraints. + - Upgrading will change the default chracter encoding of all tables. + For most tables, this should be an effective noop, but may result in + transitory table locks. For the ``allocations`` table, it will need to + be re-written, during which the database engine will have locked the + table from being used. Operators are advised to perform test upgrades + and set expectation and upgrade plans accordingly. diff -Nru ironic-21.1.0/releasenotes/notes/catch-all-cleaning-exceptions-1317a534a1c9db56.yaml ironic-21.4.4/releasenotes/notes/catch-all-cleaning-exceptions-1317a534a1c9db56.yaml --- ironic-21.1.0/releasenotes/notes/catch-all-cleaning-exceptions-1317a534a1c9db56.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/catch-all-cleaning-exceptions-1317a534a1c9db56.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixes an issue where unexpected exceptions coming from the process to + start cleaning would not trigger the cleaning_error_handler which + performs the needful internal resets to permit cleaning to be retried + again in the future. Now any error which is encountered during the + launch of cleaning will trigger the error handler. diff -Nru ironic-21.1.0/releasenotes/notes/change-c9c01700dcfd599b.yaml ironic-21.4.4/releasenotes/notes/change-c9c01700dcfd599b.yaml --- ironic-21.1.0/releasenotes/notes/change-c9c01700dcfd599b.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/change-c9c01700dcfd599b.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,9 @@ +--- +upgrade: + - | + Two statsd metrics names have been modified to provide structural clarity + and consistency for consumers of statistics metrics. Consumers of metrics + statistics may need to update their dashboards as the + ``post_clean_step_hook`` metric is now named + ``AgentBase.post_clean_step_hook``, and the ``post_deploy_step_hook`` is + now named ``AgentBase.post_deploy_step_hook``. diff -Nru ironic-21.1.0/releasenotes/notes/checksum-before-conversion-66d273b94fa2ba4d.yaml ironic-21.4.4/releasenotes/notes/checksum-before-conversion-66d273b94fa2ba4d.yaml --- ironic-21.1.0/releasenotes/notes/checksum-before-conversion-66d273b94fa2ba4d.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/checksum-before-conversion-66d273b94fa2ba4d.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,44 @@ +--- +security: + - | + An issue in Ironic has been resolved where image checksums would not be + checked prior to the conversion of an image to a ``raw`` format image from + another image format. + + With default settings, this normally would not take place, however the + ``image_download_source`` option, which is available to be set at a + ``node`` level for a single deployment, by default for that baremetal node + in all cases, or via the ``[agent]image_download_source`` configuration + option when set to ``local``. By default, this setting is ``http``. + + This was in concert with the ``[DEFAULT]force_raw_images`` when set to + ``True``, which caused Ironic to download and convert the file. + + In a fully integrated context of Ironic's use in a larger OpenStack + deployment, where images are coming from the Glance image service, the + previous pattern was not problematic. The overall issue was introduced as + a result of the capability to supply, cache, and convert a disk image + provided as a URL by an authenticated user. + + Ironic will now validate the user supplied checksum prior to image + conversion on the conductor. This can be disabled using the + ``[conductor]disable_file_checksum`` configuration option. +fixes: + - | + Fixes a security issue where Ironic would fail to checksum disk image + files it downloads when Ironic had been requested to download and convert + the image to a raw image format. This required the + ``image_download_source`` to be explicitly set to ``local``, which is not + the default. + + This fix can be disabled by setting + ``[conductor]disable_file_checksum`` to ``True``, however this + option will be removed in new major Ironic releases. + + As a result of this, parity has been introduced to align Ironic to + Ironic-Python-Agent's support for checksums used by ``standalone`` + users of Ironic. This includes support for remote checksum files to be + supplied by URL, in order to prevent breaking existing users which may + have inadvertently been leveraging the prior code path. This support can + be disabled by setting + ``[conductor]disable_support_for_checksum_files`` to ``True``. diff -Nru ironic-21.1.0/releasenotes/notes/cinder-2019892-6b5a9de5c5f05aa6.yaml ironic-21.4.4/releasenotes/notes/cinder-2019892-6b5a9de5c5f05aa6.yaml --- ironic-21.1.0/releasenotes/notes/cinder-2019892-6b5a9de5c5f05aa6.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/cinder-2019892-6b5a9de5c5f05aa6.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,16 @@ +--- +fixes: + - | + Fixes Ironic integration with Cinder because of changes which resulted as + part of the recent Security related fix in + `bug 2004555 `_. The work in Ironic + to track this fix was logged in + `bug 2019892 `_. + Ironic now sends a service token to Cinder, which allows for access + restrictions added as part of the original CVE-2023-2088 + fix to be appropriately bypassed. Ironic was not vulnerable, + but the restrictions added as a result did impact Ironic's usage. + This is because Ironic volume attachments are not on a shared + "compute node", but instead mapped to the physical machines + and Ironic handles the attachment life-cycle after initial + attachment. diff -Nru ironic-21.1.0/releasenotes/notes/cleaning-error-5c13c33c58404b97.yaml ironic-21.4.4/releasenotes/notes/cleaning-error-5c13c33c58404b97.yaml --- ironic-21.1.0/releasenotes/notes/cleaning-error-5c13c33c58404b97.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/cleaning-error-5c13c33c58404b97.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + When aborting cleaning, the ``last_error`` field is no longer initially + empty. It is now populated on the state transition to ``clean failed``. + - | + When cleaning or deployment fails, the ``last_error`` field is no longer + temporary set to ``None`` while the power off action is running. diff -Nru ironic-21.1.0/releasenotes/notes/conductor-metric-collector-support-1b8b8c71f9f59da4.yaml ironic-21.4.4/releasenotes/notes/conductor-metric-collector-support-1b8b8c71f9f59da4.yaml --- ironic-21.1.0/releasenotes/notes/conductor-metric-collector-support-1b8b8c71f9f59da4.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/conductor-metric-collector-support-1b8b8c71f9f59da4.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,39 @@ +--- +features: + - | + Adds the ability for Ironic to send conductor process metrics + for monitoring. This requires the use of a new ``[metrics]backend`` + option value of ``collector``. This data was previously only available + through the use of statsd. This requires ``ironic-lib`` version ``5.4.0`` + or newer. This capability can be disabled using the + ``[sensor_data]enable_for_conductor`` option if set to False. + - | + Adds a ``[sensor_data]enable_for_nodes`` configuration option + to allow operators to disable sending node metric data via the + message bus notifier. + - | + Adds a new gauge metric ``ConductorManager.PowerSyncNodesCount`` + which tracks the nodes considered for power state synchrnozation. + - Adds a new gauge metric ``ConductorManager.PowerSyncRecoveryNodeCount`` + which represents the number of nodes which are being evaluated for power + state recovery checking. + - Adds a new gauge metric ``ConductorManager.SyncLocalStateNodeCount`` + which represents the number of nodes being tracked locally by the + conductor. +issues: + - Sensor data notifications to the message bus, such as using the + ``[metrics]backend`` configuration option of ``collector`` on a dedicated + API service process or instance, is not presently supported. This + functionality requires a periodic task to trigger the transmission + of metrics messages to the message bus notifier. +deprecations: + - The setting values starting with ``send_sensor`` in the ``[conductor]`` + configuration group have been deprecated and moved to a ``[sensor_data]`` + configuration group. The names have been updated to shorter, operator + friendly names.. +upgrades: + - Settings starting with ``sensor_data`` in the ``[conductor]`` + configuration group have been moved to a ``[sensor_data]`` configuration + group amd have been renamed to have shorter value names. If configuration + values are not updated, the ``oslo.config`` library will emit a warning + in the logs. diff -Nru ironic-21.1.0/releasenotes/notes/console-pid-file-6108d2775ef947fe.yaml ironic-21.4.4/releasenotes/notes/console-pid-file-6108d2775ef947fe.yaml --- ironic-21.1.0/releasenotes/notes/console-pid-file-6108d2775ef947fe.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/console-pid-file-6108d2775ef947fe.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue that when a node has console enabled but pid + file missing, the console could not be disabled as well as be + restarted, which makes the console feature unusable. diff -Nru ironic-21.1.0/releasenotes/notes/cross-link-1ffd1a4958f14fd7.yaml ironic-21.4.4/releasenotes/notes/cross-link-1ffd1a4958f14fd7.yaml --- ironic-21.1.0/releasenotes/notes/cross-link-1ffd1a4958f14fd7.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/cross-link-1ffd1a4958f14fd7.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes ``Invalid cross-device link`` in some cases when using ``file://`` + image URLs. diff -Nru ironic-21.1.0/releasenotes/notes/fakedelay-7eac23ad8881a736.yaml ironic-21.4.4/releasenotes/notes/fakedelay-7eac23ad8881a736.yaml --- ironic-21.1.0/releasenotes/notes/fakedelay-7eac23ad8881a736.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fakedelay-7eac23ad8881a736.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,8 @@ +--- +features: + - | + There are now configurable random wait times for fake drivers in a new + ironic.conf [fake] section. Each supported driver having one configuration + option controlling the delay. These delays are applied to operations which + typically block in other drivers. This allows more realistic scenarios to + be arranged for performance and functional testing of ironic itself. diff -Nru ironic-21.1.0/releasenotes/notes/file-symlink-b65bd6b407bd1683.yaml ironic-21.4.4/releasenotes/notes/file-symlink-b65bd6b407bd1683.yaml --- ironic-21.1.0/releasenotes/notes/file-symlink-b65bd6b407bd1683.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/file-symlink-b65bd6b407bd1683.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes the behavior of ``file:///`` image URLs pointing at a symlink. + Ironic no longer creates a hard link to the symlink, which could cause + confusing FileNotFoundError to happen if the symlink is relative. diff -Nru ironic-21.1.0/releasenotes/notes/fix-allocation-exception-on-list-c04e93fb9cace218.yaml ironic-21.4.4/releasenotes/notes/fix-allocation-exception-on-list-c04e93fb9cace218.yaml --- ironic-21.1.0/releasenotes/notes/fix-allocation-exception-on-list-c04e93fb9cace218.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-allocation-exception-on-list-c04e93fb9cace218.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixes an issue when listing allocations as a project scoped user when + the legacy RBAC policies have been disabled which forced an HTTP 406 + error being erroneously raised. Users attempting to list allocations + with a specific owner, different from their own, will now receive + an HTTP 403 error. diff -Nru ironic-21.1.0/releasenotes/notes/fix-console-port-conflict-6dc19688079e2c7f.yaml ironic-21.4.4/releasenotes/notes/fix-console-port-conflict-6dc19688079e2c7f.yaml --- ironic-21.1.0/releasenotes/notes/fix-console-port-conflict-6dc19688079e2c7f.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-console-port-conflict-6dc19688079e2c7f.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixes issues that auto-allocated console port could conflict on the same + host under certain circumstances related to conductor takeover. + + For more information, see `story 2010489 + `_. diff -Nru ironic-21.1.0/releasenotes/notes/fix-context-image-hardlink-16f452974abc7327.yaml ironic-21.4.4/releasenotes/notes/fix-context-image-hardlink-16f452974abc7327.yaml --- ironic-21.1.0/releasenotes/notes/fix-context-image-hardlink-16f452974abc7327.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-context-image-hardlink-16f452974abc7327.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes an issue where if selinux is enabled and enforcing, and + the published image is a hardlink, the source selinux context + is preserved, causing access denied when retrieving the image + using hardlink URL. diff -Nru ironic-21.1.0/releasenotes/notes/fix-eject-media-dvd-b1994446ea71be9c.yaml ironic-21.4.4/releasenotes/notes/fix-eject-media-dvd-b1994446ea71be9c.yaml --- ironic-21.1.0/releasenotes/notes/fix-eject-media-dvd-b1994446ea71be9c.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-eject-media-dvd-b1994446ea71be9c.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + Properly eject the virtual media from a DVD device in case this is the + only MediaType available from the Hardware, and Ironic requested CD as + the device to be used. + See `bug 2039042 `_ + for details. diff -Nru ironic-21.1.0/releasenotes/notes/fix-grub2-uefi-config-path-f1b4c5083cc97ee5.yaml ironic-21.4.4/releasenotes/notes/fix-grub2-uefi-config-path-f1b4c5083cc97ee5.yaml --- ironic-21.1.0/releasenotes/notes/fix-grub2-uefi-config-path-f1b4c5083cc97ee5.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-grub2-uefi-config-path-f1b4c5083cc97ee5.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,14 @@ +--- +fixes: + - | + Fixes the default value for the ``[DEFAULT]grub_config_path`` variable to + be the default path for UEFI bootloader configurations, where as the + default was previously the BIOS grub2 configuration path. +upgrades: + - | + The default configuration value for ``[DEFAULT]grub_config_path`` has + been changed from ``/boot/grub/grub.conf`` to ``EFI/BOOT/grub.efi`` as + the configuration parameter was for UEFI boot configuration, and the + ``/boot/grub/grub2.conf`` path is for BIOS booting. This was verified + by referencing several working UEFI virtual media examples where this + value was overridden to the new configuration value. diff -Nru ironic-21.1.0/releasenotes/notes/fix-inspectwait-finished-at-4b817af4bf4c30c2.yaml ironic-21.4.4/releasenotes/notes/fix-inspectwait-finished-at-4b817af4bf4c30c2.yaml --- ironic-21.1.0/releasenotes/notes/fix-inspectwait-finished-at-4b817af4bf4c30c2.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-inspectwait-finished-at-4b817af4bf4c30c2.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes a database API internal check to update the + ``inspection_finished_at`` field upon the completion of inspection. diff -Nru ironic-21.1.0/releasenotes/notes/fix-irmc-enforcing-snmpv3-with-fips-e45971d363925ec3.yaml ironic-21.4.4/releasenotes/notes/fix-irmc-enforcing-snmpv3-with-fips-e45971d363925ec3.yaml --- ironic-21.1.0/releasenotes/notes/fix-irmc-enforcing-snmpv3-with-fips-e45971d363925ec3.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-irmc-enforcing-snmpv3-with-fips-e45971d363925ec3.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes bug of iRMC driver in parse_driver_info where, if FIPS is enabled, + SNMP version is always required to be version 3 even though iRMC driver's + xxx_interface doesn't use SNMP actually. diff -Nru ironic-21.1.0/releasenotes/notes/fix-irmc-s6-2.00-http-incompatibility-61a31d12aa33fbd8.yaml ironic-21.4.4/releasenotes/notes/fix-irmc-s6-2.00-http-incompatibility-61a31d12aa33fbd8.yaml --- ironic-21.1.0/releasenotes/notes/fix-irmc-s6-2.00-http-incompatibility-61a31d12aa33fbd8.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-irmc-s6-2.00-http-incompatibility-61a31d12aa33fbd8.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,19 @@ +--- +upgrade: + - | + Since iRMC versions S6 2.00 and later, iRMC firmware doesn't + support HTTP connection to REST API. Operators need to set + ``[irmc] port`` in ironic.conf or ``driver_info/irmc_port`` + to 443. +features: + - | + Adds verify step and node vendor passthru method to deal with + a firmware incompatibility issue with iRMC versions S6 2.00 + and later in which HTTP connection to REST API is not supported + and HTTPS connections to REST API is required. + + Verify step checks connection to iRMC REST API and if connection + succeeds, it fetches version of iRMC firmware and store it in + ``driver_internal_info/irmc_fw_version``. Ironic operators use + node vendor passthru method to fetch & update iRMC firmware + version cached in ``driver_internal_info/irmc_fw_version``. diff -Nru ironic-21.1.0/releasenotes/notes/fix-irmc-s6-2.00-ipmi-incompatibility-118484a424df02b1.yaml ironic-21.4.4/releasenotes/notes/fix-irmc-s6-2.00-ipmi-incompatibility-118484a424df02b1.yaml --- ironic-21.1.0/releasenotes/notes/fix-irmc-s6-2.00-ipmi-incompatibility-118484a424df02b1.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-irmc-s6-2.00-ipmi-incompatibility-118484a424df02b1.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,15 @@ +--- +fixes: + - | + Fixes a firmware incompatibility issue with iRMC versions S6 2.00 + and later now doesn't support IPMI over LAN by default. + To deal with this problem, irmc driver first tries IPMI operation then, + if IPMI operation fails, it tries Redfish API of Fujitsu server. + The operator must set Redfish parameters in the ``driver_info`` + if iRMC disable or doesn't support IPMI over LAN. +upgrade: + - | + When Ironic operator uses irmc driver against Fujitsu server which runs + iRMC version S6 2.00 or later, operator may need to set Redfish parameters + in ``driver_info`` so this fix can operate properly or operator should + enable IPMI over LAN through BMC settings, if possible. diff -Nru ironic-21.1.0/releasenotes/notes/fix-nonetype-object-is-not-iterable-0592926d890d6c11.yaml ironic-21.4.4/releasenotes/notes/fix-nonetype-object-is-not-iterable-0592926d890d6c11.yaml --- ironic-21.1.0/releasenotes/notes/fix-nonetype-object-is-not-iterable-0592926d890d6c11.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-nonetype-object-is-not-iterable-0592926d890d6c11.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes ``'NoneType' object is not iterable`` in conductor logs for + ``redfish`` and ``idrac-redfish`` RAID clean and deploy steps. The message + should no longer appear. For affected nodes re-create the node or delete + ``raid_configs`` entry from ``driver_internal_info`` field. diff -Nru ironic-21.1.0/releasenotes/notes/fix-online-version-migration-db432a7b239647fa.yaml ironic-21.4.4/releasenotes/notes/fix-online-version-migration-db432a7b239647fa.yaml --- ironic-21.1.0/releasenotes/notes/fix-online-version-migration-db432a7b239647fa.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-online-version-migration-db432a7b239647fa.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,14 @@ +--- +fixes: + - | + Fixes an issue in the online upgrade logic where database models for + Node Traits and BIOS Settings resulted in an error when performing + the online data migration. This was because these tables were originally + created as extensions of the Nodes database table, and the schema + of the database was slightly different enough to result in an error + if there was data to migrate in these tables upon upgrade, + which would have occured if an early BIOS Setting adopter had + data in the database prior to upgrading to the Yoga release of Ironic. + + The online upgrade parameter now subsitutes an alternate primary key name + name when applicable. diff -Nru ironic-21.1.0/releasenotes/notes/fix-overlooked-irmc-ipmi-incompatibility-patch-situation-c246d2b59b2e8a78.yaml ironic-21.4.4/releasenotes/notes/fix-overlooked-irmc-ipmi-incompatibility-patch-situation-c246d2b59b2e8a78.yaml --- ironic-21.1.0/releasenotes/notes/fix-overlooked-irmc-ipmi-incompatibility-patch-situation-c246d2b59b2e8a78.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-overlooked-irmc-ipmi-incompatibility-patch-situation-c246d2b59b2e8a78.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,8 @@ +--- +fixes: + - | + Fixes bug in iRMC driver, where `irmc` power_interface sets and updates + `irmc_ipmi_succeed` flag which is used by rest of iRMC driver code to deal + with iRMC firmware's IPMI incompatibility but `ipmitool` power_interface + doesn't set nor update `irmc_ipmi_succeed` flag and rest of iRMC driver + code fail to handle iRMC firmware's IPMI incompatibility correctly. diff -Nru ironic-21.1.0/releasenotes/notes/fix-power-off-token-wipe-e7d605997f00d39d.yaml ironic-21.4.4/releasenotes/notes/fix-power-off-token-wipe-e7d605997f00d39d.yaml --- ironic-21.1.0/releasenotes/notes/fix-power-off-token-wipe-e7d605997f00d39d.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-power-off-token-wipe-e7d605997f00d39d.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue where an agent token could be inadvertently orphaned + if a node is already in the target power state when we attempt to turn + the node off. diff -Nru ironic-21.1.0/releasenotes/notes/fix-self-owned-node-policy-fc2dae357879dc33.yaml ironic-21.4.4/releasenotes/notes/fix-self-owned-node-policy-fc2dae357879dc33.yaml --- ironic-21.1.0/releasenotes/notes/fix-self-owned-node-policy-fc2dae357879dc33.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-self-owned-node-policy-fc2dae357879dc33.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes scope classification check with the "self_owned_node" policy + check where it was limited to check execution with only project + scoped, so system scoped users who ticked the policy endpoint would + basically get an incorrect error. diff -Nru ironic-21.1.0/releasenotes/notes/fix-system-scope-triggered-clean-22ada9b920c08365.yaml ironic-21.4.4/releasenotes/notes/fix-system-scope-triggered-clean-22ada9b920c08365.yaml --- ironic-21.1.0/releasenotes/notes/fix-system-scope-triggered-clean-22ada9b920c08365.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix-system-scope-triggered-clean-22ada9b920c08365.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,12 @@ +--- +fixes: + - | + Fixes an issue where a System Scoped user could not trigger a node into + a ``manageable`` state with cleaning enabled, as the Neutron client would + attempt to utilize their user's token to create the Neutron port for the + cleaning operation, as designed. This is because with requests made in the + ``system`` scope, there is no associated project and the request fails. + + Ironic now checks if the request has been made with a ``system`` scope, + and if so it utilizes the internal credential configuration to communicate + with Neutron. diff -Nru ironic-21.1.0/releasenotes/notes/fix_anaconda-70f4268edc255ff4.yaml ironic-21.4.4/releasenotes/notes/fix_anaconda-70f4268edc255ff4.yaml --- ironic-21.1.0/releasenotes/notes/fix_anaconda-70f4268edc255ff4.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix_anaconda-70f4268edc255ff4.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes the URL based anaconda deployment for parsing the given ``image_source`` + url. diff -Nru ironic-21.1.0/releasenotes/notes/fix_anaconda_pxe-6c75d42872424fec.yaml ironic-21.4.4/releasenotes/notes/fix_anaconda_pxe-6c75d42872424fec.yaml --- ironic-21.1.0/releasenotes/notes/fix_anaconda_pxe-6c75d42872424fec.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix_anaconda_pxe-6c75d42872424fec.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes URL based anaconda deploy to work in pxe boot. It also enables + grub based pxe anaconda deploy which is required for ``ilo`` hardware + type. diff -Nru ironic-21.1.0/releasenotes/notes/fix_secure_boot_with_anaconda_deploy-84d7c1e3bbfa40f2.yaml ironic-21.4.4/releasenotes/notes/fix_secure_boot_with_anaconda_deploy-84d7c1e3bbfa40f2.yaml --- ironic-21.1.0/releasenotes/notes/fix_secure_boot_with_anaconda_deploy-84d7c1e3bbfa40f2.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/fix_secure_boot_with_anaconda_deploy-84d7c1e3bbfa40f2.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fixes secure boot with anaconda deploy. diff -Nru ironic-21.1.0/releasenotes/notes/handle-missing-ethernetinterfaces-attr-7e52f7259fe66762.yaml ironic-21.4.4/releasenotes/notes/handle-missing-ethernetinterfaces-attr-7e52f7259fe66762.yaml --- ironic-21.1.0/releasenotes/notes/handle-missing-ethernetinterfaces-attr-7e52f7259fe66762.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/handle-missing-ethernetinterfaces-attr-7e52f7259fe66762.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixes the bug where provisioning a Redfish managed node fails if the BMC + doesn't support EthernetInterfaces attribute, even if MAC address + information is provided manually. This is done by handling of + MissingAttributeError sushy exception in get_mac_addresses() method. + This fix is needed to successfully provision machines such as Cisco UCSB + and UCSX. diff -Nru ironic-21.1.0/releasenotes/notes/irmc-add-snmp-auth-protocols-3ff7597cea7ef9dd.yaml ironic-21.4.4/releasenotes/notes/irmc-add-snmp-auth-protocols-3ff7597cea7ef9dd.yaml --- ironic-21.1.0/releasenotes/notes/irmc-add-snmp-auth-protocols-3ff7597cea7ef9dd.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/irmc-add-snmp-auth-protocols-3ff7597cea7ef9dd.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Adds ``sha256``, ``sha384`` and ``sha512`` as supported SNMPv3 + authentication protocols to iRMC driver. diff -Nru ironic-21.1.0/releasenotes/notes/irmc-align-with-ironic-default-boot-mode-dde6f65ea084c9e6.yaml ironic-21.4.4/releasenotes/notes/irmc-align-with-ironic-default-boot-mode-dde6f65ea084c9e6.yaml --- ironic-21.1.0/releasenotes/notes/irmc-align-with-ironic-default-boot-mode-dde6f65ea084c9e6.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/irmc-align-with-ironic-default-boot-mode-dde6f65ea084c9e6.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + Modify iRMC driver to use ironic.conf [deploy] default_boot_mode to determine + default boot_mode. diff -Nru ironic-21.1.0/releasenotes/notes/irmc-change-boot-interface-order-e76f5018da116a90.yaml ironic-21.4.4/releasenotes/notes/irmc-change-boot-interface-order-e76f5018da116a90.yaml --- ironic-21.1.0/releasenotes/notes/irmc-change-boot-interface-order-e76f5018da116a90.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/irmc-change-boot-interface-order-e76f5018da116a90.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,26 @@ +--- +fixes: + - | + Fixes the default boot interface order for the ``irmc`` hardware type + where previously it would prefer ``irmc-pxe`` over ``ipxe``. This + created inconsistencies for operators using multiple hardware types, + where both interfaces were enabled in the deployment. +upgrade: + - | + Operators who are upgrading should be aware that a bug was discovered + with the automatic selection of ``boot_interface`` for users of the + ``irmc`` hardware types. This was an inconsistency, resulting in + ``irmc-pxe`` being selected instead of ``ipxe`` if these boot + interfaces were enabled. Depending on the local configuration, + this may, or may not have happened and will remain static on + preexisting baremetal nodes. Some users may have been relying upon + this incorrect behavior by having mis-alligned defaults by trying to + use the ``irmc-pxe`` interface for ``ipxe``. Users wishing to continue + this usage as it was previously will need to explicitly set a + ``boot_interface`` value to either ``pxe`` or ``irmc-pxe``, depending + on the local configuration. Most operators have leveraged the default + examples, and thus did not explicitly encounter this condition. + Operators explicitly wishing to use ``pxe`` boot interfaces with + the ``ipxe`` templates and defaults set to override the defaults + for ``ironic.conf`` will need to either continue to leverage default + override configurations in their ``ironic.conf`` file. diff -Nru ironic-21.1.0/releasenotes/notes/ironic-antelope-prelude-0b77964469f56b13.yaml ironic-21.4.4/releasenotes/notes/ironic-antelope-prelude-0b77964469f56b13.yaml --- ironic-21.1.0/releasenotes/notes/ironic-antelope-prelude-0b77964469f56b13.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/ironic-antelope-prelude-0b77964469f56b13.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,14 @@ +--- +prelude: > + The Ironic team hereby announces the release of OpenStack 2023.1 + (Ironic 23.4.0). This repesents the completion of a six month development + cycle, which primarily focused on internal and scaling improvements. + Those improvements included revamping the database layer to improve + performance and ensure compatability with new versions of SQLAlchemy, + enhancing the ironic-conductor service to export application metrics to + prometheus via the ironic-prometheus-exporter, and the addition of a + new API concept of node sharding to help with scaling of services that + make frequent API calls to Ironic. + + The new Ironic release also comes with a slew of bugfixes for Ironic + services and hardware drivers. We sincerely hope you enjoy it! diff -Nru ironic-21.1.0/releasenotes/notes/limit-boot-to-disk-calls-lenovo-39763bfc98f602d8.yaml ironic-21.4.4/releasenotes/notes/limit-boot-to-disk-calls-lenovo-39763bfc98f602d8.yaml --- ironic-21.1.0/releasenotes/notes/limit-boot-to-disk-calls-lenovo-39763bfc98f602d8.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/limit-boot-to-disk-calls-lenovo-39763bfc98f602d8.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,13 @@ +--- +fixes: + - | + Fixes issues with Lenovo hardware where the system firmware may display + a blue "Boot Option Restoration" screen after the agent writes an image + to the host in UEFI boot mode, requiring manual intervention before the + deployed node boots. This issue is rooted in multiple changes being made + to the underlying NVRAM configuration of the node. Lenovo engineers + have suggested to *only* change the UEFI NVRAM and not perform + any further changes via the BMC to configure the next boot. Ironic now + does such on Lenovo hardware. More information and background on this + issue can be discovered in + `bug 2053064 `_. diff -Nru ironic-21.1.0/releasenotes/notes/lockutils-default-logging-8c38b8c0ac71043f.yaml ironic-21.4.4/releasenotes/notes/lockutils-default-logging-8c38b8c0ac71043f.yaml --- ironic-21.1.0/releasenotes/notes/lockutils-default-logging-8c38b8c0ac71043f.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/lockutils-default-logging-8c38b8c0ac71043f.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,8 @@ +--- +other: + - | + The default logging level for the ``oslo_concurrencty.lockutils`` + module logging has been changed to ``WARNING``. By default, the debug + logging was resulting in lots of noise. Operators wishing to view debug + logging for this module can tuilize the ``[DEFAULT]default_log_levels`` + configuration option. diff -Nru ironic-21.1.0/releasenotes/notes/no-recalculate-653e524fd6160e72.yaml ironic-21.4.4/releasenotes/notes/no-recalculate-653e524fd6160e72.yaml --- ironic-21.1.0/releasenotes/notes/no-recalculate-653e524fd6160e72.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/no-recalculate-653e524fd6160e72.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + No longer re-calculates checksums for images that are already raw. + Previously, it would cause significant delays in deploying raw images. diff -Nru ironic-21.1.0/releasenotes/notes/node-iso-external_http_url-c5e3fa9ae4960dd6.yaml ironic-21.4.4/releasenotes/notes/node-iso-external_http_url-c5e3fa9ae4960dd6.yaml --- ironic-21.1.0/releasenotes/notes/node-iso-external_http_url-c5e3fa9ae4960dd6.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/node-iso-external_http_url-c5e3fa9ae4960dd6.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,5 @@ +--- +fixes: + - | + The per-node ``external_http_url`` setting in the driver info is now used + for a boot ISO. Previously this setting was only used for a config floppy. diff -Nru ironic-21.1.0/releasenotes/notes/permit-conductor-to-start-without-neutron-networks-d4aa21654f9c07bf.yaml ironic-21.4.4/releasenotes/notes/permit-conductor-to-start-without-neutron-networks-d4aa21654f9c07bf.yaml --- ironic-21.1.0/releasenotes/notes/permit-conductor-to-start-without-neutron-networks-d4aa21654f9c07bf.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/permit-conductor-to-start-without-neutron-networks-d4aa21654f9c07bf.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixes an issue where the conductor service would fail to launch when + the ``neutron`` network_interface setting was enabled, and no global + ``cleaning_network`` or ``provisioning_network`` is set in `ironic.conf.` + These settings have long been able to be applied on a per-node basis via + the API. As such, the service can now be started and will error on node + validation calls, as designed for drivers missing networking parameters. diff -Nru ironic-21.1.0/releasenotes/notes/prepare-for-sqlalchemy-20-e817f340f261b1a2.yaml ironic-21.4.4/releasenotes/notes/prepare-for-sqlalchemy-20-e817f340f261b1a2.yaml --- ironic-21.1.0/releasenotes/notes/prepare-for-sqlalchemy-20-e817f340f261b1a2.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/prepare-for-sqlalchemy-20-e817f340f261b1a2.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,7 @@ +--- +upgrade: + - | + Ironic has started the process of upgrading the code base to support + SQLAlchemy 2.0 in anticipation of it's release. This results in the + minimum version of SQLAlchemy becoming 1.4.0 as it contains migration + features for the move to SQLAlchemy 2.0. diff -Nru ironic-21.1.0/releasenotes/notes/redfish-fix-raid-creation-f437066b1301c032.yaml ironic-21.4.4/releasenotes/notes/redfish-fix-raid-creation-f437066b1301c032.yaml --- ironic-21.1.0/releasenotes/notes/redfish-fix-raid-creation-f437066b1301c032.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/redfish-fix-raid-creation-f437066b1301c032.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes Raid creation issue in iLO6 and other BMC with latest schema by + removing 'VolumeType', 'Encrypted' and changing placement of 'Drives' + to inside 'Links'. diff -Nru ironic-21.1.0/releasenotes/notes/service-project-service-role-fix-e4d1a8c23856926a.yaml ironic-21.4.4/releasenotes/notes/service-project-service-role-fix-e4d1a8c23856926a.yaml --- ironic-21.1.0/releasenotes/notes/service-project-service-role-fix-e4d1a8c23856926a.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/service-project-service-role-fix-e4d1a8c23856926a.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,41 @@ +--- +fixes: + - | + Provides a fix for ``service`` role support to enable the use + case where a dedicated service project is used for cloud service + operation to facilitate actions as part of the operation of the + cloud infrastructure. + + OpenStack clouds can take a variety of configuration models + for service accounts. It is now possible to utilize the + ``[DEFAULT] rbac_service_role_elevated_access`` setting to + enable users with a ``service`` role in a dedicated ``service`` + project to act upon the API similar to a "System" scoped + "Member" where resources regardless of ``owner`` or ``lessee`` + settings are available. This is needed to enable synchronization + processes, such as ``nova-compute`` or the ``networking-baremetal`` + ML2 plugin to perform actions across the whole of an Ironic + deployment, if desirable where a "System" scoped user is also + undesirable. + + This functionality can be tuned to utilize a customized project + name aside from the default convention ``service``, for example + ``baremetal`` or ``admin``, utilizing the + ``[DEFAULT] rbac_service_project_name`` setting. + + Operators can alternatively entirely override the + ``service_role`` RBAC policy rule, if so desired, however + Ironic feels the default is both reasonable and delineates + sufficiently for the variety of Role Based Access Control + usage cases which can exist with a running Ironic deployment. +upgrades: + - | + This version of ironic includes an opt-in fix to the Role Based Access + Control logic where the "service" role in a "service" project is + able to be granted elevated access to the API surface of Ironic such that + all baremetal nodes are visible to that API consumer. This is for + deployments which have not moved to a "System" scoped user for connecting + to ironic for services like ``nova-compute`` and the + ``networking-baremetal`` Neutron plugin, and where it is desirable for + those services to be able to operate across the whole of the Ironic + deployment. diff -Nru ironic-21.1.0/releasenotes/notes/shard-support-a26f8d2ab5cca582.yaml ironic-21.4.4/releasenotes/notes/shard-support-a26f8d2ab5cca582.yaml --- ironic-21.1.0/releasenotes/notes/shard-support-a26f8d2ab5cca582.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/shard-support-a26f8d2ab5cca582.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,14 @@ +features: + - Adds support for setting a shard key on a node, and filtering node or port + lists by shard. This shard key is not used for any purpose internally in + Ironic, but instead is intended to allow API clients to filter for a + subset of nodes or ports. Being able to fetch only a subset of nodes or + ports is useful for parallelizing any operational task that needs to be + performed across all nodes or ports. + - Adds support for querying for nodes which are sharded or unsharded. This + is useful for allowing operators to find nodes which have not been + assigned a shard key. + - Adds support for querying for a list of shards via ``/v1/shards``. This + endpoint will return a list of currently assigned shard keys as well as + the count of nodes which has those keys assigned. Using this API endpoint, + operators can see a high level listing of how their nodes are sharded. diff -Nru ironic-21.1.0/releasenotes/notes/virtual-media-publisher-id-injection-c88674a31634f852.yaml ironic-21.4.4/releasenotes/notes/virtual-media-publisher-id-injection-c88674a31634f852.yaml --- ironic-21.1.0/releasenotes/notes/virtual-media-publisher-id-injection-c88674a31634f852.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/virtual-media-publisher-id-injection-c88674a31634f852.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,6 @@ +--- +fixes: + - | + Adds an ISO publisher value to ISO images which are mastered as part of + cleaning/deployment/service operations in support of a fix for + `bug 2032377 `_. diff -Nru ironic-21.1.0/releasenotes/notes/wait_hash_ring_reset-ef8bd548659e9906.yaml ironic-21.4.4/releasenotes/notes/wait_hash_ring_reset-ef8bd548659e9906.yaml --- ironic-21.1.0/releasenotes/notes/wait_hash_ring_reset-ef8bd548659e9906.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/wait_hash_ring_reset-ef8bd548659e9906.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,13 @@ +--- +fixes: + - | + When a conductor service is stopped it will now continue to respond to RPC + requests until ``[DEFAULT]hash_ring_reset_interval`` has elapsed, allowing + a hash ring reset to complete on the cluster after conductor is + unregistered. This will improve the reliability of the cluster when scaling + down or rolling out updates. + + This delay only occurs when there is more than one online conductor, + to allow fast restarts on single-node ironic installs (bifrost, + metal3). + diff -Nru ironic-21.1.0/releasenotes/notes/wipe-agent-token-upon-cleaning-timeout-c9add514fad1b02c.yaml ironic-21.4.4/releasenotes/notes/wipe-agent-token-upon-cleaning-timeout-c9add514fad1b02c.yaml --- ironic-21.1.0/releasenotes/notes/wipe-agent-token-upon-cleaning-timeout-c9add514fad1b02c.yaml 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/notes/wipe-agent-token-upon-cleaning-timeout-c9add514fad1b02c.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes an issue where an agent token was being orphaned if a baremetal node + timed out during cleaning operations, leading to issues where the node + would not be able to establish a new token with Ironic upon future + in some cases. We now always wipe the token in this case. diff -Nru ironic-21.1.0/releasenotes/source/index.rst ironic-21.4.4/releasenotes/source/index.rst --- ironic-21.1.0/releasenotes/source/index.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/releasenotes/source/index.rst 2024-10-11 15:42:16.000000000 +0000 @@ -6,6 +6,7 @@ :maxdepth: 1 unreleased + zed yoga xena wallaby diff -Nru ironic-21.1.0/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po ironic-21.4.4/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po --- ironic-21.1.0/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po 2024-10-11 15:42:16.000000000 +0000 @@ -3,15 +3,16 @@ # Andi Chandler , 2019. #zanata # Andi Chandler , 2020. #zanata # Andi Chandler , 2022. #zanata +# Andi Chandler , 2023. #zanata msgid "" msgstr "" "Project-Id-Version: Ironic Release Notes\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-09-06 22:51+0000\n" +"POT-Creation-Date: 2023-02-01 23:20+0000\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"PO-Revision-Date: 2022-09-05 10:29+0000\n" +"PO-Revision-Date: 2023-02-03 04:37+0000\n" "Last-Translator: Andi Chandler \n" "Language-Team: English (United Kingdom)\n" "Language: en_GB\n" @@ -136,9 +137,6 @@ msgid "10.1.9" msgstr "10.1.9" -msgid "11.0.0" -msgstr "11.0.0" - msgid "11.1.0" msgstr "11.1.0" @@ -157,9 +155,6 @@ msgid "11.1.4-12" msgstr "11.1.4-12" -msgid "12.0.0" -msgstr "12.0.0" - msgid "12.1.0" msgstr "12.1.0" @@ -184,9 +179,6 @@ msgid "12.1.6-3" msgstr "12.1.6-3" -msgid "12.2.0" -msgstr "12.2.0" - msgid "13.0.0" msgstr "13.0.0" @@ -211,8 +203,8 @@ msgid "13.0.7" msgstr "13.0.7" -msgid "13.0.7-25" -msgstr "13.0.7-25" +msgid "13.0.7-29" +msgstr "13.0.7-29" msgid "14.0.0" msgstr "14.0.0" @@ -226,8 +218,8 @@ msgid "15.0.2" msgstr "15.0.2" -msgid "15.0.2-17" -msgstr "15.0.2-17" +msgid "15.0.2-25" +msgstr "15.0.2-25" msgid "15.1.0" msgstr "15.1.0" @@ -253,6 +245,9 @@ msgid "16.0.5" msgstr "16.0.5" +msgid "16.0.5-11" +msgstr "16.0.5-11" + msgid "16.1.0" msgstr "16.1.0" @@ -271,8 +266,11 @@ msgid "17.0.4" msgstr "17.0.4" -msgid "17.0.4-34" -msgstr "17.0.4-34" +msgid "17.1.0" +msgstr "17.1.0" + +msgid "17.1.0-6" +msgstr "17.1.0-6" msgid "18.0.0" msgstr "18.0.0" @@ -286,8 +284,11 @@ msgid "18.2.1" msgstr "18.2.1" -msgid "18.2.1-27" -msgstr "18.2.1-27" +msgid "18.2.2" +msgstr "18.2.2" + +msgid "18.2.2-6" +msgstr "18.2.2-6" msgid "19.0.0" msgstr "19.0.0" @@ -298,8 +299,11 @@ msgid "20.1.0" msgstr "20.1.0" -msgid "20.1.0-24" -msgstr "20.1.0-24" +msgid "20.1.1" +msgstr "20.1.1" + +msgid "20.1.1-6" +msgstr "20.1.1-6" msgid "20.2.0" msgstr "20.2.0" @@ -307,9 +311,33 @@ msgid "21.0.0" msgstr "21.0.0" +msgid "21.1.0" +msgstr "21.1.0" + +msgid "21.1.0-6" +msgstr "21.1.0-6" + +msgid "21.2.0" +msgstr "21.2.0" + +msgid "21.3.0" +msgstr "21.3.0" + +msgid "21.3.0-4" +msgstr "21.3.0-4" + msgid "4.0.0 First semver release" msgstr "4.0.0 First semver release" +msgid "4.1.0" +msgstr "4.1.0" + +msgid "4.2.0" +msgstr "4.2.0" + +msgid "4.2.1" +msgstr "4.2.1" + msgid "4.2.2" msgstr "4.2.2" @@ -514,6 +542,15 @@ "masked for this request." msgid "" +"A driver that handles booting itself (for example, a driver that implements " +"booting from virtual media) should use the following to make calls to the " +"boot interface a no-op::" +msgstr "" +"A driver that handles booting itself (for example, a driver that implements " +"booting from virtual media) should use the following to make calls to the " +"boot interface a no-op::" + +msgid "" "A few major changes are worth mentioning. This is not an exhaustive list, " "and mostly includes changes from 9.0.0:" msgstr "" @@ -530,21 +567,6 @@ msgid "" "A future release will change the default value of ``[deploy]/" -"default_boot_mode`` from \"bios\" to \"uefi\". It is recommended to set an " -"explicit value for this option. For hardware types which don't support " -"setting boot mode, a future release will assume boot mode is set to UEFI if " -"no boot mode is set to node's capabilities. It is also recommended to set " -"``boot_mode`` into ``properties/capabilities`` of a node." -msgstr "" -"A future release will change the default value of ``[deploy]/" -"default_boot_mode`` from \"bios\" to \"uefi\". It is recommended to set an " -"explicit value for this option. For hardware types which don't support " -"setting boot mode, a future release will assume boot mode is set to UEFI if " -"no boot mode is set to node's capabilities. It is also recommended to set " -"``boot_mode`` into ``properties/capabilities`` of a node." - -msgid "" -"A future release will change the default value of ``[deploy]/" "default_boot_option`` from \"netboot\" to \"local\". To avoid disruptions, " "it is recommended to set an explicit value for this option." msgstr "" @@ -717,6 +739,15 @@ "for the node, merely recording the returned state instead." msgid "" +"A new option ``[agent]api_ca_file`` allows passing a CA file to the ramdisk " +"when ``redfish-virtual-media`` boot is used. Requires ironic-python-agent " +"from the Wallaby cycle." +msgstr "" +"A new option ``[agent]api_ca_file`` allows passing a CA file to the ramdisk " +"when ``redfish-virtual-media`` boot is used. Requires ironic-python-agent " +"from the Wallaby cycle." + +msgid "" "A node in the ``active`` provision state can be rescued via the ``GET /v1/" "nodes/{node_ident}/states/provision`` API, by specifying ``rescue`` as the " "``target`` value, and a ``rescue_password`` value. When the node has been " @@ -773,6 +804,21 @@ msgid "" "A permission setting has been added for ``redfish-virtual-media`` boot " "interface, which allows for explicit file permission setting when the driver " +"is being used. The default for the new ``[redfish]file_permission setting is " +"``0u644``, or 644 if manually changed using ``chmod`` on the command line. " +"Operators MAY need to adjust this if they were running the conductor with a " +"specific ``umask`` to work around the permission setting defect." +msgstr "" +"A permission setting has been added for ``redfish-virtual-media`` boot " +"interface, which allows for explicit file permission setting when the driver " +"is being used. The default for the new ``[redfish]file_permission setting is " +"``0u644``, or 644 if manually changed using ``chmod`` on the command line. " +"Operators MAY need to adjust this if they were running the conductor with a " +"specific ``umask`` to work around the permission setting defect." + +msgid "" +"A permission setting has been added for ``redfish-virtual-media`` boot " +"interface, which allows for explicit file permission setting when the driver " "is used. The default for the new ``[redfish]file_permission setting is " "``0u644``, or 644 if manually changed using ``chmod`` on the command line. " "Operators may need to check ``/httpboot/redfish`` folder permissions if " @@ -822,10 +868,6 @@ "Driver needs this verification because the machine is going to use a MAC " "that will only be specified at the profile application." -msgid "A warning is logged for any changes to immutable configuration options." -msgstr "" -"A warning is logged for any changes to immutable configuration options." - msgid "API fields to support node ``description`` and ``owner`` values." msgstr "API fields to support node ``description`` and ``owner`` values." @@ -837,18 +879,25 @@ "net/ironic/+bug/1536828 for details." msgid "" -"API version 1.57 adds a REST API endpoint for updating an existing " -"allocation. Only ``name`` and ``extra`` fields are allowed to be updated." -msgstr "" -"API version 1.57 adds a REST API endpoint for updating an existing " -"allocation. Only ``name`` and ``extra`` fields are allowed to be updated." - -msgid "" -"API version 1.58 allows backfilling allocations for existing deployed nodes " -"by providing ``node`` to ``POST /v1/allocations``." -msgstr "" -"API version 1.58 allows backfilling allocations for existing deployed nodes " -"by providing ``node`` to ``POST /v1/allocations``." +"Ability to create an allocation has been restricted by a new policy rule " +"``baremetal::allocation::create_pre_rbac`` which prevents creation of " +"allocations by any project administrator when operating with the new Role " +"Based Access Control model. The use and enforcement of this rule is disabled " +"when ``[oslo_policy]enforce_new_defaults`` is set which also makes the " +"population of a ``owner`` field for allocations to become automatically " +"populated. Most deployments should not encounter any issues with this " +"security change, and the policy rule will be removed when support for the " +"legacy ``baremetal_admin`` custom role has been removed." +msgstr "" +"Ability to create an allocation has been restricted by a new policy rule " +"``baremetal::allocation::create_pre_rbac`` which prevents creation of " +"allocations by any project administrator when operating with the new Role " +"Based Access Control model. The use and enforcement of this rule is disabled " +"when ``[oslo_policy]enforce_new_defaults`` is set which also makes the " +"population of a ``owner`` field for allocations to become automatically " +"populated. Most deployments should not encounter any issues with this " +"security change, and the policy rule will be removed when support for the " +"legacy ``baremetal_admin`` custom role has been removed." msgid "Add BIOS config to DRAC Driver" msgstr "Add BIOS config to DRAC Driver" @@ -875,13 +924,15 @@ msgstr "Add Wake-On-LAN Power Driver" msgid "" -"Add ``?detail=`` boolean query to the API list endpoints to provide a more " -"RESTful alternative to the existing ``/nodes/detail`` and similar endpoints. " -"The default is False. Now these API requests are possible:" -msgstr "" -"Add ``?detail=`` boolean query to the API list endpoints to provide a more " -"RESTful alternative to the existing ``/nodes/detail`` and similar endpoints. " -"The default is False. Now these API requests are possible:" +"Add ``anaconda`` deploy interface to Ironic. This driver will deploy the OS " +"using anaconda installer and kickstart file instead of IPA. To support this " +"feature a new configuration group ``anaconda`` is added to Ironic " +"configuration file along with ``default_ks_template`` configuration option." +msgstr "" +"Add ``anaconda`` deploy interface to Ironic. This driver will deploy the OS " +"using Anaconda installer and kickstart file instead of IPA. To support this " +"feature a new configuration group ``anaconda`` is added to Ironic " +"configuration file along with ``default_ks_template`` configuration option." msgid "" "Add ``choices`` parameter to config options. Invalid values will be rejected " @@ -988,6 +1039,9 @@ msgid "Added CORS support" msgstr "Added CORS support" +msgid "Added Cisco IMC driver" +msgstr "Added Cisco IMC driver" + msgid "" "Added configdrive support for whole disk images for iSCSI based deploy. This " "will work for UEFI only or BIOS only images. It will not work for hybrid " @@ -1025,6 +1079,24 @@ "validate iLO SSL certificates." msgid "" +"Adding ``kernel`` and ``ramdisk`` is no longer necessary for partition " +"images if ``image_type`` is set to ``partition`` and local boot is used." +msgstr "" +"Adding ``kernel`` and ``ramdisk`` is no longer necessary for partition " +"images if ``image_type`` is set to ``partition`` and local boot is used." + +msgid "" +"Adding new clean steps to ``ilo`` and ``ilo5`` hardware type - " +"``security_parameters_update``, ``update_minimum_password_length``, and " +"``update_auth_failure_logging_threshold`` which allows users to modify ilo " +"system security settings." +msgstr "" +"Adding new clean steps to ``ilo`` and ``ilo5`` hardware type - " +"``security_parameters_update``, ``update_minimum_password_length``, and " +"``update_auth_failure_logging_threshold`` which allows users to modify ilo " +"system security settings." + +msgid "" "Addition of the provision state target verb of ``adopt`` which allows an " "operator to move a node into an ``active`` state from ``manageable`` state, " "without performing a deployment operation on the node. This can be used to " @@ -1043,6 +1115,15 @@ msgstr "Additionally, adds the following API changes:" msgid "" +"Additionally, as mentioned before, `ironic.drivers.modules.pxe.PXEDeploy` " +"has moved to `ironic.drivers.modules.iscsi_deploy.ISCSIDeploy`, which will " +"break drivers that use this class." +msgstr "" +"Additionally, as mentioned before, `ironic.drivers.modules.pxe.PXEDeploy` " +"has moved to `ironic.drivers.modules.iscsi_deploy.ISCSIDeploy`, which will " +"break drivers that use this class." + +msgid "" "Addresses a condition where the Compute Service may have been unable to " "remove VIF attachment records while a baremetal node is being unprovisiond. " "This condition resulted in VIF records being orphaned, blocking future " @@ -1126,15 +1207,6 @@ "udp_transport_timeout`` allow to change the number of retries and the " "timeout values respectively for the the SNMP driver." -msgid "" -"Adds SNMPv3 message authentication and encryption features to ironic " -"``snmp`` hardware type. To enable these features, the following parameters " -"should be used in the node's ``driver_info``:" -msgstr "" -"Adds SNMPv3 message authentication and encryption features to ironic " -"``snmp`` hardware type. To enable these features, the following parameters " -"should be used in the node's ``driver_info``:" - msgid "Adds ShellinaboxConsole support for virsh SSH driver." msgstr "Adds ShellinaboxConsole support for virsh SSH driver." @@ -1161,6 +1233,15 @@ "nodes that are stuck in the rescue wait state." msgid "" +"Adds ``[conductor]clean_step_priority_override`` configuration parameter " +"which allows the operator to define a custom order in which the cleaning " +"steps are to run." +msgstr "" +"Adds ``[conductor]clean_step_priority_override`` configuration parameter " +"which allows the operator to define a custom order in which the cleaning " +"steps are to run." + +msgid "" "Adds ``[swift]/endpoint_override`` option to explicitly set the endpoint URL " "used for Swift. Ironic uses the Swift connection URL as a base for " "generation of some TempURLs. Added parameter enables operators to fix the " @@ -1188,8 +1269,11 @@ "``instance_info`` (and ``extra`` if using metalsmith), and a lessee should " "not be able to update all node attributes." -msgid "Adds ``bios`` interface to the ``redfish`` hardware type." -msgstr "Adds ``bios`` interface to the ``redfish`` hardware type." +msgid "Adds ``bios_interface`` to the node list and node show api-ref." +msgstr "Adds ``bios_interface`` to the node list and node show api-ref." + +msgid "Adds ``bios_interface`` to the node validate api-ref." +msgstr "Adds ``bios_interface`` to the node validate api-ref." msgid "" "Adds ``command_timeout`` and ``max_command_attempts`` configuration options " @@ -1199,15 +1283,6 @@ "to IPA, so when connection errors occur the command will be executed again." msgid "" -"Adds ``command_timeout`` and ``max_command_attempts`` configuration options " -"to IPA, so when connection errors occur the command will be executed again. " -"The options are located in the ``[agent]`` section." -msgstr "" -"Adds ``command_timeout`` and ``max_command_attempts`` configuration options " -"to IPA, so when connection errors occur the command will be executed again. " -"The options are located in the ``[agent]`` section." - -msgid "" "Adds ``driver_internal_info`` field to the node-related notification " "``baremetal.node.provision_set.*``, new payload version 1.16." msgstr "" @@ -1215,28 +1290,6 @@ "``baremetal.node.provision_set.*``, new payload version 1.16." msgid "" -"Adds ``external`` storage interface which is short for \"externally managed" -"\". This adds logic to allow the Bare Metal service to identify when a BFV " -"scenario is being requested based upon the configuration set for ``volume " -"targets``." -msgstr "" -"Adds ``external`` storage interface which is short for \"externally managed" -"\". This adds logic to allow the Bare Metal service to identify when a BFV " -"scenario is being requested based upon the configuration set for ``volume " -"targets``." - -msgid "" -"Adds ``get_boot_mode``, ``set_boot_mode`` and ``get_supported_boot_modes`` " -"methods to driver management interface. Drivers can override these methods " -"implementing boot mode management calls to the BMC of the baremetal nodes " -"being managed." -msgstr "" -"Adds ``get_boot_mode``, ``set_boot_mode`` and ``get_supported_boot_modes`` " -"methods to driver management interface. Drivers can override these methods " -"implementing boot mode management calls to the BMC of the baremetal nodes " -"being managed." - -msgid "" "Adds ``idrac`` hardware type support of a virtual media boot interface " "implementation that utilizes the Redfish out-of-band (OOB) management " "protocol and is compatible with the integrated Dell Remote Access Controller " @@ -1315,17 +1368,6 @@ msgstr "" "Adds ``rescue_interface`` field to the following node-related notifications:" -msgid "" -"Adds ``reset_idrac`` and ``known_good_state`` cleaning steps to hardware " -"type ``idrac``. ``reset_idrac`` actually resets the iDRAC; " -"``known_good_state`` also resets the iDRAC and clears the Lifecycle " -"Controller job queue to make sure the iDRAC is in good state." -msgstr "" -"Adds ``reset_idrac`` and ``known_good_state`` cleaning steps to hardware " -"type ``idrac``. ``reset_idrac`` actually resets the iDRAC; " -"``known_good_state`` also resets the iDRAC and clears the Lifecycle " -"Controller job queue to make sure the iDRAC is in good state." - msgid "Adds ``storage_interface`` field to the node-related notifications:" msgstr "Adds ``storage_interface`` field to the node-related notifications:" @@ -1385,27 +1427,6 @@ "notifications." msgid "" -"Adds a ``[conductor]send_sensor_data_for_undeployed_nodes`` option to enable " -"ironic to collect and transmit sensor data for all nodes for which sensor " -"data collection is available. By default, this option is not enabled which " -"aligns with the prior behavior of sensor data collection and transmission " -"where such data was only collected if an ``instance_uuid`` was present to " -"signify that the node has been or is being deployed. With this option set to " -"``True``, operators may be able to identify hardware in a faulty state " -"through the sensor data and take action before an instance workload is " -"deployed." -msgstr "" -"Adds a ``[conductor]send_sensor_data_for_undeployed_nodes`` option to enable " -"ironic to collect and transmit sensor data for all nodes for which sensor " -"data collection is available. By default, this option is not enabled which " -"aligns with the prior behaviour of sensor data collection and transmission " -"where such data was only collected if an ``instance_uuid`` was present to " -"signify that the node has been or is being deployed. With this option set to " -"``True``, operators may be able to identify hardware in a faulty state " -"through the sensor data and take action before an instance workload is " -"deployed." - -msgid "" "Adds a ``clear_job_queue`` cleaning step to the ``idrac-wsman`` management " "interface. The ``clear_job_queue`` cleaning step clears the Lifecycle " "Controller job queue including any pending jobs." @@ -1539,38 +1560,6 @@ "return tracebacks in API responses in an error condition." msgid "" -"Adds a configuration option ``[deploy]disk_erasure_concurrency`` to define " -"the target pool size used by Ironic Python Agent ramdisk to erase disk " -"devices. The number of threads created by IPA to erase disk devices is the " -"minimum value of target pool size and the number of disks to be erased. This " -"feature can greatly reduce the operation time for baremetals with multiple " -"disks. For the backwards compatibility, the default value is 1." -msgstr "" -"Adds a configuration option ``[deploy]disk_erasure_concurrency`` to define " -"the target pool size used by Ironic Python Agent ramdisk to erase disk " -"devices. The number of threads created by IPA to erase disk devices is the " -"minimum value of target pool size and the number of disks to be erased. This " -"feature can greatly reduce the operation time for baremetals with multiple " -"disks. For the backwards compatibility, the default value is 1." - -msgid "" -"Adds a configuration option ``[ipmi]disable_boot_timeout`` which is used to " -"set the default behavior whether ironic should send a raw IPMI command to " -"disable timeout. This configuration option can be overidden by the per-node " -"option ``ipmi_disable_boot_timeout`` in node's ``driver_info`` field. See " -"`story 2004266 `_ and " -"`Story 2002977 `_ for " -"additional information." -msgstr "" -"Adds a configuration option ``[ipmi]disable_boot_timeout`` which is used to " -"set the default behaviour whether ironic should send a raw IPMI command to " -"disable timeout. This configuration option can be overridden by the per-node " -"option ``ipmi_disable_boot_timeout`` in node's ``driver_info`` field. See " -"`story 2004266 `_ and " -"`Story 2002977 `_ for " -"additional information." - -msgid "" "Adds a configuration option ``webserver_verify_ca`` to support custom " "certificates to validate URLs hosted on a HTTPS webserver." msgstr "" @@ -1670,30 +1659,6 @@ "allocated from the configured port range for further use." msgid "" -"Adds a new configuration option ``[disk_utils]partprobe_attempts`` which " -"defaults to 10. This is the maximum number of times to try to read a " -"partition (if creating a config drive) via a ``partprobe`` command. " -"Previously, no retries were done which caused failures. This addresses `bug " -"1756760 `_." -msgstr "" -"Adds a new configuration option ``[disk_utils]partprobe_attempts`` which " -"defaults to 10. This is the maximum number of times to try to read a " -"partition (if creating a config drive) via a ``partprobe`` command. " -"Previously, no retries were done which caused failures. This addresses `bug " -"1756760 `_." - -msgid "" -"Adds a new configuration option ``[disk_utils]partprobe_attempts`` which " -"defaults to 10. This is the maximum number of times to try to read a " -"partition (if creating a config drive) via a ``partprobe`` command. Set it " -"to 1 if you want the previous behavior, where no retries were done." -msgstr "" -"Adds a new configuration option ``[disk_utils]partprobe_attempts`` which " -"defaults to 10. This is the maximum number of times to try to read a " -"partition (if creating a config drive) via a ``partprobe`` command. Set it " -"to 1 if you want the previous behaviour, where no retries were done." - -msgid "" "Adds a new configuration option ``[drac]boot_device_job_status_timeout`` " "that specifies the maximum amount of time (in seconds) to wait for the boot " "device configuration job to transition to the scheduled state to allow a " @@ -1790,45 +1755,6 @@ "v1/drivers/." msgid "" -"Adds an ``inspect wait`` state to handle asynchronous hardware " -"introspection. Caution should be taken due to the timeout monitoring is " -"shifted from ``inspecting`` to ``inspect wait``, please stop all running " -"asynchronous hardware inspection or wait until it is finished before " -"upgrading to the Rocky release. Otherwise nodes in asynchronous inspection " -"will be left at ``inspecting`` state forever unless the database is manually " -"updated." -msgstr "" -"Adds an ``inspect wait`` state to handle asynchronous hardware " -"introspection. Caution should be taken due to the timeout monitoring is " -"shifted from ``inspecting`` to ``inspect wait``, please stop all running " -"asynchronous hardware inspection or wait until it is finished before " -"upgrading to the Rocky release. Otherwise nodes in asynchronous inspection " -"will be left at ``inspecting`` state forever unless the database is manually " -"updated." - -msgid "" -"Adds an ``inspect wait`` state to handle asynchronous hardware " -"introspection. Returning ``INSPECTING`` from the ``inspect_hardware`` method " -"of inspect interface is deprecated, ``INSPECTWAIT`` should be returned " -"instead." -msgstr "" -"Adds an ``inspect wait`` state to handle asynchronous hardware " -"introspection. Returning ``INSPECTING`` from the ``inspect_hardware`` method " -"of inspect interface is deprecated, ``INSPECTWAIT`` should be returned " -"instead." - -msgid "" -"Adds an ``inspect wait`` state to handle asynchronous hardware " -"introspection. The ``[conductor]inspect_timeout`` configuration option is " -"deprecated for removal, please use ``[conductor]inspect_wait_timeout`` " -"instead to specify the timeout of inspection process." -msgstr "" -"Adds an ``inspect wait`` state to handle asynchronous hardware " -"introspection. The ``[conductor]inspect_timeout`` configuration option is " -"deprecated for removal, please use ``[conductor]inspect_wait_timeout`` " -"instead to specify the timeout of inspection process." - -msgid "" "Adds an `agent_iboot` driver to allow use of the Iboot power driver with the " "Agent deploy driver." msgstr "" @@ -1901,6 +1827,9 @@ msgid "Deprecated the 'parallel' option to periodic task decorator" msgstr "Deprecated the 'parallel' option to periodic task decorator" +msgid "Deprecated the bash ramdisk" +msgstr "Deprecated the bash ramdisk" + msgid "" "Drivers may optionally add a new BootInterface. This is merely a refactoring " "of the Driver API to support future improvements." @@ -1908,12 +1837,42 @@ "Drivers may optionally add a new BootInterface. This is merely a refactoring " "of the Driver API to support future improvements." +msgid "" +"Drivers using the \"agent\" deploy mechanism do not support \"rebuild --" +"preserve-ephemeral\"" +msgstr "" +"Drivers using the \"agent\" deploy mechanism do not support \"rebuild --" +"preserve-ephemeral\"" + +msgid "" +"Fix a couple of locale issues with deployments, when running on a system " +"using the Japanese locale" +msgstr "" +"Fix a couple of locale issues with deployments, when running on a system " +"using the Japanese locale" + +msgid "" +"IPMI Passwords are now obfuscated in REST API responses. This may be " +"disabled by changing API policy settings." +msgstr "" +"IPMI Passwords are now obfuscated in REST API responses. This may be " +"disabled by changing API policy settings." + msgid "Implemented a new Boot interface for drivers" msgstr "Implemented a new Boot interface for drivers" +msgid "Import Japanese translations - our first major translation addition!" +msgstr "Import Japanese translations - our first major translation addition!" + msgid "Introduce new BootInterface to the Driver API" msgstr "Introduce new BootInterface to the Driver API" +msgid "Known issues" +msgstr "Known issues" + +msgid "Liberty Series (4.0.0 - 4.2.5) Release Notes" +msgstr "Liberty Series (4.0.0 - 4.2.5) Release Notes" + msgid "Migrations from Nova \"baremetal\" have been removed" msgstr "Migrations from Nova \"baremetal\" have been removed" @@ -1926,6 +1885,23 @@ msgid "Ocata Series (7.0.0 - 7.0.x) Release Notes" msgstr "Ocata Series (7.0.0 - 7.0.x) Release Notes" +msgid "" +"Out of tree drivers may be broken by this release. The AgentDeploy and " +"ISCSIDeploy (formerly known as PXEDeploy) classes now depend on drivers to " +"utilize an instance of a BootInterface. For drivers that exist out of tree, " +"that use these deploy classes, an error will be thrown during deployment. " +"There is a simple fix. For drivers that expect these deploy classes to " +"handle PXE booting, one can add the following code to the driver's " +"`__init__` method::" +msgstr "" +"Out-of-tree drivers may be broken by this release. The AgentDeploy and " +"ISCSIDeploy (formerly known as PXEDeploy) classes now depend on drivers to " +"utilize an instance of a BootInterface. For drivers that exist out-of-tree, " +"that use these deploy classes, an error will be thrown during deployment. " +"There is a simple fix. For drivers that expect these deploy classes to " +"handle PXE booting, one can add the following code to the driver's " +"`__init__` method::" + msgid "PXE drivers now support GRUB2" msgstr "PXE drivers now support GRUB2" @@ -1971,6 +1947,13 @@ msgstr "Support for the new ENROLL workflow during Node creation" msgid "" +"The \"agent\" class of drivers now support both whole-disk and partition " +"based images." +msgstr "" +"The \"agent\" class of drivers now support both whole-disk and partition " +"based images." + +msgid "" "The Ironic API now has support for CORS requests, that may be used by, for " "example, web browser-based clients. This is configured in the [cors] section " "of ironic.conf." @@ -1979,6 +1962,33 @@ "example, web browser-based clients. This is configured in the [cors] section " "of ironic.conf." +msgid "The Ironic team apologizes profusely for this inconvenience." +msgstr "The Ironic team apologises profusely for this inconvenience." + +msgid "" +"The agent must download the tenant image in full before writing it to disk. " +"As such, the server being deployed must have enough RAM for running the " +"agent and storing the image. This is now checked before Ironic tells the " +"agent to deploy an image. An optional config [agent]memory_consumed_by_agent " +"is provided. When Ironic does this check, this config option may be set to " +"factor in the amount of RAM to reserve for running the agent." +msgstr "" +"The agent must download the tenant image in full before writing it to disk. " +"As such, the server being deployed must have enough RAM for running the " +"agent and storing the image. This is now checked before Ironic tells the " +"agent to deploy an image. An optional config [agent]memory_consumed_by_agent " +"is provided. When Ironic does this check, this config option may be set to " +"factor in the amount of RAM to reserve for running the agent." + +msgid "" +"This brings some bug fixes and small features on top of Ironic 4.0.0. Major " +"changes are listed below, and full release details are available on " +"Launchpad: https://launchpad.net/ironic/liberty/4.1.0." +msgstr "" +"This brings some bug fixes and small features on top of Ironic 4.0.0. Major " +"changes are listed below, and full release details are available on " +"Launchpad: https://launchpad.net/ironic/liberty/4.1.0." + msgid "" "This change enhances the driver interface for driver authors, and should not " "affect users of Ironic, by splitting control of booting a server from the " @@ -1993,6 +2003,15 @@ "image to a server." msgid "" +"This driver supports managing Cisco UCS C-series servers through the CIMC " +"API, rather than IPMI. Documentation is available at: https://docs.openstack." +"org/developer/ironic/drivers/cimc.html" +msgstr "" +"This driver supports managing Cisco UCS C-series servers through the CIMC " +"API, rather than IPMI. Documentation is available at: https://docs.openstack." +"org/developer/ironic/drivers/cimc.html" + +msgid "" "This is the first semver-versioned release of Ironic, created during the " "OpenStack \"Liberty\" development cycle. It marks a pivot in our versioning " "schema from date-based versioning; the previous released version was 2015.1. " @@ -2005,6 +2024,24 @@ "2015.1. Full release details are available on Launchpad: https://launchpad." "net/ironic/liberty/4.0.0." +msgid "" +"This release is a patch release on top of 4.2.0, as part of the stable " +"Liberty series. Full details are available on Launchpad: https://launchpad." +"net/ironic/liberty/4.2.1." +msgstr "" +"This release is a patch release on top of 4.2.0, as part of the stable " +"Liberty series. Full details are available on Launchpad: https://launchpad." +"net/ironic/liberty/4.2.1." + +msgid "" +"This release is proposed as the stable Liberty release for Ironic, and " +"brings with it some bug fixes and small features. Full release details are " +"available on Launchpad: https://launchpad.net/ironic/liberty/4.2.0." +msgstr "" +"This release is proposed as the stable Liberty release for Ironic, and " +"brings with it some bug fixes and small features. Full release details are " +"available on Launchpad: https://launchpad.net/ironic/liberty/4.2.0." + msgid "Train Series (12.2.0 - 13.0.x) Release Notes" msgstr "Train Series (12.2.0 - 13.0.x) Release Notes" @@ -2017,11 +2054,41 @@ msgid "Wallaby Series (16.1.0 - 17.0.x) Release Notes" msgstr "Wallaby Series (16.1.0 - 17.0.x) Release Notes" +msgid "" +"While Ironic does include a ClusteredComputeManager, which allows running " +"more than one nova-compute process with Ironic, it should be considered " +"experimental and has many known problems." +msgstr "" +"While Ironic does include a ClusteredComputeManager, which allows running " +"more than one nova-compute process with Ironic, it should be considered " +"experimental and has many known problems." + msgid "Xena Series (18.0.0 - 18.2.x) Release Notes" msgstr "Xena Series (18.0.0 - 18.2.x) Release Notes" -msgid "Yoga Series Release Notes" -msgstr "Yoga Series Release Notes" +msgid "Yoga Series (19.0.0 - 20.1.x) Release Notes" +msgstr "Yoga Series (19.0.0 - 20.1.x) Release Notes" + +msgid "Zed Series (20.2.0 - 21.1.x) Release Notes" +msgstr "Zed Series (20.2.0 - 21.1.x) Release Notes" + +msgid "" +"iLO driver documentation is available at: https://docs.openstack.org/" +"developer/ironic/drivers/ilo.html" +msgstr "" +"iLO driver documentation is available at: https://docs.openstack.org/" +"developer/ironic/drivers/ilo.html" + +msgid "" +"iLO virtual media drivers (iscsi_ilo and agent_ilo) can work standalone " +"without Swift, by configuring an HTTP(S) server for hosting the deploy/boot " +"images. A web server needs to be running on every conductor node and needs " +"to be configured in ironic.conf." +msgstr "" +"iLO virtual media drivers (iscsi_ilo and agent_ilo) can work standalone " +"without Swift, by configuring an HTTP(S) server for hosting the deploy/boot " +"images. A web server needs to be running on every conductor node and needs " +"to be configured in ironic.conf." msgid "ipmitool driver supports IPMI v1.5" msgstr "ipmitool driver supports IPMI v1.5" diff -Nru ironic-21.1.0/releasenotes/source/yoga.rst ironic-21.4.4/releasenotes/source/yoga.rst --- ironic-21.1.0/releasenotes/source/yoga.rst 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/releasenotes/source/yoga.rst 2024-10-11 15:42:16.000000000 +0000 @@ -1,6 +1,6 @@ -========================= -Yoga Series Release Notes -========================= +=========================================== +Yoga Series (19.0.0 - 20.1.x) Release Notes +=========================================== .. release-notes:: :branch: stable/yoga diff -Nru ironic-21.1.0/releasenotes/source/zed.rst ironic-21.4.4/releasenotes/source/zed.rst --- ironic-21.1.0/releasenotes/source/zed.rst 1970-01-01 00:00:00.000000000 +0000 +++ ironic-21.4.4/releasenotes/source/zed.rst 2024-10-11 15:42:16.000000000 +0000 @@ -0,0 +1,6 @@ +========================================== +Zed Series (20.2.0 - 21.1.x) Release Notes +========================================== + +.. release-notes:: + :branch: stable/zed diff -Nru ironic-21.1.0/reno.yaml ironic-21.4.4/reno.yaml --- ironic-21.1.0/reno.yaml 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/reno.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,4 +0,0 @@ ---- -# Ignore the kilo-eol tag because that branch does not work with reno -# and contains no release notes. -closed_branch_tag_re: "(.+)(?=3.1.1 # Apache-2.0 -SQLAlchemy>=1.2.19 # MIT +SQLAlchemy>=1.4.0 # MIT alembic>=1.4.2 # MIT automaton>=1.9.0 # Apache-2.0 eventlet!=0.18.3,!=0.20.1,>=0.18.2 # MIT @@ -14,7 +14,7 @@ python-cinderclient!=4.0.0,>=3.3.0 # Apache-2.0 python-glanceclient>=2.8.0 # Apache-2.0 keystoneauth1>=4.2.0 # Apache-2.0 -ironic-lib>=4.6.1 # Apache-2.0 +ironic-lib>=5.4.0 # Apache-2.0 python-swiftclient>=3.2.0 # Apache-2.0 pytz>=2013.6 # MIT stevedore>=1.29.0 # Apache-2.0 @@ -39,7 +39,7 @@ jsonpatch!=1.20,>=1.16 # BSD Jinja2>=3.0.0 # BSD License (3 clause) keystonemiddleware>=9.5.0 # Apache-2.0 -oslo.messaging>=5.29.0 # Apache-2.0 +oslo.messaging>=14.1.0 # Apache-2.0 tenacity>=6.2.0 # Apache-2.0 oslo.versionedobjects>=1.31.2 # Apache-2.0 jsonschema>=3.2.0 # MIT diff -Nru ironic-21.1.0/setup.cfg ironic-21.4.4/setup.cfg --- ironic-21.1.0/setup.cfg 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/setup.cfg 2024-10-11 15:42:16.000000000 +0000 @@ -168,6 +168,7 @@ idrac-wsman = ironic.drivers.modules.drac.vendor_passthru:DracWSManVendorPassthru idrac-redfish = ironic.drivers.modules.drac.vendor_passthru:DracRedfishVendorPassthru ilo = ironic.drivers.modules.ilo.vendor:VendorPassthru + irmc = ironic.drivers.modules.irmc.vendor:IRMCVendorPassthru ipmitool = ironic.drivers.modules.ipmitool:VendorPassthru no-vendor = ironic.drivers.modules.noop:NoVendor redfish = ironic.drivers.modules.redfish.vendor:RedfishVendorPassthru diff -Nru ironic-21.1.0/setup.py ironic-21.4.4/setup.py --- ironic-21.1.0/setup.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/setup.py 2024-10-11 15:42:16.000000000 +0000 @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools setuptools.setup( setup_requires=['pbr>=2.0.0'], - pbr=True) + pbr=True, +) diff -Nru ironic-21.1.0/test-requirements.txt ironic-21.4.4/test-requirements.txt --- ironic-21.1.0/test-requirements.txt 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/test-requirements.txt 2024-10-11 15:42:16.000000000 +0000 @@ -11,7 +11,7 @@ oslotest>=3.2.0 # Apache-2.0 stestr>=2.0.0 # Apache-2.0 psycopg2>=2.8.5 # LGPL/ZPL -testtools>=2.2.0 # MIT +testtools>=2.5.0 # MIT WebTest>=2.0.27 # MIT pysnmp>=4.4.12 bandit!=1.6.0,>=1.1.0,<2.0.0 # Apache-2.0 diff -Nru ironic-21.1.0/tools/benchmark/do_not_run_create_benchmark_data.py ironic-21.4.4/tools/benchmark/do_not_run_create_benchmark_data.py --- ironic-21.1.0/tools/benchmark/do_not_run_create_benchmark_data.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/tools/benchmark/do_not_run_create_benchmark_data.py 2024-10-11 15:42:16.000000000 +0000 @@ -10,7 +10,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +import random import sys import time @@ -20,31 +20,54 @@ from ironic.common import service from ironic.conf import CONF # noqa To Load Configuration from ironic.objects import node +from ironic.objects import port + + +NODE_COUNT = 10000 +PORTS_PER_NODE = 2 + + +# NOTE(hjensas): Mostly copy-paste from Nova +def generate_mac_address(): + """Generate an Ethernet MAC address.""" + mac = [random.randint(0x00, 0xff), + random.randint(0x00, 0xff), + random.randint(0x00, 0xff), + random.randint(0x00, 0xff), + random.randint(0x00, 0xff), + random.randint(0x00, 0xff)] + return ':'.join(map(lambda x: "%02x" % x, mac)) + + +def _create_test_node_ports(new_node): + for i in range(0, PORTS_PER_NODE): + new_port = port.Port() + new_port.node_id = new_node.id + new_port.address = generate_mac_address() + new_port.pxe_enabled = True + new_port.create() def _create_test_nodes(): print("Starting creation of fake nodes.") start = time.time() - node_count = 10000 checkin = time.time() - for i in range(0, node_count): - - new_node = node.Node({ - 'power_state': 'power off', - 'driver': 'ipmi', - 'driver_internal_info': {'test-meow': i}, - 'name': 'BenchmarkTestNode-%s' % i, - 'driver_info': { - 'ipmi_username': 'admin', - 'ipmi_password': 'admin', - 'ipmi_address': 'testhost%s.env.top.level.domain' % i}, - 'resource_class': 'CUSTOM_BAREMETAL', - 'properties': { - 'cpu': 4, - 'memory': 32, - 'cats': i, - 'meowing': True}}) + for i in range(0, NODE_COUNT): + new_node = node.Node() + new_node.power_state = 'power off' + new_node.driver = 'ipmi' + new_node.driver_internal_info = {'test-meow': i} + new_node.name = 'BenchmarkTestNode-%s' % i + new_node.driver_info = { + 'ipmi_username': 'admin', 'ipmi_password': 'admin', + 'ipmi_address': 'testhost%s.env.top.level.domain' % i} + new_node.resource_class = 'CUSTOM_BAREMETAL' + new_node.properties = {'cpu': 4, + 'memory': 32, + 'cats': i, + 'meowing': True} new_node.create() + _create_test_node_ports(new_node) delta = time.time() - checkin if delta > 10: checkin = time.time() @@ -52,7 +75,7 @@ % (i, delta, time.time() - start)) created = time.time() elapse = created - start - print('Created %s nodes in %s seconds.\n' % (node_count, elapse)) + print('Created %s nodes in %s seconds.\n' % (NODE_COUNT, elapse)) def _mix_up_nodes_data(): diff -Nru ironic-21.1.0/tools/benchmark/generate-statistics.py ironic-21.4.4/tools/benchmark/generate-statistics.py --- ironic-21.1.0/tools/benchmark/generate-statistics.py 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/tools/benchmark/generate-statistics.py 2024-10-11 15:42:16.000000000 +0000 @@ -21,6 +21,7 @@ from oslo_utils import timeutils from ironic.api.controllers.v1 import node as node_api +from ironic.api.controllers.v1 import port as port_api from ironic.api.controllers.v1 import utils as api_utils from ironic.common import context from ironic.common import service @@ -28,6 +29,7 @@ from ironic.db import api as db_api from ironic.objects import conductor from ironic.objects import node +from ironic.objects import port def _calculate_delta(start, finish): @@ -56,6 +58,24 @@ return node_count +def _assess_db_performance_ports(): + start = time.time() + dbapi = db_api.get_instance() + print('Phase - Assess DB performance - Ports') + _add_a_line() + got_connection = time.time() + ports = dbapi.get_port_list() + port_count = len(ports) + query_complete = time.time() + delta = _calculate_delta(start, got_connection) + print('Obtained DB client in %s seconds.' % delta) + delta = _calculate_delta(got_connection, query_complete) + print('Returned %s ports in python %s seconds from the DB.\n' % + (port_count, delta)) + # return node count for future use. + return port_count + + def _assess_db_and_object_performance(): print('Phase - Assess DB & Object conversion Performance') _add_a_line() @@ -88,6 +108,33 @@ observed_vendors.append(vendor) +def _assess_db_and_object_performance_ports(): + print('Phase - Assess DB & Object conversion Performance - Ports') + _add_a_line() + start = time.time() + port_list = port.Port().list(context.get_admin_context()) + got_list = time.time() + delta = _calculate_delta(start, got_list) + print('Obtained list of port objects in %s seconds.' % delta) + count = 0 + tbl_size = 0 + # In a sense, this helps provide a relative understanding if the + # database is the bottleneck, or the objects post conversion. + # converting completely to json and then measuring the size helps + # ensure that everything is "assessed" while not revealing too + # much detail. + for port_obj in port_list: + # Just looping through the entire set to count should be + # enough to ensure that the entry is loaded from the db + # and then converted to an object. + tbl_size = tbl_size + sys.getsizeof(port_obj.as_dict()) + count = count + 1 + delta = _calculate_delta(got_list, time.time()) + print('Took %s seconds to iterate through %s port objects.' % + (delta, count)) + print('Ports table is roughly %s bytes of JSON.\n' % tbl_size) + + @mock.patch('ironic.api.request') # noqa patch needed for the object model @mock.patch.object(metrics_utils, 'get_metrics_logger', lambda *_: mock.Mock) @mock.patch.object(api_utils, 'check_list_policy', lambda *_: None) @@ -155,6 +202,68 @@ 'nodes API call pattern.\n' % (delta, total_nodes)) + +@mock.patch('ironic.api.request') # noqa patch needed for the object model +@mock.patch.object(metrics_utils, 'get_metrics_logger', lambda *_: mock.Mock) +@mock.patch.object(api_utils, 'check_list_policy', lambda *_: None) +@mock.patch.object(api_utils, 'check_allow_specify_fields', lambda *_: None) +@mock.patch.object(api_utils, 'check_allowed_fields', lambda *_: None) +@mock.patch.object(oslo_policy.policy, 'LOG', autospec=True) +def _assess_db_object_and_api_performance_ports(mock_log, mock_request): + print('Phase - Assess DB & Object conversion Performance - Ports') + _add_a_line() + # Just mock it to silence it since getting the logger to update + # config seems like not a thing once started. :\ + mock_log.debug = mock.Mock() + # Internal logic requires major/minor versions and a context to + # proceed. This is just to make the NodesController respond properly. + mock_request.context = context.get_admin_context() + mock_request.version.major = 1 + mock_request.version.minor = 71 + + start = time.time() + port_api_controller = port_api.PortsController() + port_api_controller.context = context.get_admin_context() + fields = ("uuid,node_uuid,address,extra,local_link_connection," + "pxe_enabled,internal_info,physical_network," + "is_smartnic") + + total_ports = 0 + + res = port_api_controller._get_ports_collection( + resource_url='ports', + node_ident=None, + address=None, + portgroup_ident=None, + marker=None, + limit=None, + sort_key="id", + sort_dir="asc", + fields=fields.split(',')) + total_ports = len(res['ports']) + while len(res['ports']) != 1: + print(" ** Getting ports ** %s Elapsed: %s seconds." % + (total_ports, _calculate_delta(start, time.time()))) + res = port_api_controller._get_ports_collection( + resource_url='ports', + node_ident=None, + address=None, + portgroup_ident=None, + marker=res['ports'][-1]['uuid'], + limit=None, + sort_key="id", + sort_dir="asc", + fields=fields.split(',')) + new_ports = len(res['ports']) + if new_ports == 0: + break + total_ports = total_ports + new_ports + + delta = _calculate_delta(start, time.time()) + print('Took %s seconds to return all %s ports via ' + 'ports API call pattern.\n' % (delta, total_ports)) + + def _report_conductors(): print('Phase - identifying conductors/drivers') _add_a_line() @@ -190,6 +299,9 @@ _assess_db_performance() _assess_db_and_object_performance() _assess_db_object_and_api_performance() + _assess_db_performance_ports() + _assess_db_and_object_performance_ports() + _assess_db_object_and_api_performance_ports() _report_conductors() diff -Nru ironic-21.1.0/tox.ini ironic-21.4.4/tox.ini --- ironic-21.1.0/tox.ini 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/tox.ini 2024-10-11 15:42:16.000000000 +0000 @@ -1,6 +1,5 @@ [tox] minversion = 3.18.0 -skipsdist = True envlist = py3,pep8 ignore_basepython_conflict=true @@ -8,16 +7,23 @@ usedevelop = True basepython = python3 setenv = VIRTUAL_ENV={envdir} - PYTHONDONTWRITEBYTECODE = 1 + PYTHONDONTWRITEBYTECODE=1 LANGUAGE=en_US LC_ALL=en_US.UTF-8 + PYTHONUNBUFFERED=1 + SQLALCHEMY_WARN_20=true deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = stestr run --slowest {posargs} -passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY +passenv = http_proxy + HTTP_PROXY + https_proxy + HTTPS_PROXY + no_proxy + NO_PROXY [testenv:unit-with-driver-libs] deps = {[testenv]deps} @@ -38,13 +44,15 @@ Pygments>=2.2.0 # BSD bashate>=0.5.1 # Apache-2.0 allowlist_externals = bash + {toxinidir}/tools/run_bashate.sh + {toxinidir}/tools/check-releasenotes.py commands = bash tools/flake8wrap.sh {posargs} # Run bashate during pep8 runs to ensure violations are caught by # the check and gate queues. {toxinidir}/tools/run_bashate.sh {toxinidir} # Check the *.rst files - doc8 README.rst CONTRIBUTING.rst doc/source --ignore D001 + doc8 README.rst CONTRIBUTING.rst doc/source api-ref/source --ignore D001 # Check to make sure reno releasenotes created with 'reno new' {toxinidir}/tools/check-releasenotes.py @@ -77,7 +85,7 @@ [testenv:docs] # NOTE(dtantsur): documentation building process requires importing ironic deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} -r{toxinidir}/requirements.txt -r{toxinidir}/doc/requirements.txt commands = sphinx-build -b html -W doc/source doc/build/html @@ -94,7 +102,7 @@ # NOTE(Mahnoor): documentation building process requires importing ironic API modules usedevelop = False deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} -r{toxinidir}/requirements.txt -r{toxinidir}/doc/requirements.txt allowlist_externals = bash @@ -105,7 +113,7 @@ [testenv:releasenotes] usedevelop = False deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html @@ -113,7 +121,7 @@ [testenv:venv] setenv = PYTHONHASHSEED=0 deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2023.1} -r{toxinidir}/test-requirements.txt -r{toxinidir}/doc/requirements.txt commands = {posargs} diff -Nru ironic-21.1.0/zuul.d/ironic-jobs.yaml ironic-21.4.4/zuul.d/ironic-jobs.yaml --- ironic-21.1.0/zuul.d/ironic-jobs.yaml 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/zuul.d/ironic-jobs.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -85,7 +85,6 @@ q-dhcp: true q-l3: true q-meta: true - q-metering: true q-svc: true ovn-controller: false ovn-northd: false @@ -251,9 +250,9 @@ # a small root partition, so use /opt which is mounted from a bigger # ephemeral partition on such nodes LIBVIRT_STORAGE_POOL_PATH: /opt/libvirt/images - IRONIC_ANACONDA_IMAGE_REF: http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/ - IRONIC_ANACONDA_KERNEL_REF: http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/images/pxeboot/vmlinuz - IRONIC_ANACONDA_RAMDISK_REF: http://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/images/pxeboot/initrd.img + IRONIC_ANACONDA_IMAGE_REF: https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/ + IRONIC_ANACONDA_KERNEL_REF: https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/images/pxeboot/vmlinuz + IRONIC_ANACONDA_RAMDISK_REF: https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/images/pxeboot/initrd.img IRONIC_ANACONDA_INSECURE_HEARTBEAT: True IRONIC_DEPLOY_CALLBACK_WAIT_TIMEOUT: 3600 IRONIC_PXE_BOOT_RETRY_TIMEOUT: 3600 @@ -265,6 +264,7 @@ required-projects: - opendev.org/openstack/sushy-tools vars: + tempest_test_regex: test_baremetal_server_ops_wholedisk_image devstack_localrc: IRONIC_ENABLED_BOOT_INTERFACES: ipxe IRONIC_TEMPEST_WHOLE_DISK_IMAGE: True @@ -300,6 +300,9 @@ # result and makes this job VERY sensitive to heavy disk IO of the # underlying hypervisor/cloud. IRONIC_CALLBACK_TIMEOUT: 800 + IRONIC_GRUB2_SHIM_FILE: https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/EFI/BOOT/BOOTX64.EFI + IRONIC_GRUB2_FILE: https://mirror.stream.centos.org/9-stream/BaseOS/x86_64/os/EFI/BOOT/grubx64.efi + IRONIC_GRUB2_CONFIG_PATH: EFI/BOOT/grub.cfg devstack_services: s-account: True s-container: True @@ -309,7 +312,7 @@ - job: name: ironic-inspector-tempest-uefi-redfish-vmedia description: "Inspect and deploy ironic node over Redfish virtual media using UEFI" - parent: ironic-tempest-partition-uefi-redfish-vmedia + parent: ironic-tempest-uefi-redfish-vmedia required-projects: - opendev.org/openstack/ironic-inspector vars: @@ -702,7 +705,6 @@ IRONIC_IPXE_ENABLED: False IRONIC_RAMDISK_TYPE: tinyipa IRONIC_AUTOMATED_CLEAN_ENABLED: False - IRONIC_VM_SPECS_RAM: 4096 - job: # Security testing for known issues @@ -723,7 +725,6 @@ - ^ironic/tests/.*$ - ^releasenotes/.*$ - ^setup.cfg$ - - ^tools/(?!bandit\.yml).*$ - ^tox.ini$ - job: @@ -890,10 +891,12 @@ IRONIC_AUTOMATED_CLEAN_ENABLED: False Q_AGENT: openvswitch Q_ML2_TENANT_NETWORK_TYPE: vxlan + Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch SWIFT_ENABLE_TEMPURLS: True SWIFT_TEMPURL_KEY: secretkey EBTABLES_RACE_FIX: True LIBVIRT_STORAGE_POOL_PATH: /opt/libvirt/images + MYSQL_GATHER_PERFORMANCE: False old: IRONIC_VM_LOG_DIR: '{{ devstack_bases.old }}/ironic-bm-logs' grenade_localrc: @@ -910,6 +913,18 @@ cinder: False ir-api: True ir-cond: True + # Neutron services + # In the Ironic grenade job we want to explicitly enable ML2/OVS agents + # and disable OVN + q-agt: true + q-dhcp: true + q-l3: true + q-meta: true + q-svc: true + q-metering: true + ovn-controller: false + ovn-northd: false + q-ovn-metadata-agent: false tempest_plugins: - ironic-tempest-plugin tempest_test_regex: ironic_tempest_plugin.tests.scenario @@ -1072,6 +1087,7 @@ - job: name: ironic-cross-sushy + nodeset: ubuntu-jammy description: Ironic unit tests run with Sushy from source parent: openstack-tox required-projects: @@ -1086,10 +1102,11 @@ - ^tools/.*$ vars: # NOTE(dtantsur): change this every release cycle if needed. - bindep_profile: test py38 - tox_envlist: py38 + bindep_profile: test py310 + tox_envlist: py310 # This variable ensures that sushy is installed from source. tox_install_siblings: true # NOTE(dtantsur): this job will be run on sushy as well, so it's # important to set the working dir to the Ironic checkout. zuul_work_dir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/ironic'].src_dir }}" + diff -Nru ironic-21.1.0/zuul.d/project.yaml ironic-21.4.4/zuul.d/project.yaml --- ironic-21.1.0/zuul.d/project.yaml 2022-09-23 00:31:09.000000000 +0000 +++ ironic-21.4.4/zuul.d/project.yaml 2024-10-11 15:42:16.000000000 +0000 @@ -2,25 +2,38 @@ templates: - check-requirements - openstack-cover-jobs - - openstack-python3-zed-jobs - - openstack-python3-zed-jobs-arm64 + - openstack-python3-jobs + - openstack-python3-jobs-arm64 - periodic-stable-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: jobs: - - ironic-tox-unit-with-driver-libs + # NOTE(TheJulia): re-enable voting once + # https://review.opendev.org/c/openstack/ironic/+/910528 + # has merged. + - ironic-tox-unit-with-driver-libs: + voting: false - ironic-cross-sushy: voting: false - ironic-tempest-functional-python3 - - ironic-tempest-functional-rbac-scope-enforced - - ironic-grenade - - ironic-standalone - - ironic-standalone-redfish + # NOTE(JayF): This job is failing and is being fixed in master. It + # should be re-enabled in stable when it's re-enabled on master. + # commented out 2023-05-19 + - ironic-tempest-functional-rbac-scope-enforced: + voting: false + # NOTE(TheJulia): Marking non-voting until we can get + # other banches sorted breakage wise. + - ironic-grenade: + voting: false + - ironic-standalone: + voting: false + - ironic-standalone-redfish: + voting: false - ironic-tempest-bios-redfish-pxe - ironic-tempest-uefi-redfish-vmedia - - ironic-tempest-wholedisk-bios-snmp-pxe - - ironic-tempest-partition-uefi-ipmi-pxe + # NOTE(TheJulia): Disabling until we can sort out the various + # breaks with CI on this branch. # NOTE(TheJulia) Marking multinode non-voting on 20210311 # Due to a high failure rate on limestone where the compute1 # machine never appears to be able to communicate across the @@ -31,20 +44,15 @@ - ironic-tempest-bios-ipmi-direct-tinyipa - ironic-tempest-bfv - ironic-tempest-ipa-partition-uefi-pxe-grub2 - - metalsmith-integration-glance-centos8-legacy # Non-voting jobs - ironic-tox-bandit: voting: false - ironic-inspector-tempest: voting: false - - ironic-inspector-tempest-managed-non-standalone: - voting: false - ironic-inspector-tempest-uefi-redfish-vmedia: voting: false - ironic-tempest-ipa-wholedisk-bios-ipmi-direct-dib: voting: false - - ironic-tempest-ipxe-ipv6: - voting: false - ironic-standalone-anaconda: voting: false - ironic-inspector-tempest-rbac-scope-enforced: @@ -53,29 +61,34 @@ voting: false - bifrost-integration-redfish-vmedia-uefi-centos-9: voting: false - - ironic-tempest-pxe_ipmitool-postgres: - voting: false - - bifrost-benchmark-ironic: - voting: false gate: jobs: - - ironic-tox-unit-with-driver-libs + # NOTE(TheJulia): Return this to voting once + # https://review.opendev.org/c/openstack/ironic/+/910528 + # or similar change to the branch has merged + #- ironic-tox-unit-with-driver-libs - ironic-tempest-functional-python3 - - ironic-tempest-functional-rbac-scope-enforced - - ironic-grenade - - ironic-standalone - - ironic-standalone-redfish + # NOTE(JayF): This job is failing and is being fixed in master. It + # should be re-enabled in stable when it's re-enabled on master. + # commented out 2023-05-19 + #- ironic-tempest-functional-rbac-scope-enforced + # NOTE(TheJulia): Disabling grenade since the branch before is no + # longer supported. + # - ironic-grenade + # NOTE(TheJulia): Commenting out standalone jobs until we figure + # out what is causing them to fail on mulitple branches. + # - ironic-standalone + # - ironic-standalone-redfish - ironic-tempest-bios-redfish-pxe - ironic-tempest-uefi-redfish-vmedia - - ironic-tempest-wholedisk-bios-snmp-pxe - - ironic-tempest-partition-uefi-ipmi-pxe # NOTE(TheJulia): Disabled multinode on 20210311 due to Limestone # seeming to be # - ironic-tempest-ipa-wholedisk-direct-tinyipa-multinode - ironic-tempest-bios-ipmi-direct-tinyipa - ironic-tempest-bfv - ironic-tempest-ipa-partition-uefi-pxe-grub2 - - metalsmith-integration-glance-centos8-legacy + # NOTE(TheJulia): Disabling until we can the competing + # gate fixes merged experimental: jobs: # TODO(dtantsur): these jobs are useful but currently hopelessly