From 179e4ce2eb8f3ce317383dc2489e22327249c414 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 15 Sep 2025 10:08:13 +0000 Subject: [PATCH 01/11] Update .gitreview for stable/2025.2 Change-Id: I33828fe702c15bf4b5716bad49d75942b1f82abd Signed-off-by: OpenStack Release Bot Generated-By: openstack/project-config:roles/copy-release-tools-scripts/files/release-tools/functions --- .gitreview | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitreview b/.gitreview index 75c6506ca4..22a621714e 100644 --- a/.gitreview +++ b/.gitreview @@ -2,3 +2,4 @@ host=review.opendev.org port=29418 project=openstack/octavia.git +defaultbranch=stable/2025.2 From 3af236348deb5dad48746f541528898549ca6ae5 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 15 Sep 2025 10:08:17 +0000 Subject: [PATCH 02/11] Update TOX_CONSTRAINTS_FILE for stable/2025.2 Update the URL to the upper-constraints file to point to the redirect rule on releases.openstack.org so that anyone working on this branch will switch to the correct upper-constraints list automatically when the requirements repository branches. Until the requirements repository has as stable/2025.2 branch, tests will continue to use the upper-constraints list on master. Change-Id: I3157f6b72b74f34a98b4556af6f3ac7892938f44 Signed-off-by: OpenStack Release Bot Generated-By: openstack/project-config:roles/copy-release-tools-scripts/files/release-tools/functions --- elements/amphora-agent/source-repository-amphora-agent | 4 ++-- elements/octavia-lib/source-repository-octavia-lib | 2 +- tox.ini | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/elements/amphora-agent/source-repository-amphora-agent b/elements/amphora-agent/source-repository-amphora-agent index 53684a4761..08d21b025c 100644 --- a/elements/amphora-agent/source-repository-amphora-agent +++ b/elements/amphora-agent/source-repository-amphora-agent @@ -1,3 +1,3 @@ # This is used for source-based builds -amphora-agent git /opt/amphora-agent https://opendev.org/openstack/octavia -upper-constraints file /opt/upper-constraints.txt https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt +amphora-agent git /opt/amphora-agent https://opendev.org/openstack/octavia stable/2025.2 +upper-constraints file /opt/upper-constraints.txt https://releases.openstack.org/constraints/upper/2025.2 diff --git a/elements/octavia-lib/source-repository-octavia-lib b/elements/octavia-lib/source-repository-octavia-lib index 33694cbbb3..bd01430df8 100644 --- a/elements/octavia-lib/source-repository-octavia-lib +++ b/elements/octavia-lib/source-repository-octavia-lib @@ -1,2 +1,2 @@ # This is used for source-based builds -octavia-lib git /opt/octavia-lib https://opendev.org/openstack/octavia-lib +octavia-lib git /opt/octavia-lib https://opendev.org/openstack/octavia-lib stable/2025.2 diff --git a/tox.ini b/tox.ini index 50ac1e3585..67816b268f 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ setenv = install_command = pip install {opts} {packages} allowlist_externals = find -deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} +deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2025.2} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = @@ -21,7 +21,7 @@ commands = # This environment is called from CI scripts to test and publish # the API Ref to docs.openstack.org. deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2025.2} -r{toxinidir}/requirements.txt -r{toxinidir}/doc/requirements.txt allowlist_externals = rm @@ -96,7 +96,7 @@ allowlist_externals = [testenv:docs] deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2025.2} -r{toxinidir}/requirements.txt -r{toxinidir}/doc/requirements.txt allowlist_externals = rm @@ -194,7 +194,7 @@ max-line-length = 79 [testenv:releasenotes] deps = - -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2025.2} -r{toxinidir}/doc/requirements.txt allowlist_externals = rm commands = From fa44598bd8e410222a2c47b08196a2af3012fbfa Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Fri, 29 Aug 2025 22:15:58 +0000 Subject: [PATCH 03/11] Fix Amphora controller IP list update Updating the Amphora agent configuration was not triggering the health manager sender to reload it's configuration file. This meant that it would not adopt the updated controller IP and port list. This patch fixes this by signalling the health manager sender process via a multiprocessing manager queue that it needs to reload it's configuration. Closes-Bug: #2121691 Change-Id: I1c8520745bf1bb59d483cc7761929794c58e5021 Signed-off-by: Michael Johnson (cherry picked from commit 69af193d0528ad4a8a680fe8a83c38b8e8b7f9ce) --- octavia/amphorae/backends/agent/api_server/server.py | 5 ++++- octavia/cmd/agent.py | 10 +++++++--- .../amphorae/backend/agent/api_server/test_server.py | 8 +++++--- .../Fix-Amphora-Config-Update-06b649883c7a4f44.yaml | 6 ++++++ 4 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 releasenotes/notes/Fix-Amphora-Config-Update-06b649883c7a4f44.yaml diff --git a/octavia/amphorae/backends/agent/api_server/server.py b/octavia/amphorae/backends/agent/api_server/server.py index 7daa82d1cc..b46eabe3ee 100644 --- a/octavia/amphorae/backends/agent/api_server/server.py +++ b/octavia/amphorae/backends/agent/api_server/server.py @@ -56,7 +56,7 @@ def register_app_error_handler(app): class Server: - def __init__(self): + def __init__(self, hm_queue): self.app = flask.Flask(__name__) self._osutils = osutils.BaseOS.get_os_util() self._keepalived = keepalived.Keepalived() @@ -64,6 +64,7 @@ def __init__(self): self._lvs_listener = keepalivedlvs.KeepalivedLvs() self._plug = plug.Plug(self._osutils) self._amphora_info = amphora_info.AmphoraInfo(self._osutils) + self._hm_queue = hm_queue register_app_error_handler(self.app) @@ -253,6 +254,8 @@ def upload_config(self): b = stream.read(BUFFER) CONF.mutate_config_files() + # Signal to the health manager process to reload it's configuration + self._hm_queue.put('reload') except Exception as e: LOG.error("Unable to update amphora-agent configuration: %s", str(e)) diff --git a/octavia/cmd/agent.py b/octavia/cmd/agent.py index 636c5f1aaf..6475e65229 100644 --- a/octavia/cmd/agent.py +++ b/octavia/cmd/agent.py @@ -31,7 +31,6 @@ CONF = cfg.CONF -HM_SENDER_CMD_QUEUE = multiproc.Queue() class AmphoraAgent(gunicorn.app.base.BaseApplication): @@ -58,14 +57,19 @@ def main(): gmr_opts.set_defaults(CONF) gmr.TextGuruMeditation.setup_autorun(version, conf=CONF) + # Setup a multiprocessing manager and queue to share between the + # health manager sender and the workers. This allows us to reload the + # configuration into the health manager sender process. + hm_queue = multiproc.Manager().Queue() + health_sender_proc = multiproc.Process(name='HM_sender', target=health_daemon.run_sender, - args=(HM_SENDER_CMD_QUEUE,)) + args=(hm_queue,)) health_sender_proc.daemon = True health_sender_proc.start() # Initiate server class - server_instance = server.Server() + server_instance = server.Server(hm_queue) bind_ip_port = utils.ip_port_str(CONF.haproxy_amphora.bind_host, CONF.haproxy_amphora.bind_port) diff --git a/octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py b/octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py index a0c48bf72b..f2d8f0c515 100644 --- a/octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py +++ b/octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py @@ -57,16 +57,17 @@ def setUp(self): self.useFixture(fixtures.MockPatch( 'oslo_config.cfg.find_config_files', return_value=[AMP_AGENT_CONF_PATH])) + hm_queue = mock.MagicMock() with mock.patch('distro.id', return_value='ubuntu'), mock.patch( 'octavia.amphorae.backends.agent.api_server.plug.' 'Plug.plug_lo'): - self.ubuntu_test_server = server.Server() + self.ubuntu_test_server = server.Server(hm_queue) self.ubuntu_app = self.ubuntu_test_server.app.test_client() with mock.patch('distro.id', return_value='centos'), mock.patch( 'octavia.amphorae.backends.agent.api_server.plug.' 'Plug.plug_lo'): - self.centos_test_server = server.Server() + self.centos_test_server = server.Server(hm_queue) self.centos_app = self.centos_test_server.app.test_client() def test_ubuntu_haproxy(self): @@ -2887,10 +2888,11 @@ def _test_upload_config(self, distro, mock_mutate): self.assertEqual(500, rv.status_code) def test_version_discovery(self): + hm_queue = mock.MagicMock() with mock.patch('distro.id', return_value='ubuntu'), mock.patch( 'octavia.amphorae.backends.agent.api_server.plug.' 'Plug.plug_lo'): - self.test_client = server.Server().app.test_client() + self.test_client = server.Server(hm_queue).app.test_client() expected_dict = {'api_version': api_server.VERSION} rv = self.test_client.get('/') self.assertEqual(200, rv.status_code) diff --git a/releasenotes/notes/Fix-Amphora-Config-Update-06b649883c7a4f44.yaml b/releasenotes/notes/Fix-Amphora-Config-Update-06b649883c7a4f44.yaml new file mode 100644 index 0000000000..80b637b5b5 --- /dev/null +++ b/releasenotes/notes/Fix-Amphora-Config-Update-06b649883c7a4f44.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixed a bug where the Amphora configuration update would only update the + Amphora agent configuration, but the health sender would not be updated + with the new controller IP list. From 427928651408a10402f7de3a70e5e27bc3dfad79 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Wed, 15 Oct 2025 16:46:38 +0200 Subject: [PATCH 04/11] Skip 4.0.{0,1} pylint releases 4.0.0 and 4.0.1 are affected by [0] [0] https://github.com/pylint-dev/pylint/issues/10669 Change-Id: Ieb023b5cdcdbbcea5e11c6fb936bda762615e3ec Signed-off-by: Gregory Thiemonge (cherry picked from commit 6743aa7bf012612ec7d281618bafc8b208e52e0e) --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5c173c3675..afd012c126 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,7 +4,7 @@ coverage!=4.4,>=4.0 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD flake8-import-order>=0.18.0,<0.19.0 # LGPLv3 oslotest>=3.2.0 # Apache-2.0 -pylint>=2.5.3 # GPLv2 +pylint>=2.5.3,!=4.0.0,!=4.0.1 # GPLv2 stestr>=2.0.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD testtools>=2.2.0 # MIT From 39e6efbda987e4253e459da8733a989734edb9a4 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Thu, 11 Mar 2021 18:50:06 +0100 Subject: [PATCH 05/11] Don't fail if a provider driver cannot be loaded in Octavia API Fix an issue that prevents the Octavia API service to be correctly initialized when it fails to load a provider driver. When enabled_provider_drivers setting is not correctly configured (ex: if it contains a non existing driver) or when a provider driver fails to load, the exception was not caught by the Octavia API service, so the service was not properly configured and the whole Octavia API was unreachable. Now the Octavia API service skips the driver initialization in case of errors, removes the driver(s) from the enabled list, and the other provider drivers are functional. Story 2008710 Task 42044 Change-Id: I34341e1aaad1524e3e0834309c5f6897f176af53 Signed-off-by: Zachary Raines (cherry picked from commit 6fc1abb780a2fc1413d5b206cec4b033a638c3c4) --- octavia/api/app.py | 16 +++++- octavia/api/drivers/driver_factory.py | 11 ++++ octavia/api/v2/controllers/provider.py | 4 +- octavia/tests/unit/api/test_app.py | 54 +++++++++++++++++++ ...avia-api-driver-init-68b634549c12da6e.yaml | 6 +++ 5 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 octavia/tests/unit/api/test_app.py create mode 100644 releasenotes/notes/fix-octavia-api-driver-init-68b634549c12da6e.yaml diff --git a/octavia/api/app.py b/octavia/api/app.py index f5290a244a..916d576a58 100644 --- a/octavia/api/app.py +++ b/octavia/api/app.py @@ -44,8 +44,20 @@ def get_pecan_config(): def _init_drivers(): """Initialize provider drivers.""" - for provider in CONF.api_settings.enabled_provider_drivers: - driver_factory.get_driver(provider) + providers_to_remove = [] + enabled_providers = driver_factory.get_providers() + for provider in enabled_providers: + try: + driver_factory.get_driver(provider) + except Exception: + LOG.exception("Cannot load driver '%s', will remove from " + "service. Please check " + "[api_settings]/enabled_provider_drivers in " + "octavia.conf for correctness.", provider) + providers_to_remove.append(provider) + + if providers_to_remove: + driver_factory.remove_providers(providers_to_remove) def setup_app(pecan_config=None, debug=False, argv=None): diff --git a/octavia/api/drivers/driver_factory.py b/octavia/api/drivers/driver_factory.py index b67fee2d2d..31e5fb7889 100644 --- a/octavia/api/drivers/driver_factory.py +++ b/octavia/api/drivers/driver_factory.py @@ -48,3 +48,14 @@ def get_driver(provider): provider, str(e)) raise exceptions.ProviderNotFound(prov=provider) return driver + + +def remove_providers(providers): + # Presumably these provider drivers failed to load, remove so they do not + # show up in the available list + for provider in providers: + CONF.api_settings.enabled_provider_drivers.pop(provider) + + +def get_providers(): + return CONF.api_settings.enabled_provider_drivers diff --git a/octavia/api/v2/controllers/provider.py b/octavia/api/v2/controllers/provider.py index be340bcaa5..da08adc364 100644 --- a/octavia/api/v2/controllers/provider.py +++ b/octavia/api/v2/controllers/provider.py @@ -13,7 +13,6 @@ # under the License. from octavia_lib.api.drivers import exceptions as lib_exceptions -from oslo_config import cfg from oslo_log import log as logging from pecan import expose as pecan_expose from pecan import request as pecan_request @@ -26,7 +25,6 @@ from octavia.common import constants from octavia.common import exceptions -CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -46,7 +44,7 @@ def get_all(self, fields=None): self._auth_validate_action(context, context.project_id, constants.RBAC_GET_ALL) - enabled_providers = CONF.api_settings.enabled_provider_drivers + enabled_providers = driver_factory.get_providers() response_list = [ provider_types.ProviderResponse(name=key, description=value) for key, value in enabled_providers.items()] diff --git a/octavia/tests/unit/api/test_app.py b/octavia/tests/unit/api/test_app.py new file mode 100644 index 0000000000..bda51f5a93 --- /dev/null +++ b/octavia/tests/unit/api/test_app.py @@ -0,0 +1,54 @@ +# Copyright 2021 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import mock + +from oslo_config import cfg +from oslo_config import fixture as oslo_fixture + +from octavia.api import app +from octavia.api.drivers import driver_factory +import octavia.tests.unit.base as base + + +class TestApp(base.TestCase): + + def setUp(self): + super().setUp() + + @mock.patch.object(driver_factory, "get_driver") + def test__init_drivers(self, mock_get_driver): + self.CONF = self.useFixture(oslo_fixture.Config(cfg.CONF)) + self.CONF.config( + group="api_settings", + enabled_provider_drivers="provider1:desc1,provider2:desc2") + + app._init_drivers() + mock_get_driver.assert_any_call("provider1") + mock_get_driver.assert_any_call("provider2") + + @mock.patch.object(driver_factory, "get_driver") + def test__init_drivers_with_error(self, mock_get_driver): + self.CONF = self.useFixture(oslo_fixture.Config(cfg.CONF)) + self.CONF.config( + group="api_settings", + enabled_provider_drivers="provider1:desc1,provider2:desc2") + + mock_get_driver.side_effect = [True, Exception('Internal Error')] + + app._init_drivers() + mock_get_driver.assert_any_call("provider1") + mock_get_driver.assert_any_call("provider2") + enabled_providers = driver_factory.get_providers() + self.assertEqual(enabled_providers, {'provider1': 'desc1'}) diff --git a/releasenotes/notes/fix-octavia-api-driver-init-68b634549c12da6e.yaml b/releasenotes/notes/fix-octavia-api-driver-init-68b634549c12da6e.yaml new file mode 100644 index 0000000000..32f6e02ba4 --- /dev/null +++ b/releasenotes/notes/fix-octavia-api-driver-init-68b634549c12da6e.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fix an issue that prevents the Octavia API service to be correctly + initialized when it fails to load a provider driver. It will now + fail gracefully and remove the driver from the enabled list. From 7d17a6a873b86ca25fa9dd5e4a3dc20c3e1c7332 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Wed, 22 Oct 2025 16:30:11 +0100 Subject: [PATCH 06/11] Fix database connection check SQLAlchemy 2 arrived in the Dalmation release. This removed the concept of 'Connectionless execution'. See [1] for further details. Prior to this change, if the connection to the database was interrupted, the Octavia Health Worker would go into an infinite loop due to the removed method / catch all exception handler. [1] https://docs.sqlalchemy.org/en/20/changelog/migration_20.html#migration-20-implicit-execution Closes-Bug: #2129562 Note: test_api.py was partially created by Claude Code Assisted-By: Claude Code Change-Id: I5e5f3b8dc8b2a94de927c547b187f9634c572c27 Signed-off-by: Doug Szumski Signed-off-by: Gregory Thiemonge (cherry picked from commit 6fb0d272c78a101fca395afd875241e43318434c) --- octavia/db/api.py | 3 +- octavia/tests/functional/db/test_api.py | 32 +++++++++++++++++++ .../fix-bug-2129562-b5a401824426ddbb.yaml | 5 +++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 octavia/tests/functional/db/test_api.py create mode 100644 releasenotes/notes/fix-bug-2129562-b5a401824426ddbb.yaml diff --git a/octavia/db/api.py b/octavia/db/api.py index 635df05a1d..8b1b316104 100644 --- a/octavia/db/api.py +++ b/octavia/db/api.py @@ -66,7 +66,8 @@ def wait_for_connection(exit_event): while down and not exit_event.is_set(): try: LOG.debug('Trying to re-establish connection to database.') - get_engine().scalar(select([1])) + with get_engine().connect() as conn: + conn.execute(select(1)) down = False LOG.debug('Connection to database re-established.') except Exception: diff --git a/octavia/tests/functional/db/test_api.py b/octavia/tests/functional/db/test_api.py new file mode 100644 index 0000000000..6f3cb30591 --- /dev/null +++ b/octavia/tests/functional/db/test_api.py @@ -0,0 +1,32 @@ +# Copyright 2026 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import threading + +from octavia.db import api as db_api +from octavia.tests.functional.db import base + + +class TestDBAPI(base.OctaviaDBTestBase): + + def test_wait_for_connection_success(self): + exit_event = threading.Event() + + # This should complete successfully without blocking + db_api.wait_for_connection(exit_event) + + # Verify we can still execute queries after wait_for_connection + with db_api.get_engine().connect() as conn: + result = conn.execute(db_api.select(1)) + self.assertIsNotNone(result) diff --git a/releasenotes/notes/fix-bug-2129562-b5a401824426ddbb.yaml b/releasenotes/notes/fix-bug-2129562-b5a401824426ddbb.yaml new file mode 100644 index 0000000000..b407aedb76 --- /dev/null +++ b/releasenotes/notes/fix-bug-2129562-b5a401824426ddbb.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes infinite database connection retry loop in Octavia Health Worker. + `LP#2129562 `__ From 0ea1d33848e626c65acc114b85a6be3fa7a3224d Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Thu, 18 Dec 2025 08:02:00 +0100 Subject: [PATCH 07/11] Fixed missing mock in TestAmphoraAgentCMD.test_main The lack of mock for multiprocessing.Manager in this test seems to trigger random crashes when running the next test on the same process. It also displays a backtrace related to multiprocessing: Exception ignored in: Traceback (most recent call last): File /usr/lib/python3.10/multiprocessing/connection.py, line 132, in __del__ self._close() File /usr/lib/python3.10/multiprocessing/connection.py, line 361, in _close _close(self._handle) OSError: [Errno 9] Bad file descriptor Change-Id: I7c699b59954acd5fe1c56e51e8b2c55fba09cdd8 Closes-Bug: #2136683 Signed-off-by: Gregory Thiemonge (cherry picked from commit 406152a87fd7aebc25f810604b988899941475bf) --- octavia/tests/unit/cmd/test_agent.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/octavia/tests/unit/cmd/test_agent.py b/octavia/tests/unit/cmd/test_agent.py index 9656733137..ce86598268 100644 --- a/octavia/tests/unit/cmd/test_agent.py +++ b/octavia/tests/unit/cmd/test_agent.py @@ -25,14 +25,18 @@ def setUp(self): @mock.patch('octavia.amphorae.backends.agent.api_server.server.Server') @mock.patch('multiprocessing.Process') @mock.patch('octavia.common.service.prepare_service') - def test_main(self, mock_service, mock_process, mock_server, mock_amp): + @mock.patch('multiprocessing.Manager') + def test_main(self, mock_manager, mock_service, mock_process, mock_server, + mock_amp): mock_health_proc = mock.MagicMock() mock_server_instance = mock.MagicMock() mock_amp_instance = mock.MagicMock() + mock_manager_instance = mock.MagicMock() mock_process.return_value = mock_health_proc mock_server.return_value = mock_server_instance mock_amp.return_value = mock_amp_instance + mock_manager.return_value = mock_manager_instance agent.main() From 70fc906832a72d4af1485d792c911af8fc744d71 Mon Sep 17 00:00:00 2001 From: Richard Cruise Date: Mon, 9 Feb 2026 13:39:29 +0000 Subject: [PATCH 08/11] Fix issues related to pkg_resources module Fixes issues with python setuptools removing pkg_resources Update unit tests to use importlib instead of pkg_resources Pin setuptools (for pep8) in tox.ini Depends-On: https://review.opendev.org/c/openstack/requirements/+/976865 Change-Id: I04a580fcb9c53dc2cab98846f7dde704cd317943 Signed-off-by: Richard Cruise Signed-off-by: Gregory Thiemonge (cherry picked from commit 8ccea286e7619a73dba24086fc7a4eda9a97c7f7) --- octavia/tests/common/sample_certs.py | 9 ++++++--- tox.ini | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/octavia/tests/common/sample_certs.py b/octavia/tests/common/sample_certs.py index a22198effb..83a32b858a 100644 --- a/octavia/tests/common/sample_certs.py +++ b/octavia/tests/common/sample_certs.py @@ -15,7 +15,7 @@ import base64 -import pkg_resources +from importlib import resources X509_CERT_CN = 'www.example.com' @@ -813,8 +813,11 @@ def b64decode(thing): uJIQ -----END CERTIFICATE-----""" -PKCS12_BUNDLE = pkg_resources.resource_string( - 'octavia.tests.unit.common.sample_configs', 'sample_pkcs12.p12') +PKCS12_BUNDLE = ( + resources.files('octavia.tests.unit.common.sample_configs') + .joinpath('sample_pkcs12.p12') + .read_bytes() +) X509_CA_CERT_CN = 'ca.example.org' diff --git a/tox.ini b/tox.ini index 67816b268f..42f2929e8e 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ allowlist_externals = find deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/2025.2} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt + setuptools<82.0.0 commands = stestr run {posargs} stestr slowest From 5e8376a0bf4eb4b7a6f4306ff9095e04ae5ff4f7 Mon Sep 17 00:00:00 2001 From: Zachary Raines Date: Tue, 10 Feb 2026 17:31:24 -0600 Subject: [PATCH 09/11] Remove failing drivers from enabled_provider_drivers A previous patch (I34341e1aaad1524e3e0834309c5f6897f176af53) updated octavia-api so that drivers which fail to initialize are skipped instead of causing the whole API process to error out. The intention was to remove these failed drivers from the list of enabled drivers dynamically. While this works in unit tests, they are not properly removed when the api is actually running as a WSGI process. This patch switches to using set_override on the configuration object, so that modifications to the enabled_drivers is properly persisted for the duration of the process. Depends-On: I04a580fcb9c53dc2cab98846f7dde704cd317943 Change-Id: I770a70c534a5be0dd9abeb40ef3f503db63da6de Signed-off-by: Zachary Raines (cherry picked from commit 6acaef1e47b610f1a8cd21d31b7000af960ff116) --- octavia/api/drivers/driver_factory.py | 9 +++- octavia/tests/unit/api/test_app.py | 51 ++++++++++++++++--- ...bling-failed-drivers-8396741fa76246d1.yaml | 10 ++++ 3 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/fix-disabling-failed-drivers-8396741fa76246d1.yaml diff --git a/octavia/api/drivers/driver_factory.py b/octavia/api/drivers/driver_factory.py index 31e5fb7889..68862af82e 100644 --- a/octavia/api/drivers/driver_factory.py +++ b/octavia/api/drivers/driver_factory.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +import copy + from oslo_config import cfg from oslo_log import log as logging from stevedore import driver as stevedore_driver @@ -53,8 +55,13 @@ def get_driver(provider): def remove_providers(providers): # Presumably these provider drivers failed to load, remove so they do not # show up in the available list + override = copy.deepcopy(CONF.api_settings.enabled_provider_drivers) for provider in providers: - CONF.api_settings.enabled_provider_drivers.pop(provider) + override.pop(provider) + CONF.set_override( + "enabled_provider_drivers", + override, + group="api_settings") def get_providers(): diff --git a/octavia/tests/unit/api/test_app.py b/octavia/tests/unit/api/test_app.py index bda51f5a93..5adcb6ea4e 100644 --- a/octavia/tests/unit/api/test_app.py +++ b/octavia/tests/unit/api/test_app.py @@ -23,32 +23,69 @@ class TestApp(base.TestCase): - def setUp(self): super().setUp() @mock.patch.object(driver_factory, "get_driver") def test__init_drivers(self, mock_get_driver): self.CONF = self.useFixture(oslo_fixture.Config(cfg.CONF)) - self.CONF.config( + self.CONF.load_raw_values( group="api_settings", - enabled_provider_drivers="provider1:desc1,provider2:desc2") + enabled_provider_drivers="provider1:desc1,provider2:desc2", + ) app._init_drivers() mock_get_driver.assert_any_call("provider1") mock_get_driver.assert_any_call("provider2") + @staticmethod + def _fail_get_driver(provider): + if provider == "provider2": + raise Exception("Internal error") + return True + @mock.patch.object(driver_factory, "get_driver") def test__init_drivers_with_error(self, mock_get_driver): self.CONF = self.useFixture(oslo_fixture.Config(cfg.CONF)) - self.CONF.config( + self.CONF.load_raw_values( group="api_settings", - enabled_provider_drivers="provider1:desc1,provider2:desc2") + enabled_provider_drivers="provider1:desc1,provider2:desc2", + ) - mock_get_driver.side_effect = [True, Exception('Internal Error')] + mock_get_driver.side_effect = self._fail_get_driver app._init_drivers() mock_get_driver.assert_any_call("provider1") mock_get_driver.assert_any_call("provider2") enabled_providers = driver_factory.get_providers() - self.assertEqual(enabled_providers, {'provider1': 'desc1'}) + self.assertEqual(enabled_providers, {"provider1": "desc1"}) + + @mock.patch.object(driver_factory, "get_driver") + def test__init_drivers_with_error_and_register_opt(self, mock_get_driver): + """Ensure removal persists through registering new opts. + + Adding new options to CFG triggers removes cached values of the options + and causes lookups to again go to the dict initially parsed from the + config files. This ensures that the removal of failed drivers persists + even after clearing the cache, e.g., due to registering new options. + """ + self.CONF = self.useFixture(oslo_fixture.Config(cfg.CONF)) + # NOTE(raineszm): It is important to use load_raw_values here and + # not config as the latter calls set_override behind the scenes which + # shortcuts the config cache. + self.CONF.load_raw_values( + group="api_settings", + enabled_provider_drivers="provider1:desc1,provider2:desc2", + ) + + mock_get_driver.side_effect = self._fail_get_driver + + app._init_drivers() + + # Registering a new options clears the cache + self.CONF.register_opt( + cfg.BoolOpt("is_a_test", default=True), group="api_settings" + ) + + enabled_providers = driver_factory.get_providers() + self.assertEqual(enabled_providers, {"provider1": "desc1"}) diff --git a/releasenotes/notes/fix-disabling-failed-drivers-8396741fa76246d1.yaml b/releasenotes/notes/fix-disabling-failed-drivers-8396741fa76246d1.yaml new file mode 100644 index 0000000000..c5018e9624 --- /dev/null +++ b/releasenotes/notes/fix-disabling-failed-drivers-8396741fa76246d1.yaml @@ -0,0 +1,10 @@ +--- +fixes: + - | + Drivers which fail to initialize are now properly removed from the list of + enabled drivers. A previous patch updated octavia-api so that drivers which + fail to initialize are skipped instead of causing the whole API process to + error out. The intention was to remove these failed drivers from the list of + enabled drivers dynamically, but this removal was being overridden by other + configuration code. The removal of these failing drivers should now persist + until service restart. From 6db3ff69e797b212266ad885ee0853498386d9e2 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Wed, 4 Mar 2026 08:31:25 +0100 Subject: [PATCH 10/11] Disabling periodic FIPS jobs on stable branches Stable branches (<=2025.2) have limited support for Centos and the FIPS job (periodic, non voting) has been failing for a long time. We won't fix it, we will focus on restoring it on current master with Centos 10 Stream support. Change-Id: I40a4e1cb388edf07d930578e985e3e5a927543cc Signed-off-by: Gregory Thiemonge --- zuul.d/projects.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 7c5269adaf..a2b174bc4c 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -119,9 +119,6 @@ regex: ^stable/.*$ negate: true - octavia-amphora-image-build -# Putting octavia-v2-dsvm-scenario-fips in periodic as centos 8 is too slow - - octavia-v2-dsvm-scenario-fips: - voting: false experimental: jobs: - octavia-v2-dsvm-scenario-nftables From ecec6798beeac648a4dacff98797836c01249474 Mon Sep 17 00:00:00 2001 From: Gregory Thiemonge Date: Mon, 30 Mar 2026 07:50:41 +0000 Subject: [PATCH 11/11] Add SKI and AKI extensions to amphora certificates Python 3.13 enables VERIFY_X509_STRICT by default in SSL contexts, which enforces RFC 5280 compliance for X.509 certificates. This caused TLS handshake failures between octavia-worker and amphorae because the locally generated certificates were missing the Subject Key Identifier (SKI) and Authority Key Identifier (AKI) extensions. This change adds both extensions to certificates generated by the local certificate generator, ensuring compatibility with Python 3.13+ and OpenSSL strict verification mode. Closes-Bug: #2146740 Change-Id: I4fd6b76a8856fff82c5e37b279f5991ecd436ab3 Co-Authored-By: Claude Opus 4.5 Signed-off-by: Gregory Thiemonge (cherry picked from commit c86b945c21b7d5d34cf13a4e9d52810d657682da) (cherry picked from commit 2d14acc0d767de25cc840f375a7d44da192061be) --- octavia/certificates/generator/local.py | 10 ++++++++ .../unit/certificates/generator/test_local.py | 24 +++++++++++++++++++ ...sing-key-identifiers-a9980c63ce1d414c.yaml | 14 +++++++++++ 3 files changed, 48 insertions(+) create mode 100644 releasenotes/notes/fix-amphora-cert-missing-key-identifiers-a9980c63ce1d414c.yaml diff --git a/octavia/certificates/generator/local.py b/octavia/certificates/generator/local.py index fb390e2097..9235f822ef 100644 --- a/octavia/certificates/generator/local.py +++ b/octavia/certificates/generator/local.py @@ -160,6 +160,16 @@ def sign_cert(cls, csr, validity, ca_cert=None, ca_key=None, ]), critical=True ) + new_cert = new_cert.add_extension( + x509.SubjectKeyIdentifier.from_public_key( + lo_req.public_key()), + critical=False + ) + new_cert = new_cert.add_extension( + x509.AuthorityKeyIdentifier.from_issuer_public_key( + lo_cert.public_key()), + critical=False + ) signed_cert = new_cert.sign(private_key=lo_key, algorithm=algorithm, backend=backends.default_backend()) diff --git a/octavia/tests/unit/certificates/generator/test_local.py b/octavia/tests/unit/certificates/generator/test_local.py index e5dc37bec1..96040ae8e7 100644 --- a/octavia/tests/unit/certificates/generator/test_local.py +++ b/octavia/tests/unit/certificates/generator/test_local.py @@ -100,6 +100,18 @@ def test_sign_cert(self): self.assertFalse(cert.extensions.get_extension_for_class( x509.BasicConstraints).value.ca) + # Make sure SubjectKeyIdentifier extension is present + ski = cert.extensions.get_extension_for_class( + x509.SubjectKeyIdentifier) + self.assertIsNotNone(ski.value.digest) + self.assertFalse(ski.critical) + + # Make sure AuthorityKeyIdentifier extension is present + aki = cert.extensions.get_extension_for_class( + x509.AuthorityKeyIdentifier) + self.assertIsNotNone(aki.value.key_identifier) + self.assertFalse(aki.critical) + def test_sign_cert_passphrase_none(self): # Attempt sign a cert ca_private_key = self.ca_key.private_bytes( @@ -145,6 +157,18 @@ def test_sign_cert_passphrase_none(self): self.assertFalse(cert.extensions.get_extension_for_class( x509.BasicConstraints).value.ca) + # Make sure SubjectKeyIdentifier extension is present + ski = cert.extensions.get_extension_for_class( + x509.SubjectKeyIdentifier) + self.assertIsNotNone(ski.value.digest) + self.assertFalse(ski.critical) + + # Make sure AuthorityKeyIdentifier extension is present + aki = cert.extensions.get_extension_for_class( + x509.AuthorityKeyIdentifier) + self.assertIsNotNone(aki.value.key_identifier) + self.assertFalse(aki.critical) + def test_sign_cert_invalid_algorithm(self): self.assertRaises( crypto_exceptions.UnsupportedAlgorithm, diff --git a/releasenotes/notes/fix-amphora-cert-missing-key-identifiers-a9980c63ce1d414c.yaml b/releasenotes/notes/fix-amphora-cert-missing-key-identifiers-a9980c63ce1d414c.yaml new file mode 100644 index 0000000000..667f05ad03 --- /dev/null +++ b/releasenotes/notes/fix-amphora-cert-missing-key-identifiers-a9980c63ce1d414c.yaml @@ -0,0 +1,14 @@ +--- +fixes: + - | + Fixed an issue where amphora certificates generated by the local + certificate generator were missing the Subject Key Identifier (SKI) + and Authority Key Identifier (AKI) X.509 extensions. This caused TLS + handshake failures when using Python 3.13+ or OpenSSL with strict + X.509 verification enabled, as these environments now enforce RFC 5280 + compliance by default. +upgrade: + - | + Existing amphorae with certificates missing the SKI and AKI extensions + will need certificate rotation (handled by octavia-housekeeping) or + failover to obtain new compliant certificates.