From 2c58999d2253aaf034666bd933724e90991ebc82 Mon Sep 17 00:00:00 2001 From: Anna Khmelnitsky Date: Wed, 15 Apr 2020 16:58:52 -0700 Subject: [PATCH] Translate exception that grounds cluster When endpoint goes down, the user should see same exception as when the cluster is already down (detected by earlier activity). For this purpose, translate grounding exception to ServiceClusterUnavaliable. In addition, display a warning if amount of retries is less than amount of endpoints, since in this case not all endpoints will be probed. Change-Id: Ib4aa5eb95069b917c989b1f6dcd3535880b5a038 --- vmware_nsxlib/tests/unit/v3/test_cluster.py | 18 ++++++++++++++++++ vmware_nsxlib/v3/cluster.py | 16 +++++++++++++++- vmware_nsxlib/v3/config.py | 6 ++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/vmware_nsxlib/tests/unit/v3/test_cluster.py b/vmware_nsxlib/tests/unit/v3/test_cluster.py index 4b99e1f0..0facc19d 100644 --- a/vmware_nsxlib/tests/unit/v3/test_cluster.py +++ b/vmware_nsxlib/tests/unit/v3/test_cluster.py @@ -428,6 +428,24 @@ class ClusteredAPITestCase(nsxlib_testcase.NsxClientTestCase): api.get('api/v1/transport-zones') self.assertEqual(cluster.ClusterHealth.GREEN, api.health) + def test_exception_translation_on_ground_error(self): + + def server_error(): + raise requests_exceptions.ConnectionError() + + conf_managers = ['8.9.10.11', '9.10.11.12', '10.11.12.13'] + exceptions = config.ExceptionConfig() + max_attempts = 4 + api = self.mock_nsx_clustered_api( + nsx_api_managers=conf_managers, + max_attempts=max_attempts, + session_response=[server_error for i in range(0, max_attempts)]) + api.nsxlib_config.exception_config = exceptions + + self.assertRaises(nsxlib_exc.ServiceClusterUnavailable, + api.get, 'api/v1/transport-zones') + self.assertEqual(cluster.ClusterHealth.RED, api.health) + def test_cluster_proxy_connection_establish_error(self): def connect_timeout(): diff --git a/vmware_nsxlib/v3/cluster.py b/vmware_nsxlib/v3/cluster.py index b90b6fc5..a291afdc 100644 --- a/vmware_nsxlib/v3/cluster.py +++ b/vmware_nsxlib/v3/cluster.py @@ -665,7 +665,21 @@ class ClusteredAPI(object): def _proxy_stub(self, proxy_for): def _call_proxy(url, *args, **kwargs): - return self._proxy(proxy_for, url, *args, **kwargs) + try: + return self._proxy(proxy_for, url, *args, **kwargs) + except Exception as ex: + # If this was exception that grounded the cluster, + # we want to translate this exception to + # ServiceClusterUnavailable. This is in order to + # provide unified "cluster down" experience for + # the client + exc_config = self.nsxlib_config.exception_config + if exc_config.should_ground_endpoint(ex): + raise exceptions.ServiceClusterUnavailable( + cluster_id=self.cluster_id) + + raise ex + return _call_proxy def _proxy(self, proxy_for, uri, *args, **kwargs): diff --git a/vmware_nsxlib/v3/config.py b/vmware_nsxlib/v3/config.py index d1b8cf37..c6f3a248 100644 --- a/vmware_nsxlib/v3/config.py +++ b/vmware_nsxlib/v3/config.py @@ -206,6 +206,12 @@ class NsxLibConfig(object): " to True, overriding provided configuration") self.cluster_unavailable_retry = True + if len(nsx_api_managers) > self.max_attempts: + LOG.warning("max_attempts setting (%d) is lower than amount of" + " endpoints (%d), which means that not all endpoints" + " will be probed in case of retriable error", + self.max_attempts, len(nsx_api_managers)) + def extend(self, keepalive_section, validate_connection_method=None, url_base=None): if keepalive_section or validate_connection_method: